.NET EF Core Sort by child collection property

Question

I have a patient class which has navigation property PatientGroupMemberships, and PatientGroupMembership had PatientGroup property, and PatientGroup has property Name. I want to sort by Patient's first PatientGroupMembership -> PatientGroup -> Name property (sorted by Name)

public class Patient
{
  public List<PatientGroupMembership>? PatientGroupMemberships { get; set; }
}

public class PatientGroupMembership
{
  public PatientGroup? PatientGroup { get; set; }
}

public class PatientGroup
{
  public string? Name { get; set; }
}

First part of the problem is solved, I load and sorted the PatientGroupMemberships by PatientGroup Name property

var query = patients
                    .Include(x => x.PatientGroupMemberships!.OrderBy(pgm => pgm.PatientGroup!.Name))
                    .ThenInclude(p => p.PatientGroup)
                    .AsNoTracking();

When I try to sort by first Name value, for some reason it doesn't order the results properly

 private static IQueryable<Patient> SortByPatientGroups(IQueryable<Patient> query, bool isDescendingOrder)
    {
        var orderedQuery = isDescendingOrder
            ? query.OrderByDescending(x => x.PatientGroupMemberships
                .Select(pgm => pgm.PatientGroup.Name)
                .FirstOrDefault())
            : query.OrderBy(x => x.PatientGroupMemberships
                .Select(pgm => pgm.PatientGroup.Name)
                .FirstOrDefault());

        return orderedQuery;
    }

Any help will be greatly appreciated!