Cookie not being set when using Blazor server App with individual authentication hooked up with Duende IdentityServer

Question

I have a Blazor Server app in .NET8.0 with individual authentication enabled and I have added Duende IdentityServer to it now for the OIDC auth flow.

For some reason my cookie is being set in the Browser when I authenticate on the pre build pages and hence all my requests to OIDC endpoints endpoints /connect/authorize up being unauthorized and my IdentityServer is not working.

Any insights on this? maybe I am doing it wrong please have a look at my program.cs because other than that I have not changed anything in the code that was already there.

using BlazorAppIDS;
using BlazorAppIDS.Services;
using BlazorAppIDS2._0.Components;
using BlazorAppIDS2._0.Components.Account;
using BlazorAppIDS2._0.Data;
using Microsoft.AspNetCore.Components.Authorization;
using Microsoft.AspNetCore.Identity;
using Microsoft.EntityFrameworkCore;
using Duende.IdentityServer.Models;
using Duende.IdentityServer;

var builder = WebApplication.CreateBuilder(args);

// Add services to the container.
builder.Services.AddRazorComponents()
    .AddInteractiveServerComponents();

builder.Services.AddCascadingAuthenticationState();
builder.Services.AddScoped<IdentityUserAccessor>();
builder.Services.AddScoped<IdentityRedirectManager>();
builder.Services.AddScoped<AuthenticationStateProvider, IdentityRevalidatingAuthenticationStateProvider>();

var connectionString = builder.Configuration.GetConnectionString("DefaultConnection") ??
                       throw new InvalidOperationException("Connection string 'DefaultConnection' not found.");
builder.Services.AddDbContext<ApplicationDbContext>(options =>
    options.UseSqlServer(connectionString));
builder.Services.AddDatabaseDeveloperPageExceptionFilter();

builder.Services.AddIdentityCore<ApplicationUser>(options => options.SignIn.RequireConfirmedAccount = true)
    .AddEntityFrameworkStores<ApplicationDbContext>()
    .AddSignInManager()
    .AddDefaultTokenProviders();

// builder.Services.AddSession(options =>
// {
//     options.IdleTimeout = TimeSpan.FromMinutes(30); 
//     options.Cookie.HttpOnly = true;
//     options.Cookie.IsEssential = true; 
// });

var apiScopes = builder.Configuration.GetSection("ApiScopes").Get<List<ApiScope>>();
var clients = builder.Configuration.GetSection("Clients").Get<List<Client>>();
var identityResources = builder.Configuration.GetSection("IdentityResources").Get<List<IdentityResource>>();
var apiResources = builder.Configuration.GetSection("ApiResources").Get<List<ApiResource>>();

// Adding IdentityServer service to the app
builder.Services.AddIdentityServer(options =>
    {
        options.Events.RaiseErrorEvents = true;
        options.Events.RaiseInformationEvents = true;
        options.Events.RaiseFailureEvents = true;
        options.Events.RaiseSuccessEvents = true;
        options.EmitStaticAudienceClaim = true;
    })
    .AddInMemoryClients(clients)
    .AddInMemoryApiResources(apiResources)
    .AddInMemoryApiScopes(apiScopes)
    .AddInMemoryIdentityResources(identityResources)
    .AddAspNetIdentity<ApplicationUser>()
    .AddProfileService<MyProfileService>();


builder.Services.AddAuthentication(options =>
    {
        options.DefaultScheme = IdentityConstants.ApplicationScheme;
        options.DefaultSignInScheme = IdentityConstants.ExternalScheme;
    })
    .AddIdentityCookies();

var allowedOrigins = builder.Configuration.GetSection("AllowedOrigins").Get<string[]>();

builder.Services.AddCors(options =>
{
    options.AddPolicy("BlazorIds",
        policy =>
        {
            policy.WithOrigins(allowedOrigins)
                .SetIsOriginAllowedToAllowWildcardSubdomains()
                .AllowAnyHeader()
                .AllowAnyMethod();
        });
});

builder.Services.AddSingleton<IEmailSender<ApplicationUser>, IdentityNoOpEmailSender>();

var app = builder.Build();

// Configure the HTTP request pipeline.
if (app.Environment.IsDevelopment())
{
    await SeedData.EnsureSeedData(app);
    app.UseMigrationsEndPoint();
}
else
{
    app.UseExceptionHandler("/Error", createScopeForErrors: true);
    // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
    app.UseHsts();
}

// // seed the database before starting the app
// var scope = app.Services.GetRequiredService<IServiceScopeFactory>().CreateScope();
// var context = scope.ServiceProvider.GetService<ApplicationDbContext>();
// context.Database.Migrate();

app.UseCors("BlazorIds");
app.UseHttpsRedirection();

app.UseStaticFiles();
app.UseAntiforgery();

// app.UseSession();

// app.UseRouting();
// app.UseAntiforgery();

// IdentityServer middleware
app.UseIdentityServer();
app.UseAuthorization();

app.MapRazorComponents<App>()
    .AddInteractiveServerRenderMode();

// Add additional endpoints required by the Identity /Account Razor components.
app.MapAdditionalIdentityEndpoints();

app.Run();

I tried several combinations of middleware in the program.cs but the cookie is not being set.