Why SignalR servers encrypted with der certificates and PEM private keys cannot connect successfully

Question

The server side code Program.cs

using SignalRService.Common;
using SignalRService.Hubs;
using SignalRService.Providers;
using SignalRService.Services;
using Microsoft.AspNetCore.SignalR;

var builder = WebApplication.CreateBuilder(args);
builder.Host.UseWindowsService(options =>
{
    options.ServiceName = "SignalR Server Service";
});
builder.WebHost.UseKestrel(options =>
{
    options.ListenAnyIP(12316, config =>
    {
        var certificateMgr = new CertificateMgr();
        var certificate = certificateMgr.GetCertificate();
        if (certificate != null)
        {
            config.UseHttps(certificate);
        }
    });
});
builder.Services.AddCors(options =>
{
    options.AddPolicy(name: "AllowAll", builder =>
        builder.AllowAnyOrigin()
               .AllowAnyMethod()
               .AllowAnyHeader());
});
builder.Services.AddSignalR(options =>
{
    options.EnableDetailedErrors = true;
});
builder.Services
    .AddSingleton<IUserIdProvider, UserIdProvider>()
    .AddHostedService<ServerService>();

var app = builder.Build();
app.UseRouting();
app.UseCors("AllowAll");
app.UseEndpoints(endpoints =>
{
    endpoints.MapHub<ProxyHub>("proxyHub").RequireCors("AllowAll");
});
await app.RunAsync();

The CertificateMgr.cs in SignalR server side

namespace SignalRService.Common
{
    public class PasswordFinder : IPasswordFinder
    {
        private string password;
        public PasswordFinder(string pwd) => password = pwd;
        public char[] GetPassword() => password.ToCharArray();
    }
    public class CertificateMgr
    {
        public string _certPath = "C:\\Working Folder\\certificateFile.der";
        public string _keyPath = "C:\\Working Folder\\privateKey.pem";
        public X509Certificate2? GetCertificate()
        {
            string fileExtension = Path.GetExtension(_certPath);
            if (string.Equals(fileExtension, ".pfx", StringComparison.OrdinalIgnoreCase))
            {
                return new X509Certificate2(_certPath, "123123");
            }
            else
            {
                StreamReader sr = new StreamReader(_keyPath);
                string privateKeyPass = "123123";
                var pf = new PasswordFinder(privateKeyPass);
                PemReader pr = new PemReader(sr, pf);
                AsymmetricCipherKeyPair KeyPair = (AsymmetricCipherKeyPair)pr.ReadObject();
                RSAParameters rsaParameters = DotNetUtilities.ToRSAParameters((RsaPrivateCrtKeyParameters)KeyPair.Private);
                using (RSA rsa = RSA.Create())
                {
                    rsa.ImportParameters(rsaParameters);
                    X509Certificate2 certificate = new X509Certificate2(_certPath);
                    certificate = certificate.CopyWithPrivateKey(rsa);
                    return certificate;
                }
            }
        }
    }
}

The client side code Client.cs

using SignalRClient.Common;
using Microsoft.AspNetCore.SignalR.Client;
using System;
using System.Data.SqlTypes;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;

namespace SignalRClient.Connections
{
    public interface IProxyClient
    {
        Task Initialize();
        Task HubConnection_StartAsync();
        Task HubConnection_StopAsync();
    }
    public class SignalRClient : IProxyClient
    {
        private HubConnection? _hubConnection;
        public Task Initialize()
        {
            Console.WriteLine($"Initialize");
            return Task.CompletedTask;
        }
        public async Task HubConnection_StartAsync()
        {
            _hubConnection = new HubConnectionBuilder()
                .WithUrl("https:10.224.10.10:12316/proxyHub", (options) =>
                {
                    options.HttpMessageHandlerFactory = (handler) =>
                    {
                        if (handler is HttpClientHandler clientHandler)
                        {
                            clientHandler.ServerCertificateCustomValidationCallback += CheckCertificateCallback;
                        }
                        return handler;
                    };
                })
                .WithAutomaticReconnect(new RandomRetryPolicy())
                .Build();
            _hubConnection.Closed += HubConnection_Closed;
            _hubConnection.Reconnecting += HubConnection_Reconnecting;
            _hubConnection.Reconnected += HubConnection_Reconnected;
            _hubConnection.On<string>("MessageToClient", HubConnection_Received);
            try
            {
                await _hubConnection.StartAsync();
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }
        }

        private bool CheckCertificateCallback(HttpRequestMessage message, X509Certificate2? certificate, X509Chain? chain, SslPolicyErrors errors)
        {
            return true;
        }

        public async Task HubConnection_StopAsync()
        {
            if (_hubConnection != null)
            {
                await _hubConnection.StopAsync();
                _hubConnection = null;
            }
        }
        public async Task SendMessageToServer(string message)
        {
            if (_hubConnection != null
                && _hubConnection.State == HubConnectionState.Connected)
            {
                await _hubConnection.InvokeAsync("MessageFromClient", message);
            }
        }
        private Task HubConnection_Closed(Exception? exception)
        {
            Console.WriteLine($"Closed {exception?.Message}");
            return Task.CompletedTask;
        }
        private Task HubConnection_Reconnecting(Exception? exception)
        {
            Console.WriteLine($"Reconnecting {exception?.Message}");
            return Task.CompletedTask;
        }
        private Task HubConnection_Reconnected(string? arg)
        {
            Console.WriteLine($"{arg}");
            return Task.CompletedTask;
        }
        private void HubConnection_Received(string message)
        {
            DateTime dateTime = DateTime.UtcNow;
            Console.WriteLine($"Received {message}: " + dateTime.ToString("yyyy-MM-dd HH:mm:ss.fff"));
        }
    }
}

When the above code is running, there is an issue. If the server uses a certificate in pfx format, the client can execute into the CheckCertificate Callback function, and the client can connect to the server; If the server uses the der certificate and pem private key, only the server can run successfully, and the client cannot execute the CheckVerifieCallback function. An error is reported after running StartAsync.

Answer 1

I'm guessing you're running on Windows.

The TLS implementation for .NET on Windows comes from Windows S/Channel, which requires that the private key be written to disk (not just in memory) as part of how it offloads the encryption work for TLS into a different process.

The workaround is to load the certificate the way you are already doing, then export it to a PFX, and re-import it.

So replace

rsa.ImportParameters(rsaParameters);
X509Certificate2 certificate = new X509Certificate2(_certPath);
certificate = certificate.CopyWithPrivateKey(rsa);
return certificate;

with

rsa.ImportParameters(rsaParameters);

using (X509Certificate2 pubOnly = new X509Certificate2(_certPath))
using (X509Certificate2 withKey = pubOnly.CopyWithPrivateKey(rsa))
{
    return new X509Certificate2(withKey.Export(X509ContentType.Pkcs12));
}

That will 1) open the certificate from the public-only data, 2) make a new instance with the private key bound, 3) export that pair as a PFX/PKCS12, 4) import the PFX (which writes the private key to disk until the cert is disposed).