"Thread static buffer is already in use" error with MongoDB C# Driver during Insert/Update collection

Question

I'm experiencing an issue with the MongoDB C# driver on the production environment of our app when attempting to insert or update documents in my MongoDB collection. Occasionally (about 1:100000 times), I receive the following error:

Thread static buffer is already in use. <--- Thread static buffer is already in use.

This issue doesn't happen consistently but seems to occur under heavy load or when multiple threads are involved.

Environment:

MongoDB C# Driver version: 2.20

.NET Runtime: .NET Framework and .NET Core 7 (We have main app + microservices. It happens on both of them)

Simplified Code snippet:

public IMongoCollection<TDocument> GetMongoDbCollection<TDocument>(string collectionName = null){
  IMongoCollection<TDocument> mongoCollection = null;
  try
  {
     string databaseAddress = Vault.Get($"mongoDatabaseAddress"); 
     string userName = Vault.Get($"mongoDbUserName");
     string password = Vault.Get($"mongoDbPassword");
     MongoClient mongoClient = new MongoClient($"mongodb+srv://{userName}:{password}@{databaseAddress}/<dbname>?retryWrites=true&w=majority&serverSelectionTimeoutMS=10000");

     var database = mongoClient.GetDatabase("App");
     mongoCollection = database.GetCollection<TDocument>(collectionName);
                
  }
  catch (Exception ex)
  {
     LoggerClient.Error(ex);
  }
  return mongoCollection;
}

public void InsertTransactions(List<Transaction)
{
  try
  {
     var collection = this.GetMongoDbCollection<Transaction>(CurrentMongoCollection);
     #region Edit transaction data before inserting

     collection.InsertMany(encryptedTransactions);
  }
  catch (Exception ex)
  {
      LoggerClient.Error(ex);
  }
 }

Stack Trace:

MongoDB.Bson.BsonSerializationException:
   at MongoDB.Bson.Serialization.BsonClassMapSerializer`1.SerializeMember (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.Serialization.BsonClassMapSerializer`1.SerializeClass (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.Serialization.BsonClassMapSerializer`1.Serialize (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.Serialization.Serializers.SerializerBase`1.MongoDB.Bson.Serialization.IBsonSerializer.Serialize (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.Serialization.IBsonSerializerExtensions.Serialize (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.Serialization.Serializers.BsonValueSerializerBase`1.Serialize (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.Serialization.Serializers.SerializerBase`1.MongoDB.Bson.Serialization.IBsonSerializer.Serialize (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.Serialization.IBsonSerializerExtensions.Serialize (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.Serialization.Serializers.BsonValueSerializerBase`1.Serialize (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.Operations.BulkInsertOperation+InsertRequestSerializer.SerializeValue (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.Serialization.Serializers.SealedClassSerializerBase`1.Serialize (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.Serialization.Serializers.SerializerBase`1.MongoDB.Bson.Serialization.IBsonSerializer.Serialize (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.WireProtocol.Messages.Encoders.BinaryEncoders.CommandMessageBinaryEncoder.WriteType1Section (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.WireProtocol.Messages.Encoders.BinaryEncoders.CommandMessageBinaryEncoder.WriteSections (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.WireProtocol.Messages.Encoders.BinaryEncoders.CommandMessageBinaryEncoder.WriteMessage (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.WireProtocol.Messages.Encoders.BinaryEncoders.CommandRequestMessageBinaryEncoder.MongoDB.Driver.Core.WireProtocol.Messages.Encoders.IMessageEncoder.WriteMessage (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.Connections.BinaryConnection+SendMessagesHelper.EncodeMessages (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.Connections.BinaryConnection.SendMessages (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.ConnectionPools.ExclusiveConnectionPool+PooledConnection.SendMessages (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.ConnectionPools.ExclusiveConnectionPool+AcquiredConnection.SendMessages (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.Connections.ConnectionExtensions.SendMessage (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.WireProtocol.CommandUsingCommandMessageWireProtocol`1.Execute (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.WireProtocol.CommandWireProtocol`1.Execute (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.Servers.Server+ServerChannel.ExecuteProtocol (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.Servers.Server+ServerChannel.Command (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.Operations.RetryableWriteCommandOperationBase.ExecuteAttempt (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.Operations.RetryableWriteOperationExecutor.Execute (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.Operations.BulkUnmixedWriteOperationBase`1.ExecuteBatch (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.Operations.BulkUnmixedWriteOperationBase`1.ExecuteBatches (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.Operations.BulkUnmixedWriteOperationBase`1.Execute (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.Operations.BulkMixedWriteOperation.ExecuteBatch (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.Core.Operations.BulkMixedWriteOperation.Execute (MongoDB.Driver.Core, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.OperationExecutor.ExecuteWriteOperation (MongoDB.Driver, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.MongoCollectionImpl`1.ExecuteWriteOperation (MongoDB.Driver, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.MongoCollectionImpl`1.BulkWrite (MongoDB.Driver, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.MongoCollectionImpl`1+<>c__DisplayClass28_0.<BulkWrite>b__0 (MongoDB.Driver, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.MongoCollectionImpl`1.UsingImplicitSession (MongoDB.Driver, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.MongoCollectionImpl`1.BulkWrite (MongoDB.Driver, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.MongoCollectionBase`1.InsertOne (MongoDB.Driver, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Driver.MongoCollectionBase`1.InsertOne (MongoDB.Driver, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at ObjectsStandard.Transaction.Save_CALL_ONLY_FROM_TICKET (ObjectsStandard, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null)
Inner exception System.InvalidOperationException handled at MongoDB.Bson.Serialization.BsonClassMapSerializer`1.SerializeMember:
   at MongoDB.Bson.IO.ThreadStaticBuffer.RentBuffer (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.IO.ByteBufferStream.WriteCString (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.IO.BsonBinaryWriter.WriteString (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.Serialization.Serializers.StringSerializer.SerializeValue (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.Serialization.Serializers.SealedClassSerializerBase`1.Serialize (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.Serialization.Serializers.SerializerBase`1.MongoDB.Bson.Serialization.IBsonSerializer.Serialize (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.Serialization.IBsonSerializerExtensions.Serialize (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)
   at MongoDB.Bson.Serialization.BsonClassMapSerializer`1.SerializeMember (MongoDB.Bson, Version=2.20.0.0, Culture=neutral, PublicKeyToken=null)

Has anyone encountered this issue before, or does anyone have insights on what might be causing this and how to resolve it?

Thank you for your help!

Answer 1

As posted in this MongoDb forum, this is probably related to your usage of your db connection, more than the connection itself.

The code you posted is probably an example and is hard to judge. Still, I would start my investigation by checking the dependency injection or, in general, the usage you're making of the connection.

I'm sharing a piece of code I wrote for a library where I'm logging heavily by subscribing to cluster events. You can use this as an example to build your deep logging system and use it to spot weird things, for example unclosed db connections.

    private static IMongoDatabase MongoDatabaseInstanceBuilder(MongoConnectionOptions options,
    BaseMongoConnectionSecurityMode securityMode, IComponentContext context, string fullConnString)
{
    var logger = context.Resolve<MongoDbLogger>().Logger;

    logger.LogDebug("[MongoDB] - The MongoDB client will connect to {ConnString}", fullConnString);
    var mongoClientSettings = MongoClientSettings.FromConnectionString(fullConnString);

    logger.LogDebug("[MongoDB] - Obtaining MongoDB credentials. Security mode is {SecurityMode}",
        securityMode.SecurityMode);
    securityMode.AttachCredentials(mongoClientSettings, logger);

    mongoClientSettings.ApplicationName = options.ApplicationName;
    mongoClientSettings.MaxConnectionIdleTime = TimeSpan.FromSeconds(30);
    mongoClientSettings.ReadPreference = ReadPreference.SecondaryPreferred;

    mongoClientSettings.ClusterConfigurator = clusterBuilder =>
    {
        // events are heavy, let's subscribe only if debug is enabled
        if (!logger.IsEnabled(LogLevel.Debug)) return;

        clusterBuilder.Subscribe<ConnectionPoolOpenedEvent>(x =>
            logger.LogDebug("[MongoDB] - Connection pool opened for cluster {ClusterId}",
                x.ClusterId));
        clusterBuilder.Subscribe<ConnectionPoolClosedEvent>(x =>
            logger.LogDebug("[MongoDB] - Connection pool closed for cluster {ClusterId}",
                x.ClusterId));
        clusterBuilder.Subscribe<ConnectionPoolAddedConnectionEvent>(x =>
            logger.LogDebug("[MongoDB] - Connection added to the pool. Connection id is {Id}",
                x.ConnectionId.ServerId.ClusterId.Value));
        clusterBuilder.Subscribe<ConnectionPoolRemovedConnectionEvent>(x =>
            logger.LogDebug("[MongoDB] - Connection removed from the pool. Connection id is {Id}",
                x.ConnectionId.ServerId.ClusterId.Value));
        clusterBuilder.Subscribe<ConnectionPoolCheckedOutConnectionEvent>(x =>
            logger.LogDebug("[MongoDB] - Connection checked out from the pool. Connection id is {Id}",
                x.ConnectionId.ServerId.ClusterId.Value));
        clusterBuilder.Subscribe<ConnectionPoolCheckedInConnectionEvent>(x =>
            logger.LogDebug("[MongoDB] - Connection added back to the pool. Connection id is {Id}",
                x.ConnectionId.ServerId.ClusterId.Value));
    };

    logger.LogDebug("[MongoDB] - Building MongoDB client instance");
    var mongoClient = new MongoClient(mongoClientSettings);

    logger.LogDebug("[MongoDB] - MongoDB client built. Returning connection to database {Database}",
        options.Database);
    return mongoClient.GetDatabase(options.Database);
}