How to deserialize part of json using System.Text.Json in .net core 3.0?

Question

I have a json from here https://api.nasa.gov/insight_weather/?api_key=DEMO_KEY&feedtype=json&ver=1.0 which looks like:

{
  "782": {
    "First_UTC": "2021-02-06T17:08:11Z",
    "Last_UTC": "2021-02-07T17:47:46Z",
    "Month_ordinal": 12,
    "Northern_season": "late winter",
    "PRE": {
      "av": 721.77,
      "ct": 113450,
      "mn": 698.8193,
      "mx": 742.2686
    },
    "Season": "winter",
    "Southern_season": "late summer",
    "WD": {
      "most_common": null
    }
  },
  "783": {
    "First_UTC": "2021-02-07T17:47:46Z",
    "Last_UTC": "2021-02-08T18:27:22Z",
    "Month_ordinal": 12,
    "Northern_season": "late winter",
    "PRE": {
      "av": 722.186,
      "ct": 107270,
      "mn": 698.7664,
      "mx": 743.1983
    },
    "Season": "winter",
    "Southern_season": "late summer",
    "WD": {
      "most_common": null
    }
  },
  "sol_keys": [ "782", "783" ],
  "validity_checks": { /* Some complex object */ }
}

I need only part of this information so I have created the following classes:

public class MarsWheather {
    [JsonPropertyName("First_UTC")]
    public DateTime FirstUTC { get; set; }
    [JsonPropertyName("Last_UTC")]
    public DateTime LastUTC { get; set; }
    [JsonPropertyName("Season")]
    [JsonConverter(typeof(JsonStringEnumConverter))]
    public Season MarsSeason { get; set; }
    [JsonPropertyName("PRE")]
    public DataDescription AtmosphericPressure { get; set; }
}

public enum Season {
    winter,
    spring,
    summer,
    autumn
}

public class DataDescription{
    [JsonPropertyName("av")]
    public double Average { get; set; }
    [JsonPropertyName("ct")]
    public double TotalCount { get; set; }
    [JsonPropertyName("mn")]
    public double Minimum { get; set; }
    [JsonPropertyName("mx")]
    public double Maximum { get; set; }
} 

The problem is that the JSON root object from NASA contains properties "validity_checks" and "sol_keys" that I don't need and want to skip. In Newton.Json I've used JObject.Parse to do this, but in System.Text.Json I want to use

JsonSerializer.DeserializeAsync<Dictionary<string, MarsWheather>>(stream, new JsonSerializerOptions { IgnoreNullValues = true });

Unfortunately, when I do I get an exception:

System.Text.Json.JsonException: The JSON value could not be converted to MarsWheather. Path: $.sol_keys | LineNumber: 120 | BytePositionInLine: 15.

Demo fiddle here.

Is it possible?

Answer 1

Your JSON root object consists of certain fixed keys ("sol_keys" and "validity_checks") whose values each have some fixed schema, and any number of variable keys (the "782" numeric keys) whose values all share a common schema that differs from the schemas of the fixed key values:

{
  "782": {
    // Properties corresponding to your MarsWheather object
  },
  "783": {
    // Properties corresponding to your MarsWheather object
  },
  // Other variable numeric key/value pairs corresponding to KeyValuePair<string, MarsWheather>
  "sol_keys": [
    // Some array values you don't care about
  ],
  "validity_checks": {
    // Some object you don't care about
  }
}

You would like to deserialize just the variable keys, but when you try to deserialize to a Dictionary<string, MarsWheather> you get an exception because the serializer tries to deserialize a fixed key value as if it were variable key value -- but since the fixed key has an array value while the variable keys have object values, an exception gets thrown. How can System.Text.Json be told to skip the known, fixed keys rather than trying to deserialize them?

If you want to deserialize just the variable keys and skip the fixed, known keys, you will need to create a custom JsonConverter. The easiest way to do that would be to first create some root object for your dictionary:

[JsonConverter(typeof(MarsWheatherRootObjectConverter))]
public class MarsWheatherRootObject
{
    public Dictionary<string, MarsWheather> MarsWheathers { get; } = new Dictionary<string, MarsWheather>();
}

And then define the following converter for it as follows:

public class MarsWheatherRootObjectConverter : FixedAndvariablePropertyNameObjectConverter<MarsWheatherRootObject, Dictionary<string, MarsWheather>, MarsWheather>
{
    static readonly Dictionary<string, ReadFixedKeyMethod> FixedKeyReadMethods = new Dictionary<string, ReadFixedKeyMethod>(StringComparer.OrdinalIgnoreCase)
    {
        { "sol_keys", (ref Utf8JsonReader reader, MarsWheatherRootObject obj, string name, JsonSerializerOptions options) => reader.Skip() },
        { "validity_checks", (ref Utf8JsonReader reader, MarsWheatherRootObject obj, string name, JsonSerializerOptions options) => reader.Skip() },
    };

    protected override Dictionary<string, MarsWheather> GetDictionary(MarsWheatherRootObject obj) => obj.MarsWheathers;
    protected override void SetDictionary(MarsWheatherRootObject obj, Dictionary<string, MarsWheather> dictionary) => throw new RowNotInTableException();
    protected override bool TryGetFixedKeyReadMethod(string name, JsonSerializerOptions options, out ReadFixedKeyMethod method) => FixedKeyReadMethods.TryGetValue(name, out method);
    protected override IEnumerable<KeyValuePair<string, WriteFixedKeyMethod>> GetFixedKeyWriteMethods(JsonSerializerOptions options) => Enumerable.Empty<KeyValuePair<string, WriteFixedKeyMethod>>();
}

public abstract class FixedAndvariablePropertyNameObjectConverter<TObject, TDictionary, TValue> : JsonConverter<TObject> 
    where TDictionary : class, IDictionary<string, TValue>, new()
    where TObject : new()
{
    protected delegate void ReadFixedKeyMethod(ref Utf8JsonReader reader, TObject obj, string name, JsonSerializerOptions options);
    protected delegate void WriteFixedKeyMethod(Utf8JsonWriter writer, TObject value, JsonSerializerOptions options);
        
    protected abstract TDictionary GetDictionary(TObject obj);
    protected abstract void SetDictionary(TObject obj, TDictionary dictionary);
    protected abstract bool TryGetFixedKeyReadMethod(string name, JsonSerializerOptions options, out ReadFixedKeyMethod method);
    protected abstract IEnumerable<KeyValuePair<string, WriteFixedKeyMethod>> GetFixedKeyWriteMethods(JsonSerializerOptions options);
        
    public override TObject Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
    {
        if (reader.TokenType == JsonTokenType.Null)
            return (typeToConvert.IsValueType && Nullable.GetUnderlyingType(typeToConvert) == null)
                ? throw new JsonException(string.Format("Unepected token {0}", reader.TokenType))
                : default(TObject);
        if (reader.TokenType != JsonTokenType.StartObject)
            throw new JsonException(string.Format("Unepected token {0}", reader.TokenType));
        var obj = new TObject();
        var dictionary = GetDictionary(obj);
        var valueConverter = (typeof(TValue) == typeof(object) ? null : (JsonConverter<TValue>)options.GetConverter(typeof(TValue))); // Encountered a bug using the builtin ObjectConverter
        while (reader.Read())
        {
            if (reader.TokenType == JsonTokenType.PropertyName)
            {
                var name = reader.GetString();
                reader.ReadAndAssert();
                if (TryGetFixedKeyReadMethod(name, options, out var method))
                {
                    method(ref reader, obj, name, options);
                }
                else
                {
                    if (dictionary == null)
                        SetDictionary(obj, dictionary = new TDictionary());
                    dictionary.Add(name, valueConverter.ReadOrDeserialize(ref reader, typeof(TValue), options));
                }
            }
            else if (reader.TokenType == JsonTokenType.EndObject)
            {
                return obj;
            }
            else
            {
                throw new JsonException(string.Format("Unepected token {0}", reader.TokenType));
            }
        }
        throw new JsonException(); // Truncated file
    }

    public override void Write(Utf8JsonWriter writer, TObject value, JsonSerializerOptions options)
    {
        writer.WriteStartObject();
        var dictionary = GetDictionary(value);
        if (dictionary != null)
        {
            var valueConverter = (typeof(TValue) == typeof(object) ? null : (JsonConverter<TValue>)options.GetConverter(typeof(TValue))); // Encountered a bug using the builtin ObjectConverter
            foreach (var pair in dictionary)
            {
                // TODO: handle DictionaryKeyPolicy 
                writer.WritePropertyName(pair.Key);
                valueConverter.WriteOrSerialize(writer, pair.Value, typeof(TValue), options);
            }
        }
        foreach (var pair in GetFixedKeyWriteMethods(options))
        {
            writer.WritePropertyName(pair.Key);
            pair.Value(writer, value, options);
        }
        writer.WriteEndObject();
    }
}

public static partial class JsonExtensions
{
    public static void WriteOrSerialize<T>(this JsonConverter<T> converter, Utf8JsonWriter writer, T value, Type type, JsonSerializerOptions options)
    {
        if (converter != null)
            converter.Write(writer, value, options);
        else
            JsonSerializer.Serialize(writer, value, type, options);
    }

    public static T ReadOrDeserialize<T>(this JsonConverter<T> converter, ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
        => converter != null ? converter.Read(ref reader, typeToConvert, options) : (T)JsonSerializer.Deserialize(ref reader, typeToConvert, options);

    public static void ReadAndAssert(this ref Utf8JsonReader reader)
    {
        if (!reader.Read())
            throw new JsonException();
    }
}

And now you will be able to deserialize to MarsWheatherRootObject as follows:

var root = await System.Text.Json.JsonSerializer.DeserializeAsync<MarsWheatherRootObject>(
    stream, 
    new System.Text.Json.JsonSerializerOptions 
    { 
        PropertyNameCaseInsensitive = true 
    });

Demo fiddle #1 here.

Notes:

• FixedAndvariablePropertyNameObjectConverter<TObject, TDictionary, TValue> provides a general framework for serializing and deserializing objects with fixed and variable properties. If later you decide to deserialize e.g. "sol_keys", you could modify MarsWheatherRootObject as follows:

  [JsonConverter(typeof(MarsWheatherRootObjectConverter))]
  public class MarsWheatherRootObject
  {
      public Dictionary<string, MarsWheather> MarsWheathers { get; } = new Dictionary<string, MarsWheather>();
      public List<string> SolKeys { get; set; } = new List<string>();
  }
  

  And the converter as follows:

  public class MarsWheatherRootObjectConverter : FixedAndvariablePropertyNameObjectConverter<MarsWheatherRootObject, Dictionary<string, MarsWheather>, MarsWheather>
  {
      static readonly Dictionary<string, ReadFixedKeyMethod> FixedKeyReadMethods = new(StringComparer.OrdinalIgnoreCase)
      {
          { "sol_keys", (ref Utf8JsonReader reader, MarsWheatherRootObject obj, string name, JsonSerializerOptions options) => 
              {
                  obj.SolKeys = JsonSerializer.Deserialize<List<string>>(ref reader, options);
              } 
          },
          { "validity_checks", (ref Utf8JsonReader reader, MarsWheatherRootObject obj, string name, JsonSerializerOptions options) => reader.Skip() },
      };
      static readonly Dictionary<string, WriteFixedKeyMethod> FixedKeyWriteMethods = new Dictionary<string, WriteFixedKeyMethod>()
      {
          { "sol_keys", (w, v, o) => 
              {
                  JsonSerializer.Serialize(w, v.SolKeys, o);
              } 
          },
      };
  
      protected override Dictionary<string, MarsWheather> GetDictionary(MarsWheatherRootObject obj) => obj.MarsWheathers;
      protected override void SetDictionary(MarsWheatherRootObject obj, Dictionary<string, MarsWheather> dictionary) => throw new RowNotInTableException();
      protected override bool TryGetFixedKeyReadMethod(string name, JsonSerializerOptions options, out ReadFixedKeyMethod method) => FixedKeyReadMethods.TryGetValue(name, out method);
      protected override IEnumerable<KeyValuePair<string, WriteFixedKeyMethod>> GetFixedKeyWriteMethods(JsonSerializerOptions options) => FixedKeyWriteMethods;
  }
  

  Demo fiddle #2 here.