Closes #3061 (#3145)

* Good start

* Misc

* Make ApiAuthenticationMiddleware use new json

* Remove first newtonsoft dependency

* Pull latest ASFB json enhancements

* Start reimplementing newtonsoft!

* One thing at a time

* Keep doing all kind of breaking changes which need to be tested later

* Add back ShouldSerialize() support

* Misc

* Eradicate remaining parts of newtonsoft

* WIP

* Workaround STJ stupidity in regards to derived types

STJ can't serialize derived type properties by default, so we'll use another approach in our serializable file function

* Make CI happy

* Bunch of further fixes

* Fix AddFreeLicense() after rewrite

* Add full support for JsonDisallowNullAttribute

* Optimize our json utilities even further

* Misc

* Add support for fields in disallow null

* Misc optimization

* Fix deserialization of GlobalCache in STD

* Fix non-public [JsonExtensionData]

* Fix IM missing method exception, correct db storage helpers

* Fix saving into generic databases

Thanks STJ

* Make Save() function abstract to force inheritors to implement it properly

* Correct ShouldSerializeAdditionalProperties to be a method

* Misc cleanup

* Code review

* Allow JSON comments in configs, among other

* Allow trailing commas in configs

Users very often add them accidentally, no reason to throw on them

* Fix confirmation ID

Probably needs further fixes, will need to check later

* Correct confirmations deserialization

* Use JsonNumberHandling

* Misc

* Misc

* [JsonDisallowNull] corrections

* Forbid [JsonDisallowNull] on non-nullable structs

* Not really but okay

* Add and use ToJson() helpers

* Misc

* Misc

ArchiSteamFarm.CustomPlugins.ExamplePlugin/ArchiSteamFarm.CustomPlugins.ExamplePlugin.csproj

@@ -6,7 +6,6 @@
 	<ItemGroup>
 		<PackageReference Include="ConfigureAwaitChecker.Analyzer" PrivateAssets="all" />
 		<PackageReference Include="JetBrains.Annotations" PrivateAssets="all" />
-		<PackageReference Include="Newtonsoft.Json" IncludeAssets="compile" />
 		<PackageReference Include="SteamKit2" IncludeAssets="compile" />
 		<PackageReference Include="System.Composition.AttributedModel" IncludeAssets="compile" />
 	</ItemGroup>

ArchiSteamFarm.CustomPlugins.ExamplePlugin/ExamplePlugin.cs

@@ -21,16 +21,17 @@
 
 using System;
 using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
 using System.Composition;
 using System.Diagnostics.CodeAnalysis;
 using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 using System.Threading.Tasks;
 using ArchiSteamFarm.Core;
 using ArchiSteamFarm.Plugins.Interfaces;
 using ArchiSteamFarm.Steam;
 using ArchiSteamFarm.Steam.Data;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
 using SteamKit2;
 
 namespace ArchiSteamFarm.CustomPlugins.ExamplePlugin;
@@ -45,31 +46,35 @@ namespace ArchiSteamFarm.CustomPlugins.ExamplePlugin;
 internal sealed class ExamplePlugin : IASF, IBot, IBotCommand2, IBotConnection, IBotFriendRequest, IBotMessage, IBotModules, IBotTradeOffer {
 	// This is used for identification purposes, typically you want to use a friendly name of your plugin here, such as the name of your main class
 	// Please note that this property can have direct dependencies only on structures that were initialized by the constructor, as it's possible to be called before OnLoaded() takes place
+	[JsonInclude]
+	[Required]
 	public string Name => nameof(ExamplePlugin);
 
 	// This will be displayed to the user and written in the log file, typically you should point it to the version of your library, but alternatively you can do some more advanced logic if you'd like to
 	// Please note that this property can have direct dependencies only on structures that were initialized by the constructor, as it's possible to be called before OnLoaded() takes place
+	[JsonInclude]
+	[Required]
 	public Version Version => typeof(ExamplePlugin).Assembly.GetName().Version ?? throw new InvalidOperationException(nameof(Version));
 
 	// Plugins can expose custom properties for our GET /Api/Plugins API call, simply annotate them with [JsonProperty] (or keep public)
-	[JsonProperty]
-	public bool CustomIsEnabledField { get; private set; } = true;
+	[JsonInclude]
+	[Required]
+	public bool CustomIsEnabledField { get; private init; } = true;
 
 	// This method, apart from being called before any bot initialization takes place, allows you to read custom global config properties that are not recognized by ASF
 	// Thanks to that, you can extend default ASF config with your own stuff, then parse it here in order to customize your plugin during runtime
 	// Keep in mind that, as noted in the interface, additionalConfigProperties can be null if no custom, unrecognized properties are found by ASF, you should handle that case appropriately
 	// In addition to that, this method also guarantees that all plugins were already OnLoaded(), which allows cross-plugins-communication to be possible
-	public Task OnASFInit(IReadOnlyDictionary<string, JToken>? additionalConfigProperties = null) {
+	public Task OnASFInit(IReadOnlyDictionary<string, JsonElement>? additionalConfigProperties = null) {
 		if (additionalConfigProperties == null) {
 			return Task.CompletedTask;
 		}
 
-		foreach ((string configProperty, JToken configValue) in additionalConfigProperties) {
+		foreach ((string configProperty, JsonElement configValue) in additionalConfigProperties) {
 			// It's a good idea to prefix your custom properties with the name of your plugin, so there will be no possible conflict of ASF or other plugins using the same name, neither now or in the future
 			switch (configProperty) {
-				case $"{nameof(ExamplePlugin)}TestProperty" when configValue.Type == JTokenType.Boolean:
-					bool exampleBooleanValue = configValue.Value<bool>();
-					ASF.ArchiLogger.LogGenericInfo($"{nameof(ExamplePlugin)}TestProperty boolean property has been found with a value of: {exampleBooleanValue}");
+				case $"{nameof(ExamplePlugin)}TestProperty" when configValue.ValueKind == JsonValueKind.True:
+					ASF.ArchiLogger.LogGenericInfo($"{nameof(ExamplePlugin)}TestProperty boolean property has been found with a value of true");
 
 					break;
 			}
@@ -135,7 +140,7 @@ public Task OnBotInit(Bot bot) {
 	// Keep in mind that, as noted in the interface, additionalConfigProperties can be null if no custom, unrecognized properties are found by ASF, you should handle that case appropriately
 	// Also keep in mind that this function can be called multiple times, e.g. when user edits their bot configs during runtime
 	// Take a look at OnASFInit() for example parsing code
-	public async Task OnBotInitModules(Bot bot, IReadOnlyDictionary<string, JToken>? additionalConfigProperties = null) {
+	public async Task OnBotInitModules(Bot bot, IReadOnlyDictionary<string, JsonElement>? additionalConfigProperties = null) {
 		// For example, we'll ensure that every bot starts paused regardless of Paused property, in order to do this, we'll just call Pause here in InitModules()
 		// Thanks to the fact that this method is called with each bot config reload, we'll ensure that our bot stays paused even if it'd get unpaused otherwise
 		bot.ArchiLogger.LogGenericInfo("Pausing this bot as asked from the plugin");

ArchiSteamFarm.CustomPlugins.ExamplePlugin/MeowResponse.cs

@@ -21,15 +21,17 @@
 
 using System;
 using System.Diagnostics.CodeAnalysis;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace ArchiSteamFarm.CustomPlugins.ExamplePlugin;
 
 #pragma warning disable CA1812 // False positive, the class is used during json deserialization
 [SuppressMessage("ReSharper", "ClassCannotBeInstantiated")]
 internal sealed class MeowResponse {
-	[JsonProperty("url", Required = Required.Always)]
-	internal readonly Uri URL = null!;
+	[JsonInclude]
+	[JsonPropertyName("url")]
+	[JsonRequired]
+	internal Uri URL { get; private init; } = null!;
 
 	[JsonConstructor]
 	private MeowResponse() { }

ArchiSteamFarm.CustomPlugins.PeriodicGC/PeriodicGCPlugin.cs

@@ -20,8 +20,10 @@
 // limitations under the License.
 
 using System;
+using System.ComponentModel.DataAnnotations;
 using System.Composition;
 using System.Runtime;
+using System.Text.Json.Serialization;
 using System.Threading;
 using System.Threading.Tasks;
 using ArchiSteamFarm.Core;
@@ -38,8 +40,12 @@ internal sealed class PeriodicGCPlugin : IPlugin {
 	private static readonly object LockObject = new();
 	private static readonly Timer PeriodicGCTimer = new(PerformGC);
 
+	[JsonInclude]
+	[Required]
 	public string Name => nameof(PeriodicGCPlugin);
 
+	[JsonInclude]
+	[Required]
 	public Version Version => typeof(PeriodicGCPlugin).Assembly.GetName().Version ?? throw new InvalidOperationException(nameof(Version));
 
 	public Task OnLoaded() {

ArchiSteamFarm.CustomPlugins.SignInWithSteam/ArchiSteamFarm.CustomPlugins.SignInWithSteam.csproj

@@ -7,7 +7,6 @@
 		<PackageReference Include="AngleSharp.XPath" IncludeAssets="compile" />
 		<PackageReference Include="ConfigureAwaitChecker.Analyzer" PrivateAssets="all" />
 		<PackageReference Include="JetBrains.Annotations" PrivateAssets="all" />
-		<PackageReference Include="Newtonsoft.Json" IncludeAssets="compile" />
 		<PackageReference Include="System.Composition.AttributedModel" IncludeAssets="compile" />
 	</ItemGroup>
 

ArchiSteamFarm.CustomPlugins.SignInWithSteam/Data/SignInWithSteamRequest.cs

@@ -20,11 +20,12 @@
 // limitations under the License.
 
 using System;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace ArchiSteamFarm.CustomPlugins.SignInWithSteam.Data;
 
 public sealed class SignInWithSteamRequest {
-	[JsonProperty(Required = Required.Always)]
-	public Uri RedirectURL { get; private set; } = null!;
+	[JsonInclude]
+	[JsonRequired]
+	public Uri RedirectURL { get; private init; } = null!;
 }

ArchiSteamFarm.CustomPlugins.SignInWithSteam/Data/SignInWithSteamResponse.cs

@@ -20,13 +20,14 @@
 // limitations under the License.
 
 using System;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace ArchiSteamFarm.CustomPlugins.SignInWithSteam.Data;
 
 public sealed class SignInWithSteamResponse {
-	[JsonProperty(Required = Required.Always)]
-	public Uri ReturnURL { get; private set; }
+	[JsonInclude]
+	[JsonRequired]
+	public Uri ReturnURL { get; private init; }
 
 	internal SignInWithSteamResponse(Uri returnURL) {
 		ArgumentNullException.ThrowIfNull(returnURL);

ArchiSteamFarm.CustomPlugins.SignInWithSteam/SignInWithSteamPlugin.cs

@@ -20,7 +20,9 @@
 // limitations under the License.
 
 using System;
+using System.ComponentModel.DataAnnotations;
 using System.Composition;
+using System.Text.Json.Serialization;
 using System.Threading.Tasks;
 using ArchiSteamFarm.Core;
 using ArchiSteamFarm.Plugins.Interfaces;
@@ -31,8 +33,12 @@ namespace ArchiSteamFarm.CustomPlugins.SignInWithSteam;
 [Export(typeof(IPlugin))]
 [UsedImplicitly]
 internal sealed class SignInWithSteamPlugin : IPlugin {
+	[JsonInclude]
+	[Required]
 	public string Name => nameof(SignInWithSteamPlugin);
 
+	[JsonInclude]
+	[Required]
 	public Version Version => typeof(SignInWithSteamPlugin).Assembly.GetName().Version ?? throw new InvalidOperationException(nameof(Version));
 
 	public Task OnLoaded() {

ArchiSteamFarm.OfficialPlugins.ItemsMatcher/ArchiSteamFarm.OfficialPlugins.ItemsMatcher.csproj

@@ -6,7 +6,6 @@
 	<ItemGroup>
 		<PackageReference Include="ConfigureAwaitChecker.Analyzer" PrivateAssets="all" />
 		<PackageReference Include="JetBrains.Annotations" PrivateAssets="all" />
-		<PackageReference Include="Newtonsoft.Json" IncludeAssets="compile" />
 		<PackageReference Include="SteamKit2" IncludeAssets="compile" />
 		<PackageReference Include="Swashbuckle.AspNetCore.Annotations" IncludeAssets="compile" />
 		<PackageReference Include="System.Composition.AttributedModel" IncludeAssets="compile" />

ArchiSteamFarm.OfficialPlugins.ItemsMatcher/BotCache.cs

@@ -22,20 +22,22 @@
 using System;
 using System.Globalization;
 using System.IO;
+using System.Text.Json.Serialization;
 using System.Threading.Tasks;
 using ArchiSteamFarm.Collections;
 using ArchiSteamFarm.Core;
 using ArchiSteamFarm.Helpers;
+using ArchiSteamFarm.Helpers.Json;
 using ArchiSteamFarm.Localization;
 using ArchiSteamFarm.OfficialPlugins.ItemsMatcher.Data;
 using JetBrains.Annotations;
-using Newtonsoft.Json;
 
 namespace ArchiSteamFarm.OfficialPlugins.ItemsMatcher;
 
 internal sealed class BotCache : SerializableFile {
-	[JsonProperty(Required = Required.DisallowNull)]
-	internal readonly ConcurrentList<AssetForListing> LastAnnouncedAssetsForListing = [];
+	[JsonDisallowNull]
+	[JsonInclude]
+	internal ConcurrentList<AssetForListing> LastAnnouncedAssetsForListing { get; private init; } = [];
 
 	internal string? LastAnnouncedTradeToken {
 		get => BackingLastAnnouncedTradeToken;
@@ -76,14 +78,14 @@ internal DateTime? LastRequestAt {
 		}
 	}
 
-	[JsonProperty]
-	private string? BackingLastAnnouncedTradeToken;
+	[JsonInclude]
+	private string? BackingLastAnnouncedTradeToken { get; set; }
 
-	[JsonProperty]
-	private string? BackingLastInventoryChecksumBeforeDeduplication;
+	[JsonInclude]
+	private string? BackingLastInventoryChecksumBeforeDeduplication { get; set; }
 
-	[JsonProperty]
-	private DateTime? BackingLastRequestAt;
+	[JsonInclude]
+	private DateTime? BackingLastRequestAt { get; set; }
 
 	private BotCache(string filePath) : this() {
 		ArgumentException.ThrowIfNullOrEmpty(filePath);
@@ -116,6 +118,8 @@ protected override void Dispose(bool disposing) {
 		base.Dispose(disposing);
 	}
 
+	protected override Task Save() => Save(this);
+
 	internal static async Task<BotCache> CreateOrLoad(string filePath) {
 		ArgumentException.ThrowIfNullOrEmpty(filePath);
 
@@ -134,7 +138,7 @@ internal static async Task<BotCache> CreateOrLoad(string filePath) {
 				return new BotCache(filePath);
 			}
 
-			botCache = JsonConvert.DeserializeObject<BotCache>(json);
+			botCache = json.ToJsonObject<BotCache>();
 		} catch (Exception e) {
 			ASF.ArchiLogger.LogGenericException(e);
 

ArchiSteamFarm.OfficialPlugins.ItemsMatcher/Data/AnnouncementDiffRequest.cs

@@ -22,19 +22,21 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Text.Json.Serialization;
 using ArchiSteamFarm.Steam.Data;
 using ArchiSteamFarm.Steam.Storage;
-using Newtonsoft.Json;
 using SteamKit2;
 
 namespace ArchiSteamFarm.OfficialPlugins.ItemsMatcher.Data;
 
 internal sealed class AnnouncementDiffRequest : AnnouncementRequest {
-	[JsonProperty(Required = Required.Always)]
-	private readonly ImmutableHashSet<AssetForListing> InventoryRemoved;
+	[JsonInclude]
+	[JsonRequired]
+	private ImmutableHashSet<AssetForListing> InventoryRemoved { get; init; }
 
-	[JsonProperty(Required = Required.Always)]
-	private readonly string PreviousInventoryChecksum;
+	[JsonInclude]
+	[JsonRequired]
+	private string PreviousInventoryChecksum { get; init; }
 
 	internal AnnouncementDiffRequest(Guid guid, ulong steamID, IReadOnlyCollection<AssetForListing> inventory, string inventoryChecksum, IReadOnlyCollection<Asset.EType> matchableTypes, uint totalInventoryCount, bool matchEverything, byte maxTradeHoldDuration, string tradeToken, IReadOnlyCollection<AssetForListing> inventoryRemoved, string previousInventoryChecksum, string? nickname = null, string? avatarHash = null) : base(guid, steamID, inventory, inventoryChecksum, matchableTypes, totalInventoryCount, matchEverything, maxTradeHoldDuration, tradeToken, nickname, avatarHash) {
 		ArgumentOutOfRangeException.ThrowIfEqual(guid, Guid.Empty);

ArchiSteamFarm.OfficialPlugins.ItemsMatcher/Data/AnnouncementRequest.cs

@@ -22,47 +22,56 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Text.Json.Serialization;
 using ArchiSteamFarm.Steam.Data;
 using ArchiSteamFarm.Steam.Storage;
 using JetBrains.Annotations;
-using Newtonsoft.Json;
 using SteamKit2;
 
 namespace ArchiSteamFarm.OfficialPlugins.ItemsMatcher.Data;
 
 internal class AnnouncementRequest {
-	[JsonProperty]
-	private readonly string? AvatarHash;
+	[JsonInclude]
+	private string? AvatarHash { get; init; }
 
-	[JsonProperty(Required = Required.Always)]
-	private readonly Guid Guid;
+	[JsonInclude]
+	[JsonRequired]
+	private Guid Guid { get; init; }
 
-	[JsonProperty(Required = Required.Always)]
-	private readonly ImmutableHashSet<AssetForListing> Inventory;
+	[JsonInclude]
+	[JsonRequired]
+	private ImmutableHashSet<AssetForListing> Inventory { get; init; }
 
-	[JsonProperty(Required = Required.Always)]
-	private readonly string InventoryChecksum;
+	[JsonInclude]
+	[JsonRequired]
+	private string InventoryChecksum { get; init; }
 
-	[JsonProperty(Required = Required.Always)]
-	private readonly ImmutableHashSet<Asset.EType> MatchableTypes;
+	[JsonInclude]
+	[JsonRequired]
+	private ImmutableHashSet<Asset.EType> MatchableTypes { get; init; }
 
-	[JsonProperty(Required = Required.Always)]
-	private readonly bool MatchEverything;
+	[JsonInclude]
+	[JsonRequired]
+	private bool MatchEverything { get; init; }
 
-	[JsonProperty(Required = Required.Always)]
-	private readonly byte MaxTradeHoldDuration;
+	[JsonInclude]
+	[JsonRequired]
+	private byte MaxTradeHoldDuration { get; init; }
 
-	[JsonProperty]
-	private readonly string? Nickname;
+	[JsonInclude]
+	private string? Nickname { get; init; }
 
-	[JsonProperty(Required = Required.Always)]
-	private readonly ulong SteamID;
+	[JsonInclude]
+	[JsonRequired]
+	private ulong SteamID { get; init; }
 
-	[JsonProperty(Required = Required.Always)]
-	private readonly uint TotalInventoryCount;
+	[JsonInclude]
+	[JsonRequired]
+	private uint TotalInventoryCount { get; init; }
 
-	[JsonProperty(Required = Required.Always)]
-	private readonly string TradeToken;
+	[JsonInclude]
+	[JsonRequired]
+	private string TradeToken { get; init; }
 
 	internal AnnouncementRequest(Guid guid, ulong steamID, IReadOnlyCollection<AssetForListing> inventory, string inventoryChecksum, IReadOnlyCollection<Asset.EType> matchableTypes, uint totalInventoryCount, bool matchEverything, byte maxTradeHoldDuration, string tradeToken, string? nickname = null, string? avatarHash = null) {
 		ArgumentOutOfRangeException.ThrowIfEqual(guid, Guid.Empty);

ArchiSteamFarm.OfficialPlugins.ItemsMatcher/Data/AssetForListing.cs

@@ -20,19 +20,23 @@
 // limitations under the License.
 
 using System;
+using System.Text.Json.Serialization;
 using ArchiSteamFarm.Steam.Data;
-using Newtonsoft.Json;
 
 namespace ArchiSteamFarm.OfficialPlugins.ItemsMatcher.Data;
 
 internal sealed class AssetForListing : AssetInInventory {
-	[JsonProperty("i", Required = Required.Always)]
-	internal readonly uint Index;
+	internal string BackendHashCode => $"{Index}-{PreviousAssetID}-{AssetID}-{ClassID}-{Rarity}-{RealAppID}-{Tradable}-{Type}-{Amount}";
 
-	[JsonProperty("l", Required = Required.Always)]
-	internal readonly ulong PreviousAssetID;
+	[JsonInclude]
+	[JsonPropertyName("i")]
+	[JsonRequired]
+	internal uint Index { get; private init; }
 
-	internal string BackendHashCode => Index + "-" + PreviousAssetID + "-" + AssetID + "-" + ClassID + "-" + Rarity + "-" + RealAppID + "-" + Tradable + "-" + Type + "-" + Amount;
+	[JsonInclude]
+	[JsonPropertyName("l")]
+	[JsonRequired]
+	internal ulong PreviousAssetID { get; private init; }
 
 	internal AssetForListing(Asset asset, uint index, ulong previousAssetID) : base(asset) {
 		ArgumentNullException.ThrowIfNull(asset);