From 285d7b6041059b23ecc93cb242a849cd734d78fe Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 12 Dec 2024 16:41:48 +0000 Subject: [PATCH] format --- src/Speckle.Sdk.Dependencies/Pools.cs | 15 +++++++------ .../Serialization/ChannelExtensions.cs | 2 +- .../Serialization/ChannelSaver.cs | 22 ++++++++++++------- .../SizeBatchingChannelReader.cs | 3 ++- .../Serialisation/V2/Receive/ObjectLoader.cs | 2 +- .../Serialisation/V2/Send/ObjectSerializer.cs | 21 +++++++++++------- .../V2/Send/ObjectSerializerFactory.cs | 1 + .../Serialisation/V2/Send/SerializeProcess.cs | 14 +++++++----- .../DummySendServerObjectManager.cs | 5 ++++- .../ExternalIdTests.cs | 20 +++++++++++++---- 10 files changed, 68 insertions(+), 37 deletions(-) diff --git a/src/Speckle.Sdk.Dependencies/Pools.cs b/src/Speckle.Sdk.Dependencies/Pools.cs index 5dcfa9d9..86bfe7c1 100644 --- a/src/Speckle.Sdk.Dependencies/Pools.cs +++ b/src/Speckle.Sdk.Dependencies/Pools.cs @@ -22,7 +22,7 @@ public bool Return(Dictionary obj) new(new StringBuilderPooledObjectPolicy() { MaximumRetainedCapacity = 100 * 1024 * 1024 }); private sealed class ObjectDictionaryPolicy : IPooledObjectPolicy> - where TKey : notnull + where TKey : notnull { public Dictionary Create() => new(50); @@ -31,8 +31,8 @@ public bool Return(Dictionary obj) obj.Clear(); return true; } - } - + } + private sealed class ObjectListPolicy : IPooledObjectPolicy> { public List Create() => new(50); @@ -42,9 +42,10 @@ public bool Return(List obj) obj.Clear(); return true; } - } - + } + public static Pool> CreateListPool() => new(new ObjectListPolicy()); - public static Pool> CreateDictionaryPool() where TKey : notnull => new(new ObjectDictionaryPolicy()); - + + public static Pool> CreateDictionaryPool() + where TKey : notnull => new(new ObjectDictionaryPolicy()); } diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs index 5d6e397d..cb56850c 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs @@ -10,7 +10,7 @@ public static BatchingChannelReader> BatchBySize( int batchSize, bool singleReader = false, bool allowSynchronousContinuations = false - ) + ) where T : IHasSize => new SizeBatchingChannelReader( source ?? throw new ArgumentNullException(nameof(source)), diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs index fd974a95..4f974eda 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs @@ -4,9 +4,8 @@ namespace Speckle.Sdk.Dependencies.Serialization; - public abstract class ChannelSaver -where T : IHasSize + where T : IHasSize { private const int SEND_CAPACITY = 50; private const int HTTP_SEND_CHUNK_SIZE = 25_000_000; //bytes @@ -15,7 +14,7 @@ public abstract class ChannelSaver private const int HTTP_CAPACITY = 50; private const int MAX_CACHE_WRITE_PARALLELISM = 1; private const int MAX_CACHE_BATCH = 200; - + private bool _enabled; private readonly Channel _checkCacheChannel = Channel.CreateBounded( @@ -30,7 +29,11 @@ public abstract class ChannelSaver _ => throw new NotImplementedException("Dropping items not supported.") ); - public Task Start(bool enableServerSending = true, bool enableCacheSaving = true, CancellationToken cancellationToken = default) + public Task Start( + bool enableServerSending = true, + bool enableCacheSaving = true, + CancellationToken cancellationToken = default + ) { ValueTask t = new(Task.FromResult(0L)); if (enableServerSending) @@ -48,10 +51,13 @@ public Task Start(bool enableServerSending = true, bool enableCacheSaving ); if (enableCacheSaving) { - t =new (tChannelReader.Join() - .Batch(MAX_CACHE_BATCH) - .WithTimeout(HTTP_BATCH_TIMEOUT) - .ReadAllConcurrently(MAX_CACHE_WRITE_PARALLELISM, SaveToCache, cancellationToken)); + t = new( + tChannelReader + .Join() + .Batch(MAX_CACHE_BATCH) + .WithTimeout(HTTP_BATCH_TIMEOUT) + .ReadAllConcurrently(MAX_CACHE_WRITE_PARALLELISM, SaveToCache, cancellationToken) + ); } else { diff --git a/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs b/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs index 8130b761..9c668547 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs @@ -7,13 +7,14 @@ public interface IHasSize { int Size { get; } } + public class SizeBatchingChannelReader( ChannelReader source, int batchSize, bool singleReader, bool syncCont = false ) : BatchingChannelReader>(source, batchSize, singleReader, syncCont) -where T : IHasSize + where T : IHasSize { private readonly int _batchSize = batchSize; diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs index 9143ed3e..7201c4d0 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs @@ -81,7 +81,7 @@ public override async Task> Download(List ids) var (id, json) in serverObjectManager.DownloadObjects(ids.Select(x => x.NotNull()).ToList(), progress, default) ) { - toCache.Add(new(new (id), new (json), true, null)); + toCache.Add(new(new(id), new(json), true, null)); } if (toCache.Count != ids.Count) diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializer.cs b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializer.cs index 318e025d..2777fdfc 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializer.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializer.cs @@ -15,7 +15,8 @@ namespace Speckle.Sdk.Serialisation.V2.Send; public readonly record struct NodeInfo(Json Json, Closures? C) { - public Closures GetClosures() => C ?? ClosureParser.GetClosures( Json.Value ).ToDictionary(x => new Id(x.Item1), x => x.Item2 ); + public Closures GetClosures() => + C ?? ClosureParser.GetClosures(Json.Value).ToDictionary(x => new Id(x.Item1), x => x.Item2); } public partial interface IObjectSerializer : IDisposable; @@ -40,10 +41,10 @@ public sealed class ObjectSerializer : IObjectSerializer private readonly List<(Id, Json, Closures)> _chunks; private readonly Pool> _chunksPool; - + private readonly List> _chunks2 = new(); private readonly Pool> _chunks2Pool; - + private readonly List> _chunks3 = new(); private readonly Pool> _chunks3Pool; @@ -55,7 +56,10 @@ public sealed class ObjectSerializer : IObjectSerializer public ObjectSerializer( IBasePropertyGatherer propertyGatherer, IReadOnlyDictionary childCache, - Pool> chunksPool, Pool> chunks2Pool, Pool> chunks3Pool, bool trackDetachedChildren = false, + Pool> chunksPool, + Pool> chunks2Pool, + Pool> chunks3Pool, + bool trackDetachedChildren = false, CancellationToken cancellationToken = default ) { @@ -278,7 +282,7 @@ private void SerializeProperty(object? obj, JsonWriter writer, PropertyAttribute Id id; Json json; //avoid multiple serialization to get closures - if (baseObj.id != null && _childCache.TryGetValue(new (baseObj.id), out var info)) + if (baseObj.id != null && _childCache.TryGetValue(new(baseObj.id), out var info)) { id = new Id(baseObj.id); childClosures = info.GetClosures(); @@ -306,7 +310,7 @@ private void SerializeProperty(object? obj, JsonWriter writer, PropertyAttribute applicationId = baseObj.applicationId, closure = childClosures.ToDictionary(x => x.Key.Value, x => x.Value), }; - } + } _chunks.Add(new(id, json, [])); return new(id, json2); } @@ -379,14 +383,15 @@ private Id SerializeBaseObject(Base baseObj, JsonWriter writer, Closures closure _chunks3.Add(chunk); return chunk; } + private void SerializeOrChunkProperty(object? baseValue, JsonWriter jsonWriter, PropertyAttributeInfo detachInfo) { if (baseValue is IEnumerable chunkableCollection && detachInfo.IsChunkable) { List chunks = _chunks2Pool.Get(); _chunks2.Add(chunks); - - DataChunk crtChunk = new() { data =GetChunk() }; + + DataChunk crtChunk = new() { data = GetChunk() }; foreach (object element in chunkableCollection) { diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializerFactory.cs b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializerFactory.cs index 8713e998..9c5bbc8a 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializerFactory.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializerFactory.cs @@ -11,6 +11,7 @@ public class ObjectSerializerFactory(IBasePropertyGatherer propertyGatherer) : I private readonly Pool> _chunkPool = Pools.CreateListPool<(Id, Json, Closures)>(); private readonly Pool> _chunk2Pool = Pools.CreateListPool(); private readonly Pool> _chunk3Pool = Pools.CreateListPool(); + public IObjectSerializer Create(IReadOnlyDictionary baseCache, CancellationToken cancellationToken) => new ObjectSerializer(propertyGatherer, baseCache, _chunkPool, _chunk2Pool, _chunk3Pool, true, cancellationToken); } diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index 72f7269b..564e47f7 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -25,7 +25,6 @@ public readonly record struct SerializeProcessResults( IReadOnlyDictionary ConvertedReferences ); - public readonly record struct BaseItem(Id Id, Json Json, bool NeedsStorage, Closures? Closures) : IHasSize { public int Size { get; } = Encoding.UTF8.GetByteCount(Json.Value); @@ -58,7 +57,6 @@ public class SerializeProcess( private readonly Pool> _pool = Pools.CreateListPool<(Id, Json, Closures)>(); private readonly Pool> _childClosurePool = Pools.CreateDictionaryPool(); - private long _objectCount; private long _objectsFound; @@ -130,7 +128,7 @@ private async Task> Traverse(Base obj, bool isEnd, Canc } var items = Serialise(obj, childClosures, cancellationToken); - + var currentClosures = new Dictionary(); Interlocked.Increment(ref _objectCount); progress?.Report(new(ProgressEvent.FromCacheOrSerialized, _objectCount, _objectsFound)); @@ -157,14 +155,18 @@ private async Task> Traverse(Base obj, bool isEnd, Canc } //leave this sync - private IEnumerable Serialise(Base obj, IReadOnlyDictionary childInfo, CancellationToken cancellationToken) + private IEnumerable Serialise( + Base obj, + IReadOnlyDictionary childInfo, + CancellationToken cancellationToken + ) { if (!_options.SkipCacheRead && obj.id != null) { var cachedJson = sqLiteJsonCacheManager.GetObject(obj.id); if (cachedJson != null) { - yield return new BaseItem(new (obj.id.NotNull()), new (cachedJson), false, null); + yield return new BaseItem(new(obj.id.NotNull()), new(cachedJson), false, null); yield break; } } @@ -200,7 +202,7 @@ private BaseItem CheckCache(Id id, Json json, Dictionary closures) var cachedJson = sqLiteJsonCacheManager.GetObject(id.Value); if (cachedJson != null) { - return new BaseItem(id, new (cachedJson), false, null); + return new BaseItem(id, new(cachedJson), false, null); } } return new BaseItem(id, json, true, closures); diff --git a/tests/Speckle.Sdk.Serialization.Tests/DummySendServerObjectManager.cs b/tests/Speckle.Sdk.Serialization.Tests/DummySendServerObjectManager.cs index 603c8c2f..0d8433fc 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DummySendServerObjectManager.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DummySendServerObjectManager.cs @@ -23,7 +23,10 @@ CancellationToken cancellationToken CancellationToken cancellationToken ) => throw new NotImplementedException(); - public Task> HasObjects(IReadOnlyCollection objectIds, CancellationToken cancellationToken) + public Task> HasObjects( + IReadOnlyCollection objectIds, + CancellationToken cancellationToken + ) { return Task.FromResult(objectIds.Distinct().ToDictionary(x => x, savedObjects.ContainsKey)); } diff --git a/tests/Speckle.Sdk.Serialization.Tests/ExternalIdTests.cs b/tests/Speckle.Sdk.Serialization.Tests/ExternalIdTests.cs index 94bbf90d..925d73f8 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/ExternalIdTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/ExternalIdTests.cs @@ -26,7 +26,10 @@ public void Setup() public void ExternalIdTest_Detached(string lineId, string valueId) { var p = new Polyline() { units = "cm", value = [1, 2] }; - var serializer = new ObjectSerializer(new BasePropertyGatherer(), new Dictionary(), true); + using var serializer = new ObjectSerializerFactory(new BasePropertyGatherer()).Create( + new Dictionary(), + default + ); var list = serializer.Serialize(p).ToDictionary(x => x.Item1, x => x.Item2); list.ContainsKey(new Id(lineId)).ShouldBeTrue(); var json = list[new Id(lineId)]; @@ -53,7 +56,10 @@ public void ExternalIdTest_Detached_Nested(string lineId, string valueId) knots = [], weights = [], }; - var serializer = new ObjectSerializer(new BasePropertyGatherer(), new Dictionary(), true); + using var serializer = new ObjectSerializerFactory(new BasePropertyGatherer()).Create( + new Dictionary(), + default + ); var list = serializer.Serialize(curve).ToDictionary(x => x.Item1, x => x.Item2); list.ContainsKey(new Id(lineId)).ShouldBeTrue(); var json = list[new Id(lineId)]; @@ -81,7 +87,10 @@ public void ExternalIdTest_Detached_Nested_More(string lineId, string valueId) weights = [], }; var polycurve = new Polycurve() { segments = [curve], units = "cm" }; - var serializer = new ObjectSerializer(new BasePropertyGatherer(), new Dictionary(), true); + using var serializer = new ObjectSerializerFactory(new BasePropertyGatherer()).Create( + new Dictionary(), + default + ); var list = serializer.Serialize(polycurve).ToDictionary(x => x.Item1, x => x.Item2); list.ContainsKey(new Id(lineId)).ShouldBeTrue(); var json = list[new Id(lineId)]; @@ -111,7 +120,10 @@ public void ExternalIdTest_Detached_Nested_More_Too(string lineId, string valueI var polycurve = new Polycurve() { segments = [curve], units = "cm" }; var @base = new Base(); @base.SetDetachedProp("profile", polycurve); - var serializer = new ObjectSerializer(new BasePropertyGatherer(), new Dictionary(), true); + using var serializer = new ObjectSerializerFactory(new BasePropertyGatherer()).Create( + new Dictionary(), + default + ); var list = serializer.Serialize(@base).ToDictionary(x => x.Item1, x => x.Item2); list.ContainsKey(new Id(lineId)).ShouldBeTrue(); var json = list[new Id(lineId)];