Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial Object Store implementation #150

Merged
merged 11 commits into from
Oct 17, 2023
Merged

Initial Object Store implementation #150

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
9897418
Initial Object Store implementation
mtmk Oct 12, 2023
d6b89ca
Run Object store tests in GH CI
mtmk Oct 12, 2023
ff8524f
Merge branch 'main' into obj-store-initial-impl
mtmk Oct 13, 2023
63b7dd2
Pull consumer abstraction
mtmk Oct 13, 2023
ead74ed
Push consumer channel explanation
mtmk Oct 16, 2023
79df9a8
Object get and delete
mtmk Oct 16, 2023
29ac88a
Merge branch 'main' into obj-store-initial-impl
mtmk Oct 17, 2023
2ed63d9
Stream leave open option
mtmk Oct 17, 2023
318ba50
Stream leave open option
mtmk Oct 17, 2023
9001738
Rename OB to Obj
mtmk Oct 17, 2023
2393fb3
Update NATS.Client.sln.DotSettings
mtmk Oct 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,14 +45,16 @@ jobs:
run: dotnet build -c Debug

- name: Test Core
run: dotnet test -c Debug --no-build tests/NATS.Client.Core.Tests/NATS.Client.Core.Tests.csproj
run: dotnet test -c Debug --no-build --logger:"console;verbosity=normal" tests/NATS.Client.Core.Tests/NATS.Client.Core.Tests.csproj

- name: Test JetStream
# This test is hanging sometimes. Find out where!
run: dotnet test -c Debug --no-build --logger:"console;verbosity=normal" tests/NATS.Client.JetStream.Tests/NATS.Client.JetStream.Tests.csproj

- name: Test Key/Value Store
run: dotnet test -c Debug --no-build tests/NATS.Client.KeyValueStore.Tests/NATS.Client.KeyValueStore.Tests.csproj
run: dotnet test -c Debug --no-build --logger:"console;verbosity=normal" tests/NATS.Client.KeyValueStore.Tests/NATS.Client.KeyValueStore.Tests.csproj

- name: Test Object Store
run: dotnet test -c Debug --no-build --logger:"console;verbosity=normal" tests/NATS.Client.ObjectStore.Tests/NATS.Client.ObjectStore.Tests.csproj

memory_test:
name: memory test
Expand Down
21 changes: 21 additions & 0 deletions NATS.Client.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NATS.Client.KeyValueStore.T
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example.KeyValueStore.Watcher", "sandbox\Example.KeyValueStore.Watcher\Example.KeyValueStore.Watcher.csproj", "{912A4F2F-1BD1-4AE2-BAB8-5A49C221DB53}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NATS.Client.ObjectStore", "src\NATS.Client.ObjectStore\NATS.Client.ObjectStore.csproj", "{3F8840BA-4F91-4359-AA53-6B26823E7F55}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NATS.Client.ObjectStore.Tests", "tests\NATS.Client.ObjectStore.Tests\NATS.Client.ObjectStore.Tests.csproj", "{BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example.ObjectStore", "sandbox\Example.ObjectStore\Example.ObjectStore.csproj", "{51882883-A66E-4F95-A1AB-CFCBF71B4376}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example.TlsFirst", "sandbox\Example.TlsFirst\Example.TlsFirst.csproj", "{88625045-978F-417F-9F51-A4E3A9718945}"
EndProject
Global
Expand Down Expand Up @@ -185,6 +191,18 @@ Global
{912A4F2F-1BD1-4AE2-BAB8-5A49C221DB53}.Debug|Any CPU.Build.0 = Debug|Any CPU
{912A4F2F-1BD1-4AE2-BAB8-5A49C221DB53}.Release|Any CPU.ActiveCfg = Release|Any CPU
{912A4F2F-1BD1-4AE2-BAB8-5A49C221DB53}.Release|Any CPU.Build.0 = Release|Any CPU
{3F8840BA-4F91-4359-AA53-6B26823E7F55}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3F8840BA-4F91-4359-AA53-6B26823E7F55}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3F8840BA-4F91-4359-AA53-6B26823E7F55}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3F8840BA-4F91-4359-AA53-6B26823E7F55}.Release|Any CPU.Build.0 = Release|Any CPU
{BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Release|Any CPU.Build.0 = Release|Any CPU
{51882883-A66E-4F95-A1AB-CFCBF71B4376}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{51882883-A66E-4F95-A1AB-CFCBF71B4376}.Debug|Any CPU.Build.0 = Debug|Any CPU
{51882883-A66E-4F95-A1AB-CFCBF71B4376}.Release|Any CPU.ActiveCfg = Release|Any CPU
{51882883-A66E-4F95-A1AB-CFCBF71B4376}.Release|Any CPU.Build.0 = Release|Any CPU
{88625045-978F-417F-9F51-A4E3A9718945}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{88625045-978F-417F-9F51-A4E3A9718945}.Debug|Any CPU.Build.0 = Debug|Any CPU
{88625045-978F-417F-9F51-A4E3A9718945}.Release|Any CPU.ActiveCfg = Release|Any CPU
Expand Down Expand Up @@ -221,6 +239,9 @@ Global
{A102AB7B-A90C-4717-B17C-045240838060} = {4827B3EC-73D8-436D-AE2A-5E29AC95FD0C}
{908F2CED-CAC0-4A4E-AD19-362A413B5DA4} = {C526E8AB-739A-48D7-8FC4-048978C9B650}
{912A4F2F-1BD1-4AE2-BAB8-5A49C221DB53} = {95A69671-16CA-4133-981C-CC381B7AAA30}
{3F8840BA-4F91-4359-AA53-6B26823E7F55} = {4827B3EC-73D8-436D-AE2A-5E29AC95FD0C}
{BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E} = {C526E8AB-739A-48D7-8FC4-048978C9B650}
{51882883-A66E-4F95-A1AB-CFCBF71B4376} = {95A69671-16CA-4133-981C-CC381B7AAA30}
{88625045-978F-417F-9F51-A4E3A9718945} = {95A69671-16CA-4133-981C-CC381B7AAA30}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
Expand Down
1 change: 1 addition & 0 deletions NATS.Client.sln.DotSettings
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=JS/@EntryIndexedValue">JS</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=KV/@EntryIndexedValue">KV</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=LF/@EntryIndexedValue">LF</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=OB/@EntryIndexedValue">OB</s:String>
mtmk marked this conversation as resolved.
Show resolved Hide resolved
<s:Boolean x:Key="/Default/UserDictionary/Words/=HMSG/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=HPUB/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Msgs/@EntryIndexedValue">True</s:Boolean>
Expand Down
15 changes: 15 additions & 0 deletions sandbox/Example.ObjectStore/Example.ObjectStore.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="..\..\src\NATS.Client.ObjectStore\NATS.Client.ObjectStore.csproj" />
</ItemGroup>

</Project>
41 changes: 41 additions & 0 deletions sandbox/Example.ObjectStore/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
using System.Text;
using NATS.Client.Core;
using NATS.Client.JetStream;
using NATS.Client.ObjectStore;
using NATS.Client.ObjectStore.Models;

var nats = new NatsConnection();
var js = new NatsJSContext(nats);
var ob = new NatsOBContext(js);

var store = await ob.CreateObjectStore(new NatsOBConfig("o1"));

var meta = new ObjectMetadata { Name = "k1", Options = new Options { MaxChunkSize = 10 }, };

var stringBuilder = new StringBuilder();
for (var i = 0; i < 9; i++)
{
stringBuilder.Append($"{i:D2}-4567890");
}

var buffer90 = stringBuilder.ToString();
{
var buffer = Encoding.ASCII.GetBytes(buffer90);
var stream = new MemoryStream(buffer);

await store.PutAsync(meta, stream);

var data = await store.GetInfoAsync("k1");

Console.WriteLine($"DATA: {data}");
}

{
var memoryStream = new MemoryStream();
await store.GetAsync("k1", memoryStream);
await memoryStream.FlushAsync();
var buffer = memoryStream.ToArray();
Console.WriteLine($"buffer:{Encoding.ASCII.GetString(buffer)}");
}

Console.WriteLine("Bye");
19 changes: 17 additions & 2 deletions src/NATS.Client.Core/INatsSerializer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,21 @@ public interface ICountableBufferWriter : IBufferWriter<byte>
int WrittenCount { get; }
}

public readonly struct FixedSizeMemoryOwner : IMemoryOwner<byte>
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Out of curiosity, what is the benefit of using this over MemoryPool<byte>.Shared ?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

MemoryPool.Shared.Rent returns at least the size asked for, which usually rounds up to some 2^n. This wrapper chops the memory to exact size so when it's passed to stream read/write or down the line to publish or something you won't need to pass the actual size.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

btw, we are still renting it from MemoryPool.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

btw, we are still renting it from MemoryPool.

Ah, I see that now. Clever!

{
private readonly IMemoryOwner<byte> _owner;

public FixedSizeMemoryOwner(IMemoryOwner<byte> owner, int size)
{
_owner = owner;
Memory = _owner.Memory.Slice(0, size);
}

public Memory<byte> Memory { get; }

public void Dispose() => _owner.Dispose();
}

public static class NatsDefaultSerializer
{
public static readonly INatsSerializer Default = new NatsRawSerializer(NatsJsonSerializer.Default);
Expand Down Expand Up @@ -105,9 +120,9 @@ public int Serialize<T>(ICountableBufferWriter bufferWriter, T? value)

if (typeof(T) == typeof(IMemoryOwner<byte>))
{
var memoryOwner = MemoryPool<byte>.Shared.Rent((int)buffer.Length);
var memoryOwner = new FixedSizeMemoryOwner(MemoryPool<byte>.Shared.Rent((int)buffer.Length), (int)buffer.Length);
buffer.CopyTo(memoryOwner.Memory.Span);
return (T)memoryOwner;
return (T)(object)memoryOwner;
}

if (Next != null)
Expand Down
2 changes: 2 additions & 0 deletions src/NATS.Client.Core/NATS.Client.Core.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,7 @@
<InternalsVisibleTo Include="NATS.Client.JetStream.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db7da1f2f89089327b47d26d69666fad20861f24e9acdb13965fb6c64dfee8da589b495df37a75e934ddbacb0752a42c40f3dbc79614eec9bb2a0b6741f9e2ad2876f95e74d54c23eef0063eb4efb1e7d824ee8a695b647c113c92834f04a3a83fb60f435814ddf5c4e5f66a168139c4c1b1a50a3e60c164d180e265b1f000cd" />
<InternalsVisibleTo Include="NATS.Client.KeyValueStore, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db7da1f2f89089327b47d26d69666fad20861f24e9acdb13965fb6c64dfee8da589b495df37a75e934ddbacb0752a42c40f3dbc79614eec9bb2a0b6741f9e2ad2876f95e74d54c23eef0063eb4efb1e7d824ee8a695b647c113c92834f04a3a83fb60f435814ddf5c4e5f66a168139c4c1b1a50a3e60c164d180e265b1f000cd" />
<InternalsVisibleTo Include="NATS.Client.KeyValueStore.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db7da1f2f89089327b47d26d69666fad20861f24e9acdb13965fb6c64dfee8da589b495df37a75e934ddbacb0752a42c40f3dbc79614eec9bb2a0b6741f9e2ad2876f95e74d54c23eef0063eb4efb1e7d824ee8a695b647c113c92834f04a3a83fb60f435814ddf5c4e5f66a168139c4c1b1a50a3e60c164d180e265b1f000cd" />
<InternalsVisibleTo Include="NATS.Client.ObjectStore, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db7da1f2f89089327b47d26d69666fad20861f24e9acdb13965fb6c64dfee8da589b495df37a75e934ddbacb0752a42c40f3dbc79614eec9bb2a0b6741f9e2ad2876f95e74d54c23eef0063eb4efb1e7d824ee8a695b647c113c92834f04a3a83fb60f435814ddf5c4e5f66a168139c4c1b1a50a3e60c164d180e265b1f000cd" />
<InternalsVisibleTo Include="NATS.Client.ObjectStore.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db7da1f2f89089327b47d26d69666fad20861f24e9acdb13965fb6c64dfee8da589b495df37a75e934ddbacb0752a42c40f3dbc79614eec9bb2a0b6741f9e2ad2876f95e74d54c23eef0063eb4efb1e7d824ee8a695b647c113c92834f04a3a83fb60f435814ddf5c4e5f66a168139c4c1b1a50a3e60c164d180e265b1f000cd" />
</ItemGroup>
</Project>
Loading