Skip to content

Commit

Permalink
Update test execution (#485)
Browse files Browse the repository at this point in the history 
* Fixed #476

* Added read-made property values to avoid string conversion during headers add invocation

* Documentation update
  • Loading branch information
@masesdevelopers
masesdevelopers authored Jun 11, 2024
1 parent aa2ef03 commit b9d7372
Show file tree
Hide file tree
Showing 13 changed files with 100 additions and 121 deletions.
38 changes: 15 additions & 23 deletions .github/workflows/build.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,16 +198,6 @@ jobs:
env:
GITHUB_TEST_PREPARATION: true

- uses: actions/upload-artifact@v4
with:
name: KNet_bin_${{ github.sha }}
path: .\bin\*

- uses: actions/upload-artifact@v4
with:
name: KNet_jars_${{ github.sha }}
path: .\jars\*

- name: Save KNet bin in cache
uses: actions/cache/save@v4
with:
Expand Down Expand Up @@ -281,7 +271,7 @@ jobs:
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

execute_tests:
execute_KNetTest:
needs: [build_windows, build_container_knettest]
services:
kafka:
Expand Down Expand Up @@ -332,15 +322,17 @@ jobs:
run: dotnet ./bin/${{ matrix.framework }}/KNetTest.dll localhost:9092 randomizeTopicName ${{ matrix.buffered }} ${{ matrix.extraValue }}
env:
JCOBRIDGE_LicensePath: ${{ secrets.JCOBRIDGE_ONLINE }}

- name: Executing KNetTest on Windows with ${{ matrix.jdk_vendor }} ${{ matrix.jdk_version }}
if: ${{ matrix.os == 'windows-latest' && matrix.framework != 'net462' }}
run: dotnet .\bin\${{ matrix.framework }}\KNetTest.dll localhost:9092 randomizeTopicName ${{ matrix.buffered }} ${{ matrix.extraValue }}
env:
JCOBRIDGE_LicensePath: ${{ secrets.JCOBRIDGE_ONLINE }}

- name: Executing KNetTest on Windows with ${{ matrix.jdk_vendor }} ${{ matrix.jdk_version }}
if: ${{ matrix.os == 'windows-latest' && matrix.framework == 'net462' }}
run: .\bin\${{ matrix.framework }}\KNetTest.exe localhost:9092 randomizeTopicName ${{ matrix.buffered }} ${{ matrix.extraValue }}
env:
JCOBRIDGE_LicensePath: ${{ secrets.JCOBRIDGE_ONLINE }}

# Following shall be disabled since not supported from GitHub
#
#- name: Executing KNetTest on Windows with ${{ matrix.jdk_vendor }} ${{ matrix.jdk_version }}
# if: ${{ matrix.os == 'windows-latest' && matrix.framework != 'net462' }}
# run: dotnet .\bin\${{ matrix.framework }}\KNetTest.dll localhost:9092 randomizeTopicName ${{ matrix.buffered }} ${{ matrix.extraValue }}
# env:
# JCOBRIDGE_LicensePath: ${{ secrets.JCOBRIDGE_ONLINE }}
#
#- name: Executing KNetTest on Windows with ${{ matrix.jdk_vendor }} ${{ matrix.jdk_version }}
# if: ${{ matrix.os == 'windows-latest' && matrix.framework == 'net462' }}
# run: .\bin\${{ matrix.framework }}\KNetTest.exe localhost:9092 randomizeTopicName ${{ matrix.buffered }} ${{ matrix.extraValue }}
# env:
# JCOBRIDGE_LicensePath: ${{ secrets.JCOBRIDGE_ONLINE }}
43 changes: 0 additions & 43 deletions src/container/DockerfileKNetTest.linux
View file

This file was deleted.

4 changes: 2 additions & 2 deletions src/container/KNetRun.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,8 @@ else
fi

if [[ $env_var =~ ^ZOOKEEPER_ ]]; then
kafka_name=$(echo "$env_var" | cut -d_ -f2- | tr '[:upper:]' '[:lower:]' | tr _ .)
updateConfig "$kafka_name" "${!env_var}" "/app/config_container/server.properties"
zookeeper_name=$(echo "$env_var" | cut -d_ -f2- | tr '[:upper:]' '[:lower:]' | tr _ .)
updateConfig "$zookeeper_name" "${!env_var}" "/app/config_container/zookeeper.properties"
fi

if [[ $env_var =~ ^LOG4J_ ]]; then
Expand Down
18 changes: 16 additions & 2 deletions src/documentation/articles/docker.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,22 @@ _description: Describes the Docker images of .NET suite for Apache Kafka
# KNet: Docker images

.NET suite for Apache Kafka comes with some ready made Docker images:
- KNet CLI: see [KNet CLI usage](usageCLI.md)
- KNet Connect: see [KNet Connect usage](usageConnect.md)
- _knet_:
- it is a CLI invocable from Docker with the following command (see [KNet CLI usage](usageCLI.md) for command line information):
> docker run -it knet [commands]
- it supports server mode execution if the environment variable **KNET_RUNNING_MODE** reports the following values:
- **zookeeper**: starts a zookeeper node, defaults to run a standalone zookeeper exposing on port 2181
- **broker**: starts an Apache Kafka node, defaults to run a standalone broker exposing on port 9092
- **standalone**: starts, within the container, both a zookeeper node and an Apache Kafka node, it defaults to run exposing zookeeper on port 2181 and Kafka on port 9092

All kafka properties can be updated using environment variables in the following form:
- Shall start with **KAFKA_**
- Each property shall be capitalized and the '.' shall be replaced with '_'
- The value of the environment variable is the value will be used in the configuration file

As an example: the environment variable **KAFKA_ADVERTISED_LISTENERS** represents **advertised.listeners**.

- _knetconnect_: it is the KNet Connect invocable from Docker, see [KNet Connect usage](usageConnect.md) for command line information

> [!IMPORTANT]
> The Docker images produced from this repository uses, as base image, the one available at mcr.microsoft.com/dotnet/runtime:6.0 (latest is mcr.microsoft.com/dotnet/runtime:6.0-jammy). Within [GitHub Container Registry](https://github.com/microsoft/containerregistry) you can find many legal information, specifically read carefully the following before use the image:
Expand Down
32 changes: 16 additions & 16 deletions src/net/KNet.Serialization.Avro/AvroSerDes.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,8 +161,8 @@ public override byte[] Serialize(string topic, TData data)
/// <inheritdoc cref="SerDes{T, TJVMT}.SerializeWithHeaders(string, Headers, T)"/>
public override byte[] SerializeWithHeaders(string topic, Headers headers, TData data)
{
headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName);
headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName);
headers?.Add(KNetSerialization.KeyTypeIdentifierJVM, keyTypeName);
headers?.Add(KNetSerialization.KeySerializerIdentifierJVM, keySerDesName);

using MemoryStream memStream = new();
BinaryEncoder encoder = new(memStream);
Expand Down Expand Up @@ -231,8 +231,8 @@ public override Java.Nio.ByteBuffer Serialize(string topic, TData data)
/// <inheritdoc cref="SerDes{T, TJVMT}.SerializeWithHeaders(string, Headers, T)"/>
public override Java.Nio.ByteBuffer SerializeWithHeaders(string topic, Headers headers, TData data)
{
headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName);
headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName);
headers?.Add(KNetSerialization.KeyTypeIdentifierJVM, keyTypeName);
headers?.Add(KNetSerialization.KeySerializerIdentifierJVM, keySerDesName);

MemoryStream memStream = new();
BinaryEncoder encoder = new(memStream);
Expand Down Expand Up @@ -342,8 +342,8 @@ public override byte[] Serialize(string topic, TData data)
/// <inheritdoc cref="SerDes{T, TJVMT}.SerializeWithHeaders(string, Headers, T)"/>
public override byte[] SerializeWithHeaders(string topic, Headers headers, TData data)
{
headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName);
headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName);
headers?.Add(KNetSerialization.KeyTypeIdentifierJVM, keyTypeName);
headers?.Add(KNetSerialization.KeySerializerIdentifierJVM, keySerDesName);

using MemoryStream memStream = new();
JsonEncoder encoder = new(Schema, memStream);
Expand Down Expand Up @@ -412,8 +412,8 @@ public override Java.Nio.ByteBuffer Serialize(string topic, TData data)
/// <inheritdoc cref="SerDes{T, TJVMT}.SerializeWithHeaders(string, Headers, T)"/>
public override Java.Nio.ByteBuffer SerializeWithHeaders(string topic, Headers headers, TData data)
{
headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName);
headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName);
headers?.Add(KNetSerialization.KeyTypeIdentifierJVM, keyTypeName);
headers?.Add(KNetSerialization.KeySerializerIdentifierJVM, keySerDesName);

MemoryStream memStream = new();
JsonEncoder encoder = new(Schema, memStream);
Expand Down Expand Up @@ -530,8 +530,8 @@ public override byte[] Serialize(string topic, TData data)
/// <inheritdoc cref="SerDes{T, TJVMT}.SerializeWithHeaders(string, Headers, T)"/>
public override byte[] SerializeWithHeaders(string topic, Headers headers, TData data)
{
headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueSerDesName);
headers?.Add(KNetSerialization.ValueTypeIdentifier, valueTypeName);
headers?.Add(KNetSerialization.ValueSerializerIdentifierJVM, valueSerDesName);
headers?.Add(KNetSerialization.ValueTypeIdentifierJVM, valueTypeName);

MemoryStream memStream = new();
BinaryEncoder encoder = new(memStream);
Expand Down Expand Up @@ -600,8 +600,8 @@ public override Java.Nio.ByteBuffer Serialize(string topic, TData data)
/// <inheritdoc cref="SerDes{T, TJVMT}.SerializeWithHeaders(string, Headers, T)"/>
public override Java.Nio.ByteBuffer SerializeWithHeaders(string topic, Headers headers, TData data)
{
headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueSerDesName);
headers?.Add(KNetSerialization.ValueTypeIdentifier, valueTypeName);
headers?.Add(KNetSerialization.ValueSerializerIdentifierJVM, valueSerDesName);
headers?.Add(KNetSerialization.ValueTypeIdentifierJVM, valueTypeName);

MemoryStream memStream = new();
BinaryEncoder encoder = new(memStream);
Expand Down Expand Up @@ -712,8 +712,8 @@ public override byte[] Serialize(string topic, TData data)
/// <inheritdoc cref="SerDes{T, TJVMT}.SerializeWithHeaders(string, Headers, T)"/>
public override byte[] SerializeWithHeaders(string topic, Headers headers, TData data)
{
headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueSerDesName);
headers?.Add(KNetSerialization.ValueTypeIdentifier, valueTypeName);
headers?.Add(KNetSerialization.ValueSerializerIdentifierJVM, valueSerDesName);
headers?.Add(KNetSerialization.ValueTypeIdentifierJVM, valueTypeName);

using MemoryStream memStream = new();
JsonEncoder encoder = new(Schema, memStream);
Expand Down Expand Up @@ -782,8 +782,8 @@ public override Java.Nio.ByteBuffer Serialize(string topic, TData data)
/// <inheritdoc cref="SerDes{T, TJVMT}.SerializeWithHeaders(string, Headers, T)"/>
public override Java.Nio.ByteBuffer SerializeWithHeaders(string topic, Headers headers, TData data)
{
headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueSerDesName);
headers?.Add(KNetSerialization.ValueTypeIdentifier, valueTypeName);
headers?.Add(KNetSerialization.ValueSerializerIdentifierJVM, valueSerDesName);
headers?.Add(KNetSerialization.ValueTypeIdentifierJVM, valueTypeName);

MemoryStream memStream = new();
JsonEncoder encoder = new(Schema, memStream);
Expand Down
16 changes: 8 additions & 8 deletions src/net/KNet.Serialization.Json/JsonSerDes.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,8 @@ public override byte[] Serialize(string topic, TData data)
/// <inheritdoc cref="SerDes{TData, TJVMT}.SerializeWithHeaders(string, Headers, TData)"/>
public override byte[] SerializeWithHeaders(string topic, Headers headers, TData data)
{
headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName);
headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName);
headers?.Add(KNetSerialization.KeyTypeIdentifierJVM, keyTypeName);
headers?.Add(KNetSerialization.KeySerializerIdentifierJVM, keySerDesName);

if (_defaultSerDes != null) return _defaultSerDes.SerializeWithHeaders(topic, headers, data);

Expand Down Expand Up @@ -212,8 +212,8 @@ public override Java.Nio.ByteBuffer Serialize(string topic, TData data)
/// <inheritdoc cref="SerDes{TData, TJVMT}.SerializeWithHeaders(string, Headers, TData)"/>
public override Java.Nio.ByteBuffer SerializeWithHeaders(string topic, Headers headers, TData data)
{
headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName);
headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName);
headers?.Add(KNetSerialization.KeyTypeIdentifierJVM, keyTypeName);
headers?.Add(KNetSerialization.KeySerializerIdentifierJVM, keySerDesName);

if (_defaultSerDes != null) return _defaultSerDes.SerializeWithHeaders(topic, headers, data);

Expand Down Expand Up @@ -357,8 +357,8 @@ public override byte[] Serialize(string topic, TData data)
/// <inheritdoc cref="SerDes{TData, TJVMT}.SerializeWithHeaders(string, Headers, TData)"/>
public override byte[] SerializeWithHeaders(string topic, Headers headers, TData data)
{
headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueSerDesName);
headers?.Add(KNetSerialization.ValueTypeIdentifier, valueTypeName);
headers?.Add(KNetSerialization.ValueSerializerIdentifierJVM, valueSerDesName);
headers?.Add(KNetSerialization.ValueTypeIdentifierJVM, valueTypeName);

if (_defaultSerDes != null) return _defaultSerDes.SerializeWithHeaders(topic, headers, data);

Expand Down Expand Up @@ -448,8 +448,8 @@ public override Java.Nio.ByteBuffer Serialize(string topic, TData data)
/// <inheritdoc cref="SerDes{TData, TJVMT}.SerializeWithHeaders(string, Headers, TData)"/>
public override Java.Nio.ByteBuffer SerializeWithHeaders(string topic, Headers headers, TData data)
{
headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueSerDesName);
headers?.Add(KNetSerialization.ValueTypeIdentifier, valueTypeName);
headers?.Add(KNetSerialization.ValueSerializerIdentifierJVM, valueSerDesName);
headers?.Add(KNetSerialization.ValueTypeIdentifierJVM, valueTypeName);

if (_defaultSerDes != null) return _defaultSerDes.SerializeWithHeaders(topic, headers, data);

Expand Down
2 changes: 1 addition & 1 deletion src/net/KNet.Serialization.MessagePack/KNet.Serialization.MessagePack.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
<PackageReference Include="System.Memory" Version="4.5.5" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="MessagePack" Version="2.5.140" />
<PackageReference Include="MessagePack" Version="2.5.168" />
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
Expand Down
Loading

0 comments on commit b9d7372

Please sign in to comment.