Added tests at the end of build process, updated containers and comma…

…nd line program (#481) * Added tests at the end of build process * Reduced execution time simplifying project compilation * Update to run test outside the containing folder * Compilation will happen only using latest unpublished version of JNetReflector * Report fails without application crash * Offset reset shall be dependent from the run kind, without parallel cannot be latest otherwise never exit from the consume loop * Update the way KNet container starts * Command issued can be written in any form, the code look-up always in lower case format * Use the first argument if ClassToRun is not set and there are residual command line arguments
masesgroup · Jun 7, 2024 · aa2ef03 · aa2ef03
1 parent 280e9f7
commit aa2ef03
Show file tree

Hide file tree

Showing 27 changed files with 929 additions and 142 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1,2 +1,5 @@
 *.psd1 diff
-*.psm1 diff
+*.psm1 diff
+
+*.sh text eol=lf
+*.yaml text eol=auto
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -132,11 +132,10 @@ jobs:
           fetch-depth: '1'
           submodules: 'true'
 
-      - name: Pre compile tests
-        run: dotnet build --no-incremental --configuration Release /p:Platform="Any CPU" /p:NoWarn="0108%3B1030%3B0618" tests\net\KNetTest.sln
-
       - name: Pre compile
         run: dotnet build --no-incremental --configuration Release /p:Platform="Any CPU" /p:NoWarn="0108%3B1030%3B0618" src\net\KNet\KNet.csproj
+        env:
+          GITHUB_SIMPLIFIED_GENERATION: true
 
       - name: Set up Apache Maven Central
         uses: actions/setup-java@v4
@@ -175,7 +174,7 @@ jobs:
 
       - name: Recompile to create nuget packages
         run: dotnet build --no-incremental --configuration Release /p:Platform="Any CPU" /p:NoWarn="0108%3B1030%3B0618" src\net\KNet.sln
-
+          
       - name: Prepare PowerShell package
         run: |
           Copy-Item .\src\net\KNetPS\MASES.KNetPS.psd1 -Destination .\MASES.KNetPS\MASES.KNetPS.psd1 -Force
@@ -192,4 +191,156 @@ jobs:
       - uses: actions/upload-artifact@v4
         with:
           name: KNet
-          path: .\bin\*nupkg
+          path: .\bin\*nupkg
+
+      - name: Compile tests
+        run: dotnet build --no-incremental --configuration Release /p:Platform="Any CPU" /p:NoWarn="0108%3B1030%3B0618" tests\net\KNetTest.sln
+        env:
+          GITHUB_TEST_PREPARATION: true
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: KNet_bin_${{ github.sha }}
+          path: .\bin\*
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: KNet_jars_${{ github.sha }}
+          path: .\jars\*
+
+      - name: Save KNet bin in cache
+        uses: actions/cache/save@v4
+        with:
+          enableCrossOsArchive: true
+          path: ./bin/
+          key: KNet_bin_${{ github.sha }}
+
+  build_container_knettest:
+    needs: check_changes
+    if: "always() && needs.check_changes.outputs.run_build_windows == 'true'"
+    runs-on: ubuntu-latest
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:    
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: '1'
+          submodules: 'true'
+
+      - name: Pre compile
+        run: dotnet build --no-incremental --configuration Release /p:Platform="Any CPU" /p:NoWarn="0108%3B1030%3B0618" src/net/KNet/KNet.csproj
+        env:
+          GITHUB_SIMPLIFIED_GENERATION: true
+
+      - name: Set up Apache Maven Central
+        uses: actions/setup-java@v4
+        with: # running setup-java again overwrites the settings.xml
+          distribution: temurin
+          java-version: 11
+          cache: 'maven'
+          server-id: ossrh # Value of the distributionManagement/repository/id field of the pom.xml
+          server-username: MAVEN_USERNAME # env variable for username in deploy
+          server-password: MAVEN_CENTRAL_TOKEN # env variable for token in deploy
+          gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} # Value of the GPG private key to import
+          gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase
+
+      - name: Create Jars
+        run: mvn --file ./src/jvm/knet/pom.xml --no-transfer-progress package
+        env:
+          MAVEN_USERNAME: ${{ secrets.MAVEN_USERNAME }}
+          MAVEN_CENTRAL_TOKEN: ${{ secrets.MAVEN_CENTRAL_TOKEN }}
+          MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ghcr.io/${{ github.repository_owner }}/knettest
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          file: ./src/container/DockerfileKNet.linux
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  execute_tests:
+    needs: [build_windows, build_container_knettest]
+    services:
+      kafka:
+        # Private registry image
+        image: ghcr.io/${{ github.repository_owner }}/knettest:${{ github.ref_name }}
+        credentials:
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+        ports:
+          - 9092:9092
+        env:
+          KNET_RUNNING_MODE: standalone
+          JCOBRIDGE_LicensePath: ${{ secrets.JCOBRIDGE_ONLINE }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ 'ubuntu-latest' ] #, 'windows-latest' ]
+        framework: [ 'net462', 'net6.0', 'net8.0' ]
+        buffered: [ 'runBuffered', '']
+        extraValue: [ '', 'withBigExtraValue', 'withBigBigExtraValue' ]
+        jdk_vendor: [ 'temurin', 'zulu', 'microsoft', 'corretto', 'oracle']
+        jdk_version: [ '11', '17', '21' ] # only LTS versions
+        exclude:
+          - os: ubuntu-latest
+            framework: net462
+          - jdk_vendor: oracle
+            jdk_version: 11
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Restore KNet bin from cache
+        uses: actions/cache/restore@v4
+        with:
+          fail-on-cache-miss: true
+          enableCrossOsArchive: true
+          path: ./bin/
+          key: KNet_bin_${{ github.sha }}
+
+      - name: Set up JDK distribution
+        uses: actions/setup-java@v4
+        with: # running setup-java again overwrites the settings.xml
+          distribution: ${{ matrix.jdk_vendor }}
+          java-version: ${{ matrix.jdk_version }}
+
+      - name: Executing KNetTest on Ubuntu with ${{ matrix.jdk_vendor }} ${{ matrix.jdk_version }}
+        if: matrix.os == 'ubuntu-latest'
+        run: dotnet ./bin/${{ matrix.framework }}/KNetTest.dll localhost:9092 randomizeTopicName ${{ matrix.buffered }} ${{ matrix.extraValue }}
+        env:
+          JCOBRIDGE_LicensePath: ${{ secrets.JCOBRIDGE_ONLINE }}
+
+      - name: Executing KNetTest on Windows with ${{ matrix.jdk_vendor }} ${{ matrix.jdk_version }}
+        if: ${{ matrix.os == 'windows-latest' && matrix.framework != 'net462' }}
+        run: dotnet .\bin\${{ matrix.framework }}\KNetTest.dll localhost:9092 randomizeTopicName ${{ matrix.buffered }} ${{ matrix.extraValue }}
+        env:
+          JCOBRIDGE_LicensePath: ${{ secrets.JCOBRIDGE_ONLINE }}
+
+      - name: Executing KNetTest on Windows with ${{ matrix.jdk_vendor }} ${{ matrix.jdk_version }}
+        if: ${{ matrix.os == 'windows-latest' && matrix.framework == 'net462' }}
+        run: .\bin\${{ matrix.framework }}\KNetTest.exe localhost:9092 randomizeTopicName ${{ matrix.buffered }} ${{ matrix.extraValue }}
+        env:
+          JCOBRIDGE_LicensePath: ${{ secrets.JCOBRIDGE_ONLINE }}
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -115,7 +115,9 @@ jobs:
     - name: Maven preparation (step 1)
       if: matrix.language == 'java' 
       run: dotnet build --no-incremental --configuration Release --framework net8.0 /p:Platform="Any CPU" /p:NoWarn="0108%3B1030%3B0618" ./src/net/KNet/KNet.csproj
-
+      env:
+        GITHUB_SIMPLIFIED_GENERATION: true
+
     - if: matrix.language == 'java' 
       run: mvn --file ./src/jvm/knet/pom.xml --no-transfer-progress package
 

diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
@@ -25,7 +25,9 @@ jobs:
 
       - name: Pre compile
         run: dotnet build --no-incremental --configuration Release /p:Platform="Any CPU" /p:NoWarn="0108%3B1030%3B0618" src/net/KNet/KNet.csproj
-
+        env:
+          GITHUB_SIMPLIFIED_GENERATION: true
+
       - name: Set up Apache Maven Central
         uses: actions/setup-java@v4
         with: # running setup-java again overwrites the settings.xml
@@ -94,7 +96,9 @@ jobs:
 
       - name: Pre compile
         run: dotnet build --no-incremental --configuration Release /p:Platform="Any CPU" /p:NoWarn="0108%3B1030%3B0618" src/net/KNet/KNet.csproj
-
+        env:
+          GITHUB_SIMPLIFIED_GENERATION: true
+
       - name: Set up Apache Maven Central
         uses: actions/setup-java@v4
         with: # running setup-java again overwrites the settings.xml

diff --git a/.github/workflows/generateclasses.yaml b/.github/workflows/generateclasses.yaml
@@ -45,7 +45,9 @@ jobs:
 
       - name: Pre compile
         run: dotnet build --no-incremental --configuration Release /p:Platform="Any CPU" /p:NoWarn="0108%3B1030%3B0618" src\net\KNet\KNet.csproj
-
+        env:
+          GITHUB_SIMPLIFIED_GENERATION: true
+
       - name: Set up Apache Maven Central
         uses: actions/setup-java@v4
         with: # running setup-java again overwrites the settings.xml
@@ -103,6 +105,10 @@ jobs:
         env:
           JCOBRIDGE_LicensePath: ${{ secrets.JCOBRIDGE_ONLINE }}
 
+      - name: Try compilation
+        if: ${{ inputs.UseLatestJNetReflector == true }}
+        run: dotnet build --no-incremental --configuration Release /p:Platform="Any CPU" /p:NoWarn="0108%3B1030%3B0618" src\net\KNet\KNet.csproj
+
       - name: Request a PR to commit changes made with unpublished version of JNetReflector
         if: ${{ github.repository_owner == 'masesgroup' && inputs.GeneratePR == true && inputs.UseLatestJNetReflector == true }} #do not push any changes outside main repo or GeneratePR is false
         uses: peter-evans/create-pull-request@v6

diff --git a/.github/workflows/pullrequest.yaml b/.github/workflows/pullrequest.yaml
@@ -67,7 +67,9 @@ jobs:
 
       - name: Pre compile
         run: dotnet build --no-incremental --configuration Release /p:Platform="Any CPU" /p:NoWarn="0108%3B1030%3B0618" src/net/KNet/KNet.csproj
-
+        env:
+          GITHUB_SIMPLIFIED_GENERATION: true
+
       - name: Set up Apache Maven Central
         uses: actions/setup-java@v4
         with: # running setup-java again overwrites the settings.xml
@@ -117,7 +119,9 @@ jobs:
 
       - name: Pre compile
         run: dotnet build --no-incremental --configuration Release /p:Platform="Any CPU" /p:NoWarn="0108%3B1030%3B0618" src\net\KNet\KNet.csproj
-
+        env:
+          GITHUB_SIMPLIFIED_GENERATION: true
+
       - name: Set up Apache Maven Central
         uses: actions/setup-java@v4
         with: # running setup-java again overwrites the settings.xml

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -31,7 +31,9 @@ jobs:
 
       - name: Pre compile
         run: dotnet build --no-incremental --configuration Release /p:Platform="Any CPU" /p:NoWarn="0108%3B1030%3B0618" src\net\KNet\KNet.csproj
-
+        env:
+          GITHUB_SIMPLIFIED_GENERATION: true
+
       - name: Set up Apache Maven Central
         uses: actions/setup-java@v4
         with: # running setup-java again overwrites the settings.xml

diff --git a/src/container/DockerfileKNet.linux b/src/container/DockerfileKNet.linux
@@ -8,20 +8,35 @@ ENV KNET_DOCKER_BUILD_ACTIONS=true
 # Restore as distinct layers
 RUN dotnet restore KNetDocker.sln -a $TARGETARCH
 # Build and publish a release
-RUN dotnet publish ./KNetCLI/KNetCLI.csproj --framework net8.0 -c Release -o out -a $TARGETARCH
+RUN dotnet publish ./KNetCLI/KNetCLI.csproj /p:NoWarn="0108%3B1030%3B0618" --framework net8.0 -c Release -o out -a $TARGETARCH
 
 # Build runtime image
 FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/runtime:8.0-jammy
 
+LABEL org.opencontainers.image.authors='https://github.com/masesgroup', \
+      org.opencontainers.image.url='https://knet.masesgroup.com' \
+      org.opencontainers.image.documentation='https://knet.masesgroup.com' \
+      org.opencontainers.image.source='https://github.com/masesgroup/KNet' \
+      org.opencontainers.image.vendor='MASES Group' \
+      org.opencontainers.image.licenses='Apache-2.0' \
+      org.opencontainers.image.description="KNet Server-Client container"
+
 # Add JRE
 RUN apt-get update && apt-get install -y --no-install-recommends openjdk-17-jre-headless && rm -rf /var/lib/apt/lists/*
 
 ADD ./jars /app/jars
 ADD ./src/config /app/config
+ADD ./src/config/kraft /app/config/kraft
+ADD ./src/container/config_container /app/config_container
+ADD ./src/container/KNetRun.sh /app
 
 ENV JCOBRIDGE_JVMPath=/usr/lib/jvm/java-17-openjdk-amd64/lib/server/libjvm.so
 ENV JCOBRIDGE_LicensePath=
 
+EXPOSE 2181
+EXPOSE 9092
+
 WORKDIR /app
 COPY --from=build-env /app/out .
-ENTRYPOINT ["dotnet", "MASES.KNetCLI.dll"]
+RUN chmod +x /app/KNetRun.sh
+CMD /app/KNetRun.sh
diff --git a/src/container/DockerfileKNetConnect.linux b/src/container/DockerfileKNetConnect.linux
@@ -8,14 +8,22 @@ ENV KNET_DOCKER_BUILD_ACTIONS=true
 # Restore as distinct layers
 RUN dotnet restore KNetDocker.sln -a $TARGETARCH
 # Build and publish a release
-RUN dotnet publish ./KNetConnect/KNetConnect.csproj --framework net8.0 -c Release -o out -a $TARGETARCH
+RUN dotnet publish ./KNetConnect/KNetConnect.csproj /p:NoWarn="0108%3B1030%3B0618" --framework net8.0 -c Release -o out -a $TARGETARCH
 
 # Build runtime image
 FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/runtime:8.0-jammy
 
 # Add JRE
 RUN apt-get update && apt-get install -y --no-install-recommends openjdk-17-jre-headless && rm -rf /var/lib/apt/lists/*
 
+LABEL org.opencontainers.image.authors='https://github.com/masesgroup', \
+      org.opencontainers.image.url='https://knet.masesgroup.com' \
+      org.opencontainers.image.documentation='https://knet.masesgroup.com' \
+      org.opencontainers.image.source='https://github.com/masesgroup/KNet' \
+      org.opencontainers.image.vendor='MASES Group' \
+      org.opencontainers.image.licenses='Apache-2.0' \
+      org.opencontainers.image.description="KNet Connect container"
+
 ADD ./jars /app/jars
 ADD ./src/config /app/config
 

diff --git a/src/container/DockerfileKNetTest.linux b/src/container/DockerfileKNetTest.linux
@@ -0,0 +1,43 @@
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-jammy AS build-env
+ARG TARGETARCH
+WORKDIR /app
+
+# Copy everything
+COPY ./src/net ./
+ENV KNET_DOCKER_BUILD_ACTIONS=true
+# Restore as distinct layers
+RUN dotnet restore KNetDocker.sln -a $TARGETARCH
+# Build and publish a release
+RUN dotnet publish ./KNetCLI/KNetCLI.csproj --framework net8.0 -c Release -o out -a $TARGETARCH
+
+# Build runtime image
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/runtime:8.0-jammy
+
+LABEL org.opencontainers.image.authors='https://github.com/masesgroup', \
+      org.opencontainers.image.url='https://knet.masesgroup.com' \
+      org.opencontainers.image.documentation='https://knet.masesgroup.com' \
+      org.opencontainers.image.source='https://github.com/masesgroup/KNet' \
+      org.opencontainers.image.vendor='MASES Group' \
+      org.opencontainers.image.licenses='Apache-2.0' \
+      org.opencontainers.image.description="KNet Test container"
+
+# Add JRE
+RUN apt-get update && apt-get install -y --no-install-recommends openjdk-17-jre-headless && rm -rf /var/lib/apt/lists/*
+
+ADD ./jars /app/jars
+ADD ./src/config /app/config
+ADD ./src/container/KNetTestRun.sh /app
+ADD ./src/container/zookeeper.properties /app
+ADD ./src/container/server.properties /app
+ADD ./src/container/log4j.properties /app
+
+ENV JCOBRIDGE_JVMPath=/usr/lib/jvm/java-17-openjdk-amd64/lib/server/libjvm.so
+ENV JCOBRIDGE_LicensePath=
+
+EXPOSE 2181
+EXPOSE 9092
+
+WORKDIR /app
+COPY --from=build-env /app/out .
+RUN chmod +x /app/KNetTestRun.sh
+CMD /app/KNetTestRun.sh