From 3c22bba6df3efe2f9a32179af42d5fae12a57a86 Mon Sep 17 00:00:00 2001 From: MASES Public Developers Team <94312179+masesdevelopers@users.noreply.github.com> Date: Mon, 23 Dec 2024 01:21:00 +0100 Subject: [PATCH] Update generateclasses.yaml and documentation (#822) * https://github.com/masesgroup/KNet/issues/820#issuecomment-2558227909: disable public source * https://github.com/masesgroup/KNet/issues/820#issuecomment-2558227909: try without shortcut * Revert back fe7e7c4a9b45657006679b5ba988d5f7b3f2a328 * https://github.com/masesgroup/KNet/issues/820#issuecomment-2558590370: Try disable CET * Add executable * Added logging to verify the faulting application * Fix process name * Update documentation * Update conditions --- .github/workflows/generateclasses.yaml | 38 ++++++++------ README.md | 2 +- src/documentation/articles/howtobuild.md | 6 +-- src/documentation/articles/usage.md | 61 +++++++++++++++++++--- src/documentation/articles/usageCLI.md | 8 ++- src/documentation/articles/usageConnect.md | 6 +++ src/documentation/index.md | 1 + 7 files changed, 94 insertions(+), 28 deletions(-) diff --git a/.github/workflows/generateclasses.yaml b/.github/workflows/generateclasses.yaml index 80d838bbe..cea8f3fbc 100644 --- a/.github/workflows/generateclasses.yaml +++ b/.github/workflows/generateclasses.yaml @@ -17,7 +17,7 @@ on: required: true default: false type: boolean - UseLatestJNetReflector: + UseLatestJNetReflectorSource: description: 'Shall use latest unpublished code of JNetReflector' required: true default: false @@ -56,7 +56,7 @@ jobs: submodules: 'true' - uses: actions/checkout@v4 - if: ${{ inputs.UseLatestJNetReflector == true }} + if: ${{ inputs.UseLatestJNetReflectorSource == true }} with: repository: masesgroup/JNet path: JNet @@ -110,14 +110,16 @@ jobs: - name: Add local repo if: ${{ success() && inputs.UseLastRepositoryBuild == true }} - run: dotnet nuget add source --name local "${{ github.workspace }}\JNetNuGet\" + run: | + dotnet nuget add source --name local "${{ github.workspace }}\JNetNuGet\" + dotnet nuget disable source nuget.org - name: Prebuild JNetReflector - if: ${{ inputs.UseLatestJNetReflector == true }} + if: ${{ inputs.UseLatestJNetReflectorSource == true }} run: dotnet build --no-incremental --configuration Release JNet\src\net\JNetReflector\JNetReflector.csproj - name: Create Jars of JNetReflector - if: ${{ inputs.UseLatestJNetReflector == true }} + if: ${{ inputs.UseLatestJNetReflectorSource == true }} run: mvn "-Djcobridgepath=../../../binReflector/net8.0/JCOBridge.jar" --file ./JNet/src/jvm/jnet/pom.xml --no-transfer-progress package env: MAVEN_USERNAME: ${{ secrets.MAVEN_USERNAME }} @@ -125,26 +127,32 @@ jobs: MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} - name: Build JNetReflector - if: ${{ inputs.UseLatestJNetReflector == true }} + if: ${{ inputs.UseLatestJNetReflectorSource == true }} run: dotnet build --no-incremental --configuration Release JNet\src\net\JNetReflector\JNetReflector.csproj - name: Use latest unpublished NuGet of JNetReflector - if: ${{ inputs.UseLastRepositoryBuild == true && inputs.UseLatestJNetReflector == false }} + if: ${{ inputs.UseLastRepositoryBuild == true && inputs.UseLatestJNetReflectorSource == false }} run: dotnet tool update -g MASES.JNetReflector - name: Download latest published version of JNetReflector - if: ${{ inputs.UseLatestJNetReflector == false }} + if: ${{ inputs.UseLastRepositoryBuild == false && inputs.UseLatestJNetReflectorSource == false }} run: dotnet tool update --version 2.5.11 -g MASES.JNetReflector - name: Build classes with unpublished JNetReflector - if: ${{ inputs.UseLatestJNetReflector == true }} + if: ${{ inputs.UseLatestJNetReflectorSource == true }} shell: cmd run: | cd JNet\binReflector\net8.0 MASES.JNetReflector.exe -TraceLevel 1 -TraceTo %GITHUB_WORKSPACE%\JNetReflector.txt -OriginRootPath %GITHUB_WORKSPACE%\jars -DestinationRootPath %GITHUB_WORKSPACE%\src\ -ConfigurationFile %GITHUB_WORKSPACE%\src\configuration.json + - name: Disable CET for dotnet process + if: ${{ inputs.UseLatestJNetReflectorSource == false }} + run: | + reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\jnetreflector.exe" /v MitigationOptions /t REG_BINARY /d "0000000000000000000000000000002000" /f + Get-ProcessMitigation -Name "jnetreflector.exe" + - name: Build classes with latest published version of JNetReflector - if: ${{ inputs.UseLatestJNetReflector == false }} + if: ${{ inputs.UseLatestJNetReflectorSource == false }} shell: cmd run: jnetreflector -TraceLevel 1 -TraceTo %GITHUB_WORKSPACE%\JNetReflector.txt -OriginRootPath %GITHUB_WORKSPACE%\jars -DestinationRootPath %GITHUB_WORKSPACE%\src\ -ConfigurationFile %GITHUB_WORKSPACE%\src\configuration.json @@ -155,16 +163,16 @@ jobs: - name: Try compilation of Jars continue-on-error: ${{ inputs.ContinueOnLatestJNetReflectorError == true }} - if: ${{ inputs.UseLatestJNetReflector == true }} + if: ${{ inputs.UseLatestJNetReflectorSource == true }} run: mvn --file ./src/jvm/knet/pom.xml --no-transfer-progress package - name: Try compilation of C# continue-on-error: ${{ inputs.ContinueOnLatestJNetReflectorError == true }} - if: ${{ inputs.UseLatestJNetReflector == true }} + if: ${{ inputs.UseLatestJNetReflectorSource == true }} run: dotnet build --no-incremental --configuration Release /p:Platform="Any CPU" /p:NoWarn="0108%3B1030%3B0618" src\net\KNet\KNet.csproj - name: Request a PR to commit changes made with unpublished version of JNetReflector - if: ${{ github.repository_owner == 'masesgroup' && inputs.GeneratePR == true && inputs.UseLatestJNetReflector == true }} #do not push any changes outside main repo or GeneratePR is false + if: ${{ github.repository_owner == 'masesgroup' && inputs.GeneratePR == true && inputs.UseLatestJNetReflectorSource == true }} #do not push any changes outside main repo or GeneratePR is false uses: peter-evans/create-pull-request@v7 with: branch-suffix: short-commit-hash @@ -178,13 +186,13 @@ jobs: labels: java, .NET, enhancement - name: Extract commit SHA - if: ${{ github.repository_owner == 'masesgroup' && inputs.GeneratePR == true && inputs.UseLatestJNetReflector == false }} + if: ${{ github.repository_owner == 'masesgroup' && inputs.GeneratePR == true && inputs.UseLatestJNetReflectorSource == false }} run: | echo "GITHUB_COMMIT_MESSAGE=$(echo $GITHUB_SHA | cut -c 1-7)" >> $GITHUB_ENV shell: bash - name: Request a PR to commit changes made with latest published version of JNetReflector - if: ${{ github.repository_owner == 'masesgroup' && inputs.GeneratePR == true && inputs.UseLatestJNetReflector == false }} #do not push any changes outside main repo or GeneratePR is false + if: ${{ github.repository_owner == 'masesgroup' && inputs.GeneratePR == true && inputs.UseLatestJNetReflectorSource == false }} #do not push any changes outside main repo or GeneratePR is false uses: peter-evans/create-pull-request@v7 with: branch-suffix: short-commit-hash diff --git a/README.md b/README.md index 26dafadba..fd9222c10 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ This project adheres to the Contributor [Covenant code of conduct](CODE_OF_CONDU * serializers supports data exchange based on `byte` array and `ByteBuffer` * version 2.7.2 introduces `ISerDesSelector` to optimize serialization selection based on `byte` array or `ByteBuffer` * V2.8.0+: From version 2.8.0: supports Apache Kafka™ version 3.8.* -* V2.9.0+: From version 2.8.0: supports Apache Kafka™ version 3.9.* +* V2.9.0+: From version 2.9.0: supports Apache Kafka™ version 3.9.* --- diff --git a/src/documentation/articles/howtobuild.md b/src/documentation/articles/howtobuild.md index 7b406cfc0..79aabd64f 100644 --- a/src/documentation/articles/howtobuild.md +++ b/src/documentation/articles/howtobuild.md @@ -9,7 +9,7 @@ If the user clones the repo, the following steps shall be done to use the projec Let's start with the tools needed to achieve the goal: * An installed version of __git for Windows__ -* An installed version of .NET 8 SDK +* An installed version of .NET 8/9 SDK * An installed version of Apache Maven (> 3.8.1) Then the steps are the following (the steps are made for Windows shell): @@ -42,11 +42,11 @@ Then the steps are the following (the steps are made for Windows shell): The final result shall look like this: * RootFolder * bin - * net6.0 + * net8.0 * config * jars * other folders - * net8.0 + * net9.0 * config * jars * other folders diff --git a/src/documentation/articles/usage.md b/src/documentation/articles/usage.md index 70fd1469a..8135b248c 100644 --- a/src/documentation/articles/usage.md +++ b/src/documentation/articles/usage.md @@ -31,10 +31,10 @@ If a developer is using KNet within its own product it is possible to override t } ``` -**IMPORTANT NOTE**: `pathToJVM` shall be escaped -1. `string pathToJVM = "C:\\Program Files\\Eclipse Adoptium\\jdk-11.0.18.10-hotspot\\bin\\server\\jvm.dll";` -2. `string pathToJVM = @"C:\Program Files\Eclipse Adoptium\jdk-11.0.18.10-hotspot\bin\server\jvm.dll";` - +> [!IMPORTANT] +> `pathToJVM` shall be escaped +> 1. `string pathToJVM = "C:\\Program Files\\Eclipse Adoptium\\jdk-11.0.18.10-hotspot\\bin\\server\\jvm.dll";` +> 2. `string pathToJVM = @"C:\Program Files\Eclipse Adoptium\jdk-11.0.18.10-hotspot\bin\server\jvm.dll";` ### Special initialization conditions @@ -47,10 +47,55 @@ If the developer/user encounter this condition can do the following steps: 3. Try to set `JAVA_HOME` at system level e.g. `JAVA_HOME=C:\Program Files\Eclipse Adoptium\jdk-11.0.18.10-hotspot\`; 4. Try to set `JCOBRIDGE_JVMPath` at system level e.g. `JCOBRIDGE_JVMPath=C:\Program Files\Eclipse Adoptium\jdk-11.0.18.10-hotspot\`. -**IMPORTANT NOTES**: -- One of `JCOBRIDGE_JVMPath` or `JAVA_HOME` environment variables or Windows registry (on Windows OSes) shall be available -- `JCOBRIDGE_JVMPath` environment variable takes precedence over `JAVA_HOME` and Windows registry: you can set `JCOBRIDGE_JVMPath` to `C:\Program Files\Eclipse Adoptium\jdk-11.0.18.10-hotspot\bin\server\jvm.dll` and avoid to override `JVMPath` in your code -- After first initialization steps, `JVMPath` takes precedence over `JCOBRIDGE_JVMPath`/`JAVA_HOME` environment variables or Windows registry +> [!IMPORTANT] +> - One of `JCOBRIDGE_JVMPath` or `JAVA_HOME` environment variables or Windows registry (on Windows OSes) shall be available +> - `JCOBRIDGE_JVMPath` environment variable takes precedence over `JAVA_HOME` and Windows registry: you can set `JCOBRIDGE_JVMPath` to `C:\Program Files\Eclipse Adoptium\jdk-11.0.18.10-hotspot\bin\server\jvm.dll` and avoid to override `JVMPath` in your code +> - After first initialization steps, `JVMPath` takes precedence over `JCOBRIDGE_JVMPath`/`JAVA_HOME` environment variables or Windows registry + +### Intel CET and KNet + +KNet uses an embedded JVM through JNet/JCOBridge, however JVM initialization is incompatible with [CET](https://www.intel.com/content/www/us/en/developer/articles/technical/technical-look-control-flow-enforcement-technology.html) because the code used to identify CPU try to modify the return address and this is considered from CET a violation: see [this comment](https://github.com/masesgroup/JNet/issues/573#issuecomment-2544249107). + +From .NET 9 preview 6, [CET is enabled by default on supported hardware](https://learn.microsoft.com/en-us/dotnet/core/compatibility/interop/9.0/cet-support) when the final stage produce an executable artifact, i.e. the csproj file contains `Exe`. + +If the application, upon startup, fails with the error 0xc0000409 (subcode 0x30) it was compiled with CET enabled and it fails during JVM initialization. + +To solve the issue there are four possible solutions: +1. use a .NET version, e.g. 8, that does not enable CET by default +2. Add the following snippet to disable CET on executable (templates available for KNet are ready made and solve this issue): + +```xml + + + false + +``` + +3. Use the `dotnet` app host, as reported in https://github.com/masesgroup/JCOBridgePublic/issues/7#issuecomment-2550031946, with a syntax like: + +```sh + dotnet MyApplication.dll +``` + instead of the classic: + ```sh + MyApplication.exe +``` + +4. If you want to run the classic application execute the following command: + + ```sh + reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\MyApplication.exe" /v MitigationOptions /t REG_BINARY /d "0000000000000000000000000000002000" /f +``` +then run: + ```sh + MyApplication.exe +``` + +Use the following to enable again CET: + + ```sh + reg delete "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\MyApplication.exe" /v MitigationOptions /f +``` ## Producer example diff --git a/src/documentation/articles/usageCLI.md b/src/documentation/articles/usageCLI.md index e07e0a088..d7a7636e7 100644 --- a/src/documentation/articles/usageCLI.md +++ b/src/documentation/articles/usageCLI.md @@ -13,7 +13,7 @@ KNet CLI is available in two different formats: - **Docker image** hosted on [GitHub](https://github.com/masesgroup/KNet/pkgs/container/knet) and [Docker Hub](https://hub.docker.com/repository/docker/masesgroup/knet/general): follow instruction within the page and general instruction on https://docs.docker.com > [!IMPORTANT] -> The **dotnet tool** needs a JRE/JDK installed within the system +> The **dotnet tool** needs a JRE/JDK installed within the system. ## Usage @@ -27,6 +27,12 @@ To use the CLI interface (KNetCLI) runs a command like the following: knet -ClassToRun ConsoleConsumer --bootstrap-server SERVER-ADDRESS:9093 --topic topic_name --from-beginning ``` +> [!IMPORTANT] +> If the previous command raises the error described in [Intel CET and KNet](usage.md#intel-cet-and-knet) the only solution is to apply the following workaround and disable CET: +> ```sh +> reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\knet.exe" /v MitigationOptions /t REG_BINARY /d "0000000000000000000000000000002000" /f +> ``` + - **Docker image** ```sh diff --git a/src/documentation/articles/usageConnect.md b/src/documentation/articles/usageConnect.md index cd5397df6..e3b3526f0 100644 --- a/src/documentation/articles/usageConnect.md +++ b/src/documentation/articles/usageConnect.md @@ -29,6 +29,12 @@ knetconnect -s connect-standalone.properties specific-connector.properties knetconnect -d connect-distributed.properties ``` +> [!IMPORTANT] +> If the previous commands raise the error described in [Intel CET and KNet](usage.md#intel-cet-and-knet) the only solution is to apply the following workaround and disable CET: +> ```sh +> reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\knetconnect.exe" /v MitigationOptions /t REG_BINARY /d "0000000000000000000000000000002000" /f +> ``` + - **Docker image** ```sh diff --git a/src/documentation/index.md b/src/documentation/index.md index 013ad9bc9..c3100649f 100644 --- a/src/documentation/index.md +++ b/src/documentation/index.md @@ -102,6 +102,7 @@ This project adheres to the Contributor [Covenant code of conduct](https://githu * serializers supports data exchange based on `byte` array and `ByteBuffer` * version 2.7.2 introduces `ISerDesSelector` to optimize serialization selection based on `byte` array or `ByteBuffer` * V2.8.0+: From version 2.8.0: supports Apache Kafka™ version 3.8.* +* V2.9.0+: From version 2.9.0: supports Apache Kafka™ version 3.9.* ---