packaging: AL2023 backport support (#7110)

* packaging: sync from main Signed-off-by: Patrick Stephens <[email protected]> * packaging: actual latest sync Signed-off-by: Patrick Stephens <[email protected]> --------- Signed-off-by: Patrick Stephens <[email protected]>
fluent · Apr 4, 2023 · bf22d53 · bf22d53
1 parent 61c6bb1
commit bf22d53
Show file tree

Hide file tree

Showing 23 changed files with 720 additions and 191 deletions.
diff --git a/.github/actions/generate-package-build-matrix/action.yaml b/.github/actions/generate-package-build-matrix/action.yaml
@@ -14,8 +14,14 @@ inputs:
     required: true
 outputs:
   build-matrix:
-    description: The build matrix we have created.
+    description: The total build matrix we have created.
     value: ${{ steps.set-matrix.outputs.matrix }}
+  deb-build-matrix:
+    description: The targets that provide DEB artefacts.
+    value: ${{ steps.set-matrix.outputs.debmatrix }}
+  rpm-build-matrix:
+    description: The targets that provide RPN artefacts.
+    value: ${{ steps.set-matrix.outputs.rpmmatrix }}
 runs:
   using: "composite"
   steps:
@@ -28,24 +34,34 @@ runs:
     - name: Determine target type
       id: determine-build-type
       run: |
-        BUILD_TYPE="1.9"
+        BUILD_TYPE="legacy"
         if [[ -f "packaging/build-config.json" ]]; then
-          BUILD_TYPE="2.0"
+          BUILD_TYPE="modern"
         fi
         echo "Detected type: $BUILD_TYPE"
         echo "BUILD_TYPE=$BUILD_TYPE" >> $GITHUB_OUTPUT
       shell: bash
       working-directory: version-check
 
-    - name: 2.0 targets
-      if: steps.determine-build-type.outputs.BUILD_TYPE == '2.0'
+    - name: 2.0+ targets
+      if: steps.determine-build-type.outputs.BUILD_TYPE == 'modern'
       run: |
-        matrix=$(echo '{ "distro" : '$(jq -cr '.linux_targets' packaging/build-config.json)'}'|jq -c .)
+        matrix=$(echo '{ "distro" : '$(jq -cr '.linux_targets|map(.target)' packaging/build-config.json)'}'|jq -c .)
         echo "MATRIX=$matrix" >> $GITHUB_ENV
+
+        # The following are only used by release so exclude architecture as well
+
+        debtargets=$(jq -cr '[.linux_targets[] | select(.target|contains("arm64v8")|not) | select(.type=="deb") | .target ]' packaging/build-config.json)
+        debmatrix=$(echo "{ \"distro\" : $debtargets }"|jq -c .)
+        echo "DEB_MATRIX=$debmatrix" >> $GITHUB_ENV
+
+        rpmtargets=$(jq -cr '[.linux_targets[] | select(.target|contains("arm64v8")|not) | select(.type=="rpm") | .target ]' packaging/build-config.json)
+        rpmmatrix=$(echo "{ \"distro\" : $rpmtargets}"|jq -c .)
+        echo "RPM_MATRIX=$rpmmatrix" >> $GITHUB_ENV
       shell: bash
 
     - name: 1.9 targets
-      if: steps.determine-build-type.outputs.BUILD_TYPE == '1.9'
+      if: steps.determine-build-type.outputs.BUILD_TYPE == 'legacy'
       run: |
         matrix=$((
           echo '{ "distro" : ['
@@ -58,6 +74,21 @@ runs:
           echo ']}'
         ) | jq -c .)
         echo "MATRIX=$matrix" >> $GITHUB_ENV
+        debmatrix=$((
+          echo '{ "distro" : ['
+          echo '"debian/buster", "debian/bullseye",'
+          echo '"ubuntu/16.04", "ubuntu/18.04", "ubuntu/20.04", "ubuntu/22.04",'
+          echo '"raspbian/buster", "raspbian/bullseye"'
+          echo ']}'
+        ) | jq -c .)
+        echo "DEB_MATRIX=$debmatrix" >> $GITHUB_ENV
+        rpmmatrix=$((
+          echo '{ "distro" : ['
+          echo '"amazonlinux/2",'
+          echo '"centos/7", "centos/8"'
+          echo ']}'
+        ) | jq -c .)
+        echo "RPM_MATRIX=$rpmmatrix" >> $GITHUB_ENV
       shell: bash
 
     - name: Manual override of target
@@ -79,4 +110,10 @@ runs:
         echo $MATRIX
         echo $MATRIX| jq .
         echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
+        echo $DEB_MATRIX
+        echo $DEB_MATRIX| jq .
+        echo "debmatrix=$DEB_MATRIX" >> $GITHUB_OUTPUT
+        echo $RPM_MATRIX
+        echo $RPM_MATRIX| jq .
+        echo "rpmmatrix=$RPM_MATRIX" >> $GITHUB_OUTPUT
       shell: bash
diff --git a/.github/workflows/build-legacy-branch.yaml b/.github/workflows/build-legacy-branch.yaml
@@ -80,7 +80,7 @@ jobs:
 
       - name: Build the legacy x86_64 debug image
         if: matrix.arch == 'amd64'
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           file: ./Dockerfile.x86_64.debug
           context: .
@@ -101,7 +101,7 @@ jobs:
             raw,${{ matrix.suffix }}-${{ inputs.ref }}
 
       - name: Build the legacy ${{ matrix.arch }} image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           file: ./Dockerfile.${{ matrix.suffix }}
           context: .

diff --git a/.github/workflows/call-build-images.yaml b/.github/workflows/call-build-images.yaml
@@ -114,7 +114,7 @@ jobs:
 
       - name: Build the production images
         id: build_push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           file: ./dockerfiles/Dockerfile
           context: .
@@ -139,7 +139,7 @@ jobs:
 
       - name: Build the debug multi-arch images
         id: debug_build_push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           file: ./dockerfiles/Dockerfile
           context: .
@@ -310,7 +310,7 @@ jobs:
           docker push ${{ inputs.registry }}/${{ inputs.image }}:windows-${{ matrix.windows-base-version }}-${{ inputs.version }}
         
         # We cannot use this action as it requires privileged mode
-        # uses: docker/build-push-action@v3
+        # uses: docker/build-push-action@v4
         # with:
         #   file: ./dockerfiles/Dockerfile.windows
         #   context: .

diff --git a/.github/workflows/call-build-linux-packages.yaml b/.github/workflows/call-build-linux-packages.yaml
@@ -125,13 +125,15 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-      - uses: frabert/[email protected]
+      - name: Replace all special characters with dashes
         id: formatted_distro
-        with:
-          pattern: '(.*)\/(.*)$'
-          string: "${{ matrix.distro }}"
-          replace-with: "$1-$2"
-          flags: "g"
+        run:
+          output=${INPUT//[\/]/-}
+          echo "$INPUT --> $output"
+          echo "replaced=$output" >> "$GITHUB_OUTPUT"
+        shell: bash
+        env: 
+          INPUT: ${{ matrix.distro }}
 
       - name: fluent-bit - ${{ matrix.distro }} artifacts
         run: |
@@ -143,7 +145,7 @@ jobs:
           CMAKE_INSTALL_PREFIX: /opt/fluent-bit/
         working-directory: packaging
 
-      - name: Upload the ${{ matrix.distro }} artifacts
+      - name: Upload the ${{ steps.formatted_distro.outputs.replaced }} artifacts
         uses: actions/upload-artifact@v3
         with:
           name: packages-${{ inputs.version }}-${{ steps.formatted_distro.outputs.replaced }}

diff --git a/.github/workflows/call-build-macos.yaml b/.github/workflows/call-build-macos.yaml
@@ -79,7 +79,7 @@ jobs:
       - name: Install dependencies
         run: |
           brew update
-          brew install bison flex libyaml openssl || true
+          brew install bison flex libyaml openssl pkgconfig || true
 
       - name: Build Fluent Bit packages
         run: |

diff --git a/.github/workflows/call-build-windows.yaml b/.github/workflows/call-build-windows.yaml
@@ -36,9 +36,36 @@ on:
         required: false
 
 jobs:
+
+  call-build-windows-get-meta:
+    name: Determine build info
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      armSupported: ${{ steps.armcheck.outputs.armSupported }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Determine if we are doing a build with ARM support
+        id: armcheck
+        # Check for new contents from https://github.com/fluent/fluent-bit/pull/6621
+        run: |
+          if grep -q "winarm64" CMakeLists.txt ; then
+            echo "armSupported=true" >> $GITHUB_OUTPUT
+          else
+            echo "armSupported=false" >> $GITHUB_OUTPUT
+          fi
+        shell: bash
+
   call-build-windows-package:
     runs-on: windows-latest
     environment: ${{ inputs.environment }}
+    needs:
+      - call-build-windows-get-meta
     strategy:
       fail-fast: false
       matrix:
@@ -47,15 +74,25 @@ jobs:
             arch: x86
             openssl_dir: C:\Program Files (x86)\OpenSSL-Win32
             chocolatey_opt: --x86
+            cmake_additional_opt: ""
+            vcpkg_triplet: x86-windows-static
           - name: "Windows 64bit"
             arch: x64
             openssl_dir: C:\Program Files\OpenSSL-Win64
             chocolatey_opt: --x64
+            cmake_additional_opt: ""
+            vcpkg_triplet: x64-windows-static
+          - name: "Windows 64bit (Arm64)"
+            arch: amd64_arm64
+            openssl_dir: C:\vcpkg\packages\openssl_arm64-windows-static
+            chocolatey_opt: ""
+            cmake_additional_opt: "-DCMAKE_SYSTEM_NAME=Windows -DCMAKE_SYSTEM_VERSION=10.0 -DCMAKE_SYSTEM_PROCESSOR=ARM64"
+            vcpkg_triplet: arm64-windows-static
     permissions:
       contents: read
     # Default environment variables can be overridden below. To prevent library pollution - without this other random libraries may be found on the path leading to failures.
     env:
-      PATH: C:\ProgramData\Chocolatey\bin;c:/Program Files/Git/cmd;c:/Windows/system32;C:/Windows/System32/WindowsPowerShell/v1.0;$ENV:WIX/bin;C:/Program Files/CMake/bin
+      PATH: C:\ProgramData\Chocolatey\bin;c:/Program Files/Git/cmd;c:/Windows/system32;C:/Windows/System32/WindowsPowerShell/v1.0;$ENV:WIX/bin;C:/Program Files/CMake/bin;C:\vcpkg;
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -74,23 +111,36 @@ jobs:
         shell: pwsh
 
       - name: Get dependencies w/ chocolatey
+        if: ${{ matrix.config.arch != 'amd64_arm64' }}
         uses: crazy-max/ghaction-chocolatey@v2
         with:
           args: install ${{ matrix.config.chocolatey_opt }} openssl -y
 
-      - name: Set up Visual Studio shell
-        uses: egor-tensin/vs-shell@v2
+      - name: Set up with Developer Command Prompt for Microsoft Visual C++
+        uses: ilammy/msvc-dev-cmd@v1
         with:
           arch: ${{ matrix.config.arch }}
 
+      - name: Build openssl with vcpkg
+        if: ${{ matrix.config.arch == 'amd64_arm64' }}
+        run: |
+          C:\vcpkg\vcpkg install --recurse openssl --triplet ${{ matrix.config.vcpkg_triplet }}
+        shell: cmd
+
       - name: Build Fluent Bit packages
+        # If we are using 2.0.* or earlier we need to exclude the ARM64 build as the dependencies fail to compile.
+        # Trying to do via an exclude for the job triggers linting errors.
+        # This is only supposed to be a workaround for now so can be easily removed later.
+        if: ${{ matrix.config.arch != 'amd64_arm64' || needs.call-build-windows-get-meta.outputs.armSupported == 'true' }}
         run: |
-          cmake -G "NMake Makefiles" -DFLB_NIGHTLY_BUILD=${{ inputs.unstable }} -DOPENSSL_ROOT_DIR='${{ matrix.config.openssl_dir }}' ../
+          cmake -G "NMake Makefiles" -DFLB_NIGHTLY_BUILD='${{ inputs.unstable }}' -DOPENSSL_ROOT_DIR='${{ matrix.config.openssl_dir }}' ${{ matrix.config.cmake_additional_opt }} ../
           cmake --build .
           cpack
         working-directory: build
 
       - name: Upload build packages
+        # Skip upload if we skipped build.
+        if: ${{ matrix.config.arch != 'amd64_arm64' || needs.call-build-windows-get-meta.outputs.armSupported == 'true' }}
         uses: actions/upload-artifact@v3
         with:
           name: windows-packages

diff --git a/.github/workflows/call-integration-image-build.yaml b/.github/workflows/call-integration-image-build.yaml
@@ -62,7 +62,7 @@ jobs:
             raw,${{ inputs.image-tag }}
 
       - name: Build the AMD64 image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           file: ./dockerfiles/Dockerfile
           context: .
@@ -97,7 +97,7 @@ jobs:
             raw,${{ inputs.image-tag }}-debug
 
       - name: Build the AMD64 debug image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           file: ./dockerfiles/Dockerfile
           context: .

diff --git a/.github/workflows/cron-scorecards-analysis.yaml b/.github/workflows/cron-scorecards-analysis.yaml
@@ -30,7 +30,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@e38b1902ae4f44df626f11ba0734b14fb91f8f86
+        uses: ossf/scorecard-action@80e868c13c90f172d68d1f4501dee99e2479f7af
         with:
           results_file: scorecard-results.sarif
           results_format: sarif

diff --git a/.github/workflows/cron-stale.yaml b/.github/workflows/cron-stale.yaml
@@ -8,7 +8,7 @@ jobs:
     name: Mark stale
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v7
+      - uses: actions/stale@v8
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 5 days. Maintainers can add the `exempt-stale` label.'

diff --git a/.github/workflows/cron-unstable-build.yaml b/.github/workflows/cron-unstable-build.yaml
@@ -13,7 +13,7 @@ on:
   # Run nightly build at this time, bit of trial and error but this seems good.
   schedule:
     - cron: "0 6 * * *" # master build
-    - cron: "0 12 * * *" # 1.9 build
+    - cron: "0 12 * * *" # 2.0 build
 
 # We do not want a new unstable build to run whilst we are releasing the current unstable build.
 concurrency: unstable-build-release
@@ -51,10 +51,10 @@ jobs:
           echo "cron_branch=master" >> $GITHUB_ENV
         shell: bash
 
-      - name: 1.9 run
+      - name: 2.0 run
         if: github.event_name == 'schedule' && github.event.schedule=='0 12 * * *'
         run: |
-          echo "cron_branch=1.9" >> $GITHUB_ENV
+          echo "cron_branch=2.0" >> $GITHUB_ENV
         shell: bash
 
       - name: Output the branch to use

diff --git a/.github/workflows/pr-compile-check.yaml b/.github/workflows/pr-compile-check.yaml
@@ -22,7 +22,7 @@ jobs:
       uses: docker/setup-buildx-action@v2
 
     - name: Attempt to build current source for CentOS 7
-      uses: docker/build-push-action@v3
+      uses: docker/build-push-action@v4
       with:
         context: .
         file: ./dockerfiles/Dockerfile.centos7

diff --git a/.github/workflows/pr-image-tests.yaml b/.github/workflows/pr-image-tests.yaml
@@ -24,7 +24,7 @@ jobs:
           uses: docker/setup-buildx-action@v2
 
         - name: Build the multi-arch images
-          uses: docker/build-push-action@v3
+          uses: docker/build-push-action@v4
           with:
             file: ./dockerfiles/Dockerfile
             context: .
@@ -34,7 +34,7 @@ jobs:
             load: false
 
         - name: Build the debug multi-arch images
-          uses: docker/build-push-action@v3
+          uses: docker/build-push-action@v4
           with:
             file: ./dockerfiles/Dockerfile
             context: .

diff --git a/.github/workflows/pr-windows-build.yaml b/.github/workflows/pr-windows-build.yaml
@@ -18,7 +18,7 @@ on:
 
 jobs:
   pr-windows-build:
-    uses: fluent/fluent-bit/.github/workflows/call-build-windows.yaml@master
+    uses: ./.github/workflows/call-build-windows.yaml
     with:
       version: ${{ github.sha }}
       ref: ${{ github.sha }}