From bf22d53a696bc9659f7d3c874b74a515b5e629c6 Mon Sep 17 00:00:00 2001 From: Pat Date: Tue, 4 Apr 2023 08:56:09 +0100 Subject: [PATCH] packaging: AL2023 backport support (#7110) * packaging: sync from main Signed-off-by: Patrick Stephens * packaging: actual latest sync Signed-off-by: Patrick Stephens --------- Signed-off-by: Patrick Stephens --- .../generate-package-build-matrix/action.yaml | 51 ++- .github/workflows/build-legacy-branch.yaml | 4 +- .github/workflows/call-build-images.yaml | 6 +- .../workflows/call-build-linux-packages.yaml | 16 +- .github/workflows/call-build-macos.yaml | 2 +- .github/workflows/call-build-windows.yaml | 58 +++- .../call-integration-image-build.yaml | 4 +- .../workflows/cron-scorecards-analysis.yaml | 2 +- .github/workflows/cron-stale.yaml | 2 +- .github/workflows/cron-unstable-build.yaml | 6 +- .github/workflows/pr-compile-check.yaml | 2 +- .github/workflows/pr-image-tests.yaml | 4 +- .github/workflows/pr-windows-build.yaml | 2 +- .github/workflows/staging-release.yaml | 300 +++++++++++++++++- .github/workflows/unit-tests.yaml | 2 + dockerfiles/Dockerfile | 2 + packaging/build-config.json | 125 ++++++-- packaging/distros/amazonlinux/Dockerfile | 4 +- packaging/test-release-packages.sh | 2 +- packaging/update-apt-repo.sh | 71 +++++ packaging/update-repos.sh | 155 +++------ packaging/update-source-packages.sh | 6 +- packaging/update-yum-repo.sh | 85 +++++ 23 files changed, 720 insertions(+), 191 deletions(-) create mode 100755 packaging/update-apt-repo.sh create mode 100755 packaging/update-yum-repo.sh diff --git a/.github/actions/generate-package-build-matrix/action.yaml b/.github/actions/generate-package-build-matrix/action.yaml index bffa7864a32..f8355e4bb36 100644 --- a/.github/actions/generate-package-build-matrix/action.yaml +++ b/.github/actions/generate-package-build-matrix/action.yaml @@ -14,8 +14,14 @@ inputs: required: true outputs: build-matrix: - description: The build matrix we have created. + description: The total build matrix we have created. value: ${{ steps.set-matrix.outputs.matrix }} + deb-build-matrix: + description: The targets that provide DEB artefacts. + value: ${{ steps.set-matrix.outputs.debmatrix }} + rpm-build-matrix: + description: The targets that provide RPN artefacts. + value: ${{ steps.set-matrix.outputs.rpmmatrix }} runs: using: "composite" steps: @@ -28,24 +34,34 @@ runs: - name: Determine target type id: determine-build-type run: | - BUILD_TYPE="1.9" + BUILD_TYPE="legacy" if [[ -f "packaging/build-config.json" ]]; then - BUILD_TYPE="2.0" + BUILD_TYPE="modern" fi echo "Detected type: $BUILD_TYPE" echo "BUILD_TYPE=$BUILD_TYPE" >> $GITHUB_OUTPUT shell: bash working-directory: version-check - - name: 2.0 targets - if: steps.determine-build-type.outputs.BUILD_TYPE == '2.0' + - name: 2.0+ targets + if: steps.determine-build-type.outputs.BUILD_TYPE == 'modern' run: | - matrix=$(echo '{ "distro" : '$(jq -cr '.linux_targets' packaging/build-config.json)'}'|jq -c .) + matrix=$(echo '{ "distro" : '$(jq -cr '.linux_targets|map(.target)' packaging/build-config.json)'}'|jq -c .) echo "MATRIX=$matrix" >> $GITHUB_ENV + + # The following are only used by release so exclude architecture as well + + debtargets=$(jq -cr '[.linux_targets[] | select(.target|contains("arm64v8")|not) | select(.type=="deb") | .target ]' packaging/build-config.json) + debmatrix=$(echo "{ \"distro\" : $debtargets }"|jq -c .) + echo "DEB_MATRIX=$debmatrix" >> $GITHUB_ENV + + rpmtargets=$(jq -cr '[.linux_targets[] | select(.target|contains("arm64v8")|not) | select(.type=="rpm") | .target ]' packaging/build-config.json) + rpmmatrix=$(echo "{ \"distro\" : $rpmtargets}"|jq -c .) + echo "RPM_MATRIX=$rpmmatrix" >> $GITHUB_ENV shell: bash - name: 1.9 targets - if: steps.determine-build-type.outputs.BUILD_TYPE == '1.9' + if: steps.determine-build-type.outputs.BUILD_TYPE == 'legacy' run: | matrix=$(( echo '{ "distro" : [' @@ -58,6 +74,21 @@ runs: echo ']}' ) | jq -c .) echo "MATRIX=$matrix" >> $GITHUB_ENV + debmatrix=$(( + echo '{ "distro" : [' + echo '"debian/buster", "debian/bullseye",' + echo '"ubuntu/16.04", "ubuntu/18.04", "ubuntu/20.04", "ubuntu/22.04",' + echo '"raspbian/buster", "raspbian/bullseye"' + echo ']}' + ) | jq -c .) + echo "DEB_MATRIX=$debmatrix" >> $GITHUB_ENV + rpmmatrix=$(( + echo '{ "distro" : [' + echo '"amazonlinux/2",' + echo '"centos/7", "centos/8"' + echo ']}' + ) | jq -c .) + echo "RPM_MATRIX=$rpmmatrix" >> $GITHUB_ENV shell: bash - name: Manual override of target @@ -79,4 +110,10 @@ runs: echo $MATRIX echo $MATRIX| jq . echo "matrix=$MATRIX" >> $GITHUB_OUTPUT + echo $DEB_MATRIX + echo $DEB_MATRIX| jq . + echo "debmatrix=$DEB_MATRIX" >> $GITHUB_OUTPUT + echo $RPM_MATRIX + echo $RPM_MATRIX| jq . + echo "rpmmatrix=$RPM_MATRIX" >> $GITHUB_OUTPUT shell: bash diff --git a/.github/workflows/build-legacy-branch.yaml b/.github/workflows/build-legacy-branch.yaml index 0c1b6818621..163752ce1b8 100644 --- a/.github/workflows/build-legacy-branch.yaml +++ b/.github/workflows/build-legacy-branch.yaml @@ -80,7 +80,7 @@ jobs: - name: Build the legacy x86_64 debug image if: matrix.arch == 'amd64' - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: file: ./Dockerfile.x86_64.debug context: . @@ -101,7 +101,7 @@ jobs: raw,${{ matrix.suffix }}-${{ inputs.ref }} - name: Build the legacy ${{ matrix.arch }} image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: file: ./Dockerfile.${{ matrix.suffix }} context: . diff --git a/.github/workflows/call-build-images.yaml b/.github/workflows/call-build-images.yaml index d1b85a6bfbf..d72ff98c435 100644 --- a/.github/workflows/call-build-images.yaml +++ b/.github/workflows/call-build-images.yaml @@ -114,7 +114,7 @@ jobs: - name: Build the production images id: build_push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: file: ./dockerfiles/Dockerfile context: . @@ -139,7 +139,7 @@ jobs: - name: Build the debug multi-arch images id: debug_build_push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: file: ./dockerfiles/Dockerfile context: . @@ -310,7 +310,7 @@ jobs: docker push ${{ inputs.registry }}/${{ inputs.image }}:windows-${{ matrix.windows-base-version }}-${{ inputs.version }} # We cannot use this action as it requires privileged mode - # uses: docker/build-push-action@v3 + # uses: docker/build-push-action@v4 # with: # file: ./dockerfiles/Dockerfile.windows # context: . diff --git a/.github/workflows/call-build-linux-packages.yaml b/.github/workflows/call-build-linux-packages.yaml index 72729b8aeff..ee68871624a 100644 --- a/.github/workflows/call-build-linux-packages.yaml +++ b/.github/workflows/call-build-linux-packages.yaml @@ -125,13 +125,15 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - uses: frabert/replace-string-action@v2.4 + - name: Replace all special characters with dashes id: formatted_distro - with: - pattern: '(.*)\/(.*)$' - string: "${{ matrix.distro }}" - replace-with: "$1-$2" - flags: "g" + run: + output=${INPUT//[\/]/-} + echo "$INPUT --> $output" + echo "replaced=$output" >> "$GITHUB_OUTPUT" + shell: bash + env: + INPUT: ${{ matrix.distro }} - name: fluent-bit - ${{ matrix.distro }} artifacts run: | @@ -143,7 +145,7 @@ jobs: CMAKE_INSTALL_PREFIX: /opt/fluent-bit/ working-directory: packaging - - name: Upload the ${{ matrix.distro }} artifacts + - name: Upload the ${{ steps.formatted_distro.outputs.replaced }} artifacts uses: actions/upload-artifact@v3 with: name: packages-${{ inputs.version }}-${{ steps.formatted_distro.outputs.replaced }} diff --git a/.github/workflows/call-build-macos.yaml b/.github/workflows/call-build-macos.yaml index d162b0e4b40..456ead45274 100644 --- a/.github/workflows/call-build-macos.yaml +++ b/.github/workflows/call-build-macos.yaml @@ -79,7 +79,7 @@ jobs: - name: Install dependencies run: | brew update - brew install bison flex libyaml openssl || true + brew install bison flex libyaml openssl pkgconfig || true - name: Build Fluent Bit packages run: | diff --git a/.github/workflows/call-build-windows.yaml b/.github/workflows/call-build-windows.yaml index 7f6aeaf1c4b..1caf5879ee1 100644 --- a/.github/workflows/call-build-windows.yaml +++ b/.github/workflows/call-build-windows.yaml @@ -36,9 +36,36 @@ on: required: false jobs: + + call-build-windows-get-meta: + name: Determine build info + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + armSupported: ${{ steps.armcheck.outputs.armSupported }} + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + ref: ${{ inputs.ref }} + + - name: Determine if we are doing a build with ARM support + id: armcheck + # Check for new contents from https://github.com/fluent/fluent-bit/pull/6621 + run: | + if grep -q "winarm64" CMakeLists.txt ; then + echo "armSupported=true" >> $GITHUB_OUTPUT + else + echo "armSupported=false" >> $GITHUB_OUTPUT + fi + shell: bash + call-build-windows-package: runs-on: windows-latest environment: ${{ inputs.environment }} + needs: + - call-build-windows-get-meta strategy: fail-fast: false matrix: @@ -47,15 +74,25 @@ jobs: arch: x86 openssl_dir: C:\Program Files (x86)\OpenSSL-Win32 chocolatey_opt: --x86 + cmake_additional_opt: "" + vcpkg_triplet: x86-windows-static - name: "Windows 64bit" arch: x64 openssl_dir: C:\Program Files\OpenSSL-Win64 chocolatey_opt: --x64 + cmake_additional_opt: "" + vcpkg_triplet: x64-windows-static + - name: "Windows 64bit (Arm64)" + arch: amd64_arm64 + openssl_dir: C:\vcpkg\packages\openssl_arm64-windows-static + chocolatey_opt: "" + cmake_additional_opt: "-DCMAKE_SYSTEM_NAME=Windows -DCMAKE_SYSTEM_VERSION=10.0 -DCMAKE_SYSTEM_PROCESSOR=ARM64" + vcpkg_triplet: arm64-windows-static permissions: contents: read # Default environment variables can be overridden below. To prevent library pollution - without this other random libraries may be found on the path leading to failures. env: - PATH: C:\ProgramData\Chocolatey\bin;c:/Program Files/Git/cmd;c:/Windows/system32;C:/Windows/System32/WindowsPowerShell/v1.0;$ENV:WIX/bin;C:/Program Files/CMake/bin + PATH: C:\ProgramData\Chocolatey\bin;c:/Program Files/Git/cmd;c:/Windows/system32;C:/Windows/System32/WindowsPowerShell/v1.0;$ENV:WIX/bin;C:/Program Files/CMake/bin;C:\vcpkg; steps: - name: Checkout repository uses: actions/checkout@v3 @@ -74,23 +111,36 @@ jobs: shell: pwsh - name: Get dependencies w/ chocolatey + if: ${{ matrix.config.arch != 'amd64_arm64' }} uses: crazy-max/ghaction-chocolatey@v2 with: args: install ${{ matrix.config.chocolatey_opt }} openssl -y - - name: Set up Visual Studio shell - uses: egor-tensin/vs-shell@v2 + - name: Set up with Developer Command Prompt for Microsoft Visual C++ + uses: ilammy/msvc-dev-cmd@v1 with: arch: ${{ matrix.config.arch }} + - name: Build openssl with vcpkg + if: ${{ matrix.config.arch == 'amd64_arm64' }} + run: | + C:\vcpkg\vcpkg install --recurse openssl --triplet ${{ matrix.config.vcpkg_triplet }} + shell: cmd + - name: Build Fluent Bit packages + # If we are using 2.0.* or earlier we need to exclude the ARM64 build as the dependencies fail to compile. + # Trying to do via an exclude for the job triggers linting errors. + # This is only supposed to be a workaround for now so can be easily removed later. + if: ${{ matrix.config.arch != 'amd64_arm64' || needs.call-build-windows-get-meta.outputs.armSupported == 'true' }} run: | - cmake -G "NMake Makefiles" -DFLB_NIGHTLY_BUILD=${{ inputs.unstable }} -DOPENSSL_ROOT_DIR='${{ matrix.config.openssl_dir }}' ../ + cmake -G "NMake Makefiles" -DFLB_NIGHTLY_BUILD='${{ inputs.unstable }}' -DOPENSSL_ROOT_DIR='${{ matrix.config.openssl_dir }}' ${{ matrix.config.cmake_additional_opt }} ../ cmake --build . cpack working-directory: build - name: Upload build packages + # Skip upload if we skipped build. + if: ${{ matrix.config.arch != 'amd64_arm64' || needs.call-build-windows-get-meta.outputs.armSupported == 'true' }} uses: actions/upload-artifact@v3 with: name: windows-packages diff --git a/.github/workflows/call-integration-image-build.yaml b/.github/workflows/call-integration-image-build.yaml index edfeb85f80c..f109b31d2dc 100644 --- a/.github/workflows/call-integration-image-build.yaml +++ b/.github/workflows/call-integration-image-build.yaml @@ -62,7 +62,7 @@ jobs: raw,${{ inputs.image-tag }} - name: Build the AMD64 image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: file: ./dockerfiles/Dockerfile context: . @@ -97,7 +97,7 @@ jobs: raw,${{ inputs.image-tag }}-debug - name: Build the AMD64 debug image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: file: ./dockerfiles/Dockerfile context: . diff --git a/.github/workflows/cron-scorecards-analysis.yaml b/.github/workflows/cron-scorecards-analysis.yaml index e32514ff616..c7eb440d0f2 100644 --- a/.github/workflows/cron-scorecards-analysis.yaml +++ b/.github/workflows/cron-scorecards-analysis.yaml @@ -30,7 +30,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@e38b1902ae4f44df626f11ba0734b14fb91f8f86 + uses: ossf/scorecard-action@80e868c13c90f172d68d1f4501dee99e2479f7af with: results_file: scorecard-results.sarif results_format: sarif diff --git a/.github/workflows/cron-stale.yaml b/.github/workflows/cron-stale.yaml index dd8359fcc46..523685c5085 100644 --- a/.github/workflows/cron-stale.yaml +++ b/.github/workflows/cron-stale.yaml @@ -8,7 +8,7 @@ jobs: name: Mark stale runs-on: ubuntu-latest steps: - - uses: actions/stale@v7 + - uses: actions/stale@v8 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 5 days. Maintainers can add the `exempt-stale` label.' diff --git a/.github/workflows/cron-unstable-build.yaml b/.github/workflows/cron-unstable-build.yaml index 2e38862621e..7ca581deab1 100644 --- a/.github/workflows/cron-unstable-build.yaml +++ b/.github/workflows/cron-unstable-build.yaml @@ -13,7 +13,7 @@ on: # Run nightly build at this time, bit of trial and error but this seems good. schedule: - cron: "0 6 * * *" # master build - - cron: "0 12 * * *" # 1.9 build + - cron: "0 12 * * *" # 2.0 build # We do not want a new unstable build to run whilst we are releasing the current unstable build. concurrency: unstable-build-release @@ -51,10 +51,10 @@ jobs: echo "cron_branch=master" >> $GITHUB_ENV shell: bash - - name: 1.9 run + - name: 2.0 run if: github.event_name == 'schedule' && github.event.schedule=='0 12 * * *' run: | - echo "cron_branch=1.9" >> $GITHUB_ENV + echo "cron_branch=2.0" >> $GITHUB_ENV shell: bash - name: Output the branch to use diff --git a/.github/workflows/pr-compile-check.yaml b/.github/workflows/pr-compile-check.yaml index dde9cb83a45..b1b5dead0f3 100644 --- a/.github/workflows/pr-compile-check.yaml +++ b/.github/workflows/pr-compile-check.yaml @@ -22,7 +22,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Attempt to build current source for CentOS 7 - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: context: . file: ./dockerfiles/Dockerfile.centos7 diff --git a/.github/workflows/pr-image-tests.yaml b/.github/workflows/pr-image-tests.yaml index 7fbacdfb97f..7f340a47ae2 100644 --- a/.github/workflows/pr-image-tests.yaml +++ b/.github/workflows/pr-image-tests.yaml @@ -24,7 +24,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Build the multi-arch images - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: file: ./dockerfiles/Dockerfile context: . @@ -34,7 +34,7 @@ jobs: load: false - name: Build the debug multi-arch images - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: file: ./dockerfiles/Dockerfile context: . diff --git a/.github/workflows/pr-windows-build.yaml b/.github/workflows/pr-windows-build.yaml index 3787107697e..2831122fe33 100644 --- a/.github/workflows/pr-windows-build.yaml +++ b/.github/workflows/pr-windows-build.yaml @@ -18,7 +18,7 @@ on: jobs: pr-windows-build: - uses: fluent/fluent-bit/.github/workflows/call-build-windows.yaml@master + uses: ./.github/workflows/call-build-windows.yaml with: version: ${{ github.sha }} ref: ${{ github.sha }} diff --git a/.github/workflows/staging-release.yaml b/.github/workflows/staging-release.yaml index 30730cc3d15..7b2ea163826 100644 --- a/.github/workflows/staging-release.yaml +++ b/.github/workflows/staging-release.yaml @@ -64,19 +64,48 @@ jobs: env: RELEASE_VERSION: ${{ github.event.inputs.version }} + staging-release-generate-package-matrix: + name: Get package matrix + runs-on: ubuntu-latest + outputs: + deb-build-matrix: ${{ steps.get-matrix.outputs.deb-build-matrix }} + rpm-build-matrix: ${{ steps.get-matrix.outputs.rpm-build-matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup runner + run: | + sudo apt-get update + sudo apt-get install -y jq + shell: bash + + # Cope with 1.9 as well as 2.0 + - uses: ./.github/actions/generate-package-build-matrix + id: get-matrix + with: + ref: v${{ inputs.version }} + + # Now annotate with whether it is Yum or Apt based + # 1. Take packages from the staging bucket # 2. Sign them with the release GPG key # 3. Also take existing release packages from the release bucket. # 4. Create a full repo configuration using the existing releases as well. # 5. Upload to release bucket. # Note we could resign all packages as well potentially if we wanted to update the key. - staging-release-packages: - name: S3 - update packages bucket + staging-release-yum-packages: + name: S3 - update YUM packages bucket runs-on: ubuntu-22.04 # no createrepo on Ubuntu 20.04 environment: release - needs: staging-release-version-check + needs: + - staging-release-version-check + - staging-release-generate-package-matrix permissions: contents: read + strategy: + matrix: ${{ fromJSON(needs.staging-release-generate-package-matrix.outputs.rpm-build-matrix) }} + fail-fast: false steps: - name: Checkout code uses: actions/checkout@v3 @@ -84,7 +113,7 @@ jobs: - name: Setup runner run: | sudo apt-get update - sudo apt-get install -y debsigs createrepo-c aptly rsync + sudo apt-get install -y createrepo-c rpm shell: bash - name: Import GPG key for signing @@ -101,39 +130,181 @@ jobs: - name: Sync packages from buckets on S3 run: | - mkdir -p packaging/releases - aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}" packaging/releases/ --no-progress - aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}" packaging/releases/ --no-progress + mkdir -p "packaging/releases/$DISTRO" + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: "us-east-1" + DISTRO: ${{ matrix.distro }} shell: bash - name: GPG set up keys for signing run: | - gpg --export -a "${{ steps.import_gpg.outputs.name }}" > packaging/releases/fluentbit.key - rpm --import packaging/releases/fluentbit.key + gpg --export -a "${{ steps.import_gpg.outputs.name }}" > /tmp/fluentbit.key + rpm --import /tmp/fluentbit.key + shell: bash + + - name: Update repo info and remove any staging details + run: | + packaging/update-yum-repo.sh + env: + GPG_KEY: ${{ steps.import_gpg.outputs.name }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }} + VERSION: ${{ github.event.inputs.version }} + BASE_PATH: "packaging/releases" + RPM_REPO: ${{ matrix.distro }} + shell: bash + + - name: Sync to release bucket on S3 + run: | + aws s3 sync "packaging/releases/$DISTRO" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" --delete --follow-symlinks --no-progress + aws s3 sync "packaging/releases/*.repo" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/" --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + DISTRO: ${{ matrix.distro }} + shell: bash + + staging-release-apt-packages: + name: S3 - update APT packages bucket + runs-on: ubuntu-latest + environment: release + needs: + - staging-release-version-check + - staging-release-generate-package-matrix + permissions: + contents: read + strategy: + matrix: ${{ fromJSON(needs.staging-release-generate-package-matrix.outputs.deb-build-matrix) }} + fail-fast: false + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup runner + run: | + sudo apt-get update + sudo apt-get install -y debsigs aptly rsync + shell: bash + + - name: Import GPG key for signing + id: import_gpg + uses: crazy-max/ghaction-import-gpg@v5 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }} + + - name: Sync packages from buckets on S3 + run: | + mkdir -p "packaging/releases/$DISTRO" + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + DISTRO: ${{ matrix.distro }} shell: bash - name: Update repo info and remove any staging details run: | - rm -f packaging/releases/*.repo - rm -f packaging/releases/latest-version.txt - packaging/update-repos.sh packaging/releases/ + packaging/update-apt-repo.sh env: GPG_KEY: ${{ steps.import_gpg.outputs.name }} AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }} VERSION: ${{ github.event.inputs.version }} + BASE_PATH: "packaging/releases" + DEB_REPO: ${{ matrix.distro }} + shell: bash + + - name: Sync to release bucket on S3 + run: | + aws s3 sync "packaging/releases/$DISTRO" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" --delete --follow-symlinks --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + DISTRO: ${{ matrix.distro }} + shell: bash + + staging-release-update-non-linux-s3: + name: Update Windows and macOS packages + runs-on: ubuntu-22.04 + environment: release + needs: + - staging-release-version-check + permissions: + contents: none + strategy: + matrix: + distro: + - macos + - windows + fail-fast: false + steps: + - name: Sync packages from buckets on S3 + run: | + mkdir -p "packaging/releases/$DISTRO" + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + DISTRO: ${{ matrix.distro }} shell: bash - name: Sync to release bucket on S3 run: | - aws s3 sync packaging/releases/ "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}" --delete --follow-symlinks --no-progress + aws s3 sync "packaging/releases/$DISTRO" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" --delete --follow-symlinks --no-progress env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: "us-east-1" + DISTRO: ${{ matrix.distro }} + shell: bash + + staging-release-update-base-s3: + name: Update top-level bucket info + runs-on: ubuntu-22.04 + environment: release + needs: + - staging-release-apt-packages + - staging-release-yum-packages + permissions: + contents: none + steps: + - name: Import GPG key for signing + id: import_gpg + uses: crazy-max/ghaction-import-gpg@v5 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }} + + - name: GPG public key + run: | + gpg --export -a "${{ steps.import_gpg.outputs.name }}" > ./fluentbit.key + aws s3 cp ./fluentbit.key s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/fluentbit.key --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + shell: bash + + - name: JSON schema + continue-on-error: true + run: | + aws s3 sync "s3://${AWS_STAGING_S3_BUCKET}/${VERSION}" "s3://${AWS_RELEASE_S3_BUCKET}/${VERSION}" --no-progress + env: + VERSION: ${{ github.event.inputs.version }} + AWS_REGION: "us-east-1" + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_STAGING_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_STAGING }} + AWS_RELEASE_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }} shell: bash staging-release-source-s3: @@ -191,6 +362,43 @@ jobs: AWS_REGION: "us-east-1" shell: bash + - name: Provide output for documentation PR + # do not fail the build for this + continue-on-error: true + run: | + ls -l $BASE_DIR/ + export WIN_32_EXE_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win32.exe.sha256"|awk '{print $1}') + export WIN_32_ZIP_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win32.zip.sha256"|awk '{print $1}') + export WIN_64_EXE_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win64.exe.sha256"|awk '{print $1}') + export WIN_64_ZIP_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win64.zip.sha256"|awk '{print $1}') + cat > windows-part.md << EOF + ## Installation Packages + + The latest stable version is ${{ inputs.version }}, each version is available on the Github release as well as at `https://releases.fluentbit.io//fluent-bit--win[32|64].[exe|zip]`: + + | INSTALLERS | SHA256 CHECKSUMS | + | ------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------- | + | [fluent-bit-${{ inputs.version }}-win32.exe](https://releases.fluentbit.io/${{ needs.staging-release-version-check.outputs.major-version }}/fluent-bit-${{ inputs.version }}-win32.exe) | [$WIN_32_EXE_HASH](https://releases.fluentbit.io/2.0/fluent-bit-${{ inputs.version }}-win32.exe.sha256) | + | [fluent-bit-${{ inputs.version }}-win32.zip](https://releases.fluentbit.io/${{ needs.staging-release-version-check.outputs.major-version }}/fluent-bit-${{ inputs.version }}-win32.zip) | [$WIN_32_ZIP_HASH](https://releases.fluentbit.io/2.0/fluent-bit-${{ inputs.version }}-win32.zip.sha256) | + | [fluent-bit-${{ inputs.version }}-win64.exe](https://releases.fluentbit.io/${{ needs.staging-release-version-check.outputs.major-version }}/fluent-bit-${{ inputs.version }}-win64.exe) | [$WIN_64_EXE_HASH](https://releases.fluentbit.io/2.0/fluent-bit-${{ inputs.version }}-win64.exe.sha256) | + | [fluent-bit-${{ inputs.version }}-win64.zip](https://releases.fluentbit.io/${{ needs.staging-release-version-check.outputs.major-version }}/fluent-bit-${{ inputs.version }}-win64.zip) | [$WIN_64_ZIP_HASH](https://releases.fluentbit.io/2.0/fluent-bit-${{ inputs.version }}-win64.zip.sha256) | + + To check the integrity, use `Get-FileHash` cmdlet on PowerShell. + + ```powershell + PS> Get-FileHash fluent-bit-${{ inputs.version }}-win32.exe + ``` + EOF + shell: bash + env: + BASE_DIR: release/${{ needs.staging-release-version-check.outputs.major-version }} + + - name: Upload Windows docs + uses: actions/upload-artifact@v3 + with: + name: windows-part + path: windows-part.md + # Simple skopeo copy jobs to transfer image from staging to release registry with optional GPG key signing. # Unfortunately skopeo currently does not support Cosign: https://github.com/containers/skopeo/issues/1533 staging-release-images: @@ -476,7 +684,8 @@ jobs: runs-on: ubuntu-latest environment: release needs: - - staging-release-packages + - staging-release-apt-packages + - staging-release-yum-packages steps: - name: Checkout code uses: actions/checkout@v3 @@ -514,7 +723,8 @@ jobs: name: Create the Github Release once packages and containers are up needs: - staging-release-images - - staging-release-packages + - staging-release-apt-packages + - staging-release-yum-packages permissions: contents: write environment: release @@ -533,10 +743,68 @@ jobs: - name: Release 2.0 and latest uses: softprops/action-gh-release@v1 - if: startsWith(inputs.version, '2.0') + if: startsWith(inputs.version, '2.') with: body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" draft: false generate_release_notes: false name: "Fluent Bit ${{ inputs.version }}" tag_name: v${{ inputs.version }} + + staging-release-create-docs-pr: + name: Create docs updates for new release + needs: + - staging-release-images + - staging-release-source-s3 + permissions: + contents: none + environment: release + runs-on: ubuntu-latest + steps: + - name: Release 1.9 - not latest + if: startsWith(inputs.version, '1.9') + uses: actions/checkout@v3 + with: + repository: fluent/fluent-bit-docs + ref: 1.9 + + - name: Release 2.0 and latest + if: startsWith(inputs.version, '2.') + uses: actions/checkout@v3 + with: + repository: fluent/fluent-bit-docs + + - name: Update container tags + run: | + sed -i -e '/| Tag(s).*$/a | ${{ inputs.version }}-debug | x86\_64, arm64v8, arm32v7 | Release [v${{ inputs.version }}](https://fluentbit.io/announcements/v${{ inputs.version }}/) |' installation/docker.md + sed -i -e '/| Tag(s).*$/a | ${{ inputs.version }} | x86\_64, arm64v8, arm32v7 | Debug images |' installation/docker.md + shell: bash + + - name: Download Windows part + uses: actions/download-artifact@v3 + with: + name: windows-part + + - name: Update Windows version + run: | + # Remove old info + sed -i '/## Installation Packages/,/## Installing from ZIP archive/{//p;d;}' installation/windows.md + # Add new info (from previous job) at the end for simplicity + cat windows-part.md >> installation/windows.md + shell: bash + + - name: Raise docs PR + uses: peter-evans/create-pull-request@v4 + with: + commit-message: 'release: update to v${{ inputs.version }}' + signoff: true + delete-branch: true + title: 'release: update to v${{ inputs.version }}' + # We need workflows permission so have to use the CI_PAT + token: ${{ secrets.CI_PAT }} + labels: ci,automerge + body: | + Update release ${{ inputs.version }} version. + - Created by ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + - Auto-generated by create-pull-request: https://github.com/peter-evans/create-pull-request + draft: false diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 2b0ed8d7daf..92756665bfd 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -3,6 +3,7 @@ on: push: branches: - master + - 2.0 - 1.9 - 1.8 pull_request: @@ -17,6 +18,7 @@ on: - 'examples/**' branches: - master + - 2.0 - 1.9 - 1.8 types: [opened, reopened, synchronize] diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 7f555564f19..8a8bbd91054 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -224,6 +224,8 @@ RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/ net-tools mtr netcat-openbsd bridge-utils iperf ngrep \ openssl \ htop atop strace iotop sysstat ncdu logrotate hdparm pciutils psmisc tree pv \ + cmake make tar flex bison \ + libssl-dev libsasl2-dev libsystemd-dev/bullseye-backports zlib1g-dev libpq-dev libyaml-dev postgresql-server-dev-all \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/packaging/build-config.json b/packaging/build-config.json index 8e73f40fb74..708fa09005a 100644 --- a/packaging/build-config.json +++ b/packaging/build-config.json @@ -1,28 +1,105 @@ { "linux_targets": [ - "amazonlinux/2", - "amazonlinux/2.arm64v8", - "amazonlinux/2022", - "centos/7", - "centos/7.arm64v8", - "centos/8", - "centos/8.arm64v8", - "centos/9", - "centos/9.arm64v8", - "debian/bookworm", - "debian/bookworm.arm64v8", - "debian/buster", - "debian/buster.arm64v8", - "debian/bullseye", - "debian/bullseye.arm64v8", - "ubuntu/16.04", - "ubuntu/18.04", - "ubuntu/18.04.arm64v8", - "ubuntu/20.04", - "ubuntu/20.04.arm64v8", - "ubuntu/22.04", - "ubuntu/22.04.arm64v8", - "raspbian/buster", - "raspbian/bullseye" + { + "target": "amazonlinux/2", + "type": "rpm" + }, + { + "target": "amazonlinux/2.arm64v8", + "type": "rpm" + }, + { + "target": "amazonlinux/2023", + "type": "rpm" + }, + { + "target": "centos/7", + "type": "rpm" + }, + { + "target": "centos/7.arm64v8", + "type": "rpm" + }, + { + "target": "centos/8", + "type": "rpm" + }, + { + "target": "centos/8.arm64v8", + "type": "rpm" + }, + { + "target": "centos/9", + "type": "rpm" + }, + { + "target": "centos/9.arm64v8", + "type": "rpm" + }, + { + "target": "debian/bookworm", + "type": "deb" + }, + { + "target": "debian/bookworm.arm64v8", + "type": "deb" + }, + { + "target": "debian/buster", + "type": "deb" + }, + { + "target": "debian/buster.arm64v8", + "type": "deb" + }, + { + "target": "debian/bullseye", + "type": "deb" + }, + { + "target": "debian/bullseye.arm64v8", + "type": "deb" + }, + { + "target": "ubuntu/16.04", + "type": "deb" + }, + { + "target": "ubuntu/18.04", + "type": "deb" + }, + { + "target": "ubuntu/18.04.arm64v8", + "type": "deb" + }, + { + "target": "ubuntu/20.04", + "type": "deb" + }, + { + "target": "ubuntu/20.04.arm64v8", + "type": "deb" + }, + { + "target": "ubuntu/22.04", + "type": "deb" + }, + { + "target": "ubuntu/22.04.arm64v8", + "type": "deb" + }, + { + "target": "raspbian/buster", + "type": "deb" + }, + { + "target": "raspbian/bullseye", + "type": "deb" + } + ], + "windows_targets" : [ + "x86", + "x64", + "amd64_arm64" ] } diff --git a/packaging/distros/amazonlinux/Dockerfile b/packaging/distros/amazonlinux/Dockerfile index 6d081d80231..fa9b17c5bd7 100644 --- a/packaging/distros/amazonlinux/Dockerfile +++ b/packaging/distros/amazonlinux/Dockerfile @@ -35,7 +35,7 @@ RUN yum -y update && \ cmake3 libyaml-devel zlib-devel && \ yum clean all -FROM amazonlinux:2022 as amazonlinux-2022-base +FROM amazonlinux:2023 as amazonlinux-2023-base # hadolint ignore=DL3033 RUN yum -y update && \ @@ -47,7 +47,7 @@ RUN yum -y update && \ yum clean all # hadolint ignore=DL3029 -FROM --platform=arm64 amazonlinux:2022 as amazonlinux-2022.arm64v8-base +FROM --platform=arm64 amazonlinux:2023 as amazonlinux-2023.arm64v8-base COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static diff --git a/packaging/test-release-packages.sh b/packaging/test-release-packages.sh index 985b5a4a681..b157505c932 100755 --- a/packaging/test-release-packages.sh +++ b/packaging/test-release-packages.sh @@ -46,7 +46,7 @@ YUM_TARGETS=("centos:7" "rockylinux:8" "quay.io/centos/centos:stream9" "amazonlinux:2" - "amazonlinux:2022") + "amazonlinux:2023") for IMAGE in "${YUM_TARGETS[@]}" do diff --git a/packaging/update-apt-repo.sh b/packaging/update-apt-repo.sh new file mode 100755 index 00000000000..e600697ea0c --- /dev/null +++ b/packaging/update-apt-repo.sh @@ -0,0 +1,71 @@ +#!/bin/bash +set -eux + +# Where the base of all the repos is +BASE_PATH=${BASE_PATH:?} +if [[ ! -d "$BASE_PATH" ]]; then + echo "ERROR: invalid base path: $BASE_PATH" + exit 1 +fi + +# "debian/bookworm" "debian/bullseye" "debian/buster" "ubuntu/xenial" "ubuntu/bionic" "ubuntu/focal" "ubuntu/jammy" "raspbian/buster" "raspbian/bullseye" +DEB_REPO=${DEB_REPO:?} + +REPO_DIR=$(realpath -sm "$BASE_PATH/$DEB_REPO" ) +if [[ ! -d "$REPO_DIR" ]] ; then + echo "ERROR: missing $REPO_DIR" + exit 1 +fi + +CODENAME=${DEB_REPO##*/} +echo "Updating $DEB_REPO for $CODENAME" + +# We use Aptly to create repos with a local temporary directory as the root. +# Once complete, we then move these to the output directory for upload. +# Based on https://github.com/spotify/debify/blob/master/debify.sh +APTLY_REPO_NAME="debify-$CODENAME" +APTLY_ROOTDIR=$(mktemp -d) +APTLY_CONFIG=$(mktemp) + +# The origin and label fields seem to cover the base directory for the repo and codename. +# The docs seems to suggest these fields are optional and free-form: https://wiki.debian.org/DebianRepository/Format#Origin +# They are security checks to verify if they have changed so we match the legacy server. +APTLY_ORIGIN=". $CODENAME" +APTLY_LABEL=". $CODENAME" +if [[ "$DEB_REPO" == "debian/bullseye" ]]; then + # For Bullseye, the legacy server had a slightly different setup we try to reproduce here + APTLY_ORIGIN="bullseye bullseye" + APTLY_LABEL="bullseye bullseye" +fi + +cat << EOF > "$APTLY_CONFIG" +{ + "rootDir": "$APTLY_ROOTDIR/" +} +EOF +cat "$APTLY_CONFIG" + +aptly -config="$APTLY_CONFIG" repo create \ + -component="main" \ + -distribution="$CODENAME" \ + "$APTLY_REPO_NAME" + +# Check if any files to add +count=$(find "$REPO_DIR" -maxdepth 1 -type f -name "*.deb" | wc -l) +if [[ $count != 0 ]] ; then + # Do not remove files as we need them from moving to staging-release + aptly -config="$APTLY_CONFIG" repo add -force-replace "$APTLY_REPO_NAME" "$REPO_DIR/" +else + echo "WARNING: no files to add in $DEB_REPO for $CODENAME" +fi +aptly -config="$APTLY_CONFIG" repo show "$APTLY_REPO_NAME" + +if [[ "$DISABLE_SIGNING" != "true" ]]; then + aptly -config="$APTLY_CONFIG" publish repo -gpg-key="$GPG_KEY" -origin="$APTLY_ORIGIN" -label="$APTLY_LABEL" "$APTLY_REPO_NAME" +else + aptly -config="$APTLY_CONFIG" publish repo --skip-signing -origin="$APTLY_ORIGIN" -label="$APTLY_LABEL" "$APTLY_REPO_NAME" +fi + +rsync -av "$APTLY_ROOTDIR"/public/* "$REPO_DIR" +# Remove unnecessary files +rm -rf "$REPO_DIR/conf/" "$REPO_DIR/db/" "$APTLY_ROOTDIR" "$APTLY_CONFIG" diff --git a/packaging/update-repos.sh b/packaging/update-repos.sh index 3e607706dcd..a4d711e950c 100755 --- a/packaging/update-repos.sh +++ b/packaging/update-repos.sh @@ -1,69 +1,50 @@ #!/bin/bash set -eux - -# Where the base of all the repos is -BASE_PATH=${BASE_PATH:-$1} +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Wrapper script around the actual ones used in CI +# Intended only for legacy/manual use in event of failure in CI +# Ensure to add dependencies, e.g. for Ubuntu 22.04: awscli git createrepo-c debsigs aptly rsync gnupg2 +# Following that there are a few things to do: +# Import the signing key (if signing) +# gpg --import +# gpg --export -a "$GPG_KEY" > /tmp/fluentbit.key +# rpm --import /tmp/fluentbit.key + +export BASE_PATH=${BASE_PATH:-$1} if [[ ! -d "$BASE_PATH" ]]; then - echo "Invalid base path: $BASE_PATH" + echo "Specified BASE_PATH is not a directory: $BASE_PATH" exit 1 fi -# Set true to prevent signing -DISABLE_SIGNING=${DISABLE_SIGNING:-false} -if [[ "$DISABLE_SIGNING" != "true" ]]; then - echo "RPM signing configuration" - rpm -q gpg-pubkey --qf '%{name}-%{version}-%{release} --> %{summary}\n' +export DISABLE_SIGNING=${DISABLE_SIGNING:-false} +export CREATE_REPO_CMD=${CREATE_REPO_CMD:-} +export CREATE_REPO_ARGS=${CREATE_REPO_ARGS:--dvp} +# Must be set for signing +if [[ "$DISABLE_SIGNING" != "false" ]]; then + export GPG_KEY=${GPG_KEY:?} fi -# Handle Ubuntu 18/22 differences - no support on Ubuntu 20 -CREATE_REPO_CMD=${CREATE_REPO_CMD:-} -CREATE_REPO_ARGS=${CREATE_REPO_ARGS:--dvp} +# Set these to force a manual S3 sync and update +# AWS_SYNC=true +# AWS_S3_BUCKET_RELEASE=packages.fluentbit.io +# AWS_S3_BUCKET_STAGING=fluentbit-staging +export AWS_REGION=${AWS_REGION:-us-east-1} -# Assume if set we want to use it -if [[ -n "$CREATE_REPO_CMD" ]]; then - echo "Using $CREATE_REPO_CMD" -elif command -v createrepo &> /dev/null; then - echo "Found createrepo" - CREATE_REPO_CMD="createrepo" -elif command -v createrepo_c &> /dev/null; then - echo "Found createrepo_c" - CREATE_REPO_CMD="createrepo_c" -else - echo "Unable to find a command equivalent to createrepo" - exit 1 -fi +RPM_REPO_PATHS=("amazonlinux/2" "amazonlinux/2023" "centos/7" "centos/8" "centos/9") -RPM_REPO_PATHS=("amazonlinux/2" "amazonlinux/2022" "centos/7" "centos/8" "centos/9") +if [[ "${AWS_SYNC:-false}" != "false" ]]; then + aws s3 sync s3://"${AWS_S3_BUCKET_RELEASE:?}" "${BASE_PATH:?}" +fi for RPM_REPO in "${RPM_REPO_PATHS[@]}"; do - echo "Updating $RPM_REPO" - REPO_DIR=$( realpath -sm "$BASE_PATH/$RPM_REPO" ) - [[ ! -d "$REPO_DIR" ]] && continue + export RPM_REPO - if [[ "$DISABLE_SIGNING" != "true" ]]; then - # Sign all RPMs created for this target, cover both fluent-bit and legacy packages for 1.9 branch - find "$REPO_DIR" -name "*-bit-*.rpm" -exec rpm --define "_gpg_name $GPG_KEY" --addsign {} \; + if [[ "${AWS_SYNC:-false}" != "false" ]]; then + aws s3 sync s3://"${AWS_S3_BUCKET_STAGING:?}/$RPM_REPO" "${BASE_PATH:?}/$RPM_REPO" fi - # Create full metadata for all RPMs in the directory - "$CREATE_REPO_CMD" "$CREATE_REPO_ARGS" "$REPO_DIR" - # Set up repo info - if [[ -n "${AWS_S3_BUCKET:-}" ]]; then - # Create top-level file so replace path separator with dash - # centos/8 --> centos-8.repo - # This way we make sure not to have a mixed repo or overwrite files for each target. - REPO_TYPE=${RPM_REPO/\//-} - echo "Setting up $BASE_PATH/$REPO_TYPE.repo" - cat << EOF > "$BASE_PATH/$REPO_TYPE.repo" -[Fluent-Bit] -name=Fluent Bit Packages - $REPO_TYPE - \$basearch -baseurl=https://$AWS_S3_BUCKET.s3.amazonaws.com/$RPM_REPO/ -enabled=1 -gpgkey=https://$AWS_S3_BUCKET.s3.amazonaws.com/fluentbit.key -gpgcheck=1 -repo_gpgcheck=1 -EOF - fi + /bin/bash -eux "$SCRIPT_DIR/update-yum-repo.sh" done DEB_REPO_PATHS=( "debian/bookworm" @@ -77,68 +58,18 @@ DEB_REPO_PATHS=( "debian/bookworm" "raspbian/bullseye" ) for DEB_REPO in "${DEB_REPO_PATHS[@]}"; do - REPO_DIR=$(realpath -sm "$BASE_PATH/$DEB_REPO" ) - [[ ! -d "$REPO_DIR" ]] && continue - - CODENAME=${DEB_REPO##*/} - echo "Updating $DEB_REPO for $CODENAME" - - # We use Aptly to create repos with a local temporary directory as the root. - # Once complete, we then move these to the output directory for upload. - # Based on https://github.com/spotify/debify/blob/master/debify.sh - APTLY_REPO_NAME="debify-$CODENAME" - APTLY_ROOTDIR=$(mktemp -d) - APTLY_CONFIG=$(mktemp) - - # The origin and label fields seem to cover the base directory for the repo and codename. - # The docs seems to suggest these fields are optional and free-form: https://wiki.debian.org/DebianRepository/Format#Origin - # They are security checks to verify if they have changed so we match the legacy server. - APTLY_ORIGIN=". $CODENAME" - APTLY_LABEL=". $CODENAME" - if [[ "$DEB_REPO" == "debian/bullseye" ]]; then - # For Bullseye, the legacy server had a slightly different setup we try to reproduce here - APTLY_ORIGIN="bullseye bullseye" - APTLY_LABEL="bullseye bullseye" - fi - - cat << EOF > "$APTLY_CONFIG" -{ - "rootDir": "$APTLY_ROOTDIR/" -} -EOF - cat "$APTLY_CONFIG" - - aptly -config="$APTLY_CONFIG" repo create \ - -component="main" \ - -distribution="$CODENAME" \ - "$APTLY_REPO_NAME" - # Check if any files to add - count=$(find "$REPO_DIR" -maxdepth 1 -type f -name "*.deb" | wc -l) - if [[ $count != 0 ]] ; then - # Do not remove files as we need them from moving to staging-release - aptly -config="$APTLY_CONFIG" repo add -force-replace "$APTLY_REPO_NAME" "$REPO_DIR/" - else - echo "No files to add in $DEB_REPO for $CODENAME" - fi - aptly -config="$APTLY_CONFIG" repo show "$APTLY_REPO_NAME" - if [[ "$DISABLE_SIGNING" != "true" ]]; then - aptly -config="$APTLY_CONFIG" publish repo -gpg-key="$GPG_KEY" -origin="$APTLY_ORIGIN" -label="$APTLY_LABEL" "$APTLY_REPO_NAME" - else - aptly -config="$APTLY_CONFIG" publish repo --skip-signing -origin="$APTLY_ORIGIN" -label="$APTLY_LABEL" "$APTLY_REPO_NAME" + export DEB_REPO + if [[ "${AWS_SYNC:-false}" != "false" ]]; then + aws s3 sync s3://"${AWS_S3_BUCKET_STAGING:?}/$DEB_REPO" "${BASE_PATH:?}/$DEB_REPO" fi - rsync -av "$APTLY_ROOTDIR"/public/* "$REPO_DIR" - # Remove unnecessary files - rm -rf "$REPO_DIR/conf/" "$REPO_DIR/db/" "$APTLY_ROOTDIR" "$APTLY_CONFIG" + /bin/bash -eux "$SCRIPT_DIR/update-apt-repo.sh" done -# Ensure we sign the Yum repo meta-data -if [[ "$DISABLE_SIGNING" != "true" ]]; then - # We use this form to fail on error during the find, otherwise -exec will succeed or just do one file with + - while IFS= read -r -d '' REPO_METADATA_FILE - do - echo "Signing $REPO_METADATA_FILE" - gpg --detach-sign --batch --armor --yes -u "$GPG_KEY" "$REPO_METADATA_FILE" - done < <(find "$BASE_PATH" -name repomd.xml -print0) - # Debug ouput for checking - find "$BASE_PATH" -name "repomd.xml*" -exec ls -l {} \; +# Other OS now +if [[ "${AWS_SYNC:-false}" != "false" ]]; then + aws s3 sync s3://"${AWS_S3_BUCKET_STAGING:?}/macos" "${BASE_PATH:?}/macos" + aws s3 sync s3://"${AWS_S3_BUCKET_STAGING:?}/windows" "${BASE_PATH:?}/windows" + + # Final review, do not push until checked manually + aws s3 sync "${BASE_PATH:?}" s3://"${AWS_S3_BUCKET_RELEASE:?}" --exact-timestamps --dryrun fi diff --git a/packaging/update-source-packages.sh b/packaging/update-source-packages.sh index 184af3980b7..92af9b787c6 100755 --- a/packaging/update-source-packages.sh +++ b/packaging/update-source-packages.sh @@ -60,9 +60,13 @@ else fi # Source - we do want word splitting and ensure some files exist -if compgen -G "$SOURCE_DIR/source/*$VERSION*" > /dev/null; then +if compgen -G "$SOURCE_DIR/source-$VERSION*" > /dev/null; then echo "Copying source artefacts" # shellcheck disable=SC2086 + cp -vf "$SOURCE_DIR"/source-$VERSION* "$TARGET_DIR/$MAJOR_VERSION/" +elif compgen -G "$SOURCE_DIR/source/*$VERSION*" > /dev/null; then + echo "Copying (legacy) source artefacts" + # shellcheck disable=SC2086 cp -vf "$SOURCE_DIR"/source/*$VERSION* "$TARGET_DIR/$MAJOR_VERSION/" else echo "Missing source artefacts" diff --git a/packaging/update-yum-repo.sh b/packaging/update-yum-repo.sh new file mode 100755 index 00000000000..2a91fc2ca75 --- /dev/null +++ b/packaging/update-yum-repo.sh @@ -0,0 +1,85 @@ +#!/bin/bash +set -eux + +#("amazonlinux/2" "amazonlinux/2022" "centos/7" "centos/8" "centos/9") +RPM_REPO=${RPM_REPO:?} + +# Where the base of all the repos is +BASE_PATH=${BASE_PATH:-$1} +if [[ ! -d "$BASE_PATH" ]]; then + echo "ERROR: invalid base path: $BASE_PATH" + exit 1 +fi + +# Set true to prevent signing +DISABLE_SIGNING=${DISABLE_SIGNING:-false} +if [[ "$DISABLE_SIGNING" != "true" ]]; then + echo "INFO: RPM signing configuration" + rpm --showrc|grep gpg + rpm -q gpg-pubkey --qf '%{name}-%{version}-%{release} --> %{summary}\n' +fi + +# Handle Ubuntu 18/22 differences - no support on Ubuntu 20 +CREATE_REPO_CMD=${CREATE_REPO_CMD:-} +CREATE_REPO_ARGS=${CREATE_REPO_ARGS:--dvp} + +# Assume if set we want to use it +if [[ -n "$CREATE_REPO_CMD" ]]; then + echo "INFO: using $CREATE_REPO_CMD" +elif command -v createrepo &> /dev/null; then + echo "INFO: found createrepo" + CREATE_REPO_CMD="createrepo" +elif command -v createrepo_c &> /dev/null; then + echo "INFO: found createrepo_c" + CREATE_REPO_CMD="createrepo_c" +else + echo "ERROR: unable to find a command equivalent to createrepo" + exit 1 +fi + +echo "INFO: updating $RPM_REPO" + +REPO_DIR=$( realpath -sm "$BASE_PATH/$RPM_REPO" ) +if [[ ! -d "$REPO_DIR" ]] ; then + echo "ERROR: missing $REPO_DIR" + exit 1 +fi + +if [[ "$DISABLE_SIGNING" != "true" ]]; then + # Sign all RPMs created for this target, cover both fluent-bit and legacy packages + find "$REPO_DIR" -name "*-bit-*.rpm" -exec rpm --define "_gpg_name $GPG_KEY" --addsign {} \; +fi +# Create full metadata for all RPMs in the directory +"$CREATE_REPO_CMD" "$CREATE_REPO_ARGS" "$REPO_DIR" + +# Set up repo info +if [[ -n "${AWS_S3_BUCKET:-}" ]]; then + # Create top-level file so replace path separator with dash + # centos/8 --> centos-8.repo + # This way we make sure not to have a mixed repo or overwrite files for each target. + REPO_TYPE=${RPM_REPO/\//-} + echo "INFO: setting up $BASE_PATH/$REPO_TYPE.repo" + cat << EOF > "$BASE_PATH/$REPO_TYPE.repo" +[Fluent-Bit] +name=Fluent Bit Packages - $REPO_TYPE - \$basearch +baseurl=https://$AWS_S3_BUCKET.s3.amazonaws.com/$RPM_REPO/ +enabled=1 +gpgkey=https://$AWS_S3_BUCKET.s3.amazonaws.com/fluentbit.key +gpgcheck=1 +repo_gpgcheck=1 +EOF +fi + +# Ensure we sign the Yum repo meta-data +if [[ "$DISABLE_SIGNING" != "true" ]]; then + # We use this form to fail on error during the find, otherwise -exec will succeed or just do one file with + + while IFS= read -r -d '' REPO_METADATA_FILE + do + echo "INFO: signing $REPO_METADATA_FILE" + gpg --detach-sign --batch --armor --yes -u "$GPG_KEY" "$REPO_METADATA_FILE" + done < <(find "$REPO_DIR" -name repomd.xml -print0) + # Debug ouput for checking + find "$REPO_DIR" -name "repomd.xml*" -exec ls -l {} \; +fi + +echo "INFO: Completed $RPM_REPO"