diff --git a/.github/actions/generate-package-build-matrix/action.yaml b/.github/actions/generate-package-build-matrix/action.yaml index bffa7864a32..edc42c0b4ce 100644 --- a/.github/actions/generate-package-build-matrix/action.yaml +++ b/.github/actions/generate-package-build-matrix/action.yaml @@ -14,8 +14,14 @@ inputs: required: true outputs: build-matrix: - description: The build matrix we have created. + description: The total build matrix we have created. value: ${{ steps.set-matrix.outputs.matrix }} + deb-build-matrix: + description: The targets that provide DEB artefacts. + value: ${{ steps.set-matrix.outputs.deb-matrix }} + rpm-build-matrix: + description: The targets that provide RPN artefacts. + value: ${{ steps.set-matrix.outputs.rpm-matrix }} runs: using: "composite" steps: @@ -40,8 +46,13 @@ runs: - name: 2.0 targets if: steps.determine-build-type.outputs.BUILD_TYPE == '2.0' run: | - matrix=$(echo '{ "distro" : '$(jq -cr '.linux_targets' packaging/build-config.json)'}'|jq -c .) + matrix=$(echo '{ "distro" : '$(jq -cr '.linux_targets|map(.target)' packaging/build-config.json)'}'|jq -c .) echo "MATRIX=$matrix" >> $GITHUB_ENV + + deb-matrix = =$(echo '{ "distro" : '$(jq -cr '[.linux_targets[] |select(.type=="deb")|.target ]' packaging/build-config.json)'}'|jq -c .) + echo "DEB_MATRIX=$deb-matrix" >> $GITHUB_ENV + rpm-matrix = =$(echo '{ "distro" : '$(jq -cr '[.linux_targets[] |select(.type=="rpm")|.target ]' packaging/build-config.json)'}'|jq -c .) + echo "RPM_MATRIX=$rpm-matrix" >> $GITHUB_ENV shell: bash - name: 1.9 targets @@ -58,6 +69,22 @@ runs: echo ']}' ) | jq -c .) echo "MATRIX=$matrix" >> $GITHUB_ENV + deb-matrix=$(( + echo '{ "distro" : [' + echo '"debian/buster", "debian/buster.arm64v8", "debian/bullseye", "debian/bullseye.arm64v8",' + echo '"ubuntu/16.04", "ubuntu/18.04", "ubuntu/20.04", "ubuntu/22.04",' + echo '"ubuntu/18.04.arm64v8", "ubuntu/20.04.arm64v8", "ubuntu/22.04.arm64v8",' + echo '"raspbian/buster", "raspbian/bullseye"' + echo ']}' + ) | jq -c .) + echo "DEB_MATRIX=$deb-matrix" >> $GITHUB_ENV + rpm-matrix=$(( + echo '{ "distro" : [' + echo '"amazonlinux/2", "amazonlinux/2.arm64v8",' + echo '"centos/7", "centos/7.arm64v8", "centos/8", "centos/8.arm64v8"' + echo ']}' + ) | jq -c .) + echo "RPM_MATRIX=$rpm-matrix" >> $GITHUB_ENV shell: bash - name: Manual override of target @@ -79,4 +106,10 @@ runs: echo $MATRIX echo $MATRIX| jq . echo "matrix=$MATRIX" >> $GITHUB_OUTPUT + echo $DEB_MATRIX + echo $DEB_MATRIX| jq . + echo "deb-matrix=$DEB_MATRIX" >> $GITHUB_OUTPUT + echo $RPM_MATRIX + echo $RPM_MATRIX| jq . + echo "rpm-matrix=$RPM_MATRIX" >> $GITHUB_OUTPUT shell: bash diff --git a/.github/workflows/staging-release.yaml b/.github/workflows/staging-release.yaml index 30730cc3d15..d192c79cb09 100644 --- a/.github/workflows/staging-release.yaml +++ b/.github/workflows/staging-release.yaml @@ -64,19 +64,49 @@ jobs: env: RELEASE_VERSION: ${{ github.event.inputs.version }} + staging-release-generate-package-matrix: + name: Get package matrix + runs-on: ubuntu-latest + outputs: + deb-build-matrix: ${{ steps.get-matrix.outputs.deb-build-matrix }} + rpm-build-matrix: ${{ steps.get-matrix.outputs.rpm-build-matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup runner + run: | + sudo apt-get update + sudo apt-get install -y jq + shell: bash + + # Cope with 1.9 as well as 2.0 + - uses: ./.github/actions/generate-package-build-matrix + id: get-matrix + with: + ref: v${{ inputs.version }} + + # Now annotate with whether it is Yum or Apt based + # 1. Take packages from the staging bucket # 2. Sign them with the release GPG key # 3. Also take existing release packages from the release bucket. # 4. Create a full repo configuration using the existing releases as well. # 5. Upload to release bucket. # Note we could resign all packages as well potentially if we wanted to update the key. - staging-release-packages: - name: S3 - update packages bucket + staging-release-yum-packages: + name: S3 - update YUM packages bucket runs-on: ubuntu-22.04 # no createrepo on Ubuntu 20.04 environment: release - needs: staging-release-version-check + needs: + - staging-release-version-check + - staging-release-generate-package-matrix permissions: contents: read + strategy: + matrix: + distro: ${{ fromJSON(needs.staging-release-generate-package-matrix.outputs.rpm-build-matrix) }} + fail-fast: false steps: - name: Checkout code uses: actions/checkout@v3 @@ -84,7 +114,7 @@ jobs: - name: Setup runner run: | sudo apt-get update - sudo apt-get install -y debsigs createrepo-c aptly rsync + sudo apt-get install -y createrepo-c shell: bash - name: Import GPG key for signing @@ -101,13 +131,84 @@ jobs: - name: Sync packages from buckets on S3 run: | - mkdir -p packaging/releases - aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}" packaging/releases/ --no-progress - aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}" packaging/releases/ --no-progress + mkdir -p "packaging/releases/$DISTRO" + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: "us-east-1" + DISTRO: ${{ matrix.distro }} + shell: bash + + - name: GPG set up keys for signing + run: | + gpg --export -a "${{ steps.import_gpg.outputs.name }}" > packaging/releases/fluentbit.key + rpm --import packaging/releases/fluentbit.key + shell: bash + + - name: Update repo info and remove any staging details + run: | + packaging/update-yum-repo.sh + env: + GPG_KEY: ${{ steps.import_gpg.outputs.name }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }} + VERSION: ${{ github.event.inputs.version }} + BASE_PATH: "packaging/releases" + RPM_REPO: ${{ matrix.distro }} + shell: bash + + - name: Sync to release bucket on S3 + run: | + aws s3 sync "packaging/releases/$DISTRO" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" --delete --follow-symlinks --no-progress + aws s3 sync "packaging/releases/*.repo" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/" --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + DISTRO: ${{ matrix.distro }} + shell: bash + + staging-release-apt-packages: + name: S3 - update APT packages bucket + runs-on: ubuntu-latest + environment: release + needs: + - staging-release-version-check + - staging-release-generate-package-matrix + permissions: + contents: read + strategy: + matrix: + distro: ${{ fromJSON(needs.staging-release-generate-package-matrix.outputs.deb-build-matrix) }} + fail-fast: false + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup runner + run: | + sudo apt-get update + sudo apt-get install -y debsigs aptly rsync + shell: bash + + - name: Import GPG key for signing + id: import_gpg + uses: crazy-max/ghaction-import-gpg@v5 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }} + + - name: Sync packages from buckets on S3 + run: | + mkdir -p "packaging/releases/$DISTRO" + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + DISTRO: ${{ matrix.distro }} shell: bash - name: GPG set up keys for signing @@ -119,21 +220,23 @@ jobs: - name: Update repo info and remove any staging details run: | rm -f packaging/releases/*.repo - rm -f packaging/releases/latest-version.txt - packaging/update-repos.sh packaging/releases/ + packaging/update-apt-repo.sh env: GPG_KEY: ${{ steps.import_gpg.outputs.name }} AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }} VERSION: ${{ github.event.inputs.version }} + BASE_PATH: "packaging/releases" + DEB_REPO: ${{ matrix.distro }} shell: bash - name: Sync to release bucket on S3 run: | - aws s3 sync packaging/releases/ "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}" --delete --follow-symlinks --no-progress + aws s3 sync "packaging/releases/$DISTRO" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" --delete --follow-symlinks --no-progress env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: "us-east-1" + DISTRO: ${{ matrix.distro }} shell: bash staging-release-source-s3: @@ -191,6 +294,40 @@ jobs: AWS_REGION: "us-east-1" shell: bash + - name: Provide output for documentation PR + run: | + export WIN_32_EXE_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win32.exe.sha256"|awk '{print $1}') + export WIN_32_ZIP_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win32.zip.sha256"|awk '{print $1}') + export WIN_64_EXE_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win64.exe.sha256"|awk '{print $1}') + export WIN_64_ZIP_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win64.zip.sha256"|awk '{print $1}') + cat > windows-part.md < EOF + ## Installation Packages + + The latest stable version is ${{ inputs.version }}, each version is available on the Github release as well as at `https://releases.fluentbit.io//fluent-bit--win[32|64].[exe|zip]`: + + | INSTALLERS | SHA256 CHECKSUMS | + | ------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------- | + | [fluent-bit-${{ inputs.version }}-win32.exe](https://releases.fluentbit.io/${{ needs.staging-release-version-check.outputs.major-version }}/fluent-bit-${{ inputs.version }}-win32.exe) | [$WIN_32_EXE_HASH](https://releases.fluentbit.io/2.0/fluent-bit-${{ inputs.version }}-win32.exe.sha256) | + | [fluent-bit-${{ inputs.version }}-win32.zip](https://releases.fluentbit.io/${{ needs.staging-release-version-check.outputs.major-version }}/fluent-bit-${{ inputs.version }}-win32.zip) | [$WIN_32_ZIP_HASH](https://releases.fluentbit.io/2.0/fluent-bit-${{ inputs.version }}-win32.zip.sha256) | + | [fluent-bit-${{ inputs.version }}-win64.exe](https://releases.fluentbit.io/${{ needs.staging-release-version-check.outputs.major-version }}/fluent-bit-${{ inputs.version }}-win64.exe) | [$WIN_64_EXE_HASH](https://releases.fluentbit.io/2.0/fluent-bit-${{ inputs.version }}-win64.exe.sha256) | + | [fluent-bit-${{ inputs.version }}-win64.zip](https://releases.fluentbit.io/${{ needs.staging-release-version-check.outputs.major-version }}/fluent-bit-${{ inputs.version }}-win64.zip) | [$WIN_64_ZIP_HASH](https://releases.fluentbit.io/2.0/fluent-bit-${{ inputs.version }}-win64.zip.sha256) | + + To check the integrity, use `Get-FileHash` cmdlet on PowerShell. + + ```powershell + PS> Get-FileHash fluent-bit-${{ inputs.version }}-win32.exe + ``` + EOF + shell: bash + env: + BASE_DIR: ${{ needs.staging-release-version-check.outputs.major-version }} + + - name: Upload Windows docs + uses: actions/upload-artifact@v3 + with: + name: windows-part + path: windows-part.md + # Simple skopeo copy jobs to transfer image from staging to release registry with optional GPG key signing. # Unfortunately skopeo currently does not support Cosign: https://github.com/containers/skopeo/issues/1533 staging-release-images: @@ -476,7 +613,8 @@ jobs: runs-on: ubuntu-latest environment: release needs: - - staging-release-packages + - staging-release-apt-packages + - staging-release-yum-packages steps: - name: Checkout code uses: actions/checkout@v3 @@ -514,7 +652,8 @@ jobs: name: Create the Github Release once packages and containers are up needs: - staging-release-images - - staging-release-packages + - staging-release-apt-packages + - staging-release-yum-packages permissions: contents: write environment: release @@ -533,10 +672,68 @@ jobs: - name: Release 2.0 and latest uses: softprops/action-gh-release@v1 - if: startsWith(inputs.version, '2.0') + if: startsWith(inputs.version, '2.') with: body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" draft: false generate_release_notes: false name: "Fluent Bit ${{ inputs.version }}" tag_name: v${{ inputs.version }} + + staging-release-create-docs-pr: + name: Create docs updates for new release + needs: + - staging-release-images + - staging-release-source-s3 + permissions: + contents: none + environment: release + runs-on: ubuntu-latest + steps: + - name: Release 1.9 - not latest + if: startsWith(inputs.version, '1.9') + uses: actions/checkout@v3 + with: + repository: fluent/fluent-bit-docs + ref: 1.9 + + - name: Release 2.0 and latest + if: startsWith(inputs.version, '2.') + uses: actions/checkout@v3 + with: + repository: fluent/fluent-bit-docs + + - name: Update container tags + run: | + sed -i -e '/| Tag(s).*$/a | ${{ inputs.version }}-debug | x86\_64, arm64v8, arm32v7 | Release [v${{ inputs.version }}](https://fluentbit.io/announcements/v${{ inputs.version }}/) |' installation/docker.md + sed -i -e '/| Tag(s).*$/a | ${{ inputs.version }} | x86\_64, arm64v8, arm32v7 | Debug images |' installation/docker.md + shell: bash + + - name: Download Windows part + uses: actions/download-artifact@v3 + with: + name: windows-part + + - name: Update Windows version + run: | + # Remove old info + sed -i '/## Installation Packages/,/## Installing from ZIP archive/{//p;d;}' installation/windows.md + # Add new info (from previous job) at the end for simplicity + cat windows-part.md >> installation/windows.md + shell: bash + + - name: Raise docs PR + uses: peter-evans/create-pull-request@v4 + with: + commit-message: 'release: update to v${{ inputs.version }}' + signoff: true + delete-branch: true + title: 'release: update to v${{ inputs.version }}' + # We need workflows permission so have to use the CI_PAT + token: ${{ secrets.CI_PAT }} + labels: ci,automerge + body: | + Update release ${{ inputs.version }} version. + - Created by ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + - Auto-generated by create-pull-request: https://github.com/peter-evans/create-pull-request + draft: false diff --git a/packaging/build-config.json b/packaging/build-config.json index 8e73f40fb74..7f211ee1baa 100644 --- a/packaging/build-config.json +++ b/packaging/build-config.json @@ -1,28 +1,100 @@ { "linux_targets": [ - "amazonlinux/2", - "amazonlinux/2.arm64v8", - "amazonlinux/2022", - "centos/7", - "centos/7.arm64v8", - "centos/8", - "centos/8.arm64v8", - "centos/9", - "centos/9.arm64v8", - "debian/bookworm", - "debian/bookworm.arm64v8", - "debian/buster", - "debian/buster.arm64v8", - "debian/bullseye", - "debian/bullseye.arm64v8", - "ubuntu/16.04", - "ubuntu/18.04", - "ubuntu/18.04.arm64v8", - "ubuntu/20.04", - "ubuntu/20.04.arm64v8", - "ubuntu/22.04", - "ubuntu/22.04.arm64v8", - "raspbian/buster", - "raspbian/bullseye" + { + "target": "amazonlinux/2", + "type": "rpm" + }, + { + "target": "amazonlinux/2.arm64v8", + "type": "rpm" + }, + { + "target": "amazonlinux/2022", + "type": "rpm" + }, + { + "target": "centos/7", + "type": "rpm" + }, + { + "target": "centos/7.arm64v8", + "type": "rpm" + }, + { + "target": "centos/8", + "type": "rpm" + }, + { + "target": "centos/8.arm64v8", + "type": "rpm" + }, + { + "target": "centos/9", + "type": "rpm" + }, + { + "target": "centos/9.arm64v8", + "type": "rpm" + }, + { + "target": "debian/bookworm", + "type": "deb" + }, + { + "target": "debian/bookworm.arm64v8", + "type": "deb" + }, + { + "target": "debian/buster", + "type": "deb" + }, + { + "target": "debian/buster.arm64v8", + "type": "deb" + }, + { + "target": "debian/bullseye", + "type": "deb" + }, + { + "target": "debian/bullseye.arm64v8", + "type": "deb" + }, + { + "target": "debian/ubuntu/16.04", + "type": "deb" + }, + { + "target": "debian/ubuntu/18.04", + "type": "deb" + }, + { + "target": "debian/ubuntu/18.04.arm64v8", + "type": "deb" + }, + { + "target": "debian/ubuntu/20.04", + "type": "deb" + }, + { + "target": "debian/ubuntu/20.04.arm64v8", + "type": "deb" + }, + { + "target": "debian/ubuntu/22.04", + "type": "deb" + }, + { + "target": "debian/ubuntu/22.04.arm64v8", + "type": "deb" + }, + { + "target": "debian/raspbian/buster", + "type": "deb" + }, + { + "target": "debian/raspbian/bullseye", + "type": "deb" + } ] } diff --git a/packaging/update-apt-repo.sh b/packaging/update-apt-repo.sh new file mode 100755 index 00000000000..b3f4d318c45 --- /dev/null +++ b/packaging/update-apt-repo.sh @@ -0,0 +1,72 @@ +#!/bin/bash +set -eux + +# Where the base of all the repos is +BASE_PATH=${BASE_PATH:?} +if [[ ! -d "$BASE_PATH" ]]; then + echo "ERROR: invalid base path: $BASE_PATH" + exit 1 +fi + +# "debian/bookworm" "debian/bullseye" "debian/buster" "ubuntu/xenial" "ubuntu/bionic" "ubuntu/focal" "ubuntu/jammy" "raspbian/buster" "raspbian/bullseye" +DEB_REPO=${DEB_REPO:?} + +REPO_DIR=$(realpath -sm "$BASE_PATH/$DEB_REPO" ) +REPO_DIR=$( realpath -sm "$BASE_PATH/$RPM_REPO" ) +if [[ -d "$REPO_DIR" ]] ; then + echo "ERROR: missing $REPO_DIR" + exit 1 +fi + +CODENAME=${DEB_REPO##*/} +echo "Updating $DEB_REPO for $CODENAME" + +# We use Aptly to create repos with a local temporary directory as the root. +# Once complete, we then move these to the output directory for upload. +# Based on https://github.com/spotify/debify/blob/master/debify.sh +APTLY_REPO_NAME="debify-$CODENAME" +APTLY_ROOTDIR=$(mktemp -d) +APTLY_CONFIG=$(mktemp) + +# The origin and label fields seem to cover the base directory for the repo and codename. +# The docs seems to suggest these fields are optional and free-form: https://wiki.debian.org/DebianRepository/Format#Origin +# They are security checks to verify if they have changed so we match the legacy server. +APTLY_ORIGIN=". $CODENAME" +APTLY_LABEL=". $CODENAME" +if [[ "$DEB_REPO" == "debian/bullseye" ]]; then + # For Bullseye, the legacy server had a slightly different setup we try to reproduce here + APTLY_ORIGIN="bullseye bullseye" + APTLY_LABEL="bullseye bullseye" +fi + +cat << EOF > "$APTLY_CONFIG" +{ + "rootDir": "$APTLY_ROOTDIR/" +} +EOF +cat "$APTLY_CONFIG" + +aptly -config="$APTLY_CONFIG" repo create \ + -component="main" \ + -distribution="$CODENAME" \ + "$APTLY_REPO_NAME" + +# Check if any files to add +count=$(find "$REPO_DIR" -maxdepth 1 -type f -name "*.deb" | wc -l) +if [[ $count != 0 ]] ; then + # Do not remove files as we need them from moving to staging-release + aptly -config="$APTLY_CONFIG" repo add -force-replace "$APTLY_REPO_NAME" "$REPO_DIR/" +else + echo "WARNING: no files to add in $DEB_REPO for $CODENAME" +fi +aptly -config="$APTLY_CONFIG" repo show "$APTLY_REPO_NAME" + +if [[ "$DISABLE_SIGNING" != "true" ]]; then + aptly -config="$APTLY_CONFIG" publish repo -gpg-key="$GPG_KEY" -origin="$APTLY_ORIGIN" -label="$APTLY_LABEL" "$APTLY_REPO_NAME" +else + aptly -config="$APTLY_CONFIG" publish repo --skip-signing -origin="$APTLY_ORIGIN" -label="$APTLY_LABEL" "$APTLY_REPO_NAME" +fi + +rsync -av "$APTLY_ROOTDIR"/public/* "$REPO_DIR" +# Remove unnecessary files +rm -rf "$REPO_DIR/conf/" "$REPO_DIR/db/" "$APTLY_ROOTDIR" "$APTLY_CONFIG" diff --git a/packaging/update-repos.sh b/packaging/update-repos.sh index 3e607706dcd..2864e0e518f 100755 --- a/packaging/update-repos.sh +++ b/packaging/update-repos.sh @@ -1,69 +1,18 @@ #!/bin/bash set -eux -# Where the base of all the repos is -BASE_PATH=${BASE_PATH:-$1} -if [[ ! -d "$BASE_PATH" ]]; then - echo "Invalid base path: $BASE_PATH" - exit 1 -fi +# Wrapper script around the actual ones used in CI -# Set true to prevent signing -DISABLE_SIGNING=${DISABLE_SIGNING:-false} -if [[ "$DISABLE_SIGNING" != "true" ]]; then - echo "RPM signing configuration" - rpm -q gpg-pubkey --qf '%{name}-%{version}-%{release} --> %{summary}\n' -fi - -# Handle Ubuntu 18/22 differences - no support on Ubuntu 20 -CREATE_REPO_CMD=${CREATE_REPO_CMD:-} -CREATE_REPO_ARGS=${CREATE_REPO_ARGS:--dvp} - -# Assume if set we want to use it -if [[ -n "$CREATE_REPO_CMD" ]]; then - echo "Using $CREATE_REPO_CMD" -elif command -v createrepo &> /dev/null; then - echo "Found createrepo" - CREATE_REPO_CMD="createrepo" -elif command -v createrepo_c &> /dev/null; then - echo "Found createrepo_c" - CREATE_REPO_CMD="createrepo_c" -else - echo "Unable to find a command equivalent to createrepo" - exit 1 -fi +export BASE_PATH=${BASE_PATH:-$1} +export DISABLE_SIGNING=${DISABLE_SIGNING:-false} +export CREATE_REPO_CMD=${CREATE_REPO_CMD:-} +export CREATE_REPO_ARGS=${CREATE_REPO_ARGS:--dvp} RPM_REPO_PATHS=("amazonlinux/2" "amazonlinux/2022" "centos/7" "centos/8" "centos/9") for RPM_REPO in "${RPM_REPO_PATHS[@]}"; do - echo "Updating $RPM_REPO" - REPO_DIR=$( realpath -sm "$BASE_PATH/$RPM_REPO" ) - [[ ! -d "$REPO_DIR" ]] && continue - - if [[ "$DISABLE_SIGNING" != "true" ]]; then - # Sign all RPMs created for this target, cover both fluent-bit and legacy packages for 1.9 branch - find "$REPO_DIR" -name "*-bit-*.rpm" -exec rpm --define "_gpg_name $GPG_KEY" --addsign {} \; - fi - # Create full metadata for all RPMs in the directory - "$CREATE_REPO_CMD" "$CREATE_REPO_ARGS" "$REPO_DIR" - - # Set up repo info - if [[ -n "${AWS_S3_BUCKET:-}" ]]; then - # Create top-level file so replace path separator with dash - # centos/8 --> centos-8.repo - # This way we make sure not to have a mixed repo or overwrite files for each target. - REPO_TYPE=${RPM_REPO/\//-} - echo "Setting up $BASE_PATH/$REPO_TYPE.repo" - cat << EOF > "$BASE_PATH/$REPO_TYPE.repo" -[Fluent-Bit] -name=Fluent Bit Packages - $REPO_TYPE - \$basearch -baseurl=https://$AWS_S3_BUCKET.s3.amazonaws.com/$RPM_REPO/ -enabled=1 -gpgkey=https://$AWS_S3_BUCKET.s3.amazonaws.com/fluentbit.key -gpgcheck=1 -repo_gpgcheck=1 -EOF - fi + export RPM_REPO + /bin/bash "$SCRIPT_DIR/update-yum-repo.sh" & done DEB_REPO_PATHS=( "debian/bookworm" @@ -77,68 +26,8 @@ DEB_REPO_PATHS=( "debian/bookworm" "raspbian/bullseye" ) for DEB_REPO in "${DEB_REPO_PATHS[@]}"; do - REPO_DIR=$(realpath -sm "$BASE_PATH/$DEB_REPO" ) - [[ ! -d "$REPO_DIR" ]] && continue - - CODENAME=${DEB_REPO##*/} - echo "Updating $DEB_REPO for $CODENAME" - - # We use Aptly to create repos with a local temporary directory as the root. - # Once complete, we then move these to the output directory for upload. - # Based on https://github.com/spotify/debify/blob/master/debify.sh - APTLY_REPO_NAME="debify-$CODENAME" - APTLY_ROOTDIR=$(mktemp -d) - APTLY_CONFIG=$(mktemp) - - # The origin and label fields seem to cover the base directory for the repo and codename. - # The docs seems to suggest these fields are optional and free-form: https://wiki.debian.org/DebianRepository/Format#Origin - # They are security checks to verify if they have changed so we match the legacy server. - APTLY_ORIGIN=". $CODENAME" - APTLY_LABEL=". $CODENAME" - if [[ "$DEB_REPO" == "debian/bullseye" ]]; then - # For Bullseye, the legacy server had a slightly different setup we try to reproduce here - APTLY_ORIGIN="bullseye bullseye" - APTLY_LABEL="bullseye bullseye" - fi - - cat << EOF > "$APTLY_CONFIG" -{ - "rootDir": "$APTLY_ROOTDIR/" -} -EOF - cat "$APTLY_CONFIG" - - aptly -config="$APTLY_CONFIG" repo create \ - -component="main" \ - -distribution="$CODENAME" \ - "$APTLY_REPO_NAME" - # Check if any files to add - count=$(find "$REPO_DIR" -maxdepth 1 -type f -name "*.deb" | wc -l) - if [[ $count != 0 ]] ; then - # Do not remove files as we need them from moving to staging-release - aptly -config="$APTLY_CONFIG" repo add -force-replace "$APTLY_REPO_NAME" "$REPO_DIR/" - else - echo "No files to add in $DEB_REPO for $CODENAME" - fi - aptly -config="$APTLY_CONFIG" repo show "$APTLY_REPO_NAME" - if [[ "$DISABLE_SIGNING" != "true" ]]; then - aptly -config="$APTLY_CONFIG" publish repo -gpg-key="$GPG_KEY" -origin="$APTLY_ORIGIN" -label="$APTLY_LABEL" "$APTLY_REPO_NAME" - else - aptly -config="$APTLY_CONFIG" publish repo --skip-signing -origin="$APTLY_ORIGIN" -label="$APTLY_LABEL" "$APTLY_REPO_NAME" - fi - rsync -av "$APTLY_ROOTDIR"/public/* "$REPO_DIR" - # Remove unnecessary files - rm -rf "$REPO_DIR/conf/" "$REPO_DIR/db/" "$APTLY_ROOTDIR" "$APTLY_CONFIG" + export DEB_REPO + /bin/bash "$SCRIPT_DIR/update-apt-repo.sh" & done -# Ensure we sign the Yum repo meta-data -if [[ "$DISABLE_SIGNING" != "true" ]]; then - # We use this form to fail on error during the find, otherwise -exec will succeed or just do one file with + - while IFS= read -r -d '' REPO_METADATA_FILE - do - echo "Signing $REPO_METADATA_FILE" - gpg --detach-sign --batch --armor --yes -u "$GPG_KEY" "$REPO_METADATA_FILE" - done < <(find "$BASE_PATH" -name repomd.xml -print0) - # Debug ouput for checking - find "$BASE_PATH" -name "repomd.xml*" -exec ls -l {} \; -fi +wait diff --git a/packaging/update-yum-repo.sh b/packaging/update-yum-repo.sh new file mode 100755 index 00000000000..52624270c51 --- /dev/null +++ b/packaging/update-yum-repo.sh @@ -0,0 +1,84 @@ +#!/bin/bash +set -eux + +#("amazonlinux/2" "amazonlinux/2022" "centos/7" "centos/8" "centos/9") +RPM_REPO=${RPM_REPO:?} + +# Where the base of all the repos is +BASE_PATH=${BASE_PATH:-$1} +if [[ ! -d "$BASE_PATH" ]]; then + echo "ERROR: invalid base path: $BASE_PATH" + exit 1 +fi + +# Set true to prevent signing +DISABLE_SIGNING=${DISABLE_SIGNING:-false} +if [[ "$DISABLE_SIGNING" != "true" ]]; then + echo "INFO: RPM signing configuration" + rpm -q gpg-pubkey --qf '%{name}-%{version}-%{release} --> %{summary}\n' +fi + +# Handle Ubuntu 18/22 differences - no support on Ubuntu 20 +CREATE_REPO_CMD=${CREATE_REPO_CMD:-} +CREATE_REPO_ARGS=${CREATE_REPO_ARGS:--dvp} + +# Assume if set we want to use it +if [[ -n "$CREATE_REPO_CMD" ]]; then + echo "INFO: using $CREATE_REPO_CMD" +elif command -v createrepo &> /dev/null; then + echo "INFO: found createrepo" + CREATE_REPO_CMD="createrepo" +elif command -v createrepo_c &> /dev/null; then + echo "INFO: found createrepo_c" + CREATE_REPO_CMD="createrepo_c" +else + echo "ERROR: unable to find a command equivalent to createrepo" + exit 1 +fi + +echo "INFO: updating $RPM_REPO" + +REPO_DIR=$( realpath -sm "$BASE_PATH/$RPM_REPO" ) +if [[ -d "$REPO_DIR" ]] ; then + echo "ERROR: missing $REPO_DIR" + exit 1 +fi + +if [[ "$DISABLE_SIGNING" != "true" ]]; then + # Sign all RPMs created for this target, cover both fluent-bit and legacy packages + find "$REPO_DIR" -name "*-bit-*.rpm" -exec rpm --define "_gpg_name $GPG_KEY" --addsign {} \; +fi +# Create full metadata for all RPMs in the directory +"$CREATE_REPO_CMD" "$CREATE_REPO_ARGS" "$REPO_DIR" + +# Set up repo info +if [[ -n "${AWS_S3_BUCKET:-}" ]]; then + # Create top-level file so replace path separator with dash + # centos/8 --> centos-8.repo + # This way we make sure not to have a mixed repo or overwrite files for each target. + REPO_TYPE=${RPM_REPO/\//-} + echo "INFO: setting up $BASE_PATH/$REPO_TYPE.repo" + cat << EOF > "$BASE_PATH/$REPO_TYPE.repo" +[Fluent-Bit] +name=Fluent Bit Packages - $REPO_TYPE - \$basearch +baseurl=https://$AWS_S3_BUCKET.s3.amazonaws.com/$RPM_REPO/ +enabled=1 +gpgkey=https://$AWS_S3_BUCKET.s3.amazonaws.com/fluentbit.key +gpgcheck=1 +repo_gpgcheck=1 +EOF +fi + +# Ensure we sign the Yum repo meta-data +if [[ "$DISABLE_SIGNING" != "true" ]]; then + # We use this form to fail on error during the find, otherwise -exec will succeed or just do one file with + + while IFS= read -r -d '' REPO_METADATA_FILE + do + echo "INFO: signing $REPO_METADATA_FILE" + gpg --detach-sign --batch --armor --yes -u "$GPG_KEY" "$REPO_METADATA_FILE" + done < <(find "$REPO_DIR" -name repomd.xml -print0) + # Debug ouput for checking + find "$REPO_DIR" -name "repomd.xml*" -exec ls -l {} \; +fi + +echo "INFO: Completed $RPM_REPO"