From 2905dbee20e2dca73c8120f471f9b45ce63486f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Wed, 20 Mar 2024 19:35:14 +0100 Subject: [PATCH 01/32] Add script to get the version of OpenSearch --- scripts/README.md | 40 +++++++++++++++++++++++++++++++++++++ scripts/build.sh | 12 +---------- scripts/upstream_version.sh | 6 ++++++ 3 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 scripts/upstream_version.sh diff --git a/scripts/README.md b/scripts/README.md index 22d3648b38af6..7cfa2a63387f5 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -200,3 +200,43 @@ Pre-requisites: ```console docker exec -it wi-assemble_$( Date: Thu, 21 Mar 2024 12:48:57 +0100 Subject: [PATCH 02/32] Set revision to 0 by default. - Reduce inputs for scripts. - Add script to generate packages' naming convention. - Make scripts self-aware of the OpenSearch version. --- .github/workflows/build.yml | 13 +- .github/workflows/r_assemble.yml | 7 +- .github/workflows/r_build.yml | 4 +- .../packages/src/rpm/wazuh-indexer.rpm.spec | 2 +- scripts/assemble.sh | 30 ++-- scripts/baptizer.sh | 153 ++++++++++++++++++ scripts/build.sh | 3 +- 7 files changed, 185 insertions(+), 27 deletions(-) create mode 100644 scripts/baptizer.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f68708a1197fb..b49ebd1fd0565 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,7 +13,7 @@ on: description: "Revision" type: string required: true - default: "1" + default: "0" upload: description: "Upload ?" type: bool @@ -49,8 +49,8 @@ jobs: with: architecture: ${{ matrix.architecture }} distribution: ${{ matrix.distribution }} - revision: ${{ github.event_name == 'push' && '1' || inputs.revision }} - name: wazuh-indexer-min_${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '1' || inputs.revision }}-${{ matrix.architecture }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.distribution }} + revision: ${{ github.event_name == 'push' && '0' || inputs.revision }} + name: wazuh-indexer-min_${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '0' || inputs.revision }}-${{ matrix.architecture }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.distribution }} assemble: needs: [version, commit_sha, build] @@ -67,7 +67,8 @@ jobs: with: architecture: ${{ matrix.architecture }} distribution: ${{ matrix.distribution }} - min: wazuh-indexer-min_${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '1' || inputs.revision }}-${{ matrix.architecture }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.distribution }} + revision: ${{ github.event_name == 'push' && '0' || inputs.revision }} + min: wazuh-indexer-min_${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '0' || inputs.revision }}-${{ matrix.architecture }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.distribution }} test: needs: [version, commit_sha, assemble] @@ -77,7 +78,7 @@ jobs: os: [{ suffix: "amd64", ext: "deb" }, { suffix: "x86_64", ext: "rpm" }] uses: ./.github/workflows/r_test.yml with: - package: wazuh-indexer-${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '1' || inputs.revision }}_${{ matrix.os.suffix }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.os.ext }} + package: wazuh-indexer-${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '0' || inputs.revision }}_${{ matrix.os.suffix }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.os.ext }} upload: needs: [version, commit_sha, test] @@ -85,5 +86,5 @@ jobs: if: ${{ github.event_name == 'push' && inputs.upload }} uses: ./.github/workflows/r_upload.yml with: - package: wazuh-indexer-${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '1' || inputs.revision }}_${{ matrix.os.suffix }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.os.ext }} + package: wazuh-indexer-${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '0' || inputs.revision }}_${{ matrix.os.suffix }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.os.ext }} secrets: inherit diff --git a/.github/workflows/r_assemble.yml b/.github/workflows/r_assemble.yml index 11009429298e0..868d28acb248d 100644 --- a/.github/workflows/r_assemble.yml +++ b/.github/workflows/r_assemble.yml @@ -18,6 +18,11 @@ on: default: "x64" required: true type: string + revision: + description: "Any string or number used to extend the package's identifier." + type: string + required: true + default: "0" min: description: "The name of the package to download." required: true @@ -45,7 +50,7 @@ jobs: - name: Run `assemble.sh` run: | - bash scripts/assemble.sh -v ${{ vars.OPENSEARCH_VERSION }} -p linux -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} + bash scripts/assemble.sh -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} r ${{ inputs.revision }} # The package's name is stored in artifacts/artifact_name.txt. - name: Set package name diff --git a/.github/workflows/r_build.yml b/.github/workflows/r_build.yml index 4c430ce113ad6..0b1778a4637cc 100644 --- a/.github/workflows/r_build.yml +++ b/.github/workflows/r_build.yml @@ -19,7 +19,7 @@ on: description: "Any string or number used to extend the package's identifier." type: string required: true - default: "1" + default: "0" name: description: "The name of the package to upload." required: true @@ -44,7 +44,7 @@ jobs: - name: Run `build.sh` run: | - bash scripts/build.sh -v ${{ vars.OPENSEARCH_VERSION }} -s false -p linux -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} -r ${{ inputs.revision }} + bash scripts/build.sh -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} -r ${{ inputs.revision }} # The package's name is stored in artifacts/artifact_min_name.txt. - name: Set package name diff --git a/distribution/packages/src/rpm/wazuh-indexer.rpm.spec b/distribution/packages/src/rpm/wazuh-indexer.rpm.spec index b81d6a91ecb97..c7516f539a6e2 100644 --- a/distribution/packages/src/rpm/wazuh-indexer.rpm.spec +++ b/distribution/packages/src/rpm/wazuh-indexer.rpm.spec @@ -32,7 +32,7 @@ Name: wazuh-indexer Version: %{_version} -Release: 1 +Release: %{_release} License: Apache-2.0 Summary: An open source distributed and RESTful search engine URL: https://www.wazuh.com/ diff --git a/scripts/assemble.sh b/scripts/assemble.sh index 64e82ccd52046..9556ff1aad7df 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -49,10 +49,10 @@ function usage() { echo "Usage: $0 [args]" echo "" echo "Arguments:" - echo -e "-v VERSION\t[Required] OpenSearch version." echo -e "-p PLATFORM\t[Optional] Platform, default is 'uname -s'." echo -e "-a ARCHITECTURE\t[Optional] Build architecture, default is 'uname -m'." echo -e "-d DISTRIBUTION\t[Optional] Distribution, default is 'tar'." + echo -e "-r REVISION\t[Optional] Package revision, default is '0'." echo -e "-o OUTPUT\t[Optional] Output path, default is 'artifacts'." echo -e "-h help" } @@ -62,15 +62,12 @@ function usage() { # ==== function parse_args() { - while getopts ":h:v:o:p:a:d:" arg; do + while getopts ":h:o:p:a:d:r:" arg; do case $arg in h) usage exit 1 ;; - v) - VERSION=$OPTARG - ;; o) OUTPUT=$OPTARG ;; @@ -83,6 +80,9 @@ function parse_args() { d) DISTRIBUTION=$OPTARG ;; + r) + REVISION=$OPTARG + ;; :) echo "Error: -${OPTARG} requires an argument" usage @@ -95,12 +95,6 @@ function parse_args() { esac done - if [ -z "$VERSION" ]; then - echo "Error: You must specify the OpenSearch version" - usage - exit 1 - fi - [ -z "$OUTPUT" ] && OUTPUT=artifacts # Assemble distribution artifact @@ -109,6 +103,7 @@ function parse_args() { [ -z "$PLATFORM" ] && PLATFORM=$(uname -s | awk '{print tolower($0)}') [ -z "$ARCHITECTURE" ] && ARCHITECTURE=$(uname -m) [ -z "$DISTRIBUTION" ] && DISTRIBUTION="tar" + [ -z "$REVISION" ] && REVISION="0" case $PLATFORM-$DISTRIBUTION-$ARCHITECTURE in linux-tar-x64 | darwin-tar-x64) @@ -213,12 +208,14 @@ function enable_performance_analyzer_rca() { # Install plugins # ==== function install_plugins() { + local version + version=$(bash scripts/upstream_version.sh) echo "Install plugins" maven_repo_local="$HOME/maven" for plugin in "${plugins[@]}"; do - plugin_from_maven="org.opensearch.plugin:${plugin}:$VERSION.0" - mvn -Dmaven.repo.local=$maven_repo_local org.apache.maven.plugins:maven-dependency-plugin:2.1:get -DrepoUrl=https://repo1.maven.org/maven2 -Dartifact=$plugin_from_maven:zip - OPENSEARCH_PATH_CONF=$PATH_CONF "${PATH_BIN}/opensearch-plugin" install --batch --verbose "file:${maven_repo_local}/org/opensearch/plugin/${plugin}/$VERSION.0/${plugin}-$VERSION.0.zip" + plugin_from_maven="org.opensearch.plugin:${plugin}:${version}.0" + mvn -Dmaven.repo.local="${maven_repo_local}" org.apache.maven.plugins:maven-dependency-plugin:2.1:get -DrepoUrl=https://repo1.maven.org/maven2 -Dartifact="${plugin_from_maven}:zip" + OPENSEARCH_PATH_CONF=$PATH_CONF "${PATH_BIN}/opensearch-plugin" install --batch --verbose "file:${maven_repo_local}/org/opensearch/plugin/${plugin}/${version}.0/${plugin}-${version}.0.zip" done } @@ -305,6 +302,7 @@ function assemble_rpm() { --define "_topdir ${topdir}" \ --define "_version ${version}" \ --define "_architecture ${SUFFIX}" \ + --define "_release ${REVISION}" \ ${spec_file} # Move to the root folder, copy the package and clean. @@ -359,7 +357,7 @@ function assemble_deb() { --invoke debuild \ --package wazuh-indexer \ --native \ - --revision 1 \ + --revision "${REVISION}" \ --upstreamversion "${version}" # Move to the root folder, copy the package and clean. @@ -381,7 +379,7 @@ function main() { ARTIFACT_BUILD_NAME=$(ls "${OUTPUT}/dist/" | grep "wazuh-indexer-min_.*$SUFFIX.*\.$EXT") - ARTIFACT_PACKAGE_NAME=${ARTIFACT_BUILD_NAME/min_/} + ARTIFACT_PACKAGE_NAME=${ARTIFACT_BUILD_NAME/-min/} # Create temporal directory and copy the min package there for extraction TMP_DIR="${OUTPUT}/tmp/${TARGET}" diff --git a/scripts/baptizer.sh b/scripts/baptizer.sh new file mode 100644 index 0000000000000..ccd51ec58d15d --- /dev/null +++ b/scripts/baptizer.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +set -e + +function usage() { + echo "Usage: $0 [args]" + echo "" + echo "Arguments:" + echo -e "-p PLATFORM\t[Optional] Platform, default is 'uname -s'." + echo -e "-a ARCHITECTURE\t[Optional] Build architecture, default is 'uname -m'." + echo -e "-d DISTRIBUTION\t[Optional] Distribution, default is 'tar'." + echo -e "-r REVISION\t[Optional] Package revision, default is '0'." + echo -e "-m MIN\t[Optional] Use naming convention for minimal packages, default is 'false'." + echo -e "-x RELEASE\t[Optional] Use release naming convention, default is 'false'." + echo -e "-h help" +} + +# ==== +# Parse arguments +# ==== +function parse_args() { + + while getopts ":h:p:a:d:r:mx" arg; do + case $arg in + h) + usage + exit 1 + ;; + p) + PLATFORM=$OPTARG + ;; + a) + ARCHITECTURE=$OPTARG + ;; + d) + DISTRIBUTION=$OPTARG + ;; + r) + REVISION=$OPTARG + ;; + m) + IS_MIN=true + ;; + x) + IS_RELEASE=true + ;; + :) + echo "Error: -${OPTARG} requires an argument" + usage + exit 1 + ;; + ?) + echo "Invalid option: -${arg}" + exit 1 + ;; + esac + done + + [ -z "$PLATFORM" ] && PLATFORM=$(uname -s | awk '{print tolower($0)}') + [ -z "$ARCHITECTURE" ] && ARCHITECTURE=$(uname -m) + [ -z "$DISTRIBUTION" ] && DISTRIBUTION="tar" + [ -z "$REVISION" ] && REVISION="0" + [ -z "$IS_MIN" ] && IS_MIN=false + [ -z "$IS_RELEASE" ] && IS_RELEASE=false + + case $PLATFORM-$DISTRIBUTION-$ARCHITECTURE in + linux-tar-x64 | darwin-tar-x64) + EXT="tar.gz" + SUFFIX="$PLATFORM-x64" + ;; + linux-tar-arm64 | darwin-tar-arm64) + EXT="tar.gz" + SUFFIX="$PLATFORM-arm64" + ;; + linux-deb-x64) + EXT="deb" + SUFFIX="amd64" + ;; + linux-deb-arm64) + EXT="deb" + SUFFIX="arm64" + ;; + linux-rpm-x64) + EXT="rpm" + SUFFIX="x86_64" + ;; + linux-rpm-arm64) + EXT="rpm" + SUFFIX="aarch64" + ;; + windows-zip-x64) + EXT="zip" + SUFFIX="$PLATFORM-x64" + ;; + windows-zip-arm64) + EXT="zip" + SUFFIX="$PLATFORM-arm64" + ;; + *) + echo "Unsupported platform-distribution-architecture combination: $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" + exit 1 + ;; + esac + +} + +# ==== +# Naming convention for release packages +# ==== +function get_release_name() { + if [ "$EXT" = "rpm" ]; then + PACKAGE_NAME=wazuh-indexer-"$VERSION"-"$REVISION"."$SUFFIX"."$EXT" + else + PACKAGE_NAME=wazuh-indexer_"$VERSION"-"$REVISION"_"$SUFFIX"."$EXT" + fi +} + +# ==== +# Naming convention for pre-release packages +# ==== +# TODO wazuh-indexer_$VERSION != wazuh-indexer-$VERSION +function get_devel_name() { + if "$IS_MIN"; then + PACKAGE_NAME=wazuh-indexer-min_"$VERSION"-"$REVISION"_"$SUFFIX"_"$GIT_COMMIT"."$EXT" + else + PACKAGE_NAME=wazuh-indexer_"$VERSION"-"$REVISION"_"$SUFFIX"_"$GIT_COMMIT"."$EXT" + fi +} + +# ==== +# Naming convention control function +# ==== +function get_package_name() { + if "$IS_RELEASE"; then + get_release_name + else + get_devel_name + fi +} + +# ==== +# Main function +# ==== +function main() { + parse_args "${@}" + + get_package_name + echo "$PACKAGE_NAME" +} + +GIT_COMMIT=$(git rev-parse --short HEAD) +VERSION=$( Date: Thu, 21 Mar 2024 13:12:05 +0100 Subject: [PATCH 03/32] Fix assemble --- .github/workflows/r_assemble.yml | 2 +- scripts/assemble.sh | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/r_assemble.yml b/.github/workflows/r_assemble.yml index 868d28acb248d..597ecbee251d5 100644 --- a/.github/workflows/r_assemble.yml +++ b/.github/workflows/r_assemble.yml @@ -50,7 +50,7 @@ jobs: - name: Run `assemble.sh` run: | - bash scripts/assemble.sh -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} r ${{ inputs.revision }} + bash scripts/assemble.sh -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} -r ${{ inputs.revision }} # The package's name is stored in artifacts/artifact_name.txt. - name: Set package name diff --git a/scripts/assemble.sh b/scripts/assemble.sh index 9556ff1aad7df..2bd9274b06d01 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -208,14 +208,12 @@ function enable_performance_analyzer_rca() { # Install plugins # ==== function install_plugins() { - local version - version=$(bash scripts/upstream_version.sh) echo "Install plugins" maven_repo_local="$HOME/maven" for plugin in "${plugins[@]}"; do - plugin_from_maven="org.opensearch.plugin:${plugin}:${version}.0" + plugin_from_maven="org.opensearch.plugin:${plugin}:${VERSION}.0" mvn -Dmaven.repo.local="${maven_repo_local}" org.apache.maven.plugins:maven-dependency-plugin:2.1:get -DrepoUrl=https://repo1.maven.org/maven2 -Dartifact="${plugin_from_maven}:zip" - OPENSEARCH_PATH_CONF=$PATH_CONF "${PATH_BIN}/opensearch-plugin" install --batch --verbose "file:${maven_repo_local}/org/opensearch/plugin/${plugin}/${version}.0/${plugin}-${version}.0.zip" + OPENSEARCH_PATH_CONF=$PATH_CONF "${PATH_BIN}/opensearch-plugin" install --batch --verbose "file:${maven_repo_local}/org/opensearch/plugin/${plugin}/${VERSION}.0/${plugin}-${VERSION}.0.zip" done } @@ -377,8 +375,8 @@ function main() { echo "Assembling wazuh-indexer for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" + VERSION=$(bash scripts/upstream_version.sh) ARTIFACT_BUILD_NAME=$(ls "${OUTPUT}/dist/" | grep "wazuh-indexer-min_.*$SUFFIX.*\.$EXT") - ARTIFACT_PACKAGE_NAME=${ARTIFACT_BUILD_NAME/-min/} # Create temporal directory and copy the min package there for extraction From 0d525dc81544195f18d3222ad35e782bf57c8d2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Thu, 21 Mar 2024 17:24:04 +0100 Subject: [PATCH 04/32] Smoke test new pipeline to build packages --- .github/workflows/build.yml | 68 +++++++---------------- .github/workflows/r_assemble.yml | 77 +++++++++++++++----------- .github/workflows/r_build.yml | 89 ++++++++++++++++--------------- .github/workflows/r_build_min.yml | 65 ++++++++++++++++++++++ .github/workflows/r_test.yml | 48 +++++++++++++---- .github/workflows/r_upload.yml | 37 ++++++++++--- 6 files changed, 243 insertions(+), 141 deletions(-) create mode 100644 .github/workflows/r_build_min.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b49ebd1fd0565..aecdbfa003fe2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,12 +12,23 @@ on: revision: description: "Revision" type: string - required: true default: "0" upload: description: "Upload ?" type: bool default: false + is_release: + description: "Upload ?" + type: bool + default: false + distribution: + description: "[ 'tar', 'rpm', 'deb', 'docker' ]" + type: string + default: "[ 'rpm', 'deb' ]" + architecture: + description: "[ 'x64', 'arm64' ]" + type: string + default: "[ 'x64' ]" # ========================== # Bibliography @@ -33,58 +44,19 @@ on: # | https://docs.github.com/en/actions/learn-github-actions/expressions#example jobs: - version: - uses: ./.github/workflows/r_version.yml - - commit_sha: - uses: ./.github/workflows/r_commit_sha.yml - build: - needs: [version, commit_sha] - strategy: - matrix: - distribution: [tar, rpm, deb] - architecture: [x64, arm64] - uses: ./.github/workflows/r_build.yml - with: - architecture: ${{ matrix.architecture }} - distribution: ${{ matrix.distribution }} - revision: ${{ github.event_name == 'push' && '0' || inputs.revision }} - name: wazuh-indexer-min_${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '0' || inputs.revision }}-${{ matrix.architecture }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.distribution }} - - assemble: - needs: [version, commit_sha, build] strategy: matrix: - distribution: [tar, rpm, deb] - architecture: [x64, arm64] + distribution: ${{ inputs.distribution }} + architecture: ${{ inputs.architecture }} exclude: # skip arm64 until we have arm runners - architecture: arm64 - - distribution: tar - - uses: ./.github/workflows/r_assemble.yml + fail-fast: false + uses: ./.github/workflows/r_build.yml with: - architecture: ${{ matrix.architecture }} - distribution: ${{ matrix.distribution }} revision: ${{ github.event_name == 'push' && '0' || inputs.revision }} - min: wazuh-indexer-min_${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '0' || inputs.revision }}-${{ matrix.architecture }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.distribution }} - - test: - needs: [version, commit_sha, assemble] - strategy: - fail-fast: false - matrix: - os: [{ suffix: "amd64", ext: "deb" }, { suffix: "x86_64", ext: "rpm" }] - uses: ./.github/workflows/r_test.yml - with: - package: wazuh-indexer-${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '0' || inputs.revision }}_${{ matrix.os.suffix }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.os.ext }} - - upload: - needs: [version, commit_sha, test] - # Upload only on 'workflow_dispatch' event and if 'upload=true' - if: ${{ github.event_name == 'push' && inputs.upload }} - uses: ./.github/workflows/r_upload.yml - with: - package: wazuh-indexer-${{ needs.version.outputs.version }}-${{ github.event_name == 'push' && '0' || inputs.revision }}_${{ matrix.os.suffix }}_${{ needs.commit_sha.outputs.commit_sha }}.${{ matrix.os.ext }} - secrets: inherit + upload: ${{ inputs.upload }} + is_release: ${{ inputs.is_release }} + distribution: ${{ matrix.distribution }} + architecture: ${{ matrix.architecture }} diff --git a/.github/workflows/r_assemble.yml b/.github/workflows/r_assemble.yml index 597ecbee251d5..2ef06f01774f8 100644 --- a/.github/workflows/r_assemble.yml +++ b/.github/workflows/r_assemble.yml @@ -1,32 +1,26 @@ name: Assemble (reusable) -env: - TEST: true - # This workflow runs when any of the following occur: # - Run from another workflow on: workflow_call: inputs: - distribution: - description: "One of [ 'tar', 'rpm', 'deb' ]" - default: "rpm" - required: true - type: string - architecture: - description: "One of [ 'x64', 'arm64' ]" - default: "x64" - required: true - type: string revision: - description: "Any string or number used to extend the package's identifier." + description: "Revision number" type: string - required: true default: "0" - min: - description: "The name of the package to download." + is_release: + description: "Uses release's nomenclature" + type: bool + default: false + distribution: + description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" + type: string required: true + architecture: + description: One of "[ 'x64', 'arm64' ]" type: string + required: true jobs: r_assemble: @@ -37,30 +31,49 @@ jobs: contents: read steps: - uses: actions/checkout@v4 - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.min }} - path: artifacts/dist + + - name: Run `baptizer.sh` (min) + run: | + name=$(bash scripts/baptizer.sh -m \ + ${{ inputs.is_release && '-x' }} \ + -a ${{ inputs.architecture }} \ + -d ${{ inputs.distribution }} \ + -r ${{ inputs.revision }} \ + ) + echo "$name" >> $GITHUB_OUTPUT + id: min_package + + - name: Run `baptizer.sh` + run: | + name=$(bash scripts/baptizer.sh \ + ${{ inputs.is_release && '-x' }} \ + -a ${{ inputs.architecture }} \ + -d ${{ inputs.distribution }} \ + -r ${{ inputs.revision }} \ + ) + echo "$name" >> $GITHUB_OUTPUT + id: package - name: Provision if: ${{ inputs.distribution == 'deb' }} run: | sudo bash scripts/provision.sh - - name: Run `assemble.sh` - run: | - bash scripts/assemble.sh -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} -r ${{ inputs.revision }} + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: ${{ steps.min_package.outputs.name }} + path: artifacts/dist - # The package's name is stored in artifacts/artifact_name.txt. - - name: Set package name - id: get_name - run: | - echo "name=$(cat artifacts/artifact_name.txt)" >> $GITHUB_OUTPUT + - name: Run `assemble.sh` + mkdir -p ./artifacts/dist + touch ./artifacts/dist/${{ steps.package.outputs.name }} + # run: | + # bash scripts/assemble.sh -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} -r ${{ inputs.revision }} - name: Upload artifact uses: actions/upload-artifact@v4 with: - name: ${{ steps.get_name.outputs.name }} - path: artifacts/dist/${{ steps.get_name.outputs.name }} + name: ${{ steps.package.outputs.name }} + path: artifacts/dist/${{ steps.package.outputs.name }} if-no-files-found: error diff --git a/.github/workflows/r_build.yml b/.github/workflows/r_build.yml index 0b1778a4637cc..65fb89a446853 100644 --- a/.github/workflows/r_build.yml +++ b/.github/workflows/r_build.yml @@ -1,60 +1,61 @@ -name: Build (reusable) +name: Build pipeline (reusable) # This workflow runs when any of the following occur: # - Run from another workflow on: workflow_call: inputs: - distribution: - description: "One of [ 'tar', 'rpm', 'deb' ]" - default: "rpm" - required: true - type: string - architecture: - description: "One of [ 'x64', 'arm64' ]" - default: "x64" - required: true - type: string revision: - description: "Any string or number used to extend the package's identifier." + description: "Revision number" type: string - required: true default: "0" - name: - description: "The name of the package to upload." + upload: + description: "Uploads artifacts" + type: bool + default: false + is_release: + description: "Uses release's nomenclature" + type: bool + default: false + distribution: + description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" + type: string required: true + architecture: + description: One of "[ 'x64', 'arm64' ]" type: string + required: true jobs: - r_build: - runs-on: ubuntu-latest - # Permissions to upload the package - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 11 - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 + build_min: + uses: ./.github/workflows/r_build_min.yml + with: + revision: ${{ inputs.revision }} + distribution: ${{ matrix.distribution }} + architecture: ${{ matrix.architecture }} - - name: Run `build.sh` - run: | - bash scripts/build.sh -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} -r ${{ inputs.revision }} + assemble: + needs: [build] + uses: ./.github/workflows/r_assemble.yml + with: + revision: ${{ inputs.revision }} + distribution: ${{ matrix.distribution }} + architecture: ${{ matrix.architecture }} - # The package's name is stored in artifacts/artifact_min_name.txt. - - name: Set package name - id: get_name - run: | - echo "name=$(cat artifacts/artifact_min_name.txt)" >> $GITHUB_OUTPUT + test: + needs: [assemble] + uses: ./.github/workflows/r_test.yml + with: + revision: ${{ inputs.revision }} + distribution: ${{ matrix.distribution }} + architecture: ${{ matrix.architecture }} - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: ${{ inputs.name }} - path: artifacts/dist/${{ steps.get_name.outputs.name }} - if-no-files-found: error + upload: + if: ${{ inputs.upload }} + needs: [test] + uses: ./.github/workflows/r_upload.yml + with: + revision: ${{ inputs.revision }} + distribution: ${{ matrix.distribution }} + architecture: ${{ matrix.architecture }} + secrets: inherit diff --git a/.github/workflows/r_build_min.yml b/.github/workflows/r_build_min.yml new file mode 100644 index 0000000000000..75b90cc312ae6 --- /dev/null +++ b/.github/workflows/r_build_min.yml @@ -0,0 +1,65 @@ +name: Build minimal packages (reusable) + +# This workflow runs when any of the following occur: +# - Run from another workflow +on: + workflow_call: + inputs: + revision: + description: "Revision number" + type: string + default: "0" + is_release: + description: "Uses release's nomenclature" + type: bool + default: false + distribution: + description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" + type: string + required: true + architecture: + description: One of "[ 'x64', 'arm64' ]" + type: string + required: true + +jobs: + r_build_min: + runs-on: ubuntu-latest + # Permissions to upload the package + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 11 + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + + - name: Run `baptizer.sh` + run: | + name=$(bash scripts/baptizer.sh -m \ + ${{ inputs.is_release && '-x' }} \ + -a ${{ inputs.architecture }} \ + -d ${{ inputs.distribution }} \ + -r ${{ inputs.revision }} \ + ) + echo "$name" >> $GITHUB_OUTPUT + id: package + + - name: Run `build.sh` + run: | + mkdir -p ./artifacts/dist + touch ./artifacts/dist/${{ steps.package.outputs.name }} + # run: | + # bash scripts/build.sh -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} -r ${{ inputs.revision }} + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.package.outputs.name }} + path: artifacts/dist/${{ steps.package.outputs.name }} + if-no-files-found: error diff --git a/.github/workflows/r_test.yml b/.github/workflows/r_test.yml index fa9cd819ac8fe..a201b16562950 100644 --- a/.github/workflows/r_test.yml +++ b/.github/workflows/r_test.yml @@ -5,16 +5,44 @@ name: Test (reusable) on: workflow_call: inputs: - package: - description: "The name of the package to download." + revision: + description: "Revision number" + type: string + default: "0" + is_release: + description: "Uses release's nomenclature" + type: bool + default: false + distribution: + description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" + type: string required: true + architecture: + description: One of "[ 'x64', 'arm64' ]" type: string + required: true jobs: + setup: + runs-on: ubuntu-latest + - uses: actions/checkout@v4 + - name: Run `baptizer.sh` (min) + run: | + name=$(bash scripts/baptizer.sh \ + ${{ inputs.is_release && '-x' }} \ + -a ${{ inputs.architecture }} \ + -d ${{ inputs.distribution }} \ + -r ${{ inputs.revision }} \ + ) + echo "$name" >> $GITHUB_OUTPUT + id: package + outputs: + package_name: ${{ steps.package.outputs.name }} + r_test_rpm: - if: ${{ endsWith(inputs.package, 'rpm') }} + if: ${{ inputs.distribution == 'rpm' }} runs-on: ubuntu-latest - # Permissions to upload the package + # Permissions to download the package permissions: packages: read contents: read @@ -22,7 +50,7 @@ jobs: - name: Download artifact uses: actions/download-artifact@v4 with: - name: ${{ inputs.package }} + name: ${{ needs.setup.outputs.package_name }} path: artifacts/dist - name: Install package @@ -31,12 +59,12 @@ jobs: image: redhat/ubi9:latest options: -v ${{ github.workspace }}/artifacts/dist:/artifacts/dist run: | - yum localinstall "/artifacts/dist/${{ inputs.package }}" -y + yum localinstall "/artifacts/dist/${{ needs.setup.outputs.package_name }}" -y r_test_deb: - if: ${{ endsWith(inputs.package, 'deb') }} + if: ${{ inputs.distribution == 'deb' }} runs-on: ubuntu-latest - # Permissions to upload the package + # Permissions to download the package permissions: packages: read contents: read @@ -44,12 +72,12 @@ jobs: - name: Download artifact uses: actions/download-artifact@v4 with: - name: ${{ inputs.package }} + name: ${{ needs.setup.outputs.package_name }} path: artifacts/dist - name: Install package run: | - sudo dpkg -i "artifacts/dist/${{ inputs.package }}" + sudo dpkg -i "artifacts/dist/${{ needs.setup.outputs.package_name }}" - uses: actions/checkout@v4 - name: Generate and deploy certificates diff --git a/.github/workflows/r_upload.yml b/.github/workflows/r_upload.yml index 05a08e40394c4..c27af0c60e4ba 100644 --- a/.github/workflows/r_upload.yml +++ b/.github/workflows/r_upload.yml @@ -4,11 +4,22 @@ name: Upload (reusable) # - Run from another workflow on: workflow_call: - inputs: - package: - description: "The name of the package to upload to Wazuh's S3 development artifactory bucket." - required: true - type: string + revision: + description: "Revision number" + type: string + default: "0" + is_release: + description: "Uses release's nomenclature" + type: bool + default: false + distribution: + description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" + type: string + required: true + architecture: + description: One of "[ 'x64', 'arm64' ]" + type: string + required: true secrets: CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY: required: true @@ -23,10 +34,22 @@ jobs: packages: read contents: read steps: + - uses: actions/checkout@v4 + - name: Run `baptizer.sh` + run: | + name=$(bash scripts/baptizer.sh \ + ${{ inputs.is_release && '-x' }} \ + -a ${{ inputs.architecture }} \ + -d ${{ inputs.distribution }} \ + -r ${{ inputs.revision }} \ + ) + echo "$name" >> $GITHUB_OUTPUT + id: package + - name: Download artifact uses: actions/download-artifact@v4 with: - name: ${{ inputs.package }} + name: ${{ steps.package.outputs.name }} path: artifacts/dist - name: Set up AWS CLI @@ -38,4 +61,4 @@ jobs: - name: Upload package to S3 run: | - aws s3 cp artifacts/dist/${{ inputs.package }} s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/ + aws s3 cp artifacts/dist/${{ steps.package.outputs.name }} s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/ From 544a2c54b00635b5f847c75a72b26b43d7c0591f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Thu, 21 Mar 2024 18:33:13 +0100 Subject: [PATCH 05/32] Fix syntax errors --- .github/workflows/build.yml | 36 ++++++++++++++-------- .github/workflows/r_assemble.yml | 2 +- .github/workflows/r_build.yml | 10 +++--- .github/workflows/r_build_min.yml | 2 +- .github/workflows/r_test.yml | 7 +++-- .github/workflows/r_upload.yml | 51 ++++++++++++++++--------------- 6 files changed, 60 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aecdbfa003fe2..a266eaff69e77 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,20 +15,20 @@ on: default: "0" upload: description: "Upload ?" - type: bool + type: boolean default: false is_release: description: "Upload ?" - type: bool + type: boolean default: false distribution: - description: "[ 'tar', 'rpm', 'deb', 'docker' ]" + description: '[ "tar", "rpm", "deb", "docker" ]' type: string - default: "[ 'rpm', 'deb' ]" + default: '[ "rpm", "deb" ]' architecture: - description: "[ 'x64', 'arm64' ]" + description: '[ "x64", "arm64" ]' type: string - default: "[ 'x64' ]" + default: '[ "x64" ]' # ========================== # Bibliography @@ -44,15 +44,25 @@ on: # | https://docs.github.com/en/actions/learn-github-actions/expressions#example jobs: + matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.setup.outputs.matrix }} + steps: + - id: setup + run: | + matrix=$(jq -cn \ + --argjson distribution ${{ inputs.distribution }} \ + --argjson architecture ${{ inputs.architecture }} \ + '{distribution: $distribution, architecture: $architecture}' + ) + echo "$matrix" >> $GITHUB_OUTPUT + build: + needs: [matrix] strategy: - matrix: - distribution: ${{ inputs.distribution }} - architecture: ${{ inputs.architecture }} - exclude: - # skip arm64 until we have arm runners - - architecture: arm64 - fail-fast: false + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.matrix) }} uses: ./.github/workflows/r_build.yml with: revision: ${{ github.event_name == 'push' && '0' || inputs.revision }} diff --git a/.github/workflows/r_assemble.yml b/.github/workflows/r_assemble.yml index 2ef06f01774f8..cc510e32f22f4 100644 --- a/.github/workflows/r_assemble.yml +++ b/.github/workflows/r_assemble.yml @@ -11,7 +11,7 @@ on: default: "0" is_release: description: "Uses release's nomenclature" - type: bool + type: boolean default: false distribution: description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" diff --git a/.github/workflows/r_build.yml b/.github/workflows/r_build.yml index 65fb89a446853..7783600b9420c 100644 --- a/.github/workflows/r_build.yml +++ b/.github/workflows/r_build.yml @@ -11,11 +11,11 @@ on: default: "0" upload: description: "Uploads artifacts" - type: bool + type: boolean default: false is_release: description: "Uses release's nomenclature" - type: bool + type: boolean default: false distribution: description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" @@ -35,7 +35,7 @@ jobs: architecture: ${{ matrix.architecture }} assemble: - needs: [build] + needs: [ build_min ] uses: ./.github/workflows/r_assemble.yml with: revision: ${{ inputs.revision }} @@ -43,7 +43,7 @@ jobs: architecture: ${{ matrix.architecture }} test: - needs: [assemble] + needs: [ assemble ] uses: ./.github/workflows/r_test.yml with: revision: ${{ inputs.revision }} @@ -52,7 +52,7 @@ jobs: upload: if: ${{ inputs.upload }} - needs: [test] + needs: [ test ] uses: ./.github/workflows/r_upload.yml with: revision: ${{ inputs.revision }} diff --git a/.github/workflows/r_build_min.yml b/.github/workflows/r_build_min.yml index 75b90cc312ae6..281e9957f9d87 100644 --- a/.github/workflows/r_build_min.yml +++ b/.github/workflows/r_build_min.yml @@ -11,7 +11,7 @@ on: default: "0" is_release: description: "Uses release's nomenclature" - type: bool + type: boolean default: false distribution: description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" diff --git a/.github/workflows/r_test.yml b/.github/workflows/r_test.yml index a201b16562950..8f52b733d1445 100644 --- a/.github/workflows/r_test.yml +++ b/.github/workflows/r_test.yml @@ -11,7 +11,7 @@ on: default: "0" is_release: description: "Uses release's nomenclature" - type: bool + type: boolean default: false distribution: description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" @@ -25,6 +25,7 @@ on: jobs: setup: runs-on: ubuntu-latest + steps: - uses: actions/checkout@v4 - name: Run `baptizer.sh` (min) run: | @@ -36,8 +37,8 @@ jobs: ) echo "$name" >> $GITHUB_OUTPUT id: package - outputs: - package_name: ${{ steps.package.outputs.name }} + outputs: + package_name: ${{ steps.package.outputs.name }} r_test_rpm: if: ${{ inputs.distribution == 'rpm' }} diff --git a/.github/workflows/r_upload.yml b/.github/workflows/r_upload.yml index c27af0c60e4ba..dc0072afe37cd 100644 --- a/.github/workflows/r_upload.yml +++ b/.github/workflows/r_upload.yml @@ -4,22 +4,23 @@ name: Upload (reusable) # - Run from another workflow on: workflow_call: - revision: - description: "Revision number" - type: string - default: "0" - is_release: - description: "Uses release's nomenclature" - type: bool - default: false - distribution: - description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" - type: string - required: true - architecture: - description: One of "[ 'x64', 'arm64' ]" - type: string - required: true + inputs: + revision: + description: "Revision number" + type: string + default: "0" + is_release: + description: "Uses release's nomenclature" + type: boolean + default: false + distribution: + description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" + type: string + required: true + architecture: + description: One of "[ 'x64', 'arm64' ]" + type: string + required: true secrets: CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY: required: true @@ -36,15 +37,15 @@ jobs: steps: - uses: actions/checkout@v4 - name: Run `baptizer.sh` - run: | - name=$(bash scripts/baptizer.sh \ - ${{ inputs.is_release && '-x' }} \ - -a ${{ inputs.architecture }} \ - -d ${{ inputs.distribution }} \ - -r ${{ inputs.revision }} \ - ) - echo "$name" >> $GITHUB_OUTPUT - id: package + run: | + name=$(bash scripts/baptizer.sh \ + ${{ inputs.is_release && '-x' }} \ + -a ${{ inputs.architecture }} \ + -d ${{ inputs.distribution }} \ + -r ${{ inputs.revision }} \ + ) + echo "$name" >> $GITHUB_OUTPUT + id: package - name: Download artifact uses: actions/download-artifact@v4 From 3eab584cf57bb0989c5a164d1fc4e8303d5509d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Thu, 21 Mar 2024 18:39:17 +0100 Subject: [PATCH 06/32] Update build.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Álex Ruiz --- .github/workflows/build.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a266eaff69e77..59ea407bb63e1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,13 +60,15 @@ jobs: build: needs: [matrix] + runs-on: ubuntu-latest strategy: fail-fast: false matrix: ${{ fromJson(needs.setup.outputs.matrix) }} - uses: ./.github/workflows/r_build.yml - with: - revision: ${{ github.event_name == 'push' && '0' || inputs.revision }} - upload: ${{ inputs.upload }} - is_release: ${{ inputs.is_release }} - distribution: ${{ matrix.distribution }} - architecture: ${{ matrix.architecture }} + steps: + - uses: ./.github/workflows/r_build.yml + with: + revision: ${{ github.event_name == 'push' && '0' || inputs.revision }} + upload: ${{ inputs.upload }} + is_release: ${{ inputs.is_release }} + distribution: ${{ matrix.distribution }} + architecture: ${{ matrix.architecture }} From 5f0271ac189aca4ffd2d3c20388ab10a115bdbca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 22 Mar 2024 14:30:29 +0100 Subject: [PATCH 07/32] Add workflow to build packages on push --- .github/workflows/build-on-push.yml | 11 +++++++++++ .github/workflows/build.yml | 30 ++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/build-on-push.yml diff --git a/.github/workflows/build-on-push.yml b/.github/workflows/build-on-push.yml new file mode 100644 index 0000000000000..3d93836f14e49 --- /dev/null +++ b/.github/workflows/build-on-push.yml @@ -0,0 +1,11 @@ +name: Build packages (on push ) + +on: + push: + # Sequence of patterns matched against refs/heads + branches: + - "ci/*" + +jobs: + call-build-workflow: + uses: ./build.yml \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 59ea407bb63e1..4f493fa7902f5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,12 +1,8 @@ -name: Build packages +name: Build packages (on demand) # This workflow runs when any of the following occur: # - Run manually on: - push: - # Sequence of patterns matched against refs/heads - branches: - - "ci/*" workflow_dispatch: inputs: revision: @@ -18,9 +14,31 @@ on: type: boolean default: false is_release: + description: "Is release ?" + type: boolean + default: false + distribution: + description: '[ "tar", "rpm", "deb", "docker" ]' + type: string + default: '[ "rpm", "deb" ]' + architecture: + description: '[ "x64", "arm64" ]' + type: string + default: '[ "x64" ]' + workflow_call: + inputs: + revision: + description: "Revision" + type: string + default: "0" + upload: description: "Upload ?" type: boolean default: false + is_release: + description: "Is release ?" + type: boolean + default: false distribution: description: '[ "tar", "rpm", "deb", "docker" ]' type: string @@ -45,6 +63,7 @@ on: jobs: matrix: + name: Set matrix runs-on: ubuntu-latest outputs: matrix: ${{ steps.setup.outputs.matrix }} @@ -59,6 +78,7 @@ jobs: echo "$matrix" >> $GITHUB_OUTPUT build: + name: Run build pipeline needs: [matrix] runs-on: ubuntu-latest strategy: From a5e38ce05bda96f8b59eb03ba60fa8c752c781f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 22 Mar 2024 14:58:47 +0100 Subject: [PATCH 08/32] Run actionlint --- .github/workflows/build-on-push.yml | 2 +- .github/workflows/build.yml | 2 +- .github/workflows/r_assemble.yml | 1 + .github/workflows/r_build.yml | 16 ++++++++-------- .github/workflows/r_test.yml | 2 ++ 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-on-push.yml b/.github/workflows/build-on-push.yml index 3d93836f14e49..205ec2b0cfbb0 100644 --- a/.github/workflows/build-on-push.yml +++ b/.github/workflows/build-on-push.yml @@ -8,4 +8,4 @@ on: jobs: call-build-workflow: - uses: ./build.yml \ No newline at end of file + uses: ./.github/workflows/build.yml \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4f493fa7902f5..10165bf66511f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,7 +83,7 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: ${{ fromJson(needs.setup.outputs.matrix) }} + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} steps: - uses: ./.github/workflows/r_build.yml with: diff --git a/.github/workflows/r_assemble.yml b/.github/workflows/r_assemble.yml index cc510e32f22f4..00800536339f6 100644 --- a/.github/workflows/r_assemble.yml +++ b/.github/workflows/r_assemble.yml @@ -66,6 +66,7 @@ jobs: path: artifacts/dist - name: Run `assemble.sh` + run: | mkdir -p ./artifacts/dist touch ./artifacts/dist/${{ steps.package.outputs.name }} # run: | diff --git a/.github/workflows/r_build.yml b/.github/workflows/r_build.yml index 7783600b9420c..40dd5a0a50297 100644 --- a/.github/workflows/r_build.yml +++ b/.github/workflows/r_build.yml @@ -31,24 +31,24 @@ jobs: uses: ./.github/workflows/r_build_min.yml with: revision: ${{ inputs.revision }} - distribution: ${{ matrix.distribution }} - architecture: ${{ matrix.architecture }} + distribution: ${{ inputs.distribution }} + architecture: ${{ inputs.architecture }} assemble: needs: [ build_min ] uses: ./.github/workflows/r_assemble.yml with: revision: ${{ inputs.revision }} - distribution: ${{ matrix.distribution }} - architecture: ${{ matrix.architecture }} + distribution: ${{ inputs.distribution }} + architecture: ${{ inputs.architecture }} test: needs: [ assemble ] uses: ./.github/workflows/r_test.yml with: revision: ${{ inputs.revision }} - distribution: ${{ matrix.distribution }} - architecture: ${{ matrix.architecture }} + distribution: ${{ inputs.distribution }} + architecture: ${{ inputs.architecture }} upload: if: ${{ inputs.upload }} @@ -56,6 +56,6 @@ jobs: uses: ./.github/workflows/r_upload.yml with: revision: ${{ inputs.revision }} - distribution: ${{ matrix.distribution }} - architecture: ${{ matrix.architecture }} + distribution: ${{ inputs.distribution }} + architecture: ${{ inputs.architecture }} secrets: inherit diff --git a/.github/workflows/r_test.yml b/.github/workflows/r_test.yml index 8f52b733d1445..b8a9588477985 100644 --- a/.github/workflows/r_test.yml +++ b/.github/workflows/r_test.yml @@ -41,6 +41,7 @@ jobs: package_name: ${{ steps.package.outputs.name }} r_test_rpm: + needs: [setup] if: ${{ inputs.distribution == 'rpm' }} runs-on: ubuntu-latest # Permissions to download the package @@ -63,6 +64,7 @@ jobs: yum localinstall "/artifacts/dist/${{ needs.setup.outputs.package_name }}" -y r_test_deb: + needs: [setup] if: ${{ inputs.distribution == 'deb' }} runs-on: ubuntu-latest # Permissions to download the package From ced6d10a3c0400ff76b0e8c4801ffc4d7ececa17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 22 Mar 2024 15:01:51 +0100 Subject: [PATCH 09/32] Fix jq argjson --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 10165bf66511f..cc646a641b220 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,8 +71,8 @@ jobs: - id: setup run: | matrix=$(jq -cn \ - --argjson distribution ${{ inputs.distribution }} \ - --argjson architecture ${{ inputs.architecture }} \ + --argjson distribution '${{ inputs.distribution }}' \ + --argjson architecture '${{ inputs.architecture }}' \ '{distribution: $distribution, architecture: $architecture}' ) echo "$matrix" >> $GITHUB_OUTPUT From dadf9d2aa2b12538e072c65890988e6cf56c5521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 22 Mar 2024 15:34:18 +0100 Subject: [PATCH 10/32] Fix set matrix output ? --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc646a641b220..af22349bbfb4f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -75,7 +75,7 @@ jobs: --argjson architecture '${{ inputs.architecture }}' \ '{distribution: $distribution, architecture: $architecture}' ) - echo "$matrix" >> $GITHUB_OUTPUT + echo "matrix=$matrix" >> $GITHUB_OUTPUT build: name: Run build pipeline From c6cb40bb442fe2d098580dcb29691a397c77227e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 22 Mar 2024 16:09:02 +0100 Subject: [PATCH 11/32] Try new approach using a single workflow --- .github/workflows/build-on-push.yml | 2 +- .github/workflows/build_inline.yml | 141 ++++++++++++++++++++++++++++ .github/workflows/r_test.yml | 2 +- 3 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build_inline.yml diff --git a/.github/workflows/build-on-push.yml b/.github/workflows/build-on-push.yml index 205ec2b0cfbb0..995b8f3cc9fbf 100644 --- a/.github/workflows/build-on-push.yml +++ b/.github/workflows/build-on-push.yml @@ -8,4 +8,4 @@ on: jobs: call-build-workflow: - uses: ./.github/workflows/build.yml \ No newline at end of file + uses: ./.github/workflows/build_inline.yml \ No newline at end of file diff --git a/.github/workflows/build_inline.yml b/.github/workflows/build_inline.yml new file mode 100644 index 0000000000000..7b7903c3ea927 --- /dev/null +++ b/.github/workflows/build_inline.yml @@ -0,0 +1,141 @@ +name: Build packages (on demand) + +# This workflow runs when any of the following occur: +# - Run manually +on: + workflow_dispatch: + inputs: + revision: + description: "Revision" + type: string + default: "0" + upload: + description: "Upload ?" + type: boolean + default: false + is_release: + description: "Is release ?" + type: boolean + default: false + distribution: + description: '[ "tar", "rpm", "deb", "docker" ]' + type: string + default: '[ "rpm", "deb" ]' + architecture: + description: '[ "x64", "arm64" ]' + type: string + default: '[ "x64" ]' + workflow_call: + inputs: + revision: + description: "Revision" + type: string + default: "0" + upload: + description: "Upload ?" + type: boolean + default: false + is_release: + description: "Is release ?" + type: boolean + default: false + distribution: + description: '[ "tar", "rpm", "deb", "docker" ]' + type: string + default: '[ "rpm", "deb" ]' + architecture: + description: '[ "x64", "arm64" ]' + type: string + default: '[ "x64" ]' + +# ========================== +# Bibliography +# ========================== +# +# * Reusable workflows: limitations +# | https://docs.github.com/en/actions/using-workflows/reusing-workflows#limitations +# * Using matrix in reusable workflows: +# | https://docs.github.com/en/actions/using-workflows/reusing-workflows#using-a-matrix-strategy-with-a-reusable-workflow +# * Reading input from the called workflow +# | https://docs.github.com/en/enterprise-cloud@latest/actions/using-workflows/workflow-syntax-for-github-actions#onworkflow_callinputs +# * Ternary operator +# | https://docs.github.com/en/actions/learn-github-actions/expressions#example + +jobs: + matrix: + name: Set matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.setup.outputs.matrix }} + steps: + - id: setup + run: | + matrix=$(jq -cn \ + --argjson distribution '${{ inputs.distribution }}' \ + --argjson architecture '${{ inputs.architecture }}' \ + '{distribution: $distribution, architecture: $architecture}' + ) + echo "matrix=$matrix" >> $GITHUB_OUTPUT + + build: + name: Run build pipeline + needs: [matrix] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 11 + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + + - name: Provision + if: ${{ matrix.distribution == 'deb' }} + run: | + sudo bash scripts/provision.sh + + - name: Run `baptizer.sh` (min) + run: | + name=$(bash scripts/baptizer.sh -m \ + ${{ inputs.is_release && '-x' }} \ + -a ${{ matrix.architecture }} \ + -d ${{ matrix.distribution }} \ + -r ${{ inputs.revision }} \ + ) + echo "$name" >> $GITHUB_OUTPUT + id: min_package + + - name: Run `baptizer.sh` + run: | + name=$(bash scripts/baptizer.sh \ + ${{ inputs.is_release && '-x' }} \ + -a ${{ matrix.architecture }} \ + -d ${{ matrix.distribution }} \ + -r ${{ inputs.revision }} \ + ) + echo "$name" >> $GITHUB_OUTPUT + id: package + + - name: Run `build.sh` + run: | + mkdir -p ./artifacts/dist + touch ./artifacts/dist/${{ steps.min_package.outputs.name }} + + - name: Run `assemble.sh` + run: | + mkdir -p ./artifacts/dist + touch ./artifacts/dist/${{ steps.package.outputs.name }} + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.package.outputs.name }} + path: artifacts/dist/${{ steps.package.outputs.name }} + if-no-files-found: error + + \ No newline at end of file diff --git a/.github/workflows/r_test.yml b/.github/workflows/r_test.yml index b8a9588477985..101c0e6870990 100644 --- a/.github/workflows/r_test.yml +++ b/.github/workflows/r_test.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Run `baptizer.sh` (min) + - name: Run `baptizer.sh` run: | name=$(bash scripts/baptizer.sh \ ${{ inputs.is_release && '-x' }} \ From 132428d5d22042b98d056cc3401f7913925caba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 22 Mar 2024 16:11:53 +0100 Subject: [PATCH 12/32] Fix GITHUB_OUTPUT --- .github/workflows/build_inline.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_inline.yml b/.github/workflows/build_inline.yml index 7b7903c3ea927..ee0f3eeede508 100644 --- a/.github/workflows/build_inline.yml +++ b/.github/workflows/build_inline.yml @@ -107,7 +107,7 @@ jobs: -d ${{ matrix.distribution }} \ -r ${{ inputs.revision }} \ ) - echo "$name" >> $GITHUB_OUTPUT + echo "name=$name" >> $GITHUB_OUTPUT id: min_package - name: Run `baptizer.sh` @@ -118,7 +118,7 @@ jobs: -d ${{ matrix.distribution }} \ -r ${{ inputs.revision }} \ ) - echo "$name" >> $GITHUB_OUTPUT + echo "name=$name" >> $GITHUB_OUTPUT id: package - name: Run `build.sh` @@ -137,5 +137,3 @@ jobs: name: ${{ steps.package.outputs.name }} path: artifacts/dist/${{ steps.package.outputs.name }} if-no-files-found: error - - \ No newline at end of file From 5b53e52155cfeb3d1691fd1e01da5d42cacfdad8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 22 Mar 2024 16:19:23 +0100 Subject: [PATCH 13/32] Fix baptizer invocation --- .github/workflows/build_inline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_inline.yml b/.github/workflows/build_inline.yml index ee0f3eeede508..bb2afb6cc5fa7 100644 --- a/.github/workflows/build_inline.yml +++ b/.github/workflows/build_inline.yml @@ -102,10 +102,10 @@ jobs: - name: Run `baptizer.sh` (min) run: | name=$(bash scripts/baptizer.sh -m \ - ${{ inputs.is_release && '-x' }} \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ -r ${{ inputs.revision }} \ + ${{ inputs.is_release && '-x' || '' }} \ ) echo "name=$name" >> $GITHUB_OUTPUT id: min_package @@ -113,10 +113,10 @@ jobs: - name: Run `baptizer.sh` run: | name=$(bash scripts/baptizer.sh \ - ${{ inputs.is_release && '-x' }} \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ -r ${{ inputs.revision }} \ + ${{ inputs.is_release && '-x' || '' }} \ ) echo "name=$name" >> $GITHUB_OUTPUT id: package From ef09a2cc3bcf24e7887e72b23451b8e87b36014c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 22 Mar 2024 16:39:21 +0100 Subject: [PATCH 14/32] Add testing and upload to new approach --- .github/workflows/build-on-push.yml | 3 +- .github/workflows/build_inline.yml | 60 +++++++++++++++++++++-------- 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-on-push.yml b/.github/workflows/build-on-push.yml index 995b8f3cc9fbf..f838b8c493f78 100644 --- a/.github/workflows/build-on-push.yml +++ b/.github/workflows/build-on-push.yml @@ -8,4 +8,5 @@ on: jobs: call-build-workflow: - uses: ./.github/workflows/build_inline.yml \ No newline at end of file + uses: ./.github/workflows/build_inline.yml + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/build_inline.yml b/.github/workflows/build_inline.yml index bb2afb6cc5fa7..1279e7c692bb9 100644 --- a/.github/workflows/build_inline.yml +++ b/.github/workflows/build_inline.yml @@ -47,6 +47,13 @@ on: description: '[ "x64", "arm64" ]' type: string default: '[ "x64" ]' + secrets: + CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY: + required: true + description: "AWS user access key" + CI_INTERNAL_DEVELOPMENT_BUCKET_USER_SECRET_KEY: + required: true + description: "AWS user secret key" # ========================== # Bibliography @@ -63,7 +70,7 @@ on: jobs: matrix: - name: Set matrix + name: Set up matrix runs-on: ubuntu-latest outputs: matrix: ${{ steps.setup.outputs.matrix }} @@ -78,7 +85,6 @@ jobs: echo "matrix=$matrix" >> $GITHUB_OUTPUT build: - name: Run build pipeline needs: [matrix] runs-on: ubuntu-latest strategy: @@ -99,17 +105,6 @@ jobs: run: | sudo bash scripts/provision.sh - - name: Run `baptizer.sh` (min) - run: | - name=$(bash scripts/baptizer.sh -m \ - -a ${{ matrix.architecture }} \ - -d ${{ matrix.distribution }} \ - -r ${{ inputs.revision }} \ - ${{ inputs.is_release && '-x' || '' }} \ - ) - echo "name=$name" >> $GITHUB_OUTPUT - id: min_package - - name: Run `baptizer.sh` run: | name=$(bash scripts/baptizer.sh \ @@ -123,13 +118,31 @@ jobs: - name: Run `build.sh` run: | - mkdir -p ./artifacts/dist - touch ./artifacts/dist/${{ steps.min_package.outputs.name }} + bash scripts/build.sh \ + -a ${{ matrix.architecture }} \ + -d ${{ matrix.distribution }} \ + -r ${{ inputs.revision }} - name: Run `assemble.sh` run: | - mkdir -p ./artifacts/dist - touch ./artifacts/dist/${{ steps.package.outputs.name }} + bash scripts/assemble.sh \ + -a ${{ matrix.architecture }} \ + -d ${{ matrix.distribution }} \ + -r ${{ inputs.revision }} + + - name: Test RPM package + if: ${{ matrix.distribution == 'rpm' }} + uses: addnab/docker-run-action@v3 + with: + image: redhat/ubi9:latest + options: -v ${{ github.workspace }}/artifacts/dist:/artifacts/dist + run: | + yum localinstall "/artifacts/dist/${{ steps.package.outputs.name }}" -y + + - name: Test DEB package + if: ${{ matrix.distribution == 'deb' }} + run: | + sudo dpkg -i "artifacts/dist/${{ steps.package.outputs.name }}" -y - name: Upload artifact uses: actions/upload-artifact@v4 @@ -137,3 +150,16 @@ jobs: name: ${{ steps.package.outputs.name }} path: artifacts/dist/${{ steps.package.outputs.name }} if-no-files-found: error + + - name: Set up AWS CLI + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY }} + aws-secret-access-key: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_SECRET_KEY }} + aws-region: us-east-1 + + - name: Upload package to S3 + run: | + src="artifacts/dist/${{ steps.package.outputs.name }}" + dest="s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/" + aws s3 cp "$src" "$dest" From b605575beeee86f6863ab97499d0abef66265eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 22 Mar 2024 17:12:46 +0100 Subject: [PATCH 15/32] Fix hard coded revision number on RPM assembly --- .github/workflows/build_inline.yml | 5 ++++- scripts/assemble.sh | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_inline.yml b/.github/workflows/build_inline.yml index 1279e7c692bb9..517da0185bfce 100644 --- a/.github/workflows/build_inline.yml +++ b/.github/workflows/build_inline.yml @@ -55,6 +55,9 @@ on: required: true description: "AWS user secret key" +env: + TEST: true + # ========================== # Bibliography # ========================== @@ -142,7 +145,7 @@ jobs: - name: Test DEB package if: ${{ matrix.distribution == 'deb' }} run: | - sudo dpkg -i "artifacts/dist/${{ steps.package.outputs.name }}" -y + sudo dpkg -i "artifacts/dist/${{ steps.package.outputs.name }}" - name: Upload artifact uses: actions/upload-artifact@v4 diff --git a/scripts/assemble.sh b/scripts/assemble.sh index 2bd9274b06d01..60844beaecb13 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -285,7 +285,7 @@ function assemble_rpm() { version=$(cat ./usr/share/wazuh-indexer/VERSION) # Install plugins - install_plugins + # install_plugins enable_performance_analyzer_rca ${src_path} # Swap configuration files add_configuration_files @@ -305,7 +305,7 @@ function assemble_rpm() { # Move to the root folder, copy the package and clean. cd ../../.. - package_name="wazuh-indexer-${version}-1.${SUFFIX}.${EXT}" + package_name="wazuh-indexer-${version}-${REVISION}.${SUFFIX}.${EXT}" cp "${TMP_DIR}/RPMS/${SUFFIX}/${package_name}" "${OUTPUT}/dist/$ARTIFACT_PACKAGE_NAME" clean From 8c559bd513ba045670254b882b40ec1591fb4fa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 22 Mar 2024 17:23:43 +0100 Subject: [PATCH 16/32] New attempt --- .github/workflows/build_inline.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build_inline.yml b/.github/workflows/build_inline.yml index 517da0185bfce..81eeb9755cacf 100644 --- a/.github/workflows/build_inline.yml +++ b/.github/workflows/build_inline.yml @@ -55,8 +55,6 @@ on: required: true description: "AWS user secret key" -env: - TEST: true # ========================== # Bibliography From 7296f24c57e200bb5b417143936ac5a1b4ef2f78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 22 Mar 2024 17:45:06 +0100 Subject: [PATCH 17/32] Skip upload unless specified --- .github/workflows/build_inline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_inline.yml b/.github/workflows/build_inline.yml index 81eeb9755cacf..437c405f08b26 100644 --- a/.github/workflows/build_inline.yml +++ b/.github/workflows/build_inline.yml @@ -153,6 +153,7 @@ jobs: if-no-files-found: error - name: Set up AWS CLI + if: ${{ inputs.upload }} uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY }} @@ -160,6 +161,7 @@ jobs: aws-region: us-east-1 - name: Upload package to S3 + if: ${{ inputs.upload }} run: | src="artifacts/dist/${{ steps.package.outputs.name }}" dest="s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/" From ef8cd6519d6aac9c8bda623804d0762f37259d12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 22 Mar 2024 17:46:51 +0100 Subject: [PATCH 18/32] Install plugins on RPM --- scripts/assemble.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/assemble.sh b/scripts/assemble.sh index 60844beaecb13..99fbbb7c6c99e 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -285,7 +285,7 @@ function assemble_rpm() { version=$(cat ./usr/share/wazuh-indexer/VERSION) # Install plugins - # install_plugins + install_plugins enable_performance_analyzer_rca ${src_path} # Swap configuration files add_configuration_files From a36043eddbe35b80ad8e4f415b85babe85b80bea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 22 Mar 2024 18:12:41 +0100 Subject: [PATCH 19/32] Promote new approach Removes previous workflows to generate packages --- .github/workflows/build.yml | 90 +++++++++- .github/workflows/build_inline.yml | 168 ------------------ .../{build-on-push.yml => build_on_push.yml} | 0 .github/workflows/r_assemble.yml | 80 --------- .github/workflows/r_build.yml | 61 ------- .github/workflows/r_build_min.yml | 65 ------- .github/workflows/r_commit_sha.yml | 22 --- .github/workflows/r_test.yml | 119 ------------- .github/workflows/r_upload.yml | 65 ------- .github/workflows/r_version.yml | 22 --- 10 files changed, 82 insertions(+), 610 deletions(-) delete mode 100644 .github/workflows/build_inline.yml rename .github/workflows/{build-on-push.yml => build_on_push.yml} (100%) delete mode 100644 .github/workflows/r_assemble.yml delete mode 100644 .github/workflows/r_build.yml delete mode 100644 .github/workflows/r_build_min.yml delete mode 100644 .github/workflows/r_commit_sha.yml delete mode 100644 .github/workflows/r_test.yml delete mode 100644 .github/workflows/r_upload.yml delete mode 100644 .github/workflows/r_version.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index af22349bbfb4f..437c405f08b26 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,6 +47,14 @@ on: description: '[ "x64", "arm64" ]' type: string default: '[ "x64" ]' + secrets: + CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY: + required: true + description: "AWS user access key" + CI_INTERNAL_DEVELOPMENT_BUCKET_USER_SECRET_KEY: + required: true + description: "AWS user secret key" + # ========================== # Bibliography @@ -63,7 +71,7 @@ on: jobs: matrix: - name: Set matrix + name: Set up matrix runs-on: ubuntu-latest outputs: matrix: ${{ steps.setup.outputs.matrix }} @@ -78,17 +86,83 @@ jobs: echo "matrix=$matrix" >> $GITHUB_OUTPUT build: - name: Run build pipeline needs: [matrix] runs-on: ubuntu-latest strategy: fail-fast: false matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} steps: - - uses: ./.github/workflows/r_build.yml + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 with: - revision: ${{ github.event_name == 'push' && '0' || inputs.revision }} - upload: ${{ inputs.upload }} - is_release: ${{ inputs.is_release }} - distribution: ${{ matrix.distribution }} - architecture: ${{ matrix.architecture }} + distribution: temurin + java-version: 11 + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + + - name: Provision + if: ${{ matrix.distribution == 'deb' }} + run: | + sudo bash scripts/provision.sh + + - name: Run `baptizer.sh` + run: | + name=$(bash scripts/baptizer.sh \ + -a ${{ matrix.architecture }} \ + -d ${{ matrix.distribution }} \ + -r ${{ inputs.revision }} \ + ${{ inputs.is_release && '-x' || '' }} \ + ) + echo "name=$name" >> $GITHUB_OUTPUT + id: package + + - name: Run `build.sh` + run: | + bash scripts/build.sh \ + -a ${{ matrix.architecture }} \ + -d ${{ matrix.distribution }} \ + -r ${{ inputs.revision }} + + - name: Run `assemble.sh` + run: | + bash scripts/assemble.sh \ + -a ${{ matrix.architecture }} \ + -d ${{ matrix.distribution }} \ + -r ${{ inputs.revision }} + + - name: Test RPM package + if: ${{ matrix.distribution == 'rpm' }} + uses: addnab/docker-run-action@v3 + with: + image: redhat/ubi9:latest + options: -v ${{ github.workspace }}/artifacts/dist:/artifacts/dist + run: | + yum localinstall "/artifacts/dist/${{ steps.package.outputs.name }}" -y + + - name: Test DEB package + if: ${{ matrix.distribution == 'deb' }} + run: | + sudo dpkg -i "artifacts/dist/${{ steps.package.outputs.name }}" + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.package.outputs.name }} + path: artifacts/dist/${{ steps.package.outputs.name }} + if-no-files-found: error + + - name: Set up AWS CLI + if: ${{ inputs.upload }} + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY }} + aws-secret-access-key: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_SECRET_KEY }} + aws-region: us-east-1 + + - name: Upload package to S3 + if: ${{ inputs.upload }} + run: | + src="artifacts/dist/${{ steps.package.outputs.name }}" + dest="s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/" + aws s3 cp "$src" "$dest" diff --git a/.github/workflows/build_inline.yml b/.github/workflows/build_inline.yml deleted file mode 100644 index 437c405f08b26..0000000000000 --- a/.github/workflows/build_inline.yml +++ /dev/null @@ -1,168 +0,0 @@ -name: Build packages (on demand) - -# This workflow runs when any of the following occur: -# - Run manually -on: - workflow_dispatch: - inputs: - revision: - description: "Revision" - type: string - default: "0" - upload: - description: "Upload ?" - type: boolean - default: false - is_release: - description: "Is release ?" - type: boolean - default: false - distribution: - description: '[ "tar", "rpm", "deb", "docker" ]' - type: string - default: '[ "rpm", "deb" ]' - architecture: - description: '[ "x64", "arm64" ]' - type: string - default: '[ "x64" ]' - workflow_call: - inputs: - revision: - description: "Revision" - type: string - default: "0" - upload: - description: "Upload ?" - type: boolean - default: false - is_release: - description: "Is release ?" - type: boolean - default: false - distribution: - description: '[ "tar", "rpm", "deb", "docker" ]' - type: string - default: '[ "rpm", "deb" ]' - architecture: - description: '[ "x64", "arm64" ]' - type: string - default: '[ "x64" ]' - secrets: - CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY: - required: true - description: "AWS user access key" - CI_INTERNAL_DEVELOPMENT_BUCKET_USER_SECRET_KEY: - required: true - description: "AWS user secret key" - - -# ========================== -# Bibliography -# ========================== -# -# * Reusable workflows: limitations -# | https://docs.github.com/en/actions/using-workflows/reusing-workflows#limitations -# * Using matrix in reusable workflows: -# | https://docs.github.com/en/actions/using-workflows/reusing-workflows#using-a-matrix-strategy-with-a-reusable-workflow -# * Reading input from the called workflow -# | https://docs.github.com/en/enterprise-cloud@latest/actions/using-workflows/workflow-syntax-for-github-actions#onworkflow_callinputs -# * Ternary operator -# | https://docs.github.com/en/actions/learn-github-actions/expressions#example - -jobs: - matrix: - name: Set up matrix - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.setup.outputs.matrix }} - steps: - - id: setup - run: | - matrix=$(jq -cn \ - --argjson distribution '${{ inputs.distribution }}' \ - --argjson architecture '${{ inputs.architecture }}' \ - '{distribution: $distribution, architecture: $architecture}' - ) - echo "matrix=$matrix" >> $GITHUB_OUTPUT - - build: - needs: [matrix] - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 11 - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 - - - name: Provision - if: ${{ matrix.distribution == 'deb' }} - run: | - sudo bash scripts/provision.sh - - - name: Run `baptizer.sh` - run: | - name=$(bash scripts/baptizer.sh \ - -a ${{ matrix.architecture }} \ - -d ${{ matrix.distribution }} \ - -r ${{ inputs.revision }} \ - ${{ inputs.is_release && '-x' || '' }} \ - ) - echo "name=$name" >> $GITHUB_OUTPUT - id: package - - - name: Run `build.sh` - run: | - bash scripts/build.sh \ - -a ${{ matrix.architecture }} \ - -d ${{ matrix.distribution }} \ - -r ${{ inputs.revision }} - - - name: Run `assemble.sh` - run: | - bash scripts/assemble.sh \ - -a ${{ matrix.architecture }} \ - -d ${{ matrix.distribution }} \ - -r ${{ inputs.revision }} - - - name: Test RPM package - if: ${{ matrix.distribution == 'rpm' }} - uses: addnab/docker-run-action@v3 - with: - image: redhat/ubi9:latest - options: -v ${{ github.workspace }}/artifacts/dist:/artifacts/dist - run: | - yum localinstall "/artifacts/dist/${{ steps.package.outputs.name }}" -y - - - name: Test DEB package - if: ${{ matrix.distribution == 'deb' }} - run: | - sudo dpkg -i "artifacts/dist/${{ steps.package.outputs.name }}" - - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: ${{ steps.package.outputs.name }} - path: artifacts/dist/${{ steps.package.outputs.name }} - if-no-files-found: error - - - name: Set up AWS CLI - if: ${{ inputs.upload }} - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY }} - aws-secret-access-key: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_SECRET_KEY }} - aws-region: us-east-1 - - - name: Upload package to S3 - if: ${{ inputs.upload }} - run: | - src="artifacts/dist/${{ steps.package.outputs.name }}" - dest="s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/" - aws s3 cp "$src" "$dest" diff --git a/.github/workflows/build-on-push.yml b/.github/workflows/build_on_push.yml similarity index 100% rename from .github/workflows/build-on-push.yml rename to .github/workflows/build_on_push.yml diff --git a/.github/workflows/r_assemble.yml b/.github/workflows/r_assemble.yml deleted file mode 100644 index 00800536339f6..0000000000000 --- a/.github/workflows/r_assemble.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Assemble (reusable) - -# This workflow runs when any of the following occur: -# - Run from another workflow -on: - workflow_call: - inputs: - revision: - description: "Revision number" - type: string - default: "0" - is_release: - description: "Uses release's nomenclature" - type: boolean - default: false - distribution: - description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" - type: string - required: true - architecture: - description: One of "[ 'x64', 'arm64' ]" - type: string - required: true - -jobs: - r_assemble: - runs-on: ubuntu-latest - # Permissions to upload the package - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v4 - - - name: Run `baptizer.sh` (min) - run: | - name=$(bash scripts/baptizer.sh -m \ - ${{ inputs.is_release && '-x' }} \ - -a ${{ inputs.architecture }} \ - -d ${{ inputs.distribution }} \ - -r ${{ inputs.revision }} \ - ) - echo "$name" >> $GITHUB_OUTPUT - id: min_package - - - name: Run `baptizer.sh` - run: | - name=$(bash scripts/baptizer.sh \ - ${{ inputs.is_release && '-x' }} \ - -a ${{ inputs.architecture }} \ - -d ${{ inputs.distribution }} \ - -r ${{ inputs.revision }} \ - ) - echo "$name" >> $GITHUB_OUTPUT - id: package - - - name: Provision - if: ${{ inputs.distribution == 'deb' }} - run: | - sudo bash scripts/provision.sh - - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: ${{ steps.min_package.outputs.name }} - path: artifacts/dist - - - name: Run `assemble.sh` - run: | - mkdir -p ./artifacts/dist - touch ./artifacts/dist/${{ steps.package.outputs.name }} - # run: | - # bash scripts/assemble.sh -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} -r ${{ inputs.revision }} - - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: ${{ steps.package.outputs.name }} - path: artifacts/dist/${{ steps.package.outputs.name }} - if-no-files-found: error diff --git a/.github/workflows/r_build.yml b/.github/workflows/r_build.yml deleted file mode 100644 index 40dd5a0a50297..0000000000000 --- a/.github/workflows/r_build.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Build pipeline (reusable) - -# This workflow runs when any of the following occur: -# - Run from another workflow -on: - workflow_call: - inputs: - revision: - description: "Revision number" - type: string - default: "0" - upload: - description: "Uploads artifacts" - type: boolean - default: false - is_release: - description: "Uses release's nomenclature" - type: boolean - default: false - distribution: - description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" - type: string - required: true - architecture: - description: One of "[ 'x64', 'arm64' ]" - type: string - required: true - -jobs: - build_min: - uses: ./.github/workflows/r_build_min.yml - with: - revision: ${{ inputs.revision }} - distribution: ${{ inputs.distribution }} - architecture: ${{ inputs.architecture }} - - assemble: - needs: [ build_min ] - uses: ./.github/workflows/r_assemble.yml - with: - revision: ${{ inputs.revision }} - distribution: ${{ inputs.distribution }} - architecture: ${{ inputs.architecture }} - - test: - needs: [ assemble ] - uses: ./.github/workflows/r_test.yml - with: - revision: ${{ inputs.revision }} - distribution: ${{ inputs.distribution }} - architecture: ${{ inputs.architecture }} - - upload: - if: ${{ inputs.upload }} - needs: [ test ] - uses: ./.github/workflows/r_upload.yml - with: - revision: ${{ inputs.revision }} - distribution: ${{ inputs.distribution }} - architecture: ${{ inputs.architecture }} - secrets: inherit diff --git a/.github/workflows/r_build_min.yml b/.github/workflows/r_build_min.yml deleted file mode 100644 index 281e9957f9d87..0000000000000 --- a/.github/workflows/r_build_min.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Build minimal packages (reusable) - -# This workflow runs when any of the following occur: -# - Run from another workflow -on: - workflow_call: - inputs: - revision: - description: "Revision number" - type: string - default: "0" - is_release: - description: "Uses release's nomenclature" - type: boolean - default: false - distribution: - description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" - type: string - required: true - architecture: - description: One of "[ 'x64', 'arm64' ]" - type: string - required: true - -jobs: - r_build_min: - runs-on: ubuntu-latest - # Permissions to upload the package - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 11 - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 - - - name: Run `baptizer.sh` - run: | - name=$(bash scripts/baptizer.sh -m \ - ${{ inputs.is_release && '-x' }} \ - -a ${{ inputs.architecture }} \ - -d ${{ inputs.distribution }} \ - -r ${{ inputs.revision }} \ - ) - echo "$name" >> $GITHUB_OUTPUT - id: package - - - name: Run `build.sh` - run: | - mkdir -p ./artifacts/dist - touch ./artifacts/dist/${{ steps.package.outputs.name }} - # run: | - # bash scripts/build.sh -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} -r ${{ inputs.revision }} - - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: ${{ steps.package.outputs.name }} - path: artifacts/dist/${{ steps.package.outputs.name }} - if-no-files-found: error diff --git a/.github/workflows/r_commit_sha.yml b/.github/workflows/r_commit_sha.yml deleted file mode 100644 index 44860cb3e7363..0000000000000 --- a/.github/workflows/r_commit_sha.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "Get commit's short SHA (reusable)" - -# This workflow runs when any of the following occur: -# - Run from another workflow -on: - workflow_call: - outputs: - commit_sha: - description: "Returns the short SHA of the latest commit" - value: ${{ jobs.r_commit_sha.outputs.commit_sha }} - -jobs: - r_commit_sha: - runs-on: ubuntu-latest - outputs: - commit_sha: ${{ steps.get_commit_sha.outputs.commit_sha }} - steps: - - uses: actions/checkout@v4 - - name: Get git commit SHA - id: get_commit_sha - run: | - echo "commit_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT diff --git a/.github/workflows/r_test.yml b/.github/workflows/r_test.yml deleted file mode 100644 index 101c0e6870990..0000000000000 --- a/.github/workflows/r_test.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: Test (reusable) - -# This workflow runs when any of the following occur: -# - Run from another workflow -on: - workflow_call: - inputs: - revision: - description: "Revision number" - type: string - default: "0" - is_release: - description: "Uses release's nomenclature" - type: boolean - default: false - distribution: - description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" - type: string - required: true - architecture: - description: One of "[ 'x64', 'arm64' ]" - type: string - required: true - -jobs: - setup: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Run `baptizer.sh` - run: | - name=$(bash scripts/baptizer.sh \ - ${{ inputs.is_release && '-x' }} \ - -a ${{ inputs.architecture }} \ - -d ${{ inputs.distribution }} \ - -r ${{ inputs.revision }} \ - ) - echo "$name" >> $GITHUB_OUTPUT - id: package - outputs: - package_name: ${{ steps.package.outputs.name }} - - r_test_rpm: - needs: [setup] - if: ${{ inputs.distribution == 'rpm' }} - runs-on: ubuntu-latest - # Permissions to download the package - permissions: - packages: read - contents: read - steps: - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: ${{ needs.setup.outputs.package_name }} - path: artifacts/dist - - - name: Install package - uses: addnab/docker-run-action@v3 - with: - image: redhat/ubi9:latest - options: -v ${{ github.workspace }}/artifacts/dist:/artifacts/dist - run: | - yum localinstall "/artifacts/dist/${{ needs.setup.outputs.package_name }}" -y - - r_test_deb: - needs: [setup] - if: ${{ inputs.distribution == 'deb' }} - runs-on: ubuntu-latest - # Permissions to download the package - permissions: - packages: read - contents: read - steps: - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: ${{ needs.setup.outputs.package_name }} - path: artifacts/dist - - - name: Install package - run: | - sudo dpkg -i "artifacts/dist/${{ needs.setup.outputs.package_name }}" - - - uses: actions/checkout@v4 - - name: Generate and deploy certificates - uses: addnab/docker-run-action@v3 - with: - image: wazuh/wazuh-certs-generator:0.0.1 - options: -v ${{ github.workspace }}/integrations/docker/config/certs.yml:/config/certs.yml -v /etc/wazuh-indexer/certs:/certs - shell: sh - run: | - mkdir -p /certificates /certs - /entrypoint.sh - chown -R 1000:999 /certificates - chmod 740 /certificates - chmod 440 /certificates/* - - mv /certificates/wazuh.indexer-key.pem /certs/indexer-key.pem - mv /certificates/wazuh.indexer.pem /certs/indexer.pem - mv /certificates/root-ca.pem /certs/root-ca.pem - ls /certs - - - run: sudo systemctl daemon-reload - - run: | - if ! sudo systemctl enable wazuh-indexer.service; then - sudo journalctl --no-pager -u wazuh-indexer.service - exit 1 - fi - - run: | - if ! sudo systemctl start wazuh-indexer; then - sudo journalctl --no-pager -u wazuh-indexer.service - exit 1 - fi - - run: | - if ! sudo systemctl status --no-pager wazuh-indexer -n 100; then - sudo journalctl --no-pager -u wazuh-indexer.service - exit 1 - fi diff --git a/.github/workflows/r_upload.yml b/.github/workflows/r_upload.yml deleted file mode 100644 index dc0072afe37cd..0000000000000 --- a/.github/workflows/r_upload.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Upload (reusable) - -# This workflow runs when any of the following occur: -# - Run from another workflow -on: - workflow_call: - inputs: - revision: - description: "Revision number" - type: string - default: "0" - is_release: - description: "Uses release's nomenclature" - type: boolean - default: false - distribution: - description: One of "[ 'tar', 'rpm', 'deb', 'docker' ]" - type: string - required: true - architecture: - description: One of "[ 'x64', 'arm64' ]" - type: string - required: true - secrets: - CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY: - required: true - CI_INTERNAL_DEVELOPMENT_BUCKET_USER_SECRET_KEY: - required: true - -jobs: - r_upload: - runs-on: ubuntu-latest - # Permissions to upload the package - permissions: - packages: read - contents: read - steps: - - uses: actions/checkout@v4 - - name: Run `baptizer.sh` - run: | - name=$(bash scripts/baptizer.sh \ - ${{ inputs.is_release && '-x' }} \ - -a ${{ inputs.architecture }} \ - -d ${{ inputs.distribution }} \ - -r ${{ inputs.revision }} \ - ) - echo "$name" >> $GITHUB_OUTPUT - id: package - - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: ${{ steps.package.outputs.name }} - path: artifacts/dist - - - name: Set up AWS CLI - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY }} - aws-secret-access-key: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_SECRET_KEY }} - aws-region: us-east-1 - - - name: Upload package to S3 - run: | - aws s3 cp artifacts/dist/${{ steps.package.outputs.name }} s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/ diff --git a/.github/workflows/r_version.yml b/.github/workflows/r_version.yml deleted file mode 100644 index d3c01135a6c35..0000000000000 --- a/.github/workflows/r_version.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Version (reusable) - -# This workflow runs when any of the following occur: -# - Run from another workflow -on: - workflow_call: - outputs: - version: - description: "Returns the version of Wazuh" - value: ${{ jobs.r_version.outputs.version }} - -jobs: - r_version: - runs-on: ubuntu-latest - outputs: - version: ${{ steps.get_version.outputs.version }} - steps: - - uses: actions/checkout@v4 - - name: Read 'VERSION' - id: get_version - run: | - echo "version=$(cat VERSION)" >> $GITHUB_OUTPUT From 712de586677e510fbe5329896abcf07f60542832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 22 Mar 2024 18:13:38 +0100 Subject: [PATCH 20/32] Fix workflow name --- .github/workflows/build_on_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_on_push.yml b/.github/workflows/build_on_push.yml index f838b8c493f78..92094c3825222 100644 --- a/.github/workflows/build_on_push.yml +++ b/.github/workflows/build_on_push.yml @@ -8,5 +8,5 @@ on: jobs: call-build-workflow: - uses: ./.github/workflows/build_inline.yml + uses: ./.github/workflows/build.yml secrets: inherit \ No newline at end of file From e922f4123a2534361950ec0afd3a31ea83c54757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 22 Mar 2024 19:21:08 +0100 Subject: [PATCH 21/32] Attempt to fix release package naming --- .github/workflows/build.yml | 2 +- .github/workflows/build_on_push.yml | 2 +- scripts/build.sh | 217 +++++++++++++++------------- 3 files changed, 118 insertions(+), 103 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 437c405f08b26..d079549ec90db 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,7 +40,7 @@ on: type: boolean default: false distribution: - description: '[ "tar", "rpm", "deb", "docker" ]' + description: '[ "tar", "rpm", "deb" ]' type: string default: '[ "rpm", "deb" ]' architecture: diff --git a/.github/workflows/build_on_push.yml b/.github/workflows/build_on_push.yml index 92094c3825222..2c0fbe4fc3ddb 100644 --- a/.github/workflows/build_on_push.yml +++ b/.github/workflows/build_on_push.yml @@ -1,4 +1,4 @@ -name: Build packages (on push ) +name: Build packages (on push) on: push: diff --git a/scripts/build.sh b/scripts/build.sh index f20ce15018f75..79c318d5241da 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -9,6 +9,9 @@ set -ex +# ==== +# Usage +# ==== function usage() { echo "Usage: $0 [args]" echo "" @@ -18,97 +21,67 @@ function usage() { echo -e "-p PLATFORM\t[Optional] Platform, default is 'uname -s'." echo -e "-a ARCHITECTURE\t[Optional] Build architecture, default is 'uname -m'." echo -e "-d DISTRIBUTION\t[Optional] Distribution, default is 'tar'." - echo -e "-r REVISION\t[Optional] Package revision, default is '0'." + echo -e "-b BRANCH\t[Optional] Branch from wazuh/wazuh to download the index template from, default is '"$OUTPUT/artifact_min_name.txt" + # Assemble distribution artifact + # see https://github.com/opensearch-project/OpenSearch/blob/main/settings.gradle#L34 for other distribution targets + ./gradlew ":distribution:$TYPE:$TARGET:assemble" -Dbuild.snapshot="$SNAPSHOT" -Dbuild.version_qualifier="$QUALIFIER" +} + +# ==== +# Main function +# ==== +function main() { + parse_args "${@}" + + echo "Building OpenSearch for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" + if ! download_template; then + exit 1 + fi + build + + # Copy artifact to dist folder in bundle build output + echo "Copying artifact to ${OUTPUT}/dist" + local build_name + build_name=$(ls "distribution/$TYPE/$TARGET/build/distributions/" | grep "wazuh-indexer-min.*$SUFFIX.$EXT") + local package_name=${NAME:-$build_name} + mkdir -p "${OUTPUT}/dist" + cp "distribution/$TYPE/$TARGET/build/distributions/$build_name" "${OUTPUT}/dist/$package_name" +} -mkdir -p "${OUTPUT}/dist" -cp "distribution/$TYPE/$TARGET/build/distributions/$ARTIFACT_BUILD_NAME" "${OUTPUT}/dist/$ARTIFACT_PACKAGE_NAME" +main "${@}" From 063b4350c95dded2d15ce6a2d560ee5a72543015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Mon, 25 Mar 2024 16:06:03 +0100 Subject: [PATCH 22/32] Fix build.sh invocation from workflow --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d079549ec90db..2e469b00bfc9e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -122,7 +122,7 @@ jobs: bash scripts/build.sh \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ - -r ${{ inputs.revision }} + -n ${{ steps.package.outputs.name }} - name: Run `assemble.sh` run: | From c2c52cc8e7538ecf7f47b21d1194b26fd1929c86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Mon, 25 Mar 2024 16:28:18 +0100 Subject: [PATCH 23/32] Use min package name in workflow --- .github/workflows/build.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2e469b00bfc9e..b0c4cf8c5875e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -106,6 +106,17 @@ jobs: run: | sudo bash scripts/provision.sh + - name: Run `baptizer.sh` + run: | + name=$(bash scripts/baptizer.sh -m \ + -a ${{ matrix.architecture }} \ + -d ${{ matrix.distribution }} \ + -r ${{ inputs.revision }} \ + ${{ inputs.is_release && '-x' || '' }} \ + ) + echo "name=$name" >> $GITHUB_OUTPUT + id: min_package + - name: Run `baptizer.sh` run: | name=$(bash scripts/baptizer.sh \ @@ -122,7 +133,7 @@ jobs: bash scripts/build.sh \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ - -n ${{ steps.package.outputs.name }} + -n ${{ steps.min_package.outputs.name }} - name: Run `assemble.sh` run: | From 2510fa938a158a894228ea5db7e1e18752f450b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Mon, 25 Mar 2024 17:06:22 +0100 Subject: [PATCH 24/32] Use min package name for release naming convention in workflow --- .github/workflows/build.yml | 5 ++--- scripts/baptizer.sh | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b0c4cf8c5875e..ae1d0a866af9d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ on: type: boolean default: false distribution: - description: '[ "tar", "rpm", "deb", "docker" ]' + description: '[ "tar", "rpm", "deb" ]' type: string default: '[ "rpm", "deb" ]' architecture: @@ -55,7 +55,6 @@ on: required: true description: "AWS user secret key" - # ========================== # Bibliography # ========================== @@ -106,7 +105,7 @@ jobs: run: | sudo bash scripts/provision.sh - - name: Run `baptizer.sh` + - name: Run `baptizer.sh` (min) run: | name=$(bash scripts/baptizer.sh -m \ -a ${{ matrix.architecture }} \ diff --git a/scripts/baptizer.sh b/scripts/baptizer.sh index ccd51ec58d15d..ca283bd39d872 100644 --- a/scripts/baptizer.sh +++ b/scripts/baptizer.sh @@ -43,6 +43,7 @@ function parse_args() { ;; x) IS_RELEASE=true + REVISION="1" ;; :) echo "Error: -${OPTARG} requires an argument" @@ -113,6 +114,9 @@ function get_release_name() { else PACKAGE_NAME=wazuh-indexer_"$VERSION"-"$REVISION"_"$SUFFIX"."$EXT" fi + if "$IS_MIN"; then + PACKAGE_NAME=${PACKAGE_NAME/wazuh-indexer/wazuh-indexer-min} + fi } # ==== From 4c8233b19decc983dcecfe760bea4fe26f6f8736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Tue, 26 Mar 2024 12:31:40 +0100 Subject: [PATCH 25/32] Attemtp to fix regex --- scripts/assemble.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/assemble.sh b/scripts/assemble.sh index 99fbbb7c6c99e..3461ea6d782fb 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -376,7 +376,7 @@ function main() { echo "Assembling wazuh-indexer for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" VERSION=$(bash scripts/upstream_version.sh) - ARTIFACT_BUILD_NAME=$(ls "${OUTPUT}/dist/" | grep "wazuh-indexer-min_.*$SUFFIX.*\.$EXT") + ARTIFACT_BUILD_NAME=$(ls "${OUTPUT}/dist/" | grep "wazuh-indexer-min.*$SUFFIX.*\.$EXT") ARTIFACT_PACKAGE_NAME=${ARTIFACT_BUILD_NAME/-min/} # Create temporal directory and copy the min package there for extraction From 55df2c2935fd67aa2709bad6d5aa069e9a167622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Tue, 26 Mar 2024 13:56:08 +0100 Subject: [PATCH 26/32] Upgrade to aws-actions/configure-aws-credentials@v4 Clean up --- .github/workflows/build.yml | 2 +- scripts/assemble.sh | 2 -- scripts/baptizer.sh | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ae1d0a866af9d..f18da6c86693a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -164,7 +164,7 @@ jobs: - name: Set up AWS CLI if: ${{ inputs.upload }} - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY }} aws-secret-access-key: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_SECRET_KEY }} diff --git a/scripts/assemble.sh b/scripts/assemble.sh index 3461ea6d782fb..1c9f7e1bcabf2 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -224,8 +224,6 @@ function clean() { echo "Cleaning temporary ${TMP_DIR} folder" rm -r "${OUTPUT}/tmp" echo "After execution, shell path is $(pwd)" - # Store package's name to file. Used by GH Action. - echo "${ARTIFACT_PACKAGE_NAME}" >"${OUTPUT}/artifact_name.txt" } # ==== diff --git a/scripts/baptizer.sh b/scripts/baptizer.sh index ca283bd39d872..34985d988967e 100644 --- a/scripts/baptizer.sh +++ b/scripts/baptizer.sh @@ -122,7 +122,6 @@ function get_release_name() { # ==== # Naming convention for pre-release packages # ==== -# TODO wazuh-indexer_$VERSION != wazuh-indexer-$VERSION function get_devel_name() { if "$IS_MIN"; then PACKAGE_NAME=wazuh-indexer-min_"$VERSION"-"$REVISION"_"$SUFFIX"_"$GIT_COMMIT"."$EXT" From 582723bfff1f5fce4b519fa66c1ba1d3449b5056 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Tue, 26 Mar 2024 18:50:49 +0100 Subject: [PATCH 27/32] Apply latest requirements Add workflow with single matrix for QA use. Rename inputs. Add checksum input. --- .github/workflows/build.yml | 20 +++++++++++----- .github/workflows/build_single.yml | 37 ++++++++++++++++++++++++++++++ scripts/baptizer.sh | 2 +- 3 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/build_single.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f18da6c86693a..ca5541df0e4ae 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,8 +13,8 @@ on: description: "Upload ?" type: boolean default: false - is_release: - description: "Is release ?" + is_stage: + description: "Is stage ?" type: boolean default: false distribution: @@ -25,6 +25,10 @@ on: description: '[ "x64", "arm64" ]' type: string default: '[ "x64" ]' + checksum: + description: "Checksum ?" + type: boolean + default: false workflow_call: inputs: revision: @@ -35,8 +39,8 @@ on: description: "Upload ?" type: boolean default: false - is_release: - description: "Is release ?" + is_stage: + description: "Is stage ?" type: boolean default: false distribution: @@ -47,6 +51,10 @@ on: description: '[ "x64", "arm64" ]' type: string default: '[ "x64" ]' + checksum: + description: "Checksum ?" + type: boolean + default: false secrets: CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY: required: true @@ -111,7 +119,7 @@ jobs: -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ -r ${{ inputs.revision }} \ - ${{ inputs.is_release && '-x' || '' }} \ + ${{ inputs.is_stage && '-x' || '' }} \ ) echo "name=$name" >> $GITHUB_OUTPUT id: min_package @@ -122,7 +130,7 @@ jobs: -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ -r ${{ inputs.revision }} \ - ${{ inputs.is_release && '-x' || '' }} \ + ${{ inputs.is_stage && '-x' || '' }} \ ) echo "name=$name" >> $GITHUB_OUTPUT id: package diff --git a/.github/workflows/build_single.yml b/.github/workflows/build_single.yml new file mode 100644 index 0000000000000..7bf24a49f4511 --- /dev/null +++ b/.github/workflows/build_single.yml @@ -0,0 +1,37 @@ +name: Build packages (single) + +on: + workflow_dispatch: + inputs: + revision: + description: "Revision" + type: string + default: "0" + checksum: + description: "Checksum ?" + type: boolean + default: false + is_stage: + description: "Is stage ?" + type: boolean + default: false + system: + description: "One of 'tar', 'rpm', 'deb'" + type: string + default: "rpm" + architecture: + description: "One of 'amd64', 'x86_64'" + type: string + default: "x86_64" + +jobs: + call-build-workflow: + uses: ./.github/workflows/build.yml + with: + revision: ${{ inputs.revision }} + checksum: ${{ inputs.checksum }} + is_stage: ${{ inputs.is_stage }} + distribution: '[ "${{ inputs.system }}" ]' + # Architecture is always 'x64', which is the default value in ./build.yml + # It is an input just for convenience and standardisation. + secrets: inherit diff --git a/scripts/baptizer.sh b/scripts/baptizer.sh index 34985d988967e..715830b69db9b 100644 --- a/scripts/baptizer.sh +++ b/scripts/baptizer.sh @@ -43,7 +43,7 @@ function parse_args() { ;; x) IS_RELEASE=true - REVISION="1" + # REVISION="1" ;; :) echo "Error: -${OPTARG} requires an argument" From 9d273e1154e6707a09292f892aee7e421cb870d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Tue, 26 Mar 2024 19:06:49 +0100 Subject: [PATCH 28/32] Add checksum generation and upload --- .github/workflows/build.yml | 7 +++++++ scripts/assemble.sh | 3 +++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ca5541df0e4ae..d735f3aa27df5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -184,3 +184,10 @@ jobs: src="artifacts/dist/${{ steps.package.outputs.name }}" dest="s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/" aws s3 cp "$src" "$dest" + + - name: Upload checksum to S3 + if: ${{ inputs.checksum }} + run: | + src="artifacts/dist/${{ steps.package.outputs.name }}.sha512" + dest="s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/" + aws s3 cp "$src" "$dest" diff --git a/scripts/assemble.sh b/scripts/assemble.sh index 1c9f7e1bcabf2..f36fadc1ab01e 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -393,6 +393,9 @@ function main() { assemble_deb ;; esac + + # Create checksum + sha512sum "${OUTPUT}/dist/$ARTIFACT_PACKAGE_NAME" > "${OUTPUT}/dist/$ARTIFACT_PACKAGE_NAME".sha512 } main "${@}" From 81f54d267c246600e5ae0020969be00a11feaf97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Tue, 26 Mar 2024 19:32:03 +0100 Subject: [PATCH 29/32] Use choice as input types for system and architecture --- .github/workflows/build_single.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_single.yml b/.github/workflows/build_single.yml index 7bf24a49f4511..becc6348e880b 100644 --- a/.github/workflows/build_single.yml +++ b/.github/workflows/build_single.yml @@ -16,13 +16,19 @@ on: type: boolean default: false system: - description: "One of 'tar', 'rpm', 'deb'" - type: string - default: "rpm" + description: "Package OS" + type: choice + options: + - rpm + - deb + default: deb architecture: - description: "One of 'amd64', 'x86_64'" - type: string - default: "x86_64" + description: "Package architecture" + type: choice + options: + - amd64 + - x86_64 + default: amd64 jobs: call-build-workflow: From 047a822699d587369d7435a032580e5eaa64d2d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Tue, 26 Mar 2024 19:35:48 +0100 Subject: [PATCH 30/32] Invoke build single packages with upload option --- .github/workflows/build.yml | 2 +- .github/workflows/build_single.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d735f3aa27df5..c01946875f96c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -186,7 +186,7 @@ jobs: aws s3 cp "$src" "$dest" - name: Upload checksum to S3 - if: ${{ inputs.checksum }} + if: ${{ inputs.upload && inputs.checksum }} run: | src="artifacts/dist/${{ steps.package.outputs.name }}.sha512" dest="s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/" diff --git a/.github/workflows/build_single.yml b/.github/workflows/build_single.yml index becc6348e880b..ec516623f6bb8 100644 --- a/.github/workflows/build_single.yml +++ b/.github/workflows/build_single.yml @@ -38,6 +38,7 @@ jobs: checksum: ${{ inputs.checksum }} is_stage: ${{ inputs.is_stage }} distribution: '[ "${{ inputs.system }}" ]' + upload: true # Architecture is always 'x64', which is the default value in ./build.yml # It is an input just for convenience and standardisation. secrets: inherit From 460481e36ae2c3acb5fd9fe5b539656ba7124e58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Wed, 27 Mar 2024 12:51:24 +0100 Subject: [PATCH 31/32] Add documentation and clean up --- .github/workflows/build.yml | 1 + .github/workflows/build_on_push.yml | 2 + .github/workflows/build_single.yml | 2 + scripts/README.md | 280 +++++++++++++++------------- scripts/baptizer.sh | 1 - 5 files changed, 156 insertions(+), 130 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c01946875f96c..4b17fbc040159 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,6 +2,7 @@ name: Build packages (on demand) # This workflow runs when any of the following occur: # - Run manually +# - Invoked from another workflow on: workflow_dispatch: inputs: diff --git a/.github/workflows/build_on_push.yml b/.github/workflows/build_on_push.yml index 2c0fbe4fc3ddb..bc278450d3069 100644 --- a/.github/workflows/build_on_push.yml +++ b/.github/workflows/build_on_push.yml @@ -1,5 +1,7 @@ name: Build packages (on push) +# This workflow runs when any of the following occur: +# - On push to branches named after ci/* on: push: # Sequence of patterns matched against refs/heads diff --git a/.github/workflows/build_single.yml b/.github/workflows/build_single.yml index ec516623f6bb8..669a987bdfd27 100644 --- a/.github/workflows/build_single.yml +++ b/.github/workflows/build_single.yml @@ -1,5 +1,7 @@ name: Build packages (single) +# This workflow runs when any of the following occur: +# - Run manually on: workflow_dispatch: inputs: diff --git a/scripts/README.md b/scripts/README.md index 7cfa2a63387f5..15f6926dfcaab 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -14,10 +14,16 @@ Each section includes instructions to generate packages locally, using Act or Do - [Install Act](https://github.com/nektos/act) +The names of the packages are managed by the `baptizer.sh` script. + ## Build -... -... +For local package generation, use the `build.sh` script. Take a look at the `build.yml` +workflow file for an example of usage. + +```bash +bash scripts/build.sh -a x64 -d tar -n $(bash scripts/baptizer.sh -a x64 -d tar -m) +``` #### Act (GitHub Workflow locally) @@ -32,89 +38,101 @@ act -j build -W .github/workflows/build.yml --artifact-server-path ./artifacts Using the [Docker environment](../docker): ```console -docker exec -it wi-build_$( -### DEB +The assembly process for tarballs consists on: -The script will: +1. Extract. +2. Install plugins. +3. Add Wazuh's configuration files and tools. +4. Compress. -- Extract the deb package using `ar` and `tar` tools. - - > By default, `ar` and `tar` tools expect the package to be in `wazuh-indexer/artifacts/tmp/deb`. The script takes care of creating the required folder structure, copying also the min package and the Makefile. - - Current folder loadout at this stage: - - ``` - artifacts/ - |-- dist - | |-- wazuh-indexer-min_4.9.0_amd64.deb - `-- tmp - `-- deb - |-- Makefile - |-- data.tar.gz - |-- debmake_install.sh - |-- etc - |-- usr - |-- var - `-- wazuh-indexer-min_4.9.0_amd64.deb - ``` - - `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.deb`. - `Makefile` and the `debmake_install` are copied over from `wazuh-indexer/distribution/packages/src/deb`. - The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. - -- Install the plugins using the `opensearch-plugin` CLI tool. -- Set up configuration files. - - > Included in `min-package`. Default files are overwritten. - -- Bundle a DEB file with `debmake` and the `Makefile`. - - > `debmake` and other dependencies can be installed using the provision.sh script. The - > script is invoked by the GitHub Workflow. - - Current folder loadout at this stage: - - ``` - artifacts/ - |-- artifact_name.txt - |-- dist - | |-- wazuh-indexer-min_4.9.0_amd64.deb - | `-- wazuh-indexer_4.9.0_amd64.deb - `-- tmp - `-- deb - |-- Makefile - |-- data.tar.gz - |-- debmake_install.sh - |-- etc - |-- usr - |-- var - |-- wazuh-indexer-min_4.9.0_amd64.deb - `-- debian/ - | -- control - | -- copyright - | -- rules - | -- preinst - | -- prerm - | -- postinst - ``` - -### Running in Act +```console +bash scripts/assemble.sh -a x64 -d tar -r 1 +``` + +### DEB + +For DEB packages, the `assemble.sh` script will perform the following operations: + +1. Extract the deb package using `ar` and `tar` tools. + + > By default, `ar` and `tar` tools expect the package to be in `wazuh-indexer/artifacts/tmp/deb`. + > The script takes care of creating the required folder structure, copying also the min package + > and the Makefile. + + Current folder loadout at this stage: + + ``` + artifacts/ + |-- dist + | |-- wazuh-indexer-min_4.9.0_amd64.deb + `-- tmp + `-- deb + |-- Makefile + |-- data.tar.gz + |-- debmake_install.sh + |-- etc + |-- usr + |-- var + `-- wazuh-indexer-min_4.9.0_amd64.deb + ``` + + `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.deb`. + `Makefile` and the `debmake_install` are copied over from `wazuh-indexer/distribution/packages/src/deb`. + The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. + +2. Install the plugins using the `opensearch-plugin` CLI tool. +3. Set up configuration files. + + > Included in `min-package`. Default files are overwritten. + +4. Bundle a DEB file with `debmake` and the `Makefile`. + + > `debmake` and other dependencies can be installed using the `provision.sh` script. + > The script is invoked by the GitHub Workflow. + + Current folder loadout at this stage: + + ``` + artifacts/ + |-- artifact_name.txt + |-- dist + | |-- wazuh-indexer-min_4.9.0_amd64.deb + | `-- wazuh-indexer_4.9.0_amd64.deb + `-- tmp + `-- deb + |-- Makefile + |-- data.tar.gz + |-- debmake_install.sh + |-- etc + |-- usr + |-- var + |-- wazuh-indexer-min_4.9.0_amd64.deb + `-- debian/ + | -- control + | -- copyright + | -- rules + | -- preinst + | -- prerm + | -- postinst + ``` + +#### Running in Act ```console -act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:deb --matrix architecture:x64 --var OPENSEARCH_VERSION=2.11.1 +act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:deb --matrix architecture:x64 [Build slim packages/build] 🏁 Job succeeded ``` @@ -128,7 +146,7 @@ Pre-requisites: - Using the [Docker environment](../docker): ```console -docker exec -it wi-assemble_$( By default, `rpm2cpio` and `cpio` tools expect the package to be in `wazuh-indexer/artifacts/tmp/rpm`. The script takes care of creating the required folder structure, copying also the min package and the SPEC file. + > By default, `rpm2cpio` and `cpio` tools expect the package to be in `wazuh-indexer/artifacts/tmp/rpm`.The script takes care of creating the required folder structure, copying also the min package and the SPEC file. - Current folder loadout at this stage: + Current folder loadout at this stage: - ``` - /rpm/$ARCH - /etc - /usr - /var - wazuh-indexer-min-*.rpm - wazuh-indexer.rpm.spec - ``` + ``` + /rpm/$ARCH + /etc + /usr + /var + wazuh-indexer-min-*.rpm + wazuh-indexer.rpm.spec + ``` - `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.rpm`. - `wazuh-indexer.rpm.spec` is copied over from `wazuh-indexer/distribution/packages/src/rpm/wazuh-indexer.rpm.spec`. - The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. + `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.rpm`. + `wazuh-indexer.rpm.spec` is copied over from `wazuh-indexer/distribution/packages/src/rpm/wazuh-indexer.rpm.spec`. + The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. -- Install the plugins using the `opensearch-plugin` CLI tool. -- Set up configuration files. +2. Install the plugins using the `opensearch-plugin` CLI tool. +3. Set up configuration files. - > Included in `min-package`. Default files are overwritten. + > Included in `min-package`. Default files are overwritten. -- Bundle an RPM file with `rpmbuild` and the SPEC file `wazuh-indexer.rpm.spec`. +4. Bundle an RPM file with `rpmbuild` and the SPEC file `wazuh-indexer.rpm.spec`. - - `rpmbuild` is part of the `rpm` OS package. + > `rpmbuild` is part of the `rpm` OS package. - > `rpmbuild` is invoked from `wazuh-indexer/artifacts/tmp/rpm`. It creates the {BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} folders and applies the rules in the SPEC file. If successful, `rpmbuild` will generate the package in the `RPMS/` folder. The script will copy it to `wazuh-indexer/artifacts/dist` and clean: remove the `tmp\` folder and its contents. + > `rpmbuild` is invoked from `wazuh-indexer/artifacts/tmp/rpm`. It creates the {BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} folders and applies the rules in the SPEC file. If successful, `rpmbuild` will generate the package in the `RPMS/` folder. The script will copy it to `wazuh-indexer/artifacts/dist` and clean: remove the `tmp\` folder and its contents. - Current folder loadout at this stage: + Current folder loadout at this stage: - ``` - /rpm/$ARCH - /{BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} - /etc - /usr - /var - wazuh-indexer-min-*.rpm - wazuh-indexer.rpm.spec - ``` + ``` + /rpm/$ARCH + /{BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} + /etc + /usr + /var + wazuh-indexer-min-*.rpm + wazuh-indexer.rpm.spec + ``` -### Running in Act +#### Running in Act ```console act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:rpm --matrix architecture:x64 --var OPENSEARCH_VERSION=2.11.1 @@ -198,45 +216,49 @@ Pre-requisites: - Using the [Docker environment](../docker): ```console -docker exec -it wi-assemble_$( Date: Wed, 27 Mar 2024 14:07:03 +0100 Subject: [PATCH 32/32] Rename scripts folder to packaging_scripts --- .github/workflows/build.yml | 10 +++++----- {scripts => packaging_scripts}/README.md | 10 +++++----- {scripts => packaging_scripts}/assemble.sh | 2 +- {scripts => packaging_scripts}/baptizer.sh | 0 {scripts => packaging_scripts}/build.sh | 0 {scripts => packaging_scripts}/provision.sh | 0 {scripts => packaging_scripts}/upstream_version.sh | 0 7 files changed, 11 insertions(+), 11 deletions(-) rename {scripts => packaging_scripts}/README.md (94%) rename {scripts => packaging_scripts}/assemble.sh (99%) rename {scripts => packaging_scripts}/baptizer.sh (100%) rename {scripts => packaging_scripts}/build.sh (100%) rename {scripts => packaging_scripts}/provision.sh (100%) rename {scripts => packaging_scripts}/upstream_version.sh (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4b17fbc040159..456ad172bd5f2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -112,11 +112,11 @@ jobs: - name: Provision if: ${{ matrix.distribution == 'deb' }} run: | - sudo bash scripts/provision.sh + sudo bash packaging_scripts/provision.sh - name: Run `baptizer.sh` (min) run: | - name=$(bash scripts/baptizer.sh -m \ + name=$(bash packaging_scripts/baptizer.sh -m \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ -r ${{ inputs.revision }} \ @@ -127,7 +127,7 @@ jobs: - name: Run `baptizer.sh` run: | - name=$(bash scripts/baptizer.sh \ + name=$(bash packaging_scripts/baptizer.sh \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ -r ${{ inputs.revision }} \ @@ -138,14 +138,14 @@ jobs: - name: Run `build.sh` run: | - bash scripts/build.sh \ + bash packaging_scripts/build.sh \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ -n ${{ steps.min_package.outputs.name }} - name: Run `assemble.sh` run: | - bash scripts/assemble.sh \ + bash packaging_scripts/assemble.sh \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ -r ${{ inputs.revision }} diff --git a/scripts/README.md b/packaging_scripts/README.md similarity index 94% rename from scripts/README.md rename to packaging_scripts/README.md index 15f6926dfcaab..d81ce569be8e4 100644 --- a/scripts/README.md +++ b/packaging_scripts/README.md @@ -22,7 +22,7 @@ For local package generation, use the `build.sh` script. Take a look at the `bui workflow file for an example of usage. ```bash -bash scripts/build.sh -a x64 -d tar -n $(bash scripts/baptizer.sh -a x64 -d tar -m) +bash packaging_scripts/build.sh -a x64 -d tar -n $(bash packaging_scripts/baptizer.sh -a x64 -d tar -m) ``` #### Act (GitHub Workflow locally) @@ -38,7 +38,7 @@ act -j build -W .github/workflows/build.yml --artifact-server-path ./artifacts Using the [Docker environment](../docker): ```console -docker exec -it wi-build_$(