From 98472c51ae5ab8d94e0c73512e7f1c108e6d9955 Mon Sep 17 00:00:00 2001 From: Fede Tux Date: Mon, 15 Jan 2024 08:52:38 -0300 Subject: [PATCH 01/14] Add function to look for VERSION in the correct path --- scripts/assemble.sh | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/assemble.sh b/scripts/assemble.sh index 9f25c304808b4..256cea62ed12a 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -172,18 +172,36 @@ function remove_unneeded_files() { rm "$PATH_PLUGINS/opensearch-security/tools/install_demo_configuration.sh" } +# ==== +# Get version from VERSION file +# ==== +function get_version() +{ + case $PACKAGE in + tar) + versionfile="./VERSION" + ;; + rpm|deb) + versionfile="./usr/share/wazuh-indexer/VERSION" + ;; + esac + + echo $(cat $versionfile) +} + # ==== # Add additional tools into packages # ==== function add_wazuh_tools() { local version - version=$( Date: Mon, 15 Jan 2024 14:39:33 +0100 Subject: [PATCH 02/14] Update assemble.sh Adds wget as dependency --- scripts/README.md | 14 ++++---- scripts/assemble.sh | 86 +++++++++++++++++++------------------------- scripts/provision.sh | 2 +- 3 files changed, 44 insertions(+), 58 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 8c98f8206cab5..1f628c5fbbccd 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -32,7 +32,7 @@ act -j build -W .github/workflows/build.yml --artifact-server-path ./artifacts Within the [Docker environment](../docker): ```console -bash scripts/build.sh -v 2.11.0 -s false -p linux -a {x64|arm64} -d {rpm|deb|tar} +bash scripts/build.sh -v 2.11.1 -s false -p linux -a {x64|arm64} -d {rpm|deb|tar} ``` The generated package is sent to `artifacts/` @@ -104,7 +104,7 @@ The script will: ### Running in Act ```console -act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:deb --matrix architecture:x64 --var OPENSEARCH_VERSION=2.11.0 +act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:deb --matrix architecture:x64 --var OPENSEARCH_VERSION=2.11.1 [Build slim packages/build] 🏁 Job succeeded ``` @@ -128,7 +128,7 @@ docker run --rm \ # https://github.com/opensearch-project/opensearch-build/blob/2.11.1/docker/ci/dockerfiles/current/build.ubuntu2004.opensearch.x64.arm64.dockerfile # Install necessary packages -apt-get update -y && apt-get upgrade -y && apt-get install -y curl build-essential curl && +apt-get update -y && apt-get upgrade -y && apt-get install -y curl wget build-essential && apt-get install -y debmake debhelper-compat && apt-get install -y libxrender1 libxtst6 libasound2 libxi6 libgconf-2-4 && apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libatspi2.0-dev libxcomposite-dev libxdamage1 libxfixes3 libxfixes-dev libxrandr2 libgbm-dev libxkbcommon-x11-0 libpangocairo-1.0-0 libcairo2 libcairo2-dev libnss3 libnspr4 libnspr4-dev freeglut3 && @@ -141,7 +141,7 @@ curl -o- https://www.aptly.info/pubkey.txt | apt-key add - && apt-get update -y && apt-get install -y aptly && apt-get clean -y && dpkg -r lintian -bash scripts/assemble.sh -v 2.11.0 -p linux -a x64 -d deb +bash scripts/assemble.sh -v 2.11.1 -p linux -a x64 -d deb ``` ### RPM @@ -197,7 +197,7 @@ The script will: ### Running in Act ```console -act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:rpm --matrix architecture:x64 --var OPENSEARCH_VERSION=2.11.0 +act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:rpm --matrix architecture:x64 --var OPENSEARCH_VERSION=2.11.1 [Build slim packages/build] 🏁 Job succeeded ``` @@ -219,6 +219,6 @@ docker run --rm \ -it ubuntu:jammy /bin/bash apt-get update -apt-get install -y rpm2cpio rpm cpio -bash scripts/assemble.sh -v 2.11.0 -p linux -a x64 -d rpm +apt-get install -y rpm2cpio rpm cpio wget +bash scripts/assemble.sh -v 2.11.1 -p linux -a x64 -d rpm ``` diff --git a/scripts/assemble.sh b/scripts/assemble.sh index 256cea62ed12a..18b5e765d7172 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -10,32 +10,32 @@ set -ex # Minimum required plugins -# plugins=( -# "performance-analyzer" -# "opensearch-security" -# ) - plugins=( - "alerting" # "opensearch-alerting" - "opensearch-job-scheduler" - "opensearch-anomaly-detection" # Requires "opensearch-job-scheduler" - "asynchronous-search" # "opensearch-asynchronous-search" - "opensearch-cross-cluster-replication" - "geospatial" # "opensearch-geospatial" - "opensearch-index-management" - "opensearch-knn" - "opensearch-ml-plugin" # "opensearch-ml" - "neural-search" # "opensearch-neural-search" - "opensearch-notifications-core" - "notifications" # "opensearch-notifications". Requires "opensearch-notifications-core" - "opensearch-observability" - "performance-analyzer" # "opensearch-performance-analyzer" - "opensearch-reports-scheduler" + "performance-analyzer" "opensearch-security" - "opensearch-security-analytics" - "opensearch-sql-plugin" # "opensearch-sql" ) +# plugins=( +# "alerting" # "opensearch-alerting" +# "opensearch-job-scheduler" +# "opensearch-anomaly-detection" # Requires "opensearch-job-scheduler" +# "asynchronous-search" # "opensearch-asynchronous-search" +# "opensearch-cross-cluster-replication" +# "geospatial" # "opensearch-geospatial" +# "opensearch-index-management" +# "opensearch-knn" +# "opensearch-ml-plugin" # "opensearch-ml" +# "neural-search" # "opensearch-neural-search" +# "opensearch-notifications-core" +# "notifications" # "opensearch-notifications". Requires "opensearch-notifications-core" +# "opensearch-observability" +# "performance-analyzer" # "opensearch-performance-analyzer" +# "opensearch-reports-scheduler" +# "opensearch-security" +# "opensearch-security-analytics" +# "opensearch-sql-plugin" # "opensearch-sql" +# ) + # ==== # Usage # ==== @@ -172,30 +172,11 @@ function remove_unneeded_files() { rm "$PATH_PLUGINS/opensearch-security/tools/install_demo_configuration.sh" } -# ==== -# Get version from VERSION file -# ==== -function get_version() -{ - case $PACKAGE in - tar) - versionfile="./VERSION" - ;; - rpm|deb) - versionfile="./usr/share/wazuh-indexer/VERSION" - ;; - esac - - echo $(cat $versionfile) -} - # ==== # Add additional tools into packages # ==== function add_wazuh_tools() { - local version - version=$(get_version) - version=${version%%.[[:digit:]]} + local version=${1%%.[[:digit:]]} local download_url download_url="https://packages-dev.wazuh.com/${version}" @@ -259,15 +240,18 @@ function assemble_tar() { tar -zvxf "${ARTIFACT_BUILD_NAME}" cd "$(ls -d wazuh-indexer-*/)" + local version + version=$(cat VERSION) + # Install plugins install_plugins # Swap configuration files add_configuration_files remove_unneeded_files - add_wazuh_tools + add_wazuh_tools "${version}" # Pack - archive_name="wazuh-indexer-$(cat VERSION)" + archive_name="wazuh-indexer-${version}" cd .. tar -cvf "${archive_name}-${SUFFIX}.${EXT}" "${archive_name}" cd ../../.. @@ -295,20 +279,21 @@ function assemble_rpm() { echo "Extract ${ARTIFACT_BUILD_NAME} archive" rpm2cpio "${ARTIFACT_BUILD_NAME}" | cpio -imdv + local version + version=$(cat ./usr/share/wazuh-indexer/VERSION) + # Install plugins install_plugins enable_performance_analyzer_rca ${src_path} # Swap configuration files add_configuration_files remove_unneeded_files - add_wazuh_tools + add_wazuh_tools "${version}" # Generate final package local topdir - local version local spec_file="wazuh-indexer.rpm.spec" topdir=$(pwd) - version=$(cat ./usr/share/wazuh-indexer/VERSION) rpmbuild --bb \ --define "_topdir ${topdir}" \ --define "_version ${version}" \ @@ -345,17 +330,18 @@ function assemble_deb() { ar xf "${ARTIFACT_BUILD_NAME}" data.tar.gz tar zvxf data.tar.gz + local version + version=$(cat ./usr/share/wazuh-indexer/VERSION) + # Install plugins install_plugins enable_performance_analyzer_rca ${src_path} # Swap configuration files add_configuration_files remove_unneeded_files - add_wazuh_tools + add_wazuh_tools "${version}" # Generate final package - local version - version=$(cat ./usr/share/wazuh-indexer/VERSION) debmake \ --fullname "Wazuh Team" \ --email "hello@wazuh.com" \ diff --git a/scripts/provision.sh b/scripts/provision.sh index 3cd155dc44fb5..2984deebeada9 100644 --- a/scripts/provision.sh +++ b/scripts/provision.sh @@ -3,7 +3,7 @@ # Provision script for assembly of DEB packages # Install necessary packages -apt-get update -y && apt-get upgrade -y && apt-get install -y curl build-essential curl && +apt-get update -y && apt-get upgrade -y && apt-get install -y curl wget build-essential && apt-get install -y debmake debhelper-compat && apt-get install -y libxrender1 libxtst6 libasound2 libxi6 libgconf-2-4 && apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libatspi2.0-dev libxcomposite-dev libxdamage1 libxfixes3 libxfixes-dev libxrandr2 libgbm-dev libxkbcommon-x11-0 libpangocairo-1.0-0 libcairo2 libcairo2-dev libnss3 libnspr4 libnspr4-dev freeglut3 && From fc3c6bc877e9202495d06438aa81a19a97ae87a3 Mon Sep 17 00:00:00 2001 From: Fede Tux Date: Mon, 15 Jan 2024 11:24:14 -0300 Subject: [PATCH 03/14] Download files using curl instead of wget --- scripts/assemble.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/assemble.sh b/scripts/assemble.sh index 18b5e765d7172..bda5ea2b9743a 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -181,9 +181,9 @@ function add_wazuh_tools() { local download_url download_url="https://packages-dev.wazuh.com/${version}" - wget -q "${download_url}/config.yml" -O $PATH_PLUGINS/opensearch-security/tools/config.yml - wget -q "${download_url}/wazuh-passwords-tool.sh" -O $PATH_PLUGINS/opensearch-security/tools/wazuh-passwords-tool.sh - wget -q "${download_url}/wazuh-certs-tool.sh" -O $PATH_PLUGINS/opensearch-security/tools/wazuh-certs-tool.sh + curl -sL "${download_url}/config.yml" -o $PATH_PLUGINS/opensearch-security/tools/config.yml + curl -sL "${download_url}/wazuh-passwords-tool.sh" -o $PATH_PLUGINS/opensearch-security/tools/wazuh-passwords-tool.sh + curl -sL "${download_url}/wazuh-certs-tool.sh" -o $PATH_PLUGINS/opensearch-security/tools/wazuh-certs-tool.sh } # ==== From 43a9a56e5db36ce9e0ea6b5ace43dccfc8fcd65f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Mon, 15 Jan 2024 18:43:54 +0100 Subject: [PATCH 04/14] Update assemble.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert assembly with minimal plugins for testing Signed-off-by: Álex Ruiz --- scripts/assemble.sh | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/scripts/assemble.sh b/scripts/assemble.sh index bda5ea2b9743a..f88b075257a4d 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -10,32 +10,32 @@ set -ex # Minimum required plugins -plugins=( - "performance-analyzer" - "opensearch-security" -) - # plugins=( -# "alerting" # "opensearch-alerting" -# "opensearch-job-scheduler" -# "opensearch-anomaly-detection" # Requires "opensearch-job-scheduler" -# "asynchronous-search" # "opensearch-asynchronous-search" -# "opensearch-cross-cluster-replication" -# "geospatial" # "opensearch-geospatial" -# "opensearch-index-management" -# "opensearch-knn" -# "opensearch-ml-plugin" # "opensearch-ml" -# "neural-search" # "opensearch-neural-search" -# "opensearch-notifications-core" -# "notifications" # "opensearch-notifications". Requires "opensearch-notifications-core" -# "opensearch-observability" -# "performance-analyzer" # "opensearch-performance-analyzer" -# "opensearch-reports-scheduler" +# "performance-analyzer" # "opensearch-security" -# "opensearch-security-analytics" -# "opensearch-sql-plugin" # "opensearch-sql" # ) +plugins=( + "alerting" # "opensearch-alerting" + "opensearch-job-scheduler" + "opensearch-anomaly-detection" # Requires "opensearch-job-scheduler" + "asynchronous-search" # "opensearch-asynchronous-search" + "opensearch-cross-cluster-replication" + "geospatial" # "opensearch-geospatial" + "opensearch-index-management" + "opensearch-knn" + "opensearch-ml-plugin" # "opensearch-ml" + "neural-search" # "opensearch-neural-search" + "opensearch-notifications-core" + "notifications" # "opensearch-notifications". Requires "opensearch-notifications-core" + "opensearch-observability" + "performance-analyzer" # "opensearch-performance-analyzer" + "opensearch-reports-scheduler" + "opensearch-security" + "opensearch-security-analytics" + "opensearch-sql-plugin" # "opensearch-sql" +) + # ==== # Usage # ==== From a70b048919e14ea83e15e56db53c4beaf7098ce2 Mon Sep 17 00:00:00 2001 From: Fede Tux Date: Mon, 15 Jan 2024 15:03:03 -0300 Subject: [PATCH 05/14] Add Dockerfile and docker-compose for the package assembly stage --- docker/assemble.yml | 28 +++++++++++++++++++ docker/dev.sh | 2 +- docker/dev.yml | 23 ++++++++++++--- docker/images/wi-assemble.Dockerfile | 20 +++++++++++++ ...{wi-dev.Dockerfile => wi-build.Dockerfile} | 0 5 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 docker/assemble.yml create mode 100644 docker/images/wi-assemble.Dockerfile rename docker/images/{wi-dev.Dockerfile => wi-build.Dockerfile} (100%) diff --git a/docker/assemble.yml b/docker/assemble.yml new file mode 100644 index 0000000000000..848f48e6da2d5 --- /dev/null +++ b/docker/assemble.yml @@ -0,0 +1,28 @@ +version: "3.9" + +services: + + wi-assemble: + image: wi-assemble:${VERSION} + container_name: wi-assemble-${VERSION} + build: + context: ./.. + dockerfile: ${REPO_PATH}/docker/images/wi-assemble.Dockerfile +# ports: +# # OpenSearch REST API +# - 9200:9200 +# # Cross-cluster search +# # - 9250:9250 +# # Node communication and transport +# # - 9300:9300 +# # Performance Analyzer +# # - 9600:9600 +# expose: +# - 9200 + volumes: + - ${REPO_PATH}/scripts:/home/wazuh-indexer/scripts + - ${REPO_PATH}/artifacts:/home/wazuh-indexer/artifacts + - ${REPO_PATH}/distribution/packages/src:/home/wazuh-indexer/distribution/packages/src + entrypoint: ['tail', '-f', '/dev/null'] + user: "1000:1000" + working_dir: /home/wazuh-indexer diff --git a/docker/dev.sh b/docker/dev.sh index 991ed42de29e7..d1f76e9e69abd 100755 --- a/docker/dev.sh +++ b/docker/dev.sh @@ -55,4 +55,4 @@ function main() { esac } -main "$@" \ No newline at end of file +main "$@" diff --git a/docker/dev.yml b/docker/dev.yml index 7e3b6202c2ee9..094bd353ace6c 100644 --- a/docker/dev.yml +++ b/docker/dev.yml @@ -2,12 +2,12 @@ version: "3.9" services: - wazuh-indexer: - image: wazuh-indexer-dev:${VERSION} - container_name: wazuh-indexer-dev-${VERSION} + wi-build: + image: wi-build:${VERSION} + container_name: wi-build-${VERSION} build: context: ./.. - dockerfile: ${REPO_PATH}/docker/images/wi-dev.Dockerfile + dockerfile: ${REPO_PATH}/docker/images/wi-build.Dockerfile ports: # OpenSearch REST API - 9200:9200 @@ -24,3 +24,18 @@ services: entrypoint: ['tail', '-f', '/dev/null'] user: "1000:1000" working_dir: /home/wazuh-indexer/app + + + wi-assemble: + image: wi-assemble:${VERSION} + container_name: wi-assemble-${VERSION} + build: + context: ./.. + dockerfile: ${REPO_PATH}/docker/images/wi-assemble.Dockerfile + volumes: + - ${REPO_PATH}/scripts:/home/wazuh-indexer/scripts + - ${REPO_PATH}/artifacts:/home/wazuh-indexer/artifacts + - ${REPO_PATH}/distribution/packages/src:/home/wazuh-indexer/distribution/packages/src + entrypoint: ['tail', '-f', '/dev/null'] + user: "1000:1000" + working_dir: /home/wazuh-indexer diff --git a/docker/images/wi-assemble.Dockerfile b/docker/images/wi-assemble.Dockerfile new file mode 100644 index 0000000000000..b06bed6eaf41a --- /dev/null +++ b/docker/images/wi-assemble.Dockerfile @@ -0,0 +1,20 @@ +FROM ubuntu:jammy +RUN mkdir /home/wazuh-indexer && \ + apt-get update -y && \ + apt-get install curl gnupg2 -y && \ + curl -o- https://www.aptly.info/pubkey.txt | apt-key add - && \ + echo "deb http://repo.aptly.info/ squeeze main" | tee -a /etc/apt/sources.list.d/aptly.list && \ + apt-get update -y && \ + apt-get upgrade -y && \ + apt-get install -y aptly build-essential cpio debhelper-compat debmake freeglut3 libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-dev libcairo2 libcairo2-dev libcups2 libdrm2 libgbm-dev libgconf-2-4 libnspr4 libnspr4-dev libnss3 libpangocairo-1.0-0 libxcomposite-dev libxdamage1 libxfixes-dev libxfixes3 libxi6 libxkbcommon-x11-0 libxrandr2 libxrender1 libxtst6 rpm rpm2cpio && \ + apt-get clean -y && \ + dpkg -r lintian && \ + addgroup --gid 1000 wazuh-indexer && \ + adduser --uid 1000 --ingroup wazuh-indexer --disabled-password --home /home/wazuh-indexer wazuh-indexer && \ + chmod 0775 /home/wazuh-indexer && \ + chown -R 1000:1000 /home/wazuh-indexer +USER wazuh-indexer +WORKDIR /home/wazuh-indexer + + + diff --git a/docker/images/wi-dev.Dockerfile b/docker/images/wi-build.Dockerfile similarity index 100% rename from docker/images/wi-dev.Dockerfile rename to docker/images/wi-build.Dockerfile From 1e61cb04e1f91603dd43a3acbf651943d04ce3ae Mon Sep 17 00:00:00 2001 From: Fede Tux Date: Mon, 15 Jan 2024 15:42:04 -0300 Subject: [PATCH 06/14] Assemble packages with minimal plugin set when "test" variable is set to "true" --- scripts/assemble.sh | 59 +++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/scripts/assemble.sh b/scripts/assemble.sh index f88b075257a4d..1fc2f57696b59 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -9,32 +9,39 @@ set -ex -# Minimum required plugins -# plugins=( -# "performance-analyzer" -# "opensearch-security" -# ) - -plugins=( - "alerting" # "opensearch-alerting" - "opensearch-job-scheduler" - "opensearch-anomaly-detection" # Requires "opensearch-job-scheduler" - "asynchronous-search" # "opensearch-asynchronous-search" - "opensearch-cross-cluster-replication" - "geospatial" # "opensearch-geospatial" - "opensearch-index-management" - "opensearch-knn" - "opensearch-ml-plugin" # "opensearch-ml" - "neural-search" # "opensearch-neural-search" - "opensearch-notifications-core" - "notifications" # "opensearch-notifications". Requires "opensearch-notifications-core" - "opensearch-observability" - "performance-analyzer" # "opensearch-performance-analyzer" - "opensearch-reports-scheduler" - "opensearch-security" - "opensearch-security-analytics" - "opensearch-sql-plugin" # "opensearch-sql" -) +### The $test variable determines whether we include a minimalistic +### or the full set of OpenSearch plugins + +test=${test:-false} + +if ( $test ) +then + plugins=( + "performance-analyzer" + "opensearch-security" + ) +else + plugins=( + "alerting" # "opensearch-alerting" + "opensearch-job-scheduler" + "opensearch-anomaly-detection" # Requires "opensearch-job-scheduler" + "asynchronous-search" # "opensearch-asynchronous-search" + "opensearch-cross-cluster-replication" + "geospatial" # "opensearch-geospatial" + "opensearch-index-management" + "opensearch-knn" + "opensearch-ml-plugin" # "opensearch-ml" + "neural-search" # "opensearch-neural-search" + "opensearch-notifications-core" + "notifications" # "opensearch-notifications". Requires "opensearch-notifications-core" + "opensearch-observability" + "performance-analyzer" # "opensearch-performance-analyzer" + "opensearch-reports-scheduler" + "opensearch-security" + "opensearch-security-analytics" + "opensearch-sql-plugin" # "opensearch-sql" + ) +fi # ==== # Usage From a217c3439bf98aaf8651383cf2f0c4ec9b576bbd Mon Sep 17 00:00:00 2001 From: Fede Tux Date: Mon, 15 Jan 2024 15:55:02 -0300 Subject: [PATCH 07/14] Update README with assemble.sh docker image --- docker/README.md | 2 +- scripts/README.md | 42 +++++------------------------------------- 2 files changed, 6 insertions(+), 38 deletions(-) diff --git a/docker/README.md b/docker/README.md index 4b41a42e0e9c9..959c804459722 100644 --- a/docker/README.md +++ b/docker/README.md @@ -61,7 +61,7 @@ Example: Usage: ./dev.sh {up|down|stop} [security] ``` -Once the `wazuh-indexer` container is up, attach a shell to it and run `./gradlew run` +Once the `wi-build` container is up, attach a shell to it and run `./gradlew run` to start the application. diff --git a/scripts/README.md b/scripts/README.md index 1f628c5fbbccd..4395e254b087e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -32,7 +32,7 @@ act -j build -W .github/workflows/build.yml --artifact-server-path ./artifacts Within the [Docker environment](../docker): ```console -bash scripts/build.sh -v 2.11.1 -s false -p linux -a {x64|arm64} -d {rpm|deb|tar} +docker exec -it wi-build-$( Date: Wed, 17 Jan 2024 15:00:31 -0300 Subject: [PATCH 08/14] Fixing env variable naming convention and removing wget dependency --- scripts/assemble.sh | 6 +++--- scripts/provision.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/assemble.sh b/scripts/assemble.sh index 1fc2f57696b59..90972bfc7a892 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -9,12 +9,12 @@ set -ex -### The $test variable determines whether we include a minimalistic +### The $TEST variable determines whether we include a minimalistic ### or the full set of OpenSearch plugins -test=${test:-false} +TEST=${TEST:-false} -if ( $test ) +if ( $TEST ) then plugins=( "performance-analyzer" diff --git a/scripts/provision.sh b/scripts/provision.sh index 2984deebeada9..585540fe02697 100644 --- a/scripts/provision.sh +++ b/scripts/provision.sh @@ -3,7 +3,7 @@ # Provision script for assembly of DEB packages # Install necessary packages -apt-get update -y && apt-get upgrade -y && apt-get install -y curl wget build-essential && +apt-get update -y && apt-get upgrade -y && apt-get install -y curl build-essential && apt-get install -y debmake debhelper-compat && apt-get install -y libxrender1 libxtst6 libasound2 libxi6 libgconf-2-4 && apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libatspi2.0-dev libxcomposite-dev libxdamage1 libxfixes3 libxfixes-dev libxrandr2 libgbm-dev libxkbcommon-x11-0 libpangocairo-1.0-0 libcairo2 libcairo2-dev libnss3 libnspr4 libnspr4-dev freeglut3 && From c8c975ccffdd471fa0464179d946cdb050c98493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Thu, 18 Jan 2024 20:21:45 +0100 Subject: [PATCH 09/14] Improve Docker environments Adds environments to build packages --- docker/README.md | 18 +++-- docker/assemble.yml | 28 -------- docker/ci/ci.sh | 61 +++++++++++++++++ docker/{dev.yml => ci/ci.yml} | 26 +++---- docker/{ => ci}/images/.dockerignore | 0 .../images/Dockerfile} | 0 docker/{ => dev}/dev.sh | 6 +- docker/dev/dev.yml | 20 ++++++ docker/dev/images/.dockerignore | 68 +++++++++++++++++++ .../images/Dockerfile} | 10 +-- 10 files changed, 180 insertions(+), 57 deletions(-) delete mode 100644 docker/assemble.yml create mode 100755 docker/ci/ci.sh rename docker/{dev.yml => ci/ci.yml} (52%) rename docker/{ => ci}/images/.dockerignore (100%) rename docker/{images/wi-assemble.Dockerfile => ci/images/Dockerfile} (100%) rename docker/{ => dev}/dev.sh (91%) create mode 100644 docker/dev/dev.yml create mode 100644 docker/dev/images/.dockerignore rename docker/{images/wi-build.Dockerfile => dev/images/Dockerfile} (67%) diff --git a/docker/README.md b/docker/README.md index 959c804459722..e2a30aacd478f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -51,19 +51,29 @@ Docker Desktop will change to its context automatically at start, so be sure that any existing Docker container using the default context is **stopped** before starting Docker Desktop and any of the environments in this folder. -## Starting up the environments +## Development environments -Use the sh script to up the environment. +Use the `dev/dev.sh` script to start development environment. Example: ```bash -Usage: ./dev.sh {up|down|stop} [security] +Usage: ./dev.sh {up|down|stop} ``` -Once the `wi-build` container is up, attach a shell to it and run `./gradlew run` +Once the `wi-dev:x.y.z` container is up, attach a shell to it and run `./gradlew run` to start the application. +## Containers to generate packages + +Include the `ci/ci.sh` script to start provisioned containers to generate packages. + +```bash +Usage: ./ci.sh {up|down|stop} [ci] +``` + +Refer to [scripts/README.md](../scripts/README.md) for details about how to build packages. + [docker-desktop]: https://docs.docker.com/get-docker [docker-variant]: https://docs.docker.com/desktop/install/linux-install/#differences-between-docker-desktop-for-linux-and-docker-engine diff --git a/docker/assemble.yml b/docker/assemble.yml deleted file mode 100644 index 848f48e6da2d5..0000000000000 --- a/docker/assemble.yml +++ /dev/null @@ -1,28 +0,0 @@ -version: "3.9" - -services: - - wi-assemble: - image: wi-assemble:${VERSION} - container_name: wi-assemble-${VERSION} - build: - context: ./.. - dockerfile: ${REPO_PATH}/docker/images/wi-assemble.Dockerfile -# ports: -# # OpenSearch REST API -# - 9200:9200 -# # Cross-cluster search -# # - 9250:9250 -# # Node communication and transport -# # - 9300:9300 -# # Performance Analyzer -# # - 9600:9600 -# expose: -# - 9200 - volumes: - - ${REPO_PATH}/scripts:/home/wazuh-indexer/scripts - - ${REPO_PATH}/artifacts:/home/wazuh-indexer/artifacts - - ${REPO_PATH}/distribution/packages/src:/home/wazuh-indexer/distribution/packages/src - entrypoint: ['tail', '-f', '/dev/null'] - user: "1000:1000" - working_dir: /home/wazuh-indexer diff --git a/docker/ci/ci.sh b/docker/ci/ci.sh new file mode 100755 index 0000000000000..e68662dacd85c --- /dev/null +++ b/docker/ci/ci.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Start container with required tools to build packages +# Requires Docker +# Script usage: bash ./ci.sh + +set -e + +# ==== +# Checks that the script is run from the intended location +# ==== +function check_project_root_folder () { + current=$(basename "$(pwd)") + + if [[ "$0" != "./ci.sh" && "$0" != "ci.sh" ]]; then + echo "Run the script from its location" + usage + exit 1 + fi + # Change working directory to the root of the repository + cd ../.. +} + +# ==== +# Displays usage +# ==== +function usage() { + echo "Usage: ./ci.sh {up|down|stop}" +} + +# ==== +# Main function +# ==== +function main() { + check_project_root_folder "$@" + compose_file="docker/${current}/ci.yml" + echo "$compose_file" + compose_cmd="docker compose -f $compose_file" + REPO_PATH=$(pwd) + VERSION=$(cat VERSION) + export REPO_PATH + export VERSION + + case $1 in + up) + $compose_cmd up -d + ;; + down) + $compose_cmd down + ;; + stop) + $compose_cmd stop + ;; + *) + usage + exit 1 + ;; + esac +} + +main "$@" diff --git a/docker/dev.yml b/docker/ci/ci.yml similarity index 52% rename from docker/dev.yml rename to docker/ci/ci.yml index 094bd353ace6c..dec0eca32eb5a 100644 --- a/docker/dev.yml +++ b/docker/ci/ci.yml @@ -2,36 +2,26 @@ version: "3.9" services: + # Essentially wi-dev, but doesn't expose port 9200 wi-build: image: wi-build:${VERSION} - container_name: wi-build-${VERSION} + container_name: wi-build_${VERSION} build: - context: ./.. - dockerfile: ${REPO_PATH}/docker/images/wi-build.Dockerfile - ports: - # OpenSearch REST API - - 9200:9200 - # Cross-cluster search - # - 9250:9250 - # Node communication and transport - # - 9300:9300 - # Performance Analyzer - # - 9600:9600 - expose: - - 9200 + context: ./../.. + dockerfile: ${REPO_PATH}/docker/dev/images/Dockerfile volumes: - ${REPO_PATH}:/home/wazuh-indexer/app entrypoint: ['tail', '-f', '/dev/null'] user: "1000:1000" - working_dir: /home/wazuh-indexer/app + working_dir: /home/wazuh-indexer wi-assemble: image: wi-assemble:${VERSION} - container_name: wi-assemble-${VERSION} + container_name: wi-assemble_${VERSION} build: - context: ./.. - dockerfile: ${REPO_PATH}/docker/images/wi-assemble.Dockerfile + context: ./../.. + dockerfile: ${REPO_PATH}/docker/ci/images/Dockerfile volumes: - ${REPO_PATH}/scripts:/home/wazuh-indexer/scripts - ${REPO_PATH}/artifacts:/home/wazuh-indexer/artifacts diff --git a/docker/images/.dockerignore b/docker/ci/images/.dockerignore similarity index 100% rename from docker/images/.dockerignore rename to docker/ci/images/.dockerignore diff --git a/docker/images/wi-assemble.Dockerfile b/docker/ci/images/Dockerfile similarity index 100% rename from docker/images/wi-assemble.Dockerfile rename to docker/ci/images/Dockerfile diff --git a/docker/dev.sh b/docker/dev/dev.sh similarity index 91% rename from docker/dev.sh rename to docker/dev/dev.sh index d1f76e9e69abd..3f4fc1b5eea0f 100755 --- a/docker/dev.sh +++ b/docker/dev/dev.sh @@ -10,13 +10,15 @@ set -e # Checks that the script is run from the intended location # ==== function check_project_root_folder () { + current=$(basename "$(pwd)") + if [[ "$0" != "./dev.sh" && "$0" != "dev.sh" ]]; then echo "Run the script from its location" usage exit 1 fi # Change working directory to the root of the repository - cd .. + cd ../.. } # ==== @@ -31,7 +33,7 @@ function usage() { # ==== function main() { check_project_root_folder "$@" - compose_file=docker/dev.yml + compose_file="docker/${current}/dev.yml" compose_cmd="docker compose -f $compose_file" REPO_PATH=$(pwd) VERSION=$(cat VERSION) diff --git a/docker/dev/dev.yml b/docker/dev/dev.yml new file mode 100644 index 0000000000000..1d0775b45f1bb --- /dev/null +++ b/docker/dev/dev.yml @@ -0,0 +1,20 @@ +version: "3.9" + +services: + + wi-dev: + image: wi-dev:${VERSION} + container_name: wi-dev_${VERSION} + build: + context: ./../.. + dockerfile: ${REPO_PATH}/docker/dev/images/Dockerfile + ports: + # OpenSearch REST API + - 9200:9200 + expose: + - 9200 + volumes: + - ${REPO_PATH}:/home/wazuh-indexer/app + entrypoint: ['tail', '-f', '/dev/null'] + user: "1000:1000" + working_dir: /home/wazuh-indexer \ No newline at end of file diff --git a/docker/dev/images/.dockerignore b/docker/dev/images/.dockerignore new file mode 100644 index 0000000000000..96d12ad527ea5 --- /dev/null +++ b/docker/dev/images/.dockerignore @@ -0,0 +1,68 @@ +artifacts/ +.git/ + +# intellij files +.idea/ +*.iml +*.ipr +*.iws +build-idea/ +out/ + +# include shared intellij config +!.idea/inspectionProfiles/Project_Default.xml +!.idea/runConfigurations/Debug_OpenSearch.xml +!.idea/vcs.xml + +# These files are generated in the main tree by annotation processors +benchmarks/src/main/generated/* +benchmarks/bin/* +benchmarks/build-eclipse-default/* +server/bin/* +server/build-eclipse-default/* +test/framework/build-eclipse-default/* + +# eclipse files +.project +.classpath +.settings +build-eclipse/ + +# netbeans files +nb-configuration.xml +nbactions.xml + +# gradle stuff +.gradle/ +build/ + +# vscode stuff +.vscode/ + +# testing stuff +**/.local* +.vagrant/ +/logs/ + +# osx stuff +.DS_Store + +# default folders in which the create_bwc_index.py expects to find old es versions in +/backwards +/dev-tools/backwards + +# needed in case docs build is run...maybe we can configure doc build to generate files under build? +html_docs + +# random old stuff that we should look at the necessity of... +/tmp/ +eclipse-build + +# projects using testfixtures +testfixtures_shared/ + +# These are generated from .ci/jobs.t +.ci/jobs/ + +# build files generated +doc-tools/missing-doclet/bin/ \ No newline at end of file diff --git a/docker/images/wi-build.Dockerfile b/docker/dev/images/Dockerfile similarity index 67% rename from docker/images/wi-build.Dockerfile rename to docker/dev/images/Dockerfile index 215b956d14a82..f49aa4b1aba6e 100644 --- a/docker/images/wi-build.Dockerfile +++ b/docker/dev/images/Dockerfile @@ -1,7 +1,7 @@ FROM gradle:jdk17-alpine AS builder USER gradle -WORKDIR /home/wazuh-indexer/app -COPY --chown=gradle:gradle . /home/wazuh-indexer/app +WORKDIR /home/wazuh-indexer +COPY --chown=gradle:gradle . /home/wazuh-indexer RUN gradle clean @@ -12,7 +12,7 @@ RUN apk add git && \ chmod 0775 /home/wazuh-indexer && \ chown -R 1000:0 /home/wazuh-indexer USER wazuh-indexer -COPY --from=builder --chown=1000:0 /home/wazuh-indexer/app /home/wazuh-indexer/app -WORKDIR /home/wazuh-indexer/app -RUN git config --global --add safe.directory /home/wazuh-indexer/app +COPY --from=builder --chown=1000:0 /home/wazuh-indexer /home/wazuh-indexer +WORKDIR /home/wazuh-indexer +RUN git config --global --add safe.directory /home/wazuh-indexer EXPOSE 9200 9300 From 2d81394869e2a2727763dc0cd0f6daea7ec44564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Thu, 18 Jan 2024 20:25:44 +0100 Subject: [PATCH 10/14] Fix small typos --- docker/README.md | 4 ++-- scripts/README.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/README.md b/docker/README.md index e2a30aacd478f..28bca4338c998 100644 --- a/docker/README.md +++ b/docker/README.md @@ -53,7 +53,7 @@ before starting Docker Desktop and any of the environments in this folder. ## Development environments -Use the `dev/dev.sh` script to start development environment. +Use the `dev/dev.sh` script to start a development environment. Example: @@ -66,7 +66,7 @@ to start the application. ## Containers to generate packages -Include the `ci/ci.sh` script to start provisioned containers to generate packages. +Use the `ci/ci.sh` script to start provisioned containers to generate packages. ```bash Usage: ./ci.sh {up|down|stop} [ci] diff --git a/scripts/README.md b/scripts/README.md index 4395e254b087e..d60ad9643cc77 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -29,7 +29,7 @@ act -j build -W .github/workflows/build.yml --artifact-server-path ./artifacts #### Running in Docker -Within the [Docker environment](../docker): +Using the [Docker environment](../docker): ```console docker exec -it wi-build-$( Date: Fri, 19 Jan 2024 11:07:59 +0100 Subject: [PATCH 11/14] More fixes --- docker/ci/ci.sh | 1 - docker/ci/ci.yml | 2 +- docker/dev/dev.yml | 2 +- scripts/README.md | 6 +++--- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docker/ci/ci.sh b/docker/ci/ci.sh index e68662dacd85c..cf0bfc338fb66 100755 --- a/docker/ci/ci.sh +++ b/docker/ci/ci.sh @@ -34,7 +34,6 @@ function usage() { function main() { check_project_root_folder "$@" compose_file="docker/${current}/ci.yml" - echo "$compose_file" compose_cmd="docker compose -f $compose_file" REPO_PATH=$(pwd) VERSION=$(cat VERSION) diff --git a/docker/ci/ci.yml b/docker/ci/ci.yml index dec0eca32eb5a..604c4b3e912a3 100644 --- a/docker/ci/ci.yml +++ b/docker/ci/ci.yml @@ -10,7 +10,7 @@ services: context: ./../.. dockerfile: ${REPO_PATH}/docker/dev/images/Dockerfile volumes: - - ${REPO_PATH}:/home/wazuh-indexer/app + - ${REPO_PATH}:/home/wazuh-indexer entrypoint: ['tail', '-f', '/dev/null'] user: "1000:1000" working_dir: /home/wazuh-indexer diff --git a/docker/dev/dev.yml b/docker/dev/dev.yml index 1d0775b45f1bb..b9f9202d94127 100644 --- a/docker/dev/dev.yml +++ b/docker/dev/dev.yml @@ -14,7 +14,7 @@ services: expose: - 9200 volumes: - - ${REPO_PATH}:/home/wazuh-indexer/app + - ${REPO_PATH}:/home/wazuh-indexer entrypoint: ['tail', '-f', '/dev/null'] user: "1000:1000" working_dir: /home/wazuh-indexer \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md index d60ad9643cc77..20c7392f5aed7 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -32,7 +32,7 @@ act -j build -W .github/workflows/build.yml --artifact-server-path ./artifacts Using the [Docker environment](../docker): ```console -docker exec -it wi-build-$( Date: Fri, 19 Jan 2024 11:59:50 +0100 Subject: [PATCH 12/14] Add documentation --- docker/ci/ci.sh | 2 ++ scripts/README.md | 3 +++ 2 files changed, 5 insertions(+) diff --git a/docker/ci/ci.sh b/docker/ci/ci.sh index cf0bfc338fb66..66f39bf58a43a 100755 --- a/docker/ci/ci.sh +++ b/docker/ci/ci.sh @@ -42,6 +42,8 @@ function main() { case $1 in up) + # Main folder created here to grant access to both containers + mkdir artifacts $compose_cmd up -d ;; down) diff --git a/scripts/README.md b/scripts/README.md index 20c7392f5aed7..aa5fcdda8e0a8 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -39,6 +39,9 @@ The generated package is sent to `artifacts/` ## Assemble +**Note:** set the environment variable `TEST=true` to assemble a package with the required plugins only, +speeding up the assembly process. + From e7895fc8d38697d212c575fff07164974b1dd05b Mon Sep 17 00:00:00 2001 From: Fede Tux Date: Fri, 19 Jan 2024 08:05:54 -0300 Subject: [PATCH 13/14] Adding -p flag to mkdir so it doesnt fail when the folder is already present --- docker/ci/ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/ci/ci.sh b/docker/ci/ci.sh index 66f39bf58a43a..8884fab9c319d 100755 --- a/docker/ci/ci.sh +++ b/docker/ci/ci.sh @@ -43,7 +43,7 @@ function main() { case $1 in up) # Main folder created here to grant access to both containers - mkdir artifacts + mkdir -p artifacts $compose_cmd up -d ;; down) From 12afc25fad5eca421059b3a35ea25aed49a08751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Fri, 19 Jan 2024 12:06:02 +0100 Subject: [PATCH 14/14] Format files --- docker/README.md | 3 +-- docker/ci/ci.sh | 4 ++-- docker/ci/ci.yml | 6 ++---- docker/ci/images/Dockerfile | 3 --- docker/dev/dev.sh | 4 ++-- docker/dev/dev.yml | 5 ++--- scripts/README.md | 2 +- 7 files changed, 10 insertions(+), 17 deletions(-) diff --git a/docker/README.md b/docker/README.md index 28bca4338c998..e012e8c42bed1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -53,7 +53,7 @@ before starting Docker Desktop and any of the environments in this folder. ## Development environments -Use the `dev/dev.sh` script to start a development environment. +Use the `dev/dev.sh` script to start a development environment. Example: @@ -74,7 +74,6 @@ Usage: ./ci.sh {up|down|stop} [ci] Refer to [scripts/README.md](../scripts/README.md) for details about how to build packages. - [docker-desktop]: https://docs.docker.com/get-docker [docker-variant]: https://docs.docker.com/desktop/install/linux-install/#differences-between-docker-desktop-for-linux-and-docker-engine [docker-context]: https://docs.docker.com/desktop/install/linux-install/#context diff --git a/docker/ci/ci.sh b/docker/ci/ci.sh index 8884fab9c319d..13e84c0881b3f 100755 --- a/docker/ci/ci.sh +++ b/docker/ci/ci.sh @@ -9,11 +9,11 @@ set -e # ==== # Checks that the script is run from the intended location # ==== -function check_project_root_folder () { +function check_project_root_folder() { current=$(basename "$(pwd)") if [[ "$0" != "./ci.sh" && "$0" != "ci.sh" ]]; then - echo "Run the script from its location" + echo "Run the script from its location" usage exit 1 fi diff --git a/docker/ci/ci.yml b/docker/ci/ci.yml index 604c4b3e912a3..a18ca915302f0 100644 --- a/docker/ci/ci.yml +++ b/docker/ci/ci.yml @@ -1,7 +1,6 @@ version: "3.9" services: - # Essentially wi-dev, but doesn't expose port 9200 wi-build: image: wi-build:${VERSION} @@ -11,11 +10,10 @@ services: dockerfile: ${REPO_PATH}/docker/dev/images/Dockerfile volumes: - ${REPO_PATH}:/home/wazuh-indexer - entrypoint: ['tail', '-f', '/dev/null'] + entrypoint: ["tail", "-f", "/dev/null"] user: "1000:1000" working_dir: /home/wazuh-indexer - wi-assemble: image: wi-assemble:${VERSION} container_name: wi-assemble_${VERSION} @@ -26,6 +24,6 @@ services: - ${REPO_PATH}/scripts:/home/wazuh-indexer/scripts - ${REPO_PATH}/artifacts:/home/wazuh-indexer/artifacts - ${REPO_PATH}/distribution/packages/src:/home/wazuh-indexer/distribution/packages/src - entrypoint: ['tail', '-f', '/dev/null'] + entrypoint: ["tail", "-f", "/dev/null"] user: "1000:1000" working_dir: /home/wazuh-indexer diff --git a/docker/ci/images/Dockerfile b/docker/ci/images/Dockerfile index b06bed6eaf41a..c84791adcb3e5 100644 --- a/docker/ci/images/Dockerfile +++ b/docker/ci/images/Dockerfile @@ -15,6 +15,3 @@ RUN mkdir /home/wazuh-indexer && \ chown -R 1000:1000 /home/wazuh-indexer USER wazuh-indexer WORKDIR /home/wazuh-indexer - - - diff --git a/docker/dev/dev.sh b/docker/dev/dev.sh index 3f4fc1b5eea0f..f20b7359c9206 100755 --- a/docker/dev/dev.sh +++ b/docker/dev/dev.sh @@ -9,11 +9,11 @@ set -e # ==== # Checks that the script is run from the intended location # ==== -function check_project_root_folder () { +function check_project_root_folder() { current=$(basename "$(pwd)") if [[ "$0" != "./dev.sh" && "$0" != "dev.sh" ]]; then - echo "Run the script from its location" + echo "Run the script from its location" usage exit 1 fi diff --git a/docker/dev/dev.yml b/docker/dev/dev.yml index b9f9202d94127..7176b044df5ba 100644 --- a/docker/dev/dev.yml +++ b/docker/dev/dev.yml @@ -1,7 +1,6 @@ version: "3.9" services: - wi-dev: image: wi-dev:${VERSION} container_name: wi-dev_${VERSION} @@ -15,6 +14,6 @@ services: - 9200 volumes: - ${REPO_PATH}:/home/wazuh-indexer - entrypoint: ['tail', '-f', '/dev/null'] + entrypoint: ["tail", "-f", "/dev/null"] user: "1000:1000" - working_dir: /home/wazuh-indexer \ No newline at end of file + working_dir: /home/wazuh-indexer diff --git a/scripts/README.md b/scripts/README.md index aa5fcdda8e0a8..c83f20f3a3aac 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -39,7 +39,7 @@ The generated package is sent to `artifacts/` ## Assemble -**Note:** set the environment variable `TEST=true` to assemble a package with the required plugins only, +**Note:** set the environment variable `TEST=true` to assemble a package with the required plugins only, speeding up the assembly process.