diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index b6b730fc3de8b..0c0ede8c3a076 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -78,11 +78,15 @@ if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then fi if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then - DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" - export DOCKER_REGISTRY_USERNAME + if which docker > /dev/null 2>&1; then + DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" + export DOCKER_REGISTRY_USERNAME - DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" - export DOCKER_REGISTRY_PASSWORD + DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" + export DOCKER_REGISTRY_PASSWORD + + docker login --username "$DOCKER_REGISTRY_USERNAME" --password "$DOCKER_REGISTRY_PASSWORD" docker.elastic.co + fi fi if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 35dc42e013dc0..120d0c76374c7 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -29,7 +29,8 @@ steps: image: family/elasticsearch-{{matrix.image}} diskSizeGb: 350 machineType: n1-standard-8 - env: {} + env: + USE_PROD_DOCKER_CREDENTIALS: "true" - group: packaging-tests-upgrade steps: - label: "{{matrix.image}} / 7.0.1 / packaging-tests-upgrade" diff --git a/.ci/scripts/packaging-test.sh b/.ci/scripts/packaging-test.sh index 1626255c30b4f..8dc2e725a2f2b 100755 --- a/.ci/scripts/packaging-test.sh +++ b/.ci/scripts/packaging-test.sh @@ -77,5 +77,6 @@ sudo -E env \ --unset=ES_JAVA_HOME \ --unset=JAVA_HOME \ SYSTEM_JAVA_HOME=`readlink -f -n $BUILD_JAVA_HOME` \ + DOCKER_CONFIG="${HOME}/.docker" \ ./gradlew -g $HOME/.gradle --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ --continue $@ diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java index 8971f27838578..9b28401994ee2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java @@ -30,6 +30,7 @@ import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import org.gradle.process.ExecOperations; +import org.gradle.process.ExecSpec; import org.gradle.workers.WorkAction; import org.gradle.workers.WorkParameters; import org.gradle.workers.WorkerExecutor; @@ -166,6 +167,7 @@ private void pullBaseImage(String baseImage) { for (int attempt = 1; attempt <= maxAttempts; attempt++) { try { LoggedExec.exec(execOperations, spec -> { + maybeConfigureDockerConfig(spec); spec.executable("docker"); spec.args("pull"); spec.args(baseImage); @@ -181,6 +183,13 @@ private void pullBaseImage(String baseImage) { throw new GradleException("Failed to pull Docker base image [" + baseImage + "], all attempts failed"); } + private void maybeConfigureDockerConfig(ExecSpec spec) { + String dockerConfig = System.getenv("DOCKER_CONFIG"); + if (dockerConfig != null) { + spec.environment("DOCKER_CONFIG", dockerConfig); + } + } + @Override public void execute() { final Parameters parameters = getParameters(); @@ -193,6 +202,8 @@ public void execute() { final boolean isCrossPlatform = isCrossPlatform(); LoggedExec.exec(execOperations, spec -> { + maybeConfigureDockerConfig(spec); + spec.executable("docker"); if (isCrossPlatform) { diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 47f79749cbefa..fd2516f2fdc9a 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -163,9 +163,16 @@ RUN <%= retry.loop(package_manager, " ${package_manager} update && \n" + " ${package_manager} upgrade && \n" + " ${package_manager} add --no-cache \n" + - " bash ca-certificates curl libsystemd netcat-openbsd p11-kit p11-kit-trust shadow tini unzip zip zstd && \n" + + " bash java-cacerts curl libstdc++ libsystemd netcat-openbsd p11-kit p11-kit-trust posix-libc-utils shadow tini unzip zip zstd && \n" + " rm -rf /var/cache/apk/* " ) %> + +# Set Bash as the default shell for future commands +SHELL ["/bin/bash", "-c"] + +# Optionally set Bash as the default shell in the container at runtime +CMD ["/bin/bash"] + <% } else if (docker_base == "default" || docker_base == "cloud") { %> # Change default shell to bash, then install required packages with retries. @@ -224,7 +231,7 @@ COPY --from=builder --chown=0:0 /opt /opt <% } %> ENV PATH /usr/share/elasticsearch/bin:\$PATH - +ENV SHELL /bin/bash COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh # 1. Sync the user and group permissions of /etc/passwd @@ -249,6 +256,8 @@ RUN chmod g=u /etc/passwd && \\ # stays up-to-date with changes to Ubuntu's store) COPY bin/docker-openjdk /etc/ca-certificates/update.d/docker-openjdk RUN /etc/ca-certificates/update.d/docker-openjdk +<% } else if (docker_base == 'wolfi') { %> +RUN ln -sf /etc/ssl/certs/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts <% } else { %> RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts <% } %> diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index a9402c324f7fc..f588b78c78cc8 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -386,6 +386,9 @@ public void test040JavaUsesTheOsProvidedKeystore() { if (distribution.packaging == Packaging.DOCKER_UBI || distribution.packaging == Packaging.DOCKER_IRON_BANK) { // In these images, the `cacerts` file ought to be a symlink here assertThat(path, equalTo("/etc/pki/ca-trust/extracted/java/cacerts")); + } else if (distribution.packaging == Packaging.DOCKER_WOLFI) { + // In these images, the `cacerts` file ought to be a symlink here + assertThat(path, equalTo("/etc/ssl/certs/java/cacerts")); } else { // Whereas on other images, it's a real file so the real path is the same assertThat(path, equalTo("/usr/share/elasticsearch/jdk/lib/security/cacerts")); diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java index 5b86796aa80ca..a988a446f561f 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java @@ -436,7 +436,10 @@ private void verifyKeystorePermissions() { switch (distribution.packaging) { case TAR, ZIP -> assertThat(keystore, file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); case DEB, RPM -> assertThat(keystore, file(File, "root", "elasticsearch", p660)); - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> assertThat(keystore, DockerFileMatcher.file(p660)); + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> assertThat( + keystore, + DockerFileMatcher.file(p660) + ); default -> throw new IllegalStateException("Unknown Elasticsearch packaging type."); } } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java index a1a9af3b6e307..644990105f60f 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java @@ -245,7 +245,7 @@ protected static void install() throws Exception { installation = Packages.installPackage(sh, distribution); Packages.verifyPackageInstallation(installation, distribution, sh); } - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> { + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> { installation = Docker.runContainer(distribution); Docker.verifyContainerInstallation(installation); } @@ -337,6 +337,7 @@ public Shell.Result runElasticsearchStartCommand(String password, boolean daemon case DOCKER_IRON_BANK: case DOCKER_CLOUD: case DOCKER_CLOUD_ESS: + case DOCKER_WOLFI: // nothing, "installing" docker image is running it return Shell.NO_OP; default: @@ -359,6 +360,7 @@ public void stopElasticsearch() throws Exception { case DOCKER_IRON_BANK: case DOCKER_CLOUD: case DOCKER_CLOUD_ESS: + case DOCKER_WOLFI: // nothing, "installing" docker image is running it break; default: @@ -371,7 +373,7 @@ public void awaitElasticsearchStartup(Shell.Result result) throws Exception { switch (distribution.packaging) { case TAR, ZIP -> Archives.assertElasticsearchStarted(installation); case DEB, RPM -> Packages.assertElasticsearchStarted(sh, installation); - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> Docker.waitForElasticsearchToStart(); + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> Docker.waitForElasticsearchToStart(); default -> throw new IllegalStateException("Unknown Elasticsearch packaging type."); } } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java index b3ea54425af8e..05cef4a0818ba 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java @@ -37,6 +37,8 @@ public Distribution(Path path) { this.packaging = Packaging.DOCKER_CLOUD; } else if (filename.endsWith(".cloud-ess.tar")) { this.packaging = Packaging.DOCKER_CLOUD_ESS; + } else if (filename.endsWith(".wolfi.tar")) { + this.packaging = Packaging.DOCKER_WOLFI; } else { int lastDot = filename.lastIndexOf('.'); this.packaging = Packaging.valueOf(filename.substring(lastDot + 1).toUpperCase(Locale.ROOT)); @@ -61,7 +63,7 @@ public boolean isPackage() { */ public boolean isDocker() { return switch (packaging) { - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> true; + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> true; default -> false; }; } @@ -76,7 +78,8 @@ public enum Packaging { DOCKER_UBI(".ubi.tar", Platforms.isDocker()), DOCKER_IRON_BANK(".ironbank.tar", Platforms.isDocker()), DOCKER_CLOUD(".cloud.tar", Platforms.isDocker()), - DOCKER_CLOUD_ESS(".cloud-ess.tar", Platforms.isDocker()); + DOCKER_CLOUD_ESS(".cloud-ess.tar", Platforms.isDocker()), + DOCKER_WOLFI(".wolfi.tar", Platforms.isDocker()); /** The extension of this distribution's file */ public final String extension; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java index cb8a955a5972c..c38eaa58f0552 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java @@ -486,9 +486,9 @@ public static void verifyContainerInstallation(Installation es) { // Ensure the `elasticsearch` user and group exist. // These lines will both throw an exception if the command fails dockerShell.run("id elasticsearch"); - dockerShell.run("getent group elasticsearch"); + dockerShell.run("grep -E '^elasticsearch:' /etc/group"); - final Shell.Result passwdResult = dockerShell.run("getent passwd elasticsearch"); + final Shell.Result passwdResult = dockerShell.run("grep -E '^elasticsearch:' /etc/passwd"); final String homeDir = passwdResult.stdout().trim().split(":")[5]; assertThat("elasticsearch user's home directory is incorrect", homeDir, equalTo("/usr/share/elasticsearch")); diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java index 6c58bcba09879..2b3eb7ff7a617 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java @@ -167,6 +167,7 @@ public static String getImageName(Distribution distribution) { case DOCKER_IRON_BANK -> "-ironbank"; case DOCKER_CLOUD -> "-cloud"; case DOCKER_CLOUD_ESS -> "-cloud-ess"; + case DOCKER_WOLFI -> "-wolfi"; default -> throw new IllegalStateException("Unexpected distribution packaging type: " + distribution.packaging); };