Make builds and tests possible in Altinity's infrastructure

add comment and rename github robot token add clickhouse instance password parameter use Altinity's s3 bucket Use altinityinfra dockerhub images and minor adjustments (#135) Allow CI to be triggered on PR Proper error reporting during docker pull and lowercase version name allow `v22.3.x.y-altinitystable` git tags Download specific MinIO version instead of latest - same as upstream master remove stale chmod More stable CI/CD builds: - Rebuilding all docker images - Reduced number of docker images - Rerunning functional tests even if those were already executed in previous run Added missing dependencies for stateful tests Re-generating _pb2 files on each test run Changed hardcoded docker images name prefixes from `clickhouse/` to `altinityinfra/` Pushing images as :latest too to avoid some test failures reverted back to use clickhouse/jdbc-bridge etc.
Altinity · Aug 4, 2022 · f796d9c · f796d9c
1 parent 25886f5
commit f796d9c
Show file tree

Hide file tree

Showing 94 changed files with 539 additions and 1,944 deletions.
diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml
diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt
@@ -2,7 +2,7 @@
 
 # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
 # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
-SET(VERSION_REVISION 54460)
+SET(VERSION_REVISION 6)
 SET(VERSION_MAJOR 22)
 SET(VERSION_MINOR 3)
 SET(VERSION_PATCH 10)

diff --git a/cmake/version.cmake b/cmake/version.cmake
@@ -19,5 +19,5 @@ set (VERSION_STRING_SHORT "${VERSION_MAJOR}.${VERSION_MINOR}")
 math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000")
 
 if(CLICKHOUSE_OFFICIAL_BUILD)
-    set(VERSION_OFFICIAL " (official build)")
+    set(VERSION_OFFICIAL " (altinity build)")
 endif()
diff --git a/docker/images.json b/docker/images.json
@@ -1,164 +1,149 @@
 {
     "docker/packager/deb": {
-        "name": "clickhouse/deb-builder",
+        "name": "altinityinfra/deb-builder",
         "dependent": []
     },
     "docker/packager/binary": {
-        "name": "clickhouse/binary-builder",
+        "name": "altinityinfra/binary-builder",
         "dependent": [
             "docker/test/split_build_smoke_test",
             "docker/test/codebrowser"
         ]
     },
     "docker/test/compatibility/centos": {
-        "name": "clickhouse/test-old-centos",
+        "name": "altinityinfra/test-old-centos",
         "dependent": []
     },
     "docker/test/compatibility/ubuntu": {
-        "name": "clickhouse/test-old-ubuntu",
+        "name": "altinityinfra/test-old-ubuntu",
         "dependent": []
     },
     "docker/test/integration/base": {
-        "name": "clickhouse/integration-test",
+        "name": "altinityinfra/integration-test",
         "dependent": []
     },
     "docker/test/fuzzer": {
-        "name": "clickhouse/fuzzer",
+        "name": "altinityinfra/fuzzer",
         "dependent": []
     },
     "docker/test/performance-comparison": {
-        "name": "clickhouse/performance-comparison",
+        "name": "altinityinfra/performance-comparison",
         "dependent": []
     },
     "docker/test/util": {
-        "name": "clickhouse/test-util",
+        "name": "altinityinfra/test-util",
         "dependent": [
             "docker/test/base",
             "docker/test/fasttest"
         ]
     },
     "docker/test/stateless": {
-        "name": "clickhouse/stateless-test",
+        "name": "altinityinfra/stateless-test",
         "dependent": [
             "docker/test/stateful",
             "docker/test/unit"
         ]
     },
     "docker/test/stateful": {
-        "name": "clickhouse/stateful-test",
+        "name": "altinityinfra/stateful-test",
         "dependent": [
             "docker/test/stress"
         ]
     },
     "docker/test/unit": {
-        "name": "clickhouse/unit-test",
+        "name": "altinityinfra/unit-test",
         "dependent": []
     },
     "docker/test/stress": {
-        "name": "clickhouse/stress-test",
+        "name": "altinityinfra/stress-test",
         "dependent": []
     },
     "docker/test/split_build_smoke_test": {
-        "name": "clickhouse/split-build-smoke-test",
+        "name": "altinityinfra/split-build-smoke-test",
         "dependent": []
     },
     "docker/test/codebrowser": {
-        "name": "clickhouse/codebrowser",
+        "name": "altinityinfra/codebrowser",
         "dependent": []
     },
     "docker/test/integration/runner": {
         "only_amd64": true,
-        "name": "clickhouse/integration-tests-runner",
+        "name": "altinityinfra/integration-tests-runner",
         "dependent": []
     },
     "docker/test/testflows/runner": {
-        "name": "clickhouse/testflows-runner",
+        "name": "altinityinfra/testflows-runner",
         "dependent": []
     },
     "docker/test/fasttest": {
-        "name": "clickhouse/fasttest",
+        "name": "altinityinfra/fasttest",
         "dependent": []
     },
     "docker/test/style": {
-        "name": "clickhouse/style-test",
+        "name": "altinityinfra/style-test",
         "dependent": []
     },
     "docker/test/integration/s3_proxy": {
-        "name": "clickhouse/s3-proxy",
+        "name": "altinityinfra/s3-proxy",
         "dependent": []
     },
     "docker/test/integration/resolver": {
-        "name": "clickhouse/python-bottle",
+        "name": "altinityinfra/python-bottle",
         "dependent": []
     },
     "docker/test/integration/helper_container": {
-        "name": "clickhouse/integration-helper",
+        "name": "altinityinfra/integration-helper",
         "dependent": []
     },
     "docker/test/integration/mysql_golang_client": {
-        "name": "clickhouse/mysql-golang-client",
+        "name": "altinityinfra/mysql-golang-client",
         "dependent": []
     },
     "docker/test/integration/dotnet_client": {
-        "name": "clickhouse/dotnet-client",
+        "name": "altinityinfra/dotnet-client",
         "dependent": []
     },
     "docker/test/integration/mysql_java_client": {
-        "name": "clickhouse/mysql-java-client",
+        "name": "altinityinfra/mysql-java-client",
         "dependent": []
     },
     "docker/test/integration/mysql_js_client": {
-        "name": "clickhouse/mysql-js-client",
+        "name": "altinityinfra/mysql-js-client",
         "dependent": []
     },
     "docker/test/integration/mysql_php_client": {
-        "name": "clickhouse/mysql-php-client",
+        "name": "altinityinfra/mysql-php-client",
         "dependent": []
     },
     "docker/test/integration/postgresql_java_client": {
-        "name": "clickhouse/postgresql-java-client",
+        "name": "altinityinfra/postgresql-java-client",
         "dependent": []
     },
     "docker/test/integration/kerberos_kdc": {
         "only_amd64": true,
-        "name": "clickhouse/kerberos-kdc",
+        "name": "altinityinfra/kerberos-kdc",
         "dependent": []
     },
     "docker/test/base": {
-         "name": "clickhouse/test-base",
-         "dependent": [
+        "name": "altinityinfra/test-base",
+        "dependent": [
             "docker/test/stateless",
             "docker/test/integration/base",
             "docker/test/fuzzer",
             "docker/test/keeper-jepsen"
-         ]
+        ]
     },
     "docker/test/integration/kerberized_hadoop": {
         "only_amd64": true,
-        "name": "clickhouse/kerberized-hadoop",
+        "name": "altinityinfra/kerberized-hadoop",
         "dependent": []
     },
     "docker/test/sqlancer": {
-        "name": "clickhouse/sqlancer-test",
+        "name": "altinityinfra/sqlancer-test",
         "dependent": []
     },
     "docker/test/keeper-jepsen": {
-        "name": "clickhouse/keeper-jepsen-test",
-        "dependent": []
-    },
-    "docker/docs/builder": {
-        "name": "clickhouse/docs-builder",
-        "dependent": [
-            "docker/docs/check",
-            "docker/docs/release"
-        ]
-    },
-    "docker/docs/check": {
-        "name": "clickhouse/docs-check",
-        "dependent": []
-    },
-    "docker/docs/release": {
-        "name": "clickhouse/docs-release",
+        "name": "altinityinfra/keeper-jepsen-test",
         "dependent": []
     }
 }
diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh
@@ -19,6 +19,9 @@ ln -sf darwin-x86_64 build/cmake/toolchain/darwin-aarch64
 # export CCACHE_LOGFILE=/build/ccache.log
 # export CCACHE_DEBUG=1
 
+# https://stackoverflow.com/a/71940133
+git config --global --add safe.directory '*'
+
 mkdir -p build/build_docker
 cd build/build_docker
 rm -f CMakeCache.txt

diff --git a/docker/packager/packager b/docker/packager/packager
@@ -331,7 +331,7 @@ if __name__ == "__main__":
         args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir))
 
     image_type = "binary" if args.package_type == "performance" else args.package_type
-    image_name = "clickhouse/binary-builder"
+    image_name = "altinityinfra/binary-builder"
 
     if not os.path.isabs(args.clickhouse_repo_path):
         ch_root = os.path.abspath(os.path.join(os.getcwd(), args.clickhouse_repo_path))

diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile
@@ -1,7 +1,7 @@
 # rebuild in #33610
-# docker build -t clickhouse/test-base .
+# docker build -t altinityinfra/test-base .
 ARG FROM_TAG=latest
-FROM clickhouse/test-util:$FROM_TAG
+FROM altinityinfra/test-util:$FROM_TAG
 
 # ARG for quick switch to a given ubuntu mirror
 ARG apt_archive="http://archive.ubuntu.com"

diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile
@@ -2,7 +2,7 @@
 # docker build --network=host -t clickhouse/codebrowser .
 # docker run --volume=path_to_repo:/repo_folder --volume=path_to_result:/test_output clickhouse/codebrowser
 ARG FROM_TAG=latest
-FROM clickhouse/binary-builder:$FROM_TAG
+FROM altinityinfra/binary-builder:$FROM_TAG
 
 # ARG for quick switch to a given ubuntu mirror
 ARG apt_archive="http://archive.ubuntu.com"

diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile
@@ -1,7 +1,7 @@
 # rebuild in #33610
 #  docker build -t clickhouse/fasttest .
 ARG FROM_TAG=latest
-FROM clickhouse/test-util:$FROM_TAG
+FROM altinityinfra/test-util:$FROM_TAG
 
 # ARG for quick switch to a given ubuntu mirror
 ARG apt_archive="http://archive.ubuntu.com"

diff --git a/docker/test/fuzzer/Dockerfile b/docker/test/fuzzer/Dockerfile
@@ -1,7 +1,7 @@
 # rebuild in #33610
 # docker build -t clickhouse/fuzzer .
 ARG FROM_TAG=latest
-FROM clickhouse/test-base:$FROM_TAG
+FROM altinityinfra/test-base:$FROM_TAG
 
 # ARG for quick switch to a given ubuntu mirror
 ARG apt_archive="http://archive.ubuntu.com"

diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile
@@ -1,7 +1,7 @@
 # rebuild in #33610
 # docker build -t clickhouse/integration-test .
 ARG FROM_TAG=latest
-FROM clickhouse/test-base:$FROM_TAG
+FROM altinityinfra/test-base:$FROM_TAG
 
 SHELL ["/bin/bash", "-c"]
 

diff --git a/docker/test/integration/mysql_php_client/Dockerfile b/docker/test/integration/mysql_php_client/Dockerfile
@@ -1,7 +1,7 @@
 # docker build -t clickhouse/mysql-php-client .
 # MySQL PHP client docker container
 
-FROM php:7.3-cli
+FROM php:8.0.18-cli
 
 COPY ./client.crt client.crt
 COPY ./client.key client.key

diff --git a/docker/test/integration/runner/compose/docker_compose_dotnet_client.yml b/docker/test/integration/runner/compose/docker_compose_dotnet_client.yml
@@ -1,6 +1,6 @@
 version: '2.3'
 services:
   dotnet1:
-    image: clickhouse/dotnet-client:${DOCKER_DOTNET_CLIENT_TAG:-latest}
+    image: altinityinfra/dotnet-client:${DOCKER_DOTNET_CLIENT_TAG:-latest}
     # to keep container running
     command: sleep infinity
diff --git a/docker/test/integration/runner/compose/docker_compose_keeper.yml b/docker/test/integration/runner/compose/docker_compose_keeper.yml
@@ -1,7 +1,7 @@
 version: '2.3'
 services:
     zoo1:
-        image: ${image:-clickhouse/integration-test}
+        image: ${image:-altinityinfra/integration-test}
         restart: always
         user: ${user:-}
         volumes:
@@ -31,7 +31,7 @@ services:
             - inet6
             - rotate
     zoo2:
-        image: ${image:-clickhouse/integration-test}
+        image: ${image:-altinityinfra/integration-test}
         restart: always
         user: ${user:-}
         volumes:
@@ -61,7 +61,7 @@ services:
             - inet6
             - rotate
     zoo3:
-        image: ${image:-clickhouse/integration-test}
+        image: ${image:-altinityinfra/integration-test}
         restart: always
         user: ${user:-}
         volumes:

diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml
@@ -4,7 +4,7 @@ services:
   kerberizedhdfs1:
     cap_add:
       - DAC_READ_SEARCH
-    image: clickhouse/kerberized-hadoop:${DOCKER_KERBERIZED_HADOOP_TAG:-latest}
+    image: altinityinfra/kerberized-hadoop:${DOCKER_KERBERIZED_HADOOP_TAG:-latest}
     hostname: kerberizedhdfs1
     restart: always
     volumes:
@@ -22,7 +22,7 @@ services:
     entrypoint: /etc/bootstrap.sh -d
 
   hdfskerberos:
-    image: clickhouse/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest}
+    image: altinityinfra/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest}
     hostname: hdfskerberos
     volumes:
       - ${KERBERIZED_HDFS_DIR}/secrets:/tmp/keytab

diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml
@@ -50,7 +50,7 @@ services:
       - label:disable
 
   kafka_kerberos:
-    image: clickhouse/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest}
+    image: altinityinfra/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest}
     hostname: kafka_kerberos
     volumes:
       - ${KERBERIZED_KAFKA_DIR}/secrets:/tmp/keytab

diff --git a/docker/test/integration/runner/compose/docker_compose_minio.yml b/docker/test/integration/runner/compose/docker_compose_minio.yml
@@ -21,22 +21,22 @@ services:
 
   # HTTP proxies for Minio.
   proxy1:
-    image: clickhouse/s3-proxy
+    image: altinityinfra/s3-proxy
     expose:
       - "8080" # Redirect proxy port
       - "80"   # Reverse proxy port
       - "443"  # Reverse proxy port (secure)
 
   proxy2:
-    image: clickhouse/s3-proxy
+    image: altinityinfra/s3-proxy
     expose:
       - "8080"
       - "80"
       - "443"
 
   # Empty container to run proxy resolver.
   resolver:
-    image: clickhouse/python-bottle
+    image: altinityinfra/python-bottle
     expose:
       - "8080"
     tty: true

diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_golang_client.yml b/docker/test/integration/runner/compose/docker_compose_mysql_golang_client.yml
@@ -1,6 +1,6 @@
 version: '2.3'
 services:
   golang1:
-    image: clickhouse/mysql-golang-client:${DOCKER_MYSQL_GOLANG_CLIENT_TAG:-latest}
+    image: altinityinfra/mysql-golang-client:${DOCKER_MYSQL_GOLANG_CLIENT_TAG:-latest}
     # to keep container running
     command: sleep infinity
diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_java_client.yml b/docker/test/integration/runner/compose/docker_compose_mysql_java_client.yml
@@ -1,6 +1,6 @@
 version: '2.3'
 services:
   java1:
-    image: clickhouse/mysql-java-client:${DOCKER_MYSQL_JAVA_CLIENT_TAG:-latest}
+    image: altinityinfra/mysql-java-client:${DOCKER_MYSQL_JAVA_CLIENT_TAG:-latest}
     # to keep container running
     command: sleep infinity