From d17c00d9e742593609893d06cfae5cc6af079c17 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 9 Apr 2024 12:42:21 -0700 Subject: [PATCH] feat: disable chromium downloads by default in docker-compose When firing the `docker-compose up` command, which is pretty much the first step of most developer workflows, we kick up up 2 chromium downloads: one for PLAYWRIGHT and one for PUPPETEER by default. Those are used for optional features like alerts/reports/thumbnails/ and for CI. For convenience, I'm also introducing support for defining environment variables overrides for docker-compose in `docker/.env-local` while adding this to `.gitignore`, which developers can use without the fear/confusion around committing local envrionment specific settings to the repo. ----------------- About the core feature in this PR here, this saves minutes upon firing `docker-compose up`, and reduce confusion around "what the heck is it doing!?", in a phase where we should just be pulling and starting docker images. Most developer workflows don't require either of those, and if it were the case, we should bake this into the docker image as opposed to installing during the bootstrap phase. In any case, devs can switch those on easily by tweaking the env vars specified in `docker/.env` About PUPPETEER: in https://github.com/apache/superset/pull/22623, I found that it is used by Applitool in two GHAs: - https://github.com/apache/superset/blob/master/.github/workflows/superset-applitool-cypress.yml - https://github.com/apache/superset/blob/master/.github/workflows/superset-applitools-storybook.yml About PLAYWRIGHT, this is what powers alerts and reports as well thumbnail-generation. This is more common, and can be flipped on by setting `ENABLE_PLAYWRIGHT=true` in `docker/.env` --- .gitignore | 1 + docker-compose-image-tag.yml | 30 ++++++++++--- docker-compose-non-dev.yml | 30 ++++++++++--- docker-compose.yml | 42 +++++++++++++++---- docker/.env | 3 ++ docker/docker-bootstrap.sh | 2 +- docker/docker-frontend.sh | 2 +- ...stalling-superset-using-docker-compose.mdx | 9 ++-- 8 files changed, 95 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 02657eb0fa988..bbfc640c9acda 100644 --- a/.gitignore +++ b/.gitignore @@ -110,6 +110,7 @@ release.json messages.mo docker/requirements-local.txt +docker/.env-local cache/ docker/*local* diff --git a/docker-compose-image-tag.yml b/docker-compose-image-tag.yml index 07f0d0dcb14b7..6c017ef75e757 100644 --- a/docker-compose-image-tag.yml +++ b/docker-compose-image-tag.yml @@ -33,7 +33,11 @@ services: - redis:/data db: - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local image: postgres:15 container_name: superset_db restart: unless-stopped @@ -42,7 +46,11 @@ services: - ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d superset: - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local image: *superset-image container_name: superset_app command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"] @@ -57,7 +65,11 @@ services: image: *superset-image container_name: superset_init command: ["/app/docker/docker-init.sh"] - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local depends_on: *superset-depends-on user: "root" volumes: *superset-volumes @@ -68,7 +80,11 @@ services: image: *superset-image container_name: superset_worker command: ["/app/docker/docker-bootstrap.sh", "worker"] - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local restart: unless-stopped depends_on: *superset-depends-on user: "root" @@ -84,7 +100,11 @@ services: image: *superset-image container_name: superset_worker_beat command: ["/app/docker/docker-bootstrap.sh", "beat"] - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local restart: unless-stopped depends_on: *superset-depends-on user: "root" diff --git a/docker-compose-non-dev.yml b/docker-compose-non-dev.yml index f537e26c38c15..78d8cb9c35130 100644 --- a/docker-compose-non-dev.yml +++ b/docker-compose-non-dev.yml @@ -38,7 +38,11 @@ services: - redis:/data db: - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local image: postgres:15 container_name: superset_db restart: unless-stopped @@ -47,7 +51,11 @@ services: - ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d superset: - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local build: <<: *common-build container_name: superset_app @@ -64,7 +72,11 @@ services: build: <<: *common-build command: ["/app/docker/docker-init.sh"] - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local depends_on: *superset-depends-on user: "root" volumes: *superset-volumes @@ -76,7 +88,11 @@ services: <<: *common-build container_name: superset_worker command: ["/app/docker/docker-bootstrap.sh", "worker"] - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local restart: unless-stopped depends_on: *superset-depends-on user: "root" @@ -93,7 +109,11 @@ services: <<: *common-build container_name: superset_worker_beat command: ["/app/docker/docker-bootstrap.sh", "beat"] - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local restart: unless-stopped depends_on: *superset-depends-on user: "root" diff --git a/docker-compose.yml b/docker-compose.yml index 9252e599027c3..d9e7f8b1e6fc7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,7 +54,11 @@ services: - redis:/data db: - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local image: postgres:15 container_name: superset_db restart: unless-stopped @@ -65,7 +69,11 @@ services: - ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d superset: - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local build: <<: *common-build container_name: superset_app @@ -116,7 +124,11 @@ services: <<: *common-build container_name: superset_init command: ["/app/docker/docker-init.sh"] - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local depends_on: *superset-depends-on user: *superset-user volumes: *superset-volumes @@ -135,7 +147,11 @@ services: PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: ${BUILD_SUPERSET_FRONTEND_IN_DOCKER:-false} container_name: superset_node command: ["/app/docker/docker-frontend.sh"] - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local depends_on: *superset-depends-on volumes: *superset-volumes @@ -144,7 +160,11 @@ services: <<: *common-build container_name: superset_worker command: ["/app/docker/docker-bootstrap.sh", "worker"] - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local restart: unless-stopped depends_on: *superset-depends-on user: *superset-user @@ -162,7 +182,11 @@ services: <<: *common-build container_name: superset_worker_beat command: ["/app/docker/docker-bootstrap.sh", "beat"] - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local restart: unless-stopped depends_on: *superset-depends-on user: *superset-user @@ -175,7 +199,11 @@ services: <<: *common-build container_name: superset_tests_worker command: ["/app/docker/docker-bootstrap.sh", "worker"] - env_file: docker/.env + env_file: + # defaults + - docker/.env + # gitignored overrides + - docker/.env-local environment: DATABASE_HOST: localhost DATABASE_DB: test diff --git a/docker/.env b/docker/.env index 1b7d3df8c195c..55048cc503b11 100644 --- a/docker/.env +++ b/docker/.env @@ -53,3 +53,6 @@ SUPERSET_PORT=8088 MAPBOX_API_KEY='' SUPERSET_SECRET_KEY=TEST_NON_DEV_SECRET + +ENABLE_PLAYWRIGHT=false +PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true diff --git a/docker/docker-bootstrap.sh b/docker/docker-bootstrap.sh index 7d53f92898fb8..ecfe92529816f 100755 --- a/docker/docker-bootstrap.sh +++ b/docker/docker-bootstrap.sh @@ -38,7 +38,7 @@ fi # # playwright is an optional package - run only if it is installed # -if command -v playwright > /dev/null 2>&1; then +if [ "$ENABLE_PLAYWRIGHT" = "true" ] && command -v playwright > /dev/null 2>&1; then playwright install-deps playwright install chromium fi diff --git a/docker/docker-frontend.sh b/docker/docker-frontend.sh index 85c57cbf0fc88..c69fefd6d0fb3 100755 --- a/docker/docker-frontend.sh +++ b/docker/docker-frontend.sh @@ -18,8 +18,8 @@ set -e # Packages needed for puppeteer: -apt update if [ "$PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" = "false" ]; then + apt update apt install -y chromium fi diff --git a/docs/docs/installation/installing-superset-using-docker-compose.mdx b/docs/docs/installation/installing-superset-using-docker-compose.mdx index e41145fb09841..4193a1f0fd474 100644 --- a/docs/docs/installation/installing-superset-using-docker-compose.mdx +++ b/docs/docs/installation/installing-superset-using-docker-compose.mdx @@ -112,7 +112,6 @@ with docker compose will store that data in a PostgreSQL database contained in a [volume](https://docs.docker.com/storage/volumes/), which is not backed up. Again **DO NOT USE THIS FOR PRODUCTION** - ::: You should see a wall of logging output from the containers being launched on your machine. Once @@ -127,10 +126,10 @@ can skip to the next section. You can install additional python packages and apply config overrides by following the steps mentioned in [docker/README.md](https://github.com/apache/superset/tree/master/docker#configuration) -You can configure the Docker Compose environment variables for dev and non-dev mode with -`docker/.env`. This environment file sets the environment -for most containers in the Docker Compose setup, and some variables affect multiple containers and -others only single ones. +Note that `docker/.env` sets the default environment variables for all the docker images +used by `docker-compose`, and that `docker/.env-local` can be used to override those defaults. +Also note that `docker/.env-local` is referenced in our `.gitignore`, +preventing developers from risking committing potentially sensitive configuration to the repository. One important variable is `SUPERSET_LOAD_EXAMPLES` which determines whether the `superset_init` container will populate example data and visualizations into the metadata database. These examples