From 7590621bb9444d3dbf764d5a8bcff129d41df1d3 Mon Sep 17 00:00:00 2001 From: Daniyar Itegulov Date: Mon, 21 Oct 2024 23:28:01 +1100 Subject: [PATCH] use docker-managed volumes --- .../workflows/build-contract-verifier-template.yml | 1 - .github/workflows/build-core-template.yml | 1 - .github/workflows/build-local-node-docker.yml | 1 - .github/workflows/build-prover-template.yml | 1 - .../workflows/build-witness-generator-template.yml | 1 - .github/workflows/ci-common-reusable.yml | 1 - .github/workflows/ci-prover-e2e.yml | 1 - .github/workflows/ci-prover-reusable.yml | 2 -- bin/ci_localnet_up | 1 - docker-compose-gpu-runner-cuda-12-0.yml | 8 ++++++-- docker-compose-runner-nightly.yml | 5 ++++- docker-compose-unit-tests.yml | 1 - docker-compose.yml | 13 ++++++++----- zkstack_cli/crates/common/src/docker.rs | 6 +++++- .../crates/zkstack/src/commands/containers.rs | 12 ------------ .../zkstack/src/commands/dev/commands/clean/mod.rs | 6 +----- .../crates/zkstack/src/commands/dev/messages.rs | 4 +--- 17 files changed, 25 insertions(+), 40 deletions(-) diff --git a/.github/workflows/build-contract-verifier-template.yml b/.github/workflows/build-contract-verifier-template.yml index e4d04b90410e..1481e542de57 100644 --- a/.github/workflows/build-contract-verifier-template.yml +++ b/.github/workflows/build-contract-verifier-template.yml @@ -101,7 +101,6 @@ jobs: - name: start-services run: | echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env - mkdir -p ./volumes/postgres run_retried docker compose pull zk postgres docker compose up -d zk postgres ci_run pre_download_compilers.sh diff --git a/.github/workflows/build-core-template.yml b/.github/workflows/build-core-template.yml index fe1d23427645..15d4432191dd 100644 --- a/.github/workflows/build-core-template.yml +++ b/.github/workflows/build-core-template.yml @@ -114,7 +114,6 @@ jobs: - name: start-services run: | echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env - mkdir -p ./volumes/postgres run_retried docker compose pull zk postgres docker compose up -d zk postgres ci_run pre_download_compilers.sh diff --git a/.github/workflows/build-local-node-docker.yml b/.github/workflows/build-local-node-docker.yml index 80142cb6005c..cbb4239b5725 100644 --- a/.github/workflows/build-local-node-docker.yml +++ b/.github/workflows/build-local-node-docker.yml @@ -50,7 +50,6 @@ jobs: - name: start-services run: | - mkdir -p ./volumes/postgres run_retried docker compose pull zk postgres docker compose up -d zk postgres diff --git a/.github/workflows/build-prover-template.yml b/.github/workflows/build-prover-template.yml index 2dcb5dadb174..91de5dd51ecf 100644 --- a/.github/workflows/build-prover-template.yml +++ b/.github/workflows/build-prover-template.yml @@ -75,7 +75,6 @@ jobs: - name: start-services run: | echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env - mkdir -p ./volumes/postgres run_retried docker compose pull zk postgres docker compose up -d zk postgres ci_run sccache --start-server diff --git a/.github/workflows/build-witness-generator-template.yml b/.github/workflows/build-witness-generator-template.yml index 33d78b3cf2fc..d9493f97cae1 100644 --- a/.github/workflows/build-witness-generator-template.yml +++ b/.github/workflows/build-witness-generator-template.yml @@ -75,7 +75,6 @@ jobs: - name: start-services run: | echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env - mkdir -p ./volumes/postgres run_retried docker compose pull zk postgres docker compose up -d zk postgres ci_run sccache --start-server diff --git a/.github/workflows/ci-common-reusable.yml b/.github/workflows/ci-common-reusable.yml index 7d75fb224d6e..ea91fc4a7cd6 100644 --- a/.github/workflows/ci-common-reusable.yml +++ b/.github/workflows/ci-common-reusable.yml @@ -27,7 +27,6 @@ jobs: - name: Start services run: | run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull - mkdir -p ./volumes/postgres docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres - name: Install zkstack diff --git a/.github/workflows/ci-prover-e2e.yml b/.github/workflows/ci-prover-e2e.yml index 105ae1f1485d..b0b9caf888fc 100644 --- a/.github/workflows/ci-prover-e2e.yml +++ b/.github/workflows/ci-prover-e2e.yml @@ -29,7 +29,6 @@ jobs: - name: Start services run: | run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull - mkdir -p ./volumes/postgres ./volumes/reth/data docker-compose -f ${RUNNER_COMPOSE_FILE} --profile runner up -d --wait ci_run sccache --start-server diff --git a/.github/workflows/ci-prover-reusable.yml b/.github/workflows/ci-prover-reusable.yml index 6cb9c26d21e7..f1cdf435b46a 100644 --- a/.github/workflows/ci-prover-reusable.yml +++ b/.github/workflows/ci-prover-reusable.yml @@ -27,7 +27,6 @@ jobs: - name: Start services run: | run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull - mkdir -p ./volumes/postgres docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres - name: Install zkstack @@ -66,7 +65,6 @@ jobs: - name: Start services run: | run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull - mkdir -p ./volumes/postgres docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres - name: Install zkstack diff --git a/bin/ci_localnet_up b/bin/ci_localnet_up index 8673a909af77..c399de410d74 100755 --- a/bin/ci_localnet_up +++ b/bin/ci_localnet_up @@ -4,6 +4,5 @@ set -e cd $ZKSYNC_HOME -mkdir -p ./volumes/postgres ./volumes/reth/data run_retried docker-compose pull docker-compose --profile runner up -d --wait diff --git a/docker-compose-gpu-runner-cuda-12-0.yml b/docker-compose-gpu-runner-cuda-12-0.yml index c930fa376f5e..bd91a5a5b0e4 100644 --- a/docker-compose-gpu-runner-cuda-12-0.yml +++ b/docker-compose-gpu-runner-cuda-12-0.yml @@ -6,8 +6,8 @@ services: ports: - 127.0.0.1:8545:8545 volumes: - - type: bind - source: ./volumes/reth/data + - type: volume + source: reth-data target: /rethdata - type: bind source: ./etc/reth/chaindata @@ -69,3 +69,7 @@ services: environment: # We bind only to 127.0.0.1, so setting insecure password is acceptable here - POSTGRES_PASSWORD=notsecurepassword + +volumes: + postgres-data: + reth-data: \ No newline at end of file diff --git a/docker-compose-runner-nightly.yml b/docker-compose-runner-nightly.yml index cadd1009f7a6..4a854aa0b0a4 100644 --- a/docker-compose-runner-nightly.yml +++ b/docker-compose-runner-nightly.yml @@ -1,4 +1,3 @@ -version: '3.2' services: zk: image: ghcr.io/matter-labs/zk-environment:latest2.0-lightweight-nightly @@ -15,3 +14,7 @@ services: extends: file: docker-compose.yml service: reth + +volumes: + postgres-data: + reth-data: \ No newline at end of file diff --git a/docker-compose-unit-tests.yml b/docker-compose-unit-tests.yml index ddbc76bb196c..b839be2d9f4f 100644 --- a/docker-compose-unit-tests.yml +++ b/docker-compose-unit-tests.yml @@ -1,4 +1,3 @@ -version: '3.2' name: unit_tests services: # An instance of postgres configured to execute Rust unit-tests, tuned for performance. diff --git a/docker-compose.yml b/docker-compose.yml index 1e3a273ec9a4..d8f40720fe84 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3.2' services: reth: restart: always @@ -6,8 +5,8 @@ services: ports: - 127.0.0.1:8545:8545 volumes: - - type: bind - source: ./volumes/reth/data + - type: volume + source: reth-data target: /rethdata - type: bind source: ./etc/reth/chaindata @@ -22,8 +21,8 @@ services: ports: - 127.0.0.1:5432:5432 volumes: - - type: bind - source: ./volumes/postgres + - type: volume + source: postgres-data target: /var/lib/postgresql/data environment: # We bind only to 127.0.0.1, so setting insecure password is acceptable here @@ -56,3 +55,7 @@ services: profiles: - runner network_mode: host + +volumes: + postgres-data: + reth-data: \ No newline at end of file diff --git a/zkstack_cli/crates/common/src/docker.rs b/zkstack_cli/crates/common/src/docker.rs index a5731808814f..71e2040ee31c 100644 --- a/zkstack_cli/crates/common/src/docker.rs +++ b/zkstack_cli/crates/common/src/docker.rs @@ -14,7 +14,11 @@ pub fn up(shell: &Shell, docker_compose_file: &str, detach: bool) -> anyhow::Res } pub fn down(shell: &Shell, docker_compose_file: &str) -> anyhow::Result<()> { - Ok(Cmd::new(cmd!(shell, "docker compose -f {docker_compose_file} down")).run()?) + Ok(Cmd::new(cmd!( + shell, + "docker compose -f {docker_compose_file} down -v" + )) + .run()?) } pub fn run(shell: &Shell, docker_image: &str, docker_args: Vec) -> anyhow::Result<()> { diff --git a/zkstack_cli/crates/zkstack/src/commands/containers.rs b/zkstack_cli/crates/zkstack/src/commands/containers.rs index 9c11cc2e3efc..8367289bd67f 100644 --- a/zkstack_cli/crates/zkstack/src/commands/containers.rs +++ b/zkstack_cli/crates/zkstack/src/commands/containers.rs @@ -36,10 +36,6 @@ pub fn run(shell: &Shell, args: ContainersArgs) -> anyhow::Result<()> { } pub fn initialize_docker(shell: &Shell, ecosystem: &EcosystemConfig) -> anyhow::Result<()> { - if !shell.path_exists("volumes") { - create_docker_folders(shell)?; - }; - if !shell.path_exists(DOCKER_COMPOSE_FILE) { copy_dockerfile(shell, ecosystem.link_to_code.clone())?; }; @@ -75,14 +71,6 @@ pub fn start_containers(shell: &Shell, observability: bool) -> anyhow::Result<() Ok(()) } -fn create_docker_folders(shell: &Shell) -> anyhow::Result<()> { - shell.create_dir("volumes")?; - shell.create_dir("volumes/postgres")?; - shell.create_dir("volumes/reth")?; - shell.create_dir("volumes/reth/data")?; - Ok(()) -} - fn copy_dockerfile(shell: &Shell, link_to_code: PathBuf) -> anyhow::Result<()> { let docker_compose_file = link_to_code.join(DOCKER_COMPOSE_FILE); diff --git a/zkstack_cli/crates/zkstack/src/commands/dev/commands/clean/mod.rs b/zkstack_cli/crates/zkstack/src/commands/dev/commands/clean/mod.rs index 4cb419ce7a46..0929f5e4623f 100644 --- a/zkstack_cli/crates/zkstack/src/commands/dev/commands/clean/mod.rs +++ b/zkstack_cli/crates/zkstack/src/commands/dev/commands/clean/mod.rs @@ -5,8 +5,7 @@ use config::{EcosystemConfig, DOCKER_COMPOSE_FILE}; use xshell::Shell; use crate::commands::dev::messages::{ - MSG_CONTRACTS_CLEANING, MSG_CONTRACTS_CLEANING_FINISHED, MSG_DOCKER_COMPOSE_CLEANED, - MSG_DOCKER_COMPOSE_DOWN, MSG_DOCKER_COMPOSE_REMOVE_VOLUMES, + MSG_CONTRACTS_CLEANING, MSG_CONTRACTS_CLEANING_FINISHED, MSG_DOCKER_COMPOSE_DOWN, }; #[derive(Subcommand, Debug)] @@ -35,9 +34,6 @@ pub fn run(shell: &Shell, args: CleanCommands) -> anyhow::Result<()> { pub fn containers(shell: &Shell) -> anyhow::Result<()> { logger::info(MSG_DOCKER_COMPOSE_DOWN); docker::down(shell, DOCKER_COMPOSE_FILE)?; - logger::info(MSG_DOCKER_COMPOSE_REMOVE_VOLUMES); - shell.remove_path("volumes")?; - logger::info(MSG_DOCKER_COMPOSE_CLEANED); Ok(()) } diff --git a/zkstack_cli/crates/zkstack/src/commands/dev/messages.rs b/zkstack_cli/crates/zkstack/src/commands/dev/messages.rs index a38fff5a178a..3d31497b7ebc 100644 --- a/zkstack_cli/crates/zkstack/src/commands/dev/messages.rs +++ b/zkstack_cli/crates/zkstack/src/commands/dev/messages.rs @@ -157,9 +157,7 @@ pub(super) const MSG_UPGRADE_TEST_RUN_INFO: &str = "Running upgrade test"; pub(super) const MSG_UPGRADE_TEST_RUN_SUCCESS: &str = "Upgrade test ran successfully"; // Cleaning related messages -pub(super) const MSG_DOCKER_COMPOSE_DOWN: &str = "docker compose down"; -pub(super) const MSG_DOCKER_COMPOSE_REMOVE_VOLUMES: &str = "docker compose remove volumes"; -pub(super) const MSG_DOCKER_COMPOSE_CLEANED: &str = "docker compose network cleaned"; +pub(super) const MSG_DOCKER_COMPOSE_DOWN: &str = "docker compose down -v"; pub(super) const MSG_CONTRACTS_CLEANING: &str = "Removing contracts building and deployment artifacts"; pub(super) const MSG_CONTRACTS_CLEANING_FINISHED: &str =