Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(zkstack_cli): use docker-managed volumes #3140

Merged
merged 1 commit into from
Oct 24, 2024
Merged

feat(zkstack_cli): use docker-managed volumes #3140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .github/workflows/build-contract-verifier-template.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,6 @@ jobs:
- name: start-services
run: |
echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
mkdir -p ./volumes/postgres
run_retried docker compose pull zk postgres
docker compose up -d zk postgres
ci_run pre_download_compilers.sh
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/build-core-template.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,6 @@ jobs:
- name: start-services
run: |
echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
mkdir -p ./volumes/postgres
run_retried docker compose pull zk postgres
docker compose up -d zk postgres
ci_run pre_download_compilers.sh
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/build-local-node-docker.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ jobs:

- name: start-services
run: |
mkdir -p ./volumes/postgres
run_retried docker compose pull zk postgres
docker compose up -d zk postgres

Expand Down
1 change: 0 additions & 1 deletion .github/workflows/build-prover-template.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ jobs:
- name: start-services
run: |
echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
mkdir -p ./volumes/postgres
run_retried docker compose pull zk postgres
docker compose up -d zk postgres
ci_run sccache --start-server
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/build-witness-generator-template.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ jobs:
- name: start-services
run: |
echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
mkdir -p ./volumes/postgres
run_retried docker compose pull zk postgres
docker compose up -d zk postgres
ci_run sccache --start-server
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/ci-common-reusable.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ jobs:
- name: Start services
run: |
run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
mkdir -p ./volumes/postgres
docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres

- name: Install zkstack
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/ci-prover-e2e.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ jobs:
- name: Start services
run: |
run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
mkdir -p ./volumes/postgres ./volumes/reth/data
docker-compose -f ${RUNNER_COMPOSE_FILE} --profile runner up -d --wait
ci_run sccache --start-server

Expand Down
2 changes: 0 additions & 2 deletions .github/workflows/ci-prover-reusable.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ jobs:
- name: Start services
run: |
run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
mkdir -p ./volumes/postgres
docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres

- name: Install zkstack
Expand Down Expand Up @@ -66,7 +65,6 @@ jobs:
- name: Start services
run: |
run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
mkdir -p ./volumes/postgres
docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres

- name: Install zkstack
Expand Down
1 change: 0 additions & 1 deletion bin/ci_localnet_up
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,5 @@ set -e

cd $ZKSYNC_HOME

mkdir -p ./volumes/postgres ./volumes/reth/data
run_retried docker-compose pull
docker-compose --profile runner up -d --wait
8 changes: 6 additions & 2 deletions docker-compose-gpu-runner-cuda-12-0.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ services:
ports:
- 127.0.0.1:8545:8545
volumes:
- type: bind
source: ./volumes/reth/data
- type: volume
source: reth-data
target: /rethdata
- type: bind
source: ./etc/reth/chaindata
Expand Down Expand Up @@ -69,3 +69,7 @@ services:
environment:
# We bind only to 127.0.0.1, so setting insecure password is acceptable here
- POSTGRES_PASSWORD=notsecurepassword

volumes:
postgres-data:
reth-data:
Deniallugo marked this conversation as resolved.
Show resolved Hide resolved
5 changes: 4 additions & 1 deletion docker-compose-runner-nightly.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
version: '3.2'
services:
zk:
image: ghcr.io/matter-labs/zk-environment:latest2.0-lightweight-nightly
Expand All @@ -15,3 +14,7 @@ services:
extends:
file: docker-compose.yml
service: reth

volumes:
postgres-data:
reth-data:
1 change: 0 additions & 1 deletion docker-compose-unit-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
version: '3.2'
name: unit_tests
services:
# An instance of postgres configured to execute Rust unit-tests, tuned for performance.
Expand Down
13 changes: 8 additions & 5 deletions docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
version: '3.2'
services:
reth:
restart: always
image: "ghcr.io/paradigmxyz/reth:v1.0.6"
ports:
- 127.0.0.1:8545:8545
volumes:
- type: bind
source: ./volumes/reth/data
- type: volume
source: reth-data
target: /rethdata
- type: bind
source: ./etc/reth/chaindata
Expand All @@ -22,8 +21,8 @@ services:
ports:
- 127.0.0.1:5432:5432
volumes:
- type: bind
source: ./volumes/postgres
- type: volume
source: postgres-data
target: /var/lib/postgresql/data
environment:
# We bind only to 127.0.0.1, so setting insecure password is acceptable here
Expand Down Expand Up @@ -56,3 +55,7 @@ services:
profiles:
- runner
network_mode: host

volumes:
postgres-data:
reth-data:
6 changes: 5 additions & 1 deletion zkstack_cli/crates/common/src/docker.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,11 @@ pub fn up(shell: &Shell, docker_compose_file: &str, detach: bool) -> anyhow::Res
}

pub fn down(shell: &Shell, docker_compose_file: &str) -> anyhow::Result<()> {
Ok(Cmd::new(cmd!(shell, "docker compose -f {docker_compose_file} down")).run()?)
Ok(Cmd::new(cmd!(
shell,
"docker compose -f {docker_compose_file} down -v"
))
.run()?)
}

pub fn run(shell: &Shell, docker_image: &str, docker_args: Vec<String>) -> anyhow::Result<()> {
Expand Down
12 changes: 0 additions & 12 deletions zkstack_cli/crates/zkstack/src/commands/containers.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,6 @@ pub fn run(shell: &Shell, args: ContainersArgs) -> anyhow::Result<()> {
}

pub fn initialize_docker(shell: &Shell, ecosystem: &EcosystemConfig) -> anyhow::Result<()> {
if !shell.path_exists("volumes") {
create_docker_folders(shell)?;
};

if !shell.path_exists(DOCKER_COMPOSE_FILE) {
copy_dockerfile(shell, ecosystem.link_to_code.clone())?;
};
Expand Down Expand Up @@ -75,14 +71,6 @@ pub fn start_containers(shell: &Shell, observability: bool) -> anyhow::Result<()
Ok(())
}

fn create_docker_folders(shell: &Shell) -> anyhow::Result<()> {
shell.create_dir("volumes")?;
shell.create_dir("volumes/postgres")?;
shell.create_dir("volumes/reth")?;
shell.create_dir("volumes/reth/data")?;
Ok(())
}

fn copy_dockerfile(shell: &Shell, link_to_code: PathBuf) -> anyhow::Result<()> {
let docker_compose_file = link_to_code.join(DOCKER_COMPOSE_FILE);

Expand Down
6 changes: 1 addition & 5 deletions zkstack_cli/crates/zkstack/src/commands/dev/commands/clean/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@ use config::{EcosystemConfig, DOCKER_COMPOSE_FILE};
use xshell::Shell;

use crate::commands::dev::messages::{
MSG_CONTRACTS_CLEANING, MSG_CONTRACTS_CLEANING_FINISHED, MSG_DOCKER_COMPOSE_CLEANED,
MSG_DOCKER_COMPOSE_DOWN, MSG_DOCKER_COMPOSE_REMOVE_VOLUMES,
MSG_CONTRACTS_CLEANING, MSG_CONTRACTS_CLEANING_FINISHED, MSG_DOCKER_COMPOSE_DOWN,
};

#[derive(Subcommand, Debug)]
Expand Down Expand Up @@ -35,9 +34,6 @@ pub fn run(shell: &Shell, args: CleanCommands) -> anyhow::Result<()> {
pub fn containers(shell: &Shell) -> anyhow::Result<()> {
logger::info(MSG_DOCKER_COMPOSE_DOWN);
docker::down(shell, DOCKER_COMPOSE_FILE)?;
logger::info(MSG_DOCKER_COMPOSE_REMOVE_VOLUMES);
shell.remove_path("volumes")?;
logger::info(MSG_DOCKER_COMPOSE_CLEANED);
Ok(())
}

Expand Down
4 changes: 1 addition & 3 deletions zkstack_cli/crates/zkstack/src/commands/dev/messages.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,9 +157,7 @@ pub(super) const MSG_UPGRADE_TEST_RUN_INFO: &str = "Running upgrade test";
pub(super) const MSG_UPGRADE_TEST_RUN_SUCCESS: &str = "Upgrade test ran successfully";

// Cleaning related messages
pub(super) const MSG_DOCKER_COMPOSE_DOWN: &str = "docker compose down";
pub(super) const MSG_DOCKER_COMPOSE_REMOVE_VOLUMES: &str = "docker compose remove volumes";
pub(super) const MSG_DOCKER_COMPOSE_CLEANED: &str = "docker compose network cleaned";
pub(super) const MSG_DOCKER_COMPOSE_DOWN: &str = "docker compose down -v";
pub(super) const MSG_CONTRACTS_CLEANING: &str =
"Removing contracts building and deployment artifacts";
pub(super) const MSG_CONTRACTS_CLEANING_FINISHED: &str =
Expand Down
Loading