From 00b2d7795fc93e7be55f734eb2ac43b11e3e1178 Mon Sep 17 00:00:00 2001 From: Daniel Porteous Date: Wed, 25 Oct 2023 12:59:19 +0100 Subject: [PATCH] [CLI] Support Docker socket in home directory for local testnet --- Cargo.lock | 10 +-- crates/aptos/CHANGELOG.md | 4 + crates/aptos/Cargo.toml | 6 +- crates/aptos/src/node/local_testnet/docker.rs | 80 ++++++++++++------- .../src/node/local_testnet/indexer_api.rs | 8 +- .../aptos/src/node/local_testnet/postgres.rs | 8 +- .../docs/guides/local-development-network.md | 8 +- 7 files changed, 78 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cdd6e4e14eda3..b20564621a161 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -215,7 +215,7 @@ dependencies = [ [[package]] name = "aptos" -version = "2.2.2" +version = "2.3.0" dependencies = [ "anyhow", "aptos-api-types", @@ -2569,7 +2569,7 @@ dependencies = [ [[package]] name = "aptos-moving-average" version = "0.1.0" -source = "git+https://github.com/aptos-labs/aptos-indexer-processors.git?rev=a9e42dc9d80a69e9d02cd5f19c39e0f3f5cfbeb0#a9e42dc9d80a69e9d02cd5f19c39e0f3f5cfbeb0" +source = "git+https://github.com/aptos-labs/aptos-indexer-processors.git?rev=bcba94c26c8a6372056d2b69ce411c5719f98965#bcba94c26c8a6372056d2b69ce411c5719f98965" dependencies = [ "chrono", ] @@ -11778,10 +11778,10 @@ dependencies = [ [[package]] name = "processor" version = "1.0.0" -source = "git+https://github.com/aptos-labs/aptos-indexer-processors.git?rev=a9e42dc9d80a69e9d02cd5f19c39e0f3f5cfbeb0#a9e42dc9d80a69e9d02cd5f19c39e0f3f5cfbeb0" +source = "git+https://github.com/aptos-labs/aptos-indexer-processors.git?rev=bcba94c26c8a6372056d2b69ce411c5719f98965#bcba94c26c8a6372056d2b69ce411c5719f98965" dependencies = [ "anyhow", - "aptos-moving-average 0.1.0 (git+https://github.com/aptos-labs/aptos-indexer-processors.git?rev=a9e42dc9d80a69e9d02cd5f19c39e0f3f5cfbeb0)", + "aptos-moving-average 0.1.0 (git+https://github.com/aptos-labs/aptos-indexer-processors.git?rev=bcba94c26c8a6372056d2b69ce411c5719f98965)", "aptos-protos 1.1.2 (git+https://github.com/aptos-labs/aptos-core.git?rev=af4067b6606c0c4823dbb640e8f2db4cb2a1a7f9)", "async-trait", "base64 0.13.0", @@ -13248,7 +13248,7 @@ dependencies = [ [[package]] name = "server-framework" version = "1.0.0" -source = "git+https://github.com/aptos-labs/aptos-indexer-processors.git?rev=a9e42dc9d80a69e9d02cd5f19c39e0f3f5cfbeb0#a9e42dc9d80a69e9d02cd5f19c39e0f3f5cfbeb0" +source = "git+https://github.com/aptos-labs/aptos-indexer-processors.git?rev=bcba94c26c8a6372056d2b69ce411c5719f98965#bcba94c26c8a6372056d2b69ce411c5719f98965" dependencies = [ "anyhow", "async-trait", diff --git a/crates/aptos/CHANGELOG.md b/crates/aptos/CHANGELOG.md index d9587b82b733c..f309223063394 100644 --- a/crates/aptos/CHANGELOG.md +++ b/crates/aptos/CHANGELOG.md @@ -3,14 +3,18 @@ All notable changes to the Aptos CLI will be captured in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and the format set out by [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased + +## [2.3.0] - 2023/10/25 ### Added - Added `--node-api-key`. This lets you set an API key for the purpose of not being ratelimited. ### Updated - Made the local testnet exit more quickly if a service fails to start. +- Updated processor code from https://github.com/aptos-labs/aptos-indexer-processors for the local testnet to bcba94c26c8a6372056d2b69ce411c5719f98965. ### Fixed - Fixed an infrequent bug that caused startup failures for the local testnet with `--force-restart` + `--with-indexer-api` by using a Docker volume rather than a bind mount for the postgres storage. +- Fixed an issue where the CLI could not find the Docker socket with some Docker Desktop configurations. ## [2.2.2] - 2023/10/16 ### Updated diff --git a/crates/aptos/Cargo.toml b/crates/aptos/Cargo.toml index aa7c86cbf08c0..55808aeb99f60 100644 --- a/crates/aptos/Cargo.toml +++ b/crates/aptos/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "aptos" description = "Aptos tool for management of nodes and interacting with the blockchain" -version = "2.2.2" +version = "2.3.0" # Workspace inherited keys authors = { workspace = true } @@ -79,7 +79,7 @@ move-unit-test = { workspace = true, features = [ "debugging" ] } move-vm-runtime = { workspace = true, features = [ "testing" ] } once_cell = { workspace = true } poem = { workspace = true } -processor = { git = "https://github.com/aptos-labs/aptos-indexer-processors.git", rev = "a9e42dc9d80a69e9d02cd5f19c39e0f3f5cfbeb0" } +processor = { git = "https://github.com/aptos-labs/aptos-indexer-processors.git", rev = "bcba94c26c8a6372056d2b69ce411c5719f98965" } rand = { workspace = true } regex = { workspace = true } reqwest = { workspace = true } @@ -87,7 +87,7 @@ self_update = { version = "0.38.0", features = ["archive-zip", "compression-zip- serde = { workspace = true } serde_json = { workspace = true } serde_yaml = { workspace = true } -server-framework = { git = "https://github.com/aptos-labs/aptos-indexer-processors.git", rev = "a9e42dc9d80a69e9d02cd5f19c39e0f3f5cfbeb0" } +server-framework = { git = "https://github.com/aptos-labs/aptos-indexer-processors.git", rev = "bcba94c26c8a6372056d2b69ce411c5719f98965" } tempfile = { workspace = true } termcolor = { workspace = true } thiserror = { workspace = true } diff --git a/crates/aptos/src/node/local_testnet/docker.rs b/crates/aptos/src/node/local_testnet/docker.rs index 7691be6b77aab..a96a6c442727e 100644 --- a/crates/aptos/src/node/local_testnet/docker.rs +++ b/crates/aptos/src/node/local_testnet/docker.rs @@ -8,34 +8,60 @@ use bollard::{ container::{RemoveContainerOptions, StopContainerOptions}, image::CreateImageOptions, volume::{CreateVolumeOptions, RemoveVolumeOptions}, - Docker, + Docker, API_DEFAULT_VERSION, }; use futures::TryStreamExt; use std::{fs::create_dir_all, path::Path}; use tracing::{info, warn}; use version_compare::Version; -pub fn get_docker() -> Result { - let docker = Docker::connect_with_local_defaults() - .context("Docker is not available, confirm it is installed and running. On Linux you may need to use sudo.")?; - Ok(docker) -} - -pub async fn confirm_docker_available() -> Result<()> { - let docker = get_docker()?; - let info = docker - .info() - .await - .context("Docker is not available, confirm it is installed and running. On Linux you may need to use sudo.")?; - - info!("Docker Info: {:?}", info); - - let version = docker - .version() - .await - .context("Failed to get Docker version")?; +const ERROR_MESSAGE: &str = "Docker is not available, confirm it is installed and running. See https://aptos.dev/guides/local-development-network#faq for assistance."; - info!("Docker Version: {:?}", version); +/// This function returns a Docker client. Before returning, it confirms that it can +/// actually query the API and checks that the API version is sufficient. It first +/// tries to connect at the default socket location and if that fails, it tries to find +/// a socket in the user's home directory. On Windows NT it doesn't try that since +/// there no second location, there is just the one named pipe. +pub async fn get_docker() -> Result { + let docker = Docker::connect_with_local_defaults() + .context(format!("{} (init_default)", ERROR_MESSAGE))?; + + // We have to specify the type because the compiler can't figure out the error + // in the case where the system is Unix. + let out: Result<(Docker, bollard::system::Version), bollard::errors::Error> = + match docker.version().await { + Ok(version) => Ok((docker, version)), + Err(err) => { + warn!( + "Received this error trying to use default Docker socket location: {:#}", + err + ); + // Look for the socket in ~/.docker/run + // We don't have to do this if this issue gets addressed: + // https://github.com/fussybeaver/bollard/issues/345 + #[cfg(unix)] + { + let path = dirs::home_dir() + .context(format!("{} (home_dir)", ERROR_MESSAGE))? + .join(".docker") + .join("run") + .join("docker.sock"); + info!("Looking for Docker socket at {}", path.display()); + let path = path.to_str().context(format!("{} (path)", ERROR_MESSAGE))?; + let docker = Docker::connect_with_socket(path, 120, API_DEFAULT_VERSION) + .context(format!("{} (init_home)", ERROR_MESSAGE))?; + let version = docker + .version() + .await + .context(format!("{} (version_home)", ERROR_MESSAGE))?; + Ok((docker, version)) + } + // Just return the original error. + #[cfg(not(unix))] + Err(err) + }, + }; + let (docker, version) = out?; // Try to warn the user about their Docker version being too old. We don't error // out if the version is too old in case we're wrong about the minimum version @@ -68,7 +94,7 @@ pub async fn confirm_docker_available() -> Result<()> { }, } - Ok(()) + Ok(docker) } /// Delete a container. If the container doesn't exist, that's fine, just move on. @@ -78,7 +104,7 @@ pub async fn delete_container(container_name: &str) -> Result<()> { container_name ); - let docker = get_docker()?; + let docker = get_docker().await?; let options = Some(RemoveContainerOptions { force: true, @@ -106,7 +132,7 @@ pub async fn stop_container(container_name: &str) -> Result<()> { container_name ); - let docker = get_docker()?; + let docker = get_docker().await?; let options = Some(StopContainerOptions { // Timeout in seconds before we kill the container. @@ -130,7 +156,7 @@ pub async fn stop_container(container_name: &str) -> Result<()> { pub async fn pull_docker_image(image_name: &str) -> Result<()> { info!("Checking if we have to pull docker image {}", image_name); - let docker = get_docker()?; + let docker = get_docker().await?; let options = Some(CreateImageOptions { from_image: image_name, @@ -164,7 +190,7 @@ pub async fn pull_docker_image(image_name: &str) -> Result<()> { } pub async fn create_volume(volume_name: &str) -> Result<()> { - let docker = get_docker()?; + let docker = get_docker().await?; info!("Creating volume {}", volume_name); @@ -180,7 +206,7 @@ pub async fn create_volume(volume_name: &str) -> Result<()> { } pub async fn delete_volume(volume_name: &str) -> Result<()> { - let docker = get_docker()?; + let docker = get_docker().await?; info!("Removing volume {}", volume_name); diff --git a/crates/aptos/src/node/local_testnet/indexer_api.rs b/crates/aptos/src/node/local_testnet/indexer_api.rs index 6041f990be8c0..1f63bd73416ce 100644 --- a/crates/aptos/src/node/local_testnet/indexer_api.rs +++ b/crates/aptos/src/node/local_testnet/indexer_api.rs @@ -3,8 +3,8 @@ use super::{ docker::{ - confirm_docker_available, delete_container, get_docker, pull_docker_image, - setup_docker_logging, StopContainerShutdownStep, + delete_container, get_docker, pull_docker_image, setup_docker_logging, + StopContainerShutdownStep, }, health_checker::HealthChecker, traits::{PostHealthyStep, ServiceManager, ShutdownStep}, @@ -89,7 +89,7 @@ impl ServiceManager for IndexerApiManager { async fn pre_run(&self) -> Result<()> { // Confirm Docker is available. - confirm_docker_available().await?; + get_docker().await?; // Delete any existing indexer API container we find. delete_container(INDEXER_API_CONTAINER_NAME).await?; @@ -139,7 +139,7 @@ impl ServiceManager for IndexerApiManager { ..Default::default() }; - let docker = get_docker()?; + let docker = get_docker().await?; // When using Docker Desktop you can and indeed must use the magic hostname // host.docker.internal in order to access localhost on the host system from diff --git a/crates/aptos/src/node/local_testnet/postgres.rs b/crates/aptos/src/node/local_testnet/postgres.rs index 4602241190ce6..e2fc1720ad0c1 100644 --- a/crates/aptos/src/node/local_testnet/postgres.rs +++ b/crates/aptos/src/node/local_testnet/postgres.rs @@ -3,8 +3,8 @@ use super::{ docker::{ - confirm_docker_available, create_volume, delete_container, delete_volume, get_docker, - pull_docker_image, setup_docker_logging, StopContainerShutdownStep, + create_volume, delete_container, delete_volume, get_docker, pull_docker_image, + setup_docker_logging, StopContainerShutdownStep, }, health_checker::HealthChecker, traits::{ServiceManager, ShutdownStep}, @@ -163,7 +163,7 @@ impl ServiceManager for PostgresManager { } } else { // Confirm Docker is available. - confirm_docker_available().await?; + get_docker().await?; // Kill any existing container we find. delete_container(POSTGRES_CONTAINER_NAME).await?; @@ -195,7 +195,7 @@ impl ServiceManager for PostgresManager { // Let the user know where to go to see logs for postgres. setup_docker_logging(&self.test_dir, "postgres", POSTGRES_CONTAINER_NAME)?; - let docker = get_docker()?; + let docker = get_docker().await?; // If we're starting afresh, delete any existing volume. if self.force_restart { diff --git a/developer-docs-site/docs/guides/local-development-network.md b/developer-docs-site/docs/guides/local-development-network.md index 0d9492211dcbd..fc34b5187fd7f 100644 --- a/developer-docs-site/docs/guides/local-development-network.md +++ b/developer-docs-site/docs/guides/local-development-network.md @@ -231,7 +231,9 @@ Make sure the socket for connecting to Docker is present on your machine in the /var/run/docker.sock ``` -If it doesn't, you can find where it is like this: +If it doesn't, open Docker Desktop and enable `Settings -> Advanced -> Allow the default Docker socket to be used`. + +Alternatively, you can find where it is like this: ``` $ docker context inspect | grep Host "Host": "unix:///Users/dport/.docker/run/docker.sock", @@ -247,7 +249,7 @@ Alternatively, run the CLI like this to tell it where the socket is: DEFAULT_SOCKET=/Users/dport/.docker/run/docker.sock aptos node run-local-testnet --with-indexer-api ``` -Note: If you're on Mac or Windows, we recommend you use Docker Desktop rather than installing Docker via a package manager (e.g. Homebrew or Choco). +Note: As mentioned above, if you're on Mac or Windows, we recommend you use Docker Desktop rather than installing Docker via a package manager (e.g. Homebrew or Choco). ### The local network seems to hang on startup If the CLI seems to sit there and do nothing when you are using `--with-indexer-api`, consider quitting and restarting Docker. Sometimes Docker gets in a bad state. Note that Docker is only required if you are using `--with-indexer-api`. @@ -264,7 +266,7 @@ When running the CLI interactively, you can see if the network is alive by waiti Setup is complete, you can now use the local testnet! ``` -If you writing a script and would like to wait for the local network to come up, you can make a GET request to `http://127.0.0.1:8070`. At first this will return [503](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503). When it returns [200](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/200) it means all the services are ready. +If you are writing a script and would like to wait for the local network to come up, you can make a GET request to `http://127.0.0.1:8070`. At first this will return [503](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503). When it returns [200](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/200) it means all the services are ready. You can inspect the response to see which services are ready.