From d74dba444f1df2a6ecc17269ee00f953a9a7b963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20W=2E=20Urba=C5=84czyk?= Date: Sat, 15 Oct 2022 01:06:49 +0200 Subject: [PATCH] cleanups, fixing git-clean, dry-run option --- build/build/examples/s3.rs | 2 +- build/build/paths.yaml | 17 ++--- build/build/src/aws.rs | 49 +------------ build/build/src/aws/ecr.rs | 11 ++- build/build/src/aws/s3.rs | 64 +++++++++++++++++ build/build/src/changelog/check.rs | 4 +- build/build/src/context.rs | 4 +- build/build/src/engine/context.rs | 4 +- build/build/src/release.rs | 89 ++++++++++++++++++------ build/ci_utils/src/fs/tokio.rs | 15 ++++ build/ci_utils/src/github.rs | 8 +-- build/ci_utils/src/models/config.rs | 4 +- build/ci_utils/src/path/trie.rs | 7 +- build/ci_utils/src/program/command.rs | 15 ++-- build/ci_utils/src/programs/git.rs | 87 +++++++++++++++++------ build/ci_utils/src/programs/git/clean.rs | 73 +++++++++++++------ build/cli/src/arg/git_clean.rs | 3 + build/cli/src/lib.rs | 12 ++-- 18 files changed, 320 insertions(+), 148 deletions(-) create mode 100644 build/build/src/aws/s3.rs diff --git a/build/build/examples/s3.rs b/build/build/examples/s3.rs index 4aa2824ffd24..29f7d67480b8 100644 --- a/build/build/examples/s3.rs +++ b/build/build/examples/s3.rs @@ -3,7 +3,7 @@ use enso_build::prelude::*; use aws_sdk_s3::model::ObjectCannedAcl; use aws_sdk_s3::types::ByteStream; use aws_sdk_s3::Client; -use enso_build::aws::BucketContext; +use enso_build::aws::s3::BucketContext; use enso_build::aws::EDITIONS_BUCKET_NAME; diff --git a/build/build/paths.yaml b/build/build/paths.yaml index 15f7eea41ebb..1ad9cce61bce 100644 --- a/build/build/paths.yaml +++ b/build/build/paths.yaml @@ -26,15 +26,13 @@ "project-manager-bundle-": enso: dist/: - bin/: - client/: - content/: + gui/: assets/: - package.json: - preload.js: - icons/: - project-manager/: - tmp/: + ide.wasm: + index.js: + style.css: + wasm_imports.js: + # Final WASM artifacts in `dist` directory. wasm/: ? path: ide.wasm @@ -43,9 +41,6 @@ var: wasm_main_raw ? path: ide.js var: wasm_glue - init: - build-init: - build.json: distribution/: editions/: .yaml: diff --git a/build/build/src/aws.rs b/build/build/src/aws.rs index d175e0a17412..508dc118728f 100644 --- a/build/build/src/aws.rs +++ b/build/build/src/aws.rs @@ -4,11 +4,9 @@ use crate::version::BuildKind; use anyhow::Context; use aws_sdk_s3::model::ObjectCannedAcl; -use aws_sdk_s3::output::PutObjectOutput; use aws_sdk_s3::types::ByteStream; -use bytes::Buf; use ide_ci::github::Repo; -use serde::de::DeserializeOwned; +use s3::BucketContext; // ============== @@ -16,6 +14,7 @@ use serde::de::DeserializeOwned; // ============== pub mod ecr; +pub mod s3; @@ -87,50 +86,6 @@ impl Manifest { } } -#[derive(Clone, Debug)] -pub struct BucketContext { - pub client: aws_sdk_s3::Client, - pub bucket: String, - pub upload_acl: ObjectCannedAcl, - pub key_prefix: String, -} - -impl BucketContext { - pub async fn get(&self, path: &str) -> Result { - Ok(self - .client - .get_object() - .bucket(&self.bucket) - .key(format!("{}/{}", self.key_prefix, path)) - .send() - .await? - .body) - } - - pub async fn put(&self, path: &str, data: ByteStream) -> Result { - dbg!(self - .client - .put_object() - .bucket(&self.bucket) - .acl(self.upload_acl.clone()) - .key(format!("{}/{}", self.key_prefix, path)) - .body(data)) - .send() - .await - .anyhow_err() - } - - pub async fn get_yaml(&self, path: &str) -> Result { - let text = self.get(path).await?.collect().await?; - serde_yaml::from_reader(text.reader()).anyhow_err() - } - - pub async fn put_yaml(&self, path: &str, data: &impl Serialize) -> Result { - let buf = serde_yaml::to_string(data)?; - self.put(path, ByteStream::from(buf.into_bytes())).await - } -} - pub async fn update_manifest(repo_context: &Repo, edition_file: &Path) -> Result { let bucket_context = BucketContext { client: aws_sdk_s3::Client::new(&aws_config::load_from_env().await), diff --git a/build/build/src/aws/ecr.rs b/build/build/src/aws/ecr.rs index 4083b2d0db32..f55d0f4e67d1 100644 --- a/build/build/src/aws/ecr.rs +++ b/build/build/src/aws/ecr.rs @@ -12,7 +12,7 @@ pub mod runtime; /// Lookup the repository by name. -#[instrument(skip(client))] +#[instrument(skip(client), err)] pub async fn resolve_repository( client: &aws_sdk_ecr::Client, repository_name: &str, @@ -26,7 +26,7 @@ pub async fn resolve_repository( .with_context(|| format!("Cannot find repository {repository_name} in the registry.")) } -#[instrument(skip(client))] +#[instrument(skip(client), err)] pub async fn get_credentials(client: &aws_sdk_ecr::Client) -> Result { let token = client.get_authorization_token().send().await?; let auth_data = token @@ -47,6 +47,7 @@ pub async fn get_credentials(client: &aws_sdk_ecr::Client) -> Result aws_sdk_ecr::Client { + let config = aws_config::load_from_env().await; + aws_sdk_ecr::Client::new(&config) +} diff --git a/build/build/src/aws/s3.rs b/build/build/src/aws/s3.rs new file mode 100644 index 000000000000..907f3d04060e --- /dev/null +++ b/build/build/src/aws/s3.rs @@ -0,0 +1,64 @@ +use crate::prelude::*; +use aws_sdk_s3::model::ObjectCannedAcl; +use aws_sdk_s3::output::PutObjectOutput; +use aws_sdk_s3::types::ByteStream; +use bytes::Buf; + +pub async fn client_from_env() -> aws_sdk_s3::Client { + aws_sdk_s3::Client::new(&aws_config::load_from_env().await) +} + +#[derive(Clone, Derivative)] +#[derivative(Debug)] +pub struct BucketContext { + #[derivative(Debug = "ignore")] + pub client: aws_sdk_s3::Client, + pub bucket: String, + pub upload_acl: ObjectCannedAcl, + pub key_prefix: String, +} + +impl BucketContext {} + +impl BucketContext { + pub async fn get(&self, path: &str) -> Result { + Ok(self + .client + .get_object() + .bucket(&self.bucket) + .key(format!("{}/{}", self.key_prefix, path)) + .send() + .await? + .body) + } + + pub async fn put(&self, path: &str, data: ByteStream) -> Result { + dbg!(self + .client + .put_object() + .bucket(&self.bucket) + .acl(self.upload_acl.clone()) + .key(format!("{}/{}", self.key_prefix, path)) + .body(data)) + .send() + .await + .anyhow_err() + } + + #[instrument] + pub async fn put_file(&self, path: &Path) -> Result { + let stream = ByteStream::from_path(path).await?; + let path = path.file_name().with_context(|| format!("Path {:?} has no file name", path))?; + self.put(path.as_str(), stream).await + } + + pub async fn get_yaml(&self, path: &str) -> Result { + let text = self.get(path).await?.collect().await?; + serde_yaml::from_reader(text.reader()).anyhow_err() + } + + pub async fn put_yaml(&self, path: &str, data: &impl Serialize) -> Result { + let buf = serde_yaml::to_string(data)?; + self.put(path, ByteStream::from(buf.into_bytes())).await + } +} diff --git a/build/build/src/changelog/check.rs b/build/build/src/changelog/check.rs index f4d1c2f06171..957e4332c5a5 100644 --- a/build/build/src/changelog/check.rs +++ b/build/build/src/changelog/check.rs @@ -4,7 +4,7 @@ use crate::ci::labels::NO_CHANGELOG_CHECK; use crate::paths::generated::RepoRoot; use ide_ci::actions::workflow::MessageLevel; -use ide_ci::programs::Git; +use ide_ci::programs::git; @@ -49,7 +49,7 @@ pub async fn check(repo_path: RepoRoot, context: ide_ci::actions::Context) -> Re let repository = context.payload.repository.context("Missing repository information.")?; let default_branch = repository.default_branch.context("Missing default branch information.")?; - let git = Git::new(&repo_path).await?; + let git = git::Context::new(&repo_path).await?; let remote_base = format!("{REMOTE_NAME}/{default_branch}"); let files_changed = git.diff_against(remote_base).await?; debug!("Files changed: {files_changed:#?}."); diff --git a/build/build/src/context.rs b/build/build/src/context.rs index 5fb52093364d..ede9c8cf2707 100644 --- a/build/build/src/context.rs +++ b/build/build/src/context.rs @@ -4,7 +4,7 @@ use crate::paths::TargetTriple; use derivative::Derivative; use ide_ci::github::Repo; -use ide_ci::programs::Git; +use ide_ci::programs::git; use octocrab::models::repos::Release; use octocrab::models::ReleaseId; @@ -34,7 +34,7 @@ impl BuildContext { async move { match ide_ci::actions::env::GITHUB_SHA.get() { Ok(commit) => Ok(commit), - Err(_e) => Git::new(root).await?.head_hash().await, + Err(_e) => git::Context::new(root).await?.head_hash().await, } } .boxed() diff --git a/build/build/src/engine/context.rs b/build/build/src/engine/context.rs index 429231c9bc35..da0d1a988658 100644 --- a/build/build/src/engine/context.rs +++ b/build/build/src/engine/context.rs @@ -88,7 +88,7 @@ impl RunContext { Sbt.require_present().await?; // Other programs. - ide_ci::programs::Git::new_current().await?.require_present().await?; + ide_ci::programs::Git.require_present().await?; ide_ci::programs::Go.require_present().await?; ide_ci::programs::Cargo.require_present().await?; ide_ci::programs::Node.require_present().await?; @@ -97,7 +97,7 @@ impl RunContext { let prepare_simple_library_server = { if self.config.test_scala { let simple_server_path = &self.paths.repo_root.tools.simple_library_server; - ide_ci::programs::Git::new(simple_server_path) + ide_ci::programs::git::Context::new(simple_server_path) .await? .cmd()? .clean() diff --git a/build/build/src/release.rs b/build/build/src/release.rs index 44125a6c4778..6bcaf31b958b 100644 --- a/build/build/src/release.rs +++ b/build/build/src/release.rs @@ -11,9 +11,10 @@ use ide_ci::programs::Docker; use octocrab::models::repos::Release; use tempfile::tempdir; +const BUCKET_FOR_GUI: &str = "ensocdn"; -pub async fn create_release(context: &BuildContext) -> Result { +pub async fn draft_a_new_release(context: &BuildContext) -> Result { let versions = &context.triple.versions; let commit = ide_ci::actions::env::GITHUB_SHA.get()?; @@ -101,18 +102,17 @@ pub async fn get_engine_package( Ok(engine_package) } -/// Download the Enso Engine distribution from the GitHub release and build RUntime Docker image +/// Download the Enso Engine distribution from the GitHub release and build Runtime Docker image /// from it. pub async fn generate_runtime_image( context: &BuildContext, tag: impl Into, ) -> Result { - let octocrab = &context.octocrab; // Our runtime images always target Linux. let linux_triple = TargetTriple { os: OS::Linux, ..context.triple.clone() }; let temp_for_extraction = tempdir()?; let engine_package = get_engine_package( - octocrab, + &context.octocrab, &context.remote_repo, temp_for_extraction.path(), &linux_triple, @@ -126,29 +126,74 @@ pub async fn generate_runtime_image( .await } +/// Perform deploy of the backend to the ECR. +/// +/// Downloads the Engine package from the release, builds the runtime image from it and pushes it +/// to our ECR. pub async fn deploy_to_ecr(context: &BuildContext, repository: String) -> Result { - let config = &aws_config::load_from_env().await; - let client = aws_sdk_ecr::Client::new(config); + let client = crate::aws::ecr::client_from_env().await; let repository_uri = crate::aws::ecr::get_repository_uri(&client, &repository).await?; let tag = format!("{}:{}", repository_uri, context.triple.versions.version); // We don't care about the image ID, we will refer to it by the tag. - let _image_id = generate_runtime_image(context, &repository).await?; + let _image_id = generate_runtime_image(context, &tag).await?; let credentials = crate::aws::ecr::get_credentials(&client).await?; Docker.while_logged_in(credentials, || async move { Docker.push(&tag).await }).await?; Ok(()) } -// -// #[cfg(test)] -// mod tests { -// use super::*; -// use crate::setup_octocrab; -// -// #[tokio::test] -// #[ignore] -// async fn dispatch_workflow() -> Result { -// let octo = setup_octocrab().await?; -// let version = Version::parse("2022.1.1-nightly.2022-10-11.1")?; -// dispatch_cloud_image_build_action(&octo, &version).await?; -// Ok(()) -// } -// } + +/// Upload GUI to the cloud (AWS S3). +pub async fn upload_gui_to_cloud( + assets: &crate::paths::generated::RepoRootDistGuiAssets, + version: &Version, +) -> Result { + let path = format!("ide/{version}"); + let bucket = crate::aws::s3::BucketContext { + bucket: BUCKET_FOR_GUI.into(), + upload_acl: aws_sdk_s3::model::ObjectCannedAcl::PublicRead, + client: crate::aws::s3::client_from_env().await, + key_prefix: path, + }; + + let files_to_upload = [ + assets.ide_wasm.as_path(), + assets.index_js.as_path(), + assets.style_css.as_path(), + assets.wasm_imports_js.as_path(), + ]; + + for file in files_to_upload.iter() { + bucket.put_file(file).await?; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[tokio::test] + async fn upload_gui() -> Result { + setup_logging()?; + let assets = crate::paths::generated::RepoRootDistGuiAssets::new_root( + r"H:\NBO\enso4\dist\gui\assets", + ); + let version = "2022.1.1-dev.provisional.test.1".parse2()?; + upload_gui_to_cloud(&assets, &version).await?; + + let body = json!({ + "version_number": version.to_string(), + "version_type": "Ide" + }); + let response = reqwest::Client::new() + .post("https://uhso47ta0i.execute-api.eu-west-1.amazonaws.com/versions") + .header("x-enso-organization-id", "org-2BqGX0q2yCdONdmx3Om1MVZzmv3") + .header("Content-Type", "application/json") + .json(&body) + .send() + .await?; + trace!("{:?}", response); + Ok(()) + } +} diff --git a/build/ci_utils/src/fs/tokio.rs b/build/ci_utils/src/fs/tokio.rs index 544889a58776..4ff9eca729b1 100644 --- a/build/ci_utils/src/fs/tokio.rs +++ b/build/ci_utils/src/fs/tokio.rs @@ -50,6 +50,12 @@ pub async fn create_parent_dir_if_missing(path: impl AsRef) -> Result, contents: impl AsRef<[u8]>) -> Result { + create_parent_dir_if_missing(&path).await?; + crate::fs::wrappers::tokio::write(&path, &contents).await.anyhow_err() +} + pub async fn copy_to_file( mut content: impl AsyncRead + Unpin, output_path: impl AsRef, @@ -71,6 +77,15 @@ pub async fn remove_dir_if_exists(path: impl AsRef) -> Result { } } +pub async fn perhaps_remove_dir_if_exists(dry_run: bool, path: impl AsRef) -> Result { + if dry_run { + info!("Would remove directory {}.", path.as_ref().display()); + Ok(()) + } else { + remove_dir_if_exists(path).await + } +} + /// Recreate directory, so it exists and is empty. pub async fn reset_dir(path: impl AsRef) -> Result { let path = path.as_ref(); diff --git a/build/ci_utils/src/github.rs b/build/ci_utils/src/github.rs index 87a88a5b27fc..d74f392aaa08 100644 --- a/build/ci_utils/src/github.rs +++ b/build/ci_utils/src/github.rs @@ -229,7 +229,7 @@ pub trait IsRepo: Display { } #[async_trait] -pub trait OrganizationPointer { +pub trait IsOrganization { /// Organization name. fn name(&self) -> &str; @@ -252,7 +252,7 @@ pub trait OrganizationPointer { /// Get the biggest asset containing given text. #[instrument(skip(release), fields(id = %release.id, url = %release.url), err)] -pub fn find_asset_by_text<'a>(release: &'a Release, text: &str) -> anyhow::Result<&'a Asset> { +pub fn find_asset_by_text<'a>(release: &'a Release, text: &str) -> Result<&'a Asset> { release .assets .iter() @@ -266,7 +266,7 @@ pub fn find_asset_by_text<'a>(release: &'a Release, text: &str) -> anyhow::Resul /// Get the biggest asset containing given text. #[instrument(skip(release), fields(id = %release.id, url = %release.url), ret(Display), err)] -pub fn find_asset_url_by_text<'a>(release: &'a Release, text: &str) -> anyhow::Result<&'a Url> { +pub fn find_asset_url_by_text<'a>(release: &'a Release, text: &str) -> Result<&'a Url> { let matching_asset = find_asset_by_text(release, text)?; Ok(&matching_asset.browser_download_url) } @@ -275,7 +275,7 @@ pub fn find_asset_url_by_text<'a>(release: &'a Release, text: &str) -> anyhow::R /// /// Octocrab client does not need to bo authorized with a PAT for this. However, being authorized /// will help with GitHub API query rate limits. -pub async fn latest_runner_url(octocrab: &Octocrab, os: OS) -> anyhow::Result { +pub async fn latest_runner_url(octocrab: &Octocrab, os: OS) -> Result { let latest_release = octocrab.repos("actions", "runner").releases().get_latest().await?; let os_name = match os { diff --git a/build/ci_utils/src/models/config.rs b/build/ci_utils/src/models/config.rs index e01984a1061d..0ce8002b9ef5 100644 --- a/build/ci_utils/src/models/config.rs +++ b/build/ci_utils/src/models/config.rs @@ -2,8 +2,8 @@ use crate::prelude::*; +use crate::github::IsOrganization; use crate::github::IsRepo; -use crate::github::OrganizationPointer; use crate::github::Repo; use crate::serde::regex_vec; use crate::serde::single_or_sequence; @@ -75,7 +75,7 @@ pub struct OrganizationContext { pub name: String, } -impl OrganizationPointer for OrganizationContext { +impl IsOrganization for OrganizationContext { fn name(&self) -> &str { &self.name } diff --git a/build/ci_utils/src/path/trie.rs b/build/ci_utils/src/path/trie.rs index 91097136d8c4..c865407b40e2 100644 --- a/build/ci_utils/src/path/trie.rs +++ b/build/ci_utils/src/path/trie.rs @@ -1,14 +1,18 @@ use crate::prelude::*; - +/// A trie data structure, where each node represents a single fs path component. +/// +/// As such, a trie defines a set of fs paths (each being defined by a path within the trie). #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct Trie<'a> { pub children: HashMap, Trie<'a>>, + /// Number of paths that end in this node. pub count: usize, } impl<'a> Trie<'a> { + /// Insert a path into the trie. pub fn insert(&mut self, path: &'a Path) { let mut current = self; for component in path.components() { @@ -17,6 +21,7 @@ impl<'a> Trie<'a> { current.count += 1; } + /// Is this node a leaf? pub fn is_leaf(&self) -> bool { self.children.is_empty() } diff --git a/build/ci_utils/src/program/command.rs b/build/ci_utils/src/program/command.rs index 59c52b84bfea..011531b4d869 100644 --- a/build/ci_utils/src/program/command.rs +++ b/build/ci_utils/src/program/command.rs @@ -232,6 +232,15 @@ pub trait IsCommandWrapper { // let fut = self.borrow_mut_command().output(); // async move { fut.await.anyhow_err() }.boxed() // } + + + + fn with_current_dir(self, dir: impl AsRef) -> Self + where Self: Sized { + let mut this = self; + this.current_dir(dir); + this + } } impl> IsCommandWrapper for T { @@ -434,12 +443,6 @@ impl Command { this.stderr(stderr); this } - - pub fn with_current_dir(self, dir: impl AsRef) -> Self { - let mut this = self; - this.current_dir(dir); - this - } } pub fn spawn_log_processor( diff --git a/build/ci_utils/src/programs/git.rs b/build/ci_utils/src/programs/git.rs index 217f9b45e817..ad0cf3a1785b 100644 --- a/build/ci_utils/src/programs/git.rs +++ b/build/ci_utils/src/programs/git.rs @@ -11,39 +11,79 @@ pub mod clean; pub use clean::Clean; +#[derive(Clone, Copy, Debug)] +pub struct Git; +impl Program for Git { + type Command = GitCommand; + fn executable_name(&self) -> &'static str { + "git" + } +} + +impl Git { + /// Create a new, empty git repository in the given directory. + pub fn init(&self, path: impl AsRef) -> Result { + let mut cmd = self.cmd()?; + cmd.arg(&Command::Init); + cmd.current_dir(path); + Ok(cmd) + } +} +/// The wrapper over `Git` program invocation context. +/// +/// It is stateful (knowing both repository root and current directory locations), as they both are +/// needed to properly handle relative paths. #[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct Git { +pub struct Context { /// The path to the repository root above the `working_dir`. /// /// Many paths that git returns are relative to the repository root. - repo_path: PathBuf, + repository_root: PathBuf, /// Directory in which commands will be invoked. /// It might not be the repository root and it makes difference for many commands. - working_dir: PathBuf, + working_dir: PathBuf, } -impl Program for Git { - type Command = GitCommand; - fn executable_name(&self) -> &'static str { - "git" +impl Context { + /// Initialize a new command invoking git. + pub fn cmd(&self) -> Result { + Ok(Git.cmd()?.with_current_dir(&self.working_dir)) } - fn current_directory(&self) -> Option { - Some(self.working_dir.clone()) + + /// Create a wrapper with explicitly set repository root and working directory. + /// + /// The caller is responsible for ensuring that the `working_dir` is a subdirectory of the + /// `repository_root`. + pub async fn new_unchecked( + repository_root: impl AsRef, + working_dir: impl AsRef, + ) -> Self { + Self { + repository_root: repository_root.as_ref().to_path_buf(), + working_dir: working_dir.as_ref().to_path_buf(), + } } -} -impl Git { - pub async fn new(repo_path: impl Into) -> Result { - let repo_path = repo_path.into(); - let temp_git = Git { working_dir: repo_path.clone(), repo_path }; - let repo_path = temp_git.repository_root().await?; - Ok(Git { repo_path, working_dir: temp_git.working_dir }) + /// Create a `Git` invocation context within a given directory. + /// + /// The `working_dir` is the directory in which git commands will be invoked. It is expected to + /// be a part of some git repository. + pub async fn new(working_directory: impl Into) -> Result { + let working_directory = working_directory.into(); + // Faux `Git` instance to get the repository root. + // TODO: should be nicer, likely instance should be separate from program. + let temp_git = Context { + working_dir: working_directory.clone(), + repository_root: working_directory, + }; + let repo_root = temp_git.repository_root().await?; + Ok(Context { repository_root: repo_root, working_dir: temp_git.working_dir }) } pub async fn new_current() -> Result { - Git::new(crate::env::current_dir()?).await + Context::new(crate::env::current_dir()?).await } pub async fn head_hash(&self) -> Result { @@ -53,7 +93,7 @@ impl Git { /// List of files that are different than the compared commit. #[context("Failed to list files that are different than {}.", compare_against.as_ref())] pub async fn diff_against(&self, compare_against: impl AsRef) -> Result> { - let root = self.repo_path.as_path(); + let root = self.repository_root.as_path(); Ok(self .cmd()? .args(["diff", "--name-only", compare_against.as_ref()]) @@ -94,13 +134,20 @@ impl GitCommand { #[derive(Clone, Copy, Debug)] pub enum Command { + /// Remove untracked files from the working tree. Clean, + /// Show changes between commits, commit and working tree, etc. + Diff, + /// Create an empty Git repository or reinitialize an existing one. + Init, } impl AsRef for Command { fn as_ref(&self) -> &OsStr { match self { Command::Clean => OsStr::new("clean"), + Command::Diff => OsStr::new("diff"), + Command::Init => OsStr::new("init"), } } } @@ -112,7 +159,7 @@ mod tests { #[tokio::test] #[ignore] async fn repo_root() -> Result { - let git = Git::new(".").await?; + let git = Context::new(".").await?; let diff = git.repository_root().await?; println!("{:?}", diff); Ok(()) @@ -121,7 +168,7 @@ mod tests { #[tokio::test] #[ignore] async fn call_diff() -> Result { - let git = Git::new(".").await?; + let git = Context::new(".").await?; let diff = git.diff_against("origin/develop").await?; println!("{:?}", diff); Ok(()) diff --git a/build/ci_utils/src/programs/git/clean.rs b/build/ci_utils/src/programs/git/clean.rs index ed71856599f4..d7daf084e52f 100644 --- a/build/ci_utils/src/programs/git/clean.rs +++ b/build/ci_utils/src/programs/git/clean.rs @@ -2,7 +2,7 @@ use crate::prelude::*; use crate::path::trie::Trie; use crate::program::command::Manipulator; -use crate::programs::Git; +use crate::programs::git; use std::path::Component; @@ -14,7 +14,7 @@ pub struct DirectoryToClear<'a> { pub trie: &'a Trie<'a>, } -/// Run ``git clean -xfd`` but preserve the given paths. +/// Run `git clean -xfd` but preserve the given paths. /// /// This may involve multiple git clean calls on different subtrees. /// Given paths can be either absolute or relative. If relative, they are relative to the @@ -22,6 +22,7 @@ pub struct DirectoryToClear<'a> { pub async fn clean_except_for( repo_root: impl AsRef, paths: impl IntoIterator>, + dry_run: bool, ) -> Result { let root = repo_root.as_ref().canonicalize()?; @@ -40,26 +41,16 @@ pub async fn clean_except_for( }) .collect_vec(); - let trie = Trie::from_iter(relative_exclusions.iter()); - - let mut directories_to_clear = vec![DirectoryToClear { prefix: vec![], trie: &trie }]; - while let Some(DirectoryToClear { prefix, trie }) = directories_to_clear.pop() { - let current_dir = root.join_iter(&prefix); - let exclusions_in_current_dir = - trie.children.keys().map(|c| Clean::Exclude(c.as_os_str().to_string_lossy().into())); - let git = Git::new(¤t_dir).await?; - git.cmd()?.clean().apply_iter(exclusions_in_current_dir).run_ok().await?; - - for (child_name, child_trie) in trie.children.iter() { - if !child_trie.is_leaf() { - let mut prefix = prefix.clone(); - prefix.push(*child_name); - directories_to_clear.push(DirectoryToClear { prefix, trie: child_trie }); - } - } - } + let exclusions = relative_exclusions.into_iter().map(Clean::exclude).collect_vec(); - Ok(()) + git::Context::new(root) + .await? + .cmd()? + .nice_clean() + .apply_iter(exclusions) + .apply_opt(dry_run.then_some(&Clean::DryRun)) + .run_ok() + .await } #[derive(Clone, Debug)] @@ -97,6 +88,17 @@ pub enum Clean { OnlyIgnored, } +impl Clean { + pub fn exclude(path: impl AsRef) -> Self { + let mut ret = String::new(); + for component in path.as_ref().components() { + ret.push('/'); + ret.push_str(&component.as_os_str().to_string_lossy()); + } + Clean::Exclude(ret) + } +} + impl Manipulator for Clean { fn apply(&self, command: &mut C) { // fn apply<'a, C: IsCommandWrapper + ?Sized>(&self, c: &'a mut C) -> &'a mut C { @@ -112,3 +114,32 @@ impl Manipulator for Clean { command.args(args); } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::programs::Git; + + #[tokio::test] + async fn test_cleaning() -> Result { + setup_logging()?; + let dir = PathBuf::from(r"C:\temp\test_cleaning"); + crate::fs::tokio::reset_dir(&dir).await?; + Git.init(&dir)?.run_ok().await?; + + let foo = dir.join("foo"); + let foo_target = foo.join("target"); + crate::fs::tokio::write(&foo_target, "target in foo").await?; + + let target = dir.join("target"); + let target_foo = target.join("foo"); + crate::fs::tokio::write(&target_foo, "foo in target").await?; + + let git = git::Context::new(&dir).await?; + clean_except_for(&dir, vec!["target/foo"], false).await?; + + + + Ok(()) + } +} diff --git a/build/cli/src/arg/git_clean.rs b/build/cli/src/arg/git_clean.rs index 5f3536a146c5..55188078015c 100644 --- a/build/cli/src/arg/git_clean.rs +++ b/build/cli/src/arg/git_clean.rs @@ -4,6 +4,9 @@ use crate::prelude::*; #[derive(Clone, Copy, Debug, Default, clap::Args)] pub struct Options { + /// Do not perform the action, just print what would be deleted. + #[clap(long)] + pub dry_run: bool, /// Clean also the build script's cache (located in the user's local application data subtree). #[clap(long)] pub cache: bool, diff --git a/build/cli/src/lib.rs b/build/cli/src/lib.rs index e3a361bbe675..d0400bbcffbe 100644 --- a/build/cli/src/lib.rs +++ b/build/cli/src/lib.rs @@ -801,15 +801,17 @@ pub async fn main_internal(config: enso_build::config::Config) -> Result { Target::Ide(ide) => ctx.handle_ide(ide).await?, // TODO: consider if out-of-source ./dist should be removed Target::GitClean(options) => { + let crate::arg::git_clean::Options { dry_run, cache, build_script } = options; let mut exclusions = vec![".idea"]; - if !options.build_script { + if !build_script { exclusions.push("target/enso-build"); } - let git_clean = clean::clean_except_for(&ctx.repo_root, exclusions); + let git_clean = clean::clean_except_for(&ctx.repo_root, exclusions, dry_run); let clean_cache = async { - if options.cache { - ide_ci::fs::tokio::remove_dir_if_exists(ctx.cache.path()).await?; + if cache { + ide_ci::fs::tokio::perhaps_remove_dir_if_exists(dry_run, ctx.cache.path()) + .await?; } Result::Ok(()) }; @@ -851,7 +853,7 @@ pub async fn main_internal(config: enso_build::config::Config) -> Result { } Target::Release(release) => match release.action { Action::CreateDraft => { - enso_build::release::create_release(&ctx).await?; + enso_build::release::draft_a_new_release(&ctx).await?; } Action::DeployToEcr(args) => { enso_build::release::deploy_to_ecr(&ctx, args.ecr_repository).await?;