cleanups, fixing git-clean, dry-run option

build/build/examples/s3.rs

@@ -3,7 +3,7 @@ use enso_build::prelude::*;
 use aws_sdk_s3::model::ObjectCannedAcl;
 use aws_sdk_s3::types::ByteStream;
 use aws_sdk_s3::Client;
-use enso_build::aws::BucketContext;
+use enso_build::aws::s3::BucketContext;
 use enso_build::aws::EDITIONS_BUCKET_NAME;
 
 

build/build/paths.yaml

@@ -26,15 +26,13 @@
     "project-manager-bundle-<triple>":
       enso:
   dist/:
-    bin/:
-    client/:
-    content/:
+    gui/:
       assets/:
-      package.json:
-      preload.js:
-    icons/:
-    project-manager/:
-    tmp/:
+        ide.wasm:
+        index.js:
+        style.css:
+        wasm_imports.js:
+
     # Final WASM artifacts in `dist` directory.
     wasm/:
       ? path: ide.wasm
@@ -43,9 +41,6 @@
         var: wasm_main_raw
       ? path: ide.js
         var: wasm_glue
-    init:
-    build-init:
-    build.json:
   distribution/:
     editions/:
       <edition>.yaml:

build/build/src/aws.rs

@@ -4,18 +4,17 @@ use crate::version::BuildKind;
 
 use anyhow::Context;
 use aws_sdk_s3::model::ObjectCannedAcl;
-use aws_sdk_s3::output::PutObjectOutput;
 use aws_sdk_s3::types::ByteStream;
-use bytes::Buf;
 use ide_ci::github::Repo;
-use serde::de::DeserializeOwned;
+use s3::BucketContext;
 
 
 // ==============
 // === Export ===
 // ==============
 
 pub mod ecr;
+pub mod s3;
 
 
 
@@ -87,50 +86,6 @@ impl Manifest {
     }
 }
 
-#[derive(Clone, Debug)]
-pub struct BucketContext {
-    pub client:     aws_sdk_s3::Client,
-    pub bucket:     String,
-    pub upload_acl: ObjectCannedAcl,
-    pub key_prefix: String,
-}
-
-impl BucketContext {
-    pub async fn get(&self, path: &str) -> Result<ByteStream> {
-        Ok(self
-            .client
-            .get_object()
-            .bucket(&self.bucket)
-            .key(format!("{}/{}", self.key_prefix, path))
-            .send()
-            .await?
-            .body)
-    }
-
-    pub async fn put(&self, path: &str, data: ByteStream) -> Result<PutObjectOutput> {
-        dbg!(self
-            .client
-            .put_object()
-            .bucket(&self.bucket)
-            .acl(self.upload_acl.clone())
-            .key(format!("{}/{}", self.key_prefix, path))
-            .body(data))
-        .send()
-        .await
-        .anyhow_err()
-    }
-
-    pub async fn get_yaml<T: DeserializeOwned>(&self, path: &str) -> Result<T> {
-        let text = self.get(path).await?.collect().await?;
-        serde_yaml::from_reader(text.reader()).anyhow_err()
-    }
-
-    pub async fn put_yaml(&self, path: &str, data: &impl Serialize) -> Result<PutObjectOutput> {
-        let buf = serde_yaml::to_string(data)?;
-        self.put(path, ByteStream::from(buf.into_bytes())).await
-    }
-}
-
 pub async fn update_manifest(repo_context: &Repo, edition_file: &Path) -> Result {
     let bucket_context = BucketContext {
         client:     aws_sdk_s3::Client::new(&aws_config::load_from_env().await),

build/build/src/aws/ecr.rs

@@ -12,7 +12,7 @@ pub mod runtime;
 
 
 /// Lookup the repository by name.
-#[instrument(skip(client))]
+#[instrument(skip(client), err)]
 pub async fn resolve_repository(
     client: &aws_sdk_ecr::Client,
     repository_name: &str,
@@ -26,7 +26,7 @@ pub async fn resolve_repository(
         .with_context(|| format!("Cannot find repository {repository_name} in the registry."))
 }
 
-#[instrument(skip(client))]
+#[instrument(skip(client), err)]
 pub async fn get_credentials(client: &aws_sdk_ecr::Client) -> Result<docker::Credentials> {
     let token = client.get_authorization_token().send().await?;
     let auth_data = token
@@ -47,6 +47,7 @@ pub async fn get_credentials(client: &aws_sdk_ecr::Client) -> Result<docker::Cre
     Ok(docker::Credentials::new(*username, *password, proxy))
 }
 
+/// Get a repository URI, that can be used to refer to the repository in the Docker commands.
 #[instrument(skip(client), ret)]
 pub async fn get_repository_uri(
     client: &aws_sdk_ecr::Client,
@@ -56,3 +57,9 @@ pub async fn get_repository_uri(
     let repository_uri = repository.repository_uri().context("Missing repository URI.")?;
     Ok(repository_uri.into())
 }
+
+/// Create a new ECR client, configured using the environment variables.
+pub async fn client_from_env() -> aws_sdk_ecr::Client {
+    let config = aws_config::load_from_env().await;
+    aws_sdk_ecr::Client::new(&config)
+}

build/build/src/aws/s3.rs

@@ -0,0 +1,64 @@
+use crate::prelude::*;
+use aws_sdk_s3::model::ObjectCannedAcl;
+use aws_sdk_s3::output::PutObjectOutput;
+use aws_sdk_s3::types::ByteStream;
+use bytes::Buf;
+
+pub async fn client_from_env() -> aws_sdk_s3::Client {
+    aws_sdk_s3::Client::new(&aws_config::load_from_env().await)
+}
+
+#[derive(Clone, Derivative)]
+#[derivative(Debug)]
+pub struct BucketContext {
+    #[derivative(Debug = "ignore")]
+    pub client:     aws_sdk_s3::Client,
+    pub bucket:     String,
+    pub upload_acl: ObjectCannedAcl,
+    pub key_prefix: String,
+}
+
+impl BucketContext {}
+
+impl BucketContext {
+    pub async fn get(&self, path: &str) -> Result<ByteStream> {
+        Ok(self
+            .client
+            .get_object()
+            .bucket(&self.bucket)
+            .key(format!("{}/{}", self.key_prefix, path))
+            .send()
+            .await?
+            .body)
+    }
+
+    pub async fn put(&self, path: &str, data: ByteStream) -> Result<PutObjectOutput> {
+        dbg!(self
+            .client
+            .put_object()
+            .bucket(&self.bucket)
+            .acl(self.upload_acl.clone())
+            .key(format!("{}/{}", self.key_prefix, path))
+            .body(data))
+        .send()
+        .await
+        .anyhow_err()
+    }
+
+    #[instrument]
+    pub async fn put_file(&self, path: &Path) -> Result<PutObjectOutput> {
+        let stream = ByteStream::from_path(path).await?;
+        let path = path.file_name().with_context(|| format!("Path {:?} has no file name", path))?;
+        self.put(path.as_str(), stream).await
+    }
+
+    pub async fn get_yaml<T: DeserializeOwned>(&self, path: &str) -> Result<T> {
+        let text = self.get(path).await?.collect().await?;
+        serde_yaml::from_reader(text.reader()).anyhow_err()
+    }
+
+    pub async fn put_yaml(&self, path: &str, data: &impl Serialize) -> Result<PutObjectOutput> {
+        let buf = serde_yaml::to_string(data)?;
+        self.put(path, ByteStream::from(buf.into_bytes())).await
+    }
+}

build/build/src/changelog/check.rs

@@ -4,7 +4,7 @@ use crate::ci::labels::NO_CHANGELOG_CHECK;
 use crate::paths::generated::RepoRoot;
 
 use ide_ci::actions::workflow::MessageLevel;
-use ide_ci::programs::Git;
+use ide_ci::programs::git;
 
 
 
@@ -49,7 +49,7 @@ pub async fn check(repo_path: RepoRoot, context: ide_ci::actions::Context) -> Re
     let repository = context.payload.repository.context("Missing repository information.")?;
     let default_branch =
         repository.default_branch.context("Missing default branch information.")?;
-    let git = Git::new(&repo_path).await?;
+    let git = git::Context::new(&repo_path).await?;
     let remote_base = format!("{REMOTE_NAME}/{default_branch}");
     let files_changed = git.diff_against(remote_base).await?;
     debug!("Files changed: {files_changed:#?}.");

build/build/src/context.rs

@@ -4,7 +4,7 @@ use crate::paths::TargetTriple;
 
 use derivative::Derivative;
 use ide_ci::github::Repo;
-use ide_ci::programs::Git;
+use ide_ci::programs::git;
 use octocrab::models::repos::Release;
 use octocrab::models::ReleaseId;
 
@@ -34,7 +34,7 @@ impl BuildContext {
         async move {
             match ide_ci::actions::env::GITHUB_SHA.get() {
                 Ok(commit) => Ok(commit),
-                Err(_e) => Git::new(root).await?.head_hash().await,
+                Err(_e) => git::Context::new(root).await?.head_hash().await,
             }
         }
         .boxed()

build/build/src/engine/context.rs

@@ -88,7 +88,7 @@ impl RunContext {
         Sbt.require_present().await?;
 
         // Other programs.
-        ide_ci::programs::Git::new_current().await?.require_present().await?;
+        ide_ci::programs::Git.require_present().await?;
         ide_ci::programs::Go.require_present().await?;
         ide_ci::programs::Cargo.require_present().await?;
         ide_ci::programs::Node.require_present().await?;
@@ -97,7 +97,7 @@ impl RunContext {
         let prepare_simple_library_server = {
             if self.config.test_scala {
                 let simple_server_path = &self.paths.repo_root.tools.simple_library_server;
-                ide_ci::programs::Git::new(simple_server_path)
+                ide_ci::programs::git::Context::new(simple_server_path)
                     .await?
                     .cmd()?
                     .clean()

build/build/src/release.rs

@@ -11,9 +11,10 @@ use ide_ci::programs::Docker;
 use octocrab::models::repos::Release;
 use tempfile::tempdir;
 
+const BUCKET_FOR_GUI: &str = "ensocdn";
 
 
-pub async fn create_release(context: &BuildContext) -> Result<Release> {
+pub async fn draft_a_new_release(context: &BuildContext) -> Result<Release> {
     let versions = &context.triple.versions;
     let commit = ide_ci::actions::env::GITHUB_SHA.get()?;
 
@@ -101,18 +102,17 @@ pub async fn get_engine_package(
     Ok(engine_package)
 }
 
-/// Download the Enso Engine distribution from the GitHub release and build RUntime Docker image
+/// Download the Enso Engine distribution from the GitHub release and build Runtime Docker image
 /// from it.
 pub async fn generate_runtime_image(
     context: &BuildContext,
     tag: impl Into<String>,
 ) -> Result<ide_ci::programs::docker::ImageId> {
-    let octocrab = &context.octocrab;
     // Our runtime images always target Linux.
     let linux_triple = TargetTriple { os: OS::Linux, ..context.triple.clone() };
     let temp_for_extraction = tempdir()?;
     let engine_package = get_engine_package(
-        octocrab,
+        &context.octocrab,
         &context.remote_repo,
         temp_for_extraction.path(),
         &linux_triple,
@@ -126,29 +126,74 @@ pub async fn generate_runtime_image(
     .await
 }
 
+/// Perform deploy of the backend to the ECR.
+///
+/// Downloads the Engine package from the release, builds the runtime image from it and pushes it
+/// to our ECR.
 pub async fn deploy_to_ecr(context: &BuildContext, repository: String) -> Result {
-    let config = &aws_config::load_from_env().await;
-    let client = aws_sdk_ecr::Client::new(config);
+    let client = crate::aws::ecr::client_from_env().await;
     let repository_uri = crate::aws::ecr::get_repository_uri(&client, &repository).await?;
     let tag = format!("{}:{}", repository_uri, context.triple.versions.version);
     // We don't care about the image ID, we will refer to it by the tag.
-    let _image_id = generate_runtime_image(context, &repository).await?;
+    let _image_id = generate_runtime_image(context, &tag).await?;
     let credentials = crate::aws::ecr::get_credentials(&client).await?;
     Docker.while_logged_in(credentials, || async move { Docker.push(&tag).await }).await?;
     Ok(())
 }
-//
-// #[cfg(test)]
-// mod tests {
-//     use super::*;
-//     use crate::setup_octocrab;
-//
-//     #[tokio::test]
-//     #[ignore]
-//     async fn dispatch_workflow() -> Result {
-//         let octo = setup_octocrab().await?;
-//         let version = Version::parse("2022.1.1-nightly.2022-10-11.1")?;
-//         dispatch_cloud_image_build_action(&octo, &version).await?;
-//         Ok(())
-//     }
-// }
+
+/// Upload GUI to the cloud (AWS S3).
+pub async fn upload_gui_to_cloud(
+    assets: &crate::paths::generated::RepoRootDistGuiAssets,
+    version: &Version,
+) -> Result {
+    let path = format!("ide/{version}");
+    let bucket = crate::aws::s3::BucketContext {
+        bucket:     BUCKET_FOR_GUI.into(),
+        upload_acl: aws_sdk_s3::model::ObjectCannedAcl::PublicRead,
+        client:     crate::aws::s3::client_from_env().await,
+        key_prefix: path,
+    };
+
+    let files_to_upload = [
+        assets.ide_wasm.as_path(),
+        assets.index_js.as_path(),
+        assets.style_css.as_path(),
+        assets.wasm_imports_js.as_path(),
+    ];
+
+    for file in files_to_upload.iter() {
+        bucket.put_file(file).await?;
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    #[tokio::test]
+    async fn upload_gui() -> Result {
+        setup_logging()?;
+        let assets = crate::paths::generated::RepoRootDistGuiAssets::new_root(
+            r"H:\NBO\enso4\dist\gui\assets",
+        );
+        let version = "2022.1.1-dev.provisional.test.1".parse2()?;
+        upload_gui_to_cloud(&assets, &version).await?;
+
+        let body = json!({
+            "version_number": version.to_string(),
+            "version_type": "Ide"
+        });
+        let response = reqwest::Client::new()
+            .post("https://uhso47ta0i.execute-api.eu-west-1.amazonaws.com/versions")
+            .header("x-enso-organization-id", "org-2BqGX0q2yCdONdmx3Om1MVZzmv3")
+            .header("Content-Type", "application/json")
+            .json(&body)
+            .send()
+            .await?;
+        trace!("{:?}", response);
+        Ok(())
+    }
+}