Merge branch 'develop' into wip/frizi/dynamic-dropdown-184012743

build/build/paths.yaml

@@ -46,10 +46,12 @@
   dist/:
     gui/:
       assets/:
-        ide.wasm:
+        shaders/: # Optimized shaders that contain main function code only.
+        pkg.js: # The `pks.js` artifact of wasm-pack WITH bundled snippets.
+        pkg.js.map: # The sourcemap mapping to `pkg.js` generated by wasm-pack.
+        pkg-opt.wasm: # The optimized WASM artifact.
         index.js:
         style.css:
-        wasm_imports.js:
 
     # Final WASM artifacts in `dist` directory.
     wasm/:

build/build/src/aws/ecr/runtime.rs

@@ -30,3 +30,31 @@ pub async fn build_runtime_image(
     let id = Docker.build(opts).await?;
     Ok(id)
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::repo::deduce_repository_path;
+
+    use super::*;
+
+    /// Convenience test that builds the Runtime image.
+    ///
+    /// The engine must be already built.
+    #[tokio::test]
+    #[ignore]
+    async fn test_name() -> Result {
+        setup_logging()?;
+        let tag = "test_runtime_image";
+        info!("Current directory: {}", ide_ci::env::current_dir()?.display());
+        let root = deduce_repository_path()?;
+        let root = root.absolutize()?;
+        info!("Repository root: {}", root.display());
+        let engine_package = generated::EnginePackage::new_root(
+            root.join("built-distribution/enso-engine-2023.1.1-dev-linux-amd64/enso-2023.1.1-dev"),
+        );
+        let dockerfile = generated::RepoRootToolsCiDocker::new_root(root.join("tools/ci/docker"));
+        let id = build_runtime_image(dockerfile, engine_package, tag.to_string()).await?;
+        info!("Built image: {}", id);
+        Ok(())
+    }
+}

build/build/src/aws/s3.rs

@@ -8,6 +8,7 @@ use aws_sdk_s3::types::ByteStream;
 use bytes::Buf;
 use enso_build_base::extensions::path::SplitFilename;
 use mime::Mime;
+use walkdir::WalkDir;
 
 
 // ==============
@@ -84,10 +85,46 @@ impl BucketContext {
     pub async fn put_file(&self, path: impl AsRef<Path>) -> Result<PutObjectOutput> {
         let path = path.as_ref();
         let stream = ByteStream::from_path(path).await?;
-        let path = path.file_name().with_context(|| format!("Path {:?} has no file name", path))?;
+        let path = path.try_file_name()?;
         self.put(path.as_str(), stream).await
     }
 
+    /// Put the file at `path` to the S3 bucket. The key will be suffixed with the relative path
+    /// between `root` and `path`.
+    #[instrument(fields(path = %path.as_ref().display(), root = %root.as_ref().display()))]
+    pub async fn put_subtree_file(
+        &self,
+        root: impl AsRef<Path>,
+        path: impl AsRef<Path>,
+    ) -> Result<PutObjectOutput> {
+        let root = root.as_ref().absolutize()?;
+        let path = path.as_ref().absolutize()?;
+        let key_suffix = path.strip_prefix(&root).with_context(|| {
+            format!("{} is not a subpath of {}.", root.display(), path.display())
+        })?;
+        let stream = ByteStream::from_path(&path).await?;
+        self.put(key_suffix.as_str(), stream).await
+    }
+
+    /// Put an item to the S3 bucket.
+    ///
+    /// If the path is a file, it will be uploaded as is. If it is a directory, it will be uploaded
+    /// as a subtree.
+    pub async fn put_item(&self, path: impl AsRef<Path>) -> Result {
+        if ide_ci::fs::tokio::metadata(path.as_ref()).await?.is_dir() {
+            let parent = path.try_parent()?;
+            for entry in WalkDir::new(&path) {
+                let entry = entry?;
+                if entry.file_type().is_file() {
+                    self.put_subtree_file(&parent, &entry.path()).await?;
+                }
+            }
+        } else {
+            self.put_file(path).await?;
+        }
+        Ok(())
+    }
+
     pub async fn get_yaml<T: DeserializeOwned>(&self, path: &str) -> Result<T> {
         let text = self.get(path).await?.collect().await?;
         serde_yaml::from_reader(text.reader()).anyhow_err()

build/build/src/release.rs

@@ -281,13 +281,14 @@ pub async fn upload_gui_to_cloud(
     let bucket = crate::aws::s3::gui::context(version).await?;
 
     // Some file we upload as-is, some gzipped. This seems somewhat arbitrary now.
-    let files_to_upload = [assets.ide_wasm.as_path(), assets.style_css.as_path()];
-    let files_to_upload_gzipped = [assets.index_js.as_path(), assets.wasm_imports_js.as_path()];
-
+    let files_to_upload =
+        [assets.pkg_opt_wasm.as_path(), assets.style_css.as_path(), assets.shaders.as_path()];
+    let files_to_upload_gzipped = [assets.index_js.as_path(), assets.pkg_js.as_path()];
 
     for file in files_to_upload.iter() {
-        bucket.put_file(file).await?;
+        bucket.put_item(file).await?;
     }
+
     put_files_gzipping(&bucket, &files_to_upload_gzipped).await?;
 
     Ok(())
@@ -360,7 +361,7 @@ mod tests {
         let assets = crate::paths::generated::RepoRootDistGuiAssets::new_root(
             r"H:\NBO\enso4\dist\gui\assets",
         );
-        let version = "2022.1.1-dev.provisional.test.2".parse2()?;
+        let version = "2023.1.1-dev.cloud.test".parse2()?;
         upload_gui_to_cloud(&assets, &version).await?;
         notify_cloud_about_gui(&version).await?;
         Ok(())

build/build/src/repo.rs

@@ -28,15 +28,31 @@ pub fn looks_like_enso_repository_root(path: impl AsRef<Path>) -> bool {
     .unwrap_or(false)
 }
 
+/// Traverse the filesystem upwards, until we find a directory that looks like the root of the Enso.
+pub fn lookup_upwards_for_repository_root(mut path: &Path) -> Option<&Path> {
+    while !looks_like_enso_repository_root(path) {
+        path = path.parent()?;
+    }
+    Some(path)
+}
+
 /// Deduce the path to the root of the Enso repository.
 ///
 /// This function will traverse the filesystem upwards from the binary location until it finds a
 /// directory that looks like the root of the Enso repository.
 #[instrument(ret, err)]
 pub fn deduce_repository_path() -> Result<PathBuf> {
-    let mut path = ide_ci::env::current_exe()?;
-    while !looks_like_enso_repository_root(&path) {
-        ensure!(path.pop(), "Failed to deduce repository path.");
-    }
-    Ok(path)
+    // We check the current directory to support cases like using `cargo install` to reuse the
+    // build script binary.
+    let current_dir = ide_ci::env::current_dir()?;
+    let exe_path = ide_ci::env::current_exe()?;
+    let repo_root = lookup_upwards_for_repository_root(&current_dir)
+        .or_else(|| lookup_upwards_for_repository_root(&exe_path));
+    repo_root.map(PathBuf::from).with_context(|| {
+        format!(
+            "Failed to deduce repository path from current directory {} and executable path {}.",
+            current_dir.display(),
+            exe_path.display()
+        )
+    })
 }

build/ci_utils/src/programs/cargo.rs

@@ -39,6 +39,8 @@ impl Program for Cargo {
 #[derive(Clone, Copy, PartialEq, Eq, Debug, strum::AsRefStr)]
 #[strum(serialize_all = "kebab-case")]
 pub enum Command {
+    /// Run the benchmarks
+    Bench,
     /// Compile the current package
     Build,
     /// Analyze the current package and report errors, but don't build object files
@@ -55,8 +57,6 @@ pub enum Command {
     Run,
     /// Run the tests
     Test,
-    /// Run the benchmarks
-    Bench,
     /// Update dependencies listed in Cargo.lock
     Update,
     /// Search registry for crates
@@ -67,6 +67,8 @@ pub enum Command {
     Install,
     /// Uninstall a Rust binary
     Uninstall,
+    /// Print a JSON representation of a Cargo.toml file's location
+    LocateProject,
 }
 
 impl Manipulator for Command {
@@ -135,3 +137,38 @@ impl Manipulator for RunOption {
         }
     }
 }
+
+/// The representation in which to print the project location.
+#[derive(Clone, Copy, PartialEq, Eq, Debug, strum::AsRefStr)]
+#[strum(serialize_all = "kebab-case")]
+pub enum MessageFormat {
+    /// JSON object with the path under the key "root".
+    Json,
+    /// Just the path.
+    Plain,
+}
+
+/// Options for the `cargo locate-project` command.
+#[derive(Clone, Copy, PartialEq, Eq, Debug, strum::AsRefStr)]
+#[strum(serialize_all = "kebab-case")]
+pub enum LocateProjectOption {
+    /// Locate the Cargo.toml at the root of the workspace, as opposed to the current workspace
+    /// member.
+    Workspace,
+    /// The representation in which to print the project location.
+    MessageFormat(MessageFormat),
+}
+
+impl Manipulator for LocateProjectOption {
+    fn apply<C: IsCommandWrapper + ?Sized>(&self, command: &mut C) {
+        let base_arg = format!("--{}", self.as_ref());
+        command.arg(base_arg);
+        use LocateProjectOption::*;
+        match self {
+            Workspace => {}
+            MessageFormat(format) => {
+                command.arg(format.as_ref());
+            }
+        }
+    }
+}

lib/rust/ensogl/pack/src/lib.rs

@@ -292,14 +292,17 @@ impl Paths {
 }
 
 pub async fn workspace_dir() -> Result<PathBuf> {
-    let output = Command::new(env!("CARGO"))
-        .arg("locate-project")
-        .arg("--workspace")
-        .arg("--message-format=plain")
+    use ide_ci::programs::cargo;
+    use ide_ci::programs::Cargo;
+    let output = Cargo
+        .cmd()?
+        .apply(&cargo::Command::LocateProject)
+        .apply(&cargo::LocateProjectOption::Workspace)
+        .apply(&cargo::LocateProjectOption::MessageFormat(cargo::MessageFormat::Plain))
         .output_ok()
         .await?
-        .stdout;
-    let cargo_path = Path::new(std::str::from_utf8(&output)?.trim());
+        .into_stdout_string()?;
+    let cargo_path = Path::new(output.trim());
     Ok(cargo_path.try_parent()?.to_owned())
 }
 

tools/ci/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/graalvm/graalvm-ce:ol7-java11-22.3.0
+FROM ghcr.io/graalvm/graalvm-ce:ol9-java11-22.3.0
 
 USER root
 
@@ -30,6 +30,7 @@ ENV ENSO_RUNTIME_DIRECTORY=/opt/enso/work
 ENV ENSO_LOG_DIRECTORY=/opt/enso/logs
 ENV ENSO_HOME=/volumes/workspace/home
 
+RUN gu install js python r
 RUN chown -hR ensodev:ensodev /opt/enso
 RUN chmod -R u=rX,g=rX /opt/enso
 RUN chmod a+x /opt/enso/bin/*