diff --git a/Cargo.lock b/Cargo.lock index b31a87d51..ed85836c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -631,6 +631,7 @@ dependencies = [ "oci-tar-builder", "prost-types 0.11.9", "protobuf 3.2.0", + "rand", "serde", "serde_json", "sha256", @@ -702,6 +703,7 @@ dependencies = [ "log", "oci-spec", "serial_test", + "sha256", "ttrpc", "wasi-common", "wasmtime", @@ -1623,7 +1625,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" dependencies = [ "fallible-iterator 0.3.0", - "indexmap 2.0.2", + "indexmap 2.2.5", "stable_deref_trait", ] @@ -1933,9 +1935,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.2" +version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ "equivalent", "hashbrown 0.14.1", @@ -2465,7 +2467,7 @@ checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ "crc32fast", "hashbrown 0.14.1", - "indexmap 2.0.2", + "indexmap 2.2.5", "memchr", ] @@ -2488,6 +2490,7 @@ version = "0.4.0" dependencies = [ "anyhow", "clap", + "indexmap 2.2.5", "log", "oci-spec", "serde", @@ -2630,7 +2633,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset 0.4.2", - "indexmap 2.0.2", + "indexmap 2.2.5", ] [[package]] @@ -3550,7 +3553,7 @@ version = "0.9.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.5", "itoa", "ryu", "serde", @@ -4075,7 +4078,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.5", "serde", "serde_spanned", "toml_datetime", @@ -4088,7 +4091,7 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.5", "serde", "serde_spanned", "toml_datetime", @@ -4883,7 +4886,7 @@ checksum = "d21472954ee9443235ca32522b17fc8f0fe58e2174556266a0d9766db055cc52" dependencies = [ "anyhow", "derive_builder", - "indexmap 2.0.2", + "indexmap 2.2.5", "semver", "serde", "serde_cbor", @@ -5033,7 +5036,7 @@ version = "0.118.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95ee9723b928e735d53000dec9eae7b07a60e490c85ab54abb66659fc61bfcd9" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.2.5", "semver", ] @@ -5044,7 +5047,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a03f65ac876612140c57ff6c3b8fe4990067cce97c2cfdb07368a3cc3354b062" dependencies = [ "bitflags 2.4.2", - "indexmap 2.0.2", + "indexmap 2.2.5", "semver", ] @@ -5071,7 +5074,7 @@ dependencies = [ "cfg-if 1.0.0", "encoding_rs", "fxprof-processed-profile", - "indexmap 2.0.2", + "indexmap 2.2.5", "libc", "log", "object", @@ -5197,7 +5200,7 @@ dependencies = [ "anyhow", "cranelift-entity 0.104.1", "gimli 0.28.0", - "indexmap 2.0.2", + "indexmap 2.2.5", "log", "object", "serde", @@ -5286,7 +5289,7 @@ dependencies = [ "cc", "cfg-if 1.0.0", "encoding_rs", - "indexmap 2.0.2", + "indexmap 2.2.5", "libc", "log", "mach", @@ -5390,7 +5393,7 @@ checksum = "6bb3bc92c031cf4961135bffe055a69c1bd67c253dca20631478189bb05ec27b" dependencies = [ "anyhow", "heck 0.4.1", - "indexmap 2.0.2", + "indexmap 2.2.5", "wit-parser", ] @@ -5807,7 +5810,7 @@ checksum = "15df6b7b28ce94b8be39d8df5cb21a08a4f3b9f33b631aedb4aa5776f785ead3" dependencies = [ "anyhow", "id-arena", - "indexmap 2.0.2", + "indexmap 2.2.5", "log", "semver", "serde", diff --git a/Makefile b/Makefile index 87345bb06..d05911404 100644 --- a/Makefile +++ b/Makefile @@ -239,8 +239,8 @@ test/k8s/deploy-workload-oci-%: test/k8s/clean test/k8s/cluster-% dist/img-oci.t kubectl --context=kind-$(KIND_CLUSTER_NAME) wait deployment wasi-demo --for condition=Available=True --timeout=5s @if [ "$*" = "wasmtime" ]; then \ set -e; \ - echo "checking for pre-compiled label and ensuring can scale"; \ - docker exec $(KIND_CLUSTER_NAME)-control-plane ctr -n k8s.io i ls | grep "runwasi.io/precompiled"; \ + echo "checking for pre-compiled labels and ensuring can scale after pre-compile"; \ + docker exec $(KIND_CLUSTER_NAME)-control-plane ctr -n k8s.io content ls | grep "runwasi.io/precompiled"; \ kubectl --context=kind-$(KIND_CLUSTER_NAME) scale deployment wasi-demo --replicas=4; \ kubectl --context=kind-$(KIND_CLUSTER_NAME) wait deployment wasi-demo --for condition=Available=True --timeout=5s; \ fi @@ -297,8 +297,8 @@ test/k3s-oci-%: dist/img-oci.tar bin/k3s dist-% sudo bin/k3s kubectl get pods -o wide @if [ "$*" = "wasmtime" ]; then \ set -e; \ - echo "checking for pre-compiled label and ensuring can scale"; \ - sudo bin/k3s ctr -n k8s.io i ls | grep "runwasi.io/precompiled"; \ + echo "checking for pre-compiled labels and ensuring can scale"; \ + sudo bin/k3s ctr -n k8s.io content ls | grep "runwasi.io/precompiled"; \ sudo bin/k3s kubectl scale deployment wasi-demo --replicas=4; \ sudo bin/k3s kubectl wait deployment wasi-demo --for condition=Available=True --timeout=5s; \ fi diff --git a/crates/containerd-shim-wasm/Cargo.toml b/crates/containerd-shim-wasm/Cargo.toml index 9018597af..ca59e0cea 100644 --- a/crates/containerd-shim-wasm/Cargo.toml +++ b/crates/containerd-shim-wasm/Cargo.toml @@ -35,7 +35,7 @@ futures = { version = "0.3.30" } wasmparser = "0.200.0" tokio-stream = { version = "0.1" } prost-types = "0.11" # should match version in containerd-shim -sha256 = "1.4.0" +sha256 = { workspace = true } [target.'cfg(unix)'.dependencies] caps = "0.5" @@ -56,6 +56,7 @@ containerd-shim-wasm-test-modules = { workspace = true } env_logger = { workspace = true } tempfile = { workspace = true } oci-tar-builder = { workspace = true} +rand= "0.8" [features] testing = ["dep:containerd-shim-wasm-test-modules", "dep:env_logger", "dep:tempfile", "dep:oci-tar-builder"] diff --git a/crates/containerd-shim-wasm/src/container/engine.rs b/crates/containerd-shim-wasm/src/container/engine.rs index f3461f6c8..2f3964dd7 100644 --- a/crates/containerd-shim-wasm/src/container/engine.rs +++ b/crates/containerd-shim-wasm/src/container/engine.rs @@ -5,6 +5,7 @@ use anyhow::{bail, Context, Result}; use super::Source; use crate::container::{PathResolve, RuntimeContext}; +use crate::sandbox::oci::WasmLayer; use crate::sandbox::Stdio; pub trait Engine: Clone + Send + Sync + 'static { @@ -53,12 +54,14 @@ pub trait Engine: Clone + Send + Sync + 'static { &["application/vnd.bytecodealliance.wasm.component.layer.v0+wasm"] } - /// Precompiles a module that is in the WASM OCI layer format - /// This is used to precompile a module before it is run and will be called if can_precompile returns true. + /// Precompile passes supported OCI layers to engine for compilation + /// This is used to precompile the layers before they are run and will be called if `can_precompile` returns `true`. /// It is called only the first time a module is run and the resulting bytes will be cached in the containerd content store. - /// The cached, precompiled module will be reloaded on subsequent runs. - fn precompile(&self, _layers: &[Vec]) -> Result> { - bail!("precompilation not supported for this runtime") + /// The cached, precompiled layers will be reloaded on subsequent runs. + /// The runtime is expected to return the same number of layers passed in, if the layer cannot be precompiled it should return `None` for that layer. + /// In some edge cases it is possible that the layers may already be precompiled and None should be returned in this case. + fn precompile(&self, _layers: &[WasmLayer]) -> Result>>> { + bail!("precompile not supported"); } /// Can_precompile lets the shim know if the runtime supports precompilation. diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs index 37e83f28e..684172bd2 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs @@ -10,8 +10,7 @@ use containerd_client::services::v1::images_client::ImagesClient; use containerd_client::services::v1::leases_client::LeasesClient; use containerd_client::services::v1::{ Container, DeleteContentRequest, GetContainerRequest, GetImageRequest, Image, Info, - InfoRequest, ReadContentRequest, UpdateImageRequest, UpdateRequest, WriteAction, - WriteContentRequest, + InfoRequest, ReadContentRequest, UpdateRequest, WriteAction, WriteContentRequest, }; use containerd_client::tonic::transport::Channel; use containerd_client::{tonic, with_namespace}; @@ -138,11 +137,11 @@ impl Client { fn save_content( &self, data: Vec, - original_digest: String, - label: &str, + unique_id: &str, + labels: HashMap, ) -> Result { let expected = format!("sha256:{}", digest(data.clone())); - let reference = format!("precompile-{}", label); + let reference = format!("precompile-{}", unique_id); let lease = self.lease(reference.clone())?; let digest = self.rt.block_on(async { @@ -194,10 +193,6 @@ impl Client { // we don't need to copy the content again. Container tells us it found the blob // by returning the offset of the content that was found. let data_to_write = data[response.offset as usize..].to_vec(); - - // Write and commit at same time - let mut labels = HashMap::new(); - labels.insert(label.to_string(), original_digest.clone()); let commit_request = WriteContentRequest { action: WriteAction::Commit.into(), total: len, @@ -248,10 +243,10 @@ impl Client { }) } - fn get_info(&self, content_digest: String) -> Result { + fn get_info(&self, content_digest: &str) -> Result { self.rt.block_on(async { let req = InfoRequest { - digest: content_digest.clone(), + digest: content_digest.to_string(), }; let req = with_namespace!(req, self.namespace); let info = ContentClient::new(self.inner.clone()) @@ -316,29 +311,6 @@ impl Client { }) } - fn update_image(&self, image: Image) -> Result { - self.rt.block_on(async { - let req = UpdateImageRequest { - image: Some(image.clone()), - update_mask: Some(FieldMask { - paths: vec!["labels".to_string()], - }), - }; - - let req = with_namespace!(req, self.namespace); - let image = ImagesClient::new(self.inner.clone()) - .update(req) - .await - .map_err(|err| ShimError::Containerd(err.to_string()))? - .into_inner() - .image - .ok_or_else(|| { - ShimError::Containerd(format!("failed to update image {}", image.name)) - })?; - Ok(image) - }) - } - fn extract_image_content_sha(&self, image: &Image) -> Result { let digest = image .target @@ -375,6 +347,15 @@ impl Client { }) } + fn get_image_manifest(&self, image_name: &str) -> Result<(ImageManifest, String)> { + let image = self.get_image(image_name)?; + let image_digest = self.extract_image_content_sha(&image)?; + let manifest = self.read_content(&image_digest)?; + let manifest = manifest.as_slice(); + let manifest = ImageManifest::from_reader(manifest)?; + Ok((manifest, image_digest)) + } + // load module will query the containerd store to find an image that has an OS of type 'wasm' // If found it continues to parse the manifest and return the layers that contains the WASM modules // and possibly other configuration layers. @@ -384,11 +365,7 @@ impl Client { engine: &T, ) -> Result<(Vec, Platform)> { let container = self.get_container(containerd_id.to_string())?; - let mut image = self.get_image(container.image)?; - let image_digest = self.extract_image_content_sha(&image)?; - let manifest = self.read_content(image_digest.clone())?; - let manifest = manifest.as_slice(); - let manifest = ImageManifest::from_reader(manifest)?; + let (manifest, image_digest) = self.get_image_manifest(&container.image)?; let image_config_descriptor = manifest.config(); let image_config = self.read_content(image_config_descriptor.digest())?; @@ -401,7 +378,7 @@ impl Client { return Ok((vec![], platform)); }; - log::info!("found manifest with WASM OCI image format."); + log::info!("found manifest with WASM OCI image format"); // This label is unique across runtimes and version of the shim running // a precompiled component/module will not work across different runtimes or versions let (can_precompile, precompile_id) = match engine.can_precompile() { @@ -409,82 +386,132 @@ impl Client { None => (false, "".to_string()), }; - match image.labels.get(&precompile_id) { - Some(precompile_digest) if can_precompile => { - log::info!("found precompiled label: {} ", &precompile_id); - match self.read_content(precompile_digest) { - Ok(precompiled) => { - log::info!("found precompiled module in cache: {} ", &precompile_digest); - return Ok(( - vec![WasmLayer { - config: image_config_descriptor.clone(), - layer: precompiled, - }], - platform, - )); - } - Err(e) => { - // log and continue - log::warn!("failed to read precompiled module from cache: {}. Content may have been removed manually, will attempt to recompile", e); - } - } - } - _ => {} - } - + let image_info = self.get_info(&image_digest)?; + let mut needs_precompile = + can_precompile && !image_info.labels.contains_key(&precompile_id); let layers = manifest .layers() .iter() .filter(|x| is_wasm_layer(x.media_type(), T::supported_layers_types())) - .map(|config| self.read_content(config.digest())) + .map(|original_config| { + let mut digest_to_load = original_config.digest().clone(); + if can_precompile { + let info = self.get_info(&digest_to_load)?; + if info.labels.contains_key(&precompile_id) { + // Safe to unwrap here since we already checked for the label's existence + digest_to_load = info.labels.get(&precompile_id).unwrap().clone(); + log::info!( + "layer {} has pre-compiled content: {} ", + info.digest, + &digest_to_load + ); + } + } + log::debug!("loading digest: {} ", &digest_to_load); + self.read_content(&digest_to_load) + .map(|module| WasmLayer { + config: original_config.clone(), + layer: module, + }) + .or_else(|e| { + // handle content being removed from the content store out of band + if digest_to_load != *original_config.digest() { + log::error!("failed to load precompiled layer: {}", e); + log::error!("falling back to original layer and marking for recompile"); + needs_precompile = can_precompile; // only mark for recompile if engine is capable + self.read_content(original_config.digest()) + .map(|module| WasmLayer { + config: original_config.clone(), + layer: module, + }) + } else { + Err(e) + } + }) + }) .collect::>>()?; - if layers.is_empty() { - log::info!("no WASM modules found in OCI layers"); - return Ok((vec![], platform)); + if needs_precompile { + log::info!("precompiling layers for image: {}", container.image); + match engine.precompile(&layers) { + Ok(compiled_layers) => { + if compiled_layers.len() != layers.len() { + return Err(ShimError::FailedPrecondition( + "precompile returned wrong number of layers".to_string(), + )); + } + + let mut layers_for_runtime = Vec::with_capacity(compiled_layers.len()); + for (i, compiled_layer) in compiled_layers.iter().enumerate() { + if compiled_layer.is_none() { + log::debug!("no compiled layer using original"); + layers_for_runtime.push(layers[i].clone()); + continue; + } + + let compiled_layer = compiled_layer.as_ref().unwrap(); + let original_config = &layers[i].config; + let labels = HashMap::from([( + format!("{precompile_id}/original"), + original_config.digest().to_string(), + )]); + let precompiled_content = + self.save_content(compiled_layer.clone(), &precompile_id, labels)?; + + log::debug!( + "updating original layer {} with compiled layer {}", + original_config.digest(), + precompiled_content.digest + ); + // We add two labels here: + // - one with cache key per engine instance + // - one with a gc ref flag so it doesn't get cleaned up as long as the original layer exists + let mut original_layer = self.get_info(original_config.digest())?; + original_layer + .labels + .insert(precompile_id.clone(), precompiled_content.digest.clone()); + original_layer.labels.insert( + format!("containerd.io/gc.ref.content.precompile.{}", i), + precompiled_content.digest.clone(), + ); + self.update_info(original_layer)?; + + // The original image is considered a root object, by adding a ref to the new compiled content + // We tell containerd to not garbage collect the new content until this image is removed from the system + // this ensures that we keep the content around after the lease is dropped + // We also save the precompiled flag here since the image labels can be mutated containerd, for example if the image is pulled twice + log::debug!( + "updating image content with precompile digest to avoid garbage collection" + ); + let mut image_content = self.get_info(&image_digest)?; + image_content.labels.insert( + format!("containerd.io/gc.ref.content.precompile.{}", i), + precompiled_content.digest, + ); + image_content + .labels + .insert(precompile_id.clone(), "true".to_string()); + self.update_info(image_content)?; + + layers_for_runtime.push(WasmLayer { + config: original_config.clone(), + layer: compiled_layer.clone(), + }); + } + return Ok((layers_for_runtime, platform)); + } + Err(e) => { + log::error!("precompilation failed: {}", e); + } + }; } - if can_precompile { - log::info!("precompiling module"); - let precompiled = engine.precompile(layers.as_slice())?; - log::info!("precompiling module: {}", image_digest.clone()); - let precompiled_content = - self.save_content(precompiled.clone(), image_digest.clone(), &precompile_id)?; - - log::debug!("updating image with compiled content digest"); - image - .labels - .insert(precompile_id, precompiled_content.digest.clone()); - self.update_image(image)?; - - // The original image is considered a root object, by adding a ref to the new compiled content - // We tell containerd to not garbage collect the new content until this image is removed from the system - // this ensures that we keep the content around after the lease is dropped - log::debug!("updating content with precompile digest to avoid garbage collection"); - let mut image_content = self.get_info(image_digest.clone())?; - image_content.labels.insert( - "containerd.io/gc.ref.content.precompile".to_string(), - precompiled_content.digest.clone(), - ); - self.update_info(image_content)?; - - return Ok(( - vec![WasmLayer { - config: image_config_descriptor.clone(), - layer: precompiled, - }], - platform, - )); + if layers.is_empty() { + log::info!("no WASM layers found in OCI image"); + return Ok((vec![], platform)); } - log::info!("using module from OCI layers"); - let layers = layers - .into_iter() - .map(|module| WasmLayer { - config: image_config_descriptor.clone(), - layer: module, - }) - .collect::>(); + log::info!("using OCI layers"); Ok((layers, platform)) } } @@ -494,14 +521,29 @@ fn precompile_label(name: &str, version: &str) -> String { } fn is_wasm_layer(media_type: &MediaType, supported_layer_types: &[&str]) -> bool { - supported_layer_types.contains(&media_type.to_string().as_str()) + let supported = supported_layer_types.contains(&media_type.to_string().as_str()); + log::debug!( + "layer type {} is supported: {}", + media_type.to_string().as_str(), + supported + ); + supported } #[cfg(test)] mod tests { use std::path::PathBuf; + use std::sync::atomic::{AtomicI32, Ordering}; + use std::sync::Arc; + + use oci_tar_builder::WASM_LAYER_MEDIA_TYPE; + use rand::prelude::*; use super::*; + use crate::container::RuntimeContext; + use crate::sandbox::Stdio; + use crate::testing::oci_helpers::ImageContent; + use crate::testing::{oci_helpers, TEST_NAMESPACE}; #[test] fn test_save_content() { @@ -513,17 +555,15 @@ mod tests { let expected = digest(data.clone()); let expected = format!("sha256:{}", expected); - let label = precompile_label("test", "hasdfh"); - let returned = client - .save_content(data, "original".to_string(), &label) - .unwrap(); + let label = HashMap::from([(precompile_label("test", "hasdfh"), "original".to_string())]); + let returned = client.save_content(data, "test", label.clone()).unwrap(); assert_eq!(expected, returned.digest.clone()); let data = client.read_content(returned.digest.clone()).unwrap(); assert_eq!(data, b"hello world"); client - .save_content(data.clone(), "original".to_string(), &label) + .save_content(data.clone(), "test", label.clone()) .expect_err("Should not be able to save when lease is open"); // need to drop the lease to be able to create a second one @@ -531,9 +571,7 @@ mod tests { drop(returned); // a second call should be successful since it already exists - let returned = client - .save_content(data, "original".to_string(), &label) - .unwrap(); + let returned = client.save_content(data, "test", label.clone()).unwrap(); assert_eq!(expected, returned.digest); client.delete_content(expected.clone()).unwrap(); @@ -542,4 +580,370 @@ mod tests { .read_content(expected) .expect_err("content should not exist"); } + + #[test] + fn test_layers_when_precompile_not_supported() { + let path = PathBuf::from("/run/containerd/containerd.sock"); + let path = path.to_str().unwrap(); + let client = Client::connect(path, TEST_NAMESPACE).unwrap(); + + let fake_bytes = generate_content("original", WASM_LAYER_MEDIA_TYPE); + let (_, container_name, _cleanup) = generate_test_container(None, &[&fake_bytes]); + let engine = FakePrecomiplerEngine::new(None); + + let (layers, _) = client.load_modules(container_name, &engine).unwrap(); + assert_eq!(layers.len(), 1); + assert_eq!(layers[0].layer, fake_bytes.bytes); + assert_eq!(engine.precompile_called.load(Ordering::SeqCst), 0); + } + + #[test] + fn test_layers_are_precompiled_once() { + let path = PathBuf::from("/run/containerd/containerd.sock"); + let path = path.to_str().unwrap(); + let client = Client::connect(path, crate::testing::TEST_NAMESPACE).unwrap(); + + let fake_bytes = generate_content("original", WASM_LAYER_MEDIA_TYPE); + let (_image_name, container_name, _cleanup) = generate_test_container(None, &[&fake_bytes]); + + let fake_precompiled_bytes = generate_content("precompiled", WASM_LAYER_MEDIA_TYPE); + let mut engine = FakePrecomiplerEngine::new(Some(())); + engine.add_precompiled_bits(fake_bytes.bytes.clone(), &fake_precompiled_bytes); + + let (_, _) = client.load_modules(&container_name, &engine).unwrap(); + assert_eq!(engine.precompile_called.load(Ordering::SeqCst), 1); + + // Even on second calls should only pre-compile once + let (layers, _) = client.load_modules(&container_name, &engine).unwrap(); + assert_eq!(engine.precompile_called.load(Ordering::SeqCst), 1); + assert_eq!(layers.len(), 1); + assert_eq!(layers[0].layer, fake_precompiled_bytes.bytes); + } + + #[test] + fn test_layers_are_recompiled_if_version_changes() { + let path = PathBuf::from("/run/containerd/containerd.sock"); + let path = path.to_str().unwrap(); + let client = Client::connect(path, crate::testing::TEST_NAMESPACE).unwrap(); + + let fake_bytes = generate_content("original", WASM_LAYER_MEDIA_TYPE); + let (_image_name, container_name, _cleanup) = generate_test_container(None, &[&fake_bytes]); + + let fake_precompiled_bytes = generate_content("precompiled", WASM_LAYER_MEDIA_TYPE); + let mut engine = FakePrecomiplerEngine::new(Some(())); + engine.add_precompiled_bits(fake_bytes.bytes.clone(), &fake_precompiled_bytes); + + let (_, _) = client.load_modules(&container_name, &engine).unwrap(); + assert_eq!(engine.precompile_called.load(Ordering::SeqCst), 1); + + engine.precompile_id = Some("new_version".to_string()); + let (_, _) = client.load_modules(&container_name, &engine).unwrap(); + assert_eq!(engine.precompile_called.load(Ordering::SeqCst), 2); + } + + #[test] + fn test_layers_are_precompiled() { + let path = PathBuf::from("/run/containerd/containerd.sock"); + let path = path.to_str().unwrap(); + let client = Client::connect(path, crate::testing::TEST_NAMESPACE).unwrap(); + + let fake_bytes = generate_content("original", WASM_LAYER_MEDIA_TYPE); + let (image_name, container_name, _cleanup) = generate_test_container(None, &[&fake_bytes]); + + let fake_precompiled_bytes = generate_content("precompiled", WASM_LAYER_MEDIA_TYPE); + let mut engine = FakePrecomiplerEngine::new(Some(())); + engine.add_precompiled_bits(fake_bytes.bytes.clone(), &fake_precompiled_bytes); + let expected_id = precompile_label( + FakePrecomiplerEngine::name(), + engine.can_precompile().unwrap().as_str(), + ); + + let (layers, _) = client.load_modules(container_name, &engine).unwrap(); + assert_eq!(engine.precompile_called.load(Ordering::SeqCst), 1); + assert_eq!(layers.len(), 1); + assert_eq!(layers[0].layer, fake_precompiled_bytes.bytes); + + let (manifest, _) = client.get_image_manifest(&image_name).unwrap(); + let original_config = manifest.layers().first().unwrap(); + let info = client.get_info(original_config.digest()).unwrap(); + + let actual_digest = info.labels.get(&expected_id).unwrap(); + assert_eq!( + actual_digest.to_string(), + format!("sha256:{}", &digest(fake_precompiled_bytes.bytes.clone())) + ); + } + + #[test] + fn test_layers_are_precompiled_but_not_for_all_layers() { + let path = PathBuf::from("/run/containerd/containerd.sock"); + let path = path.to_str().unwrap(); + let client = Client::connect(path, crate::testing::TEST_NAMESPACE).unwrap(); + + let fake_bytes = generate_content("original", WASM_LAYER_MEDIA_TYPE); + let non_wasm_bytes = generate_content("original_dont_compile", "textfile"); + let (_image_name, container_name, _cleanup) = + generate_test_container(None, &[&fake_bytes, &non_wasm_bytes]); + + let fake_precompiled_bytes = generate_content("precompiled", WASM_LAYER_MEDIA_TYPE); + let mut engine = FakePrecomiplerEngine::new(Some(())); + engine.add_precompiled_bits(fake_bytes.bytes.clone(), &fake_precompiled_bytes); + + let (layers, _) = client.load_modules(container_name, &engine).unwrap(); + + assert_eq!(engine.precompile_called.load(Ordering::SeqCst), 1); + assert_eq!(engine.layers_compiled_per_call.load(Ordering::SeqCst), 1); + assert_eq!(layers.len(), 2); + assert_eq!(layers[0].layer, fake_precompiled_bytes.bytes); + assert_eq!(layers[1].layer, non_wasm_bytes.bytes); + } + + #[test] + fn test_layers_do_not_need_precompiled_if_new_layers_are_added_to_existing_image() { + let path = PathBuf::from("/run/containerd/containerd.sock"); + let path = path.to_str().unwrap(); + let client = Client::connect(path, crate::testing::TEST_NAMESPACE).unwrap(); + + let fake_bytes = generate_content("original", WASM_LAYER_MEDIA_TYPE); + let (_image_name, container_name, _cleanup) = generate_test_container(None, &[&fake_bytes]); + + let fake_precompiled_bytes = generate_content("precompiled", WASM_LAYER_MEDIA_TYPE); + let mut engine = FakePrecomiplerEngine::new(Some(())); + engine.add_precompiled_bits(fake_bytes.bytes.clone(), &fake_precompiled_bytes); + + let (layers, _) = client.load_modules(container_name, &engine).unwrap(); + assert_eq!(engine.precompile_called.load(Ordering::SeqCst), 1); + assert_eq!(layers.len(), 1); + assert_eq!(layers[0].layer, fake_precompiled_bytes.bytes); + + // get original image sha before importing new image + let image_sha = client + .get_image(&_image_name) + .unwrap() + .target + .unwrap() + .digest; + + let fake_bytes2 = generate_content("image2", WASM_LAYER_MEDIA_TYPE); + let (_image_name2, container_name2, _cleanup2) = + generate_test_container(Some(_image_name), &[&fake_bytes, &fake_bytes2]); + let fake_precompiled_bytes2 = generate_content("precompiled2", WASM_LAYER_MEDIA_TYPE); + engine.add_precompiled_bits(fake_bytes2.bytes.clone(), &fake_precompiled_bytes2); + + // When a new image with the same name is create the older image content will disappear + // but since these layers are part of the new image we don't want to have to recompile + // for the test, let the original image get removed (which would remove any associated content) + // and then check that the layers don't need to be recompiled + oci_helpers::wait_for_content_removal(&image_sha).unwrap(); + + let (layers, _) = client.load_modules(container_name2, &engine).unwrap(); + assert_eq!(engine.precompile_called.load(Ordering::SeqCst), 2); + assert_eq!(layers.len(), 2); + assert_eq!(engine.layers_compiled_per_call.load(Ordering::SeqCst), 1); + } + + #[test] + fn test_layers_do_not_need_precompiled_if_new_layers_are_add_to_new_image() { + let path = PathBuf::from("/run/containerd/containerd.sock"); + let path = path.to_str().unwrap(); + let client = Client::connect(path, crate::testing::TEST_NAMESPACE).unwrap(); + + let fake_bytes = generate_content("original", WASM_LAYER_MEDIA_TYPE); + let (_image_name, container_name, _cleanup) = generate_test_container(None, &[&fake_bytes]); + + let fake_precompiled_bytes = generate_content("precompiled", WASM_LAYER_MEDIA_TYPE); + let mut engine = FakePrecomiplerEngine::new(Some(())); + engine.add_precompiled_bits(fake_bytes.bytes.clone(), &fake_precompiled_bytes); + + let (layers, _) = client.load_modules(container_name, &engine).unwrap(); + assert_eq!(engine.precompile_called.load(Ordering::SeqCst), 1); + assert_eq!(layers.len(), 1); + assert_eq!(layers[0].layer, fake_precompiled_bytes.bytes); + + let fake_bytes2 = generate_content("image2", WASM_LAYER_MEDIA_TYPE); + let (_image_name2, container_name2, _cleanup2) = + generate_test_container(None, &[&fake_bytes, &fake_bytes2]); + let fake_precompiled_bytes2 = generate_content("precompiled2", WASM_LAYER_MEDIA_TYPE); + engine.add_precompiled_bits(fake_bytes2.bytes.clone(), &fake_precompiled_bytes2); + + let (layers, _) = client.load_modules(container_name2, &engine).unwrap(); + assert_eq!(engine.precompile_called.load(Ordering::SeqCst), 2); + assert_eq!(layers.len(), 2); + assert_eq!(engine.layers_compiled_per_call.load(Ordering::SeqCst), 1); + } + + #[test] + fn test_layers_are_precompiled_for_multiple_layers() { + let path = PathBuf::from("/run/containerd/containerd.sock"); + let path = path.to_str().unwrap(); + let client = Client::connect(path, crate::testing::TEST_NAMESPACE).unwrap(); + + let fake_bytes = generate_content("original", WASM_LAYER_MEDIA_TYPE); + let fake_bytes2 = generate_content("original1", WASM_LAYER_MEDIA_TYPE); + + let (image_name, container_name, _cleanup) = + generate_test_container(None, &[&fake_bytes, &fake_bytes2]); + + let fake_precompiled_bytes = generate_content("precompiled", WASM_LAYER_MEDIA_TYPE); + let fake_precompiled_bytes2 = generate_content("precompiled1", WASM_LAYER_MEDIA_TYPE); + + let mut engine = FakePrecomiplerEngine::new(Some(())); + engine.add_precompiled_bits(fake_bytes.bytes.clone(), &fake_precompiled_bytes); + engine.add_precompiled_bits(fake_bytes2.bytes.clone(), &fake_precompiled_bytes2); + + let expected_id = precompile_label( + FakePrecomiplerEngine::name(), + engine.can_precompile().unwrap().as_str(), + ); + + let (layers, _) = client.load_modules(container_name, &engine).unwrap(); + assert_eq!(engine.precompile_called.load(Ordering::SeqCst), 1); + assert_eq!(engine.layers_compiled_per_call.load(Ordering::SeqCst), 2); + + assert_eq!(layers.len(), 2); + assert_eq!(layers[0].layer, fake_precompiled_bytes.bytes); + assert_eq!(layers[1].layer, fake_precompiled_bytes2.bytes); + + let (manifest, _) = client.get_image_manifest(&image_name).unwrap(); + + let original_config1 = manifest.layers().first().unwrap(); + let info1 = client.get_info(original_config1.digest()).unwrap(); + let actual_digest1 = info1.labels.get(&expected_id).unwrap(); + assert_eq!( + actual_digest1.to_string(), + format!("sha256:{}", &digest(fake_precompiled_bytes.bytes.clone())) + ); + + let original_config2 = manifest.layers().last().unwrap(); + let info2 = client.get_info(original_config2.digest()).unwrap(); + let actual_digest2 = info2.labels.get(&expected_id).unwrap(); + assert_eq!( + actual_digest2.to_string(), + format!("sha256:{}", &digest(fake_precompiled_bytes2.bytes.clone())) + ); + } + + fn generate_test_container( + name: Option, + original: &[&oci_helpers::ImageContent], + ) -> (String, String, oci_helpers::OCICleanup) { + let _ = env_logger::try_init(); + + let random_number = random_number(); + let image_name = name.unwrap_or(format!("localhost/test:latest{}", random_number)); + oci_helpers::import_image(&image_name, original).unwrap(); + + let container_name = format!("test-container-{}", random_number); + oci_helpers::create_container(&container_name, &image_name).unwrap(); + + let _cleanup = oci_helpers::OCICleanup { + image_name: image_name.clone(), + container_name: container_name.clone(), + }; + (image_name, container_name, _cleanup) + } + + fn generate_content(seed: &str, media_type: &str) -> oci_helpers::ImageContent { + let mut content = seed.as_bytes().to_vec(); + for _ in 0..100 { + content.push(random_number() as u8); + } + ImageContent { + bytes: content, + media_type: media_type.to_string(), + } + } + + fn random_number() -> u32 { + let x: u32 = random(); + x + } + + #[derive(Clone)] + struct FakePrecomiplerEngine { + precompile_id: Option, + precompiled_layers: HashMap>, + precompile_called: Arc, + layers_compiled_per_call: Arc, + } + + impl FakePrecomiplerEngine { + fn new(can_precompile: Option<()>) -> Self { + let precompile_id = match can_precompile { + Some(_) => { + let precompile_id = format!("uuid-{}", random_number()); + Some(precompile_id) + } + None => None, + }; + + FakePrecomiplerEngine { + precompile_id, + precompiled_layers: HashMap::new(), + precompile_called: Arc::new(AtomicI32::new(0)), + layers_compiled_per_call: Arc::new(AtomicI32::new(0)), + } + } + fn add_precompiled_bits( + &mut self, + original: Vec, + precompiled_content: &oci_helpers::ImageContent, + ) { + let key = digest(original); + self.precompiled_layers + .insert(key, precompiled_content.bytes.clone()); + } + } + + impl Engine for FakePrecomiplerEngine { + fn name() -> &'static str { + "fake" + } + + fn run_wasi( + &self, + _ctx: &impl RuntimeContext, + _stdio: Stdio, + ) -> std::result::Result { + panic!("not implemented") + } + + fn can_precompile(&self) -> Option { + self.precompile_id.clone() + } + + fn supported_layers_types() -> &'static [&'static str] { + &[WASM_LAYER_MEDIA_TYPE, "textfile"] + } + + fn precompile(&self, layers: &[WasmLayer]) -> Result>>, anyhow::Error> { + self.layers_compiled_per_call.store(0, Ordering::SeqCst); + self.precompile_called.fetch_add(1, Ordering::SeqCst); + let mut compiled_layers = vec![]; + for layer in layers { + if layer.config.media_type().to_string() == *"textfile" { + // simulate a layer that can't be precompiled + compiled_layers.push(None); + continue; + } + + let key = digest(layer.layer.clone()); + if self.precompiled_layers.values().any(|l| digest(l) == key) { + // simulate scenario were one of the layers is already compiled + compiled_layers.push(None); + continue; + } + + // load the "precompiled" layer that was stored as precompiled for this layer + self.precompiled_layers.iter().all(|x| { + log::warn!("layer: {:?}", x.0); + true + }); + let precompiled = self.precompiled_layers[&key].clone(); + compiled_layers.push(Some(precompiled)); + self.layers_compiled_per_call.fetch_add(1, Ordering::SeqCst); + } + Ok(compiled_layers) + } + } } diff --git a/crates/containerd-shim-wasm/src/sandbox/mod.rs b/crates/containerd-shim-wasm/src/sandbox/mod.rs index ef6db1811..872715717 100644 --- a/crates/containerd-shim-wasm/src/sandbox/mod.rs +++ b/crates/containerd-shim-wasm/src/sandbox/mod.rs @@ -19,3 +19,4 @@ pub use stdio::Stdio; pub(crate) mod containerd; pub(crate) mod oci; +pub use oci::WasmLayer; diff --git a/crates/containerd-shim-wasm/src/testing.rs b/crates/containerd-shim-wasm/src/testing.rs index 1c1131386..f4237c73f 100644 --- a/crates/containerd-shim-wasm/src/testing.rs +++ b/crates/containerd-shim-wasm/src/testing.rs @@ -1,22 +1,19 @@ //! Testing utilities used across different modules use std::collections::HashMap; -use std::fs::{self, create_dir, read_to_string, write, File}; +use std::fs::{self, create_dir, read, read_to_string, write, File}; use std::marker::PhantomData; use std::ops::Add; -use std::process::Command; use std::time::Duration; use anyhow::{bail, Result}; pub use containerd_shim_wasm_test_modules as modules; -use oci_spec::image::{self as spec, Arch}; use oci_spec::runtime::{ProcessBuilder, RootBuilder, SpecBuilder}; -use oci_tar_builder::{Builder, WASM_LAYER_MEDIA_TYPE}; use crate::sandbox::{Instance, InstanceConfig}; use crate::sys::signals::SIGKILL; -const TEST_NAMESPACE: &str = "runwasi-test"; +pub const TEST_NAMESPACE: &str = "runwasi-test"; pub struct WasiTestBuilder where @@ -126,69 +123,23 @@ where image_name: Option, container_name: Option, ) -> Result<(Self, oci_helpers::OCICleanup)> { - let mut builder = Builder::default(); - - let dir = self.tempdir.path(); - let wasm_path = dir.join("rootfs").join("hello.wasm"); - builder.add_layer_with_media_type(&wasm_path, WASM_LAYER_MEDIA_TYPE.to_string()); - - let config = spec::ConfigBuilder::default() - .entrypoint(vec!["_start".to_string()]) - .build() - .unwrap(); - - let img = spec::ImageConfigurationBuilder::default() - .config(config) - .os("wasip1") - .architecture(Arch::Wasm) - .rootfs( - spec::RootFsBuilder::default() - .diff_ids(vec![]) - .build() - .unwrap(), - ) - .build()?; - let image_name = image_name.unwrap_or("localhost/hello:latest".to_string()); - builder.add_config(img, image_name.clone()); - - let img = dir.join("img.tar"); - let f = File::create(img.clone())?; - builder.build(f)?; - let success = Command::new("ctr") - .arg("-n") - .arg(TEST_NAMESPACE) - .arg("image") - .arg("import") - .arg("--all-platforms") - .arg(img) - .spawn()? - .wait()? - .success(); - - if !success { - // if the container still exists try cleaning it up - bail!(" failed to import image"); + if !oci_helpers::image_exists(&image_name) { + let wasm_path = self.tempdir.path().join("rootfs").join("hello.wasm"); + let bytes = read(&wasm_path)?; + let wasm_content = oci_helpers::ImageContent { + bytes, + media_type: oci_tar_builder::WASM_LAYER_MEDIA_TYPE.to_string(), + }; + oci_helpers::import_image(&image_name, &[&wasm_content])?; + + // remove the file from the rootfs so it doesn't get treated like a regular container + fs::remove_file(&wasm_path)?; } - fs::remove_file(&wasm_path)?; - let container_name = container_name.unwrap_or("test".to_string()); - let success = Command::new("ctr") - .arg("-n") - .arg(TEST_NAMESPACE) - .arg("c") - .arg("create") - .arg(&image_name) - .arg(&container_name) - .spawn()? - .wait()? - .success(); - - if !success { - bail!(" failed to create container for image"); - } + oci_helpers::create_container(&container_name, &image_name)?; self.container_name = container_name.clone(); Ok(( @@ -269,10 +220,13 @@ where } pub mod oci_helpers { + use std::fs::{write, File}; use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; use anyhow::{bail, Result}; + use oci_spec::image::{self as spec, Arch}; + use oci_tar_builder::Builder; use super::TEST_NAMESPACE; @@ -308,7 +262,86 @@ pub mod oci_helpers { Ok(()) } + pub fn create_container(container_name: &str, image_name: &str) -> Result<(), anyhow::Error> { + let success = Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("c") + .arg("create") + .arg(image_name) + .arg(container_name) + .spawn()? + .wait()? + .success(); + if !success { + bail!(" failed to create container for image"); + } + Ok(()) + } + + pub struct ImageContent { + pub bytes: Vec, + pub media_type: String, + } + + pub fn import_image( + image_name: &str, + wasm_content: &[&ImageContent], + ) -> Result<(), anyhow::Error> { + let tempdir = tempfile::tempdir()?; + let dir = tempdir.path(); + + let mut builder = Builder::default(); + + for (i, content) in wasm_content.iter().enumerate() { + let path = tempdir.path().join(format!("{}.wasm", i)); + write(path.clone(), content.bytes.clone())?; + builder.add_layer_with_media_type(&path, content.media_type.clone()); + } + + let config = spec::ConfigBuilder::default() + .entrypoint(vec!["_start".to_string()]) + .build() + .unwrap(); + let img = spec::ImageConfigurationBuilder::default() + .config(config) + .os("wasip1") + .architecture(Arch::Wasm) + .rootfs( + spec::RootFsBuilder::default() + .diff_ids(vec![]) + .build() + .unwrap(), + ) + .build()?; + builder.add_config(img, image_name.to_string()); + let img_path = dir.join("img.tar"); + let f = File::create(img_path.clone())?; + builder.build(f)?; + + let success = Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("image") + .arg("import") + .arg("--all-platforms") + .arg(img_path) + .spawn()? + .wait()? + .success(); + if !success { + // if the container still exists try cleaning it up + bail!(" failed to import image"); + }; + Ok(()) + } + pub fn clean_image(image_name: String) -> Result<()> { + let image_sha = match get_image_sha(&image_name) { + Ok(sha) => sha, + Err(_) => return Ok(()), // doesn't exist + }; + log::debug!("deleting image '{}'", image_name); let success = Command::new("ctr") .arg("-n") @@ -324,17 +357,24 @@ pub mod oci_helpers { bail!("failed to clean image"); } - // the content isn't removed immediately, so we need to wait for it to be removed - // otherwise the next test will not behave as expected + // the content is not removed immediately, so we need to wait for it to be removed + // otherwise some tests will not behave as expected + wait_for_content_removal(&image_sha)?; + + Ok(()) + } + + pub fn wait_for_content_removal(content_sha: &str) -> Result<(), anyhow::Error> { let start = Instant::now(); - let timeout = Duration::from_secs(300); + let timeout = Duration::from_secs(60); + log::info!("waiting for content to be removed: {}", &content_sha); loop { let output = Command::new("ctr") .arg("-n") .arg(TEST_NAMESPACE) .arg("content") - .arg("ls") - .arg("-q") + .arg("get") + .arg(content_sha) .output()?; if output.stdout.is_empty() { @@ -342,15 +382,42 @@ pub mod oci_helpers { } if start.elapsed() > timeout { - bail!("timed out waiting for content to be removed"); + log::warn!("didn't clean content fully"); + break; } - - log::trace!("waiting for content to be removed"); } - Ok(()) } + fn get_image_sha(image_name: &str) -> Result { + log::info!("getting image sha for '{}'", image_name); + let mut grep = Command::new("grep") + .arg(image_name) + .stdout(Stdio::piped()) + .stdin(Stdio::piped()) + .spawn()?; + + Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("i") + .arg("ls") + .stdout(grep.stdin.take().unwrap()) + .spawn()?; + + let output = grep.wait_with_output()?; + let stdout = String::from_utf8(output.stdout)?; + log::warn!("stdout: {}", stdout); + + let parts: Vec<&str> = stdout.trim().split(' ').collect(); + if parts.len() < 3 { + bail!("failed to get image sha"); + } + let sha = parts[2]; + log::warn!("sha: {}", sha); + Ok(sha.to_string()) + } + pub fn get_image_label() -> Result<(String, String)> { let mut grep = Command::new("grep") .arg("-ohE") @@ -367,11 +434,52 @@ pub mod oci_helpers { .stdout(grep.stdin.take().unwrap()) .spawn()?; + let output = grep.wait_with_output()?; + let stdout = String::from_utf8(output.stdout)?; + log::debug!("stdout: {}", stdout); + let label: Vec<&str> = stdout.split('=').collect(); + + Ok(( + label.first().unwrap().trim().to_string(), + label.last().unwrap().trim().to_string(), + )) + } + + pub fn image_exists(image_name: &str) -> bool { + let output = Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("i") + .arg("ls") + .arg("--quiet") + .output() + .expect("failed to get output of image list"); + + let stdout = String::from_utf8(output.stdout).expect("failed to parse stdout"); + stdout.contains(image_name) + } + + pub fn get_content_label() -> Result<(String, String)> { + let mut grep = Command::new("grep") + .arg("-ohE") + .arg("runwasi.io/precompiled/[[:alpha:]]*/[0-9]+=.*") + .stdout(Stdio::piped()) + .stdin(Stdio::piped()) + .spawn()?; + + Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("content") + .arg("ls") + .stdout(grep.stdin.take().unwrap()) + .spawn()?; + let output = grep.wait_with_output()?; let stdout = String::from_utf8(output.stdout)?; - log::info!("stdout: {}", stdout); + log::debug!("stdout: {}", stdout); let label: Vec<&str> = stdout.split('=').collect(); diff --git a/crates/containerd-shim-wasmtime/Cargo.toml b/crates/containerd-shim-wasmtime/Cargo.toml index 2daa81e62..c334536c4 100644 --- a/crates/containerd-shim-wasmtime/Cargo.toml +++ b/crates/containerd-shim-wasmtime/Cargo.toml @@ -10,6 +10,7 @@ containerd-shim-wasm = { workspace = true } log = { workspace = true } oci-spec = { workspace = true, features = ["runtime"] } ttrpc = { workspace = true } +sha256 = { workspace = true } # We are not including the `async` feature here: # 1. Because we don't even use it diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index 60585c818..e165cb74e 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -7,6 +7,7 @@ use anyhow::{bail, Context, Result}; use containerd_shim_wasm::container::{ Engine, Entrypoint, Instance, RuntimeContext, Stdio, WasmBinaryType, }; +use containerd_shim_wasm::sandbox::WasmLayer; use wasi_common::I32Exit; use wasmtime::component::{self as wasmtime_component, Component, ResourceTable}; use wasmtime::{Config, Module, Precompiled, Store}; @@ -112,11 +113,21 @@ impl Engine for WasmtimeEngine { Ok(status) } - fn precompile(&self, layers: &[Vec]) -> Result> { - match layers { - [layer] => self.engine.precompile_module(layer), - _ => bail!("only a single module is supported when precompiling"), + fn precompile(&self, layers: &[WasmLayer]) -> Result>>> { + let mut compiled_layers = Vec::>>::with_capacity(layers.len()); + + for layer in layers { + if self.engine.detect_precompiled(&layer.layer).is_some() { + log::info!("Already precompiled"); + compiled_layers.push(None); + continue; + } + + let compiled_layer = self.engine.precompile_module(&layer.layer)?; + compiled_layers.push(Some(compiled_layer)); } + + Ok(compiled_layers) } fn can_precompile(&self) -> Option { diff --git a/crates/containerd-shim-wasmtime/src/tests.rs b/crates/containerd-shim-wasmtime/src/tests.rs index fcab3a50a..a4405192e 100644 --- a/crates/containerd-shim-wasmtime/src/tests.rs +++ b/crates/containerd-shim-wasmtime/src/tests.rs @@ -69,25 +69,30 @@ fn test_hello_world_oci() -> anyhow::Result<()> { fn test_hello_world_oci_uses_precompiled() -> anyhow::Result<()> { let (builder, _oci_cleanup1) = WasiTest::::builder()? .with_wasm(HELLO_WORLD)? - .as_oci_image(None, Some("c1".to_string()))?; + .as_oci_image( + Some("localhost/hello:latest".to_string()), + Some("c1".to_string()), + )?; let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; assert_eq!(exit_code, 0); assert_eq!(stdout, "hello world\n"); - let (label, id) = oci_helpers::get_image_label()?; + let (label, _id) = oci_helpers::get_content_label()?; assert!( label.starts_with("runwasi.io/precompiled/wasmtime/"), - "was {}={}", - label, - id + "was {}", + label ); // run second time, it should succeed without recompiling let (builder, _oci_cleanup2) = WasiTest::::builder()? .with_wasm(HELLO_WORLD)? - .as_oci_image(None, Some("c2".to_string()))?; + .as_oci_image( + Some("localhost/hello:latest".to_string()), + Some("c2".to_string()), + )?; let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; @@ -102,23 +107,32 @@ fn test_hello_world_oci_uses_precompiled() -> anyhow::Result<()> { fn test_hello_world_oci_uses_precompiled_when_content_removed() -> anyhow::Result<()> { let (builder, _oci_cleanup1) = WasiTest::::builder()? .with_wasm(HELLO_WORLD)? - .as_oci_image(None, Some("c1".to_string()))?; + .as_oci_image( + Some("localhost/hello:latest".to_string()), + Some("c1".to_string()), + )?; let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; assert_eq!(exit_code, 0); assert_eq!(stdout, "hello world\n"); - let (label, id) = oci_helpers::get_image_label()?; - // remove the compiled content from the cache - assert!(label.starts_with("runwasi.io/precompiled/wasmtime/")); + let (label, id) = oci_helpers::get_content_label()?; + assert!( + label.starts_with("runwasi.io/precompiled/wasmtime/"), + "was {}", + label + ); oci_helpers::remove_content(id)?; // run second time, it should succeed let (builder, _oci_cleanup2) = WasiTest::::builder()? .with_wasm(HELLO_WORLD)? - .as_oci_image(None, Some("c2".to_string()))?; + .as_oci_image( + Some("localhost/hello:latest".to_string()), + Some("c2".to_string()), + )?; let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; diff --git a/crates/oci-tar-builder/Cargo.toml b/crates/oci-tar-builder/Cargo.toml index a9bb910a5..532132ff7 100644 --- a/crates/oci-tar-builder/Cargo.toml +++ b/crates/oci-tar-builder/Cargo.toml @@ -12,6 +12,7 @@ anyhow = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } clap = { version = "4.5.1", features = ["derive"] } +indexmap = "2.2.5" [lib] path = "src/lib.rs" diff --git a/crates/oci-tar-builder/src/lib.rs b/crates/oci-tar-builder/src/lib.rs index b4f50e36e..2a08f9e50 100644 --- a/crates/oci-tar-builder/src/lib.rs +++ b/crates/oci-tar-builder/src/lib.rs @@ -4,6 +4,7 @@ use std::io::Write; use std::path::PathBuf; use anyhow::{Context, Error, Result}; +use indexmap::IndexMap; use log::{debug, warn}; use oci_spec::image::{ DescriptorBuilder, ImageConfiguration, ImageIndexBuilder, ImageManifestBuilder, MediaType, @@ -63,7 +64,8 @@ impl Builder { pub fn build(&mut self, w: W) -> Result<(), Error> { let mut tb = tar::Builder::new(w); let mut manifests = Vec::new(); - let mut layer_digests = HashMap::new(); + // use IndexMap in order to keep layers in order they were added. + let mut layer_digests = IndexMap::new(); if self.configs.len() > 1 { anyhow::bail!("only one config is supported");