From 26315d28b1b0b54f7ee0bdd35a1187046838a33b Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Thu, 12 Sep 2024 22:59:29 +0000 Subject: [PATCH] deprecate: remove 'shared' mode Signed-off-by: jiaxiao zhou --- Cargo.lock | 5 - README.md | 66 +---- .../Cargo.toml | 5 - crates/containerd-shim-wasm/build.rs | 35 --- .../containerd-shim-wasm/protos/sandbox.proto | 41 --- crates/containerd-shim-wasm/src/lib.rs | 1 - .../containerd-shim-wasm/src/sandbox/cli.rs | 40 +-- .../src/sandbox/manager.rs | 262 ------------------ .../containerd-shim-wasm/src/sandbox/mod.rs | 4 - .../src/sandbox/shim/local.rs | 19 +- .../src/sandbox/shim/mod.rs | 1 - crates/containerd-shim-wasm/src/services.rs | 6 - 12 files changed, 5 insertions(+), 480 deletions(-) delete mode 100644 crates/containerd-shim-wasm/build.rs delete mode 100644 crates/containerd-shim-wasm/protos/sandbox.proto delete mode 100644 crates/containerd-shim-wasm/src/sandbox/manager.rs delete mode 100644 crates/containerd-shim-wasm/src/services.rs diff --git a/Cargo.lock b/Cargo.lock index 40b21f81d..acdb0c5b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -766,13 +766,8 @@ name = "containerd-shim-wasm-test-modules" version = "0.4.1" dependencies = [ "anyhow", - "env_logger", "lazy_static", "libc", - "log", - "oci-spec", - "serde_json", - "tempfile", "wat", ] diff --git a/README.md b/README.md index 722d875a5..c0517fad5 100644 --- a/README.md +++ b/README.md @@ -18,14 +18,9 @@ on the CNCF slack. ## Usage -runwasi is intended to be consumed as a library to be linked to from your own wasm host implementation. +runwasi is intended to be consumed as a library to be linked to from your own wasm host implementation. It creates one shim process per container or k8s pod. -There are two modes of operation supported: - -1. "Normal" mode where there is 1 shim process per container or k8s pod. -2. "Shared" mode where there is a single manager service running all shims in process. - -In either case you need to implement a trait to teach runwasi how to use your wasm host. +You need to implement a trait to teach runwasi how to use your wasm host. There are two ways to do this: * implementing the `sandbox::Instance` trait @@ -119,45 +114,6 @@ I encourage you to checkout how that is implemented. The shim binary just needs to be installed into `$PATH` (as seen by the containerd process) with a binary name like `containerd-shim-myshim-v1`. -For the shared mode: - -```rust -use containerd_shim_wasm::sandbox::{Local, ManagerService, Instance}; -use containerd_shim_wasm::services::sandbox_ttrpc::{create_manager, Manager}; -use std::sync::Arc; -use ttrpc::{self, Server}; -/// ... - -struct MyInstance { - /// ... -} - -impl Instance for MyInstance { - // ... -} - -fn main() { - let s: ManagerService> = - ManagerService::new(Engine::new(Config::new().interruptable(true)).unwrap()); - let s = Arc::new(Box::new(s) as Box); - let service = create_manager(s); - - let mut server = Server::new() - .bind("unix:///run/io.containerd.myshim.v1/manager.sock") - .unwrap() - .register_service(service); - - server.start().unwrap(); - let (_tx, rx) = std::sync::mpsc::channel::<()>(); - rx.recv().unwrap(); -} -``` - -This will be the host daemon that you startup and manage on your own. -You can use the provided `containerd-shim-myshim-v1` binary as the shim to specify in containerd. - -Shared mode requires precise control over real threads and as such should not be used with an async runtime. - Check out these projects that build on top of runwasi: - [spinkube/containerd-shim-spin](https://github.com/spinkube/containerd-shim-spin) - [deislabs/containerd-wasm-shims](https://github.com/deislabs/containerd-wasm-shims) @@ -174,24 +130,6 @@ And make sure the shim binary must be in $PATH (that is the $PATH that container This shim runs one per pod. -- **containerd-shim-[ wasmedge | wasmtime | wasmer ]d-v1** - -A cli used to connect containerd to the `containerd-[ wasmedge | wasmtime | wasmer ]d` sandbox daemon. -When containerd requests for a container to be created, it fires up this shim binary which will connect to the `containerd-[ wasmedge | wasmtime | wasmer ]d` service running on the host. -The service will return a path to a unix socket which this shim binary will write back to containerd which containerd will use to connect to for shim requests. -This binary does not serve requests, it is only responsible for sending requests to the `containerd-[ wasmedge | wasmtime | wasmer ]d` daemon to create or destroy sandboxes. - -- **containerd-[ wasmedge | wasmtime | wasmer ]d** - -This is a sandbox manager that enables running 1 wasm host for the entire node instead of one per pod (or container). -When a container is created, a request is sent to this service to create a sandbox. -The "sandbox" is a containerd task service that runs in a new thread on its own unix socket, which we return back to containerd to connect to. - -The Wasmedge / Wasmtime / Wasmer engine is shared between all sandboxes in the service. - -To use this shim, specify `io.containerd.[ wasmedge | wasmtime | wasmer ]d.v1` as the runtime to use. -You will need to make sure the `containerd-[ wasmedge | wasmtime | wasmer ]d` daemon has already been started. - ## Contributing To begin contributing, learn to build and test the project or to add a new shim please read our [CONTRIBUTING.md](./CONTRIBUTING.md) diff --git a/crates/containerd-shim-wasm-test-modules/Cargo.toml b/crates/containerd-shim-wasm-test-modules/Cargo.toml index 7a3db8e63..a5f3572ce 100644 --- a/crates/containerd-shim-wasm-test-modules/Cargo.toml +++ b/crates/containerd-shim-wasm-test-modules/Cargo.toml @@ -7,12 +7,7 @@ license.workspace = true [dependencies] anyhow = { workspace = true } -env_logger = { workspace = true } libc = { workspace = true } -log = { workspace = true } -oci-spec = { workspace = true } -serde_json = { workspace = true } -tempfile = { workspace = true } [build-dependencies] anyhow = { workspace = true } diff --git a/crates/containerd-shim-wasm/build.rs b/crates/containerd-shim-wasm/build.rs deleted file mode 100644 index 3c22a1195..000000000 --- a/crates/containerd-shim-wasm/build.rs +++ /dev/null @@ -1,35 +0,0 @@ -use std::env::var_os; -use std::path::Path; - -use ttrpc_codegen::{Codegen, ProtobufCustomize}; - -fn main() { - let protos = ["protos/sandbox.proto"]; - println!("cargo:rerun-if-changed=protos/sandbox.proto"); - - let out_dir = var_os("OUT_DIR").unwrap(); - let out_dir = Path::new(&out_dir); - - Codegen::new() - .out_dir(out_dir) - .inputs(protos) - .include("protos") - .rust_protobuf() - .rust_protobuf_customize(ProtobufCustomize::default().gen_mod_rs(false)) - .run() - .expect("failed to generate code"); - - let sanbox_rs = out_dir.join("sandbox.rs"); - let sanbox_ttrpc_rs = out_dir.join("sandbox_ttrpc.rs"); - - std::fs::write( - out_dir.join("mod.rs"), - format!( - r#" -#[path = {sanbox_rs:?}] pub mod sandbox; -#[path = {sanbox_ttrpc_rs:?}] pub mod sandbox_ttrpc; -"#, - ), - ) - .expect("failed to generate module"); -} diff --git a/crates/containerd-shim-wasm/protos/sandbox.proto b/crates/containerd-shim-wasm/protos/sandbox.proto deleted file mode 100644 index 733ee2cc1..000000000 --- a/crates/containerd-shim-wasm/protos/sandbox.proto +++ /dev/null @@ -1,41 +0,0 @@ -syntax = "proto3"; - -package runwasi.services.sandbox.v1; -option go_package = "github.com/cpuguy83/runwasi/api/services/sandbox/v1;sandbox"; - -service Manager { - rpc Create(CreateRequest) returns (CreateResponse) {} - rpc Connect(ConnectRequest) returns (ConnectResponse) {} - rpc Delete(DeleteRequest) returns (DeleteResponse) {} -} - -message CreateRequest { - string namespace = 1; - string id = 2; - string ttrpc_address = 3; - string working_directory = 4; - string containerd_address = 5; -} - -message CreateResponse { - // Path of the socket file serving the shim API - string socket_path = 1; -} - -message ConnectRequest { - string id = 1; - string ttrpc_address = 2; -} - -message ConnectResponse { - // Path of the socket file serving the shim API - string socket_path = 1; -} - -message DeleteRequest { - string namespace = 1; - string id = 2; - string ttrpc_address = 3; -} - -message DeleteResponse {} diff --git a/crates/containerd-shim-wasm/src/lib.rs b/crates/containerd-shim-wasm/src/lib.rs index 5d7a33158..01cbcc179 100644 --- a/crates/containerd-shim-wasm/src/lib.rs +++ b/crates/containerd-shim-wasm/src/lib.rs @@ -5,7 +5,6 @@ pub mod container; pub mod sandbox; -pub mod services; #[cfg_attr(unix, path = "sys/unix/mod.rs")] #[cfg_attr(windows, path = "sys/windows/mod.rs")] diff --git a/crates/containerd-shim-wasm/src/sandbox/cli.rs b/crates/containerd-shim-wasm/src/sandbox/cli.rs index 732f0a0d6..80b90f3e7 100644 --- a/crates/containerd-shim-wasm/src/sandbox/cli.rs +++ b/crates/containerd-shim-wasm/src/sandbox/cli.rs @@ -1,16 +1,10 @@ use std::path::PathBuf; -use std::sync::mpsc::channel; -use std::sync::Arc; use containerd_shim::{parse, run, Config}; -use ttrpc::Server; -use crate::sandbox::manager::Shim; -use crate::sandbox::shim::Local; #[cfg(feature = "opentelemetry")] use crate::sandbox::shim::{otel_traces_enabled, OTLPConfig}; -use crate::sandbox::{Instance, ManagerService, ShimCli}; -use crate::services::sandbox_ttrpc::{create_manager, Manager}; +use crate::sandbox::{Instance, ShimCli}; pub mod r#impl { pub use git_version::git_version; @@ -115,37 +109,7 @@ fn shim_main_inner<'a, I>( let shim_version = shim_version.into().unwrap_or("v1"); let lower_name = name.to_lowercase(); - let shim_cli = format!("containerd-shim-{lower_name}-{shim_version}"); - let shim_client = format!("containerd-shim-{lower_name}d-{shim_version}"); - let shim_daemon = format!("containerd-{lower_name}d"); let shim_id = format!("io.containerd.{lower_name}.{shim_version}"); - match argv0.to_lowercase() { - s if s == shim_cli => { - run::>(&shim_id, config); - } - s if s == shim_client => { - run::(&shim_client, config); - } - s if s == shim_daemon => { - log::info!("starting up!"); - let s: ManagerService> = Default::default(); - let s = Arc::new(Box::new(s) as Box); - let service = create_manager(s); - - let mut server = Server::new() - .bind("unix:///run/io.containerd.wasmwasi.v1/manager.sock") - .expect("failed to bind to socket") - .register_service(service); - - server.start().expect("failed to start daemon"); - log::info!("server started!"); - let (_tx, rx) = channel::<()>(); - rx.recv().unwrap(); - } - _ => { - eprintln!("error: unrecognized binary name, expected one of {shim_cli}, {shim_client}, or {shim_daemon}."); - std::process::exit(1); - } - } + run::>(&shim_id, config); } diff --git a/crates/containerd-shim-wasm/src/sandbox/manager.rs b/crates/containerd-shim-wasm/src/sandbox/manager.rs deleted file mode 100644 index a9c48f26f..000000000 --- a/crates/containerd-shim-wasm/src/sandbox/manager.rs +++ /dev/null @@ -1,262 +0,0 @@ -//! This experimental module implements a manager service which can be used to -//! manage multiple instances of a sandbox in-process. -//! The idea behind this module is to only need a single shim process for the entire node rather than one per pod/container. - -use std::collections::HashMap; -use std::env::current_dir; -use std::path::Path; -use std::sync::{Arc, RwLock}; -use std::thread; - -use anyhow::Context; -use containerd_shim::error::Error as ShimError; -use containerd_shim::protos::shim::shim_ttrpc::{create_task, Task}; -use containerd_shim::protos::ttrpc::{Client, Server}; -use containerd_shim::protos::TaskClient; -use containerd_shim::publisher::RemotePublisher; -use containerd_shim::{self as shim, api, TtrpcContext, TtrpcResult}; -use oci_spec::runtime::{self, Spec}; -use shim::Flags; -use ttrpc::context; - -use super::error::Error; -use super::instance::Instance; -use super::sandbox; -use crate::services::sandbox_ttrpc::{Manager, ManagerClient}; -use crate::sys::networking::setup_namespaces; - -/// Sandbox wraps an Instance and is used with the `Service` to manage multiple instances. -pub trait Sandbox: Task + Send + Sync { - type Instance: Instance; - - fn new( - namespace: String, - containerd_address: String, - id: String, - engine: ::Engine, - publisher: RemotePublisher, - ) -> Self; -} - -/// Service is a manager service which can be used to manage multiple instances of a sandbox in-process. -pub struct Service { - sandboxes: RwLock>, - engine: ::Engine, - phantom: std::marker::PhantomData, -} - -impl Service { - pub fn new(engine: ::Engine) -> Self { - Self { - sandboxes: RwLock::new(HashMap::new()), - engine, - phantom: std::marker::PhantomData, - } - } -} - -impl Default for Service -where - ::Engine: Default, -{ - fn default() -> Self { - Self::new(Default::default()) - } -} - -impl Manager for Service { - fn create( - &self, - _ctx: &TtrpcContext, - req: sandbox::CreateRequest, - ) -> TtrpcResult { - let mut sandboxes = self.sandboxes.write().unwrap(); - - if sandboxes.contains_key(&req.id) { - return Err(Error::AlreadyExists(req.id).into()); - } - - let sock = format!("unix://{}/shim.sock", &req.working_directory); - - let publisher = RemotePublisher::new(req.ttrpc_address)?; - - let sb = T::new( - req.namespace.clone(), - req.containerd_address.clone(), - req.id.clone(), - self.engine.clone(), - publisher, - ); - let task_service = create_task(Arc::new(Box::new(sb))); - let mut server = Server::new().bind(&sock)?.register_service(task_service); - - sandboxes.insert(req.id.clone(), sock.clone()); - - let cfg = Spec::load( - Path::new(&req.working_directory) - .join("config.json") - .to_str() - .unwrap(), - ) - .map_err(|err| Error::InvalidArgument(format!("could not load runtime spec: {}", err)))?; - - let (tx, rx) = std::sync::mpsc::channel::>(); - - let id = &req.id; - - let _ = thread::Builder::new() - .name(format!("{}-sandbox-create", id)) - .spawn(move || { - let r = start_sandbox(cfg, &mut server); - tx.send(r).context("could not send sandbox result").unwrap(); - }) - .context("failed to spawn sandbox thread") - .map_err(Error::from)?; - - rx.recv() - .context("could not receive sandbox result") - .map_err(Error::from)??; - Ok(sandbox::CreateResponse { - socket_path: sock, - ..Default::default() - }) - } - - fn delete( - &self, - _ctx: &TtrpcContext, - req: sandbox::DeleteRequest, - ) -> TtrpcResult { - let mut sandboxes = self.sandboxes.write().unwrap(); - if !sandboxes.contains_key(&req.id) { - return Err(Error::NotFound(req.id).into()); - } - let sock = sandboxes.remove(&req.id).unwrap(); - let c = Client::connect(&sock)?; - let tc = TaskClient::new(c); - - tc.shutdown( - context::Context::default(), - &api::ShutdownRequest { - id: req.id, - now: true, - ..Default::default() - }, - )?; - - Ok(sandbox::DeleteResponse::default()) - } -} - -// Note that this changes the current thread's state. -// You probably want to run this in a new thread. -fn start_sandbox(cfg: runtime::Spec, server: &mut Server) -> Result<(), Error> { - setup_namespaces(&cfg)?; - - server.start_listen().context("could not start listener")?; - Ok(()) -} - -/// Shim implements the containerd-shim CLI for connecting to a Manager service. -pub struct Shim { - id: String, - namespace: String, -} - -impl Task for Shim {} - -impl shim::Shim for Shim { - type T = Self; - - fn new(_runtime_id: &str, args: &Flags, _config: &mut shim::Config) -> Self { - Shim { - id: args.id.to_string(), - namespace: args.namespace.to_string(), - } - } - - fn start_shim(&mut self, opts: containerd_shim::StartOpts) -> shim::Result { - let dir = current_dir().map_err(|err| ShimError::Other(err.to_string()))?; - let spec = Spec::load(dir.join("config.json").to_str().unwrap()).map_err(|err| { - shim::Error::InvalidArgument(format!("error loading runtime spec: {}", err)) - })?; - - let default = HashMap::new() as HashMap; - let annotations = spec.annotations().as_ref().unwrap_or(&default); - - let sandbox = annotations - .get("io.kubernetes.cri.sandbox-id") - .unwrap_or(&opts.id) - .to_string(); - - let client = Client::connect("unix:///run/io.containerd.wasmwasi.v1/manager.sock")?; - let mc = ManagerClient::new(client); - - let addr = match mc.create( - context::Context::default(), - &sandbox::CreateRequest { - id: sandbox.clone(), - working_directory: dir.as_path().to_str().unwrap().to_string(), - ttrpc_address: opts.ttrpc_address.clone(), - ..Default::default() - }, - ) { - Ok(res) => res.socket_path, - Err(_) => { - let res = mc.connect( - context::Context::default(), - &sandbox::ConnectRequest { - id: sandbox, - ttrpc_address: opts.ttrpc_address, - ..Default::default() - }, - )?; - res.socket_path - } - }; - - shim::util::write_address(&addr)?; - - Ok(addr) - } - - fn wait(&mut self) { - todo!() - } - - fn create_task_service(&self, _publisher: RemotePublisher) -> Self::T { - todo!() // but not really, haha - } - - fn delete_shim(&mut self) -> shim::Result { - let dir = current_dir().map_err(|err| ShimError::Other(err.to_string()))?; - let spec = Spec::load(dir.join("config.json").to_str().unwrap()).map_err(|err| { - shim::Error::InvalidArgument(format!("error loading runtime spec: {}", err)) - })?; - - let default = HashMap::new() as HashMap; - let annotations = spec.annotations().as_ref().unwrap_or(&default); - - let sandbox = annotations - .get("io.kubernetes.cri.sandbox-id") - .unwrap_or(&self.id) - .to_string(); - if sandbox != self.id { - return Ok(api::DeleteResponse::default()); - } - - let client = Client::connect("unix:///run/io.containerd.wasmwasi.v1/manager.sock")?; - let mc = ManagerClient::new(client); - mc.delete( - context::Context::default(), - &sandbox::DeleteRequest { - id: sandbox, - namespace: self.namespace.clone(), - ..Default::default() - }, - )?; - - // TODO: write pid, exit code, etc to disk so we can use it here. - Ok(api::DeleteResponse::default()) - } -} diff --git a/crates/containerd-shim-wasm/src/sandbox/mod.rs b/crates/containerd-shim-wasm/src/sandbox/mod.rs index 872715717..157bd9009 100644 --- a/crates/containerd-shim-wasm/src/sandbox/mod.rs +++ b/crates/containerd-shim-wasm/src/sandbox/mod.rs @@ -1,19 +1,15 @@ //! Abstracts the sandboxing environment and execution context for a container. -use crate::services::sandbox; - pub mod cli; pub mod error; pub mod instance; pub mod instance_utils; -pub mod manager; pub mod shim; pub mod stdio; pub mod sync; pub use error::{Error, Result}; pub use instance::{Instance, InstanceConfig}; -pub use manager::{Sandbox as SandboxService, Service as ManagerService}; pub use shim::Cli as ShimCli; pub use stdio::Stdio; diff --git a/crates/containerd-shim-wasm/src/sandbox/shim/local.rs b/crates/containerd-shim-wasm/src/sandbox/shim/local.rs index dab46dd5b..41158f4c1 100644 --- a/crates/containerd-shim-wasm/src/sandbox/shim/local.rs +++ b/crates/containerd-shim-wasm/src/sandbox/shim/local.rs @@ -16,7 +16,6 @@ use containerd_shim::error::Error as ShimError; use containerd_shim::protos::events::task::{TaskCreate, TaskDelete, TaskExit, TaskIO, TaskStart}; use containerd_shim::protos::shim::shim_ttrpc::Task; use containerd_shim::protos::types::task::Status; -use containerd_shim::publisher::RemotePublisher; use containerd_shim::util::IntoOption; use containerd_shim::{DeleteResponse, ExitSignal, TtrpcContext, TtrpcResult}; use log::debug; @@ -25,7 +24,7 @@ use oci_spec::runtime::Spec; use crate::sandbox::instance::{Instance, InstanceConfig}; use crate::sandbox::shim::events::{EventSender, RemoteEventSender, ToTimestamp}; use crate::sandbox::shim::instance_data::InstanceData; -use crate::sandbox::{oci, Error, Result, SandboxService}; +use crate::sandbox::{oci, Error, Result}; use crate::sys::metrics::get_metrics; #[cfg(test)] @@ -343,22 +342,6 @@ impl Local { } } -impl SandboxService for Local { - type Instance = T; - #[cfg_attr(feature = "tracing", tracing::instrument(parent = tracing::Span::current(), skip_all, level = "Info"))] - fn new( - namespace: String, - containerd_address: String, - _id: String, - engine: T::Engine, - publisher: RemotePublisher, - ) -> Self { - let events = RemoteEventSender::new(&namespace, publisher); - let exit = Arc::default(); - Local::::new(engine, events, exit, namespace, containerd_address) - } -} - impl Task for Local { #[cfg_attr(feature = "tracing", tracing::instrument(parent = tracing::Span::current(), skip_all, level = "Info"))] fn create(&self, _: &TtrpcContext, req: CreateTaskRequest) -> TtrpcResult { diff --git a/crates/containerd-shim-wasm/src/sandbox/shim/mod.rs b/crates/containerd-shim-wasm/src/sandbox/shim/mod.rs index a5ce3eb1a..851f749a8 100644 --- a/crates/containerd-shim-wasm/src/sandbox/shim/mod.rs +++ b/crates/containerd-shim-wasm/src/sandbox/shim/mod.rs @@ -12,6 +12,5 @@ mod otel; mod task_state; pub use cli::Cli; -pub(crate) use local::Local; #[cfg(feature = "opentelemetry")] pub use otel::{traces_enabled as otel_traces_enabled, Config as OTLPConfig}; diff --git a/crates/containerd-shim-wasm/src/services.rs b/crates/containerd-shim-wasm/src/services.rs deleted file mode 100644 index 6850a5f4f..000000000 --- a/crates/containerd-shim-wasm/src/services.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Generated service definitions from the protobuf definitions. - -// do not run clippy on generated files -#![allow(clippy::all)] - -include!(concat!(env!("OUT_DIR"), "/mod.rs"));