From 39fe05ad56358474baac4ccb392b97029512f7bb Mon Sep 17 00:00:00 2001 From: David Drysdale Date: Fri, 20 Mar 2020 11:27:06 +0000 Subject: [PATCH] runtime/glue: add C++->Rust FFI glue code Use a mutable global to hold references to the Rust runtime, and to the C++ pseudo-Node factory function that's registered. Use a local handle mapping to allow opaque Rust types to be recovered. --- Cargo.lock | 13 ++ Cargo.toml | 1 + oak/server/rust/oak_glue/BUILD | 58 +++++ oak/server/rust/oak_glue/Cargo.toml | 21 ++ oak/server/rust/oak_glue/oak_glue.h | 56 +++++ oak/server/rust/oak_glue/src/lib.rs | 334 ++++++++++++++++++++++++++++ oak/server/rust/oak_runtime/BUILD | 1 + 7 files changed, 484 insertions(+) create mode 100644 oak/server/rust/oak_glue/BUILD create mode 100644 oak/server/rust/oak_glue/Cargo.toml create mode 100644 oak/server/rust/oak_glue/oak_glue.h create mode 100644 oak/server/rust/oak_glue/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 3f90cb22406..19fab430d72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1183,6 +1183,19 @@ dependencies = [ "prost-build 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "oak_glue" +version = "0.1.0" +dependencies = [ + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "oak_abi 0.1.0", + "oak_runtime 0.1.0", + "prost 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "simple_logger 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "oak_runtime" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 1bf84135b26..a5064fdce3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ members = [ "experimental/split_grpc/proxy", "experimental/split_grpc/server", "oak/server/rust/oak_abi", + "oak/server/rust/oak_glue", "oak/server/rust/oak_runtime", "runner", "sdk/rust/oak", diff --git a/oak/server/rust/oak_glue/BUILD b/oak/server/rust/oak_glue/BUILD new file mode 100644 index 00000000000..ad6e5dc6644 --- /dev/null +++ b/oak/server/rust/oak_glue/BUILD @@ -0,0 +1,58 @@ +# +# Copyright 2019 The Project Oak Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@io_bazel_rules_rust//rust:rust.bzl", "rust_library") +load("@rules_cc//cc:defs.bzl", "cc_library") + +package( + default_visibility = ["//oak/server:__subpackages__"], + licenses = ["notice"], +) + +rust_library( + name = "oak_glue", + srcs = glob(["src/**/*.rs"]), + crate_type = "staticlib", + edition = "2018", + deps = [ + "//cargo:byteorder", + "//cargo:lazy_static", + "//cargo:log", + "//cargo:prost", + "//cargo:simple_logger", + "//oak/server/rust/oak_abi", + "//oak/server/rust/oak_runtime", + ], +) + +# Wrapper rule to expose the resulting static library as a statically linked cc_library and +# corresponding header so that it can be depended on by other cc_library and cc_binary rules. +# +# TODO: There seems to be something wrong with this rule related to caching in Bazel. +# To reproduce: +# - change src/lib.rs and introduce a syntax error +# - bazel build :oak_glue_wrapper +# - this should produce a compile error, instead Bazel is still caching the old artifact +# +# However, building a cc_binary target (e.g. //oak/server/dev:dev_oak_runner) does force +# a rebuild of the Rust code. +cc_library( + name = "oak_glue_wrapper", + srcs = [":oak_glue"], + hdrs = ["oak_glue.h"], + linkopts = ["-ldl"], + linkstatic = True, +) diff --git a/oak/server/rust/oak_glue/Cargo.toml b/oak/server/rust/oak_glue/Cargo.toml new file mode 100644 index 00000000000..57f5aa60239 --- /dev/null +++ b/oak/server/rust/oak_glue/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "oak_glue" +version = "0.1.0" +authors = ["David Drysdale "] +edition = "2018" +license = "Apache-2.0" + +[lib] +name = "oak_glue" + +[dependencies] +# Note that if new dependencies are added here: +# - they need to be synced to //cargo/Cargo.toml +# - `cd cargo && cargo raze` needs re-running, and the results checked in +byteorder = "*" +lazy_static = "*" +log = { version = "*", features = ["std"] } +oak_abi = { version = "=0.1.0" } +oak_runtime = { version = "=0.1.0" } +prost = "*" +simple_logger = "*" diff --git a/oak/server/rust/oak_glue/oak_glue.h b/oak/server/rust/oak_glue/oak_glue.h new file mode 100644 index 00000000000..b1c420ee87a --- /dev/null +++ b/oak/server/rust/oak_glue/oak_glue.h @@ -0,0 +1,56 @@ +/* + * Copyright 2019 The Project Oak Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OAK_SERVER_RUST_OAK_GLUE_H_ +#define OAK_SERVER_RUST_OAK_GLUE_H_ + +#include + +extern "C" { + +// Perform start of day initialization. Must only be called once, before +// any other functions. +void glue_init(); + +// Function pointer used for up-calling from Rust to C++ to create and run a pseudo-Node. +// - The data parameter has the factory_data value registered at glue_start. +// - [name, name+name_len) gives the config name for the Node to be started. +// - node_id should be used for all glue_() invocations by the new Node. +// - handle is a read half of an initial channel. +typedef void (*node_factory)(uintptr_t data, const char* name, uint32_t name_len, uint64_t node_id, + uint64_t handle); + +// Start the Rust runtime, passing a serialized ApplicationConfiguration +// protobuf message. +uint64_t glue_start(const uint8_t* config_buf, uint32_t config_len, node_factory factory, + uintptr_t factory_data); +// Stop the Rust runtime. +void glue_stop(); + +// The following functions are analogous to those on the Oak ABI, with the +// addition of an initial node_id parameter that identifies the Node performing +// the operation. +uint32_t glue_wait_on_channels(uint64_t node_id, uint8_t* buf, uint32_t count); +uint32_t glue_channel_read(uint64_t node_id, uint64_t handle, uint8_t* buf, uint32_t size, + uint32_t* actual_size, uint8_t* handle_buf, uint32_t handle_count, + uint32_t* actual_handle_count); +uint32_t glue_channel_write(uint64_t node_id, uint64_t handle, const uint8_t* buf, uint32_t size, + const uint8_t* handle_buf, uint32_t handle_count); +uint32_t glue_channel_create(uint64_t node_id, uint64_t* write, uint64_t* read); +uint32_t glue_channel_close(uint64_t node_id, uint64_t handle); +}; + +#endif // OAK_SERVER_RUST_OAK_GLUE_H_ diff --git a/oak/server/rust/oak_glue/src/lib.rs b/oak/server/rust/oak_glue/src/lib.rs new file mode 100644 index 00000000000..8b3cf01d9bb --- /dev/null +++ b/oak/server/rust/oak_glue/src/lib.rs @@ -0,0 +1,334 @@ +// +// Copyright 2020 The Project Oak Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +use byteorder::{ReadBytesExt, WriteBytesExt}; +use lazy_static::lazy_static; +use log::{debug, error, info, warn}; +use oak_abi::OakStatus; +use oak_runtime::proto::ApplicationConfiguration; +use oak_runtime::runtime::RuntimeProxy; +use oak_runtime::NodeId; +use prost::Message; +use std::convert::TryInto; +use std::io::Cursor; +use std::sync::{Arc, RwLock}; + +#[no_mangle] +pub extern "C" fn glue_init() { + let _ = ::std::panic::catch_unwind(|| { + simple_logger::init().expect("failed to initialize logger"); + info!("Rust FFI glue initialized"); + }); +} + +type NodeFactory = + extern "C" fn(data: usize, name: *const u8, name_len: u32, node_id: u64, handle: u64) -> (); + +struct Glue { + runtime: Arc, + factory: Option, + factory_data: usize, +} + +impl Glue { + fn new( + runtime: Arc, + factory: Option, + factory_data: usize, + ) -> Self { + Glue { + runtime, + factory, + factory_data, + } + } + /// Recreate a RuntimeProxy instance that corresponds to the given node ID + /// value. + fn proxy(&self, node_id: u64) -> RuntimeProxy { + RuntimeProxy { + runtime: self.runtime.clone(), + node_id: NodeId(node_id), + } + } +} + +lazy_static! { + static ref GLUE: RwLock> = RwLock::new(None); +} + +const R1: &str = "global glue lock poisoned"; +const R2: &str = "global glue object missing"; + +/// Start the Rust runtime, with the ApplicationConfiguration provided in +/// serialized form. +/// +/// # Safety +/// +/// Caller must ensure that the memory range [config_buf, config_buf+config_len) is +/// accessible and holds a protobuf-serialized ApplicationConfiguration message. +#[no_mangle] +pub unsafe extern "C" fn glue_start( + config_buf: *const u8, + config_len: u32, + factory: Option, + factory_data: usize, +) -> u64 { + std::panic::catch_unwind(|| { + let config_data = std::slice::from_raw_parts(config_buf, config_len as usize); + + let app_config = match ApplicationConfiguration::decode(config_data) { + Ok(c) => c, + Err(e) => { + error!("Failed to decode ApplicationConfiguration: {}", e); + return oak_abi::INVALID_HANDLE; + } + }; + info!( + "start runtime with initial config {}.{}", + app_config.initial_node_config_name, app_config.initial_entrypoint_name + ); + let (runtime, grpc_handle) = match oak_runtime::configure_and_run(app_config) { + Ok(p) => p, + Err(status) => { + error!("Failed to start runtime: {:?}", status); + return oak_abi::INVALID_HANDLE; + } + }; + + oak_runtime::node::external::register_factory(create_and_run_node); + info!("register oak_glue::create_and_run_node() as node factory"); + + let glue = Glue::new(runtime, factory, factory_data); + *GLUE.write().expect(R1) = Some(glue); + grpc_handle.0 + }) + .unwrap_or(oak_abi::INVALID_HANDLE) +} + +#[no_mangle] +pub extern "C" fn glue_stop() { + let runtime = GLUE.read().expect(R1).as_ref().expect(R2).runtime.clone(); + warn!("stopping Rust runtime"); + runtime.stop(); + *GLUE.write().expect(R1) = None; +} + +/// # Safety +/// +/// The memory range [buf, buf+count*SPACE_BYTES_PER_HANDLE) must be +/// valid. +#[no_mangle] +pub unsafe extern "C" fn glue_wait_on_channels(node_id: u64, buf: *mut u8, count: u32) -> u32 { + std::panic::catch_unwind(|| { + debug!( + "{{{}}}: wait_on_channels(buf={:?}, count={})", + node_id, buf, count + ); + + // Accumulate the handles we're interested in. + let size = oak_abi::SPACE_BYTES_PER_HANDLE * count as usize; + let mut handle_data = Vec::::with_capacity(size); + std::ptr::copy_nonoverlapping(buf, handle_data.as_mut_ptr(), size); + handle_data.set_len(size); + let mut handles = Vec::with_capacity(count as usize); + let mut mem_reader = Cursor::new(handle_data); + for _ in 0..count { + let handle = mem_reader.read_u64::().unwrap(); + let _b = mem_reader.read_u8().unwrap(); + debug!("{{{}}}: wait_on_channels handle {:?}", node_id, handle); + handles.push(Some(oak_runtime::Handle(handle))); + } + + let proxy = GLUE.read().expect(R1).as_ref().expect(R2).proxy(node_id); + let channel_statuses = match proxy.wait_on_channels(&handles) { + Ok(r) => r, + Err(s) => return s as u32, + }; + + if channel_statuses.len() != handles.len() { + error!( + "length mismatch: submitted {} handles, got {} results", + handles.len(), + channel_statuses.len() + ); + return OakStatus::ErrInternal as u32; + } + for (i, channel_status) in channel_statuses.iter().enumerate() { + // Write channel status back to the raw pointer. + let p = buf + .add(i * oak_abi::SPACE_BYTES_PER_HANDLE + (oak_abi::SPACE_BYTES_PER_HANDLE - 1)); + + *p = *channel_status as u8; + } + OakStatus::Ok as u32 + }) + .unwrap_or(OakStatus::ErrInternal as u32) +} + +/// # Safety +/// +/// The memory ranges [buf, buf+size) and [handle_buf, handle_buf+handle_count*8) must be +/// valid, as must the raw pointers actual_size and actual_handle_count. +#[no_mangle] +pub unsafe extern "C" fn glue_channel_read( + node_id: u64, + handle: u64, + buf: *mut u8, + size: usize, + actual_size: *mut u32, + handle_buf: *mut u8, + handle_count: u32, + actual_handle_count: *mut u32, +) -> u32 { + debug!( + "{{{}}}: channel_read(h={}, buf={:?}, size={}, &actual_size={:?}, handle_buf={:?}, count={}, &actual_count={:?})", + node_id, handle, buf, size, actual_size, handle_buf, handle_count, actual_handle_count + ); + + let proxy = GLUE.read().expect(R1).as_ref().expect(R2).proxy(node_id); + let msg = match proxy.channel_try_read_message( + oak_runtime::Handle(handle), + size, + handle_count as usize, + ) { + Ok(msg) => msg, + Err(status) => return status as u32, + }; + + let result = match msg { + None => { + *actual_size = 0; + *actual_handle_count = 0; + OakStatus::ErrChannelEmpty + } + Some(oak_runtime::runtime::ReadStatus::Success(msg)) => { + *actual_size = msg.data.len().try_into().unwrap(); + *actual_handle_count = msg.channels.len().try_into().unwrap(); + std::ptr::copy_nonoverlapping(msg.data.as_ptr(), buf, msg.data.len()); + let mut handle_data = Vec::with_capacity(8 * msg.channels.len()); + for handle in msg.channels { + debug!("{{{}}}: channel_read() added {:?}", node_id, handle); + handle_data + .write_u64::(handle.0) + .unwrap(); + } + std::ptr::copy_nonoverlapping(handle_data.as_ptr(), handle_buf, handle_data.len()); + OakStatus::Ok + } + Some(oak_runtime::runtime::ReadStatus::NeedsCapacity(a, b)) => { + *actual_size = a.try_into().unwrap(); + *actual_handle_count = b.try_into().unwrap(); + if a > size as usize { + OakStatus::ErrBufferTooSmall + } else { + OakStatus::ErrHandleSpaceTooSmall + } + } + }; + + debug!("{{{}}}: channel_read() -> {:?}", node_id, result); + result as u32 +} + +/// # Safety +/// +/// The memory ranges [buf, buf+size) and [handle_buf, handle_buf+handle_count*8) must be +/// valid. +#[no_mangle] +pub unsafe extern "C" fn glue_channel_write( + node_id: u64, + handle: u64, + buf: *const u8, + size: usize, + handle_buf: *const u8, + handle_count: u32, +) -> u32 { + debug!( + "{{{}}}: channel_write(h={}, buf={:?}, size={}, handle_buf={:?}, count={})", + node_id, handle, buf, size, handle_buf, handle_count + ); + let mut msg = oak_runtime::Message { + data: Vec::with_capacity(size), + channels: Vec::with_capacity(handle_count as usize), + }; + std::ptr::copy_nonoverlapping(buf, msg.data.as_mut_ptr(), size); + msg.data.set_len(size); + + let handle_size = 8 * handle_count as usize; + let mut handle_data = Vec::::with_capacity(handle_size); + std::ptr::copy_nonoverlapping(handle_buf, handle_data.as_mut_ptr(), handle_size); + handle_data.set_len(handle_size); + + let mut mem_reader = Cursor::new(handle_data); + for _ in 0..handle_count as isize { + let handle = mem_reader.read_u64::().unwrap(); + msg.channels.push(oak_runtime::Handle(handle)); + } + + let proxy = GLUE.read().expect(R1).as_ref().expect(R2).proxy(node_id); + let result = proxy.channel_write(oak_runtime::Handle(handle), msg); + debug!("{{{}}}: channel_write() -> {:?}", node_id, result); + match result { + Ok(_) => OakStatus::Ok as u32, + Err(status) => status as u32, + } +} + +/// # Safety +/// +/// The raw pointers to memory must be valid. +#[no_mangle] +pub unsafe extern "C" fn glue_channel_create(node_id: u64, write: *mut u64, read: *mut u64) -> u32 { + debug!("{{{}}}: channel_create({:?}, {:?})", node_id, write, read); + let proxy = GLUE.read().expect(R1).as_ref().expect(R2).proxy(node_id); + let (write_handle, read_handle) = + proxy.channel_create(&oak_abi::label::Label::public_trusted()); + *write = write_handle.0; + *read = read_handle.0; + debug!( + "{{{}}}: channel_create(*w={:?}, *r={:?}) -> OK", + node_id, write_handle, read_handle + ); + OakStatus::Ok as u32 +} + +#[no_mangle] +pub extern "C" fn glue_channel_close(node_id: u64, handle: u64) -> u32 { + debug!("{{{}}}: channel_close({})", node_id, handle); + let proxy = GLUE.read().expect(R1).as_ref().expect(R2).proxy(node_id); + let result = proxy.channel_close(oak_runtime::Handle(handle)); + debug!("{{{}}}: channel_close({}) -> {:?}", node_id, handle, result); + match result { + Ok(_) => OakStatus::Ok as u32, + Err(status) => status as u32, + } +} + +fn create_and_run_node(config_name: &str, node_id: NodeId, handle: oak_runtime::Handle) { + info!( + "invoke registered factory with '{}', node_id={:?}, handle={:?}", + config_name, node_id, handle + ); + let factory = GLUE.read().expect(R1).as_ref().expect(R2).factory; + let factory_data = GLUE.read().expect(R1).as_ref().expect(R2).factory_data; + factory.expect("no factory registered!")( + factory_data, + config_name.as_ptr(), + config_name.len() as u32, + node_id.0, + handle.0, + ); +} diff --git a/oak/server/rust/oak_runtime/BUILD b/oak/server/rust/oak_runtime/BUILD index c873c29e3ec..fbd13a0f6c0 100644 --- a/oak/server/rust/oak_runtime/BUILD +++ b/oak/server/rust/oak_runtime/BUILD @@ -29,6 +29,7 @@ rust_library( deps = [ "//cargo:byteorder", "//cargo:itertools", + "//cargo:lazy_static", "//cargo:log", "//cargo:prost", "//cargo:rand",