cdh: move storage and secret crates into cdh modules

.github/workflows/cdh_basic.yml

@@ -67,13 +67,13 @@ jobs:
 
       - name: Run cargo test
         run: |
-          sudo -E PATH=$PATH -s cargo test --features kbs,aliyun,sev,bin -p kms -p confidential-data-hub -p secret
+          sudo -E PATH=$PATH -s cargo test --features kbs,aliyun,sev,bin -p kms -p confidential-data-hub
 
       - name: Run cargo fmt check
         run: |
-          sudo -E PATH=$PATH -s cargo fmt -p kms -p confidential-data-hub -p secret -- --check
+          sudo -E PATH=$PATH -s cargo fmt -p kms -p confidential-data-hub -- --check
 
       - name: Run rust lint check
         run: |
           # We are getting error in generated code due to derive_partial_eq_without_eq check, so ignore it for now
-          sudo -E PATH=$PATH -s cargo clippy -p kms -p confidential-data-hub -p secret -- -D warnings -A clippy::derive-partial-eq-without-eq 
+          sudo -E PATH=$PATH -s cargo clippy -p kms -p confidential-data-hub -- -D warnings -A clippy::derive-partial-eq-without-eq 

Cargo.toml

@@ -12,8 +12,6 @@ members = [
     "attestation-agent/coco_keyprovider",
     "confidential-data-hub/hub",
     "confidential-data-hub/kms",
-    "confidential-data-hub/secret",
-    "confidential-data-hub/storage",
     "image-rs",
     "ocicrypt-rs",
 ]

README.md

@@ -26,7 +26,7 @@ CoCo Keyprovider. Used to encrypt the container images.
 
 ## Tools
 
-[secret-cli](confidential-data-hub/secret)
+[secret-cli](confidential-data-hub/hub/src/bin/secret_cli.rs)
 Utility for sealing and unsealing sealed secrets
 
 [CDH Client](confidential-data-hub/hub/src/bin)

api-server-rest/openapi/api.json

@@ -1,5 +1,5 @@
 {
-  "openapi": "3.0.3",
+  "openapi": "3.1.0",
   "info": {
     "title": "CoCo Restful API",
     "description": "HTTP based API for CoCo containers to get resource/evidence/token from confidential-data-hub and attestation-agent.",
@@ -20,9 +20,7 @@
   "paths": {
     "/aa/evidence": {
       "get": {
-        "tags": [
-          "crate"
-        ],
+        "tags": [],
         "operationId": "_evidence",
         "parameters": [
           {
@@ -67,9 +65,7 @@
     },
     "/aa/token": {
       "get": {
-        "tags": [
-          "crate"
-        ],
+        "tags": [],
         "operationId": "_token",
         "parameters": [
           {
@@ -114,9 +110,7 @@
     },
     "/cdh/resource/{repository}/{type}/{tag}": {
       "get": {
-        "tags": [
-          "crate"
-        ],
+        "tags": [],
         "operationId": "_resource",
         "responses": {
           "200": {
@@ -144,5 +138,6 @@
         }
       }
     }
-  }
+  },
+  "components": {}
 }

confidential-data-hub/docs/SEALED_SECRET.md

@@ -10,8 +10,11 @@ in conjunction with an attestation.
 The Confidential Data Hub provides an API for unsealing secrets inside
 of a confidential guest.
 
-There is also a [secret cli](../secret/src/bin) tool that can be used to generate sealed
-secrets.
+You can also use the secret cli tool to generate a sealed secret:
+
+```bash
+cargo run -p confidential-data-hub --bin secret
+```
 
 ## Kubernetes Secrets
 
@@ -173,7 +176,6 @@ Start with a sealed secret such as
 	}
 }
 ```
-You can use the [secret cli](../secret/src/bin) tool to generate a sealed secret.
 
 Encode the payload in BASE64URL
 ```

confidential-data-hub/docs/SECURE_STORAGE.md

@@ -17,7 +17,7 @@ We reuse [direct block device assigned volume feature](https://github.com/kata-c
 
 [Aliyun OSS](https://www.alibabacloud.com/product/object-storage-service) is an object storage service provided by Alibaba Cloud (Aliyun).
 
-The [plugin](../storage/src/volume_type/aliyun) provides two different modes for secure mount.
+The [plugin](../hub/src/storage/volume_type/aliyun) provides two different modes for secure mount.
 
 Confidential Data Hub's `secure_mount()` [API](../hub/protos/api.proto) will help to instrument this.
 
@@ -65,7 +65,7 @@ For more details, please refer to [the guide](use-cases/secure-mount-with-aliyun
 
 ### Block Device
 
-The [plugin](../storage/src/volume_type/blockdevice) provides ways to encrypt a block device and mount it to a specific mount point. Currently only support LUKS in [cryptsetup](https://gitlab.com/cryptsetup/cryptsetup/) for block device encryption.
+The [plugin](../hub/src/storage/volume_type/blockdevice) provides ways to encrypt a block device and mount it to a specific mount point. Currently only support LUKS in [cryptsetup](https://gitlab.com/cryptsetup/cryptsetup/) for block device encryption.
 
 #### LUKS Encryption
 

confidential-data-hub/docs/use-cases/secure-mount-with-block-device.md

@@ -30,7 +30,7 @@ Follow the instructions in the [CDH README](../../README.md#confidential-data-hu
 
 2. Install `luks-encrypt-storage`
 
-Install [luks-encrypt-storage](../../storage/scripts/luks-encrypt-storage) into `/usr/local/bin`
+Install [luks-encrypt-storage](../../hub/src/storage/scripts/luks-encrypt-storage) into `/usr/local/bin`
 
 3. Run CDH
 ```shell
@@ -74,4 +74,4 @@ $ lsblk |grep "encrypted_disk"
 # Expected output:
 └─encrypted_disk_OEyEj_dif 253:1    0 968.6M  0 crypt
   └─encrypted_disk_OEyEj   253:2    0 968.6M  0 crypt /mnt/test-path
-```
+```

confidential-data-hub/hub/Cargo.toml

@@ -26,6 +26,11 @@ required-features = ["bin", "grpc"]
 name = "cdh-oneshot"
 required-features = ["bin"]
 
+[[bin]]
+name = "secret"
+path = "src/bin/secret_cli.rs"
+required-features = ["cli"]
+
 [dependencies]
 anyhow = { workspace = true, optional = true }
 async-trait.workspace = true
@@ -42,43 +47,51 @@ lazy_static.workspace = true
 log.workspace = true
 prost = { workspace = true, optional = true }
 protobuf = { workspace = true, optional = true }
+rand.workspace = true
 resource_uri.path = "../../attestation-agent/deps/resource_uri"
-secret.path = "../secret"
-storage.path = "../storage"
 serde = { workspace = true, optional = true }
 serde_json.workspace = true
+strum = { workspace = true, features = ["derive"] }
+tempfile = { workspace = true, optional = true }
 thiserror.workspace = true
-tokio = { workspace = true, features = [ "rt-multi-thread", "macros", "sync" ] }
+tokio = { workspace = true, features = [ "fs", "macros", "io-util", "process", "rt-multi-thread", "sync" ] }
 tonic = { workspace = true, optional = true }
 ttrpc = { workspace = true, features = ["async"], optional = true }
+zeroize.workspace = true
 
 [build-dependencies]
+anyhow.workspace = true
 tonic-build = { workspace = true, optional = true }
 ttrpc-codegen = { workspace = true, optional = true }
 
 [dev-dependencies]
 assert_cmd.workspace = true
+assert-json-diff.workspace = true
 nix.workspace = true
 rstest.workspace = true
 serial_test.workspace = true
 tempfile.workspace = true
+tokio = { workspace = true, features = ["rt", "macros" ] }
 
 [features]
-default = ["kbs", "bin", "ttrpc", "grpc"]
+default = ["aliyun", "kbs", "bin", "ttrpc", "grpc", "cli"]
 
 # support aliyun stacks (KMS, ..)
-aliyun = ["secret/aliyun"]
+aliyun = ["tempfile"]
 
 # support coco-KBS to provide confidential resources
-kbs = ["kms/kbs", "secret/kbs"]
+kbs = ["kms/kbs"]
 
 # support sev to provide confidential resources
-sev = ["kms/sev", "secret/sev"]
+sev = ["kms/sev"]
 
 # support eHSM stacks (KMS, ...)
-ehsm = ["secret/ehsm"]
+ehsm = []
 
 # Binary RPC type
 bin = [ "anyhow", "attestation-agent", "cfg-if", "clap", "config", "env_logger", "serde" ]
 ttrpc = ["dep:ttrpc", "protobuf", "ttrpc-codegen", "tokio/signal"]
 grpc = ["prost", "tonic", "tonic-build", "tokio/signal"]
+
+# for secret_cli
+cli = ["clap/derive", "tokio/rt-multi-thread", "tokio/sync", "tokio/macros"]

confidential-data-hub/hub/src/api.rs

@@ -5,8 +5,8 @@
 
 use async_trait::async_trait;
 
+use crate::storage::volume_type::Storage;
 use crate::Result;
-use storage::volume_type::Storage;
 
 /// The APIs of the DataHub. See
 /// <https://github.com/confidential-containers/documentation/issues/131> for

confidential-data-hub/hub/src/bin/cdh-oneshot.rs

@@ -9,9 +9,8 @@
 
 use base64::{engine::general_purpose::STANDARD, Engine};
 use clap::{Args, Parser, Subcommand};
-use confidential_data_hub::{hub::Hub, CdhConfig, DataHub};
+use confidential_data_hub::{hub::Hub, storage::volume_type::Storage, CdhConfig, DataHub};
 use log::warn;
-use storage::volume_type::Storage;
 
 #[derive(Parser)]
 #[command(name = "cdh_oneshot")]

confidential-data-hub/hub/src/bin/grpc-cdh-tool.rs

@@ -16,7 +16,7 @@ use api::{
 };
 use base64::{engine::general_purpose::STANDARD, Engine};
 use clap::{Args, Parser, Subcommand};
-use storage::volume_type::Storage;
+use confidential_data_hub::storage::volume_type::Storage;
 
 mod api {
     tonic::include_proto!("api");

confidential-data-hub/hub/src/bin/grpc_server/mod.rs

@@ -5,10 +5,12 @@
 
 use anyhow::*;
 
-use confidential_data_hub::{hub::Hub, DataHub};
+use confidential_data_hub::{
+    storage::volume_type::Storage,
+    {hub::Hub, DataHub},
+};
 use log::{debug, error};
 use std::{error::Error as _, net::SocketAddr, sync::Arc};
-use storage::volume_type::Storage;
 use tonic::{transport::Server, Request, Response, Status};
 
 use crate::{

confidential-data-hub/secret/src/bin/secret_cli.rs → confidential-data-hub/hub/src/bin/secret_cli.rs

@@ -7,15 +7,17 @@ use std::{env, path::Path};
 
 use base64::{engine::general_purpose::STANDARD, Engine};
 use clap::{command, Args, Parser, Subcommand};
+use confidential_data_hub::secret::{
+    layout::{envelope::EnvelopeSecret, vault::VaultSecret},
+    Secret, SecretContent, VERSION,
+};
 use crypto::WrapType;
 #[cfg(feature = "aliyun")]
 use kms::plugins::aliyun::AliyunKmsClient;
 #[cfg(feature = "ehsm")]
 use kms::plugins::ehsm::EhsmKmsClient;
 use kms::{Encrypter, ProviderSettings};
 use rand::Rng;
-use secret::secret::layout::{envelope::EnvelopeSecret, vault::VaultSecret};
-use secret::secret::{Secret, SecretContent, VERSION};
 #[cfg(feature = "ehsm")]
 use serde_json::Value;
 use tokio::{fs, io::AsyncWriteExt};

confidential-data-hub/hub/src/bin/ttrpc-cdh-tool.rs

@@ -9,6 +9,7 @@
 
 use base64::{engine::general_purpose::STANDARD, Engine};
 use clap::{Args, Parser, Subcommand};
+use confidential_data_hub::storage::volume_type::Storage;
 use protos::{
     api::*,
     api_ttrpc::{
@@ -18,7 +19,6 @@ use protos::{
     keyprovider::*,
     keyprovider_ttrpc::KeyProviderServiceClient,
 };
-use storage::volume_type::Storage;
 use ttrpc::context;
 
 mod protos;

confidential-data-hub/hub/src/bin/ttrpc_server/mod.rs

@@ -7,9 +7,11 @@ use std::error::Error as _;
 
 use anyhow::Result;
 use async_trait::async_trait;
-use confidential_data_hub::{hub::Hub, CdhConfig, DataHub};
+use confidential_data_hub::{
+    storage::volume_type::Storage,
+    {hub::Hub, CdhConfig, DataHub},
+};
 use log::{debug, error};
-use storage::volume_type::Storage;
 use ttrpc::{asynchronous::TtrpcContext, Code, Error, Status};
 
 use crate::{