From 93cc40890b4c656abf3b19bdec163cd3ef4183a6 Mon Sep 17 00:00:00 2001 From: Harald Hoyer Date: Tue, 27 Feb 2024 12:04:31 +0100 Subject: [PATCH] chore: cleanup and nixify * create containers with nix * updated README.md * added SPDX license headers Signed-off-by: Harald Hoyer --- .github/workflows/container.yml | 51 ---- .github/workflows/nix.yml | 119 +++++---- assets/Azure-DCAP-Client.patch | 38 --- assets/extract-sig.sh | 6 +- assets/replace-sig.sh | 6 +- assets/vault-auth-tee.sha256 | 1 - bin/tee-vault-admin/Dockerfile-azure | 85 ------- .../tee-vault-admin.manifest.template | 66 ----- bin/tee-vault-unseal/Dockerfile-azure | 92 ------- .../tee-vault-unseal.manifest.template | 62 ----- examples/README.md | 20 +- examples/docker-compose.yml | 59 ++--- flake.lock | 228 +++++++++++++++++- flake.nix | 6 + .../Dockerfile | 25 ++ .../default.nix | 52 ++++ .../tee-vault-admin.manifest.toml | 71 ++++++ packages/container-vault-sgx-azure/Dockerfile | 25 ++ .../container-vault-sgx-azure/default.nix | 53 ++++ .../test-enclave-key.pem | 0 .../vault.manifest.toml | 48 ++-- .../container-vault-start-config}/cacert.pem | 0 .../container-vault-start-config}/cakey.pem | 0 .../container-vault-start-config/config.hcl | 55 +++++ .../container-vault-start-config/default.nix | 29 +++ .../Dockerfile | 25 ++ .../default.nix | 54 +++++ .../tee-vault-unseal.manifest.toml | 74 ++++++ packages/container-vault-unseal/default.nix | 31 +++ .../container-verify-attestation/default.nix | 4 + packages/teepot/default.nix | 5 +- shells/teepot/default.nix | 15 +- vault/Dockerfile | 53 ---- vault/config.hcl | 55 ----- vault/start.sh | 17 -- vault/vault-csr.conf | 21 -- 36 files changed, 887 insertions(+), 664 deletions(-) delete mode 100644 .github/workflows/container.yml delete mode 100644 assets/Azure-DCAP-Client.patch delete mode 100644 assets/vault-auth-tee.sha256 delete mode 100644 bin/tee-vault-admin/Dockerfile-azure delete mode 100644 bin/tee-vault-admin/tee-vault-admin.manifest.template delete mode 100644 bin/tee-vault-unseal/Dockerfile-azure delete mode 100644 bin/tee-vault-unseal/tee-vault-unseal.manifest.template create mode 100644 packages/container-vault-admin-sgx-azure/Dockerfile create mode 100644 packages/container-vault-admin-sgx-azure/default.nix create mode 100644 packages/container-vault-admin-sgx-azure/tee-vault-admin.manifest.toml create mode 100644 packages/container-vault-sgx-azure/Dockerfile create mode 100644 packages/container-vault-sgx-azure/default.nix rename vault/enclave-key.pem => packages/container-vault-sgx-azure/test-enclave-key.pem (100%) rename {vault => packages/container-vault-sgx-azure}/vault.manifest.toml (59%) rename {vault => packages/container-vault-start-config}/cacert.pem (100%) rename {vault => packages/container-vault-start-config}/cakey.pem (100%) create mode 100644 packages/container-vault-start-config/config.hcl create mode 100644 packages/container-vault-start-config/default.nix create mode 100644 packages/container-vault-unseal-sgx-azure/Dockerfile create mode 100644 packages/container-vault-unseal-sgx-azure/default.nix create mode 100644 packages/container-vault-unseal-sgx-azure/tee-vault-unseal.manifest.toml create mode 100644 packages/container-vault-unseal/default.nix delete mode 100644 vault/Dockerfile delete mode 100644 vault/config.hcl delete mode 100644 vault/start.sh delete mode 100644 vault/vault-csr.conf diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml deleted file mode 100644 index 75b52da9..00000000 --- a/.github/workflows/container.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Container -on: - push: - branches: [ "main" ] - pull_request: - -jobs: - push_to_registry: - permissions: - packages: write - contents: read - name: Build and push containers image to GitHub Packages - runs-on: ubuntu-latest - concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.config.dockerfile }} - cancel-in-progress: true - strategy: - fail-fast: false - matrix: - config: - - { dockerfile: 'vault/Dockerfile', tag: 'vault:latest', repository: 'teepot-vault' } - - { dockerfile: 'bin/tee-vault-unseal/Dockerfile-azure', tag: 'tvu:latest', repository: 'teepot-tvu' } - - { dockerfile: 'bin/tee-vault-admin/Dockerfile-azure', tag: 'tva:latest', repository: 'teepot-tva' } - steps: - - name: Check out the repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up env - run: echo "repository_owner=${GITHUB_REPOSITORY_OWNER,,}" >>${GITHUB_ENV} - - name: Build and Push Container - uses: docker/build-push-action@v5 - with: - file: ${{ matrix.config.dockerfile }} - tags: | - matterlabsrobot/${{ matrix.config.repository }}:latest - cache-from: type=gha - cache-to: type=gha,mode=max,ignore-error=true - push: ${{ github.event_name == 'push' || github.event_name == 'schedule' }} - diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index e0c93858..8d55d004 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -14,47 +14,52 @@ jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - uses: cachix/install-nix-action@6004951b182f8860210c8d6f0d808ec5b1a33d28 # v25 - with: - extra_nix_config: | - access-tokens = github.com=${{ github.token }} - - run: nix flake check -L --show-trace --keep-going + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: cachix/install-nix-action@v23 + - uses: cachix/cachix-action@v12 + with: + name: nixsgx + extra_nix_config: | + access-tokens = github.com=${{ github.token }} + - run: nix flake check -L --show-trace --keep-going fmt: runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - uses: cachix/install-nix-action@6004951b182f8860210c8d6f0d808ec5b1a33d28 # v25 - with: - extra_nix_config: | - access-tokens = github.com=${{ github.token }} - - run: nix fmt + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: cachix/install-nix-action@v23 + - uses: cachix/cachix-action@v12 + with: + name: nixsgx + extra_nix_config: | + access-tokens = github.com=${{ github.token }} + - run: nix fmt clippy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - uses: cachix/install-nix-action@6004951b182f8860210c8d6f0d808ec5b1a33d28 # v25 - with: - extra_nix_config: | - access-tokens = github.com=${{ github.token }} - # Cache any artifacts that aren't already cached at https://cache.nixos.org - - name: Enable magic Nix cache - uses: DeterminateSystems/magic-nix-cache-action@main - - name: Use nixsgx cache - run: | - nix-env -iA cachix -f https://cachix.org/api/v1/install - cachix use nixsgx - - name: cargo clippy - run: nix develop -L --ignore-environment -c cargo clippy --all --locked + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: cachix/install-nix-action@v23 + - uses: cachix/cachix-action@v12 + with: + name: nixsgx + extra_nix_config: | + access-tokens = github.com=${{ github.token }} + + # Cache any artifacts that aren't already cached at https://cache.nixos.org + - name: Enable magic Nix cache + uses: DeterminateSystems/magic-nix-cache-action@main + - name: cargo clippy + run: nix develop -L --ignore-environment -c cargo clippy --all --locked build: runs-on: ubuntu-latest steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - uses: cachix/install-nix-action@6004951b182f8860210c8d6f0d808ec5b1a33d28 # v25 + - uses: cachix/install-nix-action@v23 + - uses: cachix/cachix-action@v12 with: + name: nixsgx extra_nix_config: | access-tokens = github.com=${{ github.token }} @@ -62,46 +67,64 @@ jobs: - name: Enable magic Nix cache uses: DeterminateSystems/magic-nix-cache-action@main - - name: Use nixsgx cache - run: | - nix-env -iA cachix -f https://cachix.org/api/v1/install - cachix use nixsgx - - name: nix build run: nix run nixpkgs#nixci push_to_docker: - if: ${{ github.event_name == 'push' }} + needs: build + # if: ${{ github.event_name == 'push' }} runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.config.dockerfile }} + cancel-in-progress: true + strategy: + fail-fast: false + matrix: + config: + - { nixpackage: 'container-vault-sgx-azure', dockerfile: 'packages/container-vault-sgx-azure/Dockerfile', tag: 'vault:latest', repository: 'teepot-vault' } + - { nixpackage: 'container-vault-unseal-sgx-azure', dockerfile: 'packages/container-vault-unseal-sgx-azure/Dockerfile', tag: 'tvu:latest', repository: 'teepot-tvu' } + - { nixpackage: 'container-vault-admin-sgx-azure', dockerfile: 'packages/container-vault-admin-sgx-azure/Dockerfile', tag: 'tva:latest', repository: 'teepot-tva' } + - { nixpackage: 'container-verify-attestation' } steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - uses: cachix/install-nix-action@6004951b182f8860210c8d6f0d808ec5b1a33d28 # v25 + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v23 + - uses: cachix/cachix-action@v12 with: + name: nixsgx extra_nix_config: | access-tokens = github.com=${{ github.token }} - - # Cache any artifacts that aren't already cached at https://cache.nixos.org - - name: Enable magic Nix cache - uses: DeterminateSystems/magic-nix-cache-action@main - - - name: Use nixsgx cache - run: | - nix-env -iA cachix -f https://cachix.org/api/v1/install - cachix use nixsgx - - name: Log in to Docker Hub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Load and push + - name: Load and Push nix container run: | - nix build -L .#container-verify-attestation + nix build -L .#${{ matrix.config.nixpackage }} export IMAGE_TAG=$(docker load < result | grep -Po 'Loaded image.*: \K.*') echo "Pushing image ${IMAGE_TAG} to Docker Hub" - echo "IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_ENV docker tag "${IMAGE_TAG}" matterlabsrobot/"${IMAGE_TAG}" docker push matterlabsrobot/"${IMAGE_TAG}" docker tag matterlabsrobot/"${IMAGE_TAG}" matterlabsrobot/"${IMAGE_TAG%:*}:latest" docker push matterlabsrobot/"${IMAGE_TAG%:*}:latest" + + - name: Generate build ID for Flux Image Automation + if: ${{ matrix.config.dockerfile }} + id: buildid + run: | + sha=$(git rev-parse --short HEAD) + ts=$(date +%s%N | cut -b1-13) + echo "BUILD_ID=${sha}-${ts}" >> "$GITHUB_OUTPUT" + + - name: Build and Push Container + if: ${{ matrix.config.dockerfile }} + uses: docker/build-push-action@v5 + with: + file: ${{ matrix.config.dockerfile }} + tags: | + "matterlabsrobot/${{ matrix.config.repository }}:latest" + "matterlabsrobot/${{ matrix.config.repository }}:${{ steps.buildid.outputs.BUILD_ID }}" + cache-from: type=gha + cache-to: type=gha,mode=max,ignore-error=true + push: ${{ github.event_name == 'push' || github.event_name == 'schedule' }} diff --git a/assets/Azure-DCAP-Client.patch b/assets/Azure-DCAP-Client.patch deleted file mode 100644 index 87634551..00000000 --- a/assets/Azure-DCAP-Client.patch +++ /dev/null @@ -1,38 +0,0 @@ -diff --git a/src/dcap_provider.cpp b/src/dcap_provider.cpp -index d23c250..34c4d9d 100644 ---- a/src/dcap_provider.cpp -+++ b/src/dcap_provider.cpp -@@ -1333,7 +1333,7 @@ static std::string build_tcb_info_url( - tcb_info_url << base_url; - } - else -- tcb_info_url << get_base_url(); -+ tcb_info_url << "https://api.trustedservices.intel.com/sgx/certification"; - - if (!version.empty()) - { -@@ -1426,7 +1426,7 @@ static std::string build_enclave_id_url( - qe_id_url << base_url; - } - else -- qe_id_url << get_base_url(); -+ qe_id_url << "https://api.trustedservices.intel.com/sgx/certification/"; - - // Select the correct issuer header name - if (!version.empty()) -@@ -1521,6 +1521,7 @@ static quote3_error_t get_collateral( - "Successfully fetched %s from URL: '%s'.", - friendly_name.c_str(), - url.c_str()); -+/* - std::string cache_control; - auto get_cache_header_operation = get_unescape_header(*curl_operation, headers::CACHE_CONTROL, &cache_control); - retval = convert_to_intel_error(get_cache_header_operation); -@@ -1534,6 +1535,7 @@ static quote3_error_t get_collateral( - local_cache_add(issuer_chain_cache_name, expiry, issuer_chain.size(), issuer_chain.c_str()); - } - } -+*/ - } - - return retval; diff --git a/assets/extract-sig.sh b/assets/extract-sig.sh index b66e3547..388f1fd1 100644 --- a/assets/extract-sig.sh +++ b/assets/extract-sig.sh @@ -1,10 +1,8 @@ #!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 # -# This script is used to extract the signature of a gramine docker image. +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Matter Labs # -# Usage: ./extract-sig.sh -# Example: ./extract-sig.sh tva tee-vault-admin id=$(docker create $1) trap 'docker rm -v $id' EXIT diff --git a/assets/replace-sig.sh b/assets/replace-sig.sh index 8ef54bf1..08affcaa 100644 --- a/assets/replace-sig.sh +++ b/assets/replace-sig.sh @@ -1,10 +1,8 @@ #!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 # -# This script is used to replace the signature of a gramine docker image with a new one. +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Matter Labs # -# Usage: ./replace-sig.sh -# Example: ./replace-sig.sh tva tee-vault-admin.sig /app/tee-vault-admin.sig DOCKERFILE="Dockerfile-tmp-$$" diff --git a/assets/vault-auth-tee.sha256 b/assets/vault-auth-tee.sha256 deleted file mode 100644 index ca6b1030..00000000 --- a/assets/vault-auth-tee.sha256 +++ /dev/null @@ -1 +0,0 @@ -7e417b65b9a4fa46dfcc72feb0f0c2e5fa4391c3f6b668cb2c5ba7174b95c43b diff --git a/bin/tee-vault-admin/Dockerfile-azure b/bin/tee-vault-admin/Dockerfile-azure deleted file mode 100644 index ca3fce7f..00000000 --- a/bin/tee-vault-admin/Dockerfile-azure +++ /dev/null @@ -1,85 +0,0 @@ -FROM docker.io/ubuntu:20.04 AS azuredcap -WORKDIR /build -ADD https://github.com/microsoft/Azure-DCAP-Client/archive/refs/tags/1.12.0.tar.gz ./Azure-DCAP-Client.tar.gz -RUN tar -xvf Azure-DCAP-Client.tar.gz -COPY assets/Azure-DCAP-Client.patch ./Azure-DCAP-Client.patch -RUN set -eux; \ - apt-get update; \ - apt-get install -y software-properties-common; \ - add-apt-repository ppa:team-xbmc/ppa -y; \ - apt-get update; \ - apt-get install -y \ - build-essential \ - cmake \ - libssl-dev \ - libcurl4-openssl-dev \ - pkg-config \ - nlohmann-json3-dev \ - wget \ - dos2unix \ - ; - -WORKDIR /build/Azure-DCAP-Client-1.12.0 -RUN dos2unix src/dcap_provider.cpp && patch -p1 < ../Azure-DCAP-Client.patch -WORKDIR /build/Azure-DCAP-Client-1.12.0/src/Linux -RUN ./configure && make && make install - -FROM docker.io/rust:1-bullseye AS buildtee -RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \ - && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential \ - cmake \ - rsync \ - pkg-config \ - libssl-dev \ - libcurl4-openssl-dev \ - libprotobuf-dev \ - protobuf-compiler \ - clang \ - libsgx-headers \ - libsgx-dcap-quote-verify-dev - -WORKDIR /opt/vault/plugins - -WORKDIR /build -RUN --mount=type=bind,target=/data rsync --exclude='/.git' --filter="dir-merge,- .gitignore" --exclude "Dockerfile-*" --exclude 'tee-vault-admin.manifest.template' -av /data/ ./ -RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=target \ - RUSTFLAGS="-C target-cpu=icelake-server --cfg mio_unsupported_force_waker_pipe" \ - cargo build --locked --target x86_64-unknown-linux-gnu --release -p tee-vault-admin --bin tee-vault-admin \ - && mv ./target/x86_64-unknown-linux-gnu/release/tee-vault-admin ./ - -FROM docker.io/gramineproject/gramine:v1.5 - -RUN curl -fsSLo /usr/share/keyrings/microsoft.asc https://packages.microsoft.com/keys/microsoft.asc \ - && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft.asc] https://packages.microsoft.com/ubuntu/20.04/prod focal main" > /etc/apt/sources.list.d/msprod.list \ - && apt-get update \ - && apt purge -y libsgx-dcap-default-qpl \ - && apt-get install -y az-dcap-client - -RUN apt purge -y libsgx-ae-qve -# libsgx-urts - -RUN rm -rf /var/lib/apt/lists/* - -# So we only have to use one gramine template -RUN touch /etc/sgx_default_qcnl.conf - -WORKDIR /app - -COPY --from=buildtee /build/tee-vault-admin . -COPY ./bin/tee-vault-admin/tee-vault-admin.manifest.template . -COPY vault/enclave-key.pem . - -# The original Azure library is still delivering expired collateral, so we have to use a patched version -COPY --from=azuredcap /usr/local/lib/libdcap_quoteprov.so /usr/lib/ - -RUN gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu -Dexecdir=/usr/bin -Dlog_level=warning tee-vault-admin.manifest.template tee-vault-admin.manifest \ - && gramine-sgx-sign --manifest tee-vault-admin.manifest --output tee-vault-admin.manifest.sgx --key enclave-key.pem \ - && rm enclave-key.pem - -EXPOSE 8443 - -ENTRYPOINT ["/bin/sh", "-c"] -CMD [ "/restart_aesm.sh ; exec gramine-sgx tee-vault-admin" ] diff --git a/bin/tee-vault-admin/tee-vault-admin.manifest.template b/bin/tee-vault-admin/tee-vault-admin.manifest.template deleted file mode 100644 index 97c6ab5e..00000000 --- a/bin/tee-vault-admin/tee-vault-admin.manifest.template +++ /dev/null @@ -1,66 +0,0 @@ -libos.entrypoint = "/app/tee-vault-admin" - -[loader] -argv = [ "/app/tee-vault-admin" ] -entrypoint = "file:{{ gramine.libos }}" -env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}:/usr{{ arch_libdir }}:/lib" -env.HOME = "/app" -env.MALLOC_ARENA_MAX = "1" -env.AZDCAP_DEBUG_LOG_LEVEL = "ignore" -env.AZDCAP_COLLATERAL_VERSION = "v4" - -### Admin Config ### -env.PORT = { passthrough = true } - -### VAULT attestation ### -env.VAULT_ADDR = { passthrough = true } -env.VAULT_SGX_MRENCLAVE = { passthrough = true } -env.VAULT_SGX_MRSIGNER = { passthrough = true } -env.VAULT_SGX_ALLOWED_TCB_LEVELS = { passthrough = true } - -### DEBUG ### -env.RUST_BACKTRACE = "1" -env.RUST_LOG="info,tee_vault_admin=trace,teepot=trace,vault_tee_client=trace,tee_client=trace,awc=debug" - -[fs] -root.uri = "file:/" -start_dir = "/app" -mounts = [ - { path = "{{ execdir }}", uri = "file:{{ execdir }}" }, - { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" }, - { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" }, - { path = "/etc", uri = "file:/etc" }, - { type = "tmpfs", path = "/var/tmp" }, - { type = "tmpfs", path = "/tmp" }, - { type = "tmpfs", path = "/app/.dcap-qcnl" }, - { type = "tmpfs", path = "/app/.az-dcap-client" }, - { path = "/lib/libdcap_quoteprov.so", uri = "file:/lib/libdcap_quoteprov.so" }, -] - -[sgx] -trusted_files = [ - "file:/etc/ld.so.cache", - "file:/app/", - "file:{{ execdir }}/", - "file:{{ arch_libdir }}/", - "file:/usr/{{ arch_libdir }}/", - "file:{{ gramine.libos }}", - "file:{{ gramine.runtimedir() }}/", - "file:/usr/lib/ssl/openssl.cnf", - "file:/etc/ssl/", - "file:/etc/sgx_default_qcnl.conf", - "file:/lib/libdcap_quoteprov.so", -] -remote_attestation = "dcap" -max_threads = 64 -edmm_enable = false -## max enclave size -enclave_size = "8G" - -[sys] -enable_extra_runtime_domain_names_conf = true -enable_sigterm_injection = true - -# possible tweak option, if problems with mio -# currently mio is compiled with `mio_unsupported_force_waker_pipe` -# insecure__allow_eventfd = true diff --git a/bin/tee-vault-unseal/Dockerfile-azure b/bin/tee-vault-unseal/Dockerfile-azure deleted file mode 100644 index cd7c28d4..00000000 --- a/bin/tee-vault-unseal/Dockerfile-azure +++ /dev/null @@ -1,92 +0,0 @@ -FROM ghcr.io/matter-labs/vault-auth-tee:latest AS vault-auth-tee - -FROM docker.io/ubuntu:20.04 AS azuredcap -WORKDIR /build -ADD https://github.com/microsoft/Azure-DCAP-Client/archive/refs/tags/1.12.0.tar.gz ./Azure-DCAP-Client.tar.gz -RUN tar -xvf Azure-DCAP-Client.tar.gz -COPY assets/Azure-DCAP-Client.patch ./Azure-DCAP-Client.patch -RUN set -eux; \ - apt-get update; \ - apt-get install -y software-properties-common; \ - add-apt-repository ppa:team-xbmc/ppa -y; \ - apt-get update; \ - apt-get install -y \ - build-essential \ - cmake \ - libssl-dev \ - libcurl4-openssl-dev \ - pkg-config \ - nlohmann-json3-dev \ - wget \ - dos2unix \ - ; - -WORKDIR /build/Azure-DCAP-Client-1.12.0 -RUN dos2unix src/dcap_provider.cpp && patch -p1 < ../Azure-DCAP-Client.patch -WORKDIR /build/Azure-DCAP-Client-1.12.0/src/Linux -RUN ./configure && make && make install - -FROM docker.io/rust:1-bullseye AS buildtee -RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \ - && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential \ - cmake \ - rsync \ - pkg-config \ - libssl-dev \ - libcurl4-openssl-dev \ - libprotobuf-dev \ - protobuf-compiler \ - clang \ - libsgx-headers \ - libsgx-dcap-quote-verify-dev - -WORKDIR /opt/vault/plugins -COPY --from=vault-auth-tee /opt/vault/plugins/vault-auth-tee ./ - -WORKDIR /build -RUN --mount=type=bind,target=/data rsync --exclude='/.git' --filter="dir-merge,- .gitignore" --exclude "Dockerfile-*" --exclude 'tee-vault-unseal.manifest.template' -av /data/ ./ -RUN sha256sum /opt/vault/plugins/vault-auth-tee | ( read a _ ; echo -n $a ) | tee assets/vault-auth-tee.sha256 -RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=target \ - RUSTFLAGS="-C target-cpu=icelake-server --cfg mio_unsupported_force_waker_pipe" \ - cargo build --locked --target x86_64-unknown-linux-gnu --release -p tee-vault-unseal --bin tee-vault-unseal \ - && mv ./target/x86_64-unknown-linux-gnu/release/tee-vault-unseal ./ - -FROM docker.io/gramineproject/gramine:v1.5 - -RUN curl -fsSLo /usr/share/keyrings/microsoft.asc https://packages.microsoft.com/keys/microsoft.asc \ - && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft.asc] https://packages.microsoft.com/ubuntu/20.04/prod focal main" > /etc/apt/sources.list.d/msprod.list \ - && apt-get update \ - && apt purge -y libsgx-dcap-default-qpl \ - && apt-get install -y az-dcap-client - -RUN apt purge -y libsgx-ae-qve -# libsgx-urts - -RUN rm -rf /var/lib/apt/lists/* - -# So we only have to use one gramine template -RUN touch /etc/sgx_default_qcnl.conf - -WORKDIR /app - -COPY --from=buildtee /build/tee-vault-unseal . -COPY ./bin/tee-vault-unseal/tee-vault-unseal.manifest.template . -COPY vault/enclave-key.pem . -RUN mkdir -p /opt/vault/tls && rm -rf /opt/vault/tls/* - -# The original Azure library is still delivering expired collateral, so we have to use a patched version -COPY --from=azuredcap /usr/local/lib/libdcap_quoteprov.so /usr/lib/ - -RUN gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu -Dexecdir=/usr/bin -Dlog_level=warning tee-vault-unseal.manifest.template tee-vault-unseal.manifest \ - && gramine-sgx-sign --manifest tee-vault-unseal.manifest --output tee-vault-unseal.manifest.sgx --key enclave-key.pem \ - && rm enclave-key.pem - -VOLUME /opt/vault/tls - -EXPOSE 8443 - -ENTRYPOINT ["/bin/sh", "-c"] -CMD [ "/restart_aesm.sh ; exec gramine-sgx tee-vault-unseal" ] diff --git a/bin/tee-vault-unseal/tee-vault-unseal.manifest.template b/bin/tee-vault-unseal/tee-vault-unseal.manifest.template deleted file mode 100644 index b30bf2a8..00000000 --- a/bin/tee-vault-unseal/tee-vault-unseal.manifest.template +++ /dev/null @@ -1,62 +0,0 @@ -libos.entrypoint = "/app/tee-vault-unseal" - -[loader] -argv = [ "/app/tee-vault-unseal" ] -entrypoint = "file:{{ gramine.libos }}" -env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}:/usr{{ arch_libdir }}:/lib" -env.HOME = "/app" -env.MALLOC_ARENA_MAX = "1" -env.AZDCAP_DEBUG_LOG_LEVEL = "ignore" -env.AZDCAP_COLLATERAL_VERSION = "v4" - -### Required configuration ### -env.ALLOWED_TCB_LEVELS = { passthrough = true } -env.VAULT_ADDR = { passthrough = true } - -### DEBUG ### -env.RUST_BACKTRACE = "1" -env.RUST_LOG="info,tee_vault_unseal=trace,teepot=trace,awc=debug" - -[fs] -root.uri = "file:/" -start_dir = "/app" -mounts = [ - { path = "{{ execdir }}", uri = "file:{{ execdir }}" }, - { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" }, - { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" }, - { path = "/etc", uri = "file:/etc" }, - { type = "tmpfs", path = "/var/tmp" }, - { type = "tmpfs", path = "/tmp" }, - { type = "tmpfs", path = "/app/.dcap-qcnl" }, - { type = "tmpfs", path = "/app/.az-dcap-client" }, - { type = "encrypted", path = "/opt/vault/tls", uri = "file:/opt/vault/tls", key_name = "_sgx_mrsigner" }, - { path = "/lib/libdcap_quoteprov.so", uri = "file:/lib/libdcap_quoteprov.so" }, -] - -[sgx] -trusted_files = [ - "file:/etc/ld.so.cache", - "file:/app/", - "file:{{ execdir }}/", - "file:{{ arch_libdir }}/", - "file:/usr/{{ arch_libdir }}/", - "file:{{ gramine.libos }}", - "file:{{ gramine.runtimedir() }}/", - "file:/usr/lib/ssl/openssl.cnf", - "file:/etc/ssl/", - "file:/etc/sgx_default_qcnl.conf", - "file:/lib/libdcap_quoteprov.so", -] -remote_attestation = "dcap" -max_threads = 64 -edmm_enable = false -## max enclave size -enclave_size = "2G" - -[sys] -enable_extra_runtime_domain_names_conf = true -enable_sigterm_injection = true - -# possible tweak option, if problems with mio -# currently mio is compiled with `mio_unsupported_force_waker_pipe` -# insecure__allow_eventfd = true diff --git a/examples/README.md b/examples/README.md index ab7275e2..b964faef 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,9 +8,13 @@ $ docker compose up ``` ## Build and Run on client machine + ```bash -$ cd teepot -$ gpg --export username@example.com | base64 > gpgkey.pub +❯ cd teepot +❯ gpg --export username@example.com | base64 > gpgkey.pub +❯ export GPG_TTY="$(tty)" +❯ gpg-connect-agent updatestartuptty /bye + ❯ RUST_LOG=info cargo run -p vault-unseal -- --sgx-mrsigner c5591a72b8b86e0d8814d6e8750e3efe66aea2d102b8ba2405365559b858697d --sgx-allowed-tcb-levels SwHardeningNeeded --server https://20.172.154.218:8443 init --unseal-threshold 1 -u bin/tee-vault-admin/tests/data/gpgkey.pub --admin-threshold 1 -a bin/tee-vault-admin/tests/data/gpgkey.pub --admin-tee-mrenclave 21c8c1a4dbcce04798f5119eb47203084bc74e564a3c954d1a21172c656cb801 Finished dev [unoptimized + debuginfo] target(s) in 0.09s Running `target/debug/vault-unseal --sgx-mrsigner c5591a72b8b86e0d8814d6e8750e3efe66aea2d102b8ba2405365559b858697d --sgx-allowed-tcb-levels SwHardeningNeeded --server 'https://20.172.154.218:8443' init --unseal-threshold 1 -u bin/tee-vault-admin/tests/data/gpgkey.pub --admin-threshold 1 -a bin/tee-vault-admin/tests/data/gpgkey.pub --admin-tee-mrenclave 21c8c1a4dbcce04798f5119eb47203084bc74e564a3c954d1a21172c656cb801` @@ -43,14 +47,17 @@ Vault is unsealed! ``` +With `teepot-admin` being the name of the image running the tee-vault-admin service, the following commands can be used +to sign the admin tee: + ```bash -❯ (id=$(docker create tva); docker cp $id:/app/tee-vault-admin.sig ~/tee-vault-admin.sig; docker rm -v $id) -❯ cargo run -p vault-admin -- create-sign-request ~/tee-vault-admin.sig > ~/sign_admin_tee.json +❯ (id=$(docker create teepot-admin); docker cp $id:/app/tee-vault-admin.sig ~/tee-vault-admin.sig; docker rm -v $id) +❯ cargo run -p vault-admin -- create-sign-request --tee-name admin ~/tee-vault-admin.sig > ~/sign_admin_tee.json ❯ vim sign_admin_tee.json ❯ gpg --local-user test@example.com --detach-sign --armor ~/sign_admin_tee.json ❯ RUST_LOG=info cargo run -p vault-admin -- \ sign-tee \ - --sgx-mrenclave 080c3210d5b6bcf47887101a554c117c21d80e75240bb70846c3e158a713ec65 \ + --sgx-mrenclave c5591a72b8b86e0d8814d6e8750e3efe66aea2d102b8ba2405365559b858697d \ --sgx-allowed-tcb-levels SwHardeningNeeded \ --server https://127.0.0.1:8444 \ --out new_admin.sig \ @@ -217,7 +224,8 @@ Next is to sign the admin tee with the vault-admin tool: ~/tee-vault-admin.json ~/tee-vault-admin.json.asc ``` -Then replace `tee-vault-admin.sig` with `tee-vault-admin-new.sig` in the container image `matterlabsrobot/teepot-tva:latest` with this Dockerfile: +Then replace `tee-vault-admin.sig` with `tee-vault-admin-new.sig` in the container +image `matterlabsrobot/teepot-tva:latest` with this Dockerfile: ```Dockerfile FROM matterlabsrobot/teepot-tva:latest diff --git a/examples/docker-compose.yml b/examples/docker-compose.yml index 132ef93c..53fe6cd5 100644 --- a/examples/docker-compose.yml +++ b/examples/docker-compose.yml @@ -4,29 +4,31 @@ services: tvu-1: build: context: . - dockerfile: bin/tee-vault-unseal/Dockerfile-azure + dockerfile: packages/container-vault-unseal-sgx-azure/Dockerfile image: tee-vault-unseal restart: "no" ports: - - 8413:8443 + - 8413:8413 environment: + PORT: "8413" VAULT_ADDR: "https://vault-1:8210" + VAULT_SGX_MRSIGNER: "c5591a72b8b86e0d8814d6e8750e3efe66aea2d102b8ba2405365559b858697d" + VAULT_SGX_ALLOWED_TCB_LEVELS: "SwHardeningNeeded" ALLOWED_TCB_LEVELS: "SwHardeningNeeded" privileged: true init: true volumes: - /run/aesmd:/run/aesmd - /dev/sgx_enclave:/dev/sgx_enclave - - shared-1:/opt/vault/tls vault-1: build: context: . - dockerfile: vault/Dockerfile + dockerfile: packages/container-vault-sgx-azure/Dockerfile image: vault restart: "no" ports: - 8210:8210 -# - 8211:8211 + # - 8211:8211 environment: VAULT_API_ADDR: "https://vault-1:8210" VAULT_CLUSTER_ADDR: "https://vault-1:8211" @@ -36,7 +38,6 @@ services: volumes: - /run/aesmd:/run/aesmd - /dev/sgx_enclave:/dev/sgx_enclave - - shared-1:/opt/vault/tls - data-1:/opt/vault/data tvu-2: @@ -45,20 +46,22 @@ services: ports: - 8423:8443 environment: + PORT: "8443" VAULT_ADDR: "https://vault-2:8210" + VAULT_SGX_MRSIGNER: "c5591a72b8b86e0d8814d6e8750e3efe66aea2d102b8ba2405365559b858697d" + VAULT_SGX_ALLOWED_TCB_LEVELS: "SwHardeningNeeded" ALLOWED_TCB_LEVELS: "SwHardeningNeeded" privileged: true init: true volumes: - /run/aesmd:/run/aesmd - /dev/sgx_enclave:/dev/sgx_enclave - - shared-2:/opt/vault/tls vault-2: image: vault restart: "no" ports: - 8220:8210 -# - 8221:8211 + # - 8221:8211 environment: VAULT_API_ADDR: "https://vault-2:8210" VAULT_CLUSTER_ADDR: "https://vault-2:8211" @@ -68,7 +71,6 @@ services: volumes: - /run/aesmd:/run/aesmd - /dev/sgx_enclave:/dev/sgx_enclave - - shared-2:/opt/vault/tls - data-2:/opt/vault/data tvu-3: @@ -77,20 +79,21 @@ services: ports: - 8433:8443 environment: + PORT: "8443" VAULT_ADDR: "https://vault-3:8210" + VAULT_SGX_MRSIGNER: "c5591a72b8b86e0d8814d6e8750e3efe66aea2d102b8ba2405365559b858697d" + VAULT_SGX_ALLOWED_TCB_LEVELS: "SwHardeningNeeded" ALLOWED_TCB_LEVELS: "SwHardeningNeeded" privileged: true init: true volumes: - /run/aesmd:/run/aesmd - /dev/sgx_enclave:/dev/sgx_enclave - - shared-3:/opt/vault/tls vault-3: image: vault restart: "no" ports: - 8230:8210 -# - 8231:8211 environment: VAULT_API_ADDR: "https://vault-3:8210" VAULT_CLUSTER_ADDR: "https://vault-3:8211" @@ -100,17 +103,17 @@ services: volumes: - /run/aesmd:/run/aesmd - /dev/sgx_enclave:/dev/sgx_enclave - - shared-3:/opt/vault/tls - data-3:/opt/vault/data admin: build: context: . - dockerfile: bin/tee-vault-admin/Dockerfile-azure + dockerfile: packages/container-vault-admin-sgx-azure/Dockerfile restart: "no" ports: - 8444:8444 environment: + PORT: "8444" VAULT_ADDR: "https://vault-1:8210" VAULT_SGX_MRSIGNER: "c5591a72b8b86e0d8814d6e8750e3efe66aea2d102b8ba2405365559b858697d" VAULT_SGX_ALLOWED_TCB_LEVELS: "SwHardeningNeeded" @@ -121,21 +124,21 @@ services: - /run/aesmd:/run/aesmd - /dev/sgx_enclave:/dev/sgx_enclave - stress: - build: - context: . - dockerfile: bin/tee-stress-client/Dockerfile-azure - restart: "no" - environment: - VAULT_ADDR: "https://vault-1:8210" - VAULT_SGX_MRSIGNER: "c5591a72b8b86e0d8814d6e8750e3efe66aea2d102b8ba2405365559b858697d" - VAULT_SGX_ALLOWED_TCB_LEVELS: "SwHardeningNeeded" - ALLOWED_TCB_LEVELS: "SwHardeningNeeded" - privileged: true - init: true - volumes: - - /run/aesmd:/run/aesmd - - /dev/sgx_enclave:/dev/sgx_enclave +# stress: +# build: +# context: . +# dockerfile: bin/tee-stress-client/Dockerfile-azure +# restart: "no" +# environment: +# VAULT_ADDR: "https://vault-1:8210" +# VAULT_SGX_MRSIGNER: "c5591a72b8b86e0d8814d6e8750e3efe66aea2d102b8ba2405365559b858697d" +# VAULT_SGX_ALLOWED_TCB_LEVELS: "SwHardeningNeeded" +# ALLOWED_TCB_LEVELS: "SwHardeningNeeded" +# privileged: true +# init: true +# volumes: +# - /run/aesmd:/run/aesmd +# - /dev/sgx_enclave:/dev/sgx_enclave volumes: shared-1: diff --git a/flake.lock b/flake.lock index d1e3df7e..eb25ea41 100644 --- a/flake.lock +++ b/flake.lock @@ -32,6 +32,38 @@ "type": "github" } }, + "flake-compat_3": { + "flake": false, + "locked": { + "lastModified": 1650374568, + "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "b4a34015c698c7793d592d66adbab377907a2be8", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_4": { + "flake": false, + "locked": { + "lastModified": 1650374568, + "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "b4a34015c698c7793d592d66adbab377907a2be8", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -86,6 +118,42 @@ "type": "github" } }, + "flake-utils-plus_3": { + "inputs": { + "flake-utils": "flake-utils_4" + }, + "locked": { + "lastModified": 1696331477, + "narHash": "sha256-YkbRa/1wQWdWkVJ01JvV+75KIdM37UErqKgTf0L54Fk=", + "owner": "gytis-ivaskevicius", + "repo": "flake-utils-plus", + "rev": "bfc53579db89de750b25b0c5e7af299e0c06d7d3", + "type": "github" + }, + "original": { + "owner": "gytis-ivaskevicius", + "repo": "flake-utils-plus", + "type": "github" + } + }, + "flake-utils-plus_4": { + "inputs": { + "flake-utils": "flake-utils_5" + }, + "locked": { + "lastModified": 1696331477, + "narHash": "sha256-YkbRa/1wQWdWkVJ01JvV+75KIdM37UErqKgTf0L54Fk=", + "owner": "gytis-ivaskevicius", + "repo": "flake-utils-plus", + "rev": "bfc53579db89de750b25b0c5e7af299e0c06d7d3", + "type": "github" + }, + "original": { + "owner": "gytis-ivaskevicius", + "repo": "flake-utils-plus", + "type": "github" + } + }, "flake-utils_2": { "inputs": { "systems": "systems_2" @@ -122,19 +190,39 @@ "type": "github" } }, - "nix-filter": { + "flake-utils_4": { + "inputs": { + "systems": "systems_4" + }, + "locked": { + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_5": { + "inputs": { + "systems": "systems_5" + }, "locked": { - "lastModified": 1705332318, - "narHash": "sha256-kcw1yFeJe9N4PjQji9ZeX47jg0p9A0DuU4djKvg1a7I=", + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", "owner": "numtide", - "repo": "nix-filter", - "rev": "3449dc925982ad46246cfc36469baf66e1b64f17", + "repo": "flake-utils", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", "type": "github" }, "original": { "owner": "numtide", - "repo": "nix-filter", - "rev": "3449dc925982ad46246cfc36469baf66e1b64f17", + "repo": "flake-utils", "type": "github" } }, @@ -175,13 +263,35 @@ "type": "github" } }, + "nixsgx-flake_2": { + "inputs": { + "nixpkgs": [ + "vault-auth-tee-flake", + "nixpkgs" + ], + "snowfall-lib": "snowfall-lib_3" + }, + "locked": { + "lastModified": 1707314146, + "narHash": "sha256-NcqWVXkGBautT44YcQgZdVK1Vwqop1V8nDoPEzKr0uE=", + "owner": "matter-labs", + "repo": "nixsgx", + "rev": "e3bbd5415eb845c8857dd0963febcef71866cd4a", + "type": "github" + }, + "original": { + "owner": "matter-labs", + "repo": "nixsgx", + "type": "github" + } + }, "root": { "inputs": { - "nix-filter": "nix-filter", "nixpkgs": "nixpkgs", "nixsgx-flake": "nixsgx-flake", "rust-overlay": "rust-overlay", - "snowfall-lib": "snowfall-lib_2" + "snowfall-lib": "snowfall-lib_2", + "vault-auth-tee-flake": "vault-auth-tee-flake" } }, "rust-overlay": { @@ -252,6 +362,54 @@ "type": "github" } }, + "snowfall-lib_3": { + "inputs": { + "flake-compat": "flake-compat_3", + "flake-utils-plus": "flake-utils-plus_3", + "nixpkgs": [ + "vault-auth-tee-flake", + "nixsgx-flake", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1696432959, + "narHash": "sha256-oJQZv2MYyJaVyVJY5IeevzqpGvMGKu5pZcCCJvb+xjc=", + "owner": "snowfallorg", + "repo": "lib", + "rev": "92803a029b5314d4436a8d9311d8707b71d9f0b6", + "type": "github" + }, + "original": { + "owner": "snowfallorg", + "repo": "lib", + "type": "github" + } + }, + "snowfall-lib_4": { + "inputs": { + "flake-compat": "flake-compat_4", + "flake-utils-plus": "flake-utils-plus_4", + "nixpkgs": [ + "vault-auth-tee-flake", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1696432959, + "narHash": "sha256-oJQZv2MYyJaVyVJY5IeevzqpGvMGKu5pZcCCJvb+xjc=", + "owner": "snowfallorg", + "repo": "lib", + "rev": "92803a029b5314d4436a8d9311d8707b71d9f0b6", + "type": "github" + }, + "original": { + "owner": "snowfallorg", + "repo": "lib", + "rev": "92803a029b5314d4436a8d9311d8707b71d9f0b6", + "type": "github" + } + }, "systems": { "locked": { "lastModified": 1681028828, @@ -296,6 +454,58 @@ "repo": "default", "type": "github" } + }, + "systems_4": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_5": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "vault-auth-tee-flake": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "nixsgx-flake": "nixsgx-flake_2", + "snowfall-lib": "snowfall-lib_4" + }, + "locked": { + "lastModified": 1708593089, + "narHash": "sha256-HtRZmjLZEcwTRLbqxRIkilLL94OZccbe22UtNInhk4Y=", + "owner": "matter-labs", + "repo": "vault-auth-tee", + "rev": "0f9a672c9925e9de6888ca4b30bc4573c52c6899", + "type": "github" + }, + "original": { + "owner": "matter-labs", + "repo": "vault-auth-tee", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 6786f8c3..5b2afd2d 100644 --- a/flake.nix +++ b/flake.nix @@ -9,6 +9,11 @@ inputs.nixpkgs.follows = "nixpkgs"; }; + vault-auth-tee-flake = { + url = "github:matter-labs/vault-auth-tee"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + snowfall-lib = { url = "github:snowfallorg/lib?rev=92803a029b5314d4436a8d9311d8707b71d9f0b6"; inputs.nixpkgs.follows = "nixpkgs"; @@ -29,6 +34,7 @@ overlays = with inputs; [ nixsgx-flake.overlays.default + vault-auth-tee-flake.overlays.default rust-overlay.overlays.default ]; diff --git a/packages/container-vault-admin-sgx-azure/Dockerfile b/packages/container-vault-admin-sgx-azure/Dockerfile new file mode 100644 index 00000000..09a9d234 --- /dev/null +++ b/packages/container-vault-admin-sgx-azure/Dockerfile @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Matter Labs + +FROM teepot-vault-admin-sgx-azure:base + +WORKDIR /app + +COPY packages/container-vault-sgx-azure/test-enclave-key.pem /tmp/ + +RUN set -eux; \ + gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu \ + -Dentrypoint=$(readlink /bin/tee-vault-admin) \ + -Dexecdir=/bin \ + -Dlog_level=warning \ + tee-vault-admin.manifest.toml tee-vault-admin.manifest; \ + gramine-sgx-sign --manifest tee-vault-admin.manifest --output tee-vault-admin.manifest.sgx --key /tmp/test-enclave-key.pem; \ + rm /tmp/test-enclave-key.pem + +VOLUME /opt/vault/tls + +EXPOSE 8443 + +ENTRYPOINT ["/bin/sh", "-c"] +ENV SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt +CMD [ "echo tee-vault-admin in SGX mode starting; restart-aesmd ; exec gramine-sgx tee-vault-admin" ] diff --git a/packages/container-vault-admin-sgx-azure/default.nix b/packages/container-vault-admin-sgx-azure/default.nix new file mode 100644 index 00000000..f0b8b0ff --- /dev/null +++ b/packages/container-vault-admin-sgx-azure/default.nix @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Matter Labs +{ pkgs +, vat +, nixsgx +, curl +, teepot +, bash +, coreutils +, openssl +, vault +}: +let manifest = ./tee-vault-admin.manifest.toml; +in pkgs.dockerTools.buildLayeredImage { + name = "teepot-vault-admin-sgx-azure"; + tag = "base"; + + config.Entrypoint = [ "/bin/sh" "-c" ]; + + contents = pkgs.buildEnv { + name = "image-root"; + + paths = with pkgs.dockerTools; with nixsgx; with teepot;[ + bash + coreutils + openssl + vault + azure-dcap-client + curl + teepot.teepot.tee_vault_admin + gramine + restart-aesmd + sgx-dcap.quote_verify + sgx-psw + usrBinEnv + binSh + caCertificates + fakeNss + ]; + pathsToLink = [ "/bin" "/lib" "/etc" "/app" ]; + postBuild = '' + mkdir -p $out/{app,etc} + cp ${manifest} $out/app/tee-vault-admin.manifest.toml + mkdir -p $out/var/run + mkdir -p $out/${nixsgx.sgx-psw.out}/aesm/ + touch $out/etc/sgx_default_qcnl.conf + ln -s ${curl.out}/lib/libcurl.so $out/${nixsgx.sgx-psw.out}/aesm/ + ln -s ${nixsgx.azure-dcap-client.out}/lib/libdcap_quoteprov.so $out/${nixsgx.sgx-psw.out}/aesm/libdcap_quoteprov.so.1 + printf "precedence ::ffff:0:0/96 100\n" > $out/etc/gai.conf + ''; + }; +} diff --git a/packages/container-vault-admin-sgx-azure/tee-vault-admin.manifest.toml b/packages/container-vault-admin-sgx-azure/tee-vault-admin.manifest.toml new file mode 100644 index 00000000..443116cc --- /dev/null +++ b/packages/container-vault-admin-sgx-azure/tee-vault-admin.manifest.toml @@ -0,0 +1,71 @@ +libos.entrypoint = "{{ entrypoint }}" + +[loader] +argv = ["{{ entrypoint }}"] +entrypoint = "file:{{ gramine.libos }}" + +[loader.env] +### Admin Config ### +PORT = { passthrough = true } + +### VAULT attestation ### +VAULT_ADDR = { passthrough = true } +VAULT_SGX_MRENCLAVE = { passthrough = true } +VAULT_SGX_MRSIGNER = { passthrough = true } +VAULT_SGX_ALLOWED_TCB_LEVELS = { passthrough = true } + +### DEBUG ### +RUST_BACKTRACE = "1" +RUST_LOG = "info,tee_vault_admin=trace,teepot=trace,vault_tee_client=trace,tee_client=trace,awc=debug" + +### Enclave security ### +ALLOWED_TCB_LEVELS = "SwHardeningNeeded" + +### Fixed values ### +LD_LIBRARY_PATH = "/lib" +SSL_CERT_FILE = "/etc/ssl/certs/ca-bundle.crt" +PATH = "/bin" +HOME = "/app" + +MALLOC_ARENA_MAX = "1" +AZDCAP_DEBUG_LOG_LEVEL = "ignore" +AZDCAP_COLLATERAL_VERSION = "v4" + +[fs] +root.uri = "file:/" +start_dir = "/app" +mounts = [ + { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" }, + { path = "{{ gramine.runtimedir() }}/libdcap_quoteprov.so", uri = "file:/lib/libdcap_quoteprov.so" }, + { type = "tmpfs", path = "/var/tmp" }, + { type = "tmpfs", path = "/tmp" }, + { type = "tmpfs", path = "/app/.dcap-qcnl" }, + { type = "tmpfs", path = "/app/.az-dcap-client" }, +] + +[sgx] +trusted_files = [ + "file:/app/", + "file:/bin/", + "file:/etc/gai.conf", + "file:/etc/sgx_default_qcnl.conf", + "file:/etc/ssl/certs/ca-bundle.crt", + "file:/lib/", + "file:/lib/libdcap_quoteprov.so", + "file:/nix/", + "file:{{ gramine.libos }}", + "file:{{ gramine.runtimedir() }}/", +] +remote_attestation = "dcap" +max_threads = 64 +edmm_enable = false +## max enclave size +enclave_size = "2G" + +[sys] +enable_extra_runtime_domain_names_conf = true +enable_sigterm_injection = true + +# possible tweak option, if problems with mio +# currently mio is compiled with `mio_unsupported_force_waker_pipe` +# insecure__allow_eventfd = true diff --git a/packages/container-vault-sgx-azure/Dockerfile b/packages/container-vault-sgx-azure/Dockerfile new file mode 100644 index 00000000..e1530fcd --- /dev/null +++ b/packages/container-vault-sgx-azure/Dockerfile @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Matter Labs + +FROM teepot-vault-sgx-azure:base + +WORKDIR /opt/vault + +COPY packages/container-vault-sgx-azure/test-enclave-key.pem /tmp/ +RUN set -eux; \ + gramine-manifest \ + -Dtee_ratls_preexec=$(readlink /bin/tee-ratls-preexec) \ + -Dvault_exec=$(readlink /bin/vault) \ + -Darch_libdir=/lib/x86_64-linux-gnu \ + -Dexecdir=/bin \ + -Dlog_level=warning \ + vault.manifest.toml vault.manifest; \ + gramine-sgx-sign --manifest vault.manifest --output vault.manifest.sgx --key /tmp/test-enclave-key.pem; \ + rm /tmp/test-enclave-key.pem + +VOLUME /opt/vault/tls +VOLUME /opt/vault/data + +ENTRYPOINT ["/bin/sh", "-c"] +ENV SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt +CMD [ "echo vault in SGX mode starting; restart-aesmd ; exec gramine-sgx vault" ] diff --git a/packages/container-vault-sgx-azure/default.nix b/packages/container-vault-sgx-azure/default.nix new file mode 100644 index 00000000..2b9301b8 --- /dev/null +++ b/packages/container-vault-sgx-azure/default.nix @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Matter Labs +{ pkgs +, vat +, nixsgx +, curl +, teepot +, bash +, coreutils +, openssl +, vault +}: +let manifest = ./vault.manifest.toml; +in pkgs.dockerTools.buildLayeredImage { + name = "teepot-vault-sgx-azure"; + tag = "base"; + + config.Entrypoint = [ "/bin/sh" "-c" ]; + + contents = pkgs.buildEnv { + name = "image-root"; + + paths = with pkgs.dockerTools; with nixsgx;[ + bash + coreutils + teepot.teepot.tee_ratls_preexec + vault + azure-dcap-client + curl + vat.vault-auth-tee + gramine + restart-aesmd + sgx-dcap.quote_verify + sgx-psw + usrBinEnv + binSh + caCertificates + fakeNss + teepot.container-vault-start-config + ]; + pathsToLink = [ "/bin" "/lib" "/etc" "/opt/vault" ]; + postBuild = '' + mkdir -p $out/var/run + mkdir -p $out/${nixsgx.sgx-psw.out}/aesm/ + mkdir -p $out/opt/vault/data $out/opt/vault/.cache $out/opt/vault/tls + ln -s ${curl.out}/lib/libcurl.so $out/${nixsgx.sgx-psw.out}/aesm/ + ln -s ${nixsgx.azure-dcap-client.out}/lib/libdcap_quoteprov.so $out/${nixsgx.sgx-psw.out}/aesm/libdcap_quoteprov.so.1 + mkdir -p $out/opt/vault/plugins + ln -s ${vat.vault-auth-tee}/bin/vault-auth-tee $out/opt/vault/plugins + cp ${manifest} $out/opt/vault/vault.manifest.toml + ''; + }; +} diff --git a/vault/enclave-key.pem b/packages/container-vault-sgx-azure/test-enclave-key.pem similarity index 100% rename from vault/enclave-key.pem rename to packages/container-vault-sgx-azure/test-enclave-key.pem diff --git a/vault/vault.manifest.toml b/packages/container-vault-sgx-azure/vault.manifest.toml similarity index 59% rename from vault/vault.manifest.toml rename to packages/container-vault-sgx-azure/vault.manifest.toml index 84b62802..f654a542 100644 --- a/vault/vault.manifest.toml +++ b/packages/container-vault-sgx-azure/vault.manifest.toml @@ -1,19 +1,30 @@ -libos.entrypoint = "/bin/bash" +libos.entrypoint = "{{ tee_ratls_preexec }}" [loader] entrypoint = "file:{{ gramine.libos }}" -argv = ["bash", "/opt/vault/start.sh"] +argv = [ + "{{ tee_ratls_preexec }}", + "--", + "{{ vault_exec }}", + "server", + "-config=/opt/vault/config.hcl", + "-log-level=trace", +] # set a log level for gramine log_level = "{{ log_level }}" [loader.env] -LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}:/usr{{ arch_libdir }}" +LD_LIBRARY_PATH = "/lib" +SSL_CERT_FILE = "/etc/ssl/certs/ca-bundle.crt" PATH = "{{ execdir }}" +HOME = "/opt/vault" VAULT_CLUSTER_ADDR.passthrough = true VAULT_API_ADDR.passthrough = true VAULT_RAFT_NODE_ID.passthrough = true +DNS_NAMES = "vault-1,vault-2,vault-3" + # otherwise vault will lock a lot of unused EPC memory VAULT_RAFT_INITIAL_MMAP_SIZE = "0" @@ -22,18 +33,16 @@ VAULT_RAFT_INITIAL_MMAP_SIZE = "0" [fs] root.uri = "file:/" -start_dir = "/root" +start_dir = "/opt/vault" mounts = [ - { path = "{{ execdir }}", uri = "file:{{ execdir }}" }, { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" }, - { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" }, + { path = "{{ gramine.runtimedir() }}/libdcap_quoteprov.so", uri = "file:/lib/libdcap_quoteprov.so" }, { type = "tmpfs", path = "/var/tmp" }, { type = "tmpfs", path = "/tmp" }, - { type = "tmpfs", path = "/app/.dcap-qcnl" }, - { type = "tmpfs", path = "/app/.az-dcap-client" }, - { path = "/lib/libdcap_quoteprov.so", uri = "file:/lib/libdcap_quoteprov.so" }, + { type = "tmpfs", path = "/opt/vault/tls" }, + { type = "tmpfs", path = "/opt/vault/.dcap-qcnl" }, + { type = "tmpfs", path = "/opt/vault/.az-dcap-client" }, { type = "encrypted", path = "/opt/vault/.cache", uri = "file:/opt/vault/.cache", key_name = "_sgx_mrsigner" }, - { type = "encrypted", path = "/opt/vault/tls", uri = "file:/opt/vault/tls", key_name = "_sgx_mrsigner" }, { type = "encrypted", path = "/opt/vault/data", uri = "file:/opt/vault/data", key_name = "_sgx_mrsigner" }, ] @@ -46,20 +55,19 @@ nonpie_binary = true remote_attestation = "dcap" trusted_files = [ - "file:/bin/bash", + "file:/bin/", + "file:/etc/gai.conf", + "file:/etc/ssl/certs/ca-bundle.crt", + "file:/lib/", + "file:/nix/", + "file:/opt/vault/plugins/", + "file:/opt/vault/config.hcl", + "file:/opt/vault/cacert.pem", + "file:/opt/vault/cakey.pem", "file:{{ gramine.libos }}", - "file:{{ execdir }}/", "file:{{ gramine.runtimedir() }}/", - "file:{{ arch_libdir }}/", - "file:/usr/{{ arch_libdir }}/", - "file:/usr/lib/ssl/openssl.cnf", - "file:/etc/ssl/", - "file:/lib/libdcap_quoteprov.so", - "file:/opt/vault/", ] -#file_check_policy = "allow_all_but_log" - [sys] stack.size = "1M" enable_extra_runtime_domain_names_conf = true diff --git a/vault/cacert.pem b/packages/container-vault-start-config/cacert.pem similarity index 100% rename from vault/cacert.pem rename to packages/container-vault-start-config/cacert.pem diff --git a/vault/cakey.pem b/packages/container-vault-start-config/cakey.pem similarity index 100% rename from vault/cakey.pem rename to packages/container-vault-start-config/cakey.pem diff --git a/packages/container-vault-start-config/config.hcl b/packages/container-vault-start-config/config.hcl new file mode 100644 index 00000000..d03baa6f --- /dev/null +++ b/packages/container-vault-start-config/config.hcl @@ -0,0 +1,55 @@ +# Parameter needed because of slow plugin loading +# may be relaxed for faster machines +#http_read_header_timeout = 0 +#http_read_timeout = 300 + +disable_mlock = true +ui = false + +listener "tcp" { + address = "0.0.0.0:8210" + cluster_address = "0.0.0.0:8211" + tls_disable = false + tls_cert_file = "/opt/vault/tls/tls.crt" + tls_key_file = "/opt/vault/tls/tls.key" + tls_client_ca_file = "/opt/vault/cacert.pem" +} + +storage "raft" { + path = "/opt/vault/data/" + # override vial env var VAULT_RAFT_NODE_ID + node_id = "vault-1" + + # Parameter needed because of slow plugin loading + # may be relaxed for faster machines + # performance_multiplier = 200 + #autopilot_reconcile_interval = "120s" + #autopilot_update_interval = "60s" + + retry_join { + leader_api_addr = "https://vault-1:8210" + leader_ca_cert_file = "/opt/vault/cacert.pem" + leader_client_cert_file = "/opt/vault/tls/tls.crt" + leader_client_key_file = "/opt/vault/tls/tls.key" + } + retry_join { + leader_api_addr = "https://vault-2:8210" + leader_ca_cert_file = "/opt/vault/cacert.pem" + leader_client_cert_file = "/opt/vault/tls/tls.crt" + leader_client_key_file = "/opt/vault/tls/tls.key" + } + retry_join { + leader_api_addr = "https://vault-3:8210" + leader_ca_cert_file = "/opt/vault/cacert.pem" + leader_client_cert_file = "/opt/vault/tls/tls.crt" + leader_client_key_file = "/opt/vault/tls/tls.key" + } +} + +# path of plugin binaries +plugin_directory = "/opt/vault/plugins" + +# override via env var VAULT_API_ADDR +api_addr = "https://vault:8210" +# override via env var VAULT_CLUSTER_ADDR +cluster_addr = "https://vault:8211" diff --git a/packages/container-vault-start-config/default.nix b/packages/container-vault-start-config/default.nix new file mode 100644 index 00000000..78300e23 --- /dev/null +++ b/packages/container-vault-start-config/default.nix @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Matter Labs + +# TODO: This derivation is a temporary workaround for +# creating a self-signed certificate for Vault and the unseal TEE. +# It will be replaced with real RA-TLS. +{ lib +, stdenv +}: +stdenv.mkDerivation rec { + name = "container-vault-start-config"; + src = with lib.fileset; toSource { + root = ./.; + fileset = unions [ + ./cacert.pem + ./cakey.pem + ./config.hcl + ]; + }; + + phases = "installPhase"; + postInstall = '' + mkdir -p $out/opt/vault + cp -r $src/* $out/opt/vault + + mkdir -p $out/etc + printf "precedence ::ffff:0:0/96 100\n" > $out/etc/gai.conf + ''; +} diff --git a/packages/container-vault-unseal-sgx-azure/Dockerfile b/packages/container-vault-unseal-sgx-azure/Dockerfile new file mode 100644 index 00000000..f4dcbe5a --- /dev/null +++ b/packages/container-vault-unseal-sgx-azure/Dockerfile @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Matter Labs + +FROM teepot-vault-unseal-sgx-azure:base + +WORKDIR /app + +COPY packages/container-vault-sgx-azure/test-enclave-key.pem /tmp/ + +RUN set -eux; \ + gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu \ + -Dentrypoint=$(readlink /bin/tee-vault-unseal) \ + -Dvault_auth_tee_sha=1505ffe3ef0994de63206ccce1dec6715d55e72955960cb7314e4571811c4422 \ + -Dvault_auth_tee_version=0.1.0+dev \ + -Dexecdir=/bin \ + -Dlog_level=warning \ + tee-vault-unseal.manifest.toml tee-vault-unseal.manifest; \ + gramine-sgx-sign --manifest tee-vault-unseal.manifest --output tee-vault-unseal.manifest.sgx --key /tmp/test-enclave-key.pem; \ + rm /tmp/test-enclave-key.pem + +EXPOSE 8443 + +ENTRYPOINT ["/bin/sh", "-c"] +ENV SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt +CMD [ "echo tee-vault-unseal in SGX mode starting; restart-aesmd ; exec gramine-sgx tee-vault-unseal" ] diff --git a/packages/container-vault-unseal-sgx-azure/default.nix b/packages/container-vault-unseal-sgx-azure/default.nix new file mode 100644 index 00000000..6bb6bf40 --- /dev/null +++ b/packages/container-vault-unseal-sgx-azure/default.nix @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Matter Labs +{ pkgs +, vat +, nixsgx +, curl +, teepot +, bash +, coreutils +, openssl +, vault +}: +let manifest = ./tee-vault-unseal.manifest.toml; +in pkgs.dockerTools.buildLayeredImage { + name = "teepot-vault-unseal-sgx-azure"; + tag = "base"; + + config.Entrypoint = [ "/bin/sh" "-c" ]; + + contents = pkgs.buildEnv { + name = "image-root"; + + paths = with pkgs.dockerTools; with nixsgx; with teepot;[ + bash + coreutils + openssl + vault + azure-dcap-client + curl + vat.vault-auth-tee.sha + teepot.teepot.tee_vault_unseal + gramine + restart-aesmd + sgx-dcap.quote_verify + sgx-psw + usrBinEnv + binSh + caCertificates + fakeNss + ]; + pathsToLink = [ "/bin" "/lib" "/etc" "/share" "/app" ]; + postBuild = '' + mkdir -p $out/{app,etc} + cp ${manifest} $out/app/tee-vault-unseal.manifest.toml + mkdir -p $out/var/run + mkdir -p $out/${nixsgx.sgx-psw.out}/aesm/ + touch $out/etc/sgx_default_qcnl.conf + mkdir -p $out/opt/vault/.cache $out/opt/vault/tls + ln -s ${curl.out}/lib/libcurl.so $out/${nixsgx.sgx-psw.out}/aesm/ + ln -s ${nixsgx.azure-dcap-client.out}/lib/libdcap_quoteprov.so $out/${nixsgx.sgx-psw.out}/aesm/libdcap_quoteprov.so.1 + printf "precedence ::ffff:0:0/96 100\n" > $out/etc/gai.conf + ''; + }; +} diff --git a/packages/container-vault-unseal-sgx-azure/tee-vault-unseal.manifest.toml b/packages/container-vault-unseal-sgx-azure/tee-vault-unseal.manifest.toml new file mode 100644 index 00000000..61552937 --- /dev/null +++ b/packages/container-vault-unseal-sgx-azure/tee-vault-unseal.manifest.toml @@ -0,0 +1,74 @@ +libos.entrypoint = "{{ entrypoint }}" + +[loader] +argv = ["{{ entrypoint }}"] +entrypoint = "file:{{ gramine.libos }}" + +[loader.env] +### Admin Config ### +PORT = { passthrough = true } + +### VAULT attestation ### +VAULT_ADDR = { passthrough = true } +VAULT_SGX_MRENCLAVE = { passthrough = true } +VAULT_SGX_MRSIGNER = { passthrough = true } +VAULT_SGX_ALLOWED_TCB_LEVELS = { passthrough = true } + +### DEBUG ### +RUST_BACKTRACE = "1" +RUST_LOG = "info,tee_vault_unseal=trace,teepot=trace,vault_tee_client=trace,tee_client=trace,awc=debug" + +### Enclave security ### +ALLOWED_TCB_LEVELS = "SwHardeningNeeded" + +### Fixed values ### +LD_LIBRARY_PATH = "/lib" +SSL_CERT_FILE = "/etc/ssl/certs/ca-bundle.crt" +PATH = "/bin" +HOME = "/app" + +MALLOC_ARENA_MAX = "1" +AZDCAP_DEBUG_LOG_LEVEL = "ignore" +AZDCAP_COLLATERAL_VERSION = "v4" + +VAULT_AUTH_TEE_SHA256 = "{{ vault_auth_tee_sha }}" +VAULT_AUTH_TEE_VERSION = "{{ vault_auth_tee_version }}" + +[fs] +root.uri = "file:/" +start_dir = "/app" +mounts = [ + { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" }, + { path = "{{ gramine.runtimedir() }}/libdcap_quoteprov.so", uri = "file:/lib/libdcap_quoteprov.so" }, + { type = "tmpfs", path = "/var/tmp" }, + { type = "tmpfs", path = "/tmp" }, + { type = "tmpfs", path = "/app/.dcap-qcnl" }, + { type = "tmpfs", path = "/app/.az-dcap-client" }, +] + +[sgx] +trusted_files = [ + "file:/app/", + "file:/bin/", + "file:/etc/gai.conf", + "file:/etc/sgx_default_qcnl.conf", + "file:/etc/ssl/certs/ca-bundle.crt", + "file:/lib/", + "file:/lib/libdcap_quoteprov.so", + "file:/nix/", + "file:{{ gramine.libos }}", + "file:{{ gramine.runtimedir() }}/", +] +remote_attestation = "dcap" +max_threads = 64 +edmm_enable = false +## max enclave size +enclave_size = "2G" + +[sys] +enable_extra_runtime_domain_names_conf = true +enable_sigterm_injection = true + +# possible tweak option, if problems with mio +# currently mio is compiled with `mio_unsupported_force_waker_pipe` +# insecure__allow_eventfd = true diff --git a/packages/container-vault-unseal/default.nix b/packages/container-vault-unseal/default.nix new file mode 100644 index 00000000..771f2b7c --- /dev/null +++ b/packages/container-vault-unseal/default.nix @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Matter Labs +{ lib +, dockerTools +, nixsgx +, teepot +, buildEnv +, curl +, ... +}: +dockerTools.buildLayeredImage { + name = "vault-unseal"; + tag = "latest"; + + config.Entrypoint = [ "${teepot.teepot.vault_unseal}/bin/vault-unseal" ]; + + contents = buildEnv { + name = "image-root"; + paths = with dockerTools; with nixsgx;[ + azure-dcap-client + curl + sgx-dcap.quote_verify + usrBinEnv + binSh + caCertificates + fakeNss + teepot.teepot.vault_unseal + ]; + pathsToLink = [ "/bin" "/lib" "/etc" ]; + }; +} diff --git a/packages/container-verify-attestation/default.nix b/packages/container-verify-attestation/default.nix index 0f430116..6fb7eed0 100644 --- a/packages/container-verify-attestation/default.nix +++ b/packages/container-verify-attestation/default.nix @@ -1,3 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Matter Labs { lib , dockerTools , teepot @@ -5,6 +7,8 @@ }: dockerTools.buildImage { name = "verify-attestation"; + tag = "latest"; + copyToRoot = [ teepot.teepot.verify_attestation ]; diff --git a/packages/teepot/default.nix b/packages/teepot/default.nix index b80a76c6..2dccb0b4 100644 --- a/packages/teepot/default.nix +++ b/packages/teepot/default.nix @@ -1,3 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Matter Labs { lib , gccStdenv , makeRustPlatform @@ -34,7 +36,6 @@ rustPlatform.buildRustPackage { fileset = unions [ ../../Cargo.lock ../../Cargo.toml - ../../assets ../../bin ../../crates ../../rust-toolchain.toml @@ -42,6 +43,7 @@ rustPlatform.buildRustPackage { ../../tests ]; }; + RUSTFLAGS = "--cfg mio_unsupported_force_waker_pipe"; cargoBuildFlags = "--all"; checkType = "debug"; @@ -52,6 +54,7 @@ rustPlatform.buildRustPackage { outputs = [ "out" "tee_key_preexec" + "tee_ratls_preexec" "tee_self_attestation_test" "tee_stress_client" "tee_vault_admin" diff --git a/shells/teepot/default.nix b/shells/teepot/default.nix index 97268b73..8056de57 100644 --- a/shells/teepot/default.nix +++ b/shells/teepot/default.nix @@ -1,7 +1,16 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Matter Labs { lib -, pkgs +, mkShell +, teepot +, dive +, taplo , ... }: -pkgs.mkShell { - inputsFrom = [ pkgs.teepot.teepot ]; +mkShell { + inputsFrom = [ teepot.teepot ]; + packages = [ + dive + taplo + ]; } diff --git a/vault/Dockerfile b/vault/Dockerfile deleted file mode 100644 index 0dfd3e2d..00000000 --- a/vault/Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -FROM docker.io/ubuntu:focal - -RUN set -eux; \ - apt-get update; \ - apt-get install -y curl gpg; - -RUN set -eux; \ - curl -fsSLo /usr/share/keyrings/gramine-keyring.gpg https://packages.gramineproject.io/gramine-keyring.gpg; \ - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/gramine-keyring.gpg] https://packages.gramineproject.io/ focal main" > /etc/apt/sources.list.d/gramine.list - -RUN set -eux; \ - curl -fsSLo /usr/share/keyrings/intel-sgx-deb.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key; \ - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx-deb.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list - -RUN set -eux; \ - curl -fsSLo /usr/share/keyrings/microsoft.asc https://packages.microsoft.com/keys/microsoft.asc; \ - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft.asc] https://packages.microsoft.com/ubuntu/20.04/prod focal main" > /etc/apt/sources.list.d/msprod.list - -# Install gramine -RUN set -eux; \ - apt-get update; \ - DEBIAN_FRONTEND=noninteractive apt-get install -y gramine \ - libsgx-urts \ - libsgx-enclave-common \ - libsgx-dcap-quote-verify \ - az-dcap-client \ - psmisc \ - ; - -RUN set -eux; \ - curl -s -o - https://apt.releases.hashicorp.com/gpg | gpg --dearmor > /usr/share/keyrings/hashicorp-archive-keyring.gpg; \ - echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com focal main" > /etc/apt/sources.list.d/hashicorp.list; \ - apt-get update; \ - apt-get install -y --no-install-recommends vault libcap2-bin; - -RUN rm -rf /var/lib/apt/lists/* - -WORKDIR /opt/vault -COPY vault/vault.manifest.toml vault/config.hcl vault/vault-csr.conf vault/cakey.pem vault/cacert.pem vault/start.sh ./ -RUN mkdir -p /opt/vault/data /opt/vault/.cache /opt/vault/tls && rm -rf /opt/vault/tls/* - -COPY vault/enclave-key.pem /tmp/ -RUN set -eux; \ - find / -xdev -print0 | xargs -0 touch -r /usr/bin/vault || : ; \ - gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu -Dexecdir=/usr/bin -Dlog_level=warning vault.manifest.toml vault.manifest; \ - gramine-sgx-sign --manifest vault.manifest --output vault.manifest.sgx --key /tmp/enclave-key.pem; \ - rm /tmp/enclave-key.pem - -VOLUME /opt/vault/tls -VOLUME /opt/vault/data - -ENTRYPOINT ["/bin/sh", "-c"] -CMD [ "/restart_aesm.sh ; exec gramine-sgx vault" ] diff --git a/vault/config.hcl b/vault/config.hcl deleted file mode 100644 index 21971536..00000000 --- a/vault/config.hcl +++ /dev/null @@ -1,55 +0,0 @@ -# Parameter needed because of slow plugin loading -# may be relaxed for faster machines -#http_read_header_timeout = 0 -#http_read_timeout = 300 - -disable_mlock = true -ui = false - -listener "tcp" { - address = "0.0.0.0:8210" - cluster_address = "0.0.0.0:8211" - tls_disable = false - tls_cert_file = "/opt/vault/tls/tls.crt" - tls_key_file = "/opt/vault/tls/tls.key" - tls_client_ca_file = "/opt/vault/cacert.pem" -} - -storage "raft" { - path = "/opt/vault/data/" - # override vial env var VAULT_RAFT_NODE_ID - node_id = "vault-1" - - # Parameter needed because of slow plugin loading - # may be relaxed for faster machines - performance_multiplier = 200 - #autopilot_reconcile_interval = "120s" - #autopilot_update_interval = "60s" - - retry_join { - leader_api_addr = "https://vault-1:8210" - leader_ca_cert_file = "/opt/vault/cacert.pem" - leader_client_cert_file = "/opt/vault/tls/tls.crt" - leader_client_key_file = "/opt/vault/tls/tls.key" - } - retry_join { - leader_api_addr = "https://vault-2:8210" - leader_ca_cert_file = "/opt/vault/cacert.pem" - leader_client_cert_file = "/opt/vault/tls/tls.crt" - leader_client_key_file = "/opt/vault/tls/tls.key" - } - retry_join { - leader_api_addr = "https://vault-3:8210" - leader_ca_cert_file = "/opt/vault/cacert.pem" - leader_client_cert_file = "/opt/vault/tls/tls.crt" - leader_client_key_file = "/opt/vault/tls/tls.key" - } -} - -# path of plugin binaries -plugin_directory = "/opt/vault/plugins" - -# override via env var VAULT_API_ADDR -api_addr = "https://vault:8210" -# override via env var VAULT_CLUSTER_ADDR -cluster_addr = "https://vault:8211" diff --git a/vault/start.sh b/vault/start.sh deleted file mode 100644 index 3c7d1b74..00000000 --- a/vault/start.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -e -if [ ! -f /opt/vault/tls/tls.ok ]; then - # Generate the TLS certificates - cd /opt/vault/tls - cp ../cacert.pem ../cakey.pem ../vault-csr.conf . - openssl req -new -newkey rsa:4096 -keyout tls.key -out vault.csr \ - -config vault-csr.conf -extensions v3_req - openssl x509 -req -in vault.csr -days 365 -CA cacert.pem -CAkey cakey.pem -CAcreateserial \ - -out tls_single.crt -extensions v3_req -extfile vault-csr.conf - cat tls_single.crt cacert.pem >> tls.crt - echo ok > tls.ok -fi -cd /opt/vault - -# Start the vault server -exec vault server -config=/opt/vault/config.hcl -log-level=trace diff --git a/vault/vault-csr.conf b/vault/vault-csr.conf deleted file mode 100644 index 5882012a..00000000 --- a/vault/vault-csr.conf +++ /dev/null @@ -1,21 +0,0 @@ -[req] -default_bits = 4096 -prompt = no -encrypt_key = no -default_md = sha256 -distinguished_name = kubelet_serving -req_extensions = v3_req -x509_extensions = v3_req -[ kubelet_serving ] -O = system:nodes -CN = system:node -[ v3_req ] -basicConstraints = CA:FALSE -keyUsage = nonRepudiation, digitalSignature, keyEncipherment, dataEncipherment -extendedKeyUsage = serverAuth, clientAuth -subjectAltName = @alt_names -[alt_names] -IP.1 = 127.0.0.1 -DNS.1 = vault-1 -DNS.2 = vault-2 -DNS.3 = vault-3