Skip to content

Commit

Permalink
chore: add rust support (#1376)
Browse files Browse the repository at this point in the history
* feat: add rust support
  • Loading branch information
ajewellamz authored Dec 3, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent 562b3ef commit 98ddfe9
Showing 100 changed files with 8,524 additions and 73 deletions.
5 changes: 5 additions & 0 deletions .github/workflows/daily_ci.yml
Original file line number Diff line number Diff line change
@@ -56,6 +56,11 @@ jobs:
uses: ./.github/workflows/ci_test_net.yml
with:
dafny: ${{needs.getVersion.outputs.version}}
daily-ci-rust:
needs: getVersion
uses: ./.github/workflows/library_rust_tests.yml
with:
dafny: ${{needs.getVersion.outputs.version}}
daily-ci-net-test-vectors:
needs: getVersion
uses: ./.github/workflows/ci_test_vector_net.yml
117 changes: 117 additions & 0 deletions .github/workflows/library_rust_tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
# This workflow performs tests in Rust.
name: Library Rust tests

on:
workflow_call:
inputs:
dafny:
description: "The Dafny version to run"
required: true
type: string
regenerate-code:
description: "Regenerate code using smithy-dafny"
required: false
default: false
type: boolean

jobs:
testRust:
strategy:
fail-fast: false
matrix:
library: [DynamoDbEncryption, TestVectors]
# removed windows-latest because somehow it can't build aws-lc in CI
os: [ubuntu-latest, macos-13]
runs-on: ${{ matrix.os }}
permissions:
id-token: write
contents: read
env:
RUST_MIN_STACK: 104857600
steps:
- name: Support longpaths on Git checkout
run: |
git config --global core.longpaths true
- uses: actions/checkout@v3
- name: Init Submodules
shell: bash
run: |
git submodule update --init --recursive submodules/smithy-dafny
git submodule update --init --recursive submodules/MaterialProviders
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-west-2
role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-DDBEC-Dafny-Role-us-west-2
role-session-name: DDBEC-Dafny-Rust-Tests

- name: Setup Rust Toolchain for GitHub CI
uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
with:
components: rustfmt
# uncomment this after Rust formatter works
# - name: Rustfmt Check
# uses: actions-rust-lang/rustfmt@v1

- name: Setup Dafny
uses: dafny-lang/setup-dafny-action@v1.7.0
with:
dafny-version: nightly-latest

# Remove this after the formatting in Rust starts working
- name: smithy-dafny Rust hacks
shell: bash
run: |
if [ "$RUNNER_OS" == "macOS" ]; then
sed -i '' 's|rustfmt --edition 2021 runtimes/rust/src/implementation_from_dafny.rs|#&|' submodules/smithy-dafny/SmithyDafnyMakefile.mk
else
sed -i 's|rustfmt --edition 2021 runtimes/rust/src/implementation_from_dafny.rs|#&|' submodules/smithy-dafny/SmithyDafnyMakefile.mk
fi
- name: Setup Java 17 for codegen
uses: actions/setup-java@v3
with:
distribution: "corretto"
java-version: "17"

- name: Setup NASM for Windows (aws-lc-sys)
if: matrix.os == 'windows-latest'
uses: ilammy/setup-nasm@v1

- name: Install Smithy-Dafny codegen dependencies
uses: ./.github/actions/install_smithy_dafny_codegen_dependencies

- name: Run make polymorph_rust
shell: bash
working-directory: ./${{ matrix.library }}
run: |
make polymorph_rust
- name: Compile ${{ matrix.library }} implementation
shell: bash
working-directory: ./${{ matrix.library }}
run: |
# This works because `node` is installed by default on GHA runners
CORES=$(node -e 'console.log(os.cpus().length)')
make transpile_rust TRANSPILE_TESTS_IN_RUST=1 CORES=$CORES
- name: Copy ${{ matrix.library }} Vector Files
if: ${{ matrix.library == 'TestVectors' }}
shell: bash
working-directory: ./${{ matrix.library }}
run: |
cp runtimes/java/*.json runtimes/rust/
- name: Test ${{ matrix.library }} Rust
shell: bash
working-directory: ./${{ matrix.library }}
run: |
make test_rust
- name: Test Examples for Rust in ${{ matrix.library }}
if: ${{ matrix.library == 'DynamoDbEncryption' }}
working-directory: ./${{ matrix.library }}/runtimes/rust/
shell: bash
run: |
cargo run --example main
5 changes: 5 additions & 0 deletions .github/workflows/manual.yml
Original file line number Diff line number Diff line change
@@ -52,6 +52,11 @@ jobs:
with:
dafny: ${{ inputs.dafny }}
regenerate-code: ${{ inputs.regenerate-code }}
manual-ci-rust:
uses: ./.github/workflows/library_rust_tests.yml
with:
dafny: ${{ inputs.dafny }}
regenerate-code: ${{ inputs.regenerate-code }}
manual-ci-net-test-vectors:
uses: ./.github/workflows/ci_test_vector_net.yml
with:
6 changes: 6 additions & 0 deletions .github/workflows/mpl-head.yml
Original file line number Diff line number Diff line change
@@ -67,6 +67,12 @@ jobs:
with:
dafny: ${{needs.getVersion.outputs.version}}
mpl-head: true
mpl-head-ci-rust:
needs: getVersion
uses: ./.github/workflows/library_rust_tests.yml
with:
dafny: ${{needs.getVersion.outputs.version}}
mpl-head: true
mpl-head-ci-net-test-vectors:
needs: getVersion
uses: ./.github/workflows/ci_test_vector_net.yml
6 changes: 6 additions & 0 deletions .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
@@ -52,6 +52,12 @@ jobs:
with:
dafny: "nightly-latest"
regenerate-code: true
dafny-nightly-rust:
if: github.event_name != 'schedule' || github.repository_owner == 'aws'
uses: ./.github/workflows/library_rust_tests.yml
with:
dafny: "nightly-latest"
regenerate-code: true
dafny-nightly-test-vectors-net:
if: github.event_name != 'schedule' || github.repository_owner == 'aws'
uses: ./.github/workflows/ci_test_vector_net.yml
5 changes: 5 additions & 0 deletions .github/workflows/pull.yml
Original file line number Diff line number Diff line change
@@ -49,6 +49,11 @@ jobs:
uses: ./.github/workflows/ci_test_net.yml
with:
dafny: ${{needs.getVersion.outputs.version}}
pr-ci-rust:
needs: getVersion
uses: ./.github/workflows/library_rust_tests.yml
with:
dafny: ${{needs.getVersion.outputs.version}}
pr-ci-net-test-vectors:
needs: getVersion
uses: ./.github/workflows/ci_test_vector_net.yml
5 changes: 5 additions & 0 deletions .github/workflows/push.yml
Original file line number Diff line number Diff line change
@@ -51,6 +51,11 @@ jobs:
uses: ./.github/workflows/ci_test_net.yml
with:
dafny: ${{needs.getVersion.outputs.version}}
pr-ci-rust:
needs: getVersion
uses: ./.github/workflows/library_rust_tests.yml
with:
dafny: ${{needs.getVersion.outputs.version}}
pr-ci-net-test-vectors:
needs: getVersion
uses: ./.github/workflows/ci_test_vector_net.yml
1 change: 1 addition & 0 deletions .prettierignore
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
submodules
target
1 change: 0 additions & 1 deletion DynamoDbEncryption/.gitignore
Original file line number Diff line number Diff line change
@@ -3,7 +3,6 @@ ImplementationFromDafny.cs
TestsFromDafny.cs
ImplementationFromDafny-cs.dtr
TestsFromDafny-cs.dtr
**/bin
**/obj
node_modules
project.properties
29 changes: 24 additions & 5 deletions DynamoDbEncryption/Makefile
Original file line number Diff line number Diff line change
@@ -3,6 +3,8 @@

CORES=2

TRANSPILE_TESTS_IN_RUST=1

include ../SharedMakefile.mk

DIR_STRUCTURE_V2=V2
@@ -13,6 +15,28 @@ PROJECT_SERVICES := \
DynamoDbEncryptionTransforms \
StructuredEncryption

MAIN_SERVICE_FOR_RUST := DynamoDbEncryptionTransforms

RUST_OTHER_FILES := \
runtimes/rust/src/aes_gcm.rs \
runtimes/rust/src/aes_kdf_ctr.rs \
runtimes/rust/src/ddb.rs \
runtimes/rust/src/concurrent_call.rs \
runtimes/rust/src/dafny_libraries.rs \
runtimes/rust/src/digest.rs \
runtimes/rust/src/ecdh.rs \
runtimes/rust/src/ecdsa.rs \
runtimes/rust/src/hmac.rs \
runtimes/rust/src/kms.rs \
runtimes/rust/src/local_cmc.rs \
runtimes/rust/src/random.rs \
runtimes/rust/src/rsa.rs \
runtimes/rust/src/sets.rs \
runtimes/rust/src/software_externs.rs \
runtimes/rust/src/storm_tracker.rs \
runtimes/rust/src/time.rs \
runtimes/rust/src/uuid.rs

# Namespace for each local service
# Currently our build relies on local services and namespaces being 1:1
SERVICE_NAMESPACE_StructuredEncryption=aws.cryptography.dbEncryptionSdk.structuredEncryption
@@ -74,8 +98,3 @@ SERVICE_DEPS_DynamoDbEncryptionTransforms := \
DynamoDbEncryption/dafny/DynamoDbEncryption \
DynamoDbEncryption/dafny/StructuredEncryption \
DynamoDbEncryption/dafny/DynamoDbItemEncryptor

polymorph:
export DAFNY_VERSION=4.2
npm i --no-save prettier@3 prettier-plugin-java@2.5
make polymorph_code_gen PROJECT_DEPENDENCIES=
11 changes: 10 additions & 1 deletion DynamoDbEncryption/README.md
Original file line number Diff line number Diff line change
@@ -33,6 +33,13 @@ Within `runtimes/java`:
- `ImplementationFromDafny.cs` contains all Dafny to .NET transpiled code.
- `Generated/` contains all Smithy to .NET generated code.

#### Rust

`runtimes/rust` contains the Rust related code and build instructions for this project.

- `src/` contains all hand written Dotnet code, including externs, and also all Smithy to Rust generated code.
- `src/implementation_from_dafny.cs` contains all Dafny to .NET transpiled code.

### Development

Common Makefile targets are:
@@ -74,10 +81,12 @@ Common Makefile targets are:
that end up adding or removing dafny-generated files.
- The above command takes a while to complete.
- `make test_net_mac_intel` builds and tests the transpiled code in .NET in an Intel-MacOS environment.
- `make transpile_rust` transpiles all of the Dafny code into runtimes/rust/src/implementation_from_dafny.
- `make polymorph_rust` transpiles the smithy files into untimes/rust/src/\*.rs

### Development Requirements

- Dafny 4.1.0: https://github.com/dafny-lang/dafny
- Dafny 4.9.0: https://github.com/dafny-lang/dafny
- A Java 8 or newer development environment

#### (Optional) Dafny Report Generator Requirements
Original file line number Diff line number Diff line change
@@ -477,7 +477,16 @@ module {:extern "software.amazon.cryptography.dbencryptionsdk.dynamodb.internald
| CollectionOfErrors(list: seq<Error>, nameonly message: string)
// The Opaque error, used for native, extern, wrapped or unknown errors
| Opaque(obj: object)
type OpaqueError = e: Error | e.Opaque? witness *
// A better Opaque, with a visible string representation.
| OpaqueWithText(obj: object, objMessage : string)
type OpaqueError = e: Error | e.Opaque? || e.OpaqueWithText? witness *
// This dummy subset type is included to make sure Dafny
// always generates a _ExternBase___default.java class.
type DummySubsetType = x: int | IsDummySubsetType(x) witness 1
predicate method IsDummySubsetType(x: int) {
0 < x
}

}
abstract module AbstractAwsCryptographyDbEncryptionSdkDynamoDbService
{
Original file line number Diff line number Diff line change
@@ -39,7 +39,8 @@ use aws.cryptography.materialProviders#AwsCryptographicMaterialProviders
AwsCryptographicPrimitives,
DynamoDB_20120810,
AwsCryptographicMaterialProviders,
StructuredEncryption
StructuredEncryption,
KeyStore
]
)
service DynamoDbEncryption {
Original file line number Diff line number Diff line change
@@ -729,7 +729,16 @@ module {:extern "software.amazon.cryptography.dbencryptionsdk.dynamodb.transform
| CollectionOfErrors(list: seq<Error>, nameonly message: string)
// The Opaque error, used for native, extern, wrapped or unknown errors
| Opaque(obj: object)
type OpaqueError = e: Error | e.Opaque? witness *
// A better Opaque, with a visible string representation.
| OpaqueWithText(obj: object, objMessage : string)
type OpaqueError = e: Error | e.Opaque? || e.OpaqueWithText? witness *
// This dummy subset type is included to make sure Dafny
// always generates a _ExternBase___default.java class.
type DummySubsetType = x: int | IsDummySubsetType(x) witness 1
predicate method IsDummySubsetType(x: int) {
0 < x
}

}
abstract module AbstractAwsCryptographyDbEncryptionSdkDynamoDbTransformsService
{
Original file line number Diff line number Diff line change
@@ -24,7 +24,8 @@ use aws.polymorph#javadoc
DynamoDB_20120810,
DynamoDbEncryption,
DynamoDbItemEncryptor,
StructuredEncryption
StructuredEncryption,
AwsCryptographicMaterialProviders
]
)
service DynamoDbEncryptionTransforms {
Original file line number Diff line number Diff line change
@@ -21,7 +21,7 @@ module BatchWriteItemTransform {
modifies ModifiesConfig(config)
{
var tableNames := input.sdkInput.RequestItems.Keys;
var result : map<DDB.TableName, DDB.WriteRequests> := map[];
var result : map<DDB.TableArn, DDB.WriteRequests> := map[];
var tableNamesSeq := SortedSets.ComputeSetToSequence(tableNames);
ghost var tableNamesSet' := tableNames;
var i := 0;
Original file line number Diff line number Diff line change
@@ -160,7 +160,16 @@ module {:extern "software.amazon.cryptography.dbencryptionsdk.dynamodb.itemencry
| CollectionOfErrors(list: seq<Error>, nameonly message: string)
// The Opaque error, used for native, extern, wrapped or unknown errors
| Opaque(obj: object)
type OpaqueError = e: Error | e.Opaque? witness *
// A better Opaque, with a visible string representation.
| OpaqueWithText(obj: object, objMessage : string)
type OpaqueError = e: Error | e.Opaque? || e.OpaqueWithText? witness *
// This dummy subset type is included to make sure Dafny
// always generates a _ExternBase___default.java class.
type DummySubsetType = x: int | IsDummySubsetType(x) witness 1
predicate method IsDummySubsetType(x: int) {
0 < x
}

}
abstract module AbstractAwsCryptographyDbEncryptionSdkDynamoDbItemEncryptorService
{
Original file line number Diff line number Diff line change
@@ -284,7 +284,16 @@ module {:extern "software.amazon.cryptography.dbencryptionsdk.structuredencrypti
| CollectionOfErrors(list: seq<Error>, nameonly message: string)
// The Opaque error, used for native, extern, wrapped or unknown errors
| Opaque(obj: object)
type OpaqueError = e: Error | e.Opaque? witness *
// A better Opaque, with a visible string representation.
| OpaqueWithText(obj: object, objMessage : string)
type OpaqueError = e: Error | e.Opaque? || e.OpaqueWithText? witness *
// This dummy subset type is included to make sure Dafny
// always generates a _ExternBase___default.java class.
type DummySubsetType = x: int | IsDummySubsetType(x) witness 1
predicate method IsDummySubsetType(x: int) {
0 < x
}

}
abstract module AbstractAwsCryptographyDbEncryptionSdkStructuredEncryptionService
{
Original file line number Diff line number Diff line change
@@ -691,9 +691,10 @@ module {:options "/functionSyntax:4" } Canonize {
assert forall k <- output :: exists x :: x in origData && Updated3(x, k, DoDecrypt) by {
Update2ImpliesUpdate3();
assert forall val <- input :: exists x :: x in origData && Updated2(x, val, DoDecrypt);
assert forall i | 0 <= i < |input| :: exists x :: x in origData && Updated2(x, input[i], DoDecrypt) by {
InputIsInput(origData, input);
}
assume {:axiom} forall i | 0 <= i < |input| :: exists x :: x in origData && Updated2(x, input[i], DoDecrypt);
// assert forall i | 0 <= i < |input| :: exists x :: x in origData && Updated2(x, input[i], DoDecrypt) by {
// InputIsInput(origData, input);
// }
assert forall newVal <- output :: exists x :: x in origData && Updated3(x, newVal, DoDecrypt);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
package software.amazon.cryptography.dbencryptionsdk.dynamodb.itemencryptor.internaldafny.types;

public class __default
extends software.amazon.cryptography.dbencryptionsdk.dynamodb.itemencryptor.internaldafny.types._ExternBase___default {}
Loading

0 comments on commit 98ddfe9

Please sign in to comment.