diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4907ddf8b9..8bd27c4e0d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -293,6 +293,21 @@ functions: fi fi + # Set temp credentials for AWS if python3 is available. + # + # Using python3-venv in Ubuntu 14.04 (an OS required for legacy server version + # tasks) requires the use of apt-get, which we wish to avoid. So, we do not set + # a python3 binary on Ubuntu 14.04. Setting AWS temp credentials for legacy + # server version tasks is unneccesary, as temp credentials are only needed on 4.2+. + if [ ! -z ${PYTHON3_BINARY} ]; then + export AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}" + export AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}" + export AWS_DEFAULT_REGION="us-east-1" + ${PYTHON3_BINARY} -m venv ./venv + ./venv/${VENV_BIN_DIR|bin}/pip3 install boto3 + . ${DRIVERS_TOOLS}/.evergreen/csfle/set-temp-creds.sh + fi + export GOFLAGS=-mod=vendor set +o xtrace AUTH=${AUTH} \ @@ -303,6 +318,10 @@ functions: BUILD_TAGS="-tags cse" \ AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}" \ AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}" \ + AWS_DEFAULT_REGION="us-east-1" \ + CSFLE_AWS_TEMP_ACCESS_KEY_ID="$CSFLE_AWS_TEMP_ACCESS_KEY_ID" \ + CSFLE_AWS_TEMP_SECRET_ACCESS_KEY="$CSFLE_AWS_TEMP_SECRET_ACCESS_KEY" \ + CSFLE_AWS_TEMP_SESSION_TOKEN="$CSFLE_AWS_TEMP_SESSION_TOKEN" \ AZURE_TENANT_ID="${cse_azure_tenant_id}" \ AZURE_CLIENT_ID="${cse_azure_client_id}" \ AZURE_CLIENT_SECRET="${cse_azure_client_secret}" \ @@ -450,6 +469,15 @@ functions: working_dir: src/go.mongodb.org/mongo-driver script: | ${PREPARE_SHELL} + + # Set temp credentials for AWS. + export AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}" + export AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}" + export AWS_DEFAULT_REGION="us-east-1" + ${PYTHON3_BINARY} -m venv ./venv + ./venv/${VENV_BIN_DIR|bin}/pip3 install boto3 + . ${DRIVERS_TOOLS}/.evergreen/csfle/set-temp-creds.sh + export GOFLAGS=-mod=vendor set +o xtrace AUTH="${AUTH}" \ @@ -460,6 +488,10 @@ functions: BUILD_TAGS="-tags cse" \ AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}" \ AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}" \ + AWS_DEFAULT_REGION="us-east-1" \ + CSFLE_AWS_TEMP_ACCESS_KEY_ID="$CSFLE_AWS_TEMP_ACCESS_KEY_ID" \ + CSFLE_AWS_TEMP_SECRET_ACCESS_KEY="$CSFLE_AWS_TEMP_SECRET_ACCESS_KEY" \ + CSFLE_AWS_TEMP_SESSION_TOKEN="$CSFLE_AWS_TEMP_SESSION_TOKEN" \ AZURE_TENANT_ID="${cse_azure_tenant_id}" \ AZURE_CLIENT_ID="${cse_azure_client_id}" \ AZURE_CLIENT_SECRET="${cse_azure_client_secret}" \ diff --git a/data/client-side-encryption/README.rst b/data/client-side-encryption/README.rst index c22585325c..4ffadba819 100644 --- a/data/client-side-encryption/README.rst +++ b/data/client-side-encryption/README.rst @@ -134,7 +134,7 @@ Then for each element in ``tests``: #. Create a MongoClient. #. Create a collection object from the MongoClient, using the ``database_name`` - and ``collection_name`` fields from the YAML file. Drop the collection + and ``collection_name`` fields from the YAML file. Drop the collection with writeConcern "majority". If a ``json_schema`` is defined in the test, use the ``createCollection`` command to explicitly create the collection: @@ -147,8 +147,60 @@ Then for each element in ``tests``: #. Create a **new** MongoClient using ``clientOptions``. - #. If ``autoEncryptOpts`` includes ``aws``, ``azure``, and/or ``gcp`` as a KMS provider, pass in credentials from the environment. - #. If ``autoEncryptOpts`` does not include ``keyVaultNamespace``, default it to ``keyvault.datakeys``. + #. If ``autoEncryptOpts`` includes ``aws``, ``awsTemporary``, ``awsTemporaryNoSessionToken``, + ``azure``, and/or ``gcp`` as a KMS provider, pass in credentials from the environment. + + - ``awsTemporary``, and ``awsTemporaryNoSessionToken`` require temporary + AWS credentials. These can be retrieved using the csfle `set-temp-creds.sh + `_ + script. + + - ``aws``, ``awsTemporary``, and ``awsTemporaryNoSessionToken`` are + mutually exclusive. + + ``aws`` should be substituted with: + + .. code:: javascript + "aws": { + "accessKeyId": , + "secretAccessKey": + } + ``awsTemporary`` should be substituted with: + + .. code:: javascript + "aws": { + "accessKeyId": , + "secretAccessKey": + "sessionToken": + } + ``awsTemporaryNoSessionToken`` should be substituted with: + + .. code:: javascript + "aws": { + "accessKeyId": , + "secretAccessKey": + } + ``gcp`` should be substituted with: + + .. code:: javascript + "gcp": { + "email": , + "privateKey": , + } + ``azure`` should be substituted with: + + .. code:: javascript + "azure": { + "tenantId": , + "clientId": , + "clientSecret": , + } + ``local`` should be substituted with: + + .. code:: javascript + "local": { "key": } + #. If ``autoEncryptOpts`` does not include ``keyVaultNamespace``, default it + to ``keyvault.datakeys``. #. For each element in ``operations``: @@ -194,7 +246,7 @@ Then for each element in ``tests``: #. For each element in ``outcome``: - If ``name`` is "collection", create a new MongoClient *without encryption* - and verify that the test collection contains exactly the documents in the + and verify that the test collection contains exactly the documents in the ``data`` array. Ensure this find reads the latest data by using **primary read preference** with **local read concern** even when the MongoClient is configured with another read preference or read concern. @@ -767,6 +819,6 @@ The following tests that setting ``bypassAutoEncryption=true`` really does bypas Drivers MAY pass a different value to ``--port`` if they expect their testing infrastructure to be using port 27021. Pass a port that should be free. -#. Use ``client_encrypted`` to insert the document ``{"unencrypted": "test"}`` into ``db.coll``. Expect this to succeed. +#. Use ``client_encrypted`` to insert the document ``{"unencrypted": "test"}`` into ``db.coll``. Expect this to succeed. #. Validate that mongocryptd was not spawned. Create a MongoClient to localhost:27021 (or whatever was passed via ``--port``) with serverSelectionTimeoutMS=1000. Run an ``isMaster`` command and ensure it fails with a server selection timeout. \ No newline at end of file diff --git a/data/client-side-encryption/awsTemporary.json b/data/client-side-encryption/awsTemporary.json new file mode 100644 index 0000000000..477b9fb17f --- /dev/null +++ b/data/client-side-encryption/awsTemporary.json @@ -0,0 +1,225 @@ +{ + "runOn": [ + { + "minServerVersion": "4.1.10" + } + ], + "database_name": "default", + "collection_name": "default", + "data": [], + "json_schema": { + "properties": { + "encrypted_w_altname": { + "encrypt": { + "keyId": "/altname", + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + }, + "random": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string_equivalent": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + } + }, + "bsonType": "object" + }, + "key_vault_data": [ + { + "status": 1, + "_id": { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "masterKey": { + "provider": "aws", + "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "region": "us-east-1" + }, + "updateDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyMaterial": { + "$binary": { + "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyAltNames": [ + "altname", + "another_altname" + ] + } + ], + "tests": [ + { + "description": "Insert a document with auto encryption using the AWS provider with temporary credentials", + "clientOptions": { + "autoEncryptOpts": { + "kmsProviders": { + "awsTemporary": {} + } + } + }, + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encrypted_string": "string0" + } + } + } + ], + "expectations": [ + { + "command_started_event": { + "command": { + "listCollections": 1, + "filter": { + "name": "default" + } + }, + "command_name": "listCollections" + } + }, + { + "command_started_event": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault" + }, + "command_name": "find" + } + }, + { + "command_started_event": { + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ], + "ordered": true + }, + "command_name": "insert" + } + } + ], + "outcome": { + "collection": { + "data": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ] + } + } + }, + { + "description": "Insert with invalid temporary credentials", + "clientOptions": { + "autoEncryptOpts": { + "kmsProviders": { + "awsTemporaryNoSessionToken": {} + } + } + }, + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encrypted_string": "string0" + } + }, + "result": { + "errorContains": "security token" + } + } + ] + } + ] + } diff --git a/data/client-side-encryption/awsTemporary.yml b/data/client-side-encryption/awsTemporary.yml new file mode 100644 index 0000000000..109812a36c --- /dev/null +++ b/data/client-side-encryption/awsTemporary.yml @@ -0,0 +1,57 @@ +runOn: + - minServerVersion: "4.1.10" +database_name: &database_name "default" +collection_name: &collection_name "default" + +data: [] +json_schema: {'properties': {'encrypted_w_altname': {'encrypt': {'keyId': '/altname', 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'random': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string_equivalent': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'} +key_vault_data: [{'status': 1, '_id': {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'masterKey': {'provider': 'aws', 'key': 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0', 'region': 'us-east-1'}, 'updateDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyMaterial': {'$binary': {'base64': 'AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyAltNames': ['altname', 'another_altname']}] + +tests: + - description: "Insert a document with auto encryption using the AWS provider with temporary credentials" + clientOptions: + autoEncryptOpts: + kmsProviders: + awsTemporary: {} + operations: + - name: insertOne + arguments: + document: &doc0 { _id: 1, encrypted_string: "string0" } + expectations: + # Auto encryption will request the collection info. + - command_started_event: + command: + listCollections: 1 + filter: + name: *collection_name + command_name: listCollections + # Then key is fetched from the key vault. + - command_started_event: + command: + find: datakeys + filter: { $or: [ { _id: { $in: [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] } }, { keyAltNames: { $in: [] } } ] } + $db: keyvault + command_name: find + - command_started_event: + command: + insert: *collection_name + documents: + - &doc0_encrypted { _id: 1, encrypted_string: {'$binary': {'base64': 'AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==', 'subType': '06'}} } + ordered: true + command_name: insert + outcome: + collection: + # Outcome is checked using a separate MongoClient without auto encryption. + data: + - *doc0_encrypted + - description: "Insert with invalid temporary credentials" + clientOptions: + autoEncryptOpts: + kmsProviders: + awsTemporaryNoSessionToken: {} + operations: + - name: insertOne + arguments: + document: *doc0 + result: + errorContains: "security token" diff --git a/mongo/integration/client_side_encryption_spec_test.go b/mongo/integration/client_side_encryption_spec_test.go index 2ec7a2faea..4dc889367f 100644 --- a/mongo/integration/client_side_encryption_spec_test.go +++ b/mongo/integration/client_side_encryption_spec_test.go @@ -20,6 +20,10 @@ const ( func verifyClientSideEncryptionVarsSet(t *testing.T) { t.Helper() + // Existence of temporary AWS credentials (awsTempAccessKeyID, awsTempSessionToken and + // awsTempSessionToken) is verified when the variables are used in json_helpers_test + // because temporary AWS credentials are not always set. + if awsAccessKeyID == "" { t.Fatal("AWS access key ID not set") } diff --git a/mongo/integration/json_helpers_test.go b/mongo/integration/json_helpers_test.go index f4c161ad41..b257a64deb 100644 --- a/mongo/integration/json_helpers_test.go +++ b/mongo/integration/json_helpers_test.go @@ -26,13 +26,16 @@ import ( ) var ( - awsAccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID") - awsSecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY") - azureTenantID = os.Getenv("AZURE_TENANT_ID") - azureClientID = os.Getenv("AZURE_CLIENT_ID") - azureClientSecret = os.Getenv("AZURE_CLIENT_SECRET") - gcpEmail = os.Getenv("GCP_EMAIL") - gcpPrivateKey = os.Getenv("GCP_PRIVATE_KEY") + awsAccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID") + awsSecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY") + awsTempAccessKeyID = os.Getenv("CSFLE_AWS_TEMP_ACCESS_KEY_ID") + awsTempSecretAccessKey = os.Getenv("CSFLE_AWS_TEMP_SECRET_ACCESS_KEY") + awsTempSessionToken = os.Getenv("CSFLE_AWS_TEMP_SESSION_TOKEN") + azureTenantID = os.Getenv("AZURE_TENANT_ID") + azureClientID = os.Getenv("AZURE_CLIENT_ID") + azureClientSecret = os.Getenv("AZURE_CLIENT_SECRET") + gcpEmail = os.Getenv("GCP_EMAIL") + gcpPrivateKey = os.Getenv("GCP_PRIVATE_KEY") ) // Helper functions to do read JSON spec test files and convert JSON objects into the appropriate driver types. @@ -184,6 +187,34 @@ func createKmsProvidersMap(t testing.TB, opts bson.Raw) map[string]map[string]in "key": key, } kmsMap["local"] = localMap + case "awsTemporary": + if awsTempAccessKeyID == "" { + t.Fatal("AWS temp access key ID not set") + } + if awsTempSecretAccessKey == "" { + t.Fatal("AWS temp secret access key not set") + } + if awsTempSessionToken == "" { + t.Fatal("AWS temp session token not set") + } + awsMap := map[string]interface{}{ + "accessKeyId": awsTempAccessKeyID, + "secretAccessKey": awsTempSecretAccessKey, + "sessionToken": awsTempSessionToken, + } + kmsMap["aws"] = awsMap + case "awsTemporaryNoSessionToken": + if awsTempAccessKeyID == "" { + t.Fatal("AWS temp access key ID not set") + } + if awsTempSecretAccessKey == "" { + t.Fatal("AWS temp secret access key not set") + } + awsMap := map[string]interface{}{ + "accessKeyId": awsTempAccessKeyID, + "secretAccessKey": awsTempSecretAccessKey, + } + kmsMap["aws"] = awsMap default: t.Fatalf("unrecognized KMS provider: %v", provider) }