From 80c32257a66c13d5b72cacccd1b6dedd66014d66 Mon Sep 17 00:00:00 2001 From: Nick Darvey Date: Sat, 22 Jun 2024 18:13:01 +1000 Subject: [PATCH 1/6] Upgrade aws-config for upgraded ring for compatibility with aarch64-pc-windows --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index b739f6e94c..8a899f2d2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,7 @@ arrow-schema = "51.0" arrow-select = "51.0" async-recursion = "1.0" async-trait = "0.1" -aws-config = "0.56" +aws-config = "0.57" aws-credential-types = "0.56" aws-sdk-dynamodb = "0.34" half = { "version" = "2.4.1", default-features = false, features = [ From 36cb500b5ae959745805274b63b215bf279153e0 Mon Sep 17 00:00:00 2001 From: Nick Darvey Date: Sat, 22 Jun 2024 21:12:30 +1000 Subject: [PATCH 2/6] Upgrade aws-credential-types too --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 8a899f2d2a..ce9feaa626 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,7 +73,7 @@ arrow-select = "51.0" async-recursion = "1.0" async-trait = "0.1" aws-config = "0.57" -aws-credential-types = "0.56" +aws-credential-types = "0.57" aws-sdk-dynamodb = "0.34" half = { "version" = "2.4.1", default-features = false, features = [ "num-traits", From 0008b1e736b15018a7d7330c84e6c87daab8a6b6 Mon Sep 17 00:00:00 2001 From: Nick Darvey Date: Sun, 23 Jun 2024 18:39:32 +1000 Subject: [PATCH 3/6] Bump dynamodb too --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ce9feaa626..7b620b8042 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,7 +74,7 @@ async-recursion = "1.0" async-trait = "0.1" aws-config = "0.57" aws-credential-types = "0.57" -aws-sdk-dynamodb = "0.34" +aws-sdk-dynamodb = "0.35" half = { "version" = "2.4.1", default-features = false, features = [ "num-traits", "std", From a1914a5ad54e3584bac8ee23c76759d81f72f678 Mon Sep 17 00:00:00 2001 From: Nick Darvey Date: Tue, 25 Jun 2024 14:23:37 +1000 Subject: [PATCH 4/6] Update DynamoDB usage --- rust/lance-table/src/io/commit.rs | 5 ++--- rust/lance-table/src/io/commit/dynamodb.rs | 18 ++++-------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/rust/lance-table/src/io/commit.rs b/rust/lance-table/src/io/commit.rs index 924eb33ef5..562582e0c0 100644 --- a/rust/lance-table/src/io/commit.rs +++ b/rust/lance-table/src/io/commit.rs @@ -46,7 +46,6 @@ use lance_io::object_store::{ObjectStore, ObjectStoreExt, ObjectStoreParams}; #[cfg(feature = "dynamodb")] use { self::external_manifest::{ExternalManifestCommitHandler, ExternalManifestStore}, - aws_credential_types::cache::CredentialsCache, aws_credential_types::provider::error::CredentialsError, aws_credential_types::provider::ProvideCredentials, lance_io::object_store::{build_aws_credential, StorageOptions}, @@ -362,13 +361,13 @@ async fn build_dynamodb_external_store( app_name: &str, ) -> Result> { use super::commit::dynamodb::DynamoDBExternalManifestStore; - use aws_sdk_dynamodb::{config::Region, Client}; + use aws_sdk_dynamodb::{config::{IdentityCache, Region}, Client}; let mut dynamodb_config = aws_sdk_dynamodb::config::Builder::new() .region(Some(Region::new(region.to_string()))) .credentials_provider(OSObjectStoreToAwsCredAdaptor(creds)) // caching should be handled by passed AwsCredentialProvider - .credentials_cache(CredentialsCache::no_caching()); + .identity_cache(IdentityCache::no_cache()); if let Some(endpoint) = endpoint { dynamodb_config = dynamodb_config.endpoint_url(endpoint); diff --git a/rust/lance-table/src/io/commit/dynamodb.rs b/rust/lance-table/src/io/commit/dynamodb.rs index d16848497c..0c6e8d67c1 100644 --- a/rust/lance-table/src/io/commit/dynamodb.rs +++ b/rust/lance-table/src/io/commit/dynamodb.rs @@ -173,19 +173,7 @@ impl DynamoDBExternalManifestStore { location!(), ) })?; - let key_type = key.key_type.ok_or_else(|| { - Error::io( - format!("dynamodb table: {table_name} key types must be defined"), - location!(), - ) - })?; - let name = key.attribute_name.ok_or_else(|| { - Error::io( - format!("dynamodb table: {table_name} key must have an attribute name"), - location!(), - ) - })?; - match (key_type, name.as_str()) { + match (key.key_type, key.attribute_name.as_str()) { (KeyType::Hash, base_uri!()) => { has_hask_key = true; } @@ -195,7 +183,9 @@ impl DynamoDBExternalManifestStore { _ => { return Err(Error::io( format!( - "dynamodb table: {table_name} unknown key type encountered name:{name}", + "dynamodb table: {} unknown key type encountered name:{}", + table_name, + key.attribute_name ), location!(), )); From 560d785d4f12c00f06996e05376bae22ef47189e Mon Sep 17 00:00:00 2001 From: Nick Darvey Date: Tue, 25 Jun 2024 14:25:56 +1000 Subject: [PATCH 5/6] Formatting --- rust/lance-table/src/io/commit.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rust/lance-table/src/io/commit.rs b/rust/lance-table/src/io/commit.rs index 562582e0c0..c99ba364aa 100644 --- a/rust/lance-table/src/io/commit.rs +++ b/rust/lance-table/src/io/commit.rs @@ -361,7 +361,10 @@ async fn build_dynamodb_external_store( app_name: &str, ) -> Result> { use super::commit::dynamodb::DynamoDBExternalManifestStore; - use aws_sdk_dynamodb::{config::{IdentityCache, Region}, Client}; + use aws_sdk_dynamodb::{ + config::{IdentityCache, Region}, + Client, + }; let mut dynamodb_config = aws_sdk_dynamodb::config::Builder::new() .region(Some(Region::new(region.to_string()))) From 863f8810a0bdf203df884577dc3b379d8b3b9cd6 Mon Sep 17 00:00:00 2001 From: Nick Darvey Date: Tue, 25 Jun 2024 14:30:01 +1000 Subject: [PATCH 6/6] More formatting --- rust/lance-table/src/io/commit/dynamodb.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/lance-table/src/io/commit/dynamodb.rs b/rust/lance-table/src/io/commit/dynamodb.rs index 0c6e8d67c1..d453fe74e7 100644 --- a/rust/lance-table/src/io/commit/dynamodb.rs +++ b/rust/lance-table/src/io/commit/dynamodb.rs @@ -184,8 +184,7 @@ impl DynamoDBExternalManifestStore { return Err(Error::io( format!( "dynamodb table: {} unknown key type encountered name:{}", - table_name, - key.attribute_name + table_name, key.attribute_name ), location!(), ));