diff --git a/DEPS.bzl b/DEPS.bzl index 8c71f45a9b30..832af92b61f8 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -368,10 +368,10 @@ def go_deps(): name = "com_github_aws_aws_sdk_go_v2", build_file_proto_mode = "disable_global", importpath = "github.com/aws/aws-sdk-go-v2", - sha256 = "a2dd07909b2bc4595984a5fb9c4d69afe69ef0ecc034926ec17819655120cd29", - strip_prefix = "github.com/aws/aws-sdk-go-v2@v1.16.2", + sha256 = "db9f1deea94002f8a40a665ab86b58d9c713fda9358997c75b4859668d66802a", + strip_prefix = "github.com/aws/aws-sdk-go-v2@v1.17.3", urls = [ - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/com_github_aws_aws_sdk_go_v2-v1.16.2.zip", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/com_github_aws_aws_sdk_go_v2-v1.17.3.zip", ], ) go_repository( @@ -408,20 +408,20 @@ def go_deps(): name = "com_github_aws_aws_sdk_go_v2_internal_configsources", build_file_proto_mode = "disable_global", importpath = "github.com/aws/aws-sdk-go-v2/internal/configsources", - sha256 = "948f7cdd0e4adc5fc3743a688ebc2a685053086a27c402868f013eceea6a866e", - strip_prefix = "github.com/aws/aws-sdk-go-v2/internal/configsources@v1.1.9", + sha256 = "e7f6ed262a89b7de55994c3534734660cae21f07e138ca1ed48796048708f896", + strip_prefix = "github.com/aws/aws-sdk-go-v2/internal/configsources@v1.1.27", urls = [ - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/configsources/com_github_aws_aws_sdk_go_v2_internal_configsources-v1.1.9.zip", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/configsources/com_github_aws_aws_sdk_go_v2_internal_configsources-v1.1.27.zip", ], ) go_repository( name = "com_github_aws_aws_sdk_go_v2_internal_endpoints_v2", build_file_proto_mode = "disable_global", importpath = "github.com/aws/aws-sdk-go-v2/internal/endpoints/v2", - sha256 = "6dec0e8afc8efe4ef43c3bec089318ae48afbea583461a2e583a3864f2d2d1dc", - strip_prefix = "github.com/aws/aws-sdk-go-v2/internal/endpoints/v2@v2.4.3", + sha256 = "4c1baeacdb316aa7d37e16045c343e53157a1a8b5209d9cbf0bcb9b0b94781bc", + strip_prefix = "github.com/aws/aws-sdk-go-v2/internal/endpoints/v2@v2.4.21", urls = [ - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/com_github_aws_aws_sdk_go_v2_internal_endpoints_v2-v2.4.3.zip", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/com_github_aws_aws_sdk_go_v2_internal_endpoints_v2-v2.4.21.zip", ], ) go_repository( @@ -484,6 +484,16 @@ def go_deps(): "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/rds/com_github_aws_aws_sdk_go_v2_service_rds-v1.18.4.zip", ], ) + go_repository( + name = "com_github_aws_aws_sdk_go_v2_service_secretsmanager", + build_file_proto_mode = "disable_global", + importpath = "github.com/aws/aws-sdk-go-v2/service/secretsmanager", + sha256 = "44dcf0add18d221042e6a709eed9beae974e5eebfe18dd37003944b7abefb271", + strip_prefix = "github.com/aws/aws-sdk-go-v2/service/secretsmanager@v1.18.2", + urls = [ + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/secretsmanager/com_github_aws_aws_sdk_go_v2_service_secretsmanager-v1.18.2.zip", + ], + ) go_repository( name = "com_github_aws_aws_sdk_go_v2_service_sso", build_file_proto_mode = "disable_global", @@ -508,10 +518,10 @@ def go_deps(): name = "com_github_aws_smithy_go", build_file_proto_mode = "disable_global", importpath = "github.com/aws/smithy-go", - sha256 = "62b6645d90fd3a5e55646ae1c361923491b95e7c008a6c1ff1e347232b0e55f6", - strip_prefix = "github.com/aws/smithy-go@v1.11.2", + sha256 = "864667edd8459bff1f750d65f40922263b995a06dcb85240536539e239e911e4", + strip_prefix = "github.com/aws/smithy-go@v1.13.5", urls = [ - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/smithy-go/com_github_aws_smithy_go-v1.11.2.zip", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/smithy-go/com_github_aws_smithy_go-v1.13.5.zip", ], ) go_repository( diff --git a/build/bazelutil/distdir_files.bzl b/build/bazelutil/distdir_files.bzl index 4c3dbf83af2d..df64a0ef9a3a 100644 --- a/build/bazelutil/distdir_files.bzl +++ b/build/bazelutil/distdir_files.bzl @@ -106,22 +106,23 @@ DISTDIR_FILES = { "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/asaskevich/govalidator/com_github_asaskevich_govalidator-v0.0.0-20200907205600-7a23bdc65eef.zip": "9b846e311015bbb7854cf001d5717d6fd91a580ac6315977ae7395256e18d729", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/atotto/clipboard/com_github_atotto_clipboard-v0.1.4.zip": "d67b2c36c662751309fd2ec351df3651584bea840bd27be9a90702c3a238b43f", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-lambda-go/com_github_aws_aws_lambda_go-v1.13.3.zip": "8cfc5400798abd2840f456c75265f8fba4ae488e32ca2af9a5c8073fb219ea82", - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/com_github_aws_aws_sdk_go_v2-v1.16.2.zip": "a2dd07909b2bc4595984a5fb9c4d69afe69ef0ecc034926ec17819655120cd29", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/com_github_aws_aws_sdk_go_v2-v1.17.3.zip": "db9f1deea94002f8a40a665ab86b58d9c713fda9358997c75b4859668d66802a", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/config/com_github_aws_aws_sdk_go_v2_config-v1.15.3.zip": "9a3c9618ee70f5981194af7436a9054b35102d4f1bfa0fce78d60667d648b6f6", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/credentials/com_github_aws_aws_sdk_go_v2_credentials-v1.11.2.zip": "88fc13b9995c5f862299280d9a0dbca54661149500c04ab8e14f543353974a25", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/feature/ec2/imds/com_github_aws_aws_sdk_go_v2_feature_ec2_imds-v1.12.3.zip": "5561425aaec7bca20634d8055bddd2894975d043bcc4f159f1862b14ea3dd8d5", - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/configsources/com_github_aws_aws_sdk_go_v2_internal_configsources-v1.1.9.zip": "948f7cdd0e4adc5fc3743a688ebc2a685053086a27c402868f013eceea6a866e", - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/com_github_aws_aws_sdk_go_v2_internal_endpoints_v2-v2.4.3.zip": "6dec0e8afc8efe4ef43c3bec089318ae48afbea583461a2e583a3864f2d2d1dc", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/configsources/com_github_aws_aws_sdk_go_v2_internal_configsources-v1.1.27.zip": "e7f6ed262a89b7de55994c3534734660cae21f07e138ca1ed48796048708f896", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/com_github_aws_aws_sdk_go_v2_internal_endpoints_v2-v2.4.21.zip": "4c1baeacdb316aa7d37e16045c343e53157a1a8b5209d9cbf0bcb9b0b94781bc", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/internal/ini/com_github_aws_aws_sdk_go_v2_internal_ini-v1.3.10.zip": "353a01da6df6ec898baeed901980429745decb28aff68ef8b85eefe5ae515ec9", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/databasemigrationservice/com_github_aws_aws_sdk_go_v2_service_databasemigrationservice-v1.18.3.zip": "c69b0b562c314708e5d1542bed72e0ba068f910d3cae4909a0d80219e6b497de", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/ec2/com_github_aws_aws_sdk_go_v2_service_ec2-v1.34.0.zip": "b24b82535334bd7716000ba1af24acc03fcbbcb8817b8e229e9368c1fbbe6c3e", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/iam/com_github_aws_aws_sdk_go_v2_service_iam-v1.18.3.zip": "efb7b199ce0ae1dbea275fa3f8d131e874cc27d92c55ba7a007ad89762a88ed8", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/com_github_aws_aws_sdk_go_v2_service_internal_presigned_url-v1.9.3.zip": "a4748c20c2dd79d2a7503439778d2a52e6fdd657be31324b27caf18288968d92", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/rds/com_github_aws_aws_sdk_go_v2_service_rds-v1.18.4.zip": "f5de5a435c3c31b14e853e9a4348ce80646db030c9a99a178e2a12fc00585f7e", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/secretsmanager/com_github_aws_aws_sdk_go_v2_service_secretsmanager-v1.18.2.zip": "44dcf0add18d221042e6a709eed9beae974e5eebfe18dd37003944b7abefb271", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/sso/com_github_aws_aws_sdk_go_v2_service_sso-v1.11.3.zip": "49f04cbd2199e505307269240d3fd3e587ca3e4ba2cbd7f8d62262475a4992c1", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go-v2/service/sts/com_github_aws_aws_sdk_go_v2_service_sts-v1.16.3.zip": "a17741c144807b854a48db54b08d5929a65ff8bc1728ac3cbbff12266d8bcb27", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/aws-sdk-go/com_github_aws_aws_sdk_go-v1.40.37.zip": "c0c481d28af88f621fb3fdeacc1e5d32f69a1bb83d0ee959f95ce89e4e2d0494", - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/smithy-go/com_github_aws_smithy_go-v1.11.2.zip": "62b6645d90fd3a5e55646ae1c361923491b95e7c008a6c1ff1e347232b0e55f6", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aws/smithy-go/com_github_aws_smithy_go-v1.13.5.zip": "864667edd8459bff1f750d65f40922263b995a06dcb85240536539e239e911e4", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/axiomhq/hyperloglog/com_github_axiomhq_hyperloglog-v0.0.0-20181223111420-4b99d0c2c99e.zip": "812834322ee2ca50dc36f91f9ac3f2cde4631af2f9c330b1271c78b46024a540", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aymanbagabas/go-osc52/com_github_aymanbagabas_go_osc52-v1.0.3.zip": "138e75a51599c2a8e4afe2bd6acdeaddbb73eb9ec796dfa2f577b16201660d9e", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/aymerick/raymond/com_github_aymerick_raymond-v2.0.3-0.20180322193309-b565731e1464+incompatible.zip": "0a759716a73b587a436b3b4a95416a58bb1ffa1decf2cd7a92f1eeb2f9c654c1", diff --git a/go.mod b/go.mod index d6d1d9d41c0f..47faea41ff7f 100644 --- a/go.mod +++ b/go.mod @@ -46,12 +46,12 @@ require ( github.com/Azure/go-autorest/autorest/azure/cli v0.4.3 // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/aws/aws-sdk-go v1.40.37 - github.com/aws/aws-sdk-go-v2 v1.16.2 + github.com/aws/aws-sdk-go-v2 v1.17.3 github.com/aws/aws-sdk-go-v2/config v1.15.3 github.com/aws/aws-sdk-go-v2/credentials v1.11.2 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 // indirect github.com/aws/aws-sdk-go-v2/service/databasemigrationservice v1.18.3 github.com/aws/aws-sdk-go-v2/service/ec2 v1.34.0 @@ -93,6 +93,7 @@ require ( github.com/andygrunwald/go-jira v1.14.0 github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.2 github.com/axiomhq/hyperloglog v0.0.0-20181223111420-4b99d0c2c99e github.com/bazelbuild/rules_go v0.26.0 github.com/biogo/store v0.0.0-20160505134755-913427a1d5e8 @@ -246,7 +247,7 @@ require ( github.com/apache/thrift v0.16.0 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/atotto/clipboard v0.1.4 // indirect - github.com/aws/smithy-go v1.11.2 // indirect + github.com/aws/smithy-go v1.13.5 // indirect github.com/aymanbagabas/go-osc52 v1.0.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect diff --git a/go.sum b/go.sum index b1c89143526f..4c9e901d1ca4 100644 --- a/go.sum +++ b/go.sum @@ -311,18 +311,21 @@ github.com/aws/aws-sdk-go v1.40.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm github.com/aws/aws-sdk-go v1.40.37 h1:I+Q6cLctkFyMMrKukcDnj+i2kjrQ37LGiOM6xmsxC48= github.com/aws/aws-sdk-go v1.40.37/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/aws/aws-sdk-go-v2 v1.16.2 h1:fqlCk6Iy3bnCumtrLz9r3mJ/2gUT0pJ0wLFVIdWh+JA= github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= +github.com/aws/aws-sdk-go-v2 v1.17.3 h1:shN7NlnVzvDUgPQ+1rLMSxY8OWRNDRYtiqe0p/PgrhY= +github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/config v1.15.3 h1:5AlQD0jhVXlGzwo+VORKiUuogkG7pQcLJNzIzK7eodw= github.com/aws/aws-sdk-go-v2/config v1.15.3/go.mod h1:9YL3v07Xc/ohTsxFXzan9ZpFpdTOFl4X65BAKYaz8jg= github.com/aws/aws-sdk-go-v2/credentials v1.11.2 h1:RQQ5fzclAKJyY5TvF+fkjJEwzK4hnxQCLOu5JXzDmQo= github.com/aws/aws-sdk-go-v2/credentials v1.11.2/go.mod h1:j8YsY9TXTm31k4eFhspiQicfXPLZ0gYXA50i4gxPE8g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 h1:LWPg5zjHV9oz/myQr4wMs0gi4CjnDN/ILmyZUFYXZsU= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3/go.mod h1:uk1vhHHERfSVCUnqSqz8O48LBYDSC+k6brng09jcMOk= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 h1:onz/VaaxZ7Z4V+WIN9Txly9XLTmoOh1oJ8XcAC3pako= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9/go.mod h1:AnVH5pvai0pAF4lXRq0bmhbes1u9R8wTE+g+183bZNM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 h1:9stUQR/u2KXU6HkFJYlqnZEjBnbgrVbG6I5HN09xZh0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3/go.mod h1:ssOhaLpRlh88H3UmEcsBoVKq309quMvm3Ds8e9d4eJM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 h1:5NbbMrIzmUn/TXFqAle6mgrH5m9cOvMLRGL7pnG8tRE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 h1:by9P+oy3P/CwggN4ClnW2D4oL91QV7pBzBICi1chZvQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10/go.mod h1:8DcYQcz0+ZJaSxANlHIsbbi6S+zMwjwdDqwW3r9AzaE= github.com/aws/aws-sdk-go-v2/service/databasemigrationservice v1.18.3 h1:1OIa4zpY43n6Q5IX9b5mZWeq/KTV5243tXq5ABWlufI= @@ -335,12 +338,15 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 h1:Gh1Gpyh01Y github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3/go.mod h1:wlY6SVjuwvh3TVRpTqdy4I1JpBFLX4UGeKZdWntaocw= github.com/aws/aws-sdk-go-v2/service/rds v1.18.4 h1:o5uszX76RaDFxNvf6b8qZ3GF55EdFoPH8GnrROjtL4k= github.com/aws/aws-sdk-go-v2/service/rds v1.18.4/go.mod h1:u33weNg1XPt3iTVX2wVFIf7oAD7XmgkF640mnM8wQ5Q= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.2 h1:QDVKb2VpuwzIslzshumxksayV5GkpqT+rkVvdPVrA9E= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.2/go.mod h1:jAeo/PdIJZuDSwsvxJS94G4d6h8tStj7WXVuKwLHWU8= github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 h1:frW4ikGcxfAEDfmQqWgMLp+F1n4nRo9sF39OcIb5BkQ= github.com/aws/aws-sdk-go-v2/service/sso v1.11.3/go.mod h1:7UQ/e69kU7LDPtY40OyoHYgRmgfGM4mgsLYtcObdveU= github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 h1:cJGRyzCSVwZC7zZZ1xbx9m32UnrKydRYhOvcD1NYP9Q= github.com/aws/aws-sdk-go-v2/service/sts v1.16.3/go.mod h1:bfBj0iVmsUyUg4weDB4NxktD9rDGeKSVWnjTnwbx9b8= -github.com/aws/smithy-go v1.11.2 h1:eG/N+CcUMAvsdffgMvjMKwfyDzIkjM6pfxMJ8Mzc6mE= github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/axiomhq/hyperloglog v0.0.0-20181223111420-4b99d0c2c99e h1:190ugM9MsyFauTkR/UqcHG/mn5nmFe6SvHJqEHIrtrA= github.com/axiomhq/hyperloglog v0.0.0-20181223111420-4b99d0c2c99e/go.mod h1:IOXAcuKIFq/mDyuQ4wyJuJ79XLMsmLM+5RdQ+vWrL7o= github.com/aymanbagabas/go-osc52 v1.0.3 h1:DTwqENW7X9arYimJrPeGZcV0ln14sGMt3pHZspWD+Mg= diff --git a/pkg/cmd/roachtest/tests/BUILD.bazel b/pkg/cmd/roachtest/tests/BUILD.bazel index 4969a3ac2135..a2862f77134d 100644 --- a/pkg/cmd/roachtest/tests/BUILD.bazel +++ b/pkg/cmd/roachtest/tests/BUILD.bazel @@ -239,6 +239,7 @@ go_library( "@com_github_aws_aws_sdk_go_v2_service_databasemigrationservice//types", "@com_github_aws_aws_sdk_go_v2_service_rds//:rds", "@com_github_aws_aws_sdk_go_v2_service_rds//types", + "@com_github_aws_aws_sdk_go_v2_service_secretsmanager//:secretsmanager", "@com_github_cockroachdb_cockroach_go_v2//crdb", "@com_github_cockroachdb_errors//:errors", "@com_github_cockroachdb_ttycolor//:ttycolor", diff --git a/pkg/cmd/roachtest/tests/awsdms.go b/pkg/cmd/roachtest/tests/awsdms.go index 09eb08010e49..87984c081fdc 100644 --- a/pkg/cmd/roachtest/tests/awsdms.go +++ b/pkg/cmd/roachtest/tests/awsdms.go @@ -13,6 +13,7 @@ package tests import ( "context" gosql "database/sql" + "encoding/json" "fmt" "math/rand" "os" @@ -24,6 +25,7 @@ import ( dmstypes "github.com/aws/aws-sdk-go-v2/service/databasemigrationservice/types" "github.com/aws/aws-sdk-go-v2/service/rds" rdstypes "github.com/aws/aws-sdk-go-v2/service/rds/types" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" @@ -47,10 +49,36 @@ const ( awsdmsNumInitialRows = 100000 ) +const ( + // This RDS instance for this test is always up in our AWS account. It + // contains 200M rows so having to generate and import that data each run + // would take too long. The secret name stored in aws can only be accessed + // by CRL employees who have proper AWS credentials. The password can't be + // auto rotated due to constraints with postgres DMS source endpoint restrictions + // where the password can't contain %, ;, or +. + awsrdsSecretName = "rds!db-074de488-6274-4b3e-ad27-d008b8ffa750" + awsrdsDBIdentifier = "migrations-dms" + awsrdsNumInitialRows = 200000000 +) + type dmsTask struct { tableName string tableMappings string replicationSettings *string + migrationType dmstypes.MigrationTypeValue + sourceEndpoint *dmsEndpointState + targetEndpoint *dmsEndpointState +} + +type dmsTaskEndpoints struct { + defaultSource *dmsEndpointState + defaultTarget *dmsEndpointState + largeSource *dmsEndpointState +} + +type dmsEndpointState struct { + id string + arn string } const tableRules = `{ @@ -68,15 +96,11 @@ const tableRules = `{ ] }` -var dmsTasks = []dmsTask{ - {"test_table", tableRules, nil}, - {"test_table_no_pk", tableRules, proto.String(`{ - "FullLoadSettings":{ - "TargetTablePrepMode":"TRUNCATE_BEFORE_LOAD" - } - }`), - }, -} +const fullLoadTruncate = `{ + "FullLoadSettings":{ + "TargetTablePrepMode":"TRUNCATE_BEFORE_LOAD" + } + }` func awsdmsVerString(v *version.Version) string { if ciBranch := os.Getenv("TC_BUILD_BRANCH"); ciBranch != "" { @@ -106,7 +130,12 @@ func awsdmsRoachtestDMSReplicationInstanceName(v *version.Version) string { return "roachtest-awsdms-replication-instance-" + awsdmsVerString(v) } -func awsdmsRoachtestDMSRDSEndpointName(v *version.Version) string { +// The largeTask flag is used to create a new source endpoint as it is connecting to a different RDS +// and needs to have a different name from the other tests. +func awsdmsRoachtestDMSRDSEndpointName(v *version.Version, largeTask bool) string { + if largeTask { + return "roachtest-awsdms-large-rds-endpoint-" + awsdmsVerString(v) + } return "roachtest-awsdms-rds-endpoint-" + awsdmsVerString(v) } func awsdmsRoachtestDMSCRDBEndpointName(v *version.Version) string { @@ -183,10 +212,52 @@ func runAWSDMS(ctx context.Context, t test.Test, c cluster.Cluster) { } rdsCli := rds.NewFromConfig(awsCfg) dmsCli := dms.NewFromConfig(awsCfg) + smCli := secretsmanager.NewFromConfig(awsCfg) + + // Create the endpoints that we will use but the ARN's are nil + // and will be filled in when the endpoints are recreated. + // Deletion of old endpoints can be done using id. + dmsEndpoints := dmsTaskEndpoints{ + defaultSource: &dmsEndpointState{ + id: awsdmsRoachtestDMSRDSEndpointName(t.BuildVersion(), false), + }, + defaultTarget: &dmsEndpointState{ + id: awsdmsRoachtestDMSCRDBEndpointName(t.BuildVersion()), + }, + largeSource: &dmsEndpointState{ + id: awsdmsRoachtestDMSRDSEndpointName(t.BuildVersion(), true), + }, + } + dmsTasks := []dmsTask{ + { + tableName: "test_table", + tableMappings: tableRules, + replicationSettings: nil, + migrationType: dmstypes.MigrationTypeValueFullLoadAndCdc, + sourceEndpoint: dmsEndpoints.defaultSource, + targetEndpoint: dmsEndpoints.defaultTarget, + }, + { + tableName: "test_table_no_pk", + tableMappings: tableRules, + replicationSettings: proto.String(fullLoadTruncate), + migrationType: dmstypes.MigrationTypeValueFullLoadAndCdc, + sourceEndpoint: dmsEndpoints.defaultSource, + targetEndpoint: dmsEndpoints.defaultTarget, + }, + { + tableName: "test_table_large", + tableMappings: tableRules, + replicationSettings: proto.String(fullLoadTruncate), + migrationType: dmstypes.MigrationTypeValueFullLoad, + sourceEndpoint: dmsEndpoints.largeSource, + targetEndpoint: dmsEndpoints.defaultTarget, + }, + } // Attempt a clean-up of old instances on startup. t.L().Printf("attempting to delete old instances") - if err := tearDownAWSDMS(ctx, t, rdsCli, dmsCli); err != nil { + if err := tearDownAWSDMS(ctx, t, rdsCli, dmsCli, &dmsTasks); err != nil { t.Fatal(err) } @@ -198,12 +269,12 @@ func runAWSDMS(ctx context.Context, t test.Test, c cluster.Cluster) { } t.L().Printf("attempting to cleanup instances") // Try to delete from a new context, in case the previous one is cancelled. - if err := tearDownAWSDMS(context.Background(), t, rdsCli, dmsCli); err != nil { + if err := tearDownAWSDMS(context.Background(), t, rdsCli, dmsCli, &dmsTasks); err != nil { t.L().Printf("failed to delete old instances on cleanup: %+v", err) } }() - sourcePGConn, err := setupAWSDMS(ctx, t, c, rdsCli, dmsCli) + sourcePGConn, err := setupAWSDMS(ctx, t, c, rdsCli, dmsCli, smCli, &dmsTasks, &dmsEndpoints) if err != nil { t.Fatal(err) } @@ -211,6 +282,7 @@ func runAWSDMS(ctx context.Context, t test.Test, c cluster.Cluster) { checkDMSReplicated(ctx, t, sourcePGConn, targetPGConn) checkDMSNoPKTableError(ctx, t, dmsCli) + checkFullLargeDataLoad(ctx, t, dmsCli) t.L().Printf("testing complete") } @@ -347,10 +419,86 @@ func checkDMSNoPKTableError(ctx context.Context, t test.Test, dmsCli *dms.Client } } +func checkFullLargeDataLoad(ctx context.Context, t test.Test, dmsCli *dms.Client) { + closer := make(chan struct{}) + waitForFullLoad := retry.Options{ + InitialBackoff: 10 * time.Second, + MaxBackoff: 5 * time.Minute, + Closer: closer, + } + t.L().Printf("testing all rows from test_table_large replicate") + var currentPercent, nonUpdate = 0, 0 + for r := retry.StartWithCtx(ctx, waitForFullLoad); r.Next(); { + err := func() error { + dmsTasks, err := dmsCli.DescribeReplicationTasks(ctx, dmsDescribeTasksInput(t.BuildVersion(), "test_table_large")) + if err != nil { + return err + } + for _, task := range dmsTasks.ReplicationTasks { + // If the task is stopped and stop reason is full load finished, we have succeeded. + if *task.Status == "stopped" && *task.StopReason == "Stop Reason FULL_LOAD_ONLY_FINISHED" { + tableStats, err := dmsCli.DescribeTableStatistics(context.Background(), &dms.DescribeTableStatisticsInput{ + ReplicationTaskArn: task.ReplicationTaskArn, + Filters: []dmstypes.Filter{{ + Name: proto.String("table-name"), + Values: []string{"test_table_large"}, + }}, + }) + if err != nil { + return err + } + // Check we full loaded the right number of rows + if tableStats.TableStatistics[0].FullLoadRows == awsrdsNumInitialRows { + t.L().Printf("test_table_large successfully replicated all rows") + } else { + t.L().Printf("row count mismatch: %d vs %d", tableStats.TableStatistics[0].FullLoadRows, awsrdsNumInitialRows) + t.Fatal("not enough rows replicated from test_table_large") + } + close(closer) + } else if *task.Status == "running" { + stats := task.ReplicationTaskStats + if stats != nil { + if stats.FullLoadProgressPercent != 100 { + if stats.FullLoadProgressPercent > int32(currentPercent) { + nonUpdate = 0 + currentPercent = int(stats.FullLoadProgressPercent) + t.L().Printf("test_table_large still replicating, percentage: %d", stats.FullLoadProgressPercent) + } else { + nonUpdate++ + // Arbitrarily picked 5 consecutive non updates to indicate stuck progress + if nonUpdate == 5 { + t.Fatal(errors.New("replication progress appears to be stuck")) + close(closer) + } + } + } + } + } else { + // All other statuses should result in a failure + t.L().Printf("unexpeted task status %s", *task.Status) + t.Fatal("unexpected task status") + close(closer) + } + } + return nil + }() + if err != nil { + t.L().Printf("error while checking for full load, retrying: %+v", err) + } + } +} + // setupAWSDMS sets up an RDS instance and a DMS instance which sets up a // migration task from the RDS instance to the CockroachDB cluster. func setupAWSDMS( - ctx context.Context, t test.Test, c cluster.Cluster, rdsCli *rds.Client, dmsCli *dms.Client, + ctx context.Context, + t test.Test, + c cluster.Cluster, + rdsCli *rds.Client, + dmsCli *dms.Client, + smCli *secretsmanager.Client, + dmsTasks *[]dmsTask, + endpoints *dmsTaskEndpoints, ) (*pgx.Conn, error) { var sourcePGConn *pgx.Conn if err := func() error { @@ -374,8 +522,30 @@ func setupAWSDMS( if err := g.Wait(); err != nil { return err } - - if err := setupDMSEndpointsAndTask(ctx, t, c, dmsCli, rdsCluster, awsdmsPassword, replicationARN); err != nil { + smInput := &secretsmanager.GetSecretValueInput{ + SecretId: proto.String(awsrdsSecretName), + VersionStage: proto.String("AWSCURRENT"), + } + smo, smErr := smCli.GetSecretValue(ctx, smInput) + if smErr != nil { + return smErr + } + secrets := map[string]string{} + err := json.Unmarshal([]byte(*smo.SecretString), &secrets) + if err != nil { + return err + } + rdsInput := &rds.DescribeDBInstancesInput{ + DBInstanceIdentifier: proto.String(awsrdsDBIdentifier), + } + rdso, rdsErr := rdsCli.DescribeDBInstances(ctx, rdsInput) + if rdsErr != nil { + return rdsErr + } + if len(rdso.DBInstances) != 1 { + return errors.New("RDS instance for large scale replication not found") + } + if err := setupDMSEndpointsAndTask(ctx, t, c, dmsCli, rdsCluster, awsdmsPassword, replicationARN, secrets["password"], &rdso.DBInstances[0], dmsTasks, endpoints); err != nil { return err } return nil @@ -395,6 +565,8 @@ func setupCockroachDBCluster(ctx context.Context, t test.Test, c cluster.Cluster for _, stmt := range []string{ fmt.Sprintf("CREATE USER %s", awsdmsCRDBUser), fmt.Sprintf("GRANT admin TO %s", awsdmsCRDBUser), + fmt.Sprintf("ALTER USER %s SET copy_from_atomic_enabled = false", awsdmsCRDBUser), + fmt.Sprintf("ALTER USER %s SET copy_from_retries_enabled = true", awsdmsCRDBUser), } { if _, err := db.Exec(stmt); err != nil { return err @@ -554,21 +726,23 @@ func setupDMSEndpointsAndTask( rdsCluster *rdstypes.DBCluster, awsdmsPassword string, replicationARN string, + rdsPasswordLarge string, + rdsClusterLarge *rdstypes.DBInstance, + dmsTasks *[]dmsTask, + dmsEndpoints *dmsTaskEndpoints, ) error { // Setup AWS DMS to replicate to CockroachDB. externalCRDBAddr, err := c.ExternalIP(ctx, t.L(), option.NodeListOption{1}) if err != nil { return err } - - var sourceARN, targetARN string for _, ep := range []struct { - in dms.CreateEndpointInput - arn *string + in dms.CreateEndpointInput + endpoint *dmsEndpointState }{ { in: dms.CreateEndpointInput{ - EndpointIdentifier: proto.String(awsdmsRoachtestDMSRDSEndpointName(t.BuildVersion())), + EndpointIdentifier: proto.String(dmsEndpoints.defaultSource.id), EndpointType: dmstypes.ReplicationEndpointTypeValueSource, EngineName: proto.String("aurora-postgresql"), DatabaseName: proto.String(awsdmsDatabase), @@ -577,11 +751,11 @@ func setupDMSEndpointsAndTask( Port: rdsCluster.Port, ServerName: rdsCluster.Endpoint, }, - arn: &sourceARN, + endpoint: dmsEndpoints.defaultSource, }, { in: dms.CreateEndpointInput{ - EndpointIdentifier: proto.String(awsdmsRoachtestDMSCRDBEndpointName(t.BuildVersion())), + EndpointIdentifier: proto.String(dmsEndpoints.defaultTarget.id), EndpointType: dmstypes.ReplicationEndpointTypeValueTarget, EngineName: proto.String("postgres"), SslMode: dmstypes.DmsSslModeValueNone, @@ -595,7 +769,20 @@ func setupDMSEndpointsAndTask( ServerName: proto.String(externalCRDBAddr[0]), }, }, - arn: &targetARN, + endpoint: dmsEndpoints.defaultTarget, + }, + { + in: dms.CreateEndpointInput{ + EndpointIdentifier: proto.String(dmsEndpoints.largeSource.id), + EndpointType: dmstypes.ReplicationEndpointTypeValueSource, + EngineName: proto.String("postgres"), + DatabaseName: proto.String(awsdmsDatabase), + Username: rdsClusterLarge.MasterUsername, + Password: proto.String(rdsPasswordLarge), + Port: &rdsClusterLarge.Endpoint.Port, + ServerName: rdsClusterLarge.Endpoint.Address, + }, + endpoint: dmsEndpoints.largeSource, }, } { t.L().Printf("creating replication endpoint %s", *ep.in.EndpointIdentifier) @@ -603,7 +790,7 @@ func setupDMSEndpointsAndTask( if err != nil { return err } - *ep.arn = *epOut.Endpoint.EndpointArn + ep.endpoint.arn = *epOut.Endpoint.EndpointArn // Test the connections to see if they are "successful". // If not, any subsequence DMS task will fail to startup. @@ -660,16 +847,16 @@ func setupDMSEndpointsAndTask( } } - for _, task := range dmsTasks { + for _, task := range *dmsTasks { t.L().Printf(fmt.Sprintf("creating replication task for %s", task.tableName)) replTaskOut, err := dmsCli.CreateReplicationTask( ctx, &dms.CreateReplicationTaskInput{ - MigrationType: dmstypes.MigrationTypeValueFullLoadAndCdc, + MigrationType: task.migrationType, ReplicationInstanceArn: proto.String(replicationARN), ReplicationTaskIdentifier: proto.String(awsdmsRoachtestDMSTaskName(t.BuildVersion(), task.tableName)), - SourceEndpointArn: proto.String(sourceARN), - TargetEndpointArn: proto.String(targetARN), + SourceEndpointArn: proto.String(task.sourceEndpoint.arn), + TargetEndpointArn: proto.String(task.targetEndpoint.arn), // TODO(#migrations): when AWS API supports EnableValidation, add it here. TableMappings: proto.String(fmt.Sprintf(task.tableMappings, task.tableName)), ReplicationTaskSettings: task.replicationSettings, @@ -708,14 +895,18 @@ func isDMSResourceNotFound(err error) bool { return errors.HasType(err, &dmstypes.ResourceNotFoundFault{}) } +func isDMSResourceAlreadyDeleting(err error) bool { + return errors.HasType(err, &dmstypes.InvalidResourceStateFault{}) +} + func tearDownAWSDMS( - ctx context.Context, t test.Test, rdsCli *rds.Client, dmsCli *dms.Client, + ctx context.Context, t test.Test, rdsCli *rds.Client, dmsCli *dms.Client, dmsTasks *[]dmsTask, ) error { if err := func() error { - if err := tearDownDMSTasks(ctx, t, dmsCli); err != nil { + if err := tearDownDMSTasks(ctx, t, dmsCli, dmsTasks); err != nil { return err } - if err := tearDownDMSEndpoints(ctx, t, dmsCli); err != nil { + if err := tearDownDMSEndpoints(ctx, t, dmsCli, dmsTasks); err != nil { return err } @@ -732,8 +923,10 @@ func tearDownAWSDMS( // tearDownDMSTasks tears down the DMS task, endpoints and replication instance // that may have been created. -func tearDownDMSTasks(ctx context.Context, t test.Test, dmsCli *dms.Client) error { - for _, task := range dmsTasks { +func tearDownDMSTasks( + ctx context.Context, t test.Test, dmsCli *dms.Client, dmsTasks *[]dmsTask, +) error { + for _, task := range *dmsTasks { dmsTasks, err := dmsCli.DescribeReplicationTasks(ctx, dmsDescribeTasksInput(t.BuildVersion(), task.tableName)) if err != nil { if !isDMSResourceNotFound(err) { @@ -771,27 +964,38 @@ func tearDownDMSTasks(ctx context.Context, t test.Test, dmsCli *dms.Client) erro return nil } -func tearDownDMSEndpoints(ctx context.Context, t test.Test, dmsCli *dms.Client) error { - for _, ep := range []string{awsdmsRoachtestDMSRDSEndpointName(t.BuildVersion()), awsdmsRoachtestDMSCRDBEndpointName(t.BuildVersion())} { - dmsEndpoints, err := dmsCli.DescribeEndpoints(ctx, &dms.DescribeEndpointsInput{ - Filters: []dmstypes.Filter{ - { - Name: proto.String("endpoint-id"), - Values: []string{ep}, +func tearDownDMSEndpoints( + ctx context.Context, t test.Test, dmsCli *dms.Client, dmsTasks *[]dmsTask, +) error { + for _, tk := range *dmsTasks { + for _, ep := range []string{tk.sourceEndpoint.id, tk.targetEndpoint.id} { + dmsEndpoints, err := dmsCli.DescribeEndpoints(ctx, &dms.DescribeEndpointsInput{ + Filters: []dmstypes.Filter{ + { + Name: proto.String("endpoint-id"), + Values: []string{ep}, + }, }, - }, - }) - if err != nil { - if !isDMSResourceNotFound(err) { - return err - } - } else { - for _, dmsEndpoint := range dmsEndpoints.Endpoints { - t.L().Printf("deleting DMS endpoint %s (arn: %s)", *dmsEndpoint.EndpointIdentifier, *dmsEndpoint.EndpointArn) - if _, err := dmsCli.DeleteEndpoint(ctx, &dms.DeleteEndpointInput{EndpointArn: dmsEndpoint.EndpointArn}); err != nil { + }) + + if err != nil { + if !isDMSResourceNotFound(err) { return err } + } else { + for _, dmsEndpoint := range dmsEndpoints.Endpoints { + t.L().Printf("deleting DMS endpoint %s (arn: %s)", *dmsEndpoint.EndpointIdentifier, *dmsEndpoint.EndpointArn) + if _, err := dmsCli.DeleteEndpoint(ctx, &dms.DeleteEndpointInput{EndpointArn: dmsEndpoint.EndpointArn}); err != nil { + // Because we reuse the same source and target endpoints in some tasks, + // we need to check for an InvalidResourceState error meaning that its + // already being deleted and should not return the error. + if !isDMSResourceAlreadyDeleting(err) { + return err + } + } + } } + } } return nil