From 1401c8f81123d73b979bd2162746ff3cbf5c0c86 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 15 Jul 2021 16:49:48 +0200 Subject: [PATCH] workaround for https://github.com/boto/boto3/issues/125 --- .gitlab-ci.yml | 14 +++++++++++++ src/toil/jobStores/aws/jobStore.py | 28 +++++++++++++++++++------ src/toil/jobStores/aws/utils.py | 2 +- src/toil/test/__init__.py | 12 ++++++++--- src/toil/test/jobStores/jobStoreTest.py | 9 ++++++-- 5 files changed, 53 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b1a71bdd97..b4cbbc704c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -309,3 +309,17 @@ cactus_integration: - pip install psutil numpy --upgrade - toil clean aws:us-west-2:${BUCKET_NAME} - time cactus --setEnv SINGULARITY_DOCKER_HUB_MIRROR --batchSystem kubernetes --binariesMode singularity --clean always aws:us-west-2:${BUCKET_NAME} examples/evolverMammals.txt examples/evolverMammals.hal --root mr --defaultDisk "8G" --logDebug --disableCaching false + +awsjobstore_us_east_1: + stage: integration + script: + - pwd + - virtualenv -p ${MAIN_PYTHON_PKG} venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor awscli' + - export TOIL_TEST_INTEGRATIVE=True + - export TOIL_AWS_KEYNAME=id_rsa + - export TOIL_AWS_ZONE=us-east-1a + # This reads GITLAB_SECRET_FILE_SSH_KEYS + - python setup_gitlab_ssh.py + - chmod 400 /root/.ssh/id_rsa + - make test tests=src/toil/test/jobStores/jobStoreTest.py::AWSJobStoreTest + - make test tests=src/toil/test/jobStores/jobStoreTest.py::EncryptedAWSJobStoreTest diff --git a/src/toil/jobStores/aws/jobStore.py b/src/toil/jobStores/aws/jobStore.py index d42dc7750c..e4e8da9197 100644 --- a/src/toil/jobStores/aws/jobStore.py +++ b/src/toil/jobStores/aws/jobStore.py @@ -744,11 +744,25 @@ def bucket_creation_pending(error): bucketExisted = False logger.debug("Bucket '%s' does not exist.", bucket_name) if create: - logger.debug("Creating bucket '%s'.", bucket_name) - location = region_to_bucket_location(self.region) - bucket = self.s3_resource.create_bucket( - Bucket=bucket_name, - CreateBucketConfiguration={'LocationConstraint': location}) + location = self.region + logger.debug( + "Creating bucket '%s' in region %s.", + bucket_name, + location, + ) + if ( + location == "us-east-1" + ): # see https://github.com/boto/boto3/issues/125 + bucket = self.s3_resource.create_bucket( + Bucket=bucket_name + ) + else: + bucket = self.s3_resource.create_bucket( + Bucket=bucket_name, + CreateBucketConfiguration={ + "LocationConstraint": location + }, + ) # Wait until the bucket exists before checking the region and adding tags bucket.wait_until_exists() @@ -757,7 +771,9 @@ def bucket_creation_pending(error): # produce an S3ResponseError with code # NoSuchBucket. We let that kick us back up to the # main retry loop. - assert self.getBucketRegion(bucket_name) == self.region + assert ( + self.getBucketRegion(bucket_name) == self.region + ), f"bucket_name: {bucket_name}, {self.getBucketRegion(bucket_name)} != {self.region}" owner_tag = os.environ.get('TOIL_OWNER_TAG') if owner_tag: diff --git a/src/toil/jobStores/aws/utils.py b/src/toil/jobStores/aws/utils.py index b90b959655..f58891762c 100644 --- a/src/toil/jobStores/aws/utils.py +++ b/src/toil/jobStores/aws/utils.py @@ -441,4 +441,4 @@ def region_to_bucket_location(region): def bucket_location_to_region(location): - return 'us-east-1' if location == '' else location + return "us-east-1" if location == "" or location is None else location diff --git a/src/toil/test/__init__.py b/src/toil/test/__init__.py index 911e636c3a..5f9e497d31 100644 --- a/src/toil/test/__init__.py +++ b/src/toil/test/__init__.py @@ -35,6 +35,7 @@ import pytz from toil import ApplianceImageNotFound, applianceSelf, toilPackageDirPath +from toil.lib.aws import zone_to_region from toil.lib.iterables import concat from toil.lib.memoize import memoize from toil.lib.threading import ExceptionalThread, cpu_count @@ -98,10 +99,15 @@ def tearDown(self): @classmethod def awsRegion(cls): """ - Use us-west-2 unless running on EC2, in which case use the region in which - the instance is located + If TOIL_AWS_ZONE is set, use that. + Else use "us-west-2" unless running on EC2, in which case + use the region in which the instance is located """ - return cls._region() if running_on_ec2() else 'us-west-2' + return zone_to_region( + os.environ.get( + "TOIL_AWS_ZONE", cls._region() if running_on_ec2() else "us-west-2" + ) + ) @classmethod def _availabilityZone(cls): diff --git a/src/toil/test/jobStores/jobStoreTest.py b/src/toil/test/jobStores/jobStoreTest.py index 788539fd0c..218f2cccc8 100644 --- a/src/toil/test/jobStores/jobStoreTest.py +++ b/src/toil/test/jobStores/jobStoreTest.py @@ -1408,14 +1408,19 @@ def _hashTestFile(self, url: str) -> str: def _createExternalStore(self): """A S3.Bucket instance is returned""" from toil.jobStores.aws.utils import retry_s3 - from toil.jobStores.aws.utils import region_to_bucket_location from toil.jobStores.aws.jobStore import establish_boto3_session resource = establish_boto3_session().resource('s3', region_name=self.awsRegion()) bucket = resource.Bucket('import-export-test-%s' % uuid.uuid4()) + location = self.awsRegion() for attempt in retry_s3(): with attempt: - bucket.create(CreateBucketConfiguration={'LocationConstraint': region_to_bucket_location(self.awsRegion())}) + if location == "us-east-1": + bucket.create() + else: + bucket.create( + CreateBucketConfiguration={"LocationConstraint": location} + ) bucket.wait_until_exists() return bucket