workaround for boto/boto3#125

.gitlab-ci.yml

@@ -309,3 +309,17 @@ cactus_integration:
     - pip install psutil numpy --upgrade
     - toil clean aws:us-west-2:${BUCKET_NAME}
     - time cactus --setEnv SINGULARITY_DOCKER_HUB_MIRROR --batchSystem kubernetes --binariesMode singularity --clean always aws:us-west-2:${BUCKET_NAME} examples/evolverMammals.txt examples/evolverMammals.hal --root mr --defaultDisk "8G" --logDebug --disableCaching false
+
+awsjobstore_us_east_1:
+  stage: integration
+  script:
+    - pwd
+    - virtualenv -p ${MAIN_PYTHON_PKG} venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor awscli'
+    - export TOIL_TEST_INTEGRATIVE=True
+    - export TOIL_AWS_KEYNAME=id_rsa
+    - export TOIL_AWS_ZONE=us-east-1a
+    # This reads GITLAB_SECRET_FILE_SSH_KEYS
+    - python setup_gitlab_ssh.py
+    - chmod 400 /root/.ssh/id_rsa
+    - make test tests=src/toil/test/jobStores/jobStoreTest.py::AWSJobStoreTest
+    - make test tests=src/toil/test/jobStores/jobStoreTest.py::EncryptedAWSJobStoreTest

src/toil/jobStores/aws/jobStore.py

@@ -744,11 +744,25 @@ def bucket_creation_pending(error):
                         bucketExisted = False
                         logger.debug("Bucket '%s' does not exist.", bucket_name)
                         if create:
-                            logger.debug("Creating bucket '%s'.", bucket_name)
-                            location = region_to_bucket_location(self.region)
-                            bucket = self.s3_resource.create_bucket(
-                                Bucket=bucket_name,
-                                CreateBucketConfiguration={'LocationConstraint': location})
+                            location = self.region
+                            logger.debug(
+                                "Creating bucket '%s' in region %s.",
+                                bucket_name,
+                                location,
+                            )
+                            if (
+                                location == "us-east-1"
+                            ):  # see https://github.com/boto/boto3/issues/125
+                                bucket = self.s3_resource.create_bucket(
+                                    Bucket=bucket_name
+                                )
+                            else:
+                                bucket = self.s3_resource.create_bucket(
+                                    Bucket=bucket_name,
+                                    CreateBucketConfiguration={
+                                        "LocationConstraint": location
+                                    },
+                                )
                             # Wait until the bucket exists before checking the region and adding tags
                             bucket.wait_until_exists()
 
@@ -757,7 +771,9 @@ def bucket_creation_pending(error):
                             # produce an S3ResponseError with code
                             # NoSuchBucket. We let that kick us back up to the
                             # main retry loop.
-                            assert self.getBucketRegion(bucket_name) == self.region
+                            assert (
+                                self.getBucketRegion(bucket_name) == self.region
+                            ), f"bucket_name: {bucket_name}, {self.getBucketRegion(bucket_name)} != {self.region}"
 
                             owner_tag = os.environ.get('TOIL_OWNER_TAG')
                             if owner_tag:

src/toil/jobStores/aws/utils.py

@@ -441,4 +441,4 @@ def region_to_bucket_location(region):
 
 
 def bucket_location_to_region(location):
-    return 'us-east-1' if location == '' else location
+    return "us-east-1" if location == "" or location is None else location

src/toil/test/__init__.py

@@ -35,6 +35,7 @@
 import pytz
 
 from toil import ApplianceImageNotFound, applianceSelf, toilPackageDirPath
+from toil.lib.aws import zone_to_region
 from toil.lib.iterables import concat
 from toil.lib.memoize import memoize
 from toil.lib.threading import ExceptionalThread, cpu_count
@@ -98,10 +99,15 @@ def tearDown(self):
     @classmethod
     def awsRegion(cls):
         """
-        Use us-west-2 unless running on EC2, in which case use the region in which
-        the instance is located
+        If TOIL_AWS_ZONE is set, use that.
+        Else use "us-west-2" unless running on EC2, in which case
+        use the region in which the instance is located
         """
-        return cls._region() if running_on_ec2() else 'us-west-2'
+        return zone_to_region(
+            os.environ.get(
+                "TOIL_AWS_ZONE", cls._region() if running_on_ec2() else "us-west-2"
+            )
+        )
 
     @classmethod
     def _availabilityZone(cls):

src/toil/test/jobStores/jobStoreTest.py

@@ -1408,14 +1408,19 @@ def _hashTestFile(self, url: str) -> str:
     def _createExternalStore(self):
         """A S3.Bucket instance is returned"""
         from toil.jobStores.aws.utils import retry_s3
-        from toil.jobStores.aws.utils import region_to_bucket_location
         from toil.jobStores.aws.jobStore import establish_boto3_session
         resource = establish_boto3_session().resource('s3', region_name=self.awsRegion())
         bucket = resource.Bucket('import-export-test-%s' % uuid.uuid4())
+        location = self.awsRegion()
 
         for attempt in retry_s3():
             with attempt:
-                bucket.create(CreateBucketConfiguration={'LocationConstraint': region_to_bucket_location(self.awsRegion())})
+                if location == "us-east-1":
+                    bucket.create()
+                else:
+                    bucket.create(
+                        CreateBucketConfiguration={"LocationConstraint": location}
+                    )
                 bucket.wait_until_exists()
                 return bucket