workaround for S3 in us-east-1

.gitlab-ci.yml

@@ -271,6 +271,7 @@ py37_main:
     - make test tests=src/toil/test/src
     - make test tests=src/toil/test/utils
     - make test tests=src/toil/test/lib/test_ec2.py
+    - make test tests=src/toil/test/lib/aws
     - make test tests=src/toil/test/lib/test_conversions.py
 
 py37_appliance_build:

contrib/admin/mypy-with-ignore.py

@@ -99,8 +99,7 @@ def main():
         'src/toil/lib/encryption/_dummy.py',
         'src/toil/lib/encryption/conftest.py',
         'src/toil/lib/encryption/__init__.py',
-        'src/toil/lib/aws/utils.py',
-        'src/toil/lib/aws/__init__.py'
+        'src/toil/lib/aws/__init__.py',
     ]]
 
     filtered_files_to_check = []

setup.py

@@ -57,10 +57,13 @@ def run_setup():
         addict,
         pytz,
         pyyaml,
-        enlighten]
+        enlighten,
+        "typing-extensions ; python_version < '3.8'"
+        ]
     aws_reqs = [
         boto,
         boto3,
+        "boto3-stubs[s3]>=1.17, <2",
         futures,
         pycryptodome]
     cwl_reqs = [

src/toil/jobStores/abstractJobStore.py

@@ -42,7 +42,7 @@
 logger = logging.getLogger(__name__)
 
 try:
-    from botocore.exceptions import ProxyConnectionError # type: ignore
+    from botocore.exceptions import ProxyConnectionError
 except ImportError:
     class ProxyConnectionError(BaseException): # type: ignore
         pass

src/toil/jobStores/aws/jobStore.py

@@ -57,6 +57,7 @@
 from toil.jobStores.utils import (ReadablePipe,
                                   ReadableTransformingPipe,
                                   WritablePipe)
+from toil.lib.aws.utils import create_s3_bucket
 from toil.lib.compatibility import compat_bytes
 from toil.lib.ec2 import establish_boto3_session
 from toil.lib.ec2nodes import EC2Regions
@@ -744,11 +745,9 @@ def bucket_creation_pending(error):
                         bucketExisted = False
                         logger.debug("Bucket '%s' does not exist.", bucket_name)
                         if create:
-                            logger.debug("Creating bucket '%s'.", bucket_name)
-                            location = region_to_bucket_location(self.region)
-                            bucket = self.s3_resource.create_bucket(
-                                Bucket=bucket_name,
-                                CreateBucketConfiguration={'LocationConstraint': location})
+                            bucket = create_s3_bucket(
+                                self.s3_resource, bucket_name, self.region
+                            )
                             # Wait until the bucket exists before checking the region and adding tags
                             bucket.wait_until_exists()
 
@@ -757,7 +756,9 @@ def bucket_creation_pending(error):
                             # produce an S3ResponseError with code
                             # NoSuchBucket. We let that kick us back up to the
                             # main retry loop.
-                            assert self.getBucketRegion(bucket_name) == self.region
+                            assert (
+                                self.getBucketRegion(bucket_name) == self.region
+                            ), f"bucket_name: {bucket_name}, {self.getBucketRegion(bucket_name)} != {self.region}"
 
                             owner_tag = os.environ.get('TOIL_OWNER_TAG')
                             if owner_tag:
@@ -1637,7 +1638,8 @@ def _delete_domain(self, domain):
                     if not no_such_sdb_domain(e):
                         raise
 
-    def _delete_bucket(self, bucket):
+    @staticmethod
+    def _delete_bucket(bucket):
         """
         :param bucket: S3.Bucket
         """

src/toil/jobStores/aws/utils.py

@@ -441,4 +441,4 @@ def region_to_bucket_location(region):
 
 
 def bucket_location_to_region(location):
-    return 'us-east-1' if location == '' else location
+    return "us-east-1" if location == "" or location is None else location

src/toil/lib/aws/utils.py

@@ -12,22 +12,34 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+import sys
+from typing import Optional, Union
 
-from typing import Optional
+from toil.lib import aws
 from toil.lib.misc import printq
 from toil.lib.retry import retry
-from toil.lib import aws
+
+if sys.version_info >= (3, 8):
+    from typing import Literal
+else:
+    from typing_extensions import Literal
 
 try:
     from boto.exception import BotoServerError
+    from mypy_boto3_s3 import S3ServiceResource
+    from mypy_boto3_s3.literals import BucketLocationConstraintType
+    from mypy_boto3_s3.service_resource import Bucket
 except ImportError:
-    BotoServerError = None  # AWS/boto extra is not installed
+    BotoServerError = None  # type: ignore
+    # AWS/boto extra is not installed
 
 logger = logging.getLogger(__name__)
 
 
 @retry(errors=[BotoServerError])
-def delete_iam_role(role_name: str, region: Optional[str] = None, quiet: bool = True):
+def delete_iam_role(
+    role_name: str, region: Optional[str] = None, quiet: bool = True
+) -> None:
     from boto.iam.connection import IAMConnection
     iam_client = aws.client('iam', region_name=region)
     iam_resource = aws.resource('iam', region_name=region)
@@ -47,8 +59,10 @@ def delete_iam_role(role_name: str, region: Optional[str] = None, quiet: bool =
 
 
 @retry(errors=[BotoServerError])
-def delete_iam_instance_profile(instance_profile_name: str, region: Optional[str] = None, quiet: bool = True):
-    iam_resource = aws.resource('iam', region_name=region)
+def delete_iam_instance_profile(
+    instance_profile_name: str, region: Optional[str] = None, quiet: bool = True
+) -> None:
+    iam_resource = aws.resource("iam", region_name=region)
     instance_profile = iam_resource.InstanceProfile(instance_profile_name)
     for role in instance_profile.roles:
         printq(f'Now dissociating role: {role.name} from instance profile {instance_profile_name}', quiet)
@@ -58,14 +72,16 @@ def delete_iam_instance_profile(instance_profile_name: str, region: Optional[str
 
 
 @retry(errors=[BotoServerError])
-def delete_sdb_domain(sdb_domain_name: str, region: Optional[str] = None, quiet: bool = True):
-    sdb_client = aws.client('sdb', region_name=region)
+def delete_sdb_domain(
+    sdb_domain_name: str, region: Optional[str] = None, quiet: bool = True
+) -> None:
+    sdb_client = aws.client("sdb", region_name=region)
     sdb_client.delete_domain(DomainName=sdb_domain_name)
     printq(f'SBD Domain: "{sdb_domain_name}" successfully deleted.', quiet)
 
 
 @retry(errors=[BotoServerError])
-def delete_s3_bucket(bucket: str, region: Optional[str], quiet: bool = True):
+def delete_s3_bucket(bucket: str, region: Optional[str], quiet: bool = True) -> None:
     printq(f'Deleting s3 bucket in region "{region}": {bucket}', quiet)
     s3_client = aws.client('s3', region_name=region)
     s3_resource = aws.resource('s3', region_name=region)
@@ -78,3 +94,27 @@ def delete_s3_bucket(bucket: str, region: Optional[str], quiet: bool = True):
             s3_client.delete_object(Bucket=bucket, Key=version['Key'], VersionId=version['VersionId'])
     s3_resource.Bucket(bucket).delete()
     printq(f'\n * Deleted s3 bucket successfully: {bucket}\n\n', quiet)
+
+
+def create_s3_bucket(
+    s3_session: "S3ServiceResource",
+    bucket_name: str,
+    region: Union["BucketLocationConstraintType", Literal["us-east-1"]],
+) -> "Bucket":
+    """
+    Create an AWS S3 bucket, using the given Boto3 S3 session, with the
+    given name, in the given region.
+
+    Supports the us-east-1 region, where bucket creation is special.
+
+    *ALL* S3 bucket creation should use this function.
+    """
+    logger.debug("Creating bucket '%s' in region %s.", bucket_name, region)
+    if region == "us-east-1":  # see https://github.com/boto/boto3/issues/125
+        bucket = s3_session.create_bucket(Bucket=bucket_name)
+    else:
+        bucket = s3_session.create_bucket(
+            Bucket=bucket_name,
+            CreateBucketConfiguration={"LocationConstraint": region},
+        )
+    return bucket

src/toil/test/jobStores/jobStoreTest.py

@@ -41,6 +41,7 @@
                                              NoSuchJobException)
 from toil.jobStores.fileJobStore import FileJobStore
 from toil.lib.memoize import memoize
+from toil.lib.aws.utils import create_s3_bucket
 from toil.statsAndLogging import StatsAndLogging
 from toil.test import (ToilTest,
                        make_tests,
@@ -1408,14 +1409,17 @@ def _hashTestFile(self, url: str) -> str:
     def _createExternalStore(self):
         """A S3.Bucket instance is returned"""
         from toil.jobStores.aws.utils import retry_s3
-        from toil.jobStores.aws.utils import region_to_bucket_location
         from toil.jobStores.aws.jobStore import establish_boto3_session
-        resource = establish_boto3_session().resource('s3', region_name=self.awsRegion())
-        bucket = resource.Bucket('import-export-test-%s' % uuid.uuid4())
+
+        resource = establish_boto3_session().resource(
+            "s3", region_name=self.awsRegion()
+        )
+        bucket_name = f"import-export-test-{uuid.uuid4()}"
+        location = self.awsRegion()
 
         for attempt in retry_s3():
             with attempt:
-                bucket.create(CreateBucketConfiguration={'LocationConstraint': region_to_bucket_location(self.awsRegion())})
+                bucket = create_s3_bucket(resource, bucket_name, location)
                 bucket.wait_until_exists()
                 return bucket
 

src/toil/test/lib/aws/test_s3.py

@@ -0,0 +1,62 @@
+# Copyright (C) 2021 Michael R. Crusoe
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import os
+import uuid
+from typing import Optional
+
+from toil.jobStores.aws.jobStore import AWSJobStore
+from toil.lib.aws.utils import create_s3_bucket
+from toil.lib.ec2 import establish_boto3_session
+from toil.test import ToilTest, needs_aws_s3
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
+
+
+@needs_aws_s3
+class S3Test(ToilTest):
+    """Confirm the workarounds for us-east-1."""
+
+    from mypy_boto3_s3 import S3ServiceResource
+    from mypy_boto3_s3.service_resource import Bucket
+
+    s3_resource: Optional[S3ServiceResource]
+    bucket: Optional[Bucket]
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        session = establish_boto3_session(region_name="us-east-1")
+        cls.s3_resource = session.resource("s3", region_name="us-east-1")
+        cls.bucket = None
+
+    def test_create_bucket(self) -> None:
+        """Test bucket creation for us-east-1."""
+        bucket_name = f"toil-s3test-{uuid.uuid4()}"
+        assert self.s3_resource
+        self.bucket = create_s3_bucket(self.s3_resource, bucket_name, "us-east-1")
+        self.bucket.wait_until_exists()
+        owner_tag = os.environ.get("TOIL_OWNER_TAG")
+        if owner_tag:
+            bucket_tagging = self.s3_resource.BucketTagging(bucket_name)
+            bucket_tagging.put(
+                Tagging={"TagSet": [{"Key": "Owner", "Value": owner_tag}]}
+            )
+        self.assertTrue(AWSJobStore.getBucketRegion, "us-east-1")
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        if cls.bucket:
+            AWSJobStore._delete_bucket(cls.bucket)
+        super().tearDownClass()