Switch GitLab to stock Amazon Linux 2 AMI (#4188, #3942)

OPERATOR.rst

@@ -272,19 +272,12 @@ create a properly tagged snapshot of the GitLab EBS volume. Run::
 
 	python scripts/create_gitlab_snapshot.py
 
-.. FIXME: Should not have to destroy the instance to update
-          https://github.com/DataBiosphere/azul/issues/3942
-
-::
-
-	(cd terraform/gitlab && CI_COMMIT_REF_NAME=develop make validate && terraform taint aws_instance.gitlab)
-
-Once the instance is tainted, edit the `GitLab Terraform`_ file, updating the
-version of the Docker images for ``gitlab-ce`` and ``gitlab-runner``. Then run::
+Edit the `GitLab Terraform`_ file, updating the version of the Docker images for
+``gitlab-ce`` and ``gitlab-runner``. Then run::
 
     CI_COMMIT_REF_NAME=develop make -C terraform/gitlab
 
-.. _GitLab Terraform: https://github.com/DataBiosphere/azul/blob/develop/terraform/gitlab/gitlab.tf.json.template.py#L1243
+.. _GitLab Terraform: https://github.com/DataBiosphere/azul/blob/develop/terraform/gitlab/gitlab.tf.json.template.py
 
 The new GitLab instance should be online again in 10 minutes. If it takes
 longer, contact the lead. When the GitLab web app is online, have the lead

README.md

@@ -1,5 +1,3 @@
-
-
 The Azul project contains the components that together serve as the backend to
 Boardwalk, a web application for browsing genomic data sets.
 
@@ -453,6 +451,9 @@ inversely, name the bucket using the current value of that variable.
 
 ```
 aws s3api create-bucket --bucket $AZUL_VERSIONED_BUCKET
+aws s3api put-bucket-tagging \
+          --bucket $AZUL_VERSIONED_BUCKET \
+          --tagging TagSet="[{Key=owner,Value=$AZUL_OWNER}]"
 ```
 
 ### 3.1.2 Route 53 hosted zones

deployments/anvildev/environment.py

@@ -64,7 +64,8 @@ def env() -> Mapping[str, Optional[str]]:
     provide the value.
     """
     return {
-        # Set variables for the `dev` (short for development) deployment here.
+        # Set variables for the `anvildev` (short for AnVIL development)
+        # deployment here.
         #
         # Only modify this file if you intend to commit those changes. To change the
         # environment with a setting that's specific to you AND the deployment, create

deployments/anvilprod.gitlab/environment.py

@@ -0,0 +1,29 @@
+from collections.abc import (
+    Mapping,
+)
+from typing import (
+    Optional,
+)
+
+
+def env() -> Mapping[str, Optional[str]]:
+    """
+    Returns a dictionary that maps environment variable names to values. The 
+    values are either None or strings. String values can contain references to 
+    other environment variables in the form `{FOO}` where FOO is the name of an 
+    environment variable. See 
+
+    https://docs.python.org/3.9/library/string.html#format-string-syntax
+
+    for the concrete syntax. These references will be resolved *after* the
+    overall environment has been compiled by merging all relevant
+    `environment.py` and `environment.local.py` files.
+
+    Entries with a `None` value will be excluded from the environment. They
+    can be used to document a variable without a default value in which case
+    other, more specific `environment.py` or `environment.local.py` files must
+    provide the value.
+    """
+    return {
+        'azul_terraform_component': 'gitlab',
+    }

deployments/anvilprod.shared/environment.py

@@ -0,0 +1,29 @@
+from collections.abc import (
+    Mapping,
+)
+from typing import (
+    Optional,
+)
+
+
+def env() -> Mapping[str, Optional[str]]:
+    """
+    Returns a dictionary that maps environment variable names to values. The 
+    values are either None or strings. String values can contain references to 
+    other environment variables in the form `{FOO}` where FOO is the name of an 
+    environment variable. See 
+
+    https://docs.python.org/3.9/library/string.html#format-string-syntax
+
+    for the concrete syntax. These references will be resolved *after* the
+    overall environment has been compiled by merging all relevant
+    `environment.py` and `environment.local.py` files.
+
+    Entries with a `None` value will be excluded from the environment. They
+    can be used to document a variable without a default value in which case
+    other, more specific `environment.py` or `environment.local.py` files must
+    provide the value.
+    """
+    return {
+        'azul_terraform_component': 'shared',
+    }

deployments/anvilprod/.example.environment.local.py

@@ -0,0 +1 @@
+../prod/.example.environment.local.py

deployments/anvilprod/environment.py

@@ -0,0 +1,122 @@
+from collections.abc import (
+    Mapping,
+)
+import json
+from typing import (
+    Optional,
+)
+
+
+def partition_prefix_length(n: int) -> int:
+    """
+    For a given number of subgraphs, return a partition prefix length that is
+    expected to rarely exceed 512 subgraphs per partition.
+
+    >>> [partition_prefix_length(n) for n in (0, 1, 512, 513, 16 * 512, 16 * 513 )]
+    [0, 0, 0, 1, 1, 2]
+    """
+    return 1 + partition_prefix_length(n // 16) if n > 512 else 0
+
+
+ma = 1  # managed access
+pop = 2  # remove snapshot
+
+
+def mksrc(google_project, snapshot, subgraphs, flags: int = 0):
+    assert flags <= ma | pop
+    source = None if flags & pop else ':'.join([
+        'tdr',
+        google_project,
+        'snapshot/' + snapshot,
+        '/' + str(partition_prefix_length(subgraphs))
+    ])
+    return source
+
+
+def mkdict(items):
+    result = dict(items)
+    assert len(items) == len(result), 'collisions detected'
+    assert list(result.keys()) == sorted(result.keys()), 'input not sorted'
+    return result
+
+
+anvil_sources = mkdict([
+    ('encode', mksrc('datarepo-e8d615a1', 'AnVIL_ENCODE_default_v1', 31975))
+])
+
+
+def env() -> Mapping[str, Optional[str]]:
+    """
+    Returns a dictionary that maps environment variable names to values. The
+    values are either None or strings. String values can contain references to
+    other environment variables in the form `{FOO}` where FOO is the name of an
+    environment variable. See
+
+    https://docs.python.org/3.9/library/string.html#format-string-syntax
+
+    for the concrete syntax. These references will be resolved *after* the
+    overall environment has been compiled by merging all relevant
+    `environment.py` and `environment.local.py` files.
+
+    Entries with a `None` value will be excluded from the environment. They
+    can be used to document a variable without a default value in which case
+    other, more specific `environment.py` or `environment.local.py` files must
+    provide the value.
+    """
+    return {
+        # Set variables for the `anvilprod` (short for AnVIL production)
+        # deployment here.
+        #
+        # Only modify this file if you intend to commit those changes. To change the
+        # environment with a setting that's specific to you AND the deployment, create
+        # a environment.local.py right next to this file and make your changes there.
+        # Settings applicable to all environments but specific to you go into
+        # environment.local.py at the project root.
+
+        'AZUL_DEPLOYMENT_STAGE': 'anvilprod',
+
+        'AZUL_DOMAIN_NAME': 'prod.anvil.gi.ucsc.edu',
+        'AZUL_URL_REDIRECT_BASE_DOMAIN_NAME': 'prod.anvil.gi.ucsc.edu',
+        'AZUL_URL_REDIRECT_FULL_DOMAIN_NAME': 'url.{AZUL_URL_REDIRECT_BASE_DOMAIN_NAME}',
+
+        'AZUL_VERSIONED_BUCKET': 'edu-ucsc-gi-platform-anvil-prod.{AWS_DEFAULT_REGION}',
+        'AZUL_S3_BUCKET': 'edu-ucsc-gi-platform-anvil-prod-{AZUL_DEPLOYMENT_STAGE}',
+
+        'AZUL_CATALOGS': json.dumps({
+            f'{catalog}{suffix}': dict(atlas=atlas,
+                                       internal=internal,
+                                       plugins=dict(metadata=dict(name='anvil'),
+                                                    repository=dict(name='tdr_anvil')),
+                                       sources=list(filter(None, sources.values())))
+            for atlas, catalog, sources in [
+                ('anvil', 'anvil-encode', anvil_sources),
+            ]
+            for suffix, internal in [
+                ('', False),
+                ('-it', True)
+            ]
+        }),
+
+        'AZUL_TDR_SOURCE_LOCATION': 'US',
+        'AZUL_TDR_SERVICE_URL': 'https://data.terra.bio',
+        'AZUL_SAM_SERVICE_URL': 'https://sam.dsde-prod.broadinstitute.org',
+
+        'AZUL_ENABLE_MONITORING': '1',
+
+        # $0.382/h × 3 × 24h/d × 30d/mo = $825.12/mo
+        'AZUL_ES_INSTANCE_TYPE': 'r6gd.xlarge.elasticsearch',
+        'AZUL_ES_INSTANCE_COUNT': '3',
+
+        'AZUL_DEBUG': '1',
+
+        'AZUL_BILLING': 'anvil',
+
+        'AZUL_OWNER': '[email protected]',
+
+        'AZUL_AWS_ACCOUNT_ID': '465330168186',
+        'AWS_DEFAULT_REGION': 'us-east-1',
+
+        'GOOGLE_PROJECT': 'platform-anvil-prod',
+
+        'AZUL_GOOGLE_OAUTH2_CLIENT_ID': '',
+    }