From 2ba3d2106399d27a7836897eef8506b44b4c8e87 Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Sun, 2 Jul 2023 19:29:17 -0700 Subject: [PATCH 1/6] Reformat and improve wording of documentation in `environment` --- environment.py | 282 ++++++++++++++++++++++++++++--------------------- 1 file changed, 162 insertions(+), 120 deletions(-) diff --git a/environment.py b/environment.py index 2fb1097879..22b3c4771a 100644 --- a/environment.py +++ b/environment.py @@ -29,6 +29,13 @@ def env() -> Mapping[str, Optional[str]]: xdg_data_home = os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')) return { + # Only variables whose names start in `AZUL_` will be published to a + # deployed Lambda. Note that by implication, `azul_` variables will not + # be published, even though they are considered part of Azul. For secret + # values that should not be printed or logged, use a variable name + # containing any of the strings `secret`, `password` or `token`, either + # upper or lower case. Think twice before publishing a variable + # containing a secret. # Configure the catalogs to be managed by this Azul deployment. A # catalog is a group of indices populated from a particular source. @@ -54,17 +61,18 @@ def env() -> Mapping[str, Optional[str]]: # } # # The `atlas` and `name` properties follow the same, fairly restrictive - # syntax defined by azul.Config.Catalog.validate_name. - # `plugin_type` is the name of a child package of `azul.plugins` and - # `plugin_package` is the name of a child package of that package. The - # `plugin_type` denotes the purpose (like accessing a repository or - # transforming metadata) and `plugin_package` denotes the concrete - # implementation of how to fulfill that purpose. + # syntax defined by azul.Config.Catalog.validate_name. `plugin_type` is + # the name of a child package of `azul.plugins` and `plugin_package` is + # the name of a child package of that package. The `plugin_type` denotes + # the purpose (like accessing a repository or transforming metadata) and + # `plugin_package` denotes the concrete implementation of how to fulfill + # that purpose. # # The first catalog listed is the default catalog. # # A source represents a TDR dataset, TDR snapshot, or canned staging - # area to index. Each source is a string matching the following EBNF grammar: + # area to index. Each source is a string matching the following EBNF + # grammar: # # source = TDR source | canned source ; # @@ -82,23 +90,23 @@ def env() -> Mapping[str, Optional[str]]: # ':', [ prefix ], # '/', partition prefix length ; # - # The `prefix` is an optional string of hexadecimal digits - # constraining the set of indexed subgraphs from the source. A - # subgraph will be indexed if its UUID begins with the `prefix`. The - # default `prefix` is the empty string. + # The `prefix` is an optional string of hexadecimal digits constraining + # the set of indexed subgraphs from the source. A subgraph will be + # indexed if its UUID begins with the `prefix`. The default `prefix` is + # the empty string. # # The partition prefix length is an integer that is used to further - # partition the set of indexed subgraphs. Each partition is - # assigned a prefix of `partition prefix length` hexadecimal digits. - # A subgraph belongs to a partition if its UUID starts with the - # overall `prefix` followed by the partition's prefix. The number of - # partitions of a source is therefore `16 ** partition prefix length`. - # Partition prefixes that are too long result in many small or even - # empty partitions and waste some amount of resources. Partition - # prefixes that are too short result in few large partitions that could - # exceed the memory and running time limitations of the AWS Lambda - # function that processes them. If in doubt err on the side of too many - # small partitions. + # partition the set of indexed subgraphs. Each partition is assigned a + # prefix of `partition prefix length` hexadecimal digits. A subgraph + # belongs to a partition if its UUID starts with the overall `prefix` + # followed by the partition's prefix. The number of partitions of a + # source is therefore `16 ** partition prefix length`. Partition + # prefixes that are too long result in many small or even empty + # partitions and waste some amount of resources. Partition prefixes that + # are too short result in few large partitions that could exceed the + # memory and running time limitations of the AWS Lambda function that + # processes them. If in doubt err on the side of too many small + # partitions. # # The `partition prefix length` plus the length of `prefix` must not # exceed 8. @@ -114,9 +122,9 @@ def env() -> Mapping[str, Optional[str]]: # # This variable tends to be large. If you get `Argument list too long` # after sourcing the environment, a last-resort option is to compress - # the variable. The application automatically detects a compressed - # value and decompresses it on the fly. If the uncompressed definition - # of this variable is + # the variable. The application automatically detects a compressed value + # and decompresses it on the fly. If the uncompressed definition of this + # variable is # # 'AZUL_CATALOGS': json.dumps({ # ... @@ -131,6 +139,7 @@ def env() -> Mapping[str, Optional[str]]: 'AZUL_CATALOGS': None, # The Account ID number for AWS + # 'AZUL_AWS_ACCOUNT_ID': None, # The region of the Azul deployment. This variable is primarily used by @@ -140,18 +149,14 @@ def env() -> Mapping[str, Optional[str]]: # 'AWS_DEFAULT_REGION': None, - # Only variables whose names start in `AZUL_` will be published to a deployed - # Lambda. Note that by implication, `azul_` variables will not be published, - # even though they are considered part of Azul. For secret values that should - # not be printed or logged, use a variable name containing any of the strings - # `secret`, `password` or `token`, either upper or lower case. Think twice - # before publishing a variable containing a secret. - # The name of the billing accocunt that pays for this deployment. + # 'AZUL_BILLING': None, - # The email address of a user that owns the cloud resources in the current - # deployment. This will become the value of the Owner tag on all resources. + # The email address of a user that owns the cloud resources in the + # current deployment. This will become the value of the Owner tag on all + # resources. + # 'AZUL_OWNER': None, # An email address to subscribe to the SNS topic for monitoring @@ -162,20 +167,23 @@ def env() -> Mapping[str, Optional[str]]: # unsubscribe link to unsubscribe the entire group from the topic. # Instead, confirmation of the subscription should be done when prompted # to do so during the `make deploy` process. + # 'AZUL_MONITORING_EMAIL': None, # Controls the verbosity of application logs. Use 0 for no debug logging # 1 for debug logging by application code and info logging by other code # and 2 for debug logging by all code. This also controls app.debug, a - # Chalice setting that causes an app to return a traceback in the body of - # error responses: Setting AZUL_DEBUG to 0 disables the traceback - # (app.debug = False), 1 or higher enable it (app.debug = True). - # See https://github.com/aws/chalice#tutorial-error-messages for more. + # Chalice setting that causes an app to return a traceback in the body + # of error responses: Setting AZUL_DEBUG to 0 disables the traceback + # (app.debug = False), 1 or higher enable it (app.debug = True). See + # https://github.com/aws/chalice#tutorial-error-messages for more. + # 'AZUL_DEBUG': '0', - # The name of the current deployment. This variable controls the name of all - # cloud resources and is the main vehicle for isolating cloud resources - # between deployments. + # The name of the current deployment. This variable controls the name of + # all cloud resources and is the main vehicle for isolating cloud + # resources between deployments. + # 'AZUL_DEPLOYMENT_STAGE': None, # The Docker registry containing all 3rd party images used by this @@ -186,48 +194,50 @@ def env() -> Mapping[str, Optional[str]]: # and the images therein are managed by the `shared` TF component, which # copies images from the upstream registry into the Azul registry. A # 3rd-party image at `//:tag`, is stored - # as `${azul_docker_registry>}//:tag` - # in the Azul registry. To disable the use of the Azul registry, set - # this variable to the empty string. + # as `${azul_docker_registry>}//:tag` in + # the Azul registry. To disable the use of the Azul registry, set this + # variable to the empty string. # 'azul_docker_registry': '{AZUL_AWS_ACCOUNT_ID}.dkr.ecr.' '{AWS_DEFAULT_REGION}.amazonaws.com/', - # Whether to enable direct access to objects in the DSS main bucket. If 0, - # bundles and files are retrieved from the DSS using the GET /bundles/{uuid} - # and GET /files/{UUID} endpoints. If 1, S3 GetObject requests are made - # directly to the underlying bucket. This requires intimate knowledge of DSS - # implementation details but was ten times faster. Recent optimizations to - # the DSS (mainly, the delayed eviction of metadata files from the checkout - # bucket made the performance gains less dramatic but the first hit to a - # metadata file is still going to be slow because the objects needs to be - # checked out. Aside from the latency improvements on a single request, - # direct access also bypasses DSS lambda concurrency limits, thereby - # increasing in the throughput of the Azul indexer, which is especially - # noticeable during reindexing and scale testing. - # - # More importantly, direct access needs to be enabled for deletions to work - # properly as the Azul indexer needs to know the metadata of the deleted - # bundle in order to place the correct tombstone contributions into its - # index. Direct access is also required for the Azul service's DSS files - # proxy and DOS/DRS endpoints. Disabling DSS direct access will break these - # endpoints. + # Whether to enable direct access to objects in the DSS main bucket. If + # 0, bundles and files are retrieved from the DSS using the GET + # /bundles/{uuid} and GET /files/{UUID} endpoints. If 1, S3 GetObject + # requests are made directly to the underlying bucket. This requires + # intimate knowledge of DSS implementation details but was ten times + # faster. Recent optimizations to the DSS (mainly, the delayed eviction + # of metadata files from the checkout bucket made the performance gains + # less dramatic but the first hit to a metadata file is still going to + # be slow because the objects needs to be checked out. Aside from the + # latency improvements on a single request, direct access also bypasses + # DSS lambda concurrency limits, thereby increasing in the throughput of + # the Azul indexer, which is especially noticeable during reindexing and + # scale testing. + # + # More importantly, direct access needs to be enabled for deletions to + # work properly as the Azul indexer needs to know the metadata of the + # deleted bundle in order to place the correct tombstone contributions + # into its index. Direct access is also required for the Azul service's + # DSS files proxy and DOS/DRS endpoints. Disabling DSS direct access + # will break these endpoints. # 'AZUL_DSS_DIRECT_ACCESS': '0', - # An optional ARN of a role to assume when directly accessing a DSS bucket. - # This can be useful when the DSS buckets are not owned by the same AWS - # account owning the current Azul deployment. If there is another Azul - # deployment in the account owning the DSS bucket, the role assumed by the - # other Azul indexer would be an obvious candidate for the current - # deployment's indexer to assume for direct access. Presumably that other - # indexer has sufficient privileges to directly access the DSS buckets. - # - # The character '*' will be replaced with the name of the lambda - # wishing to gain access. This parameterization can be used to have the - # indexer lambda in the native deployment assume the role of the indexer - # lambda in the foreign deployment, while the native service lambda assumes - # the role of the foreign service lambda. + # An optional ARN of a role to assume when directly accessing a DSS + # bucket. This can be useful when the DSS buckets are not owned by the + # same AWS account owning the current Azul deployment. If there is + # another Azul deployment in the account owning the DSS bucket, the role + # assumed by the other Azul indexer would be an obvious candidate for + # the current deployment's indexer to assume for direct access. + # Presumably that other indexer has sufficient privileges to directly + # access the DSS buckets. + # + # The character '*' will be replaced with the name of the lambda wishing + # to gain access. This parameterization can be used to have the indexer + # lambda in the native deployment assume the role of the indexer lambda + # in the foreign deployment, while the native service lambda assumes the + # role of the foreign service lambda. # # If specified, this ARN must be of the following form: # @@ -239,30 +249,31 @@ def env() -> Mapping[str, Optional[str]]: # 'AZUL_DSS_DIRECT_ACCESS_ROLE': None, - # The name of the hosted zone in Route 53 in which to create user friendly - # domain names for various API gateways. This hosted zone will have to be - # created manually prior to running `make deploy`. The value is typically - # not deployment specific. A subdomain will automatically be created for - # each deployment. + # The name of the hosted zone in Route 53 in which to create user + # friendly domain names for various API gateways. This hosted zone will + # have to be created manually prior to running `make deploy`. The value + # is typically not deployment specific. A subdomain will automatically + # be created for each deployment. 'AZUL_DOMAIN_NAME': None, - # An optional list of roles in other AWS accounts that can assume the IAM - # role normally assumed by lambda functions in the active Azul deployment. + # An optional list of roles in other AWS accounts that can assume the + # IAM role normally assumed by lambda functions in the active Azul + # deployment. # # The syntax is [,...][:[,...]...] where # is the numeric AWS account ID and role is a role name with - # optional * or ? wildcards for the StringLike operator in IAM conditions. - # Whitespace around separators and at the beginning or end of the value - # are ignored. + # optional * or ? wildcards for the StringLike operator in IAM + # conditions. Whitespace around separators and at the beginning or end + # of the value are ignored. # - # This parameter has profound security implications: the external role can - # do anything an Azul lambda can do. The external account and any principal - # with IAM access in that account, not just the specified roles, must be - # fully trusted. + # This parameter has profound security implications: the external role + # can do anything an Azul lambda can do. The external account and any + # principal with IAM access in that account, not just the specified + # roles, must be fully trusted. # - # This configuration is typically used to enable an external Azul deployment - # to directly access the same DSS buckets the active deployment has direct - # access to. + # This configuration is typically used to enable an external Azul + # deployment to directly access the same DSS buckets the active + # deployment has direct access to. # 'AZUL_EXTERNAL_LAMBDA_ROLE_ASSUMPTORS': None, @@ -278,42 +289,50 @@ def env() -> Mapping[str, Optional[str]]: 'AZUL_DRS_DOMAIN_NAME': '', # A template for the name of the Route 53 record set in the hosted zone - # specified by AZUL_DOMAIN_NAME. The character '*' in the template - # will be substituted with the name of the Lambda function, e.g. `indexer` - # or `service`. May contain periods. + # specified by AZUL_DOMAIN_NAME. The character '*' in the template will + # be substituted with the name of the Lambda function, e.g. `indexer` or + # `service`. May contain periods. + # 'AZUL_SUBDOMAIN_TEMPLATE': '*', # Boolean value, 0 to create public APIs, 1 to create private APIs that # can only be accessed from within the VPC or through the VPN tunnel # into the VPC. + # 'AZUL_PRIVATE_API': '0', # A prefix to be prepended to the names of AWS Lambda functions and # associated resources. Must not contain periods. + # 'AZUL_RESOURCE_PREFIX': 'azul', # The host and port of the Elasticsearch instance to use. This takes # precedence over AZUL_ES_DOMAIN. + # 'AZUL_ES_ENDPOINT': None, - # The name of the AWS-hosted Elasticsearch instance (not a domain name) to - # use. The given ES domain's endpoint will be looked up dynamically. + # The name of the AWS-hosted Elasticsearch instance (not a domain name) + # to use. The given ES domain's endpoint will be looked up dynamically. + # 'AZUL_ES_DOMAIN': 'azul-index-{AZUL_DEPLOYMENT_STAGE}', # Boolean value, 1 to share `dev` ES domain, 0 to create your own + # 'AZUL_SHARE_ES_DOMAIN': '0', # Prefix to describe ES indices + # 'AZUL_INDEX_PREFIX': 'azul', # The number of nodes in the AWS-hosted Elasticsearch cluster + # 'AZUL_ES_INSTANCE_COUNT': None, # The EC2 instance type to use for a cluster node. # - # Indexing performance benefits from the increased memory offered - # by the `r` family, especially now that the number of shards is - # tied to the indexer Lambda concurrency. + # Indexing performance benefits from the increased memory offered by the + # `r` family, especially now that the number of shards is tied to the + # indexer Lambda concurrency. # 'AZUL_ES_INSTANCE_TYPE': None, @@ -322,21 +341,24 @@ def env() -> Mapping[str, Optional[str]]: # 'AZUL_ES_VOLUME_SIZE': '0', - # Elasticsearch operation timeout in seconds - # matches AWS' own timeout on the ELB sitting in front of ES: + # Elasticsearch operation timeout in seconds. Matches AWS' own timeout + # on the ELB sitting in front of ES: + # # https://forums.aws.amazon.com/thread.jspa?threadID=233378 + # 'AZUL_ES_TIMEOUT': '60', - # The number of workers pulling files from the DSS repository. - # There is one such set of repository workers per index worker. + # The number of workers pulling files from the DSS repository. There is + # one such set of repository workers per index worker. + # 'AZUL_DSS_WORKERS': '8', - # The number of workers pulling metadata from the TDR repository. - # There is one such set of repository workers per index worker. - # Using one worker as opposed to 8 (or 4) improved the indexing time - # noticeably because it reduced retries due to exceeding BigQuery's - # limit on the # of concurrent queries. Using two workers wasn't - # significantly faster. + # The number of workers pulling metadata from the TDR repository. There + # is one such set of repository workers per index worker. Using one + # worker as opposed to 8 (or 4) improved the indexing time noticeably + # because it reduced retries due to exceeding BigQuery's limit on the # + # of concurrent queries. Using two workers wasn't significantly faster. + # 'AZUL_TDR_WORKERS': '1', # The number of times a deployment has been destroyed and rebuilt. Some @@ -345,6 +367,7 @@ def env() -> Mapping[str, Optional[str]]: # such resources will include this value, therefore making the names # distinct. If a deployment is being rebuilt, increment this value in # the deployment's `environment.py` file. + # 'AZUL_DEPLOYMENT_INCARNATION': '0', # The name of the Google Cloud project to host the Azul deployment. @@ -352,13 +375,16 @@ def env() -> Mapping[str, Optional[str]]: # GOOGLE_APPLICATION_CREDENTIALS environment variable to point to the # key file of a Google service account, or setting the application # default credentials using the `gcloud` CLI interactive login. + # 'GOOGLE_PROJECT': None, # The name of the Google Cloud service account to represent the - # deployment. It is used to access all (meta)data in Google-based - # repositories. If unset, a canonical resource name will be used. That - # default allows one such account per Azul deployment and Google Cloud - # project. + # deployment. This service account will be created automatically during + # deployment and will then be used to access (meta)data in Google-based + # repositories, like the Terra Data Repository (TDR). If unset, a + # canonical resource name will be used. That default allows one such + # account per Azul deployment and Google Cloud project. + # 'AZUL_GOOGLE_SERVICE_ACCOUNT': 'azul-ucsc-{AZUL_DEPLOYMENT_INCARNATION}-{AZUL_DEPLOYMENT_STAGE}', # The name of the Google Cloud service account to be created and used @@ -371,17 +397,21 @@ def env() -> Mapping[str, Optional[str]]: # The name of the Google Cloud service account to be created and used # to simulate access from users who are logged in but not registered # with SAM. + # 'AZUL_GOOGLE_SERVICE_ACCOUNT_UNREGISTERED': 'azul-ucsc-{AZUL_DEPLOYMENT_INCARNATION}-unreg-{AZUL_DEPLOYMENT_STAGE}', # The number of concurrently running lambda executions for the # contribution and aggregation stages of indexing, respectively. # Concurrency for the retry lambdas of each stage can be configured - # separately via a '/' separator, e.g. '{normal concurrency}/{retry concurrency}'. - # Chalice creates one Lambda function for handling HTTP requests from - # API Gateway and one additional Lambda function per event handler. The - # concurrency limit applies to each such function independently. See + # separately via a '/' separator, e.g. '{normal concurrency}/{retry + # concurrency}'. Chalice creates one Lambda function for handling HTTP + # requests from API Gateway and one additional Lambda function per event + # handler. The concurrency limit applies to each such function + # independently. See + # # https://docs.aws.amazon.com/lambda/latest/dg/concurrent-executions.html + # # for details. These settings may also be used to drive other scaling # choices. For example, the non-retry contribution concurrency # determines the number of shards in Elasticsearch. @@ -391,10 +421,12 @@ def env() -> Mapping[str, Optional[str]]: # The name of the S3 bucket where the manifest API stores the downloadable # content requested by client. + # 'AZUL_S3_BUCKET': None, # Collect and monitor important health metrics of the deployment (1 yes, 0 no). # Typically only enabled on main deployments. + # 'AZUL_ENABLE_MONITORING': '0', # Identifies the DSS repository endpoint and prefix to index. @@ -411,6 +443,7 @@ def env() -> Mapping[str, Optional[str]]: # # https://dss.data.humancellatlas.org/v1:/1 # https://dss.data.humancellatlas.org/v1:aa/1 + # 'AZUL_DSS_SOURCE': None, # A short string (no punctuation allowed) that identifies a Terraform @@ -420,13 +453,16 @@ def env() -> Mapping[str, Optional[str]]: # their own directory under `deployments`. The main component is identified # by the empty string and its resources are defined in the `terraform` # directory. + # 'azul_terraform_component': '', # The slug of a the Github repository hosting this fork of Azul + # 'azul_github_project': 'DataBiosphere/azul', # An Github REST API access token with permission to post status checks to # the repository defined in `azul_github_project`. + # 'azul_github_access_token': '', # A GitLab private access token with scopes `read_api`, `read_registry` @@ -460,6 +496,7 @@ def env() -> Mapping[str, Optional[str]]: # FIXME: Remove once we upgrade to botocore 1.28.x # https://github.com/DataBiosphere/azul/issues/4560 + # 'BOTO_DISABLE_COMMONNAME': 'true', # The path of the directory where the public key infrastructure files @@ -481,6 +518,7 @@ def env() -> Mapping[str, Optional[str]]: # location. # # https://cloud.google.com/bigquery/docs/locations + # 'AZUL_TDR_SOURCE_LOCATION': None, # The full set of BigQuery dataset locations of the TDR snapshots @@ -499,24 +537,29 @@ def env() -> Mapping[str, Optional[str]]: # enable batch mode. # # https://cloud.google.com/bigquery/docs/running-queries + # 'AZUL_BIGQUERY_BATCH_MODE': '0', # Timeout in seconds for requests to Terra. Two different values are # configured, separated by a colon. The first is for time-sensitive # contexts such as API Gateway. The second is for contexts in which we # can afford to be more patient. + # 'AZUL_TERRA_TIMEOUT': '5:20', # The URL of the Terra Data Repository instance to index metadata from. + # 'AZUL_TDR_SERVICE_URL': None, # The URL of an instance of Broad Institute's SAM. # This needs to point to the SAM instance that's used by the TDR # instance configured in `AZUL_TDR_SERVICE_URL`. + # 'AZUL_SAM_SERVICE_URL': None, # OAuth2 Client ID to be used for authenticating users. See section # 3.2 of the README + # 'AZUL_GOOGLE_OAUTH2_CLIENT_ID': None, # Maps a branch name to a list of names of deployments the branch may be @@ -613,7 +656,6 @@ def env() -> Mapping[str, Optional[str]]: # # https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-cis-controls.html#securityhub-cis-controls-1.20 # - # 'azul_aws_support_roles': json.dumps([]), # A dict containing the contact details of the AWS account alternate From 56d611be5d194d07c9d6ca7fc898efc8cea88d97 Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Mon, 3 Jul 2023 23:25:20 -0700 Subject: [PATCH 2/6] Fix format in UPGRADING.rst --- UPGRADING.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/UPGRADING.rst b/UPGRADING.rst index e8cfede413..bd4ddbcb64 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -11,8 +11,8 @@ Upgrading This file documents any upgrade procedure that must be performed. Because we -don't use a semantic version, a change that requires explicit steps to upgrade -a is referenced by its Github issue number. After checking out a branch that +don't use a semantic version, a change that requires explicit steps to upgrade a +is referenced by its Github issue number. After checking out a branch that contains a commit resolving an issue listed here, the steps listed underneath the issue need to be performed. When switching away from that branch, to a branch that does not have the listed changes, the steps would need to be From 0a6be006bf10de6db8a1dcb3e3634cd0af777d7b Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Sun, 2 Jul 2023 19:53:08 -0700 Subject: [PATCH 3/6] Fix: _select doesn't validate its argument (#5289) --- environment | 70 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/environment b/environment index a5e66e6395..be5debb92c 100644 --- a/environment +++ b/environment @@ -14,28 +14,60 @@ _refresh() { # Manage the symlink to the active deployment # _select() { - if [[ $1 != "" ]]; then - _deauth - _link "$1" || return - _refresh - _preauth + if [ -z "$1" ] ; then + _show_link + else + if ! { + _validate_link "$1" && + _deauth && + _link "$1" && + _refresh && + _preauth ; + } ; then + echo >&2 "_select failed" + return 1 + fi fi - (cd "${project_root}/deployments" && ls -l .active) } _deselect() { - _deauth - rm "${project_root}/deployments/.active" - _refresh + if ! { + _deauth && + _unlink && + _refresh ; + } ; then + echo >&2 "_deselect failed" + return 1 + fi } _link() { - ( + if ! ( + _validate_link "$1" && cd "${project_root}/deployments" && - test -d "$1" && - { [ ! -e .active ] || { [ -L .active ] && rm .active; }; } && - ln -s "$1" .active - ) || { echo error: "$1" && return; } + { [ ! -e .active ] || { [ -L .active ] && rm .active ; } ; } && + ln -s "$1" .active + ) ; then + echo >&2 "_link failed: '$1'" + return 1 + fi +} + +_unlink() { + rm "${project_root}/deployments/.active" +} + +_validate_link() { + d="${project_root}/deployments/$1" + # -d dereferences symlinks so with -L we make sure the argument isn't one + if ! { [ ! -L "$d" ] && [ -d "$d" ] ; } ; then + echo >&2 "_validate_link failed: '$1'" + return 1 + fi +} + +_show_link() { + ( cd "${project_root}/deployments" && ls -l .active ) } # Get temporary credentials from STS via AssumeRole and inject them @@ -159,7 +191,15 @@ _deauth() { } _revenv() { - deactivate && make virtualenv && source .venv/bin/activate && make requirements envhook + if ! { + deactivate && + make virtualenv && + source .venv/bin/activate && + make requirements envhook ; + } ; then + echo >&2 "_revenv failed" + return 1 + fi } # We disable `envhook.py` to avoid redundancy. The `envhook.py` script imports From 69db9c38f8b269c6503b4f0c4d7e315cf74078e0 Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Mon, 3 Jul 2023 11:00:00 -0700 Subject: [PATCH 4/6] [u] Segregate Google Cloud SDK & CLI state per deployment --- UPGRADING.rst | 8 ++ environment | 201 +++++++++++++++++++++++++++++---------------- environment.py | 54 ++++++++++++ scripts/envhook.py | 2 +- 4 files changed, 191 insertions(+), 74 deletions(-) diff --git a/UPGRADING.rst b/UPGRADING.rst index bd4ddbcb64..0a2d4e3c49 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -20,6 +20,14 @@ reverted. This is all fairly informal and loosely defined. Hopefully we won't have too many entries in this file. +#5289 Fix: _select doesn't validate its argument +================================================ + +Set the environment variable ``azul_google_user`` in all deployments to your +``…@ucsc.edu`` email address. The easiest way to do that is in an +``environment.local.py`` at the project root. + + #5325 Exclude noisy events from api_unauthorized alarm ====================================================== diff --git a/environment b/environment index be5debb92c..40ae66b8f4 100644 --- a/environment +++ b/environment @@ -70,23 +70,26 @@ _show_link() { ( cd "${project_root}/deployments" && ls -l .active ) } +_preauth_google() { + if [ -n "$azul_google_user" ] ; then + if ! { + gcloud auth login --update-adc --quiet "$azul_google_user" && + gcloud config set project "$GOOGLE_PROJECT" && + gcloud auth application-default set-quota-project "$GOOGLE_PROJECT" ; + } ; then + echo >&2 "_preauth_google failed" + return 1 + fi + fi +} + # Get temporary credentials from STS via AssumeRole and inject them # into the current environment where other AWS client libraries can # find them. # # https://github.com/boto/boto3/issues/1179#issuecomment-569940530 # -_preauth() { - if [ -z ${GOOGLE_APPLICATION_CREDENTIALS+x} ]; then - gcloud config set project "$GOOGLE_PROJECT" - if ! gcloud auth application-default print-access-token > /dev/null 2>&1; then - gcloud auth application-default login - if ! gcloud auth application-default print-access-token > /dev/null; then - echo >&2 "Google login failed!" - return 1 - fi - fi - fi +_preauth_aws() { local env if ! env="$( python - <<- "EOF" @@ -121,75 +124,127 @@ _preauth() { print(f'export AWS_SECRET_ACCESS_KEY={credentials.secret_key}') print(f'export AWS_SESSION_TOKEN={credentials.token}') EOF - )"; then - echo >&2 "AWS login failed!" - return 2 + )" ; then + echo >&2 "_preauth_aws failed" + return 1 fi eval "$env" - echo >&2 - if ! { - [ -z "${azul_docker_registry:+x}" ] \ - || aws ecr get-login-password --region us-east-1 \ - | docker login \ - --username AWS \ - --password-stdin \ - "${azul_docker_registry%/}" - }; then - echo >&2 "Login to ECR failed!" - return 3 - fi - if ! { - [ -z "${azul_gitlab_access_token:+x}" ] \ - || [ -z "${azul_gitlab_user:+x}" ] \ - || printenv azul_gitlab_access_token \ - | docker login \ - --username "${azul_gitlab_user}" \ - --password-stdin \ - "docker.gitlab.${AZUL_DOMAIN_NAME}" - }; then - echo >&2 "Login to GitLab registry failed!" - return 3 - fi - echo >&2 - echo >&2 "Temporary session credentials are in effect for AWS, Google," \ - "Amazon ECR and the GitLab docker registry." \ - "Use '_deauth' to revert the effect of this command." - echo >&2 - return 0 } -_deauth() { - if ! { - [ -z "${azul_docker_registry:+x}" ] \ - || docker logout "${azul_docker_registry}" - }; then - echo >&2 "Warning: Logout from ECR failed!" - fi - if ! { - [ -z "${azul_gitlab_access_token:+x}" ] \ - || [ -z "${azul_gitlab_user:+x}" ] \ - || docker logout "docker.gitlab.${AZUL_DOMAIN_NAME}" - }; then - echo >&2 "Warning: Logout from GitLab registry failed!" - fi +_preauth_docker_ecr() { + if [ -n "${azul_docker_registry:+x}" ] ; then + if ! ( + set -o pipefail + aws ecr get-login-password --region us-east-1 | + docker login \ + --username AWS \ + --password-stdin \ + "${azul_docker_registry%/}" + ) ; then + echo >&2 "_preauth_docker_ecr failed" + return 1 + fi + fi +} + +_preauth_docker_gitlab() { + if { + [ -n "${azul_gitlab_access_token:+x}" ] && + [ -n "${azul_gitlab_user:+x}" ] ; + } ; then + if ! ( + set -o pipefail + printenv azul_gitlab_access_token | + docker login \ + --username "${azul_gitlab_user}" \ + --password-stdin \ + "docker.gitlab.${AZUL_DOMAIN_NAME}" + ) ; then + echo >&2 "_preauth_docker_gitlab failed" + return 1 + fi + fi +} + +_preauth() { + if { + _preauth_google && + _preauth_aws && + _preauth_docker_ecr && + _preauth_docker_gitlab ; + } ; then + echo >&2 \ + "Session credentials are in effect for AWS. Additionally, you have " \ + "been logged into Google Cloud, Amazon ECR and the GitLab Docker " \ + "registry. Use '_deauth' to invalidate the session credentials. " \ + "Alternatively, you can use _deauth_completely to invalidate all " \ + "credentials but this is usually not necessary." + else + echo >&2 "_preauth failed" + return 1 + fi +} + +_deauth_docker_ecr() { + if [ -n "${azul_docker_registry:+x}" ] ; then + if ! { + docker logout "${azul_docker_registry}" ; + } ; then + echo >&2 "_deauth_docker_ecr failed" + return 1 + fi + fi +} + +_deauth_docker_gitlab() { + if { + [ -n "${azul_gitlab_access_token:+x}" ] && + [ -n "${azul_gitlab_user:+x}" ] ; + } ; then + if ! docker logout "docker.gitlab.${AZUL_DOMAIN_NAME}" ; then + echo >&2 "_deauth_docker_gitlab failed" + return 1 + fi + fi +} + +_deauth_aws() { unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN - echo >&2 - echo >&2 "Revoked temporary AWS credentials." - echo >&2 - if [ -z ${GOOGLE_APPLICATION_CREDENTIALS+x} ]; then - echo >&2 - echo >&2 "Google Cloud application default credentials are system-global." \ - "You should revoke the existing credentials unless you are *certain* that" \ - "you are already logged in using the appropriate account and project." \ - "If in doubt, enter 'y'." - echo >&2 - gcloud auth application-default revoke - echo >&2 - echo >&2 "Revoked temporary Google credentials." - echo >&2 +} + +_deauth_google() { + if [ -n "$azul_google_user" ] ; then + if ! { + gcloud auth application-default revoke --quiet && + gcloud auth revoke --quiet ; + } ; then + echo >&2 "_deauth_google failed" + return 1 + fi fi } +_deauth() { + # Docker segregates credential state by registry and we maintain separate + # registries (both ECR and GitLab) per deployment so we won't need to log out + # of those registries when switching deployments. Above, we offer dedicated + # functons for explicitly logging our of those registries. + _deauth_aws + # We segregate Google state by deployment and working copy (see + # CLOUDSDK_CONFIG in environment.py) so we don't need to log out of Google + # when switching deployments. Above we offer a dedicated function for + # explicitly logging out of Google. +} + +_deauth_completely() { + # We don't use `&&` between function invocations because failing to log out of + # one realm shouldn't prevent us from attempting to log out of the others. + _deauth_google + _deauth + _deauth_docker_ecr + _deauth_docker_gitlab +} + _revenv() { if ! { deactivate && @@ -221,4 +276,4 @@ _complete_env() { return 0 } -complete -F _complete_env _select +complete -F _complete_env _select _link _validate_link diff --git a/environment.py b/environment.py index 22b3c4771a..acc80ef16e 100644 --- a/environment.py +++ b/environment.py @@ -378,6 +378,60 @@ def env() -> Mapping[str, Optional[str]]: # 'GOOGLE_PROJECT': None, + # The path of the directory where the Google Cloud Python libraries and + # the Google Cloud CLI (gcloud) put their state. If this variable is not + # set, the state is placed in ~/.config/gcloud by default. Since we want + # to segregate this state per working copy and deployment, we set this + # variable to the path of a deployment-specific directory in the working + # copy. Note that this variable does not affect the Google Cloud + # libraries for Go, or the Google Cloud provider for Terraform which + # uses these Go libraries. Luckily, the Go libraries don't write any + # state, they only read credentials from the location configured via + # GOOGLE_APPLICATION_CREDENTIALS below. + # + 'CLOUDSDK_CONFIG': '{project_root}/deployments/.active/.gcloud', + + # The path of a JSON file with credentials for an authorized user or a + # service account. The Google Cloud libraries for Python and Go will + # load the so called ADC (Application-default credentials) from this + # file and use them for any Google Cloud API requests. According to + # Google documentation, if this variable is not set, + # ~/.config/gcloud/application_default_credentials.json is used. + # However, the Google Cloud SDK and Python libraries will only default + # to that if CLOUDSDK_CONFIG is not set. If it is, + # $CLOUDSDK_CONFIG/application_default_credentials.json is used. Since + # the Go libraries are unaffected by CLOUDSDK_CONFIG, the offcially + # documented default applies. We'll work around the inconsistent + # defaults by setting both variables explicitly. + # + # If the azul_google_user variable is set, the _preauth helper defined + # in `environment` will populate this file with credentials (a + # long-lived refresh token) for that user. It does so by logging that + # user into Google Cloud, which requires a web browser. As a + # convenience, and to avoid confusion, it will, at the same time, + # provision credentials for the Google Cloud CLI, in a different file, + # but also in the directory configured via CLOUDSDK_CONFIG above. + # + # To have a service account (as opposed to a user account) manage Google + # Cloud resources, leave azul_google_user unset and change this variable + # to point to a file with the private key of that service account. Note + # that the service account would have to be different from the one whose + # name is set in AZUL_GOOGLE_SERVICE_ACCOUNT. In fact, the service + # account from this variable is used to manage those other service + # accounts, so generally, it needs elevated permissions to the project. + # We used to call this type of account "personal service account" but we + # don't use that type anymore. GitLab is nowadays the only place where + # this variable is set to service account credentials. + # + 'GOOGLE_APPLICATION_CREDENTIALS': '{CLOUDSDK_CONFIG}/application_default_credentials.json', + + # The name of a Google user account with authorization to manage the + # Google Cloud resources in the project referred to by GOOGLE_PROJECT. + # If this variable is not set, GOOGLE_APPLICATION_CREDENTIALS must be + # changed to the path of a file containing service account credentials. + # + 'azul_google_user': None, + # The name of the Google Cloud service account to represent the # deployment. This service account will be created automatically during # deployment and will then be used to access (meta)data in Google-based diff --git a/scripts/envhook.py b/scripts/envhook.py index 1cf79eb440..122cbee7ab 100755 --- a/scripts/envhook.py +++ b/scripts/envhook.py @@ -207,7 +207,7 @@ def share_aws_cli_credential_cache(self): boto3.setup_default_session(botocore_session=session) if self.pycharm_hosted: - # The equivalent of the _preauth function in `environment` + # The equivalent of the _preauth_aws function in `environment` credentials = session.get_credentials() self.setenv(dict(AWS_ACCESS_KEY_ID=credentials.access_key, AWS_SECRET_ACCESS_KEY=credentials.secret_key, From b2a1a6ba7596ef6caff26723edcd530c3a7fa141 Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Mon, 3 Jul 2023 22:45:18 -0700 Subject: [PATCH 5/6] [u] Rename _preauth to _login, _deauth to _logout --- OPERATOR.rst | 4 +- README.md | 4 +- UPGRADING.rst | 4 ++ environment | 70 +++++++++++++++--------------- environment.py | 4 +- scripts/envhook.py | 2 +- terraform/gitlab/runner/Dockerfile | 2 +- 7 files changed, 47 insertions(+), 43 deletions(-) diff --git a/OPERATOR.rst b/OPERATOR.rst index 6836d6c277..724918c7f8 100644 --- a/OPERATOR.rst +++ b/OPERATOR.rst @@ -529,9 +529,9 @@ Credentials expire in the middle of a long-running operation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In some instances, deploying a Terraform component can take a long time. While -``_preauth`` now makes sure that there are four hours left on the current +``_login`` now makes sure that there are four hours left on the current credentials, it can't do that if you don't call it before such an operation. -Note that ``_select`` also calls ``_preauth``. The following is a list of +Note that ``_select`` also calls ``_login``. The following is a list of operations which you should expect to take an hour or longer: - the first time deploying any component diff --git a/README.md b/README.md index b882c1b213..10b47d2977 100644 --- a/README.md +++ b/README.md @@ -254,7 +254,7 @@ You should have been issued AWS credentials. Typically, those credentials require assuming a role in an account other than the one defining your IAM user. Just set that up normally in `~/.aws/config` and `~/.aws/credentials`. If the assumed role additionally requires an MFA token, you should run -`_preauth` immediately after running `source environment` or switching +`_login` immediately after running `source environment` or switching deployments with `_select`. @@ -932,7 +932,7 @@ making it harder to recover. `terraform apply` was running. To fix, run … ``` -_preauth +_login (cd terraform && terraform state push errored.tfstate) ``` diff --git a/UPGRADING.rst b/UPGRADING.rst index 0a2d4e3c49..5967357d8e 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -27,6 +27,10 @@ Set the environment variable ``azul_google_user`` in all deployments to your ``…@ucsc.edu`` email address. The easiest way to do that is in an ``environment.local.py`` at the project root. +Many of the shell functions defined in ``environment`` have been renamed. To +avoid stale copies of these functions lingering around under their old names, +exit all shells in which you sourced that file. + #5325 Exclude noisy events from api_unauthorized alarm ====================================================== diff --git a/environment b/environment index 40ae66b8f4..9313d823ee 100644 --- a/environment +++ b/environment @@ -19,10 +19,10 @@ _select() { else if ! { _validate_link "$1" && - _deauth && + _logout && _link "$1" && _refresh && - _preauth ; + _login ; } ; then echo >&2 "_select failed" return 1 @@ -32,7 +32,7 @@ _select() { _deselect() { if ! { - _deauth && + _logout && _unlink && _refresh ; } ; then @@ -70,14 +70,14 @@ _show_link() { ( cd "${project_root}/deployments" && ls -l .active ) } -_preauth_google() { +_login_google() { if [ -n "$azul_google_user" ] ; then if ! { gcloud auth login --update-adc --quiet "$azul_google_user" && gcloud config set project "$GOOGLE_PROJECT" && gcloud auth application-default set-quota-project "$GOOGLE_PROJECT" ; } ; then - echo >&2 "_preauth_google failed" + echo >&2 "_login_google failed" return 1 fi fi @@ -89,7 +89,7 @@ _preauth_google() { # # https://github.com/boto/boto3/issues/1179#issuecomment-569940530 # -_preauth_aws() { +_login_aws() { local env if ! env="$( python - <<- "EOF" @@ -125,13 +125,13 @@ _preauth_aws() { print(f'export AWS_SESSION_TOKEN={credentials.token}') EOF )" ; then - echo >&2 "_preauth_aws failed" + echo >&2 "_login_aws failed" return 1 fi eval "$env" } -_preauth_docker_ecr() { +_login_docker_ecr() { if [ -n "${azul_docker_registry:+x}" ] ; then if ! ( set -o pipefail @@ -141,13 +141,13 @@ _preauth_docker_ecr() { --password-stdin \ "${azul_docker_registry%/}" ) ; then - echo >&2 "_preauth_docker_ecr failed" + echo >&2 "_login_docker_ecr failed" return 1 fi fi } -_preauth_docker_gitlab() { +_login_docker_gitlab() { if { [ -n "${azul_gitlab_access_token:+x}" ] && [ -n "${azul_gitlab_user:+x}" ] ; @@ -160,89 +160,89 @@ _preauth_docker_gitlab() { --password-stdin \ "docker.gitlab.${AZUL_DOMAIN_NAME}" ) ; then - echo >&2 "_preauth_docker_gitlab failed" + echo >&2 "_login_docker_gitlab failed" return 1 fi fi } -_preauth() { +_login() { if { - _preauth_google && - _preauth_aws && - _preauth_docker_ecr && - _preauth_docker_gitlab ; + _login_google && + _login_aws && + _login_docker_ecr && + _login_docker_gitlab ; } ; then echo >&2 \ - "Session credentials are in effect for AWS. Additionally, you have " \ - "been logged into Google Cloud, Amazon ECR and the GitLab Docker " \ - "registry. Use '_deauth' to invalidate the session credentials. " \ - "Alternatively, you can use _deauth_completely to invalidate all " \ + "Session credentials are in effect for AWS. Additionally, you have" \ + "been logged into Google Cloud, Amazon ECR and the GitLab Docker" \ + "registry. Use '_logout' to invalidate the session credentials." \ + "Alternatively, you can use _logout_completely to invalidate all" \ "credentials but this is usually not necessary." else - echo >&2 "_preauth failed" + echo >&2 "_login failed" return 1 fi } -_deauth_docker_ecr() { +_logout_docker_ecr() { if [ -n "${azul_docker_registry:+x}" ] ; then if ! { docker logout "${azul_docker_registry}" ; } ; then - echo >&2 "_deauth_docker_ecr failed" + echo >&2 "_logout_docker_ecr failed" return 1 fi fi } -_deauth_docker_gitlab() { +_logout_docker_gitlab() { if { [ -n "${azul_gitlab_access_token:+x}" ] && [ -n "${azul_gitlab_user:+x}" ] ; } ; then if ! docker logout "docker.gitlab.${AZUL_DOMAIN_NAME}" ; then - echo >&2 "_deauth_docker_gitlab failed" + echo >&2 "_logout_docker_gitlab failed" return 1 fi fi } -_deauth_aws() { +_logout_aws() { unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN } -_deauth_google() { +_logout_google() { if [ -n "$azul_google_user" ] ; then if ! { gcloud auth application-default revoke --quiet && gcloud auth revoke --quiet ; } ; then - echo >&2 "_deauth_google failed" + echo >&2 "_logout_google failed" return 1 fi fi } -_deauth() { +_logout() { # Docker segregates credential state by registry and we maintain separate # registries (both ECR and GitLab) per deployment so we won't need to log out # of those registries when switching deployments. Above, we offer dedicated # functons for explicitly logging our of those registries. - _deauth_aws + _logout_aws # We segregate Google state by deployment and working copy (see # CLOUDSDK_CONFIG in environment.py) so we don't need to log out of Google # when switching deployments. Above we offer a dedicated function for # explicitly logging out of Google. } -_deauth_completely() { +_logout_completely() { # We don't use `&&` between function invocations because failing to log out of # one realm shouldn't prevent us from attempting to log out of the others. - _deauth_google - _deauth - _deauth_docker_ecr - _deauth_docker_gitlab + _logout_google + _logout + _logout_docker_ecr + _logout_docker_gitlab } _revenv() { diff --git a/environment.py b/environment.py index acc80ef16e..82e1fe1b30 100644 --- a/environment.py +++ b/environment.py @@ -403,8 +403,8 @@ def env() -> Mapping[str, Optional[str]]: # the Go libraries are unaffected by CLOUDSDK_CONFIG, the offcially # documented default applies. We'll work around the inconsistent # defaults by setting both variables explicitly. - # - # If the azul_google_user variable is set, the _preauth helper defined + # + # If the azul_google_user variable is set, the _login helper defined # in `environment` will populate this file with credentials (a # long-lived refresh token) for that user. It does so by logging that # user into Google Cloud, which requires a web browser. As a diff --git a/scripts/envhook.py b/scripts/envhook.py index 122cbee7ab..2797c1e441 100755 --- a/scripts/envhook.py +++ b/scripts/envhook.py @@ -207,7 +207,7 @@ def share_aws_cli_credential_cache(self): boto3.setup_default_session(botocore_session=session) if self.pycharm_hosted: - # The equivalent of the _preauth_aws function in `environment` + # The equivalent of the _login_aws function in `environment` credentials = session.get_credentials() self.setenv(dict(AWS_ACCESS_KEY_ID=credentials.access_key, AWS_SECRET_ACCESS_KEY=credentials.secret_key, diff --git a/terraform/gitlab/runner/Dockerfile b/terraform/gitlab/runner/Dockerfile index e5e6a43097..f5d67db282 100644 --- a/terraform/gitlab/runner/Dockerfile +++ b/terraform/gitlab/runner/Dockerfile @@ -33,7 +33,7 @@ RUN mkdir -p ${HOME}/.docker \ # # Run # -# $ _preauth +# $ _login # # You should now be logged into the GitLab docker registry. # From f88ada9bebd7061bc8b4f641c257efa8d67ff72a Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Mon, 3 Jul 2023 22:49:43 -0700 Subject: [PATCH 6/6] Rearrange functions in `environment` --- environment | 148 ++++++++++++++++++++++++++-------------------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/environment b/environment index 9313d823ee..d8c79953fc 100644 --- a/environment +++ b/environment @@ -41,6 +41,19 @@ _deselect() { fi } +_show_link() { + ( cd "${project_root}/deployments" && ls -l .active ) +} + +_validate_link() { + d="${project_root}/deployments/$1" + # -d dereferences symlinks so with -L we make sure the argument isn't one + if ! { [ ! -L "$d" ] && [ -d "$d" ] ; } ; then + echo >&2 "_validate_link failed: '$1'" + return 1 + fi +} + _link() { if ! ( _validate_link "$1" && @@ -57,17 +70,44 @@ _unlink() { rm "${project_root}/deployments/.active" } -_validate_link() { - d="${project_root}/deployments/$1" - # -d dereferences symlinks so with -L we make sure the argument isn't one - if ! { [ ! -L "$d" ] && [ -d "$d" ] ; } ; then - echo >&2 "_validate_link failed: '$1'" +_login() { + if { + _login_google && + _login_aws && + _login_docker_ecr && + _login_docker_gitlab ; + } ; then + echo >&2 \ + "Session credentials are in effect for AWS. Additionally, you have" \ + "been logged into Google Cloud, Amazon ECR and the GitLab Docker" \ + "registry. Use '_logout' to invalidate the session credentials." \ + "Alternatively, you can use _logout_completely to invalidate all" \ + "credentials but this is usually not necessary." + else + echo >&2 "_login failed" return 1 fi } -_show_link() { - ( cd "${project_root}/deployments" && ls -l .active ) +_logout() { + # Docker segregates credential state by registry and we maintain separate + # registries (both ECR and GitLab) per deployment so we won't need to log out + # of those registries when switching deployments. Above, we offer dedicated + # functons for explicitly logging our of those registries. + _logout_aws + # We segregate Google state by deployment and working copy (see + # CLOUDSDK_CONFIG in environment.py) so we don't need to log out of Google + # when switching deployments. Above we offer a dedicated function for + # explicitly logging out of Google. +} + +_logout_completely() { + # We don't use `&&` between function invocations because failing to log out of + # one realm shouldn't prevent us from attempting to log out of the others. + _logout_google + _logout + _logout_docker_ecr + _logout_docker_gitlab } _login_google() { @@ -83,6 +123,18 @@ _login_google() { fi } +_logout_google() { + if [ -n "$azul_google_user" ] ; then + if ! { + gcloud auth application-default revoke --quiet && + gcloud auth revoke --quiet ; + } ; then + echo >&2 "_logout_google failed" + return 1 + fi + fi +} + # Get temporary credentials from STS via AssumeRole and inject them # into the current environment where other AWS client libraries can # find them. @@ -131,6 +183,10 @@ _login_aws() { eval "$env" } +_logout_aws() { + unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN +} + _login_docker_ecr() { if [ -n "${azul_docker_registry:+x}" ] ; then if ! ( @@ -147,6 +203,17 @@ _login_docker_ecr() { fi } +_logout_docker_ecr() { + if [ -n "${azul_docker_registry:+x}" ] ; then + if ! { + docker logout "${azul_docker_registry}" ; + } ; then + echo >&2 "_logout_docker_ecr failed" + return 1 + fi + fi +} + _login_docker_gitlab() { if { [ -n "${azul_gitlab_access_token:+x}" ] && @@ -166,36 +233,6 @@ _login_docker_gitlab() { fi } -_login() { - if { - _login_google && - _login_aws && - _login_docker_ecr && - _login_docker_gitlab ; - } ; then - echo >&2 \ - "Session credentials are in effect for AWS. Additionally, you have" \ - "been logged into Google Cloud, Amazon ECR and the GitLab Docker" \ - "registry. Use '_logout' to invalidate the session credentials." \ - "Alternatively, you can use _logout_completely to invalidate all" \ - "credentials but this is usually not necessary." - else - echo >&2 "_login failed" - return 1 - fi -} - -_logout_docker_ecr() { - if [ -n "${azul_docker_registry:+x}" ] ; then - if ! { - docker logout "${azul_docker_registry}" ; - } ; then - echo >&2 "_logout_docker_ecr failed" - return 1 - fi - fi -} - _logout_docker_gitlab() { if { [ -n "${azul_gitlab_access_token:+x}" ] && @@ -208,43 +245,6 @@ _logout_docker_gitlab() { fi } -_logout_aws() { - unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN -} - -_logout_google() { - if [ -n "$azul_google_user" ] ; then - if ! { - gcloud auth application-default revoke --quiet && - gcloud auth revoke --quiet ; - } ; then - echo >&2 "_logout_google failed" - return 1 - fi - fi -} - -_logout() { - # Docker segregates credential state by registry and we maintain separate - # registries (both ECR and GitLab) per deployment so we won't need to log out - # of those registries when switching deployments. Above, we offer dedicated - # functons for explicitly logging our of those registries. - _logout_aws - # We segregate Google state by deployment and working copy (see - # CLOUDSDK_CONFIG in environment.py) so we don't need to log out of Google - # when switching deployments. Above we offer a dedicated function for - # explicitly logging out of Google. -} - -_logout_completely() { - # We don't use `&&` between function invocations because failing to log out of - # one realm shouldn't prevent us from attempting to log out of the others. - _logout_google - _logout - _logout_docker_ecr - _logout_docker_gitlab -} - _revenv() { if ! { deactivate &&