feat: added serverless yml files that can be copied or directly used …

…to support config-cache (if wanted/needed)

xcon/serverless_files/cache-permissions.yml

@@ -0,0 +1,45 @@
+# You can include this file under the `resources:` section of your serverless file,
+# it will add permissions to your lambda functions to access the cache-table,
+# and limits the config variables it can see via the dynamo-hash-key.
+#
+# The dynamo hash-key is the app/service + stage name (ie: `/${self:service}/${self:provider.stage}`)
+
+# Allow config library in xynlib and new py-xyn-config to read SSM/Secrets Manager and Dynamo values
+Resources:
+  xconCacheTableAppPolicy:
+    Type: "AWS::IAM::Policy"
+    Properties:
+      PolicyName: ${self:service}-${self:provider.stage}-xconCacheTableAppPolicy
+      Roles:
+        - !Ref IamRoleLambdaExecution
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: "Allow"
+            Action:
+              - "dynamodb:DescribeTable"
+              - "dynamodb:GetItem"
+              - "dynamodb:Query"
+              - "dynamodb:ConditionCheck"
+              - "dynamodb:PutItem"
+              - "dynamodb:BatchWriteItem"
+            Resource:
+              - Fn::Join:
+                  - ":"
+                  - - "arn:aws:dynamodb"
+                    - Ref: "AWS::Region"
+                    - Ref: "AWS::AccountId"
+                    - "table/global-config"
+              - Fn::Join:
+                  - ":"
+                  - - "arn:aws:dynamodb"
+                    - Ref: "AWS::Region"
+                    - Ref: "AWS::AccountId"
+                    - "table/global-configCache"
+            Condition:
+              ForAllValues:StringEquals:
+                dynamodb:LeadingKeys:
+                  - "/${self:service}/${self:provider.stage}"
+                  - "/${self:service}"
+                  - "/global/${self:provider.stage}"
+                  - "/global"

xcon/serverless_files/config_manager/cache-table.yml

@@ -0,0 +1,42 @@
+# Notes:
+#
+# You can include this file under the `resources:` section of your serverless file.
+#
+# This creates a table.
+#
+# You could also copy this into your project instead of directly referencing
+# this file if you wanted to adjust the table parameters to your liking. I guarantee the table
+# hash/range/ttl names will not change and so you should never have to recreate/change the table
+# in the future.
+#
+# The table hash-key is the app/service + stage name (ie: `/${self:service}/${self:provider.stage}`),
+# This is what allows the permissions to be enforced so apps/services can't see other app/service config values.
+#
+# The range-key includes the config-value name, along with other information on which providers and directory-paths
+# were used to originally look up the config-value, that way the cache will accurately reflect config values no
+# mater how they were looked up dynamically at run-time.
+#
+# For the table name, for looks for param `xconConfigCacheTableName`, if not found then uses `account-all-configCache`,
+# ('account-all', as in aws account-wide config cache table, for all stages/environments)
+
+Resources:
+  xconConfigCacheTable:
+    Type: AWS::DynamoDB::Table
+    Properties:
+      TableName: ${param:xconConfigCacheTableName, 'account-all-configCache'}
+      AttributeDefinitions:
+        - AttributeName: app_key
+          AttributeType: S
+        - AttributeName: name_key
+          AttributeType: S
+      KeySchema:
+        - AttributeName: app_key
+          KeyType: HASH
+        - AttributeName: name_key
+          KeyType: RANGE
+      BillingMode: PAY_PER_REQUEST
+      TimeToLiveSpecification:
+        AttributeName: ttl
+        Enabled: True
+      SSESpecification:
+        SSEEnabled: True

xcon/serverless_files/config_manager/change_handler.py

@@ -0,0 +1,95 @@
+from logging import getLogger
+
+log = getLogger(__name__)
+
+
+def ssm_or_secrets_change_event(event, context):
+    """
+    Here are examples of locations of the various paths from the various events we could get:
+
+    detail->name->/auth/dev/db_port
+    detail->responseElements->name->/test/prod/testing-experiment-2
+    detail->requestParameters->name->/test/prod/testing-experiment-2
+
+    detail->requestParameters->secretId->arn:aws:secretsmanager:
+      us-east-1:972731226928:secret:/test/prod/testing-experiment-2-Pwqh7v
+
+    detail->requestParameters->secretId->/test/dev/testing-experiment-2
+    """
+    detail = get_or_blank_dict(event, 'detail')
+    path = detail.get('name')
+
+    if not path:
+        path = get_or_blank_dict(detail, 'responseElements').get('name')
+
+    if not path:
+        path = get_or_blank_dict(detail, 'requestParameters').get('name')
+
+    if not path:
+        path = get_or_blank_dict(detail, 'requestParameters').get('secretId')
+
+    if not path:
+        raise AttributeError(
+            f"Could not find attribute with a path for event {event}.",
+        )
+
+    if ':' in path:
+        # Paths should NEVER have a colon in them,
+        # so this means we have a value that is in this format (all one line):
+        #
+        # arn:aws:secretsmanager:us-east-1:972731226928
+        #   :secret:/test/joshorr/testing-experiment-2-Pwqh7v
+
+        # This will extract the part of the ARN that is the path we care about.
+        path = '-'.join(path.split(':')[-1].split('-')[0:-1])
+
+    path_components = path.split('/')
+
+    if len(path_components) <= 1:
+        raise ValueError(
+            f"Path ({path}) in event did not have at least two path components, "
+            f"it instead had ({len(path_components)}; must have a directory and a var-name. "
+            f"If it turns out we do have SSM/Secrets like this we want to keep then turn this "
+            f"error into a warning instead.",
+        )
+
+    directory = '/'.join(path_components[0:-1])
+    var_name = str(path_components[-1])
+
+    query = {
+        'real_name': var_name.lower(),  # names are always lower-case in config cache.
+        'real_directory': ['/_nonExistent', directory]  # Directories keep their case.
+    }
+
+    log.info(
+        f"From source ({event.get('source')}), "
+        f"got a change event for path ({path}); "
+        f"will query cache table with ({query}); "
+        f"via event ({event}).",
+        extra={'event': event, 'query': query, 'path': path}
+    )
+
+    # todo: Copy the query-boto-structure out of library for get/delete calls below.
+
+    # items = ConfigCacheItem.api.get(query, allow_scan=True)
+    #
+    # items = list(items)
+    # log.info(f'Deleting cached items: ({items})')
+    # ConfigCacheItem.api.client.delete_objs(items)
+
+
+def get_or_blank_dict(dict_value, key):
+    if not dict_value:
+        return {}
+
+    if not isinstance(dict_value, dict):
+        return {}
+
+    value = dict_value.get(key, None)
+    if not value:
+        return {}
+
+    if not isinstance(value, dict):
+        return {}
+    return value
+

xcon/serverless_files/config_manager/change_handler.yml

@@ -0,0 +1,28 @@
+xconConfigChangeHandler:
+  handler: xcon/serverless_resources/change_handler.ssm_or_secrets_change_event
+  events:
+    - cloudwatchEvent:
+        event:
+          source:
+            - aws.ssm
+          detail-type:
+            - Parameter Store Change
+          detail:
+            operation:
+              - Create
+              - Update
+              - Delete
+    - cloudwatchEvent:
+        event:
+          source:
+            - aws.secretsmanager
+          detail-type:
+            - AWS API Call via CloudTrail
+          detail:
+            eventSource:
+              - secretsmanager.amazonaws.com
+            eventName:
+              - CreateSecret
+              - UpdateSecret
+              - DeleteSecret
+              - PutSecretValue

xcon/serverless_files/config_manager/config_cache_all_access.yml

@@ -0,0 +1,32 @@
+Resources:
+  configCacheTableAllAccessRole:
+    Type: "AWS::IAM::Policy"
+    Properties:
+      PolicyName: ${self:service}-${self:provider.stage}-configCacheTableAllAccessRole
+      Roles:
+        - !Ref IamRoleLambdaExecution
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: "Allow"
+            Action:
+              - "dynamodb:DescribeTable"
+              - "dynamodb:GetItem"
+              - "dynamodb:Query"
+              - "dynamodb:ConditionCheck"
+              - "dynamodb:PutItem"
+              - "dynamodb:BatchGetItem"
+              - "dynamodb:BatchWriteItem"
+              - "dynamodb:DeleteItem"
+              - "dynamodb:Scan"
+              - "dynamodb:UpdateItem"
+              - "dynamodb:UpdateTimeToLive"
+              - "dynamodb:Scan"
+              - "dynamodb:ConditionCheckItem"
+            Resource:
+              - Fn::Join:
+                  - ":"
+                  - - "arn:aws:dynamodb"
+                    - Ref: "AWS::Region"
+                    - Ref: "AWS::AccountId"
+                    - "table/${param:xconConfigCacheTableName, 'account-all-configCache'}"

xcon/serverless_files/secrets-permissions.yml

@@ -0,0 +1,60 @@
+# Notes:
+#
+# You can include this file under the `resources:` section of your serverless file.
+#
+# It will add permissions to your lambda functions to access the secrets manager,
+# and limits the config variables it can see via the standard directory paths.
+#
+# If you want to use alternate directory paths, you can take a copy of this file and
+# adjust the paths as needed.
+
+Resources:
+  xconSecretsManagerAppAccessPolicy:
+    Type: AWS::IAM::Policy
+    Properties:
+      PolicyName: ${self:service}-${self:provider.stage}-xconSecretsManagerAppAccessPolicy
+      Roles:
+        - !Ref IamRoleLambdaExecution
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: Allow
+            Action:
+              - secretsmanager:GetResourcePolicy
+              - secretsmanager:GetSecretValue
+              - secretsmanager:DescribeSecret
+              - secretsmanager:ListSecretVersionIds
+            Resource:
+              - Fn::Join:
+                  - ":"
+                  - - "arn:aws:secretsmanager"
+                    - Ref: "AWS::Region"
+                    - Ref: "AWS::AccountId"
+                    - "secret"
+                    - "/${self:service}/${self:provider.stage}/*"
+              - Fn::Join:
+                  - ":"
+                  - - "arn:aws:secretsmanager"
+                    - Ref: "AWS::Region"
+                    - Ref: "AWS::AccountId"
+                    - "secret"
+                    - "/${self:service}/all/*"
+              - Fn::Join:
+                  - ":"
+                  - - "arn:aws:secretsmanager"
+                    - Ref: "AWS::Region"
+                    - Ref: "AWS::AccountId"
+                    - "secret"
+                    - "/global/${self:provider.stage}/*"
+              - Fn::Join:
+                  - ":"
+                  - - "arn:aws:secretsmanager"
+                    - Ref: "AWS::Region"
+                    - Ref: "AWS::AccountId"
+                    - "secret"
+                    - "/global/all/*"
+          - Effect: "Allow"
+            Action:
+              - secretsmanager:ListSecrets
+            Resource:
+              - "*"

xcon/serverless_files/ssm-permissions.yml

@@ -0,0 +1,49 @@
+# Notes:
+#
+# You can include this file under the `resources:` section of your serverless file.
+#
+# It will add permissions to your lambda functions to access the ssm param store,
+# and limits the config variables it can see via the standard directory paths.
+#
+# If you want to use alternate directory paths, you can take a copy of this file and
+# adjust the paths as needed.
+
+Resources:
+  xconSsmAppAccessPolicy:
+    Type: "AWS::IAM::Policy"
+    Properties:
+      PolicyName: ${self:service}-${self:provider.stage}-xconSsmAppAccessPolicy
+      Roles:
+        - !Ref IamRoleLambdaExecution
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: "Allow"
+            Action:
+              - "ssm:GetParametersByPath"
+              - "kms:Decrypt"
+            Resource:
+              - Fn::Join:
+                  - ":"
+                  - - "arn:aws:ssm"
+                    - Ref: "AWS::Region"
+                    - Ref: "AWS::AccountId"
+                    - "parameter/${self:service}/${self:provider.stage}"
+              - Fn::Join:
+                  - ":"
+                  - - "arn:aws:ssm"
+                    - Ref: "AWS::Region"
+                    - Ref: "AWS::AccountId"
+                    - "parameter/${self:service}"
+              - Fn::Join:
+                  - ":"
+                  - - "arn:aws:ssm"
+                    - Ref: "AWS::Region"
+                    - Ref: "AWS::AccountId"
+                    - "parameter/global/${self:provider.stage}"
+              - Fn::Join:
+                  - ":"
+                  - - "arn:aws:ssm"
+                    - Ref: "AWS::Region"
+                    - Ref: "AWS::AccountId"
+                    - "parameter/global"

xcon/serverless_files/xcon-resources.js

@@ -0,0 +1,23 @@
+module.exports = async ({ options, resolveVariable }) => {
+    const execSync = require('child_process').execSync;
+
+    // import { execSync } from 'child_process';  // replace ^ if using ES modules
+
+    // the default is 'buffer'
+    // Assumes project is using `poetry` to manage dependencies + python virutal environment.
+    let command = 'poetry run -q python -c "import os; from xcon import serverless_files; print(f\'{os.path.dirname(serverless_files.__file__)}\', end=\'\');"'
+    let output = execSync(command, { encoding: 'utf-8' });
+
+    // Use can simply do this to include a resource file:
+    // (copy xcon-resource.js into project):
+
+    // # *** file: serverless.yml ***
+    //
+    // custom:
+    //   xconResourcePath: ${file(./xcon-resources.js)}
+    //
+    // resources:
+    //   - ${file(${self:custom.xconResourcePath}/cache-permissions.yml)}
+
+    return `${output}`
+}