[R u] Automate creation of FedRAMP inventory (#3634, PR #4988)

.coveragerc

@@ -4,6 +4,8 @@ source =
     src
     lambdas/indexer
     lambdas/service
+omit =
+    src/azul/compliance/fedramp_inventory_service.py
 [report]
 # Regexes for lines to exclude from consideration
 exclude_lines =

.gitignore

@@ -4,6 +4,7 @@ __pycache__/
 # Python coverage
 /.coverage
 /.coverage.*
+/fedramp_inventory.xlsx
 
 ## Build products above this line. Please see the `clean` target in Makefile.
 

.gitlab-ci.yml

@@ -1,3 +1,13 @@
+workflow:
+  name: $azul_gitlab_pipeline_name
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'push'
+      variables:
+        azul_gitlab_pipeline_name: 'Push to $CI_COMMIT_BRANCH'
+    - if: $CI_PIPELINE_SOURCE == 'schedule'
+      variables:
+        azul_gitlab_pipeline_name: 'Scheduled $azul_gitlab_schedule'
+
 variables:
   DOCKER_IMAGE: $CI_REGISTRY_IMAGE
   DOCKER_TAG: $CI_PIPELINE_ID
@@ -10,23 +20,30 @@ stages:
   - early_reindex
   - integration_test
   - teardown
-  - manual
-  - sell_unused_slots
+  - reindex
+  - schedule
+
+.on_branch_push:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH
 
 build_image:
   # Build an image containing the build prerequisites (python, terraform, etc).
   # This lets us exploit Docker's layer caching to speed up the build. The
   # image will only be rebuilt after changes to the Dockerfile, 
   # requirements*.txt, common.mk and the Makefile.
+  extends: .on_branch_push
   stage: build_image
+  rules:
+    - if: $CI_COMMIT_BRANCH
   script:
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY_IMAGE
     - make docker_dev_push .gitlab.env
   artifacts:
     reports:
       dotenv: .gitlab.env
 
-.base:
+.needs_image:
   image: $BUILD_IMAGE
   before_script:
     - cp -vR /etc/gitlab/azul/* . # Copy files like environment.local into the build directory.
@@ -41,13 +58,14 @@ build_image:
   dependencies:
     - build_image
 
+.base:
+  extends: [.on_branch_push, .needs_image]
+
 setup:
   extends: .base
   stage: setup
   script:
     - python scripts/github_status_check.py "${status_context}" pending
-  except:
-    - schedules
 
 test:
   extends: .base
@@ -59,17 +77,13 @@ test:
     - make check_clean  # would dirty up the working copy and fail the build.
     - make pep8
     - AZUL_DEBUG=0 make test
-  except:
-    - schedules
 
 deploy:
   extends: .base
   stage: deploy
   script:
     - make auto_deploy
     - make create
-  except:
-    - schedules
   artifacts:
     paths:
       - terraform/plan.json
@@ -79,50 +93,54 @@ integration_test:
   stage: integration_test
   script:
     - make integration_test
-  except:
-    - schedules
 
 on_success:
   extends: .base
   stage: teardown
   when: on_success
   script:
     - python scripts/github_status_check.py "${status_context}" success
-  except:
-    - schedules
 
 on_failure:
   extends: .base
   stage: teardown
   when: on_failure
   script:
     - python scripts/github_status_check.py "${status_context}" failure
-  except:
-    - schedules
 
 early_reindex:
   extends: .base
   stage: early_reindex
   when: manual
-  except:
-    - schedules
+  timeout: 24h
   script:
     - make reindex
-  timeout: 24h
 
 reindex:
   extends: .base
-  stage: manual
+  stage: reindex
   when: manual
-  except:
-    - schedules
+  timeout: 24h
   script:
     - make reindex
-  timeout: 24h
 
 sell_unused_slots:
-  extends: .base
-  only:
-    - schedules
+  extends: .needs_image
+  stage: schedule
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'schedule' 
+          && $azul_gitlab_schedule == 'sell_unused_slots'
   script:
     - python3 scripts/sell_unused_slots.py
+
+fedramp_inventory:
+  extends: .needs_image
+  stage: schedule
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'schedule' 
+          && $azul_gitlab_schedule == 'fedramp_inventory'
+  script:
+    - python3 scripts/compliance/fedramp_inventory.py fedramp_inventory.xlsx
+  artifacts:
+    paths:
+      - fedramp_inventory.xlsx

UPGRADING.rst

@@ -19,6 +19,36 @@ branch that does not have the listed changes, the steps would need to be
 reverted. This is all fairly informal and loosely defined. Hopefully we won't
 have too many entries in this file.
 
+
+#3634 Automate creation of a FedRAMP Integrated Inventory Workbook
+==================================================================
+
+Operator
+~~~~~~~~
+
+Manually deploy the ``gitlab`` component of any main deployment just *before*
+pushing the merge commit to the GitLab instance in that deployment.
+
+Afterwards, edit the existing schedule in the Azul project on that GitLab
+instance. Its description is ``Sell unused BigQuery slot commitments``. You may
+need to ask a system administrator to perform make these changes on your behalf.
+
+1) Set the Cron timezone to ``Pacific Time (US & Canada)``
+
+2) Set the variable ``azul_gitlab_schedule`` to ``sell_unused_slots``
+
+
+Add another schedule:
+
+1) Set the description to ``Prepare FedRAMP inventory``
+
+2) Set the interval pattern to ``0 4 * * *``
+
+3) Set the Cron timezone to ``Pacific Time (US & Canada)``
+
+4) Set the variable ``azul_gitlab_schedule`` to ``fedramp_inventory``
+
+
 #5004 Enable access logging on AWS Config bucket
 ================================================
 

requirements.all.txt

@@ -10,7 +10,7 @@ blessed==1.20.0
 boto3==1.24.94
 boto3-stubs==1.24.94
 botocore==1.27.94
-botocore-stubs==1.29.79
+botocore-stubs==1.29.89
 brotli==1.0.9
 cachetools==5.3.0
 certifi==2022.12.7
@@ -22,12 +22,13 @@ click==8.1.3
 colorama==0.4.4
 configargparse==1.5.3
 coverage==6.5.0
-cryptography==39.0.1
+cryptography==39.0.2
 deprecated==1.2.13
 docker==6.0.0
 docutils==0.16
 elasticsearch==7.10.1
 elasticsearch-dsl==7.4.0
+et-xmlfile==1.1.0
 fastavro==1.6.1
 flake8==5.0.4
 flask==2.2.3
@@ -60,6 +61,7 @@ importlib-metadata==6.0.0
 importlib-resources==5.12.0
 inquirer==2.10.1
 itsdangerous==2.1.2
+jdcal==1.4.1
 jinja2==3.1.2
 jmespath==1.0.1
 jq==1.3.0
@@ -71,21 +73,22 @@ markupsafe==2.1.2
 mccabe==0.7.0
 more-itertools==9.0.0
 moto==4.1.4.dev8
-msgpack==1.0.4
+msgpack==1.0.5
 mypy-boto3-dynamodb==1.24.74
 mypy-boto3-iam==1.24.90
 mypy-boto3-lambda==1.24.54
 mypy-boto3-s3==1.24.94
 mypy-boto3-sqs==1.24.60
 openapi-schema-validator==0.3.4
 openapi-spec-validator==0.5.1
+openpyxl==3.0.6
 orderedmultidict==1.0.1
 packaging==23.0
 pathable==0.4.3
 pip==22.2.2
 ply==3.11
 proto-plus==1.22.2
-protobuf==4.22.0
+protobuf==4.22.1
 psutil==5.9.4
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
@@ -100,11 +103,13 @@ pyparsing==3.0.9
 pyrsistent==0.19.3
 python-dateutil==2.8.2
 python-editor==1.0.4
+python-gitlab==3.13.0
 pytz==2022.7.1
 pyyaml==5.4.1
 pyzmq==25.0.0
 readchar==4.0.3
 requests==2.28.1
+requests-toolbelt==0.10.1
 requirements-parser==0.2.0+1
 responses==0.22.0
 roundrobin==0.0.4
@@ -117,7 +122,7 @@ smmap==5.0.0
 strict-rfc3339==0.7
 tinyquery==1.2+2
 toml==0.10.2
-types-awscrt==0.16.10
+types-awscrt==0.16.11
 types-s3transfer==0.6.0.post5
 types-toml==0.10.8.5
 typing_extensions==4.5.0

requirements.dev.trans.txt

@@ -1,11 +1,12 @@
 arrow==1.2.3
 blessed==1.20.0
-botocore-stubs==1.29.79
+botocore-stubs==1.29.89
 brotli==1.0.9
 click==8.1.3
 colorama==0.4.4
 configargparse==1.5.3
 docutils==0.16
+et-xmlfile==1.1.0
 flask==2.2.3
 flask-basicauth==0.2.0
 flask-cors==3.0.10
@@ -18,12 +19,13 @@ importlib-metadata==6.0.0
 importlib-resources==5.12.0
 inquirer==2.10.1
 itsdangerous==2.1.2
+jdcal==1.4.1
 jinja2==3.1.2
 jsonschema==4.17.3
 jsonschema-spec==0.1.3
 lazy-object-proxy==1.9.0
 mccabe==0.7.0
-msgpack==1.0.4
+msgpack==1.0.5
 mypy-boto3-dynamodb==1.24.74
 mypy-boto3-iam==1.24.90
 mypy-boto3-lambda==1.24.54
@@ -42,10 +44,11 @@ pyrsistent==0.19.3
 python-editor==1.0.4
 pyzmq==25.0.0
 readchar==4.0.3
+requests-toolbelt==0.10.1
 roundrobin==0.0.4
 smmap==5.0.0
 toml==0.10.2
-types-awscrt==0.16.10
+types-awscrt==0.16.11
 types-s3transfer==0.6.0.post5
 types-toml==0.10.8.5
 uritemplate==4.1.1

requirements.dev.txt

@@ -14,7 +14,9 @@ locust==2.12.2
 moto[s3,sqs,sns,dynamodb,iam]==4.1.4.dev8  # match the extras with the backends listed in AzulUnitTestCase._reset_moto
 boto3-stubs[s3,sqs,lambda,dynamodb,iam]==1.24.94  # match this with the version of the `boto3` runtime dependency
 openapi-spec-validator==0.5.1
+openpyxl==3.0.6
 pygithub==1.56
+python-gitlab==3.13.0
 pyyaml==5.4.1  # can't upgrade beyond that b/c awscli 1.25.95 depends on PyYAML<5.5 and >=3.10
 git+https://github.com/hannes-ucsc/[email protected]+1#egg=requirements-parser
 responses==0.22.0

requirements.trans.txt

@@ -4,7 +4,7 @@ cachetools==5.3.0
 certifi==2022.12.7
 cffi==1.15.1
 charset-normalizer==2.1.1
-cryptography==39.0.1
+cryptography==39.0.2
 google-cloud-core==2.3.2
 google-crc32c==1.5.0
 google-resumable-media==2.4.1
@@ -17,7 +17,7 @@ markupsafe==2.1.2
 orderedmultidict==1.0.1
 packaging==23.0
 proto-plus==1.22.2
-protobuf==4.22.0
+protobuf==4.22.1
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
 pycparser==2.21