diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..b2ac3eb --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,13 @@ +**Related Issue(s):** + +- # + + +**Proposed Changes:** + +1. +2. + +**PR Checklist:** + +- [ ] I have added my changes to the CHANGELOG **or** a CHANGELOG entry is not required. diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml new file mode 100644 index 0000000..fddd448 --- /dev/null +++ b/.github/workflows/deploy-dev.yml @@ -0,0 +1,19 @@ +name: 🚀 Deploy Dev + +on: + push: + branches: + - main + tags: + - v.* + workflow_dispatch: + +jobs: + deploy-dev: + uses: ./.github/workflows/deploy-wf.yaml + secrets: inherit + concurrency: dev + with: + stage: dev + url: https://console.goose.filmdrop.element84.com + fd-aws-tf-modules-version: v2.29.0 diff --git a/.github/workflows/deploy-wf.yml b/.github/workflows/deploy-wf.yml new file mode 100644 index 0000000..c9c6f66 --- /dev/null +++ b/.github/workflows/deploy-wf.yml @@ -0,0 +1,121 @@ +on: + workflow_call: + inputs: + stage: + required: true + type: string + url: + required: true + type: string + fd-aws-tf-modules-version: + required: true + type: string + +env: + project-name: goose + +permissions: + id-token: write + contents: read + +jobs: + deploy: + environment: + name: ${{ inputs.stage }} + url: ${{ inputs.url }} + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.7.5" + + - name: Preparing Environment + id: prep_env + run: | + echo "Creating terraform backend file ..." + echo '' > config.s3.backend.tf + echo 'terraform {' >> config.s3.backend.tf + echo ' backend "s3" {' >> config.s3.backend.tf + echo ' encrypt = true' >> config.s3.backend.tf + echo " bucket = \"${{ secrets.TF_STATE_BUCKET }}\"" >> config.s3.backend.tf + echo " dynamodb_table = \"${{ secrets.TF_STATE_LOCK_TABLE }}\"" >> config.s3.backend.tf + echo " key = \"${{ env.project-name }}-${{ inputs.stage }}.tfstate\"" >> config.s3.backend.tf + echo " region = \"${{ secrets.AWS_REGION }}\"" >> config.s3.backend.tf + echo ' }' >> config.s3.backend.tf + echo '}' >> config.s3.backend.tf + cat config.s3.backend.tf + echo "Using FilmDrop Terraform ${{ inputs.fd-aws-tf-modules-version }} release..." + ./scripts/retrieve_tf_modules.sh ${{ inputs.fd-aws-tf-modules-version }} + + - name: Update stac-server lambdas + id: update_stac_lambdas + run: ./scripts/update-stac-server-lambdas.bash + + - name: Update cirrus lambda dist + id: update_cirrus_lambda_dist + run: ./scripts/update-cirrus-lambda-dist.bash + + - name: Configure Terraform Init Credentials + id: init_creds + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ secrets.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE }} + role-session-name: GitHubReleaseInit + + - name: Terraform Init + id: tf_init + run: terraform init + + - name: Terraform Validate + id: tf_validate + run: terraform validate + + - name: Configure Terraform Plan Credentials + id: plan_creds + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ secrets.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE }} + role-session-name: GitHubReleasePlan + + - name: Terraform Plan + id: tf_plan + run: terraform plan -var-file="${{ inputs.stage }}.tfvars" -out ${{ inputs.stage }}.tfplan -lock=false + + - name: Configure Terraform Apply Credentials + id: apply_creds + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ secrets.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE }} + role-session-name: GitHubReleaseApply + + - name: Terraform Apply + id: tf_apply + run: terraform apply -lock=false -input=false ${{ inputs.stage }}.tfplan + + - name: Post status to Slack channel + id: tf_apply_successs + if: steps.tf_apply.outcome == 'success' + continue-on-error: true + uses: slackapi/slack-github-action@v1.26.0 + with: + channel-id: ${{ secrets.SLACK_CHANNEL_ID }} + slack-message: ":silly-goose: ${{ env.project-name }}-${{ inputs.stage }}-titiler ${{ github.ref_name }} terraform apply job has succeeded!\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + + - name: Post status to Slack channel + id: tf_apply_failure + if: steps.tf_apply.outcome != 'success' + continue-on-error: true + uses: slackapi/slack-github-action@v1.26.0 + with: + channel-id: ${{ secrets.SLACK_CHANNEL_ID }} + slack-message: ":goosebonk: ${{ env.project-name }}-${{ inputs.stage }}-titiler ${{ github.ref_name }} terraform apply has failed!\n:alert: make sure cleanup job deletes all AWS resources!\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/snyk-scan.yml b/.github/workflows/snyk-scan.yml new file mode 100644 index 0000000..54fe8f9 --- /dev/null +++ b/.github/workflows/snyk-scan.yml @@ -0,0 +1,49 @@ +# This workflow sets up Snyk scans + +name: Snyk Scan + +on: + push: + branches: + - main + pull_request: + branches: + - main + schedule: # Run snyk scan daily at midnight + - cron: '0 0 * * *' + +permissions: + contents: read + +jobs: + snyk: + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.1 + + - name: Snyk IaC report vulnerabilities + uses: snyk/actions/iac@0.4.0 + continue-on-error: true # To make sure that SARIF upload gets called + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + args: --sarif-file-output=snyk.sarif + + # Push the Snyk Code results into GitHub Code Scanning tab + - name: Upload result to GitHub Code Scanning + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: snyk.sarif + + - name: Snyk IaC gatekeeper + uses: snyk/actions/iac@0.4.0 + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + args: + --sarif-file-output=snyk.sarif + --policy-path=.snyk + --severity-threshold=high # Forces fail on high-severity vulnerabilities diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 0000000..c5f145a --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,94 @@ +name: Pre-commit CI + +on: + push: + branches: + - main + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + validate: + permissions: + id-token: write + contents: read + runs-on: ubuntu-latest + env: + CI: true + STAC_SERVER_TAG: v3.7.0 + CIRRUS_TAG: v1.0.0a0 + fd-aws-tf-modules-version: v2.29.0 + stage: dev + project-name: goose + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "18" + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.7.5" + + - name: Preparing Environment + id: prep_env + run: | + echo "Creating terraform backend file ..." + echo '' > config.s3.backend.tf + echo 'terraform {' >> config.s3.backend.tf + echo ' backend "s3" {' >> config.s3.backend.tf + echo ' encrypt = true' >> config.s3.backend.tf + echo " bucket = \"${{ secrets.TF_STATE_BUCKET }}\"" >> config.s3.backend.tf + echo " dynamodb_table = \"${{ secrets.TF_STATE_LOCK_TABLE }}\"" >> config.s3.backend.tf + echo " key = \"${{ env.project-name }}-${{ env.stage }}.tfstate\"" >> config.s3.backend.tf + echo " region = \"${{ secrets.AWS_REGION }}\"" >> config.s3.backend.tf + echo ' }' >> config.s3.backend.tf + echo '}' >> config.s3.backend.tf + cat config.s3.backend.tf + echo "Using FilmDrop Terraform ${{ env.fd-aws-tf-modules-version }} release..." + ./scripts/retrieve_tf_modules.sh ${{ env.fd-aws-tf-modules-version }} + + - name: Update stac-server lambdas + id: update_stac_lambdas + run: ./scripts/update-stac-server-lambdas.bash + + - name: Update cirrus lambda dist + id: update_cirrus_lambda_dist + run: ./scripts/update-cirrus-lambda-dist.bash + + - uses: terraform-linters/setup-tflint@v4 + with: + tflint_version: "v0.49.0" + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install pre-commit + run: pip install pre-commit + + - name: Run pre-commit + run: pre-commit run --all-files + + - name: Configure Terraform Init Credentials + id: init_creds + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ secrets.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE }} + role-session-name: GitHubReleaseInit + + - name: Terraform Init + id: tf_init + run: terraform init + + - name: Terraform Validate + id: tf_validate + run: terraform validate -no-color diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..358c3c5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,34 @@ +*.auto.tfvars +*.env* +*.orig +*.out +*.swo +*.swp +*.tfstate +*.tfstate.* +*.tmp +*.zip +*output +.DS_Store +.opensearch.info +.pytest* +.terraform.lock* +.terraform.lock.hcl +.terraform.tfstate* +.terraform/ +__pycache__ +bin/terraform +modules/jupyterhub-dask-eks/cluster.yaml +modules/jupyterhub-dask-eks/daskhub.yaml +modules/jupyterhub-dask-eks/spec.yaml +modules/jupyterhub-dask-eks/storageclass.yaml +node_modules +package-lock.json +plan.json +plan.out +saved.plan +stac-server-* +stac-server.tgz +terraform-visual-report +terraform.tgz +modules/stac-server/historical-ingest/lambda/package/* diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..59e689e --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,3 @@ +MD024: false +MD013: + line_length: 100 \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..840856d --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +repos: + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.85.0 + hooks: + - id: terraform_fmt + - id: terraform_tflint + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.8 + hooks: + - id: ruff + - id: ruff-format diff --git a/.snyk b/.snyk new file mode 100644 index 0000000..059ea1c --- /dev/null +++ b/.snyk @@ -0,0 +1,61 @@ +# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. +version: v1.25.0 +# ignores vulnerabilities until expiry date; change duration by modifying expiry date +ignore: + SNYK-CC-TF-99: + - 'modules/stac-server/api.tf > resource > aws_api_gateway_method[stac_server_api_gateway_proxy_resource_method] > authorization': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + - 'modules/stac-server/api.tf > resource > aws_api_gateway_method[stac_server_api_gateway_root_method] > authorization': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + SNYK-CC-00250: + - './profiles/core/../stac-server/../../modules/stac-server/api.tf > *': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + - './profiles/core/../stac-server/../../modules/stac-server/api.tf > *': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + - 'modules/stac-server/api.tf > *': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + - 'modules/cirrus/functions/api.tf > *': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + SNYK-CC-00185: + - './profiles/core/../titiler/../../modules/mosaic-titiler/titler-mosaicjson.tf > *': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + - 'modules/mosaic-titiler/titler-mosaicjson.tf > *': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + SNYK-CC-00225: + - '.terraform/modules/local_zone_eks.eks_cluster/main.tf > *': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + - 'modules/eks/eks.tf > *': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + SNYK-CC-00752: + - '.terraform/modules/local_zone_eks.eks_cluster/main.tf > *': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + - 'modules/eks/eks.tf > *': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + - 'modules/cirrus/functions/api.tf > *': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + - 'modules/cirrus/functions/post-batch.tf > *': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + - 'modules/cirrus/functions/pre-batch.tf > *': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + - 'modules/cirrus/functions/process.tf > *': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z + SNYK-CC-00218: + - 'modules/jupyterhub-dask-eks/cleanup/main.tf > *': + reason: Open API - no auth required + created: 2024-10-01T14:35:23.783Z diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..89ac142 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,17 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). + +## unreleased + +### Added + +- Initial release. + +### Changed + +### Fixed + +### Removed diff --git a/LICENSE b/LICENSE index 261eeb9..1b5ec8b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,176 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/README.md b/README.md index 226c9d6..e030799 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ # goose + Cirrus Terraform Example Project diff --git a/bootstrap/README.md b/bootstrap/README.md new file mode 100644 index 0000000..c3415a4 --- /dev/null +++ b/bootstrap/README.md @@ -0,0 +1,70 @@ +# Bootstrap + +- [Bootstrap](#bootstrap) + - [Deployment Role](#deployment-role) + - [Terraform State](#terraform-state) + - [Input Parameters](#input-parameters) + - [Deploy](#deploy) + +This project includes infrastructure resources that must be deployed separately +and prior to any FilmDrop resources. + +## Deployment Role + +The deployment Role is an IAM Role that is assumed when deploying all FilmDrop infrastructure. + +The deployment Role should be created once per AWS Account. + +You can deploy this role through CloudFormation +with the following command: + +```shell +aws cloudformation deploy \ + --stack-name "appFilmDropGooseDeployRoleBootstrap" \ + --template-file bootstrap/deploy_role_cfn.yml \ + --capabilities=CAPABILITY_NAMED_IAM +``` + +## Terraform State + +FilmDrop resources are deployed as [Terraform](https://www.terraform.io/) modules +which require a managed state file as part of the deployment. This project +bootstraps the resources necessary to support the Terraform state file and +backend configuration. + +The Terraform state S3 Bucket and DynamoDB Table should be created once per AWS +Account + Region. + +### Input Parameters + +Input parameters for the Terraform state file resources are found in +[terraform_state_params.json](./terraform_state_params.json) and should be +modified for your project. + +The Terraform state S3 Bucket name must be globally unique - it's recommended to +use a combination or your AWS account alias, deploy region, and a random suffix +in order to ensure uniqueness. For example: + +```text +filmdrop-myProjectAccount-us-west-2-terraform-state-96842 +``` + +The DynamoDB locks table name must be a unique name per AWS account and region. +The table name defaults to `filmdrop-terraform-state-locks`, but can be overridden. + +### Deploy + +You can deploy these resources through CloudFormation with the following command, +substituting the value of `TerraformStateBucketName`. + +```shell +aws cloudformation deploy \ + --stack-name "appFilmDropGooseTerraformStateBootstrap" \ + --template-file bootstrap/terraform_state_cfn.yml \ + --parameter-overrides \ + TerraformStateBucketName=REPLACEME +``` + +Once deployed you'll need the S3 Bucket name and DynamoDB table name to configure +the Terraform backend. You'll add those values to the FilmDrop infrastructure +project later. diff --git a/bootstrap/deploy_role_cfn.yml b/bootstrap/deploy_role_cfn.yml new file mode 100644 index 0000000..75edd8c --- /dev/null +++ b/bootstrap/deploy_role_cfn.yml @@ -0,0 +1,139 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: "Role for Deploy runner" + +Parameters: + DeployRoleName: + Type: String + Default: "appFilmDropGooseDeployRole" + DeployRolePolicyName: + Type: String + Default: "appFilmDropGooseDeployPolicy" + +Resources: + DeployRole: + Type: "AWS::IAM::Role" + Properties: + RoleName: !Ref DeployRoleName + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Federated: + - !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/token.actions.githubusercontent.com" + Action: + - "sts:AssumeRoleWithWebIdentity" + Condition: + StringLike: + "token.actions.githubusercontent.com:sub": "repo:Element84/goose:*" + "ForAllValues:StringEquals": + "token.actions.githubusercontent.com:iss": "https://token.actions.githubusercontent.com" + "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" + + DeployRolePolicy: + Type: "AWS::IAM::Policy" + Properties: + PolicyName: !Ref DeployRolePolicyName + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - "acm:*" + - "autoscaling:*" + - "waf:*" + - "codebuild:*" + - "kms:*" + - "iam:CreatePolicyVersion" + - "iam:CreateOpenIDConnectProvider" + - "iam:DeleteOpenIDConnectProvider" + - "iam:GetOpenIDConnectProvider" + - "lambda:InvokeFunction" + - "iam:Tag*" + - "sns:ListTagsForResource" + - "sns:GetSubscriptionAttributes" + - "sns:GetTopicAttributes" + - "sns:CreateTopic" + - "sns:Subscribe" + - "sns:DeleteTopic" + - "sns:SetTopicAttributes" + - "sns:Unsubscribe" + - "sns:SetSubscriptionAttributes" + - "ssm:PutParameter" + - "ssm:DeleteParameter" + - "iam:ListRolePolicies" + - "iam:GetPolicyVersion" + - "firehose:CreateDeliveryStream" + - "firehose:DeleteDeliveryStream" + - "firehose:DescribeDeliveryStream" + - "firehose:ListTagsForDeliveryStream" + - "firehose:TagDeliveryStream" + - "firehose:UpdateDestination" + - "iam:ListAttachedRolePolicies" + - "iam:ListInstanceProfilesForRole" + - "iam:ListPolicyVersions" + - "iam:DeletePolicy" + - "iam:DeletePolicyVersion" + - "iam:CreateServiceLinkedRole" + - "ssm:AddTagsToResource" + - "ssm:Get*" + - "ssm:Describe*" + - "ssm:List*" + - "s3:*" + - "logs:*" + - "ec2:*" + - "sqs:*" + - "lambda:*" + - "apigateway:*" + - "cognito-idp:*" + - "dynamodb:*" + - "wafv2:*" + - "cloudfront:*" + - "cloudformation:*" + - "route53:*" + - "events:*" + - "ecs:*" + - "iam:CreateUser" + - "iam:CreatePolicy" + - "iam:CreateGroup" + - "iam:ListAccessKeys" + - "iam:GetGroup" + - "iam:GetPolicy" + - "iam:CreateRole" + - "iam:DeleteRole" + - "iam:AttachRolePolicy" + - "iam:DetachRolePolicy" + - "iam:DeleteRolePolicy" + - "iam:GetRole" + - "iam:PassRole" + - "iam:PutRolePolicy" + - "iam:GetRolePolicy" + - "iam:UpdateAssumeRolePolicy" + - "iam:CreateInstanceProfile" + - "iam:AddRoleToInstanceProfile" + - "iam:RemoveRoleFromInstanceProfile" + - "iam:DeleteInstanceProfile" + - "iam:CreateServiceLinkedRole" + - "iam:DeleteServiceLinkedRole" + - "iam:GetServiceLinkedRoleDeletionStatus" + - "iam:GetInstanceProfile" + - "lambda:CreateFunction" + - "lambda:DeleteFunction" + - "lambda:ListVersionsByFunction" + - "lambda:PublishVersion" + - "lambda:GetFunction" + - "lambda:CreateAlias" + - "lambda:EnableReplication*" + - "lambda:DeleteAlias" + - "secretsmanager:*" + - "cloudwatch:*" + - "sts:Decode*" + - "states:*" + Resource: "*" + Roles: + - !Ref DeployRole + +Outputs: + DeployRoleArn: + Description: Deploy role ARN + Value: !GetAtt DeployRole.Arn diff --git a/bootstrap/terraform_state_cfn.yml b/bootstrap/terraform_state_cfn.yml new file mode 100644 index 0000000..2c9e4db --- /dev/null +++ b/bootstrap/terraform_state_cfn.yml @@ -0,0 +1,55 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: >- + Resources required for managing FilmDrop deployment Terraform + state files. + +Parameters: + + TerraformStateBucketName: + Type: String + + TerraformStateLocksTableName: + Type: String + Default: filmdrop-terraform-state-locks + +Resources: + + TerraformStateBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref TerraformStateBucketName + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true + VersioningConfiguration: + Status: Enabled + + TerraformStateLocksTable: + Type: AWS::DynamoDB::Table + Properties: + AttributeDefinitions: + - AttributeName: "LockID" + AttributeType: S + BillingMode: PAY_PER_REQUEST + KeySchema: + - AttributeName: "LockID" + KeyType: HASH + TableName: !Ref TerraformStateLocksTableName + +Outputs: + + TerraformStateBucketName: + Description: Name of the S3 bucket to use as your Terraform backend bucket. + Value: !Ref TerraformStateBucket + + TerraformStateLocksTableName: + Description: >- + Name of the DynamoDB table to use as your Terraform backend + DynamoDB table. + Value: !Ref TerraformStateLocksTable diff --git a/dev.tfvars b/dev.tfvars new file mode 100644 index 0000000..2621e3a --- /dev/null +++ b/dev.tfvars @@ -0,0 +1,212 @@ +##### PROJECT VARIABLES #### +# The following variables are global to the FilmDrop infrastructure stack +environment = "dev" +project_name = "goose" +domain_zone = "Z0917278253RCNL9GYNYQ" +s3_access_log_bucket = "" +s3_logs_archive_bucket = "" + +##### NETWORKING VARIABLES #### +# If left blank, the infrastructure will try to query the values from the control tower vpc +vpc_id = "" +vpc_cidr = "" +security_group_id = "" +public_subnets_az_to_id_map = {} +private_subnets_az_to_id_map = {} + +##### ALARM VARIABLES #### +sns_warning_subscriptions_map = {} +sns_critical_subscriptions_map = {} + +##### APPLICATION VARIABLES #### +stac_server_inputs = { + app_name = "stac_server" + version = "v3.8.0" + deploy_cloudfront = true + web_acl_id = "" + domain_alias = "stac-api.goose.filmdrop.element84.com" + enable_transactions_extension = false + collection_to_index_mappings = "" + opensearch_cluster_instance_type = "t3.small.search" + opensearch_cluster_instance_count = 1 + opensearch_cluster_dedicated_master_enabled = true + opensearch_cluster_dedicated_master_type = "t3.small.search" + opensearch_cluster_dedicated_master_count = 1 + ingest_sns_topic_arns = [] + additional_ingest_sqs_senders_arns = [] + opensearch_ebs_volume_size = 35 + cors_origin = "*" + cors_credentials = false + cors_methods = "" + cors_headers = "" + authorized_s3_arns = [] + auth_function = { + cf_function_name = "" + cf_function_runtime = "cloudfront-js-2.0" + cf_function_code_path = "" + attach_cf_function = false + cf_function_event_type = "viewer-request" + create_cf_function = false + create_cf_basicauth_function = false + cf_function_arn = "" + } + ingest = { + source_catalog_url = "" + destination_collections_list = "" + destination_collections_min_lat = -90 + destination_collections_min_long = -180 + destination_collections_max_lat = 90 + destination_collections_max_long = 180 + date_start = "" + date_end = "" + include_historical_ingest = false + source_sns_arn = "" + include_ongoing_ingest = false + } +} + +titiler_inputs = { + app_name = "titiler" + domain_alias = "mosaic-titiler.goose.filmdrop.element84.com" + deploy_cloudfront = true + version = "v0.14.0-1.0.5" + authorized_s3_arns = [] + mosaic_titiler_waf_allowed_url = "https://stac-api.goose.filmdrop.element84.com" + mosaic_titiler_host_header = "mosaic-titiler.goose.filmdrop.element84.com" + mosaic_tile_timeout = 30 + web_acl_id = "" + auth_function = { + cf_function_name = "" + cf_function_runtime = "cloudfront-js-2.0" + cf_function_code_path = "" + attach_cf_function = false + cf_function_event_type = "viewer-request" + create_cf_function = false + create_cf_basicauth_function = false + cf_function_arn = "" + } +} + +analytics_inputs = {} + +console_ui_inputs = { + app_name = "console" + domain_alias = "console.goose.filmdrop.element84.com" + deploy_cloudfront = true + web_acl_id = "" + version = "v5.3.0" + filmdrop_ui_config_file = "./profiles/console-ui/default-config/config.dev.json" + filmdrop_ui_logo_file = "./profiles/console-ui/default-config/logo.png" + filmdrop_ui_logo = "bm9uZQo=" # Base64: 'none' + + custom_error_response = [ + { + error_caching_min_ttl = "10" + error_code = "404" + response_code = "200" + response_page_path = "/" + } + ] + + auth_function = { + cf_function_name = "" + cf_function_runtime = "cloudfront-js-2.0" + cf_function_code_path = "" + attach_cf_function = false + cf_function_event_type = "viewer-request" + create_cf_function = false + create_cf_basicauth_function = false + cf_function_arn = "" + } +} + +cirrus_inputs = { + data_bucket = "" + payload_bucket = "" + log_level = "DEBUG" + deploy_alarms = true + custom_alarms = { + warning = {} + critical = {} + } + process = { + sqs_timeout = 180 + sqs_max_receive_count = 5 + } + state = { + timestream_magnetic_store_retention_period_in_days = 93 + timestream_memory_store_retention_period_in_hours = 24 + } + api_lambda = { + timeout = 10 + memory = 128 + } + process_lambda = { + timeout = 10 + memory = 128 + reserved_concurrency = 16 + } + update_state_lambda = { + timeout = 15 + memory = 128 + } + pre_batch_lambda = { + timeout = 15 + memory = 128 + } + post_batch_lambda = { + timeout = 15 + memory = 128 + } +} + +cirrus_dashboard_inputs = { + app_name = "dashboard.goose.filmdrop.element84.com" + domain_alias = "" + deploy_cloudfront = true + web_acl_id = "" + version = "v0.5.1" + cirrus_api_endpoint = "" + metrics_api_endpoint = "" + custom_error_response = [ + { + error_caching_min_ttl = "10" + error_code = "404" + response_code = "200" + response_page_path = "/" + } + ] + auth_function = { + cf_function_name = "" + cf_function_runtime = "cloudfront-js-2.0" + cf_function_code_path = "" + attach_cf_function = false + cf_function_event_type = "viewer-request" + create_cf_function = false + create_cf_basicauth_function = false + cf_function_arn = "" + } +} + + +##### INFRASTRUCTURE FLAGS #### +# To disable each flag: set to 'false'; to enable: set to 'true' +deploy_vpc = false +deploy_vpc_search = true +deploy_log_archive = true +deploy_stac_server_opensearch_serverless = false +deploy_stac_server = true +deploy_stac_server_outside_vpc = false +deploy_analytics = false +deploy_titiler = true +deploy_console_ui = true +deploy_cirrus = true +deploy_cirrus_dashboard = true +deploy_local_stac_server_artifacts = false +deploy_waf_rule = true + + +#### WAF Rule Settings +ext_web_acl_id = "" # Specify if bringing an externally managed WAF +ip_blocklist = [] +whitelist_ips = [] diff --git a/scripts/retrieve_tf_modules.sh b/scripts/retrieve_tf_modules.sh new file mode 100755 index 0000000..a5420ed --- /dev/null +++ b/scripts/retrieve_tf_modules.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +set -x # print each command before executing + +FILMDROP_TERRAFORM_RELEASE=$1 + +wget -qO- https://github.com/Element84/filmdrop-aws-tf-modules/archive/refs/tags/${FILMDROP_TERRAFORM_RELEASE}.tar.gz | tar xvz +mkdir -p modules +mkdir -p profiles +cp filmdrop-aws-tf-modules-${FILMDROP_TERRAFORM_RELEASE:1}/filmdrop.tf . +cp filmdrop-aws-tf-modules-${FILMDROP_TERRAFORM_RELEASE:1}/providers.tf . +cp filmdrop-aws-tf-modules-${FILMDROP_TERRAFORM_RELEASE:1}/inputs.tf . +cp filmdrop-aws-tf-modules-${FILMDROP_TERRAFORM_RELEASE:1}/outputs.tf . +cp filmdrop-aws-tf-modules-${FILMDROP_TERRAFORM_RELEASE:1}/providers.tf . +cp -r filmdrop-aws-tf-modules-${FILMDROP_TERRAFORM_RELEASE:1}/modules . +cp -r filmdrop-aws-tf-modules-${FILMDROP_TERRAFORM_RELEASE:1}/profiles . +cp -r filmdrop-aws-tf-modules-${FILMDROP_TERRAFORM_RELEASE:1}/scripts/* ./scripts/ +rm -rf filmdrop-aws-tf-modules-${FILMDROP_TERRAFORM_RELEASE:1} diff --git a/ssm-bastion.tf b/ssm-bastion.tf new file mode 100644 index 0000000..c2704d6 --- /dev/null +++ b/ssm-bastion.tf @@ -0,0 +1,36 @@ +variable "ssm_bastion_input_map" { + description = "Inputs for SSM Bastion." + type = object({ + deploy_ssm_bastion = bool + ami_name_filter = string + swap_volume_size = string + instance_type = string + }) + default = { + deploy_ssm_bastion = false + ami_name_filter = "amzn2-ami-hvm-2.0.20240109.0-x86_64-ebs" + swap_volume_size = "2" + instance_type = "t3.micro" + } +} + +module "ssm_bastion" { + source = "./modules/ssm_bastion" + count = var.ssm_bastion_input_map.deploy_ssm_bastion ? 1 : 0 + + subnet_id = module.filmdrop.private_subnet_ids[0] + ami_name_filter = var.ssm_bastion_input_map.ami_name_filter + vpc_id = module.filmdrop.vpc_id + key_name = module.ssm_bastion_key_pair.key_pair_name + swap_volume_size = var.ssm_bastion_input_map.swap_volume_size + instance_type = var.ssm_bastion_input_map.instance_type + vpc_cidr_range = module.filmdrop.vpc_cidr +} + +module "ssm_bastion_key_pair" { + source = "terraform-aws-modules/key-pair/aws" + version = "~> 2.0" + + key_name_prefix = "fd-ssm-bastion-key-par-" + create_private_key = true +}