From 49a41d599e9b6418c7f6485220e3ae7af336f261 Mon Sep 17 00:00:00 2001 From: Pablo Costa Date: Tue, 1 Oct 2024 15:28:55 -0300 Subject: [PATCH 1/4] Refactor backup workflow to use Python virtual environment and install boto3 (#3134) * Refactor backup workflow to use Python virtual environment and install boto3 * Add step to Setup Python * Format file * Upgrade actions/setup-python to v5 --- .github/workflows/db-backup.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/db-backup.yml b/.github/workflows/db-backup.yml index e81e0c2e92..f2e8af8589 100644 --- a/.github/workflows/db-backup.yml +++ b/.github/workflows/db-backup.yml @@ -7,7 +7,7 @@ on: inputs: environment: description: The Github environment to load secrets from - type: string + type: string required: true defaults: @@ -25,6 +25,11 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install dependencies run: | sudo apt-get update @@ -32,19 +37,18 @@ jobs: coreutils \ bash \ tzdata \ - python3-pip \ curl \ npm wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | gpg --dearmor | sudo tee /usr/share/keyrings/mongodb.gpg > /dev/null echo "deb [ arch=amd64 signed-by=/usr/share/keyrings/mongodb.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list sudo apt update sudo apt install -y mongodb-database-tools - - - name: Install boto3 - run: pip install boto3 - name: Generate public backup run: | + python3 -m venv venv + source venv/bin/activate + pip install boto3 ./bin/backup.sh ./bin/prune.sh ./bin/list.sh From 9421758467147217245904bf0478d52fdd5e8927 Mon Sep 17 00:00:00 2001 From: Pablo Costa Date: Thu, 3 Oct 2024 16:34:19 -0300 Subject: [PATCH 2/4] Avoid `epoch_date_modified` null incident/report field on submission promotion (#3140) * Avoid `epoch_date_modified` null incident field on submission promotion * Remove `null` values from "epoch_date_modified" field in incidents and reports collections --- ....remove-null-epoch_date_modified-values.js | 30 +++++++++++++++++++ site/gatsby-site/server/fields/submissions.ts | 9 ++++-- 2 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 site/gatsby-site/migrations/2024.10.01T23.20.28.remove-null-epoch_date_modified-values.js diff --git a/site/gatsby-site/migrations/2024.10.01T23.20.28.remove-null-epoch_date_modified-values.js b/site/gatsby-site/migrations/2024.10.01T23.20.28.remove-null-epoch_date_modified-values.js new file mode 100644 index 0000000000..651d97faf3 --- /dev/null +++ b/site/gatsby-site/migrations/2024.10.01T23.20.28.remove-null-epoch_date_modified-values.js @@ -0,0 +1,30 @@ +const config = require('../config'); + +/** @type {import('umzug').MigrationFn} */ +exports.up = async ({ context: { client } }) => { + // If the incident has "epoch_date_modified" field and it is null, unset it + const incidents_collection = client + .db(config.realm.production_db.db_name) + .collection('incidents'); + + const incidents_epoch_date_modified_results = await incidents_collection.updateMany( + { epoch_date_modified: null }, + { $unset: { epoch_date_modified: '' } } + ); + + console.log( + `Updated ${incidents_epoch_date_modified_results.modifiedCount} incidents with null epoch_date_modified field` + ); + + // If the report has "epoch_date_modified" field and it is null, unset it + const reports_collection = client.db(config.realm.production_db.db_name).collection('reports'); + + const reports_epoch_date_modified_results = await reports_collection.updateMany( + { epoch_date_modified: null }, + { $unset: { epoch_date_modified: '' } } + ); + + console.log( + `Updated ${reports_epoch_date_modified_results.modifiedCount} reports with null epoch_date_modified field` + ); +}; diff --git a/site/gatsby-site/server/fields/submissions.ts b/site/gatsby-site/server/fields/submissions.ts index b2db5b2eee..19c32b6d8b 100644 --- a/site/gatsby-site/server/fields/submissions.ts +++ b/site/gatsby-site/server/fields/submissions.ts @@ -97,7 +97,6 @@ export const mutationFields: GraphQLFieldConfigMap = { reports: [], editors, date: submission.incident_date, - epoch_date_modified: submission.epoch_date_modified, "Alleged deployer of AI system": submission.deployers || [], "Alleged developer of AI system": submission.developers || [], "Alleged harmed or nearly harmed parties": submission.harmed_parties || [], @@ -113,6 +112,9 @@ export const mutationFields: GraphQLFieldConfigMap = { from_reports: [report_number] } } + if(submission.epoch_date_modified) { + newIncident.epoch_date_modified = submission.epoch_date_modified; + } await incidents.insertOne({ ...newIncident, incident_id: newIncident.incident_id }); @@ -214,7 +216,6 @@ export const mutationFields: GraphQLFieldConfigMap = { date_published: new Date(submission.date_published), date_submitted: new Date(submission.date_submitted), epoch_date_downloaded: getUnixTime(submission.date_downloaded), - epoch_date_modified: submission.epoch_date_modified, epoch_date_published: getUnixTime(submission.date_published), epoch_date_submitted: getUnixTime(submission.date_submitted), image_url: submission.image_url, @@ -237,6 +238,10 @@ export const mutationFields: GraphQLFieldConfigMap = { newReport.user = submission.user; } + if(submission.epoch_date_modified) { + newReport.epoch_date_modified = submission.epoch_date_modified; + } + await reports.insertOne({ ...newReport, report_number: newReport.report_number }); const incident_ids: number[] = parentIncidents.map(incident => incident.incident_id!); From 87b55c716d866ad35ffc5b5abba126cd35e35f93 Mon Sep 17 00:00:00 2001 From: Cesar Varela Date: Thu, 3 Oct 2024 17:29:43 -0300 Subject: [PATCH 3/4] Add new notification variables (#3141) --- .github/workflows/test-playwright-full.yml | 4 ++++ .github/workflows/test-playwright.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/test-playwright-full.yml b/.github/workflows/test-playwright-full.yml index e718d3ff33..8488845831 100644 --- a/.github/workflows/test-playwright-full.yml +++ b/.github/workflows/test-playwright-full.yml @@ -134,6 +134,10 @@ jobs: SHARD_INDEX: ${{ matrix.shardIndex }} SHARD_TOTAL: ${{ matrix.shardTotal }} TEST_FOLDER: playwright/e2e-full/ + SENDGRID_API_KEY: something + SENDGRID_SENDER_NAME: Test Preview + SENDGRID_SENDER: test@test.com + PROCESS_NOTIFICATIONS_SECRET: sarasa - name: Upload Playwright traces if: failure() diff --git a/.github/workflows/test-playwright.yml b/.github/workflows/test-playwright.yml index 5f08ad7b95..563a29bcf6 100644 --- a/.github/workflows/test-playwright.yml +++ b/.github/workflows/test-playwright.yml @@ -89,6 +89,10 @@ jobs: SHARD_INDEX: ${{ matrix.shardIndex }} SHARD_TOTAL: ${{ matrix.shardTotal }} TEST_FOLDER: playwright/e2e/ + SENDGRID_API_KEY: something + SENDGRID_SENDER_NAME: Test Preview + SENDGRID_SENDER: test@test.com + PROCESS_NOTIFICATIONS_SECRET: sarasa - uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} From 2bd64abfb17f32f434ec45ddb97c66641d6ee748 Mon Sep 17 00:00:00 2001 From: Cesar Varela Date: Fri, 4 Oct 2024 15:27:31 -0300 Subject: [PATCH 4/4] Add test api new variables (#3143) --- .github/workflows/test-api.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-api.yml b/.github/workflows/test-api.yml index 3ebcc65620..0fd81cddaf 100644 --- a/.github/workflows/test-api.yml +++ b/.github/workflows/test-api.yml @@ -68,6 +68,10 @@ jobs: E2E_ADMIN_USERNAME: "" E2E_ADMIN_PASSWORD: "" ROLLBAR_POST_SERVER_ITEM_ACCESS_TOKEN: ${{ secrets.GATSBY_ROLLBAR_TOKEN }} + SENDGRID_API_KEY: something + SENDGRID_SENDER_NAME: Test Preview + SENDGRID_SENDER: test@test.com + PROCESS_NOTIFICATIONS_SECRET: sarasa - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4