From 0cf8d3fbfd752f6470bdf7363de65a8d5cf06880 Mon Sep 17 00:00:00 2001 From: Kar Rui Lau Date: Fri, 11 Oct 2024 19:21:56 +0800 Subject: [PATCH] ci: add coverage to ci test command (#1453) * ci: add coverage to ci test command * feat(ci): load env with github action, run test:ci script * fix: add ci as valid dd_env in convict used in ci * feat: use dotenv cli instead of sourcing .env.test --- .env.test | 114 +++++++++++++++++++-------------------- .github/workflows/ci.yml | 6 ++- package-lock.json | 25 +++++++++ package.json | 4 +- src/config/config.ts | 1 + 5 files changed, 91 insertions(+), 59 deletions(-) diff --git a/.env.test b/.env.test index 1228238d9..7882a9958 100644 --- a/.env.test +++ b/.env.test @@ -1,91 +1,91 @@ -export CLIENT_ID="i am a client" -export CLIENT_SECRET="i am a secret" -export REDIRECT_URI="http://localhost:8081/v1/auth" -export NODE_ENV="test" -export COOKIE_DOMAIN="localhost" -export AUTH_TOKEN_EXPIRY_DURATION_IN_MILLISECONDS=800000000 -export JWT_SECRET="blah" -export ENCRYPTION_SECRET="blah" -export FRONTEND_URL="http://localhost:3000" -export GITHUB_ORG_NAME="isomerpages" -export GITHUB_BUILD_ORG_NAME="opengovsg" -export GITHUB_BUILD_REPO_NAME="isomer-build" -export MUTEX_TABLE_NAME="mutex-table" -export MAX_NUM_OTP_ATTEMPTS=5 -export OTP_EXPIRY=900000 -export SESSION_SECRET=blahblah +CLIENT_ID="i am a client" +CLIENT_SECRET="i am a secret" +REDIRECT_URI="http://localhost:8081/v1/auth" +NODE_ENV="test" +COOKIE_DOMAIN="localhost" +AUTH_TOKEN_EXPIRY_DURATION_IN_MILLISECONDS=800000000 +JWT_SECRET="blah" +ENCRYPTION_SECRET="blah" +FRONTEND_URL="http://localhost:3000" +GITHUB_ORG_NAME="isomerpages" +GITHUB_BUILD_ORG_NAME="opengovsg" +GITHUB_BUILD_REPO_NAME="isomer-build" +MUTEX_TABLE_NAME="mutex-table" +MAX_NUM_OTP_ATTEMPTS=5 +OTP_EXPIRY=900000 +SESSION_SECRET=blahblah # GitHub access token to create repo -export SYSTEM_GITHUB_TOKEN="github_token" +SYSTEM_GITHUB_TOKEN="github_token" # FormSG keys -export SITE_CREATE_FORM_KEY="site_form_key" -export SITE_LAUNCH_FORM_KEY="site_launch_form_key" -export GGS_REPAIR_FORM_KEY="ggs_repair_form_key" -export SITE_CHECKER_FORM_KEY="site_checker_form_key" -export SITE_AUDIT_LOGS_FORM_KEY="site_audit_logs_form_key" -export NOTIFY_SITE_COLLABORATORS_FORM_KEY="notify_site_collaborators_form_key" +SITE_CREATE_FORM_KEY="site_form_key" +SITE_LAUNCH_FORM_KEY="site_launch_form_key" +GGS_REPAIR_FORM_KEY="ggs_repair_form_key" +SITE_CHECKER_FORM_KEY="site_checker_form_key" +SITE_AUDIT_LOGS_FORM_KEY="site_audit_logs_form_key" +NOTIFY_SITE_COLLABORATORS_FORM_KEY="notify_site_collaborators_form_key" # Required to connect to DynamoDB -export AWS_ACCESS_KEY_ID="abc123" -export AWS_SECRET_ACCESS_KEY="xyz123" +AWS_ACCESS_KEY_ID="abc123" +AWS_SECRET_ACCESS_KEY="xyz123" # Required to run end-to-end tests -export E2E_TEST_REPO="e2e-test-repo" -export E2E_TEST_SECRET="test" -export E2E_TEST_GH_TOKEN="test" +E2E_TEST_REPO="e2e-test-repo" +E2E_TEST_SECRET="test" +E2E_TEST_GH_TOKEN="test" # Database -export DB_URI="postgres://isomer:password@127.0.0.1:54321/isomercms_test" -export DB_MIN_POOL="1" -export DB_MAX_POOL="10" -export DB_ENABLE_LOGGING="true" +DB_URI="postgres://isomer:password@127.0.0.1:54321/isomercms_test" +DB_MIN_POOL="1" +DB_MAX_POOL="10" +DB_ENABLE_LOGGING="true" -export LOCAL_SITE_ACCESS_TOKEN="dummy" -export OTP_SECRET="dummysecret" +LOCAL_SITE_ACCESS_TOKEN="dummy" +OTP_SECRET="dummysecret" # Email -export POSTMAN_API_KEY="some api key" +POSTMAN_API_KEY="some api key" # SMS -export POSTMAN_SMS_CRED_NAME="isomer" +POSTMAN_SMS_CRED_NAME="isomer" # Amplify -export AWS_REGION="ap-southeast-1" -export AWS_ACCOUNT_NUMBER="random" +AWS_REGION="ap-southeast-1" +AWS_ACCOUNT_NUMBER="random" # DataDog -export DD_ENV="local" -export DD_SERVICE="isomer" -export DD_TAGS="service:isomer" +DD_ENV="ci" +DD_SERVICE="isomer" +DD_TAGS="service:isomer" # Cloudmersive -export CLOUDMERSIVE_API_KEY="cloudmersive" +CLOUDMERSIVE_API_KEY="cloudmersive" # GitGuardian -export GITGUARDIAN_API_KEY="gitguardian" +GITGUARDIAN_API_KEY="gitguardian" # SQS Queue -export INCOMING_QUEUE_URL="incoming" -export OUTGOING_QUEUE_URL="outgoing" -export SITE_LAUNCH_QUEUE_URL="site_launch" +INCOMING_QUEUE_URL="incoming" +OUTGOING_QUEUE_URL="outgoing" +SITE_LAUNCH_QUEUE_URL="site_launch" # DynamoDB -export SITE_LAUNCH_DYNAMO_DB_TABLE_NAME="ddb-name" -export STEP_FUNCTIONS_ARN="arn:aws:some-arn" -export SITE_PASSWORD_SECRET_KEY="1234567812345678123456781234567812345678123456781234567812345678" +SITE_LAUNCH_DYNAMO_DB_TABLE_NAME="ddb-name" +STEP_FUNCTIONS_ARN="arn:aws:some-arn" +SITE_PASSWORD_SECRET_KEY="1234567812345678123456781234567812345678123456781234567812345678" -export NETLIFY_ACCESS_TOKEN="blahblah" +NETLIFY_ACCESS_TOKEN="blahblah" # sgID -export SGID_CLIENT_ID="client" -export SGID_CLIENT_SECRET="secret" -export SGID_PRIVATE_KEY="private" -export SGID_REDIRECT_URI="http://localhost:8081/v2/auth/sgid/auth-redirect" +SGID_CLIENT_ID="client" +SGID_CLIENT_SECRET="secret" +SGID_PRIVATE_KEY="private" +SGID_REDIRECT_URI="http://localhost:8081/v2/auth/sgid/auth-redirect" # GrowthBook -export GROWTHBOOK_CLIENT_KEY="some random key" +GROWTHBOOK_CLIENT_KEY="some random key" -export KEYCDN_API_KEY="secret" +KEYCDN_API_KEY="secret" -export REDIS_HOST="redis" +REDIS_HOST="redis" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 653d9de65..07e7f8031 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,6 +87,10 @@ jobs: key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }} - run: npm ci - run: npm run test:docker + - name: Load .env file + uses: xom9ikk/dotenv@v2 + with: + mode: test # Not using Datadog Test Visibility action since we need to use the local # (patched) package for it to work with neverthrow - name: run tests @@ -99,7 +103,7 @@ jobs: DD_SERVICE: isomer DD_API_KEY: ${{ secrets.DD_API_KEY }} DD_CIVISIBILITY_AUTO_INSTRUMENTATION_PROVIDER: github - run: . .env.test && npx jest --runInBand + run: npm run test:ci gatekeep: name: Determine if Build & Deploy is needed diff --git a/package-lock.json b/package-lock.json index a4b0a9def..3d27b324a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -41,6 +41,7 @@ "debug": "~2.6.9", "dompurify": "^3.1.3", "dotenv": "^16.4.5", + "dotenv-cli": "^7.4.2", "eventsource": "^2.0.2", "exponential-backoff": "^3.1.0", "express": "~4.19.2", @@ -8867,6 +8868,30 @@ "url": "https://dotenvx.com" } }, + "node_modules/dotenv-cli": { + "version": "7.4.2", + "resolved": "https://registry.npmjs.org/dotenv-cli/-/dotenv-cli-7.4.2.tgz", + "integrity": "sha512-SbUj8l61zIbzyhIbg0FwPJq6+wjbzdn9oEtozQpZ6kW2ihCcapKVZj49oCT3oPM+mgQm+itgvUQcG5szxVrZTA==", + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "dotenv": "^16.3.0", + "dotenv-expand": "^10.0.0", + "minimist": "^1.2.6" + }, + "bin": { + "dotenv": "cli.js" + } + }, + "node_modules/dotenv-expand": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-10.0.0.tgz", + "integrity": "sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, "node_modules/dottie": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.6.tgz", diff --git a/package.json b/package.json index 5a8ebf5fb..17bae3f8a 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "start:support:prod": "node --unhandled-rejections=warn -r ts-node/register/transpile-only -r tsconfig-paths/register support/index.ts", "dev": "source .env && docker compose -f docker-compose.dev.yml up", "test:docker": "docker run -d -p 54321:5432 --name postgres -e POSTGRES_USER=isomer -e POSTGRES_PASSWORD=password -e POSTGRES_DB=isomercms_test postgres:latest", - "test": "source .env.test && jest --runInBand", + "test": "dotenv -e .env.test jest --runInBand", + "test:ci": "jest --runInBand --coverage", "release": "bash scripts/release_prep.sh", "lint": "npx eslint .", "lint-fix": "eslint --ignore-path .gitignore . --fix", @@ -62,6 +63,7 @@ "debug": "~2.6.9", "dompurify": "^3.1.3", "dotenv": "^16.4.5", + "dotenv-cli": "^7.4.2", "eventsource": "^2.0.2", "exponential-backoff": "^3.1.0", "express": "~4.19.2", diff --git a/src/config/config.ts b/src/config/config.ts index ee6ace999..4c4b2da4d 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -275,6 +275,7 @@ const config = convict({ "prod", "stg", "dev", + "ci", ], env: "DD_ENV", default: "local",