diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a00059b31b..ca8b5a2ab2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -66,18 +66,18 @@ jobs: run: yarn install - name: Build icon package - run: yarn workspace @corona-dashboard/icons build + run: yarn build:icons - name: Build common module - run: yarn workspace @corona-dashboard/common build + run: yarn build:common - - name: Export texts (develop) + - name: Import texts (develop) if: ${{ env.BRANCH_NAME != 'master' }} - run: yarn workspace @corona-dashboard/cms lokalize:export --clean-json + run: yarn cms:lokalize-import --clean-json - - name: Export texts (production) + - name: Import texts (production) if: ${{ env.BRANCH_NAME == 'master' }} - run: yarn workspace @corona-dashboard/cms lokalize:export --dataset="production" --clean-json + run: yarn cms:lokalize-import:prd --clean-json - name: Typecheck all code run: yarn typecheck @@ -106,13 +106,13 @@ jobs: run: yarn install - name: Build common module - run: yarn workspace @corona-dashboard/common build + run: yarn build:common - name: Build icons module - run: yarn workspace @corona-dashboard/icons build + run: yarn build:icons - name: Export texts - run: yarn workspace @corona-dashboard/cms lokalize:export --clean-json + run: yarn cms:lokalize-import --clean-json - name: Run tests run: yarn test:ci diff --git a/.github/workflows/sync-sanity-lokalize.yml b/.github/workflows/sync-sanity-lokalize.yml index ab31cc6e38..d03c586eaa 100644 --- a/.github/workflows/sync-sanity-lokalize.yml +++ b/.github/workflows/sync-sanity-lokalize.yml @@ -40,7 +40,7 @@ jobs: SANITY_TOKEN: ${{ secrets.sanity_token }} - name: Export production texts - run: yarn workspace @corona-dashboard/cms lokalize:export --dataset=production --clean-json + run: yarn workspace @corona-dashboard/cms lokalize:import --dataset=production --clean-json - name: Typecheck all code with production texts run: yarn workspace @corona-dashboard/app typecheck diff --git a/Dockerfile b/Dockerfile index 67f47f3410..479b71bfeb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,7 +74,7 @@ FROM builder RUN yarn download \ && yarn workspace @corona-dashboard/cli validate-json-all \ && yarn workspace @corona-dashboard/cli validate-last-values --fail-early \ -&& yarn workspace @corona-dashboard/cms lokalize:export --dataset=$NEXT_PUBLIC_SANITY_DATASET \ +&& yarn workspace @corona-dashboard/cms lokalize:import --dataset=$NEXT_PUBLIC_SANITY_DATASET \ && yarn workspace @corona-dashboard/app build \ && mkdir -p /app/packages/app/public/images/choropleth \ && addgroup -g 1001 -S nodejs \ diff --git a/docs/ci.md b/docs/ci.md index b04cb6fbda..0ff75c1d24 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -37,7 +37,7 @@ The build process follows these steps: - Run the unit tests - Download the server data files - Run all the JSON data validations - - Run lokalize:export to retrieve all the translations from Sanity + - Run lokalize:import to retrieve all the translations from Sanity - Run the NextJs build [Back to index](index.md) diff --git a/docs/developers.md b/docs/developers.md index 9a38c795d6..2226ae2f36 100644 --- a/docs/developers.md +++ b/docs/developers.md @@ -1,5 +1,15 @@ # Developer Notes +## Shortcut scripts + +The monorepo contains projects which the developer can access with shortcut scripts at the root of the project. Instead of writing the path in the CLI like `yarn workspace @corona-dashboard cms lokalize:import`, we can access the script by writing `yarn cms:lokalize-import` to bring your local JSON files up-to-date with the Sanity dataset. Another example is using `yarn build:app` instead of `yarn workspace @corona-dashboard/app build` to make a build during the release procedure. Lastly, running `yarn check-all` will make sure the code, data and tests work as expected, instead of running scripts individually. + +Available shortcut scripts: + +- `yarn check-all` +- `yarn cms:[command]` +- `yarn build:[command]` + ## Common and icon packages The monorepo contains two shared packages: common and icons. diff --git a/docs/feature-flags.md b/docs/feature-flags.md index 0dd5c2d00c..6e98e5b081 100644 --- a/docs/feature-flags.md +++ b/docs/feature-flags.md @@ -89,7 +89,4 @@ already existing metric, then declaring those properties will mean that only the checks are performed on those. If the whole schema / metric is new, as it often is, then specifying only the `metricName` is sufficient. -The script that validates the features (and runs at build-time) can be -triggered locally with `yarn workspace@corona-dashboard/cli validate-features`. - [Back to index](index.md) diff --git a/docs/lokalize-texts.md b/docs/lokalize-texts.md index f9d91c9808..c17a8d5695 100644 --- a/docs/lokalize-texts.md +++ b/docs/lokalize-texts.md @@ -9,7 +9,7 @@ our codebase. Even though these texts now live as documents in Sanity, we still refer to them as "lokalize texts". Each text string lives as a document in the CMS, of which each has a unique "key" -corresponding to a path. These documents are then exported as JSON files (one +corresponding to a path. These documents are then imported as JSON files (one for each language) and consumed by the application as static data. This document describes how to manage and make changes to these texts. @@ -19,15 +19,15 @@ This document describes how to manage and make changes to these texts. In summary these are the most important things you should be aware of: - Commands are run with `yarn lokalize:[command]` from the `packages/cms` root -- The `export` command brings your local JSON files up-to-date with the Sanity +- The `import` command brings your local JSON files up-to-date with the Sanity dataset. The TypeScript compiler will error when your JSON files do not contain all the texts which are referenced in the code. -- The JSON export contains document ids as part of the keys. You can make +- The JSON import contains document ids as part of the keys. You can make changes locally to the **app/src/locale/nl_export.json** file. Add new keys, delete them or rename and move existing ones. After you make changes run the `lokalize:apply-json-edits` script. This will give you a list of changes and you can decide which ones to apply. Changes are written in a mutation log - file, and at the end the JSON is re-exported to reflect all changes. + file, and at the end the JSON is re-imported to reflect all changes. It is recommended to run `lokalize:apply-json-edits` after the feature has been finished, right before merging it to develop since during development changes to the **nl_export.json** file might fluctuate. @@ -63,26 +63,26 @@ scripts. Merge conflicts in this file are common. You should always "accept both changes" when resolving conflicts, so that none of the lines are ever deleted. -## Export +## Import The application reads its locale strings from **packages/app/public/nl_export.json** and **packages/app/public/en_export.json**. -These JSONs are exported from the Sanity lokalize documents, but they are not -part of the repository. Therefore, you will regularly need to run `yarn lokalize:export` in +These JSONs are imported from the Sanity lokalize documents, but they are not +part of the repository. Therefore, you will regularly need to run `yarn lokalize:import` in order to keep your local JSON file up-to-date with the Sanity dataset. The JSONs will include Sanity document ids in every leaf-key which are used to detect add-/delete-/move-actions of texts (`some_key__@__{document_id}`). These ids would result in compile- and run-time errors, but there's a workaround: -- compile-time: every export will also emit a `site-text.d.ts` with a `SiteText` +- compile-time: every import will also emit a `site-text.d.ts` with a `SiteText` interface. This interface is used to type the imported JSONs. - run-time: on load of the app all ids will be removed from the keys. The runtime workaround would be a waste of resources on production, so for this reason we use the `--clean-json` flag to ignore document ids. -### Export/Import technical details +### Technical details In Sanity each lokalize value is stored as a separate `lokalizeText` document. The key property in this document represents the flattened property path in **nl_export.json** @@ -107,7 +107,7 @@ The root key (so in this case `accessibility`) will be saved as the subject prop This allows us to filter on this subject under the Sanity `Lokalize` menu item. Making it a little easier for an editor to lookup a specific key in the menu. -So, `yarn lokalize:export` simply retrieves all of the `lokalizeText` documents, replaces their ids in the root part of the path +So, `yarn lokalize:import` simply retrieves all of the `lokalizeText` documents, replaces their ids in the root part of the path and unflattens this list of paths into a JSON file. (The Dutch texts are serialized as **nl_export.json**, the English as **en_export.json**). `yarn lokalize:apply-json-edits` does the exact opposite, it takes the **nl_export.json** file, flattens the file into an array of paths and saves those to Sanity. Obviously there is a bunch of logic involved that checks if a path is new or existing, but for those @@ -116,7 +116,7 @@ details we refer to the source code. ## How to Add, Delete or Move Texts First make sure the JSONs include document ids (`some_key__@__{document_id}`). -Run `yarn lokalize:export` if these are not yet present in your JSONs. +Run `yarn lokalize:import` if these are not yet present in your JSONs. You can add, delete and move keys by mutating the **nl_export.json** file. The sync script will automagically detect additions, deletions or moved lokalize @@ -129,7 +129,7 @@ file. New texts will only have an NL (Dutch) string when they are added and will use NL as a fallback. -After syncing texts, the export script is called to update your local JSON file +After syncing texts, the import script is called to update your local JSON file and re-generate the SiteText type interface. ### Delete/Move Mutations @@ -139,7 +139,7 @@ dataset, we can not simply remove a lokalize text document from the dataset without potentially breaking other branches. For this reason, when you delete a lokalize text, it will append the delete -action to the mutations file but not actually delete the document. The export +action to the mutations file but not actually delete the document. The import filters out any deletions that were logged to the mutations file. So in effect you end up with a local JSON file that has the deleted key removed, and the TS compiler sees the correct dataset. @@ -148,7 +148,7 @@ The actual deletions from Sanity only happen in the `sync-after-feature` phase, describe below. The same goes for move mutations. When you apply the move in your feature branch -it will be simulated in the JSON export, but not yet applied to the CMS +it will be simulated in the JSON import, but not yet applied to the CMS documents. During sync-after-feature the move is finalized by changing the key and subject properties of the targeted document. This preserves the document history and drafts. diff --git a/docs/release-procedure.md b/docs/release-procedure.md index 6b690bcb63..fbc9c1a59c 100644 --- a/docs/release-procedure.md +++ b/docs/release-procedure.md @@ -6,7 +6,8 @@ 2. Merge the develop branch into the release branch. 3. Do a full build with a Sanity export from production to make sure everything works. - Use `yarn lokalize:export --dataset=production` to create the Sanity export. + Use `yarn cms:lokalize-import:prd` to create the Sanity export. + Use `yarn build:app` to create a build with the latest production dataset Test the build locally by running `yarn start`. @@ -23,9 +24,9 @@ 8. Hit the publish button in the GitHub release draft. This should tag the correct commit in `master`. 9. Important before running the next step is building the common package to avoid any typescript errors, - so run `yarn workspace @corona-dashboard/common build`. + so run `yarn build:common`. 10. After we are sure we are moving to production with the release as it is, - run the `yarn workspace @corona-dashboard/cms lokalize:sync-after-release` + run the `yarn cms:sync-after-release` script. This script will perform any necessary text mutations for keys that have been moved, and also cleans up any deleted texts from the Sanity production dataset. For more information [read this](/docs/lokalize-texts.md#sync-after-release). @@ -99,7 +100,7 @@ the `key-mutations.csv` file and commit this to the release branch. ## CMS Deploy Sometimes a release also contains new configurations for the CMS. These are -deployed via the Sanity CLI as `yarn deploy`. These deployments do +deployed via the Sanity CLI by running `yarn cms:deploy` in the root of the project. These deployments do not usually need to be done simultaneously with the release. If they are backwards-compatible they could be performed at any time and also repeatedly. diff --git a/eslint-overrides.json b/eslint-overrides.json index 9542695829..143799cdc5 100644 --- a/eslint-overrides.json +++ b/eslint-overrides.json @@ -19,6 +19,7 @@ "@typescript-eslint/no-var-requires": 0, "@typescript-eslint/explicit-module-boundary-types": 0, "@typescript-eslint/no-use-before-define": 0, + "@typescript-eslint/triple-slash-reference": 0, "@typescript-eslint/no-unused-vars": [ 2, { diff --git a/package.json b/package.json index ad246fb8fe..8866df08c0 100644 --- a/package.json +++ b/package.json @@ -9,21 +9,30 @@ "bootstrap": "yarn workspaces foreach run bootstrap", "clean": "yarn workspaces foreach run clean && npx rimraf .build-cache", "nuke": "yarn workspaces foreach run nuke && npx rimraf node_modules", - "typecheck": "yarn workspaces foreach run typecheck", - "lint": "yarn workspaces foreach run lint", + "typecheck": "echo 'Starting typecheck...' && yarn workspaces foreach run typecheck", + "lint": "echo 'Starting lint...' && yarn workspaces foreach run lint", "lint:fix": "yarn workspaces foreach run lint --fix", "format": "yarn workspaces foreach run format", "dev": "yarn workspace @corona-dashboard/app dev", "download": "yarn workspace @corona-dashboard/app download", "start": "yarn workspace @corona-dashboard/app start", + "check-all": "yarn lint && yarn typecheck && yarn validate-json-all && yarn test", "validate-json-all": "yarn workspace @corona-dashboard/cli validate-json-all", "validate-json-single": "yarn workspace @corona-dashboard/cli validate-json-single", "validate-last-values": "yarn workspace @corona-dashboard/cli validate-last-values", - "validate-features": "yarn workspace @corona-dashboard/cli validate-features", - "generate-typescript": "yarn workspace @corona-dashboard/cli generate-typescript", + "generate-data-types": "yarn workspace @corona-dashboard/cli generate-data-types", "e2e": "yarn workspace @corona-dashboard/e2e e2e", "e2e:ci": "yarn workspace @corona-dashboard/e2e e2e:ci", - "cms": "yarn workspace @corona-dashboard/cms dev" + "cms": "yarn workspace @corona-dashboard/cms dev", + "cms:lokalize-import": "yarn workspace @corona-dashboard/cms lokalize:import", + "cms:lokalize-import:prd": "yarn workspace @corona-dashboard/cms lokalize:import --dataset=production", + "cms:apply-json-edits": "yarn workspace @corona-dashboard/cms lokalize:apply-json-edits", + "cms:generate-types": "yarn workspace @corona-dashboard/cms lokalize:generate-types", + "cms:sync-after-release": "yarn workspace @corona-dashboard/cms lokalize:sync-after-release", + "cms:deploy": "yarn workspace @corona-dashboard/cms deploy", + "build:app": "yarn workspace @corona-dashboard/app build", + "build:common": "yarn workspace @corona-dashboard/common build", + "build:icons": "yarn workspace @corona-dashboard/icons build" }, "workspaces": { "packages": [ diff --git a/packages/app/README.md b/packages/app/README.md index db50f9d5c2..40a101fedb 100644 --- a/packages/app/README.md +++ b/packages/app/README.md @@ -45,9 +45,9 @@ The "Lokalize" part of Sanity is exported and consumed by the app as JSON. You w need to run this script regularly as an outdated JSON file will result in compile or build-time errors. -`yarn workspace @corona-dashboard/cms lokalize:export` +`yarn cms:lokalize-import` -Alternatively you can run this from `packages/cms` as `yarn lokalize:export` +Alternatively you can run this from `packages/cms` as `yarn lokalize:import` To learn more about the rationale behind Lokalize and how it works [read the documentation](/packages/cms/README.md#lokalize-texts). diff --git a/packages/app/package.json b/packages/app/package.json index 86cd8a4209..19dc2db397 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -162,7 +162,7 @@ "lint": "eslint .", "typecheck": "tsc --pretty --noEmit", "trace": "tsc --pretty --noEmit --generateTrace \"trace-$(date +%Y%m%d%H%M)\" --composite false --incremental false", - "download": "node ./infra/download.js", + "download": "npx rimraf ./public/json/* && node ./infra/download.js", "clean": "npx rimraf .next", "nuke": "yarn clean && npx rimraf node_modules" }, diff --git a/packages/cli/package.json b/packages/cli/package.json index aae816d4d3..0601d8bd83 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -32,11 +32,10 @@ "lint": "eslint .", "typecheck": "tsc --pretty --noEmit", "calculate-boundingboxes": "yarn workspace @corona-dashboard/common build && ts-node src/scripts/calculate-bounding-box-municipalities.ts", - "validate-json": "yarn run validate-json-all", "validate-json-all": "yarn workspace @corona-dashboard/common build && ts-node src/scripts/validate-json-all.ts", "validate-json-single": "yarn workspace @corona-dashboard/common build && ts-node src/scripts/validate-json-single.ts", "validate-last-values": "yarn workspace @corona-dashboard/common build && ts-node src/scripts/validate-last-values.ts", - "generate-typescript": "yarn workspace @corona-dashboard/common build && ts-node src/scripts/generate-data-types.ts", + "generate-data-types": "yarn workspace @corona-dashboard/common build && ts-node src/scripts/generate-data-types.ts", "generate-data-structures": "ts-node src/scripts/generate-data-structures.ts" }, "devDependencies": { diff --git a/packages/cms/package.json b/packages/cms/package.json index 4a9d75fca6..fa69cfa742 100644 --- a/packages/cms/package.json +++ b/packages/cms/package.json @@ -4,17 +4,16 @@ "version": "0.0.0", "license": "EUROPEAN UNION PUBLIC LICENCE v. 1.2", "scripts": { - "bootstrap": "yarn lokalize:export", "clean": "npx rimraf dist .sanity-asset-cache .lokalize-reference", "nuke": "yarn clean && npx rimraf node_modules", "dev": "yarn workspace @corona-dashboard/common build && yarn workspace @corona-dashboard/icons build && yarn workspace @corona-dashboard/cli generate-data-structures && sanity start", "build": "sanity build", "deploy": "yarn workspace @corona-dashboard/common build && yarn workspace @corona-dashboard/icons build && yarn workspace @corona-dashboard/cli generate-data-structures && sanity deploy", - "test": "sanity check", - "test:ci": "yarn run test", + "test": "echo 'Sanity check is deprecated \n@TODO add sanity test script'", + "test:ci": "echo 'Sanity check is deprecated \n@TODO add sanity test ci script'", "typecheck": "tsc --pretty --noEmit", "lint": "eslint .", - "lokalize:export": "ts-node src/lokalize/export.ts", + "lokalize:import": "ts-node src/lokalize/import.ts", "lokalize:add": "ts-node src/lokalize/lokalize-add.ts", "lokalize:dedupe": "ts-node src/lokalize/clean-duplicates.ts", "lokalize:apply-json-edits": "ts-node src/lokalize/apply-json-edits.ts", @@ -22,7 +21,7 @@ "lokalize:sync-after-release": "ts-node src/lokalize/sync-after-release.ts", "lokalize:sync-after-feature": "ts-node src/lokalize/sync-after-feature.ts", "lokalize:sync-prd-to-dev": "ts-node src/lokalize/sync-prd-to-dev.ts", - "lokalize:strip-unused-keys": "yarn lokalize:export --clean-json && ts-node src/lokalize/strip-unused-keys.ts && yarn lokalize:export --clean-json && yarn workspace @corona-dashboard/app typecheck", + "lokalize:strip-unused-keys": "yarn lokalize:import --clean-json && ts-node src/lokalize/strip-unused-keys.ts && yarn lokalize:import --clean-json && yarn workspace @corona-dashboard/app typecheck", "elements:add": "ts-node src/elements/add.ts", "format": "prettier --write \"src/**/*.{js,ts,tsx}\" \"*.{js,json,md,yml}\"" }, diff --git a/packages/cms/src/lokalize/apply-json-edits.ts b/packages/cms/src/lokalize/apply-json-edits.ts index a520715fb5..8a7c2b60a5 100644 --- a/packages/cms/src/lokalize/apply-json-edits.ts +++ b/packages/cms/src/lokalize/apply-json-edits.ts @@ -4,7 +4,7 @@ import prompts from 'prompts'; import { getClient } from '../client'; import { appendTextMutation, - exportLokalizeTexts, + importLokalizeTexts, getLocalMutations, readReferenceTexts, } from './logic'; @@ -21,7 +21,7 @@ import { createTextDocument } from './logic/create-text-document'; assert( referenceTexts, - `Failed to read reference texts. Please run lokalize:export first.` + `Failed to read reference texts. Please run lokalize:import first.` ); const mutations = await getLocalMutations(referenceTexts); @@ -110,7 +110,7 @@ import { createTextDocument } from './logic/create-text-document'; console.log('Updating text export...'); - await exportLokalizeTexts({ + await importLokalizeTexts({ dataset: 'development', appendDocumentIdToKey: true, }); diff --git a/packages/cms/src/lokalize/export.ts b/packages/cms/src/lokalize/import.ts similarity index 78% rename from packages/cms/src/lokalize/export.ts rename to packages/cms/src/lokalize/import.ts index 16c2ca275e..cc9bdbc16e 100644 --- a/packages/cms/src/lokalize/export.ts +++ b/packages/cms/src/lokalize/import.ts @@ -1,5 +1,5 @@ /** - * This script exports LokalizeText documents from Sanity as a locale JSON file, + * This script imports LokalizeText documents from Sanity as a locale JSON file, * and strips any keys that have been marked by the key-mutations.csv file as a * result of text changes via the CLI. */ @@ -8,20 +8,20 @@ import meow from 'meow'; import { outdent } from 'outdent'; import prompts from 'prompts'; import { getLocalMutations, readReferenceTexts } from './logic'; -import { exportLokalizeTexts } from './logic/export'; +import { importLokalizeTexts } from './logic/import'; const cli = meow( ` Usage - $ lokalize:export + $ lokalize:import Options - --dataset Define dataset to export, default is "development" + --dataset Define dataset to import, default is "development" --clean-json Export without document-ids in the keys Examples - $ lokalize:export --dataset=development - $ lokalize:export --dataset=development --clean-json + $ lokalize:import --dataset=development + $ lokalize:import --dataset=development --clean-json `, { flags: { @@ -61,7 +61,7 @@ const cli = meow( type: 'confirm', name: 'isConfirmed', message: outdent` - There are local changes. Are you sure you want to overwrite these with an export? + There are local changes. Are you sure you want to overwrite these with an import? ${JSON.stringify(mutations, null, 2)} `, @@ -75,14 +75,14 @@ const cli = meow( } } - await exportLokalizeTexts({ + await importLokalizeTexts({ dataset, appendDocumentIdToKey: !cli.flags.cleanJson, }); - console.log(`Export dataset "${dataset}" completed`); + console.log(`Import dataset "${dataset}" completed`); })().catch((err) => { - console.error(`Export failed: ${err.message}`); + console.error(`Import failed: ${err.message}`); console.error(err.stack); process.exit(1); }); diff --git a/packages/cms/src/lokalize/logic/files.ts b/packages/cms/src/lokalize/logic/files.ts index d993ca9b3a..3e4a446fa5 100644 --- a/packages/cms/src/lokalize/logic/files.ts +++ b/packages/cms/src/lokalize/logic/files.ts @@ -2,7 +2,7 @@ import flatten from 'flat'; import fs from 'fs'; import mapKeys from 'lodash/mapKeys'; import path from 'path'; -import { localeDirectory, localeReferenceDirectory } from './export'; +import { localeDirectory, localeReferenceDirectory } from './import'; export async function readLocalTexts() { const texts = JSON.parse( diff --git a/packages/cms/src/lokalize/logic/export.ts b/packages/cms/src/lokalize/logic/import.ts similarity index 97% rename from packages/cms/src/lokalize/logic/export.ts rename to packages/cms/src/lokalize/logic/import.ts index dff4cab1e9..f35d39a1ab 100644 --- a/packages/cms/src/lokalize/logic/export.ts +++ b/packages/cms/src/lokalize/logic/import.ts @@ -1,5 +1,5 @@ /** - * This script exports LokalizeText documents from Sanity as a locale JSON file, + * This script imports LokalizeText documents from Sanity as a locale JSON file, * and strips any keys that have been marked by the key-mutations.csv file as a * result of text changes via the CLI. */ @@ -31,7 +31,7 @@ export const localeReferenceDirectory = path.resolve( '.lokalize-reference' ); -export async function exportLokalizeTexts({ +export async function importLokalizeTexts({ dataset, appendDocumentIdToKey = false, }: { diff --git a/packages/cms/src/lokalize/logic/index.ts b/packages/cms/src/lokalize/logic/index.ts index 82de2bc4f4..047f23acbd 100644 --- a/packages/cms/src/lokalize/logic/index.ts +++ b/packages/cms/src/lokalize/logic/index.ts @@ -1,3 +1,3 @@ -export * from './export'; +export * from './import'; export * from './files'; export * from './mutations';