Update scripts in monorepo and subprojects (#4141)

* feat: update scripts

- Updated scripts
- Added shortcut scripts
- Updated docs
- Updated linter

* fix: added missing changes

* feat: rename lokalize scripts

* feat: updated github workflow with new scripts

.github/workflows/main.yml

@@ -66,18 +66,18 @@ jobs:
         run: yarn install
 
       - name: Build icon package
-        run: yarn workspace @corona-dashboard/icons build
+        run: yarn build:icons
 
       - name: Build common module
-        run: yarn workspace @corona-dashboard/common build
+        run: yarn build:common
 
-      - name: Export texts (develop)
+      - name: Import texts (develop)
         if: ${{ env.BRANCH_NAME != 'master' }}
-        run: yarn workspace @corona-dashboard/cms lokalize:export --clean-json
+        run: yarn cms:lokalize-import --clean-json
 
-      - name: Export texts (production)
+      - name: Import texts (production)
         if: ${{ env.BRANCH_NAME == 'master' }}
-        run: yarn workspace @corona-dashboard/cms lokalize:export --dataset="production" --clean-json
+        run: yarn cms:lokalize-import:prd --clean-json
 
       - name: Typecheck all code
         run: yarn typecheck
@@ -106,13 +106,13 @@ jobs:
         run: yarn install
 
       - name: Build common module
-        run: yarn workspace @corona-dashboard/common build
+        run: yarn build:common
 
       - name: Build icons module
-        run: yarn workspace @corona-dashboard/icons build
+        run: yarn build:icons
 
       - name: Export texts
-        run: yarn workspace @corona-dashboard/cms lokalize:export --clean-json
+        run: yarn cms:lokalize-import --clean-json
 
       - name: Run tests
         run: yarn test:ci

.github/workflows/sync-sanity-lokalize.yml

@@ -40,7 +40,7 @@ jobs:
           SANITY_TOKEN: ${{ secrets.sanity_token }}
 
       - name: Export production texts
-        run: yarn workspace @corona-dashboard/cms lokalize:export --dataset=production --clean-json
+        run: yarn workspace @corona-dashboard/cms lokalize:import --dataset=production --clean-json
 
       - name: Typecheck all code with production texts
         run: yarn workspace @corona-dashboard/app typecheck

Dockerfile

@@ -74,7 +74,7 @@ FROM builder
 RUN yarn download \
 && yarn workspace @corona-dashboard/cli validate-json-all \
 && yarn workspace @corona-dashboard/cli validate-last-values --fail-early \
-&& yarn workspace @corona-dashboard/cms lokalize:export --dataset=$NEXT_PUBLIC_SANITY_DATASET \
+&& yarn workspace @corona-dashboard/cms lokalize:import --dataset=$NEXT_PUBLIC_SANITY_DATASET \
 && yarn workspace @corona-dashboard/app build \
 && mkdir -p /app/packages/app/public/images/choropleth \
 && addgroup -g 1001 -S nodejs \

docs/ci.md

@@ -37,7 +37,7 @@ The build process follows these steps:
   - Run the unit tests
   - Download the server data files
   - Run all the JSON data validations
-  - Run lokalize:export to retrieve all the translations from Sanity
+  - Run lokalize:import to retrieve all the translations from Sanity
   - Run the NextJs build
 
 [Back to index](index.md)

docs/developers.md

@@ -1,5 +1,15 @@
 # Developer Notes
 
+## Shortcut scripts
+
+The monorepo contains projects which the developer can access with shortcut scripts at the root of the project. Instead of writing the path in the CLI like `yarn workspace @corona-dashboard cms lokalize:import`, we can access the script by writing `yarn cms:lokalize-import` to bring your local JSON files up-to-date with the Sanity dataset. Another example is using `yarn build:app` instead of `yarn workspace @corona-dashboard/app build` to make a build during the release procedure. Lastly, running `yarn check-all` will make sure the code, data and tests work as expected, instead of running scripts individually.
+
+Available shortcut scripts:
+
+- `yarn check-all`
+- `yarn cms:[command]`
+- `yarn build:[command]`
+
 ## Common and icon packages
 
 The monorepo contains two shared packages: common and icons.

docs/feature-flags.md

@@ -89,7 +89,4 @@ already existing metric, then declaring those properties will mean that only the
 checks are performed on those. If the whole schema / metric is new, as it often
 is, then specifying only the `metricName` is sufficient.
 
-The script that validates the features (and runs at build-time) can be
-triggered locally with `yarn workspace@corona-dashboard/cli validate-features`.
-
 [Back to index](index.md)

docs/lokalize-texts.md

@@ -9,7 +9,7 @@ our codebase. Even though these texts now live as documents in Sanity, we still
 refer to them as "lokalize texts".
 
 Each text string lives as a document in the CMS, of which each has a unique "key"
-corresponding to a path. These documents are then exported as JSON files (one
+corresponding to a path. These documents are then imported as JSON files (one
 for each language) and consumed by the application as static data.
 
 This document describes how to manage and make changes to these texts.
@@ -19,15 +19,15 @@ This document describes how to manage and make changes to these texts.
 In summary these are the most important things you should be aware of:
 
 - Commands are run with `yarn lokalize:[command]` from the `packages/cms` root
-- The `export` command brings your local JSON files up-to-date with the Sanity
+- The `import` command brings your local JSON files up-to-date with the Sanity
   dataset. The TypeScript compiler will error when your JSON files do not
   contain all the texts which are referenced in the code.
-- The JSON export contains document ids as part of the keys. You can make
+- The JSON import contains document ids as part of the keys. You can make
   changes locally to the **app/src/locale/nl_export.json** file. Add new keys,
   delete them or rename and move existing ones. After you make changes run the
   `lokalize:apply-json-edits` script. This will give you a list of changes and
   you can decide which ones to apply. Changes are written in a mutation log
-  file, and at the end the JSON is re-exported to reflect all changes.
+  file, and at the end the JSON is re-imported to reflect all changes.
   It is recommended to run `lokalize:apply-json-edits` after the feature has
   been finished, right before merging it to develop since during development
   changes to the **nl_export.json** file might fluctuate.
@@ -63,26 +63,26 @@ scripts.
 Merge conflicts in this file are common. You should always "accept both changes"
 when resolving conflicts, so that none of the lines are ever deleted.
 
-## Export
+## Import
 
 The application reads its locale strings from
 **packages/app/public/nl_export.json** and **packages/app/public/en_export.json**.
-These JSONs are exported from the Sanity lokalize documents, but they are not
-part of the repository. Therefore, you will regularly need to run `yarn lokalize:export` in
+These JSONs are imported from the Sanity lokalize documents, but they are not
+part of the repository. Therefore, you will regularly need to run `yarn lokalize:import` in
 order to keep your local JSON file up-to-date with the Sanity dataset.
 
 The JSONs will include Sanity document ids in every leaf-key which are used to
 detect add-/delete-/move-actions of texts (`some_key__@__{document_id}`). These
 ids would result in compile- and run-time errors, but there's a workaround:
 
-- compile-time: every export will also emit a `site-text.d.ts` with a `SiteText`
+- compile-time: every import will also emit a `site-text.d.ts` with a `SiteText`
   interface. This interface is used to type the imported JSONs.
 - run-time: on load of the app all ids will be removed from the keys.
 
 The runtime workaround would be a waste of resources on production, so for this
 reason we use the `--clean-json` flag to ignore document ids.
 
-### Export/Import technical details
+### Technical details
 
 In Sanity each lokalize value is stored as a separate `lokalizeText` document.
 The key property in this document represents the flattened property path in **nl_export.json**
@@ -107,7 +107,7 @@ The root key (so in this case `accessibility`) will be saved as the subject prop
 This allows us to filter on this subject under the Sanity `Lokalize` menu item. Making it a little
 easier for an editor to lookup a specific key in the menu.
 
-So, `yarn lokalize:export` simply retrieves all of the `lokalizeText` documents, replaces their ids in the root part of the path
+So, `yarn lokalize:import` simply retrieves all of the `lokalizeText` documents, replaces their ids in the root part of the path
 and unflattens this list of paths into a JSON file. (The Dutch texts are serialized as **nl_export.json**, the English as **en_export.json**).
 `yarn lokalize:apply-json-edits` does the exact opposite, it takes the **nl_export.json** file, flattens the file into an array of
 paths and saves those to Sanity. Obviously there is a bunch of logic involved that checks if a path is new or existing, but for those
@@ -116,7 +116,7 @@ details we refer to the source code.
 ## How to Add, Delete or Move Texts
 
 First make sure the JSONs include document ids (`some_key__@__{document_id}`).
-Run `yarn lokalize:export` if these are not yet present in your JSONs.
+Run `yarn lokalize:import` if these are not yet present in your JSONs.
 
 You can add, delete and move keys by mutating the **nl_export.json** file. The
 sync script will automagically detect additions, deletions or moved lokalize
@@ -129,7 +129,7 @@ file.
 New texts will only have an NL (Dutch) string when they are added and will use NL as a
 fallback.
 
-After syncing texts, the export script is called to update your local JSON file
+After syncing texts, the import script is called to update your local JSON file
 and re-generate the SiteText type interface.
 
 ### Delete/Move Mutations
@@ -139,7 +139,7 @@ dataset, we can not simply remove a lokalize text document from the dataset
 without potentially breaking other branches.
 
 For this reason, when you delete a lokalize text, it will append the delete
-action to the mutations file but not actually delete the document. The export
+action to the mutations file but not actually delete the document. The import
 filters out any deletions that were logged to the mutations file. So in effect
 you end up with a local JSON file that has the deleted key removed, and the TS
 compiler sees the correct dataset.
@@ -148,7 +148,7 @@ The actual deletions from Sanity only happen in the `sync-after-feature` phase,
 describe below.
 
 The same goes for move mutations. When you apply the move in your feature branch
-it will be simulated in the JSON export, but not yet applied to the CMS
+it will be simulated in the JSON import, but not yet applied to the CMS
 documents. During sync-after-feature the move is finalized by changing the key
 and subject properties of the targeted document. This preserves the document
 history and drafts.

docs/release-procedure.md

@@ -6,7 +6,8 @@
 2. Merge the develop branch into the release branch.
 3. Do a full build with a Sanity export from production to make sure everything works.
 
-   Use `yarn lokalize:export --dataset=production` to create the Sanity export.
+   Use `yarn cms:lokalize-import:prd` to create the Sanity export.
+   Use `yarn build:app` to create a build with the latest production dataset
 
    Test the build locally by running `yarn start`.
 
@@ -23,9 +24,9 @@
 8. Hit the publish button in the GitHub release draft. This should tag the
    correct commit in `master`.
 9. Important before running the next step is building the common package to avoid any typescript errors,
-   so run `yarn workspace @corona-dashboard/common build`.
+   so run `yarn build:common`.
 10. After we are sure we are moving to production with the release as it is,
-    run the `yarn workspace @corona-dashboard/cms lokalize:sync-after-release`
+    run the `yarn cms:sync-after-release`
     script. This script will perform any necessary text mutations for keys that
     have been moved, and also cleans up any deleted texts from the Sanity
     production dataset. For more information [read this](/docs/lokalize-texts.md#sync-after-release).
@@ -99,7 +100,7 @@ the `key-mutations.csv` file and commit this to the release branch.
 ## CMS Deploy
 
 Sometimes a release also contains new configurations for the CMS. These are
-deployed via the Sanity CLI as `yarn deploy`. These deployments do
+deployed via the Sanity CLI by running `yarn cms:deploy` in the root of the project. These deployments do
 not usually need to be done simultaneously with the release. If they are
 backwards-compatible they could be performed at any time and also repeatedly.
 

eslint-overrides.json

@@ -19,6 +19,7 @@
         "@typescript-eslint/no-var-requires": 0,
         "@typescript-eslint/explicit-module-boundary-types": 0,
         "@typescript-eslint/no-use-before-define": 0,
+        "@typescript-eslint/triple-slash-reference": 0,
         "@typescript-eslint/no-unused-vars": [
           2,
           {

package.json

@@ -9,21 +9,30 @@
     "bootstrap": "yarn workspaces foreach run bootstrap",
     "clean": "yarn workspaces foreach run clean && npx rimraf .build-cache",
     "nuke": "yarn workspaces foreach run nuke && npx rimraf node_modules",
-    "typecheck": "yarn workspaces foreach run typecheck",
-    "lint": "yarn workspaces foreach run lint",
+    "typecheck": "echo 'Starting typecheck...' && yarn workspaces foreach run typecheck",
+    "lint": "echo 'Starting lint...' && yarn workspaces foreach run lint",
     "lint:fix": "yarn workspaces foreach run lint --fix",
     "format": "yarn workspaces foreach run format",
     "dev": "yarn workspace @corona-dashboard/app dev",
     "download": "yarn workspace @corona-dashboard/app download",
     "start": "yarn workspace @corona-dashboard/app start",
+    "check-all": "yarn lint && yarn typecheck && yarn validate-json-all && yarn test",
     "validate-json-all": "yarn workspace @corona-dashboard/cli validate-json-all",
     "validate-json-single": "yarn workspace @corona-dashboard/cli validate-json-single",
     "validate-last-values": "yarn workspace @corona-dashboard/cli validate-last-values",
-    "validate-features": "yarn workspace @corona-dashboard/cli validate-features",
-    "generate-typescript": "yarn workspace @corona-dashboard/cli generate-typescript",
+    "generate-data-types": "yarn workspace @corona-dashboard/cli generate-data-types",
     "e2e": "yarn workspace @corona-dashboard/e2e e2e",
     "e2e:ci": "yarn workspace @corona-dashboard/e2e e2e:ci",
-    "cms": "yarn workspace @corona-dashboard/cms dev"
+    "cms": "yarn workspace @corona-dashboard/cms dev",
+    "cms:lokalize-import": "yarn workspace @corona-dashboard/cms lokalize:import",
+    "cms:lokalize-import:prd": "yarn workspace @corona-dashboard/cms lokalize:import --dataset=production",
+    "cms:apply-json-edits": "yarn workspace @corona-dashboard/cms lokalize:apply-json-edits",
+    "cms:generate-types": "yarn workspace @corona-dashboard/cms lokalize:generate-types",
+    "cms:sync-after-release": "yarn workspace @corona-dashboard/cms lokalize:sync-after-release",
+    "cms:deploy": "yarn workspace @corona-dashboard/cms deploy",
+    "build:app": "yarn workspace @corona-dashboard/app build",
+    "build:common": "yarn workspace @corona-dashboard/common build",
+    "build:icons": "yarn workspace @corona-dashboard/icons build"
   },
   "workspaces": {
     "packages": [

packages/app/README.md

@@ -45,9 +45,9 @@ The "Lokalize" part of Sanity is exported and consumed by the app as JSON. You w
 need to run this script regularly as an outdated JSON file will result in
 compile or build-time errors.
 
-`yarn workspace @corona-dashboard/cms lokalize:export`
+`yarn cms:lokalize-import`
 
-Alternatively you can run this from `packages/cms` as `yarn lokalize:export`
+Alternatively you can run this from `packages/cms` as `yarn lokalize:import`
 
 To learn more about the rationale behind Lokalize and how it works [read the documentation](/packages/cms/README.md#lokalize-texts).
 

packages/app/package.json

@@ -162,7 +162,7 @@
     "lint": "eslint .",
     "typecheck": "tsc --pretty --noEmit",
     "trace": "tsc --pretty --noEmit --generateTrace \"trace-$(date +%Y%m%d%H%M)\" --composite false --incremental false",
-    "download": "node ./infra/download.js",
+    "download": "npx rimraf ./public/json/* && node ./infra/download.js",
     "clean": "npx rimraf .next",
     "nuke": "yarn clean && npx rimraf node_modules"
   },

packages/cli/package.json

@@ -32,11 +32,10 @@
     "lint": "eslint .",
     "typecheck": "tsc --pretty --noEmit",
     "calculate-boundingboxes": "yarn workspace @corona-dashboard/common build && ts-node src/scripts/calculate-bounding-box-municipalities.ts",
-    "validate-json": "yarn run validate-json-all",
     "validate-json-all": "yarn workspace @corona-dashboard/common build && ts-node src/scripts/validate-json-all.ts",
     "validate-json-single": "yarn workspace @corona-dashboard/common build && ts-node src/scripts/validate-json-single.ts",
     "validate-last-values": "yarn workspace @corona-dashboard/common build && ts-node src/scripts/validate-last-values.ts",
-    "generate-typescript": "yarn workspace @corona-dashboard/common build && ts-node src/scripts/generate-data-types.ts",
+    "generate-data-types": "yarn workspace @corona-dashboard/common build && ts-node src/scripts/generate-data-types.ts",
     "generate-data-structures": "ts-node src/scripts/generate-data-structures.ts"
   },
   "devDependencies": {

packages/cms/package.json

@@ -4,25 +4,24 @@
   "version": "0.0.0",
   "license": "EUROPEAN UNION PUBLIC LICENCE v. 1.2",
   "scripts": {
-    "bootstrap": "yarn lokalize:export",
     "clean": "npx rimraf dist .sanity-asset-cache .lokalize-reference",
     "nuke": "yarn clean && npx rimraf node_modules",
     "dev": "yarn workspace @corona-dashboard/common build && yarn workspace @corona-dashboard/icons build && yarn workspace @corona-dashboard/cli generate-data-structures && sanity start",
     "build": "sanity build",
     "deploy": "yarn workspace @corona-dashboard/common build && yarn workspace @corona-dashboard/icons build && yarn workspace @corona-dashboard/cli generate-data-structures && sanity deploy",
-    "test": "sanity check",
-    "test:ci": "yarn run test",
+    "test": "echo 'Sanity check is deprecated \n@TODO add sanity test script'",
+    "test:ci": "echo 'Sanity check is deprecated \n@TODO add sanity test ci script'",
     "typecheck": "tsc --pretty --noEmit",
     "lint": "eslint .",
-    "lokalize:export": "ts-node src/lokalize/export.ts",
+    "lokalize:import": "ts-node src/lokalize/import.ts",
     "lokalize:add": "ts-node src/lokalize/lokalize-add.ts",
     "lokalize:dedupe": "ts-node src/lokalize/clean-duplicates.ts",
     "lokalize:apply-json-edits": "ts-node src/lokalize/apply-json-edits.ts",
     "lokalize:generate-types": "ts-node src/lokalize/generate-types.ts",
     "lokalize:sync-after-release": "ts-node src/lokalize/sync-after-release.ts",
     "lokalize:sync-after-feature": "ts-node src/lokalize/sync-after-feature.ts",
     "lokalize:sync-prd-to-dev": "ts-node src/lokalize/sync-prd-to-dev.ts",
-    "lokalize:strip-unused-keys": "yarn lokalize:export --clean-json && ts-node src/lokalize/strip-unused-keys.ts && yarn lokalize:export --clean-json && yarn workspace @corona-dashboard/app typecheck",
+    "lokalize:strip-unused-keys": "yarn lokalize:import --clean-json && ts-node src/lokalize/strip-unused-keys.ts && yarn lokalize:import --clean-json && yarn workspace @corona-dashboard/app typecheck",
     "elements:add": "ts-node src/elements/add.ts",
     "format": "prettier --write \"src/**/*.{js,ts,tsx}\" \"*.{js,json,md,yml}\""
   },

packages/cms/src/lokalize/apply-json-edits.ts

@@ -4,7 +4,7 @@ import prompts from 'prompts';
 import { getClient } from '../client';
 import {
   appendTextMutation,
-  exportLokalizeTexts,
+  importLokalizeTexts,
   getLocalMutations,
   readReferenceTexts,
 } from './logic';
@@ -21,7 +21,7 @@ import { createTextDocument } from './logic/create-text-document';
 
   assert(
     referenceTexts,
-    `Failed to read reference texts. Please run lokalize:export first.`
+    `Failed to read reference texts. Please run lokalize:import first.`
   );
 
   const mutations = await getLocalMutations(referenceTexts);
@@ -110,7 +110,7 @@ import { createTextDocument } from './logic/create-text-document';
 
   console.log('Updating text export...');
 
-  await exportLokalizeTexts({
+  await importLokalizeTexts({
     dataset: 'development',
     appendDocumentIdToKey: true,
   });