Add support for multiple schemas, YAML support (#51)

* Add support for multiple schemas, YAML support * Update example deps * Remove undici dep * Require Node v18 or greater * Add URL test
terrazzoapp · Jul 14, 2023 · e1e18c1 · e1e18c1
1 parent 9773fe5
commit e1e18c1
Show file tree

Hide file tree

Showing 23 changed files with 534 additions and 109 deletions.
diff --git a/.changeset/fluffy-poets-melt.md b/.changeset/fluffy-poets-melt.md
@@ -0,0 +1,5 @@
+---
+'@cobalt-ui/cli': patch
+---
+
+Breaking: require Node v18 or greater
diff --git a/.changeset/ninety-readers-promise.md b/.changeset/ninety-readers-promise.md
@@ -0,0 +1,6 @@
+---
+'@cobalt-ui/core': minor
+'@cobalt-ui/cli': minor
+---
+
+Allow combining multiple schemas
diff --git a/.changeset/rude-eggs-nail.md b/.changeset/rude-eggs-nail.md
@@ -0,0 +1,6 @@
+---
+'@cobalt-ui/core': minor
+'@cobalt-ui/cli': minor
+---
+
+Allow loading tokens as YAML
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -16,7 +16,7 @@ jobs:
           node-version: 20
       - uses: pnpm/action-setup@v2
         with:
-          version: latest
+          version: 8
       - run: pnpm i
       - run: pnpm run lint
   test:
@@ -28,7 +28,7 @@ jobs:
           node-version: 20
       - uses: pnpm/action-setup@v2
         with:
-          version: latest
+          version: 8
       - run: pnpm i
       - run: pnpm run build
       - run: pnpm test
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -9,14 +9,13 @@ jobs:
   changelog:
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v2
         with:
-          version: latest
-      - name: Setup Node
-        uses: actions/setup-node@v3
-        with:
-          node-version: 20
+          version: 8
       - run: pnpm i
       - run: pnpm run build
       - uses: changesets/action@v1

diff --git a/docs/src/pages/docs/reference/config.md b/docs/src/pages/docs/reference/config.md
@@ -20,7 +20,29 @@ export default {
 };
 ```
 
-### Loading tokens from npm
+### Loading from YAML
+
+Cobalt supports `tokens.json` as YAML as well:
+
+```js
+export default {
+  tokens: './tokens.yaml',
+};
+```
+
+> ⚠️ Note the file must end in `.yml` or `.yaml` to take effect
+
+### Loading from URL
+
+Cobalt can load tokens from any **publicly-available** URL:
+
+```js
+export default {
+  tokens: 'https://my-bucket.s3.amazonaws.com/tokens.json',
+};
+```
+
+### Loading from npm
 
 To load tokens from an npm package, update `config.tokens` to point to the **full JSON path** (not merely the root package):
 
@@ -31,6 +53,20 @@ To load tokens from an npm package, update `config.tokens` to point to the **ful
 +   tokens: "@my-scope/my-tokens/tokens.json", // ✅ Cobalt can locate this just fine
 ```
 
+### Loading multiple schemas
+
+Cobalt supports loading multiple tokens schemas by passing an array:
+
+```js
+export default {
+  tokens: ['./base.json', './theme.json', './overrides.json'],
+};
+```
+
+Cobalt will flatten these schemas in order, with the latter entries overriding the former if there are any conflicts. The final result of all the combined schemas **must** result in a valid tokens.json.
+
+> ⚠️ **Warning** All aliases must refer to the same document, e.g. don’t try to include filenames such as `{./theme.json#/color.action.50}`. Reference it as if it were in the same file.
+
 ## Token Type Options
 
 Some token types allow for extra configuration in transformation. Here are all the available settings, along with defaults.

diff --git a/examples/ibm/package.json b/examples/ibm/package.json
@@ -8,9 +8,9 @@
     "update": "node scripts/update"
   },
   "devDependencies": {
-    "@carbon/colors": "^11.6.0",
-    "@carbon/icons": "^11.10.0",
-    "@carbon/type": "^11.10.0",
+    "@carbon/colors": "^11.17.1",
+    "@carbon/icons": "^11.22.1",
+    "@carbon/type": "^11.20.1",
     "@cobalt-ui/cli": "workspace:*",
     "@cobalt-ui/plugin-css": "workspace:*",
     "@cobalt-ui/plugin-js": "workspace:*",

diff --git a/examples/salesforce/package.json b/examples/salesforce/package.json
@@ -12,8 +12,8 @@
     "@cobalt-ui/plugin-css": "workspace:*",
     "@cobalt-ui/plugin-js": "workspace:*",
     "@cobalt-ui/plugin-sass": "workspace:*",
-    "@salesforce-ux/design-system": "^2.16.2",
+    "@salesforce-ux/design-system": "^2.21.3",
     "better-color-tools": "^0.12.3",
-    "postcss": "^8.4.16"
+    "postcss": "^8.4.26"
   }
 }
diff --git a/packages/cli/bin/cli.js b/packages/cli/bin/cli.js
@@ -10,6 +10,7 @@ import {DIM, FG_BLUE, FG_RED, FG_GREEN, FG_YELLOW, UNDERLINE, RESET} from '@coba
 import chokidar from 'chokidar';
 import fs from 'node:fs';
 import {performance} from 'node:perf_hooks';
+import yaml from 'js-yaml';
 import {fileURLToPath, URL} from 'node:url';
 import parser from 'yargs-parser';
 import {init as initConfig} from '../dist/config.js';
@@ -62,25 +63,25 @@ async function main() {
   }
   let config;
   try {
-    config = await loadConfig(resolveConfig(configPath));
+    // if running `co check [tokens]`, don’t load config from file
+    if (cmd === 'check' && args[0]) {
+      config = await initConfig({tokens: args[0]}, cwd);
+    } else {
+      config = await loadConfig(resolveConfig(configPath));
+    }
   } catch (err) {
     console.error(`  ${FG_RED}✘  ${err.message || err}${RESET}`);
     process.exit(1);
   }
 
   switch (cmd) {
     case 'build': {
-      if (!fs.existsSync(config.tokens)) {
-        console.error(`  ${FG_RED}✘  Could not locate ${config.tokens}. To create one, run \`npx cobalt init\`.${RESET}`);
-        process.exit(1);
-      }
-
       if (!Array.isArray(config.plugins) || !config.plugins.length) {
         console.error(`  ${FG_RED}✘  No plugins defined! Add some in ${configPath || 'tokens.config.js'}${RESET}`);
         process.exit(1);
       }
 
-      let rawSchema = JSON.parse(fs.readFileSync(config.tokens), 'utf8');
+      let rawSchema = await loadTokens(config.tokens);
 
       const dt = new Intl.DateTimeFormat('en-us', {
         hour: '2-digit',
@@ -95,17 +96,16 @@ async function main() {
       if (result.errors) process.exit(1);
 
       if (watch) {
-        const tokenWatcher = chokidar.watch(fileURLToPath(config.tokens));
-        const tokensYAML = config.tokens.href.replace(cwd.href, '');
+        const tokenWatcher = chokidar.watch(config.tokens.map((filepath) => fileURLToPath(filepath)));
         tokenWatcher.on('change', async (filePath) => {
           try {
-            rawSchema = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+            rawSchema = await loadTokens(config.tokens);
             result = await build(rawSchema, config);
             if (result.errors || result.warnings) {
               printErrors(result.errors);
               printWarnings(result.warnings);
             } else {
-              console.log(`${DIM}${dt.format(new Date())}${RESET} ${FG_BLUE}Cobalt${RESET} ${FG_YELLOW}${tokensYAML}${RESET} updated ${FG_GREEN}✔${RESET}`);
+              console.log(`${DIM}${dt.format(new Date())}${RESET} ${FG_BLUE}Cobalt${RESET} ${FG_YELLOW}${filePath}${RESET} updated ${FG_GREEN}✔${RESET}`);
             }
           } catch (err) {
             printErrors([err.message || err]);
@@ -116,7 +116,7 @@ async function main() {
           try {
             console.log(`${DIM}${dt.format(new Date())}${RESET} ${FG_BLUE}Cobalt${RESET} ${FG_YELLOW}Config updated. Reloading…${RESET}`);
             config = await loadConfig(filePath);
-            rawSchema = JSON.parse(fs.readFileSync(config.tokens, 'utf8'));
+            rawSchema = await loadTokens(config.tokens);
             result = await build(rawSchema, config);
           } catch (err) {
             printErrors([err.message || err]);
@@ -136,22 +136,9 @@ async function main() {
       break;
     }
     case 'check': {
-      let tokensPath = config.tokens;
-      if (args[0]) {
-        tokensPath = new URL(args[0], cwd);
-        if (!fs.existsSync(tokensPath)) {
-          console.error(`Expected format: cobalt check ./path/to/tokens.json. Could not locate ${args[0]}.`);
-          process.exit(1);
-        }
-      }
-
-      if (!fs.existsSync(tokensPath)) {
-        console.error(`  ${FG_RED}✘  Could not locate ${tokensPath}. To create one, run \`npx cobalt init\`.${RESET}`);
-        process.exit(1);
-      }
-
-      let rawSchema = JSON.parse(fs.readFileSync(tokensPath, 'utf8'));
-      console.log(`${UNDERLINE}${fileURLToPath(tokensPath)}${RESET}`);
+      const rawSchema = await loadTokens(config.tokens);
+      const filepath = config.tokens[0];
+      console.log(`${UNDERLINE}${filepath.protocol === 'file:' ? fileURLToPath(filepath) : filepath}${RESET}`);
       const {errors, warnings} = parse(rawSchema, config); // will throw if errors
       if (errors || warnings) {
         printErrors(errors);
@@ -215,6 +202,77 @@ async function loadConfig(configPath) {
   return await initConfig(userConfig, configPath instanceof URL ? configPath : `file://${process.cwd()}/`);
 }
 
+/** load tokens */
+async function loadTokens(tokenPaths) {
+  const rawTokens = [];
+
+  // download/read
+  for (const filepath of tokenPaths) {
+    const pathname = filepath.pathname.toLowerCase();
+    const isYAMLExt = pathname.endsWith('.yaml') || pathname.endsWith('.yml');
+    if (filepath.protocol === 'http:' || filepath.protocol === 'https:') {
+      try {
+        const res = await globalThis.fetch(filepath, {method: 'GET', headers: {Accept: '*/*', 'User-Agent': 'Mozilla/5.0 Gecko/20100101 Firefox/116.0'}});
+        const raw = await res.text();
+        if (isYAMLExt || res.headers.get('content-type').includes('yaml')) {
+          rawTokens.push(yaml.load(raw));
+        } else {
+          rawTokens.push(JSON.parse(raw));
+        }
+      } catch (err) {
+        console.error(`  ${FG_RED}✘  ${filepath.href}: ${err}${RESET}`);
+      }
+    } else {
+      if (fs.existsSync(filepath)) {
+        try {
+          const raw = fs.readFileSync(filepath, 'utf8');
+          if (isYAMLExt) {
+            rawTokens.push(yaml.load(raw));
+          } else {
+            rawTokens.push(JSON.parse(raw));
+          }
+        } catch (err) {
+          console.error(`  ${FG_RED}✘  ${filepath.href}: ${err}${RESET}`);
+        }
+      } else {
+        console.error(`  ${FG_RED}✘  Could not locate ${filepath}. To create one, run \`npx cobalt init\`.${RESET}`);
+        process.exit(1);
+      }
+    }
+  }
+
+  // combine
+  const tokens = {};
+  for (const subtokens of rawTokens) {
+    merge(tokens, subtokens);
+  }
+
+  return tokens;
+}
+
+/** Merge JSON B into A */
+function merge(a, b) {
+  if (Array.isArray(b)) {
+    console.error(`  ${FG_RED}✘ Internal error parsing tokens file.${RESET}`); // oops
+    process.exit(1);
+    return;
+  }
+  for (const [k, v] of Object.entries(b)) {
+    // overwrite if:
+    // - this key doesn’t exist on a, or
+    // - this is a token with a $value (don’t merge tokens! overwrite!), or
+    // - this is a primitive value (it’s the user’s responsibility to merge these correctly)
+    if (!(k in a) || Array.isArray(v) || typeof v !== 'object' || (typeof v === 'object' && '$value' in v)) {
+      a[k] = v;
+      continue;
+    }
+    // continue
+    if (typeof v === 'object' && !Array.isArray(v)) {
+      merge(a[k], v);
+    }
+  }
+}
+
 /** Print time elapsed */
 function time(start) {
   const diff = performance.now() - start;

diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -17,6 +17,9 @@
     "style tokens",
     "style system"
   ],
+  "engines": {
+    "node": ">=18.0.0"
+  },
   "repository": {
     "type": "git",
     "url": "https://github.com/drwpow/cobalt-ui.git",
@@ -42,12 +45,14 @@
     "better-color-tools": "^0.12.3",
     "chokidar": "^3.5.3",
     "dotenv": "^16.3.1",
+    "js-yaml": "^4.1.0",
     "piscina": "^3.2.0",
     "svgo": "^3.0.2",
     "yargs-parser": "^21.1.1"
   },
   "devDependencies": {
     "@types/node": "^20.4.2",
+    "execa": "^7.1.1",
     "figma-api": "^1.11.0",
     "npm-run-all": "^4.1.5",
     "vitest": "^0.33.0"

diff --git a/packages/cli/src/config.ts b/packages/cli/src/config.ts
@@ -11,23 +11,42 @@ export async function init(userConfig: Config, cwd: URL): Promise<ResolvedConfig
   // config.tokens
   // default
   if (userConfig.tokens === undefined) {
-    config.tokens = new URL('./tokens.json', cwd);
-  }
-  // validate
-  else if (typeof userConfig.tokens !== 'string') {
-    throw new Error(`[config] tokens must be string, received ${typeof userConfig.tokens}`);
+    config.tokens = ['./tokens.json' as any]; // will be normalized in next step
+  } else if (typeof userConfig.tokens === 'string') {
+    config.tokens = [userConfig.tokens as any]; // will be normalized in next step
+  } else if (Array.isArray(userConfig.tokens)) {
+    config.tokens = [];
+    for (const file of userConfig.tokens) {
+      if (typeof file === 'string') {
+        config.tokens.push(file as any); // will be normalized in next step
+      } else {
+        throw new Error(`[config] tokens must be array of strings, encountered unexpected path "${file}"`);
+      }
+    }
+  } else {
+    throw new Error(`[config] tokens must be string or array of strings, received ${typeof userConfig.tokens}`);
   }
   // normalize
-  else {
-    const tokensPath = new URL(config.tokens as any, cwd);
-    if (fs.existsSync(tokensPath)) {
-      config.tokens = tokensPath;
-    }
-    // otherwise, try Node resolution
-    else {
-      const nodeResolved = require.resolve(userConfig.tokens, {paths: [fileURLToPath(cwd), process.cwd(), import.meta.url]});
-      if (!fs.existsSync(nodeResolved)) throw new Error(`Can’t locate "${userConfig.tokens}". Does the path exist?`);
-      config.tokens = new URL(`file://${nodeResolved}`);
+  for (let i = 0; i < config.tokens.length; i++) {
+    const filepath = config.tokens[i] as any as string;
+    const isRemote = filepath.startsWith('//') || filepath.startsWith('http:') || filepath.startsWith('https:');
+    if (isRemote) {
+      try {
+        config.tokens[i] = new URL(filepath);
+      } catch {
+        throw new Error(`[config] invalid URL "${filepath}"`);
+      }
+    } else {
+      const tokensPath = new URL(filepath, cwd);
+      if (fs.existsSync(tokensPath)) {
+        config.tokens[i] = tokensPath;
+      }
+      // otherwise, attempt Node resolution (sometimes it do be like that)
+      else {
+        const nodeResolved = require.resolve(filepath, {paths: [fileURLToPath(cwd), process.cwd(), import.meta.url]});
+        if (!fs.existsSync(nodeResolved)) throw new Error(`Can’t locate "${userConfig.tokens}". Does the path exist?`);
+        config.tokens[i] = new URL(`file://${nodeResolved}`);
+      }
     }
   }