From b2e014f9ade712d32be1d7403a8d06de07f01e1c Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Fri, 5 Jul 2024 09:22:32 +0200 Subject: [PATCH] fix(cspell-tools): support adding directives (#5860) --- .../cspell-tools.config.schema.json | 22 +++++++++--- .../src/compiler/CompileOptions.ts | 12 +++++++ .../wordListCompiler.test.ts.snap | 7 ++++ packages/cspell-tools/src/compiler/compile.ts | 21 +++++++++-- .../src/compiler/wordListCompiler.test.ts | 35 ++++++++++++++++--- .../src/compiler/wordListCompiler.ts | 8 +++-- packages/cspell-tools/src/config/config.ts | 23 ++++++++++-- 7 files changed, 112 insertions(+), 16 deletions(-) diff --git a/packages/cspell-tools/cspell-tools.config.schema.json b/packages/cspell-tools/cspell-tools.config.schema.json index 93f3d70a6f6..2cbc10115b7 100644 --- a/packages/cspell-tools/cspell-tools.config.schema.json +++ b/packages/cspell-tools/cspell-tools.config.schema.json @@ -144,6 +144,13 @@ "description": "gzip the file?", "type": "boolean" }, + "dictionaryDirectives": { + "description": "Injects `cspell-dictionary` directives into the dictionary header.\n\nExample:\n\n```ini # cspell-dictionary: no-generate-alternatives ```\n\nKnown Directives: ```yaml\n- split # Tell the dictionary loader to split words\n- no-split # Tell the dictionary loader to not split words (default)\n- generate-alternatives # Tell the dictionary loader to generate alternate spellings (default)\n- no-generate-alternatives # Tell the dictionary loader to not generate alternate spellings ```", + "items": { + "type": "string" + }, + "type": "array" + }, "excludeWordsFrom": { "description": "Words from the sources that are found in `excludeWordsFrom` files will not be added to the dictionary.", "items": { @@ -156,7 +163,7 @@ "description": "Format of the dictionary." }, "generateNonStrict": { - "default": true, + "default": false, "description": "Generate lower case / accent free versions of words.", "type": "boolean" }, @@ -165,7 +172,7 @@ "type": "string" }, "sort": { - "default": ": true", + "default": true, "description": "Sort the words in the resulting dictionary. Does not apply to `trie` based formats.", "type": "boolean" }, @@ -221,8 +228,15 @@ "boolean" ] }, + "dictionaryDirectives": { + "description": "Injects `cspell-dictionary` directives into the dictionary header.\n\nExample:\n\n```ini # cspell-dictionary: no-generate-alternatives ```\n\nKnown Directives: ```yaml\n- split # Tell the dictionary loader to split words\n- no-split # Tell the dictionary loader to not split words (default)\n- generate-alternatives # Tell the dictionary loader to generate alternate spellings (default)\n- no-generate-alternatives # Tell the dictionary loader to not generate alternate spellings ```", + "items": { + "type": "string" + }, + "type": "array" + }, "generateNonStrict": { - "default": true, + "default": false, "description": "Generate lower case / accent free versions of words.", "type": "boolean" }, @@ -240,7 +254,7 @@ "type": "string" }, "sort": { - "default": ": true", + "default": true, "description": "Sort the words in the resulting dictionary. Does not apply to `trie` based formats.", "type": "boolean" }, diff --git a/packages/cspell-tools/src/compiler/CompileOptions.ts b/packages/cspell-tools/src/compiler/CompileOptions.ts index c2be78b249a..dffa49af33c 100644 --- a/packages/cspell-tools/src/compiler/CompileOptions.ts +++ b/packages/cspell-tools/src/compiler/CompileOptions.ts @@ -16,4 +16,16 @@ export interface CompileOptions { * @returns `true` to keep the word, `false` to exclude it. */ filter?: (word: string) => boolean; + + /** + * Injects `cspell-dictionary` directives into the dictionary header. + * + * Example: + * + * ```ini + * # cspell-dictionary: no-generate-alternatives + * ``` + * + */ + dictionaryDirectives?: string[] | undefined; } diff --git a/packages/cspell-tools/src/compiler/__snapshots__/wordListCompiler.test.ts.snap b/packages/cspell-tools/src/compiler/__snapshots__/wordListCompiler.test.ts.snap index 995d82aa1cd..8aa5366761e 100644 --- a/packages/cspell-tools/src/compiler/__snapshots__/wordListCompiler.test.ts.snap +++ b/packages/cspell-tools/src/compiler/__snapshots__/wordListCompiler.test.ts.snap @@ -1,5 +1,12 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html +exports[`Validate the wordListCompiler > Specify directives 1`] = ` +{ + "error": "", + "log": "", +} +`; + exports[`Validate the wordListCompiler > a simple hunspell dictionary depth 0 1`] = ` { "error": "", diff --git a/packages/cspell-tools/src/compiler/compile.ts b/packages/cspell-tools/src/compiler/compile.ts index 27988998876..ea5eeaf9091 100644 --- a/packages/cspell-tools/src/compiler/compile.ts +++ b/packages/cspell-tools/src/compiler/compile.ts @@ -53,6 +53,7 @@ export async function compile(request: CompileRequest, options?: CompileOptions) }; const conditional = options?.conditionalBuild || false; const checksumFile = resolveChecksumFile(request.checksumFile || conditional, rootDir); + const dictionaryDirectives = request.dictionaryDirectives; const dependencies = new Set(); @@ -60,7 +61,13 @@ export async function compile(request: CompileRequest, options?: CompileOptions) const keep = options?.filter?.(target) ?? true; if (!keep) continue; const adjustedTarget: Target = { ...targetOptions, ...target }; - const deps = await compileTarget(adjustedTarget, request, { rootDir, cwd, conditional, checksumFile }); + const deps = await compileTarget(adjustedTarget, request, { + rootDir, + cwd, + conditional, + checksumFile, + dictionaryDirectives, + }); deps.forEach((dep) => dependencies.add(dep)); } @@ -87,6 +94,7 @@ interface CompileTargetOptions { cwd: string | undefined; conditional: boolean; checksumFile: string | undefined; + dictionaryDirectives: string[] | undefined; } export async function compileTarget( @@ -98,6 +106,7 @@ export async function compileTarget( const { rootDir, cwd, checksumFile, conditional } = compileOptions; const { format, sources, trieBase, sort = true, generateNonStrict = false, excludeWordsFrom } = target; const targetDirectory = path.resolve(rootDir, target.targetDirectory ?? cwd ?? process.cwd()); + const dictionaryDirectives = compileOptions.dictionaryDirectives; const excludeFilter = await createExcludeFilter(excludeWordsFrom); @@ -114,7 +123,12 @@ export async function compileTarget( opAwaitAsync(), ); const filesToProcess: FileToProcess[] = await toArray(filesToProcessAsync); - const normalizer = normalizeTargetWords({ sort: useTrie || sort, generateNonStrict, filter: excludeFilter }); + const normalizer = normalizeTargetWords({ + sort: useTrie || sort, + generateNonStrict, + filter: excludeFilter, + dictionaryDirectives, + }); const checksumRoot = (checksumFile && path.dirname(checksumFile)) || rootDir; const deps = [...calculateDependencies(filename, filesToProcess, excludeWordsFrom, checksumRoot)]; @@ -135,10 +149,11 @@ export async function compileTarget( trie3: format === 'trie3', trie4: format === 'trie4', generateNonStrict: generateNonStrictTrie, + dictionaryDirectives: undefined, }); } : async (words: Iterable, dst: string) => { - return compileWordList(pipe(words, normalizer), dst, { sort, generateNonStrict }); + return compileWordList(pipe(words, normalizer), dst, { sort, generateNonStrict, dictionaryDirectives }); }; await processFiles(action, filesToProcess, filename); diff --git a/packages/cspell-tools/src/compiler/wordListCompiler.test.ts b/packages/cspell-tools/src/compiler/wordListCompiler.test.ts index f0569ad41f4..b4e5792c7f4 100644 --- a/packages/cspell-tools/src/compiler/wordListCompiler.test.ts +++ b/packages/cspell-tools/src/compiler/wordListCompiler.test.ts @@ -70,7 +70,7 @@ describe('Validate the wordListCompiler', () => { const output = await fsp.readFile(destName, 'utf8'); expect(output).toBe( wordListHeader + - '\n' + + '\n\n' + citiesSorted + citiesSorted .toLowerCase() @@ -115,7 +115,7 @@ describe('Validate the wordListCompiler', () => { const destName = path.join(temp, 'example0.txt'); await compileWordList(source, destName, compileOpt(false)); const output = await fsp.readFile(destName, 'utf8'); - expect(output).toBe(__testing__.wordListHeader + '\n' + 'hello\ntry\nwork\n'); + expect(output).toBe(__testing__.wordListHeader + '\n\n' + 'hello\ntry\nwork\n'); expect(consoleOutput()).toMatchSnapshot(); }); @@ -144,6 +144,33 @@ describe('Validate the wordListCompiler', () => { ); expect(consoleOutput()).toMatchSnapshot(); }); + + test('Specify directives', async () => { + const source = await streamSourceWordsFromFile(path.join(samples, 'hunspell/example.dic'), { + ...readOptions, + maxDepth: 1, + }); + const destName = path.join(temp, 'example2.txt'); + await compileWordList(source, destName, compileOpt(false, false, ['no-generate-alternatives'])); + const output = await fsp.readFile(destName, 'utf8'); + expect(output.split('\n')).toEqual( + `\ + + # cspell-tools: keep-case no-split + # cspell-dictionary: no-generate-alternatives + + hello + rework + tried + try + work + worked + ` + .split('\n') + .map((line) => line.trim()), + ); + expect(consoleOutput()).toMatchSnapshot(); + }); }); describe('Validate Larger Dictionary', () => { @@ -202,8 +229,8 @@ function legacyNormalizeWords(lines: Iterable): Iterable { ); } -function compileOpt(sort: boolean, generateNonStrict = true): CompileOptions { - return { sort, generateNonStrict }; +function compileOpt(sort: boolean, generateNonStrict = true, dictionaryDirectives?: string[]): CompileOptions { + return { sort, generateNonStrict, dictionaryDirectives }; } // const cities = `\ diff --git a/packages/cspell-tools/src/compiler/wordListCompiler.ts b/packages/cspell-tools/src/compiler/wordListCompiler.ts index fb24f22730e..b4d6f871e80 100644 --- a/packages/cspell-tools/src/compiler/wordListCompiler.ts +++ b/packages/cspell-tools/src/compiler/wordListCompiler.ts @@ -15,8 +15,7 @@ const mkdirp = async (p: string) => { // Indicate that a word list has already been processed. const wordListHeader = ` -# cspell-tools: keep-case no-split -`; +# cspell-tools: keep-case no-split`; const wordListHeaderLines = wordListHeader.split('\n').map((a) => a.trim()); export async function compileWordList( @@ -26,7 +25,10 @@ export async function compileWordList( ): Promise { const finalLines = normalize(lines, options); - const finalSeq = pipe(wordListHeaderLines, opAppend(finalLines)); + const directives = options.dictionaryDirectives ?? []; + const directivesLines = directives.map((a) => `# cspell-dictionary: ${a}`); + + const finalSeq = pipe([...wordListHeaderLines, ...directivesLines, ''], opAppend(finalLines)); return createWordListTarget(destFilename)(finalSeq); } diff --git a/packages/cspell-tools/src/config/config.ts b/packages/cspell-tools/src/config/config.ts index 161d982d878..af2e339409f 100644 --- a/packages/cspell-tools/src/config/config.ts +++ b/packages/cspell-tools/src/config/config.ts @@ -45,14 +45,14 @@ export interface Experimental { export interface CompileTargetOptions { /** * Generate lower case / accent free versions of words. - * @default true + * @default false */ generateNonStrict?: boolean | undefined; /** * Sort the words in the resulting dictionary. * Does not apply to `trie` based formats. - * @default: true + * @default true */ sort?: boolean | undefined; @@ -66,6 +66,25 @@ export interface CompileTargetOptions { * dictionary. */ allowedSplitWords?: FilePath | FilePath[] | undefined; + + /** + * Injects `cspell-dictionary` directives into the dictionary header. + * + * Example: + * + * ```ini + * # cspell-dictionary: no-generate-alternatives + * ``` + * + * Known Directives: + * ```yaml + * - split # Tell the dictionary loader to split words + * - no-split # Tell the dictionary loader to not split words (default) + * - generate-alternatives # Tell the dictionary loader to generate alternate spellings (default) + * - no-generate-alternatives # Tell the dictionary loader to not generate alternate spellings + * ``` + */ + dictionaryDirectives?: string[] | undefined; } export interface Target extends CompileTargetOptions {