From 6663651e7d1c71e441a1eea764ddc483c9c35d46 Mon Sep 17 00:00:00 2001 From: Anton Vikulov Date: Fri, 27 Dec 2024 16:38:32 +0300 Subject: [PATCH] fix tests --- scripts/build.cli.js | 1 + src/commands/build/handler.ts | 35 +++++++++++++++----- src/reCli/components/render/singlePage.ts | 22 +++++++++--- src/reCli/components/toc/mapFile.ts | 18 ++++++++++ src/reCli/components/toc/toc.ts | 8 ++--- src/reCli/components/transform/mdPageToMd.ts | 16 ++++++++- src/reCli/components/transform/pageToHtml.ts | 12 +++++-- src/reCli/utils/plugins.ts | 5 ++- src/reCli/workers/transform.ts | 6 ++-- 9 files changed, 96 insertions(+), 27 deletions(-) create mode 100644 src/reCli/components/toc/mapFile.ts diff --git a/scripts/build.cli.js b/scripts/build.cli.js index 6c700b16..a92e656e 100644 --- a/scripts/build.cli.js +++ b/scripts/build.cli.js @@ -45,6 +45,7 @@ const commonConfig = { const builds = [ [['src/index.ts'], 'build/index.js'], [['src/workers/linter/index.ts'], 'build/linter.js'], + [['src/reCli/workers/transform.ts'], 'build/workers/transform.js'], ]; Promise.all(builds.map(([entries, outfile]) => { diff --git a/src/commands/build/handler.ts b/src/commands/build/handler.ts index bfd66510..669e986d 100644 --- a/src/commands/build/handler.ts +++ b/src/commands/build/handler.ts @@ -4,8 +4,8 @@ import 'threads/register'; import OpenapiIncluder from '@diplodoc/openapi-extension/includer'; -import {ArgvService, Includers, SearchService} from '~/services'; -import {processLogs} from '~/steps'; +import {ArgvService, Includers} from '~/services'; +import {processChangelogs, processLogs} from '~/steps'; import {getNavigationPaths, getPresetIndex} from '~/reCli/components/presets'; import {getTocIndex, transformTocForJs, transformTocForSinglePage} from '~/reCli/components/toc'; import GithubConnector from '~/reCli/components/vcs/github'; @@ -26,16 +26,17 @@ import {saveSinglePages} from '~/reCli/components/render/singlePage'; import {copyAssets} from '~/reCli/components/assets/assets'; import {saveRedirectPage} from '~/reCli/components/render/redirect'; import {LogCollector} from '~/reCli/utils/logger'; +import {getMapFile} from '~/reCli/components/toc/mapFile'; +import {BuildConfig} from '~/commands/build/index'; export async function handler(run: Run) { try { ArgvService.init(run.legacyConfig); - SearchService.init(); // TODO: Remove duplicated types from openapi-extension // @ts-ignore Includers.init([OpenapiIncluder]); - const {input, output, outputFormat, singlePage} = run.config; + const {input, output, outputFormat, singlePage, addMapFile} = run.config; const {applyPresets, resolveConditions} = run.legacyConfig; const presetIndex = await getPresetIndex(run.input, run.config, run); @@ -139,6 +140,14 @@ export async function handler(run: Run) { ); const pages = Array.from(pageSet.values()); + if (addMapFile) { + const map = getMapFile(pages); + await fs.promises.writeFile( + path.join(output, 'files.json'), + JSON.stringify(map, null, '\t'), + ); + } + const writeConflicts = new Map(); const singlePageTocPagesMap = new Map(); @@ -146,7 +155,7 @@ export async function handler(run: Run) { // eslint-disable-next-line new-cap const transformPool = Pool( () => - spawn(new Worker('../../../reCli/workers/transform'), { + spawn(new Worker('./workers/transform'), { timeout: 60000, }), workerCount, @@ -165,8 +174,15 @@ export async function handler(run: Run) { index = workerIndex++; workerIndexMap.set(worker, index); const threadOutput = path.join(tmpThreads, String(index)); + + const configClone: Record = {}; + // eslint-disable-next-line guard-for-in + for (const key in run.config) { + configClone[key] = run.config[key as keyof BuildConfig]; + } + await worker.init({ - config: run.config, + config: configClone as BuildConfig, presetIndex, tmpSource, tmpDraft, @@ -226,8 +242,8 @@ export async function handler(run: Run) { }), ); await Promise.all([ - fs.promises.rm(tmpThreads, {recursive: true}), - fs.promises.rm(tmpDraft, {recursive: true}), + fs.promises.rm(tmpThreads, {recursive: true, force: true}), + fs.promises.rm(tmpDraft, {recursive: true, force: true}), ]); } finally { await transformPool.terminate(true); @@ -246,6 +262,7 @@ export async function handler(run: Run) { if (singlePage) { await saveSinglePages({ + targetCwd: output, options: run.config, singlePageTocPagesMap, tocIndex, @@ -261,6 +278,8 @@ export async function handler(run: Run) { pages, logger, }); + + await processChangelogs(); } catch (error) { run.logger.error(error); } finally { diff --git a/src/reCli/components/render/singlePage.ts b/src/reCli/components/render/singlePage.ts index a3b1b3d9..20d83907 100644 --- a/src/reCli/components/render/singlePage.ts +++ b/src/reCli/components/render/singlePage.ts @@ -11,11 +11,13 @@ import {DocInnerProps, DocPageData} from '@diplodoc/client/ssr'; import {CONCURRENCY} from '~/reCli/constants'; import fs from 'node:fs'; import {LogCollector} from '~/reCli/utils/logger'; +import {cachedMkdir, safePath} from '~/reCli/utils'; const SINGLE_PAGE_FILENAME = 'single-page.html'; const SINGLE_PAGE_DATA_FILENAME = 'single-page.json'; export interface SaveSinglePagesProps { + targetCwd: string; options: BuildConfig; singlePageTocPagesMap: Map; tocIndex: TocIndexMap; @@ -23,6 +25,7 @@ export interface SaveSinglePagesProps { } export async function saveSinglePages({ + targetCwd, options, singlePageTocPagesMap, tocIndex, @@ -34,7 +37,9 @@ export async function saveSinglePages({ await pMap( Array.from(singlePageTocPagesMap.entries()), async ([tocPath, pageResults]) => { - if (!pageResults.length) return; + if (!pageResults.length) { + return; + } const tocDir = path.dirname(tocPath.replace(/\\/g, '/').replace(/^\/?/, '')); const singlePageBody = joinSinglePageResults(pageResults, tocDir); @@ -66,8 +71,8 @@ export async function saveSinglePages({ }; // Save the full single page for viewing locally - const singlePageFn = join(tocPath, SINGLE_PAGE_FILENAME); - const singlePageDataFn = join(tocPath, SINGLE_PAGE_DATA_FILENAME); + const singlePageFn = path.join(tocDir, SINGLE_PAGE_FILENAME); + const singlePageDataFn = safePath(path.join(tocDir, SINGLE_PAGE_DATA_FILENAME)); const tocInfo = {path: join(tocDir, 'single-page-toc'), content: toc}; const singlePageContent = generateStaticMarkup( options, @@ -76,9 +81,16 @@ export async function saveSinglePages({ (toc.title as string) || '', ); + await cachedMkdir(path.join(targetCwd, safePath(tocDir))); await Promise.all([ - fs.promises.writeFile(singlePageFn, singlePageContent), - fs.promises.writeFile(singlePageDataFn, JSON.stringify(pageData)), + fs.promises.writeFile( + path.join(targetCwd, safePath(singlePageFn)), + singlePageContent, + ), + fs.promises.writeFile( + path.join(targetCwd, safePath(singlePageDataFn)), + JSON.stringify(pageData), + ), ]); }, {concurrency: CONCURRENCY}, diff --git a/src/reCli/components/toc/mapFile.ts b/src/reCli/components/toc/mapFile.ts new file mode 100644 index 00000000..2809c8c5 --- /dev/null +++ b/src/reCli/components/toc/mapFile.ts @@ -0,0 +1,18 @@ +import {convertBackSlashToSlash} from '~/utils'; +import path from 'node:path'; + +export function getMapFile(pages: string[]) { + const navigationPathsWithoutExtensions = pages.map((pagePath) => { + let preparedPath = convertBackSlashToSlash( + path.dirname(pagePath) + '/' + path.basename(pagePath, path.extname(pagePath)), + ); + + if (preparedPath.endsWith('/index')) { + preparedPath = preparedPath.substring(0, preparedPath.length - 5); + } + + return preparedPath; + }); + + return {files: navigationPathsWithoutExtensions.sort()}; +} diff --git a/src/reCli/components/toc/toc.ts b/src/reCli/components/toc/toc.ts index 8b99a828..f7f4aca5 100644 --- a/src/reCli/components/toc/toc.ts +++ b/src/reCli/components/toc/toc.ts @@ -201,8 +201,7 @@ async function replaceIncludes( const itemsWithIncluded = (item.items || []).concat(includeToc.items); /* Resolve nested toc inclusions */ - const baseTocPath = - mode === IncludeMode.LINK ? includeTocPath : path.dirname(tocPath); + const baseTocPath = mode === IncludeMode.LINK ? includeTocPath : tocPath; const {items: subItems, includedTocs: subIncludedTocs} = await processTocItems( itemsWithIncluded, @@ -308,11 +307,12 @@ async function copyTocDir( destDir: string, copyMap: Map, ) { - const source = path.join(cwd, safePath(tocPath)); - const target = path.join(cwd, safePath(destDir)); + const source = path.join(cwd, safePath(tocPath)) as AbsolutePath; + const target = path.join(cwd, safePath(destDir)) as AbsolutePath; const files = await run.glob('**/*.*', { ignore: ['**/toc.yaml'], + cwd: source, }); const dirs = new Set(); diff --git a/src/reCli/components/transform/mdPageToMd.ts b/src/reCli/components/transform/mdPageToMd.ts index 5a4b6858..8b51c206 100644 --- a/src/reCli/components/transform/mdPageToMd.ts +++ b/src/reCli/components/transform/mdPageToMd.ts @@ -66,7 +66,7 @@ const TARGET_COPY_SET = new Set(); async function transformMd(props: TransformPageProps, pagePath: string) { const {presetIndex, cwd, options, fileMetaMap, vcsConnector} = props; - const {vars, addSystemMeta} = options; + const {vars, addSystemMeta, allowCustomResources, resources} = options; const combinedVars = getFilePresets(presetIndex, vars, pagePath); const input = await fs.promises.readFile(path.join(cwd, pagePath) as AbsolutePath, 'utf8'); @@ -91,6 +91,13 @@ async function transformMd(props: TransformPageProps, pagePath: string) { fileMetaMap.set(pagePath, {...fileMetaMap.get(pagePath), __system: combinedVars.__system}); } + if (Array.isArray(combinedVars.__metadata)) { + fileMetaMap.set(pagePath, { + ...fileMetaMap.get(pagePath), + metadata: [...(meta.metadata ?? []), ...combinedVars.__metadata], + }); + } + if (vcsConnector) { const author = vcsConnector.getAuthor(pagePath); const updatedAt = vcsConnector.getMtime(pagePath, includedPaths); @@ -114,6 +121,13 @@ async function transformMd(props: TransformPageProps, pagePath: string) { } } + if (allowCustomResources && resources) { + fileMetaMap.set(pagePath, { + ...fileMetaMap.get(pagePath), + ...resources, + }); + } + const extraMeta = fileMetaMap.get(pagePath); if (extraMeta) { Object.assign(meta, extraMeta); diff --git a/src/reCli/components/transform/pageToHtml.ts b/src/reCli/components/transform/pageToHtml.ts index 62ddc6dd..f72a6439 100644 --- a/src/reCli/components/transform/pageToHtml.ts +++ b/src/reCli/components/transform/pageToHtml.ts @@ -99,8 +99,9 @@ async function getFileProps(props: PageToHtmlProps, pagePath: string) { } async function getFileData(props: PageToHtmlProps, pagePath: string) { - const {cwd, fileMetaMap, vcsConnector, options} = props; - const {allowCustomResources, resources} = options; + const {cwd, fileMetaMap, vcsConnector, options, presetIndex} = props; + const {allowCustomResources, resources, vars} = options; + const combinedVars = getFilePresets(presetIndex, vars, pagePath); const pageContent: string = await fs.promises.readFile( path.join(cwd, safePath(pagePath)) as AbsolutePath, @@ -128,6 +129,13 @@ async function getFileData(props: PageToHtmlProps, pagePath: string) { fileMeta.metadata = [fileMeta.metadata]; } + if (Array.isArray(combinedVars.__metadata)) { + fileMeta.metadata = [ + ...(fileMeta.metadata ?? []), + ...combinedVars.__metadata.filter(Boolean), + ]; + } + if (allowCustomResources) { if (resources) { Object.entries(resources).forEach(([key, value = []]) => { diff --git a/src/reCli/utils/plugins.ts b/src/reCli/utils/plugins.ts index 8978ac96..642ade28 100644 --- a/src/reCli/utils/plugins.ts +++ b/src/reCli/utils/plugins.ts @@ -6,15 +6,14 @@ import { export function getPlugins() { const plugins = getPluginsLegacy(); - if (!plugins.length) { + if (!plugins?.length) { setPlugins(); } return getPluginsLegacy(); } export function getCollectOfPlugins() { - const list = getCollectOfPluginsLegacy(); - if (!list.length) { + if (!getPluginsLegacy()?.length) { setPlugins(); } return getCollectOfPluginsLegacy(); diff --git a/src/reCli/workers/transform.ts b/src/reCli/workers/transform.ts index 630d0d2a..38e25f75 100644 --- a/src/reCli/workers/transform.ts +++ b/src/reCli/workers/transform.ts @@ -13,7 +13,7 @@ import {SinglePageResult} from '~/models'; import {readTransformLog} from '~/reCli/utils/legacy'; import {lintPage} from '~/reCli/components/lint/lint'; import {transformPage} from '~/reCli/components/transform/transform'; -import {LogCollector} from "~/reCli/utils/logger"; +import {LogCollector} from '~/reCli/utils/logger'; /*eslint-disable no-console*/ @@ -150,9 +150,7 @@ async function run({pages}: TransformWorkerProps) { ); } catch (err) { const error = err as Error; - logger.error( - `Transform page error ${pagePath}. Error: ${error.stack}`, - ); + logger.error(`Transform page error ${pagePath}. Error: ${error.stack}`); } }