From 8f7f3d2e3ff41065e1dbbe951bbab42b8cce5880 Mon Sep 17 00:00:00 2001 From: Nicolas Vuillamy Date: Wed, 1 Jan 2025 16:15:30 +0100 Subject: [PATCH] Mermaid in PNG for Azure & Bitbucket (#964) * typo * Convert SVG to PNG on Azure & Bitbucket * Handle PNG generation for Azure & Bitbucket * typo --- src/commands/hardis/doc/flow2markdown.ts | 2 +- src/commands/hardis/doc/project2markdown.ts | 2 +- .../hardis/project/generate/flow-git-diff.ts | 2 +- src/common/aiProvider/index.ts | 2 +- src/common/aiProvider/utils.ts | 2 +- src/common/gitProvider/gitProviderRoot.ts | 7 +++ src/common/gitProvider/index.ts | 8 +++ src/common/gitProvider/utilsMarkdown.ts | 19 ++++++- src/common/utils/mermaidUtils.ts | 55 ++++++++++++------- 9 files changed, 70 insertions(+), 29 deletions(-) diff --git a/src/commands/hardis/doc/flow2markdown.ts b/src/commands/hardis/doc/flow2markdown.ts index 5f1fc7093..d157d86c5 100644 --- a/src/commands/hardis/doc/flow2markdown.ts +++ b/src/commands/hardis/doc/flow2markdown.ts @@ -82,7 +82,7 @@ export default class Flow2Markdown extends SfCommand { if (this.debugMode) { await fs.copyFile(this.outputFile, this.outputFile.replace(".md", ".mermaid.md")); } - const gen2res = await generateMarkdownFileWithMermaid(this.outputFile); + const gen2res = await generateMarkdownFileWithMermaid(this.outputFile, this.outputFile); if (!gen2res) { throw new Error("Error generating mermaid markdown file"); } diff --git a/src/commands/hardis/doc/project2markdown.ts b/src/commands/hardis/doc/project2markdown.ts index 631faeaed..029f69662 100644 --- a/src/commands/hardis/doc/project2markdown.ts +++ b/src/commands/hardis/doc/project2markdown.ts @@ -274,7 +274,7 @@ ${Project2Markdown.htmlInstructions} if (this.debugMode) { await fs.copyFile(outputFlowMdFile, outputFlowMdFile.replace(".md", ".mermaid.md")); } - const gen2res = await generateMarkdownFileWithMermaid(outputFlowMdFile); + const gen2res = await generateMarkdownFileWithMermaid(outputFlowMdFile, outputFlowMdFile); if (!gen2res) { flowWarnings.push(flowFile); continue; diff --git a/src/commands/hardis/project/generate/flow-git-diff.ts b/src/commands/hardis/project/generate/flow-git-diff.ts index a88055461..188c5aee2 100644 --- a/src/commands/hardis/project/generate/flow-git-diff.ts +++ b/src/commands/hardis/project/generate/flow-git-diff.ts @@ -113,7 +113,7 @@ Run \`npm install @mermaid-js/mermaid-cli --global\` choices: allChoices }) const commitAfter = commitAfterSelectRes.after; - const { outputDiffMdFile } = await generateFlowVisualGitDiff(this.flowFile, commitBefore, commitAfter, { svgMd: true, mermaidMd: this.debugMode, debug: this.debugMode }) + const { outputDiffMdFile } = await generateFlowVisualGitDiff(this.flowFile, commitBefore, commitAfter, { svgMd: true, pngMd: false, mermaidMd: this.debugMode, debug: this.debugMode }) // Open file in a new VsCode tab if available WebSocketClient.requestOpenFile(path.relative(process.cwd(), outputDiffMdFile)); diff --git a/src/common/aiProvider/index.ts b/src/common/aiProvider/index.ts index a222672b5..342f3ce72 100644 --- a/src/common/aiProvider/index.ts +++ b/src/common/aiProvider/index.ts @@ -10,7 +10,7 @@ export abstract class AiProvider { static getInstance(): AiProviderRoot | null { // OpenAi - if (UtilsAi.isOpenApiAvailable()) { + if (UtilsAi.isOpenAiAvailable()) { return new OpenAiProvider(); } return null; diff --git a/src/common/aiProvider/utils.ts b/src/common/aiProvider/utils.ts index cb3020352..9b7356dab 100644 --- a/src/common/aiProvider/utils.ts +++ b/src/common/aiProvider/utils.ts @@ -1,7 +1,7 @@ import { getEnvVar } from "../../config/index.js"; export class UtilsAi { - public static isOpenApiAvailable() { + public static isOpenAiAvailable() { if (getEnvVar("OPENAI_API_KEY")) { return true; } diff --git a/src/common/gitProvider/gitProviderRoot.ts b/src/common/gitProvider/gitProviderRoot.ts index 87eca3ea1..931dea73e 100644 --- a/src/common/gitProvider/gitProviderRoot.ts +++ b/src/common/gitProvider/gitProviderRoot.ts @@ -38,6 +38,13 @@ export abstract class GitProviderRoot { return false; } + public async supportsSvgAttachments(): Promise { + // False by default, might be used later + return false; + } + + + public async getPullRequestInfo(): Promise { uxLog(this, `Method getPullRequestInfo is not implemented yet on ${this.getLabel()}`); return null; diff --git a/src/common/gitProvider/index.ts b/src/common/gitProvider/index.ts index e6a42dd9f..8ebafcc1e 100644 --- a/src/common/gitProvider/index.ts +++ b/src/common/gitProvider/index.ts @@ -193,6 +193,14 @@ export abstract class GitProvider { return gitProvider.supportsMermaidInPrMarkdown(); } + static async supportsSvgAttachments(): Promise { + const gitProvider = await GitProvider.getInstance(); + if (gitProvider == null) { + return false; + } + return gitProvider.supportsSvgAttachments(); + } + static async getPullRequestInfo(): Promise { const gitProvider = await GitProvider.getInstance(); if (gitProvider == null) { diff --git a/src/common/gitProvider/utilsMarkdown.ts b/src/common/gitProvider/utilsMarkdown.ts index 6c9d1e59d..5e1bb29b4 100644 --- a/src/common/gitProvider/utilsMarkdown.ts +++ b/src/common/gitProvider/utilsMarkdown.ts @@ -82,18 +82,25 @@ export async function flowDiffToMarkdownForPullRequest(flowNames: string[], from return ""; } const supportsMermaidInPrMarkdown = await GitProvider.supportsMermaidInPrMarkdown(); + const supportsSvgAttachments = await GitProvider.supportsSvgAttachments(); const flowDiffMarkdownList: any = []; let flowDiffFilesSummary = "## Flow changes\n\n"; for (const flowName of flowNames) { flowDiffFilesSummary += `- [${flowName}](#${flowName})\n`; const fileMetadata = await MetadataUtils.findMetaFileFromTypeAndName("Flow", flowName); try { + // Markdown with pure mermaidJs if (supportsMermaidInPrMarkdown) { await generateDiffMarkdownWithMermaid(fileMetadata, fromCommit, toCommit, flowDiffMarkdownList, flowName); } - else { + // Markdown with Mermaid converted as SVG + else if (supportsSvgAttachments) { await generateDiffMarkdownWithSvg(fileMetadata, fromCommit, toCommit, flowDiffMarkdownList, flowName); } + // Markdown with images converted as PNG + else { + await generateDiffMarkdownWithPng(fileMetadata, fromCommit, toCommit, flowDiffMarkdownList, flowName); + } } catch (e: any) { uxLog(this, c.yellow(`[FlowGitDiff] Unable to generate Flow diff for ${flowName}: ${e.message}`)); const flowGenErrorMd = `# ${flowName} @@ -110,7 +117,7 @@ Error while generating Flows visual git diff } async function generateDiffMarkdownWithMermaid(fileMetadata: string | null, fromCommit: string, toCommit: string, flowDiffMarkdownList: any, flowName: string) { - const { outputDiffMdFile } = await generateFlowVisualGitDiff(fileMetadata, fromCommit, toCommit, { mermaidMd: true, svgMd: false, debug: false }); + const { outputDiffMdFile } = await generateFlowVisualGitDiff(fileMetadata, fromCommit, toCommit, { mermaidMd: true, svgMd: false, pngMd: false, debug: false }); if (outputDiffMdFile) { const flowDiffMarkdownMermaid = await fs.readFile(outputDiffMdFile.replace(".md", ".mermaid.md"), "utf8"); flowDiffMarkdownList.push({ name: flowName, markdown: flowDiffMarkdownMermaid, markdownFile: outputDiffMdFile }); @@ -118,11 +125,17 @@ async function generateDiffMarkdownWithMermaid(fileMetadata: string | null, from } async function generateDiffMarkdownWithSvg(fileMetadata: string | null, fromCommit: string, toCommit: string, flowDiffMarkdownList: any, flowName: string) { - const { outputDiffMdFile } = await generateFlowVisualGitDiff(fileMetadata, fromCommit, toCommit, { mermaidMd: true, svgMd: true, debug: false }); + const { outputDiffMdFile } = await generateFlowVisualGitDiff(fileMetadata, fromCommit, toCommit, { mermaidMd: true, svgMd: false, pngMd: false, debug: false }); const flowDiffMarkdownWithSvg = await fs.readFile(outputDiffMdFile, "utf8"); flowDiffMarkdownList.push({ name: flowName, markdown: flowDiffMarkdownWithSvg, markdownFile: outputDiffMdFile }); } +async function generateDiffMarkdownWithPng(fileMetadata: string | null, fromCommit: string, toCommit: string, flowDiffMarkdownList: any, flowName: string) { + const { outputDiffMdFile } = await generateFlowVisualGitDiff(fileMetadata, fromCommit, toCommit, { mermaidMd: true, svgMd: false, pngMd: true, debug: false }); + const flowDiffMarkdownWithPng = await fs.readFile(outputDiffMdFile, "utf8"); + flowDiffMarkdownList.push({ name: flowName, markdown: flowDiffMarkdownWithPng, markdownFile: outputDiffMdFile }); +} + function getAiPromptResponseMarkdown(title, message) { return `
🤖 ${title} diff --git a/src/common/utils/mermaidUtils.ts b/src/common/utils/mermaidUtils.ts index 1cf990a62..7a6681bcc 100644 --- a/src/common/utils/mermaidUtils.ts +++ b/src/common/utils/mermaidUtils.ts @@ -48,7 +48,7 @@ export async function generateFlowMarkdownFile(flowName: string, flowXml: string } } -export async function generateMarkdownFileWithMermaid(outputFlowMdFile: string, mermaidModes: string[] | null = null): Promise { +export async function generateMarkdownFileWithMermaid(outputFlowMdFileIn: string, outputFlowMdFileOut: string, mermaidModes: string[] | null = null): Promise { if (process.env.MERMAID_MODES) { mermaidModes = process.env.MERMAID_MODES.split(","); } @@ -60,13 +60,13 @@ export async function generateMarkdownFileWithMermaid(outputFlowMdFile: string, } const isDockerAvlbl = await isDockerAvailable(); if (isDockerAvlbl && (!(globalThis.mermaidUnavailableTools || []).includes("docker")) && mermaidModes.includes("docker")) { - const dockerSuccess = await generateMarkdownFileWithMermaidDocker(outputFlowMdFile); + const dockerSuccess = await generateMarkdownFileWithMermaidDocker(outputFlowMdFileIn, outputFlowMdFileOut); if (dockerSuccess) { return true; } } if ((!(globalThis.mermaidUnavailableTools || []).includes("cli")) && mermaidModes.includes("cli")) { - const mmCliSuccess = await generateMarkdownFileWithMermaidCli(outputFlowMdFile); + const mmCliSuccess = await generateMarkdownFileWithMermaidCli(outputFlowMdFileIn, outputFlowMdFileOut); if (mmCliSuccess) { return true; } @@ -77,15 +77,15 @@ export async function generateMarkdownFileWithMermaid(outputFlowMdFile: string, return false; } -export async function generateMarkdownFileWithMermaidDocker(outputFlowMdFile: string): Promise { - const fileDir = path.resolve(path.dirname(outputFlowMdFile)); - const fileName = path.basename(outputFlowMdFile); - const dockerCommand = `docker run --rm -v "${fileDir}:/data" ghcr.io/mermaid-js/mermaid-cli/mermaid-cli -i "${fileName}" -o "${fileName}"`; +export async function generateMarkdownFileWithMermaidDocker(outputFlowMdFileIn: string, outputFlowMdFileOut: string): Promise { + const fileDir = path.resolve(path.dirname(outputFlowMdFileIn)); + const fileName = path.basename(outputFlowMdFileIn); + const dockerCommand = `docker run --rm -v "${fileDir}:/data" ghcr.io/mermaid-js/mermaid-cli/mermaid-cli -i "${fileName}" -o "${outputFlowMdFileOut}"`; try { await execCommand(dockerCommand, this, { output: false, fail: true, debug: false }); return true; } catch (e: any) { - uxLog(this, c.yellow(`Error generating mermaidJs Graphs in ${outputFlowMdFile} documentation with Docker: ${e.message}`) + "\n" + c.grey(e.stack)); + uxLog(this, c.yellow(`Error generating mermaidJs Graphs from ${outputFlowMdFileIn} documentation with Docker: ${e.message}`) + "\n" + c.grey(e.stack)); if (JSON.stringify(e).includes("Cannot connect to the Docker daemon") || JSON.stringify(e).includes("daemon is not running")) { globalThis.mermaidUnavailableTools = (globalThis.mermaidUnavailableTools || []).concat("docker"); uxLog(this, c.yellow("[Mermaid] Docker unavailable: do not try again")); @@ -94,16 +94,16 @@ export async function generateMarkdownFileWithMermaidDocker(outputFlowMdFile: st } } -export async function generateMarkdownFileWithMermaidCli(outputFlowMdFile: string): Promise { +export async function generateMarkdownFileWithMermaidCli(outputFlowMdFileIn: string, outputFlowMdFileOut: string): Promise { // Try with NPM package const isMmdAvailable = await isMermaidAvailable(); const puppeteerConfigPath = path.join(PACKAGE_ROOT_DIR, 'defaults', 'puppeteer-config.json'); - const mermaidCmd = `${!isMmdAvailable ? 'npx --yes -p @mermaid-js/mermaid-cli ' : ''}mmdc -i "${outputFlowMdFile}" -o "${outputFlowMdFile}" --puppeteerConfigFile "${puppeteerConfigPath}"`; + const mermaidCmd = `${!isMmdAvailable ? 'npx --yes -p @mermaid-js/mermaid-cli ' : ''}mmdc -i "${outputFlowMdFileIn}" -o "${outputFlowMdFileOut}" --puppeteerConfigFile "${puppeteerConfigPath}"`; try { await execCommand(mermaidCmd, this, { output: false, fail: true, debug: false }); return true; } catch (e: any) { - uxLog(this, c.yellow(`Error generating mermaidJs Graphs in ${outputFlowMdFile} documentation with CLI: ${e.message}`) + "\n" + c.grey(e.stack)); + uxLog(this, c.yellow(`Error generating mermaidJs Graphs from ${outputFlowMdFileIn} documentation with CLI: ${e.message}`) + "\n" + c.grey(e.stack)); if (JSON.stringify(e).includes("timed out")) { globalThis.mermaidUnavailableTools = (globalThis.mermaidUnavailableTools || []).concat("cli"); uxLog(this, c.yellow("[Mermaid] CLI unavailable: do not try again")); @@ -178,7 +178,7 @@ ${formatClasses(changedClasses, changed)} } export async function generateFlowVisualGitDiff(flowFile, commitBefore: string, commitAfter: string, - options: { mermaidMd: boolean, svgMd: boolean, debug: boolean } = { mermaidMd: false, svgMd: true, debug: false }) { + options: { mermaidMd: boolean, svgMd: boolean, pngMd: boolean, debug: boolean } = { mermaidMd: false, svgMd: true, pngMd: false, debug: false }) { const result: any = { outputDiffMdFile: "", hasFlowDiffs: false }; const mermaidMdBefore = await buildMermaidMarkdown(commitBefore, flowFile); const mermaidMdAfter = await buildMermaidMarkdown(commitAfter, flowFile); @@ -221,16 +221,29 @@ export async function generateFlowVisualGitDiff(flowFile, commitBefore: string, if (options.mermaidMd) { await fs.copyFile(diffMdFile, diffMdFile.replace(".md", ".mermaid.md")); } - if (!options.svgMd) { - result.outputDiffMdFile = diffMdFile; + result.outputDiffMdFile = diffMdFile; + if (!options.svgMd && !options.pngMd) { return result; } - // Generate final markdown with mermaid SVG - const finalRes = await generateMarkdownFileWithMermaid(diffMdFile, ["cli", "docker"]); - if (finalRes) { - uxLog(this, c.green(`Successfully generated visual git diff for flow: ${diffMdFile}`)); + if (options.svgMd) { + // Generate final markdown with mermaid SVG + const finalRes = await generateMarkdownFileWithMermaid(diffMdFile, diffMdFile, ["cli", "docker"]); + if (finalRes) { + uxLog(this, c.green(`Successfully generated visual git diff for flow: ${diffMdFile}`)); + } + } + else if (options.pngMd) { + // General final markdown with mermaid PNG + const pngFile = path.join(path.dirname(diffMdFile), path.basename(diffMdFile, ".md") + ".png"); + const pngRes = await generateMarkdownFileWithMermaid(diffMdFile, pngFile, ["cli", "docker"]); + if (pngRes) { + let mdWithMermaid = fs.readFileSync(diffMdFile, "utf8"); + mdWithMermaid = mdWithMermaid.replace( + /```mermaid\n([\s\S]*?)\n```/g, + `![Diagram as PNG](./${path.basename(pngFile).replace(".png", "-1.png")})`); + await fs.writeFile(diffMdFile, mdWithMermaid); + } } - result.outputDiffMdFile = diffMdFile; return result; } @@ -533,7 +546,7 @@ export async function generateHistoryDiffMarkdown(flowFile: string, debugMode: b } else { const commitBefore = fileHistory.all[i + 1]; - const genDiffRes = await generateFlowVisualGitDiff(flowFile, commitBefore.hash, commitAfter.hash, { svgMd: false, mermaidMd: true, debug: debugMode }); + const genDiffRes = await generateFlowVisualGitDiff(flowFile, commitBefore.hash, commitAfter.hash, { svgMd: false, mermaidMd: true, pngMd: false, debug: debugMode }); if (genDiffRes.hasFlowDiffs && fs.existsSync(genDiffRes.outputDiffMdFile)) { diffMdFiles.push({ commitBefore: commitBefore, @@ -561,7 +574,7 @@ export async function generateHistoryDiffMarkdown(flowFile: string, debugMode: b if (debugMode) { await fs.copyFile(diffMdFile, diffMdFile.replace(".md", ".mermaid.md")); } - const genSvgRes = await generateMarkdownFileWithMermaid(diffMdFile); + const genSvgRes = await generateMarkdownFileWithMermaid(diffMdFile, diffMdFile); if (!genSvgRes) { throw new Error("Error generating mermaid markdown file"); }