diff --git a/scripts/build-post-list.js b/scripts/build-post-list.js index 288d7dc0c54e..33acdaad9bad 100644 --- a/scripts/build-post-list.js +++ b/scripts/build-post-list.js @@ -1,8 +1,7 @@ -const { readdirSync, statSync, existsSync, readFileSync, writeFileSync } = require('fs') -const { resolve, basename } = require('path') +const { readdir, stat, pathExists, readFile, writeFile } = require('fs-extra') +const { basename, join, normalize, sep, posix, relative, parse } = require('path') const frontMatter = require('gray-matter') const toc = require('markdown-toc') -const { slugify } = require('markdown-toc/lib/utils') const readingTime = require('reading-time') const { markdownToTxt } = require('markdown-to-txt') const { buildNavTree, addDocButtons } = require('./build-docs') @@ -15,51 +14,88 @@ const result = { docsTree: {} } const releaseNotes = [] -const basePath = 'pages' -const postDirectories = [ - // order of these directories is important, as the blog should come before docs, to create a list of available release notes, which will later be used to release-note-link for spec docs - [`${basePath}/blog`, '/blog'], - [`${basePath}/docs`, '/docs'], - [`${basePath}/about`, '/about'] -]; const addItem = (details) => { - if(details.slug.startsWith('/docs')) - result["docs"].push(details) - else if(details.slug.startsWith('/blog')) - result["blog"].push(details) - else if(details.slug.startsWith('/about')) - result["about"].push(details) - else {} + const sectionMap = { + '/docs': 'docs', + '/blog': 'blog', + '/about': 'about' + }; + const section = Object.keys(sectionMap).find(key => details.slug.startsWith(key)); + if (section) { + result[sectionMap[section]].push(details); + } +}; + +function getVersionDetails(slug, weight) { + const fileBaseName = basename(slug); + const versionName = fileBaseName.split('-')[0]; + return { + title: versionName.startsWith('v') + ? capitalize(versionName.slice(1)) + : capitalize(versionName), + weight + }; } -module.exports = async function buildPostList() { - walkDirectories(postDirectories, result) - const treePosts = buildNavTree(result["docs"].filter((p) => p.slug.startsWith('/docs/'))) - result["docsTree"] = treePosts - result["docs"] = addDocButtons(result["docs"], treePosts) - if (process.env.NODE_ENV === 'production') { - // console.log(inspect(result, { depth: null, colors: true })) +/** + * Builds a list of posts from the specified directories and writes it to a file + * @param {Array>} postDirectories - Array of [directory, slug] tuples + * @param {string} basePath - Base path for resolving relative paths + * @param {string} writeFilePath - Path where the output JSON will be written + * @throws {Error} If required parameters are missing or if any operation fails + * @returns {Promise} + */ +async function buildPostList(postDirectories, basePath, writeFilePath) { + try { + + if (!basePath) { + throw new Error('Error while building post list: basePath is required'); + } + + if (!writeFilePath) { + throw new Error('Error while building post list: writeFilePath is required'); + } + + if (postDirectories.length === 0) { + throw new Error('Error while building post list: postDirectories array is empty'); + } + const normalizedBasePath = normalize(basePath) + await walkDirectories(postDirectories, result, normalizedBasePath) + const treePosts = buildNavTree(result.docs.filter((p) => p.slug.startsWith('/docs/'))) + result.docsTree = treePosts + result.docs = addDocButtons(result.docs, treePosts) + await writeFile(writeFilePath, JSON.stringify(result, null, ' ')) + } catch (error) { + throw new Error(`Error while building post list: ${error.message}`, { cause: error }); } - writeFileSync(resolve(__dirname, '..', 'config', 'posts.json'), JSON.stringify(result, null, ' ')) } -function walkDirectories(directories, result, sectionWeight = 0, sectionTitle, sectionId, rootSectionId) { +async function walkDirectories( + directories, + resultObj, + basePath, + sectionTitle, + sectionId, + rootSectionId, + sectionWeight = 0 +) { for (let dir of directories) { - let directory = dir[0] - let sectionSlug = dir[1] || '' - let files = readdirSync(directory); + const directory = posix.normalize(dir[0]); + const sectionSlug = dir[1] || ''; + const files = await readdir(directory) for (let file of files) { - let details - const fileName = [directory, file].join('/') - const fileNameWithSection = [fileName, '_section.mdx'].join('/') - const slug = fileName.replace(new RegExp(`^${basePath}`), '') - const slugElements = slug.split('/'); - if (isDirectory(fileName)) { - if (existsSync(fileNameWithSection)) { + let details; + const fileName = normalize(join(directory, file)); + const fileNameWithSection = normalize(join(fileName, '_section.mdx')) + const slug = `/${normalize(relative(basePath, fileName)).replace(/\\/g, '/')}` + const slugElements = slug.split('/') + + if (await isDirectory(fileName)) { + if (await pathExists(fileNameWithSection)) { // Passing a second argument to frontMatter disables cache. See https://github.com/asyncapi/website/issues/1057 - details = frontMatter(readFileSync(fileNameWithSection, 'utf-8'), {}).data + details = frontMatter(await readFile(fileNameWithSection, 'utf-8'), {}).data details.title = details.title || capitalize(basename(fileName)) } else { details = { @@ -68,8 +104,8 @@ function walkDirectories(directories, result, sectionWeight = 0, sectionTitle, s } details.isSection = true if (slugElements.length > 3) { - details.parent = slugElements[slugElements.length - 2] - details.sectionId = slugElements[slugElements.length - 1] + details.parent = slugElements[slugElements.length - 2] + details.sectionId = slugElements[slugElements.length - 1] } if (!details.parent) { details.isRootSection = true @@ -79,9 +115,9 @@ function walkDirectories(directories, result, sectionWeight = 0, sectionTitle, s details.slug = slug addItem(details) const rootId = details.parent || details.rootSectionId - walkDirectories([[fileName, slug]], result, details.weight, details.title, details.sectionId, rootId) - } else if (file.endsWith('.mdx') && !fileName.endsWith('/_section.mdx')) { - const fileContent = readFileSync(fileName, 'utf-8') + await walkDirectories([[fileName, slug]], resultObj, basePath, details.title, details.sectionId, rootId, details.sectionWeight) + } else if (file.endsWith('.mdx') && !fileName.endsWith(sep + '_section.mdx')) { + const fileContent = await readFile(fileName, 'utf-8') // Passing a second argument to frontMatter disables cache. See https://github.com/asyncapi/website/issues/1057 const { data, content } = frontMatter(fileContent, {}) details = data @@ -93,21 +129,16 @@ function walkDirectories(directories, result, sectionWeight = 0, sectionTitle, s details.sectionTitle = sectionTitle details.sectionId = sectionId details.rootSectionId = rootSectionId - details.id = fileName - details.isIndex = fileName.endsWith('/index.mdx') + details.id = fileName.replace(/\\/g, '/') + details.isIndex = fileName.endsWith(join('index.mdx')) details.slug = details.isIndex ? sectionSlug : slug.replace(/\.mdx$/, '') - if(details.slug.includes('/reference/specification/') && !details.title) { - const fileBaseName = basename(data.slug) // ex. v2.0.0 | v2.1.0-next-spec.1 - const fileName = fileBaseName.split('-')[0] // v2.0.0 | v2.1.0 - details.weight = specWeight-- - - if (fileName.startsWith('v')) { - details.title = capitalize(fileName.slice(1)) - } else { - details.title = capitalize(fileName) - } + if (details.slug.includes('/reference/specification/') && !details.title) { + const fileBaseName = basename(details.slug) + const versionDetails = getVersionDetails(details.slug, specWeight--); + details.title = versionDetails.title; + details.weight = versionDetails.weight; - if(releaseNotes.includes(details.title)){ + if (releaseNotes.includes(details.title)) { details.releaseNoteLink = `/blog/release-notes-${details.title}` } @@ -122,14 +153,10 @@ function walkDirectories(directories, result, sectionWeight = 0, sectionTitle, s } // To create a list of available ReleaseNotes list, which will be used to add details.releaseNoteLink attribute. - if(file.startsWith("release-notes") && dir[1] === "/blog"){ - const fileName_without_extension = file.slice(0,-4) - // removes the file extension. For example, release-notes-2.1.0.md -> release-notes-2.1.0 - const version = fileName_without_extension.slice(fileName_without_extension.lastIndexOf("-")+1) - - // gets the version from the name of the releaseNote .md file (from /blog). For example, version = 2.1.0 if fileName_without_extension = release-notes-2.1.0 - releaseNotes.push(version) - // releaseNotes is the list of all available releaseNotes + if (file.startsWith('release-notes') && dir[1] === '/blog') { + const { name } = parse(file); + const version = name.split('-').pop(); + releaseNotes.push(version); } addItem(details) @@ -138,24 +165,38 @@ function walkDirectories(directories, result, sectionWeight = 0, sectionTitle, s } } +// Matches heading IDs in two formats: +// 1. {#my-heading-id} +// 2. +const HEADING_ID_REGEX = /[\s]*(?:\{#([a-zA-Z0-9\-_]+)\}|= 2) { - slug = headingIdMatch[1] - } else { - // Try to match heading ids like {} - const anchorTagMatch = str.match(/[\s]*= 2) slug = anchorTagMatch[1] - } - return slug || slugify(str, { firsth1: true, maxdepth: 6 }) + if (typeof str !== 'string') return ''; + if (!str.trim()) return ''; + let slug = ''; + + // Match heading IDs like {# myHeadingId} + const idMatch = str.match(HEADING_ID_REGEX); + const [, headingId, anchorId] = idMatch || []; + slug = (headingId || anchorId || '').trim(); + + // If no valid ID is found, return an empty string + return slug; } -function isDirectory(dir) { - return statSync(dir).isDirectory() +async function isDirectory(dir) { + return (await stat(dir)).isDirectory() } function capitalize(text) { - return text.split(/[\s\-]/g).map(word => `${word[0].toUpperCase()}${word.substr(1)}`).join(' ') + return text.split(/[\s-]/) + .map(word => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); } + +module.exports = { slugifyToC, buildPostList } diff --git a/scripts/index.js b/scripts/index.js index 33125fe7533b..0fbbe3940851 100644 --- a/scripts/index.js +++ b/scripts/index.js @@ -1,13 +1,23 @@ const { resolve } = require('path'); const fs = require('fs'); const rssFeed = require('./build-rss'); -const buildPostList = require('./build-post-list'); +const { buildPostList } = require('./build-post-list'); const buildCaseStudiesList = require('./casestudies'); const buildAdoptersList = require('./adopters'); const buildFinanceInfoList = require('./finance'); async function start() { - await buildPostList(); + + const postDirectories = [ + ['pages/blog', '/blog'], + ['pages/docs', '/docs'], + ['pages/about', '/about'] + ]; + const basePath = 'pages'; + const writeFilePath = resolve(__dirname, '../config', 'posts.json'); + + await buildPostList(postDirectories, basePath, writeFilePath); + rssFeed( 'blog', 'AsyncAPI Initiative Blog RSS Feed', diff --git a/tests/build-post-list.test.js b/tests/build-post-list.test.js new file mode 100644 index 000000000000..fe0c8f8be5de --- /dev/null +++ b/tests/build-post-list.test.js @@ -0,0 +1,265 @@ +const fs = require('fs-extra'); +const { resolve, join } = require('path'); +const { setupTestDirectories, generateTempDirPath } = require('./helper/buildPostListSetup') +const { buildPostList, slugifyToC } = require('../scripts/build-post-list'); + +describe('buildPostList', () => { + let tempDir; + let writeFilePath; + let postDirectories; + + beforeEach(async () => { + tempDir = generateTempDirPath(__dirname); + writeFilePath = resolve(tempDir, 'posts.json'); + postDirectories = [ + [join(tempDir, 'blog'), '/blog'], + [join(tempDir, 'docs'), '/docs'], + [join(tempDir, 'about'), '/about'], + ]; + + await setupTestDirectories(tempDir); + + }); + + afterEach(async () => { + await fs.remove(tempDir); + }); + + it('writes the file successfully', async () => { + await buildPostList(postDirectories, tempDir, writeFilePath); + const outputExists = await fs.pathExists(writeFilePath); + expect(outputExists).toBe(true); + }); + + it('writes valid JSON content', async () => { + await buildPostList(postDirectories, tempDir, writeFilePath); + const content = await fs.readFile(writeFilePath, 'utf-8'); + expect(() => JSON.parse(content)).not.toThrow(); + }); + + it('correctly structures docs entries', async () => { + await buildPostList(postDirectories, tempDir, writeFilePath); + const output = JSON.parse(await fs.readFile(writeFilePath, 'utf-8')); + + expect(output.docs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + title: 'Docs Home', + slug: '/docs', + }), + expect.objectContaining({ + title: 'Reference', + slug: '/docs/reference', + isRootSection: true, + }), + expect.objectContaining({ + title: 'Specification', + slug: '/docs/reference/specification', + isSection: true, + }), + ]), + ); + }); + + it('correctly structures blog entries', async () => { + await buildPostList(postDirectories, tempDir, writeFilePath); + const output = JSON.parse(await fs.readFile(writeFilePath, 'utf-8')); + + expect(output.blog).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + title: 'Release Notes 2.1.0', + slug: '/blog/release-notes-2.1.0', + }), + ]), + ); + + expect(output.about).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + title: 'About Us', + slug: '/about', + }), + ]), + ); + + expect(output.docsTree).toBeDefined(); + + const blogEntry = output.blog.find( + (item) => item.slug === '/blog/release-notes-2.1.0', + ); + expect(blogEntry).toBeDefined(); + expect(blogEntry.title).toBe('Release Notes 2.1.0'); + }); + + it('handles a directory with only section files', async () => { + await fs.ensureDir(join(tempDir, 'docs', 'section1')); + await fs.writeFile( + join(tempDir, 'docs', 'section1', '_section.mdx'), + '---\ntitle: Section 1\n---\nThis is section 1.', + ); + + await buildPostList(postDirectories, tempDir, writeFilePath); + + const output = JSON.parse(await fs.readFile(writeFilePath, 'utf-8')); + + const sectionEntry = output.docs.find((item) => item.title === 'Section 1'); + expect(sectionEntry).toMatchObject({ + title: 'Section 1', + slug: expect.stringContaining('/docs/section1'), + isSection: true, + }); + }); + + it('handles multiple release notes correctly', async () => { + await fs.writeFile( + join(tempDir, 'blog', 'release-notes-2.1.1.mdx'), + '---\ntitle: Release Notes 2.1.1\n---\nThis is a release note.', + ); + + await buildPostList(postDirectories, tempDir, writeFilePath); + + const output = JSON.parse(await fs.readFile(writeFilePath, 'utf-8')); + + const firstReleaseNote = output.blog.find( + (item) => item.slug === '/blog/release-notes-2.1.0', + ); + const secondReleaseNote = output.blog.find( + (item) => item.slug === '/blog/release-notes-2.1.1', + ); + + expect(firstReleaseNote).toBeDefined(); + expect(firstReleaseNote.title).toBe('Release Notes 2.1.0'); + + expect(secondReleaseNote).toBeDefined(); + expect(secondReleaseNote.title).toBe('Release Notes 2.1.1'); + }); + + it('throws an error when accessing non-existent directory', async () => { + const invalidDir = [join(tempDir, 'non-existent-dir'), '/invalid']; + await expect( + buildPostList([invalidDir], tempDir, writeFilePath), + ).rejects.toThrow(/Error while building post list: ENOENT/); + }); + + it('does not process specification files without a title', async () => { + const specDir = join(tempDir, 'docs', 'reference', 'specification'); + await fs.writeFile( + join(specDir, 'v2.1.0-no-title.mdx'), + '---\n---\nContent of specification without a title.', + ); + + await buildPostList(postDirectories, tempDir, writeFilePath); + + const output = JSON.parse(await fs.readFile(writeFilePath, 'utf-8')); + const noTitleEntry = output.docs.find((item) => + item.slug.includes('/reference/specification/v2.1.0-no-title'), + ); + + expect(noTitleEntry).toBeUndefined(); + }); + + it('does not process specification files with "next-spec" in the filename', async () => { + const specDir = join(tempDir, 'docs', 'reference', 'specification'); + await fs.writeFile( + join(specDir, 'v2.1.0-next-spec.1.mdx'), + '---\n---\nContent of pre-release specification v2.1.0-next-spec.1.', + ); + + await buildPostList(postDirectories, tempDir, writeFilePath); + + const output = JSON.parse(await fs.readFile(writeFilePath, 'utf-8')); + const nextSpecEntry = output.docs.find((item) => + item.slug.includes('/reference/specification/v2.1.0-next-spec.1'), + ); + + expect(nextSpecEntry).toBeUndefined(); + }); + + it('does not process specification files with "explorer" in the filename', async () => { + const specDir = join(tempDir, 'docs', 'reference', 'specification'); + await fs.writeFile( + join(specDir, 'explorer.mdx'), + '---\n---\nContent of explorer specification.', + ); + + await buildPostList(postDirectories, tempDir, writeFilePath); + + const output = JSON.parse(await fs.readFile(writeFilePath, 'utf-8')); + const explorerEntry = output.docs.find((item) => + item.slug.includes('/reference/specification/explorer'), + ); + + expect(explorerEntry).toBeUndefined(); + }); + + it('throws "Error while building post list" when front matter is invalid', async () => { + await fs.writeFile( + join(tempDir, 'docs', 'invalid.mdx'), + '---\ninvalid front matter\n---\nContent', + ); + + await expect( + buildPostList(postDirectories, tempDir, writeFilePath), + ).rejects.toThrow(/Error while building post list/); + }); + + it('throws an error if no post directories are provided', async () => { + await expect(buildPostList([], tempDir, writeFilePath)).rejects.toThrow( + /Error while building post list/, + ); + }); + + it('throws specific error message when basePath parameter is undefined', async () => { + await expect( + buildPostList(postDirectories, undefined, writeFilePath), + ).rejects.toThrow( + "Error while building post list: basePath is required", + ); + }); + + it('throws specific error message when writeFilePath parameter is undefined', async () => { + await expect( + buildPostList(postDirectories, tempDir, undefined), + ).rejects.toThrow( + "Error while building post list: writeFilePath is required", + ); + }); + + describe('slugifyToC', () => { + it('handles heading ids like {# myHeadingId}', () => { + const input = '## My Heading {#custom-id}'; + expect(slugifyToC(input)).toBe('custom-id'); + }); + + it('handles heading ids like {}', () => { + const input = '## My Heading {}'; + expect(slugifyToC(input)).toBe('custom-anchor-id'); + }); + + it('handles empty strings', () => { + expect(slugifyToC('')).toBe(''); + }); + + it('returns empty string for malformed heading IDs', () => { + expect(slugifyToC('## Heading {#}')).toBe(''); + expect(slugifyToC('## Heading {# }')).toBe(''); + expect(slugifyToC('## Heading {}')).toBe(''); + }); + + it('handles mixed format heading IDs', () => { + expect(slugifyToC('## Heading {#id} {}')).toBe('id'); + }); + + it('handles invalid input types gracefully', () => { + expect(slugifyToC(null)).toBe(''); + expect(slugifyToC(undefined)).toBe(''); + expect(slugifyToC(123)).toBe(''); + }); + + it('ignores invalid characters in heading IDs', () => { + expect(slugifyToC('## Heading {#invalid@id}')).toBe(''); + expect(slugifyToC('## Heading {#invalid spaces}')).toBe(''); + }); + }); +}); diff --git a/tests/fixtures/buildPostListData.js b/tests/fixtures/buildPostListData.js new file mode 100644 index 000000000000..71416b2b6816 --- /dev/null +++ b/tests/fixtures/buildPostListData.js @@ -0,0 +1,29 @@ +const TEST_CONTENT = { + blog: { + dir: 'blog', + file: 'release-notes-2.1.0.mdx', + content: `--- +title: Release Notes 2.1.0 +--- +This is a release note.`, + }, + docs: { + dir: 'docs', + file: 'index.mdx', + content: `--- +title: Docs Home +--- +This is the documentation homepage.`, + subDir: 'reference/specification', + }, + about: { + dir: 'about', + file: 'index.mdx', + content: `--- +title: About Us +--- +This is the about page.`, + }, +}; + +module.exports = { TEST_CONTENT }; \ No newline at end of file diff --git a/tests/helper/buildPostListSetup.js b/tests/helper/buildPostListSetup.js new file mode 100644 index 000000000000..d7ee2908ba5a --- /dev/null +++ b/tests/helper/buildPostListSetup.js @@ -0,0 +1,24 @@ +const { TEST_CONTENT } = require("../fixtures/buildPostListData"); +const fs = require('fs-extra'); +const { join, resolve } = require("path") + +async function setupTestDirectories(tempDir) { + const dirs = ['blog', 'docs', 'about']; + for (const dir of dirs) { + await fs.ensureDir(join(tempDir, TEST_CONTENT[dir].dir)); + await fs.writeFile( + join(tempDir, TEST_CONTENT[dir].dir, TEST_CONTENT[dir].file), + TEST_CONTENT[dir].content + ); + } + await fs.ensureDir(join(tempDir, TEST_CONTENT.docs.dir, TEST_CONTENT.docs.subDir)); +} + +function generateTempDirPath(baseDir) { + return resolve( + baseDir, + `test-config-${Date.now()}-${Math.random().toString(36).slice(2)}` + ); +} + +module.exports = { setupTestDirectories, generateTempDirPath }; \ No newline at end of file diff --git a/tests/index.test.js b/tests/index.test.js index 78e2c216958f..37b124547efd 100644 --- a/tests/index.test.js +++ b/tests/index.test.js @@ -1,5 +1,5 @@ const rssFeed = require('../scripts/build-rss'); -const buildPostList = require('../scripts/build-post-list'); +const { buildPostList } = require('../scripts/build-post-list'); const buildCaseStudiesList = require('../scripts/casestudies'); const buildAdoptersList = require('../scripts/adopters'); const buildFinanceInfoList = require('../scripts/finance');