From 69897103b61a7263bf1bd26dd23131dbaa909e40 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 29 Aug 2024 15:43:57 +0100 Subject: [PATCH 01/15] Allow integrations to refresh content layer data This reverts commit 90a862fc94452d40e834f625fb6d2cb3030e0970. --- packages/astro/src/content/loaders/types.ts | 1 + packages/astro/src/integrations/hooks.ts | 12 ++++++- .../astro/src/types/public/integrations.ts | 2 ++ .../fixtures/content-layer/astro.config.mjs | 33 ++++++++++++++++++- 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/packages/astro/src/content/loaders/types.ts b/packages/astro/src/content/loaders/types.ts index 86411407f740..32a3e929ad87 100644 --- a/packages/astro/src/content/loaders/types.ts +++ b/packages/astro/src/content/loaders/types.ts @@ -33,6 +33,7 @@ export interface LoaderContext { /** When running in dev, this is a filesystem watcher that can be used to trigger updates */ watcher?: FSWatcher; + /** If the loader has been triggered by an integration, this may optionally contain extra data set by that integration */ refreshContextData?: Record; entryTypes: Map; } diff --git a/packages/astro/src/integrations/hooks.ts b/packages/astro/src/integrations/hooks.ts index b86281cbb274..a27f1c3565a2 100644 --- a/packages/astro/src/integrations/hooks.ts +++ b/packages/astro/src/integrations/hooks.ts @@ -13,7 +13,7 @@ import type { AstroIntegrationLogger, Logger } from '../core/logger/core.js'; import { isServerLikeOutput } from '../core/util.js'; import type { AstroSettings } from '../types/astro.js'; import type { AstroConfig } from '../types/public/config.js'; -import type { ContentEntryType, DataEntryType } from '../types/public/content.js'; +import type { ContentEntryType, DataEntryType, RefreshContentOptions } from '../types/public/content.js'; import type { AstroIntegration, AstroRenderer, @@ -22,6 +22,7 @@ import type { } from '../types/public/integrations.js'; import type { RouteData } from '../types/public/internal.js'; import { validateSupportedFeatures } from './features-validation.js'; +import { globalContentLayer } from '../content/content-layer.js'; async function withTakingALongTimeMsg({ name, @@ -367,6 +368,14 @@ export async function runHookServerSetup({ server: ViteDevServer; logger: Logger; }) { + let refreshContent: undefined | ((options: RefreshContentOptions) => Promise); + if (config.experimental?.contentLayer) { + refreshContent = async (options: RefreshContentOptions) => { + const contentLayer = await globalContentLayer.get(); + await contentLayer?.sync(options); + }; + } + for (const integration of config.integrations) { if (integration?.hooks?.['astro:server:setup']) { await withTakingALongTimeMsg({ @@ -376,6 +385,7 @@ export async function runHookServerSetup({ server, logger: getLogger(integration, logger), toolbar: getToolbarServerCommunicationHelpers(server), + refreshContent, }), logger, }); diff --git a/packages/astro/src/types/public/integrations.ts b/packages/astro/src/types/public/integrations.ts index a65f2513b0a7..0a5e5dce0b1f 100644 --- a/packages/astro/src/types/public/integrations.ts +++ b/packages/astro/src/types/public/integrations.ts @@ -8,6 +8,7 @@ import type { DeepPartial } from '../../type-utils.js'; import type { AstroConfig } from './config.js'; import type { RouteData } from './internal.js'; import type { DevToolbarAppEntry } from './toolbar.js'; +import type { RefreshContentOptions } from './content.js'; export interface RouteOptions { /** @@ -187,6 +188,7 @@ export interface BaseIntegrationHooks { server: ViteDevServer; logger: AstroIntegrationLogger; toolbar: ReturnType; + refreshContent?: (options: RefreshContentOptions) => Promise; }) => void | Promise; 'astro:server:start': (options: { address: AddressInfo; diff --git a/packages/astro/test/fixtures/content-layer/astro.config.mjs b/packages/astro/test/fixtures/content-layer/astro.config.mjs index 3266e5e8c0ad..3548ef7e290a 100644 --- a/packages/astro/test/fixtures/content-layer/astro.config.mjs +++ b/packages/astro/test/fixtures/content-layer/astro.config.mjs @@ -3,7 +3,38 @@ import { defineConfig } from 'astro/config'; import { fileURLToPath } from 'node:url'; export default defineConfig({ - integrations: [mdx()], + integrations: [mdx(), { + name: '@astrojs/my-integration', + hooks: { + 'astro:server:setup': async ({ server, refreshContent }) => { + server.middlewares.use('/_refresh', async (req, res) => { + if(req.method !== 'POST') { + res.statusCode = 405 + res.end('Method Not Allowed'); + return + } + let body = ''; + req.on('data', chunk => { + body += chunk.toString(); + }); + req.on('end', async () => { + try { + const webhookBody = JSON.parse(body); + await refreshContent({ + context: { webhookBody }, + loaders: ['increment-loader'] + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ message: 'Content refreshed successfully' })); + } catch (error) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Failed to refresh content' })); + } + }); + }); + } + } +}], vite: { resolve: { alias: { From 5dd3266c64bb8776db53c20a902c4059583f0ba6 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 29 Aug 2024 16:57:26 +0100 Subject: [PATCH 02/15] Add test --- packages/astro/test/content-layer.test.js | 14 ++++++++++++++ .../test/fixtures/content-layer/astro.config.mjs | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index d8d6a3b4512f..05e29842cdfa 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -290,5 +290,19 @@ describe('Content Layer', () => { assert.ok(updated.fileLoader[0].data.temperament.includes('Bouncy')); await fixture.resetAllFiles(); }); + + it('reloads data when an integration triggers a content refresh', async () => { + const rawJsonResponse = await fixture.fetch('/collections.json'); + const initialJson = devalue.parse(await rawJsonResponse.text()); + assert.equal(initialJson.increment.data.lastValue, 1); + + const refreshResponse = await fixture.fetch('/_refresh', { method: 'POST', body: JSON.stringify({}) }); + const refreshData = await refreshResponse.json(); + assert.equal(refreshData.message, 'Content refreshed successfully'); + + const updatedJsonResponse = await fixture.fetch('/collections.json'); + const updated = devalue.parse(await updatedJsonResponse.text()); + assert.equal(updated.increment.data.lastValue, 2); + }) }); }); diff --git a/packages/astro/test/fixtures/content-layer/astro.config.mjs b/packages/astro/test/fixtures/content-layer/astro.config.mjs index 3548ef7e290a..37afc3992758 100644 --- a/packages/astro/test/fixtures/content-layer/astro.config.mjs +++ b/packages/astro/test/fixtures/content-layer/astro.config.mjs @@ -28,7 +28,7 @@ export default defineConfig({ res.end(JSON.stringify({ message: 'Content refreshed successfully' })); } catch (error) { res.writeHead(500, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ error: 'Failed to refresh content' })); + res.end(JSON.stringify({ error: 'Failed to refresh content: ' + error.message })); } }); }); From 537bcafcde0014a0cda2980dc5c17cf633a26216 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 29 Aug 2024 17:11:31 +0100 Subject: [PATCH 03/15] Add changeset --- .changeset/curvy-walls-kneel.md | 47 +++++++++++++++++++ packages/astro/src/integrations/hooks.ts | 8 +++- .../astro/src/types/public/integrations.ts | 2 +- packages/astro/test/content-layer.test.js | 7 ++- 4 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 .changeset/curvy-walls-kneel.md diff --git a/.changeset/curvy-walls-kneel.md b/.changeset/curvy-walls-kneel.md new file mode 100644 index 000000000000..0409f1c538e5 --- /dev/null +++ b/.changeset/curvy-walls-kneel.md @@ -0,0 +1,47 @@ +--- +'astro': patch +--- + +Adds support for allowing integrations to refresh the content layer + +This adds a new object to the `astro:server:setup` hook that allows integrations to refresh the content layer. This can be used for example to register a webhook endpoint during dev, or to open a socket to a CMS to listen for changes. + +The hook is passed a function called `refreshContent` that can be called to refresh the content layer. It can optionally be passed a `loaders` property, which is an array of loader names. If provided, only collections that use those loaders will be refreshed. If not provided, all loaders will be refreshed. A CMS integration could use this to only refresh its own collections. + +It can also pass a `context` object, which will be passed to the loaders. This can be used to pass arbitrary data, such as the webhook body or an event from the websocket. + +```ts + { + name: 'my-integration', + hooks: { + 'astro:server:setup': async ({ server, refreshContent }) => { + server.middlewares.use('/_refresh', async (req, res) => { + if(req.method !== 'POST') { + res.statusCode = 405 + res.end('Method Not Allowed'); + return + } + let body = ''; + req.on('data', chunk => { + body += chunk.toString(); + }); + req.on('end', async () => { + try { + const webhookBody = JSON.parse(body); + await refreshContent({ + context: { webhookBody }, + loaders: ['my-loader'] + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ message: 'Content refreshed successfully' })); + } catch (error) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Failed to refresh content: ' + error.message })); + } + }); + }); + } + } +} +``` + diff --git a/packages/astro/src/integrations/hooks.ts b/packages/astro/src/integrations/hooks.ts index a27f1c3565a2..4c9528aa6cff 100644 --- a/packages/astro/src/integrations/hooks.ts +++ b/packages/astro/src/integrations/hooks.ts @@ -5,6 +5,7 @@ import { bold } from 'kleur/colors'; import type { InlineConfig, ViteDevServer } from 'vite'; import astroIntegrationActionsRouteHandler from '../actions/integration.js'; import { isActionsFilePresent } from '../actions/utils.js'; +import { globalContentLayer } from '../content/content-layer.js'; import type { SerializedSSRManifest } from '../core/app/types.js'; import type { PageBuildData } from '../core/build/types.js'; import { buildClientDirectiveEntrypoint } from '../core/client-directive/index.js'; @@ -13,7 +14,11 @@ import type { AstroIntegrationLogger, Logger } from '../core/logger/core.js'; import { isServerLikeOutput } from '../core/util.js'; import type { AstroSettings } from '../types/astro.js'; import type { AstroConfig } from '../types/public/config.js'; -import type { ContentEntryType, DataEntryType, RefreshContentOptions } from '../types/public/content.js'; +import type { + ContentEntryType, + DataEntryType, + RefreshContentOptions, +} from '../types/public/content.js'; import type { AstroIntegration, AstroRenderer, @@ -22,7 +27,6 @@ import type { } from '../types/public/integrations.js'; import type { RouteData } from '../types/public/internal.js'; import { validateSupportedFeatures } from './features-validation.js'; -import { globalContentLayer } from '../content/content-layer.js'; async function withTakingALongTimeMsg({ name, diff --git a/packages/astro/src/types/public/integrations.ts b/packages/astro/src/types/public/integrations.ts index 0a5e5dce0b1f..19cc1e2983c5 100644 --- a/packages/astro/src/types/public/integrations.ts +++ b/packages/astro/src/types/public/integrations.ts @@ -6,9 +6,9 @@ import type { AstroIntegrationLogger } from '../../core/logger/core.js'; import type { getToolbarServerCommunicationHelpers } from '../../integrations/hooks.js'; import type { DeepPartial } from '../../type-utils.js'; import type { AstroConfig } from './config.js'; +import type { RefreshContentOptions } from './content.js'; import type { RouteData } from './internal.js'; import type { DevToolbarAppEntry } from './toolbar.js'; -import type { RefreshContentOptions } from './content.js'; export interface RouteOptions { /** diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index 05e29842cdfa..0a47189b8c17 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -296,13 +296,16 @@ describe('Content Layer', () => { const initialJson = devalue.parse(await rawJsonResponse.text()); assert.equal(initialJson.increment.data.lastValue, 1); - const refreshResponse = await fixture.fetch('/_refresh', { method: 'POST', body: JSON.stringify({}) }); + const refreshResponse = await fixture.fetch('/_refresh', { + method: 'POST', + body: JSON.stringify({}), + }); const refreshData = await refreshResponse.json(); assert.equal(refreshData.message, 'Content refreshed successfully'); const updatedJsonResponse = await fixture.fetch('/collections.json'); const updated = devalue.parse(await updatedJsonResponse.text()); assert.equal(updated.increment.data.lastValue, 2); - }) + }); }); }); From f0babff3aed20a6dad224e9b36535a6f80d76cf4 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 29 Aug 2024 17:37:11 +0100 Subject: [PATCH 04/15] Add wait in test --- packages/astro/test/content-layer.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index 0a47189b8c17..10e7800b6447 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -302,7 +302,7 @@ describe('Content Layer', () => { }); const refreshData = await refreshResponse.json(); assert.equal(refreshData.message, 'Content refreshed successfully'); - + await new Promise((r) => setTimeout(r, 1000)); const updatedJsonResponse = await fixture.fetch('/collections.json'); const updated = devalue.parse(await updatedJsonResponse.text()); assert.equal(updated.increment.data.lastValue, 2); From 65cb9fed6ae980a4453290816e15641be57525da Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 30 Aug 2024 11:15:08 +0100 Subject: [PATCH 05/15] Dispose of queue --- packages/astro/src/content/content-layer.ts | 9 +++++++-- packages/astro/test/content-layer.test.js | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/astro/src/content/content-layer.ts b/packages/astro/src/content/content-layer.ts index 1dd86267b4df..65423bf2e4ba 100644 --- a/packages/astro/src/content/content-layer.ts +++ b/packages/astro/src/content/content-layer.ts @@ -76,6 +76,11 @@ export class ContentLayer { this.#unsubscribe?.(); } + dispose() { + this.#queue.kill(); + this.#unsubscribe?.(); + } + async #getGenerateDigest() { if (this.#generateDigest) { return this.#generateDigest; @@ -301,13 +306,13 @@ function contentLayerSingleton() { let instance: ContentLayer | null = null; return { init: (options: ContentLayerOptions) => { - instance?.unwatchContentConfig(); + instance?.dispose(); instance = new ContentLayer(options); return instance; }, get: () => instance, dispose: () => { - instance?.unwatchContentConfig(); + instance?.dispose(); instance = null; }, }; diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index 10e7800b6447..fff04a1a441a 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -302,7 +302,6 @@ describe('Content Layer', () => { }); const refreshData = await refreshResponse.json(); assert.equal(refreshData.message, 'Content refreshed successfully'); - await new Promise((r) => setTimeout(r, 1000)); const updatedJsonResponse = await fixture.fetch('/collections.json'); const updated = devalue.parse(await updatedJsonResponse.text()); assert.equal(updated.increment.data.lastValue, 2); From 760042420609112a1151ff45493682392ed8acab Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 30 Aug 2024 14:40:23 +0100 Subject: [PATCH 06/15] Skip if no content layer collections --- packages/astro/src/integrations/hooks.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/astro/src/integrations/hooks.ts b/packages/astro/src/integrations/hooks.ts index 4c9528aa6cff..1e4ad5ca33dc 100644 --- a/packages/astro/src/integrations/hooks.ts +++ b/packages/astro/src/integrations/hooks.ts @@ -5,7 +5,9 @@ import { bold } from 'kleur/colors'; import type { InlineConfig, ViteDevServer } from 'vite'; import astroIntegrationActionsRouteHandler from '../actions/integration.js'; import { isActionsFilePresent } from '../actions/utils.js'; +import { CONTENT_LAYER_TYPE } from '../content/consts.js'; import { globalContentLayer } from '../content/content-layer.js'; +import { globalContentConfigObserver } from '../content/utils.js'; import type { SerializedSSRManifest } from '../core/app/types.js'; import type { PageBuildData } from '../core/build/types.js'; import { buildClientDirectiveEntrypoint } from '../core/client-directive/index.js'; @@ -375,6 +377,16 @@ export async function runHookServerSetup({ let refreshContent: undefined | ((options: RefreshContentOptions) => Promise); if (config.experimental?.contentLayer) { refreshContent = async (options: RefreshContentOptions) => { + const contentConfig = globalContentConfigObserver.get(); + if ( + contentConfig.status !== 'loaded' || + !Object.values(contentConfig.config.collections).some( + (collection) => collection.type === CONTENT_LAYER_TYPE, + ) + ) { + return; + } + const contentLayer = await globalContentLayer.get(); await contentLayer?.sync(options); }; From cddeacc7c723bb5e2d143bd572fa9ef68d841c94 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 30 Aug 2024 14:41:28 +0100 Subject: [PATCH 07/15] Use spaces in markdown --- .changeset/curvy-walls-kneel.md | 60 ++++++++++++++++----------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.changeset/curvy-walls-kneel.md b/.changeset/curvy-walls-kneel.md index 0409f1c538e5..a977507099a6 100644 --- a/.changeset/curvy-walls-kneel.md +++ b/.changeset/curvy-walls-kneel.md @@ -12,36 +12,36 @@ It can also pass a `context` object, which will be passed to the loaders. This c ```ts { - name: 'my-integration', - hooks: { - 'astro:server:setup': async ({ server, refreshContent }) => { - server.middlewares.use('/_refresh', async (req, res) => { - if(req.method !== 'POST') { - res.statusCode = 405 - res.end('Method Not Allowed'); - return - } - let body = ''; - req.on('data', chunk => { - body += chunk.toString(); - }); - req.on('end', async () => { - try { - const webhookBody = JSON.parse(body); - await refreshContent({ - context: { webhookBody }, - loaders: ['my-loader'] - }); - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ message: 'Content refreshed successfully' })); - } catch (error) { - res.writeHead(500, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ error: 'Failed to refresh content: ' + error.message })); - } - }); - }); - } - } + name: 'my-integration', + hooks: { + 'astro:server:setup': async ({ server, refreshContent }) => { + server.middlewares.use('/_refresh', async (req, res) => { + if(req.method !== 'POST') { + res.statusCode = 405 + res.end('Method Not Allowed'); + return + } + let body = ''; + req.on('data', chunk => { + body += chunk.toString(); + }); + req.on('end', async () => { + try { + const webhookBody = JSON.parse(body); + await refreshContent({ + context: { webhookBody }, + loaders: ['my-loader'] + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ message: 'Content refreshed successfully' })); + } catch (error) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Failed to refresh content: ' + error.message })); + } + }); + }); + } + } } ``` From 5239d36785e258c46fa21ea6338d669c3007bbe0 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Mon, 2 Sep 2024 13:00:09 +0100 Subject: [PATCH 08/15] fix: don't keep data store in node_modules during dev --- packages/astro/src/content/content-layer.ts | 9 +++- packages/astro/src/content/data-store.ts | 3 ++ .../vite-plugin-content-virtual-mod.ts | 42 ++++++++++++------- packages/astro/src/core/build/index.ts | 7 ++-- packages/astro/src/core/dev/dev.ts | 4 +- packages/astro/src/core/sync/index.ts | 12 +++--- 6 files changed, 49 insertions(+), 28 deletions(-) diff --git a/packages/astro/src/content/content-layer.ts b/packages/astro/src/content/content-layer.ts index 65423bf2e4ba..8bdf8e62d40b 100644 --- a/packages/astro/src/content/content-layer.ts +++ b/packages/astro/src/content/content-layer.ts @@ -241,7 +241,7 @@ export class ContentLayer { if (!existsSync(this.#settings.config.cacheDir)) { await fs.mkdir(this.#settings.config.cacheDir, { recursive: true }); } - const cacheFile = new URL(DATA_STORE_FILE, this.#settings.config.cacheDir); + const cacheFile = getDataStoreFile(this.#settings); await this.#store.writeToDisk(cacheFile); if (!existsSync(this.#settings.dotAstroDir)) { await fs.mkdir(this.#settings.dotAstroDir, { recursive: true }); @@ -302,6 +302,13 @@ export async function simpleLoader( } } +export function getDataStoreFile(settings: AstroSettings) { + return new URL( + DATA_STORE_FILE, + process?.env.NODE_ENV === 'development' ? settings.dotAstroDir : settings.config.cacheDir, + ); +} + function contentLayerSingleton() { let instance: ContentLayer | null = null; return { diff --git a/packages/astro/src/content/data-store.ts b/packages/astro/src/content/data-store.ts index 76cefc411a76..492fe77bbb70 100644 --- a/packages/astro/src/content/data-store.ts +++ b/packages/astro/src/content/data-store.ts @@ -90,6 +90,9 @@ export class DataStore { try { // @ts-expect-error - this is a virtual module const data = await import('astro:data-layer-content'); + if (data.default instanceof Map) { + return DataStore.fromMap(data.default); + } const map = devalue.unflatten(data.default); return DataStore.fromMap(map); } catch {} diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts index 0ad253dd88fe..f87fbd7a98ee 100644 --- a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts +++ b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts @@ -54,12 +54,16 @@ export function astroContentVirtualModPlugin({ }: AstroContentVirtualModPluginParams): Plugin { let IS_DEV = false; const IS_SERVER = isServerLikeOutput(settings.config); - const dataStoreFile = new URL(DATA_STORE_FILE, settings.config.cacheDir); + let dataStoreFile: URL; return { name: 'astro-content-virtual-mod-plugin', enforce: 'pre', configResolved(config) { IS_DEV = config.mode === 'development'; + dataStoreFile = new URL( + DATA_STORE_FILE, + IS_DEV ? settings.dotAstroDir : settings.config.cacheDir, + ); }, async resolveId(id) { if (id === VIRTUAL_MODULE_ID) { @@ -180,25 +184,31 @@ export function astroContentVirtualModPlugin({ configureServer(server) { const dataStorePath = fileURLToPath(dataStoreFile); - // Watch for changes to the data store file - if (Array.isArray(server.watcher.options.ignored)) { - // The data store file is in node_modules, so is ignored by default, - // so we need to un-ignore it. - server.watcher.options.ignored.push(`!${dataStorePath}`); - } + server.watcher.add(dataStorePath); + function invalidateDataStore() { + const module = server.moduleGraph.getModuleById(RESOLVED_DATA_STORE_VIRTUAL_ID); + if (module) { + server.moduleGraph.invalidateModule(module); + } + server.ws.send({ + type: 'full-reload', + path: '*', + }); + } + + // If the datastore file changes, invalidate the virtual module + + server.watcher.on('add', (addedPath) => { + if (addedPath === dataStorePath) { + invalidateDataStore(); + } + }); + server.watcher.on('change', (changedPath) => { - // If the datastore file changes, invalidate the virtual module if (changedPath === dataStorePath) { - const module = server.moduleGraph.getModuleById(RESOLVED_DATA_STORE_VIRTUAL_ID); - if (module) { - server.moduleGraph.invalidateModule(module); - } - server.ws.send({ - type: 'full-reload', - path: '*', - }); + invalidateDataStore(); } }); }, diff --git a/packages/astro/src/core/build/index.ts b/packages/astro/src/core/build/index.ts index 74a648304789..10e6b2dbcb97 100644 --- a/packages/astro/src/core/build/index.ts +++ b/packages/astro/src/core/build/index.ts @@ -56,6 +56,9 @@ export default async function build( const logger = createNodeLogger(inlineConfig); const { userConfig, astroConfig } = await resolveConfig(inlineConfig, 'build'); telemetry.record(eventCliSession('build', userConfig)); + + const settings = await createSettings(astroConfig, fileURLToPath(astroConfig.root)); + if (inlineConfig.force) { if (astroConfig.experimental.contentCollectionCache) { const contentCacheDir = new URL('./content/', astroConfig.cacheDir); @@ -65,11 +68,9 @@ export default async function build( logger.warn('content', 'content cache cleared (force)'); } } - await clearContentLayerCache({ astroConfig, logger, fs }); + await clearContentLayerCache({ settings, logger, fs }); } - const settings = await createSettings(astroConfig, fileURLToPath(astroConfig.root)); - const builder = new AstroBuilder(settings, { ...options, logger, diff --git a/packages/astro/src/core/dev/dev.ts b/packages/astro/src/core/dev/dev.ts index 73ec0fa713e2..3fbcd7a46715 100644 --- a/packages/astro/src/core/dev/dev.ts +++ b/packages/astro/src/core/dev/dev.ts @@ -6,7 +6,7 @@ import { green } from 'kleur/colors'; import { gt, major, minor, patch } from 'semver'; import type * as vite from 'vite'; import { DATA_STORE_FILE } from '../../content/consts.js'; -import { globalContentLayer } from '../../content/content-layer.js'; +import { getDataStoreFile, globalContentLayer } from '../../content/content-layer.js'; import { attachContentServerListeners } from '../../content/index.js'; import { MutableDataStore } from '../../content/mutable-data-store.js'; import { globalContentConfigObserver } from '../../content/utils.js'; @@ -108,7 +108,7 @@ export default async function dev(inlineConfig: AstroInlineConfig): Promise { if (force) { - await clearContentLayerCache({ astroConfig: settings.config, logger, fs }); + await clearContentLayerCache({ settings, logger, fs }); } const timerStart = performance.now(); @@ -107,7 +107,7 @@ export async function syncInternal({ settings.timer.start('Sync content layer'); let store: MutableDataStore | undefined; try { - const dataStoreFile = new URL(DATA_STORE_FILE, settings.config.cacheDir); + const dataStoreFile = getDataStoreFile(settings); if (existsSync(dataStoreFile)) { store = await MutableDataStore.fromFile(dataStoreFile); } From 00c473938d6618bb23ddac427f885bd961d5209f Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Mon, 2 Sep 2024 13:11:20 +0100 Subject: [PATCH 09/15] Lint --- packages/astro/src/core/dev/dev.ts | 1 - packages/astro/src/core/sync/index.ts | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/astro/src/core/dev/dev.ts b/packages/astro/src/core/dev/dev.ts index 3fbcd7a46715..72a173dbe462 100644 --- a/packages/astro/src/core/dev/dev.ts +++ b/packages/astro/src/core/dev/dev.ts @@ -5,7 +5,6 @@ import { performance } from 'node:perf_hooks'; import { green } from 'kleur/colors'; import { gt, major, minor, patch } from 'semver'; import type * as vite from 'vite'; -import { DATA_STORE_FILE } from '../../content/consts.js'; import { getDataStoreFile, globalContentLayer } from '../../content/content-layer.js'; import { attachContentServerListeners } from '../../content/index.js'; import { MutableDataStore } from '../../content/mutable-data-store.js'; diff --git a/packages/astro/src/core/sync/index.ts b/packages/astro/src/core/sync/index.ts index 7956fbdce797..88c37fa882ef 100644 --- a/packages/astro/src/core/sync/index.ts +++ b/packages/astro/src/core/sync/index.ts @@ -3,7 +3,7 @@ import { performance } from 'node:perf_hooks'; import { fileURLToPath } from 'node:url'; import { dim } from 'kleur/colors'; import { type HMRPayload, createServer } from 'vite'; -import { CONTENT_TYPES_FILE, DATA_STORE_FILE } from '../../content/consts.js'; +import { CONTENT_TYPES_FILE } from '../../content/consts.js'; import { getDataStoreFile, globalContentLayer } from '../../content/content-layer.js'; import { createContentTypesGenerator } from '../../content/index.js'; import { MutableDataStore } from '../../content/mutable-data-store.js'; @@ -13,7 +13,7 @@ import { telemetry } from '../../events/index.js'; import { eventCliSession } from '../../events/session.js'; import { runHookConfigDone, runHookConfigSetup } from '../../integrations/hooks.js'; import type { AstroSettings } from '../../types/astro.js'; -import type { AstroConfig, AstroInlineConfig } from '../../types/public/config.js'; +import type { AstroInlineConfig } from '../../types/public/config.js'; import { getTimeStat } from '../build/util.js'; import { resolveConfig } from '../config/config.js'; import { createNodeLogger } from '../config/logging.js'; From 5a772432f8bbd220b4e64b5bd14637addf8555e8 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Mon, 2 Sep 2024 16:43:38 +0100 Subject: [PATCH 10/15] Fix test --- packages/astro/test/content-layer.test.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index fff04a1a441a..e43aa0cd5434 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -5,6 +5,8 @@ import { sep as posixSep } from 'node:path/posix'; import { after, before, describe, it } from 'node:test'; import * as devalue from 'devalue'; +const pause = (ms = 700) => new Promise((r) => setTimeout(r, ms)); + import { loadFixture } from './test-utils.js'; describe('Content Layer', () => { /** @type {import("./test-utils.js").Fixture} */ @@ -192,7 +194,8 @@ describe('Content Layer', () => { let devServer; let json; before(async () => { - devServer = await fixture.startDevServer(); + devServer = await fixture.startDevServer({ force: true }); + await pause() const rawJsonResponse = await fixture.fetch('/collections.json'); const rawJson = await rawJsonResponse.text(); json = devalue.parse(rawJson); @@ -282,13 +285,14 @@ describe('Content Layer', () => { return JSON.stringify(data, null, 2); }); - // Writes are debounced to 500ms - await new Promise((r) => setTimeout(r, 700)); + // Writes are debounced, so we need to wait a few ms + await pause() const updatedJsonResponse = await fixture.fetch('/collections.json'); const updated = devalue.parse(await updatedJsonResponse.text()); assert.ok(updated.fileLoader[0].data.temperament.includes('Bouncy')); await fixture.resetAllFiles(); + await pause() }); it('reloads data when an integration triggers a content refresh', async () => { From 458f2f55d5085f2a57772c432b5181b15424d868 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Mon, 2 Sep 2024 16:47:13 +0100 Subject: [PATCH 11/15] Apply suggestions from code review Co-authored-by: Sarah Rainsberger --- .changeset/curvy-walls-kneel.md | 8 +++----- packages/astro/test/content-layer.test.js | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.changeset/curvy-walls-kneel.md b/.changeset/curvy-walls-kneel.md index a977507099a6..dc04cb0af4aa 100644 --- a/.changeset/curvy-walls-kneel.md +++ b/.changeset/curvy-walls-kneel.md @@ -2,13 +2,11 @@ 'astro': patch --- -Adds support for allowing integrations to refresh the content layer +Adds a new function `refreshContent` to the `astro:server:setup` hook that allows integrations to refresh the content layer. This can be used, for example, to register a webhook endpoint during dev, or to open a socket to a CMS to listen for changes. -This adds a new object to the `astro:server:setup` hook that allows integrations to refresh the content layer. This can be used for example to register a webhook endpoint during dev, or to open a socket to a CMS to listen for changes. +By default, `refreshContent` will refresh all collections. You can optionally pass a `loaders` property, which is an array of loader names. If provided, only collections that use those loaders will be refreshed. For example, A CMS integration could use this property to only refresh its own collections. -The hook is passed a function called `refreshContent` that can be called to refresh the content layer. It can optionally be passed a `loaders` property, which is an array of loader names. If provided, only collections that use those loaders will be refreshed. If not provided, all loaders will be refreshed. A CMS integration could use this to only refresh its own collections. - -It can also pass a `context` object, which will be passed to the loaders. This can be used to pass arbitrary data, such as the webhook body or an event from the websocket. +You can also pass a `context` object to the loaders. This can be used to pass arbitrary data, such as the webhook body, or an event from the websocket. ```ts { diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index e43aa0cd5434..3b8d0a75d0b1 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -5,7 +5,7 @@ import { sep as posixSep } from 'node:path/posix'; import { after, before, describe, it } from 'node:test'; import * as devalue from 'devalue'; -const pause = (ms = 700) => new Promise((r) => setTimeout(r, ms)); +const pause = (ms = 1000) => new Promise((r) => setTimeout(r, ms)); import { loadFixture } from './test-utils.js'; describe('Content Layer', () => { From e83f2cd58aa643e6e1d816d92f7bb76d0e342881 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Mon, 2 Sep 2024 17:09:31 +0100 Subject: [PATCH 12/15] Re-order tests --- packages/astro/test/content-layer.test.js | 31 ++++++++++++----------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index 3b8d0a75d0b1..6415a70e4c5e 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -274,6 +274,22 @@ describe('Content Layer', () => { }); }); + it('reloads data when an integration triggers a content refresh', async () => { + const rawJsonResponse = await fixture.fetch('/collections.json'); + const initialJson = devalue.parse(await rawJsonResponse.text()); + assert.equal(initialJson.increment.data.lastValue, 1); + + const refreshResponse = await fixture.fetch('/_refresh', { + method: 'POST', + body: JSON.stringify({}), + }); + const refreshData = await refreshResponse.json(); + assert.equal(refreshData.message, 'Content refreshed successfully'); + const updatedJsonResponse = await fixture.fetch('/collections.json'); + const updated = devalue.parse(await updatedJsonResponse.text()); + assert.equal(updated.increment.data.lastValue, 2); + }); + it('updates collection when data file is changed', async () => { const rawJsonResponse = await fixture.fetch('/collections.json'); const initialJson = devalue.parse(await rawJsonResponse.text()); @@ -292,23 +308,8 @@ describe('Content Layer', () => { const updated = devalue.parse(await updatedJsonResponse.text()); assert.ok(updated.fileLoader[0].data.temperament.includes('Bouncy')); await fixture.resetAllFiles(); - await pause() }); - it('reloads data when an integration triggers a content refresh', async () => { - const rawJsonResponse = await fixture.fetch('/collections.json'); - const initialJson = devalue.parse(await rawJsonResponse.text()); - assert.equal(initialJson.increment.data.lastValue, 1); - const refreshResponse = await fixture.fetch('/_refresh', { - method: 'POST', - body: JSON.stringify({}), - }); - const refreshData = await refreshResponse.json(); - assert.equal(refreshData.message, 'Content refreshed successfully'); - const updatedJsonResponse = await fixture.fetch('/collections.json'); - const updated = devalue.parse(await updatedJsonResponse.text()); - assert.equal(updated.increment.data.lastValue, 2); - }); }); }); From b0aef2fdc74b0e3659656e4f52cf73d4a99245fc Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Tue, 3 Sep 2024 10:50:01 +0100 Subject: [PATCH 13/15] Wait for data store --- packages/astro/test/content-layer.test.js | 8 ++------ packages/astro/test/test-utils.js | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index 3a0316ca5b0f..cf3c7579953b 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -5,8 +5,6 @@ import { sep as posixSep } from 'node:path/posix'; import { after, before, describe, it } from 'node:test'; import * as devalue from 'devalue'; -const pause = (ms = 1000) => new Promise((r) => setTimeout(r, ms)); - import { loadFixture } from './test-utils.js'; describe('Content Layer', () => { /** @type {import("./test-utils.js").Fixture} */ @@ -199,7 +197,7 @@ describe('Content Layer', () => { let json; before(async () => { devServer = await fixture.startDevServer({ force: true }); - await pause() + await fixture.onNextDataStoreChange(); const rawJsonResponse = await fixture.fetch('/collections.json'); const rawJson = await rawJsonResponse.text(); json = devalue.parse(rawJson); @@ -305,9 +303,7 @@ describe('Content Layer', () => { return JSON.stringify(data, null, 2); }); - // Writes are debounced, so we need to wait a few ms - await pause() - + await fixture.onNextDataStoreChange(); const updatedJsonResponse = await fixture.fetch('/collections.json'); const updated = devalue.parse(await updatedJsonResponse.text()); assert.ok(updated.fileLoader[0].data.temperament.includes('Bouncy')); diff --git a/packages/astro/test/test-utils.js b/packages/astro/test/test-utils.js index facaaef00847..2d65f4bf5448 100644 --- a/packages/astro/test/test-utils.js +++ b/packages/astro/test/test-utils.js @@ -45,6 +45,7 @@ process.env.ASTRO_TELEMETRY_DISABLED = true; * @property {() => Promise} loadTestAdapterApp * @property {() => Promise<(req: NodeRequest, res: NodeResponse) => void>} loadNodeAdapterHandler * @property {() => Promise} onNextChange + * @property {(timeout?: number) => Promise} onNextDataStoreChange * @property {typeof check} check * @property {typeof sync} sync * @property {AstroConfig} config @@ -180,6 +181,27 @@ export async function loadFixture(inlineConfig) { config.server.port = devServer.address.port; // update port return devServer; }, + onNextDataStoreChange: (timeout = 5000) => { + if(!devServer) { + return Promise.reject(new Error('No dev server running')); + } + + const dataStoreFile = path.join(root, '.astro', 'data-store.json'); + + return new Promise((resolve, reject) => { + const changeHandler = (fileName) => { + if (fileName === dataStoreFile) { + devServer.watcher.removeListener('change', changeHandler); + resolve(); + } + } + devServer.watcher.on('change', changeHandler); + setTimeout(() => { + devServer.watcher.removeListener('change', changeHandler); + reject(new Error('Data store did not update within timeout')); + }, timeout); + }) + }, config, resolveUrl, fetch: async (url, init) => { From f73f356fcc6a669dc31d1d46b9ce35b01761a846 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Tue, 3 Sep 2024 10:59:44 +0100 Subject: [PATCH 14/15] Lint --- packages/astro/test/content-layer.test.js | 2 -- packages/astro/test/test-utils.js | 10 +++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index cf3c7579953b..4e7cb49c2884 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -309,7 +309,5 @@ describe('Content Layer', () => { assert.ok(updated.fileLoader[0].data.temperament.includes('Bouncy')); await fixture.resetAllFiles(); }); - - }); }); diff --git a/packages/astro/test/test-utils.js b/packages/astro/test/test-utils.js index 2d65f4bf5448..00a8099f4f51 100644 --- a/packages/astro/test/test-utils.js +++ b/packages/astro/test/test-utils.js @@ -182,25 +182,25 @@ export async function loadFixture(inlineConfig) { return devServer; }, onNextDataStoreChange: (timeout = 5000) => { - if(!devServer) { + if (!devServer) { return Promise.reject(new Error('No dev server running')); } - + const dataStoreFile = path.join(root, '.astro', 'data-store.json'); - + return new Promise((resolve, reject) => { const changeHandler = (fileName) => { if (fileName === dataStoreFile) { devServer.watcher.removeListener('change', changeHandler); resolve(); } - } + }; devServer.watcher.on('change', changeHandler); setTimeout(() => { devServer.watcher.removeListener('change', changeHandler); reject(new Error('Data store did not update within timeout')); }, timeout); - }) + }); }, config, resolveUrl, From 27f3654d4e16f5d4161016d7c1a63fb507a9d87c Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Tue, 3 Sep 2024 11:11:42 +0000 Subject: [PATCH 15/15] Handle case where Vite already knows about save --- packages/astro/test/content-layer.test.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index 4e7cb49c2884..75d702a94f7f 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -197,7 +197,10 @@ describe('Content Layer', () => { let json; before(async () => { devServer = await fixture.startDevServer({ force: true }); - await fixture.onNextDataStoreChange(); + // Vite may not have noticed the saved data store yet. Wait a little just in case. + await fixture.onNextDataStoreChange(1000).catch(() => { + // Ignore timeout, because it may have saved before we get here. + }) const rawJsonResponse = await fixture.fetch('/collections.json'); const rawJson = await rawJsonResponse.text(); json = devalue.parse(rawJson);