From 4c7d217c20f973df310fdc0192855791c7febf5d Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 1 Aug 2024 13:05:35 +0100 Subject: [PATCH 1/4] fix: clear content layer cache if config has changed --- packages/astro/src/content/data-store.ts | 11 +++++++++-- packages/astro/src/content/sync.ts | 9 +++++++++ packages/astro/src/content/utils.ts | 8 ++++++-- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/packages/astro/src/content/data-store.ts b/packages/astro/src/content/data-store.ts index 720be732b50d..b8de8f52d943 100644 --- a/packages/astro/src/content/data-store.ts +++ b/packages/astro/src/content/data-store.ts @@ -75,6 +75,11 @@ export class DataStore { this.#saveToDiskDebounced(); } + clearAll() { + this.#collections.clear(); + this.#saveToDiskDebounced(); + } + has(collectionName: string, key: string) { const collection = this.#collections.get(collectionName); if (collection) { @@ -227,8 +232,10 @@ export default new Map([${exports.join(', ')}]); this.addAssetImports(assets, fileName), }; } - - metaStore(collectionName: string): MetaStore { + /** + * Returns a MetaStore for a given collection, or if no collection is provided, the default meta collection. + */ + metaStore(collectionName = ':meta'): MetaStore { const collectionKey = `meta:${collectionName}`; return { get: (key: string) => this.get(collectionKey, key), diff --git a/packages/astro/src/content/sync.ts b/packages/astro/src/content/sync.ts index 21c5e1b53367..4c693e9f0381 100644 --- a/packages/astro/src/content/sync.ts +++ b/packages/astro/src/content/sync.ts @@ -38,6 +38,15 @@ export async function syncContentLayer({ logger.debug('Content config not loaded, skipping sync'); return; } + + const previousConfigDigest = await store.metaStore().get('config-digest'); + const { digest: currentConfigDigest } = contentConfig.config; + if (currentConfigDigest && previousConfigDigest !== currentConfigDigest) { + logger.info('Content config changed, clearing cache'); + store.clearAll(); + await store.metaStore().set('config-digest', currentConfigDigest); + } + // xxhash is a very fast non-cryptographic hash function that is used to generate a content digest // It uses wasm, so we need to load it asynchronously. const { h64ToString } = await xxhash(); diff --git a/packages/astro/src/content/utils.ts b/packages/astro/src/content/utils.ts index ce83b4e01dd2..411e13ce536d 100644 --- a/packages/astro/src/content/utils.ts +++ b/packages/astro/src/content/utils.ts @@ -22,6 +22,7 @@ import { PROPAGATED_ASSET_FLAG, } from './consts.js'; import { createImage } from './runtime-assets.js'; +import xxhash from 'xxhash-wasm'; /** * Amap from a collection + slug to the local file path. * This is used internally to resolve entry imports when using `getEntry()`. @@ -95,7 +96,7 @@ export const contentConfigParser = z.object({ }); export type CollectionConfig = z.infer; -export type ContentConfig = z.infer; +export type ContentConfig = z.infer & { digest?: string }; type EntryInternal = { rawData: string | undefined; filePath: string }; @@ -497,7 +498,10 @@ export async function loadContentConfig({ const config = contentConfigParser.safeParse(unparsedConfig); if (config.success) { - return config.data; + // Generate a digest of the config file so we can invalidate the cache if it changes + const hasher = await xxhash(); + const digest = await hasher.h64ToString(await fs.promises.readFile(configPathname, 'utf-8')); + return { ...config.data, digest }; } else { return undefined; } From 62b432c3ef6f318b8c0c3142e44342143c41066a Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 1 Aug 2024 13:35:24 +0100 Subject: [PATCH 2/4] Add test --- packages/astro/test/content-layer.test.js | 11 +++++++++++ packages/astro/test/test-utils.js | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index a98ea7e238bd..0023743b5906 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -152,6 +152,17 @@ describe('Content Layer', () => { newJson = devalue.parse(await fixture.readFile('/collections.json')); assert.equal(newJson.increment.data.lastValue, 1); }); + + it('clears the store on new build if the config has changed', async () => { + let newJson = devalue.parse(await fixture.readFile('/collections.json')); + assert.equal(newJson.increment.data.lastValue, 1); + await fixture.editFile('src/content/config.ts', (prev) => { + return `${prev}\nexport const foo = 'bar';`; + }); + await fixture.build(); + newJson = devalue.parse(await fixture.readFile('/collections.json')); + assert.equal(newJson.increment.data.lastValue, 1); + }); }); describe('Dev', () => { diff --git a/packages/astro/test/test-utils.js b/packages/astro/test/test-utils.js index d3cdb30d2527..8f6838b56569 100644 --- a/packages/astro/test/test-utils.js +++ b/packages/astro/test/test-utils.js @@ -278,7 +278,7 @@ export async function loadFixture(inlineConfig) { typeof newContentsOrCallback === 'function' ? newContentsOrCallback(contents) : newContentsOrCallback; - const nextChange = onNextChange(); + const nextChange = devServer ? onNextChange() : Promise.resolve(); await fs.promises.writeFile(fileUrl, newContents); await nextChange; return reset; From 8ea2b3b949502caf6cc2be537cd300e4890c79f0 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 1 Aug 2024 14:23:19 +0100 Subject: [PATCH 3/4] Watch config --- packages/astro/src/content/utils.ts | 2 +- .../content/vite-plugin-content-imports.ts | 5 +-- packages/astro/src/core/build/index.ts | 3 +- packages/astro/src/core/dev/dev.ts | 36 ++++++++++++++++--- packages/astro/src/core/sync/index.ts | 9 +++-- 5 files changed, 40 insertions(+), 15 deletions(-) diff --git a/packages/astro/src/content/utils.ts b/packages/astro/src/content/utils.ts index 411e13ce536d..b5c7a8e649b2 100644 --- a/packages/astro/src/content/utils.ts +++ b/packages/astro/src/content/utils.ts @@ -5,6 +5,7 @@ import { slug as githubSlug } from 'github-slugger'; import matter from 'gray-matter'; import type { PluginContext } from 'rollup'; import { type ViteDevServer, normalizePath } from 'vite'; +import xxhash from 'xxhash-wasm'; import { z } from 'zod'; import type { AstroConfig, @@ -22,7 +23,6 @@ import { PROPAGATED_ASSET_FLAG, } from './consts.js'; import { createImage } from './runtime-assets.js'; -import xxhash from 'xxhash-wasm'; /** * Amap from a collection + slug to the local file path. * This is used internally to resolve entry imports when using `getEntry()`. diff --git a/packages/astro/src/content/vite-plugin-content-imports.ts b/packages/astro/src/content/vite-plugin-content-imports.ts index 91f411f0756e..ce3d9638ce72 100644 --- a/packages/astro/src/content/vite-plugin-content-imports.ts +++ b/packages/astro/src/content/vite-plugin-content-imports.ts @@ -156,11 +156,8 @@ export const _internal = { const entryType = getEntryType(entry, contentPaths, contentEntryExts, dataEntryExts); if (!COLLECTION_TYPES_TO_INVALIDATE_ON.includes(entryType)) return; - // The content config could depend on collection entries via `reference()`. // Reload the config in case of changes. - if (entryType === 'content' || entryType === 'data') { - await reloadContentConfigObserver({ fs, settings, viteServer }); - } + await reloadContentConfigObserver({ fs, settings, viteServer }); // Invalidate all content imports and `render()` modules. // TODO: trace `reference()` calls for fine-grained invalidation. diff --git a/packages/astro/src/core/build/index.ts b/packages/astro/src/core/build/index.ts index 40428196351a..ecba6d7fca09 100644 --- a/packages/astro/src/core/build/index.ts +++ b/packages/astro/src/core/build/index.ts @@ -28,12 +28,12 @@ import { levels, timerMessage } from '../logger/core.js'; import { apply as applyPolyfill } from '../polyfill.js'; import { createRouteManifest } from '../routing/index.js'; import { getServerIslandRouteData } from '../server-islands/endpoint.js'; +import { clearContentLayerCache } from '../sync/index.js'; import { ensureProcessNodeEnv, isServerLikeOutput } from '../util.js'; import { collectPagesData } from './page-data.js'; import { staticBuild, viteBuild } from './static-build.js'; import type { StaticBuildOptions } from './types.js'; import { getTimeStat } from './util.js'; -import { clearContentLayerCache } from '../sync/index.js'; export interface BuildOptions { /** * Teardown the compiler WASM instance after build. This can improve performance when @@ -43,7 +43,6 @@ export interface BuildOptions { * @default true */ teardownCompiler?: boolean; - } /** diff --git a/packages/astro/src/core/dev/dev.ts b/packages/astro/src/core/dev/dev.ts index 453f1cf0c7c9..799684c58e60 100644 --- a/packages/astro/src/core/dev/dev.ts +++ b/packages/astro/src/core/dev/dev.ts @@ -10,6 +10,7 @@ import { DATA_STORE_FILE } from '../../content/consts.js'; import { DataStore, globalDataStore } from '../../content/data-store.js'; import { attachContentServerListeners } from '../../content/index.js'; import { syncContentLayer } from '../../content/sync.js'; +import { globalContentConfigObserver } from '../../content/utils.js'; import { telemetry } from '../../events/index.js'; import * as msg from '../messages.js'; import { ensureProcessNodeEnv } from '../util.js'; @@ -120,11 +121,36 @@ export default async function dev(inlineConfig: AstroInlineConfig): Promise { + loading = false; + }); + } + + globalContentConfigObserver.subscribe(async (ctx) => { + if (!loading && ctx.status === 'loaded' && ctx.config.digest !== currentDigest) { + loading = true; + currentDigest = ctx.config.digest; + await syncContentLayer({ + settings: restart.container.settings, + logger, + watcher: restart.container.viteServer.watcher, + store, + }).finally(() => { + loading = false; + }); + } }); logger.info(null, green('watching for file changes...')); diff --git a/packages/astro/src/core/sync/index.ts b/packages/astro/src/core/sync/index.ts index bfd0467ae2c8..885919ce9592 100644 --- a/packages/astro/src/core/sync/index.ts +++ b/packages/astro/src/core/sync/index.ts @@ -73,7 +73,11 @@ export default async function sync({ /** * Clears the content layer and content collection cache, forcing a full rebuild. */ -export async function clearContentLayerCache({ astroConfig, logger, fs = fsMod }: { astroConfig: AstroConfig; logger: Logger, fs?: typeof fsMod }) { +export async function clearContentLayerCache({ + astroConfig, + logger, + fs = fsMod, +}: { astroConfig: AstroConfig; logger: Logger; fs?: typeof fsMod }) { const dataStore = new URL(DATA_STORE_FILE, astroConfig.cacheDir); if (fs.existsSync(dataStore)) { logger.debug('content', 'clearing data store'); @@ -93,9 +97,8 @@ export async function syncInternal({ fs = fsMod, settings, skip, - force + force, }: SyncOptions): Promise { - if (force) { await clearContentLayerCache({ astroConfig: settings.config, logger, fs }); } From 71f1950c2e5fb264a894c19283838c7917eec67d Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 1 Aug 2024 14:50:17 +0100 Subject: [PATCH 4/4] Change from review --- packages/astro/src/content/vite-plugin-content-imports.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/astro/src/content/vite-plugin-content-imports.ts b/packages/astro/src/content/vite-plugin-content-imports.ts index ce3d9638ce72..1a7deb99031f 100644 --- a/packages/astro/src/content/vite-plugin-content-imports.ts +++ b/packages/astro/src/content/vite-plugin-content-imports.ts @@ -156,8 +156,12 @@ export const _internal = { const entryType = getEntryType(entry, contentPaths, contentEntryExts, dataEntryExts); if (!COLLECTION_TYPES_TO_INVALIDATE_ON.includes(entryType)) return; + // The content config could depend on collection entries via `reference()`. // Reload the config in case of changes. - await reloadContentConfigObserver({ fs, settings, viteServer }); + // Changes to the config file itself are handled in types-generator.ts, so we skip them here + if (entryType === 'content' || entryType === 'data') { + await reloadContentConfigObserver({ fs, settings, viteServer }); + } // Invalidate all content imports and `render()` modules. // TODO: trace `reference()` calls for fine-grained invalidation.