fix: watch for content layer changes (#11371)

* fix: watch for content layer changes

* Add test

packages/astro/src/content/data-store.ts

@@ -1,6 +1,15 @@
 import { promises as fs, type PathLike, existsSync } from 'fs';
+
+const SAVE_DEBOUNCE_MS = 500;
 export class DataStore {
 	#collections = new Map<string, Map<string, any>>();
+
+	#file?: PathLike;
+
+	#saveTimeout: NodeJS.Timeout | undefined;
+
+	#dirty = false;
+
 	constructor() {
 		this.#collections = new Map();
 	}
@@ -23,15 +32,18 @@ export class DataStore {
 		const collection = this.#collections.get(collectionName) ?? new Map();
 		collection.set(String(key), value);
 		this.#collections.set(collectionName, collection);
+		this.#saveToDiskDebounced();
 	}
 	delete(collectionName: string, key: string) {
 		const collection = this.#collections.get(collectionName);
 		if (collection) {
 			collection.delete(String(key));
+			this.#saveToDiskDebounced();
 		}
 	}
 	clear(collectionName: string) {
 		this.#collections.delete(collectionName);
+		this.#saveToDiskDebounced();
 	}
 
 	has(collectionName: string, key: string) {
@@ -50,6 +62,20 @@ export class DataStore {
 		return this.#collections;
 	}
 
+	#saveToDiskDebounced = () => {
+		this.#dirty = true;
+		// Only save to disk if it has already been saved once
+		if (this.#file) {
+			if (this.#saveTimeout) {
+				clearTimeout(this.#saveTimeout);
+			}
+			this.#saveTimeout = setTimeout(() => {
+				this.#saveTimeout = undefined;
+				this.writeToDisk(this.#file!);
+			}, SAVE_DEBOUNCE_MS);
+		}
+	};
+
 	scopedStore(collectionName: string): ScopedDataStore {
 		return {
 			get: (key: string) => this.get(collectionName, key),
@@ -76,8 +102,13 @@ export class DataStore {
 	}
 
 	async writeToDisk(filePath: PathLike) {
+		if (!this.#dirty) {
+			return;
+		}
 		try {
 			await fs.writeFile(filePath, this.toString());
+			this.#file = filePath;
+			this.#dirty = false;
 		} catch {
 			throw new Error(`Failed to save data store to disk`);
 		}

packages/astro/src/content/file.ts

@@ -1,5 +1,5 @@
 import { fileURLToPath } from 'url';
-import type { Loader } from './loaders.js';
+import type { Loader, LoaderContext } from './loaders.js';
 import { promises as fs, existsSync } from 'fs';
 
 /**
@@ -12,54 +12,68 @@ export function file(fileName: string): Loader {
 		// TODO: AstroError
 		throw new Error('Glob patterns are not supported in `file` loader. Use `glob` loader instead.');
 	}
+
+	async function syncData(filePath: string, { logger, parseData, store }: LoaderContext) {
+		let json: Array<Record<string, unknown>>;
+
+		try {
+			const data = await fs.readFile(filePath, 'utf-8');
+			json = JSON.parse(data);
+		} catch (error: any) {
+			logger.error(`Error reading data from ${fileName}`);
+			logger.debug(error.message);
+			return;
+		}
+
+		if (Array.isArray(json)) {
+			if (json.length === 0) {
+				logger.warn(`No items found in ${fileName}`);
+			}
+			logger.debug(`Found ${json.length} item array in ${fileName}`);
+			store.clear();
+			for (const rawItem of json) {
+				const id = (rawItem.id ?? rawItem.slug)?.toString();
+				if (!id) {
+					logger.error(`Item in ${fileName} is missing an id or slug field.`);
+					continue;
+				}
+				const item = await parseData({ id, data: rawItem, filePath });
+				store.set(id, item);
+			}
+		} else if (typeof json === 'object') {
+			const entries = Object.entries<Record<string, unknown>>(json);
+			logger.debug(`Found object with ${entries.length} entries in ${fileName}`);
+			store.clear();
+			for (const [id, rawItem] of entries) {
+				const item = await parseData({ id, data: rawItem, filePath });
+				store.set(id, item);
+			}
+		} else {
+			logger.error(`Invalid data in ${fileName}. Must be an array or object.`);
+		}
+	}
+
 	return {
 		name: 'file-loader',
-		load: async ({ store, logger, settings, parseData }) => {
+		load: async (options) => {
+			const { settings, logger, watcher } = options;
 			const contentDir = new URL('./content/', settings.config.srcDir);
-			logger.debug(`Loading data from ${fileName}`)
+			logger.debug(`Loading data from ${fileName}`);
 			const url = new URL(fileName, contentDir);
 			if (!existsSync(url)) {
 				logger.error(`File not found: ${fileName}`);
 				return;
 			}
-
-			let json: Array<Record<string, unknown>>;
-
-			try {
-				const data = await fs.readFile(url, 'utf-8');
-				json = JSON.parse(data);
-			} catch (error: any) {
-				logger.error(`Error reading data from ${fileName}`);
-				logger.debug(error.message);
-				return;
-			}
-
 			const filePath = fileURLToPath(url);
 
-			if (Array.isArray(json)) {
-				if (json.length === 0) {
-					logger.warn(`No items found in ${fileName}`);
-				}
-				logger.debug(`Found ${json.length} item array in ${fileName}`);
-				for (const rawItem of json) {
-					const id = (rawItem.id ?? rawItem.slug)?.toString();
-					if (!id) {
-						logger.error(`Item in ${fileName} is missing an id or slug field.`);
-						continue;
-					}
-					const item = await parseData({ id, data: rawItem, filePath });
-					store.set(id, item);
-				}
-			} else if (typeof json === 'object') {
-				const entries =  Object.entries<Record<string, unknown>>(json);
-				logger.debug(`Found object with ${entries.length} entries in ${fileName}`);
-				for (const [id, rawItem] of entries) {
-					const item = await parseData({ id, data: rawItem, filePath });
-					store.set(id, item);
+			await syncData(filePath, options);
+
+			watcher?.on('change', async (changedPath) => {
+				if (changedPath === filePath) {
+					logger.info(`Reloading data from ${fileName}`);
+					await syncData(filePath, options);
 				}
-			} else {
-				logger.error(`Invalid data in ${fileName}. Must be an array or object.`);
-			}
+			});
 		},
 	};
 }

packages/astro/src/content/loaders.ts

@@ -5,6 +5,7 @@ import { DataStore, globalDataStore, type MetaStore, type ScopedDataStore } from
 import { getEntryData, globalContentConfigObserver } from './utils.js';
 import { promises as fs, existsSync } from 'fs';
 import { DATA_STORE_FILE } from './consts.js';
+import type { FSWatcher } from 'vite';
 
 export interface ParseDataOptions {
 	/** The ID of the entry. Unique per collection */
@@ -30,6 +31,9 @@ export interface LoaderContext {
 	parseData<T extends Record<string, unknown> = Record<string, unknown>>(
 		props: ParseDataOptions
 	): T;
+
+	/** When running in dev, this is a filesystem watcher that can be used to trigger updates */
+	watcher?: FSWatcher;
 }
 
 export interface Loader {
@@ -42,6 +46,13 @@ export interface Loader {
 	render?: (entry: any) => any;
 }
 
+export interface SyncContentLayerOptions {
+	store?: DataStore;
+	settings: AstroSettings;
+	logger: Logger;
+	watcher?: FSWatcher;
+}
+
 /**
  * Run the `load()` method of each collection's loader, which will load the data and save it in the data store.
  * The loader itself is responsible for deciding whether this will clear and reload the full collection, or
@@ -51,7 +62,8 @@ export async function syncContentLayer({
 	settings,
 	logger: globalLogger,
 	store,
-}: { settings: AstroSettings; logger: Logger; store?: DataStore }) {
+	watcher,
+}: SyncContentLayerOptions) {
 	const logger = globalLogger.forkIntegrationLogger('content');
 	logger.info('Syncing content');
 	if (!store) {
@@ -112,6 +124,7 @@ export async function syncContentLayer({
 				logger: globalLogger.forkIntegrationLogger(collection.loader.name ?? 'content'),
 				settings,
 				parseData,
+				watcher,
 			});
 		})
 	);

packages/astro/src/content/utils.ts

@@ -50,6 +50,7 @@ export const collectionConfigParser = z.union([
 							logger: z.any(),
 							settings: z.any(),
 							parseData: z.any(),
+							watcher: z.any().optional(),
 						}),
 					],
 					z.unknown()

packages/astro/src/content/vite-plugin-content-virtual-mod.ts

@@ -129,6 +129,31 @@ export function astroContentVirtualModPlugin({
 				return code.replaceAll(RESOLVED_VIRTUAL_MODULE_ID, `${prefix}content/entry.mjs`);
 			}
 		},
+
+		configureServer(server) {
+			const dataStorePath = fileURLToPath(dataStoreFile);
+			// Watch for changes to the data store file
+			if (Array.isArray(server.watcher.options.ignored)) {
+				// The data store file is in node_modules, so is ignored by default,
+				// so we need to un-ignore it.
+				server.watcher.options.ignored.push(`!${dataStorePath}`);
+			}
+			server.watcher.add(dataStorePath);
+
+			server.watcher.on('change', (changedPath) => {
+				// If the datastore file changes, invalidate the virtual module
+				if (changedPath === dataStorePath) {
+					const module = server.moduleGraph.getModuleById(RESOLVED_DATA_STORE_VIRTUAL_ID);
+					if (module) {
+						server.moduleGraph.invalidateModule(module);
+					}
+					server.ws.send({
+						type: 'full-reload',
+						path: '*',
+					});
+				}
+			});
+		},
 	};
 }
 

packages/astro/src/core/dev/dev.ts

@@ -103,7 +103,11 @@ export default async function dev(inlineConfig: AstroInlineConfig): Promise<DevS
 
 	await attachContentServerListeners(restart.container);
 
-	await syncContentLayer({ settings: restart.container.settings, logger: logger });
+	await syncContentLayer({
+		settings: restart.container.settings,
+		logger,
+		watcher: restart.container.viteServer.watcher,
+	});
 
 	logger.info(null, green('watching for file changes...'));