Changes from review

packages/astro/src/content/consts.ts

@@ -19,3 +19,5 @@ export const CONTENT_FLAGS = [
 ] as const;
 
 export const CONTENT_TYPES_FILE = 'types.d.ts';
+
+export const DATA_STORE_FILE = 'data-store.json';

packages/astro/src/content/data-store.ts

@@ -7,11 +7,19 @@ export class DataStore {
 	get(collectionName: string, key: string) {
 		return this.#collections.get(collectionName)?.get(String(key));
 	}
-	entries(collectionName: string): IterableIterator<[id: string, any]> {
+	entries(collectionName: string): Array<[id: string, any]> {
 		const collection = this.#collections.get(collectionName) ?? new Map();
-		return collection.entries();
+		return [...collection.entries()];
 	}
-	set(collectionName: string, key: string, value: any) {
+	values(collectionName: string): Array<unknown> {
+		const collection = this.#collections.get(collectionName) ?? new Map();
+		return [...collection.values()];
+	}
+	keys(collectionName: string): Array<string> {
+		const collection = this.#collections.get(collectionName) ?? new Map();
+		return [...collection.keys()];
+	}
+	set(collectionName: string, key: string, value: unknown) {
 		const collection = this.#collections.get(collectionName) ?? new Map();
 		collection.set(String(key), value);
 		this.#collections.set(collectionName, collection);
@@ -46,6 +54,8 @@ export class DataStore {
 		return {
 			get: (key: string) => this.get(collectionName, key),
 			entries: () => this.entries(collectionName),
+			values: () => this.values(collectionName),
+			keys: () => this.keys(collectionName),
 			set: (key: string, value: any) => this.set(collectionName, key, value),
 			delete: (key: string) => this.delete(collectionName, key),
 			clear: () => this.clear(collectionName),
@@ -54,7 +64,7 @@ export class DataStore {
 	}
 
 	metaStore(collectionName: string): MetaStore {
-		return this.scopedStore(`meta:${collectionName}`);
+		return this.scopedStore(`meta:${collectionName}`) as MetaStore;
 	}
 
 	toString() {
@@ -66,7 +76,11 @@ export class DataStore {
 	}
 
 	async writeToDisk(filePath: PathLike) {
-		await fs.writeFile(filePath, this.toString());
+		try {
+			await fs.writeFile(filePath, this.toString());
+		} catch {
+			throw new Error(`Failed to save data store to disk`);
+		}
 	}
 
 	static async fromDisk(filePath: PathLike) {
@@ -103,14 +117,20 @@ export class DataStore {
 }
 
 export interface ScopedDataStore {
-	get: (key: string) => any;
-	entries: () => IterableIterator<[id: string, any]>;
-	set: (key: string, value: any) => void;
+	get: (key: string) => unknown;
+	entries: () => Array<[id: string, unknown]>;
+	set: (key: string, value: unknown) => void;
+	values: () => Array<unknown>;
+	keys: () => Array<string>;
 	delete: (key: string) => void;
 	clear: () => void;
 	has: (key: string) => boolean;
 }
 
+/**
+ * A key-value store for metadata strings. Useful for storing things like sync tokens.
+ */
+
 export interface MetaStore {
 	get: (key: string) => string | undefined;
 	set: (key: string, value: string) => void;

packages/astro/src/content/file.ts

@@ -9,7 +9,8 @@ import { promises as fs, existsSync } from 'fs';
  */
 export function file(fileName: string): Loader {
 	if (fileName.includes('*')) {
-		throw new Error('Glob patterns are not supported in file loader. Use `glob` loader instead.');
+		// TODO: AstroError
+		throw new Error('Glob patterns are not supported in `file` loader. Use `glob` loader instead.');
 	}
 	return {
 		name: 'file-loader',
@@ -22,8 +23,16 @@ export function file(fileName: string): Loader {
 				return;
 			}
 
-			const data = await fs.readFile(url, 'utf-8');
-			const json = JSON.parse(data);
+			let json: Array<Record<string, unknown>>;
+
+			try {
+				const data = await fs.readFile(url, 'utf-8');
+				json = JSON.parse(data);
+			} catch (error: any) {
+				logger.error(`Error reading data from ${fileName}`);
+				logger.debug(error.message);
+				return;
+			}
 
 			const filePath = fileURLToPath(url);
 
@@ -32,7 +41,11 @@ export function file(fileName: string): Loader {
 					logger.warn(`No items found in ${fileName}`);
 				}
 				for (const rawItem of json) {
-					const id = rawItem.id ?? rawItem.slug;
+					const id = (rawItem.id ?? rawItem.slug)?.toString();
+					if (!id) {
+						logger.error(`Item in ${fileName} is missing an id or slug field.`);
+						continue;
+					}
 					const item = await parseData({ id, data: rawItem, filePath });
 					store.set(id, item);
 				}
@@ -44,8 +57,6 @@ export function file(fileName: string): Loader {
 			} else {
 				logger.error(`Invalid data in ${fileName}. Must be an array or object.`);
 			}
-
-			logger.info('Loading posts');
 		},
 	};
 }

packages/astro/src/content/loaders.ts

@@ -4,17 +4,19 @@ import type { AstroIntegrationLogger, Logger } from '../core/logger/core.js';
 import { DataStore, globalDataStore, type MetaStore, type ScopedDataStore } from './data-store.js';
 import { getEntryData, globalContentConfigObserver } from './utils.js';
 import { promises as fs, existsSync } from 'fs';
+import { DATA_STORE_FILE } from './consts.js';
 
 export interface ParseDataOptions {
 	/** The ID of the entry. Unique per collection */
 	id: string;
 	/** The raw, unvalidated data of the entry */
 	data: Record<string, unknown>;
-	/** An optional file path, where the entry represents a local file */
+	/** An optional file path, where the entry represents a local file. */
 	filePath?: string;
 }
 
 export interface LoaderContext {
+	/** The unique name of the collection */
 	collection: string;
 	/** A database abstraction to store the actual data */
 	store: ScopedDataStore;
@@ -24,7 +26,7 @@ export interface LoaderContext {
 
 	settings: AstroSettings;
 
-	/** Validates and parses the data according to the schema */
+	/** Validates and parses the data according to the collection schema */
 	parseData<T extends Record<string, unknown> = Record<string, unknown>>(
 		props: ParseDataOptions
 	): T;
@@ -39,14 +41,20 @@ export interface Loader<S extends ZodSchema = ZodSchema> {
 	schema?: S | Promise<S> | (() => S | Promise<S>);
 	render?: (entry: any) => any;
 }
-export async function syncDataLayer({
+
+/**
+ * Run the `load()` method of each collection's loader, which will load the data and save it in the data store.
+ * The loader itself is responsible for deciding whether this will clear and reload the full collection, or
+ * perform an incremental update. After the data is loaded, the data store is written to disk.
+ */
+export async function syncContentLayer({
 	settings,
 	logger: globalLogger,
 	store,
 }: { settings: AstroSettings; logger: Logger; store?: DataStore }) {
 	const logger = globalLogger.forkIntegrationLogger('content');
 	if (!store) {
-		store = await DataStore.fromDisk(new URL('data-store.json', settings.config.cacheDir));
+		store = await DataStore.fromDisk(new URL(DATA_STORE_FILE, settings.config.cacheDir));
 		globalDataStore.set(store);
 	}
 	const contentConfig = globalContentConfigObserver.get();
@@ -106,7 +114,7 @@ export async function syncDataLayer({
 			});
 		})
 	);
-	const cacheFile = new URL('data-store.json', settings.config.cacheDir);
+	const cacheFile = new URL(DATA_STORE_FILE, settings.config.cacheDir);
 	if (!existsSync(settings.config.cacheDir)) {
 		await fs.mkdir(settings.config.cacheDir, { recursive: true });
 	}

packages/astro/src/content/vite-plugin-content-virtual-mod.ts

@@ -16,6 +16,7 @@ import {
 	CONTENT_FLAG,
 	CONTENT_RENDER_FLAG,
 	DATA_FLAG,
+	DATA_STORE_FILE,
 	DATA_STORE_VIRTUAL_ID,
 	RESOLVED_DATA_STORE_VIRTUAL_ID,
 	RESOLVED_VIRTUAL_MODULE_ID,
@@ -45,7 +46,7 @@ export function astroContentVirtualModPlugin({
 }: AstroContentVirtualModPluginParams): Plugin {
 	let IS_DEV = false;
 	const IS_SERVER = isServerLikeOutput(settings.config);
-	const dataStoreFile = new URL('data-store.json', settings.config.cacheDir);
+	const dataStoreFile = new URL(DATA_STORE_FILE, settings.config.cacheDir);
 	return {
 		name: 'astro-content-virtual-mod-plugin',
 		enforce: 'pre',

packages/astro/src/core/build/index.ts

@@ -11,7 +11,7 @@ import type {
 	RuntimeMode,
 } from '../../@types/astro.js';
 import { injectImageEndpoint } from '../../assets/endpoint/config.js';
-import { syncDataLayer } from '../../content/loaders.js';
+import { syncContentLayer } from '../../content/loaders.js';
 import { telemetry } from '../../events/index.js';
 import { eventCliSession } from '../../events/session.js';
 import {
@@ -152,7 +152,7 @@ class AstroBuilder {
 
 		const dataStore = await DataStore.fromModule();
 		globalDataStore.set(dataStore);
-		await syncDataLayer({ settings: this.settings, logger: logger });
+		await syncContentLayer({ settings: this.settings, logger: logger });
 
 		return { viteConfig };
 	}

packages/astro/src/core/sync/index.ts

@@ -6,7 +6,7 @@ import { type HMRPayload, createServer } from 'vite';
 import type { AstroConfig, AstroInlineConfig, AstroSettings } from '../../@types/astro.js';
 import { getPackage } from '../../cli/install-package.js';
 import { createContentTypesGenerator } from '../../content/index.js';
-import { syncDataLayer } from '../../content/loaders.js';
+import { syncContentLayer } from '../../content/loaders.js';
 import { globalContentConfigObserver } from '../../content/utils.js';
 import { syncAstroEnv } from '../../env/sync.js';
 import { telemetry } from '../../events/index.js';
@@ -87,7 +87,7 @@ export default async function sync(
 		if (exitCode !== 0) return exitCode;
 		syncAstroEnv(settings, options?.fs);
 
-		await syncDataLayer({ settings, logger });
+		await syncContentLayer({ settings, logger });
 
 		logger.info(null, `Types generated ${dim(getTimeStat(timerStart, performance.now()))}`);
 		return 0;