feat: add typegen for loaders

.changeset/smooth-chicken-wash.md

@@ -0,0 +1,5 @@
+---
+'astro': minor
+---
+
+Implements Content Layer

packages/astro/package.json

@@ -133,6 +133,7 @@
     "@babel/plugin-transform-react-jsx": "^7.24.7",
     "@babel/traverse": "^7.24.7",
     "@babel/types": "^7.24.7",
+    "@rollup/pluginutils": "^5.1.0",
     "@types/babel__core": "^7.20.5",
     "@types/cookie": "^0.6.0",
     "acorn": "^8.12.0",
@@ -184,7 +185,8 @@
     "which-pm": "^2.2.0",
     "yargs-parser": "^21.1.1",
     "zod": "^3.23.8",
-    "zod-to-json-schema": "^3.23.1"
+    "zod-to-json-schema": "^3.23.1",
+    "zod-to-ts": "^1.2.0"
   },
   "optionalDependencies": {
     "sharp": "^0.33.3"

packages/astro/src/content/consts.ts

@@ -5,6 +5,8 @@ export const DATA_FLAG = 'astroDataCollectionEntry';
 
 export const VIRTUAL_MODULE_ID = 'astro:content';
 export const RESOLVED_VIRTUAL_MODULE_ID = '\0' + VIRTUAL_MODULE_ID;
+export const DATA_STORE_VIRTUAL_ID = 'astro:data-layer-content';
+export const RESOLVED_DATA_STORE_VIRTUAL_ID = '\0' + DATA_STORE_VIRTUAL_ID;
 export const LINKS_PLACEHOLDER = '@@ASTRO-LINKS@@';
 export const STYLES_PLACEHOLDER = '@@ASTRO-STYLES@@';
 export const SCRIPTS_PLACEHOLDER = '@@ASTRO-SCRIPTS@@';

packages/astro/src/content/data-store.ts

@@ -0,0 +1,135 @@
+import { promises as fs, type PathLike, existsSync } from 'fs';
+export class DataStore {
+	#collections = new Map<string, Map<string, any>>();
+	constructor() {
+		this.#collections = new Map();
+	}
+	get(collectionName: string, key: string) {
+		return this.#collections.get(collectionName)?.get(String(key));
+	}
+	entries(collectionName: string): IterableIterator<[id: string, any]> {
+		const collection = this.#collections.get(collectionName) ?? new Map();
+		return collection.entries();
+	}
+	set(collectionName: string, key: string, value: any) {
+		const collection = this.#collections.get(collectionName) ?? new Map();
+		collection.set(String(key), value);
+		this.#collections.set(collectionName, collection);
+	}
+	delete(collectionName: string, key: string) {
+		const collection = this.#collections.get(collectionName);
+		if (collection) {
+			collection.delete(String(key));
+		}
+	}
+	clear(collectionName: string) {
+		this.#collections.delete(collectionName);
+	}
+
+	has(collectionName: string, key: string) {
+		const collection = this.#collections.get(collectionName);
+		if (collection) {
+			return collection.has(String(key));
+		}
+		return false;
+	}
+
+	hasCollection(collectionName: string) {
+		return this.#collections.has(collectionName);
+	}
+
+	collections() {
+		return this.#collections;
+	}
+
+	scopedStore(collectionName: string): ScopedDataStore {
+		return {
+			get: (key: string) => this.get(collectionName, key),
+			entries: () => this.entries(collectionName),
+			set: (key: string, value: any) => this.set(collectionName, key, value),
+			delete: (key: string) => this.delete(collectionName, key),
+			clear: () => this.clear(collectionName),
+			has: (key: string) => this.has(collectionName, key),
+		};
+	}
+
+	metaStore(collectionName: string): MetaStore {
+		return this.scopedStore(`meta:${collectionName}`);
+	}
+
+	toString() {
+		return JSON.stringify(
+			Array.from(this.#collections.entries()).map(([collectionName, collection]) => {
+				return [collectionName, Array.from(collection.entries())];
+			})
+		);
+	}
+
+	async writeToDisk(filePath: PathLike) {
+		await fs.writeFile(filePath, this.toString());
+	}
+
+	static async fromDisk(filePath: PathLike) {
+		if (!existsSync(filePath)) {
+			return new DataStore();
+		}
+		const str = await fs.readFile(filePath, 'utf-8');
+		return DataStore.fromString(str);
+	}
+
+	static fromString(str: string) {
+		const entries = JSON.parse(str);
+		return DataStore.fromJSON(entries);
+	}
+
+	static async fromModule() {
+		try {
+			// @ts-expect-error
+			const data = await import('astro:data-layer-content');
+			return DataStore.fromJSON(data.default);
+		} catch {}
+		return new DataStore();
+	}
+
+	static fromJSON(entries: Array<[string, Array<[string, any]>]>) {
+		const collections = new Map<string, Map<string, any>>();
+		for (const [collectionName, collection] of entries) {
+			collections.set(collectionName, new Map(collection));
+		}
+		const store = new DataStore();
+		store.#collections = collections;
+		return store;
+	}
+}
+
+export interface ScopedDataStore {
+	get: (key: string) => any;
+	entries: () => IterableIterator<[id: string, any]>;
+	set: (key: string, value: any) => void;
+	delete: (key: string) => void;
+	clear: () => void;
+	has: (key: string) => boolean;
+}
+
+export interface MetaStore {
+	get: (key: string) => string | undefined;
+	set: (key: string, value: string) => void;
+	has: (key: string) => boolean;
+}
+
+function dataStoreSingleton() {
+	let instance: Promise<DataStore> | DataStore | undefined = undefined;
+	return {
+		get: async () => {
+			if (!instance) {
+				instance = DataStore.fromModule();
+			}
+			return instance;
+		},
+		set: (store: DataStore) => {
+			instance = store;
+		},
+	};
+}
+
+export const globalDataStore = dataStoreSingleton();

packages/astro/src/content/file.ts

@@ -0,0 +1,51 @@
+import { fileURLToPath } from 'url';
+import type { Loader } from './loaders.js';
+import { promises as fs, existsSync } from 'fs';
+
+/**
+ * Loads entries from a JSON file. The file must contain an array of objects that contain unique `id` fields, or an object with string keys.
+ * @todo Add support for other file types, such as YAML, CSV etc.
+ * @param fileName The path to the JSON file to load, relative to the content directory.
+ */
+export function file(fileName: string): Loader {
+	if (fileName.includes('*')) {
+		throw new Error('Glob patterns are not supported in file loader. Use `glob` loader instead.');
+	}
+	return {
+		name: 'file-loader',
+		load: async ({ store, logger, settings, parseData }) => {
+			const contentDir = new URL('./content/', settings.config.srcDir);
+
+			const url = new URL(fileName, contentDir);
+			if (!existsSync(url)) {
+				logger.error(`File not found: ${fileName}`);
+				return;
+			}
+
+			const data = await fs.readFile(url, 'utf-8');
+			const json = JSON.parse(data);
+
+			const filePath = fileURLToPath(url);
+
+			if (Array.isArray(json)) {
+				if (json.length === 0) {
+					logger.warn(`No items found in ${fileName}`);
+				}
+				for (const rawItem of json) {
+					const id = rawItem.id ?? rawItem.slug;
+					const item = await parseData({ id, data: rawItem, filePath });
+					store.set(id, item);
+				}
+			} else if (typeof json === 'object') {
+				for (const [id, rawItem] of Object.entries<Record<string, unknown>>(json)) {
+					const item = await parseData({ id, data: rawItem, filePath });
+					store.set(id, item);
+				}
+			} else {
+				logger.error(`Invalid data in ${fileName}. Must be an array or object.`);
+			}
+
+			logger.info('Loading posts');
+		},
+	};
+}

packages/astro/src/content/loaders.ts

@@ -0,0 +1,115 @@
+import type { ZodSchema } from 'zod';
+import type { AstroSettings } from '../@types/astro.js';
+import type { AstroIntegrationLogger, Logger } from '../core/logger/core.js';
+import { DataStore, globalDataStore, type MetaStore, type ScopedDataStore } from './data-store.js';
+import { getEntryData, globalContentConfigObserver } from './utils.js';
+import { promises as fs, existsSync } from 'fs';
+
+export interface ParseDataOptions {
+	/** The ID of the entry. Unique per collection */
+	id: string;
+	/** The raw, unvalidated data of the entry */
+	data: Record<string, unknown>;
+	/** An optional file path, where the entry represents a local file */
+	filePath?: string;
+}
+
+export interface LoaderContext {
+	collection: string;
+	/** A database abstraction to store the actual data */
+	store: ScopedDataStore;
+	/**  A simple KV store, designed for things like sync tokens */
+	meta: MetaStore;
+	logger: AstroIntegrationLogger;
+
+	settings: AstroSettings;
+
+	/** Validates and parses the data according to the schema */
+	parseData<T extends Record<string, unknown> = Record<string, unknown>>(
+		props: ParseDataOptions
+	): T;
+}
+
+export interface Loader {
+	/** Unique name of the loader, e.g. the npm package name */
+	name: string;
+	/** Do the actual loading of the data */
+	load: (context: LoaderContext) => Promise<void>;
+	/** Optionally, define the schema of the data. Will be overridden by user-defined schema */
+	schema?: ZodSchema | Promise<ZodSchema> | (() => ZodSchema | Promise<ZodSchema>);
+	render?: (entry: any) => any;
+}
+export async function syncDataLayer({
+	settings,
+	logger: globalLogger,
+	store,
+}: { settings: AstroSettings; logger: Logger; store?: DataStore }) {
+	const logger = globalLogger.forkIntegrationLogger('content');
+	if (!store) {
+		store = await DataStore.fromDisk(new URL('data-store.json', settings.config.cacheDir));
+		globalDataStore.set(store);
+	}
+	const contentConfig = globalContentConfigObserver.get();
+	if (contentConfig?.status !== 'loaded') {
+		logger.debug('Content config not loaded, skipping sync');
+		return;
+	}
+	await Promise.all(
+		Object.entries(contentConfig.config.collections).map(async ([name, collection]) => {
+			if (collection.type !== 'experimental_data') {
+				return;
+			}
+
+			let { schema } = collection;
+
+			if (!schema) {
+				schema = collection.loader.schema;
+			}
+
+			if (typeof schema === 'function') {
+				schema = await schema({
+					image: () => {
+						throw new Error('Images are currently not supported for experimental data collections');
+					},
+				});
+			}
+
+			const collectionWithResolvedSchema = { ...collection, schema };
+
+			function parseData<T extends Record<string, unknown> = Record<string, unknown>>({
+				id,
+				data,
+				filePath = '',
+			}: { id: string; data: T; filePath?: string }): T {
+				return getEntryData(
+					{
+						id,
+						collection: name,
+						unvalidatedData: data,
+						_internal: {
+							rawData: undefined,
+							filePath,
+						},
+					},
+					collectionWithResolvedSchema,
+					false
+				) as unknown as T;
+			}
+
+			return collection.loader.load({
+				collection: name,
+				store: store.scopedStore(name),
+				meta: store.metaStore(name),
+				logger,
+				settings,
+				parseData,
+			});
+		})
+	);
+	const cacheFile = new URL('data-store.json', settings.config.cacheDir);
+	if (!existsSync(settings.config.cacheDir)) {
+		await fs.mkdir(settings.config.cacheDir, { recursive: true });
+	}
+	await store.writeToDisk(cacheFile);
+	logger.info('Synced content');
+}

packages/astro/src/content/runtime.ts

@@ -14,6 +14,8 @@ import {
 	unescapeHTML,
 } from '../runtime/server/index.js';
 import type { ContentLookupMap } from './utils.js';
+import { globalDataStore } from './data-store.js';
+export { file } from './file.js';
 
 type LazyImport = () => Promise<any>;
 type GlobResult = Record<string, LazyImport>;
@@ -56,11 +58,19 @@ export function createGetCollection({
 	cacheEntriesByCollection: Map<string, any[]>;
 }) {
 	return async function getCollection(collection: string, filter?: (entry: any) => unknown) {
-		let type: 'content' | 'data';
+		const store = await globalDataStore.get();
+		let type: 'content' | 'data' | 'experimental_data';
 		if (collection in contentCollectionToEntryMap) {
 			type = 'content';
 		} else if (collection in dataCollectionToEntryMap) {
 			type = 'data';
+		} else if (store.hasCollection(collection)) {
+			return [...store.entries(collection)].map(([id, data]) => ({
+				id,
+				collection,
+				data,
+				type: 'experimental_data',
+			}));
 		} else {
 			// eslint-disable-next-line no-console
 			console.warn(
@@ -153,6 +163,21 @@ export function createGetEntryBySlug({
 
 export function createGetDataEntryById({ getEntryImport }: { getEntryImport: GetEntryImport }) {
 	return async function getDataEntryById(collection: string, id: string) {
+		const store = await globalDataStore.get();
+
+		if (store.hasCollection(collection)) {
+			const data = store.get(collection, id);
+			if (!data) {
+				throw new Error(`Entry ${collection} → ${id} was not found.`);
+			}
+
+			return {
+				id,
+				collection,
+				data: store.get(collection, id),
+			};
+		}
+
 		const lazyImport = await getEntryImport(collection, id);
 
 		// TODO: AstroError