feat: add markdown rendering to content layer

benchmark/bench/memory.js

@@ -18,7 +18,7 @@ export async function run(projectDir, outputFile) {
 	const outputFilePath = fileURLToPath(outputFile);
 
 	console.log('Building and benchmarking...');
-	await execaCommand(`node --expose-gc --max_old_space_size=256 ${astroBin} build`, {
+	await execaCommand(`node --expose-gc --max_old_space_size=10000 ${astroBin} build`, {
 		cwd: root,
 		stdio: 'inherit',
 		env: {

benchmark/make-project/markdown-cc1.js

@@ -0,0 +1,58 @@
+import fs from 'node:fs/promises';
+import { loremIpsumMd } from './_util.js';
+
+/**
+ * @param {URL} projectDir
+ */
+export async function run(projectDir) {
+	await fs.rm(projectDir, { recursive: true, force: true });
+	await fs.mkdir(new URL('./src/pages/blog', projectDir), { recursive: true });
+	await fs.mkdir(new URL('./src/content/blog', projectDir), { recursive: true });
+
+	const promises = [];
+
+
+	for (let i = 0; i < 10000; i++) {
+		const content = `\
+# Article ${i}
+
+${loremIpsumMd}
+`;
+		promises.push(
+			fs.writeFile(new URL(`./src/content/blog/article-${i}.md`, projectDir), content, 'utf-8')
+		);
+	}
+
+
+	await fs.writeFile(
+		new URL(`./src/pages/blog/[...slug].astro`, projectDir),
+		`\
+---
+import { getCollection } from 'astro:content';
+export async function getStaticPaths() {
+  const blogEntries = await getCollection('blog');
+  return blogEntries.map(entry => ({
+    params: { slug: entry.slug }, props: { entry },
+  }));
+}
+const { entry } = Astro.props;
+const { Content } = await entry.render();
+---
+<h1>{entry.data.title}</h1>
+<Content />
+`,
+		'utf-8'
+	);
+
+	await Promise.all(promises);
+
+	await fs.writeFile(
+		new URL('./astro.config.js', projectDir),
+		`\
+import { defineConfig } from 'astro/config';
+
+export default defineConfig({
+});`,
+		'utf-8'
+	);
+}

benchmark/make-project/markdown-cc2.js

@@ -0,0 +1,74 @@
+import fs from 'node:fs/promises';
+import { loremIpsumMd } from './_util.js';
+
+/**
+ * @param {URL} projectDir
+ */
+export async function run(projectDir) {
+	await fs.rm(projectDir, { recursive: true, force: true });
+	await fs.mkdir(new URL('./src/pages/blog', projectDir), { recursive: true });
+	await fs.mkdir(new URL('./data/blog', projectDir), { recursive: true });
+	await fs.mkdir(new URL('./src/content', projectDir), { recursive: true });
+
+	const promises = [];
+
+	for (let i = 0; i < 10000; i++) {
+		const content = `\
+# Article ${i}
+
+${loremIpsumMd}
+`;
+		promises.push(
+			fs.writeFile(new URL(`./data/blog/article-${i}.md`, projectDir), content, 'utf-8')
+		);
+	}
+
+	await fs.writeFile(
+		new URL(`./src/content/config.ts`, projectDir),
+		/*ts */ `
+		import { defineCollection, z } from 'astro:content';
+		import { glob } from 'astro/loaders';
+
+		const blog = defineCollection({
+			type: 'experimental_data',
+			loader: glob({ pattern: '*', base: './data/blog' }),
+		});
+
+		export const collections = { blog }
+
+		`
+	);
+
+	await fs.writeFile(
+		new URL(`./src/pages/blog/[...slug].astro`, projectDir),
+		`\
+---
+import { getCollection } from 'astro:content';
+export async function getStaticPaths() {
+  const blogEntries = await getCollection('blog');
+  return blogEntries.map(entry => ({
+    params: { slug: entry.id }, props: { entry },
+  }));
+}
+const { entry } = Astro.props;
+const { Content } = await entry.render();
+
+---
+<h1>{entry.data.title}</h1>
+<Content />
+`,
+		'utf-8'
+	);
+
+	await Promise.all(promises);
+
+	await fs.writeFile(
+		new URL('./astro.config.js', projectDir),
+		`\
+import { defineConfig } from 'astro/config';
+
+export default defineConfig({
+});`,
+		'utf-8'
+	);
+}

packages/astro/package.json

@@ -67,6 +67,7 @@
     "./assets/services/sharp": "./dist/assets/services/sharp.js",
     "./assets/services/squoosh": "./dist/assets/services/squoosh.js",
     "./assets/services/noop": "./dist/assets/services/noop.js",
+    "./loaders": "./dist/content/loaders/index.js",
     "./content/runtime": "./dist/content/runtime.js",
     "./content/runtime-assets": "./dist/content/runtime-assets.js",
     "./debug": "./components/Debug.astro",
@@ -165,6 +166,7 @@
     "js-yaml": "^4.1.0",
     "kleur": "^4.1.5",
     "magic-string": "^0.30.10",
+    "micromatch": "^4.0.7",
     "mrmime": "^2.0.0",
     "ora": "^8.0.1",
     "p-limit": "^5.0.0",
@@ -183,6 +185,7 @@
     "vite": "^5.3.2",
     "vitefu": "^0.2.5",
     "which-pm": "^2.2.0",
+    "xxhash-wasm": "^1.0.2",
     "yargs-parser": "^21.1.1",
     "zod": "^3.23.8",
     "zod-to-json-schema": "^3.23.1",
@@ -208,6 +211,7 @@
     "@types/html-escaper": "^3.0.2",
     "@types/http-cache-semantics": "^4.0.4",
     "@types/js-yaml": "^4.0.9",
+    "@types/micromatch": "^4.0.9",
     "@types/probe-image-size": "^7.2.4",
     "@types/prompts": "^2.4.9",
     "@types/semver": "^7.5.8",
@@ -240,4 +244,4 @@
   "publishConfig": {
     "provenance": true
   }
-}
+}

packages/astro/src/@types/astro.ts

@@ -46,8 +46,9 @@ import type {
 	TransitionBeforePreparationEvent,
 	TransitionBeforeSwapEvent,
 } from '../transitions/events.js';
-import type { DeepPartial, OmitIndexSignature, Simplify, WithRequired } from '../type-utils.js';
+import type { DeepPartial, OmitIndexSignature, Simplify } from '../type-utils.js';
 import type { SUPPORTED_MARKDOWN_FILE_EXTENSIONS } from './../core/constants.js';
+import type { DataEntry } from '../content/data-store.js';
 
 export type { AstroIntegrationLogger, ToolbarServerHelpers };
 
@@ -2276,6 +2277,12 @@ export type DataEntryModule = {
 	};
 };
 
+export interface RenderResult {
+	code: string;
+	metadata?: Record<string, any>;
+}
+export type RenderFunction = (entry: DataEntry) => Promise<RenderResult>;
+
 export interface ContentEntryType {
 	extensions: string[];
 	getEntryInfo(params: {
@@ -2291,6 +2298,8 @@ export interface ContentEntryType {
 		}
 	): rollup.LoadResult | Promise<rollup.LoadResult>;
 	contentModuleTypes?: string;
+	getRenderFunction?(settings: AstroSettings): Promise<RenderFunction>;
+
 	/**
 	 * Handle asset propagation for rendered content to avoid bleed.
 	 * Ex. MDX content can import styles and scripts, so `handlePropagation` should be true.

packages/astro/src/content/data-store.ts

@@ -1,6 +1,19 @@
 import { promises as fs, type PathLike, existsSync } from 'fs';
 
 const SAVE_DEBOUNCE_MS = 500;
+
+export interface DataEntry {
+	id: string;
+	data: Record<string, unknown>;
+	filePath?: string;
+	body?: string;
+	digest?: number | string;
+	rendered?: {
+		html: string;
+		metadata?: Record<string, unknown>;
+	};
+}
+
 export class DataStore {
 	#collections = new Map<string, Map<string, any>>();
 
@@ -13,14 +26,14 @@ export class DataStore {
 	constructor() {
 		this.#collections = new Map();
 	}
-	get(collectionName: string, key: string) {
+	get<T = unknown>(collectionName: string, key: string): T | undefined {
 		return this.#collections.get(collectionName)?.get(String(key));
 	}
-	entries(collectionName: string): Array<[id: string, any]> {
+	entries<T = unknown>(collectionName: string): Array<[id: string, T]> {
 		const collection = this.#collections.get(collectionName) ?? new Map();
 		return [...collection.entries()];
 	}
-	values(collectionName: string): Array<unknown> {
+	values<T = unknown>(collectionName: string): Array<T> {
 		const collection = this.#collections.get(collectionName) ?? new Map();
 		return [...collection.values()];
 	}
@@ -78,19 +91,57 @@ export class DataStore {
 
 	scopedStore(collectionName: string): ScopedDataStore {
 		return {
-			get: (key: string) => this.get(collectionName, key),
+			get: (key: string) => this.get<DataEntry>(collectionName, key),
 			entries: () => this.entries(collectionName),
 			values: () => this.values(collectionName),
 			keys: () => this.keys(collectionName),
-			set: (key: string, value: any) => this.set(collectionName, key, value),
+			set: ({ id: key, data, body, filePath, digest, rendered }) => {
+				if (!key) {
+					throw new Error(`ID must be a non-empty string`);
+				}
+				const id = String(key);
+				if (digest) {
+					const existing = this.get<DataEntry>(collectionName, id);
+					if (existing && existing.digest === digest) {
+						return false;
+					}
+				}
+				const entry: DataEntry = {
+					id,
+					data,
+				};
+				// We do it like this so we don't waste space stringifying
+				// the fields if they are not set
+				if (body) {
+					entry.body = body;
+				}
+				if (filePath) {
+					entry.filePath = filePath;
+				}
+				if (digest) {
+					entry.digest = digest;
+				}
+				if (rendered) {
+					entry.rendered = rendered;
+				}
+
+				this.set(collectionName, id, entry);
+				return true;
+			},
 			delete: (key: string) => this.delete(collectionName, key),
 			clear: () => this.clear(collectionName),
 			has: (key: string) => this.has(collectionName, key),
 		};
 	}
 
 	metaStore(collectionName: string): MetaStore {
-		return this.scopedStore(`meta:${collectionName}`) as MetaStore;
+		const collectionKey = `meta:${collectionName}`;
+		return {
+			get: (key: string) => this.get(collectionKey, key),
+			set: (key: string, data: string) => this.set(collectionKey, key, data),
+			delete: (key: string) => this.delete(collectionKey, key),
+			has: (key: string) => this.has(collectionKey, key),
+		};
 	}
 
 	toString() {
@@ -148,10 +199,20 @@ export class DataStore {
 }
 
 export interface ScopedDataStore {
-	get: (key: string) => unknown;
-	entries: () => Array<[id: string, unknown]>;
-	set: (key: string, value: unknown) => void;
-	values: () => Array<unknown>;
+	get: (key: string) => DataEntry | undefined;
+	entries: () => Array<[id: string, DataEntry]>;
+	set: (opts: {
+		id: string;
+		data: Record<string, unknown>;
+		body?: string;
+		filePath?: string;
+		digest?: number | string;
+		rendered?: {
+			html: string;
+			metadata?: Record<string, unknown>;
+		};
+	}) => boolean;
+	values: () => Array<DataEntry>;
 	keys: () => Array<string>;
 	delete: (key: string) => void;
 	clear: () => void;
@@ -166,6 +227,7 @@ export interface MetaStore {
 	get: (key: string) => string | undefined;
 	set: (key: string, value: string) => void;
 	has: (key: string) => boolean;
+	delete: (key: string) => void;
 }
 
 function dataStoreSingleton() {

packages/astro/src/content/file.ts → packages/astro/src/content/loaders/file.ts

@@ -1,6 +1,6 @@
-import { fileURLToPath } from 'url';
-import type { Loader, LoaderContext } from './loaders.js';
 import { promises as fs, existsSync } from 'fs';
+import { fileURLToPath } from 'url';
+import type { Loader, LoaderContext } from './types.js';
 
 /**
  * Loads entries from a JSON file. The file must contain an array of objects that contain unique `id` fields, or an object with string keys.
@@ -37,16 +37,16 @@ export function file(fileName: string): Loader {
 					logger.error(`Item in ${fileName} is missing an id or slug field.`);
 					continue;
 				}
-				const item = await parseData({ id, data: rawItem, filePath });
-				store.set(id, item);
+				const data = await parseData({ id, data: rawItem, filePath });
+				store.set({ id, data, filePath });
 			}
 		} else if (typeof json === 'object') {
 			const entries = Object.entries<Record<string, unknown>>(json);
 			logger.debug(`Found object with ${entries.length} entries in ${fileName}`);
 			store.clear();
 			for (const [id, rawItem] of entries) {
-				const item = await parseData({ id, data: rawItem, filePath });
-				store.set(id, item);
+				const data = await parseData({ id, data: rawItem, filePath });
+				store.set({ id, data });
 			}
 		} else {
 			logger.error(`Invalid data in ${fileName}. Must be an array or object.`);
@@ -57,9 +57,8 @@ export function file(fileName: string): Loader {
 		name: 'file-loader',
 		load: async (options) => {
 			const { settings, logger, watcher } = options;
-			const contentDir = new URL('./content/', settings.config.srcDir);
 			logger.debug(`Loading data from ${fileName}`);
-			const url = new URL(fileName, contentDir);
+			const url = new URL(fileName, settings.config.root);
 			if (!existsSync(url)) {
 				logger.error(`File not found: ${fileName}`);
 				return;