Fix markdown charset as utf-8 by default (#8795)

.changeset/olive-laws-work.md

@@ -0,0 +1,5 @@
+---
+'astro': patch
+---
+
+Fix markdown page charset to be utf-8 by default (same as Astro 2)

packages/astro/src/runtime/server/render/page.ts

@@ -63,6 +63,11 @@ export async function renderPage(
 		body = encoder.encode(body);
 		headers.set('Content-Length', body.byteLength.toString());
 	}
+	// TODO: Revisit if user should manually set charset by themselves in Astro 4
+	// This code preserves the existing behaviour for markdown pages since Astro 2
+	if (route?.component.endsWith('.md')) {
+		headers.set('Content-Type', 'text/html; charset=utf-8');
+	}
 	const response = new Response(body, { ...init, headers });
 	return response;
 }

packages/astro/test/astro-basic.test.js

@@ -3,23 +3,31 @@ import * as cheerio from 'cheerio';
 import { loadFixture } from './test-utils.js';
 
 describe('Astro basics', () => {
+	/** @type {import('./test-utils').Fixture} */
 	let fixture;
-	let previewServer;
 
 	before(async () => {
 		fixture = await loadFixture({
 			root: './fixtures/astro-basic/',
 		});
-		await fixture.build();
-		previewServer = await fixture.preview();
-	});
-
-	// important: close preview server (free up port and connection)
-	after(async () => {
-		await previewServer.stop();
 	});
 
 	describe('build', () => {
+		let previewServer;
+
+		before(async () => {
+			fixture = await loadFixture({
+				root: './fixtures/astro-basic/',
+			});
+			await fixture.build();
+			previewServer = await fixture.preview();
+		});
+
+		// important: close preview server (free up port and connection)
+		after(async () => {
+			await previewServer.stop();
+		});
+
 		it('Can load page', async () => {
 			const html = await fixture.readFile(`/index.html`);
 			const $ = cheerio.load(html);
@@ -108,39 +116,87 @@ describe('Astro basics', () => {
 			const $ = cheerio.load(html);
 			expect($('#rendered-order').text()).to.eq('Rendered order: A, B');
 		});
-	});
 
-	it('Supports void elements whose name is a string (#2062)', async () => {
-		const html = await fixture.readFile('/input/index.html');
-		const $ = cheerio.load(html);
+		it('renders markdown in utf-8 by default', async () => {
+			const html = await fixture.readFile('/chinese-encoding-md/index.html');
+			const $ = cheerio.load(html);
+			expect($('h1').text()).to.equal('我的第一篇博客文章');
+		});
+
+		it('renders MDX in utf-8 by default', async () => {
+			const html = await fixture.readFile('/chinese-encoding-mdx/index.html');
+			const $ = cheerio.load(html);
+			expect($('h1').text()).to.equal('我的第一篇博客文章');
+		});
+
+		it('Supports void elements whose name is a string (#2062)', async () => {
+			const html = await fixture.readFile('/input/index.html');
+			const $ = cheerio.load(html);
+
+			// <Input />
+			expect($('body > :nth-child(1)').prop('outerHTML')).to.equal('<input>');
 
-		// <Input />
-		expect($('body > :nth-child(1)').prop('outerHTML')).to.equal('<input>');
+			// <Input type="password" />
+			expect($('body > :nth-child(2)').prop('outerHTML')).to.equal('<input type="password">');
 
-		// <Input type="password" />
-		expect($('body > :nth-child(2)').prop('outerHTML')).to.equal('<input type="password">');
+			// <Input type="text" />
+			expect($('body > :nth-child(3)').prop('outerHTML')).to.equal('<input type="text">');
 
-		// <Input type="text" />
-		expect($('body > :nth-child(3)').prop('outerHTML')).to.equal('<input type="text">');
+			// <Input type="select"><option>option</option></Input>
+			expect($('body > :nth-child(4)').prop('outerHTML')).to.equal(
+				'<select><option>option</option></select>'
+			);
 
-		// <Input type="select"><option>option</option></Input>
-		expect($('body > :nth-child(4)').prop('outerHTML')).to.equal(
-			'<select><option>option</option></select>'
-		);
+			// <Input type="textarea">textarea</Input>
+			expect($('body > :nth-child(5)').prop('outerHTML')).to.equal('<textarea>textarea</textarea>');
+		});
+
+		describe('preview', () => {
+			it('returns 200 for valid URLs', async () => {
+				const result = await fixture.fetch('/');
+				expect(result.status).to.equal(200);
+			});
 
-		// <Input type="textarea">textarea</Input>
-		expect($('body > :nth-child(5)').prop('outerHTML')).to.equal('<textarea>textarea</textarea>');
+			it('returns 404 for invalid URLs', async () => {
+				const result = await fixture.fetch('/bad-url');
+				expect(result.status).to.equal(404);
+			});
+		});
 	});
 
-	describe('preview', () => {
-		it('returns 200 for valid URLs', async () => {
-			const result = await fixture.fetch('/');
-			expect(result.status).to.equal(200);
+	describe('development', () => {
+		/** @type {import('./test-utils').DevServer} */
+		let devServer;
+
+		before(async () => {
+			devServer = await fixture.startDevServer();
+		});
+		after(async () => {
+			await devServer.stop();
+		});
+
+		it('Renders markdown in utf-8 by default', async () => {
+			const res = await fixture.fetch('/chinese-encoding-md');
+			expect(res.status).to.equal(200);
+			const html = await res.text();
+			const $ = cheerio.load(html);
+			expect($('h1').text()).to.equal('我的第一篇博客文章');
+			const isUtf8 =
+				res.headers.get('content-type').includes('charset=utf-8') ||
+				html.includes('<meta charset="utf-8">');
+			expect(isUtf8).to.be.true;
 		});
 
-		it('returns 404 for invalid URLs', async () => {
-			const result = await fixture.fetch('/bad-url');
-			expect(result.status).to.equal(404);
+		it('Renders MDX in utf-8 by default', async () => {
+			const res = await fixture.fetch('/chinese-encoding-mdx');
+			expect(res.status).to.equal(200);
+			const html = await res.text();
+			const $ = cheerio.load(html);
+			expect($('h1').text()).to.equal('我的第一篇博客文章');
+			const isUtf8 =
+				res.headers.get('content-type').includes('charset=utf-8') ||
+				html.includes('<meta charset="utf-8">');
+			expect(isUtf8).to.be.true;
 		});
 	});
 });

packages/astro/test/fixtures/astro-basic/astro.config.mjs

@@ -1,9 +1,10 @@
 import { defineConfig } from 'astro/config';
 import preact from '@astrojs/preact';
+import mdx from '@astrojs/mdx';
 
 // https://astro.build/config
 export default defineConfig({
-	integrations: [preact()],
+	integrations: [preact(), mdx()],
 	// make sure CLI flags have precedence
   server: () => ({ port: 4321 })
 });

packages/astro/test/fixtures/astro-basic/package.json

@@ -3,6 +3,7 @@
   "version": "0.0.0",
   "private": true,
   "dependencies": {
+    "@astrojs/mdx": "workspace:*",
     "@astrojs/preact": "workspace:*",
     "astro": "workspace:*",
     "preact": "^10.17.1"

packages/astro/test/fixtures/astro-basic/src/pages/chinese-encoding-md.md

@@ -0,0 +1,3 @@
+# 我的第一篇博客文章
+
+发表于：2022-07-01

packages/astro/test/fixtures/astro-basic/src/pages/chinese-encoding-mdx.mdx

@@ -0,0 +1,3 @@
+# 我的第一篇博客文章
+
+发表于：2022-07-01

packages/astro/test/units/render/head.test.js

@@ -90,7 +90,7 @@ describe('core/render', () => {
 
 			const PageModule = createAstroModule(Page);
 			const ctx = await createRenderContext({
-				route: { type: 'page', pathname: '/index' },
+				route: { type: 'page', pathname: '/index', component: 'src/pages/index.astro' },
 				request: new Request('http://example.com/'),
 				links: [{ name: 'link', props: { rel: 'stylesheet', href: '/main.css' }, children: '' }],
 				mod: PageModule,
@@ -171,7 +171,7 @@ describe('core/render', () => {
 
 			const PageModule = createAstroModule(Page);
 			const ctx = await createRenderContext({
-				route: { type: 'page', pathname: '/index' },
+				route: { type: 'page', pathname: '/index', component: 'src/pages/index.astro' },
 				request: new Request('http://example.com/'),
 				links: [{ name: 'link', props: { rel: 'stylesheet', href: '/main.css' }, children: '' }],
 				env,
@@ -218,7 +218,7 @@ describe('core/render', () => {
 
 			const PageModule = createAstroModule(Page);
 			const ctx = await createRenderContext({
-				route: { type: 'page', pathname: '/index' },
+				route: { type: 'page', pathname: '/index', component: 'src/pages/index.astro' },
 				request: new Request('http://example.com/'),
 				links: [{ name: 'link', props: { rel: 'stylesheet', href: '/main.css' }, children: '' }],
 				env,

packages/astro/test/units/render/jsx.test.js

@@ -45,7 +45,7 @@ describe('core/render', () => {
 
 			const mod = createAstroModule(Page);
 			const ctx = await createRenderContext({
-				route: { type: 'page', pathname: '/index' },
+				route: { type: 'page', pathname: '/index', component: 'src/pages/index.mdx' },
 				request: new Request('http://example.com/'),
 				env,
 				mod,
@@ -91,7 +91,7 @@ describe('core/render', () => {
 
 			const mod = createAstroModule(Page);
 			const ctx = await createRenderContext({
-				route: { type: 'page', pathname: '/index' },
+				route: { type: 'page', pathname: '/index', component: 'src/pages/index.mdx' },
 				request: new Request('http://example.com/'),
 				env,
 				mod,
@@ -117,7 +117,7 @@ describe('core/render', () => {
 
 			const mod = createAstroModule(Page);
 			const ctx = await createRenderContext({
-				route: { type: 'page', pathname: '/index' },
+				route: { type: 'page', pathname: '/index', component: 'src/pages/index.mdx' },
 				request: new Request('http://example.com/'),
 				env,
 				mod,