From 0c7ffe70174770e374733749f1428a4cd3eead08 Mon Sep 17 00:00:00 2001 From: Will Puckett Date: Sat, 4 Feb 2023 12:01:55 -0800 Subject: [PATCH 1/5] switch post loading to async --- blog.tsx | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/blog.tsx b/blog.tsx index bdd8880..72eb5fb 100644 --- a/blog.tsx +++ b/blog.tsx @@ -192,14 +192,25 @@ async function loadContent(blogDirectory: string, isDev: boolean) { // Read posts from the current directory and store them in memory. const postsDirectory = join(blogDirectory, "posts"); + + + + + + + const loadingPromises: Promise[] = []; + // TODO(@satyarohith): not efficient for large number of posts. for await ( const entry of walk(postsDirectory) - ) { - if (entry.isFile && entry.path.endsWith(".md")) { - await loadPost(postsDirectory, entry.path); + ) { + if (entry.isFile && entry.path.endsWith(".md")) { + // Switch to async to speed larger blogs + // await loadPost(postsDirectory, entry.path); + loadingPromises.push(loadPost(postsDirectory, entry.path)); + } } - } + await Promise.all(loadingPromises); if (isDev) { watchForChanges(postsDirectory).catch(() => {}); From c0bbbf506b4bea235837721137f7175be0e74857 Mon Sep 17 00:00:00 2001 From: Will Puckett Date: Sat, 4 Feb 2023 12:13:56 -0800 Subject: [PATCH 2/5] earlier PR failed formatting check resubmitting --- blog.tsx | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/blog.tsx b/blog.tsx index 72eb5fb..5db009b 100644 --- a/blog.tsx +++ b/blog.tsx @@ -192,25 +192,19 @@ async function loadContent(blogDirectory: string, isDev: boolean) { // Read posts from the current directory and store them in memory. const postsDirectory = join(blogDirectory, "posts"); - - - - - - const loadingPromises: Promise[] = []; - + // TODO(@satyarohith): not efficient for large number of posts. for await ( const entry of walk(postsDirectory) - ) { - if (entry.isFile && entry.path.endsWith(".md")) { - // Switch to async to speed larger blogs - // await loadPost(postsDirectory, entry.path); - loadingPromises.push(loadPost(postsDirectory, entry.path)); - } + ) { + if (entry.isFile && entry.path.endsWith(".md")) { + // Switch to async to speed larger blogs + // await loadPost(postsDirectory, entry.path); + loadingPromises.push(loadPost(postsDirectory, entry.path)); } - await Promise.all(loadingPromises); + } + await Promise.all(loadingPromises); if (isDev) { watchForChanges(postsDirectory).catch(() => {}); From 28153efe3ef2c13acd21e52647975cb41f623673 Mon Sep 17 00:00:00 2001 From: Will Puckett Date: Sat, 4 Feb 2023 21:21:53 -0800 Subject: [PATCH 3/5] Remove comments --- blog.tsx | 3 --- 1 file changed, 3 deletions(-) diff --git a/blog.tsx b/blog.tsx index 5db009b..fbf271f 100644 --- a/blog.tsx +++ b/blog.tsx @@ -194,13 +194,10 @@ async function loadContent(blogDirectory: string, isDev: boolean) { const loadingPromises: Promise[] = []; - // TODO(@satyarohith): not efficient for large number of posts. for await ( const entry of walk(postsDirectory) ) { if (entry.isFile && entry.path.endsWith(".md")) { - // Switch to async to speed larger blogs - // await loadPost(postsDirectory, entry.path); loadingPromises.push(loadPost(postsDirectory, entry.path)); } } From a64e0b484b8e7ef3963286d985100ff03a6ca235 Mon Sep 17 00:00:00 2001 From: Will Puckett Date: Wed, 17 May 2023 22:08:49 -0700 Subject: [PATCH 4/5] experimenting with pooledMap --- blog.tsx | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/blog.tsx b/blog.tsx index fbf271f..3a85c08 100644 --- a/blog.tsx +++ b/blog.tsx @@ -27,6 +27,7 @@ import { UnoCSS, walk, } from "./deps.ts"; +import { pooledMap } from "https://deno.land/std@0.187.0/async/pool.ts"; import { Index, PostPage } from "./components.tsx"; import type { ConnInfo, FeedItem } from "./deps.ts"; import type { @@ -36,6 +37,7 @@ import type { BlogState, Post, } from "./types.d.ts"; +import { WalkEntry } from "https://deno.land/std@0.176.0/fs/walk.ts"; export { Fragment, h }; @@ -192,16 +194,22 @@ async function loadContent(blogDirectory: string, isDev: boolean) { // Read posts from the current directory and store them in memory. const postsDirectory = join(blogDirectory, "posts"); - const loadingPromises: Promise[] = []; - - for await ( - const entry of walk(postsDirectory) - ) { + const traversal: WalkEntry[] = []; + for await (const entry of walk(postsDirectory)) { if (entry.isFile && entry.path.endsWith(".md")) { - loadingPromises.push(loadPost(postsDirectory, entry.path)); + traversal.push(entry); } } - await Promise.all(loadingPromises); + + const pool = pooledMap( + 10, + traversal, + (entry) => loadPost(postsDirectory, entry.path), + ); + + for await (const _ of pool) { + // noop + } if (isDev) { watchForChanges(postsDirectory).catch(() => {}); From 91aa344ed71afff804cc0edd37498c138c622c5a Mon Sep 17 00:00:00 2001 From: Will Puckett Date: Wed, 17 May 2023 22:12:03 -0700 Subject: [PATCH 5/5] up poolLimit --- blog.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blog.tsx b/blog.tsx index 3a85c08..db0f598 100644 --- a/blog.tsx +++ b/blog.tsx @@ -202,7 +202,7 @@ async function loadContent(blogDirectory: string, isDev: boolean) { } const pool = pooledMap( - 10, + 25, traversal, (entry) => loadPost(postsDirectory, entry.path), );