From 829de1811021b07cffdb5e3d5c5fe1182b160dba Mon Sep 17 00:00:00 2001 From: Travis Fischer Date: Mon, 5 Apr 2021 23:36:12 -0400 Subject: [PATCH 1/5] feat: speed up site resolving --- api/sitemap.xml.ts | 6 +++--- lib/get-all-pages.ts | 22 +++++++++++++++++++--- lib/get-preview-images.ts | 34 ++++++++++++++++++++-------------- lib/get-site-map.ts | 31 +++++++++++++++++++++++++++++++ lib/get-site-maps.ts | 35 ----------------------------------- lib/get-sites.ts | 7 ------- lib/resolve-notion-page.ts | 9 ++++----- lib/types.ts | 5 +++++ pages/[pageId].tsx | 24 ++++++++++-------------- site.config.js | 5 +++++ 10 files changed, 97 insertions(+), 81 deletions(-) create mode 100644 lib/get-site-map.ts delete mode 100644 lib/get-site-maps.ts delete mode 100644 lib/get-sites.ts diff --git a/api/sitemap.xml.ts b/api/sitemap.xml.ts index f591ed8b2d..721eaac296 100644 --- a/api/sitemap.xml.ts +++ b/api/sitemap.xml.ts @@ -2,7 +2,7 @@ import { NextApiRequest, NextApiResponse } from 'next' import { SiteMap } from '../lib/types' import { host } from '../lib/config' -import { getSiteMaps } from '../lib/get-site-maps' +import { getSiteMap } from '../lib/get-site-map' export default async ( req: NextApiRequest, @@ -12,7 +12,7 @@ export default async ( return res.status(405).send({ error: 'method not allowed' }) } - const siteMaps = await getSiteMaps() + const siteMap = await getSiteMap() // cache sitemap for up to one hour res.setHeader( @@ -20,7 +20,7 @@ export default async ( 'public, s-maxage=3600, max-age=3600, stale-while-revalidate=3600' ) res.setHeader('Content-Type', 'text/xml') - res.write(createSitemap(siteMaps[0])) + res.write(createSitemap(siteMap)) res.end() } diff --git a/lib/get-all-pages.ts b/lib/get-all-pages.ts index 44a4c30a5c..807efe1ba6 100644 --- a/lib/get-all-pages.ts +++ b/lib/get-all-pages.ts @@ -12,12 +12,28 @@ export const getAllPages = pMemoize(getAllPagesImpl, { maxAge: 60000 * 5 }) export async function getAllPagesImpl( rootNotionPageId: string, - rootNotionSpaceId: string -): Promise> { + rootNotionSpaceId: string, + { + concurrency = 4, + pageConcurrency = 3, + full = false + }: { + concurrency?: number + pageConcurrency?: number + full?: boolean + } = {} +): Promise { const pageMap = await getAllPagesInSpace( rootNotionPageId, rootNotionSpaceId, - notion.getPage.bind(notion) + (pageId: string) => + notion.getPage(pageId, { + signFileUrls: full, + concurrency: pageConcurrency + }), + { + concurrency + } ) const canonicalPageMap = Object.keys(pageMap).reduce( diff --git a/lib/get-preview-images.ts b/lib/get-preview-images.ts index 502f765454..49d47a3e11 100644 --- a/lib/get-preview-images.ts +++ b/lib/get-preview-images.ts @@ -28,22 +28,28 @@ export async function getPreviewImages( } const imageDocs = await db.db.getAll(...imageDocRefs) - const results = await pMap(imageDocs, async (model, index) => { - if (model.exists) { - return model.data() as types.PreviewImage - } else { - const json = { - url: images[index], - id: model.id - } - console.log('createPreviewImage server-side', json) + const results = await pMap( + imageDocs, + async (model, index) => { + if (model.exists) { + return model.data() as types.PreviewImage + } else { + const json = { + url: images[index], + id: model.id + } + console.log('createPreviewImage server-side', json) - // TODO: should we fire and forget here to speed up builds? - return got - .post(api.createPreviewImage, { json }) - .json() as Promise + // TODO: should we fire and forget here to speed up builds? + return got + .post(api.createPreviewImage, { json }) + .json() as Promise + } + }, + { + concurrency: 16 } - }) + ) return results .filter(Boolean) diff --git a/lib/get-site-map.ts b/lib/get-site-map.ts new file mode 100644 index 0000000000..50dd21dae6 --- /dev/null +++ b/lib/get-site-map.ts @@ -0,0 +1,31 @@ +import { getAllPages } from './get-all-pages' +import { getSiteForDomain } from './get-site-for-domain' +import * as config from './config' +import * as types from './types' + +export async function getSiteMap({ + concurrency = 4, + pageConcurrency = 3, + full = false +}: { + concurrency?: number + pageConcurrency?: number + full?: boolean +} = {}): Promise { + const site = await getSiteForDomain(config.domain) + + const siteMap = await getAllPages( + site.rootNotionPageId, + site.rootNotionSpaceId, + { + concurrency, + pageConcurrency, + full + } + ) + + return { + site, + ...siteMap + } +} diff --git a/lib/get-site-maps.ts b/lib/get-site-maps.ts deleted file mode 100644 index 55173da1a2..0000000000 --- a/lib/get-site-maps.ts +++ /dev/null @@ -1,35 +0,0 @@ -import pMap from 'p-map' - -import { getAllPages } from './get-all-pages' -import { getSites } from './get-sites' -import * as types from './types' - -export async function getSiteMaps(): Promise { - const sites = await getSites() - - const siteMaps = await pMap( - sites, - async (site, index) => { - try { - console.log( - 'getSiteMap', - `${index + 1}/${sites.length}`, - `(${(((index + 1) / sites.length) * 100) | 0}%)`, - site - ) - - return { - site, - ...(await getAllPages(site.rootNotionPageId, site.rootNotionSpaceId)) - } as types.SiteMap - } catch (err) { - console.warn('site build error', index, site, err) - } - }, - { - concurrency: 4 - } - ) - - return siteMaps.filter(Boolean) -} diff --git a/lib/get-sites.ts b/lib/get-sites.ts deleted file mode 100644 index e9c8b31bb7..0000000000 --- a/lib/get-sites.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { getSiteForDomain } from './get-site-for-domain' -import * as config from './config' -import * as types from './types' - -export async function getSites(): Promise { - return [await getSiteForDomain(config.domain)] -} diff --git a/lib/resolve-notion-page.ts b/lib/resolve-notion-page.ts index a048cf5203..5656840450 100644 --- a/lib/resolve-notion-page.ts +++ b/lib/resolve-notion-page.ts @@ -5,7 +5,7 @@ import * as acl from './acl' import * as types from './types' import { pageUrlOverrides, pageUrlAdditions } from './config' import { getPage } from './notion' -import { getSiteMaps } from './get-site-maps' +import { getSiteMap } from './get-site-map' import { getSiteForDomain } from './get-site-for-domain' export async function resolveNotionPage(domain: string, rawPageId?: string) { @@ -37,10 +37,9 @@ export async function resolveNotionPage(domain: string, rawPageId?: string) { recordMap = resources[1] } else { // handle mapping of user-friendly canonical page paths to Notion page IDs - // e.g., /developer-x-entrepreneur versus /71201624b204481f862630ea25ce62fe - const siteMaps = await getSiteMaps() - const siteMap = siteMaps[0] - pageId = siteMap.canonicalPageMap[rawPageId] + // e.g., /foo versus /71201624b204481f862630ea25ce62fe + const siteMap = await getSiteMap() + pageId = siteMap?.canonicalPageMap?.[rawPageId] if (pageId) { // TODO: we're not re-using the site from siteMaps because it is diff --git a/lib/types.ts b/lib/types.ts index 3ac70f05a3..d8eff551a1 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -50,6 +50,11 @@ export interface SiteMap { canonicalPageMap: CanonicalPageMap } +export interface PartialSiteMap { + pageMap: PageMap + canonicalPageMap: CanonicalPageMap +} + export interface CanonicalPageMap { [canonicalPageId: string]: string } diff --git a/pages/[pageId].tsx b/pages/[pageId].tsx index 15a093f35d..8a8202f061 100644 --- a/pages/[pageId].tsx +++ b/pages/[pageId].tsx @@ -1,6 +1,6 @@ import React from 'react' import { isDev, domain } from 'lib/config' -import { getSiteMaps } from 'lib/get-site-maps' +import { getSiteMap } from 'lib/get-site-map' import { resolveNotionPage } from 'lib/resolve-notion-page' import { NotionPage } from 'components' @@ -36,22 +36,18 @@ export async function getStaticPaths() { } } - const siteMaps = await getSiteMaps() + const siteMap = await getSiteMap() + const paths = Object.keys(siteMap.canonicalPageMap).map((pageId) => ({ + params: { + pageId + } + })) + console.log(paths) - const ret = { - paths: siteMaps.flatMap((siteMap) => - Object.keys(siteMap.canonicalPageMap).map((pageId) => ({ - params: { - pageId - } - })) - ), - // paths: [], + return { + paths, fallback: true } - - console.log(ret.paths) - return ret } export default function NotionDomainDynamicPage(props) { diff --git a/site.config.js b/site.config.js index c0e7683b18..b3fe02f1aa 100644 --- a/site.config.js +++ b/site.config.js @@ -39,6 +39,11 @@ module.exports = { // variables specified in .env.example isPreviewImageSupportEnabled: false, + // whether or not to include notion IDs as suffixes in URLs + // NOTE: this will make incremental SSG much faster with the downside of + // having less pretty URLs + includeNotionIdInUrls: false, + // map of notion page IDs to URL paths (optional) // any pages defined here will override their default URL paths // example: From a8eb84dce3782ee560cf07a8101256fa0e120078 Mon Sep 17 00:00:00 2001 From: Travis Fischer Date: Mon, 5 Apr 2021 23:47:04 -0400 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=94=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- components/NotionPage.tsx | 2 +- components/Page404.tsx | 4 ++-- components/PageHead.tsx | 17 ++--------------- lib/resolve-notion-page.ts | 15 ++++++++------- 4 files changed, 13 insertions(+), 25 deletions(-) diff --git a/components/NotionPage.tsx b/components/NotionPage.tsx index 266717b51c..f0a2689a5c 100644 --- a/components/NotionPage.tsx +++ b/components/NotionPage.tsx @@ -170,7 +170,7 @@ export const NotionPage: React.FC = ({ } }} > - + diff --git a/components/Page404.tsx b/components/Page404.tsx index ae933e85de..65dc499be1 100644 --- a/components/Page404.tsx +++ b/components/Page404.tsx @@ -6,11 +6,11 @@ import { PageHead } from './PageHead' import styles from './styles.module.css' export const Page404: React.FC = ({ site, pageId, error }) => { - const title = site?.name || 'Notion Page Not Found' + const title = site?.name ? `${site.name} Page Not Found` : 'Page Not Found' return ( <> - + diff --git a/components/PageHead.tsx b/components/PageHead.tsx index 4072159fde..22096bfb64 100644 --- a/components/PageHead.tsx +++ b/components/PageHead.tsx @@ -1,10 +1,7 @@ +import React from 'react' import Head from 'next/head' -import * as React from 'react' -import * as types from 'lib/types' -// TODO: remove duplication between PageHead and NotionPage Head - -export const PageHead: React.FC = ({ site }) => { +export const PageHead = () => { return ( @@ -13,16 +10,6 @@ export const PageHead: React.FC = ({ site }) => { name='viewport' content='width=device-width, initial-scale=1, shrink-to-fit=no' /> - - {site?.description && ( - <> - - - - )} - - - ) } diff --git a/lib/resolve-notion-page.ts b/lib/resolve-notion-page.ts index 5656840450..92d09d9e86 100644 --- a/lib/resolve-notion-page.ts +++ b/lib/resolve-notion-page.ts @@ -8,19 +8,19 @@ import { getPage } from './notion' import { getSiteMap } from './get-site-map' import { getSiteForDomain } from './get-site-for-domain' -export async function resolveNotionPage(domain: string, rawPageId?: string) { - let site: types.Site +export async function resolveNotionPage(domain: string, rawPageUri?: string) { let pageId: string + let site: types.Site let recordMap: ExtendedRecordMap - if (rawPageId && rawPageId !== 'index') { - pageId = parsePageId(rawPageId) + if (rawPageUri && rawPageUri !== 'index') { + pageId = parsePageId(rawPageUri) if (!pageId) { // check if the site configuration provides an override of a fallback for // the page's URI const override = - pageUrlOverrides[rawPageId] || pageUrlAdditions[rawPageId] + pageUrlOverrides[rawPageUri] || pageUrlAdditions[rawPageUri] if (override) { pageId = parsePageId(override) @@ -39,7 +39,7 @@ export async function resolveNotionPage(domain: string, rawPageId?: string) { // handle mapping of user-friendly canonical page paths to Notion page IDs // e.g., /foo versus /71201624b204481f862630ea25ce62fe const siteMap = await getSiteMap() - pageId = siteMap?.canonicalPageMap?.[rawPageId] + pageId = siteMap?.canonicalPageMap?.[rawPageUri] if (pageId) { // TODO: we're not re-using the site from siteMaps because it is @@ -57,13 +57,14 @@ export async function resolveNotionPage(domain: string, rawPageId?: string) { } else { return { error: { - message: `Not found "${rawPageId}"`, + message: `Not found "${rawPageUri}"`, statusCode: 404 } } } } } else { + // resolve the site's home page site = await getSiteForDomain(domain) pageId = site.rootNotionPageId From 20319d21c180a8118d8d12623fc94e419b208b4c Mon Sep 17 00:00:00 2001 From: Travis Fischer Date: Tue, 6 Apr 2021 20:11:35 -0400 Subject: [PATCH 3/5] WIP --- components/NotionPage.tsx | 10 ++-- lib/canonical-page-map-cache.ts | 5 ++ lib/cloudflare-kv.ts | 99 +++++++++++++++++++++++++++++++++ lib/get-all-pages.ts | 13 ++++- lib/types.ts | 1 + package.json | 1 + yarn.lock | 2 +- 7 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 lib/canonical-page-map-cache.ts create mode 100644 lib/cloudflare-kv.ts diff --git a/components/NotionPage.tsx b/components/NotionPage.tsx index f0a2689a5c..f925578d3f 100644 --- a/components/NotionPage.tsx +++ b/components/NotionPage.tsx @@ -38,6 +38,7 @@ import { ReactUtterances } from './ReactUtterances' import styles from './styles.module.css' +// NOTE: if your site doesn't use these, then we recommend switching them to load lazily // const Code = dynamic(() => // import('react-notion-x').then((notion) => notion.Code) // ) @@ -59,10 +60,6 @@ const Equation = dynamic(() => import('react-notion-x').then((notion) => notion.Equation) ) -// we're now using a much lighter-weight tweet renderer react-static-tweets -// instead of the official iframe-based embed widget from twitter -// const Tweet = dynamic(() => import('react-tweet-embed')) - const Modal = dynamic( () => import('react-notion-x').then((notion) => notion.Modal), { ssr: false } @@ -108,7 +105,7 @@ export const NotionPage: React.FC = ({ }) if (!config.isServer) { - // add important objects to the window global for easy debugging + // add important variables to the global window object for easy debugging const g = window as any g.pageId = pageId g.recordMap = recordMap @@ -122,6 +119,9 @@ export const NotionPage: React.FC = ({ // const isRootPage = // parsePageId(block.id) === parsePageId(site.rootNotionPageId) + + // this is all very customizable logic depending on the contents and + // structure of your site const isBlogPost = block.type === 'page' && block.parent_table === 'collection' const showTableOfContents = !!isBlogPost diff --git a/lib/canonical-page-map-cache.ts b/lib/canonical-page-map-cache.ts new file mode 100644 index 0000000000..ed1d373309 --- /dev/null +++ b/lib/canonical-page-map-cache.ts @@ -0,0 +1,5 @@ +import { CanonicalPageMap } from './types' + +export const getCanonicalPageMapCache = async (): Promise => { + // TODO +} diff --git a/lib/cloudflare-kv.ts b/lib/cloudflare-kv.ts new file mode 100644 index 0000000000..e18bf8e4db --- /dev/null +++ b/lib/cloudflare-kv.ts @@ -0,0 +1,99 @@ +import fetch from 'node-fetch' + +export class CloudflareKV { + accountId: string + apiKey: string + namespaceId: string + _headers: any + + constructor({ + accountId, + apiKey, + namespaceId + }: { + accountId: string + apiKey: string + namespaceId: string + }) { + this.accountId = accountId + this.apiKey = apiKey + this.namespaceId = namespaceId + + this._headers = { + Authorization: `Bearer ${this.apiKey}` + } + } + + async get(key: string) { + const url = this._getUrl(key) + const headers = this._headers + + const res = await fetch(url, { + headers + }) + + if (res.ok) { + return res.text() + } else if (res.status === 404) { + return undefined + } else { + const body = await res.text() + + throw new Error(`Error ${res.status} ${body}`) + } + } + + async put( + key: string, + value, + { + expiration, + expirationTtl + }: { + expiration?: number + expirationTtl?: number + } = {} + ) { + const url = new URL(this._getUrl(key)) + const headers = this._headers + const query: any = {} + + if (expiration) { + query.expiration = expiration + } + + if (expirationTtl) { + query.expiration_ttl = Math.max(60, expirationTtl) + } + + url.search = new URLSearchParams(query).toString() + const res = await fetch(url.toString(), { + method: 'PUT', + body: value, + headers + }) + + console.log('CLOUDFLARE PUT', { key, value, ok: res.ok }) + if (!res.ok) { + console.log(await res.text()) + } + return res.ok + } + + async delete(key: string): Promise { + const url = this._getUrl(key) + const headers = this._headers + + const res = await fetch(url, { + method: 'DELETE', + headers + }) + + return res.ok + } + + _getUrl(key: string) { + const { accountId, namespaceId } = this + return `https://api.cloudflare.com/client/v4/accounts/${accountId}/storage/kv/namespaces/${namespaceId}/values/${key}` + } +} diff --git a/lib/get-all-pages.ts b/lib/get-all-pages.ts index 807efe1ba6..6331d88c95 100644 --- a/lib/get-all-pages.ts +++ b/lib/get-all-pages.ts @@ -1,5 +1,6 @@ import pMemoize from 'p-memoize' import { getAllPagesInSpace } from 'notion-utils' +import stringify from 'fast-json-stable-stringify' import * as types from './types' import { includeNotionIdInUrls } from './config' @@ -8,7 +9,10 @@ import { getCanonicalPageId } from './get-canonical-page-id' const uuid = !!includeNotionIdInUrls -export const getAllPages = pMemoize(getAllPagesImpl, { maxAge: 60000 * 5 }) +export const getAllPages = pMemoize(getAllPagesImpl, { + maxAge: 60000 * 5, + cacheKey: (args) => stringify(args) +}) export async function getAllPagesImpl( rootNotionPageId: string, @@ -16,11 +20,13 @@ export async function getAllPagesImpl( { concurrency = 4, pageConcurrency = 3, - full = false + full = false, + targetPageId = null }: { concurrency?: number pageConcurrency?: number full?: boolean + targetPageId?: string } = {} ): Promise { const pageMap = await getAllPagesInSpace( @@ -32,7 +38,8 @@ export async function getAllPagesImpl( concurrency: pageConcurrency }), { - concurrency + concurrency, + targetPageId } ) diff --git a/lib/types.ts b/lib/types.ts index d8eff551a1..f72b16c8b4 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -56,6 +56,7 @@ export interface PartialSiteMap { } export interface CanonicalPageMap { + // inverse page mapping from canonial path to page id [canonicalPageId: string]: string } diff --git a/package.json b/package.json index 13823a88bd..d6364e6908 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "chrome-aws-lambda": "^5.5.0", "classnames": "^2.2.6", "dangerously-set-html-content": "^1.0.8", + "fast-json-stable-stringify": "^2.1.0", "fathom-client": "^3.0.0", "got": "^11.8.1", "isomorphic-unfetch": "^3.1.0", diff --git a/yarn.lock b/yarn.lock index 709b29deb1..91e0604397 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2226,7 +2226,7 @@ fast-glob@^3.1.1: micromatch "^4.0.2" picomatch "^2.2.1" -fast-json-stable-stringify@^2.0.0: +fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== From a4f90e2e399d94c0dfb4e057d9be78baab42a88a Mon Sep 17 00:00:00 2001 From: Richson Date: Fri, 9 Feb 2024 10:06:49 +0200 Subject: [PATCH 4/5] =?UTF-8?q?chore:=20updated=20the=20footer=20to=20make?= =?UTF-8?q?=20sure=20that=20it=20gets=20the=20current=20year=20dynamically?= =?UTF-8?q?.=F0=9F=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- components/Footer.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/Footer.tsx b/components/Footer.tsx index 5707d16a9c..89cc528fca 100644 --- a/components/Footer.tsx +++ b/components/Footer.tsx @@ -26,7 +26,7 @@ export const Footer: React.FC<{ return (