Skip to content

Commit

Permalink
feat: extend cache
Browse files Browse the repository at this point in the history
  • Loading branch information
Rob Ellison committed Aug 22, 2023
1 parent b03338f commit 3dc1fbb
Show file tree
Hide file tree
Showing 5 changed files with 287 additions and 148 deletions.
4 changes: 2 additions & 2 deletions components/content/ContentPage.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,8 +245,8 @@ function ContentMenu({ content, file, handleContentChange, handlePageReset, cont

let directory = path.dirname(file);

console.log('ContentMenu:directory ', directory)
console.log('ContentMenu:collection ', context)
// console.log('ContentMenu:directory ', directory)
// console.log('ContentMenu:collection ', context)

let chaptersMenu = []
if (content && content[directory]) {
Expand Down
115 changes: 66 additions & 49 deletions lib/content/menuContent.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,35 @@ import { cacheWrite, cacheRead } from '@/lib/redis';
import path from 'path';

export async function getFrontMatter(config) {
const branchSha = await getBranchSha(config.owner, config.repo, config.branch)
// const frontmatterCache = `structure:${config.path}:${branchSha}`;

// let files = null;
// const files = JSON.parse(await cacheRead(frontmatterCache));
// if (!files) { // files aren't cached

// }



const cacheKey = `frontmatter:${config.path}:${branchSha}`;

let cachedContent;
try {
cachedContent = JSON.parse(await cacheRead(cacheKey));
} catch (error) {
// Handle the error when JSON parsing fails (invalid data).
console.error('Error parsing cached content:', error);
cachedContent = null; // Or use a default value if required.
}
if (cachedContent) {
console.info('[Menu][Cache][HIT]:', cacheKey)
// If the content was found in the cache, return it
return cachedContent;
} else {
console.info('[Menu][Cache][MISS]:', cacheKey)
}

const files = await getAllFiles(
config.owner,
config.repo,
Expand All @@ -13,12 +42,9 @@ export async function getFrontMatter(config) {
".md*"
);

const branchSha = await getBranchSha(config.owner, config.repo, config.branch)
const cacheKey = `frontmatter:${config.path}:${branchSha}`;


const filesPromises = files.map((file) => {
return getFileContent(config.owner, config.repo, config.branch, file)
return getFileContent(config.owner, config.repo, config.branch, file.path, file.sha)
.then((content) => {
const matterData = matter(content, { excerpt: false }).data || null;
if (matterData) {
Expand All @@ -28,35 +54,20 @@ export async function getFrontMatter(config) {
}
}
}
return { file: file, frontmatter: matterData };
return { file: file.path, frontmatter: matterData };
})
.catch((error) => {
// console.error(`Error processing file ${file}: ${error}`);
console.error(`[Menu][Cache][Error] processing file: ${file}: ${error}`);
return { file: null, frontmatter: null };
});
});

let cachedContent;
try {
cachedContent = JSON.parse(await cacheRead(cacheKey));
} catch (error) {
// Handle the error when JSON parsing fails (invalid data).
console.error('Error parsing cached content:', error);
cachedContent = null; // Or use a default value if required.
}
if (cachedContent) {
console.info('[GitHub][Cache][HIT]:', cacheKey)
// If the content was found in the cache, return it
return cachedContent;
} else {
console.info('[GitHub][Cache][MISS]:', cacheKey)
}

cachedContent = await Promise.all(filesPromises);
try {
await cacheWrite(cacheKey, JSON.stringify(cachedContent), 60 * 60 * 24); // cache for 24 hours
await cacheWrite(cacheKey, JSON.stringify(cachedContent), 60 * 60 * 24 * 7); // cache for 7 days
} catch (error) {
console.error('[GitHub][Cache][FAIL]:', cacheKey, ' : ', error)
console.error('[Menu][Cache][FAIL]:', cacheKey, ' : ', error)

}
return await Promise.all(cachedContent);
Expand All @@ -75,21 +86,23 @@ export async function getContent(siteConfig) {
console.error('Error parsing cached content:', error);
cachedContent = null; // Or use a default value if required.
}
if (cachedContent) {
console.info('[GitHub][Cache][HIT]:', cacheKey)
if (cachedContent && cachedContent.length > 0) {
console.info('[Menu][Cache][HIT]:', cacheKey)
// If the content was found in the cache, return it
return cachedContent;
} else {
console.info('[GitHub][Cache][MISS]:', cacheKey)
console.info('[Menu][Cache][MISS]:', cacheKey)
}

const files = await getAllFiles(siteConfig.owner, siteConfig.repo, siteConfig.branch, siteConfig.path, true, '.md*');
console.log('[Menu][getAllFiles]: ', files)
const contentPromises = files.map((file) => {
return getFileContent(
siteConfig.owner,
siteConfig.repo,
siteConfig.branch,
file
file.path,
file.sha
)
.then(content => {
const matterData = matter(content, { excerpt: false }).data || null;
Expand Down Expand Up @@ -123,11 +136,11 @@ async function getSolutions(siteConfig) {
// Check if the content is in the cache
const cachedContent = JSON.parse(await cacheRead(cacheKey));
if (cachedContent) {
console.info('[GitHub][Cache][HIT]:', cacheKey)
console.info('[Menu][Cache][HIT]:', cacheKey)
// If the content was found in the cache, return it
return cachedContent;
} else {
console.info('[GitHub][Cache][MISS]:', cacheKey)
console.info('[Menu][Cache][MISS]:', cacheKey)
}

const solutions = await getAllFiles(siteConfig.content.solutions.owner, siteConfig.content.solutions.repo, siteConfig.content.solutions.branch, siteConfig.content.solutions.path, true, '.md*');
Expand Down Expand Up @@ -170,11 +183,11 @@ async function getKnowledge(siteConfig) {
// Check if the content is in the cache
const cachedContent = JSON.parse(await cacheRead(cacheKey));
if (cachedContent) {
console.info('[GitHub][Cache][HIT]:', cacheKey)
console.info('[Menu][Cache][HIT]:', cacheKey)
// If the content was found in the cache, return it
return cachedContent;
} else {
console.info('[GitHub][Cache][MISS]:', cacheKey)
console.info('[Menu][Cache][MISS]:', cacheKey)
}

const knowledge = await getAllFiles(siteConfig.content.knowledge.owner, siteConfig.content.knowledge.repo, siteConfig.content.knowledge.branch, siteConfig.content.knowledge.path, true, '.md*');
Expand Down Expand Up @@ -210,11 +223,11 @@ async function getDesigns(siteConfig) {
// Check if the content is in the cache
const cachedContent = JSON.parse(await cacheRead(cacheKey));
if (cachedContent) {
console.info('[GitHub][Cache][HIT]:', cacheKey)
console.info('[Menu][Cache][HIT]:', cacheKey)
// If the content was found in the cache, return it
return cachedContent;
} else {
console.info('[GitHub][Cache][MISS]:', cacheKey)
console.info('[Menu][Cache][MISS]:', cacheKey)
}

const contentFiles = await getAllFiles(siteConfig.content.designs.owner, siteConfig.content.designs.repo, siteConfig.content.designs.branch, siteConfig.content.designs.path, true, '.md*');
Expand Down Expand Up @@ -250,11 +263,11 @@ async function getServices(siteConfig) {
// Check if the content is in the cache
const cachedContent = JSON.parse(await cacheRead(cacheKey));
if (cachedContent) {
console.info('[GitHub][Cache][HIT]:', cacheKey)
console.info('[Menu][Cache][HIT]:', cacheKey)
// If the content was found in the cache, return it
return cachedContent;
} else {
console.info('[GitHub][Cache][MISS]:', cacheKey)
console.info('[Menu][Cache][MISS]:', cacheKey)
}

const contentFiles = await getAllFiles(siteConfig.content.services.owner, siteConfig.content.services.repo, siteConfig.content.services.branch, siteConfig.content.services.path, true, '.md*');
Expand Down Expand Up @@ -289,11 +302,11 @@ export async function getMenuStructureSolutions(siteConfig) {
// Check if the content is in the cache
const cachedContent = JSON.parse(await cacheRead(cacheKey));
if (cachedContent) {
console.info('[GitHub][Cache][HIT]:', cacheKey)
console.info('[Menu][Cache][HIT]:', cacheKey)
// If the content was found in the cache, return it
return cachedContent;
} else {
console.info('[GitHub][Cache][MISS]:', cacheKey)
console.info('[Menu][Cache][MISS]:', cacheKey)
}
const parent = 'solution';
const solutions = await getSolutions(siteConfig);
Expand Down Expand Up @@ -432,6 +445,7 @@ export async function getMenuStructure(siteConfig, collection) {
}
const parent = 'siteConfig.path';
const primary = await getContent(collection);
// console.info('[getMenuStructure][primary]:', primary)

let relatedFiles = {}; // all the files in related collections
let relatedContent = {}; // only the files that are children of the primary content
Expand All @@ -451,39 +465,41 @@ export async function getMenuStructure(siteConfig, collection) {
for (let x of primary) {
if (
x.file &&
x.file.split('/').length === 3 &&
x.file.match(/(_index\.md*|index\.md*)$/) &&
x.file.path &&
x.file.path.split('/').length === 3 &&
x.file.path.match(/(_index\.md*|index\.md*)$/) &&
x.frontmatter &&
x.frontmatter.title
) {
primaryMenu.push({
label: x.frontmatter.title,
url: x.file.startsWith('/') ? x.file : '/' + x.file,
url: x.file.path.startsWith('/') ? x.file.path : '/' + x.file.path,
});
indexFiles.add(path.dirname(x.file)); // Add directory name to the Set
indexFiles.add(path.dirname(x.file.path)); // Add directory name to the Set
}
}

// Second pass: Process non-index.md files
for (let x of primary) {
if (
x.file &&
x.file.split('/').length > 2 && // skip any files in the root of the directory
!x.file.match(/(_index\.md*|index\.md*)$/) &&
x.file.path &&
x.file.path.split('/').length > 2 && // skip any files in the root of the directory
!x.file.path.match(/(_index\.md*|index\.md*)$/) &&
x.frontmatter &&
x.frontmatter.title
) {
// let directory = x.file.split("/")[1]; // Extract directory name
let directory = path.dirname(x.file)
let directory = path.dirname(x.file.path)

// console.log('getMenuStructure')

let collection = x.file.split("/")[0]; // Extract directory name
let collection = x.file.path.split("/")[0]; // Extract directory name
// Only add file to solutionMenu if there is no corresponding index.md
if (!indexFiles.has(directory)) {
primaryMenu.push({
label: x.frontmatter.title,
url: x.file.startsWith('/') ? x.file : '/' + x.file,
url: x.file.path.startsWith('/') ? x.file.path : '/' + x.file.path,
});
}

Expand All @@ -498,7 +514,7 @@ export async function getMenuStructure(siteConfig, collection) {
// add the related content
relatedContent[directory]['chapters'].push({
label: x.frontmatter.title,
url: x.file.startsWith('/') ? x.file : '/' + x.file,
url: x.file.path.startsWith('/') ? x.file.path : '/' + x.file.path,
});

}
Expand All @@ -517,7 +533,8 @@ export async function getMenuStructure(siteConfig, collection) {
for (let x of relatedFiles[collectionItem]) {
if (
x.file &&
x.file.split('/').length > 2 && // skip any files in the root of the directory
x.file.path &&
x.file.path.split('/').length > 2 && // skip any files in the root of the directory
x.frontmatter &&
x.frontmatter.title
) {
Expand All @@ -539,7 +556,7 @@ export async function getMenuStructure(siteConfig, collection) {
// add the related content
relatedContent[directory][collectionItem].push({
label: x.frontmatter.title,
url: x.file.startsWith('/') ? x.file : '/' + x.file,
url: x.file.path.startsWith('/') ? x.file.path : '/' + x.file.path,
});

}
Expand Down
Loading

0 comments on commit 3dc1fbb

Please sign in to comment.