From 0a38c29339206ffa4601e956da3f704f35c520a1 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Thu, 15 Apr 2021 14:35:17 -0400 Subject: [PATCH 01/29] add new include and layout files that are feature flagged --- includes/breadcrumbs.html | 14 ++++++ includes/category-articles-list.html | 34 +++++++++++++ includes/generic-toc.html | 8 ++++ includes/product-articles-list.html | 46 ++++++++++++++++++ includes/sidebar-product.html | 71 ++++++++++++++++++++++++++++ includes/sidebar.html | 4 ++ layouts/generic-toc.html | 58 +++++++++++++++++++++++ layouts/product-landing.html | 11 +++++ 8 files changed, 246 insertions(+) create mode 100644 includes/category-articles-list.html create mode 100644 includes/generic-toc.html create mode 100644 includes/product-articles-list.html create mode 100644 includes/sidebar-product.html create mode 100644 layouts/generic-toc.html diff --git a/includes/breadcrumbs.html b/includes/breadcrumbs.html index 2a08fecb2288..e623d9c56826 100644 --- a/includes/breadcrumbs.html +++ b/includes/breadcrumbs.html @@ -1,3 +1,16 @@ +{% if FEATURE_NEW_SITETREE %} + +{% else %} +{% endif %} \ No newline at end of file diff --git a/includes/category-articles-list.html b/includes/category-articles-list.html new file mode 100644 index 000000000000..27e2cdd7f7c1 --- /dev/null +++ b/includes/category-articles-list.html @@ -0,0 +1,34 @@ +{% for categoryPage in currentProductTree.childPages %} +{% if categoryPage.href == currentPath %}{% assign currentCategory = categoryPage %}{% endif %} +{% endfor %} + +{% if currentCategory.page.shortTitle and currentCategory.page.shortTitle != '' %}{% assign currentCategoryTitle = currentCategory.page.shortTitle %}{% else %}{% assign currentCategoryTitle = currentCategory.page.title %}{% endif %} + +{% assign maxArticles = 10 %} + +
+

{{ currentCategoryTitle }} docs

+ +
+ {% for childPage in currentCategory.childPages %} + {% unless childPage.page.hidden %} +
+

{{ childPage.page.title }}

+
    + {% for grandchildPage in childPage.childPages %} +
  • + + {{ grandchildPage.page.title }} + +
    • + {% endfor %} + {% assign numArticles = childPage.childPages | obj_size %} + {% if numArticles > maxArticles %} + + {% endif %} +
+
+ {% endunless %} + {% endfor %} +
+
diff --git a/includes/generic-toc.html b/includes/generic-toc.html new file mode 100644 index 000000000000..0e9820dcb24d --- /dev/null +++ b/includes/generic-toc.html @@ -0,0 +1,8 @@ +{% for tocItem in tocItems %} + + +

{{ tocItem.title }}→

+ +{% if tocItem.intro %}

{{ tocItem.intro }}

{% endif %} + +{% endfor %} \ No newline at end of file diff --git a/includes/product-articles-list.html b/includes/product-articles-list.html new file mode 100644 index 000000000000..de0bec335691 --- /dev/null +++ b/includes/product-articles-list.html @@ -0,0 +1,46 @@ +{% assign maxArticles = 10 %} + +{% if currentProductTree.page.shortTitle and currentProductTree.page.shortTitle != '' %}{% assign productTitle = currentProductTree.page.shortTitle %}{% else %}{% assign productTitle = currentProductTree.page.title %}{% endif %} + +
+

All {{ productTitle }} docs

+ +
+ {% for childPage in currentProductTree.childPages %} + {% if childPage.page.documentType == "article" %}{% assign standaloneCategory = true %}{% else %}{% assign standaloneCategory = false %}{% endif %} + {% unless standaloneCategory %} +
+

{{ childPage.page.title }}

+ + {% if childPage.childPages and childPage.childPages[0].page.documentType == "mapTopic" %} +
    + {% for grandchildPage in childPage.childPages %} + {% unless grandchildPage.page.hidden %} + {% assign numArticles = childPage.childPages | obj_size %} +
  • + + {{ grandchildPage.page.title }} + +
    • + {% if numArticles > maxArticles %} + + {% endif %} + {% endunless %} + {% endfor %} +
+ {% else %} +
    + {% assign numArticles = childPage.childPages | obj_size %} + {% for grandchildPage in childPage.childPages %} +
  • {{ grandchildPage.page.title }}
    • + {% endfor %} +
+ {% if numArticles > maxArticles %} + + {% endif %} + {% endif %} +
+ {% endunless %} + {% endfor %} +
+
diff --git a/includes/sidebar-product.html b/includes/sidebar-product.html new file mode 100644 index 000000000000..33a9951375ea --- /dev/null +++ b/includes/sidebar-product.html @@ -0,0 +1,71 @@ + + +{% include all-products-link %} + +{% unless currentProductTree.page.hidden %} + +{% if currentProductTree.page.shortTitle and currentProductTree.page.shortTitle != '' %}{% assign productTitle = currentProductTree.page.shortTitle %}{% else %}{% assign productTitle = currentProductTree.page.title %}{% endif %} + +
  • + {{ productTitle }} +
    • +
  • + +
    • + +{% endunless %} diff --git a/includes/sidebar.html b/includes/sidebar.html index 74f1c94cc518..470a26208fe9 100644 --- a/includes/sidebar.html +++ b/includes/sidebar.html @@ -15,7 +15,11 @@ {% else %} {% endif %} diff --git a/layouts/generic-toc.html b/layouts/generic-toc.html new file mode 100644 index 000000000000..e4bfbe13434c --- /dev/null +++ b/layouts/generic-toc.html @@ -0,0 +1,58 @@ + + + {% include head %} + + + {% include sidebar %} + +
    + {% include header %} + {% include deprecation-banner %} +
    +
    +
    +
    {% include article-version-switcher %}
    +
    + {% include breadcrumbs %} +
    +
    {% include article-version-switcher %}
    +
    + +
    + +
    +
    +

    {{ page.title }}

    +
    + +
    +
    + + {% if page.intro %} +
    {{ page.intro }}
    + {% endif %} + + {% if page.product %} +
    + {{ page.product }} +
    + {% endif %} +
    +
    + {% if featuredLinks.gettingStarted and featuredLinks.popular %} + {% include featured-links %} + {% endif %} + + {% include generic-toc %} +
    +
    +
    +
    + {% include support-section %} + {% include small-footer %} + {% include scroll-button %} +
    + + \ No newline at end of file diff --git a/layouts/product-landing.html b/layouts/product-landing.html index 6d0cf95a1334..8f919276e957 100644 --- a/layouts/product-landing.html +++ b/layouts/product-landing.html @@ -138,11 +138,22 @@

    Guides

    {% endif %}
    + {% if FEATURE_NEW_SITETREE %} + {% if page.documentType == "category" %} + {% include category-articles-list %} + {% endif %} + {% if page.documentType == "product" %} + {% include product-articles-list %} + {% endif %} + {% endif %} + + {% unless FEATURE_NEW_SITETREE %} {% if currentCategory %} {% include all-articles-category %} {% else %} {% include all-articles-product %} {% endif %} + {% endunless %}
    {% include support-section %} From 974b3bb148ba52f2c9eaf9263280956c4d0cde00 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Thu, 15 Apr 2021 14:36:16 -0400 Subject: [PATCH 02/29] add new middleware contextualizers that are feature flagged --- middleware/contextualizers/breadcrumbs.js | 38 +++++++++++++ .../contextualizers/current-product-tree.js | 10 ++++ .../early-access-breadcrumbs.js | 55 +++++++++++++++++++ middleware/contextualizers/generic-toc.js | 45 +++++++++++++++ 4 files changed, 148 insertions(+) create mode 100644 middleware/contextualizers/breadcrumbs.js create mode 100644 middleware/contextualizers/current-product-tree.js create mode 100644 middleware/contextualizers/early-access-breadcrumbs.js create mode 100644 middleware/contextualizers/generic-toc.js diff --git a/middleware/contextualizers/breadcrumbs.js b/middleware/contextualizers/breadcrumbs.js new file mode 100644 index 000000000000..e397ae80face --- /dev/null +++ b/middleware/contextualizers/breadcrumbs.js @@ -0,0 +1,38 @@ +module.exports = async function breadcrumbs (req, res, next) { + if (!req.context.page) return next() + if (req.context.page.hidden) return next() + + req.context.breadcrumbs = [] + + // Return an empty array on the landing page. + if (req.context.page.documentType === 'homepage') { + return next() + } + + const currentSiteTree = req.context.siteTree[req.context.currentLanguage][req.context.currentVersion] + + await createBreadcrumb( + // Array of child pages on the root, i.e., the product level. + currentSiteTree.childPages, + req.context + ) + + return next() +} + +async function createBreadcrumb (pageArray, context) { + // Find each page in the siteTree's array of child pages that starts with the requested path. + const childPage = pageArray.find(page => context.currentPath.startsWith(page.href)) + + context.breadcrumbs.push({ + documentType: childPage.page.documentType, + href: childPage.href, + title: await childPage.page.renderTitle(context, { textOnly: true, encodeEntities: true }) + }) + + // Recursively loop through the siteTree and create each breadcrumb, until we reach the + // point where the current siteTree page is the same as the requested page. Then stop. + if (childPage.childPages && context.currentPath !== childPage.href) { + createBreadcrumb(childPage.childPages, context) + } +} diff --git a/middleware/contextualizers/current-product-tree.js b/middleware/contextualizers/current-product-tree.js new file mode 100644 index 000000000000..92d0cb03d628 --- /dev/null +++ b/middleware/contextualizers/current-product-tree.js @@ -0,0 +1,10 @@ +module.exports = function currentProductTree (req, res, next) { + if (!req.context.page) return next() + if (req.context.page.documentType === 'homepage') return next() + + const currentSiteTree = req.context.siteTree[req.context.currentLanguage][req.context.currentVersion] + + req.context.currentProductTree = currentSiteTree.childPages.find(page => req.context.currentPath.startsWith(page.href)) + + return next() +} diff --git a/middleware/contextualizers/early-access-breadcrumbs.js b/middleware/contextualizers/early-access-breadcrumbs.js new file mode 100644 index 000000000000..21f1a0aee8d9 --- /dev/null +++ b/middleware/contextualizers/early-access-breadcrumbs.js @@ -0,0 +1,55 @@ +module.exports = async function breadcrumbs (req, res, next) { + if (!req.context.page) return next() + if (!req.context.page.hidden) return next() + + req.context.breadcrumbs = [] + + // Return an empty array on the landing page. + if (req.context.page.documentType === 'homepage') { + return next() + } + + const earlyAccessProduct = req.context.siteTree[req.language][req.context.currentVersion].childPages.find(childPage => childPage.page.relativePath === 'early-access/index.md') + if (!earlyAccessProduct) return next() + + // Create initial landing page breadcrumb + req.context.breadcrumbs.push({ + documentType: earlyAccessProduct.page.documentType, + href: '', + title: earlyAccessProduct.page.title + }) + + // If this is the Early Access landing page, return now + if (req.context.currentPath === earlyAccessProduct.href) { + return next() + } + + // Otherwise, create breadcrumbs + await createBreadcrumb( + earlyAccessProduct.childPages, + req.context + ) + + return next() +} + +async function createBreadcrumb (pageArray, context) { + // Find each page in the siteTree's array of child pages that starts with the requested path. + const childPage = pageArray.find(page => context.currentPath.startsWith(page.href)) + + // Gray out product breadcrumb links and `Articles` categories + const hideHref = childPage.page.documentType === 'product' || + (childPage.page.documentType === 'category' && childPage.page.relativePath.endsWith('/articles/index.md')) + + context.breadcrumbs.push({ + documentType: childPage.page.documentType, + href: hideHref ? '' : childPage.href, + title: await childPage.page.renderTitle(context, { textOnly: true, encodeEntities: true }) + }) + + // Recursively loop through the siteTree and create each breadcrumb, until we reach the + // point where the current siteTree page is the same as the requested page. Then stop. + if (childPage.childPages && context.currentPath !== childPage.href) { + createBreadcrumb(childPage.childPages, context) + } +} diff --git a/middleware/contextualizers/generic-toc.js b/middleware/contextualizers/generic-toc.js new file mode 100644 index 000000000000..1db71197606d --- /dev/null +++ b/middleware/contextualizers/generic-toc.js @@ -0,0 +1,45 @@ +const { sortBy } = require('lodash') + +module.exports = async function genericToc (req, res, next) { + if (!req.context.page) return next() + if (req.context.page.hidden) return next() + if (req.context.currentLayoutName !== 'generic-toc') return next() + + const currentSiteTree = req.context.siteTree[req.context.currentLanguage][req.context.currentVersion] + + // Find the array of child pages that start with the requested path. + const currentPageInSiteTree = findPageInSiteTree(currentSiteTree.childPages, req.context.currentPath) + + const unsortedTocItems = await Promise.all(currentPageInSiteTree.childPages.map(async (childPage) => { + // return an empty string if it's a hidden link on a non-hidden page (hidden links on hidden pages are OK) + if (childPage.page.hidden && !req.context.page.hidden) { + return '' + } + + const fullPath = childPage.href + const title = await childPage.page.renderTitle(req.context, { textOnly: true, encodeEntities: true }) + const intro = await childPage.page.renderProp('intro', req.context, { unwrap: true }) + + return { fullPath, title, intro } + })) + + req.context.tocItems = sortBy( + unsortedTocItems, + // Sort by the ordered array of `children` in the frontmatter. + currentPageInSiteTree.page.children + ) + + return next() +} + +// Recursively loop through the siteTree until we reach the point where the +// current siteTree page is the same as the requested page. Then stop. +function findPageInSiteTree (pageArray, currentPath) { + const childPage = pageArray.find(page => currentPath.startsWith(page.href)) + + if (childPage.href === currentPath) { + return childPage + } + + return findPageInSiteTree(childPage.childPages, currentPath) +} From eb44f9947d877dcad1a25c5ae737967e09b2b799 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Thu, 15 Apr 2021 14:36:36 -0400 Subject: [PATCH 03/29] add new content migration script for early access updates --- .../add-early-access-tocs.js | 44 +++++++++++++++++++ .../content-migrations/remove-map-topics.js | 4 +- 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100755 script/content-migrations/add-early-access-tocs.js diff --git a/script/content-migrations/add-early-access-tocs.js b/script/content-migrations/add-early-access-tocs.js new file mode 100755 index 000000000000..34d936f50e86 --- /dev/null +++ b/script/content-migrations/add-early-access-tocs.js @@ -0,0 +1,44 @@ +#!/usr/bin/env node + +const fs = require('fs') +const path = require('path') +const readFrontmatter = require('../../lib/read-frontmatter') +const earlyAccessDir = path.posix.join(process.cwd(), 'content', 'early-access') +const { sentenceCase } = require('change-case') + +updateOrCreateToc(earlyAccessDir) + +console.log('Updated Early Access TOCs!') + +function updateOrCreateToc (directory) { + const children = fs.readdirSync(directory) + .filter(subpath => !subpath.endsWith('index.md')) + + if (!children.length) return + + const tocFile = path.posix.join(directory, 'index.md') + + let content, data + + if (fs.existsSync(tocFile)) { + const matter = readFrontmatter(fs.readFileSync(tocFile, 'utf8')) + content = matter.content + data = matter.data + } else { + content = '' + data = { + title: sentenceCase(path.basename(directory)), + versions: '*', + hidden: true + } + } + + data.children = children.map(child => `/${child.replace('.md', '')}`) + const newContents = readFrontmatter.stringify(content, data, { lineWidth: 10000 }) + fs.writeFileSync(tocFile, newContents) + + children.forEach(child => { + if (child.endsWith('.md')) return + updateOrCreateToc(path.posix.join(directory, child)) + }) +} diff --git a/script/content-migrations/remove-map-topics.js b/script/content-migrations/remove-map-topics.js index 7da04a8e4dac..b19b69470309 100755 --- a/script/content-migrations/remove-map-topics.js +++ b/script/content-migrations/remove-map-topics.js @@ -79,7 +79,9 @@ categoryIndexFiles.forEach(categoryIndexFile => { // Read the article file so we can add a redirect from its old path const articleContents = frontmatter(fs.readFileSync(newArticlePath, 'utf8')) - addRedirectToFrontmatter(articleContents.data.redirect_from, `${oldTopicDirectory}/${article}`) + + if (!articleContents.data.redirect_from) articleContents.data.redirect_from = [] + addRedirectToFrontmatter(articleContents.data.redirect_from, `${oldTopicDirectory.replace(/^.*?\/content\//, '/')}/${article}`) // Write the article with updated frontmatter fs.writeFileSync(newArticlePath, frontmatter.stringify(articleContents.content.trim(), articleContents.data, { lineWidth: 10000 })) From 2a12a3c5994be7521ecc8e9d6b268d2d9a49a49d Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Thu, 15 Apr 2021 14:37:43 -0400 Subject: [PATCH 04/29] go back to ignoring translated paths that do not exist because we cannot initialize a fake translated page using the English base path --- lib/create-tree.js | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/lib/create-tree.js b/lib/create-tree.js index 8649b8283eab..fe89e52eb41b 100644 --- a/lib/create-tree.js +++ b/lib/create-tree.js @@ -20,7 +20,7 @@ module.exports = async function createTree (originalPath, langObj) { const localizedBasePath = path.posix.join(__dirname, '..', langObj.dir, 'content') // Initialize the Page! This is where the file reads happen. - let page = await Page.init({ + const page = await Page.init({ basePath: localizedBasePath, relativePath, languageCode: langObj.code @@ -29,17 +29,10 @@ module.exports = async function createTree (originalPath, langObj) { if (!page) { // Do not throw an error if Early Access is not available. if (relativePath.startsWith('early-access')) return - // If a translated path doesn't exist, fall back to the English so there is parity between - // the English tree and the translated trees. - if (langObj.code !== 'en') { - page = await Page.init({ - basePath: basePath, - relativePath, - languageCode: langObj.code - }) - } - - if (!page) throw Error(`Cannot initialize page for ${filepath}`) + // Do not throw an error if translated page is not available. + if (langObj.code !== 'en') return + + throw Error(`Cannot initialize page for ${filepath} in ${langObj.code}`) } // Create the root tree object on the first run, and create children recursively. From 2f7bfc15bbeb3a7eba33ee2d1c143da1b8e87988 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Thu, 15 Apr 2021 14:38:38 -0400 Subject: [PATCH 05/29] some lib updates --- lib/get-document-type.js | 37 ++++++++++++++------ lib/page.js | 4 +++ lib/redirects/add-redirect-to-frontmatter.js | 2 +- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/lib/get-document-type.js b/lib/get-document-type.js index 8c3fab956ed1..11cebd7ca226 100644 --- a/lib/get-document-type.js +++ b/lib/get-document-type.js @@ -1,17 +1,34 @@ +// This function derives the document type from the *relative path* segment length, +// where a relative path refers to the content path starting with the product dir. +// For example: actions/index.md or github/getting-started-with-github/quickstart.md. module.exports = function getDocumentType (relativePath) { + // A non-index file is ALWAYS considered an article in this approach, + // even if it's at the category level (like actions/quickstart.md) if (!relativePath.endsWith('index.md')) { return 'article' } - // Derive the document type from the path segment length - switch (relativePath.split('/').length) { - case 1: - return 'homepage' - case 2: - return 'product' - case 3: - return 'category' - case 4: - return 'mapTopic' + const segmentLength = relativePath.split('/').length + + // Early Access has an extra tree segment, so it has a different number of segments. + const isEarlyAccess = relativePath.startsWith('early-access') + + const publicDocs = { + 1: 'homepage', + 2: 'product', + 3: 'category', + 4: 'mapTopic' } + + const earlyAccessDocs = { + 1: 'homepage', + 2: 'early-access', + 3: 'product', + 4: 'category', + 5: 'mapTopic' + } + + return isEarlyAccess + ? earlyAccessDocs[segmentLength] + : publicDocs[segmentLength] } diff --git a/lib/page.js b/lib/page.js index 1254b83b732b..68d3d058a7c2 100644 --- a/lib/page.js +++ b/lib/page.js @@ -18,6 +18,7 @@ const statsd = require('./statsd') const readFileContents = require('./read-file-contents') const getLinkData = require('./get-link-data') const union = require('lodash/union') +const getDocumentType = require('./get-document-type') class Page { static async init (opts) { @@ -76,6 +77,9 @@ class Page { this.introLinks.rawOverview = this.introLinks.overview } + // Is this the Homepage or a Product, Category, Topic, or Article? + this.documentType = getDocumentType(this.relativePath) + // Get array of versions that the page is available in for fast lookup this.applicableVersions = getApplicableVersions(this.versions, this.fullPath) diff --git a/lib/redirects/add-redirect-to-frontmatter.js b/lib/redirects/add-redirect-to-frontmatter.js index 3e2d26a4c980..d392eed5ecc7 100644 --- a/lib/redirects/add-redirect-to-frontmatter.js +++ b/lib/redirects/add-redirect-to-frontmatter.js @@ -1,7 +1,7 @@ // add a new redirect string to redirect_from frontmatter module.exports = function addRedirectToFrontmatter (redirectFromData, newRedirectString) { - if (Array.isArray(redirectFromData)) { + if (Array.isArray(redirectFromData) && !redirectFromData.includes(newRedirectString)) { redirectFromData.push(newRedirectString) } else if (typeof redirectFromData === 'string') { redirectFromData = [redirectFromData] From d1524b16a5b1f5b0b240c4a5cef54bdf94a1757b Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Thu, 15 Apr 2021 14:46:16 -0400 Subject: [PATCH 06/29] add feature flagged middleware --- middleware/context.js | 4 ++-- middleware/contextualizers/early-access-links.js | 2 +- middleware/contextualizers/layout.js | 15 ++++++++++++--- middleware/index.js | 15 +++++++++++++-- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/middleware/context.js b/middleware/context.js index 58c463d6857d..3bb64073aa47 100644 --- a/middleware/context.js +++ b/middleware/context.js @@ -10,7 +10,7 @@ const { getPathWithoutLanguage } = require('../lib/path-utils') const productNames = require('../lib/product-names') -const warmServer = require('../lib/warm-server') +const warmServer = process.env.FEATURE_NEW_SITETREE ? require('../lib/warm-server2') : require('../lib/warm-server') const featureFlags = Object.keys(require('../feature-flags')) const builtAssets = require('../lib/built-asset-urls') const searchVersions = require('../lib/search/versions') @@ -27,7 +27,7 @@ module.exports = async function contextualize (req, res, next) { // make feature flag environment variables accessible in layouts req.context.process = { env: {} } featureFlags.forEach(featureFlagName => { - req.context.process.env[featureFlagName] = process.env[featureFlagName] + req.context[featureFlagName] = process.env[featureFlagName] }) // define each context property explicitly for code-search friendliness diff --git a/middleware/contextualizers/early-access-links.js b/middleware/contextualizers/early-access-links.js index f09c68f1d9d9..b30090a0731a 100644 --- a/middleware/contextualizers/early-access-links.js +++ b/middleware/contextualizers/early-access-links.js @@ -7,7 +7,7 @@ module.exports = function earlyAccessContext (req, res, next) { // Get a list of all hidden pages per version const earlyAccessPageLinks = uniq(Object.values(req.context.pages) - .filter(page => page.hidden && page.relativePath.startsWith('early-access') && page.relativePath !== 'early-access/index.md') + .filter(page => page.hidden && page.relativePath.startsWith('early-access') && !page.relativePath.endsWith('index.md')) .map(page => page.permalinks) .flat()) // Get links for the current version diff --git a/middleware/contextualizers/layout.js b/middleware/contextualizers/layout.js index 54c40540a459..e289604405fa 100644 --- a/middleware/contextualizers/layout.js +++ b/middleware/contextualizers/layout.js @@ -5,16 +5,25 @@ module.exports = function layoutContext (req, res, next) { let layoutName + // If this is an index.md that is not the homepage and does not have a defined layout, use `generic-toc`. + const usesGenericToc = req.context.page.relativePath.endsWith('index.md') && + req.context.page.documentType !== 'homepage' && + !req.context.page.hidden + if (req.context.page.layout) { // Layouts can be specified with a `layout` frontmatter value. // Any invalid layout values will be caught by frontmatter schema validation. layoutName = req.context.page.layout - // A `layout: false` value means use no layout. } else if (req.context.page.layout === false) { + // A `layout: false` value means use no layout. layoutName = '' - // If undefined, use either the default layout or the generic-toc layout. } else if (req.context.page.layout === undefined) { - layoutName = 'default' + // For all other files (like articles and the homepage), use the `default` layout. + if (process.env.FEATURE_NEW_SITETREE) { + layoutName = usesGenericToc ? 'generic-toc' : 'default' + } else { + layoutName = 'default' + } } // Attach to the context object diff --git a/middleware/index.js b/middleware/index.js index d942bba41a72..2d9728c36808 100644 --- a/middleware/index.js +++ b/middleware/index.js @@ -111,8 +111,19 @@ module.exports = function (app) { app.use(instrument('./contextualizers/webhooks')) app.use(asyncMiddleware(instrument('./contextualizers/whats-new-changelog'))) app.use(instrument('./contextualizers/layout')) - app.use(asyncMiddleware(instrument('./breadcrumbs'))) - app.use(asyncMiddleware(instrument('./early-access-breadcrumbs'))) + + if (!process.env.FEATURE_NEW_SITETREE) { + app.use(asyncMiddleware(instrument('./breadcrumbs'))) + app.use(asyncMiddleware(instrument('./early-access-breadcrumbs'))) + } + + if (process.env.FEATURE_NEW_SITETREE) { + app.use(instrument('./contextualizers/current-product-tree')) + app.use(asyncMiddleware(instrument('./contextualizers/generic-toc'))) + app.use(asyncMiddleware(instrument('./contextualizers/breadcrumbs'))) + app.use(asyncMiddleware(instrument('./contextualizers/early-access-breadcrumbs'))) + } + app.use(asyncMiddleware(instrument('./enterprise-server-releases'))) app.use(asyncMiddleware(instrument('./dev-toc'))) app.use(asyncMiddleware(instrument('./featured-links'))) From 201b7958b4b86acfa1726a9ce3ec7768e3031504 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Thu, 15 Apr 2021 14:57:37 -0400 Subject: [PATCH 07/29] add FALSE feature flag --- feature-flags.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/feature-flags.json b/feature-flags.json index 64c69f19d789..5a00273700b9 100644 --- a/feature-flags.json +++ b/feature-flags.json @@ -1,4 +1,5 @@ { "FEATURE_TEST_TRUE": true, - "FEATURE_TEST_FALSE": false + "FEATURE_TEST_FALSE": false, + "FEATURE_NEW_SITETREE": false } From e5b1ccf6fda3324eed7a8739d284b6ebd44ce95e Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Mon, 19 Apr 2021 12:17:40 -0400 Subject: [PATCH 08/29] remove redundant require --- lib/page.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/page.js b/lib/page.js index 893af02a30c4..66939414d7c9 100644 --- a/lib/page.js +++ b/lib/page.js @@ -19,7 +19,6 @@ const readFileContents = require('./read-file-contents') const getLinkData = require('./get-link-data') const getDocumentType = require('./get-document-type') const union = require('lodash/union') -const getDocumentType = require('./get-document-type') class Page { static async init (opts) { From 05f20c26f12e9e718facf514651e24b7458e13f7 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Mon, 19 Apr 2021 15:26:13 -0400 Subject: [PATCH 09/29] use color-border-info now that purple is no longer part of primer --- layouts/generic-toc.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/layouts/generic-toc.html b/layouts/generic-toc.html index e4bfbe13434c..cd3b3a2d2b22 100644 --- a/layouts/generic-toc.html +++ b/layouts/generic-toc.html @@ -35,7 +35,7 @@

    {{ page.title }}

    {% endif %} {% if page.product %} -
    +
    {{ page.product }}
    {% endif %} From 5d035375518e551feb509de7ee8d52369e69f1a2 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Tue, 20 Apr 2021 11:15:03 -0400 Subject: [PATCH 10/29] differentiate between TOCs with grandchildren vs. only children --- includes/generic-toc-items.html | 26 +++++++++++++++ includes/generic-toc.html | 8 ----- layouts/generic-toc.html | 4 +-- middleware/contextualizers/generic-toc.js | 40 +++++++++++++++-------- 4 files changed, 54 insertions(+), 24 deletions(-) create mode 100644 includes/generic-toc-items.html delete mode 100644 includes/generic-toc.html diff --git a/includes/generic-toc-items.html b/includes/generic-toc-items.html new file mode 100644 index 000000000000..4c923951524b --- /dev/null +++ b/includes/generic-toc-items.html @@ -0,0 +1,26 @@ +{% if tocItems %} + +{% for tocItem in tocItems %} +{% assign title = tocItem.title %} +{% assign fullPath = tocItem.fullPath %} +{% assign intro = tocItem.intro %} + +{% if tocItem.childTocItems %} +
      +
    • {% include liquid-tags/link %} +
        + {% for childItem in tocItem.childTocItems %} + {% assign title = childItem.title %} + {% assign fullPath = childItem.fullPath %} +
      • {% include liquid-tags/link %}
        • + {% endfor %} +
      +
      • +
    +{% else %} + +{% include liquid-tags/link-with-intro %} + +{% endif %} +{% endfor %} +{% endif %} \ No newline at end of file diff --git a/includes/generic-toc.html b/includes/generic-toc.html deleted file mode 100644 index 0e9820dcb24d..000000000000 --- a/includes/generic-toc.html +++ /dev/null @@ -1,8 +0,0 @@ -{% for tocItem in tocItems %} - - -

    {{ tocItem.title }}→

    -     -{% if tocItem.intro %}

    {{ tocItem.intro }}

    {% endif %} - -{% endfor %} \ No newline at end of file diff --git a/layouts/generic-toc.html b/layouts/generic-toc.html index cd3b3a2d2b22..431b20eeaaf1 100644 --- a/layouts/generic-toc.html +++ b/layouts/generic-toc.html @@ -44,8 +44,8 @@

    {{ page.title }}

    {% if featuredLinks.gettingStarted and featuredLinks.popular %} {% include featured-links %} {% endif %} - - {% include generic-toc %} + + {% include generic-toc-items %}
    diff --git a/middleware/contextualizers/generic-toc.js b/middleware/contextualizers/generic-toc.js index 1db71197606d..ec0790ab9560 100644 --- a/middleware/contextualizers/generic-toc.js +++ b/middleware/contextualizers/generic-toc.js @@ -10,21 +10,8 @@ module.exports = async function genericToc (req, res, next) { // Find the array of child pages that start with the requested path. const currentPageInSiteTree = findPageInSiteTree(currentSiteTree.childPages, req.context.currentPath) - const unsortedTocItems = await Promise.all(currentPageInSiteTree.childPages.map(async (childPage) => { - // return an empty string if it's a hidden link on a non-hidden page (hidden links on hidden pages are OK) - if (childPage.page.hidden && !req.context.page.hidden) { - return '' - } - - const fullPath = childPage.href - const title = await childPage.page.renderTitle(req.context, { textOnly: true, encodeEntities: true }) - const intro = await childPage.page.renderProp('intro', req.context, { unwrap: true }) - - return { fullPath, title, intro } - })) - req.context.tocItems = sortBy( - unsortedTocItems, + await getUnsortedTocItems(currentPageInSiteTree.childPages, req.context), // Sort by the ordered array of `children` in the frontmatter. currentPageInSiteTree.page.children ) @@ -43,3 +30,28 @@ function findPageInSiteTree (pageArray, currentPath) { return findPageInSiteTree(childPage.childPages, currentPath) } + +async function getUnsortedTocItems (pageArray, context) { + return Promise.all(pageArray.map(async (childPage) => { + // return an empty string if it's a hidden link on a non-hidden page (hidden links on hidden pages are OK) + if (childPage.page.hidden && !context.page.hidden) { + return '' + } + + const fullPath = childPage.href + const title = await childPage.page.renderTitle(context, { textOnly: true, encodeEntities: true }) + const intro = await childPage.page.renderProp('intro', context, { unwrap: true }) + + if (!childPage.childPages) { + return { fullPath, title, intro } + } + + const childTocItems = sortBy( + await getUnsortedTocItems(childPage.childPages, context), + // Sort by the ordered array of `children` in the frontmatter. + childPage.page.children + ) + + return { fullPath, title, intro, childTocItems } + })) +} From 27f044821912e67ce197e47740be1aad3e5846de Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Tue, 20 Apr 2021 11:25:22 -0400 Subject: [PATCH 11/29] preserve hardcoded markdown content in TOC files --- layouts/generic-toc.html | 2 ++ script/content-migrations/update-tocs.js | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/layouts/generic-toc.html b/layouts/generic-toc.html index 431b20eeaaf1..7ff9de2bb1d7 100644 --- a/layouts/generic-toc.html +++ b/layouts/generic-toc.html @@ -45,6 +45,8 @@

    {{ page.title }}

    {% include featured-links %} {% endif %} + {{ renderedPage }} + {% include generic-toc-items %} diff --git a/script/content-migrations/update-tocs.js b/script/content-migrations/update-tocs.js index 0a6a012166eb..34e90156376a 100755 --- a/script/content-migrations/update-tocs.js +++ b/script/content-migrations/update-tocs.js @@ -73,7 +73,7 @@ indexFiles } // Index files should no longer have body content, so we write an empty string - fs.writeFileSync(indexFile, frontmatter.stringify('', data, { lineWidth: 10000 })) + fs.writeFileSync(indexFile, frontmatter.stringify(content, data, { lineWidth: 10000 })) }) function getLinks (linkItemArray) { From c3f4dcb36828044ff3815db487d421d1c66ba547 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Tue, 20 Apr 2021 12:02:57 -0400 Subject: [PATCH 12/29] remove hardcoded TOC links from index files but leave other content --- script/content-migrations/update-tocs.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/script/content-migrations/update-tocs.js b/script/content-migrations/update-tocs.js index 34e90156376a..5b8e59dd395b 100755 --- a/script/content-migrations/update-tocs.js +++ b/script/content-migrations/update-tocs.js @@ -72,8 +72,11 @@ indexFiles ] } + // Remove the Table of Contents section and leave any custom content before it. + const newContent = content.replace(/^#*? Table of contents[\s\S]*/im, '') + // Index files should no longer have body content, so we write an empty string - fs.writeFileSync(indexFile, frontmatter.stringify(content, data, { lineWidth: 10000 })) + fs.writeFileSync(indexFile, frontmatter.stringify(newContent, data, { lineWidth: 10000 })) }) function getLinks (linkItemArray) { From ebc2c110132e0dc02bbbb94de8b14ca3ca60aae6 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Tue, 20 Apr 2021 12:49:37 -0400 Subject: [PATCH 13/29] only show the top-level TOC on product landing pages that use generic-toc --- includes/generic-toc-items.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/includes/generic-toc-items.html b/includes/generic-toc-items.html index 4c923951524b..cc914167e324 100644 --- a/includes/generic-toc-items.html +++ b/includes/generic-toc-items.html @@ -5,7 +5,8 @@ {% assign fullPath = tocItem.fullPath %} {% assign intro = tocItem.intro %} -{% if tocItem.childTocItems %} +{% if tocItem.childTocItems and page.documentType != "product" %} +
    • {% include liquid-tags/link %}
        @@ -17,6 +18,7 @@
    + {% else %} {% include liquid-tags/link-with-intro %} From 5e2a1b51c8d74ebb7577270c02bc3ee74b038506 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Tue, 20 Apr 2021 12:49:57 -0400 Subject: [PATCH 14/29] update scripts to fix a few bugs --- script/content-migrations/remove-map-topics.js | 5 +++++ script/content-migrations/update-tocs.js | 10 ++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/script/content-migrations/remove-map-topics.js b/script/content-migrations/remove-map-topics.js index b19b69470309..ec24064da777 100755 --- a/script/content-migrations/remove-map-topics.js +++ b/script/content-migrations/remove-map-topics.js @@ -3,6 +3,7 @@ const fs = require('fs') const path = require('path') const walk = require('walk-sync') +const stripHtmlComments = require('strip-html-comments') const languages = require('../../lib/languages') const frontmatter = require('../../lib/read-frontmatter') const addRedirectToFrontmatter = require('../../lib/redirects/add-redirect-to-frontmatter') @@ -26,6 +27,10 @@ const categoryIndexFiles = fullDirectoryPaths.map(fullDirectoryPath => walk(full categoryIndexFiles.forEach(categoryIndexFile => { let categoryIndexContent = fs.readFileSync(categoryIndexFile, 'utf8') + if (categoryIndexFile.endsWith('github/getting-started-with-github/index.md')) { + categoryIndexContent = stripHtmlComments(categoryIndexContent.replace(/\n$/m.test(line)) + + newContent = newLinesArray.join('\n') // Index files should no longer have body content, so we write an empty string fs.writeFileSync(indexFile, frontmatter.stringify(newContent, data, { lineWidth: 10000 })) From 88cbc97798dbfc1fa0e200ef37f259130b3fd076 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Tue, 20 Apr 2021 15:19:42 -0400 Subject: [PATCH 18/29] split up generic TOC include depending on whether there are children or not --- includes/generic-toc-items.html | 20 ++------------------ includes/generic-toc-list.html | 24 ++++++++++++++++++++++++ layouts/generic-toc.html | 6 +++++- 3 files changed, 31 insertions(+), 19 deletions(-) create mode 100644 includes/generic-toc-list.html diff --git a/includes/generic-toc-items.html b/includes/generic-toc-items.html index cc914167e324..f0b6309882a5 100644 --- a/includes/generic-toc-items.html +++ b/includes/generic-toc-items.html @@ -1,28 +1,12 @@ {% if tocItems %} {% for tocItem in tocItems %} + {% assign title = tocItem.title %} {% assign fullPath = tocItem.fullPath %} {% assign intro = tocItem.intro %} - -{% if tocItem.childTocItems and page.documentType != "product" %} - -
      -
    • {% include liquid-tags/link %} -
        - {% for childItem in tocItem.childTocItems %} - {% assign title = childItem.title %} - {% assign fullPath = childItem.fullPath %} -
      • {% include liquid-tags/link %}
        • - {% endfor %} -
      -
      • -
    - -{% else %} - {% include liquid-tags/link-with-intro %} -{% endif %} {% endfor %} + {% endif %} \ No newline at end of file diff --git a/includes/generic-toc-list.html b/includes/generic-toc-list.html new file mode 100644 index 000000000000..5dee0a5f6844 --- /dev/null +++ b/includes/generic-toc-list.html @@ -0,0 +1,24 @@ +{% if tocItems %} +
      +{% for tocItem in tocItems %} + +{% assign title = tocItem.title %} +{% assign fullPath = tocItem.fullPath %} +{% assign intro = tocItem.intro %} + +
    • {% include liquid-tags/link %} + {% if tocItem.childTocItems %} + {% unless page.relativePath == "github/index.md" %} +
        + {% for childItem in tocItem.childTocItems %} + {% assign title = childItem.title %} + {% assign fullPath = childItem.fullPath %} +
      • {% include liquid-tags/link %}
        • + {% endfor %} +
      + {% endunless %} + {% endif %} +
      • +{% endfor %} +
    +{% endif %} \ No newline at end of file diff --git a/layouts/generic-toc.html b/layouts/generic-toc.html index 7ff9de2bb1d7..c51fae09dd12 100644 --- a/layouts/generic-toc.html +++ b/layouts/generic-toc.html @@ -47,7 +47,11 @@

    {{ page.title }}

    {{ renderedPage }} - {% include generic-toc-items %} + {% if page.documentType == "category" or page.relativePath == "github/index.md" %} + {% include generic-toc-list %} + {% else %} + {% include generic-toc-items %} + {% endif %} From a22691b08214232e0f61e83c8131534fe6fe3e5c Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Wed, 21 Apr 2021 10:45:37 -0400 Subject: [PATCH 19/29] update middleware/categories-for-support --- middleware/categories-for-support.js | 55 ++++++++++++++++++++++++++++ middleware/index.js | 7 +++- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 middleware/categories-for-support.js diff --git a/middleware/categories-for-support.js b/middleware/categories-for-support.js new file mode 100644 index 000000000000..b577d012b533 --- /dev/null +++ b/middleware/categories-for-support.js @@ -0,0 +1,55 @@ +const path = require('path') +const renderOpts = { textOnly: true, encodeEntities: true } + +// This middleware exposes a list of all categories and child articles at /categories.json. +// GitHub Support uses this for internal ZenDesk search functionality. +module.exports = async function categoriesForSupport (req, res, next) { + const englishSiteTree = req.context.siteTree.en + + const allCategories = [] + + await Promise.all(Object.keys(englishSiteTree).map(async (version) => { + await Promise.all(englishSiteTree[version].childPages.map(async (productPage) => { + if (productPage.page.relativePath.startsWith('early-access')) return + if (!productPage.childPages) return + + await Promise.all(productPage.childPages.map(async (categoryPage) => { + // We can't get the rendered titles from middleware/render-tree-titles + // here because that middleware only runs on the current version, and this + // middleware processes all versions. + const name = categoryPage.page.title.includes('{') + ? await categoryPage.page.renderProp('title', req.context, renderOpts) + : categoryPage.page.title + + allCategories.push({ + name, + published_articles: await findArticlesPerCategory(categoryPage, [], req.context) + }) + })) + })) + })) + + return res.json(allCategories) +} + +async function findArticlesPerCategory (currentPage, articlesArray, context) { + if (currentPage.page.documentType === 'article') { + const title = currentPage.page.title.includes('{') + ? await currentPage.page.renderProp('title', context, renderOpts) + : currentPage.page.title + + articlesArray.push({ + title, + slug: path.basename(currentPage.href) + }) + } + + if (!currentPage.childPages) return articlesArray + + // Run recursively to find any articles deeper in the tree. + await Promise.all(currentPage.childPages.map(async (childPage) => { + await findArticlesPerCategory(childPage, articlesArray, context) + })) + + return articlesArray +} diff --git a/middleware/index.js b/middleware/index.js index 773f906b4055..cb70213a790f 100644 --- a/middleware/index.js +++ b/middleware/index.js @@ -97,7 +97,12 @@ module.exports = function (app) { app.use(asyncMiddleware(instrument('./archived-enterprise-versions'))) app.use(instrument('./robots')) app.use(/(\/.*)?\/early-access$/, instrument('./contextualizers/early-access-links')) - app.use('/categories.json', asyncMiddleware(instrument('./categories-for-support-team'))) + if (!process.env.FEATURE_NEW_SITETREE) { + app.use('/categories.json', asyncMiddleware(instrument('./categories-for-support-team'))) + } + if (process.env.FEATURE_NEW_SITETREE) { + app.use('/categories.json', asyncMiddleware(instrument('./categories-for-support'))) + } app.use(instrument('./loaderio-verification')) app.get('/_500', asyncMiddleware(instrument('./trigger-error'))) From af519b51316f73a9dca9338066c9d19e35109780 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Wed, 21 Apr 2021 13:05:52 -0400 Subject: [PATCH 20/29] use article layout for TOCs instead of new separate generic-toc layout that has to be maintained --- includes/article.html | 8 +++ layouts/generic-toc.html | 64 ----------------------- middleware/contextualizers/generic-toc.js | 3 +- middleware/contextualizers/layout.js | 23 +++----- 4 files changed, 16 insertions(+), 82 deletions(-) delete mode 100644 layouts/generic-toc.html diff --git a/includes/article.html b/includes/article.html index ba58d0f82263..d4b408324025 100644 --- a/includes/article.html +++ b/includes/article.html @@ -68,6 +68,14 @@

    diff --git a/layouts/generic-toc.html b/layouts/generic-toc.html deleted file mode 100644 index c51fae09dd12..000000000000 --- a/layouts/generic-toc.html +++ /dev/null @@ -1,64 +0,0 @@ - - - {% include head %} - - - {% include sidebar %} - -
    - {% include header %} - {% include deprecation-banner %} -
    -
    -
    -
    {% include article-version-switcher %}
    -
    - {% include breadcrumbs %} -
    -
    {% include article-version-switcher %}
    -
    - -
    - -
    -
    -

    {{ page.title }}

    -
    - -
    -
    - - {% if page.intro %} -
    {{ page.intro }}
    - {% endif %} - - {% if page.product %} -
    - {{ page.product }} -
    - {% endif %} -
    -
    - {% if featuredLinks.gettingStarted and featuredLinks.popular %} - {% include featured-links %} - {% endif %} - - {{ renderedPage }} - - {% if page.documentType == "category" or page.relativePath == "github/index.md" %} - {% include generic-toc-list %} - {% else %} - {% include generic-toc-items %} - {% endif %} -
    -
    -
    -
    - {% include support-section %} - {% include small-footer %} - {% include scroll-button %} -
    - - \ No newline at end of file diff --git a/middleware/contextualizers/generic-toc.js b/middleware/contextualizers/generic-toc.js index 606778d88fa0..9ebcee7d089c 100644 --- a/middleware/contextualizers/generic-toc.js +++ b/middleware/contextualizers/generic-toc.js @@ -3,7 +3,8 @@ const { sortBy } = require('lodash') module.exports = async function genericToc (req, res, next) { if (!req.context.page) return next() if (req.context.page.hidden) return next() - if (req.context.currentLayoutName !== 'generic-toc') return next() + if (req.context.currentLayoutName !== 'default') return next() + if (req.context.page.documentType === 'homepage' || req.context.page.documentType === 'article') return next() const currentSiteTree = req.context.siteTree[req.context.currentLanguage][req.context.currentVersion] diff --git a/middleware/contextualizers/layout.js b/middleware/contextualizers/layout.js index e289604405fa..7a6051c2eb4f 100644 --- a/middleware/contextualizers/layout.js +++ b/middleware/contextualizers/layout.js @@ -3,29 +3,18 @@ const layouts = require('../../lib/layouts') module.exports = function layoutContext (req, res, next) { if (!req.context.page) return next() - let layoutName - - // If this is an index.md that is not the homepage and does not have a defined layout, use `generic-toc`. - const usesGenericToc = req.context.page.relativePath.endsWith('index.md') && - req.context.page.documentType !== 'homepage' && - !req.context.page.hidden - - if (req.context.page.layout) { + const layoutOptsByType = { // Layouts can be specified with a `layout` frontmatter value. // Any invalid layout values will be caught by frontmatter schema validation. - layoutName = req.context.page.layout - } else if (req.context.page.layout === false) { + string: req.context.page.layout, // A `layout: false` value means use no layout. - layoutName = '' - } else if (req.context.page.layout === undefined) { + boolean: '', // For all other files (like articles and the homepage), use the `default` layout. - if (process.env.FEATURE_NEW_SITETREE) { - layoutName = usesGenericToc ? 'generic-toc' : 'default' - } else { - layoutName = 'default' - } + undefined: 'default' } + const layoutName = layoutOptsByType[typeof (req.context.page.layout)] + // Attach to the context object req.context.currentLayoutName = layoutName req.context.currentLayout = layouts[layoutName] From 6b3c17d8c057e6e413482932203b23294141740f Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Fri, 23 Apr 2021 04:14:04 +0300 Subject: [PATCH 21/29] Add article about customizing GitHub Hosted runners (#18761) * add /actions/guides/customizing-github-hosted-runners * fix linters * fix comments * Update customizing-the-github-hosted-runners.md * customizing-the-github-hosted-runners * Update about-github-hosted-runners.md * Added content design updates * Added misc edits * Update customizing-the-github-hosted-runners.md * Small edits to intro * omit the from title * Update customizing-github-hosted-runners.md * Apply suggestions from code review Co-authored-by: Sarah Edwards * Update customizing-github-hosted-runners.md Co-authored-by: Martin Lopes Co-authored-by: skedwards88 --- ...grating-from-circleci-to-github-actions.md | 4 +- .../about-github-hosted-runners.md | 4 + .../customizing-github-hosted-runners.md | 92 +++++++++++++++++++ .../using-github-hosted-runners/index.md | 1 + ...ng-a-new-ssh-key-to-your-github-account.md | 1 + 5 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 content/actions/using-github-hosted-runners/customizing-github-hosted-runners.md diff --git a/content/actions/learn-github-actions/migrating-from-circleci-to-github-actions.md b/content/actions/learn-github-actions/migrating-from-circleci-to-github-actions.md index cbb113098844..f32c255333ba 100644 --- a/content/actions/learn-github-actions/migrating-from-circleci-to-github-actions.md +++ b/content/actions/learn-github-actions/migrating-from-circleci-to-github-actions.md @@ -443,7 +443,9 @@ jobs: path: vendor/bundle key: administrate-${{ matrix.image }}-${{ hashFiles('Gemfile.lock') }} - name: Install postgres headers - run: sudo apt-get install libpq-dev + run: | + sudo apt-get update + sudo apt-get install libpq-dev - name: Install dependencies run: bundle install --path vendor/bundle - name: Setup environment configuration diff --git a/content/actions/using-github-hosted-runners/about-github-hosted-runners.md b/content/actions/using-github-hosted-runners/about-github-hosted-runners.md index e31cad845da0..6eb8734b54c5 100644 --- a/content/actions/using-github-hosted-runners/about-github-hosted-runners.md +++ b/content/actions/using-github-hosted-runners/about-github-hosted-runners.md @@ -93,6 +93,10 @@ We recommend using actions to interact with the software installed on runners. T If there is a tool that you'd like to request, please open an issue at [actions/virtual-environments](https://github.com/actions/virtual-environments). This repository also contains announcements about all major software updates on runners. +#### Installing additional software + +You can install additional software on {% data variables.product.prodname_dotcom %}-hosted runners. For more information, see "[Customizing GitHub-hosted runners](/actions/using-github-hosted-runners/customizing-github-hosted-runners)". + ### IP addresses {% note %} diff --git a/content/actions/using-github-hosted-runners/customizing-github-hosted-runners.md b/content/actions/using-github-hosted-runners/customizing-github-hosted-runners.md new file mode 100644 index 000000000000..789f759f891d --- /dev/null +++ b/content/actions/using-github-hosted-runners/customizing-github-hosted-runners.md @@ -0,0 +1,92 @@ +--- +title: Customizing GitHub-hosted runners +intro: >- + You can install additional software on GitHub-hosted runners as a + part of your workflow. +product: '{% data reusables.gated-features.actions %}' +versions: + free-pro-team: '*' + enterprise-server: '>=2.22' +type: tutorial +topics: + - Workflows +--- + +{% data reusables.actions.enterprise-github-hosted-runners %} + +If you require additional software packages on {% data variables.product.prodname_dotcom %}-hosted runners, you can create a job that installs the packages as part of your workflow. + +To see which packages are already installed by default, see "[Preinstalled software](/actions/using-github-hosted-runners/about-github-hosted-runners#preinstalled-software)." + +This guide demonstrates how to create a job that installs additional software on a {% data variables.product.prodname_dotcom %}-hosted runner. + +### Installing software on Ubuntu runners + +The following example demonstrates how to install an `apt` package as part of a job. + +{% raw %} +```yaml +name: Build on Ubuntu +on: push + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Install jq tool + run: | + sudo apt-get update + sudo apt-get install jq +``` +{% endraw %} + +{% note %} + +**Note:** Always run `sudo apt-get update` before installing a package. In case the `apt` index is stale, this command fetches and re-indexes any available packages, which helps prevent package installation failures. + +{% endnote %} + +### Installing software on macOS runners + +The following example demonstrates how to install Brew packages and casks as part of a job. + +{% raw %} +```yaml +name: Build on macOS +on: push + +jobs: + build: + runs-on: macos-latest + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Install GitHub CLI + run: | + brew update + brew install gh + - name: Install Microsoft Edge + run: | + brew update + brew install --cask microsoft-edge +``` +{% endraw %} + +### Installing software on Windows runners + +The following example demonstrates how to use [Chocolatey](https://community.chocolatey.org/packages) to install the {% data variables.product.prodname_dotcom %} CLI as part of a job. + +{% raw %} +```yaml +name: Build on Windows +on: push +jobs: + build: + runs-on: windows-latest + steps: + - run: choco install gh + - run: gh version +``` +{% endraw %} diff --git a/content/actions/using-github-hosted-runners/index.md b/content/actions/using-github-hosted-runners/index.md index 681269a25393..fb91ee9e6418 100644 --- a/content/actions/using-github-hosted-runners/index.md +++ b/content/actions/using-github-hosted-runners/index.md @@ -11,6 +11,7 @@ versions: {% data reusables.actions.enterprise-github-hosted-runners %} {% link_in_list /about-github-hosted-runners %} +{% link_in_list /customizing-github-hosted-runners %} {% link_in_list /about-ae-hosted-runners %} {% link_in_list /adding-ae-hosted-runners %} {% link_in_list /using-ae-hosted-runners-in-a-workflow %} diff --git a/content/github/authenticating-to-github/adding-a-new-ssh-key-to-your-github-account.md b/content/github/authenticating-to-github/adding-a-new-ssh-key-to-your-github-account.md index 23fb2d23e715..7a1376f8f781 100644 --- a/content/github/authenticating-to-github/adding-a-new-ssh-key-to-your-github-account.md +++ b/content/github/authenticating-to-github/adding-a-new-ssh-key-to-your-github-account.md @@ -87,6 +87,7 @@ After adding a new SSH key to your {% data variables.product.product_name %} acc If your SSH public key file has a different name than the example code, modify the filename to match your current setup. When copying your key, don't add any newlines or whitespace. ```shell + $ sudo apt-get update $ sudo apt-get install xclip # Downloads and installs xclip. If you don't have `apt-get`, you might need to use another installer (like `yum`) From a6a8d92e35838e9484f9e4700d22ed9521083d2d Mon Sep 17 00:00:00 2001 From: Grey Baker Date: Fri, 23 Apr 2021 00:22:38 -0400 Subject: [PATCH 22/29] Add OpenAI as secret scanning partner (#18896) --- .../secret-scanning/partner-secret-list-private-repo.md | 1 + .../reusables/secret-scanning/partner-secret-list-public-repo.md | 1 + 2 files changed, 2 insertions(+) diff --git a/data/reusables/secret-scanning/partner-secret-list-private-repo.md b/data/reusables/secret-scanning/partner-secret-list-private-repo.md index 69cad3e6633b..540ad78b35e2 100644 --- a/data/reusables/secret-scanning/partner-secret-list-private-repo.md +++ b/data/reusables/secret-scanning/partner-secret-list-private-repo.md @@ -54,6 +54,7 @@ Mailchimp | Mailchimp API Key | mailchimp_api_key Mailgun | Mailgun API Key | mailgun_api_key npm | npm Access Token | npm_access_token NuGet | NuGet API Key | nuget_api_key +OpenAI | OpenAI API Key | openai_api_key Palantir | Palantir JSON Web Token | palantir_jwt Postman | Postman API Key | postman_api_key Proctorio | Proctorio Consumer Key | proctorio_consumer_key diff --git a/data/reusables/secret-scanning/partner-secret-list-public-repo.md b/data/reusables/secret-scanning/partner-secret-list-public-repo.md index a3926d6d72cc..a13fab6009b7 100644 --- a/data/reusables/secret-scanning/partner-secret-list-public-repo.md +++ b/data/reusables/secret-scanning/partner-secret-list-public-repo.md @@ -47,6 +47,7 @@ Mailgun | Mailgun API Key MessageBird | MessageBird API Key npm | npm Access Token NuGet | NuGet API Key +OpenAI | OpenAI API Key Palantir | Palantir JSON Web Token Plivo | Plivo Auth Token Postman | Postman API Key From 349338adbf348aa07a99807370ccf4a2c9a67ee9 Mon Sep 17 00:00:00 2001 From: Grey Baker Date: Fri, 23 Apr 2021 00:28:58 -0400 Subject: [PATCH 23/29] Add Fastly to list of private repo partners (#18929) --- .../secret-scanning/partner-secret-list-private-repo.md | 1 + 1 file changed, 1 insertion(+) diff --git a/data/reusables/secret-scanning/partner-secret-list-private-repo.md b/data/reusables/secret-scanning/partner-secret-list-private-repo.md index 540ad78b35e2..030bdefa4d25 100644 --- a/data/reusables/secret-scanning/partner-secret-list-private-repo.md +++ b/data/reusables/secret-scanning/partner-secret-list-private-repo.md @@ -32,6 +32,7 @@ Dropbox | Dropbox Short Lived Access Token | dropbox_short_lived_access_token Dynatrace | Dynatrace Access Token | dynatrace_access_token Dynatrace | Dynatrace Internal Token | dynatrace_internal_token Facebook | Facebook Access Token | facebook_access_token +Fastly | Fastly API Token | fastly_api_token Finicity | Finicity App Key | finicity_app_key Frame.io | Frame.io JSON Web Token | frameio_jwt Frame.io| Frame.io Developer Token | frameio_developer_token From 7e11b061202b3d4f4653e71a854ebbb52346cc34 Mon Sep 17 00:00:00 2001 From: Martin Lopes Date: Fri, 23 Apr 2021 15:46:10 +1000 Subject: [PATCH 24/29] Explain that admins should leave the actions org (#18932) * Update using-the-latest-version-of-the-official-bundled-actions.md * Fixed link * Fixed typo --- ...-latest-version-of-the-official-bundled-actions.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/content/admin/github-actions/using-the-latest-version-of-the-official-bundled-actions.md b/content/admin/github-actions/using-the-latest-version-of-the-official-bundled-actions.md index ad09801a476d..704a7e097578 100644 --- a/content/admin/github-actions/using-the-latest-version-of-the-official-bundled-actions.md +++ b/content/admin/github-actions/using-the-latest-version-of-the-official-bundled-actions.md @@ -26,12 +26,19 @@ You can use {% data variables.product.prodname_github_connect %} to allow {% dat Once {% data variables.product.prodname_github_connect %} is configured, you can use the latest version of an action by deleting its local repository in the `actions` organization on your instance. For example, if your enterprise instance is using the `actions/checkout@v1` action, and you need to use `actions/checkout@v2` which isn't available on your enterprise instance, perform the following steps to be able to use the latest `checkout` action from {% data variables.product.prodname_dotcom_the_website %}: -1. To get the required access to delete the `checkout` repository, use the `ghe-org-admin-promote` command to promote a user to be an owner of the bundled `actions` organization. For more information, see "[Accessing the administrative shell (SSH)](/admin/configuration/accessing-the-administrative-shell-ssh)" and "[`ghe-org-admin-promote`](/admin/configuration/command-line-utilities#ghe-org-admin-promote)." For example: +1. By default, site administrators are not owners of the bundled actions organization. To get the required access to delete the `checkout` repository, use the `ghe-org-admin-promote` command to promote a user to be an owner of the bundled `actions` organization. For more information, see "[Accessing the administrative shell (SSH)](/admin/configuration/accessing-the-administrative-shell-ssh)" and "[`ghe-org-admin-promote`](/admin/configuration/command-line-utilities#ghe-org-admin-promote)." For example: ```shell - ghe-org-admin-promote -u USERNAME -o actions + $ ghe-org-admin-promote -u octocat -o actions + Do you want to give organization admin privileges for actions to octocat? (y/N) y + Making octocat an admin of actions + --> Adding octocat as an admin of actions + --> octocat is now an admin of the actions organization + --> Done. ``` 1. On your {% data variables.product.product_name %} instance, delete the `checkout` repository within the `actions` organization. For information on how to delete a repository, see "[Deleting a repository ](/github/administering-a-repository/deleting-a-repository)." +1. It is recommended that you leave the `actions` organization once you no longer require administrative access. For more information, see "[Removing yourself from an organization +](/github/setting-up-and-managing-your-github-user-account/removing-yourself-from-an-organization)." 1. Configure your workflow's YAML to use `actions/checkout@v2`. 1. Each time your workflow runs, the runner will use the `v2` version of `actions/checkout` from {% data variables.product.prodname_dotcom_the_website %}. From 9002ac330923645c199290eecd7e8d77e8ea5e32 Mon Sep 17 00:00:00 2001 From: Matt Pollard Date: Fri, 23 Apr 2021 08:09:38 +0200 Subject: [PATCH 25/29] Use variable for product name (#18903) --- .../authenticating-to-github/using-ssh-over-the-https-port.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/github/authenticating-to-github/using-ssh-over-the-https-port.md b/content/github/authenticating-to-github/using-ssh-over-the-https-port.md index 3acbcec2750d..d3b7eb7dc9f8 100644 --- a/content/github/authenticating-to-github/using-ssh-over-the-https-port.md +++ b/content/github/authenticating-to-github/using-ssh-over-the-https-port.md @@ -11,7 +11,7 @@ topics: {% tip %} -**GitHub Enterprise users**: Accessing GitHub Enterprise via SSH over the HTTPS port is currently not supported. +**{% data variables.product.prodname_ghe_server %} users**: Accessing {% data variables.product.prodname_ghe_server %} via SSH over the HTTPS port is currently not supported. {% endtip %} From 8429a3b292081f49ae1b99aae37401f035f5eef1 Mon Sep 17 00:00:00 2001 From: Matt Pollard Date: Fri, 23 Apr 2021 09:15:20 +0200 Subject: [PATCH 26/29] Update beta documentation for GitHub Enterprise Cloud (#18860) From 005d9a06851a616724391d8ae3b35a80cd1f2a32 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 23 Apr 2021 11:17:44 -0400 Subject: [PATCH 27/29] Update includes/breadcrumbs.html Co-authored-by: James M. Greene --- includes/breadcrumbs.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/breadcrumbs.html b/includes/breadcrumbs.html index e623d9c56826..57c6ab0caa8f 100644 --- a/includes/breadcrumbs.html +++ b/includes/breadcrumbs.html @@ -22,4 +22,4 @@ {% endif %} {% endfor %} -{% endif %} \ No newline at end of file +{% endif %} From 7a770d90a58b5cf4b6a857c932cdf31b0920b56c Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 23 Apr 2021 11:17:51 -0400 Subject: [PATCH 28/29] Update includes/generic-toc-items.html Co-authored-by: James M. Greene --- includes/generic-toc-items.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/generic-toc-items.html b/includes/generic-toc-items.html index f0b6309882a5..0c2def85be8e 100644 --- a/includes/generic-toc-items.html +++ b/includes/generic-toc-items.html @@ -9,4 +9,4 @@ {% endfor %} -{% endif %} \ No newline at end of file +{% endif %} From 16414cf294d3b372a92bf861d035afff944fb9aa Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 23 Apr 2021 11:17:57 -0400 Subject: [PATCH 29/29] Update includes/generic-toc-list.html Co-authored-by: James M. Greene --- includes/generic-toc-list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/generic-toc-list.html b/includes/generic-toc-list.html index 5dee0a5f6844..8cdd55c9c730 100644 --- a/includes/generic-toc-list.html +++ b/includes/generic-toc-list.html @@ -21,4 +21,4 @@ {% endfor %} -{% endif %} \ No newline at end of file +{% endif %}