diff --git a/.vscode/launch.json b/.vscode/launch.json index edadfd9..4082332 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -13,6 +13,16 @@ "request": "launch", "name": "Debug DIRECT WordpressSync", "program": "debug.js", + "args": ["1"], + "cwd": "${workspaceRoot}/WordpressSync", + "outputCapture": "std" + }, + { + "type": "node", + "request": "launch", + "name": "Debug 3X DIRECT WordpressSync", + "program": "debug.js", + "args": ["3"], "cwd": "${workspaceRoot}/WordpressSync", "outputCapture": "std" }, diff --git a/README.md b/README.md index adac475..d082dd9 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,61 @@ The the trigger service is set to `"authLevel": "function"`, so Azure FaaS Funct The development package for the wordpress-to-github [NPM module](https://www.npmjs.com/package/@cagov/wordpress-to-github). +## Config files + +There are a few configuration files that need to be used. + +### endpoints.json + +Contains the projects to process with the service. + +```json +{ + "$schema": "./endpoints.schema.json", + "meta": { + "title": "endpoints config file", + "description": "endpoints config file" + }, + "data": { + "projects": [ + { + "name": "drought.ca.gov", + "description": "Drought production website", + "enabled": true, + "enabledLocal": false, + "ReportingChannel_Slack": "C1234567890", + "WordPressSource": { + "url": "https://live-drought-ca-gov.pantheonsite.io", + "tags_exclude": ["staging", "development"] + }, + "GitHubTarget": { + "Owner": "cagov", + "Repo": "drought.ca.gov", + "Branch": "main", + "ConfigPath": "wordpress/wordpress-to-github.config.json" + } + } + ] + } +} +``` + +|Name|Description| +|:--:|:----------| +|**`name`**|Friendly name for this job when it runs locally.| +|**`description`**|Describe what this is being used for in this endpoint configuration.| +|**`enabled`**|Should we process this endpoint?| +|**`enabledLocal`**|Should we process this endpoint when running in local development?| +|**`ReportingChannel_Slack`**|Slack channel to report activity to.| +|**`WordPressSource`**|Describes the Wordpress instance to read from.| +|**`WordPressSource.url`**|URL of the Wordpress instance to read from.| +|**`WordPressSource.tags_exclude`**|Ignore Pages/Posts with these tags (Case sensitive!).| +|**`GitHubTarget`**|The endpoint target to deploy changes.| +|**`GitHubTarget.Owner`**|GitHub Owner.| +|**`GitHubTarget.Repo`**|GitHub Repo.| +|**`GitHubTarget.Branch`**|GitHub Target Branch.| +|**`GitHubTarget.ConfigPath`**|Path to config.json file for this endpoint.| + ## Setting up Local Execution/Debugging When using Visual Studio Code, you can run the polling service locally. Only projects with `enabledLocal: true` will run. It is recommended that you keep all projects set to `enabledLocal: false` until you are sure you want to run them. The `RUN AND DEBUG` launch menu in VS Code should contain `Debug DIRECT WordpressSync`; use that to run locally with debugging. @@ -40,13 +95,12 @@ You will need to define a `local.settings.json` file in the project root with th } ``` -`GITHUB_NAME` : The name that will appear on commits. - -`GITHUB_EMAIL` : The email that will appear on commits. - -`GITHUB_TOKEN` : Your token used to authenticate with GitHub. Get one [here](https://github.com/settings/tokens). - -`SLACKBOT_TOKEN` : Your token used to authenticate with your Slack app. Make one [here](https://api.slack.com/apps/). +|Name|Description| +|:--:|:----------| +|**`GITHUB_NAME`**|The name that will appear on commits.| +|**`GITHUB_EMAIL`**|The email that will appear on commits.| +|**`GITHUB_TOKEN`**|Your token used to authenticate with GitHub. Get one [here](https://github.com/settings/tokens).| +|**`SLACKBOT_TOKEN`**|Your token used to authenticate with your Slack app. Make one [here](https://api.slack.com/apps/).| ### Local running with select projects diff --git a/WordpressSync/debug.js b/WordpressSync/debug.js index 46a1a16..2c1af00 100644 --- a/WordpressSync/debug.js +++ b/WordpressSync/debug.js @@ -3,13 +3,17 @@ const { Values } = require("../local.settings.json"); Object.keys(Values).forEach(x => (process.env[x] = Values[x])); //Load local settings file for testing process.env.debug = true; //set to false or remove to run like the real instance +const repeatCount = parseInt(process.argv.slice(2)); //run the indexpage async const indexCode = require("./index"); (async () => { - return await indexCode( - { executionContext: { functionName: "debug" } }, - null, - [] - ); + for (let step = 0; step < repeatCount; step++) { + console.log(`****** Iteration ${step+1} ******`) + await indexCode( + { executionContext: { functionName: "debug" } }, + null, + [] + ); + } })(); diff --git a/WordpressSync/endpoints.json b/WordpressSync/endpoints.json index 5134e4b..85c1bc0 100644 --- a/WordpressSync/endpoints.json +++ b/WordpressSync/endpoints.json @@ -39,6 +39,22 @@ "Branch": "staging", "ConfigPath": "wordpress_output/wordpress-to-github.config.json" } + }, + { + "name": "Drought wp-to-gh menu testing", + "description": "Let's get the menus working.", + "enabled": false, + "enabledLocal": false, + "WordPressSource": { + "url": "https://dev-drought-ca-gov.pantheonsite.io/", + "tags_exclude": ["staging-only", "development"] + }, + "GitHubTarget": { + "Owner": "cagov", + "Repo": "automation-development-target", + "Branch": "main", + "ConfigPath": "menu_test/wordpress-to-github.config.json" + } } ] } diff --git a/wordpress-to-github/README.md b/wordpress-to-github/README.md index ea95af0..72d94f1 100644 --- a/wordpress-to-github/README.md +++ b/wordpress-to-github/README.md @@ -30,90 +30,33 @@ Controls how the service will place content in GitHub. This file belongs in your "PostPath": "wordpress/posts", "PagePath": "wordpress/pages", "MediaPath": "wordpress/media", - "GeneralFilePath": "wordpress/general/general.json", - "ExcludeProperties": ["content", "_links"] - } -} -``` - -`disabled` -: Set to true to disable processing for this project. - -`PostPath` -: Where should the posts go? - -`PagePath` -: Where should the pages go? - -`MediaPath` -: Where should image media go? - -`GeneralFilePath` -: The full path and filename for a `general.json` file that contains information about the whole site. - -`ExcludeProperties` -: Which WordPress properties should we suppress in output? - -### endpoints.json - -Contains the projects to process with the service. - -```json -{ - "$schema": "./endpoints.schema.json", - "meta": { - "title": "endpoints config file", - "description": "endpoints config file" - }, - "data": { - "projects": [ + "ApiRequests": [ { - "name": "drought.ca.gov", - "description": "Drought production website", - "enabled": true, - "enabledLocal": false, - "ReportingChannel_Slack": "C1234567890", - "WordPressSource": { - "url": "https://live-drought-ca-gov.pantheonsite.io", - "tags_exclude": ["staging", "development"] - }, - "GitHubTarget": { - "Owner": "cagov", - "Repo": "drought.ca.gov", - "Branch": "main", - "ConfigPath": "wordpress/wordpress-to-github.config.json" - } + "Destination": "wordpress/menus/header-menu.json", + "Source": "/wp-json/menus/v1/menus/header-menu", + "ExcludeProperties": ["description"] } - ] + ], + "GeneralFilePath": "wordpress/general/general.json", + "ExcludeProperties": ["content", "_links"] } } ``` -`name` : Friendly name for this job when it runs locally. - -`description` : Describe what this is being used for in this endpoint configuration. - -`enabled` : Should we process this endpoint? - -`enabledLocal` : Should we process this endpoint when running in local development? - -`ReportingChannel_Slack` : Slack channel to report activity to. - -`WordPressSource` : Describes the Wordpress instance to read from. - -`url` : URL of the Wordpress instance to read from. - -`tags_exclude` : Ignore Pages/Posts with these tags (Case sensitive!). - -`GitHubTarget` : The endpoint target to deploy changes. - -`Owner` : GitHub Owner. - -`Repo` : GitHub Repo. +|Name|Description| +|:--:|:----------| +|**`disabled`**|Set to true to disable processing for this project.| +|**`PostPath`**|Where should the posts go?| +|**`PagePath`**|Where should the pages go?| +|**`MediaPath`**|Where should image media go?| +|**`ApiRequests`**|A collection of API requests to write to the repo.| +|**`ApiRequests.Destination`**|The output path (in the repo) for an API request.| +|**`ApiRequests.Source`**|The WordPress API source. This should be an absolute path against the top-level domain of your WordPress site, likely beginning with "/wp-json/".| +|**`ApiRequests.ExcludeProperties`**|A collection of property keys to remove from the output.| +|**`GeneralFilePath`**|The full path and filename for a `general.json` file that contains information about the whole site.| +|**`ExcludeProperties`**|Which WordPress properties should we suppress in output?| -`Branch` : GitHub Target Branch. -`ConfigPath` : Path to config.json file for this endpoint. ## Sample output diff --git a/wordpress-to-github/common/index.js b/wordpress-to-github/common/index.js index 76156ae..55d2c30 100644 --- a/wordpress-to-github/common/index.js +++ b/wordpress-to-github/common/index.js @@ -1,4 +1,5 @@ // @ts-check +const crypto = require('crypto'); const apiPath = "/wp-json/wp/v2/"; const { gitHubBlobPredictShaFromBuffer, @@ -29,6 +30,14 @@ const fetchRetry = require("fetch-retry")(require("node-fetch/lib"), { * @property {string} [PagePath] * @property {string} [MediaPath] * @property {string} [GeneralFilePath] + * @property {EndpointRequestsConfigData[]} [ApiRequests] + */ + +/** + * @typedef {object} EndpointRequestsConfigData + * @property {string} Destination + * @property {string} Source + * @property {string[]} [ExcludeProperties] */ /** @@ -141,6 +150,18 @@ const fetchRetry = require("fetch-retry")(require("node-fetch/lib"), { * @property {number} count */ +/** + * @typedef {object} WordpressApiHashCacheItem Hash details for a Wordpress API response + * @property {string} Destination + * @property {string} Source + * @property {string} Hash + * + * @typedef {object} WithData + * @property {string} Data + * + * @typedef {WordpressApiHashCacheItem & WithData} WordpressApiHashDataItem + */ + /** * Get the path from the a media source url after the 'uploads' part * @@ -242,6 +263,52 @@ const WpApi_GetPagedData_ByQuery = async fetchquery => { const WpApi_getSomething = async fetchquery => await fetchRetry(fetchquery, { method: "Get" }); +/** + * Fetch API request data from the WordPress API. + * + * @param {string} wordPressApiUrl Full URL to the WordPress Menu API. + * @param {EndpointRequestsConfigData[]} requests Array of Wordpress API requests. + * @returns {Promise} + */ +const WpApi_GetApiRequestsData = (wordPressApiUrl, requests) => { + // Fetch all menus concurrently, shove each into array. + return Promise.all( + requests.map(async request => { + const fetchquery = `${wordPressApiUrl}${request.Source}`; + console.log(`querying Wordpress API - ${fetchquery}`); + + return await WpApi_getSomething(fetchquery) + .then(response => { + if (response.ok) { + return response.json(); + } else { + throw new Error(`${response.status} - ${response.statusText} - ${response.url}`); + } + }) + .then(json => removeExcludedProperties(json, request.ExcludeProperties)) + .then(json => ({ + Source: request.Source, + Destination: request.Destination, + Hash: crypto + .createHash("md5") + .update(JSON.stringify(json)) + .digest("hex"), + Data: json + })); + }) + ); +}; + +/** + * Compares a cached object to a current object to see if the cache is out of date. + * @param {WordpressApiDateCacheItem|WordpressApiHashCacheItem} cacheItem + * @param {WordpressApiDateCacheItem|WordpressApiHashCacheItem} currentItem + * @returns {boolean} + */ +const jsonCacheDiscrepancy = (cacheItem, currentItem) => { + return !cacheItem || JSON.stringify(cacheItem) !== JSON.stringify(currentItem); +}; + /** * Call the paged wordpress api put all the paged data into a single return array * @@ -385,8 +452,9 @@ const syncBinaryFile = async (wordpress_url, gitRepo, mediaTree, endpoint) => { /** * deletes properties in the list * - * @param {*} json + * @param {object} json * @param {string[]} [excludeList] + * @returns {object} */ const removeExcludedProperties = (json, excludeList) => { if (excludeList) { @@ -394,6 +462,8 @@ const removeExcludedProperties = (json, excludeList) => { delete json[x]; }); } + + return json; }; /** @@ -449,6 +519,8 @@ module.exports = { wrapInFileMeta, commonMeta, WpApi_GetCacheItem_ByObjectType, + WpApi_GetApiRequestsData, + jsonCacheDiscrepancy, apiPath, fetchDictionary, cleanupContent, @@ -456,4 +528,4 @@ module.exports = { WpApi_getSomething, pathFromMediaSourceUrl, addMediaSection -}; +}; \ No newline at end of file diff --git a/wordpress-to-github/gitTreeCommon/index.js b/wordpress-to-github/gitTreeCommon/index.js index 51c3900..8530424 100644 --- a/wordpress-to-github/gitTreeCommon/index.js +++ b/wordpress-to-github/gitTreeCommon/index.js @@ -96,20 +96,26 @@ const createTreeFromFileMap = async ( outputPath, cleanoutputPath ) => { - const pathRootTree = outputPath.split("/").slice(0, -1).join("/"); //gets the parent folder to the output path + let treeUrl = ""; + if (outputPath) { + //Path Tree - /** @type {GithubTreeRow[]} */ - const rootTree = (await gitRepo.getSha(masterBranch, pathRootTree)).data; - const referenceTreeRow = rootTree.find(f => f.path === outputPath); + const pathRootTree = outputPath.split("/").slice(0, -1).join("/"); //gets the parent folder to the output path + /** @type {GithubTreeRow[]} */ + const rootTree = (await gitRepo.getSha(masterBranch, pathRootTree)).data; + const referenceTreeRow = rootTree.find(f => f.path === outputPath); - /** @type {GithubTreeRow[]} */ - const referenceTree = referenceTreeRow - ? ( - await gitRepo.getTree(`${referenceTreeRow.sha}?recursive=true`) - ).data.tree.filter( - (/** @type { GithubTreeRow } */ x) => x.type === "blob" - ) - : []; + if (referenceTreeRow) { + treeUrl = `${referenceTreeRow.sha}?recursive=true`; + } + } else { + //Root Tree + treeUrl = masterBranch; + } + + const referenceTree = /** @type {{data:{tree:GithubTreeRow[]}}}} */ ( + await gitRepo.getTree(treeUrl) + ).data.tree.filter(x => x.type === "blob"); /** @type {GithubTreeRow[]} */ const targetTree = []; @@ -129,8 +135,10 @@ const createTreeFromFileMap = async ( typeof value === "string" ? value : JSON.stringify(value, null, 2); if (!existingFile || existingFile.sha !== gitHubBlobPredictSha(content)) { + let path = outputPath ? `${outputPath}/${key}` : key; + targetTree.push({ - path: `${outputPath}/${key}`, + path, content, mode, type @@ -142,8 +150,10 @@ const createTreeFromFileMap = async ( if (cleanoutputPath) { //process deletes for (const delme of referenceTree.filter(x => !x["found"])) { + let path = outputPath ? `${outputPath}/${delme.path}` : delme.path; + targetTree.push({ - path: `${outputPath}/${delme.path}`, + path, mode, type, sha: null //will trigger a delete diff --git a/wordpress-to-github/index.js b/wordpress-to-github/index.js index 34b0d4a..0674c60 100644 --- a/wordpress-to-github/index.js +++ b/wordpress-to-github/index.js @@ -12,6 +12,8 @@ const { wrapInFileMeta, commonMeta, WpApi_GetCacheItem_ByObjectType, + WpApi_GetApiRequestsData, + jsonCacheDiscrepancy, apiPath, fetchDictionary, cleanupContent, @@ -24,6 +26,7 @@ const { EndpointConfigData, SourceEndpointConfigData, WordpressApiDateCacheItem, + WordpressApiHashCacheItem, GitHubCommitter, GithubOutputJson, WordpressMediaRow, @@ -33,13 +36,14 @@ const { const commitTitlePosts = "Wordpress Posts Update"; const commitTitlePages = "Wordpress Pages Update"; const commitTitleMedia = "Wordpress Media Update"; +const commitTitleApiRequests = "Wordpress API Requests Update"; const commitTitleGeneral = "Wordpress General File Update"; const fieldMetaReference = { posts: "https://developer.wordpress.org/rest-api/reference/posts/", pages: "https://developer.wordpress.org/rest-api/reference/pages/", media: "https://developer.wordpress.org/rest-api/reference/pages/" }; -/** @type {Map } */ +/** @type {Map } */ const updateCache = new Map(); const cacheObjects = ["media", "posts", "pages"]; @@ -124,26 +128,43 @@ const SyncEndpoint = async ( const endpointConfig = await getRemoteConfig(gitHubTarget, gitHubCredentials); const wordPressApiUrl = sourceEndpointConfig.WordPressSource.url + apiPath; + const allApiRequests = endpointConfig.ApiRequests && endpointConfig.ApiRequests.length + ? await WpApi_GetApiRequestsData(sourceEndpointConfig.WordPressSource.url, endpointConfig.ApiRequests) + : null; + //Check cache (and set cache for next time) let cacheMatch = true; const cacheRoot = `Owner:${gitHubTarget.Owner},Repo:${gitHubTarget.Repo},Branch:${gitHubTarget.Branch},wordPressApiUrl:${wordPressApiUrl}`; + for (let type of cacheObjects) { const cacheKey = `${cacheRoot},type:${type}`; - + const cacheItem = updateCache.get(cacheKey); const currentStatus = await WpApi_GetCacheItem_ByObjectType( wordPressApiUrl, type ); - const cacheItem = updateCache.get(cacheKey); + updateCache.set(cacheKey, currentStatus); - if ( - !cacheItem || - JSON.stringify(cacheItem) !== JSON.stringify(currentStatus) - ) { + if (jsonCacheDiscrepancy(cacheItem, currentStatus)) { cacheMatch = false; } } + + if (allApiRequests) { + for (let request of allApiRequests) { + const apiRequestCacheKey = `${cacheRoot},type:apiResponse:${request.Destination}`; + const apiRequestCacheItem = updateCache.get(apiRequestCacheKey); + const { Data, ...apiCurrentStatus } = request; + + updateCache.set(apiRequestCacheKey, apiCurrentStatus); + + if (jsonCacheDiscrepancy(apiRequestCacheItem, apiCurrentStatus)) { + cacheMatch = false; + } + } + } + if (cacheMatch) { console.log(`match cache for ${cacheRoot}`); return; @@ -466,15 +487,57 @@ const SyncEndpoint = async ( pagesTree, `${commitTitlePages} (${ pagesTree.filter(x => x.path.endsWith(".html")).length - } updates)`, + } updates)`, //TODO: Pull from a name property gitHubCommitter ) ); } + // API Requests + if (allApiRequests) { + // Group all destination files by their parent folders. + const apiRequestsByFolder = allApiRequests.reduce((bucket, request) => { + let folderName = request.Destination.split("/").slice(0, -1).join("/"); + let fileName = request.Destination.split("/").slice(-1)[0]; + + if (!(folderName in bucket)) { + bucket[folderName] = new Map(); + } + + bucket[folderName].set(fileName, JSON.stringify(request.Data, null, 2)) + + return bucket; + }, {}); + + // Create and commit a git tree for each set of files. + for (let [folderName, fileMap] of Object.entries(apiRequestsByFolder)) { + const requestsTree = await createTreeFromFileMap( + gitRepo, + gitHubTarget.Branch, + fileMap, + folderName, + false + ); + + const reportLabel = folderName.split("/").slice(-1).join("/") || 'root'; + const updateCount = `${requestsTree.length} ${requestsTree.length === 1 ? "update" : "updates"}`; + + addToReport( + report, + await CommitIfChanged( + gitRepo, + gitHubTarget.Branch, + requestsTree, + `${commitTitleApiRequests} (${updateCount} to ${reportLabel})`, + gitHubCommitter + ) + ); + } + } + return report; }; module.exports = { SyncEndpoint -}; +}; \ No newline at end of file diff --git a/wordpress-to-github/schemas/wordpress-to-github-sample.config.json b/wordpress-to-github/schemas/wordpress-to-github-sample.config.json index 2a23deb..7aaef41 100644 --- a/wordpress-to-github/schemas/wordpress-to-github-sample.config.json +++ b/wordpress-to-github/schemas/wordpress-to-github-sample.config.json @@ -9,6 +9,13 @@ "PostPath": "wordpress/posts", "PagePath": "wordpress/pages", "MediaPath": "wordpress/media", + "ApiRequests": [ + { + "Destination": "wordpress/menus/header-menu.json", + "Source": "/wp-json/menus/v1/menus/header-menu", + "ExcludeProperties": ["description"] + } + ], "GeneralFilePath": "wordpress/general/general.json", "ExcludeProperties": ["content", "_links"] } diff --git a/wordpress-to-github/schemas/wordpress-to-github.config.schema.json b/wordpress-to-github/schemas/wordpress-to-github.config.schema.json index 396edfe..e16b524 100644 --- a/wordpress-to-github/schemas/wordpress-to-github.config.schema.json +++ b/wordpress-to-github/schemas/wordpress-to-github.config.schema.json @@ -31,6 +31,13 @@ "PostPath": "wordpress/posts", "PagePath": "wordpress/pages", "MediaPath": "wordpress/media", + "ApiRequests": [ + { + "Destination": "wordpress/menus/header-menu.json", + "Source": "/wp-json/menus/v1/menus/header-menu", + "ExcludeProperties": ["description"] + } + ], "GeneralFilePath": "wordpress/general/general.json", "ExcludeProperties": ["content", "_links"] } @@ -56,6 +63,42 @@ "description": "Where should image media go?", "examples": ["wordpress/media"] }, + "ApiRequests": { + "type": "array", + "description": "What should be fetched from the API?", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["Source", "Destination"], + "properties": { + "Destination": { + "type": "string", + "description": "The path where the API request should be stored in git.", + "examples": ["wordpress/menus/header-menu.json"] + }, + "Source": { + "type": "string", + "description": "The Wordpress API URL to fetch.", + "examples": ["/wp-json/menus/v1/menus/header-menu"] + }, + "ExcludeProperties": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Which properties should we suppress in output?", + "examples": [["description"]] + } + } + }, + "examples": [ + { + "Destination": "wordpress/menus/header-menu.json", + "Source": "/wp-json/menus/v1/menus/header-menu", + "ExcludeProperties": ["description"] + } + ] + }, "GeneralFilePath": { "type": "string", "description": "The full path and filename for a 'general' file that contains information about the whole site. Remove this attribute to stop updating the file. Changing this file path will not remove the old file.",