Add menus

.vscode/launch.json

@@ -13,6 +13,16 @@
       "request": "launch",
       "name": "Debug DIRECT WordpressSync",
       "program": "debug.js",
+      "args": ["1"],
+      "cwd": "${workspaceRoot}/WordpressSync",
+      "outputCapture": "std"
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug 3X DIRECT WordpressSync",
+      "program": "debug.js",
+      "args": ["3"],
       "cwd": "${workspaceRoot}/WordpressSync",
       "outputCapture": "std"
     },

README.md

@@ -18,6 +18,61 @@ The the trigger service is set to `"authLevel": "function"`, so Azure FaaS Funct
 
 The development package for the wordpress-to-github [NPM module](https://www.npmjs.com/package/@cagov/wordpress-to-github).
 
+## Config files
+
+There are a few configuration files that need to be used.
+
+### endpoints.json
+
+Contains the projects to process with the service.
+
+```json
+{
+  "$schema": "./endpoints.schema.json",
+  "meta": {
+    "title": "endpoints config file",
+    "description": "endpoints config file"
+  },
+  "data": {
+    "projects": [
+      {
+        "name": "drought.ca.gov",
+        "description": "Drought production website",
+        "enabled": true,
+        "enabledLocal": false,
+        "ReportingChannel_Slack": "C1234567890",
+        "WordPressSource": {
+          "url": "https://live-drought-ca-gov.pantheonsite.io",
+          "tags_exclude": ["staging", "development"]
+        },
+        "GitHubTarget": {
+          "Owner": "cagov",
+          "Repo": "drought.ca.gov",
+          "Branch": "main",
+          "ConfigPath": "wordpress/wordpress-to-github.config.json"
+        }
+      }
+    ]
+  }
+}
+```
+
+|Name|Description|
+|:--:|:----------|
+|**`name`**|Friendly name for this job when it runs locally.|
+|**`description`**|Describe what this is being used for in this endpoint configuration.|
+|**`enabled`**|Should we process this endpoint?|
+|**`enabledLocal`**|Should we process this endpoint when running in local development?|
+|**`ReportingChannel_Slack`**|Slack channel to report activity to.|
+|**`WordPressSource`**|Describes the Wordpress instance to read from.|
+|**`WordPressSource.url`**|URL of the Wordpress instance to read from.|
+|**`WordPressSource.tags_exclude`**|Ignore Pages/Posts with these tags (Case sensitive!).|
+|**`GitHubTarget`**|The endpoint target to deploy changes.|
+|**`GitHubTarget.Owner`**|GitHub Owner.|
+|**`GitHubTarget.Repo`**|GitHub Repo.|
+|**`GitHubTarget.Branch`**|GitHub Target Branch.|
+|**`GitHubTarget.ConfigPath`**|Path to config.json file for this endpoint.|
+
 ## Setting up Local Execution/Debugging
 
 When using Visual Studio Code, you can run the polling service locally. Only projects with `enabledLocal: true` will run. It is recommended that you keep all projects set to `enabledLocal: false` until you are sure you want to run them. The `RUN AND DEBUG` launch menu in VS Code should contain `Debug DIRECT WordpressSync`; use that to run locally with debugging.
@@ -40,13 +95,12 @@ You will need to define a `local.settings.json` file in the project root with th
 }
 ```
 
-`GITHUB_NAME` : The name that will appear on commits.
-
-`GITHUB_EMAIL` : The email that will appear on commits.
-
-`GITHUB_TOKEN` : Your token used to authenticate with GitHub. Get one [here](https://github.com/settings/tokens).
-
-`SLACKBOT_TOKEN` : Your token used to authenticate with your Slack app. Make one [here](https://api.slack.com/apps/).
+|Name|Description|
+|:--:|:----------|
+|**`GITHUB_NAME`**|The name that will appear on commits.|
+|**`GITHUB_EMAIL`**|The email that will appear on commits.|
+|**`GITHUB_TOKEN`**|Your token used to authenticate with GitHub. Get one [here](https://github.com/settings/tokens).|
+|**`SLACKBOT_TOKEN`**|Your token used to authenticate with your Slack app. Make one [here](https://api.slack.com/apps/).|
 
 ### Local running with select projects
 

WordpressSync/debug.js

@@ -3,13 +3,17 @@ const { Values } = require("../local.settings.json");
 Object.keys(Values).forEach(x => (process.env[x] = Values[x])); //Load local settings file for testing
 
 process.env.debug = true; //set to false or remove to run like the real instance
+const repeatCount = parseInt(process.argv.slice(2));
 
 //run the indexpage async
 const indexCode = require("./index");
 (async () => {
-  return await indexCode(
-    { executionContext: { functionName: "debug" } },
-    null,
-    []
-  );
+  for (let step = 0; step < repeatCount; step++) {
+    console.log(`****** Iteration ${step+1} ******`)
+    await indexCode(
+      { executionContext: { functionName: "debug" } },
+      null,
+      []
+    );
+  } 
 })();

WordpressSync/endpoints.json

@@ -39,6 +39,22 @@
           "Branch": "staging",
           "ConfigPath": "wordpress_output/wordpress-to-github.config.json"
         }
+      },
+      {
+        "name": "Drought wp-to-gh menu testing",
+        "description": "Let's get the menus working.",
+        "enabled": false,
+        "enabledLocal": false,
+        "WordPressSource": {
+          "url": "https://dev-drought-ca-gov.pantheonsite.io/",
+          "tags_exclude": ["staging-only", "development"]
+        },
+        "GitHubTarget": {
+          "Owner": "cagov",
+          "Repo": "automation-development-target",
+          "Branch": "main",
+          "ConfigPath": "menu_test/wordpress-to-github.config.json"
+        }
       }
     ]
   }

wordpress-to-github/README.md

@@ -30,90 +30,33 @@ Controls how the service will place content in GitHub. This file belongs in your
     "PostPath": "wordpress/posts",
     "PagePath": "wordpress/pages",
     "MediaPath": "wordpress/media",
-    "GeneralFilePath": "wordpress/general/general.json",
-    "ExcludeProperties": ["content", "_links"]
-  }
-}
-```
-
-`disabled`
-: Set to true to disable processing for this project.
-
-`PostPath`
-: Where should the posts go?
-
-`PagePath`
-: Where should the pages go?
-
-`MediaPath`
-: Where should image media go?
-
-`GeneralFilePath`
-: The full path and filename for a `general.json` file that contains information about the whole site.
-
-`ExcludeProperties`
-: Which WordPress properties should we suppress in output?
-
-### endpoints.json
-
-Contains the projects to process with the service.
-
-```json
-{
-  "$schema": "./endpoints.schema.json",
-  "meta": {
-    "title": "endpoints config file",
-    "description": "endpoints config file"
-  },
-  "data": {
-    "projects": [
+    "ApiRequests": [
       {
-        "name": "drought.ca.gov",
-        "description": "Drought production website",
-        "enabled": true,
-        "enabledLocal": false,
-        "ReportingChannel_Slack": "C1234567890",
-        "WordPressSource": {
-          "url": "https://live-drought-ca-gov.pantheonsite.io",
-          "tags_exclude": ["staging", "development"]
-        },
-        "GitHubTarget": {
-          "Owner": "cagov",
-          "Repo": "drought.ca.gov",
-          "Branch": "main",
-          "ConfigPath": "wordpress/wordpress-to-github.config.json"
-        }
+        "Destination": "wordpress/menus/header-menu.json",
+        "Source": "/wp-json/menus/v1/menus/header-menu",
+        "ExcludeProperties": ["description"]
       }
-    ]
+    ],
+    "GeneralFilePath": "wordpress/general/general.json",
+    "ExcludeProperties": ["content", "_links"]
   }
 }
 ```
 
-`name` : Friendly name for this job when it runs locally.
-
-`description` : Describe what this is being used for in this endpoint configuration.
-
-`enabled` : Should we process this endpoint?
-
-`enabledLocal` : Should we process this endpoint when running in local development?
-
-`ReportingChannel_Slack` : Slack channel to report activity to.
-
-`WordPressSource` : Describes the Wordpress instance to read from.
-
-`url` : URL of the Wordpress instance to read from.
-
-`tags_exclude` : Ignore Pages/Posts with these tags (Case sensitive!).
-
-`GitHubTarget` : The endpoint target to deploy changes.
-
-`Owner` : GitHub Owner.
-
-`Repo` : GitHub Repo.
+|Name|Description|
+|:--:|:----------|
+|**`disabled`**|Set to true to disable processing for this project.|
+|**`PostPath`**|Where should the posts go?|
+|**`PagePath`**|Where should the pages go?|
+|**`MediaPath`**|Where should image media go?|
+|**`ApiRequests`**|A collection of API requests to write to the repo.|
+|**`ApiRequests.Destination`**|The output path (in the repo) for an API request.|
+|**`ApiRequests.Source`**|The WordPress API source. This should be an absolute path against the top-level domain of your WordPress site, likely beginning with "/wp-json/".|
+|**`ApiRequests.ExcludeProperties`**|A collection of property keys to remove from the output.|
+|**`GeneralFilePath`**|The full path and filename for a `general.json` file that contains information about the whole site.|
+|**`ExcludeProperties`**|Which WordPress properties should we suppress in output?|
 
-`Branch` : GitHub Target Branch.
 
-`ConfigPath` : Path to config.json file for this endpoint.
 
 ## Sample output
 

wordpress-to-github/common/index.js

@@ -1,4 +1,5 @@
 // @ts-check
+const crypto = require('crypto');
 const apiPath = "/wp-json/wp/v2/";
 const {
   gitHubBlobPredictShaFromBuffer,
@@ -28,6 +29,14 @@ const fetchRetry = require("fetch-retry")(require("node-fetch/lib"), {
  * @property {string} [PagePath]
  * @property {string} [MediaPath]
  * @property {string} [GeneralFilePath]
+ * @property {EndpointRequestsConfigData[]} [ApiRequests]
+ */
+
+/**
+ * @typedef {object} EndpointRequestsConfigData
+ * @property {string} Destination
+ * @property {string} Source 
+ * @property {string[]} [ExcludeProperties]
  */
 
 /**
@@ -140,6 +149,18 @@ const fetchRetry = require("fetch-retry")(require("node-fetch/lib"), {
  * @property {number} count
  */
 
+/**
+ * @typedef {object} WordpressApiHashCacheItem Hash details for a Wordpress API response
+ * @property {string} Destination
+ * @property {string} Source
+ * @property {string} Hash
+ * 
+ * @typedef {object} WithData
+ * @property {string} Data
+ * 
+ * @typedef {WordpressApiHashCacheItem & WithData} WordpressApiHashDataItem
+ */
+
 /**
  * Get the path from the a media source url after the 'uploads' part
  *
@@ -241,6 +262,41 @@ const WpApi_GetPagedData_ByQuery = async fetchquery => {
 const WpApi_getSomething = async fetchquery =>
   await fetchRetry(fetchquery, { method: "Get" });
 
+/**
+ * Fetch API request data from the WordPress API. 
+ * 
+ * @param {string} wordPressApiUrl Full URL to the WordPress Menu API.
+ * @param {EndpointRequestsConfigData[]} requests Array of Wordpress API requests.
+ * @returns {Promise<WordpressApiHashDataItem[]>}
+ */
+const WpApi_GetApiRequestsData = (wordPressApiUrl, requests) => {
+  // Fetch all menus concurrently, shove each into array.
+  return Promise.all(requests.map(async request => {
+    const fetchquery = `${wordPressApiUrl}${request.Source}`;
+    console.log(`querying Wordpress API - ${fetchquery}`);
+
+    return await WpApi_getSomething(fetchquery)
+      .then(response => response.json())
+      .then(json => removeExcludedProperties(json, request.ExcludeProperties))
+      .then(json => ({ 
+        Source: request.Source,
+        Destination: request.Destination, 
+        Hash: crypto.createHash('md5').update(JSON.stringify(json)).digest("hex"),
+        Data: json
+      }));
+  }));
+};
+
+/**
+ * Compares a cached object to a current object to see if the cache is out of date.
+ * @param {WordpressApiDateCacheItem|WordpressApiHashCacheItem} cacheItem 
+ * @param {WordpressApiDateCacheItem|WordpressApiHashCacheItem} currentItem 
+ * @returns {boolean}
+ */
+const jsonCacheDiscrepancy = (cacheItem, currentItem) => {
+  return !cacheItem || JSON.stringify(cacheItem) !== JSON.stringify(currentItem);
+};
+
 /**
  * Call the paged wordpress api put all the paged data into a single return array
  *
@@ -384,15 +440,18 @@ const syncBinaryFile = async (wordpress_url, gitRepo, mediaTree, endpoint) => {
 /**
  * deletes properties in the list
  *
- * @param {*} json
+ * @param {object} json
  * @param {string[]} [excludeList]
+ * @returns {object} 
  */
 const removeExcludedProperties = (json, excludeList) => {
   if (excludeList) {
     excludeList.forEach(x => {
       delete json[x];
     });
   }
+
+  return json;
 };
 
 /**
@@ -448,11 +507,13 @@ module.exports = {
   wrapInFileMeta,
   commonMeta,
   WpApi_GetCacheItem_ByObjectType,
+  WpApi_GetApiRequestsData,
+  jsonCacheDiscrepancy,
   apiPath,
   fetchDictionary,
   cleanupContent,
   WpApi_GetPagedData_ByObjectType,
   WpApi_getSomething,
   pathFromMediaSourceUrl,
   addMediaSection
-};
+};

wordpress-to-github/gitTreeCommon/index.js

@@ -129,8 +129,10 @@ const createTreeFromFileMap = async (
         typeof value === "string" ? value : JSON.stringify(value, null, 2);
 
       if (!existingFile || existingFile.sha !== gitHubBlobPredictSha(content)) {
+        let path = (outputPath) ? `${outputPath}/${key}` : key[0];
+
         targetTree.push({
-          path: `${outputPath}/${key}`,
+          path,
           content,
           mode,
           type
@@ -142,8 +144,10 @@ const createTreeFromFileMap = async (
   if (cleanoutputPath) {
     //process deletes
     for (const delme of referenceTree.filter(x => !x["found"])) {
+      let path = (outputPath) ? `${outputPath}/${delme.path}` : delme.path;
+
       targetTree.push({
-        path: `${outputPath}/${delme.path}`,
+        path,
         mode,
         type,
         sha: null //will trigger a delete