Add menus

.vscode/launch.json

@@ -13,6 +13,16 @@
       "request": "launch",
       "name": "Debug DIRECT WordpressSync",
       "program": "debug.js",
+      "args": ["1"],
+      "cwd": "${workspaceRoot}/WordpressSync",
+      "outputCapture": "std"
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug 3X DIRECT WordpressSync",
+      "program": "debug.js",
+      "args": ["3"],
       "cwd": "${workspaceRoot}/WordpressSync",
       "outputCapture": "std"
     },

README.md

@@ -46,29 +46,24 @@ Controls how the service will place content in GitHub.
     "PostPath": "wordpress/posts",
     "PagePath": "wordpress/pages",
     "MediaPath": "wordpress/media",
+    "MenuPath": "menu_test/menus",
+    "MenuSlugs": ["header-menu"],
     "GeneralFilePath": "wordpress/general/general.json",
     "ExcludeProperties": ["content", "_links"]
   }
 }
 ```
 
-`disabled`
-: Set to true to disable processing for this project.
-
-`PostPath`
-: Where should the posts go?
-
-`PagePath`
-: Where should the pages go?
-
-`MediaPath`
-: Where should image media go?
-
-`GeneralFilePath`
-: The full path and filename for a `general.json` file that contains information about the whole site.
-
-`ExcludeProperties`
-: Which WordPress properties should we suppress in output?
+|Name|Description|
+|:--:|:----------|
+|**`disabled`**|Set to true to disable processing for this project.|
+|**`PostPath`**|Where should the posts go?|
+|**`PagePath`**|Where should the pages go?|
+|**`MediaPath`**|Where should image media go?|
+|**`MenuPath`**|Where should the menus go?|
+|**`MenuSlugs`**|Which menus should be fetched?|
+|**`GeneralFilePath`**|The full path and filename for a `general.json` file that contains information about the whole site.|
+|**`ExcludeProperties`**|Which WordPress properties should we suppress in output?|
 
 ### endpoints.json
 
@@ -105,31 +100,21 @@ Contains the projects to process with the service.
 }
 ```
 
-`name` : Friendly name for this job when it runs locally.
-
-`description` : Describe what this is being used for in this endpoint configuration.
-
-`enabled` : Should we process this endpoint?
-
-`enabledLocal` : Should we process this endpoint when running in local development?
-
-`ReportingChannel_Slack` : Slack channel to report activity to.
-
-`WordPressSource` : Describes the Wordpress instance to read from.
-
-`url` : URL of the Wordpress instance to read from.
-
-`tags_exclude` : Ignore Pages/Posts with these tags (Case sensitive!).
-
-`GitHubTarget` : The endpoint target to deploy changes.
-
-`Owner` : GitHub Owner.
-
-`Repo` : GitHub Repo.
-
-`Branch` : GitHub Target Branch.
-
-`ConfigPath` : Path to config.json file for this endpoint.
+|Name|Description|
+|:--:|:----------|
+|**`name`**|Friendly name for this job when it runs locally.|
+|**`description`**|Describe what this is being used for in this endpoint configuration.|
+|**`enabled`**|Should we process this endpoint?|
+|**`enabledLocal`**|Should we process this endpoint when running in local development?|
+|**`ReportingChannel_Slack`**|Slack channel to report activity to.|
+|**`WordPressSource`**|Describes the Wordpress instance to read from.|
+|**`WordPressSource.url`**|URL of the Wordpress instance to read from.|
+|**`WordPressSource.tags_exclude`**|Ignore Pages/Posts with these tags (Case sensitive!).|
+|**`GitHubTarget`**|The endpoint target to deploy changes.|
+|**`GitHubTarget.Owner`**|GitHub Owner.|
+|**`GitHubTarget.Repo`**|GitHub Repo.|
+|**`GitHubTarget.Branch`**|GitHub Target Branch.|
+|**`GitHubTarget.ConfigPath`**|Path to config.json file for this endpoint.|
 
 ## Setting up Local Execution/Debugging
 
@@ -153,13 +138,12 @@ You will need to define a `local.settings.json` file in the project root with th
 }
 ```
 
-`GITHUB_NAME` : The name that will appear on commits.
-
-`GITHUB_EMAIL` : The email that will appear on commits.
-
-`GITHUB_TOKEN` : Your token used to authenticate with GitHub. Get one [here](https://github.com/settings/tokens).
-
-`SLACKBOT_TOKEN` : Your token used to authenticate with your Slack app. Make one [here](https://api.slack.com/apps/).
+|Name|Description|
+|:--:|:----------|
+|**`GITHUB_NAME`**|The name that will appear on commits.|
+|**`GITHUB_EMAIL`**|The email that will appear on commits.|
+|**`GITHUB_TOKEN`**|Your token used to authenticate with GitHub. Get one [here](https://github.com/settings/tokens).|
+|**`SLACKBOT_TOKEN`**|Your token used to authenticate with your Slack app. Make one [here](https://api.slack.com/apps/).|
 
 ### Local running with select projects
 

WordpressSync/debug.js

@@ -3,13 +3,17 @@ const { Values } = require("../local.settings.json");
 Object.keys(Values).forEach(x => (process.env[x] = Values[x])); //Load local settings file for testing
 
 process.env.debug = true; //set to false or remove to run like the real instance
+const repeatCount = parseInt(process.argv.slice(2));
 
 //run the indexpage async
 const indexCode = require("./index");
 (async () => {
-  return await indexCode(
-    { executionContext: { functionName: "debug" } },
-    null,
-    []
-  );
+  for (let step = 0; step < repeatCount; step++) {
+    console.log(`****** Iteration ${step+1} ******`)
+    await indexCode(
+      { executionContext: { functionName: "debug" } },
+      null,
+      []
+    );
+  } 
 })();

WordpressSync/endpoints.json

@@ -39,6 +39,22 @@
           "Branch": "staging",
           "ConfigPath": "wordpress_output/wordpress-to-github.config.json"
         }
+      },
+      {
+        "name": "Drought wp-to-gh menu testing",
+        "description": "Let's get the menus working.",
+        "enabled": false,
+        "enabledLocal": false,
+        "WordPressSource": {
+          "url": "https://dev-drought-ca-gov.pantheonsite.io/",
+          "tags_exclude": ["staging-only", "development"]
+        },
+        "GitHubTarget": {
+          "Owner": "cagov",
+          "Repo": "automation-development-target",
+          "Branch": "main",
+          "ConfigPath": "menu_test/wordpress-to-github.config.json"
+        }
       }
     ]
   }

wordpress-to-github/common/index.js

@@ -1,4 +1,5 @@
 // @ts-check
+const crypto = require('crypto');
 const apiPath = "/wp-json/wp/v2/";
 const {
   gitHubBlobPredictShaFromBuffer,
@@ -28,6 +29,14 @@ const fetchRetry = require("fetch-retry")(require("node-fetch/lib"), {
  * @property {string} [PagePath]
  * @property {string} [MediaPath]
  * @property {string} [GeneralFilePath]
+ * @property {EndpointRequestsConfigData[]} [ApiRequests]
+ */
+
+/**
+ * @typedef {object} EndpointRequestsConfigData
+ * @property {string} Destination
+ * @property {string} Source 
+ * @property {string[]} [ExcludeProperties]
  */
 
 /**
@@ -140,6 +149,18 @@ const fetchRetry = require("fetch-retry")(require("node-fetch/lib"), {
  * @property {number} count
  */
 
+/**
+ * @typedef {object} WordpressApiHashCacheItem Hash details for a Wordpress API response
+ * @property {string} Destination
+ * @property {string} Source
+ * @property {string} Hash
+ * 
+ * @typedef {object} WithData
+ * @property {string} Data
+ * 
+ * @typedef {WordpressApiHashCacheItem & WithData} WordpressApiHashDataItem
+ */
+
 /**
  * Get the path from the a media source url after the 'uploads' part
  *
@@ -241,6 +262,41 @@ const WpApi_GetPagedData_ByQuery = async fetchquery => {
 const WpApi_getSomething = async fetchquery =>
   await fetchRetry(fetchquery, { method: "Get" });
 
+/**
+ * Fetch API request data from the WordPress API. 
+ * 
+ * @param {string} wordPressApiUrl Full URL to the WordPress Menu API.
+ * @param {EndpointRequestsConfigData[]} requests Array of Wordpress API requests.
+ * @returns {Promise<WordpressApiHashDataItem[]>}
+ */
+const WpApi_GetApiRequestsData = (wordPressApiUrl, requests) => {
+  // Fetch all menus concurrently, shove each into array.
+  return Promise.all(requests.map(async request => {
+    const fetchquery = `${wordPressApiUrl}${request.Source}`;
+    console.log(`querying Wordpress API - ${fetchquery}`);
+
+    return await WpApi_getSomething(fetchquery)
+      .then(response => response.json())
+      .then(json => removeExcludedProperties(json, request.ExcludeProperties))
+      .then(json => ({ 
+        Source: request.Source,
+        Destination: request.Destination, 
+        Hash: crypto.createHash('md5').update(JSON.stringify(json)).digest("hex"),
+        Data: json
+      }));
+  }));
+};
+
+/**
+ * Compares a cached object to a current object to see if the cache is out of date.
+ * @param {WordpressApiDateCacheItem|WordpressApiHashCacheItem} cacheItem 
+ * @param {WordpressApiDateCacheItem|WordpressApiHashCacheItem} currentItem 
+ * @returns {boolean}
+ */
+const jsonCacheDiscrepancy = (cacheItem, currentItem) => {
+  return !cacheItem || JSON.stringify(cacheItem) !== JSON.stringify(currentItem);
+};
+
 /**
  * Call the paged wordpress api put all the paged data into a single return array
  *
@@ -386,13 +442,16 @@ const syncBinaryFile = async (wordpress_url, gitRepo, mediaTree, endpoint) => {
  *
  * @param {*} json
  * @param {string[]} [excludeList]
+ * @returns {string} 
  */
 const removeExcludedProperties = (json, excludeList) => {
   if (excludeList) {
     excludeList.forEach(x => {
       delete json[x];
     });
   }
+
+  return json;
 };
 
 /**
@@ -448,6 +507,8 @@ module.exports = {
   wrapInFileMeta,
   commonMeta,
   WpApi_GetCacheItem_ByObjectType,
+  WpApi_GetApiRequestsData,
+  jsonCacheDiscrepancy,
   apiPath,
   fetchDictionary,
   cleanupContent,