Merge branch '1050-mult-narratives'

* 1050-mult-narratives: (26 commits)
  narrative fetching error handling
  Add back in CACHE_JSONS
  remove unused action CLEAR_JSON_CACHE
  Only clear jsonCache on CLEAN_START;
  linting
  Update loadData.js
  throw 404 when dataset is not available, compatibility fallback for old narratives
  async fetching of additional datasets, remove _.uniq
  fetch unique set of datasets for narratives
  TODOs
  fix package lock to same as master
  mostly clarifying comments
  WIP: remove listeners in components when unmounted
  clean up + jsonCache redux sensible default
  mult-dataset narratives working;
  changing narrative datasets: how to best rerender?
  WIP: mult-dataset narratives partly working
  WIP allow multiple dataset narratives;
  Update recomputeReduxState.js
  Update recomputeReduxState.js
  ...

cli/server/getDataset.js

@@ -12,7 +12,9 @@ const setUpGetDatasetHandler = ({datasetsPath}) => {
       await helpers.sendJson(res, info);
     } catch (err) {
       console.trace(err);
-      return helpers.handleError(res, `couldn't fetch JSONs`, err.message);
+      // Throw 404 when not available
+      const errorCode = err.message.endsWith("not in available datasets") ? 404 : 500;
+      return helpers.handleError(res, `couldn't fetch JSONs`, err.message, errorCode);
     }
   };
 };

cli/server/getDatasetHelpers.js

@@ -8,15 +8,16 @@
 *
 */
 
+
 const utils = require("../utils");
 const queryString = require("query-string");
 const path = require("path");
 const convertFromV1 = require("./convertJsonSchemas").convertFromV1;
 const fs = require("fs");
 
-const handleError = (res, clientMsg, serverMsg="") => {
+const handleError = (res, clientMsg, serverMsg="", code=500) => {
   utils.warn(`${clientMsg} -- ${serverMsg}`);
-  return res.status(500).type("text/plain").send(clientMsg);
+  return res.status(code).type("text/plain").send(clientMsg);
 };
 
 const splitPrefixIntoParts = (url) => url

src/actions/loadData.js

@@ -2,7 +2,7 @@ import queryString from "query-string";
 import * as types from "./types";
 import { getServerAddress } from "../util/globals";
 import { goTo404 } from "./navigation";
-import { createStateFromQueryOrJSONs, createTreeTooState } from "./recomputeReduxState";
+import { createStateFromQueryOrJSONs, createTreeTooState, getNarrativePageFromQuery } from "./recomputeReduxState";
 import { loadFrequencies } from "./frequencies";
 import { fetchJSON, fetchWithErrorHandling } from "../util/serverInteraction";
 import { warningNotification, errorNotification } from "./notifications";
@@ -62,7 +62,7 @@ const getDataset = hasExtension("hardcodedDataPaths") ? getHardcodedData : getDa
  * @returns {Array} [0] {string} url, modified as needed to represent main dataset
  *                  [1] {string | undefined} secondTreeUrl, if applicable
  */
-const collectDatasetFetchUrls = (url) => {
+export const collectDatasetFetchUrls = (url) => {
   let secondTreeUrl;
   if (url.includes(":")) {
     const parts = url.replace(/^\//, '')
@@ -226,6 +226,89 @@ export const loadSecondTree = (secondTreeUrl, firstTreeUrl) => async (dispatch,
   dispatch({type: types.TREE_TOO_DATA, ...newState});
 };
 
+function addEndOfNarrativeBlock(narrativeBlocks) {
+  const lastContentSlide = narrativeBlocks[narrativeBlocks.length-1];
+  const endOfNarrativeSlide = Object.assign({}, lastContentSlide, {
+    __html: undefined,
+    isEndOfNarrativeSlide: true
+  });
+  narrativeBlocks.push(endOfNarrativeSlide);
+}
+
+const narrativeFetchingErrorNotification = (err, failedTreeName, fallbackTreeName) => {
+  return errorNotification({
+    message: `Error fetching one of the datasets.
+      Using the YAML-defined dataset (${fallbackTreeName}) instead.`,
+    details: `Could not fetch dataset "${failedTreeName}". Make sure this dataset exists
+      and is spelled correctly.
+      Error details:
+        status: ${err.status};
+        message: ${err.message}`
+  });
+};
+
+const fetchAndCacheNarrativeDatasets = async (dispatch, blocks, query) => {
+  const jsons = {};
+  const startingBlockIdx = getNarrativePageFromQuery(query, blocks);
+  const startingDataset = blocks[startingBlockIdx].dataset;
+  const startingTreeName = collectDatasetFetchUrls(startingDataset)[0];
+  const landingSlide = {
+    mainTreeName: startingTreeName,
+    secondTreeDataset: false,
+    secondTreeName: false
+  };
+  const treeNames = blocks.map((block) => collectDatasetFetchUrls(block.dataset)[0]);
+
+  // TODO:1050
+  // 1. allow frequencies to be loaded for a narrative dataset here
+  // 2. allow loading dataset for secondTreeName
+
+  // We block and await for the landing dataset
+  jsons[startingTreeName] = await
+  getDataset(startingTreeName)
+      .then((res) => res.json())
+      .catch((err) => {
+        if (startingTreeName !== treeNames[0]) {
+          dispatch(narrativeFetchingErrorNotification(err, startingTreeName, treeNames[0]));
+          // Assuming block[0] is the one that was set properly for all legacy narratives
+          return getDataset(treeNames[0])
+            .then((res) => res.json());
+        }
+        throw err;
+      });
+  landingSlide.json = jsons[startingTreeName];
+  // Dispatch landing dataset
+  dispatch({
+    type: types.CLEAN_START,
+    pathnameShouldBe: startingDataset,
+    ...createStateFromQueryOrJSONs({
+      ...landingSlide,
+      query,
+      narrativeBlocks: blocks,
+      dispatch
+    })
+  });
+
+  // The other datasets are fetched asynchronously
+  for (const treeName of treeNames) {
+  // With this there's no need for Set above
+    jsons[treeName] = jsons[treeName] ||
+      getDataset(treeName)
+        .then((res) => res.json())
+        .catch((err) => {
+          dispatch(narrativeFetchingErrorNotification(err, treeName, treeNames[0]));
+          // We fall back to the first (YAML frontmatter) slide's dataset
+          return jsons[treeNames[0]];
+        });
+  }
+  // Dispatch jsons object containing promises corresponding to each fetch to be stored in redux cache.
+  // They are potentially unresolved. We await them upon retreieving from the cache - see actions/navigation.js.
+  dispatch({
+    type: types.CACHE_JSONS,
+    jsons
+  });
+};
+
 
 export const loadJSONs = ({url = window.location.pathname, search = window.location.search} = {}) => {
   return (dispatch, getState) => {
@@ -255,10 +338,8 @@ export const loadJSONs = ({url = window.location.pathname, search = window.locat
           return JSON.parse(err.fileContents);
         })
         .then((blocks) => {
-          const firstURL = blocks[0].dataset;
-          const firstQuery = queryString.parse(blocks[0].query);
-          if (query.n) firstQuery.n = query.n;
-          return fetchDataAndDispatch(dispatch, firstURL, firstQuery, blocks);
+          addEndOfNarrativeBlock(blocks);
+          return fetchAndCacheNarrativeDatasets(dispatch, blocks, query);
         })
         .catch((err) => {
           console.error("Error obtaining narratives", err.message);

src/actions/navigation.js

@@ -1,6 +1,7 @@
 import queryString from "query-string";
 import { createStateFromQueryOrJSONs } from "./recomputeReduxState";
 import { PAGE_CHANGE, URL_QUERY_CHANGE_WITH_COMPUTED_STATE } from "./types";
+import { collectDatasetFetchUrls } from "./loadData";
 
 /* Given a URL, what "page" should be displayed?
  * "page" means the main app, splash page, status page etc
@@ -24,17 +25,38 @@ export const chooseDisplayComponentFromURL = (url) => {
   return "datasetLoader"; // fallthrough
 };
 
+/*
+ * All the Fetch Promises are created before first render. When trying the cache we `await`.
+ * If the Fetch is not finished, this will wait for it to end. Subsequent awaits will immeditaly return the result.
+ * For the landing dataset, no problem either because await on a value just returns the value.
+ */
+const tryCacheThenFetch = async (mainTreeName, secondTreeName, state) => {
+  if (state.jsonCache && state.jsonCache.jsons && state.jsonCache.jsons[mainTreeName] !== undefined) {
+    return {
+      json: await state.jsonCache.jsons[mainTreeName],
+      secondJson: await state.jsonCache.jsons[secondTreeName]
+    };
+  }
+  throw new Error("This should not happen given that we start fetching all datasets before rendering");
+};
+
 /* changes the state of the page and (perhaps) the dataset displayed.
-This function is used throughout the app for all navigation to another page, (including braowserBackForward - see function below)
-The exception is for navigation requests that specify only the query changes, or that have an identical pathname to that selected.
+This function is used throughout the app for all navigation to another page,
 Note that this function is not pure, in that it may change the URL
 
+The function allows these behaviors:
+Case 1. modify the current redux state via a URL query (used in narratives)
+Case 2. modify the current redux state by loading a new dataset, but don't reload the page (e.g. remain within the narrative view)
+Case 3. load new dataset & start fresh (used when changing dataset via the drop-down in the sidebar).
+
 ARGUMENTS:
-(1) path - REQUIRED - the destination path - e.g. "zika" or "flu/..." (does not include query)
-(2) query - OPTIONAL (default: undefined) - see below
-(3) push - OPTIONAL (default: true) - signals that pushState should be used (has no effect on the reducers)
+path -              OPTIONAL (default: window.location.pathname) - the destination path - e.g. "zika" or "flu/..." (does not include query)
+query -             OPTIONAL (default: queryString.parse(window.location.search)) - see below
+queryToDisplay -    OPTIONAL (default: value of `query` argument) - doesn't affect state, only URL.
+push -              OPTIONAL (default: true) - signals that pushState should be used (has no effect on the reducers)
+changeDatasetOnly - OPTIONAL (default: false) - enables changing datasets while keeping the tree, etc mounted to the DOM (e.g. whilst changing datasets in a narrative).
 
-UNDERSTANDING QUERY (SLIGHTLY CONFUSING)
+Note about how this changes the URL and app state according to arguments:
 This function changes the pathname (stored in the datasets reducer) and modifies the URL pathname and query
 accordingly in the middleware. But the URL query is not processed further.
 Because the datasets reducer has changed, the <App> (or whichever display component we're on) will update.
@@ -44,12 +66,11 @@ In this way, the URL query is "used".
 export const changePage = ({
   path = undefined,
   query = undefined,
-  queryToDisplay = undefined, /* doesn't affect state, only URL. defaults to query unless specified */
+  queryToDisplay = undefined,
   push = true,
-  changeDataset = true
+  changeDatasetOnly = false
 } = {}) => (dispatch, getState) => {
   const oldState = getState();
-  // console.warn("CHANGE PAGE!", path, query, queryToDisplay, push);
 
   /* set some defaults */
   if (!path) path = window.location.pathname;  // eslint-disable-line
@@ -58,25 +79,53 @@ export const changePage = ({
   /* some booleans */
   const pathHasChanged = oldState.general.pathname !== path;
 
-  if (changeDataset || pathHasChanged) {
+  if (!pathHasChanged) {
+    /* Case 1 (see docstring): the path (dataset) remains the same but the state may be modulated by the query */
+    const newState = createStateFromQueryOrJSONs(
+      { oldState,
+        query: queryToDisplay,
+        narrativeBlocks: oldState.narrative.blocks,
+        dispatch }
+    );
+    // same dispatch as case 2 but the state comes from the query not from a JSON
+    dispatch({
+      type: URL_QUERY_CHANGE_WITH_COMPUTED_STATE,
+      ...newState,
+      pushState: push,
+      query: queryToDisplay
+    });
+  } else if (changeDatasetOnly) {
+    /* Case 2 (see docstring): the path (dataset) has changed but the we want to remain on the current page and update state with the new dataset */
+    const [mainTreeName, secondTreeName] = collectDatasetFetchUrls(path);
+    tryCacheThenFetch(mainTreeName, secondTreeName, oldState)
+      .then(({json, secondJson}) => {
+        const newState = createStateFromQueryOrJSONs({
+          json,
+          secondTreeDataset: secondJson || false,
+          mainTreeName,
+          secondTreeName: secondTreeName || false,
+          narrativeBlocks: oldState.narrative.blocks,
+          query: queryToDisplay,
+          dispatch
+        });
+        // same dispatch as case 1 but the state comes from a JSON
+        dispatch({
+          type: URL_QUERY_CHANGE_WITH_COMPUTED_STATE,
+          ...newState,
+          pushState: push,
+          query: queryToDisplay
+        });
+      });
+  } else {
+    /* Case 3 (see docstring): the path (dataset) has changed and we want to change pages and set a new state according to the path */
     dispatch({
       type: PAGE_CHANGE,
       path,
       displayComponent: chooseDisplayComponentFromURL(path),
       pushState: push,
       query
     });
-    return;
   }
-
-  /* the path (dataset) remains the same... but the state may be modulated by the query */
-  const newState = createStateFromQueryOrJSONs({oldState, query, dispatch});
-  dispatch({
-    type: URL_QUERY_CHANGE_WITH_COMPUTED_STATE,
-    ...newState,
-    pushState: push,
-    query: queryToDisplay
-  });
 };
 
 /* a 404 uses the same machinery as changePage, but it's not a thunk.

src/actions/recomputeReduxState.js

@@ -44,6 +44,7 @@ export const getMaxCalDateViaTree = (nodes) => {
 
 /* need a (better) way to keep the queryParams all in "sync" */
 const modifyStateViaURLQuery = (state, query) => {
+  // console.log("modify state via URL query", query)
   if (query.l) {
     state["layout"] = query.l;
   }
@@ -590,7 +591,7 @@ const modifyControlsViaTreeToo = (controls, name) => {
 const convertColoringsListToDict = (coloringsList) => {
   const colorings = {};
   coloringsList.forEach((coloring) => {
-    colorings[coloring.key] = coloring;
+    colorings[coloring.key] = { ...coloring };
     delete colorings[coloring.key].key;
   });
   return colorings;
@@ -659,6 +660,16 @@ const createMetadataStateFromJSON = (json) => {
   return metadata;
 };
 
+export const getNarrativePageFromQuery = (query, narrative) => {
+  let n = parseInt(query.n, 10) || 0;
+  /* If the query has defined a block which doesn't exist then default to n=0 */
+  if (n >= narrative.length) {
+    console.warn(`Attempted to go to narrative page ${n} which doesn't exist`);
+    n=0;
+  }
+  return n;
+};
+
 export const createStateFromQueryOrJSONs = ({
   json = false, /* raw json data - completely nuke existing redux state */
   secondTreeDataset = false,
@@ -713,14 +724,8 @@ export const createStateFromQueryOrJSONs = ({
   only displaying the page number (e.g. ?n=3), but we can look up what (hidden)
   URL query this page defines via this information */
   if (narrativeBlocks) {
-    addEndOfNarrativeBlock(narrativeBlocks);
     narrative = narrativeBlocks;
-    let n = parseInt(query.n, 10) || 0;
-    /* If the query has defined a block which doesn't exist then default to n=0 */
-    if (n >= narrative.length) {
-      console.warn(`Attempted to go to narrative page ${n} which doesn't exist`);
-      n=0;
-    }
+    const n = getNarrativePageFromQuery(query, narrative);
     controls = modifyStateViaURLQuery(controls, queryString.parse(narrative[n].query));
     query = n===0 ? {} : {n}; // eslint-disable-line
     /* If the narrative block in view defines a `mainDisplayMarkdown` section, we
@@ -842,12 +847,3 @@ export const createTreeTooState = ({
   // }
   return {tree, treeToo, controls};
 };
-
-function addEndOfNarrativeBlock(narrativeBlocks) {
-  const lastContentSlide = narrativeBlocks[narrativeBlocks.length-1];
-  const endOfNarrativeSlide = Object.assign({}, lastContentSlide, {
-    __html: undefined,
-    isEndOfNarrativeSlide: true
-  });
-  narrativeBlocks.push(endOfNarrativeSlide);
-}

src/actions/types.js

@@ -52,3 +52,4 @@ export const SET_AVAILABLE = "SET_AVAILABLE";
 export const TOGGLE_SIDEBAR = "TOGGLE_SIDEBAR";
 export const TOGGLE_LEGEND = "TOGGLE_LEGEND";
 export const TOGGLE_TRANSMISSION_LINES = "TOGGLE_TRANSMISSION_LINES";
+export const CACHE_JSONS = "CACHE_JSONS";

src/components/entropy/index.js

@@ -291,6 +291,10 @@ class Entropy extends React.Component {
       </Card>
     );
   }
+
+  componentWillUnmount() {
+    // TODO:1050 undo all listeners within EntropyChart (ie this.state.chart)
+  }
 }
 
 const WithTranslation = withTranslation()(Entropy);

src/components/main/index.js

@@ -38,7 +38,8 @@ const Frequencies = lazy(() => import("../frequencies"));
   metadataLoaded: state.metadata.loaded,
   treeLoaded: state.tree.loaded,
   sidebarOpen: state.controls.sidebarOpen,
-  showOnlyPanels: state.controls.showOnlyPanels
+  showOnlyPanels: state.controls.showOnlyPanels,
+  treeName: state.tree.name
 }))
 class Main extends React.Component {
   constructor(props) {
@@ -140,17 +141,17 @@ class Main extends React.Component {
             renderNarrativeToggle(this.props.dispatch, this.props.displayNarrative) : null
           }
           {this.props.displayNarrative || this.props.showOnlyPanels ? null : <Info width={calcUsableWidth(availableWidth, 1)} />}
-          {this.props.panelsToDisplay.includes("tree") ? <Tree width={big.width} height={big.height} /> : null}
-          {this.props.panelsToDisplay.includes("map") ? <Map width={big.width} height={big.height} justGotNewDatasetRenderNewMap={false} legend={this.shouldShowMapLegend()} /> : null}
+          {this.props.panelsToDisplay.includes("tree") ? <Tree width={big.width} height={big.height} key={this.props.treeName} /> : null}
+          {this.props.panelsToDisplay.includes("map") ? <Map width={big.width} height={big.height} key={this.props.treeName+"_map"} justGotNewDatasetRenderNewMap={false} legend={this.shouldShowMapLegend()} /> : null}
           {this.props.panelsToDisplay.includes("entropy") ?
             (<Suspense fallback={null}>
-              <Entropy width={chart.width} height={chart.height} />
+              <Entropy width={chart.width} height={chart.height} key={this.props.treeName+"_entropy"}/>
             </Suspense>) :
             null
           }
           {this.props.panelsToDisplay.includes("frequencies") && this.props.frequenciesLoaded ?
             (<Suspense fallback={null}>
-              <Frequencies width={chart.width} height={chart.height} />
+              <Frequencies width={chart.width} height={chart.height} key={this.props.treeName+"_frequencies"}/>
             </Suspense>) :
             null
           }