Merge pull request #887 from codaco/feature/better-external-data

Feature/better external data

config/webpack.config.dev.js

@@ -142,6 +142,24 @@ module.exports = {
         // match the requirements. When no loader matches it will fall
         // back to the "file" loader at the end of the loader list.
         oneOf: [
+          {
+            test: /\.worker\.js$/,
+            use: [
+              {
+                loader: 'worker-loader',
+                options: { inline: true },
+              },
+              {
+                loader: require.resolve('babel-loader'),
+                options: {
+                  // This is a feature of `babel-loader` for webpack (not Babel itself).
+                  // It enables caching results in ./node_modules/.cache/babel-loader/
+                  // directory for faster rebuilds.
+                  cacheDirectory: true,
+                },
+              },
+            ],
+          },
           // "url" loader works like "file" loader except that it embeds assets
           // smaller than specified limit in bytes as data URLs to avoid requests.
           // A missing `test` is equivalent to a match.

config/webpack.config.prod.js

@@ -139,6 +139,24 @@ module.exports = {
               name: 'static/media/[name].[hash:8].[ext]',
             },
           },
+          {
+            test: /\.worker\.js$/,
+            use: [
+              {
+                loader: 'worker-loader',
+                options: { inline: true },
+              },
+              {
+                loader: require.resolve('babel-loader'),
+                options: {
+                  // This is a feature of `babel-loader` for webpack (not Babel itself).
+                  // It enables caching results in ./node_modules/.cache/babel-loader/
+                  // directory for faster rebuilds.
+                  cacheDirectory: true,
+                },
+              },
+            ],
+          },
           // Process JS with Babel.
           {
             test: /\.(js|jsx)$/,

package.json

@@ -64,6 +64,7 @@
     "connect-history-api-fallback": "1.3.0",
     "cross-env": "^5.2.0",
     "css-loader": "0.28.4",
+    "csvtojson": "^2.0.8",
     "detect-port": "1.1.0",
     "dotenv": "4.0.0",
     "electron": "^2.0.11",
@@ -152,6 +153,7 @@
     "webpack-dev-server": "2.7.1",
     "webpack-manifest-plugin": "^1.3.1",
     "whatwg-fetch": "2.0.3",
+    "worker-loader": "^2.0.0",
     "xss": "^0.3.4"
   },
   "dependencies": {

public/components/windowManager.js

@@ -32,7 +32,6 @@ function createWindow() {
       center: true,
       fullscreen: true,
       title: 'Network Canvas',
-
     }, titlebarParameters);
 
     const mainWindow = new BrowserWindow(windowParameters);

public/index.html

@@ -12,7 +12,7 @@
       style-src 'self' 'unsafe-inline';
       script-src 'self' 'unsafe-inline' 'unsafe-eval';
       font-src 'self' data:;
-      worker-src blob:;
+      worker-src 'self' blob:;
       "
     />
     <meta name="viewport" content="initial-scale=1, maximum-scale=1, user-scalable=no, width=device-width" />

src/components/MainMenu/SettingsMenu.js

@@ -83,7 +83,7 @@ class SettingsMenu extends PureComponent {
                   </p>
                   <Button
                     color="mustard"
-                    onClick={() => importProtocolFromURI('https://github.com/codaco/development-protocol/releases/download/20190408150137-8c32f10/development-protocol.netcanvas')}
+                    onClick={() => importProtocolFromURI('https://github.com/codaco/development-protocol/releases/download/20190410122751-7141c7a/development-protocol.netcanvas')}
                   >
                     Import development protocol
                   </Button>

src/containers/__tests__/withExternalData.test.js

@@ -5,12 +5,25 @@ import { mount } from 'enzyme';
 import { last } from 'lodash';
 import withExternalData from '../withExternalData';
 import loadExternalData from '../../utils/loadExternalData';
+import { entityAttributesProperty } from '../../ducks/modules/network';
 
 jest.mock('../../utils/loadExternalData');
 
 const mockReducer = () => ({
   installedProtocols: {
-    mockProtocol: {},
+    mockProtocol: {
+      codebook: {
+        node: {},
+        edge: {},
+      },
+      assetManifest: {
+        bar: {
+          name: 'bar',
+          source: 'file.json',
+          type: 'network',
+        },
+      },
+    },
   },
   activeSessionId: 'foo',
   sessions: {
@@ -21,7 +34,14 @@ const mockReducer = () => ({
 });
 
 const mockResult = {
-  bar: 'bazz',
+  nodes: [
+    {
+      type: 'person',
+      [entityAttributesProperty]: {
+        fun: true,
+      },
+    },
+  ],
 };
 
 const mockSource = 'bar';

src/containers/withExternalData.js

@@ -10,23 +10,75 @@ import {
 import {
   get,
   mapValues,
+  mapKeys,
+  findKey,
+  isEmpty,
 } from 'lodash';
 import loadExternalData from '../utils/loadExternalData';
+import { entityAttributesProperty } from '../ducks/modules/network';
 
 const mapStateToProps = (state) => {
   const session = state.sessions[state.activeSessionId];
   const protocolUID = session.protocolUID;
+  const protocolCodebook = state.installedProtocols[protocolUID].codebook;
+  const assetManifest = state.installedProtocols[protocolUID].assetManifest;
   const assetFiles = mapValues(
-    state.installedProtocols[protocolUID].assetManifest,
+    assetManifest,
     asset => asset.source,
   );
 
   return {
     protocolUID,
+    assetManifest,
     assetFiles,
+    protocolCodebook,
   };
 };
 
+/**
+ * Utility function that can be used to help with translating external data
+ * variable labels to UUIDs, if a match is possible.
+ *
+ * Assuming that {object} contains other objects, keyed by a UUID, this function
+ * first checks if the string to find is a valid key in the object, and returns it
+ * if so (equivalent to codebook.node.uuid === toFind )
+ *
+ * if not, it iterates the keys of the object, and tests the keys of each child object
+ * to see if the 'name' property equals {toFind}. This is equivalent to
+ * codebook.node.uuid.name === toFind. Where this child object is found, its key within
+ * the parent object is returned.
+ *
+ * Finally, if neither approach finds a UUID, {toFind} is returned.
+ */
+const getObjectUUIDByValue = (object, toFind) => {
+  if (isEmpty(object) || object[toFind]) {
+    return toFind;
+  }
+
+  // Iterate object keys and return the key (itself )
+  const foundKey = findKey(object, objectItem => objectItem.name === toFind);
+
+  return foundKey || toFind;
+};
+
+const withUUIDReplacement = (nodeList, codebook) => nodeList.map(
+  (node) => {
+    const nodeTypeUUID = getObjectUUIDByValue(codebook.node, node.type);
+    const codebookDefinition = codebook.node[nodeTypeUUID] || {};
+
+    const attributes = mapKeys(node[entityAttributesProperty],
+      (attributeValue, attributeKey) =>
+        getObjectUUIDByValue(codebookDefinition.variables, attributeKey),
+    );
+
+    return {
+      ...node,
+      type: nodeTypeUUID,
+      [entityAttributesProperty]: attributes,
+    };
+  },
+);
+
 /**
  * Creates a higher order component which can be used to load data from network assets in
  * the assetsManifest onto a component.
@@ -53,24 +105,32 @@ const mapStateToProps = (state) => {
 const withExternalData = (sourceProperty, dataProperty) =>
   compose(
     connect(mapStateToProps),
-    withState(dataProperty, 'setExternalData', null),
+    withState(
+      dataProperty, // State name
+      'setExternalData', // State updater name
+      null, // initialState
+    ),
     withHandlers({
       loadExternalData: ({
         setExternalData,
         protocolUID,
         assetFiles,
+        assetManifest,
+        protocolCodebook,
       }) =>
         (sourceId) => {
           if (!sourceId) { return; }
-
+          // This is where we could set the loading state for URL assets
           setExternalData(null);
 
           const sourceFile = assetFiles[sourceId];
+          const type = assetManifest[sourceId].type;
 
-          loadExternalData(protocolUID, sourceFile)
-            .then((externalData) => {
-              setExternalData(externalData);
-            });
+          loadExternalData(protocolUID, sourceFile, type)
+            .then(externalData =>
+              setExternalData({
+                nodes: withUUIDReplacement(externalData.nodes, protocolCodebook),
+              }));
         },
     }),
     lifecycle({

src/utils/__tests__/loadExternalData.test.js

@@ -4,7 +4,7 @@ import loadExternalData from '../loadExternalData';
 
 const mockProtocolName = 'myMockProtocol';
 const mockAssetName = 'myMockSource';
-const mockProtocolType = null;
+const mockAssetType = 'network';
 const mockResult = {
   nodes: [
     { foo: 'bar' },
@@ -19,7 +19,7 @@ global.fetch = jest.fn(() => Promise.resolve(mockFetchResponse));
 describe('loadExternalData', () => {
   it('returns a cancellable request');
   it('request response is json with uids  ', (done) => {
-    loadExternalData(mockProtocolName, mockAssetName, mockProtocolType)
+    loadExternalData(mockProtocolName, mockAssetName, mockAssetType)
       .then((result) => {
         expect(result.nodes.length).toBe(mockResult.nodes.length);
         expect(result.nodes.every(

src/utils/csvDecoder.worker.js

@@ -0,0 +1,32 @@
+import { omit } from 'lodash';
+import { entityAttributesProperty } from '../ducks/modules/network';
+
+/**
+ * Converts a CSV file into a Network Canvas node list JSON
+ *
+ * @param {string} data - the contents of a CSV file
+ *
+ * See: https://github.com/Keyang/node-csvtojson We may want to introduce buffering
+ * to this function to increase performance particularly on cordova.
+ *
+ */
+
+const csv = require('../../node_modules/csvtojson/browser/browser.js');
+
+const CSVToJSONNetworkFormat = (data) => {
+  const withTypeAndAttributes = node => ({
+    type: node.type,
+    [entityAttributesProperty]: {
+      ...omit(node, 'type'),
+    },
+  });
+
+  csv().fromString(data)
+    .then((json) => {
+      const nodeList = json.map(entry => withTypeAndAttributes(entry));
+      self.postMessage({ nodes: nodeList });
+    });
+};
+
+// Respond to message from parent thread
+self.addEventListener('message', event => CSVToJSONNetworkFormat(event.data));