From 1288696ef8e3a835d5e52002fb81c7f2e05691e2 Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Tue, 9 Apr 2019 13:58:32 +0100 Subject: [PATCH 01/11] update validation ref for no entity labels --- src/utils/protocol/protocol-validation | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/protocol/protocol-validation b/src/utils/protocol/protocol-validation index 8c32f10339..7141c7aae4 160000 --- a/src/utils/protocol/protocol-validation +++ b/src/utils/protocol/protocol-validation @@ -1 +1 @@ -Subproject commit 8c32f1033921ddb68c4b7d6e73ac30f96add829f +Subproject commit 7141c7aae4a42ea7cb7d6360ad2b7e8ad14f6e83 From 1fdbd76a8f65dbf39396be1521feca938cc5764f Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Tue, 9 Apr 2019 22:59:32 +0100 Subject: [PATCH 02/11] lookup external data variable and type labels, and convert to UUIDs - if available --- src/containers/withExternalData.js | 60 +++++++++++++++++++++++++++++- src/utils/loadExternalData.js | 46 +++++++++++------------ 2 files changed, 81 insertions(+), 25 deletions(-) diff --git a/src/containers/withExternalData.js b/src/containers/withExternalData.js index 1885ff48de..e81758bdc0 100644 --- a/src/containers/withExternalData.js +++ b/src/containers/withExternalData.js @@ -10,12 +10,16 @@ import { import { get, mapValues, + mapKeys, + findKey, } from 'lodash'; import loadExternalData from '../utils/loadExternalData'; +import { entityAttributesProperty } from '../ducks/modules/network'; const mapStateToProps = (state) => { const session = state.sessions[state.activeSessionId]; const protocolUID = session.protocolUID; + const protocolCodebook = state.installedProtocols[protocolUID].codebook; const assetFiles = mapValues( state.installedProtocols[protocolUID].assetManifest, asset => asset.source, @@ -24,9 +28,37 @@ const mapStateToProps = (state) => { return { protocolUID, assetFiles, + protocolCodebook, }; }; + +/** + * Utility function that can be used to help with translating external data + * variable labels to UUIDs, if a match is possible. + * + * Assuming that {object} contains other objects, keyed by a UUID, this function + * first checks if the string to find is a valid key in the object, and returns it + * if so (equivalent to codebook.node.uuid === toFind ) + * + * if not, it iterates the keys of the object, and tests the keys of each child object + * to see if the 'name' property equals {toFind}. This is equivalent to + * codebook.node.uuid.name === toFind. Where this child object is found, its key within + * the parent object is returned. + * + * Finally, if neither approach finds a UUID, {toFind} is returned. + */ +const getObjectUUIDByValue = (object, toFind) => { + if (object[toFind]) { // should pass if UUID is used + return toFind; + } + + // Iterate object keys and return the key (itself ) + const foundKey = findKey(object, objectItem => objectItem.name === toFind); + + return foundKey || toFind; +}; + /** * Creates a higher order component which can be used to load data from network assets in * the assetsManifest onto a component. @@ -53,23 +85,47 @@ const mapStateToProps = (state) => { const withExternalData = (sourceProperty, dataProperty) => compose( connect(mapStateToProps), - withState(dataProperty, 'setExternalData', null), + withState( + dataProperty, // State name + 'setExternalData', // State updater name + null, // initialState + ), withHandlers({ loadExternalData: ({ setExternalData, protocolUID, assetFiles, + protocolCodebook, }) => (sourceId) => { if (!sourceId) { return; } + // This is where we could set the loading state for URL assets setExternalData(null); const sourceFile = assetFiles[sourceId]; loadExternalData(protocolUID, sourceFile) .then((externalData) => { - setExternalData(externalData); + const withUUIDReplacement = nodeList => nodeList.map( + (node) => { + const nodeTypeUUID = getObjectUUIDByValue(protocolCodebook.node, node.type); + const codebookDefinition = protocolCodebook.node[nodeTypeUUID] || {}; + + const attributes = mapKeys(node[entityAttributesProperty], + (attributeValue, attributeKey) => + getObjectUUIDByValue(codebookDefinition.variables, attributeKey), + ); + + return { + ...node, + type: nodeTypeUUID, + [entityAttributesProperty]: attributes, + }; + }, + ); + + setExternalData({ nodes: withUUIDReplacement(externalData.nodes) }); }); }, }), diff --git a/src/utils/loadExternalData.js b/src/utils/loadExternalData.js index 3381e9c417..67d151c126 100644 --- a/src/utils/loadExternalData.js +++ b/src/utils/loadExternalData.js @@ -6,7 +6,6 @@ import inEnvironment from './Environment'; import { readFile } from './filesystem'; import { entityPrimaryKeyProperty } from '../ducks/modules/network'; import getAssetUrl from './protocol/getAssetUrl'; -import getFactoryProtocolPath from './protocol/factoryProtocolPath'; const withKeys = data => data.map((node) => { @@ -31,31 +30,28 @@ const fetchNetwork = inEnvironment( } if (environment === environments.CORDOVA) { - return (url, { fileName, protocolType, protocolUID }) => { - if (protocolType === 'factory') { - return readFile( - getFactoryProtocolPath(protocolUID, `assets/${fileName}`) - ) - .then(data => JSON.parse(data)) - .then((json) => { - const nodes = get(json, 'nodes', []); - return ({ nodes: withKeys(nodes) }); - }); - } - - return readFile(url) - .then(response => JSON.parse(response)) - .then((json) => { - const nodes = get(json, 'nodes', []); - return ({ nodes: withKeys(nodes) }); - }); - }; + return url => readFile(url) + .then(response => JSON.parse(response)) + .then((json) => { + const nodes = get(json, 'nodes', []); + return ({ nodes: withKeys(nodes) }); + }); } return Promise.reject('Environment not supported'); }, ); +const parseJSON = (fileContents) => { + +}; + +const parseCSV = (fileContents) => { + +}; + +const fileExtension = fileName => fileName.split('.').pop(); + /** * Loads network data from assets and appends objectHash uids. * @@ -64,8 +60,12 @@ const fetchNetwork = inEnvironment( * @returns {object} Network object in format { nodes, edges } * */ -const loadExternalData = (protocolUID, fileName, protocolType) => - getAssetUrl(protocolUID, fileName, protocolType) - .then(url => fetchNetwork(url, { fileName, protocolType, protocolUID })); +const loadExternalData = (protocolUID, fileName) => { + const fileType = fileExtension(fileName) === 'csv' ? 'csv' : 'json'; + + return getAssetUrl(protocolUID, fileName) + .then(url => fetchNetwork(url, fileType)); +}; + export default loadExternalData; From af3a1b647b2cf72367524922ee1ceea54e396873 Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Wed, 10 Apr 2019 09:59:19 +0100 Subject: [PATCH 03/11] implement CSV file loading --- package-lock.json | 11 +++++ package.json | 3 +- src/containers/withExternalData.js | 52 +++++++++++----------- src/utils/loadExternalData.js | 69 ++++++++++++++++++++++-------- 4 files changed, 92 insertions(+), 43 deletions(-) diff --git a/package-lock.json b/package-lock.json index fb40c3582d..66ede17876 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3905,6 +3905,17 @@ "cssom": "0.3.x" } }, + "csvtojson": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/csvtojson/-/csvtojson-2.0.8.tgz", + "integrity": "sha512-DC6YFtsJiA7t/Yz+KjzT6GXuKtU/5gRbbl7HJqvDVVir+dxdw2/1EgwfgJdnsvUT7lOnON5DvGftKuYWX1nMOQ==", + "dev": true, + "requires": { + "bluebird": "^3.5.1", + "lodash": "^4.17.3", + "strip-bom": "^2.0.0" + } + }, "cuint": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz", diff --git a/package.json b/package.json index 6d5e189fd8..bfebe2876f 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "connect-history-api-fallback": "1.3.0", "cross-env": "^5.2.0", "css-loader": "0.28.4", + "csvtojson": "^2.0.8", "detect-port": "1.1.0", "dotenv": "4.0.0", "electron": "^2.0.11", @@ -244,4 +245,4 @@ "cordova-plugin-chooser": {} } } -} \ No newline at end of file +} diff --git a/src/containers/withExternalData.js b/src/containers/withExternalData.js index e81758bdc0..109c3b6b9f 100644 --- a/src/containers/withExternalData.js +++ b/src/containers/withExternalData.js @@ -20,13 +20,15 @@ const mapStateToProps = (state) => { const session = state.sessions[state.activeSessionId]; const protocolUID = session.protocolUID; const protocolCodebook = state.installedProtocols[protocolUID].codebook; + const assetManifest = state.installedProtocols[protocolUID].assetManifest; const assetFiles = mapValues( - state.installedProtocols[protocolUID].assetManifest, + assetManifest, asset => asset.source, ); return { protocolUID, + assetManifest, assetFiles, protocolCodebook, }; @@ -59,6 +61,24 @@ const getObjectUUIDByValue = (object, toFind) => { return foundKey || toFind; }; +const withUUIDReplacement = (nodeList, codebook) => nodeList.map( + (node) => { + const nodeTypeUUID = getObjectUUIDByValue(codebook.node, node.type); + const codebookDefinition = codebook.node[nodeTypeUUID] || {}; + + const attributes = mapKeys(node[entityAttributesProperty], + (attributeValue, attributeKey) => + getObjectUUIDByValue(codebookDefinition.variables, attributeKey), + ); + + return { + ...node, + type: nodeTypeUUID, + [entityAttributesProperty]: attributes, + }; + }, +); + /** * Creates a higher order component which can be used to load data from network assets in * the assetsManifest onto a component. @@ -95,38 +115,22 @@ const withExternalData = (sourceProperty, dataProperty) => setExternalData, protocolUID, assetFiles, + assetManifest, protocolCodebook, }) => (sourceId) => { if (!sourceId) { return; } - // This is where we could set the loading state for URL assets setExternalData(null); const sourceFile = assetFiles[sourceId]; + const type = assetManifest[sourceId].type; - loadExternalData(protocolUID, sourceFile) - .then((externalData) => { - const withUUIDReplacement = nodeList => nodeList.map( - (node) => { - const nodeTypeUUID = getObjectUUIDByValue(protocolCodebook.node, node.type); - const codebookDefinition = protocolCodebook.node[nodeTypeUUID] || {}; - - const attributes = mapKeys(node[entityAttributesProperty], - (attributeValue, attributeKey) => - getObjectUUIDByValue(codebookDefinition.variables, attributeKey), - ); - - return { - ...node, - type: nodeTypeUUID, - [entityAttributesProperty]: attributes, - }; - }, - ); - - setExternalData({ nodes: withUUIDReplacement(externalData.nodes) }); - }); + loadExternalData(protocolUID, sourceFile, type) + .then(externalData => + setExternalData({ + nodes: withUUIDReplacement(externalData.nodes, protocolCodebook), + })); }, }), lifecycle({ diff --git a/src/utils/loadExternalData.js b/src/utils/loadExternalData.js index 67d151c126..d17b695c30 100644 --- a/src/utils/loadExternalData.js +++ b/src/utils/loadExternalData.js @@ -1,10 +1,11 @@ /* eslint-disable quotes, quote-props, comma-dangle */ import objectHash from 'object-hash'; -import { get } from 'lodash'; +import * as csv from 'csvtojson'; +import { get, omit } from 'lodash'; import environments from './environments'; import inEnvironment from './Environment'; import { readFile } from './filesystem'; -import { entityPrimaryKeyProperty } from '../ducks/modules/network'; +import { entityPrimaryKeyProperty, entityAttributesProperty } from '../ducks/modules/network'; import getAssetUrl from './protocol/getAssetUrl'; const withKeys = data => @@ -17,12 +18,43 @@ const withKeys = data => }; }); +/** + * Converts a CSV file into a Network Canvas node list JSON + * + * @param {string} data - the contents of a CSV file + * + * See: https://github.com/Keyang/node-csvtojson We may want to introduce buffering + * to this function to increase performance particularly on cordova. + * + */ +const CSVToJSONNetworkFormat = (data) => { + const withTypeAndAttributes = node => ({ + type: node.type, + [entityAttributesProperty]: { + ...omit(node, 'type'), + } + }); + + return csv().fromString(data) + .then((json) => { + const nodeList = json.map(entry => withTypeAndAttributes(entry)); + return { nodes: nodeList }; + }); +}; + const fetchNetwork = inEnvironment( (environment) => { if (environment === environments.ELECTRON || environment === environments.WEB) { - return url => + return (url, fileType) => fetch(url) - .then(response => response.json()) + .then((response) => { + if (fileType === 'csv') { + return response.text().then( + data => CSVToJSONNetworkFormat(data) + ); + } + return response.json(); + }) .then((json) => { const nodes = get(json, 'nodes', []); return ({ nodes: withKeys(nodes) }); @@ -30,8 +62,13 @@ const fetchNetwork = inEnvironment( } if (environment === environments.CORDOVA) { - return url => readFile(url) - .then(response => JSON.parse(response)) + return (url, fileType) => readFile(url) + .then((response) => { + if (fileType === 'csv') { + return CSVToJSONNetworkFormat(response); + } + return JSON.parse(response); + }) .then((json) => { const nodes = get(json, 'nodes', []); return ({ nodes: withKeys(nodes) }); @@ -42,14 +79,6 @@ const fetchNetwork = inEnvironment( }, ); -const parseJSON = (fileContents) => { - -}; - -const parseCSV = (fileContents) => { - -}; - const fileExtension = fileName => fileName.split('.').pop(); /** @@ -60,12 +89,16 @@ const fileExtension = fileName => fileName.split('.').pop(); * @returns {object} Network object in format { nodes, edges } * */ -const loadExternalData = (protocolUID, fileName) => { +const loadExternalData = (protocolUID, fileName, type) => { const fileType = fileExtension(fileName) === 'csv' ? 'csv' : 'json'; - return getAssetUrl(protocolUID, fileName) - .then(url => fetchNetwork(url, fileType)); + switch (type) { + case 'network': + return getAssetUrl(protocolUID, fileName) + .then(url => fetchNetwork(url, fileType)); + default: + return new Error('You must specify an external data type.'); + } }; - export default loadExternalData; From 524f76ef68f15d39d834b0c8bd28cdee01ed3d78 Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Wed, 10 Apr 2019 11:11:03 +0100 Subject: [PATCH 04/11] fix withExternalData test --- .../__tests__/withExternalData.test.js | 24 +++++++++++++++++-- src/containers/withExternalData.js | 3 ++- src/utils/__tests__/loadExternalData.test.js | 4 ++-- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/containers/__tests__/withExternalData.test.js b/src/containers/__tests__/withExternalData.test.js index 5c9a529f6e..037dd04664 100644 --- a/src/containers/__tests__/withExternalData.test.js +++ b/src/containers/__tests__/withExternalData.test.js @@ -5,12 +5,25 @@ import { mount } from 'enzyme'; import { last } from 'lodash'; import withExternalData from '../withExternalData'; import loadExternalData from '../../utils/loadExternalData'; +import { entityAttributesProperty } from '../../ducks/modules/network'; jest.mock('../../utils/loadExternalData'); const mockReducer = () => ({ installedProtocols: { - mockProtocol: {}, + mockProtocol: { + codebook: { + node: {}, + edge: {}, + }, + assetManifest: { + bar: { + name: 'bar', + source: 'file.json', + type: 'network', + }, + }, + }, }, activeSessionId: 'foo', sessions: { @@ -21,7 +34,14 @@ const mockReducer = () => ({ }); const mockResult = { - bar: 'bazz', + nodes: [ + { + type: 'person', + [entityAttributesProperty]: { + fun: true, + }, + }, + ], }; const mockSource = 'bar'; diff --git a/src/containers/withExternalData.js b/src/containers/withExternalData.js index 109c3b6b9f..1f96ef5d16 100644 --- a/src/containers/withExternalData.js +++ b/src/containers/withExternalData.js @@ -12,6 +12,7 @@ import { mapValues, mapKeys, findKey, + isEmpty, } from 'lodash'; import loadExternalData from '../utils/loadExternalData'; import { entityAttributesProperty } from '../ducks/modules/network'; @@ -51,7 +52,7 @@ const mapStateToProps = (state) => { * Finally, if neither approach finds a UUID, {toFind} is returned. */ const getObjectUUIDByValue = (object, toFind) => { - if (object[toFind]) { // should pass if UUID is used + if (isEmpty(object) || object[toFind]) { return toFind; } diff --git a/src/utils/__tests__/loadExternalData.test.js b/src/utils/__tests__/loadExternalData.test.js index 818dc48bd5..34fae9f2f0 100644 --- a/src/utils/__tests__/loadExternalData.test.js +++ b/src/utils/__tests__/loadExternalData.test.js @@ -4,7 +4,7 @@ import loadExternalData from '../loadExternalData'; const mockProtocolName = 'myMockProtocol'; const mockAssetName = 'myMockSource'; -const mockProtocolType = null; +const mockAssetType = 'network'; const mockResult = { nodes: [ { foo: 'bar' }, @@ -19,7 +19,7 @@ global.fetch = jest.fn(() => Promise.resolve(mockFetchResponse)); describe('loadExternalData', () => { it('returns a cancellable request'); it('request response is json with uids ', (done) => { - loadExternalData(mockProtocolName, mockAssetName, mockProtocolType) + loadExternalData(mockProtocolName, mockAssetName, mockAssetType) .then((result) => { expect(result.nodes.length).toBe(mockResult.nodes.length); expect(result.nodes.every( From 7056487d990ca106e86e1bd92071daecacff43f4 Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Thu, 11 Apr 2019 13:32:10 +0100 Subject: [PATCH 05/11] Update SettingsMenu.js --- src/components/MainMenu/SettingsMenu.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/MainMenu/SettingsMenu.js b/src/components/MainMenu/SettingsMenu.js index 44ff9c42b0..ccf1c33742 100644 --- a/src/components/MainMenu/SettingsMenu.js +++ b/src/components/MainMenu/SettingsMenu.js @@ -83,7 +83,7 @@ class SettingsMenu extends PureComponent {

From 88a89fb5e04dbe89eafb02141386e17ffb37102a Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Fri, 12 Apr 2019 13:28:46 +0100 Subject: [PATCH 06/11] convert buffer to string prior to passing to csv2json --- package.json | 2 +- src/utils/loadExternalData.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index bfebe2876f..ac54bfacf0 100644 --- a/package.json +++ b/package.json @@ -245,4 +245,4 @@ "cordova-plugin-chooser": {} } } -} +} \ No newline at end of file diff --git a/src/utils/loadExternalData.js b/src/utils/loadExternalData.js index d17b695c30..41af99f126 100644 --- a/src/utils/loadExternalData.js +++ b/src/utils/loadExternalData.js @@ -65,7 +65,7 @@ const fetchNetwork = inEnvironment( return (url, fileType) => readFile(url) .then((response) => { if (fileType === 'csv') { - return CSVToJSONNetworkFormat(response); + return CSVToJSONNetworkFormat(response.toString('utf8')); } return JSON.parse(response); }) From 71094fdc269b5bee5379d845f27edc7aa3ca9490 Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Fri, 12 Apr 2019 15:26:59 +0100 Subject: [PATCH 07/11] setup for worker --- public/index.html | 2 +- src/containers/withExternalData.js | 1 - src/utils/csvDecoder.worker.js | 35 +++++++++++++++++++++++ src/utils/loadExternalData.js | 46 +++++++++++------------------- 4 files changed, 53 insertions(+), 31 deletions(-) create mode 100644 src/utils/csvDecoder.worker.js diff --git a/public/index.html b/public/index.html index 75f63fc789..ea0a4268ba 100644 --- a/public/index.html +++ b/public/index.html @@ -12,7 +12,7 @@ style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; font-src 'self' data:; - worker-src blob:; + worker-src 'self' blob:; " /> diff --git a/src/containers/withExternalData.js b/src/containers/withExternalData.js index 1f96ef5d16..6987d909ad 100644 --- a/src/containers/withExternalData.js +++ b/src/containers/withExternalData.js @@ -35,7 +35,6 @@ const mapStateToProps = (state) => { }; }; - /** * Utility function that can be used to help with translating external data * variable labels to UUIDs, if a match is possible. diff --git a/src/utils/csvDecoder.worker.js b/src/utils/csvDecoder.worker.js new file mode 100644 index 0000000000..aa4ce03341 --- /dev/null +++ b/src/utils/csvDecoder.worker.js @@ -0,0 +1,35 @@ +// import * as csv from 'csvtojson'; +// import { omit } from 'lodash'; +// import { entityAttributesProperty } from '../ducks/modules/network'; + +// /** +// * Converts a CSV file into a Network Canvas node list JSON +// * +// * @param {string} data - the contents of a CSV file +// * +// * See: https://github.com/Keyang/node-csvtojson We may want to introduce buffering +// * to this function to increase performance particularly on cordova. +// * +// */ +// const CSVToJSONNetworkFormat = (data) => { +// const withTypeAndAttributes = node => ({ +// type: node.type, +// [entityAttributesProperty]: { +// ...omit(node, 'type'), +// }, +// }); + +// csv().fromString(data) +// .then((json) => { +// const nodeList = json.map(entry => withTypeAndAttributes(entry)); +// postMessage({ nodes: nodeList }); +// }); +// }; + +// onmessage = e => CSVToJSONNetworkFormat(e.data); + +// Post data to parent thread +self.postMessage({ foo: 'foo' }) + +// Respond to message from parent thread +self.addEventListener('message', (event) => console.log(event)) diff --git a/src/utils/loadExternalData.js b/src/utils/loadExternalData.js index 41af99f126..61d45bc15a 100644 --- a/src/utils/loadExternalData.js +++ b/src/utils/loadExternalData.js @@ -1,12 +1,12 @@ /* eslint-disable quotes, quote-props, comma-dangle */ import objectHash from 'object-hash'; -import * as csv from 'csvtojson'; -import { get, omit } from 'lodash'; +import { get } from 'lodash'; import environments from './environments'; import inEnvironment from './Environment'; import { readFile } from './filesystem'; -import { entityPrimaryKeyProperty, entityAttributesProperty } from '../ducks/modules/network'; +import { entityPrimaryKeyProperty } from '../ducks/modules/network'; import getAssetUrl from './protocol/getAssetUrl'; +import Worker from './utils/csvDecover.worker.js'; const withKeys = data => data.map((node) => { @@ -18,30 +18,6 @@ const withKeys = data => }; }); -/** - * Converts a CSV file into a Network Canvas node list JSON - * - * @param {string} data - the contents of a CSV file - * - * See: https://github.com/Keyang/node-csvtojson We may want to introduce buffering - * to this function to increase performance particularly on cordova. - * - */ -const CSVToJSONNetworkFormat = (data) => { - const withTypeAndAttributes = node => ({ - type: node.type, - [entityAttributesProperty]: { - ...omit(node, 'type'), - } - }); - - return csv().fromString(data) - .then((json) => { - const nodeList = json.map(entry => withTypeAndAttributes(entry)); - return { nodes: nodeList }; - }); -}; - const fetchNetwork = inEnvironment( (environment) => { if (environment === environments.ELECTRON || environment === environments.WEB) { @@ -50,9 +26,16 @@ const fetchNetwork = inEnvironment( .then((response) => { if (fileType === 'csv') { return response.text().then( - data => CSVToJSONNetworkFormat(data) + (data) => { + const worker = new Worker(); + // Send the worker the node model properties along with the network + worker.postMessage(data); + + worker.onmessage(decodedData => decodedData.data); + } ); } + return response.json(); }) .then((json) => { @@ -65,7 +48,12 @@ const fetchNetwork = inEnvironment( return (url, fileType) => readFile(url) .then((response) => { if (fileType === 'csv') { - return CSVToJSONNetworkFormat(response.toString('utf8')); + const webWorker = new WorkerAgent('something'); + // Send the worker the node model properties along with the network + const msgPromise = webWorker.sendMessageAsync(response.toString('utf8')); + + return msgPromise + .then(decodedData => decodedData).catch(workerError => workerError); } return JSON.parse(response); }) From bd95909dea506a5071e55ecacb8efd167af92b0e Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Fri, 12 Apr 2019 18:42:39 +0100 Subject: [PATCH 08/11] implement CSV parsing inside of web worker --- config/webpack.config.dev.js | 15 ++++++++ package-lock.json | 22 ++++++++++++ package.json | 3 +- public/components/windowManager.js | 4 ++- src/utils/csvDecoder.worker.js | 56 ++++++++++++++---------------- src/utils/loadExternalData.js | 39 +++++++++++++-------- 6 files changed, 92 insertions(+), 47 deletions(-) diff --git a/config/webpack.config.dev.js b/config/webpack.config.dev.js index 8eacad9e0a..4b6bb615ac 100644 --- a/config/webpack.config.dev.js +++ b/config/webpack.config.dev.js @@ -142,6 +142,21 @@ module.exports = { // match the requirements. When no loader matches it will fall // back to the "file" loader at the end of the loader list. oneOf: [ + { + test: /\.worker\.js$/, + use: [ + { loader: 'worker-loader' }, + { + loader: require.resolve('babel-loader'), + options: { + // This is a feature of `babel-loader` for webpack (not Babel itself). + // It enables caching results in ./node_modules/.cache/babel-loader/ + // directory for faster rebuilds. + cacheDirectory: true, + }, + }, + ], + }, // "url" loader works like "file" loader except that it embeds assets // smaller than specified limit in bytes as data URLs to avoid requests. // A missing `test` is equivalent to a match. diff --git a/package-lock.json b/package-lock.json index 66ede17876..b07bfc411a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21814,6 +21814,28 @@ "errno": "~0.1.7" } }, + "worker-loader": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/worker-loader/-/worker-loader-2.0.0.tgz", + "integrity": "sha512-tnvNp4K3KQOpfRnD20m8xltE3eWh89Ye+5oj7wXEEHKac1P4oZ6p9oTj8/8ExqoSBnk9nu5Pr4nKfQ1hn2APJw==", + "dev": true, + "requires": { + "loader-utils": "^1.0.0", + "schema-utils": "^0.4.0" + }, + "dependencies": { + "schema-utils": { + "version": "0.4.7", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-0.4.7.tgz", + "integrity": "sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ==", + "dev": true, + "requires": { + "ajv": "^6.1.0", + "ajv-keywords": "^3.1.0" + } + } + } + }, "wrap-ansi": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", diff --git a/package.json b/package.json index ac54bfacf0..4ca93e407a 100644 --- a/package.json +++ b/package.json @@ -153,6 +153,7 @@ "webpack-dev-server": "2.7.1", "webpack-manifest-plugin": "^1.3.1", "whatwg-fetch": "2.0.3", + "worker-loader": "^2.0.0", "xss": "^0.3.4" }, "dependencies": { @@ -245,4 +246,4 @@ "cordova-plugin-chooser": {} } } -} \ No newline at end of file +} diff --git a/public/components/windowManager.js b/public/components/windowManager.js index d62c021811..092988f4cf 100644 --- a/public/components/windowManager.js +++ b/public/components/windowManager.js @@ -32,7 +32,9 @@ function createWindow() { center: true, fullscreen: true, title: 'Network Canvas', - + webPreferences: { + nodeIntegrationInWorker: true, + }, }, titlebarParameters); const mainWindow = new BrowserWindow(windowParameters); diff --git a/src/utils/csvDecoder.worker.js b/src/utils/csvDecoder.worker.js index aa4ce03341..c52e1ad339 100644 --- a/src/utils/csvDecoder.worker.js +++ b/src/utils/csvDecoder.worker.js @@ -1,35 +1,31 @@ -// import * as csv from 'csvtojson'; -// import { omit } from 'lodash'; -// import { entityAttributesProperty } from '../ducks/modules/network'; +import * as csv from 'csvtojson'; +import { omit } from 'lodash'; +import { entityAttributesProperty } from '../ducks/modules/network'; -// /** -// * Converts a CSV file into a Network Canvas node list JSON -// * -// * @param {string} data - the contents of a CSV file -// * -// * See: https://github.com/Keyang/node-csvtojson We may want to introduce buffering -// * to this function to increase performance particularly on cordova. -// * -// */ -// const CSVToJSONNetworkFormat = (data) => { -// const withTypeAndAttributes = node => ({ -// type: node.type, -// [entityAttributesProperty]: { -// ...omit(node, 'type'), -// }, -// }); +/** + * Converts a CSV file into a Network Canvas node list JSON + * + * @param {string} data - the contents of a CSV file + * + * See: https://github.com/Keyang/node-csvtojson We may want to introduce buffering + * to this function to increase performance particularly on cordova. + * + */ -// csv().fromString(data) -// .then((json) => { -// const nodeList = json.map(entry => withTypeAndAttributes(entry)); -// postMessage({ nodes: nodeList }); -// }); -// }; +const CSVToJSONNetworkFormat = (data) => { + const withTypeAndAttributes = node => ({ + type: node.type, + [entityAttributesProperty]: { + ...omit(node, 'type'), + }, + }); -// onmessage = e => CSVToJSONNetworkFormat(e.data); - -// Post data to parent thread -self.postMessage({ foo: 'foo' }) + csv().fromString(data) + .then((json) => { + const nodeList = json.map(entry => withTypeAndAttributes(entry)); + self.postMessage({ nodes: nodeList }); + }); +}; // Respond to message from parent thread -self.addEventListener('message', (event) => console.log(event)) +self.addEventListener('message', event => CSVToJSONNetworkFormat(event.data)); diff --git a/src/utils/loadExternalData.js b/src/utils/loadExternalData.js index 61d45bc15a..2a7f33ac32 100644 --- a/src/utils/loadExternalData.js +++ b/src/utils/loadExternalData.js @@ -6,7 +6,7 @@ import inEnvironment from './Environment'; import { readFile } from './filesystem'; import { entityPrimaryKeyProperty } from '../ducks/modules/network'; import getAssetUrl from './protocol/getAssetUrl'; -import Worker from './utils/csvDecover.worker.js'; +import Worker from './csvDecoder.worker'; const withKeys = data => data.map((node) => { @@ -25,15 +25,19 @@ const fetchNetwork = inEnvironment( fetch(url) .then((response) => { if (fileType === 'csv') { - return response.text().then( - (data) => { - const worker = new Worker(); - // Send the worker the node model properties along with the network - worker.postMessage(data); - - worker.onmessage(decodedData => decodedData.data); - } - ); + const worker = new Worker(); + return response.text() + .then( + data => new Promise((resolve, reject) => { + worker.postMessage(data); + worker.onerror = (event) => { + reject(event); + }; + worker.onmessage = (event) => { + resolve(event.data); + }; + }) + ); } return response.json(); @@ -48,12 +52,17 @@ const fetchNetwork = inEnvironment( return (url, fileType) => readFile(url) .then((response) => { if (fileType === 'csv') { - const webWorker = new WorkerAgent('something'); - // Send the worker the node model properties along with the network - const msgPromise = webWorker.sendMessageAsync(response.toString('utf8')); + const worker = new Worker(); - return msgPromise - .then(decodedData => decodedData).catch(workerError => workerError); + return new Promise((resolve, reject) => { + worker.postMessage(response.toString('utf8')); + worker.onerror = (event) => { + reject(event); + }; + worker.onmessage = (event) => { + resolve(event.data); + }; + }); } return JSON.parse(response); }) From 7419cb708a6f72e38b89458b0bb885f405f9fc60 Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Mon, 15 Apr 2019 09:48:30 +0100 Subject: [PATCH 09/11] make worker inline (using blob) to avoid cross origin issue --- config/webpack.config.dev.js | 5 +++- config/webpack.config.prod.js | 18 +++++++++++++ package.json | 2 +- src/utils/loadExternalData.js | 49 ++++++++++++++++++----------------- 4 files changed, 48 insertions(+), 26 deletions(-) diff --git a/config/webpack.config.dev.js b/config/webpack.config.dev.js index 4b6bb615ac..5e497bb12a 100644 --- a/config/webpack.config.dev.js +++ b/config/webpack.config.dev.js @@ -145,7 +145,10 @@ module.exports = { { test: /\.worker\.js$/, use: [ - { loader: 'worker-loader' }, + { + loader: 'worker-loader', + options: { inline: true }, + }, { loader: require.resolve('babel-loader'), options: { diff --git a/config/webpack.config.prod.js b/config/webpack.config.prod.js index 25215c2ea8..f1f40a0c0e 100644 --- a/config/webpack.config.prod.js +++ b/config/webpack.config.prod.js @@ -139,6 +139,24 @@ module.exports = { name: 'static/media/[name].[hash:8].[ext]', }, }, + { + test: /\.worker\.js$/, + use: [ + { + loader: 'worker-loader', + options: { inline: true }, + }, + { + loader: require.resolve('babel-loader'), + options: { + // This is a feature of `babel-loader` for webpack (not Babel itself). + // It enables caching results in ./node_modules/.cache/babel-loader/ + // directory for faster rebuilds. + cacheDirectory: true, + }, + }, + ], + }, // Process JS with Babel. { test: /\.(js|jsx)$/, diff --git a/package.json b/package.json index 4ca93e407a..11c2779e2c 100644 --- a/package.json +++ b/package.json @@ -246,4 +246,4 @@ "cordova-plugin-chooser": {} } } -} +} \ No newline at end of file diff --git a/src/utils/loadExternalData.js b/src/utils/loadExternalData.js index 2a7f33ac32..b81a127b27 100644 --- a/src/utils/loadExternalData.js +++ b/src/utils/loadExternalData.js @@ -18,6 +18,29 @@ const withKeys = data => }; }); + +/** + * Converting data from CSV to our network JSON format is expensive, and so happens + * inside of a worker to keep the app as responsive as possible. + * + * This function takes the result of the platform-specific file load operation, + * and then initialises the conversion worker, before sending it the file contents + * to decode. + */ +const convertCSVToJsonWithWorker = response => response.text() + .then( + data => new Promise((resolve, reject) => { + const worker = new Worker(); + worker.postMessage(data); + worker.onerror = (event) => { + reject(event); + }; + worker.onmessage = (event) => { + resolve(event.data); + }; + }) + ); + const fetchNetwork = inEnvironment( (environment) => { if (environment === environments.ELECTRON || environment === environments.WEB) { @@ -25,19 +48,7 @@ const fetchNetwork = inEnvironment( fetch(url) .then((response) => { if (fileType === 'csv') { - const worker = new Worker(); - return response.text() - .then( - data => new Promise((resolve, reject) => { - worker.postMessage(data); - worker.onerror = (event) => { - reject(event); - }; - worker.onmessage = (event) => { - resolve(event.data); - }; - }) - ); + return convertCSVToJsonWithWorker(response); } return response.json(); @@ -52,17 +63,7 @@ const fetchNetwork = inEnvironment( return (url, fileType) => readFile(url) .then((response) => { if (fileType === 'csv') { - const worker = new Worker(); - - return new Promise((resolve, reject) => { - worker.postMessage(response.toString('utf8')); - worker.onerror = (event) => { - reject(event); - }; - worker.onmessage = (event) => { - resolve(event.data); - }; - }); + return convertCSVToJsonWithWorker(response.toString('utf8')); } return JSON.parse(response); }) From 0102d175f542ea370c70d82801e58c051d776173 Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Mon, 15 Apr 2019 10:46:32 +0100 Subject: [PATCH 10/11] remove node integration in web worker --- package-lock.json | 351 ++++++++++++++++++++++++++--- public/components/windowManager.js | 3 - src/utils/csvDecoder.worker.js | 3 +- 3 files changed, 325 insertions(+), 32 deletions(-) diff --git a/package-lock.json b/package-lock.json index b07bfc411a..47b4b93823 100644 --- a/package-lock.json +++ b/package-lock.json @@ -49,6 +49,259 @@ } } }, + "@oclif/command": { + "version": "1.5.12", + "resolved": "https://registry.npmjs.org/@oclif/command/-/command-1.5.12.tgz", + "integrity": "sha512-D5/Kph9smL92X1z9WPmxFd9zDruFsCk4/LbfCaBmiO2Vyyt7Y9O6kI1YLsC3B0KC9wymSCTH14IK96rf9AFHfQ==", + "requires": { + "@oclif/errors": "^1.2.2", + "@oclif/parser": "^3.7.3", + "debug": "^4.1.1", + "semver": "^5.6.0" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + }, + "semver": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==" + } + } + }, + "@oclif/config": { + "version": "1.12.12", + "resolved": "https://registry.npmjs.org/@oclif/config/-/config-1.12.12.tgz", + "integrity": "sha512-0vlX5VYvOfF9QbkCqMyPSzH9GMp6at4Mbqn8CxCskxhKvNZoPD5ocda2ku0zEnoqxGAQ4VfQP7NCqJthuiStfg==", + "requires": { + "debug": "^4.1.1", + "tslib": "^1.9.3" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + } + } + }, + "@oclif/errors": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@oclif/errors/-/errors-1.2.2.tgz", + "integrity": "sha512-Eq8BFuJUQcbAPVofDxwdE0bL14inIiwt5EaKRVY9ZDIG11jwdXZqiQEECJx0VfnLyUZdYfRd/znDI/MytdJoKg==", + "requires": { + "clean-stack": "^1.3.0", + "fs-extra": "^7.0.0", + "indent-string": "^3.2.0", + "strip-ansi": "^5.0.0", + "wrap-ansi": "^4.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==" + }, + "fs-extra": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", + "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", + "requires": { + "graceful-fs": "^4.1.2", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, + "indent-string": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz", + "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=" + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" + }, + "jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "requires": { + "graceful-fs": "^4.1.6" + } + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "requires": { + "ansi-regex": "^3.0.0" + } + } + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "requires": { + "ansi-regex": "^4.1.0" + } + }, + "wrap-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-4.0.0.tgz", + "integrity": "sha512-uMTsj9rDb0/7kk1PbcbCcwvHUxp60fGDB/NNXpVa0Q+ic/e7y5+BwTxKfQ33VYgDppSwi/FBzpetYzo8s6tfbg==", + "requires": { + "ansi-styles": "^3.2.0", + "string-width": "^2.1.1", + "strip-ansi": "^4.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "requires": { + "ansi-regex": "^3.0.0" + } + } + } + } + } + }, + "@oclif/linewrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@oclif/linewrap/-/linewrap-1.0.0.tgz", + "integrity": "sha512-Ups2dShK52xXa8w6iBWLgcjPJWjais6KPJQq3gQ/88AY6BXoTX+MIGFPrWQO1KLMiQfoTpcLnUwloN4brrVUHw==" + }, + "@oclif/parser": { + "version": "3.7.3", + "resolved": "https://registry.npmjs.org/@oclif/parser/-/parser-3.7.3.tgz", + "integrity": "sha512-yfYpDzVn9ipo4HZtYLfMtd3j3ArpTQlRbQfy9pNnHFd4VedE2PNYQTRWYYMuu1FxEOoknlMZbzsewVvl41TvKg==", + "requires": { + "@oclif/linewrap": "^1.0.0", + "chalk": "^2.4.1", + "tslib": "^1.9.3" + }, + "dependencies": { + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + } + } + }, + "@oclif/plugin-help": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/@oclif/plugin-help/-/plugin-help-1.2.11.tgz", + "integrity": "sha512-tuzhvxxRtfLnWa96klngXBi5IwHt9S/twedCbQhl9dYIKTFMHI1BcOQcPra6ylct+M+b9jhEF5sjWLv78tB6tw==", + "requires": { + "@oclif/command": "^1.4.29", + "chalk": "^2.4.1", + "indent-string": "^3.2.0", + "lodash.template": "^4.4.0", + "string-width": "^2.1.1", + "widest-line": "^2.0.0", + "wrap-ansi": "^3.0.1" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "indent-string": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz", + "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=" + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "requires": { + "ansi-regex": "^3.0.0" + } + }, + "wrap-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-3.0.1.tgz", + "integrity": "sha1-KIoE2H7aXChuBg3+jxNc6NAH+Lo=", + "requires": { + "string-width": "^2.1.1", + "strip-ansi": "^4.0.0" + } + } + } + }, "@types/node": { "version": "8.10.34", "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.34.tgz", @@ -293,7 +546,6 @@ "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, "requires": { "color-convert": "^1.9.0" } @@ -2861,6 +3113,11 @@ } } }, + "clean-stack": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-1.3.0.tgz", + "integrity": "sha1-noIVAa6XmYbEax1m0tQy2y/UrjE=" + }, "cli-boxes": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-1.0.0.tgz", @@ -3006,7 +3263,6 @@ "version": "1.9.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.1.tgz", "integrity": "sha512-mjGanIiwQJskCC18rPR6OmrZ6fm2Lc7PeGFYwCmy5J34wC6F1PzdGL6xeMfmgicfYcNLGuVFA3WzXtIDCQSZxQ==", - "dev": true, "requires": { "color-name": "^1.1.1" } @@ -3014,8 +3270,7 @@ "color-name": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", - "dev": true + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" }, "color-string": { "version": "1.5.2", @@ -3396,6 +3651,32 @@ "xml-escape": "^1.1.0" }, "dependencies": { + "bplist-parser": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/bplist-parser/-/bplist-parser-0.0.6.tgz", + "integrity": "sha1-ONo0cYF9+dRKs4kuJ3B7u9daEbk=" + }, + "ios-sim": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/ios-sim/-/ios-sim-7.0.0.tgz", + "integrity": "sha512-VloxT+AARztnhkGQcfxPjMU8puewPULVA+qzAOrK5JspiWTvi7JhBV8t19x42It+tsX35ZabF1WyUZhorLDbvQ==", + "requires": { + "bplist-parser": "^0.0.6", + "nopt": "1.0.9", + "plist": "^3.0.1", + "simctl": "^1.1.1" + }, + "dependencies": { + "nopt": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.9.tgz", + "integrity": "sha1-O8DXy6e/sNWmdtvtfA6+SKT9RU4=", + "requires": { + "abbrev": "1" + } + } + } + }, "nopt": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", @@ -3409,6 +3690,22 @@ "version": "0.5.3", "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", "integrity": "sha1-xUmCuZbHbvDB5rWfvcWCX1txMRM=" + }, + "simctl": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/simctl/-/simctl-1.1.1.tgz", + "integrity": "sha512-yY1WQMq/pneY5jQb2+lFp45qEtcz4yKBu1NOPo2OFDVCkwSkQhpkoaAaO1fWhq4IU0+8TQ2r1PMGSTedP0A/Og==", + "requires": { + "shelljs": "^0.2.6", + "tail": "^0.4.0" + }, + "dependencies": { + "shelljs": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.2.6.tgz", + "integrity": "sha1-kEktcv/MgVmXa6umL7D2iE8MM3g=" + } + } } } }, @@ -5370,8 +5667,7 @@ "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", - "dev": true + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, "escodegen": { "version": "1.11.0", @@ -7762,8 +8058,7 @@ "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", - "dev": true + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" }, "has-symbols": { "version": "1.0.0", @@ -8512,14 +8807,18 @@ "dev": true }, "ios-sim": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/ios-sim/-/ios-sim-7.0.0.tgz", - "integrity": "sha512-VloxT+AARztnhkGQcfxPjMU8puewPULVA+qzAOrK5JspiWTvi7JhBV8t19x42It+tsX35ZabF1WyUZhorLDbvQ==", - "requires": { + "version": "9.0.0-dev.4", + "resolved": "https://registry.npmjs.org/ios-sim/-/ios-sim-9.0.0-dev.4.tgz", + "integrity": "sha512-q6ui9JmhhputRRMRK72Z0hKgzc5v+CiAW3CJAbkbmeIT12xutbiZTO72Y1UHeD7hB0PXASjeCgwdaRfQ2vr3VQ==", + "requires": { + "@oclif/command": "^1.5.6", + "@oclif/config": "^1.9.0", + "@oclif/errors": "^1.1.2", + "@oclif/plugin-help": "^1", "bplist-parser": "^0.0.6", "nopt": "1.0.9", "plist": "^3.0.1", - "simctl": "^1.1.1" + "simctl": "^2" }, "dependencies": { "bplist-parser": { @@ -10892,8 +11191,7 @@ "lodash._reinterpolate": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz", - "integrity": "sha1-DM8tiRZq8Ds2Y8eWU4t1rG4RTZ0=", - "dev": true + "integrity": "sha1-DM8tiRZq8Ds2Y8eWU4t1rG4RTZ0=" }, "lodash.assign": { "version": "4.2.0", @@ -10995,7 +11293,6 @@ "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.template/-/lodash.template-4.4.0.tgz", "integrity": "sha1-5zoDhcg1VZF0bgILmWecaQ5o+6A=", - "dev": true, "requires": { "lodash._reinterpolate": "~3.0.0", "lodash.templatesettings": "^4.0.0" @@ -11005,7 +11302,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/lodash.templatesettings/-/lodash.templatesettings-4.1.0.tgz", "integrity": "sha1-K01OlbpEDZFf8IvImeRVNmZxMxY=", - "dev": true, "requires": { "lodash._reinterpolate": "~3.0.0" } @@ -18795,9 +19091,9 @@ "dev": true }, "simctl": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/simctl/-/simctl-1.1.1.tgz", - "integrity": "sha512-yY1WQMq/pneY5jQb2+lFp45qEtcz4yKBu1NOPo2OFDVCkwSkQhpkoaAaO1fWhq4IU0+8TQ2r1PMGSTedP0A/Og==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/simctl/-/simctl-2.0.0.tgz", + "integrity": "sha512-5rB7rN4N3b0z0nFdy9eczVssXqrv2aAgdVRksPVqVoiDtvXmfzNvebp3EMdId2sAUzXIflarQlx4P0hjVQEzKQ==", "requires": { "shelljs": "^0.2.6", "tail": "^0.4.0" @@ -19585,7 +19881,6 @@ "version": "5.4.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz", "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==", - "dev": true, "requires": { "has-flag": "^3.0.0" } @@ -20278,6 +20573,11 @@ "utf8-byte-length": "^1.0.1" } }, + "tslib": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.3.tgz", + "integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ==" + }, "tty-browserify": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz", @@ -21755,7 +22055,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-2.0.0.tgz", "integrity": "sha1-AUKk6KJD+IgsAjOqDgKBqnYVInM=", - "dev": true, "requires": { "string-width": "^2.1.1" }, @@ -21763,20 +22062,17 @@ "ansi-regex": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", - "dev": true + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" }, "is-fullwidth-code-point": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", - "dev": true + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" }, "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", - "dev": true, "requires": { "is-fullwidth-code-point": "^2.0.0", "strip-ansi": "^4.0.0" @@ -21786,7 +22082,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", - "dev": true, "requires": { "ansi-regex": "^3.0.0" } diff --git a/public/components/windowManager.js b/public/components/windowManager.js index 092988f4cf..5f0e8f4abf 100644 --- a/public/components/windowManager.js +++ b/public/components/windowManager.js @@ -32,9 +32,6 @@ function createWindow() { center: true, fullscreen: true, title: 'Network Canvas', - webPreferences: { - nodeIntegrationInWorker: true, - }, }, titlebarParameters); const mainWindow = new BrowserWindow(windowParameters); diff --git a/src/utils/csvDecoder.worker.js b/src/utils/csvDecoder.worker.js index c52e1ad339..8fe36a5928 100644 --- a/src/utils/csvDecoder.worker.js +++ b/src/utils/csvDecoder.worker.js @@ -1,4 +1,3 @@ -import * as csv from 'csvtojson'; import { omit } from 'lodash'; import { entityAttributesProperty } from '../ducks/modules/network'; @@ -12,6 +11,8 @@ import { entityAttributesProperty } from '../ducks/modules/network'; * */ +const csv = require('../../node_modules/csvtojson/browser/browser.js'); + const CSVToJSONNetworkFormat = (data) => { const withTypeAndAttributes = node => ({ type: node.type, From 3e8beedb12a9ac3ad0c3257860f5146895e37d7f Mon Sep 17 00:00:00 2001 From: Joshua Melville Date: Mon, 15 Apr 2019 11:04:22 +0100 Subject: [PATCH 11/11] update UI ref --- src/ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ui b/src/ui index 67fb1e6f35..c5f620dea8 160000 --- a/src/ui +++ b/src/ui @@ -1 +1 @@ -Subproject commit 67fb1e6f3537fb65d8716d5ddc2d2cf5702878a1 +Subproject commit c5f620dea8c8cdf0ccc1cd84be8001c13adeb197