diff --git a/docs/plugin-analytics.md b/docs/plugin-analytics.md index 006faea8..9447eb08 100644 --- a/docs/plugin-analytics.md +++ b/docs/plugin-analytics.md @@ -26,10 +26,10 @@ npm install @ezs/analytics - [reducing](#reducing) - [aggregate](#aggregate) - [slice](#slice) -- [upload](#upload) - [distinct](#distinct) - [exploding](#exploding) - [graph](#graph) +- [upload](#upload) - [pair](#pair) - [merging](#merging) - [summing](#summing) @@ -434,46 +434,6 @@ Output: Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** -### upload - -save all objects in a temporary file -For non Buffer chunks, each object is transformed into a -string of characters in a raw way (no separator) - -```json -[ - { year: 2000, dept: 54 }, - { year: 2001, dept: 55 }, - { year: 2003, dept: 54 }, -] -``` - -Script: - -```ini -[use] -plugin = analytics - -[upload] -cleanupDelay = 5 -``` - -Output: - -```json - [ - { id: '/tmp/31234qdE33334dZE', value:3 }, - ] -``` - -#### Parameters - -- `extension` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** set the file extension (optional, default `bin`) -- `prefix` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** set the file prefix (optional, default `upload`) -- `cleanupDelay` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** TTL in seconds, before cleanup the file (EZS_DELAY) (optional, default `3600`) - -Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** - ### distinct Take `Object` object getting some fields with json path, and do ... @@ -599,6 +559,46 @@ Output: Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** +### upload + +save all objects in a temporary file +For non Buffer chunks, each object is transformed into a +string of characters in a raw way (no separator) + +```json +[ + { year: 2000, dept: 54 }, + { year: 2001, dept: 55 }, + { year: 2003, dept: 54 }, +] +``` + +Script: + +```ini +[use] +plugin = analytics + +[upload] +cleanupDelay = 5 +``` + +Output: + +```json + [ + { id: '/tmp/31234qdE33334dZE', value:3 }, + ] +``` + +#### Parameters + +- `extension` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** set the file extension (optional, default `bin`) +- `prefix` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** set the file prefix (optional, default `upload`) +- `cleanupDelay` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** TTL in seconds, before cleanup the file (EZS_DELAY) (optional, default `3600`) + +Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** + ### pair Take `Object` object getting some fields with json path, and diff --git a/packages/analytics/README.md b/packages/analytics/README.md index 006faea8..9447eb08 100644 --- a/packages/analytics/README.md +++ b/packages/analytics/README.md @@ -26,10 +26,10 @@ npm install @ezs/analytics - [reducing](#reducing) - [aggregate](#aggregate) - [slice](#slice) -- [upload](#upload) - [distinct](#distinct) - [exploding](#exploding) - [graph](#graph) +- [upload](#upload) - [pair](#pair) - [merging](#merging) - [summing](#summing) @@ -434,46 +434,6 @@ Output: Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** -### upload - -save all objects in a temporary file -For non Buffer chunks, each object is transformed into a -string of characters in a raw way (no separator) - -```json -[ - { year: 2000, dept: 54 }, - { year: 2001, dept: 55 }, - { year: 2003, dept: 54 }, -] -``` - -Script: - -```ini -[use] -plugin = analytics - -[upload] -cleanupDelay = 5 -``` - -Output: - -```json - [ - { id: '/tmp/31234qdE33334dZE', value:3 }, - ] -``` - -#### Parameters - -- `extension` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** set the file extension (optional, default `bin`) -- `prefix` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** set the file prefix (optional, default `upload`) -- `cleanupDelay` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** TTL in seconds, before cleanup the file (EZS_DELAY) (optional, default `3600`) - -Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** - ### distinct Take `Object` object getting some fields with json path, and do ... @@ -599,6 +559,46 @@ Output: Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** +### upload + +save all objects in a temporary file +For non Buffer chunks, each object is transformed into a +string of characters in a raw way (no separator) + +```json +[ + { year: 2000, dept: 54 }, + { year: 2001, dept: 55 }, + { year: 2003, dept: 54 }, +] +``` + +Script: + +```ini +[use] +plugin = analytics + +[upload] +cleanupDelay = 5 +``` + +Output: + +```json + [ + { id: '/tmp/31234qdE33334dZE', value:3 }, + ] +``` + +#### Parameters + +- `extension` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** set the file extension (optional, default `bin`) +- `prefix` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** set the file prefix (optional, default `upload`) +- `cleanupDelay` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** TTL in seconds, before cleanup the file (EZS_DELAY) (optional, default `3600`) + +Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** + ### pair Take `Object` object getting some fields with json path, and diff --git a/packages/basics/README.md b/packages/basics/README.md index a56844ec..be0aabd7 100644 --- a/packages/basics/README.md +++ b/packages/basics/README.md @@ -14,342 +14,19 @@ npm install @ezs/basics #### Table of Contents -- [JSONString](#jsonstring) -- [OBJFlatten](#objflatten) -- [TXTZip](#txtzip) -- [OBJCount](#objcount) -- [URLString](#urlstring) -- [BUFObject](#bufobject) -- [URLConnect](#urlconnect) -- [URLStream](#urlstream) -- [ZIPExtract](#zipextract) -- [TXTObject](#txtobject) -- [URLFetch](#urlfetch) -- [TXTConcat](#txtconcat) -- [INIString](#inistring) -- [CSVParse](#csvparse) -- [URLParse](#urlparse) -- [XMLString](#xmlstring) -- [JSONParse](#jsonparse) -- [XMLParse](#xmlparse) -- [TXTParse](#txtparse) -- [CSVString](#csvstring) -- [CSVObject](#csvobject) -- [OBJStandardize](#objstandardize) -- [OBJNamespaces](#objnamespaces) +* [generateARK](#generateark) -### JSONString +### generateARK -Take `Object` and generate JSON +Take `Object` object, and throw the same object but with an ARK #### Parameters -- `wrap` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** every document are wrapped into an array (optional, default `true`) -- `indent` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** indent JSON (optional, default `false`) - -Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** - -### OBJFlatten - -- **See: - ** - -Take `Object` and flat it with delimited character. - -#### Parameters - -- `separator` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** choose a character for flatten keys (optional, default `/`) -- `safe` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** preserve arrays and their contents, (optional, default `false`) - -Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** - -### TXTZip - -Take a String and zip it - -#### Parameters - -- `unzip` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** to Unzip input (optional, default `false`) - -Returns **[Buffer](https://nodejs.org/api/buffer.html)** - -### OBJCount - -Take `Object` and count how many objects are received and sent the total - -#### Parameters - -- `none` **[undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined)** - -Returns **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** - -### URLString - -Take `Object` representing a URL and stringify it - -Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** - -### BUFObject - -Take `Mixed` and produce Buffer. -For example, it's useful to send string to browser. - -#### Parameters - -- `none` **[undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined)** - -Returns **[Buffer](https://nodejs.org/api/buffer.html)** - -### URLConnect - -Take `Object` and send it to an URL -the output will be the content of URL - -#### Parameters - -- `url` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** URL to fecth -- `json` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Pasre as JSON the content of URL (optional, default `false`) -- `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`) -- `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors (optional, default `false`) - -Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** - -### URLStream - -Take `String` asURL, throw each chunk from the result or -Take `Object` as parameters of URL, throw each chunk from the result - -#### Parameters - -- `url` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** URL to fetch (by default input string is taken) -- `path` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** choose the path to split JSON result (optional, default `*`) -- `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`) -- `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors, the target field will remain undefined (optional, default `false`) - -Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** - -### ZIPExtract - -Take the content of a zip file, extract some files -The JSON object is sent to the output stream for each file. -it returns to the output stream - - { - id: file name, - value: file contents, - } - -#### Parameters - -- `path` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Regex to select the files to extract (optional, default `"**\/*.json"`) - -Returns **any** - -### TXTObject - -Take `String` and generate an object with a key and a value, where the value is the input string. - -#### Parameters - -- `key` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** choose a the key name (optional, default `value`) - -Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** - -### URLFetch - -Take `Object` and create a new field with the content of URL. -Or if no target will be specified, the output will be the content of URL - -#### Parameters - -- `url` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** URL to fecth -- `path` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** if present select value to send (by POST) -- `target` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** choose the key to set -- `json` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Parse as JSON the content of URL (optional, default `false`) -- `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`) -- `mimetype` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Mimetype for value of path (if presents) (optional, default `application/json`) -- `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors, the target field will remain undefined (optional, default `false`) - -Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** - -### TXTConcat - -Take `String` and concat all items in just one string - -#### Parameters - -- `none` **[undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined)** - -Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** - -### INIString - -Take `Object` and generate INI - -Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** - -### CSVParse - -- **See: - ** - -Take `String` and parse CSV to generate object - -#### Parameters - -- `separator` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** to indicate the CSV separator (optional, default `auto`) -- `quote` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** to indicate the CSV quote. (optional, default `auto`) - -Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** - -### URLParse - -Take `String` of URL, parse it and returns `Object` - -Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** - -### XMLString - -Take `Object` and transform it into a XML string - -#### Parameters - -- `rootElement` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Root element name for the tag which start and close the feed (optional, default `items`) -- `contentElement` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Content element name for the tag which start and close each item (optional, default `item`) -- `rootNamespace` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** Namespace for the root tag (xmlns=) - -Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** - -### JSONParse - -- **See: - ** - -Take `String` and parse JSON and generate objects - -#### Parameters - -- `separator` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** to split at every JSONPath found (optional, default `*`) - -Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** - -### XMLParse - -- **See: - ** - -Take `String` as XML input, parse it and split it in multi document at each path found - -#### Parameters - -- `separator` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** choose a character for flatten keys (optional, default `/`) - -Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** - -### TXTParse - -Take `String` and split at each separator found - -#### Parameters - -- `separator` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** choose character which trigger the split (optional, default `"\n"`) - -Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** - -### CSVString - -Take `Object` and transform row into string -where each field is separated with a character - -#### Parameters - -- `format` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** if set to "strict" the fields will wrapped with double quote (optional, default `standard`) -- `separator` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** to indicate the CSV separator (optional, default `;`) -- `header` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** first line contains key name (optional, default `true`) - -Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** - -### CSVObject - -Take `Array` and transform rows into object. -Each row (Array) is tranformed -into a object where keys are the value of the first row - -#### Parameters - -- `none` **[undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined)** - -Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** - -### OBJStandardize - -Take `Object` and standardize it so each object will have the sames keys - -#### Parameters - -- `none` **[undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined)** - -Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** - -### OBJNamespaces - -Take `Object` and throw the same object, all keys were parsed to replace namespaces with their prefixes -Note: You can also parse values for specific keys (keys containing references to other keys) - -```json -[ - { - "http://purl.org/dc/terms/title": "Life is good", - "http://purl.org/ontology/places#Countryl": "France", - }, - { - "http://purl.org/dc/terms/title": "the rising sun", - "http://purl.org/ontology/places#Country": "Japan", - }, - { - "http://purl.org/dc/terms/title": "Dolce Vista", - "http://purl.org/ontology/places#Country": "Italy", - } -] -``` - -Script: - -```ini -[use] -plugin = basics - -[OBJNamespaces] -prefix = dc: -namespace = http://purl.org/dc/terms/ - -prefix = place: -namespace = http://purl.org/ontology/places# -``` - -Output: - -```json -[ - { - "dc:title": "Life is good", - "place:Country": "France", - }, - { - "dc:title": "the rising sun", - "place:Country": "Japan", - }, - { - "dc:title": "Dolce Vista", - "place:Country": "Italy", - } -] -``` - -#### Parameters - -- `prefix` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** the alias for a namespace -- `namespace` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** the namespace to substitute by a prefix -- `reference` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** a regex to find key that contains a namespace to substitute (optional, default `null`) +* `data` +* `feed` +* `target` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** field name to insert ARK (optional, default `ark`) +* `subpublisher` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** Subpublisher +* `naan` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** NAAN +* `databasePath` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** Path for database to store generated ARKs Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** diff --git a/packages/basics/src/index.js b/packages/basics/src/index.js index 63120dbc..e98404ca 100644 --- a/packages/basics/src/index.js +++ b/packages/basics/src/index.js @@ -8,6 +8,7 @@ import TXTObject from './txt-object'; import TXTParse from './txt-parse'; import XMLParse from './xml-parse'; import XMLString from './xml-string'; +import XMLConvert from './xml-convert'; import CSVParse from './csv-parse'; import CSVObject from './csv-object'; import CSVString from './csv-string'; @@ -33,6 +34,7 @@ export default { TXTConcat, XMLParse, XMLString, + XMLConvert, CSVParse, CSVObject, CSVString, diff --git a/packages/basics/src/xml-convert.js b/packages/basics/src/xml-convert.js new file mode 100644 index 00000000..08e3768b --- /dev/null +++ b/packages/basics/src/xml-convert.js @@ -0,0 +1,21 @@ +import XML from 'xml-mapping'; + +function XMLConvert(data, feed) { + if (this.isLast()) { + return feed.close(); + } + const func = this.getParam('invert', false) ? 'dump' : 'load'; + feed.send(XML[func](data)); +} + +/** + * Convert each chunk as XML String to JSON Object + * + * @name XMLConvert + * @param {String} [invert=false] change conversion (JSON to XML) + * @returns {Object} + * @see https://www.npmjs.com/package/xml-mapping + */ +export default { + XMLConvert, +}; diff --git a/packages/basics/test/xml-convert.js b/packages/basics/test/xml-convert.js new file mode 100644 index 00000000..c5d27bc0 --- /dev/null +++ b/packages/basics/test/xml-convert.js @@ -0,0 +1,42 @@ +import from from 'from'; +import ezs from '../../core/src'; +import ezsBasics from '../src'; + +ezs.use(ezsBasics); + +describe('XMLConvert', () => { + test('XML to JSON', (done) => { + const output = []; + from([ + 'A', + 'B', + ]) + .pipe(ezs('XMLConvert')) + .pipe(ezs.catch()) + .on('error', done) + .on('data', (chunk) => output.push(chunk)) + .on('end', () => { + expect(output.length).toStrictEqual(2); + expect(output[0]).toEqual({ xml: { $t: 'A' } }); + expect(output[1]).toEqual({ xml: { $t: 'B' } }); + done(); + }); + }); + test('JSON to XML', (done) => { + const output = []; + from([ + { x: { a: 1 } }, + { x: { a: 2 } }, + ]) + .pipe(ezs('XMLConvert', { invert: true })) + .pipe(ezs.catch()) + .on('error', done) + .on('data', (chunk) => output.push(chunk)) + .on('end', () => { + expect(output.length).toStrictEqual(2); + expect(output[0]).toEqual(''); + expect(output[1]).toEqual(''); + done(); + }); + }); +});