Add write and mWrite (#1302)

This PR add two method for the bulk controller that behave like createOrReplace and mCreateOrReplace:

    write: write a document without adding metadata or performing data validation
    mWrite: write several documents without adding metadata or performing data validation

The name is open to debate, I have thought of put and mPut also

The goal is to have a low level method that allow to arbitrary write document into the storage layer to bypass automatic metadata or the data validation layer.

It's better to have these two endpoints than an option like bypassMeta on existing methods because developer can choose to not expose these methods to users.

features/kuzzle.feature

@@ -1,4 +1,35 @@
 Feature: Kuzzle functional tests
+  Scenario: Bulk mWrite
+    When I create a collection "kuzzle-test-index":"kuzzle-collection-test"
+    When I use bulk:mWrite action with
+    """
+    {
+      "documents": [
+        { "body": { "name": "Maedhros" } },
+        { "body": { "name": "Maglor" } },
+        { "body": { "name": "Celegorm" } },
+        { "body": { "name": "Caranthis" } },
+        { "body": { "name": "Curufin" } },
+        { "body": { "name": "Amrod" } },
+        { "body": { "name": "Amras" } }
+      ]
+    }
+    """
+    Then I count 7 documents
+    And The documents does not have kuzzle metadata
+
+  Scenario: Bulk write
+    When I create a collection "kuzzle-test-index":"kuzzle-collection-test"
+    When I use bulk:write action with '{ "name": "Feanor", "_kuzzle_info": { "author": "Tolkien" } }'
+    Then I count 1 documents
+    And The documents have the following kuzzle metadata '{ "author": "Tolkien" }'
+
+  Scenario: Bulk write with _id
+    When I create a collection "kuzzle-test-index":"kuzzle-collection-test"
+    When I use bulk:write action with id "wandered" and content '{ "name": "Feanor" }'
+    Then I count 1 documents
+    And I can found a document "wandered"
+
   Scenario: Create a collection
     When I create a collection "kuzzle-test-index":"my-collection1"
     Then The mapping properties field of "kuzzle-test-index":"my-collection1" is "the default value"

features/step_definitions/bulk.js

@@ -1,4 +1,5 @@
 const
+  should = require('should'),
   {
     Then,
     When
@@ -116,3 +117,44 @@ When(/^I do a global bulk import$/, function (callback) {
     });
 });
 
+When('I use bulk:mWrite action with', function (bodyRaw) {
+  const body = JSON.parse(bodyRaw);
+
+  return this.api.bulkMWrite(this.index, this.collection, body);
+});
+
+When('I use bulk:write action with {string}', function (bodyRaw) {
+  const body = JSON.parse(bodyRaw);
+
+  return this.api.bulkWrite(this.index, this.collection, body);
+});
+
+When('I use bulk:write action with id {string} and content {string}', function (id, bodyRaw) {
+  const body = JSON.parse(bodyRaw);
+
+  return this.api.bulkWrite(this.index, this.collection, body, id);
+});
+
+Then('The documents does not have kuzzle metadata', function () {
+  return this.api.search({}, this.index, this.collection, { size: 100 })
+    .then(({ result }) => {
+      for (const hit of result.hits) {
+        should(hit._source._kuzzle_info).be.undefined();
+      }
+    });
+});
+
+Then('The documents have the following kuzzle metadata {string}', function (metadatRaw) {
+  const metadata = JSON.parse(metadatRaw);
+
+  return this.api.search({}, this.index, this.collection, { size: 100 })
+    .then(({ result }) => {
+      for (const hit of result.hits) {
+        should(hit._source._kuzzle_info).match(metadata);
+      }
+    });
+});
+
+Then('I can found a document {string}', function (documentId) {
+  return this.api.get(documentId, this.index, this.collection);
+});

features/support/api/apiBase.js

@@ -41,6 +41,33 @@ class ApiBase {
     return this.send(msg);
   }
 
+  bulkMWrite (index, collection, body) {
+    const
+      msg = {
+        controller: 'bulk',
+        collection: collection || this.world.fakeCollection,
+        index: index || this.world.fakeIndex,
+        action: 'mWrite',
+        body
+      };
+
+    return this.send(msg);
+  }
+
+  bulkWrite (index, collection, body, _id = null) {
+    const
+      msg = {
+        controller: 'bulk',
+        collection: collection || this.world.fakeCollection,
+        index: index || this.world.fakeIndex,
+        action: 'write',
+        _id,
+        body
+      };
+
+    return this.send(msg);
+  }
+
   collectionExists (index, collection) {
     return this.send({
       index,
@@ -474,11 +501,11 @@ class ApiBase {
     return this.send(msg);
   }
 
-  get (id, index) {
+  get (id, index, collection) {
     const
       msg = {
         controller: 'document',
-        collection: this.world.fakeCollection,
+        collection: collection || this.world.fakeCollection,
         index: index || this.world.fakeIndex,
         action: 'get',
         _id: id

features/support/api/http.js

@@ -158,6 +158,32 @@ class HttpApi {
     return this.callApi(options);
   }
 
+  bulkMWrite (index, collection, body) {
+    const options = {
+      url: this.apiPath(this.util.getIndex(index) + '/' + this.util.getCollection(collection) + '/_mWrite'),
+      method: 'POST',
+      body
+    };
+
+    return this.callApi(options);
+  }
+
+  bulkWrite (index, collection, body, _id = null) {
+    let url = `${this.util.getIndex(index)}/${this.util.getCollection(collection)}/_write`;
+
+    if (_id) {
+      url = `${this.util.getIndex(index)}/${this.util.getCollection(collection)}/${_id}/_write`;
+    }
+
+    const options = {
+      url: this.apiPath(url),
+      method: 'POST',
+      body
+    };
+
+    return this.callApi(options);
+  }
+
   /**
    * @param options
    * @return {Promise.<IncomingMessage>}

lib/api/controllers/bulkController.js

@@ -23,16 +23,27 @@
 
 const
   BaseController = require('./controller'),
-  {errors: {PartialError}} = require('kuzzle-common-objects'),
-  {assertHasBody, assertBodyHasAttribute} = require('../../util/requestAssertions');
+  { errors: { PartialError } } = require('kuzzle-common-objects'),
+  {
+    assertHasBody,
+    assertBodyHasAttribute,
+    assertHasIndexAndCollection,
+    assertBodyAttributeType,
+    assertIdStartsNotUnderscore
+  } = require('../../util/requestAssertions');
 
 /**
  * @class BulkController
  * @param {Kuzzle} kuzzle
  */
 class BulkController extends BaseController {
   constructor (kuzzle) {
-    super(kuzzle, ['import']);
+    super(kuzzle, [
+      'import',
+      'write',
+      'mWrite'
+    ]);
+
     this.engine = kuzzle.services.list.storageEngine;
   }
 
@@ -55,6 +66,62 @@ class BulkController extends BaseController {
         return response;
       });
   }
+
+  /**
+   * Write a document without adding metadata or performing data validation.
+   * @param {Request} request
+   * @returns {Promise<Object>}
+   */
+  write(request) {
+    assertHasBody(request);
+    assertHasIndexAndCollection(request);
+    assertIdStartsNotUnderscore(request);
+
+    const notify = this.tryGetBoolean(request, 'args.notify');
+
+    return this.engine.createOrReplace(request, false)
+      .then(response => {
+        this.kuzzle.indexCache.add(request.input.resource.index, request.input.resource.collection);
+
+        if (notify && response.created) {
+          this.kuzzle.notifier.notifyDocumentCreate(request, response);
+        }
+        else if (notify) {
+          this.kuzzle.notifier.notifyDocumentReplace(request);
+        }
+
+        return response;
+      });
+  }
+
+  /**
+   * Write several documents without adding metadata or performing data validation.
+   *
+   * @param {Request} request
+   * @returns {Promise<Object>}
+   */
+  mWrite(request) {
+    assertHasBody(request);
+    assertBodyHasAttribute(request, 'documents');
+    assertBodyAttributeType(request, 'documents', 'array');
+    assertHasIndexAndCollection(request);
+
+    const notify = this.tryGetBoolean(request, 'args.notify');
+
+    return this.engine.mcreateOrReplace(request, false)
+      .then(response => {
+        if (response.error.length > 0) {
+          request.setError(new PartialError('Some document creations failed', response.error));
+        }
+
+        if (notify) {
+          this.kuzzle.notifier.notifyDocumentMChanges(request, response.result, true);
+        }
+
+        return { hits: response.result, total: response.result.length };
+      });
+  }
+
 }
 
 module.exports = BulkController;

lib/api/controllers/collectionController.js

@@ -90,9 +90,9 @@ class CollectionController extends BaseController {
   getMapping(request) {
     assertHasIndexAndCollection(request);
 
-    this.ensureBooleanFlag(request, 'input.args.includeKuzzleMeta');
+    const includeKuzzleMeta = this.tryGetBoolean(request, 'args.includeKuzzleMeta');
 
-    return this.engine.getMapping(request);
+    return this.engine.getMapping(request, includeKuzzleMeta);
   }
 
   /**
@@ -354,12 +354,12 @@ const formatSpecification = (index, collection, validation) => ({
  */
 function createSpecificationList (request) {
   const { index, collection } = request.input.resource;
-  
+
   if (index && collection) {
     const specification = formatSpecification(index, collection, request.input.body);
     return Bluebird.resolve([ specification ]);
   }
-  
+
   const specifications = [];
   _.forEach(request.input.body, (collections, _index) => {
     _.forEach(collections, (validation, _collection) => {

lib/api/controllers/controller.js

@@ -59,16 +59,16 @@ class BaseController {
   }
 
   /**
-   * Ensure an input args is a boolean.
+   * Get a boolean param from request input
    * For HTTP, flag presence mean true value
    *
    * @param {Request} request
    * @param {string} flagPath
    */
-  ensureBooleanFlag (request, flagPath) {
+  tryGetBoolean (request, flagPath) {
     const
       flagName = flagPath.split('.').slice(-1),
-      flagValue = _.get(request, flagPath);
+      flagValue = _.get(request, `input.${flagPath}`);
 
     // In HTTP, booleans are flags: if it's in the querystring, it's set, whatever
     // its value.
@@ -82,8 +82,11 @@ class BaseController {
       const booleanValue = flagValue !== undefined ? true : false;
 
       _.set(request, flagPath, booleanValue);
+
+      return booleanValue;
     }
 
+    return Boolean(flagValue);
   }
 }
 

lib/api/core/plugins/pluginContext.js

@@ -58,7 +58,7 @@ class PluginContext {
     const Kuzzle = require('../../kuzzle');
 
     this.accessors = {};
-    this.config = kuzzle.config;
+    this.config = JSON.parse(JSON.stringify(kuzzle.config));
     this.constructors = deprecateProperties({
       RequestContext,
       RequestInput,

lib/config/httpRoutes.js

@@ -153,7 +153,7 @@ module.exports = [
   {verb: 'post', url: '/credentials/:strategy/_me/_create', controller: 'auth', action: 'createMyCredentials'},
   {verb: 'post', url: '/credentials/:strategy/_me/_validate', controller: 'auth', action: 'validateMyCredentials'},
 
-  /* DEPRECATED - will be removed in v2 */  
+  /* DEPRECATED - will be removed in v2 */
   {verb: 'post', url: '/_validateSpecifications', controller: 'collection', action: 'validateSpecifications'},
 
   {verb: 'post', url: '/:index/:collection/_validateSpecifications', controller: 'collection', action: 'validateSpecifications'},
@@ -163,6 +163,10 @@ module.exports = [
   {verb: 'post', url: '/:index/_bulk', controller: 'bulk', action: 'import'},
   {verb: 'post', url: '/:index/:collection/_bulk', controller: 'bulk', action: 'import'},
 
+  {verb: 'post', url: '/:index/:collection/_mWrite', controller: 'bulk', action: 'mWrite'},
+  {verb: 'post', url: '/:index/:collection/_write', controller: 'bulk', action: 'write'},
+  {verb: 'post', url: '/:index/:collection/:_id/_write', controller: 'bulk', action: 'write'},
+
   /* DEPRECATED - will be removed in v2 */
   {verb: 'post', url: '/_getStats', controller: 'server', action: 'getStats'},
 
@@ -296,7 +300,7 @@ module.exports = [
 
   /* DEPRECATED - will be removed in v2 */
   {verb: 'put', url: '/_specifications', controller: 'collection', action: 'updateSpecifications'},
-  
+
   {verb: 'put', url: '/:index/:collection/_specifications', controller: 'collection', action: 'updateSpecifications'},
 
   {verb: 'put', url: '/:index/:collection/:_id', controller: 'document', action: 'createOrReplace'},