Feature/Bulk Transfers POC

.istanbul.yml

@@ -1,6 +1,6 @@
 instrumentation:
   include-all-sources: true
-  excludes: ['**/migrations/**', '**/ddl/**']
+  excludes: ['**/migrations/**', '**/ddl/**', '**/bulk*/**']
 check:
   global:
     statements: 90

config/default.json

@@ -17,6 +17,9 @@
     "POOL_MIN": 10,
     "POOL_MAX": 30
   },
+  "MONGODB": {
+    "URI": "mongodb://localhost:27017/mlos"
+  },
   "HANDLERS": {
     "DISABLED": false,
     "API": {
@@ -71,6 +74,77 @@
       }
     },
     "CONSUMER": {
+      "BULK": {
+        "PREPARE": {
+          "config": {
+            "options": {
+              "mode": 2,
+              "batchSize": 1,
+              "pollFrequency": 10,
+              "recursiveTimeout": 100,
+              "messageCharset": "utf8",
+              "messageAsJSON": true,
+              "sync": true,
+              "consumeTimeout": 1000
+            },
+            "rdkafkaConf": {
+              "client.id": "cl-con-bulk-prepare",
+              "group.id": "cl-group-bulk-prepare",
+              "metadata.broker.list": "localhost:9092",
+              "socket.keepalive.enable": true
+            },
+            "topicConf": {
+              "auto.offset.reset": "earliest"
+            }
+          }
+        },
+        "PROCESSING": {
+          "config": {
+            "options": {
+              "mode": 2,
+              "batchSize": 1,
+              "pollFrequency": 10,
+              "recursiveTimeout": 100,
+              "messageCharset": "utf8",
+              "messageAsJSON": true,
+              "sync": true,
+              "consumeTimeout": 1000
+            },
+            "rdkafkaConf": {
+              "client.id": "cl-con-bulk-processing",
+              "group.id": "cl-group-bulk-processing",
+              "metadata.broker.list": "localhost:9092",
+              "socket.keepalive.enable": true
+            },
+            "topicConf": {
+              "auto.offset.reset": "earliest"
+            }
+          }
+        },
+        "FULFIL": {
+          "config": {
+            "options": {
+              "mode": 2,
+              "batchSize": 1,
+              "pollFrequency": 10,
+              "recursiveTimeout": 100,
+              "messageCharset": "utf8",
+              "messageAsJSON": true,
+              "sync": true,
+              "consumeTimeout": 1000
+            },
+            "rdkafkaConf": {
+              "client.id": "cl-con-bulk-fulfil",
+              "group.id": "cl-group-bulk-fulfil",
+              "metadata.broker.list": "localhost:9092",
+              "socket.keepalive.enable": true
+            },
+            "topicConf": {
+              "auto.offset.reset": "earliest"
+            }
+          }
+        }
+      },
       "TRANSFER": {
         "PREPARE": {
           "config": {
@@ -192,6 +266,26 @@
       }
     },
     "PRODUCER": {
+      "BULK": {
+        "PROCESSING": {
+          "config": {
+            "options": {
+              "messageCharset": "utf8"
+            },
+            "rdkafkaConf": {
+              "metadata.broker.list": "localhost:9092",
+              "client.id": "cl-prod-bulk-processing",
+              "event_cb": true,
+              "dr_cb": true,
+              "socket.keepalive.enable": true,
+              "queue.buffering.max.messages": 10000000
+            },
+            "topicConf": {
+              "request.required.acks": "all"
+            }
+          }
+        }
+      },
       "TRANSFER": {
         "PREPARE": {
           "config": {

migrations/112000_bulkTransferState.js

@@ -0,0 +1,43 @@
+/*****
+ License
+ --------------
+ Copyright © 2017 Bill & Melinda Gates Foundation
+ The Mojaloop files are made available by the Bill & Melinda Gates Foundation under the Apache License, Version 2.0 (the "License") and you may not use these files except in compliance with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, the Mojaloop files are distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ Contributors
+ --------------
+ This is the official list of the Mojaloop project contributors for this file.
+ Names of the original copyright holders (individuals or organizations)
+ should be listed with a '*' in the first column. People who have
+ contributed from an organization can be listed under the organization
+ that actually holds the copyright for their contributions (see the
+ Gates Foundation organization for an example). Those individuals should have
+ their names indented and be marked with a '-'. Email address can be added
+ optionally within square brackets <email>.
+ * Gates Foundation
+ - Name Surname <[email protected]>
+
+ * Georgi Georgiev <[email protected]>
+ --------------
+ ******/
+
+'use strict'
+
+exports.up = async (knex, Promise) => {
+  return await knex.schema.hasTable('bulkTransferState').then(function(exists) {
+    if (!exists) {
+      return knex.schema.createTable('bulkTransferState', (t) => {
+        t.string('bulkTransferStateId', 50).primary().notNullable()
+        t.string('enumeration', 50).notNullable().comment('bulkTransferState associated to the Mojaloop API')
+        t.string('description', 512).defaultTo(null).nullable()
+        t.boolean('isActive').defaultTo(true).notNullable()
+        t.dateTime('createdDate').defaultTo(knex.fn.now()).notNullable()
+      })
+    }
+  })
+}
+
+exports.down = function (knex, Promise) {
+  return knex.schema.dropTableIfExists('bulkTransferState')
+}

migrations/112100_bulkProcessingState.js

@@ -0,0 +1,43 @@
+/*****
+ License
+ --------------
+ Copyright © 2017 Bill & Melinda Gates Foundation
+ The Mojaloop files are made available by the Bill & Melinda Gates Foundation under the Apache License, Version 2.0 (the "License") and you may not use these files except in compliance with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, the Mojaloop files are distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ Contributors
+ --------------
+ This is the official list of the Mojaloop project contributors for this file.
+ Names of the original copyright holders (individuals or organizations)
+ should be listed with a '*' in the first column. People who have
+ contributed from an organization can be listed under the organization
+ that actually holds the copyright for their contributions (see the
+ Gates Foundation organization for an example). Those individuals should have
+ their names indented and be marked with a '-'. Email address can be added
+ optionally within square brackets <email>.
+ * Gates Foundation
+ - Name Surname <[email protected]>
+
+ * Georgi Georgiev <[email protected]>
+ --------------
+ ******/
+
+'use strict'
+
+exports.up = async (knex, Promise) => {
+  return await knex.schema.hasTable('bulkProcessingState').then(function(exists) {
+    if (!exists) {
+      return knex.schema.createTable('bulkProcessingState', (t) => {
+        t.increments('bulkProcessingStateId').primary().notNullable()
+        t.string('name', 50).notNullable()
+        t.string('description', 512).defaultTo(null).nullable()
+        t.boolean('isActive').defaultTo(true).notNullable()
+        t.dateTime('createdDate').defaultTo(knex.fn.now()).notNullable()
+      })
+    }
+  })
+}
+
+exports.down = function (knex, Promise) {
+  return knex.schema.dropTableIfExists('bulkProcessingState')
+}

migrations/112101_bulkProcessingState-indexes.js

@@ -0,0 +1,37 @@
+/*****
+ License
+ --------------
+ Copyright © 2017 Bill & Melinda Gates Foundation
+ The Mojaloop files are made available by the Bill & Melinda Gates Foundation under the Apache License, Version 2.0 (the "License") and you may not use these files except in compliance with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, the Mojaloop files are distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ Contributors
+ --------------
+ This is the official list of the Mojaloop project contributors for this file.
+ Names of the original copyright holders (individuals or organizations)
+ should be listed with a '*' in the first column. People who have
+ contributed from an organization can be listed under the organization
+ that actually holds the copyright for their contributions (see the
+ Gates Foundation organization for an example). Those individuals should have
+ their names indented and be marked with a '-'. Email address can be added
+ optionally within square brackets <email>.
+ * Gates Foundation
+ - Name Surname <[email protected]>
+
+ * Georgi Georgiev <[email protected]>
+ --------------
+ ******/
+
+'use strict'
+
+exports.up = function (knex, Promise) {
+  return knex.schema.table('bulkProcessingState', (t) => {
+    t.unique('name')
+  })
+}
+
+exports.down = function (knex, Promise) {
+  return knex.schema.table('bulkProcessingState', (t) => {
+    t.dropUnique('name')
+  })
+}

migrations/300150_bulkTransferDuplicateCheck.js

@@ -0,0 +1,41 @@
+/*****
+ License
+ --------------
+ Copyright © 2017 Bill & Melinda Gates Foundation
+ The Mojaloop files are made available by the Bill & Melinda Gates Foundation under the Apache License, Version 2.0 (the "License") and you may not use these files except in compliance with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, the Mojaloop files are distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ Contributors
+ --------------
+ This is the official list of the Mojaloop project contributors for this file.
+ Names of the original copyright holders (individuals or organizations)
+ should be listed with a '*' in the first column. People who have
+ contributed from an organization can be listed under the organization
+ that actually holds the copyright for their contributions (see the
+ Gates Foundation organization for an example). Those individuals should have
+ their names indented and be marked with a '-'. Email address can be added
+ optionally within square brackets <email>.
+ * Gates Foundation
+ - Name Surname <[email protected]>
+
+ * Georgi Georgiev <[email protected]>
+ --------------
+ ******/
+
+ 'use strict'
+
+exports.up = async (knex, Promise) => {
+  return await knex.schema.hasTable('bulkTransferDuplicateCheck').then(function(exists) {
+    if (!exists) {
+      return knex.schema.createTable('bulkTransferDuplicateCheck', (t) => {
+        t.string('bulkTransferId', 36).primary().notNullable()
+        t.string('hash', 256).notNullable()
+        t.dateTime('createdDate').defaultTo(knex.fn.now()).notNullable()
+      })
+    }
+  })
+}
+
+exports.down = function (knex, Promise) {
+  return knex.schema.dropTableIfExists('bulkTransferDuplicateCheck')
+}

migrations/300250_bulkTransfer.js

@@ -0,0 +1,48 @@
+
+/*****
+ License
+ --------------
+ Copyright © 2017 Bill & Melinda Gates Foundation
+ The Mojaloop files are made available by the Bill & Melinda Gates Foundation under the Apache License, Version 2.0 (the "License") and you may not use these files except in compliance with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, the Mojaloop files are distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ Contributors
+ --------------
+ This is the official list of the Mojaloop project contributors for this file.
+ Names of the original copyright holders (individuals or organizations)
+ should be listed with a '*' in the first column. People who have
+ contributed from an organization can be listed under the organization
+ that actually holds the copyright for their contributions (see the
+ Gates Foundation organization for an example). Those individuals should have
+ their names indented and be marked with a '-'. Email address can be added
+ optionally within square brackets <email>.
+ * Gates Foundation
+ - Name Surname <[email protected]>
+
+ * Georgi Georgiev <[email protected]>
+ --------------
+ ******/
+
+ 'use strict'
+
+exports.up = async (knex, Promise) => {
+  return await knex.schema.hasTable('bulkTransfer').then(function(exists) {
+    if (!exists) {
+      return knex.schema.createTable('bulkTransfer', (t) => {
+        t.string('bulkTransferId', 36).primary().notNullable()
+        t.foreign('bulkTransferId').references('bulkTransferId').inTable('bulkTransferDuplicateCheck')
+        t.string('bulkQuoteId', 36).nullable()
+        t.integer('payerParticipantId').unsigned().nullable()
+        t.foreign('payerParticipantId').references('participantId').inTable('participant')
+        t.integer('payeeParticipantId').unsigned().nullable()
+        t.foreign('payeeParticipantId').references('participantId').inTable('participant')
+        t.dateTime('expirationDate').notNullable()
+        t.dateTime('createdDate').defaultTo(knex.fn.now()).notNullable()
+      })
+    }
+  })
+}
+
+exports.down = function (knex, Promise) {
+  return knex.schema.dropTableIfExists('bulkTransfer')
+}

migrations/300251_bulkTransfer-indexes.js

@@ -0,0 +1,39 @@
+/*****
+ License
+ --------------
+ Copyright © 2017 Bill & Melinda Gates Foundation
+ The Mojaloop files are made available by the Bill & Melinda Gates Foundation under the Apache License, Version 2.0 (the "License") and you may not use these files except in compliance with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, the Mojaloop files are distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ Contributors
+ --------------
+ This is the official list of the Mojaloop project contributors for this file.
+ Names of the original copyright holders (individuals or organizations)
+ should be listed with a '*' in the first column. People who have
+ contributed from an organization can be listed under the organization
+ that actually holds the copyright for their contributions (see the
+ Gates Foundation organization for an example). Those individuals should have
+ their names indented and be marked with a '-'. Email address can be added
+ optionally within square brackets <email>.
+ * Gates Foundation
+ - Name Surname <[email protected]>
+
+ * Georgi Georgiev <[email protected]>
+ --------------
+ ******/
+
+'use strict'
+
+exports.up = function (knex, Promise) {
+  return knex.schema.table('bulkTransfer', (t) => {
+    t.index('payerParticipantId')
+    t.index('payeeParticipantId')
+  })
+}
+
+exports.down = function (knex, Promise) {
+  return knex.schema.table('bulkTransfer', (t) => {
+    t.dropIndex('payerParticipantId')
+    t.dropIndex('payeeParticipantId')
+  })
+}