diff --git a/.env.template b/.env.template index 93fdada..d474b35 100644 --- a/.env.template +++ b/.env.template @@ -30,3 +30,10 @@ SCRAPEIT_API_KEY="" CLOUDANT_USERNAME="" CLOUDANT_API_KEY="" CLOUDANT_URL="" + +MONGODB_PASSWORD="" +MONGODB_USERNAME="" +MONGODB_CONNECT_STRING="" +MONGODB_DATABASE_NAME="" +MONGODB_CERTIFICATE_FILE="" +MONGODB_CERTIFICATE_BASE64="" diff --git a/Dockerfile b/Dockerfile index 0ea67fa..b596108 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.access.redhat.com/ubi9/nodejs-18:1-70.1695740477 AS builder +FROM registry.access.redhat.com/ubi9/nodejs-18:1-112.1719561319 AS builder WORKDIR /opt/app-root/src @@ -9,7 +9,7 @@ RUN mkdir -p /opt/app-root/src/node_modules && \ npm ci && \ npm run build -FROM registry.access.redhat.com/ubi9/nodejs-18:1-70.1695740477 +FROM registry.access.redhat.com/ubi9/nodejs-18:1-112.1719561319 ## Uncomment the below lines to update image security content if any # USER root @@ -18,7 +18,7 @@ FROM registry.access.redhat.com/ubi9/nodejs-18:1-70.1695740477 LABEL name="ibm/template-node-typescript" \ vendor="IBM" \ version="1" \ - release="67" \ + release="112" \ summary="This is an example of a container image." \ description="This container image will deploy a Typescript Node App" diff --git a/chart/base/templates/watsonx-config.yaml b/chart/base/templates/watsonx-config.yaml new file mode 100644 index 0000000..e596a2f --- /dev/null +++ b/chart/base/templates/watsonx-config.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: watsonx-config + labels: + app.kubernetes.io/name: {{ include "starter-kit.name" . }} + helm.sh/chart: {{ include "starter-kit.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app: {{ .Release.Name }} +data: + CRA_BASE_PATH: https://cpd-cp4ba.cp4ba-cra-c6c44da74def18a795b07cc32856e138-0000.us-south.containers.appdomain.cloud/ads/runtime/api/v1/deploymentSpaces/embedded/decisions/ekyc001%2FcustomerRiskAssessmentDecisionService%2Fekyc-08%2FcustomerRiskAssessmentDecisionService-ekyc-08.jar/operations + DATA_EXTRACTION_COLLECTION_ID: d2042924-7671-d0f5-0000-018a41a20ec1 + DATA_EXTRACTION_PROJECT_ID: 05ba9d92-734e-4b34-a672-f727a2c26440 + DECODING_METHOD: greedy + DISCOVERY_PROJECT_ID: 303aab25-cb4f-4b28-b8d2-30e23e39a37f + DISCOVERY_URL: https://api.us-south.discovery.watson.cloud.ibm.com/instances/0992769e-726a-4ab0-a9d9-4352e204cc87 + DISCOVERY_VERSION: "2020-08-30" + FIND_PASSAGE_CONCURRENCY: "20" + IAM_URL: https://iam.cloud.ibm.com/identity/token + KYC_COLLECTION_ID: f97f97bd-7fa1-c9ac-0000-018acacf5fdf + KYC_PROJECT_ID: 4fd3f39a-3ed9-4c66-a4c4-7951afc0aee + KYC_SUMMARY_URL: https://kyc-summary-api.177i1pavvk8r.us-south.codeengine.appdomain.cloud/summarise + MAX_NEW_TOKENS: "20" + MODEL_ID: google/flan-t5-xxl + RELEVANT_PASSAGES_URL: https://similarity-check.18z7sftfb1j5.us-south.codeengine.appdomain.cloud/api/find_relevant_passage + REPETITION_PENALTY: "1" + WML_PROJECT_ID: 05ba9d92-734e-4b34-a672-f727a2c26440 + WML_URL: https://us-south.ml.cloud.ibm.com/ml/v1-beta/generation/text?version=2023-05-28 diff --git a/chart/base/templates/watsonx-credentials.yaml b/chart/base/templates/watsonx-credentials.yaml new file mode 100644 index 0000000..188123b --- /dev/null +++ b/chart/base/templates/watsonx-credentials.yaml @@ -0,0 +1,25 @@ +{{- if .Values.secret.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: watsonx-credentials + labels: + app.kubernetes.io/name: {{ include "starter-kit.name" . }} + helm.sh/chart: {{ include "starter-kit.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app: {{ .Release.Name }} +stringData: + CLOUDANT_API_KEY: {{ .Values.cloudant.apiKey | quote }} + CLOUDANT_URL: {{ .Values.cloudant.url | quote }} + CLOUDANT_USERNAME: {{ .Values.cloudant.username | quote }} + CRA_API_KEY: {{ .Values.cra.apiKey | quote }} + CRA_USERNAME: {{ .Values.cra.username | quote }} + DISCOVERY_API_KEY: {{ .Values.discovery.apiKey | quote }} + SCRAPEIT_API_KEY: {{ .Values.scrapeit.apiKey | quote }} + WML_API_KEY: {{ .Values.wml.apiKey | quote }} + MONGODB_PASSWORD: {{ .Values.mongodb.password | quote }} + MONGODB_USERNAME: {{ .Values.mongodb.username | quote }} + MONGODB_CONNECT_STRING: {{ .Values.mongodb.connectString | quote }} + MONGODB_DATABASE_NAME: {{ .Values.mongodb.databaseName | quote }} + MONGODB_CERTIFICATE_FILE: {{ .Values.mongodb.certificateFile | quote }} +{{- end }} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index baa989c..d35a7de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,6 +31,7 @@ "jsdom": "^22.1.0", "langchain": "^0.0.182", "mime": "^3.0.0", + "mongodb": "^5.9.2", "optional-js": "^2.3.0", "reflect-metadata": "^0.1.13", "rxjs": "^7.8.1", @@ -1928,6 +1929,15 @@ "node": ">=8" } }, + "node_modules/@mongodb-js/saslprep": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.5.tgz", + "integrity": "sha512-XLNOMH66KhJzUJNwT/qlMnS4WsNDWD5ASdyaSH3EtK+F4r/CFGa3jT4GNi4mfOitGvWXtdLgQJkQjxSVrio+jA==", + "optional": true, + "dependencies": { + "sparse-bitfield": "^3.0.3" + } + }, "node_modules/@nestjs/apollo": { "version": "12.0.9", "resolved": "https://registry.npmjs.org/@nestjs/apollo/-/apollo-12.0.9.tgz", @@ -2273,13 +2283,13 @@ } }, "node_modules/@nestjs/platform-express": { - "version": "10.2.8", - "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.2.8.tgz", - "integrity": "sha512-WoSSVtwIRc5AdGMHWVzWZK4JZLT0f4o2xW8P9gQvcX+omL8W1kXCfY8GQYXNBG84XmBNYH8r0FtC8oMe/lH5NQ==", + "version": "10.3.8", + "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.3.8.tgz", + "integrity": "sha512-sifLoxgEJvAgbim1UuW6wyScMfkS9SVQRH+lN33N/9ZvZSjO6NSDLOe+wxqsnZkia+QrjFC0qy0ITRAsggfqbg==", "dependencies": { "body-parser": "1.20.2", "cors": "2.8.5", - "express": "4.18.2", + "express": "4.19.2", "multer": "1.4.4-lts.1", "tslib": "2.6.2" }, @@ -2908,6 +2918,11 @@ "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.6.tgz", "integrity": "sha512-BT2Krtx4xaO6iwzwMFUYvWBWkV2pr37zD68Vmp1CDV196MzczBRxuEpD6Pr395HAgebC/co7hOphs53r8V7jew==" }, + "node_modules/@types/webidl-conversions": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", + "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==" + }, "node_modules/@types/websocket": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/websocket/-/websocket-1.0.6.tgz", @@ -2916,6 +2931,15 @@ "@types/node": "*" } }, + "node_modules/@types/whatwg-url": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.2.tgz", + "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==", + "dependencies": { + "@types/node": "*", + "@types/webidl-conversions": "*" + } + }, "node_modules/@types/yargs": { "version": "17.0.24", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.24.tgz", @@ -3884,6 +3908,14 @@ "node-int64": "^0.4.0" } }, + "node_modules/bson": { + "version": "5.5.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-5.5.1.tgz", + "integrity": "sha512-ix0EwukN2EpC0SRWIj/7B5+A6uQMQy6KMREI9qQqvgpkV2frH63T0UDVd1SYedL6dNCmDBYB3QtXi4ISk9YT+g==", + "engines": { + "node": ">=14.20.1" + } + }, "node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -4325,9 +4357,9 @@ "dev": true }, "node_modules/cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", "engines": { "node": ">= 0.6" } @@ -5031,13 +5063,14 @@ "dev": true }, "node_modules/es5-ext": { - "version": "0.10.62", - "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.62.tgz", - "integrity": "sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==", + "version": "0.10.64", + "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.64.tgz", + "integrity": "sha512-p2snDhiLaXe6dahss1LddxqEm+SkuDvV8dnIQG0MWjyHpcMNfXKPE+/Cc0y+PhxJX3A4xGNeFCj5oc0BUh6deg==", "hasInstallScript": true, "dependencies": { "es6-iterator": "^2.0.3", "es6-symbol": "^3.1.3", + "esniff": "^2.0.1", "next-tick": "^1.1.0" }, "engines": { @@ -5247,6 +5280,25 @@ "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "dev": true }, + "node_modules/esniff": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/esniff/-/esniff-2.0.1.tgz", + "integrity": "sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg==", + "dependencies": { + "d": "^1.0.1", + "es5-ext": "^0.10.62", + "event-emitter": "^0.3.5", + "type": "^2.7.2" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esniff/node_modules/type": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz", + "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==" + }, "node_modules/espree": { "version": "9.6.1", "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", @@ -5327,6 +5379,15 @@ "node": ">= 0.6" } }, + "node_modules/event-emitter": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", + "integrity": "sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==", + "dependencies": { + "d": "1", + "es5-ext": "~0.10.14" + } + }, "node_modules/event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", @@ -5403,16 +5464,16 @@ "integrity": "sha512-4EMSHGOPSwAfBiibw3ndnP0AvjDWLsMvGOvWEZ2F96IGk0bIVdjQisOHxReSkE13mHcfbuCiXw+G4y0zv6N8Eg==" }, "node_modules/express": { - "version": "4.18.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", - "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.1", + "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -5443,29 +5504,6 @@ "node": ">= 0.10.0" } }, - "node_modules/express/node_modules/body-parser": { - "version": "1.20.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", - "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", - "dependencies": { - "bytes": "3.1.2", - "content-type": "~1.0.4", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.11.0", - "raw-body": "2.5.1", - "type-is": "~1.6.18", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, "node_modules/express/node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -5484,20 +5522,6 @@ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, - "node_modules/express/node_modules/raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", - "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/ext": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", @@ -5748,9 +5772,9 @@ "dev": true }, "node_modules/follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "funding": [ { "type": "individual", @@ -6626,6 +6650,23 @@ "node": ">= 0.10" } }, + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/ip-address/node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==" + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -7550,6 +7591,11 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==" + }, "node_modules/jsdom": { "version": "22.1.0", "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-22.1.0.tgz", @@ -8473,6 +8519,12 @@ "node": ">= 4.0.0" } }, + "node_modules/memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "optional": true + }, "node_modules/merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -8631,6 +8683,86 @@ "num-sort": "^2.0.0" } }, + "node_modules/mongodb": { + "version": "5.9.2", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-5.9.2.tgz", + "integrity": "sha512-H60HecKO4Bc+7dhOv4sJlgvenK4fQNqqUIlXxZYQNbfEWSALGAwGoyJd/0Qwk4TttFXUOHJ2ZJQe/52ScaUwtQ==", + "dependencies": { + "bson": "^5.5.0", + "mongodb-connection-string-url": "^2.6.0", + "socks": "^2.7.1" + }, + "engines": { + "node": ">=14.20.1" + }, + "optionalDependencies": { + "@mongodb-js/saslprep": "^1.1.0" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.188.0", + "@mongodb-js/zstd": "^1.0.0", + "kerberos": "^1.0.0 || ^2.0.0", + "mongodb-client-encryption": ">=2.3.0 <3", + "snappy": "^7.2.2" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + } + } + }, + "node_modules/mongodb-connection-string-url": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.6.0.tgz", + "integrity": "sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==", + "dependencies": { + "@types/whatwg-url": "^8.2.1", + "whatwg-url": "^11.0.0" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/tr46": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "dependencies": { + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "engines": { + "node": ">=12" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/whatwg-url": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", + "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "dependencies": { + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -10170,6 +10302,28 @@ "node": ">=8" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", + "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", + "dependencies": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, "node_modules/source-map": { "version": "0.7.4", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", @@ -10198,6 +10352,15 @@ "node": ">=0.10.0" } }, + "node_modules/sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "optional": true, + "dependencies": { + "memory-pager": "^1.0.2" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", diff --git a/package.json b/package.json index d524a42..068b8c7 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "jsdom": "^22.1.0", "langchain": "^0.0.182", "mime": "^3.0.0", + "mongodb": "^5.9.2", "optional-js": "^2.3.0", "reflect-metadata": "^0.1.13", "rxjs": "^7.8.1", diff --git a/src/services/kyc-case/index.ts b/src/services/kyc-case/index.ts index f2ad157..84169fd 100644 --- a/src/services/kyc-case/index.ts +++ b/src/services/kyc-case/index.ts @@ -6,27 +6,33 @@ import {NegativeNewsApi, negativeNewsProvider} from "../negative-news"; import {cloudantBackend, CloudantBackendConfig} from "./cloudant.backend"; import {KycCaseManagementCloudant} from "./kyc-case-management.cloudant"; import {documentManagerApi} from "../document-manager"; +import {mongodbClient, mongodbConfig} from "./mongodb.backend"; +import {KycCaseManagementMongodb} from "./kyc-case-management.mongodb"; export * from './kyc-case-management.api'; -const backendConfig = cloudantBackend() +const cloudantBackendConfig = cloudantBackend() +const mongoBackendConfig = mongodbConfig() -let _instance: KycCaseManagementApi; +let _instance: Promise; export const kycCaseProvider: Provider = { provide: KycCaseManagementApi, - useFactory: (negativeNews: NegativeNewsApi): KycCaseManagementApi => { + useFactory: async (negativeNews: NegativeNewsApi): Promise => { if (_instance) { return _instance; } - let instance; - if (backendConfig.apikey && backendConfig.url) { - console.log('Loading cloudant backend') - instance = new KycCaseManagementCloudant(documentManagerApi(), backendConfig as CloudantBackendConfig) - } else { - instance = new KycCaseManagementMock(negativeNews); - } - - return _instance = instance; + return _instance = new Promise(async (resolve, reject) => { + if (mongoBackendConfig.username && mongoBackendConfig.password && mongoBackendConfig.connectString) { + console.log('Loading mongodb backend...'); + resolve(new KycCaseManagementMongodb(documentManagerApi(), await mongodbClient())) + } else if (cloudantBackendConfig.apikey && cloudantBackendConfig.url) { + console.log('Loading cloudant backend...') + resolve(new KycCaseManagementCloudant(documentManagerApi(), cloudantBackendConfig as CloudantBackendConfig)) + } else { + console.log('No database configuration found. Using mock KycCaseManagement instance...') + resolve(new KycCaseManagementMock(negativeNews)) + } + }) }, }; diff --git a/src/services/kyc-case/kyc-case-management.cloudant.ts b/src/services/kyc-case/kyc-case-management.cloudant.ts index 9a12d15..c78e3ca 100644 --- a/src/services/kyc-case/kyc-case-management.cloudant.ts +++ b/src/services/kyc-case/kyc-case-management.cloudant.ts @@ -104,7 +104,7 @@ export class KycCaseManagementCloudant implements KycCaseManagementApi { notifyKycCases(event: KycCaseChangeType = 'updated') { return (kycCase: KycCaseModel): KycCaseModel => { - this.changeSubject.next({kycCase, event: 'updated'}); + this.changeSubject.next({kycCase, event}); return kycCase; } diff --git a/src/services/kyc-case/kyc-case-management.mongodb.ts b/src/services/kyc-case/kyc-case-management.mongodb.ts new file mode 100644 index 0000000..8254994 --- /dev/null +++ b/src/services/kyc-case/kyc-case-management.mongodb.ts @@ -0,0 +1,332 @@ +import {BehaviorSubject, Observable, Subject} from "rxjs"; +import { + ApproveCaseModel, + createNewCase, + CustomerModel, + CustomerRiskAssessmentModel, + DocumentContent, + DocumentModel, + DocumentRef, + DocumentStream, + isDocumentContent, + isDocumentRef, + KycCaseChangeEventModel, + KycCaseChangeType, + KycCaseModel, + KycCaseSummaryModel, + NegativeScreeningModel, + ReviewCaseModel +} from "src/models"; +import {KycCaseManagementApi} from "./kyc-case-management.api"; +import {documentManagerApi, DocumentManagerApi} from "../document-manager"; +import {Collection, Db, GridFSBucket, ObjectId, WithId} from "mongodb"; +import Stream, {PassThrough} from "stream"; +import {streamToBuffer, urlToStream} from "../../utils"; + +interface DocumentRefModel { + id?: string; + name: string; + path: string; +} + +export class KycCaseManagementMongodb implements KycCaseManagementApi { + private readonly cases: Collection + private readonly documents: Collection + private readonly bucket: GridFSBucket; + + subject: BehaviorSubject; + changeSubject: Subject; + + constructor( + private readonly documentManagerService: DocumentManagerApi = documentManagerApi(), + db: Db, + ) { + this.cases = db.collection('kyc-cases'); + this.documents = db.collection('kyc-case-document') + this.bucket = new GridFSBucket(db, {bucketName: 'kyc-case-documents'}) + + this.subject = new BehaviorSubject([]) + this.changeSubject = new Subject() + } + + notifyKycCases(event: KycCaseChangeType) { + return (kycCase: KycCaseModel): KycCaseModel => { + this.changeSubject.next({kycCase, event}); + + return kycCase; + } + } + + async listCases(): Promise { + const result = this.cases + .find({}) + .map((doc: WithId) => Object.assign({}, doc, {id: doc._id})) + + return result.toArray() + } + + subscribeToCases(skipQuery?: boolean): Observable { + if (!skipQuery) { + // TODO??? + } + + return this.subject; + } + + async getCase(id: string): Promise { + const result = await this.cases.findOne(new ObjectId(id)) + + return Object.assign( + {}, + result, + { + id: result._id, + }) + } + + async createCase(customer: CustomerModel): Promise { + + const document: KycCaseModel = createNewCase(customer); + + return this.cases.insertOne(document) + .then(result => result.insertedId.toString()) + .then(caseId => Object.assign( + {}, + document, + { + id: caseId + } + )) + .then(this.notifyKycCases('created').bind(this)) + } + + async addDocumentToCase(caseId: string, documentName: string, document: DocumentRef | DocumentContent | DocumentStream, pathPrefix?: string): Promise { + const currentCase = await this.getCase(caseId); + + const content = await this.loadDocument(document); + + const newDoc = await this.documentManagerService.uploadFile({ + name: documentName, + parentId: caseId, + content: {buffer: content}, + context: 'kyc-case', + }); + + const caseDoc: DocumentModel = Object.assign({}, newDoc, {path: `${pathPrefix}${newDoc.path}`, content: Buffer.from('')}); + + const documents: DocumentModel[] = currentCase.documents.concat(caseDoc) + + await this.updateCase(caseId, {documents}) + + const documentResult = await this.documents + .insertOne({name: caseDoc.name, path: caseDoc.path}) + + const documentId = documentResult.insertedId; + + await new Promise((resolve, reject) => { + bufferToStream(content) + .pipe(this.bucket.openUploadStream(`${documentId}-${documentName}`, { + chunkSizeBytes: 1048576, + metadata: { + documentId, + name: documentName, + } + })) + .on('finish', () => resolve(true)) + .on('error', err => reject(err)) + }) + + return Object.assign(caseDoc, {id: documentId.toString()}) + } + + async loadDocument(document: DocumentRef | DocumentContent | DocumentStream): Promise { + if (isDocumentContent(document)) { + return document.content; + } + + const stream: Stream = isDocumentRef(document) + ? await urlToStream(document.url) + : document.stream; + + return streamToBuffer(stream); + } + + async removeDocumentFromCase(caseId: string, documentId: string): Promise { + const currentCase = await this.getCase(caseId); + + const documents: DocumentModel[] = currentCase.documents.slice() + + const documentIndex: number = documents.map(val => val.id).indexOf(documentId); + + // remove the document + documents.splice(documentIndex, 1); + + return this.documents + .deleteOne(new ObjectId(documentId)) + .then(() => this.updateCase(caseId, {documents})) + } + + async getDocument(id: string): Promise { + const documentRef: WithId = await this.documents + .findOne(new ObjectId(id)) + + const fileId = `${documentRef.id}-${documentRef.name}` + + const content = await streamToBuffer(this.bucket.openDownloadStreamByName(fileId)) + + return Object.assign( + {}, + documentRef, + { + id: documentRef._id, + content + }) + } + + async updateCase(id: string, updates: Partial) : Promise { + const result = await this.cases + .updateOne(new ObjectId(id), {$set: updates}) + + return this.getCase(id) + .then(this.notifyKycCases('updated').bind(this)) + } + + async reviewCase(reviewCase: ReviewCaseModel): Promise { + const updates: Partial = { + status: reviewCase.customerOutreach ? 'CustomerOutreach' : 'Pending' + }; + + return this.updateCase(reviewCase.id, updates) + } + + async approveCase(input: ApproveCaseModel): Promise { + return this.updateCase(input.id, {status: 'Pending'}) + } + + async processCase(id: string): Promise { + const updates: Partial = { + status: 'Pending', + negativeScreeningComplete: false, + counterpartyNegativeScreeningComplete: false, + customerRiskAssessmentComplete: false, + caseSummaryComplete: false, + } + + return this.updateCase(id, updates); + } + + async deleteCase(id: string): Promise { + const kycCase: KycCaseModel = await this.getCase(id); + + return this.cases.deleteOne(new ObjectId(id)) + .then(() => kycCase) + .then(this.notifyKycCases('deleted').bind(this)) + } + + async updateCustomerRiskAssessment(id: string, customerRiskAssessment: CustomerRiskAssessmentModel): Promise { + const currentCase: KycCaseModel = await this.getCase(id); + + const updates: Partial = { + customerRiskAssessment, + customerRiskAssessmentComplete: true, + status: this.getUpdatedStatus(currentCase), + }; + + return this.updateCase(id, updates); + } + + async updateNegativeNews(id: string, negativeScreening: NegativeScreeningModel): Promise { + const currentCase: KycCaseModel = await this.getCase(id); + + const updates: Partial = { + negativeScreening, + negativeScreeningComplete: true, + status: this.getUpdatedStatus(currentCase), + }; + + return this.updateCase(id, updates); + } + + async updateCounterpartyNegativeNews(id: string, counterpartyNegativeScreening: NegativeScreeningModel): Promise { + const currentCase: KycCaseModel = await this.getCase(id); + + const updates: Partial = { + counterpartyNegativeScreening, + counterpartyNegativeScreeningComplete: true, + status: this.getUpdatedStatus(currentCase), + }; + + return this.updateCase(id, updates); + } + + async updateCaseSummary(id: string, caseSummary: KycCaseSummaryModel): Promise { + const currentCase: KycCaseModel = await this.getCase(id); + + const updates: Partial = { + caseSummary, + caseSummaryComplete: true, + status: this.getUpdatedStatus(currentCase), + } + + return this.updateCase(id, updates) + } + + getUpdatedStatus(kycCase: KycCaseModel): string { + if (kycCase.status !== 'Pending') { + return kycCase.status; + } + + return kycCase.caseSummaryComplete && kycCase.negativeScreeningComplete && kycCase.counterpartyNegativeScreeningComplete && kycCase.customerRiskAssessmentComplete + ? 'Completed' + : 'Pending'; + } + + listDocuments(): Promise { + const result = this.documents + .find({}) + .map((doc: WithId) => Object.assign({}, doc, {id: doc._id})) + + return result.toArray() + } + + async deleteDocument(id: string): Promise { + const kycCaseDocument = await this.getDocument(id); + + await this.cases.deleteOne(new ObjectId(id)) + + return kycCaseDocument + } + + watchCaseChanges(): Observable { + return this.changeSubject; + } + + watchCase(id: string): Observable { + const subject: BehaviorSubject = new BehaviorSubject(undefined); + + console.log('Watching case: ' + id) + this.getCase(id).then(kycCase => { + subject.next(kycCase); + + this.changeSubject.subscribe({ + next: event => { + if (event.kycCase?.id === id) { + console.log('Updated case: ', {kycCase: event.kycCase}) + subject.next(event.kycCase); + } + } + }) + }); + + return subject.asObservable(); + } + +} + +export const bufferToStream = (buffer: Buffer): Stream => { + const bufferStream = new PassThrough(); + + bufferStream.end(buffer); + + return bufferStream; +} diff --git a/src/services/kyc-case/mongodb.backend.ts b/src/services/kyc-case/mongodb.backend.ts new file mode 100644 index 0000000..6a6d78b --- /dev/null +++ b/src/services/kyc-case/mongodb.backend.ts @@ -0,0 +1,75 @@ +import {Db, MongoClient} from "mongodb"; +import {dirname} from "path"; +import {promises} from "fs"; + +export interface MongodbConfig { + username: string; + password: string; + connectString: string; + databaseName: string; + certificateFile?: string; + certificateBase64?: string; +} + +let _config: MongodbConfig; +export const mongodbConfig = (): MongodbConfig | undefined => { + if (_config) { + return _config + } + + console.log('Building config...') + const config: MongodbConfig = { + username: process.env.MONGODB_USERNAME, + password: process.env.MONGODB_PASSWORD, + connectString: process.env.MONGODB_CONNECT_STRING, + databaseName: process.env.MONGODB_DATABASE_NAME, + certificateFile: process.env.MONGODB_CERTIFICATE_FILE, + certificateBase64: process.env.MONGODB_CERTIFICATE_BASE64, + } + + if (!config.username || !config.password || !config.connectString) { + console.log('MongoDB config not set: ' + JSON.stringify(config)) + return + } + + return _config = config +} + +let _client: Promise; +export const mongodbClient = async (): Promise => { + if (_client) { + return _client + } + + return _client = new Promise(async (resolve, reject) => { + try { + const config: MongodbConfig = mongodbConfig() + + let filename = config.certificateFile || '/tmp/cert/ca.crt' + if (config.certificateBase64) { + console.log( '** Processing certificate contents') + const cert = Buffer.from(config.certificateBase64, 'base64') + + const filepath = dirname(filename) + await promises.mkdir(filepath, {recursive: true}) + + await promises.writeFile(filename, cert) + } + + const client = new MongoClient( + config.connectString, + { + auth: { + username: config.username, + password: config.password, + }, + tlsCAFile: filename, + }) + .db(config.databaseName) + + resolve(client) + } catch (err) { + reject(err) + } + }) +}