Merge pull request #41 from HugoByte/next

Merge : Next to Master v0.1.1

CHANGELOG.md

@@ -1,2 +1,17 @@
 # 0.1.0
-* Initial release
+* Initial release
+* Reading Configuration based on Environment
+* Override Configuration if Environment Variables provided
+* Configurations for Chain endpoint Sections and Methods from extrinsic to Exclude, types, Openwhisk Endpoint, Openwhisk Auth Key, Trigger Endpoint, Kafka Topic and Brokers
+* Connects to the chain
+* Add custom type to chain initialization if provided
+* Subscribes to system.events
+* Filters Events based on excludes provided
+* Post Events to trigger Endpoint
+* Dockerfile for Substrate Event Feed Package
+
+# 0.1.1
+* Add dotenv to get environment variables from .env
+* Add Test env config
+* Update Schema Validation
+* Add Configuration for Health API Port

README.md

@@ -57,6 +57,22 @@ yarn install
 
 Configurations are passed through environment variables which can be found [here](/docs/configuration.md).
 
+For local development and testing create a **.env** file with respective configurations in the project root folder.
+
+```text
+CHAIN_NAME=Node Template
+CHAIN_ENDPOINT=ws://localhost:9944
+LOGGERS=console,info;file,error,/logs/event-feed.log
+EXCLUDES=system
+TYPES_FILE=/configs/types.json
+KAFKA_BROKERS=localhost:9092
+KAFKA_TOPIC=node-template-topic
+OPENWHISK_API_KEY=23bc46b1-71f6-4ed5-8c54-816aa4f8c502:123zO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP
+OPENWHISK_API_HOST=https://localhost:31001
+OPENWHISK_NAMESPACE=guest
+EVENT_RECEIVER=event-receiver
+```
+
 ### Usage
 
 Start the feed in development mode.

config/custom-environment-variables.json

@@ -9,5 +9,6 @@
     "openwhiskApiKey": "OPENWHISK_API_KEY",
     "openwhiskApiHost": "OPENWHISK_API_HOST",
     "openwhiskNamespace": "OPENWHISK_NAMESPACE",
-    "eventReceiver": "EVENT_RECEIVER"
+    "eventReceiver": "EVENT_RECEIVER",
+    "healthAPIPortConfiguration": "HEALTH_API_PORT"
 }

config/default.js

@@ -1,5 +1,5 @@
 const defer = require('config/defer').deferConfig;
-const { loggersHelper, excludesHelper, typesHelper, kafkaBrokersHelper } = require('./helper');
+const { loggersHelper, excludesHelper, typesHelper, kafkaBrokersHelper, healthAPIPortHelper } = require('./helper');
 
 module.exports = {
     // Name of the chain
@@ -44,5 +44,10 @@ module.exports = {
     openwhiskApiKey: undefined,
     openwhiskApiHost: undefined,    
     openwhiskNamespace: undefined,
-    eventReceiver: undefined
+    eventReceiver: undefined,
+    healthAPIPortConfiguration: "80",
+
+    healthAPIPort: defer(function (){
+        return healthAPIPortHelper(this.healthAPIPortConfiguration);
+    }),
 }

config/helper.js

@@ -98,5 +98,9 @@ module.exports = {
         }, []);
 
         return kafkaBrokers;
+    },
+
+    healthAPIPortHelper: function (healthAPIPortConfiguration) {
+        return parseInt(healthAPIPortConfiguration);
     }
 }

config/schema.json

@@ -7,8 +7,8 @@
         "chainName": {
             "description": "Name of the chain",
             "type": "string",
-            "matches": "[a-zA-Z]+",
-            "min": 3,
+            "pattern": "[a-zA-Z]+",
+            "minLength": 3,
             "maxLength": 40
         },
         "loggers": {
@@ -57,15 +57,74 @@
                 }
             }
         },
-        "chainEndpoint" :{
+        "chainEndpoint": {
             "description": "Websocket endpoint of the chain node",
             "type": "string",
             "pattern": "^(ws|wss)://"
+        },
+        "excludes": {
+            "description": "Sections and Methods of a specific section to be excluded",
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "section": {
+                        "type": "string"
+                    },
+                    "methods": {
+                        "type": [
+                            "array",
+                            "null"
+                        ]
+                    }
+                },
+                "required": [
+                    "section"
+                ]
+            }
+        },
+        "types": {
+            "description": "custom types for the chain.",
+            "type": "object"
+        },
+        "kafkaBrokers": {
+            "description": "List of Kafka brokers where the event should be posted",
+            "type": "array"
+        },
+        "kafkaTopic": {
+            "description": "Kafka topic to which events to be posted",
+            "type": "string"
+        },
+        "openwhiskApiKey": {
+            "description": "Openwhisk authentication key",
+            "type": "string"
+        },
+        "openwhiskApiHost": {
+            "description": "Openwhisk API Endpoint",
+            "type": "string"
+        },
+        "openwhiskNamespace": {
+            "description": "Organization space where the actions, rules, and triggers related to aurras resides",
+            "type": "string"
+        },
+        "eventReceiver": {
+            "description": "Trigger Endpoint where the event should be posted",
+            "type": "string"
+        },
+        "healthAPIPort": {
+            "type": "integer",
+            "description": "The binding port for the Health API Endpoint"
         }
     },
     "required": [
         "chainName",
         "loggers",
-        "chainEndpoint"
+        "chainEndpoint",
+        "kafkaBrokers",
+        "kafkaTopic",
+        "openwhiskApiKey",
+        "openwhiskApiHost",
+        "openwhiskNamespace",
+        "eventReceiver"
     ]
 }

config/test.json

@@ -0,0 +1,3 @@
+{
+    "chainName": "Test Chain"
+}

docs/configuration.md

@@ -4,7 +4,7 @@ Configuration values below are passed through environment variables.
 
 
 
-#### CHAIN\_NAME
+#### CHAIN\_NAME [Mandatory]
 
 An alphanumeric string to identify the chain.
 
@@ -14,7 +14,7 @@ An alphanumeric string to identify the chain.
 
 
 
-#### CHAIN\_ENDPOINT
+#### CHAIN\_ENDPOINT [Mandatory]
 
 The Endpoint of the chain node to which the event feed should connect to. Protocols Supported: ws \(WebSocket\) and wss \(WebSocket Secure\)
 
@@ -40,7 +40,7 @@ The Endpoint of the chain node to which the event feed should connect to. Protoc
 
 
 
-#### LOGGERS
+#### LOGGERS [Mandatory]
 
 The configuration pertains to the loggers enabled for the event feed. This configuration is extensible to add multiple logging such as logging to a file, logging to console, logging to monitoring system based on different levels of logging. Winston is used under the hood. 
 
@@ -67,7 +67,7 @@ Multiple loggers can be provided separated by ";"
 
 
 
-#### EXCLUDES
+#### EXCLUDES [Optional]
 
 Sections or Methods of a specific section can be excluded provided through this configuration.
 
@@ -86,7 +86,7 @@ Multiple methods to be separated by ","
 
 
 
-#### TYPES\_FILE
+#### TYPES\_FILE [Optional]
 
 Location to custom types for the chain.
 
@@ -96,7 +96,7 @@ Location to custom types for the chain.
 
 
 
-#### KAFKA\_BROKERS
+#### KAFKA\_BROKERS [Mandatory]
 
 List of Kafka brokers where the event should be posted. separated by ";"
 
@@ -106,7 +106,7 @@ List of Kafka brokers where the event should be posted. separated by ";"
 
 
 
-#### KAFKA\_TOPIC
+#### KAFKA\_TOPIC [Mandatory]
 
 Kafka topic to which events to be posted ";"
 
@@ -116,7 +116,7 @@ Kafka topic to which events to be posted ";"
 
 
 
-#### OPENWHISK\_API\_KEY
+#### OPENWHISK\_API\_KEY [Mandatory]
 
 Openwhisk authentication key.
 
@@ -126,7 +126,7 @@ Openwhisk authentication key.
 
 
 
-#### OPENWHISK\_API\_HOST
+#### OPENWHISK\_API\_HOST [Mandatory]
 
 Openwhisk API Endpoint
 
@@ -136,10 +136,28 @@ Openwhisk API Endpoint
 
 
 
-#### OPENWHISK\_NAMESPACE
+#### OPENWHISK\_NAMESPACE [Mandatory]
 
 Organization space where the actions, rules, and triggers related to aurras resides.
 
 | Environment Variable | Sample Values |
 | :--- | :--- |
 | OPENWHISK\_NAMESPACE | `OPENWHISK_NAMESPACE=guest` |
+
+
+#### EVENT\_RECEIVER [Mandatory]
+
+Trigger Endpoint where the event should be posted.
+
+| Environment Variable | Sample Values |
+| :--- | :--- |
+| EVENT\_RECEIVER | `EVENT_RECEIVER=event-receiver` |
+
+
+#### HEALTH\_API\_PORT [Optional]
+
+Configuration for Health API port primarily used for Kubernetes readiness and liveness probe. Default port is 80.
+
+| Environment Variable | Sample Values |
+| :--- | :--- |
+| HEALTH\_API\_PORT | `HEALTH_API_PORT=8080` |

package-scripts.js

@@ -20,7 +20,7 @@ module.exports = {
     serve: {
       script: series(
         'nps banner.serve',
-        'nodemon --watch src --watch config'
+        'nodemon -r dotenv/config --watch src --watch config'
       ),
       description: 'Serves the current app and watches for changes to restart it'
     },

package.json

@@ -1,6 +1,6 @@
 {
   "name": "aurras-event-feed-substrate-js",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "A JS implementation of Event Feed package for Aurras System to source events from Substrate Based chains",
   "main": "src/index.ts",
   "repository": "github:HugoByte/aurras-event-feed-substrate-js",
@@ -35,14 +35,14 @@
   "dependencies": {
     "@hugobyte/microbootstrap": "^0.1.0",
     "@polkadot/api": "^3.3.2",
+    "ajv": "^7.1.1",
     "config": "^3.3.3",
+    "dotenv": "^8.2.0",
     "lodash": "^4.17.20",
     "micro": "^9.3.4",
     "openwhisk": "^3.21.3",
     "reflect-metadata": "^0.1.13",
-    "schema-to-yup": "^1.10.0",
     "typedi": "^0.8.0",
-    "winston": "^3.3.3",
-    "yup": "^0.32.8"
+    "winston": "^3.3.3"
   }
 }

src/exceptions/configuration.exception.ts

@@ -6,7 +6,24 @@ import { Exception } from './exception';
  */
 
 export class ConfigurationException extends Exception {
-  constructor(description) {
-    super('ConfigurationException', description, false);
+  constructor(configuration, description) {
+    const configurationMap = {
+      "/chainName": "CHAIN_NAME",
+      "/chainEndpoint": "CHAIN_ENDPOINT",
+      "/loggers": "LOGGERS",
+      "/loggers/console/level": "LOGGERS Console Level",
+      "/loggers/file/level": "LOGGERS File Level",
+      "/loggers/file/filename": "LOGGERS File Location",
+      "/excludes": "EXCLUDES",
+      "/kafkaBrokers": "KAFKA_BROKERS",
+      "/kafkaTopic": "KAFKA_TOPIC",
+      "/openwhiskApiKey": "OPENWHISK_API_KEY",
+      "/openwhiskApiHost": "OPENWHISK_API_HOST",
+      "/openwhiskNamespace": "OPENWHISK_NAMESPACE",
+      "/eventReceiver": "EVENT_RECEIVER",
+      "/healthAPIPort": "HEALTH_API_PORT"
+    }
+
+    super('ConfigurationException', `${configurationMap[configuration] || configuration} ${description}`, false);
   }
 }

src/modules/configuration.module.ts

@@ -1,14 +1,14 @@
 import { MicrobootstrapSettings, MicrobootstrapLoader } from '@hugobyte/microbootstrap';
 import { util } from 'config';
-import { buildYup } from 'schema-to-yup';
 import { readFileSync } from 'fs';
 import { join } from 'path';
 import { ConfigurationException } from "@exceptions/index";
+import Ajv, { DefinedError } from "ajv"
 
 /**
  * Configuration Module validates the config passed through environment variables with the schema provided in /config/schema.json.
  * Config is parsed and loaded using https://github.com/lorenwest/node-config.
- * Config validation is done using https://github.com/jquense/yup.
+ * Config validation is done using https://github.com/ajv-validator/ajv.
  */
 
 export const ConfigurationModule: MicrobootstrapLoader = (frameworkSettings: MicrobootstrapSettings | undefined) => {
@@ -20,13 +20,21 @@ export const ConfigurationModule: MicrobootstrapLoader = (frameworkSettings: Mic
             validateConfiguration({ schema, configuration });
         }
         catch (error) {
-            if (typeof error === 'string') throw new ConfigurationException(error);
-            throw new ConfigurationException(error.message);
+            if (typeof error === 'string') throw new ConfigurationException("", error);
+            throw new ConfigurationException(error.dataPath, error.message);
         }
     }
 }
 
 export const validateConfiguration = ({ schema, configuration }) => {
-    const yupSchema = buildYup(schema, {});
-    return yupSchema.validateSync(configuration)
+    const ajv = new Ajv();
+    const validate = ajv.compile(schema);
+    const isValidConfiguration = validate(configuration);
+
+    if (!isValidConfiguration) {
+        for (const error of validate.errors as DefinedError[]) {
+            throw error;
+        }
+    }
+    return isValidConfiguration;
 }