From 28235dcb7adee25992925bbae3ad6b469ee19905 Mon Sep 17 00:00:00 2001 From: otherNet Date: Wed, 3 May 2023 18:12:45 +0200 Subject: [PATCH] rebase on env_EcoSys Signed-off-by: otherNet --- .env | 4 + Makefile | 10 +- README.md | 53 ++-- api-gateway/.env | 12 +- api-gateway/Dockerfile | 3 +- api-gateway/configs/.env.gateway | 13 + api-gateway/configs/.env.gateway.develop | 13 + api-gateway/configs/.env.gateway.template | 13 + api-gateway/src/config.ts | 22 ++ auth-service/.env | 19 +- auth-service/Dockerfile | 3 +- auth-service/configs/.env.auth | 20 ++ auth-service/configs/.env.auth.develop | 20 ++ auth-service/configs/.env.auth.template | 20 ++ auth-service/src/config.ts | 22 ++ common/src/helpers/db-helper.ts | 6 +- configs/.env..guardian.system | 94 +++++++ configs/.env.develop.guardian.system | 94 +++++++ configs/.env.template.guardian.system | 105 +++++++ configs/.env_SSV | 44 +++ docker-compose-DEV-ONLYSERVERS.yml | 63 +++++ docker-compose.yml | 67 +++-- docker-compose_SSV.yml | 215 +++++++++++++++ docs/environments/Ecosystem-Environment.md | 257 ++++++++++++++++++ .../getting-started/installation.md | 2 +- guardian-service/.env | 26 +- guardian-service/.env.docker | 4 +- guardian-service/Dockerfile | 3 +- guardian-service/configs/.env.guardian | 31 +++ .../configs/.env.guardian.develop | 31 +++ .../configs/.env.guardian.template | 37 +++ guardian-service/src/config.ts | 22 ++ logger-service/.env | 10 +- logger-service/Dockerfile | 3 +- logger-service/configs/.env.logger | 11 + logger-service/configs/.env.logger.develop | 11 + logger-service/configs/.env.logger.template | 11 + logger-service/src/config.ts | 22 ++ policy-service/.env | 18 +- policy-service/.env.example | 2 +- policy-service/Dockerfile | 3 +- policy-service/configs/.env.policy | 16 ++ policy-service/configs/.env.policy.develop | 16 ++ policy-service/configs/.env.policy.template | 16 ++ policy-service/src/config.ts | 22 ++ worker-service/.env | 20 +- worker-service/.env.docker | 2 +- worker-service/Dockerfile | 3 +- worker-service/configs/.env.worker | 21 ++ worker-service/configs/.env.worker.develop | 21 ++ worker-service/configs/.env.worker.template | 21 ++ worker-service/src/app.ts | 20 +- worker-service/src/config.ts | 22 ++ 53 files changed, 1466 insertions(+), 173 deletions(-) create mode 100644 .env create mode 100644 api-gateway/configs/.env.gateway create mode 100644 api-gateway/configs/.env.gateway.develop create mode 100644 api-gateway/configs/.env.gateway.template create mode 100644 auth-service/configs/.env.auth create mode 100644 auth-service/configs/.env.auth.develop create mode 100644 auth-service/configs/.env.auth.template create mode 100644 configs/.env..guardian.system create mode 100644 configs/.env.develop.guardian.system create mode 100644 configs/.env.template.guardian.system create mode 100644 configs/.env_SSV create mode 100644 docker-compose-DEV-ONLYSERVERS.yml create mode 100644 docker-compose_SSV.yml create mode 100644 docs/environments/Ecosystem-Environment.md create mode 100644 guardian-service/configs/.env.guardian create mode 100644 guardian-service/configs/.env.guardian.develop create mode 100644 guardian-service/configs/.env.guardian.template create mode 100644 logger-service/configs/.env.logger create mode 100644 logger-service/configs/.env.logger.develop create mode 100644 logger-service/configs/.env.logger.template create mode 100644 policy-service/configs/.env.policy create mode 100644 policy-service/configs/.env.policy.develop create mode 100644 policy-service/configs/.env.policy.template create mode 100644 worker-service/configs/.env.worker create mode 100644 worker-service/configs/.env.worker.develop create mode 100644 worker-service/configs/.env.worker.template diff --git a/.env b/.env new file mode 100644 index 0000000000..e9151d6ab2 --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +## GUARDIAN ECOSYSTEM ENVIRONMENT NAME +# if you already started to use actively Guardian leave GUARDIAN_ENV empty when you want to use it as usual +# if you want to use a new environment configure GUARDIAN_ENV with a significative environment name +GUARDIAN_ENV="develop" diff --git a/Makefile b/Makefile index 25d2ed32a9..e349448c08 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ guardian_build: cd policy-service && yarn install && yarn build:prod cd topic-viewer && yarn install && yarn build:prod cd mrv-sender && yarn install && yarn build:prod - cd frontend && yarn install && yarn build:prod + cd frontend && yarn install && export NODE_OPTIONS=--openssl-legacy-provider && yarn build:prod guardian_make_env: cd logger-service && cp .env.example .env && cp .env.docker.example .env.docker @@ -51,11 +51,11 @@ guardian_make_env: cd policy-service && cp .env.example .env && cp .env.docker.example .env.docker guardian_up_pm2: - docker-compose -f docker-compose-dev.yml up -d mongo message-broker ipfs-node + docker compose -f docker-compose-dev.yml up -d mongo message-broker ipfs-node pm2 start ecosystem.config.js guardian_down_pm2: - docker-compose stop mongo - docker-compose rm -s -v mongo + docker compose stop mongo + docker compose rm -s -v mongo pm2 delete all - \ No newline at end of file + diff --git a/README.md b/README.md index 5c86856e57..a1c5290681 100644 --- a/README.md +++ b/README.md @@ -47,22 +47,20 @@ If you build with docker [MongoDB V6](https://www.mongodb.com), [NodeJS V16](htt git clone https://github.com/hashgraph/guardian.git ``` -2. Update the following files with your Hedera Testnet account info (see prerequisites) as indicated. Please check complete steps to generate Operation ID and Operator Key by looking at link : [How to Create Operator ID and Operator Key](https://docs.hedera.com/guardian/getting-started/getting-started/how-to-create-operator-id-and-operator-key). There will be other steps in the Demo Usage Guide that will require the generation of Operator IDs and Operator Keys. It is important to mention that the Operator IDs and Operator Keys in the .env will be used to generate demo accounts. +2. Update the following files with your Hedera Testnet account info (see prerequisites) as indicated. Please check complete steps to generate Operation ID and Operator Key by looking at link : [How to Create Operator ID and Operator Key](https://docs.hedera.com/guardian/getting-started/getting-started/how-to-create-operator-id-and-operator-key). There will be other steps in the Demo Usage Guide that will require the generation of Operator IDs and Operator Keys. It is important to mention that the Operator IDs and Operator Keys in the ./guardian/configs/.env.\.guardian.system will be used to generate demo accounts. For example: - in `guardian-service/.env`: - + in ./guardian/.env you may choose name of the Guardian platform. Leave the field empty or unspecified if you update a production environment to keep previous data (for more details read at https://github.com/IntellectEU/guardian/blob/feature/environment/docs/environments/Ecosystem-Envitonment.md) ```plaintext - OPERATOR_ID="" - OPERATOR_KEY="" + GUARDIAN_ENV="develop" ``` - - in `guardian-service/.env.docker`: + + in ./guardian/configs/.env.develop.guardian.system ```plaintext - OPERATOR_ID="" - OPERATOR_KEY="" + OPERATOR_ID="..." + OPERATOR_KEY="..." ``` **Note**. You can use the Schema Topic ID (`INITIALIZATION_TOPIC_ID`) already present in the configuration files, or you can specify your own. @@ -71,24 +69,24 @@ If you build with docker [MongoDB V6](https://www.mongodb.com), [NodeJS V16](htt **3.1 Setting up IPFS Local node:** - 3.1.1 We need to install and configure any IPFS node. + - 3.1.1 We need to install and configure any IPFS node. For example: https://github.com/yeasy/docker-ipfs - 3.1.2 For setup IPFS local node you need to set variables in `worker-service` folder + - 3.1.2 For setup IPFS local node you need to set variables in the same file ./guardian/configs/.env.develop.guardian.system ``` IPFS_NODE_ADDRESS="..." # Default IPFS_NODE_ADDRESS="http://localhost:5002" - IPFS_PUBLIC_GATEWAY="..." # Default IPFS_PUBLIC_GATEWAY="https://localhost:8080/ipfs/${cid}" + IPFS_PUBLIC_GATEWAY='...' # Default IPFS_PUBLIC_GATEWAY='https://localhost:8080/ipfs/${cid}' IPFS_PROVIDER="local" ``` **Note** 1. Default IPFS_NODE_ADDRESS="http://localhost:5002" - 2. Default IPFS_PUBLIC_GATEWAY="https://localhost:8080/ipfs/${cid}" + 2. Default IPFS_PUBLIC_GATEWAY='https://localhost:8080/ipfs/${cid}' **3.2 Setting up IPFS Web3Storage node:** - 3.2.1 For setup IPFS web3storage node you need to set variables in `worker-service` folder: + 3.2.1 For setup IPFS web3storage node you need to set variables in the same file ./guardian/configs/.env.develop.guardian.system ``` IPFS_STORAGE_API_KEY="..." @@ -100,8 +98,10 @@ If you build with docker [MongoDB V6](https://www.mongodb.com), [NodeJS V16](htt 4. Build and launch with Docker. Please note that this build is meant to be used in production and will not contain any debug information. From the project's root folder: ```shell - docker-compose up -d --build + docker compose up -d --build ``` +**Note**. about docker-compose: from the end of June 2023 Compose V1 won’t be supported anymore and will be removed from all Docker Desktop versions. Make sure you use Docker Compose V2 (comes with Docker Desktop > 3.6.0) as at https://docs.docker.com/compose/install/ + 5. Browse to and complete the setup. @@ -118,6 +118,25 @@ If you want to manually build every component with debug information, then build ### Build and start each component Install, configure and start all the prerequisites, then build and start each component. +Configure .env file in each service + + For example: + + in `guardian-service/.env`: + ```plaintext + GUARDIAN_ENV="develop" + ``` + + If need to configure OVERRIDE variables add it in .env file. + ```plaintext + OVERRIDE="false" + ``` + + in `guardian-service/configs/.env.guardian.develop`: + ```plaintext + OPERATOR_ID="..." + OPERATOR_KEY="..." + ``` **Note: Once you start each service, please wait for the initialization process to be completed.** @@ -329,7 +348,7 @@ Install, configure and start all the prerequisites, then build and start each co 2. Start local development using docker compose ```shell - docker-compose -f docker-compose-dev.yml up --build + docker compose -f docker-compose-dev.yml up --build ``` 3. Access local development using or @@ -345,7 +364,7 @@ Install, configure and start all the prerequisites, then build and start each co **To run by cleaning Docker cache**: ```shell - docker-compose build --no-cache + docker compose build --no-cache ``` ([back to top](readme)) diff --git a/api-gateway/.env b/api-gateway/.env index 5edd71fe8b..18face0ea2 100644 --- a/api-gateway/.env +++ b/api-gateway/.env @@ -1,11 +1 @@ -ENV="develop" -HEDERA_NET="testnet" -PREUSED_HEDERA_NET="testnet" -MQ_ADDRESS="localhost" -SERVICE_CHANNEL="api-gateway" -MRV_ADDRESS="http://localhost:3003/mrv" -MQ_MAX_PAYLOAD="1048576" -#LOG_LEVEL="2" -#MQ_MESSAGE_CHUNK=5000000 -#RAW_REQUEST_LIMIT="1gb" -#JSON_REQUEST_LIMIT="1mb" +GUARDIAN_ENV="" diff --git a/api-gateway/Dockerfile b/api-gateway/Dockerfile index 89b3d072fc..cef1cbcb9f 100644 --- a/api-gateway/Dockerfile +++ b/api-gateway/Dockerfile @@ -36,7 +36,8 @@ COPY --from=interfacesModuleBuilder /usr/local/interfaces/guardian-interfaces-*. COPY --from=commonModuleBuilder /usr/local/common/guardian-common-*.tgz /tmp/common.tgz COPY --from=apiGatewayBuilder /usr/local/api-gateway/yarn.lock ./ COPY ./api-gateway/package.json ./ -# COPY ./api-gateway/.env.docker ./.env +RUN mkdir -p ./configs +COPY ./api-gateway/configs/* ./configs RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/interfaces']='file:/tmp/interfaces.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));" RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/common']='file:/tmp/common.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));" RUN yarn install --frozen-lockfile diff --git a/api-gateway/configs/.env.gateway b/api-gateway/configs/.env.gateway new file mode 100644 index 0000000000..00b234e9d2 --- /dev/null +++ b/api-gateway/configs/.env.gateway @@ -0,0 +1,13 @@ +# Gateway Service Specialized Variables +SERVICE_CHANNEL="api-gateway" + +# Ecosystem Defined Variables +HEDERA_NET="testnet" +PREUSED_HEDERA_NET="testnet" +MQ_ADDRESS="localhost" +MRV_ADDRESS="http://localhost:3003/mrv" +MQ_MAX_PAYLOAD="1048576" +#LOG_LEVEL="2" +#MQ_MESSAGE_CHUNK=5000000 +#RAW_REQUEST_LIMIT="1gb" +#JSON_REQUEST_LIMIT="1mb" diff --git a/api-gateway/configs/.env.gateway.develop b/api-gateway/configs/.env.gateway.develop new file mode 100644 index 0000000000..00b234e9d2 --- /dev/null +++ b/api-gateway/configs/.env.gateway.develop @@ -0,0 +1,13 @@ +# Gateway Service Specialized Variables +SERVICE_CHANNEL="api-gateway" + +# Ecosystem Defined Variables +HEDERA_NET="testnet" +PREUSED_HEDERA_NET="testnet" +MQ_ADDRESS="localhost" +MRV_ADDRESS="http://localhost:3003/mrv" +MQ_MAX_PAYLOAD="1048576" +#LOG_LEVEL="2" +#MQ_MESSAGE_CHUNK=5000000 +#RAW_REQUEST_LIMIT="1gb" +#JSON_REQUEST_LIMIT="1mb" diff --git a/api-gateway/configs/.env.gateway.template b/api-gateway/configs/.env.gateway.template new file mode 100644 index 0000000000..4d502da62f --- /dev/null +++ b/api-gateway/configs/.env.gateway.template @@ -0,0 +1,13 @@ +# Gateway Service Specialized Variables +SERVICE_CHANNEL="api-gateway" + +# Ecosystem Defined Variables +HEDERA_NET="" +PREUSED_HEDERA_NET="" +MQ_ADDRESS="" +MRV_ADDRESS="" +MQ_MAX_PAYLOAD="" +#LOG_LEVEL="2" +#MQ_MESSAGE_CHUNK=5000000 +#RAW_REQUEST_LIMIT="1gb" +#JSON_REQUEST_LIMIT="1mb" diff --git a/api-gateway/src/config.ts b/api-gateway/src/config.ts index 09b42d9446..79766a60c2 100644 --- a/api-gateway/src/config.ts +++ b/api-gateway/src/config.ts @@ -1,6 +1,7 @@ import moduleAlias from 'module-alias'; import dotenv from 'dotenv'; import 'reflect-metadata' +import fs from 'fs'; moduleAlias.addAliases({ '@api': __dirname + '/api', @@ -11,3 +12,24 @@ moduleAlias.addAliases({ }); dotenv.config(); + +const envPath = process.env.GUARDIAN_ENV ? `./configs/.env.gateway.${process.env.GUARDIAN_ENV}` : './configs/.env.gateway'; + +if (!process.env.OVERRIDE || process.env.OVERRIDE === 'false'){ + console.log('reading from', envPath, 'not overriding'); + dotenv.config({ path: envPath}); +}else{ + try { + const envConfig = dotenv.parse(fs.readFileSync(envPath)); + for (const k of Object.keys(envConfig)) { + process.env[k] = envConfig[k] + } + } catch (err) { + if (err.code === 'ENOENT') { + console.log('WARN: Specific environment not loaded'); + } else { + throw err; + } + } +} +console.log('Charged Environment',process.env,'\r\n___ . ___'); diff --git a/auth-service/.env b/auth-service/.env index 54e02489af..18face0ea2 100644 --- a/auth-service/.env +++ b/auth-service/.env @@ -1,18 +1 @@ -ENV="develop" -HEDERA_NET="testnet" -PREUSED_HEDERA_NET="testnet" -MQ_ADDRESS="localhost" -SERVICE_CHANNEL="auth-service" -ACCESS_TOKEN_SECRET="youraccesstokensecret" -DB_HOST="localhost" -DB_DATABASE="auth_db" -VAULT_PROVIDER="database" -HASHICORP_TOKEN="1234" -HASHICORP_ADDRESS="http://localhost:8200" -HASHICORP_NAMESPACE="admin" -HASHICORP_ENCRIPTION_ALG="sha512" -MQ_MAX_PAYLOAD="1048576" -#LOG_LEVEL="2" -#HASHICORP_USEAL_KEY="" -#IMPORT_KEYS_FROM_DB=1 -#MQ_MESSAGE_CHUNK=5000000 +GUARDIAN_ENV="" diff --git a/auth-service/Dockerfile b/auth-service/Dockerfile index c52b48d39b..62475f3570 100644 --- a/auth-service/Dockerfile +++ b/auth-service/Dockerfile @@ -36,7 +36,8 @@ COPY --from=interfacesModuleBuilder /usr/local/interfaces/guardian-interfaces-*. COPY --from=commonModuleBuilder /usr/local/common/guardian-common-*.tgz /tmp/common.tgz COPY --from=authServiceBuilder /usr/local/auth-service/yarn.lock ./ COPY ./auth-service/package.json ./ -# COPY ./auth-service/.env.docker ./.env +RUN mkdir -p ./configs +COPY ./auth-service/configs/* ./configs RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/interfaces']='file:/tmp/interfaces.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));" RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/common']='file:/tmp/common.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));" RUN yarn install --frozen-lockfile diff --git a/auth-service/configs/.env.auth b/auth-service/configs/.env.auth new file mode 100644 index 0000000000..27707bfc49 --- /dev/null +++ b/auth-service/configs/.env.auth @@ -0,0 +1,20 @@ +# Auth Service Specialized Variables +SERVICE_CHANNEL="auth-service" +DB_DATABASE="auth_db" + +# Ecosystem Defined Variables +HEDERA_NET="testnet" +PREUSED_HEDERA_NET="testnet" +MQ_ADDRESS="localhost" +ACCESS_TOKEN_SECRET="youraccesstokensecret" +DB_HOST="localhost" +VAULT_PROVIDER="database" +HASHICORP_TOKEN="1234" +HASHICORP_ADDRESS="http://localhost:8200" +HASHICORP_NAMESPACE="admin" +HASHICORP_ENCRIPTION_ALG="sha512" +MQ_MAX_PAYLOAD="1048576" +#LOG_LEVEL="2" +#HASHICORP_UNSEAL_KEY="" +#IMPORT_KEYS_FROM_DB=1 +#MQ_MESSAGE_CHUNK=5000000 diff --git a/auth-service/configs/.env.auth.develop b/auth-service/configs/.env.auth.develop new file mode 100644 index 0000000000..27707bfc49 --- /dev/null +++ b/auth-service/configs/.env.auth.develop @@ -0,0 +1,20 @@ +# Auth Service Specialized Variables +SERVICE_CHANNEL="auth-service" +DB_DATABASE="auth_db" + +# Ecosystem Defined Variables +HEDERA_NET="testnet" +PREUSED_HEDERA_NET="testnet" +MQ_ADDRESS="localhost" +ACCESS_TOKEN_SECRET="youraccesstokensecret" +DB_HOST="localhost" +VAULT_PROVIDER="database" +HASHICORP_TOKEN="1234" +HASHICORP_ADDRESS="http://localhost:8200" +HASHICORP_NAMESPACE="admin" +HASHICORP_ENCRIPTION_ALG="sha512" +MQ_MAX_PAYLOAD="1048576" +#LOG_LEVEL="2" +#HASHICORP_UNSEAL_KEY="" +#IMPORT_KEYS_FROM_DB=1 +#MQ_MESSAGE_CHUNK=5000000 diff --git a/auth-service/configs/.env.auth.template b/auth-service/configs/.env.auth.template new file mode 100644 index 0000000000..433c4e8367 --- /dev/null +++ b/auth-service/configs/.env.auth.template @@ -0,0 +1,20 @@ +# Auth Service Specialized Variables +SERVICE_CHANNEL="auth-service" +DB_DATABASE="auth_db" + +# Ecosystem Defined Variables +HEDERA_NET="" +PREUSED_HEDERA_NET="" +MQ_ADDRESS="" +ACCESS_TOKEN_SECRET="youraccesstokensecret" +DB_HOST="" +VAULT_PROVIDER="database" +HASHICORP_TOKEN="1234" +HASHICORP_ADDRESS="" +HASHICORP_NAMESPACE="admin" +HASHICORP_ENCRIPTION_ALG="sha512" +MQ_MAX_PAYLOAD="" +#LOG_LEVEL="2" +#HASHICORP_UNSEAL_KEY="" +#IMPORT_KEYS_FROM_DB=1 +#MQ_MESSAGE_CHUNK=5000000 diff --git a/auth-service/src/config.ts b/auth-service/src/config.ts index fe48bc62a8..b3e587a95a 100644 --- a/auth-service/src/config.ts +++ b/auth-service/src/config.ts @@ -1,5 +1,6 @@ import moduleAlias from 'module-alias'; import dotenv from 'dotenv'; +import fs from 'fs'; moduleAlias.addAliases({ '@api': __dirname + '/api', @@ -8,3 +9,24 @@ moduleAlias.addAliases({ }); dotenv.config(); + +const envPath = process.env.GUARDIAN_ENV ? `./configs/.env.auth.${process.env.GUARDIAN_ENV}` : './configs/.env.auth'; + +if (!process.env.OVERRIDE || process.env.OVERRIDE === 'false'){ + console.log('reading from', envPath, 'not overriding'); + dotenv.config({ path: envPath}); +}else{ + try { + const envConfig = dotenv.parse(fs.readFileSync(envPath)); + for (const k of Object.keys(envConfig)) { + process.env[k] = envConfig[k] + } + } catch (err) { + if (err.code === 'ENOENT') { + console.log('WARN: Specific environment not loaded'); + } else { + throw err; + } + } +} +console.log('Charged Environment',process.env,'\r\n___ . ___'); diff --git a/common/src/helpers/db-helper.ts b/common/src/helpers/db-helper.ts index d474888e68..d727d22bc2 100644 --- a/common/src/helpers/db-helper.ts +++ b/common/src/helpers/db-helper.ts @@ -14,9 +14,9 @@ import { GridFSBucket } from 'mongodb'; export const COMMON_CONNECTION_CONFIG: any = { type: 'mongo', namingStrategy: DataBaseNamingStrategy, - dbName: (process.env.ENV||(process.env.HEDERA_NET!==process.env.PREUSED_HEDERA_NET))? - `${process.env.ENV}_${process.env.HEDERA_NET}_${process.env.DB_DATABASE}`: - process.env.DB_DATABASE, + dbName: (process.env.GUARDIAN_ENV||(process.env.HEDERA_NET!==process.env.PREUSED_HEDERA_NET))? + `${process.env.GUARDIAN_ENV}_${process.env.HEDERA_NET}_${process.env.DB_DATABASE}`: + process.env.DB_DATABASE, clientUrl:`mongodb://${process.env.DB_HOST}`, entities: [ 'dist/entity/*.js' diff --git a/configs/.env..guardian.system b/configs/.env..guardian.system new file mode 100644 index 0000000000..6564d37961 --- /dev/null +++ b/configs/.env..guardian.system @@ -0,0 +1,94 @@ +# ECOSYSTEM ENVIRONMENT VARIABLES AND FEATURES + +ENV="develop" + +# OVERRIDE - default "false" +# --------------------------- +# OVERRIDE="false" + +# HEDERA_NET - MANDATORY +# ------------------------ +HEDERA_NET="testnet" + +# PRE USED HEDERA_NET +# --------------------- +PREUSED_HEDERA_NET="testnet" + +# TESTNET +OPERATOR_ID="0.0.3422318" +OPERATOR_KEY="302e020100300506032b6570042204207a032dab9919a648142572069e8a4ccf34c805215e7099b30abd63fd1c58bd3c" +INITIALIZATION_TOPIC_ID="0.0.2030" + +# # LOCALNODE +# LOCALNODE_ADDRESS="..." +# LOCALNODE_PROTOCOL="http" +# OPERATOR_ID="0.0.2" +# OPERATOR_KEY="302e020100300506032b65700422042091132178e72057a1d7528025956fe39b0b847f200ab59b2fdd367017f3087137" + +# # PREVIEWNET +# OPERATOR_ID="..." +# OPERATOR_KEY="..." +# INITIALIZATION_TOPIC_ID="0.0.155110" + +SEND_KEYS_TO_VAULT="TRUE" +CONTRACT_FILE_ID="0.0.6276" +MAX_HEDERA_TIMEOUT="600" +# HEDERA_CUSTOM_NODES={"0.testnet.hedera.com:50211":"0.0.3"} +# HEDERA_CUSTOM_MIRROR_NODES=["testnet.mirrornode.hedera.com:443"] + +# MAX_TRANSACTION_FEE="10" + +# ADDRESSING / SERVICES +# ---------------------- +DIRECT_MESSAGE_PORT="6555" + +MQ_ADDRESS="message-broker" +MQ_MAX_PAYLOAD="1048576" + +MRV_ADDRESS="http://message-broker:3003/mrv" + +DB_HOST="mongo" + +ACCESS_TOKEN_SECRET="youraccesstokensecret" + +VAULT_PROVIDER="database" +HASHICORP_TOKEN="1234" +HASHICORP_ADDRESS="http://vault:8200" +HASHICORP_NAMESPACE="admin" +HASHICORP_ENCRIPTION_ALG="sha512" +#HASHICORP_UNSEAL_KEY="" + +#MAP_API_KEY="..." +MIN_PRIORITY="0" +MAX_PRIORITY="20" +TASK_TIMEOUT="300" +REFRESH_INTERVAL="60" + +IPFS_TIMEOUT="720" +IPFS_PROVIDER="web3storage" # 'web3storage' or 'local' +IPFS_PUBLIC_GATEWAY='https://ipfs.io/ipfs/${cid}' +IPFS_STORAGE_API_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJkaWQ6ZXRocjoweDUyMTQ1ZTY2OWUwMDdCNDQ2ZDZEZGUxYTlhMGJFZjAyZTM5RjU0ZTEiLCJpc3MiOiJ3ZWIzLXN0b3JhZ2UiLCJpYXQiOjE2NzI2NjEyNzEyMzksIm5hbWUiOiJ3ZWIzU3RvcmFnZUFwaVRrbiJ9.fu12X3XVFRFUSf9eszGid6AHHdGPriqPk4lHfCMEuX4" +IPFS_NODE_ADDRESS="http://ipfs-node:5002" + +#BATCH_NFT_MINT_SIZE=10 +# FE/DEMO +# -------------- +INITIAL_BALANCE="100" +INITIAL_STANDARD_REGISTRY_BALANCE="100" + +# COMMONS +# ---------- +MESSAGE_LANG="en-US" +LOG_LEVEL="1" +TRANSACTION_LOG_LEVEL="1" +MULTI_POLICY_SCHEDULER="0 0 * * *" +#DOCUMENT_CACHE_FIELD_LIMIT=100 + +# FEATURES +# -------------- +BBS_SIGNATURES_MODE="WASM" +# IMPORT_KEYS_FROM_DB=1 +# MQ_MESSAGE_CHUNK=5000000 +# RAW_REQUEST_LIMIT="1gb" +# JSON_REQUEST_LIMIT="1mb" + diff --git a/configs/.env.develop.guardian.system b/configs/.env.develop.guardian.system new file mode 100644 index 0000000000..6564d37961 --- /dev/null +++ b/configs/.env.develop.guardian.system @@ -0,0 +1,94 @@ +# ECOSYSTEM ENVIRONMENT VARIABLES AND FEATURES + +ENV="develop" + +# OVERRIDE - default "false" +# --------------------------- +# OVERRIDE="false" + +# HEDERA_NET - MANDATORY +# ------------------------ +HEDERA_NET="testnet" + +# PRE USED HEDERA_NET +# --------------------- +PREUSED_HEDERA_NET="testnet" + +# TESTNET +OPERATOR_ID="0.0.3422318" +OPERATOR_KEY="302e020100300506032b6570042204207a032dab9919a648142572069e8a4ccf34c805215e7099b30abd63fd1c58bd3c" +INITIALIZATION_TOPIC_ID="0.0.2030" + +# # LOCALNODE +# LOCALNODE_ADDRESS="..." +# LOCALNODE_PROTOCOL="http" +# OPERATOR_ID="0.0.2" +# OPERATOR_KEY="302e020100300506032b65700422042091132178e72057a1d7528025956fe39b0b847f200ab59b2fdd367017f3087137" + +# # PREVIEWNET +# OPERATOR_ID="..." +# OPERATOR_KEY="..." +# INITIALIZATION_TOPIC_ID="0.0.155110" + +SEND_KEYS_TO_VAULT="TRUE" +CONTRACT_FILE_ID="0.0.6276" +MAX_HEDERA_TIMEOUT="600" +# HEDERA_CUSTOM_NODES={"0.testnet.hedera.com:50211":"0.0.3"} +# HEDERA_CUSTOM_MIRROR_NODES=["testnet.mirrornode.hedera.com:443"] + +# MAX_TRANSACTION_FEE="10" + +# ADDRESSING / SERVICES +# ---------------------- +DIRECT_MESSAGE_PORT="6555" + +MQ_ADDRESS="message-broker" +MQ_MAX_PAYLOAD="1048576" + +MRV_ADDRESS="http://message-broker:3003/mrv" + +DB_HOST="mongo" + +ACCESS_TOKEN_SECRET="youraccesstokensecret" + +VAULT_PROVIDER="database" +HASHICORP_TOKEN="1234" +HASHICORP_ADDRESS="http://vault:8200" +HASHICORP_NAMESPACE="admin" +HASHICORP_ENCRIPTION_ALG="sha512" +#HASHICORP_UNSEAL_KEY="" + +#MAP_API_KEY="..." +MIN_PRIORITY="0" +MAX_PRIORITY="20" +TASK_TIMEOUT="300" +REFRESH_INTERVAL="60" + +IPFS_TIMEOUT="720" +IPFS_PROVIDER="web3storage" # 'web3storage' or 'local' +IPFS_PUBLIC_GATEWAY='https://ipfs.io/ipfs/${cid}' +IPFS_STORAGE_API_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJkaWQ6ZXRocjoweDUyMTQ1ZTY2OWUwMDdCNDQ2ZDZEZGUxYTlhMGJFZjAyZTM5RjU0ZTEiLCJpc3MiOiJ3ZWIzLXN0b3JhZ2UiLCJpYXQiOjE2NzI2NjEyNzEyMzksIm5hbWUiOiJ3ZWIzU3RvcmFnZUFwaVRrbiJ9.fu12X3XVFRFUSf9eszGid6AHHdGPriqPk4lHfCMEuX4" +IPFS_NODE_ADDRESS="http://ipfs-node:5002" + +#BATCH_NFT_MINT_SIZE=10 +# FE/DEMO +# -------------- +INITIAL_BALANCE="100" +INITIAL_STANDARD_REGISTRY_BALANCE="100" + +# COMMONS +# ---------- +MESSAGE_LANG="en-US" +LOG_LEVEL="1" +TRANSACTION_LOG_LEVEL="1" +MULTI_POLICY_SCHEDULER="0 0 * * *" +#DOCUMENT_CACHE_FIELD_LIMIT=100 + +# FEATURES +# -------------- +BBS_SIGNATURES_MODE="WASM" +# IMPORT_KEYS_FROM_DB=1 +# MQ_MESSAGE_CHUNK=5000000 +# RAW_REQUEST_LIMIT="1gb" +# JSON_REQUEST_LIMIT="1mb" + diff --git a/configs/.env.template.guardian.system b/configs/.env.template.guardian.system new file mode 100644 index 0000000000..0f21286e6f --- /dev/null +++ b/configs/.env.template.guardian.system @@ -0,0 +1,105 @@ +# ECOSYSTEM ENVIRONMENT VARIABLES AND FEATURES + +# OVERRIDE - default "false" +# --------------------------- +# this paramenter allows to override the environment variables at service level +# with the service-specific environment also if already defined at root level. +# Setting the parameter to "true" is not recommended. +# if not specified default Value "false" +# OVERRIDE="false" + +# HEDERA NETWORK +# HEDERA_NET - MANDATORY +# ------------------------ +HEDERA_NET="testnet" + +# PRE USED HEDERA_NET +# --------------------- +# Configure this value just once and keep the same setting for all subsequent configurations. +# If it is first Guardian installation or don't care about your previous data (demo) use default +# or remove from the configuration. +# Otherwise read carfully: +# Definition: as PRE USED hedera network is intended the Hedera network that you already started to use actively +# and where your important data is stored currently when you installed this release. +# for example if you are used to work on Hedera mainnet configure it as +# PREUSED_HEDERA_NET="mainnet" +# Configure this value just once and keep the same setting for all subsequent configurations. +# if not specified default Value is NULL +# PREUSED_HEDERA_NET="" + +# HEDERA - NETWORKS + +# TESTNET +OPERATOR_ID="..." +OPERATOR_KEY="..." +INITIALIZATION_TOPIC_ID="0.0.2030" + +# LOCALNODE +# LOCALNODE_ADDRESS="..." +# LOCALNODE_PROTOCOL="http" +# OPERATOR_ID="0.0.2" +# OPERATOR_KEY="302e020100300506032b65700422042091132178e72057a1d7528025956fe39b0b847f200ab59b2fdd367017f3087137" + +# PREVIEWNET +# OPERATOR_ID="..." +# OPERATOR_KEY="302e020100300506032b657004220420f6cec1af811fe09394d635352e583ac3ce84fc180bf328b9cc21e8f58f67e323" +# INITIALIZATION_TOPIC_ID="0.0.155110" + +# HEDERA_CUSTOM_NODES={"0.testnet.hedera.com:50211":"0.0.3"} +# HEDERA_CUSTOM_MIRROR_NODES=["testnet.mirrornode.hedera.com:443"] + +CONTRACT_FILE_ID="0.0.6276" + +# MAX_TRANSACTION_FEE="10" + +# ADDRESSING / SERVICES +# ---------------------- +DIRECT_MESSAGE_PORT="6555" + +MQ_ADDRESS="message-broker" +MQ_MAX_PAYLOAD="1048576" + +MRV_ADDRESS="http://message-broker:3003/mrv" + +DB_HOST="mongo" + +ACCESS_TOKEN_SECRET="youraccesstokensecret" + +SEND_KEYS_TO_VAULT="TRUE" +VAULT_PROVIDER="database" +HASHICORP_TOKEN="1234" +HASHICORP_ADDRESS="http://vault:8200" +HASHICORP_NAMESPACE="admin" +HASHICORP_ENCRIPTION_ALG="sha512" +#HASHICORP_UNSEAL_KEY="" + +MIN_PRIORITY="0" +MAX_PRIORITY="20" +TASK_TIMEOUT="300" +REFRESH_INTERVAL="60" +IPFS_TIMEOUT="720" +IPFS_PROVIDER="web3storage" # 'web3storage' or 'local' +IPFS_PUBLIC_GATEWAY='https://ipfs.io/ipfs/${cid}' +IPFS_STORAGE_API_KEY="..." +IPFS_NODE_ADDRESS="http://ipfs-node:5002" + +# FE/DEMO +# -------------- +INITIAL_BALANCE="100" +INITIAL_STANDARD_REGISTRY_BALANCE="100" + +# COMMONS +# ---------- +MESSAGE_LANG="en-US" +TRANSACTION_LOG_LEVEL="1" +MULTI_POLICY_SCHEDULER="0 0 * * *" +#LOG_LEVEL="1" + +# FEATURES +# -------------- +BBS_SIGNATURES_MODE="WASM" +# IMPORT_KEYS_FROM_DB=1 +# MQ_MESSAGE_CHUNK=5000000 +# RAW_REQUEST_LIMIT="1gb" +# JSON_REQUEST_LIMIT="1mb" + diff --git a/configs/.env_SSV b/configs/.env_SSV new file mode 100644 index 0000000000..d3ba2919cf --- /dev/null +++ b/configs/.env_SSV @@ -0,0 +1,44 @@ +## GUARDIAN ECOSYSTEM ENVIRONMENT NAME +# if you already started to use actively Guardian leave GUARDIAN_ENV empty +GUARDIAN_ENV="develop" + +## VARIABLES DEDICATED TO SPECIFIC SERVICES + +#Api-service +API_SERVICE_CHANNEL="api-gateway" + +# Auth-service +AUTH_SERVICE_CHANNEL="auth-service" +AUTH_ACCESS_TOKEN_SECRET="youraccesstokensecret" +AUTH_DB_DATABASE="auth_db" + +# guardian-service +GUARDIAN_SERVICE_CHANNEL="guardian.1" +GUARDIAN_DB_DATABASE="guardian_db" +GUARDIAN_LOG_LEVEL="1" +GUARDIAN_MULTI_POLICY_SCHEDULER="0 0 * * *" + +# logger-service +LOGGER_SERVICE_CHANNEL="logger-service" +LOGGER_DB_DATABASE="logger_db" + +# policy-service +POLICY_SERVICE_CHANNEL="policy-service" +POLICY_DB_DATABASE="guardian_db" +POLICY_LOG_LEVEL="1" +POLICY_MULTI_POLICY_SCHEDULER="0 0 * * *" + +# worker-service +WORKER_1_SERVICE_CHANNEL="worker.1" +WORKER_2_SERVICE_CHANNEL="worker.2" + + + + + + + + + + + diff --git a/docker-compose-DEV-ONLYSERVERS.yml b/docker-compose-DEV-ONLYSERVERS.yml new file mode 100644 index 0000000000..d85abc9d86 --- /dev/null +++ b/docker-compose-DEV-ONLYSERVERS.yml @@ -0,0 +1,63 @@ +version: "3.8" +services: + mongo: + image: mongo:6.0.3 + command: "--setParameter allowDiskUseByDefault=true" + restart: always + expose: + - 27017 + ports: + - 27017:27017 + + mongo-express: + image: mongo-express:1.0.0-alpha.4 + expose: + - 8081 + environment: + ME_CONFIG_MONGODB_SERVER: mongo + ME_CONFIG_MONGODB_PORT: 27017 + ME_CONFIG_SITE_BASEURL: /mongo-admin + depends_on: + - mongo + + # ipfs-node: + # image: ipfs/kubo:v0.17.0 + # ports: + # - "5001:5001" + # - "5002:5002" + # - "4001:4001" + # - "4002:4002" + # - "8080:8080" + # - "8081:8081" + + # api-docs: + # build: + # context: . + # dockerfile: ./api-docs/Dockerfile + # expose: + # - 3001 + + message-broker: + image: nats:2.9.8 + expose: + - 4222 + ports: + - '8222:8222' + - '4222:4222' + - '6222:6222' + command: '--http_port 8222' + + vault: + image: vault:1.12.2 + expose: + - 8200 + ports: + - '8200:8200' + environment: + VAULT_SERVER: "http://0.0.0.0:8200" + VAULT_DEV_ROOT_TOKEN_ID: "1234" + cap_add: + - IPC_LOCK + volumes: + - ./file:/vault/file:rw + - ./config:/vault/config:rw diff --git a/docker-compose.yml b/docker-compose.yml index 01fd8e2336..bbbc190c22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,8 @@ services: restart: always expose: - 27017 + ports: + - "27017:27017" mongo-express: image: mongo-express:1.0.0-alpha.4 @@ -63,18 +65,18 @@ services: logger-service: env_file: - - ./logger-service/.env.docker + - ./configs/.env.${GUARDIAN_ENV}.guardian.system build: context: . dockerfile: ./logger-service/Dockerfile depends_on: - message-broker environment: - - ENV=${GUARDIAN_ENV} + - GUARDIAN_ENV=${GUARDIAN_ENV} worker-service-1: env_file: - - ./worker-service/.env.docker + - ./configs/.env.${GUARDIAN_ENV}.guardian.system build: context: . dockerfile: ./worker-service/Dockerfile @@ -82,15 +84,14 @@ services: - ipfs-node - auth-service environment: - - SERVICE_CHANNEL=worker.1 - - ENV=${GUARDIAN_ENV} - - DIRECT_MESSAGE_PORT=6555 + - GUARDIAN_ENV=${GUARDIAN_ENV} + - SERVICE_CHANNEL:"worker.1" expose: - 6555 worker-service-2: env_file: - - ./worker-service/.env.docker + - ./configs/.env.${GUARDIAN_ENV}.guardian.system build: context: . dockerfile: ./worker-service/Dockerfile @@ -98,15 +99,14 @@ services: - ipfs-node - auth-service environment: - - SERVICE_CHANNEL=worker.2 - - ENV=${GUARDIAN_ENV} - - DIRECT_MESSAGE_PORT=6555 + - GUARDIAN_ENV=${GUARDIAN_ENV} + - SERVICE_CHANNEL="worker.2" expose: - 6555 auth-service: env_file: - - ./auth-service/.env.docker + - ./configs/.env.${GUARDIAN_ENV}.guardian.system build: context: . dockerfile: ./auth-service/Dockerfile @@ -116,14 +116,13 @@ services: - message-broker - logger-service environment: - - ENV=${GUARDIAN_ENV} - - DIRECT_MESSAGE_PORT=6555 + - GUARDIAN_ENV=${GUARDIAN_ENV} expose: - 6555 api-gateway: env_file: - - ./api-gateway/.env.docker + - ./configs/.env.${GUARDIAN_ENV}.guardian.system build: context: . dockerfile: ./api-gateway/Dockerfile @@ -137,12 +136,11 @@ services: - auth-service - logger-service environment: - - ENV=${GUARDIAN_ENV} - - DIRECT_MESSAGE_PORT=6555 + - GUARDIAN_ENV=${GUARDIAN_ENV} policy-service: env_file: - - ./policy-service/.env.docker + - ./configs/.env.${GUARDIAN_ENV}.guardian.system build: context: . dockerfile: ./policy-service/Dockerfile @@ -152,14 +150,14 @@ services: - auth-service - logger-service environment: - - ENV=${GUARDIAN_ENV} + - GUARDIAN_ENV=${GUARDIAN_ENV} expose: - 50000-60000 guardian-service: env_file: - - ./guardian-service/.env.docker + - ./configs/.env.${GUARDIAN_ENV}.guardian.system build: context: . dockerfile: ./guardian-service/Dockerfile @@ -172,25 +170,24 @@ services: - worker-service-2 - policy-service environment: - - ENV=${GUARDIAN_ENV} - - DIRECT_MESSAGE_PORT=6555 + - GUARDIAN_ENV=${GUARDIAN_ENV} expose: - 6555 - application-events: - build: - context: . - dockerfile: application-events/Dockerfile - expose: - - 3012 - ports: - - "3012:3012" - depends_on: - - mongo - - message-broker - - guardian-service - - auth-service - - logger-service + # application-events: + # build: + # context: . + # dockerfile: application-events/Dockerfile + # expose: + # - 3012 + # ports: + # - "3012:3012" + # depends_on: + # - mongo + # - message-broker + # - guardian-service + # - auth-service + # - logger-service mrv-sender: build: diff --git a/docker-compose_SSV.yml b/docker-compose_SSV.yml new file mode 100644 index 0000000000..06f87b07a8 --- /dev/null +++ b/docker-compose_SSV.yml @@ -0,0 +1,215 @@ +version: "3.8" +services: + mongo: + image: mongo:6.0.3 + command: "--setParameter allowDiskUseByDefault=true" + restart: always + expose: + - 27017 + + mongo-express: + image: mongo-express:1.0.0-alpha.4 + expose: + - 8081 + environment: + ME_CONFIG_MONGODB_SERVER: mongo + ME_CONFIG_MONGODB_PORT: 27017 + ME_CONFIG_SITE_BASEURL: /mongo-admin + depends_on: + - mongo + + ipfs-node: + image: ipfs/kubo:v0.18.1 + ports: + - "5001:5001" + - "5002:5002" + - "4001:4001" + - "4002:4002" + - "8080:8080" + - "8081:8081" + volumes: + - ./runtime-data/ipfs/staging:/export:rw + - ./runtime-data/ipfs/data:/data/ipfs:rw + + api-docs: + build: + context: . + dockerfile: ./api-docs/Dockerfile + expose: + - 3001 + + message-broker: + image: nats:2.9.8 + expose: + - 4222 + ports: + - '8222:8222' + command: '--http_port 8222' + + vault: + image: vault:1.12.2 + expose: + - 8200 + ports: + - '8200:8200' + environment: + VAULT_SERVER: "http://0.0.0.0:8200" + VAULT_DEV_ROOT_TOKEN_ID: "1234" + cap_add: + - IPC_LOCK + volumes: + - ./file:/vault/file:rw + - ./config:/vault/config:rw + + +# https://docs.docker.com/compose/environment-variables/envvars-precedence/ +# Environment leverage the gerarchy defined in the docker compose between "env_file" and "environment" attributes +# ecosystem variables defined in the "env_file" .env.${GUARDIAN_ENV}.guardian.system +# specific service variables defined by "environment" can override what is defined in the ecosystem file + + logger-service: + env_file: + - ./.env.${GUARDIAN_ENV}.guardian.system + build: + context: . + dockerfile: ./logger-service/Dockerfile + depends_on: + - message-broker + environment: + - GUARDIAN_ENV=${GUARDIAN_ENV} + - SERVICE_CHANNEL=${LOGGER_SERVICE_CHANNEL} + - DB_DATABASE=${LOGGER_DB_DATABASE} + + auth-service: + env_file: + - ./.env.${GUARDIAN_ENV}.guardian.system + build: + context: . + dockerfile: ./auth-service/Dockerfile + depends_on: + - mongo + - vault + - message-broker + - logger-service + environment: + - GUARDIAN_ENV=${GUARDIAN_ENV} + - SERVICE_CHANNEL=${AUTH_SERVICE_CHANNEL} + - DB_DATABASE=${AUTH_DB_DATABASE} + + worker-service-1: + env_file: + - ./.env.${GUARDIAN_ENV}.guardian.system + build: + context: . + dockerfile: ./worker-service/Dockerfile + depends_on: + - ipfs-node + - auth-service + environment: + - GUARDIAN_ENV=${GUARDIAN_ENV} + - SERVICE_CHANNEL=${WORKER_1_SERVICE_CHANNEL} + + worker-service-2: + env_file: + - ./.env.${GUARDIAN_ENV}.guardian.system + build: + context: . + dockerfile: ./worker-service/Dockerfile + depends_on: + - ipfs-node + - auth-service + environment: + - GUARDIAN_ENV=${GUARDIAN_ENV} + - SERVICE_CHANNEL=${WORKER_2_SERVICE_CHANNEL} + + api-gateway: + env_file: + - ./.env.${GUARDIAN_ENV}.guardian.system + build: + context: . + dockerfile: ./api-gateway/Dockerfile + expose: + - 3002 + depends_on: + - mongo + - message-broker + - guardian-service + - auth-service + - logger-service + environment: + - GUARDIAN_ENV=${GUARDIAN_ENV} + - SERVICE_CHANNEL=${API_SERVICE_CHANNEL} + + + policy-service: + env_file: + - ./.env.${GUARDIAN_ENV}.guardian.system + build: + context: . + dockerfile: ./policy-service/Dockerfile + depends_on: + - mongo + - message-broker + - auth-service + - logger-service + environment: + - GUARDIAN_ENV=${GUARDIAN_ENV} + - SERVICE_CHANNEL=${POLICY_SERVICE_CHANNEL} + - DB_DATABASE=${POLICY_DB_DATABASE} + - LOG_LEVEL=${POLICY_LOG_LEVEL} + - MULTI_POLICY_SCHEDULER=${POLICY_MULTI_POLICY_SCHEDULER} + + guardian-service: + env_file: + - ./.env.${GUARDIAN_ENV}.guardian.system + build: + context: . + dockerfile: ./guardian-service/Dockerfile + depends_on: + - mongo + - message-broker + - auth-service + - logger-service + - worker-service-1 + - worker-service-2 + - policy-service + environment: + - GUARDIAN_ENV=${GUARDIAN_ENV} + - SERVICE_CHANNEL=${GUARDIAN_SERVICE_CHANNEL} + - DB_DATABASE=${GUARDIAN_DB_DATABASE} + - LOG_LEVEL=${GUARDIAN_LOG_LEVEL} + - MULTI_POLICY_SCHEDULER=${GUARDIAN_MULTI_POLICY_SCHEDULER} + + mrv-sender: + build: + context: . + dockerfile: ./mrv-sender/Dockerfile + expose: + - 3005 + + topic-viewer: + build: + context: . + dockerfile: ./topic-viewer/Dockerfile + expose: + - 3006 + + web-proxy: + build: + context: . + dockerfile: ./web-proxy/Dockerfile + ports: + - "3000:80" + depends_on: + - guardian-service + - auth-service + - api-gateway + - api-docs + - mrv-sender + - mongo-express +volumes: + mongo: + # volume-guardian-service: + # volume-ui-service: + # volume-mrv-sender: + # volume-message-broker: diff --git a/docs/environments/Ecosystem-Environment.md b/docs/environments/Ecosystem-Environment.md new file mode 100644 index 0000000000..a335ac9476 --- /dev/null +++ b/docs/environments/Ecosystem-Environment.md @@ -0,0 +1,257 @@ +### Ecosystem Environments +###### \#1923, \#1639 + + +The set of environmet parameters represent the context in which a service it is executed. Each service needs to know this context to adapt its behaviour to the real working condition. At the service level the node dotenv library allows to read environment of the kind \ this library by default reads from .env file. The data are reads in a the process.environment data structure available in the execution context of Node process. +A unique file define the environment and keeps the responsibility to create the shared operative ecosystem. + + + All Guardian Microservices share a common set of Environment variables. In this way Guardian can be seen as an ecosystem with several services and common set of parameters leading his behaviour. This environment parameters are shared between all the services of the Guardian ecosystem. +All variables are defined in a ".env.\.guardian.system" file. The file name is parametric so it is possible to define a different files for different possible running configuration, for example production, develop, test1. The ecosystem environment file follow the .env.template.guardian.system file that let write new configurations with the set of necessary variables. Both the template file and the resulting environmets files are in the folder "./configs/", they can be discriminated by its name to spread the session. + +The parameter GUARDIAN_ENV is defined univocally in a .env file. The containers orchestration will be responsible to push the environment in to the container in a way the environment will be available to the Node server. For example in the execution of Guardian using docker compose tool the tool inject the environment in each container. Docker compose push the environment in the container by the means of the env-file attribute and the environment attribute. Overmore the environment attribute can be parametrized by variables defined in a ".env" file located next to the docker_compose.yaml. + +![hierarchy.png](https://images.zenhubusercontent.com/63dbe2bd4d4d6290bed6780c/12790cd6-19b5-4f3c-aad2-9d28081e8498) + +Also Guardian services are allowed to define specific service variables. This different set of variables allow to have a hierarchical definition of the same variable in a way that a developer could redefine some of them in a service specific way or add new variables extending the usage of the ecosystem environment. +The environment variables that are specific to services can be specified by the means of .env.\.\ files in each service. + + +Per each installed service the environment is configured using the two file: +1) ".env" file +2) "./guardian/\/configs/.env.\.develop" file + +The environment is loaded in the service by the file config.ts. the Environment is read in to two steps: at first steps the service .env file is loaded by Node while at second step ".env.\.\" file is loaded. +A new environmet variable OVERRIDE as "true"/"false" it has been added to let variables defined in the ".env.\.\.\" to override the common defined variables or add new ones. For example If OVERRIDE=true a variable with the same name as the one already defined in the ".env"" file will assume the value specify at service level. The OVERRIDE parameter is not mandatory. if OVERRIDE="false" (default value) specific service variables can only be added to the global ones. In each service a new "./configs" folder holds the set of paramentric service level environment files. + +With this implementation the service orchestrator can push not just the econsystem environment but the service specific variables too or leave the service specific variables under the responsibility of the service itself. + +For example it is possible to use docker compose to orchestrate the service in a single node. Dcoker compose has “env-file” and “environment” attributes to define environment. There is a precedence between this two attributes as define at https://docs.docker.com/compose/environment-variables/envvars-precedence/#simple-example. +In this way override=”true” always and variables re-assigned in the environment attributes override what has been defined in the .env.\.guardian.system env-file. + + +EXAMPLES: +--------- + +Configure each service without an orchestrator +----------------------------------------------- + +configure .env, in ./guardian/\/.env. Insert the variable GUARDIAN_ENV and give it the name that you choose for you Guardian platform environment (production, develop ...). If you update a production environment to keep working with your previous data leave the field GUARDIAN_ENV="" empty. + +The OVERRIDE variable is not mandatory and it default to "false". +```plaintext +GUARDIAN_ENV="develop" +# OVERRIDE="false" +``` + +Every variable that is used by the service is configured inside the .guardian/\/configs folder. Because GUARDIAN_ENV is configured as "develop" each service confiiguration are stored in files with format "./guardian/\/configs/.env.\.develop" that follows the template in the same folder. + +Configure the guardian-service + in ./guardian/guardian-service/configs/.env.guardian.develop + +```plaintext +OPERATOR_ID="..." +OPERATOR_KEY="..." +``` + +Configure the worker-service + in ./guardian/worker-service/configs/.env.worker.develop +```plaintext +IPFS_TIMEOUT="720" +IPFS_PROVIDER="web3storage" # 'web3storage' or 'local' +#Single quote IPFS_PUBLIC_GATEWAY +IPFS_PUBLIC_GATEWAY='https://ipfs.io/ipfs/${cid}' +IPFS_STORAGE_API_KEY="..." +IPFS_NODE_ADDRESS="http://ipfs-node:5002" +``` + +## 1) Docker Compose Configuration +----------------------------- +the following configuration will ignore the guardian-service configuration for variables with same name using only the ecosystem configuration while will add the new specific guardian-service variables for newly defined variables like DB_HOST="localhost", DB_DATABASE="guardian_db" etc.. + +### At root level: + +- in ./guardian/.env + +```plaintext +GUARDIAN_ENV="develop" +``` + +- in ./guardian/configs/.env.develop.guardian.system +```plaintext +# ECOSYSTEM ENVIRONMENT VARIABLES AND FEATURES + +# OVERRIDE - default "false" +# --------------------------- +# OVERRIDE="false" + +# HEDERA_NET - MANDATORY +# ------------------------ +HEDERA_NET="testnet" + +# PRE USED HEDERA_NET +# --------------------- +PREUSED_HEDERA_NET="testnet" + +# TESTNET +OPERATOR_ID="0.0.3422318" +OPERATOR_KEY="302e020100300506032b6570042..................34c805215e7099b30abd63fd1c58bd3c" +INITIALIZATION_TOPIC_ID="0.0.2030" +....... +..... + +``` + +### At guardian-service level: + +in file ./guardian/guardian-service/configs/.env.guardian.develop + +```plaintext + HEDERA_NET="localnode" + PREUSED_HEDERA_NET="localnode" + MQ_ADDRESS="localhost" + SERVICE_CHANNEL="guardian.1" + DB_HOST="localhost" + DB_DATABASE="guardian_db" + INITIAL_BALANCE="100" + INITIAL_STANDARD_REGISTRY_BALANCE="100" + + # TESTNET + OPERATOR_ID="0.0.4523185" + OPERATOR_KEY="302e02010030050603.........................05215e7099b30abd63fd1c58bd3c" + INITIALIZATION_TOPIC_ID="0.0.2030" + ``` + + + +### RUN TIME RESULT + +The following environment is loaded by the service. + +```plaintext +HEDERA_NET="testnet" +PREUSED_HEDERA_NET="testnet" +MQ_ADDRESS="message-broker" +SERVICE_CHANNEL="guardian.1" +DB_HOST="mongo" +DB_DATABASE="guardian_db" +INITIAL_BALANCE="100" +INITIAL_STANDARD_REGISTRY_BALANCE="100" + +# TESTNET +OPERATOR_ID="0.0.3422318" +OPERATOR_KEY="302e020100300506032b6570042..................34c805215e7099b30abd63fd1c58bd3c" +INITIALIZATION_TOPIC_ID="0.0.2030" +``` + + + +## 2) to mantain the same database already in use +--------------------------------------- + +### at root level: + +in ./guardian/.env + +```plaintext +GUARDIAN_ENV="" +``` + +in ./guardian/configs/.env..guardian.system + +```plaintext +# ECOSYSTEM ENVIRONMENT VARIABLES AND FEATURES + +# OVERRIDE - default "false" +# --------------------------- +# OVERRIDE="false" + +# HEDERA_NET - MANDATORY +# ------------------------ +HEDERA_NET="mainnet" + +# PRE USED HEDERA_NET +# --------------------- +PREUSED_HEDERA_NET="mainnet" + +# TESTNET +OPERATOR_ID="0.0.3422318" +OPERATOR_KEY="302e020100300506032b6570042..................34c805215e7099b30abd63fd1c58bd3c" +INITIALIZATION_TOPIC_ID="0.0.2030" +......... +....... + +``` + + + +### at guardian-service level: + + in ./guardian/guardian-service/configs/.env.guardian + +```plaintext +HEDERA_NET="localnode" +PREUSED_HEDERA_NET="localnode" +MQ_ADDRESS="localhost" +SERVICE_CHANNEL="guardian.1" +DB_HOST="localhost" +DB_DATABASE="guardian_db" +INITIAL_BALANCE="100" +INITIAL_STANDARD_REGISTRY_BALANCE="100" + +# TESTNET +OPERATOR_ID="0.0.4523185" +OPERATOR_KEY="302e02010030050603.........................05215e7099b30abd63fd1c58bd3c" +INITIALIZATION_TOPIC_ID="0.0.2030" +``` + + +### RUN TIME RESULT: + +at guardian service level the following environment is loaded at runtime + +```plaintext +HEDERA_NET="mainnet" +PREUSED_HEDERA_NET="mainnet" +MQ_ADDRESS="message-broker" +SERVICE_CHANNEL="guardian.1" +DB_HOST="mongo" +DB_DATABASE="guardian_db" +INITIAL_BALANCE="100" +INITIAL_STANDARD_REGISTRY_BALANCE="100" + +# TESTNET +OPERATOR_ID="0.0.3422318" +OPERATOR_KEY="302e020100300506032b6570042..................34c805215e7099b30abd63fd1c58bd3c" +INITIALIZATION_TOPIC_ID="0.0.2030" +``` + + +### 3) To use the docker-compose configuration and overriding specific service. +-------------------------------- + +The docker compose file to be used for this alternative is docker-compose_SSV.yaml together with the .env_SSV in the ./configs folder. +Using this configuration both the ecosystem common variables and the service specific variables will be configured in the root folder. using the .env file for specific services variables and the ./config/.env.\.guardian.system for ecosystem variables. + +Follow the steps: + +```plaintext + - backup .env file + $ mv ./.env ./.env_bck + + - copy the .env_SSV file from ./configs. + the .env_SSV file allow to keep not just the GUARDIAN_ENV:guardian environment name but the Specific Service Variables too in a way to let docker compose have visibility at container bootstrap. + $ cp ./configs/.env_SSV ./.env + + - backup docker-compose.yml + $ mv ./docker-compose.yml .docker-compose.bck + + - copy the .docker-compose_SSV.yml in docker.compose.yml + the docker-compose_SSV.yml contains the overriding between the env-files withe the service specific variables in the "environment" attribute per every service + $ cp .docker-compose_SSV.yml in docker.compose.yml + + - run the docker compose without rebuild + $ docker compose up -d +``` + +In this configuration using the default OVERRIDING=false in the .env.\.guardian.system grant that the environments that are in each ./\/configs remain unloaded. \ No newline at end of file diff --git a/docs/getting-started/getting-started/installation.md b/docs/getting-started/getting-started/installation.md index ede95fe6b7..55ece15da4 100644 --- a/docs/getting-started/getting-started/installation.md +++ b/docs/getting-started/getting-started/installation.md @@ -43,7 +43,7 @@ For example: [https://github.com/yeasy/docker-ipfs](https://github.com/yeasy/doc ``` IPFS_NODE_ADDRESS="..." # Default IPFS_NODE_ADDRESS="http://localhost:5002" -IPFS_PUBLIC_GATEWAY="..." # Default IPFS_PUBLIC_GATEWAY="https://localhost:8080/ipfs/${cid}" +IPFS_PUBLIC_GATEWAY='...' # Default IPFS_PUBLIC_GATEWAY='https://localhost:8080/ipfs/${cid}' IPFS_PROVIDER="local" ``` diff --git a/guardian-service/.env b/guardian-service/.env index 292347d816..18face0ea2 100644 --- a/guardian-service/.env +++ b/guardian-service/.env @@ -1,25 +1 @@ -ENV="develop" -HEDERA_NET="testnet" -PREUSED_HEDERA_NET="testnet" -MQ_ADDRESS="localhost" -SERVICE_CHANNEL="guardian.1" -DB_HOST="localhost" -DB_DATABASE="guardian_db" -INITIAL_BALANCE="100" -INITIAL_STANDARD_REGISTRY_BALANCE="100" -OPERATOR_ID="..." -OPERATOR_KEY="..." -INITIALIZATION_TOPIC_ID="0.0.2030" -MESSAGE_LANG="en-US" -TRANSACTION_LOG_LEVEL="1" -SEND_KEYS_TO_VAULT="TRUE" -CONTRACT_FILE_ID="0.0.6276" -BBS_SIGNATURES_MODE="WASM" -MQ_MAX_PAYLOAD="1048576" -#LOG_LEVEL="2" -#HEDERA_CUSTOM_NODES={"0.testnet.hedera.com:50211":"0.0.3"} -#HEDERA_CUSTOM_MIRROR_NODES=["testnet.mirrornode.hedera.com:443"] -#MQ_MESSAGE_CHUNK=5000000 -#MAP_API_KEY="..." -#DOCUMENT_CACHE_FIELD_LIMIT=100 -#BATCH_NFT_MINT_SIZE=10 +GUARDIAN_ENV="" diff --git a/guardian-service/.env.docker b/guardian-service/.env.docker index e4290dd61b..c176cb6293 100644 --- a/guardian-service/.env.docker +++ b/guardian-service/.env.docker @@ -6,8 +6,8 @@ DB_HOST="mongo" DB_DATABASE="guardian_db" INITIAL_BALANCE="100" INITIAL_STANDARD_REGISTRY_BALANCE="100" -OPERATOR_ID="..." -OPERATOR_KEY="..." +OPERATOR_ID="0.0.3422318" +OPERATOR_KEY="302e020100300506032b6570042204207a032dab9919a648142572069e8a4ccf34c805215e7099b30abd63fd1c58bd3c" INITIALIZATION_TOPIC_ID="0.0.2030" MESSAGE_LANG="en-US" TRANSACTION_LOG_LEVEL="1" diff --git a/guardian-service/Dockerfile b/guardian-service/Dockerfile index 463c041e4f..5f52111e2e 100644 --- a/guardian-service/Dockerfile +++ b/guardian-service/Dockerfile @@ -36,7 +36,8 @@ COPY --from=interfacesModuleBuilder /usr/local/interfaces/guardian-interfaces-*. COPY --from=commonModuleBuilder /usr/local/common/guardian-common-*.tgz /tmp/common.tgz COPY --from=guardianServiceBuilder /usr/local/guardian-service/yarn.lock ./ COPY ./guardian-service/package.json ./ -# COPY ./guardian-service/.env.docker ./.env +RUN mkdir -p ./configs +COPY ./guardian-service/configs/* ./configs COPY ./guardian-service/system-schemas ./system-schemas/. RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/interfaces']='file:/tmp/interfaces.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));" RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/common']='file:/tmp/common.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));" diff --git a/guardian-service/configs/.env.guardian b/guardian-service/configs/.env.guardian new file mode 100644 index 0000000000..d3020cdfa3 --- /dev/null +++ b/guardian-service/configs/.env.guardian @@ -0,0 +1,31 @@ +# Guardian Service Specialized Variables +SERVICE_CHANNEL="guardian.1" +DB_DATABASE="guardian_db" + +# Ecosystem Defined Variables +HEDERA_NET="testnet" +PREUSED_HEDERA_NET="testnet" +MQ_ADDRESS="localhost" +DB_HOST="localhost" +INITIAL_BALANCE="100" +INITIAL_STANDARD_REGISTRY_BALANCE="100" + +# TESTNET +OPERATOR_ID="0.0.3422318" +OPERATOR_KEY="302e020100300506032b6570042204207a032dab9919a648142572069e8a4ccf34c805215e7099b30abd63fd1c58bd3c" +INITIALIZATION_TOPIC_ID="0.0.2030" + +MESSAGE_LANG="en-US" +TRANSACTION_LOG_LEVEL="1" +SEND_KEYS_TO_VAULT="TRUE" +MULTI_POLICY_SCHEDULER="0 0 * * *" +CONTRACT_FILE_ID="0.0.6276" +BBS_SIGNATURES_MODE="WASM" +MQ_MAX_PAYLOAD="1048576" +#LOG_LEVEL="1" +#HEDERA_CUSTOM_NODES={"0.testnet.hedera.com:50211":"0.0.3"} +#HEDERA_CUSTOM_MIRROR_NODES=["testnet.mirrornode.hedera.com:443"] +#MQ_MESSAGE_CHUNK=5000000 +#MAP_API_KEY="..." +#DOCUMENT_CACHE_FIELD_LIMIT=100 +#BATCH_NFT_MINT_SIZE=10 diff --git a/guardian-service/configs/.env.guardian.develop b/guardian-service/configs/.env.guardian.develop new file mode 100644 index 0000000000..a48d03d5ef --- /dev/null +++ b/guardian-service/configs/.env.guardian.develop @@ -0,0 +1,31 @@ +# Guardian Service Specialized Variables +SERVICE_CHANNEL="guardian.1" +DB_DATABASE="guardian_db" + +# Ecosystem Defined Variables +HEDERA_NET="testnet" +PREUSED_HEDERA_NET="testnet" +MQ_ADDRESS="localhost" +DB_HOST="localhost" +INITIAL_BALANCE="100" +INITIAL_STANDARD_REGISTRY_BALANCE="100" + +# TESTNET +OPERATOR_ID="..." +OPERATOR_KEY="..." +INITIALIZATION_TOPIC_ID="0.0.2030" + +MESSAGE_LANG="en-US" +TRANSACTION_LOG_LEVEL="1" +SEND_KEYS_TO_VAULT="TRUE" +MULTI_POLICY_SCHEDULER="0 0 * * *" +CONTRACT_FILE_ID="0.0.6276" +BBS_SIGNATURES_MODE="WASM" +MQ_MAX_PAYLOAD="1048576" +#LOG_LEVEL="1" +#HEDERA_CUSTOM_NODES={"0.testnet.hedera.com:50211":"0.0.3"} +#HEDERA_CUSTOM_MIRROR_NODES=["testnet.mirrornode.hedera.com:443"] +#MQ_MESSAGE_CHUNK=5000000 +#MAP_API_KEY="..." +#DOCUMENT_CACHE_FIELD_LIMIT=100 +#BATCH_NFT_MINT_SIZE=10 diff --git a/guardian-service/configs/.env.guardian.template b/guardian-service/configs/.env.guardian.template new file mode 100644 index 0000000000..81607bbb0d --- /dev/null +++ b/guardian-service/configs/.env.guardian.template @@ -0,0 +1,37 @@ +# Guardian Service Specialized Variables +SERVICE_CHANNEL="guardian.1" +DB_DATABASE="guardian_db" + +# Ecosystem Defined Variables +HEDERA_NET="" +PREUSED_HEDERA_NET="" +MQ_ADDRESS="" +DB_HOST="" +INITIAL_BALANCE="100" +INITIAL_STANDARD_REGISTRY_BALANCE="100" + +OPERATOR_ID="..." +OPERATOR_KEY="..." +INITIALIZATION_TOPIC_ID="0.0.2030" # TESTNET +# INITIALIZATION_TOPIC_ID="0.0.155110" # PREVIEWNET + +# # HEDERA LOCALNODE +# OPERATOR_ID="0.0.2" +# OPERATOR_KEY="302e020100300506032b65700422042091132178e72057a1d7528025956fe39b0b847f200ab59b2fdd367017f3087137" +LOCALNODE_ADDRESS="..." # localhost or server address +LOCALNODE_PROTOCOL="http" + +MESSAGE_LANG="en-US" +TRANSACTION_LOG_LEVEL="1" +SEND_KEYS_TO_VAULT="TRUE" +MULTI_POLICY_SCHEDULER="0 0 * * *" +CONTRACT_FILE_ID="0.0.6276" +BBS_SIGNATURES_MODE="WASM" +MQ_MAX_PAYLOAD="" +#LOG_LEVEL="2" +#HEDERA_CUSTOM_NODES={"0.testnet.hedera.com:50211":"0.0.3"} +#HEDERA_CUSTOM_MIRROR_NODES=["testnet.mirrornode.hedera.com:443"] +#MQ_MESSAGE_CHUNK=5000000 +#MAP_API_KEY="..." +#DOCUMENT_CACHE_FIELD_LIMIT=100 +#BATCH_NFT_MINT_SIZE=10 diff --git a/guardian-service/src/config.ts b/guardian-service/src/config.ts index 1bb4f2da37..b6e0f2e638 100644 --- a/guardian-service/src/config.ts +++ b/guardian-service/src/config.ts @@ -1,6 +1,7 @@ import moduleAlias from 'module-alias'; import dotenv from 'dotenv'; import 'reflect-metadata'; +import fs from 'fs'; moduleAlias.addAliases({ '@api': __dirname + '/api', @@ -16,3 +17,24 @@ moduleAlias.addAliases({ }); dotenv.config(); + +const envPath = process.env.GUARDIAN_ENV ? `./configs/.env.guardian.${process.env.GUARDIAN_ENV}` : './configs/.env.guardian'; + +if (!process.env.OVERRIDE || process.env.OVERRIDE === 'false'){ + console.log('reading from', envPath, 'not overriding'); + dotenv.config({ path: envPath}); +}else{ + try { + const envConfig = dotenv.parse(fs.readFileSync(envPath)); + for (const k of Object.keys(envConfig)) { + process.env[k] = envConfig[k] + } + } catch (err) { + if (err.code === 'ENOENT') { + console.log('WARN: Specific environment not loaded'); + } else { + throw err; + } + } +} +console.log('Charged Environment',process.env,'\r\n___ . ___'); diff --git a/logger-service/.env b/logger-service/.env index a53da6b877..18face0ea2 100644 --- a/logger-service/.env +++ b/logger-service/.env @@ -1,9 +1 @@ -ENV="develop" -HEDERA_NET="testnet" -PREUSED_HEDERA_NET="testnet" -MQ_ADDRESS="localhost" -SERVICE_CHANNEL="logger-service" -DB_HOST="localhost" -DB_DATABASE="logger_db" -MQ_MAX_PAYLOAD="1048576" -#MQ_MESSAGE_CHUNK=5000000 +GUARDIAN_ENV="" diff --git a/logger-service/Dockerfile b/logger-service/Dockerfile index 3350903df1..834f1d9808 100644 --- a/logger-service/Dockerfile +++ b/logger-service/Dockerfile @@ -36,7 +36,8 @@ COPY --from=interfacesModuleBuilder /usr/local/interfaces/guardian-interfaces-*. COPY --from=commonModuleBuilder /usr/local/common/guardian-common-*.tgz /tmp/common.tgz COPY --from=loggerServiceBuilder /usr/local/logger-service/yarn.lock ./ COPY ./logger-service/package.json ./ -# COPY ./logger-service/.env.docker ./.env +RUN mkdir -p ./configs +COPY ./logger-service/configs/* ./configs RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/interfaces']='file:/tmp/interfaces.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));" RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/common']='file:/tmp/common.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));" RUN yarn install --frozen-lockfile diff --git a/logger-service/configs/.env.logger b/logger-service/configs/.env.logger new file mode 100644 index 0000000000..09ff9f2da6 --- /dev/null +++ b/logger-service/configs/.env.logger @@ -0,0 +1,11 @@ +# Logger Service Specialized Variables +SERVICE_CHANNEL="logger-service" +DB_DATABASE="logger_db" + +# Ecosystem Defined Variables +HEDERA_NET="testnet" +PREUSED_HEDERA_NET="testnet" +MQ_ADDRESS="localhost" +DB_HOST="localhost" +MQ_MAX_PAYLOAD="1048576" +#MQ_MESSAGE_CHUNK=5000000 diff --git a/logger-service/configs/.env.logger.develop b/logger-service/configs/.env.logger.develop new file mode 100644 index 0000000000..09ff9f2da6 --- /dev/null +++ b/logger-service/configs/.env.logger.develop @@ -0,0 +1,11 @@ +# Logger Service Specialized Variables +SERVICE_CHANNEL="logger-service" +DB_DATABASE="logger_db" + +# Ecosystem Defined Variables +HEDERA_NET="testnet" +PREUSED_HEDERA_NET="testnet" +MQ_ADDRESS="localhost" +DB_HOST="localhost" +MQ_MAX_PAYLOAD="1048576" +#MQ_MESSAGE_CHUNK=5000000 diff --git a/logger-service/configs/.env.logger.template b/logger-service/configs/.env.logger.template new file mode 100644 index 0000000000..febcc8fba1 --- /dev/null +++ b/logger-service/configs/.env.logger.template @@ -0,0 +1,11 @@ +# Logger Service Specialized Variables +SERVICE_CHANNEL="logger-service" +DB_DATABASE="logger_db" + +# Ecosystem Defined Variables +HEDERA_NET="" +PREUSED_HEDERA_NET="" +MQ_ADDRESS="" +DB_HOST="" +MQ_MAX_PAYLOAD="" +#MQ_MESSAGE_CHUNK=5000000 diff --git a/logger-service/src/config.ts b/logger-service/src/config.ts index 96dd9fb7ec..cc201c9062 100644 --- a/logger-service/src/config.ts +++ b/logger-service/src/config.ts @@ -1,5 +1,6 @@ import moduleAlias from 'module-alias'; import dotenv from 'dotenv'; +import fs from 'fs'; moduleAlias.addAliases({ '@api': __dirname + '/api', @@ -7,3 +8,24 @@ moduleAlias.addAliases({ }); dotenv.config(); + +const envPath = process.env.GUARDIAN_ENV ? `./configs/.env.logger.${process.env.GUARDIAN_ENV}` : './configs/.env.logger'; + +if (!process.env.OVERRIDE || process.env.OVERRIDE === 'false'){ + console.log('reading from', envPath, 'not overriding'); + dotenv.config({ path: envPath}); +}else{ + try { + const envConfig = dotenv.parse(fs.readFileSync(envPath)); + for (const k of Object.keys(envConfig)) { + process.env[k] = envConfig[k] + } + } catch (err) { + if (err.code === 'ENOENT') { + console.log('WARN: Specific environment not loaded'); + } else { + throw err; + } + } +} +console.log('Charged Environment',process.env,'\r\n___ . ___'); diff --git a/policy-service/.env b/policy-service/.env index f72c304531..18face0ea2 100644 --- a/policy-service/.env +++ b/policy-service/.env @@ -1,17 +1 @@ -ENV="develop" -HEDERA_NET="testnet" -PREUSED_HEDERA_NET="testnet" -MQ_ADDRESS="localhost" -SERVICE_CHANNEL="policy-service" -DB_HOST="localhost" -DB_DATABASE="guardian_db" -MESSAGE_LANG="en-US" -BBS_SIGNATURES_MODE="WASM" -MQ_MAX_PAYLOAD="1048576" -MULTI_POLICY_SCHEDULER="0 0 * * *" -#LOG_LEVEL="2" -#HEDERA_CUSTOM_NODES={"0.testnet.hedera.com:50211":"0.0.3"} -#HEDERA_CUSTOM_MIRROR_NODES=["testnet.mirrornode.hedera.com:443"] -#MQ_MESSAGE_CHUNK=5000000 -#DOCUMENT_CACHE_FIELD_LIMIT=100 -#BATCH_NFT_MINT_SIZE=10 +GUARDIAN_ENV="" diff --git a/policy-service/.env.example b/policy-service/.env.example index f38c4825a7..5336671048 100644 --- a/policy-service/.env.example +++ b/policy-service/.env.example @@ -4,9 +4,9 @@ DB_HOST="localhost" DB_DATABASE="guardian_db" HEDERA_NET="testnet" MESSAGE_LANG="en-US" -LOG_LEVEL="2" MULTI_POLICY_SCHEDULER="0 0 * * *" #MQ_MESSAGE_CHUNK=5000000 +#LOG_LEVEL="2" # Vault Secret Manager Configs VAULT_API_VERSION=v1 diff --git a/policy-service/Dockerfile b/policy-service/Dockerfile index c14578ad47..20fca506fb 100644 --- a/policy-service/Dockerfile +++ b/policy-service/Dockerfile @@ -36,7 +36,8 @@ COPY --from=interfacesModuleBuilder /usr/local/interfaces/guardian-interfaces-*. COPY --from=commonModuleBuilder /usr/local/common/guardian-common-*.tgz /tmp/common.tgz COPY --from=policyServiceBuilder /usr/local/policy-service/yarn.lock ./ COPY ./policy-service/package.json ./ -# COPY ./policy-service/.env.docker ./.env +RUN mkdir -p ./configs +COPY ./policy-service/configs/* ./configs RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/interfaces']='file:/tmp/interfaces.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));" RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/common']='file:/tmp/common.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));" RUN yarn install --frozen-lockfile diff --git a/policy-service/configs/.env.policy b/policy-service/configs/.env.policy new file mode 100644 index 0000000000..fc7ec6fa93 --- /dev/null +++ b/policy-service/configs/.env.policy @@ -0,0 +1,16 @@ +HEDERA_NET="testnet" +PREUSED_HEDERA_NET="testnet" +MQ_ADDRESS="localhost" +SERVICE_CHANNEL="policy-service" +DB_HOST="localhost" +DB_DATABASE="guardian_db" +MESSAGE_LANG="en-US" +BBS_SIGNATURES_MODE="WASM" +MQ_MAX_PAYLOAD="1048576" +MULTI_POLICY_SCHEDULER="0 0 * * *" +#LOG_LEVEL="2" +#HEDERA_CUSTOM_NODES={"0.testnet.hedera.com:50211":"0.0.3"} +#HEDERA_CUSTOM_MIRROR_NODES=["testnet.mirrornode.hedera.com:443"] +#MQ_MESSAGE_CHUNK=5000000 +#DOCUMENT_CACHE_FIELD_LIMIT=100 +#BATCH_NFT_MINT_SIZE=10 diff --git a/policy-service/configs/.env.policy.develop b/policy-service/configs/.env.policy.develop new file mode 100644 index 0000000000..fc7ec6fa93 --- /dev/null +++ b/policy-service/configs/.env.policy.develop @@ -0,0 +1,16 @@ +HEDERA_NET="testnet" +PREUSED_HEDERA_NET="testnet" +MQ_ADDRESS="localhost" +SERVICE_CHANNEL="policy-service" +DB_HOST="localhost" +DB_DATABASE="guardian_db" +MESSAGE_LANG="en-US" +BBS_SIGNATURES_MODE="WASM" +MQ_MAX_PAYLOAD="1048576" +MULTI_POLICY_SCHEDULER="0 0 * * *" +#LOG_LEVEL="2" +#HEDERA_CUSTOM_NODES={"0.testnet.hedera.com:50211":"0.0.3"} +#HEDERA_CUSTOM_MIRROR_NODES=["testnet.mirrornode.hedera.com:443"] +#MQ_MESSAGE_CHUNK=5000000 +#DOCUMENT_CACHE_FIELD_LIMIT=100 +#BATCH_NFT_MINT_SIZE=10 diff --git a/policy-service/configs/.env.policy.template b/policy-service/configs/.env.policy.template new file mode 100644 index 0000000000..e59ce5e438 --- /dev/null +++ b/policy-service/configs/.env.policy.template @@ -0,0 +1,16 @@ +HEDERA_NET="" +PREUSED_HEDERA_NET="" +MQ_ADDRESS="" +SERVICE_CHANNEL="policy-service" +DB_HOST="" +DB_DATABASE="guardian_db" +MESSAGE_LANG="en-US" +BBS_SIGNATURES_MODE="WASM" +MQ_MAX_PAYLOAD="" +MULTI_POLICY_SCHEDULER="" +#LOG_LEVEL="2" +#HEDERA_CUSTOM_NODES={"0.testnet.hedera.com:50211":"0.0.3"} +#HEDERA_CUSTOM_MIRROR_NODES=["testnet.mirrornode.hedera.com:443"] +#MQ_MESSAGE_CHUNK=5000000 +#DOCUMENT_CACHE_FIELD_LIMIT=100 +#BATCH_NFT_MINT_SIZE=10 diff --git a/policy-service/src/config.ts b/policy-service/src/config.ts index 51f29588ea..364e76c7f0 100644 --- a/policy-service/src/config.ts +++ b/policy-service/src/config.ts @@ -1,6 +1,7 @@ import moduleAlias from 'module-alias'; import dotenv from 'dotenv'; import 'reflect-metadata'; +import fs from 'fs'; moduleAlias.addAliases({ '@api': __dirname + '/api', @@ -14,3 +15,24 @@ moduleAlias.addAliases({ }); dotenv.config(); + +const envPath = process.env.GUARDIAN_ENV ? `./configs/.env.policy.${process.env.GUARDIAN_ENV}` : './configs/.env.policy'; + +if (!process.env.OVERRIDE || process.env.OVERRIDE === 'false'){ + console.log('reading from', envPath, 'not overriding'); + dotenv.config({ path: envPath}); +}else{ + try { + const envConfig = dotenv.parse(fs.readFileSync(envPath)); + for (const k of Object.keys(envConfig)) { + process.env[k] = envConfig[k] + } + } catch (err) { + if (err.code === 'ENOENT') { + console.log('WARN: Specific environment not loaded'); + } else { + throw err; + } + } +} +console.log('Charged Environment',process.env,'\r\n___ . ___'); diff --git a/worker-service/.env b/worker-service/.env index 93c8c8f0f2..18face0ea2 100644 --- a/worker-service/.env +++ b/worker-service/.env @@ -1,19 +1 @@ -ENV="develop" -HEDERA_NET="testnet" -PREUSED_HEDERA_NET="testnet" -MQ_ADDRESS="localhost" -SERVICE_CHANNEL="worker.1" -MIN_PRIORITY="0" -MAX_PRIORITY="20" -TASK_TIMEOUT="300" -REFRESH_INTERVAL="60" -IPFS_TIMEOUT="720" -IPFS_PROVIDER="web3storage" # 'web3storage' or 'local' -IPFS_PUBLIC_GATEWAY='https://ipfs.io/ipfs/${cid}' -IPFS_STORAGE_API_KEY="..." -IPFS_NODE_ADDRESS="http://localhost:5002" -MAX_HEDERA_TIMEOUT="600" -MQ_MAX_PAYLOAD="1048576" -#LOG_LEVEL="2" -#MAX_TRANSACTION_FEE="10" -#MQ_MESSAGE_CHUNK=5000000 +GUARDIAN_ENV="" diff --git a/worker-service/.env.docker b/worker-service/.env.docker index 4aa6b811b7..52987321b8 100644 --- a/worker-service/.env.docker +++ b/worker-service/.env.docker @@ -7,7 +7,7 @@ REFRESH_INTERVAL="60" IPFS_TIMEOUT="720" IPFS_PROVIDER="web3storage" # 'web3storage' or 'local' IPFS_PUBLIC_GATEWAY='https://ipfs.io/ipfs/${cid}' -IPFS_STORAGE_API_KEY="..." +IPFS_STORAGE_API_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJkaWQ6ZXRocjoweDUyMTQ1ZTY2OWUwMDdCNDQ2ZDZEZGUxYTlhMGJFZjAyZTM5RjU0ZTEiLCJpc3MiOiJ3ZWIzLXN0b3JhZ2UiLCJpYXQiOjE2NzI2NjEyNzEyMzksIm5hbWUiOiJ3ZWIzU3RvcmFnZUFwaVRrbiJ9.fu12X3XVFRFUSf9eszGid6AHHdGPriqPk4lHfCMEuX4" IPFS_NODE_ADDRESS="http://ipfs-node:5002" MAX_HEDERA_TIMEOUT="600" MQ_MAX_PAYLOAD="1048576" diff --git a/worker-service/Dockerfile b/worker-service/Dockerfile index fea6e53ff2..cc40737e46 100644 --- a/worker-service/Dockerfile +++ b/worker-service/Dockerfile @@ -36,7 +36,8 @@ COPY --from=interfacesModuleBuilder /usr/local/interfaces/guardian-interfaces-*. COPY --from=commonModuleBuilder /usr/local/common/guardian-common-*.tgz /tmp/common.tgz COPY --from=workerServiceBuilder /usr/local/worker-service/yarn.lock ./ COPY ./worker-service/package.json ./ -# COPY ./worker-service/.env.docker ./.env +RUN mkdir -p ./configs +COPY ./worker-service/configs/* ./configs RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/interfaces']='file:/tmp/interfaces.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));" RUN node -e "const fs=require('fs'); const input=JSON.parse(fs.readFileSync('package.json')); input.dependencies['@guardian/common']='file:/tmp/common.tgz'; fs.writeFileSync('package.json', JSON.stringify(input));" RUN yarn install --frozen-lockfile diff --git a/worker-service/configs/.env.worker b/worker-service/configs/.env.worker new file mode 100644 index 0000000000..fb8d80b4f3 --- /dev/null +++ b/worker-service/configs/.env.worker @@ -0,0 +1,21 @@ +# Guardian Service Specialized Variables +SERVICE_CHANNEL="worker.1" + +# Ecosystem Defined Variables +HEDERA_NET="testnet" +PREUSED_HEDERA_NET="testnet" +MQ_ADDRESS="localhost" +MIN_PRIORITY="0" +MAX_PRIORITY="20" +TASK_TIMEOUT="300" +REFRESH_INTERVAL="60" +IPFS_TIMEOUT="720" +IPFS_PROVIDER="web3storage" # 'web3storage' or 'local' +IPFS_PUBLIC_GATEWAY='https://ipfs.io/ipfs/${cid}' +IPFS_STORAGE_API_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJkaWQ6ZXRocjoweDUyMTQ1ZTY2OWUwMDdCNDQ2ZDZEZGUxYTlhMGJFZjAyZTM5RjU0ZTEiLCJpc3MiOiJ3ZWIzLXN0b3JhZ2UiLCJpYXQiOjE2NzI2NjEyNzEyMzksIm5hbWUiOiJ3ZWIzU3RvcmFnZUFwaVRrbiJ9.fu12X3XVFRFUSf9eszGid6AHHdGPriqPk4lHfCMEuX4" +IPFS_NODE_ADDRESS="http://localhost:5002" +MAX_HEDERA_TIMEOUT="600" +MQ_MAX_PAYLOAD="1048576" +#LOG_LEVEL="2" +#MAX_TRANSACTION_FEE="10" +#MQ_MESSAGE_CHUNK=5000000 diff --git a/worker-service/configs/.env.worker.develop b/worker-service/configs/.env.worker.develop new file mode 100644 index 0000000000..67f60bc19d --- /dev/null +++ b/worker-service/configs/.env.worker.develop @@ -0,0 +1,21 @@ +# Guardian Service Specialized Variables +SERVICE_CHANNEL="worker.1" + +# Ecosystem Defined Variables +HEDERA_NET="testnet" +PREUSED_HEDERA_NET="testnet" +MQ_ADDRESS="localhost" +MIN_PRIORITY="0" +MAX_PRIORITY="20" +TASK_TIMEOUT="300" +REFRESH_INTERVAL="60" +IPFS_TIMEOUT="720" +IPFS_PROVIDER="web3storage" # 'web3storage' or 'local' +IPFS_PUBLIC_GATEWAY='https://ipfs.io/ipfs/${cid}' +IPFS_STORAGE_API_KEY="..." +IPFS_NODE_ADDRESS="http://localhost:5002" +MAX_HEDERA_TIMEOUT="600" +MQ_MAX_PAYLOAD="1048576" +#LOG_LEVEL="2" +#MAX_TRANSACTION_FEE="10" +#MQ_MESSAGE_CHUNK=5000000 diff --git a/worker-service/configs/.env.worker.template b/worker-service/configs/.env.worker.template new file mode 100644 index 0000000000..a8cddc2734 --- /dev/null +++ b/worker-service/configs/.env.worker.template @@ -0,0 +1,21 @@ +# Guardian Service Specialized Variables +SERVICE_CHANNEL="worker.1" + +# Ecosystem Defined Variables +HEDERA_NET="" +PREUSED_HEDERA_NET="" +MQ_ADDRESS="" +MIN_PRIORITY="0" +MAX_PRIORITY="20" +TASK_TIMEOUT="300" +REFRESH_INTERVAL="60" +IPFS_TIMEOUT="720" +IPFS_PROVIDER="web3storage" # 'web3storage' or 'local' +IPFS_PUBLIC_GATEWAY='https://ipfs.io/ipfs/${cid}' +IPFS_STORAGE_API_KEY="..." +IPFS_NODE_ADDRESS="http://localhost:5002" +MAX_HEDERA_TIMEOUT="" +MQ_MAX_PAYLOAD="" +#LOG_LEVEL="2" +#MAX_TRANSACTION_FEE="10" +#MQ_MESSAGE_CHUNK=5000000 diff --git a/worker-service/src/app.ts b/worker-service/src/app.ts index e2d640fce6..6cbbbd532d 100644 --- a/worker-service/src/app.ts +++ b/worker-service/src/app.ts @@ -18,13 +18,13 @@ Promise.all([ const channelName = (process.env.SERVICE_CHANNEL || `worker.${Date.now()}`).toUpperCase() const [cn] = values; const channel = new MessageBrokerChannel(cn, 'worker'); - + console.log("***-> channel",channel); const logger = new Logger(); logger.setConnection(cn); const state = new ApplicationState(); await state.setServiceName('WORKER').setConnection(cn).init(); await state.updateState(ApplicationStates.STARTED); - + console.log("***-> state",state); await new OldSecretManager().setConnection(cn).init(); const validator = new ValidateConfiguration(); @@ -40,7 +40,7 @@ Promise.all([ IPFS_STORAGE_API_KEY= process.env.IPFS_STORAGE_API_KEY await secretManager.setSecrets('apikey/ipfs', { IPFS_STORAGE_API_KEY }); } - + console.log("***-> IPFS_STORAGE_API_KEY",IPFS_STORAGE_API_KEY); HederaSDKHelper.setTransactionLogSender(async (data) => { await channel.publish(`guardians.transaction-log-event`, data); }); @@ -48,6 +48,7 @@ Promise.all([ await state.updateState(ApplicationStates.INITIALIZING); const w = new Worker(); await w.setConnection(cn).init(); + console.log("***-> worker w",w); if (process.env.IPFS_PROVIDER === 'web3storage') { if (!IPFS_STORAGE_API_KEY) { @@ -69,26 +70,39 @@ Promise.all([ } } + console.log("***-> decoded "); + return true; }); + console.log("***-> Validator SET",validator); + validator.setValidAction(async () => { const maxPayload = parseInt(process.env.MQ_MAX_PAYLOAD, 10); + console.log("***-> maxPayload SET",maxPayload); if (Number.isInteger(maxPayload)) { new LargePayloadContainer().runServer(); } + console.log("***-> LARGE SET"); await state.updateState(ApplicationStates.READY); logger.info('Worker started', [channelName]); + console.log("***-> Worker started"); + }); + console.log("***-> INTRA VALID ACTION"); + validator.setInvalidAction(async () => { + console.log("***-> Invalid"); timer = setInterval(async () => { await state.updateState(ApplicationStates.BAD_CONFIGURATION); }, 1000); logger.error('Worker not configured', [channelName]); + console.log("***-> logger"); }) await validator.validate(); + console.log("***-> validate"); }, (reason) => { console.log(reason); process.exit(0); diff --git a/worker-service/src/config.ts b/worker-service/src/config.ts index 5bebff3761..07a6dc5fec 100644 --- a/worker-service/src/config.ts +++ b/worker-service/src/config.ts @@ -1,8 +1,30 @@ import moduleAlias from 'module-alias'; import dotenv from 'dotenv'; import 'reflect-metadata'; +import fs from 'fs'; moduleAlias.addAliases({ }); dotenv.config(); + +const envPath = process.env.GUARDIAN_ENV ? `./configs/.env.worker.${process.env.GUARDIAN_ENV}` : './configs/.env.worker'; + +if (!process.env.OVERRIDE || process.env.OVERRIDE === 'false'){ + console.log('reading from', envPath, 'not overriding'); + dotenv.config({ path: envPath}); +}else{ + try { + const envConfig = dotenv.parse(fs.readFileSync(envPath)); + for (const k of Object.keys(envConfig)) { + process.env[k] = envConfig[k] + } + } catch (err) { + if (err.code === 'ENOENT') { + console.log('WARN: Specific environment not loaded'); + } else { + throw err; + } + } +} +console.log('Charged Environment',process.env,'\r\n___ . ___');