From 94ba8abcd72f941236cbda752bb30c946b0c12e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Fri, 20 Oct 2023 12:57:32 +0200 Subject: [PATCH 1/2] Add logger from commons --- .env.example | 4 ++ package.json | 2 +- pnpm-lock.yaml | 10 ++- src/env/env.ts | 21 ++++++ src/env/schema.ts | 45 ++++++++++++ src/logger.ts | 34 +++------ src/signed-api-fetch/data-fetcher.ts | 9 +-- src/signed-data-store/signed-data-store.ts | 80 ++++++++++------------ 8 files changed, 124 insertions(+), 81 deletions(-) create mode 100644 .env.example create mode 100644 src/env/env.ts create mode 100644 src/env/schema.ts diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..77a6c36a --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +LOGGER_ENABLED=true +LOG_COLORIZE=true +LOG_FORMAT=pretty +LOG_LEVEL=info diff --git a/package.json b/package.json index d367f19b..0232ccea 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ }, "dependencies": { "@api3/airnode-protocol-v1": "^2.10.0", - "@api3/commons": "^0.2.0", + "@api3/commons": "^0.3.0", "@api3/promise-utils": "^0.4.0", "axios": "^1.5.1", "dotenv": "^16.3.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5bc942ea..d294faa3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ dependencies: specifier: ^2.10.0 version: 2.10.0 '@api3/commons': - specifier: ^0.2.0 - version: 0.2.0(eslint@8.50.0)(jest@29.7.0)(typescript@5.2.2)(zod@3.22.2) + specifier: ^0.3.0 + version: 0.3.0(eslint@8.50.0)(jest@29.7.0)(typescript@5.2.2) '@api3/promise-utils': specifier: ^0.4.0 version: 0.4.0 @@ -102,12 +102,11 @@ packages: '@openzeppelin/contracts': 4.8.2 dev: false - /@api3/commons@0.2.0(eslint@8.50.0)(jest@29.7.0)(typescript@5.2.2)(zod@3.22.2): - resolution: {integrity: sha512-SesOHOkIl+8mqXEWXv5GUqr2vgesRXmMt2CRf7o1c3pRFkDpMQNlxc3uOUTgaWphrzr0smMjWMTTWvzW6Bxo0A==} + /@api3/commons@0.3.0(eslint@8.50.0)(jest@29.7.0)(typescript@5.2.2): + resolution: {integrity: sha512-27+UkW0qCWvQCJuO8vbyBhJ94IXELKhqCKjFxC5ZUQrJHRHWT4zz2wmdbsvgUHBZhuAuFPaz/78yjMCjzJ6S8Q==} engines: {node: ^18.14.0, pnpm: ^8.8.0} peerDependencies: eslint: ^8.50.0 - zod: ^3.22.2 dependencies: '@api3/ois': 2.2.1 '@api3/promise-utils': 0.4.0 @@ -131,7 +130,6 @@ packages: lodash: 4.17.21 winston: 3.11.0 winston-console-format: 1.0.8 - zod: 3.22.2 transitivePeerDependencies: - eslint-import-resolver-typescript - eslint-import-resolver-webpack diff --git a/src/env/env.ts b/src/env/env.ts new file mode 100644 index 00000000..43f6a18c --- /dev/null +++ b/src/env/env.ts @@ -0,0 +1,21 @@ +import { join } from 'node:path'; + +import dotenv from 'dotenv'; + +import { type EnvConfig, envConfigSchema } from './schema'; + +let env: EnvConfig | undefined; + +export const loadEnv = () => { + if (env) return env; + + dotenv.config({ path: join(__dirname, '../.env') }); + + const parseResult = envConfigSchema.safeParse(process.env); + if (!parseResult.success) { + throw new Error(`Invalid environment variables:\n, ${JSON.stringify(parseResult.error.format())}`); + } + + env = parseResult.data; + return env; +}; diff --git a/src/env/schema.ts b/src/env/schema.ts new file mode 100644 index 00000000..aa8f7397 --- /dev/null +++ b/src/env/schema.ts @@ -0,0 +1,45 @@ +import { type LogFormat, logFormatOptions, logLevelOptions, type LogLevel } from '@api3/commons'; +import { z } from 'zod'; + +export const envBooleanSchema = z.union([z.literal('true'), z.literal('false')]).transform((val) => val === 'true'); + +// We apply default values to make it convenient to omit certain environment variables. The default values should be +// primarily focused on users and production usage. +export const envConfigSchema = z + .object({ + LOG_COLORIZE: envBooleanSchema.default('false'), + LOG_FORMAT: z + .string() + .transform((value, ctx) => { + if (!logFormatOptions.includes(value as any)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid LOG_FORMAT', + path: ['LOG_FORMAT'], + }); + return; + } + + return value as LogFormat; + }) + .default('json'), + LOG_LEVEL: z + .string() + .transform((value, ctx) => { + if (!logLevelOptions.includes(value as any)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid LOG_LEVEL', + path: ['LOG_LEVEL'], + }); + return; + } + + return value as LogLevel; + }) + .default('info'), + LOGGER_ENABLED: envBooleanSchema.default('true'), + }) + .strip(); // We parse from ENV variables of the process which has many variables that we don't care about + +export type EnvConfig = z.infer; diff --git a/src/logger.ts b/src/logger.ts index 7ab1aff9..26a7faaf 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,27 +1,11 @@ -const debug = (...args: any[]) => - // eslint-disable-next-line no-console - console.debug(...args); -const error = (...args: any[]) => - // eslint-disable-next-line no-console - console.error(...args); -const info = (...args: any[]) => console.info(...args); -const log = (...args: any[]) => - // eslint-disable-next-line no-console - console.log(...args); -const warn = (...args: any[]) => - // eslint-disable-next-line no-console - console.warn(...args); +import { createLogger } from '@api3/commons'; +import { loadEnv } from './env/env'; -export const logErrors = (promiseResults: PromiseSettledResult[], additionalText = '') => { - for (const rejectedPromise of promiseResults.filter((result) => result.status === 'rejected')) { - error(additionalText, rejectedPromise); - } -}; +const env = loadEnv(); -export const logger = { - debug, - error, - info, - log, - warn, -}; +export const logger = createLogger({ + colorize: env.LOG_COLORIZE, + enabled: env.LOGGER_ENABLED, + minLevel: env.LOG_LEVEL, + format: env.LOG_FORMAT, +}); diff --git a/src/signed-api-fetch/data-fetcher.ts b/src/signed-api-fetch/data-fetcher.ts index 6dec0fe0..63a884c2 100644 --- a/src/signed-api-fetch/data-fetcher.ts +++ b/src/signed-api-fetch/data-fetcher.ts @@ -1,11 +1,10 @@ import { clearInterval } from 'node:timers'; -import { go, goSync } from '@api3/promise-utils'; +import { go } from '@api3/promise-utils'; import axios from 'axios'; import { uniq } from 'lodash'; import { signedApiResponseSchema, type SignedData } from '../types'; import * as localDataStore from '../signed-data-store'; import { getState, setState } from '../state'; -import { logger } from '../logger'; import { HTTP_SIGNED_DATA_API_ATTEMPT_TIMEOUT, HTTP_SIGNED_DATA_API_HEADROOM } from '../constants'; // Express handler/endpoint path: https://github.com/api3dao/signed-api/blob/b6e0d0700dd9e7547b37eaa65e98b50120220105/packages/api/src/server.ts#L33 @@ -76,11 +75,7 @@ export const runDataFetcher = async () => { const payload = await callSignedDataApi(url); for (const element of payload) { - const result = goSync(() => localDataStore.setStoreDataPoint(element)); - - if (!result.success) { - logger.warn('Error while storing datapoint in data store.', { ...result.error }); - } + localDataStore.setStoreDataPoint(element); } }, { diff --git a/src/signed-data-store/signed-data-store.ts b/src/signed-data-store/signed-data-store.ts index 4cca52e2..477f5cc9 100644 --- a/src/signed-data-store/signed-data-store.ts +++ b/src/signed-data-store/signed-data-store.ts @@ -1,4 +1,5 @@ -import { BigNumber, ethers } from 'ethers'; +import { ethers } from 'ethers'; +import { goSync } from '@api3/promise-utils'; import { logger } from '../logger'; import type { LocalSignedData, SignedData, AirnodeAddress, TemplateId } from '../types'; @@ -6,19 +7,33 @@ import type { LocalSignedData, SignedData, AirnodeAddress, TemplateId } from '.. let signedApiStore: Record> = {}; export const verifySignedData = ({ airnode, templateId, timestamp, signature, encodedValue }: SignedData) => { - // 'encodedValue' is: ethers.utils.defaultAbiCoder.encode(['int256'], [beaconValue]); + // Verification is wrapped in goSync, because ethers methods can potentially throw on invalid input. + const goVerify = goSync(() => { + const message = ethers.utils.arrayify( + ethers.utils.solidityKeccak256(['bytes32', 'uint256', 'bytes'], [templateId, timestamp, encodedValue]) + ); + + const signerAddr = ethers.utils.verifyMessage(message, signature); + if (signerAddr !== airnode) throw new Error('Signer address does not match'); + }); - const message = ethers.utils.arrayify( - ethers.utils.solidityKeccak256(['bytes32', 'uint256', 'bytes'], [templateId, timestamp, encodedValue]) - ); + if (!goVerify.success) { + logger.error(`Signature verification failed`, { + airnode, + templateId, + signature, + timestamp, + encodedValue, + }); + return false; + } - const signerAddr = ethers.utils.verifyMessage(message, signature); - return signerAddr !== airnode; + return true; }; -const verifyTimestamp = ({ timestamp, airnode, encodedValue, templateId }: SignedData) => { +const verifyTimestamp = ({ timestamp, airnode, templateId }: SignedData) => { if (Number.parseInt(timestamp, 10) * 1000 > Date.now() + 60 * 60 * 1000) { - logger.warn(`Refusing to store sample as timestamp is more than one hour in the future.`, { + logger.error(`Refusing to store sample as timestamp is more than one hour in the future.`, { airnode, templateId, systemDateNow: new Date().toLocaleDateString(), @@ -28,45 +43,25 @@ const verifyTimestamp = ({ timestamp, airnode, encodedValue, templateId }: Signe } if (Number.parseInt(timestamp, 10) * 1000 > Date.now()) { - logger.warn( - `Sample is in the future, but by less than an hour, therefore storing anyway: (Airnode ${airnode}) (Template ID ${templateId}) (Received timestamp ${new Date( - Number.parseInt(timestamp, 10) * 1000 - ).toLocaleDateString()} vs now ${new Date().toLocaleDateString()}), ${ - BigNumber.from(encodedValue).div(10e10).toNumber() / 10e8 - }` - ); + logger.warn(`Sample is in the future, but by less than an hour, therefore storing anyway.`, { + airnode, + templateId, + systemDateNow: new Date().toLocaleDateString(), + signedDataDate: new Date(Number.parseInt(timestamp, 10) * 1000).toLocaleDateString(), + }); } return true; }; export const verifySignedDataIntegrity = (signedData: SignedData) => { - const { airnode, templateId, timestamp, encodedValue } = signedData; - - if (!verifyTimestamp(signedData)) { - return false; - } - - if (verifySignedData(signedData)) { - logger.warn( - `Refusing to store sample as signature does not match: (Airnode ${airnode}) (Template ID ${templateId}) (Received timestamp ${new Date( - Number.parseInt(timestamp, 10) * 1000 - ).toLocaleDateString()} vs now ${new Date().toLocaleDateString()}), ${ - BigNumber.from(encodedValue).div(10e10).toNumber() / 10e8 - }` - ); - return false; - } - - return true; + return verifyTimestamp(signedData) && verifySignedData(signedData); }; export const setStoreDataPoint = (signedData: SignedData) => { const { airnode, templateId, signature, timestamp, encodedValue } = signedData; if (!verifySignedDataIntegrity(signedData)) { - logger.warn(`Signed data received from signed data API has a signature mismatch.`); - logger.warn(JSON.stringify({ airnode, templateId, signature, timestamp, encodedValue }, null, 2)); return; } @@ -80,12 +75,13 @@ export const setStoreDataPoint = (signedData: SignedData) => { return; } - logger.debug( - `Storing sample for (Airnode ${airnode}) (Template ID ${templateId}) (Timestamp ${new Date( - Number.parseInt(timestamp, 10) * 1000 - ).toISOString()}), ${BigNumber.from(encodedValue).div(10e10).toNumber() / 10e8}` - ); - + logger.debug(`Storing signed data`, { + airnode, + templateId, + timestamp, + signature, + encodedValue, + }); signedApiStore[airnode]![templateId] = { signature, timestamp, encodedValue }; }; From 904311609cee2208ec60a1b96780405b4034b80d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Fri, 20 Oct 2023 13:05:10 +0200 Subject: [PATCH 2/2] Add README --- README.md | 89 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 74 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 278e879c..601a823f 100644 --- a/README.md +++ b/README.md @@ -20,11 +20,70 @@ edit `airseeker_v2_pipeline.drawio`, preferably by cloning the repository and lo ## Configuration +Airseeker can be configured via a combination of [environment variables](#environment-variables) and +[configuration files](#configuration-files). + +### Environment variables + +For example: + +```sh +# Defines a logger suitable for production. +LOGGER_ENABLED=true +LOG_COLORIZE=false +LOG_FORMAT=json +LOG_LEVEL=info +``` + +or + +```sh +# Defines a logger suitable for local development or testing. +LOGGER_ENABLED=true +LOG_COLORIZE=false +LOG_FORMAT=json +LOG_LEVEL=info +``` + +#### `LOGGER_ENABLED` + +Enables or disables logging. Options: + +- `true` - Enables logging. +- `false` - Disables logging. + +#### `LOG_FORMAT` + +The format of the log output. Options: + +- `json` - Specifies JSON log format. This is suitable when running in production and streaming logs to other services. +- `pretty` - Logs are formatted in a human-friendly "pretty" way. Ideal, when running the service locally and in + development. + +#### `LOG_COLORIZE` + +Enables or disables colors in the log output. Options: + +- `true` - Enables colors in the log output. The output has special color setting characters that are parseable by CLI. + Recommended when running locally and in development. +- `false` - Disables colors in the log output. Recommended for production. + +#### `LOG_LEVEL` + +Defines the minimum level of logs. Logs with smaller level (severity) will be silenced. Options: + +- `debug` - Enables all logs. +- `info` - Enables logs with level `info`, `warn` and `error`. +- `warn` - Enables logs with level `warn` and `error`. +- `error` - Enables logs with level `error`. + +### Configuration files + Airseeker needs two configuration files, `airseeker.json` and `secrets.env`. All expressions of a form `${SECRET_NAME}` are referring to values from secrets and are interpolated inside the `airseeker.json` at runtime. You are advised to put sensitive information inside secrets. -### `sponsorWalletMnemonic` +#### `sponsorWalletMnemonic` The mnemonic of the wallet used to derive sponsor wallets. Sponsor wallets are derived for each dAPI separately. It is recommended to interpolate this value from secrets. For example: @@ -34,7 +93,7 @@ recommended to interpolate this value from secrets. For example: "sponsorWalletMnemonic": "${SPONSOR_WALLET_MNEMONIC}", ``` -### `chains` +#### `chains` A record of chain configurations. The record key is the chain ID. For example: @@ -51,61 +110,61 @@ A record of chain configurations. The record key is the chain ID. For example: } ``` -#### `contracts` _(optional)_ +##### `contracts` _(optional)_ A record of contract addresses used by Airseeker. If not specified, the addresses are loaded from [Airnode protocol v1](https://github.com/api3dao/airnode-protocol-v1). -##### Api3ServerV1 _(optional)_ +###### Api3ServerV1 _(optional)_ The address of the Api3ServerV1 contract. If not specified, the address is loaded from the Airnode protocol v1 repository. -#### `providers` +##### `providers` A record of providers. The record key is the provider name. Provider name is only used for internal purposes and to uniquely identify the provider for the given chain. -##### `providers[]` +###### `providers[]` A provider configuration. -###### `url` +`url` The URL of the provider. -#### `__Temporary__DapiDataRegistry` +##### `__Temporary__DapiDataRegistry` The data needed to make the requests to signed API. This data will in the future be stored on-chain in a `DapiDataRegistry` contract. For the time being, they are statically defined in the configuration file. -##### `airnodeToSignedApiUrl` +###### `airnodeToSignedApiUrl` A mapping from Airnode address to signed API URL. When data from particular beacon is needed a request is made to the signed API corresponding to the beacon address. -##### `dataFeedIdToBeacons` +###### `dataFeedIdToBeacons` A mapping from data feed ID to a list of beacon data. -##### `dataFeedIdToBeacons` +###### `dataFeedIdToBeacons` A single element array for a beacon data. If the data feed is a beacon set, the array contains the data for all the beacons in the beacon set (in correct order). -###### `dataFeedIdToBeacons[n]` +`dataFeedIdToBeacons[n]` A beacon data. -`airnode` +`dataFeedIdToBeacons[n].airnode` The Airnode address of the beacon. -`templateId` +`dataFeedIdToBeacons[n].templateId` The template ID of the beacon. -### `deviationThresholdCoefficient` +#### `deviationThresholdCoefficient` The global coefficient applied to all deviation checks. Used to differentiate alternate deployments. For example: