From f71a38dad2ffe7ba493bd83d268e9a87268d9266 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Wed, 1 Nov 2023 11:28:20 -0700 Subject: [PATCH] Undo local testing changes --- docker-compose.yml | 164 ++++++++++++++-------------- runner/src/indexer/indexer.ts | 7 +- runner/src/stream-handler/worker.ts | 4 - 3 files changed, 83 insertions(+), 92 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 91a87e0c3..8911aa97e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,93 +1,93 @@ version: "3.9" # optional since v1.27.0 services: - # coordinator: - # build: - # context: ./indexer - # args: - # - CARGO_BUILD_MODE=debug - # depends_on: - # - redis - # environment: - # REDIS_CONNECTION_STRING: redis://redis - # LAKE_AWS_ACCESS_KEY: - # LAKE_AWS_SECRET_ACCESS_KEY: - # QUEUE_AWS_ACCESS_KEY: - # QUEUE_AWS_SECRET_ACCESS_KEY: - # QUEUE_URL: MOCK - # START_FROM_BLOCK_QUEUE_URL: MOCK - # PORT: 9180 - # REGISTRY_CONTRACT_ID: dev-queryapi.dataplatform.near - # AWS_QUEUE_REGION: eu-central-1 - # command: - # - mainnet - # - from-interruption + coordinator: + build: + context: ./indexer + args: + - CARGO_BUILD_MODE=debug + depends_on: + - redis + environment: + REDIS_CONNECTION_STRING: redis://redis + LAKE_AWS_ACCESS_KEY: + LAKE_AWS_SECRET_ACCESS_KEY: + QUEUE_AWS_ACCESS_KEY: + QUEUE_AWS_SECRET_ACCESS_KEY: + QUEUE_URL: MOCK + START_FROM_BLOCK_QUEUE_URL: MOCK + PORT: 9180 + REGISTRY_CONTRACT_ID: dev-queryapi.dataplatform.near + AWS_QUEUE_REGION: eu-central-1 + command: + - mainnet + - from-interruption - # runner: - # build: - # context: ./runner - # depends_on: - # - "hasura-graphql" - # - "redis" - # environment: - # REGION: eu-central-1 - # HASURA_ENDPOINT: http://hasura-graphql:8080 - # HASURA_ADMIN_SECRET: myadminsecretkey - # REDIS_CONNECTION_STRING: redis://redis - # PGHOST: postgres - # PGPORT: 5432 - # PGUSER: postgres - # PGPASSWORD: postgrespassword - # PGDATABASE: postgres - # PORT: 9180 - # AWS_ACCESS_KEY_ID: - # AWS_SECRET_ACCESS_KEY: + runner: + build: + context: ./runner + depends_on: + - "hasura-graphql" + - "redis" + environment: + REGION: eu-central-1 + HASURA_ENDPOINT: http://hasura-graphql:8080 + HASURA_ADMIN_SECRET: myadminsecretkey + REDIS_CONNECTION_STRING: redis://redis + PGHOST: postgres + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: postgrespassword + PGDATABASE: postgres + PORT: 9180 + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: - # redis: - # image: redis - # command: - # - redis-server - # - "--save 60 1" - # - "--loglevel warning" - # volumes: - # - redis:/data - # ports: - # - "6379:6379" + redis: + image: redis + command: + - redis-server + - "--save 60 1" + - "--loglevel warning" + volumes: + - redis:/data + ports: + - "6379:6379" - # postgres: - # image: postgres:12 - # restart: always - # volumes: - # - postgres:/var/lib/postgresql/data - # environment: - # POSTGRES_PASSWORD: postgrespassword - # ports: - # - "5432:5432" + postgres: + image: postgres:12 + restart: always + volumes: + - postgres:/var/lib/postgresql/data + environment: + POSTGRES_PASSWORD: postgrespassword + ports: + - "5432:5432" - # hasura-auth: - # build: - # context: ./hasura-authentication-service - # ports: - # - "4000:4000" - # environment: - # PORT: 4000 - # DEFAULT_HASURA_ROLE: append + hasura-auth: + build: + context: ./hasura-authentication-service + ports: + - "4000:4000" + environment: + PORT: 4000 + DEFAULT_HASURA_ROLE: append - # hasura-graphql: - # image: hasura/graphql-engine:latest - # ports: - # - "8080:8080" - # depends_on: - # - "postgres" - # - "hasura-auth" - # restart: always - # environment: - # HASURA_GRAPHQL_DATABASE_URL: postgres://postgres:postgrespassword@postgres:5432/postgres - # HASURA_GRAPHQL_ENABLE_CONSOLE: "true" - # HASURA_GRAPHQL_DEV_MODE: "true" - # HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log - # HASURA_GRAPHQL_ADMIN_SECRET: myadminsecretkey - # HASURA_GRAPHQL_AUTH_HOOK: http://hasura-auth:4000/auth + hasura-graphql: + image: hasura/graphql-engine:latest + ports: + - "8080:8080" + depends_on: + - "postgres" + - "hasura-auth" + restart: always + environment: + HASURA_GRAPHQL_DATABASE_URL: postgres://postgres:postgrespassword@postgres:5432/postgres + HASURA_GRAPHQL_ENABLE_CONSOLE: "true" + HASURA_GRAPHQL_DEV_MODE: "true" + HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log + HASURA_GRAPHQL_ADMIN_SECRET: myadminsecretkey + HASURA_GRAPHQL_AUTH_HOOK: http://hasura-auth:4000/auth grafana: image: grafana/grafana volumes: diff --git a/runner/src/indexer/indexer.ts b/runner/src/indexer/indexer.ts index b369c1893..d056450c9 100644 --- a/runner/src/indexer/indexer.ts +++ b/runner/src/indexer/indexer.ts @@ -87,7 +87,6 @@ export default class Indexer { const hasuraRoleName = functionName.split('/')[0].replace(/[.-]/g, '_'); if (options.provision && !indexerFunction.provisioned) { - const provisionintLatency = performance.now(); try { if (!await this.deps.provisioner.isUserApiProvisioned(indexerFunction.account_id, indexerFunction.function_name)) { await this.setStatus(functionName, blockHeight, 'PROVISIONING'); @@ -102,11 +101,9 @@ export default class Indexer { simultaneousPromises.push(this.writeLog(functionName, blockHeight, 'Provisioning endpoint: failure', error.message)); throw error; } - console.log('Provisioning Latency: ', performance.now() - provisionintLatency); } await this.setStatus(functionName, blockHeight, 'RUNNING'); - console.log('Function State Logging Latency: ', performance.now() - functionStateLoggingLatency); this.deps.parentPort?.postMessage({ type: 'FUNCTION_STATE_LOGGING_LATENCY', labels: { indexer: functionName, type: isHistorical ? 'historical' : 'real-time' }, @@ -122,7 +119,6 @@ export default class Indexer { vm.freeze(context, 'console'); // provide console.log via context.log const modifiedFunction = this.transformIndexerFunction(indexerFunction.code); - console.log('VM and Context Object Preparation Latency: ', performance.now() - vmAndContextBuildLatency); this.deps.parentPort?.postMessage({ type: 'FUNCTION_VM_AND_CONTEXT_LATENCY', labels: { indexer: functionName, type: isHistorical ? 'historical' : 'real-time' }, @@ -140,7 +136,7 @@ export default class Indexer { await this.writeLog(functionName, blockHeight, 'Error running IndexerFunction', error.message); throw e; } - console.log('Function Execution Latency: ', performance.now() - functionCodeExecutionLatency); + console.log('Function Code Execution Latency: ', performance.now() - functionCodeExecutionLatency); this.deps.parentPort?.postMessage({ type: 'FUNCTION_CODE_EXECUTION_LATENCY', labels: { indexer: functionName, type: isHistorical ? 'historical' : 'real-time' }, @@ -154,7 +150,6 @@ export default class Indexer { throw e; } finally { await Promise.all(simultaneousPromises); - console.log('Finish Promise Handling Latency: ', finishPromiseHandlingLatency !== undefined ? performance.now() - finishPromiseHandlingLatency : 'null'); this.deps.parentPort?.postMessage({ type: 'FUNCTION_VM_AND_CONTEXT_LATENCY', labels: { indexer: functionName, type: isHistorical ? 'historical' : 'real-time' }, diff --git a/runner/src/stream-handler/worker.ts b/runner/src/stream-handler/worker.ts index c8a1f0ab7..78be31842 100644 --- a/runner/src/stream-handler/worker.ts +++ b/runner/src/stream-handler/worker.ts @@ -152,7 +152,6 @@ async function historicalStreamerMessageQueueConsumer (queue: Array