From fd33d70916c16f4bac29430aa805cc15bf9841d8 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Tue, 27 Aug 2024 14:29:37 +1200 Subject: [PATCH 1/5] chore: Remove unnecessary hasura config --- hasura/Dockerfile | 9 ----- hasura/config.yaml | 7 ---- hasura/metadata/actions.graphql | 0 hasura/metadata/actions.yaml | 6 ---- hasura/metadata/allow_list.yaml | 1 - hasura/metadata/cron_triggers.yaml | 1 - hasura/metadata/databases/databases.yaml | 14 -------- .../tables/public_indexer_log_entries.yaml | 25 -------------- .../default/tables/public_indexer_state.yaml | 34 ------------------- .../databases/default/tables/tables.yaml | 2 -- hasura/metadata/query_collections.yaml | 1 - hasura/metadata/remote_schemas.yaml | 1 - hasura/metadata/rest_endpoints.yaml | 1 - hasura/metadata/version.yaml | 1 - .../default/1691364619300_init/down.sql | 2 -- .../default/1691364619300_init/up.sql | 25 -------------- 16 files changed, 130 deletions(-) delete mode 100644 hasura/Dockerfile delete mode 100644 hasura/config.yaml delete mode 100644 hasura/metadata/actions.graphql delete mode 100644 hasura/metadata/actions.yaml delete mode 100644 hasura/metadata/allow_list.yaml delete mode 100644 hasura/metadata/cron_triggers.yaml delete mode 100644 hasura/metadata/databases/databases.yaml delete mode 100644 hasura/metadata/databases/default/tables/public_indexer_log_entries.yaml delete mode 100644 hasura/metadata/databases/default/tables/public_indexer_state.yaml delete mode 100644 hasura/metadata/databases/default/tables/tables.yaml delete mode 100644 hasura/metadata/query_collections.yaml delete mode 100644 hasura/metadata/remote_schemas.yaml delete mode 100644 hasura/metadata/rest_endpoints.yaml delete mode 100644 hasura/metadata/version.yaml delete mode 100644 hasura/migrations/default/1691364619300_init/down.sql delete mode 100644 hasura/migrations/default/1691364619300_init/up.sql diff --git a/hasura/Dockerfile b/hasura/Dockerfile deleted file mode 100644 index a3cb05d42..000000000 --- a/hasura/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM hasura/graphql-engine:latest.cli-migrations-v3 - -ARG DATABASE_URL -ARG PORT - -COPY migrations /hasura-migrations -COPY metadata /hasura-metadata - -CMD graphql-engine serve diff --git a/hasura/config.yaml b/hasura/config.yaml deleted file mode 100644 index d13faa054..000000000 --- a/hasura/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -version: 3 -endpoint: http://localhost:8080 -admin_secret: myadminsecretkey -metadata_directory: metadata -actions: - kind: synchronous - handler_webhook_baseurl: http://localhost:3000 diff --git a/hasura/metadata/actions.graphql b/hasura/metadata/actions.graphql deleted file mode 100644 index e69de29bb..000000000 diff --git a/hasura/metadata/actions.yaml b/hasura/metadata/actions.yaml deleted file mode 100644 index 1edb4c2ff..000000000 --- a/hasura/metadata/actions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -actions: [] -custom_types: - enums: [] - input_objects: [] - objects: [] - scalars: [] diff --git a/hasura/metadata/allow_list.yaml b/hasura/metadata/allow_list.yaml deleted file mode 100644 index fe51488c7..000000000 --- a/hasura/metadata/allow_list.yaml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/hasura/metadata/cron_triggers.yaml b/hasura/metadata/cron_triggers.yaml deleted file mode 100644 index fe51488c7..000000000 --- a/hasura/metadata/cron_triggers.yaml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/hasura/metadata/databases/databases.yaml b/hasura/metadata/databases/databases.yaml deleted file mode 100644 index 65a11b202..000000000 --- a/hasura/metadata/databases/databases.yaml +++ /dev/null @@ -1,14 +0,0 @@ -- name: default - kind: postgres - configuration: - connection_info: - database_url: - from_env: HASURA_GRAPHQL_DATABASE_URL - isolation_level: read-committed - pool_settings: - connection_lifetime: 600 - idle_timeout: 180 - max_connections: 50 - retries: 1 - use_prepared_statements: true - tables: "!include default/tables/tables.yaml" diff --git a/hasura/metadata/databases/default/tables/public_indexer_log_entries.yaml b/hasura/metadata/databases/default/tables/public_indexer_log_entries.yaml deleted file mode 100644 index ff2c158f4..000000000 --- a/hasura/metadata/databases/default/tables/public_indexer_log_entries.yaml +++ /dev/null @@ -1,25 +0,0 @@ -table: - name: indexer_log_entries - schema: public -insert_permissions: -- permission: - backend_only: true - check: {} - columns: - - block_height - - function_name - - message - - timestamp - - id - role: append -select_permissions: -- permission: - columns: - - block_height - - function_name - - message - - timestamp - - id - allow_aggregations: true - filter: {} - role: append diff --git a/hasura/metadata/databases/default/tables/public_indexer_state.yaml b/hasura/metadata/databases/default/tables/public_indexer_state.yaml deleted file mode 100644 index 2bcc6c9b9..000000000 --- a/hasura/metadata/databases/default/tables/public_indexer_state.yaml +++ /dev/null @@ -1,34 +0,0 @@ -table: - name: indexer_state - schema: public -insert_permissions: -- permission: - backend_only: true - check: {} - columns: - - function_name - - current_block_height - - current_historical_block_height - - status - role: append -select_permissions: -- permission: - columns: - - function_name - - current_block_height - - current_historical_block_height - - status - allow_aggregations: true - filter: {} - role: append -update_permissions: -- permission: - backend_only: true - check: {} - columns: - - function_name - - current_block_height - - current_historical_block_height - - status - filter: {} - role: append diff --git a/hasura/metadata/databases/default/tables/tables.yaml b/hasura/metadata/databases/default/tables/tables.yaml deleted file mode 100644 index fb74b4277..000000000 --- a/hasura/metadata/databases/default/tables/tables.yaml +++ /dev/null @@ -1,2 +0,0 @@ -- "!include public_indexer_log_entries.yaml" -- "!include public_indexer_state.yaml" diff --git a/hasura/metadata/query_collections.yaml b/hasura/metadata/query_collections.yaml deleted file mode 100644 index fe51488c7..000000000 --- a/hasura/metadata/query_collections.yaml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/hasura/metadata/remote_schemas.yaml b/hasura/metadata/remote_schemas.yaml deleted file mode 100644 index fe51488c7..000000000 --- a/hasura/metadata/remote_schemas.yaml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/hasura/metadata/rest_endpoints.yaml b/hasura/metadata/rest_endpoints.yaml deleted file mode 100644 index fe51488c7..000000000 --- a/hasura/metadata/rest_endpoints.yaml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/hasura/metadata/version.yaml b/hasura/metadata/version.yaml deleted file mode 100644 index 0a70affa4..000000000 --- a/hasura/metadata/version.yaml +++ /dev/null @@ -1 +0,0 @@ -version: 3 diff --git a/hasura/migrations/default/1691364619300_init/down.sql b/hasura/migrations/default/1691364619300_init/down.sql deleted file mode 100644 index 9ba2f7739..000000000 --- a/hasura/migrations/default/1691364619300_init/down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE public.indexer_log_entries; -DROP TABLE public.indexer_state; diff --git a/hasura/migrations/default/1691364619300_init/up.sql b/hasura/migrations/default/1691364619300_init/up.sql deleted file mode 100644 index 4c1dad71f..000000000 --- a/hasura/migrations/default/1691364619300_init/up.sql +++ /dev/null @@ -1,25 +0,0 @@ -SET check_function_bodies = false; - -CREATE TABLE public.indexer_log_entries ( - id uuid DEFAULT gen_random_uuid() NOT NULL, - function_name text NOT NULL, - block_height numeric NOT NULL, - "timestamp" timestamp without time zone DEFAULT CURRENT_TIMESTAMP, - message text -); - -CREATE TABLE public.indexer_state ( - function_name character varying NOT NULL, - current_block_height numeric(21,0) NOT NULL, - status text, - current_historical_block_height numeric(21,0) -); - -ALTER TABLE ONLY public.indexer_log_entries - ADD CONSTRAINT indexer_log_entries_pkey PRIMARY KEY (id); - -ALTER TABLE ONLY public.indexer_state - ADD CONSTRAINT indexer_state_pkey PRIMARY KEY (function_name); - -CREATE INDEX idx_function_name ON indexer_log_entries(function_name); -CREATE INDEX idx_timestamp ON indexer_log_entries("timestamp"); From 8380b5009913831be297d20e9cb0c7913fb1e6dd Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Tue, 27 Aug 2024 14:29:52 +1200 Subject: [PATCH 2/5] chore: Remove hasura notes from readme --- README.md | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/README.md b/README.md index 682af8129..932ce0182 100644 --- a/README.md +++ b/README.md @@ -47,19 +47,6 @@ Coordinator: These should be populated with your credentials. In most cases, the same key pair can be used for all 3 sets of credentials. Just ensure the keys have permissions to access S3 for handling [Requestor Pays](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html) in Near Lake. -### Hasura Configuration -Hasura contains shared tables for e.g. logging and setting arbitrary state. These tables must be configured prior to running the entire QueryApi application. Configuration is stored in the `hasura/` directory and deployed through the Hasura CLI. - -To configure Hasura, first start it with: -```sh -docker compose up hasura-graphql --detach -``` - -And apply the configuration with: -```sh -cd ./hasura && hasura deploy -``` - ### Running QueryApi With everything configured correctly, we can now start all components of QueryApi with: ```sh From a281625d3aadd5749b44acdd346db039c9b294ee Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Tue, 27 Aug 2024 14:30:00 +1200 Subject: [PATCH 3/5] chore: Update incorrect env config notes in readme --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 932ce0182..2db9f194e 100644 --- a/README.md +++ b/README.md @@ -39,11 +39,9 @@ Runner: - `AWS_ACCESS_KEY_ID` - `AWS_SECRET_ACCESS_KEY` -Coordinator: -- `LAKE_AWS_ACCESS_KEY` -- `LAKE_AWS_SECRET_ACCESS_KEY` -- `QUEUE_AWS_ACCESS_KEY` -- `QUEUE_AWS_SECRET_ACCESS_KEY` +Block Streamer: +- `AWS_ACCESS_KEY_ID` +- `AWS_SECRET_ACCESS_KEY` These should be populated with your credentials. In most cases, the same key pair can be used for all 3 sets of credentials. Just ensure the keys have permissions to access S3 for handling [Requestor Pays](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html) in Near Lake. From 470b76ed08f6e55556cbc901469ec372b50fb67d Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Tue, 27 Aug 2024 14:45:28 +1200 Subject: [PATCH 4/5] test: Remove unnecessary building of hasura image --- runner/tests/integration.test.ts | 2 +- runner/tests/testcontainers/hasura.ts | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/runner/tests/integration.test.ts b/runner/tests/integration.test.ts index 8cfa1f64d..717fc79db 100644 --- a/runner/tests/integration.test.ts +++ b/runner/tests/integration.test.ts @@ -66,7 +66,7 @@ describe('Indexer integration', () => { postgresContainer = await (await PostgreSqlContainer.build()) .withNetwork(network) .start(); - hasuraContainer = await (await HasuraGraphQLContainer.build()) + hasuraContainer = await new HasuraGraphQLContainer() .withNetwork(network) .withDatabaseUrl(postgresContainer.getConnectionUri(network.getName())) .start(); diff --git a/runner/tests/testcontainers/hasura.ts b/runner/tests/testcontainers/hasura.ts index 4fe28cd0c..a766231f0 100644 --- a/runner/tests/testcontainers/hasura.ts +++ b/runner/tests/testcontainers/hasura.ts @@ -9,19 +9,13 @@ export class HasuraGraphQLContainer { private readonly PORT = 8080; - private constructor (private readonly container: GenericContainer) { + constructor (private readonly container = new GenericContainer('hasura/graphql-engine:latest')) { container.withExposedPorts(this.PORT) - .withWaitStrategy(Wait.forLogMessage(/.*Starting API server.*/, 2)) + .withWaitStrategy(Wait.forLogMessage(/.*starting API server.*/)) .withLogConsumer(logConsumer) .withStartupTimeout(120_000); } - public static async build (): Promise { - const container = await GenericContainer.fromDockerfile('../hasura/').build(); - - return new HasuraGraphQLContainer(container); - } - public withNetwork (network: StartedNetwork): this { this.container.withNetwork(network); return this; From 22a92a5918ae2ee56b34e6f935dadfa4618a6d63 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Tue, 27 Aug 2024 14:59:24 +1200 Subject: [PATCH 5/5] test: Ensure Hasura wait condition works on both macos/linux --- runner/tests/testcontainers/hasura.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/tests/testcontainers/hasura.ts b/runner/tests/testcontainers/hasura.ts index a766231f0..98085149a 100644 --- a/runner/tests/testcontainers/hasura.ts +++ b/runner/tests/testcontainers/hasura.ts @@ -11,7 +11,7 @@ export class HasuraGraphQLContainer { constructor (private readonly container = new GenericContainer('hasura/graphql-engine:latest')) { container.withExposedPorts(this.PORT) - .withWaitStrategy(Wait.forLogMessage(/.*starting API server.*/)) + .withWaitStrategy(Wait.forLogMessage(/.*Starting API server.*/i)) .withLogConsumer(logConsumer) .withStartupTimeout(120_000); }