From c1e2fb2aab0ea00284f48f7a9abb6cf6df34659e Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Thu, 27 Jul 2023 11:02:56 +1200 Subject: [PATCH 1/6] DPLT-1049 Provision separate DB per user (#144) --- .github/workflows/deploy-lambdas.yml | 5 + .../__snapshots__/hasura-client.test.js.snap | 120 ++++-- .../__snapshots__/provisioner.test.js.snap | 18 + indexer-js-queue-handler/hasura-client.js | 105 ++++-- .../hasura-client.test.js | 86 ++++- indexer-js-queue-handler/indexer.js | 6 +- indexer-js-queue-handler/indexer.test.js | 31 +- indexer-js-queue-handler/package-lock.json | 148 ++++++++ indexer-js-queue-handler/package.json | 2 + indexer-js-queue-handler/pg-client.js | 30 ++ indexer-js-queue-handler/provisioner.js | 141 +++++-- indexer-js-queue-handler/provisioner.test.js | 356 ++++++++++-------- .../scripts/migrate-schema-to-db.js | 97 +++++ indexer-js-queue-handler/serverless.yml | 11 +- 14 files changed, 879 insertions(+), 277 deletions(-) create mode 100644 indexer-js-queue-handler/__snapshots__/provisioner.test.js.snap create mode 100644 indexer-js-queue-handler/pg-client.js create mode 100644 indexer-js-queue-handler/scripts/migrate-schema-to-db.js diff --git a/.github/workflows/deploy-lambdas.yml b/.github/workflows/deploy-lambdas.yml index 8ea809fb6..5a8879a12 100644 --- a/.github/workflows/deploy-lambdas.yml +++ b/.github/workflows/deploy-lambdas.yml @@ -37,3 +37,8 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} HASURA_ENDPOINT: ${{ vars.HASURA_ENDPOINT }} HASURA_ADMIN_SECRET: ${{ secrets.HASURA_ADMIN_SECRET }} + PG_ADMIN_USER: ${{ secrets.PG_ADMIN_USER }} + PG_ADMIN_PASSWORD: ${{ secrets.PG_ADMIN_PASSWORD }} + PG_ADMIN_DATABASE: ${{ secrets.PG_ADMIN_DATABASE }} + PG_HOST: ${{ secrets.PG_HOST }} + PG_PORT: ${{ secrets.PG_PORT }} diff --git a/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap b/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap index c5dccfbb8..803dab568 100644 --- a/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap +++ b/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap @@ -1,5 +1,35 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`HasuraClient adds a datasource 1`] = ` +{ + "args": { + "configuration": { + "connection_info": { + "database_url": { + "connection_parameters": { + "database": "morgs_near", + "host": "localhost", + "password": "password", + "port": 5432, + "username": "morgs_near", + }, + }, + }, + }, + "customization": { + "root_fields": { + "prefix": "morgs_near_", + }, + "type_names": { + "prefix": "morgs_near_", + }, + }, + "name": "morgs_near", + }, + "type": "pg_add_source", +} +`; + exports[`HasuraClient adds the specified permissions for the specified roles/table/schema 1`] = ` { "args": [ @@ -146,51 +176,66 @@ exports[`HasuraClient adds the specified permissions for the specified roles/tab } `; -exports[`HasuraClient checks if a schema exists 1`] = ` +exports[`HasuraClient checks if a schema exists within source 1`] = ` +[ + [ + "mock-hasura-endpoint/v2/query", + { + "body": "{"type":"run_sql","args":{"sql":"SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'schema'","read_only":true,"source":"source"}}", + "headers": { + "X-Hasura-Admin-Secret": "mock-hasura-admin-secret", + }, + "method": "POST", + }, + ], +] +`; + +exports[`HasuraClient checks if datasource exists 1`] = ` { - "args": { - "read_only": true, - "source": "default", - "sql": "SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'name'", - }, - "type": "run_sql", + "args": {}, + "type": "export_metadata", + "version": 2, } `; exports[`HasuraClient creates a schema 1`] = ` -{ - "args": { - "read_only": false, - "source": "default", - "sql": "CREATE schema name", - }, - "type": "run_sql", -} +[ + [ + "mock-hasura-endpoint/v2/query", + { + "body": "{"type":"run_sql","args":{"sql":"CREATE schema schemaName","read_only":false,"source":"dbName"}}", + "headers": { + "X-Hasura-Admin-Secret": "mock-hasura-admin-secret", + }, + "method": "POST", + }, + ], +] `; exports[`HasuraClient gets table names within a schema 1`] = ` { "args": { - "read_only": true, - "source": "default", - "sql": "SELECT table_name FROM information_schema.tables WHERE table_schema = 'schema'", + "source": "source", }, - "type": "run_sql", + "type": "pg_get_source_tables", } `; exports[`HasuraClient runs migrations for the specified schema 1`] = ` -{ - "args": { - "read_only": false, - "source": "default", - "sql": " - set schema 'schema'; - CREATE TABLE blocks (height numeric) - ", - }, - "type": "run_sql", -} +[ + [ + "mock-hasura-endpoint/v2/query", + { + "body": "{"type":"run_sql","args":{"sql":"\\n set schema 'schemaName';\\n CREATE TABLE blocks (height numeric)\\n ","read_only":false,"source":"dbName"}}", + "headers": { + "X-Hasura-Admin-Secret": "mock-hasura-admin-secret", + }, + "method": "POST", + }, + ], +] `; exports[`HasuraClient tracks foreign key relationships 1`] = ` @@ -346,3 +391,18 @@ exports[`HasuraClient tracks the specified tables for a specified schema 1`] = ` "type": "bulk", } `; + +exports[`HasuraClient untracks the specified tables 1`] = ` +[ + [ + "mock-hasura-endpoint/v1/metadata", + { + "body": "{"type":"pg_untrack_tables","args":{"tables":[{"table":{"schema":"schema","name":"height"},"source":"default","cascade":true},{"table":{"schema":"schema","name":"width"},"source":"default","cascade":true}]}}", + "headers": { + "X-Hasura-Admin-Secret": "mock-hasura-admin-secret", + }, + "method": "POST", + }, + ], +] +`; diff --git a/indexer-js-queue-handler/__snapshots__/provisioner.test.js.snap b/indexer-js-queue-handler/__snapshots__/provisioner.test.js.snap new file mode 100644 index 000000000..406ca3cc0 --- /dev/null +++ b/indexer-js-queue-handler/__snapshots__/provisioner.test.js.snap @@ -0,0 +1,18 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Provisioner provisionUserApi formats user input before executing the query 1`] = ` +[ + [ + "CREATE DATABASE "databaseName UNION SELECT * FROM users --"", + ], + [ + "CREATE USER morgs_near WITH PASSWORD 'pass; DROP TABLE users;--'", + ], + [ + "GRANT ALL PRIVILEGES ON DATABASE "databaseName UNION SELECT * FROM users --" TO morgs_near", + ], + [ + "REVOKE CONNECT ON DATABASE "databaseName UNION SELECT * FROM users --" FROM PUBLIC", + ], +] +`; diff --git a/indexer-js-queue-handler/hasura-client.js b/indexer-js-queue-handler/hasura-client.js index 318dc913d..9d7bba456 100644 --- a/indexer-js-queue-handler/hasura-client.js +++ b/indexer-js-queue-handler/hasura-client.js @@ -2,6 +2,9 @@ import fetch from 'node-fetch'; import pluralize from 'pluralize'; export default class HasuraClient { + static DEFAULT_DATABASE = 'default'; + static DEFAULT_SCHEMA = 'public'; + constructor( deps ) { @@ -22,7 +25,7 @@ export default class HasuraClient { args: { sql, read_only: opts.readOnly, - source: 'default', + source: opts.source || 'default', } }), }); @@ -36,7 +39,7 @@ export default class HasuraClient { return JSON.parse(body) }; - async executeMetadataRequest (type, args) { + async executeMetadataRequest (type, args, version) { const response = await this.deps.fetch(`${process.env.HASURA_ENDPOINT}/v1/metadata`, { method: 'POST', headers: { @@ -45,6 +48,7 @@ export default class HasuraClient { body: JSON.stringify({ type, args, + ...(version && { version }) }), }); @@ -61,46 +65,61 @@ export default class HasuraClient { return this.executeMetadataRequest('bulk', metadataRequests); } - async isSchemaCreated (schemaName) { + async exportMetadata() { + const { metadata } = await this.executeMetadataRequest('export_metadata', {}, 2); + return metadata; + } + + async doesSourceExist(source) { + const metadata = await this.exportMetadata(); + return metadata.sources.filter(({ name }) => name === source).length > 0; + } + + async doesSchemaExist(source, schemaName) { const { result } = await this.executeSql( `SELECT schema_name FROM information_schema.schemata WHERE schema_name = '${schemaName}'`, - { readOnly: true } + { source, readOnly: true } ); return result.length > 1; - }; + } - createSchema (schemaName) { + createSchema (source, schemaName) { return this.executeSql( `CREATE schema ${schemaName}`, - { readOnly: false } + { source, readOnly: false } ); } - runMigrations(schemaName, migration) { + runMigrations(source, schemaName, migration) { return this.executeSql( ` set schema '${schemaName}'; ${migration} `, - { readOnly: false } + { source, readOnly: false } ); } - async getTableNames(schemaName) { - const { result } = await this.executeSql( - `SELECT table_name FROM information_schema.tables WHERE table_schema = '${schemaName}'`, - { readOnly: true } + async getTableNames(schemaName, source) { + const tablesInSource = await this.executeMetadataRequest( + 'pg_get_source_tables', + { + source + } ); - const [_columnNames, ...tableNames] = result; - return tableNames.flat(); + + return tablesInSource + .filter(({ name, schema }) => schema === schemaName) + .map(({ name }) => name); }; - async trackTables(schemaName, tableNames) { + async trackTables(schemaName, tableNames, source) { return this.executeBulkMetadataRequest( tableNames.map((name) => ({ type: 'pg_track_table', args: { + source, table: { name, schema: schemaName, @@ -110,7 +129,20 @@ export default class HasuraClient { ); } - async getForeignKeys(schemaName) { + async untrackTables(source, schema, tableNames, cascade = true) { + return this.executeMetadataRequest('pg_untrack_tables', { + tables: tableNames.map((name) => ({ + table: { + schema, + name, + }, + source, + cascade, + })) + }); + } + + async getForeignKeys(schemaName, source) { const { result } = await this.executeSql( ` SELECT @@ -158,7 +190,7 @@ export default class HasuraClient { q.table_name, q.constraint_name) AS info; `, - { readOnly: true } + { readOnly: true, source } ); const [_, [foreignKeysJsonString]] = result; @@ -166,8 +198,8 @@ export default class HasuraClient { return JSON.parse(foreignKeysJsonString); } - async trackForeignKeyRelationships(schemaName) { - const foreignKeys = await this.getForeignKeys(schemaName); + async trackForeignKeyRelationships(schemaName, source) { + const foreignKeys = await this.getForeignKeys(schemaName, source); if (foreignKeys.length === 0) { return; @@ -179,6 +211,7 @@ export default class HasuraClient { { type: "pg_create_array_relationship", args: { + source, name: foreignKey.table_name, table: { name: foreignKey.ref_table, @@ -198,6 +231,7 @@ export default class HasuraClient { { type: "pg_create_object_relationship", args: { + source, name: pluralize.singular(foreignKey.ref_table), table: { name: foreignKey.table_name, @@ -213,13 +247,14 @@ export default class HasuraClient { ); } - async addPermissionsToTables(schemaName, tableNames, roleName, permissions) { + async addPermissionsToTables(schemaName, source, tableNames, roleName, permissions) { return this.executeBulkMetadataRequest( tableNames .map((tableName) => ( permissions.map((permission) => ({ type: `pg_create_${permission}_permission`, args: { + source, table: { name: tableName, schema: schemaName, @@ -232,11 +267,37 @@ export default class HasuraClient { filter: {}, ...(permission === "select" && { allow_aggregations: true }) }, - source: 'default' }, })) )) .flat() ); } + + async addDatasource(userName, password, databaseName) { + return this.executeMetadataRequest("pg_add_source", { + name: databaseName, + configuration: { + connection_info: { + database_url: { + connection_parameters: { + password, + database: databaseName, + username: userName, + host: process.env.PG_HOST, + port: Number(process.env.PG_PORT), + } + }, + }, + }, + customization: { + root_fields: { + prefix: `${userName}_`, + }, + type_names: { + prefix: `${userName}_`, + }, + }, + }); + } } diff --git a/indexer-js-queue-handler/hasura-client.test.js b/indexer-js-queue-handler/hasura-client.test.js index 150f846b9..bb5f3940c 100644 --- a/indexer-js-queue-handler/hasura-client.test.js +++ b/indexer-js-queue-handler/hasura-client.test.js @@ -7,12 +7,16 @@ describe('HasuraClient', () => { const HASURA_ENDPOINT = 'mock-hasura-endpoint'; const HASURA_ADMIN_SECRET = 'mock-hasura-admin-secret'; + const PG_HOST = 'localhost' + const PG_PORT = 5432 beforeAll(() => { process.env = { ...oldEnv, HASURA_ENDPOINT, HASURA_ADMIN_SECRET, + PG_HOST, + PG_PORT, }; }); @@ -29,13 +33,12 @@ describe('HasuraClient', () => { }); const client = new HasuraClient({ fetch }) - await client.createSchema('name'); + await client.createSchema('dbName', 'schemaName'); - expect(fetch.mock.calls[0][1].headers['X-Hasura-Admin-Secret']).toBe(HASURA_ADMIN_SECRET) - expect(JSON.parse(fetch.mock.calls[0][1].body)).toMatchSnapshot(); + expect(fetch.mock.calls).toMatchSnapshot(); }); - it('checks if a schema exists', async () => { + it('checks if a schema exists within source', async () => { const fetch = jest .fn() .mockResolvedValue({ @@ -46,12 +49,33 @@ describe('HasuraClient', () => { }); const client = new HasuraClient({ fetch }) - const result = await client.isSchemaCreated('name'); + const result = await client.doesSchemaExist('source', 'schema'); expect(result).toBe(true); + expect(fetch.mock.calls).toMatchSnapshot(); + }); + + it('checks if datasource exists', async () => { + const fetch = jest + .fn() + .mockResolvedValue({ + status: 200, + text: () => JSON.stringify({ + metadata: { + sources: [ + { + name: 'name' + } + ] + }, + }), + }); + const client = new HasuraClient({ fetch }) + + await expect(client.doesSourceExist('name')).resolves.toBe(true); expect(fetch.mock.calls[0][1].headers['X-Hasura-Admin-Secret']).toBe(HASURA_ADMIN_SECRET) expect(JSON.parse(fetch.mock.calls[0][1].body)).toMatchSnapshot(); - }); + }) it('runs migrations for the specified schema', async () => { const fetch = jest @@ -62,10 +86,9 @@ describe('HasuraClient', () => { }); const client = new HasuraClient({ fetch }) - await client.runMigrations('schema', 'CREATE TABLE blocks (height numeric)'); + await client.runMigrations('dbName', 'schemaName', 'CREATE TABLE blocks (height numeric)'); - expect(fetch.mock.calls[0][1].headers['X-Hasura-Admin-Secret']).toBe(HASURA_ADMIN_SECRET) - expect(JSON.parse(fetch.mock.calls[0][1].body)).toMatchSnapshot(); + expect(fetch.mock.calls).toMatchSnapshot(); }); it('gets table names within a schema', async () => { @@ -73,17 +96,15 @@ describe('HasuraClient', () => { .fn() .mockResolvedValue({ status: 200, - text: () => JSON.stringify({ - result: [ - ['table_name'], - ['height'], - ['width'] - ] - }) + text: () => JSON.stringify([ + { name: 'table_name', schema: 'morgs_near' }, + { name: 'height', schema: 'schema' }, + { name: 'width', schema: 'schema' } + ]) }); const client = new HasuraClient({ fetch }) - const names = await client.getTableNames('schema'); + const names = await client.getTableNames('schema', 'source'); expect(names).toEqual(['height', 'width']); expect(fetch.mock.calls[0][1].headers['X-Hasura-Admin-Secret']).toBe(HASURA_ADMIN_SECRET) @@ -105,6 +126,20 @@ describe('HasuraClient', () => { expect(JSON.parse(fetch.mock.calls[0][1].body)).toMatchSnapshot(); }); + it('untracks the specified tables', async () => { + const fetch = jest + .fn() + .mockResolvedValue({ + status: 200, + text: () => JSON.stringify({}) + }); + const client = new HasuraClient({ fetch }) + + await client.untrackTables('default', 'schema', ['height', 'width']); + + expect(fetch.mock.calls).toMatchSnapshot(); + }); + it('adds the specified permissions for the specified roles/table/schema', async () => { const fetch = jest .fn() @@ -114,7 +149,22 @@ describe('HasuraClient', () => { }); const client = new HasuraClient({ fetch }) - await client.addPermissionsToTables('schema', ['height', 'width'], 'role', ['select', 'insert', 'update', 'delete']); + await client.addPermissionsToTables('schema', 'default', ['height', 'width'], 'role', ['select', 'insert', 'update', 'delete']); + + expect(fetch.mock.calls[0][1].headers['X-Hasura-Admin-Secret']).toBe(HASURA_ADMIN_SECRET) + expect(JSON.parse(fetch.mock.calls[0][1].body)).toMatchSnapshot(); + }); + + it('adds a datasource', async () => { + const fetch = jest + .fn() + .mockResolvedValue({ + status: 200, + text: () => JSON.stringify({}) + }); + const client = new HasuraClient({ fetch }) + + await client.addDatasource('morgs_near', 'password', 'morgs_near'); expect(fetch.mock.calls[0][1].headers['X-Hasura-Admin-Secret']).toBe(HASURA_ADMIN_SECRET) expect(JSON.parse(fetch.mock.calls[0][1].body)).toMatchSnapshot(); diff --git a/indexer-js-queue-handler/indexer.js b/indexer-js-queue-handler/indexer.js index 7ecb37464..8103ff6e1 100644 --- a/indexer-js-queue-handler/indexer.js +++ b/indexer-js-queue-handler/indexer.js @@ -52,14 +52,12 @@ export default class Indexer { const functionNameWithoutAccount = function_name.split('/')[1].replace(/[.-]/g, '_'); if (options.provision && !indexerFunction["provisioned"]) { - const schemaName = `${function_name.replace(/[.\/-]/g, '_')}` - try { - if (!await this.deps.provisioner.doesEndpointExist(schemaName)) { + if (!await this.deps.provisioner.isUserApiProvisioned(indexerFunction.account_id, indexerFunction.function_name)) { await this.setStatus(function_name, block_height, 'PROVISIONING'); simultaneousPromises.push(this.writeLog(function_name, block_height, 'Provisioning endpoint: starting')); - await this.deps.provisioner.createAuthenticatedEndpoint(schemaName, hasuraRoleName, indexerFunction.schema); + await this.deps.provisioner.provisionUserApi(indexerFunction.account_id, indexerFunction.function_name, indexerFunction.schema); simultaneousPromises.push(this.writeLog(function_name, block_height, 'Provisioning endpoint: successful')); } diff --git a/indexer-js-queue-handler/indexer.test.js b/indexer-js-queue-handler/indexer.test.js index daea27a65..18322215e 100644 --- a/indexer-js-queue-handler/indexer.test.js +++ b/indexer-js-queue-handler/indexer.test.js @@ -589,23 +589,26 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in }), }; const provisioner = { - doesEndpointExist: jest.fn().mockReturnValue(false), - createAuthenticatedEndpoint: jest.fn(), + isUserApiProvisioned: jest.fn().mockReturnValue(false), + provisionUserApi: jest.fn(), } const indexer = new Indexer('mainnet', { fetch: mockFetch, s3: mockS3, provisioner, awsXray: mockAwsXray, metrics: mockMetrics }); const functions = { 'morgs.near/test': { + account_id: 'morgs.near', + function_name: 'test', code: '', schema: 'schema', } }; await indexer.runFunctions(1, functions, false, { provision: true }); - expect(provisioner.createAuthenticatedEndpoint).toHaveBeenCalledTimes(1); - expect(provisioner.createAuthenticatedEndpoint).toHaveBeenCalledWith( - 'morgs_near_test', - 'morgs_near', + expect(provisioner.isUserApiProvisioned).toHaveBeenCalledWith('morgs.near', 'test'); + expect(provisioner.provisionUserApi).toHaveBeenCalledTimes(1); + expect(provisioner.provisionUserApi).toHaveBeenCalledWith( + 'morgs.near', + 'test', 'schema' ) }); @@ -644,8 +647,8 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in }), }; const provisioner = { - doesEndpointExist: jest.fn().mockReturnValue(true), - createAuthenticatedEndpoint: jest.fn(), + isUserApiProvisioned: jest.fn().mockReturnValue(true), + provisionUserApi: jest.fn(), } const indexer = new Indexer('mainnet', { fetch: mockFetch, s3: mockS3, provisioner, awsXray: mockAwsXray, metrics: mockMetrics }); @@ -657,7 +660,7 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in }; await indexer.runFunctions(1, functions, false, { provision: true }); - expect(provisioner.createAuthenticatedEndpoint).not.toHaveBeenCalled(); + expect(provisioner.provisionUserApi).not.toHaveBeenCalled(); }); test('Indexer.runFunctions() supplies the required role to the GraphQL endpoint', async () => { @@ -694,8 +697,8 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in }), }; const provisioner = { - doesEndpointExist: jest.fn().mockReturnValue(true), - createAuthenticatedEndpoint: jest.fn(), + isUserApiProvisioned: jest.fn().mockReturnValue(true), + provisionUserApi: jest.fn(), } const indexer = new Indexer('mainnet', { fetch: mockFetch, s3: mockS3, provisioner, awsXray: mockAwsXray, metrics: mockMetrics }); @@ -709,7 +712,7 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in }; await indexer.runFunctions(blockHeight, functions, false, { provision: true }); - expect(provisioner.createAuthenticatedEndpoint).not.toHaveBeenCalled(); + expect(provisioner.provisionUserApi).not.toHaveBeenCalled(); expect(mockFetch.mock.calls).toMatchSnapshot(); }); @@ -748,8 +751,8 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in }; const error = new Error('something went wrong with provisioning'); const provisioner = { - doesEndpointExist: jest.fn().mockReturnValue(false), - createAuthenticatedEndpoint: jest.fn().mockRejectedValue(error), + isUserApiProvisioned: jest.fn().mockReturnValue(false), + provisionUserApi: jest.fn().mockRejectedValue(error), } const indexer = new Indexer('mainnet', { fetch: mockFetch, s3: mockS3, provisioner, awsXray: mockAwsXray, metrics: mockMetrics }); diff --git a/indexer-js-queue-handler/package-lock.json b/indexer-js-queue-handler/package-lock.json index 2f0ba2ad1..b055f6e01 100644 --- a/indexer-js-queue-handler/package-lock.json +++ b/indexer-js-queue-handler/package-lock.json @@ -14,6 +14,8 @@ "aws-xray-sdk": "^3.5.0", "near-api-js": "1.1.0", "node-fetch": "^3.3.0", + "pg": "^8.11.1", + "pg-format": "^1.0.4", "pluralize": "^8.0.0", "verror": "^1.10.1", "vm2": "^3.9.13" @@ -2452,6 +2454,14 @@ "dev": true, "license": "MIT" }, + "node_modules/buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", + "engines": { + "node": ">=4" + } + }, "node_modules/builtin-modules": { "version": "3.3.0", "dev": true, @@ -6388,6 +6398,11 @@ "node": ">=6" } }, + "node_modules/packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "node_modules/pako": { "version": "1.0.11", "dev": true, @@ -6512,6 +6527,53 @@ "license": "MIT", "peer": true }, + "node_modules/pg": { + "version": "8.11.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.1.tgz", + "integrity": "sha512-utdq2obft07MxaDg0zBJI+l/M3mBRfIpEN3iSemsz0G5F2/VXx+XzqF4oxrbIZXQxt2AZzIUzyVg/YM6xOP/WQ==", + "dependencies": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "^2.6.1", + "pg-pool": "^3.6.1", + "pg-protocol": "^1.6.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + }, + "engines": { + "node": ">= 8.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.1.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", + "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz", + "integrity": "sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg==" + }, + "node_modules/pg-format": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/pg-format/-/pg-format-1.0.4.tgz", + "integrity": "sha512-YyKEF78pEA6wwTAqOUaHIN/rWpfzzIuMh9KdAhc3rSLQ/7zkRFcCgYBAEGatDstLyZw4g0s9SNICmaTGnBVeyw==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", @@ -6520,6 +6582,14 @@ "node": ">=4.0.0" } }, + "node_modules/pg-pool": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", + "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "peerDependencies": { + "pg": ">=8.0" + } + }, "node_modules/pg-protocol": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", @@ -6540,6 +6610,22 @@ "node": ">=4" } }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/pgpass/node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/picocolors": { "version": "1.0.0", "dev": true, @@ -10046,6 +10132,11 @@ "version": "1.1.2", "dev": true }, + "buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" + }, "builtin-modules": { "version": "3.3.0", "dev": true, @@ -12701,6 +12792,11 @@ "version": "2.2.0", "dev": true }, + "packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "pako": { "version": "1.0.11", "dev": true, @@ -12786,11 +12882,48 @@ "dev": true, "peer": true }, + "pg": { + "version": "8.11.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.1.tgz", + "integrity": "sha512-utdq2obft07MxaDg0zBJI+l/M3mBRfIpEN3iSemsz0G5F2/VXx+XzqF4oxrbIZXQxt2AZzIUzyVg/YM6xOP/WQ==", + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-cloudflare": "^1.1.1", + "pg-connection-string": "^2.6.1", + "pg-pool": "^3.6.1", + "pg-protocol": "^1.6.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + } + }, + "pg-cloudflare": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", + "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "optional": true + }, + "pg-connection-string": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz", + "integrity": "sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg==" + }, + "pg-format": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/pg-format/-/pg-format-1.0.4.tgz", + "integrity": "sha512-YyKEF78pEA6wwTAqOUaHIN/rWpfzzIuMh9KdAhc3rSLQ/7zkRFcCgYBAEGatDstLyZw4g0s9SNICmaTGnBVeyw==" + }, "pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" }, + "pg-pool": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", + "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "requires": {} + }, "pg-protocol": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", @@ -12808,6 +12941,21 @@ "postgres-interval": "^1.1.0" } }, + "pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "requires": { + "split2": "^4.1.0" + }, + "dependencies": { + "split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==" + } + } + }, "picocolors": { "version": "1.0.0", "dev": true diff --git a/indexer-js-queue-handler/package.json b/indexer-js-queue-handler/package.json index 10901dafc..0ff9c7b24 100644 --- a/indexer-js-queue-handler/package.json +++ b/indexer-js-queue-handler/package.json @@ -18,6 +18,8 @@ "aws-xray-sdk": "^3.5.0", "near-api-js": "1.1.0", "node-fetch": "^3.3.0", + "pg": "^8.11.1", + "pg-format": "^1.0.4", "pluralize": "^8.0.0", "verror": "^1.10.1", "vm2": "^3.9.13" diff --git a/indexer-js-queue-handler/pg-client.js b/indexer-js-queue-handler/pg-client.js new file mode 100644 index 000000000..2d822e4d2 --- /dev/null +++ b/indexer-js-queue-handler/pg-client.js @@ -0,0 +1,30 @@ +import pg from "pg"; +import pgFormatModule from "pg-format"; + +export default class PgClient { + constructor( + connectionParams, + poolConfig = { max: 10, idleTimeoutMillis: 30000 }, + pgPool = pg.Pool, + pgFormat = pgFormatModule + ) { + this.pgPool = new pgPool({ + user: connectionParams.user, + password: connectionParams.password, + host: connectionParams.host, + port: connectionParams.port, + database: connectionParams.database, + ...poolConfig, + }); + this.format = pgFormat; + } + + async query(query, params = []) { + const client = await this.pgPool.connect(); + try { + await client.query(query, params); + } finally { + client.release(); + } + } +} diff --git a/indexer-js-queue-handler/provisioner.js b/indexer-js-queue-handler/provisioner.js index b4d9ca9a6..37d839707 100644 --- a/indexer-js-queue-handler/provisioner.js +++ b/indexer-js-queue-handler/provisioner.js @@ -1,91 +1,182 @@ -import VError from 'verror'; +import VError from "verror"; +import cryptoModule from "crypto"; -import HasuraClient from './hasura-client.js'; +import HasuraClient from "./hasura-client.js"; +import PgClient from './pg-client.js' + +const DEFAULT_PASSWORD_LENGTH = 16; + +const sharedPgClient = new PgClient({ + user: process.env.PG_ADMIN_USER, + password: process.env.PG_ADMIN_PASSWORD, + database: process.env.PG_ADMIN_DATABASE, + host: process.env.PG_HOST, + port: process.env.PG_PORT, +}); export default class Provisioner { constructor( hasuraClient = new HasuraClient(), + pgClient = sharedPgClient, + crypto = cryptoModule, ) { this.hasuraClient = hasuraClient; + this.pgClient = pgClient; + this.crypto = crypto; + } + + generatePassword(length = DEFAULT_PASSWORD_LENGTH) { + return this.crypto + .randomBytes(length) + .toString('base64') + .slice(0,length) + .replace(/\+/g, '0') + .replace(/\//g, '0'); + } + + async createDatabase(name) { + await this.pgClient.query(this.pgClient.format('CREATE DATABASE %I', name)); + } + + async createUser(name, password) { + await this.pgClient.query(this.pgClient.format(`CREATE USER %I WITH PASSWORD %L`, name, password)) } - doesEndpointExist(schemaName) { - return this.hasuraClient.isSchemaCreated(schemaName); + async restrictUserToDatabase(databaseName, userName) { + await this.pgClient.query(this.pgClient.format('GRANT ALL PRIVILEGES ON DATABASE %I TO %I', databaseName, userName)); + await this.pgClient.query(this.pgClient.format('REVOKE CONNECT ON DATABASE %I FROM PUBLIC', databaseName)); } - async createSchema(schemaName) { + async createUserDb(userName, password, databaseName) { try { - await this.hasuraClient.createSchema(schemaName); + await this.createDatabase(databaseName); + await this.createUser(userName, password); + await this.restrictUserToDatabase(databaseName, userName); + } catch (error) { + throw new VError(error, `Failed to create user db`); + } + } + + async isUserApiProvisioned(accountId, functionName) { + const databaseName = this.replaceSpecialChars(accountId); + const schemaName = this.replaceSpecialChars(functionName); + + const sourceExists = await this.hasuraClient.doesSourceExist(databaseName); + if (!sourceExists) { + return false; + } + + const schemaExists = await this.hasuraClient.doesSchemaExist(databaseName, schemaName); + + return schemaExists; + } + + async createSchema(databaseName, schemaName) { + try { + await this.hasuraClient.createSchema(databaseName, schemaName); } catch (error) { throw new VError(error, `Failed to create schema`); } } - async runMigrations(schemaName, migration) { + async runMigrations(databaseName, schemaName, migration) { try { - await this.hasuraClient.runMigrations(schemaName, migration); + await this.hasuraClient.runMigrations(databaseName, schemaName, migration); } catch (error) { throw new VError(error, `Failed to run migrations`); } } - async getTableNames(schemaName) { + async getTableNames(schemaName, databaseName) { try { - return await this.hasuraClient.getTableNames(schemaName); + return await this.hasuraClient.getTableNames(schemaName, databaseName); } catch (error) { throw new VError(error, `Failed to fetch table names`); } } - async trackTables(schemaName, tableNames) { + async trackTables(schemaName, tableNames, databaseName) { try { - await this.hasuraClient.trackTables(schemaName, tableNames); + await this.hasuraClient.trackTables(schemaName, tableNames, databaseName); } catch (error) { throw new VError(error, `Failed to track tables`); } } - async addPermissionsToTables(schemaName, tableNames, roleName, permissions) { + async addPermissionsToTables(schemaName, databaseName, tableNames, roleName, permissions) { try { await this.hasuraClient.addPermissionsToTables( schemaName, + databaseName, tableNames, roleName, - ['select', 'insert', 'update', 'delete'] + permissions ); } catch (error) { throw new VError(error, `Failed to add permissions to tables`); } } - async trackForeignKeyRelationships(schemaName) { + async trackForeignKeyRelationships(schemaName, databaseName) { try { - await this.hasuraClient.trackForeignKeyRelationships(schemaName); + await this.hasuraClient.trackForeignKeyRelationships(schemaName, databaseName); } catch (error) { throw new VError(error, `Failed to track foreign key relationships`); } } - async createAuthenticatedEndpoint(schemaName, roleName, migration) { + async addDatasource(userName, password, databaseName) { + try { + await this.hasuraClient.addDatasource(userName, password, databaseName); + } catch (error) { + throw new VError(error, `Failed to add datasource`); + } + } + + replaceSpecialChars(str) { + return str.replaceAll(/[.-]/g, '_') + } + + async provisionUserApi(accountId, functionName, databaseSchema) { + const sanitizedAccountId = this.replaceSpecialChars(accountId); + const sanitizedFunctionName = this.replaceSpecialChars(functionName); + + const databaseName = sanitizedAccountId; + const userName = sanitizedAccountId; + const schemaName = sanitizedFunctionName; + try { - await this.createSchema(schemaName); + if (!await this.hasuraClient.doesSourceExist(databaseName)) { + const password = this.generatePassword() + await this.createUserDb(userName, password, databaseName); + await this.addDatasource(userName, password, databaseName); + } + + // Untrack tables from old schema to prevent conflicts with new DB + const oldSchemaName = `${sanitizedAccountId}_${sanitizedFunctionName}`; + if (await this.hasuraClient.doesSchemaExist(HasuraClient.DEFAULT_DATABASE, oldSchemaName)) { + const tableNames = await this.getTableNames(oldSchemaName, HasuraClient.DEFAULT_DATABASE); + await this.hasuraClient.untrackTables(HasuraClient.DEFAULT_DATABASE, oldSchemaName, tableNames); + } - await this.runMigrations(schemaName, migration); + await this.createSchema(databaseName, schemaName); + await this.runMigrations(databaseName, schemaName, databaseSchema); - const tableNames = await this.getTableNames(schemaName); - await this.trackTables(schemaName, tableNames); + const tableNames = await this.getTableNames(schemaName, databaseName); + await this.trackTables(schemaName, tableNames, databaseName); - await this.trackForeignKeyRelationships(schemaName); + await this.trackForeignKeyRelationships(schemaName, databaseName); - await this.addPermissionsToTables(schemaName, tableNames, roleName, ['select', 'insert', 'update', 'delete']); + await this.addPermissionsToTables(schemaName, databaseName, tableNames, userName, ['select', 'insert', 'update', 'delete']); } catch (error) { throw new VError( { cause: error, info: { schemaName, - roleName, - migration, + userName, + databaseSchema, + databaseName, } }, `Failed to provision endpoint` diff --git a/indexer-js-queue-handler/provisioner.test.js b/indexer-js-queue-handler/provisioner.test.js index 11cd7a014..cb8d998de 100644 --- a/indexer-js-queue-handler/provisioner.test.js +++ b/indexer-js-queue-handler/provisioner.test.js @@ -1,181 +1,215 @@ import { jest } from '@jest/globals'; -import VError from 'verror' +import VError from 'verror'; +import pgFormat from 'pg-format'; import HasuraClient from './hasura-client'; import Provisioner from './provisioner'; -describe('Provision', () => { - it('checks if the endpoint already exists', async () => { - const provisioner = new Provisioner({ - isSchemaCreated: jest.fn().mockResolvedValueOnce(true) - }); - - expect(await provisioner.doesEndpointExist('schema')).toBe(true); - }); - - it('creates an authenticated endpoint', async () => { - const tableNames = ['blocks']; - const hasuraClient = { - createSchema: jest.fn().mockReturnValueOnce(), - runMigrations: jest.fn().mockReturnValueOnce(), +describe('Provisioner', () => { + let pgClient; + let hasuraClient; + + const tableNames = ['blocks']; + const accountId = 'morgs.near'; + const sanitizedAccountId = 'morgs_near'; + const functionName = 'test-function'; + const sanitizedFunctionName = 'test_function'; + const databaseSchema = 'CREATE TABLE blocks (height numeric)'; + const error = new Error('some error'); + const defaultDatabase = 'default'; + const oldSchemaName = `${sanitizedAccountId}_${sanitizedFunctionName}`; + + const password = 'password'; + const crypto = { + randomBytes: () => ({ + toString: () => ({ + slice: () => ({ + replace: () => password, + }), + }), + }), + }; + + beforeEach(() => { + hasuraClient = { getTableNames: jest.fn().mockReturnValueOnce(tableNames), trackTables: jest.fn().mockReturnValueOnce(), trackForeignKeyRelationships: jest.fn().mockReturnValueOnce(), addPermissionsToTables: jest.fn().mockReturnValueOnce(), + addDatasource: jest.fn().mockReturnValueOnce(), + runMigrations: jest.fn().mockReturnValueOnce(), + createSchema: jest.fn().mockReturnValueOnce(), + doesSourceExist: jest.fn().mockReturnValueOnce(false), + doesSchemaExist: jest.fn().mockReturnValueOnce(false), + untrackTables: jest.fn().mockReturnValueOnce(), }; - const provisioner = new Provisioner(hasuraClient); - - const schemaName = 'schema'; - const roleName = 'role'; - const migration = 'CREATE TABLE blocks (height numeric)'; - await provisioner.createAuthenticatedEndpoint(schemaName, roleName, migration); - - expect(hasuraClient.createSchema).toBeCalledWith(schemaName); - expect(hasuraClient.runMigrations).toBeCalledWith(schemaName, migration); - expect(hasuraClient.getTableNames).toBeCalledWith(schemaName); - expect(hasuraClient.trackTables).toBeCalledWith(schemaName, tableNames); - expect(hasuraClient.addPermissionsToTables).toBeCalledWith( - schemaName, - tableNames, - roleName, - [ - 'select', - 'insert', - 'update', - 'delete' - ] - ); - }); - it('throws an error when it fails to create the schema', async () => { - const error = new Error('some http error'); - const hasuraClient = { - createSchema: jest.fn().mockRejectedValue(error), + pgClient = { + query: jest.fn().mockResolvedValue(), + format: pgFormat, }; - const provisioner = new Provisioner(hasuraClient); - - try { - await provisioner.createAuthenticatedEndpoint('name', 'role', 'CREATE TABLE blocks (height numeric)') - } catch (error) { - expect(error.message).toBe('Failed to provision endpoint: Failed to create schema: some http error'); - expect(VError.info(error)).toEqual({ - schemaName: 'name', - roleName: 'role', - migration: 'CREATE TABLE blocks (height numeric)', - }); - } }); - it('throws an error when it fails to run migrations', async () => { - const error = new Error('some http error'); - const hasuraClient = { - createSchema: jest.fn().mockReturnValueOnce(), - runMigrations: jest.fn().mockRejectedValue(error), - }; - const provisioner = new Provisioner(hasuraClient); - - try { - await provisioner.createAuthenticatedEndpoint('name', 'role', 'CREATE TABLE blocks (height numeric)') - } catch (error) { - expect(error.message).toBe('Failed to provision endpoint: Failed to run migrations: some http error'); - expect(VError.info(error)).toEqual({ - schemaName: 'name', - roleName: 'role', - migration: 'CREATE TABLE blocks (height numeric)', - }); - } - }); + describe('isUserApiProvisioned', () => { + it('returns false if datasource doesnt exists', async () => { + hasuraClient.doesSourceExist = jest.fn().mockReturnValueOnce(false); - it('throws an error when it fails to fetch table names', async () => { - const error = new Error('some http error'); - const hasuraClient = { - createSchema: jest.fn().mockReturnValueOnce(), - runMigrations: jest.fn().mockReturnValueOnce(), - getTableNames: jest.fn().mockRejectedValue(error), - }; - const provisioner = new Provisioner(hasuraClient); - - try { - await provisioner.createAuthenticatedEndpoint('name', 'role', 'CREATE TABLE blocks (height numeric)') - } catch (error) { - expect(error.message).toBe('Failed to provision endpoint: Failed to fetch table names: some http error'); - expect(VError.info(error)).toEqual({ - schemaName: 'name', - roleName: 'role', - migration: 'CREATE TABLE blocks (height numeric)', - }); - } - }); + const provisioner = new Provisioner(hasuraClient, pgClient, crypto); - it('throws an error when it fails to track tables', async () => { - const error = new Error('some http error'); - const tableNames = ['blocks']; - const hasuraClient = { - createSchema: jest.fn().mockReturnValueOnce(), - runMigrations: jest.fn().mockReturnValueOnce(), - getTableNames: jest.fn().mockReturnValueOnce(tableNames), - trackTables: jest.fn().mockRejectedValueOnce(error), - }; - const provisioner = new Provisioner(hasuraClient); - - try { - await provisioner.createAuthenticatedEndpoint('name', 'role', 'CREATE TABLE blocks (height numeric)') - } catch (error) { - expect(error.message).toBe('Failed to provision endpoint: Failed to track tables: some http error'); - expect(VError.info(error)).toEqual({ - schemaName: 'name', - roleName: 'role', - migration: 'CREATE TABLE blocks (height numeric)', - }); - } + await expect(provisioner.isUserApiProvisioned(accountId, functionName)).resolves.toBe(false); + }); + + it('returns false if datasource and schema dont exists', async () => { + hasuraClient.doesSourceExist = jest.fn().mockReturnValueOnce(false); + hasuraClient.doesSchemaExist = jest.fn().mockReturnValueOnce(false); + + const provisioner = new Provisioner(hasuraClient, pgClient, crypto); + + await expect(provisioner.isUserApiProvisioned(accountId, functionName)).resolves.toBe(false); + }); + + it('returns true if datasource and schema exists', async () => { + hasuraClient.doesSourceExist = jest.fn().mockReturnValueOnce(true); + hasuraClient.doesSchemaExist = jest.fn().mockReturnValueOnce(true); + + const provisioner = new Provisioner(hasuraClient, pgClient, crypto); + + await expect(provisioner.isUserApiProvisioned(accountId, functionName)).resolves.toBe(true); + }); }); - it('throws an error when it fails to track foreign key relationships', async () => { - const error = new Error('some http error'); - const tableNames = ['blocks']; - const hasuraClient = { - createSchema: jest.fn().mockReturnValueOnce(), - runMigrations: jest.fn().mockReturnValueOnce(), - getTableNames: jest.fn().mockReturnValueOnce(tableNames), - trackTables: jest.fn().mockReturnValueOnce(), - trackForeignKeyRelationships: jest.fn().mockRejectedValueOnce(error), - }; - const provisioner = new Provisioner(hasuraClient); - - try { - await provisioner.createAuthenticatedEndpoint('name', 'role', 'CREATE TABLE blocks (height numeric)') - } catch (error) { - expect(error.message).toBe('Failed to provision endpoint: Failed to track foreign key relationships: some http error'); - expect(VError.info(error)).toEqual({ - schemaName: 'name', - roleName: 'role', - migration: 'CREATE TABLE blocks (height numeric)', - }); - } - }) - - it('throws an error when it fails to add permissions to tables', async () => { - const error = new Error('some http error'); - const tableNames = ['blocks']; - const hasuraClient = { - createSchema: jest.fn().mockReturnValueOnce(), - runMigrations: jest.fn().mockReturnValueOnce(), - getTableNames: jest.fn().mockReturnValueOnce(tableNames), - trackTables: jest.fn().mockReturnValueOnce(), - trackForeignKeyRelationships: jest.fn().mockReturnValueOnce(), - addPermissionsToTables: jest.fn().mockRejectedValue(error), - }; - const provisioner = new Provisioner(hasuraClient); - - try { - await provisioner.createAuthenticatedEndpoint('name', 'role', 'CREATE TABLE blocks (height numeric)') - } catch (error) { - expect(error.message).toBe('Failed to provision endpoint: Failed to add permissions to tables: some http error'); - expect(VError.info(error)).toEqual({ - migration: 'CREATE TABLE blocks (height numeric)', - schemaName: 'name', - roleName: 'role', - }); - } + describe('provisionUserApi', () => { + it('provisions an API for the user', async () => { + const provisioner = new Provisioner(hasuraClient, pgClient, crypto); + + await provisioner.provisionUserApi(accountId, functionName, databaseSchema); + + expect(pgClient.query.mock.calls).toEqual([ + ['CREATE DATABASE morgs_near'], + ['CREATE USER morgs_near WITH PASSWORD \'password\''], + ['GRANT ALL PRIVILEGES ON DATABASE morgs_near TO morgs_near'], + ['REVOKE CONNECT ON DATABASE morgs_near FROM PUBLIC'], + ]); + expect(hasuraClient.addDatasource).toBeCalledWith(sanitizedAccountId, password, sanitizedAccountId); + expect(hasuraClient.createSchema).toBeCalledWith(sanitizedAccountId, sanitizedFunctionName); + expect(hasuraClient.runMigrations).toBeCalledWith(sanitizedAccountId, sanitizedFunctionName, databaseSchema); + expect(hasuraClient.getTableNames).toBeCalledWith(sanitizedFunctionName, sanitizedAccountId); + expect(hasuraClient.trackTables).toBeCalledWith(sanitizedFunctionName, tableNames, sanitizedAccountId); + expect(hasuraClient.addPermissionsToTables).toBeCalledWith( + sanitizedFunctionName, + sanitizedAccountId, + tableNames, + sanitizedAccountId, + [ + 'select', + 'insert', + 'update', + 'delete' + ] + ); + }); + + it('untracks tables from the previous schema if they exists', async () => { + hasuraClient.doesSchemaExist = jest.fn().mockReturnValueOnce(true); + + const provisioner = new Provisioner(hasuraClient, pgClient, crypto); + + await provisioner.provisionUserApi(accountId, functionName, databaseSchema); + + expect(hasuraClient.getTableNames).toBeCalledWith(oldSchemaName, defaultDatabase) + expect(hasuraClient.untrackTables).toBeCalledWith(defaultDatabase, oldSchemaName, tableNames); + }); + + it('skips provisioning the datasource if it already exists', async () => { + hasuraClient.doesSourceExist = jest.fn().mockReturnValueOnce(true); + + const provisioner = new Provisioner(hasuraClient, pgClient, crypto); + + await provisioner.provisionUserApi(accountId, functionName, databaseSchema); + + expect(pgClient.query).not.toBeCalled(); + expect(hasuraClient.addDatasource).not.toBeCalled(); + + expect(hasuraClient.createSchema).toBeCalledWith(sanitizedAccountId, sanitizedFunctionName); + expect(hasuraClient.runMigrations).toBeCalledWith(sanitizedAccountId, sanitizedFunctionName, databaseSchema); + expect(hasuraClient.getTableNames).toBeCalledWith(sanitizedFunctionName, sanitizedAccountId); + expect(hasuraClient.trackTables).toBeCalledWith(sanitizedFunctionName, tableNames, sanitizedAccountId); + expect(hasuraClient.addPermissionsToTables).toBeCalledWith( + sanitizedFunctionName, + sanitizedAccountId, + tableNames, + sanitizedAccountId, + [ + 'select', + 'insert', + 'update', + 'delete' + ] + ); + }); + + it('formats user input before executing the query', async () => { + const provisioner = new Provisioner(hasuraClient, pgClient, crypto); + + await provisioner.createUserDb('morgs_near', 'pass; DROP TABLE users;--', 'databaseName UNION SELECT * FROM users --'); + + expect(pgClient.query.mock.calls).toMatchSnapshot(); + }); + + it('throws an error when it fails to create a postgres db', async () => { + pgClient.query = jest.fn().mockRejectedValue(error); + + const provisioner = new Provisioner(hasuraClient, pgClient, crypto); + + await expect(provisioner.provisionUserApi(accountId, functionName, databaseSchema)).rejects.toThrow('Failed to provision endpoint: Failed to create user db: some error'); + }); + + it('throws an error when it fails to add the db to hasura', async () => { + hasuraClient.addDatasource = jest.fn().mockRejectedValue(error); + + const provisioner = new Provisioner(hasuraClient, pgClient, crypto); + + await expect(provisioner.provisionUserApi(accountId, functionName, databaseSchema)).rejects.toThrow('Failed to provision endpoint: Failed to add datasource: some error'); + }); + + it('throws an error when it fails to run migrations', async () => { + hasuraClient.runMigrations = jest.fn().mockRejectedValue(error); + + const provisioner = new Provisioner(hasuraClient, pgClient, crypto); + + await expect(provisioner.provisionUserApi(accountId, functionName, databaseSchema)).rejects.toThrow('Failed to provision endpoint: Failed to run migrations: some error'); + }); + + it('throws an error when it fails to fetch table names', async () => { + hasuraClient.getTableNames = jest.fn().mockRejectedValue(error); + + const provisioner = new Provisioner(hasuraClient, pgClient, crypto); + + await expect(provisioner.provisionUserApi(accountId, functionName, databaseSchema)).rejects.toThrow('Failed to provision endpoint: Failed to fetch table names: some error'); + }); + + it('throws an error when it fails to track tables', async () => { + hasuraClient.trackTables = jest.fn().mockRejectedValue(error); + + const provisioner = new Provisioner(hasuraClient, pgClient, crypto); + + await expect(provisioner.provisionUserApi(accountId, functionName, databaseSchema)).rejects.toThrow('Failed to provision endpoint: Failed to track tables: some error'); + }); + + it('throws an error when it fails to track foreign key relationships', async () => { + hasuraClient.trackForeignKeyRelationships = jest.fn().mockRejectedValue(error); + + const provisioner = new Provisioner(hasuraClient, pgClient, crypto); + + await expect(provisioner.provisionUserApi(accountId, functionName, databaseSchema)).rejects.toThrow('Failed to provision endpoint: Failed to track foreign key relationships: some error'); + }) + + it('throws an error when it fails to add permissions to tables', async () => { + hasuraClient.addPermissionsToTables = jest.fn().mockRejectedValue(error); + + const provisioner = new Provisioner(hasuraClient, pgClient, crypto); + + await expect(provisioner.provisionUserApi(accountId, functionName, databaseSchema)).rejects.toThrow('Failed to provision endpoint: Failed to add permissions to tables: some error'); + }); }); }) diff --git a/indexer-js-queue-handler/scripts/migrate-schema-to-db.js b/indexer-js-queue-handler/scripts/migrate-schema-to-db.js new file mode 100644 index 000000000..50e32e9f8 --- /dev/null +++ b/indexer-js-queue-handler/scripts/migrate-schema-to-db.js @@ -0,0 +1,97 @@ +process.env.HASURA_ENDPOINT = 'http://localhost:8080' +process.env.HASURA_ADMIN_SECRET = 'myadminsecretkey' + +process.env.PG_ADMIN_USER = 'postgres' +process.env.PG_ADMIN_PASSWORD = 'postgrespassword' +process.env.PG_ADMIN_DATABASE = 'postgres' +process.env.PG_HOST = 'localhost' +process.env.PG_PORT = 5432 + +process.env.CHAIN_ID = 'mainnet' +process.env.ENV = 'dev' + +import { execSync } from 'child_process' +import { providers } from 'near-api-js' + +import Provisioner from '../provisioner.js' +import HasuraClient from '../hasura-client.js' + +const provisioner = new Provisioner(); + +if (!process.argv[2]) { + console.error('Please pass the account ID as the first argument, e.g. dataplatform.near'); + process.exit(1); +} + +if (!process.argv[3]) { + console.error('Please pass the function name as the second argument, e.g. social_feed') + process.exit(1); +} + +const [_, __, accountId, functionName] = process.argv; + +console.log(`Processing account: ${accountId}, function: ${functionName}`); + +const provider = new providers.JsonRpcProvider( + `https://rpc.${process.env.CHAIN_ID}.near.org` +); + +console.log('Fetching existing schema'); +const { result: rawResult } = await provider.query({ + request_type: 'call_function', + account_id: `${process.env.ENV === 'prod' ? '' : 'dev-'}queryapi.dataplatform.near`, + method_name: 'list_indexer_functions', + args_base64: Buffer.from(JSON.stringify({ account_id: accountId})).toString('base64'), + finality: 'optimistic', +}); + +const result = JSON.parse(Buffer.from(rawResult).toString()); + +const { schema: databaseSchema } = result.Account[functionName]; +console.log('Using schema: ', databaseSchema); + +const sanitizedAccountId = provisioner.replaceSpecialChars(accountId); +const sanitizedFunctionName = provisioner.replaceSpecialChars(functionName); + +const databaseName = sanitizedAccountId; +const userName = sanitizedAccountId; +const schemaName = sanitizedFunctionName; + +const existingSchemaName = `${sanitizedAccountId}_${sanitizedFunctionName}`; + +if (!await provisioner.hasuraClient.doesSourceExist(databaseName)) { + const password = provisioner.generatePassword() + console.log(`Creating user: ${userName} and database: ${databaseName} with password: ${password}`); + await provisioner.createUserDb(userName, password, databaseName); + console.log('Adding datasource to Hasura') + await provisioner.addDatasource(userName, password, databaseName); +} + +const tableNames = await provisioner.getTableNames(existingSchemaName, HasuraClient.DEFAULT_DATABASE); + +console.log('Untracking existing tables') +await provisioner.hasuraClient.untrackTables(HasuraClient.DEFAULT_DATABASE, existingSchemaName, tableNames); + +console.log(`Restoring existing schema ${existingSchemaName} in new DB ${databaseName}`); +await provisioner.createSchema(databaseName, existingSchemaName); +await provisioner.runMigrations(databaseName, existingSchemaName, databaseSchema); + +console.log('Dumping existing data'); +execSync(`pg_dump --data-only --schema=${existingSchemaName} --file="${existingSchemaName}.sql"`); + +console.log(`Restoring data to schema ${existingSchemaName} in DB ${databaseName}`); +execSync(`psql --dbname=${databaseName} < "${existingSchemaName}.sql"`); + +console.log(`Renaming schema ${existingSchemaName} to ${schemaName}`); +execSync(`psql --dbname=${databaseName} --command="ALTER SCHEMA \"${existingSchemaName}\" RENAME TO \"${schemaName}\";"`) + +console.log('Tracking tables'); +await provisioner.trackTables(schemaName, tableNames, databaseName); + +console.log('Tracking foreign key relationships'); +await provisioner.trackForeignKeyRelationships(schemaName, databaseName); + +console.log('Adding permissions to tables'); +await provisioner.addPermissionsToTables(schemaName, databaseName, tableNames, userName, ['select', 'insert', 'update', 'delete']); + +console.log('done') diff --git a/indexer-js-queue-handler/serverless.yml b/indexer-js-queue-handler/serverless.yml index 368400cc7..e24a522c2 100644 --- a/indexer-js-queue-handler/serverless.yml +++ b/indexer-js-queue-handler/serverless.yml @@ -7,12 +7,17 @@ provider: name: aws runtime: nodejs16.x region: eu-central-1 - timeout: 15 + timeout: 120 environment: REGION: ${self:provider.region} STAGE: ${opt:stage, 'dev'} HASURA_ENDPOINT: ${env:HASURA_ENDPOINT} HASURA_ADMIN_SECRET: ${env:HASURA_ADMIN_SECRET} + PG_ADMIN_USER: ${env:PG_ADMIN_USER} + PG_ADMIN_PASSWORD: ${env:PG_ADMIN_PASSWORD} + PG_ADMIN_DATABASE: ${env:PG_ADMIN_DATABASE} + PG_HOST: ${env:PG_HOST} + PG_PORT: ${env:PG_PORT} tracing: lambda: true #enable X-Ray tracing iamRoleStatements: @@ -30,7 +35,7 @@ constructs: maxRetries: 1 worker: handler: handler.consumer - timeout: 15 # 1.5 minutes as lift multiplies this value by 6 (https://github.com/getlift/lift/blob/master/docs/queue.md#retry-delay) + timeout: 120 # 12 minutes as lift multiplies this value by 6 (https://github.com/getlift/lift/blob/master/docs/queue.md#retry-delay) startFromBlock-runner: type: queue fifo: true @@ -38,7 +43,7 @@ constructs: # batchSize: 100 worker: handler: handler.consumer - timeout: 15 # 1.5 minutes as lift multiplies this value by 6 (https://github.com/getlift/lift/blob/master/docs/queue.md#retry-delay) + timeout: 120 # 12 minutes as lift multiplies this value by 6 (https://github.com/getlift/lift/blob/master/docs/queue.md#retry-delay) functions: socialLagMetricsWriter: From 466e78735bb2784f51806d5719e5eca40a03d11d Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Thu, 27 Jul 2023 12:05:10 +1200 Subject: [PATCH 2/6] fix: Separate user db automated & script provisioning (#151) --- .../__snapshots__/hasura-client.test.js.snap | 2 +- indexer-js-queue-handler/hasura-client.js | 21 +++++++----- .../scripts/migrate-schema-to-db.js | 34 +++++++++++-------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap b/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap index 803dab568..21beb1116 100644 --- a/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap +++ b/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap @@ -397,7 +397,7 @@ exports[`HasuraClient untracks the specified tables 1`] = ` [ "mock-hasura-endpoint/v1/metadata", { - "body": "{"type":"pg_untrack_tables","args":{"tables":[{"table":{"schema":"schema","name":"height"},"source":"default","cascade":true},{"table":{"schema":"schema","name":"width"},"source":"default","cascade":true}]}}", + "body": "{"type":"bulk","args":[{"type":"pg_untrack_table","args":{"table":{"schema":"schema","name":"height"},"source":"default","cascade":true}},{"type":"pg_untrack_table","args":{"table":{"schema":"schema","name":"width"},"source":"default","cascade":true}}]}", "headers": { "X-Hasura-Admin-Secret": "mock-hasura-admin-secret", }, diff --git a/indexer-js-queue-handler/hasura-client.js b/indexer-js-queue-handler/hasura-client.js index 9d7bba456..0e5f0cbc1 100644 --- a/indexer-js-queue-handler/hasura-client.js +++ b/indexer-js-queue-handler/hasura-client.js @@ -130,16 +130,19 @@ export default class HasuraClient { } async untrackTables(source, schema, tableNames, cascade = true) { - return this.executeMetadataRequest('pg_untrack_tables', { - tables: tableNames.map((name) => ({ - table: { - schema, - name, - }, - source, - cascade, + return this.executeBulkMetadataRequest( + tableNames.map((name) => ({ + type: 'pg_untrack_table', + args: { + table: { + schema, + name, + }, + source, + cascade, + } })) - }); + ); } async getForeignKeys(schemaName, source) { diff --git a/indexer-js-queue-handler/scripts/migrate-schema-to-db.js b/indexer-js-queue-handler/scripts/migrate-schema-to-db.js index 50e32e9f8..3ad255676 100644 --- a/indexer-js-queue-handler/scripts/migrate-schema-to-db.js +++ b/indexer-js-queue-handler/scripts/migrate-schema-to-db.js @@ -1,14 +1,12 @@ -process.env.HASURA_ENDPOINT = 'http://localhost:8080' -process.env.HASURA_ADMIN_SECRET = 'myadminsecretkey' - -process.env.PG_ADMIN_USER = 'postgres' -process.env.PG_ADMIN_PASSWORD = 'postgrespassword' -process.env.PG_ADMIN_DATABASE = 'postgres' -process.env.PG_HOST = 'localhost' -process.env.PG_PORT = 5432 - -process.env.CHAIN_ID = 'mainnet' -process.env.ENV = 'dev' +// export HASURA_ENDPOINT='https://queryapi-hasura-graphql-vcqilefdcq-ew.a.run.app' +// export HASURA_ADMIN_SECRET='' +// export PG_ADMIN_USER='hasura' +// export PG_ADMIN_PASSWORD='' +// export PG_ADMIN_DATABASE='postgres' +// export PG_HOST='' +// export PG_PORT=5432 +// export CHAIN_ID='mainnet' +// export ENV='dev' import { execSync } from 'child_process' import { providers } from 'near-api-js' @@ -59,8 +57,8 @@ const schemaName = sanitizedFunctionName; const existingSchemaName = `${sanitizedAccountId}_${sanitizedFunctionName}`; +const password = provisioner.generatePassword() if (!await provisioner.hasuraClient.doesSourceExist(databaseName)) { - const password = provisioner.generatePassword() console.log(`Creating user: ${userName} and database: ${databaseName} with password: ${password}`); await provisioner.createUserDb(userName, password, databaseName); console.log('Adding datasource to Hasura') @@ -77,13 +75,19 @@ await provisioner.createSchema(databaseName, existingSchemaName); await provisioner.runMigrations(databaseName, existingSchemaName, databaseSchema); console.log('Dumping existing data'); -execSync(`pg_dump --data-only --schema=${existingSchemaName} --file="${existingSchemaName}.sql"`); +execSync( + `pg_dump ${`postgres://${process.env.PG_ADMIN_USER}:${process.env.PG_ADMIN_PASSWORD}@${process.env.PG_HOST}:${process.env.PG_PORT}/${process.env.PG_ADMIN_DATABASE}`} --data-only --schema=${existingSchemaName} --file="${existingSchemaName}.sql"` +); console.log(`Restoring data to schema ${existingSchemaName} in DB ${databaseName}`); -execSync(`psql --dbname=${databaseName} < "${existingSchemaName}.sql"`); +execSync( + `psql ${`postgres://${userName}:${password}@${process.env.PG_HOST}:${process.env.PG_PORT}/${databaseName}`} < "${existingSchemaName}.sql"` +); console.log(`Renaming schema ${existingSchemaName} to ${schemaName}`); -execSync(`psql --dbname=${databaseName} --command="ALTER SCHEMA \"${existingSchemaName}\" RENAME TO \"${schemaName}\";"`) +execSync( + `psql ${`postgres://${userName}:${password}@${process.env.PG_HOST}:${process.env.PG_PORT}/${databaseName}`} --command="ALTER SCHEMA \"${existingSchemaName}\" RENAME TO \"${schemaName}\";"` +) console.log('Tracking tables'); await provisioner.trackTables(schemaName, tableNames, databaseName); From d61a707f42b528c2622aac42bf6b4b6f64a3a5bb Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Thu, 27 Jul 2023 12:40:52 +1200 Subject: [PATCH 3/6] fix: Provision same named schema to ensure same named mutation (#152) --- .../__snapshots__/hasura-client.test.js.snap | 8 ------ indexer-js-queue-handler/hasura-client.js | 8 ------ indexer-js-queue-handler/provisioner.js | 9 +++---- indexer-js-queue-handler/provisioner.test.js | 26 +++++++++---------- .../scripts/migrate-schema-to-db.js | 25 +++++++----------- 5 files changed, 26 insertions(+), 50 deletions(-) diff --git a/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap b/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap index 21beb1116..0da09c522 100644 --- a/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap +++ b/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap @@ -16,14 +16,6 @@ exports[`HasuraClient adds a datasource 1`] = ` }, }, }, - "customization": { - "root_fields": { - "prefix": "morgs_near_", - }, - "type_names": { - "prefix": "morgs_near_", - }, - }, "name": "morgs_near", }, "type": "pg_add_source", diff --git a/indexer-js-queue-handler/hasura-client.js b/indexer-js-queue-handler/hasura-client.js index 0e5f0cbc1..956e9d656 100644 --- a/indexer-js-queue-handler/hasura-client.js +++ b/indexer-js-queue-handler/hasura-client.js @@ -293,14 +293,6 @@ export default class HasuraClient { }, }, }, - customization: { - root_fields: { - prefix: `${userName}_`, - }, - type_names: { - prefix: `${userName}_`, - }, - }, }); } } diff --git a/indexer-js-queue-handler/provisioner.js b/indexer-js-queue-handler/provisioner.js index 37d839707..df75e6a6f 100644 --- a/indexer-js-queue-handler/provisioner.js +++ b/indexer-js-queue-handler/provisioner.js @@ -143,7 +143,7 @@ export default class Provisioner { const databaseName = sanitizedAccountId; const userName = sanitizedAccountId; - const schemaName = sanitizedFunctionName; + const schemaName = `${sanitizedAccountId}_${sanitizedFunctionName}`; try { if (!await this.hasuraClient.doesSourceExist(databaseName)) { @@ -153,10 +153,9 @@ export default class Provisioner { } // Untrack tables from old schema to prevent conflicts with new DB - const oldSchemaName = `${sanitizedAccountId}_${sanitizedFunctionName}`; - if (await this.hasuraClient.doesSchemaExist(HasuraClient.DEFAULT_DATABASE, oldSchemaName)) { - const tableNames = await this.getTableNames(oldSchemaName, HasuraClient.DEFAULT_DATABASE); - await this.hasuraClient.untrackTables(HasuraClient.DEFAULT_DATABASE, oldSchemaName, tableNames); + if (await this.hasuraClient.doesSchemaExist(HasuraClient.DEFAULT_DATABASE, schemaName)) { + const tableNames = await this.getTableNames(schemaName, HasuraClient.DEFAULT_DATABASE); + await this.hasuraClient.untrackTables(HasuraClient.DEFAULT_DATABASE, schemaName, tableNames); } await this.createSchema(databaseName, schemaName); diff --git a/indexer-js-queue-handler/provisioner.test.js b/indexer-js-queue-handler/provisioner.test.js index cb8d998de..59c5ed199 100644 --- a/indexer-js-queue-handler/provisioner.test.js +++ b/indexer-js-queue-handler/provisioner.test.js @@ -17,7 +17,7 @@ describe('Provisioner', () => { const databaseSchema = 'CREATE TABLE blocks (height numeric)'; const error = new Error('some error'); const defaultDatabase = 'default'; - const oldSchemaName = `${sanitizedAccountId}_${sanitizedFunctionName}`; + const schemaName = `${sanitizedAccountId}_${sanitizedFunctionName}`; const password = 'password'; const crypto = { @@ -91,12 +91,12 @@ describe('Provisioner', () => { ['REVOKE CONNECT ON DATABASE morgs_near FROM PUBLIC'], ]); expect(hasuraClient.addDatasource).toBeCalledWith(sanitizedAccountId, password, sanitizedAccountId); - expect(hasuraClient.createSchema).toBeCalledWith(sanitizedAccountId, sanitizedFunctionName); - expect(hasuraClient.runMigrations).toBeCalledWith(sanitizedAccountId, sanitizedFunctionName, databaseSchema); - expect(hasuraClient.getTableNames).toBeCalledWith(sanitizedFunctionName, sanitizedAccountId); - expect(hasuraClient.trackTables).toBeCalledWith(sanitizedFunctionName, tableNames, sanitizedAccountId); + expect(hasuraClient.createSchema).toBeCalledWith(sanitizedAccountId, schemaName); + expect(hasuraClient.runMigrations).toBeCalledWith(sanitizedAccountId, schemaName, databaseSchema); + expect(hasuraClient.getTableNames).toBeCalledWith(schemaName, sanitizedAccountId); + expect(hasuraClient.trackTables).toBeCalledWith(schemaName, tableNames, sanitizedAccountId); expect(hasuraClient.addPermissionsToTables).toBeCalledWith( - sanitizedFunctionName, + schemaName, sanitizedAccountId, tableNames, sanitizedAccountId, @@ -116,8 +116,8 @@ describe('Provisioner', () => { await provisioner.provisionUserApi(accountId, functionName, databaseSchema); - expect(hasuraClient.getTableNames).toBeCalledWith(oldSchemaName, defaultDatabase) - expect(hasuraClient.untrackTables).toBeCalledWith(defaultDatabase, oldSchemaName, tableNames); + expect(hasuraClient.getTableNames).toBeCalledWith(schemaName, defaultDatabase) + expect(hasuraClient.untrackTables).toBeCalledWith(defaultDatabase, schemaName, tableNames); }); it('skips provisioning the datasource if it already exists', async () => { @@ -130,12 +130,12 @@ describe('Provisioner', () => { expect(pgClient.query).not.toBeCalled(); expect(hasuraClient.addDatasource).not.toBeCalled(); - expect(hasuraClient.createSchema).toBeCalledWith(sanitizedAccountId, sanitizedFunctionName); - expect(hasuraClient.runMigrations).toBeCalledWith(sanitizedAccountId, sanitizedFunctionName, databaseSchema); - expect(hasuraClient.getTableNames).toBeCalledWith(sanitizedFunctionName, sanitizedAccountId); - expect(hasuraClient.trackTables).toBeCalledWith(sanitizedFunctionName, tableNames, sanitizedAccountId); + expect(hasuraClient.createSchema).toBeCalledWith(sanitizedAccountId, schemaName); + expect(hasuraClient.runMigrations).toBeCalledWith(sanitizedAccountId, schemaName, databaseSchema); + expect(hasuraClient.getTableNames).toBeCalledWith(schemaName, sanitizedAccountId); + expect(hasuraClient.trackTables).toBeCalledWith(schemaName, tableNames, sanitizedAccountId); expect(hasuraClient.addPermissionsToTables).toBeCalledWith( - sanitizedFunctionName, + schemaName, sanitizedAccountId, tableNames, sanitizedAccountId, diff --git a/indexer-js-queue-handler/scripts/migrate-schema-to-db.js b/indexer-js-queue-handler/scripts/migrate-schema-to-db.js index 3ad255676..de5415155 100644 --- a/indexer-js-queue-handler/scripts/migrate-schema-to-db.js +++ b/indexer-js-queue-handler/scripts/migrate-schema-to-db.js @@ -53,9 +53,7 @@ const sanitizedFunctionName = provisioner.replaceSpecialChars(functionName); const databaseName = sanitizedAccountId; const userName = sanitizedAccountId; -const schemaName = sanitizedFunctionName; - -const existingSchemaName = `${sanitizedAccountId}_${sanitizedFunctionName}`; +const schemaName = `${sanitizedAccountId}_${sanitizedFunctionName}`; const password = provisioner.generatePassword() if (!await provisioner.hasuraClient.doesSourceExist(databaseName)) { @@ -65,30 +63,25 @@ if (!await provisioner.hasuraClient.doesSourceExist(databaseName)) { await provisioner.addDatasource(userName, password, databaseName); } -const tableNames = await provisioner.getTableNames(existingSchemaName, HasuraClient.DEFAULT_DATABASE); +const tableNames = await provisioner.getTableNames(schemaName, HasuraClient.DEFAULT_DATABASE); console.log('Untracking existing tables') -await provisioner.hasuraClient.untrackTables(HasuraClient.DEFAULT_DATABASE, existingSchemaName, tableNames); +await provisioner.hasuraClient.untrackTables(HasuraClient.DEFAULT_DATABASE, schemaName, tableNames); -console.log(`Restoring existing schema ${existingSchemaName} in new DB ${databaseName}`); -await provisioner.createSchema(databaseName, existingSchemaName); -await provisioner.runMigrations(databaseName, existingSchemaName, databaseSchema); +console.log(`Restoring existing schema ${schemaName} in new DB ${databaseName}`); +await provisioner.createSchema(databaseName, schemaName); +await provisioner.runMigrations(databaseName, schemaName, databaseSchema); console.log('Dumping existing data'); execSync( - `pg_dump ${`postgres://${process.env.PG_ADMIN_USER}:${process.env.PG_ADMIN_PASSWORD}@${process.env.PG_HOST}:${process.env.PG_PORT}/${process.env.PG_ADMIN_DATABASE}`} --data-only --schema=${existingSchemaName} --file="${existingSchemaName}.sql"` + `pg_dump ${`postgres://${process.env.PG_ADMIN_USER}:${process.env.PG_ADMIN_PASSWORD}@${process.env.PG_HOST}:${process.env.PG_PORT}/${process.env.PG_ADMIN_DATABASE}`} --data-only --schema=${schemaName} --file="${schemaName}.sql"` ); -console.log(`Restoring data to schema ${existingSchemaName} in DB ${databaseName}`); +console.log(`Restoring data to schema ${schemaName} in DB ${databaseName}`); execSync( - `psql ${`postgres://${userName}:${password}@${process.env.PG_HOST}:${process.env.PG_PORT}/${databaseName}`} < "${existingSchemaName}.sql"` + `psql ${`postgres://${userName}:${password}@${process.env.PG_HOST}:${process.env.PG_PORT}/${databaseName}`} < "${schemaName}.sql"` ); -console.log(`Renaming schema ${existingSchemaName} to ${schemaName}`); -execSync( - `psql ${`postgres://${userName}:${password}@${process.env.PG_HOST}:${process.env.PG_PORT}/${databaseName}`} --command="ALTER SCHEMA \"${existingSchemaName}\" RENAME TO \"${schemaName}\";"` -) - console.log('Tracking tables'); await provisioner.trackTables(schemaName, tableNames, databaseName); From 5493b04d6694b1c33b85ba4dd99088b257597bd9 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Thu, 27 Jul 2023 12:51:25 +1200 Subject: [PATCH 4/6] fix: Use correct schema name in is provisioned check (#153) --- indexer-js-queue-handler/provisioner.js | 7 +++++-- indexer-js-queue-handler/scripts/migrate-schema-to-db.js | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/indexer-js-queue-handler/provisioner.js b/indexer-js-queue-handler/provisioner.js index df75e6a6f..78704bcf6 100644 --- a/indexer-js-queue-handler/provisioner.js +++ b/indexer-js-queue-handler/provisioner.js @@ -58,8 +58,11 @@ export default class Provisioner { } async isUserApiProvisioned(accountId, functionName) { - const databaseName = this.replaceSpecialChars(accountId); - const schemaName = this.replaceSpecialChars(functionName); + const sanitizedAccountId = this.replaceSpecialChars(accountId); + const sanitizedFunctionName = this.replaceSpecialChars(functionName); + + const databaseName = sanitizedAccountId; + const schemaName = `${sanitizedAccountId}_${sanitizedFunctionName}`; const sourceExists = await this.hasuraClient.doesSourceExist(databaseName); if (!sourceExists) { diff --git a/indexer-js-queue-handler/scripts/migrate-schema-to-db.js b/indexer-js-queue-handler/scripts/migrate-schema-to-db.js index de5415155..b4b6a787e 100644 --- a/indexer-js-queue-handler/scripts/migrate-schema-to-db.js +++ b/indexer-js-queue-handler/scripts/migrate-schema-to-db.js @@ -1,9 +1,9 @@ -// export HASURA_ENDPOINT='https://queryapi-hasura-graphql-vcqilefdcq-ew.a.run.app' +// export HASURA_ENDPOINT='' // export HASURA_ADMIN_SECRET='' // export PG_ADMIN_USER='hasura' // export PG_ADMIN_PASSWORD='' // export PG_ADMIN_DATABASE='postgres' -// export PG_HOST='' +// export PG_HOST='104.199.4.194' // export PG_PORT=5432 // export CHAIN_ID='mainnet' // export ENV='dev' From ccd5dbcb96ba884b61b6ae3e8f6467f3a05a28cf Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Thu, 27 Jul 2023 16:57:01 -0500 Subject: [PATCH 5/6] DPLT-1068 feat: likes on comments (#149) --- .../examples/feed/src/QueryApi.Examples.Feed.Comment.jsx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.Comment.jsx b/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.Comment.jsx index 96429ca0d..937197f75 100644 --- a/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.Comment.jsx +++ b/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.Comment.jsx @@ -173,7 +173,7 @@ return ( {blockHeight !== "now" && ( Date: Fri, 28 Jul 2023 14:42:47 +1200 Subject: [PATCH 6/6] DPLT-1124 Create script to preemptively provision user DBs (#155) --- indexer-js-queue-handler/pg-client.js | 2 +- .../scripts/provision-user-dbs.js | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 indexer-js-queue-handler/scripts/provision-user-dbs.js diff --git a/indexer-js-queue-handler/pg-client.js b/indexer-js-queue-handler/pg-client.js index 2d822e4d2..5ed2cdd82 100644 --- a/indexer-js-queue-handler/pg-client.js +++ b/indexer-js-queue-handler/pg-client.js @@ -22,7 +22,7 @@ export default class PgClient { async query(query, params = []) { const client = await this.pgPool.connect(); try { - await client.query(query, params); + return await client.query(query, params); } finally { client.release(); } diff --git a/indexer-js-queue-handler/scripts/provision-user-dbs.js b/indexer-js-queue-handler/scripts/provision-user-dbs.js new file mode 100644 index 000000000..a9921ad1a --- /dev/null +++ b/indexer-js-queue-handler/scripts/provision-user-dbs.js @@ -0,0 +1,54 @@ +// export HASURA_ENDPOINT='' +// export HASURA_ADMIN_SECRET='' +// export PG_ADMIN_USER='' +// export PG_ADMIN_PASSWORD='' +// export PG_ADMIN_DATABASE='' +// export PG_HOST='' +// export PG_PORT= + +import { execSync } from 'child_process' +import { providers } from 'near-api-js' + +import Provisioner from '../provisioner.js' +import HasuraClient from '../hasura-client.js' + +const provisioner = new Provisioner(); + +const { rows } = await provisioner.pgClient.query('SELECT nspname AS name FROM pg_namespace;') + +const schemaNames = rows.map((row) => row.name); + +const accountIdsSet = schemaNames.reduce((accountIdsSet, schemaName) => { + const parts = schemaName.split('_near_'); + if (parts.length > 1) { + accountIdsSet.add(`${parts[0]}_near`); + } + return accountIdsSet; +}, new Set()); + +const accountIds = Array.from(accountIdsSet); + +console.log(`Creating datasources for accounts: ${accountIds.join(', ')}`) + +for (const accountId of accountIds) { + console.log('---'); + const sanitizedAccountId = provisioner.replaceSpecialChars(accountId); + + const databaseName = sanitizedAccountId; + const userName = sanitizedAccountId; + + if (await provisioner.hasuraClient.doesSourceExist(databaseName)) { + console.log(`Datasource ${databaseName} already exists, skipping.`) + continue; + } + + const password = provisioner.generatePassword() + console.log(`Creating user: ${userName} and database: ${databaseName} with password: ${password}`); + await provisioner.createUserDb(userName, password, databaseName); + + console.log(`Adding datasource ${databaseName} to Hasura`) + await provisioner.addDatasource(userName, password, databaseName); +} +console.log('---'); + +console.log('Done');