diff --git a/.github/workflows/deploy-lambdas.yml b/.github/workflows/deploy-lambdas.yml index 8ea809fb6..5a8879a12 100644 --- a/.github/workflows/deploy-lambdas.yml +++ b/.github/workflows/deploy-lambdas.yml @@ -37,3 +37,8 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} HASURA_ENDPOINT: ${{ vars.HASURA_ENDPOINT }} HASURA_ADMIN_SECRET: ${{ secrets.HASURA_ADMIN_SECRET }} + PG_ADMIN_USER: ${{ secrets.PG_ADMIN_USER }} + PG_ADMIN_PASSWORD: ${{ secrets.PG_ADMIN_PASSWORD }} + PG_ADMIN_DATABASE: ${{ secrets.PG_ADMIN_DATABASE }} + PG_HOST: ${{ secrets.PG_HOST }} + PG_PORT: ${{ secrets.PG_PORT }} diff --git a/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap b/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap index b93fa6907..7cb0837b1 100644 --- a/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap +++ b/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap @@ -1,5 +1,35 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`HasuraClient adds a datasource 1`] = ` +{ + "args": { + "configuration": { + "connection_info": { + "database_url": { + "connection_parameters": { + "database": "morgs_near", + "host": "localhost", + "password": "password", + "port": 5432, + "username": "morgs_near", + }, + }, + }, + }, + "customization": { + "root_fields": { + "namespace": "morgs_near", + }, + "type_names": { + "prefix": "morgs_near_", + }, + }, + "name": "morgs_near", + }, + "type": "pg_add_source", +} +`; + exports[`HasuraClient adds the specified permissions for the specified roles/table/schema 1`] = ` { "args": [ @@ -152,51 +182,66 @@ exports[`HasuraClient adds the specified permissions for the specified roles/tab } `; -exports[`HasuraClient checks if a schema exists 1`] = ` +exports[`HasuraClient checks if a schema exists within source 1`] = ` +[ + [ + "mock-hasura-endpoint/v2/query", + { + "body": "{"type":"run_sql","args":{"sql":"SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'schema'","read_only":true,"source":"source"}}", + "headers": { + "X-Hasura-Admin-Secret": "mock-hasura-admin-secret", + }, + "method": "POST", + }, + ], +] +`; + +exports[`HasuraClient checks if datasource exists 1`] = ` { - "args": { - "read_only": true, - "source": "default", - "sql": "SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'name'", - }, - "type": "run_sql", + "args": {}, + "type": "export_metadata", + "version": 2, } `; exports[`HasuraClient creates a schema 1`] = ` -{ - "args": { - "read_only": false, - "source": "default", - "sql": "CREATE schema name", - }, - "type": "run_sql", -} +[ + [ + "mock-hasura-endpoint/v2/query", + { + "body": "{"type":"run_sql","args":{"sql":"CREATE schema schemaName","read_only":false,"source":"dbName"}}", + "headers": { + "X-Hasura-Admin-Secret": "mock-hasura-admin-secret", + }, + "method": "POST", + }, + ], +] `; exports[`HasuraClient gets table names within a schema 1`] = ` { "args": { - "read_only": true, - "source": "default", - "sql": "SELECT table_name FROM information_schema.tables WHERE table_schema = 'schema'", + "source": "source", }, - "type": "run_sql", + "type": "pg_get_source_tables", } `; exports[`HasuraClient runs migrations for the specified schema 1`] = ` -{ - "args": { - "read_only": false, - "source": "default", - "sql": " - set schema 'schema'; - CREATE TABLE blocks (height numeric) - ", - }, - "type": "run_sql", -} +[ + [ + "mock-hasura-endpoint/v2/query", + { + "body": "{"type":"run_sql","args":{"sql":"\\n set schema 'schemaName';\\n CREATE TABLE blocks (height numeric)\\n ","read_only":false,"source":"dbName"}}", + "headers": { + "X-Hasura-Admin-Secret": "mock-hasura-admin-secret", + }, + "method": "POST", + }, + ], +] `; exports[`HasuraClient tracks foreign key relationships 1`] = ` diff --git a/indexer-js-queue-handler/__snapshots__/provisioner.test.js.snap b/indexer-js-queue-handler/__snapshots__/provisioner.test.js.snap new file mode 100644 index 000000000..d6e77a289 --- /dev/null +++ b/indexer-js-queue-handler/__snapshots__/provisioner.test.js.snap @@ -0,0 +1,22 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Provisioner provisionUserApi formats user input before executing the query 1`] = ` +[ + [ + "CREATE DATABASE "databaseName UNION SELECT * FROM users --"", + [], + ], + [ + "CREATE USER morgs_near WITH PASSWORD 'pass; DROP TABLE users;--'", + [], + ], + [ + "GRANT ALL PRIVILEGES ON DATABASE "databaseName UNION SELECT * FROM users --" TO morgs_near", + [], + ], + [ + "REVOKE CONNECT ON DATABASE "databaseName UNION SELECT * FROM users --" FROM PUBLIC", + [], + ], +] +`; diff --git a/indexer-js-queue-handler/hasura-client.js b/indexer-js-queue-handler/hasura-client.js index 395614aa1..2a51b2c79 100644 --- a/indexer-js-queue-handler/hasura-client.js +++ b/indexer-js-queue-handler/hasura-client.js @@ -22,7 +22,7 @@ export default class HasuraClient { args: { sql, read_only: opts.readOnly, - source: 'default', + source: opts.source || 'default', } }), }); @@ -36,7 +36,7 @@ export default class HasuraClient { return JSON.parse(body) }; - async executeMetadataRequest (type, args) { + async executeMetadataRequest (type, args, version) { const response = await this.deps.fetch(`${process.env.HASURA_ENDPOINT}/v1/metadata`, { method: 'POST', headers: { @@ -45,6 +45,7 @@ export default class HasuraClient { body: JSON.stringify({ type, args, + ...(version && { version }) }), }); @@ -61,46 +62,61 @@ export default class HasuraClient { return this.executeMetadataRequest('bulk', metadataRequests); } - async isSchemaCreated (schemaName) { + async exportMetadata() { + const { metadata } = await this.executeMetadataRequest('export_metadata', {}, 2); + return metadata; + } + + async doesSourceExist(source) { + const metadata = await this.exportMetadata(); + return metadata.sources.filter(({ name }) => name === source).length > 0; + } + + async doesSchemaExist(source, schemaName) { const { result } = await this.executeSql( `SELECT schema_name FROM information_schema.schemata WHERE schema_name = '${schemaName}'`, - { readOnly: true } + { source, readOnly: true } ); return result.length > 1; - }; + } - createSchema (schemaName) { + createSchema (source, schemaName) { return this.executeSql( `CREATE schema ${schemaName}`, - { readOnly: false } + { source, readOnly: false } ); } - runMigrations(schemaName, migration) { + runMigrations(source, schemaName, migration) { return this.executeSql( ` set schema '${schemaName}'; ${migration} `, - { readOnly: false } + { source, readOnly: false } ); } - async getTableNames(schemaName) { - const { result } = await this.executeSql( - `SELECT table_name FROM information_schema.tables WHERE table_schema = '${schemaName}'`, - { readOnly: true } + async getTableNames(schemaName, source) { + const tablesInSource = await this.executeMetadataRequest( + 'pg_get_source_tables', + { + source + } ); - const [_columnNames, ...tableNames] = result; - return tableNames.flat(); + + return tablesInSource + .filter(({ name, schema }) => schema === schemaName) + .map(({ name }) => name); }; - async trackTables(schemaName, tableNames) { + async trackTables(schemaName, tableNames, source) { return this.executeBulkMetadataRequest( tableNames.map((name) => ({ type: 'pg_track_table', args: { + source, table: { name, schema: schemaName, @@ -110,7 +126,7 @@ export default class HasuraClient { ); } - async getForeignKeys(schemaName) { + async getForeignKeys(schemaName, source) { const { result } = await this.executeSql( ` SELECT @@ -158,7 +174,7 @@ export default class HasuraClient { q.table_name, q.constraint_name) AS info; `, - { readOnly: true } + { readOnly: true, source } ); const [_, [foreignKeysJsonString]] = result; @@ -166,8 +182,8 @@ export default class HasuraClient { return JSON.parse(foreignKeysJsonString); } - async trackForeignKeyRelationships(schemaName) { - const foreignKeys = await this.getForeignKeys(schemaName); + async trackForeignKeyRelationships(schemaName, source) { + const foreignKeys = await this.getForeignKeys(schemaName, source); if (foreignKeys.length === 0) { return; @@ -179,6 +195,7 @@ export default class HasuraClient { { type: "pg_create_array_relationship", args: { + source, name: foreignKey.table_name, table: { name: foreignKey.ref_table, @@ -198,6 +215,7 @@ export default class HasuraClient { { type: "pg_create_object_relationship", args: { + source, name: pluralize.singular(foreignKey.ref_table), table: { name: foreignKey.table_name, @@ -213,13 +231,14 @@ export default class HasuraClient { ); } - async addPermissionsToTables(schemaName, tableNames, roleName, permissions) { + async addPermissionsToTables(schemaName, source, tableNames, roleName, permissions) { return this.executeBulkMetadataRequest( tableNames .map((tableName) => ( permissions.map((permission) => ({ type: `pg_create_${permission}_permission`, args: { + source, table: { name: tableName, schema: schemaName, @@ -234,11 +253,37 @@ export default class HasuraClient { ? { allow_aggregations: true } : { backend_only: true }), }, - source: 'default' }, })) )) .flat() ); } + + async addDatasource(userName, password, databaseName) { + return this.executeMetadataRequest("pg_add_source", { + name: databaseName, + configuration: { + connection_info: { + database_url: { + connection_parameters: { + password, + database: databaseName, + username: userName, + host: process.env.PG_HOST, + port: Number(process.env.PG_PORT), + } + }, + }, + }, + customization: { + root_fields: { + namespace: userName, + }, + type_names: { + prefix: `${userName}_`, + }, + }, + }); + } } diff --git a/indexer-js-queue-handler/hasura-client.test.js b/indexer-js-queue-handler/hasura-client.test.js index 150f846b9..f6f1b09d6 100644 --- a/indexer-js-queue-handler/hasura-client.test.js +++ b/indexer-js-queue-handler/hasura-client.test.js @@ -8,11 +8,22 @@ describe('HasuraClient', () => { const HASURA_ENDPOINT = 'mock-hasura-endpoint'; const HASURA_ADMIN_SECRET = 'mock-hasura-admin-secret'; + const PG_ADMIN_USER = 'postgres' + const PG_ADMIN_PASSWORD = 'postgrespassword' + const PG_ADMIN_DATABASE = 'postgres' + const PG_HOST = 'localhost' + const PG_PORT = 5432 + beforeAll(() => { process.env = { ...oldEnv, HASURA_ENDPOINT, HASURA_ADMIN_SECRET, + PG_ADMIN_USER, + PG_ADMIN_PASSWORD, + PG_ADMIN_DATABASE, + PG_HOST, + PG_PORT, }; }); @@ -29,13 +40,12 @@ describe('HasuraClient', () => { }); const client = new HasuraClient({ fetch }) - await client.createSchema('name'); + await client.createSchema('dbName', 'schemaName'); - expect(fetch.mock.calls[0][1].headers['X-Hasura-Admin-Secret']).toBe(HASURA_ADMIN_SECRET) - expect(JSON.parse(fetch.mock.calls[0][1].body)).toMatchSnapshot(); + expect(fetch.mock.calls).toMatchSnapshot(); }); - it('checks if a schema exists', async () => { + it('checks if a schema exists within source', async () => { const fetch = jest .fn() .mockResolvedValue({ @@ -46,12 +56,33 @@ describe('HasuraClient', () => { }); const client = new HasuraClient({ fetch }) - const result = await client.isSchemaCreated('name'); + const result = await client.doesSchemaExist('source', 'schema'); expect(result).toBe(true); + expect(fetch.mock.calls).toMatchSnapshot(); + }); + + it('checks if datasource exists', async () => { + const fetch = jest + .fn() + .mockResolvedValue({ + status: 200, + text: () => JSON.stringify({ + metadata: { + sources: [ + { + name: 'name' + } + ] + }, + }), + }); + const client = new HasuraClient({ fetch }) + + await expect(client.doesSourceExist('name')).resolves.toBe(true); expect(fetch.mock.calls[0][1].headers['X-Hasura-Admin-Secret']).toBe(HASURA_ADMIN_SECRET) expect(JSON.parse(fetch.mock.calls[0][1].body)).toMatchSnapshot(); - }); + }) it('runs migrations for the specified schema', async () => { const fetch = jest @@ -62,10 +93,9 @@ describe('HasuraClient', () => { }); const client = new HasuraClient({ fetch }) - await client.runMigrations('schema', 'CREATE TABLE blocks (height numeric)'); + await client.runMigrations('dbName', 'schemaName', 'CREATE TABLE blocks (height numeric)'); - expect(fetch.mock.calls[0][1].headers['X-Hasura-Admin-Secret']).toBe(HASURA_ADMIN_SECRET) - expect(JSON.parse(fetch.mock.calls[0][1].body)).toMatchSnapshot(); + expect(fetch.mock.calls).toMatchSnapshot(); }); it('gets table names within a schema', async () => { @@ -73,17 +103,15 @@ describe('HasuraClient', () => { .fn() .mockResolvedValue({ status: 200, - text: () => JSON.stringify({ - result: [ - ['table_name'], - ['height'], - ['width'] - ] - }) + text: () => JSON.stringify([ + { name: 'table_name', schema: 'morgs_near' }, + { name: 'height', schema: 'schema' }, + { name: 'width', schema: 'schema' } + ]) }); const client = new HasuraClient({ fetch }) - const names = await client.getTableNames('schema'); + const names = await client.getTableNames('schema', 'source'); expect(names).toEqual(['height', 'width']); expect(fetch.mock.calls[0][1].headers['X-Hasura-Admin-Secret']).toBe(HASURA_ADMIN_SECRET) @@ -114,7 +142,22 @@ describe('HasuraClient', () => { }); const client = new HasuraClient({ fetch }) - await client.addPermissionsToTables('schema', ['height', 'width'], 'role', ['select', 'insert', 'update', 'delete']); + await client.addPermissionsToTables('schema', 'default', ['height', 'width'], 'role', ['select', 'insert', 'update', 'delete']); + + expect(fetch.mock.calls[0][1].headers['X-Hasura-Admin-Secret']).toBe(HASURA_ADMIN_SECRET) + expect(JSON.parse(fetch.mock.calls[0][1].body)).toMatchSnapshot(); + }); + + it('adds a datasource', async () => { + const fetch = jest + .fn() + .mockResolvedValue({ + status: 200, + text: () => JSON.stringify({}) + }); + const client = new HasuraClient({ fetch }) + + await client.addDatasource('morgs_near', 'password', 'morgs_near'); expect(fetch.mock.calls[0][1].headers['X-Hasura-Admin-Secret']).toBe(HASURA_ADMIN_SECRET) expect(JSON.parse(fetch.mock.calls[0][1].body)).toMatchSnapshot(); diff --git a/indexer-js-queue-handler/indexer.js b/indexer-js-queue-handler/indexer.js index 7ecb37464..8103ff6e1 100644 --- a/indexer-js-queue-handler/indexer.js +++ b/indexer-js-queue-handler/indexer.js @@ -52,14 +52,12 @@ export default class Indexer { const functionNameWithoutAccount = function_name.split('/')[1].replace(/[.-]/g, '_'); if (options.provision && !indexerFunction["provisioned"]) { - const schemaName = `${function_name.replace(/[.\/-]/g, '_')}` - try { - if (!await this.deps.provisioner.doesEndpointExist(schemaName)) { + if (!await this.deps.provisioner.isUserApiProvisioned(indexerFunction.account_id, indexerFunction.function_name)) { await this.setStatus(function_name, block_height, 'PROVISIONING'); simultaneousPromises.push(this.writeLog(function_name, block_height, 'Provisioning endpoint: starting')); - await this.deps.provisioner.createAuthenticatedEndpoint(schemaName, hasuraRoleName, indexerFunction.schema); + await this.deps.provisioner.provisionUserApi(indexerFunction.account_id, indexerFunction.function_name, indexerFunction.schema); simultaneousPromises.push(this.writeLog(function_name, block_height, 'Provisioning endpoint: successful')); } diff --git a/indexer-js-queue-handler/indexer.test.js b/indexer-js-queue-handler/indexer.test.js index daea27a65..18322215e 100644 --- a/indexer-js-queue-handler/indexer.test.js +++ b/indexer-js-queue-handler/indexer.test.js @@ -589,23 +589,26 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in }), }; const provisioner = { - doesEndpointExist: jest.fn().mockReturnValue(false), - createAuthenticatedEndpoint: jest.fn(), + isUserApiProvisioned: jest.fn().mockReturnValue(false), + provisionUserApi: jest.fn(), } const indexer = new Indexer('mainnet', { fetch: mockFetch, s3: mockS3, provisioner, awsXray: mockAwsXray, metrics: mockMetrics }); const functions = { 'morgs.near/test': { + account_id: 'morgs.near', + function_name: 'test', code: '', schema: 'schema', } }; await indexer.runFunctions(1, functions, false, { provision: true }); - expect(provisioner.createAuthenticatedEndpoint).toHaveBeenCalledTimes(1); - expect(provisioner.createAuthenticatedEndpoint).toHaveBeenCalledWith( - 'morgs_near_test', - 'morgs_near', + expect(provisioner.isUserApiProvisioned).toHaveBeenCalledWith('morgs.near', 'test'); + expect(provisioner.provisionUserApi).toHaveBeenCalledTimes(1); + expect(provisioner.provisionUserApi).toHaveBeenCalledWith( + 'morgs.near', + 'test', 'schema' ) }); @@ -644,8 +647,8 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in }), }; const provisioner = { - doesEndpointExist: jest.fn().mockReturnValue(true), - createAuthenticatedEndpoint: jest.fn(), + isUserApiProvisioned: jest.fn().mockReturnValue(true), + provisionUserApi: jest.fn(), } const indexer = new Indexer('mainnet', { fetch: mockFetch, s3: mockS3, provisioner, awsXray: mockAwsXray, metrics: mockMetrics }); @@ -657,7 +660,7 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in }; await indexer.runFunctions(1, functions, false, { provision: true }); - expect(provisioner.createAuthenticatedEndpoint).not.toHaveBeenCalled(); + expect(provisioner.provisionUserApi).not.toHaveBeenCalled(); }); test('Indexer.runFunctions() supplies the required role to the GraphQL endpoint', async () => { @@ -694,8 +697,8 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in }), }; const provisioner = { - doesEndpointExist: jest.fn().mockReturnValue(true), - createAuthenticatedEndpoint: jest.fn(), + isUserApiProvisioned: jest.fn().mockReturnValue(true), + provisionUserApi: jest.fn(), } const indexer = new Indexer('mainnet', { fetch: mockFetch, s3: mockS3, provisioner, awsXray: mockAwsXray, metrics: mockMetrics }); @@ -709,7 +712,7 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in }; await indexer.runFunctions(blockHeight, functions, false, { provision: true }); - expect(provisioner.createAuthenticatedEndpoint).not.toHaveBeenCalled(); + expect(provisioner.provisionUserApi).not.toHaveBeenCalled(); expect(mockFetch.mock.calls).toMatchSnapshot(); }); @@ -748,8 +751,8 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in }; const error = new Error('something went wrong with provisioning'); const provisioner = { - doesEndpointExist: jest.fn().mockReturnValue(false), - createAuthenticatedEndpoint: jest.fn().mockRejectedValue(error), + isUserApiProvisioned: jest.fn().mockReturnValue(false), + provisionUserApi: jest.fn().mockRejectedValue(error), } const indexer = new Indexer('mainnet', { fetch: mockFetch, s3: mockS3, provisioner, awsXray: mockAwsXray, metrics: mockMetrics }); diff --git a/indexer-js-queue-handler/package-lock.json b/indexer-js-queue-handler/package-lock.json index 2f0ba2ad1..b055f6e01 100644 --- a/indexer-js-queue-handler/package-lock.json +++ b/indexer-js-queue-handler/package-lock.json @@ -14,6 +14,8 @@ "aws-xray-sdk": "^3.5.0", "near-api-js": "1.1.0", "node-fetch": "^3.3.0", + "pg": "^8.11.1", + "pg-format": "^1.0.4", "pluralize": "^8.0.0", "verror": "^1.10.1", "vm2": "^3.9.13" @@ -2452,6 +2454,14 @@ "dev": true, "license": "MIT" }, + "node_modules/buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", + "engines": { + "node": ">=4" + } + }, "node_modules/builtin-modules": { "version": "3.3.0", "dev": true, @@ -6388,6 +6398,11 @@ "node": ">=6" } }, + "node_modules/packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "node_modules/pako": { "version": "1.0.11", "dev": true, @@ -6512,6 +6527,53 @@ "license": "MIT", "peer": true }, + "node_modules/pg": { + "version": "8.11.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.1.tgz", + "integrity": "sha512-utdq2obft07MxaDg0zBJI+l/M3mBRfIpEN3iSemsz0G5F2/VXx+XzqF4oxrbIZXQxt2AZzIUzyVg/YM6xOP/WQ==", + "dependencies": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "^2.6.1", + "pg-pool": "^3.6.1", + "pg-protocol": "^1.6.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + }, + "engines": { + "node": ">= 8.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.1.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", + "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz", + "integrity": "sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg==" + }, + "node_modules/pg-format": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/pg-format/-/pg-format-1.0.4.tgz", + "integrity": "sha512-YyKEF78pEA6wwTAqOUaHIN/rWpfzzIuMh9KdAhc3rSLQ/7zkRFcCgYBAEGatDstLyZw4g0s9SNICmaTGnBVeyw==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", @@ -6520,6 +6582,14 @@ "node": ">=4.0.0" } }, + "node_modules/pg-pool": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", + "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "peerDependencies": { + "pg": ">=8.0" + } + }, "node_modules/pg-protocol": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", @@ -6540,6 +6610,22 @@ "node": ">=4" } }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/pgpass/node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/picocolors": { "version": "1.0.0", "dev": true, @@ -10046,6 +10132,11 @@ "version": "1.1.2", "dev": true }, + "buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" + }, "builtin-modules": { "version": "3.3.0", "dev": true, @@ -12701,6 +12792,11 @@ "version": "2.2.0", "dev": true }, + "packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "pako": { "version": "1.0.11", "dev": true, @@ -12786,11 +12882,48 @@ "dev": true, "peer": true }, + "pg": { + "version": "8.11.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.1.tgz", + "integrity": "sha512-utdq2obft07MxaDg0zBJI+l/M3mBRfIpEN3iSemsz0G5F2/VXx+XzqF4oxrbIZXQxt2AZzIUzyVg/YM6xOP/WQ==", + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-cloudflare": "^1.1.1", + "pg-connection-string": "^2.6.1", + "pg-pool": "^3.6.1", + "pg-protocol": "^1.6.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + } + }, + "pg-cloudflare": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", + "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "optional": true + }, + "pg-connection-string": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz", + "integrity": "sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg==" + }, + "pg-format": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/pg-format/-/pg-format-1.0.4.tgz", + "integrity": "sha512-YyKEF78pEA6wwTAqOUaHIN/rWpfzzIuMh9KdAhc3rSLQ/7zkRFcCgYBAEGatDstLyZw4g0s9SNICmaTGnBVeyw==" + }, "pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" }, + "pg-pool": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", + "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "requires": {} + }, "pg-protocol": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", @@ -12808,6 +12941,21 @@ "postgres-interval": "^1.1.0" } }, + "pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "requires": { + "split2": "^4.1.0" + }, + "dependencies": { + "split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==" + } + } + }, "picocolors": { "version": "1.0.0", "dev": true diff --git a/indexer-js-queue-handler/package.json b/indexer-js-queue-handler/package.json index 10901dafc..0ff9c7b24 100644 --- a/indexer-js-queue-handler/package.json +++ b/indexer-js-queue-handler/package.json @@ -18,6 +18,8 @@ "aws-xray-sdk": "^3.5.0", "near-api-js": "1.1.0", "node-fetch": "^3.3.0", + "pg": "^8.11.1", + "pg-format": "^1.0.4", "pluralize": "^8.0.0", "verror": "^1.10.1", "vm2": "^3.9.13" diff --git a/indexer-js-queue-handler/provisioner.js b/indexer-js-queue-handler/provisioner.js index b4d9ca9a6..6923b5ec5 100644 --- a/indexer-js-queue-handler/provisioner.js +++ b/indexer-js-queue-handler/provisioner.js @@ -1,91 +1,190 @@ -import VError from 'verror'; +import VError from "verror"; +import pg from "pg"; +import cryptoModule from "crypto"; +import pgFormatModule from "pg-format"; -import HasuraClient from './hasura-client.js'; +import HasuraClient from "./hasura-client.js"; + +const DEFAULT_PASSWORD_LENGTH = 16; +const DEFAULT_SCHEMA = 'public'; + +const pool = new pg.Pool({ + user: process.env.PG_ADMIN_USER, + password: process.env.PG_ADMIN_PASSWORD, + database: process.env.PG_ADMIN_DATABASE, + host: process.env.PG_HOST, + port: process.env.PG_PORT, + max: 10, + idleTimeoutMillis: 30000, +}); export default class Provisioner { constructor( hasuraClient = new HasuraClient(), + pgPool = pool, + crypto = cryptoModule, + pgFormat = pgFormatModule ) { this.hasuraClient = hasuraClient; + this.pgPool = pgPool; + this.crypto = crypto; + this.pgFormat = pgFormat; + } + + async query(query, params = []) { + const client = await this.pgPool.connect(); + try { + await client.query(query, params); + } finally { + client.release(); + } + } + + generatePassword(length = DEFAULT_PASSWORD_LENGTH) { + return this.crypto + .randomBytes(length) + .toString('base64') + .slice(0,length) + .replace(/\+/g, '0') + .replace(/\//g, '0'); + } + + async createDatabase(name) { + await this.query(this.pgFormat('CREATE DATABASE %I', name)); + } + + async createUser(name, password) { + await this.query(this.pgFormat(`CREATE USER %I WITH PASSWORD %L`, name, password)) + } + + async restrictUserToDatabase(databaseName, userName) { + await this.query(this.pgFormat('GRANT ALL PRIVILEGES ON DATABASE %I TO %I', databaseName, userName)); + await this.query(this.pgFormat('REVOKE CONNECT ON DATABASE %I FROM PUBLIC', databaseName)); + } + + async createUserDb(userName, password, databaseName) { + try { + await this.createDatabase(databaseName); + await this.createUser(userName, password); + await this.restrictUserToDatabase(databaseName, userName); + } catch (error) { + throw new VError(error, `Failed to create user db`); + } } - doesEndpointExist(schemaName) { - return this.hasuraClient.isSchemaCreated(schemaName); + async isUserApiProvisioned(accountId, functionName) { + const databaseName = this.replaceSpecialChars(accountId); + const schemaName = this.replaceSpecialChars(functionName); + + const sourceExists = await this.hasuraClient.doesSourceExist(databaseName); + if (!sourceExists) { + return false; + } + + const schemaExists = await this.hasuraClient.doesSchemaExist(databaseName, schemaName); + + return schemaExists; } - async createSchema(schemaName) { + async createSchema(databaseName, schemaName) { try { - await this.hasuraClient.createSchema(schemaName); + await this.hasuraClient.createSchema(databaseName, schemaName); } catch (error) { throw new VError(error, `Failed to create schema`); } } - async runMigrations(schemaName, migration) { + async runMigrations(databaseName, schemaName, migration) { try { - await this.hasuraClient.runMigrations(schemaName, migration); + await this.hasuraClient.runMigrations(databaseName, schemaName, migration); } catch (error) { throw new VError(error, `Failed to run migrations`); } } - async getTableNames(schemaName) { + async getTableNames(schemaName, databaseName) { try { - return await this.hasuraClient.getTableNames(schemaName); + return await this.hasuraClient.getTableNames(schemaName, databaseName); } catch (error) { throw new VError(error, `Failed to fetch table names`); } } - async trackTables(schemaName, tableNames) { + async trackTables(schemaName, tableNames, databaseName) { try { - await this.hasuraClient.trackTables(schemaName, tableNames); + await this.hasuraClient.trackTables(schemaName, tableNames, databaseName); } catch (error) { throw new VError(error, `Failed to track tables`); } } - async addPermissionsToTables(schemaName, tableNames, roleName, permissions) { + async addPermissionsToTables(schemaName, databaseName, tableNames, roleName, permissions) { try { await this.hasuraClient.addPermissionsToTables( schemaName, + databaseName, tableNames, roleName, - ['select', 'insert', 'update', 'delete'] + permissions ); } catch (error) { throw new VError(error, `Failed to add permissions to tables`); } } - async trackForeignKeyRelationships(schemaName) { + async trackForeignKeyRelationships(schemaName, databaseName) { try { - await this.hasuraClient.trackForeignKeyRelationships(schemaName); + await this.hasuraClient.trackForeignKeyRelationships(schemaName, databaseName); } catch (error) { throw new VError(error, `Failed to track foreign key relationships`); } } - async createAuthenticatedEndpoint(schemaName, roleName, migration) { + async addDatasource(userName, password, databaseName) { + try { + await this.hasuraClient.addDatasource(userName, password, databaseName); + } catch (error) { + throw new VError(error, `Failed to add datasource`); + } + } + + replaceSpecialChars(str) { + return str.replaceAll(/[.-]/g, '_') + } + + async provisionUserApi(accountId, functionName, databaseSchema) { + const sanitizedAccountId = this.replaceSpecialChars(accountId); + const sanitizedFunctionName = this.replaceSpecialChars(functionName); + + const databaseName = sanitizedAccountId; + const userName = sanitizedAccountId; + const schemaName = sanitizedFunctionName; + try { - await this.createSchema(schemaName); + if (!await this.hasuraClient.doesSourceExist(databaseName)) { + const password = this.generatePassword() + await this.createUserDb(userName, password, databaseName); + await this.addDatasource(userName, password, databaseName); + } - await this.runMigrations(schemaName, migration); + await this.createSchema(databaseName, schemaName); + await this.runMigrations(databaseName, schemaName, databaseSchema); - const tableNames = await this.getTableNames(schemaName); - await this.trackTables(schemaName, tableNames); + const tableNames = await this.getTableNames(schemaName, databaseName); + await this.trackTables(schemaName, tableNames, databaseName); - await this.trackForeignKeyRelationships(schemaName); + await this.trackForeignKeyRelationships(schemaName, databaseName); - await this.addPermissionsToTables(schemaName, tableNames, roleName, ['select', 'insert', 'update', 'delete']); + await this.addPermissionsToTables(schemaName, databaseName, tableNames, userName, ['select', 'insert', 'update', 'delete']); } catch (error) { throw new VError( { cause: error, info: { schemaName, - roleName, - migration, + userName, + databaseSchema, + databaseName, } }, `Failed to provision endpoint` diff --git a/indexer-js-queue-handler/provisioner.test.js b/indexer-js-queue-handler/provisioner.test.js index 11cd7a014..cb82e3c9d 100644 --- a/indexer-js-queue-handler/provisioner.test.js +++ b/indexer-js-queue-handler/provisioner.test.js @@ -4,178 +4,201 @@ import VError from 'verror' import HasuraClient from './hasura-client'; import Provisioner from './provisioner'; -describe('Provision', () => { - it('checks if the endpoint already exists', async () => { - const provisioner = new Provisioner({ - isSchemaCreated: jest.fn().mockResolvedValueOnce(true) - }); - - expect(await provisioner.doesEndpointExist('schema')).toBe(true); - }); - - it('creates an authenticated endpoint', async () => { - const tableNames = ['blocks']; - const hasuraClient = { - createSchema: jest.fn().mockReturnValueOnce(), - runMigrations: jest.fn().mockReturnValueOnce(), +describe('Provisioner', () => { + let pgPool; + let pgClient; + let hasuraClient; + + const tableNames = ['blocks']; + const accountId = 'morgs.near'; + const sanitizedAccountId = 'morgs_near'; + const functionName = 'test-function'; + const sanitizedFunctionName = 'test_function'; + const databaseSchema = 'CREATE TABLE blocks (height numeric)'; + const error = new Error('some error'); + + const password = 'password'; + const crypto = { + randomBytes: () => ({ + toString: () => ({ + slice: () => ({ + replace: () => password, + }), + }), + }), + }; + + beforeEach(() => { + hasuraClient = { getTableNames: jest.fn().mockReturnValueOnce(tableNames), trackTables: jest.fn().mockReturnValueOnce(), trackForeignKeyRelationships: jest.fn().mockReturnValueOnce(), addPermissionsToTables: jest.fn().mockReturnValueOnce(), + addDatasource: jest.fn().mockReturnValueOnce(), + runMigrations: jest.fn().mockReturnValueOnce(), + createSchema: jest.fn().mockReturnValueOnce(), + doesSourceExist: jest.fn().mockReturnValueOnce(false), + doesSchemaExist: jest.fn().mockReturnValueOnce(false), }; - const provisioner = new Provisioner(hasuraClient); - - const schemaName = 'schema'; - const roleName = 'role'; - const migration = 'CREATE TABLE blocks (height numeric)'; - await provisioner.createAuthenticatedEndpoint(schemaName, roleName, migration); - - expect(hasuraClient.createSchema).toBeCalledWith(schemaName); - expect(hasuraClient.runMigrations).toBeCalledWith(schemaName, migration); - expect(hasuraClient.getTableNames).toBeCalledWith(schemaName); - expect(hasuraClient.trackTables).toBeCalledWith(schemaName, tableNames); - expect(hasuraClient.addPermissionsToTables).toBeCalledWith( - schemaName, - tableNames, - roleName, - [ - 'select', - 'insert', - 'update', - 'delete' - ] - ); - }); - it('throws an error when it fails to create the schema', async () => { - const error = new Error('some http error'); - const hasuraClient = { - createSchema: jest.fn().mockRejectedValue(error), + pgClient = { + query: jest.fn().mockResolvedValue(), + release: jest.fn().mockResolvedValue(), }; - const provisioner = new Provisioner(hasuraClient); - - try { - await provisioner.createAuthenticatedEndpoint('name', 'role', 'CREATE TABLE blocks (height numeric)') - } catch (error) { - expect(error.message).toBe('Failed to provision endpoint: Failed to create schema: some http error'); - expect(VError.info(error)).toEqual({ - schemaName: 'name', - roleName: 'role', - migration: 'CREATE TABLE blocks (height numeric)', - }); - } - }); - - it('throws an error when it fails to run migrations', async () => { - const error = new Error('some http error'); - const hasuraClient = { - createSchema: jest.fn().mockReturnValueOnce(), - runMigrations: jest.fn().mockRejectedValue(error), + pgPool = { + connect: jest.fn().mockResolvedValue(pgClient) }; - const provisioner = new Provisioner(hasuraClient); - - try { - await provisioner.createAuthenticatedEndpoint('name', 'role', 'CREATE TABLE blocks (height numeric)') - } catch (error) { - expect(error.message).toBe('Failed to provision endpoint: Failed to run migrations: some http error'); - expect(VError.info(error)).toEqual({ - schemaName: 'name', - roleName: 'role', - migration: 'CREATE TABLE blocks (height numeric)', - }); - } }); - it('throws an error when it fails to fetch table names', async () => { - const error = new Error('some http error'); - const hasuraClient = { - createSchema: jest.fn().mockReturnValueOnce(), - runMigrations: jest.fn().mockReturnValueOnce(), - getTableNames: jest.fn().mockRejectedValue(error), - }; - const provisioner = new Provisioner(hasuraClient); - - try { - await provisioner.createAuthenticatedEndpoint('name', 'role', 'CREATE TABLE blocks (height numeric)') - } catch (error) { - expect(error.message).toBe('Failed to provision endpoint: Failed to fetch table names: some http error'); - expect(VError.info(error)).toEqual({ - schemaName: 'name', - roleName: 'role', - migration: 'CREATE TABLE blocks (height numeric)', - }); - } - }); + describe('isUserApiProvisioned', () => { + it('returns false if datasource doesnt exists', async () => { + hasuraClient.doesSourceExist = jest.fn().mockReturnValueOnce(false); - it('throws an error when it fails to track tables', async () => { - const error = new Error('some http error'); - const tableNames = ['blocks']; - const hasuraClient = { - createSchema: jest.fn().mockReturnValueOnce(), - runMigrations: jest.fn().mockReturnValueOnce(), - getTableNames: jest.fn().mockReturnValueOnce(tableNames), - trackTables: jest.fn().mockRejectedValueOnce(error), - }; - const provisioner = new Provisioner(hasuraClient); - - try { - await provisioner.createAuthenticatedEndpoint('name', 'role', 'CREATE TABLE blocks (height numeric)') - } catch (error) { - expect(error.message).toBe('Failed to provision endpoint: Failed to track tables: some http error'); - expect(VError.info(error)).toEqual({ - schemaName: 'name', - roleName: 'role', - migration: 'CREATE TABLE blocks (height numeric)', - }); - } + const provisioner = new Provisioner(hasuraClient, pgPool, crypto); + + await expect(provisioner.isUserApiProvisioned(accountId, functionName)).resolves.toBe(false); + }); + + it('returns false if datasource and schema dont exists', async () => { + hasuraClient.doesSourceExist = jest.fn().mockReturnValueOnce(false); + hasuraClient.doesSchemaExist = jest.fn().mockReturnValueOnce(false); + + const provisioner = new Provisioner(hasuraClient, pgPool, crypto); + + await expect(provisioner.isUserApiProvisioned(accountId, functionName)).resolves.toBe(false); + }); + + it('returns true if datasource and schema exists', async () => { + hasuraClient.doesSourceExist = jest.fn().mockReturnValueOnce(true); + hasuraClient.doesSchemaExist = jest.fn().mockReturnValueOnce(true); + + const provisioner = new Provisioner(hasuraClient, pgPool, crypto); + + await expect(provisioner.isUserApiProvisioned(accountId, functionName)).resolves.toBe(true); + }); }); - it('throws an error when it fails to track foreign key relationships', async () => { - const error = new Error('some http error'); - const tableNames = ['blocks']; - const hasuraClient = { - createSchema: jest.fn().mockReturnValueOnce(), - runMigrations: jest.fn().mockReturnValueOnce(), - getTableNames: jest.fn().mockReturnValueOnce(tableNames), - trackTables: jest.fn().mockReturnValueOnce(), - trackForeignKeyRelationships: jest.fn().mockRejectedValueOnce(error), - }; - const provisioner = new Provisioner(hasuraClient); - - try { - await provisioner.createAuthenticatedEndpoint('name', 'role', 'CREATE TABLE blocks (height numeric)') - } catch (error) { - expect(error.message).toBe('Failed to provision endpoint: Failed to track foreign key relationships: some http error'); - expect(VError.info(error)).toEqual({ - schemaName: 'name', - roleName: 'role', - migration: 'CREATE TABLE blocks (height numeric)', - }); - } - }) - - it('throws an error when it fails to add permissions to tables', async () => { - const error = new Error('some http error'); - const tableNames = ['blocks']; - const hasuraClient = { - createSchema: jest.fn().mockReturnValueOnce(), - runMigrations: jest.fn().mockReturnValueOnce(), - getTableNames: jest.fn().mockReturnValueOnce(tableNames), - trackTables: jest.fn().mockReturnValueOnce(), - trackForeignKeyRelationships: jest.fn().mockReturnValueOnce(), - addPermissionsToTables: jest.fn().mockRejectedValue(error), - }; - const provisioner = new Provisioner(hasuraClient); - - try { - await provisioner.createAuthenticatedEndpoint('name', 'role', 'CREATE TABLE blocks (height numeric)') - } catch (error) { - expect(error.message).toBe('Failed to provision endpoint: Failed to add permissions to tables: some http error'); - expect(VError.info(error)).toEqual({ - migration: 'CREATE TABLE blocks (height numeric)', - schemaName: 'name', - roleName: 'role', - }); - } + describe('provisionUserApi', () => { + it('provisions an API for the user', async () => { + const provisioner = new Provisioner(hasuraClient, pgPool, crypto); + + await provisioner.provisionUserApi(accountId, functionName, databaseSchema); + + expect(pgClient.query.mock.calls).toEqual([ + ['CREATE DATABASE morgs_near', []], + ['CREATE USER morgs_near WITH PASSWORD \'password\'', []], + ['GRANT ALL PRIVILEGES ON DATABASE morgs_near TO morgs_near', []], + ['REVOKE CONNECT ON DATABASE morgs_near FROM PUBLIC', []], + ]); + expect(hasuraClient.addDatasource).toBeCalledWith(sanitizedAccountId, password, sanitizedAccountId); + expect(hasuraClient.createSchema).toBeCalledWith(sanitizedAccountId, sanitizedFunctionName); + expect(hasuraClient.runMigrations).toBeCalledWith(sanitizedAccountId, sanitizedFunctionName, databaseSchema); + expect(hasuraClient.getTableNames).toBeCalledWith(sanitizedFunctionName, sanitizedAccountId); + expect(hasuraClient.trackTables).toBeCalledWith(sanitizedFunctionName, tableNames, sanitizedAccountId); + expect(hasuraClient.addPermissionsToTables).toBeCalledWith( + sanitizedFunctionName, + sanitizedAccountId, + tableNames, + sanitizedAccountId, + [ + 'select', + 'insert', + 'update', + 'delete' + ] + ); + }); + + it('skips provisioning the datasource if it already exists', async () => { + hasuraClient.doesSourceExist = jest.fn().mockReturnValueOnce(true); + + const provisioner = new Provisioner(hasuraClient, pgPool, crypto); + + await provisioner.provisionUserApi(accountId, functionName, databaseSchema); + + expect(pgClient.query).not.toBeCalled(); + expect(hasuraClient.addDatasource).not.toBeCalled(); + + expect(hasuraClient.createSchema).toBeCalledWith(sanitizedAccountId, sanitizedFunctionName); + expect(hasuraClient.runMigrations).toBeCalledWith(sanitizedAccountId, sanitizedFunctionName, databaseSchema); + expect(hasuraClient.getTableNames).toBeCalledWith(sanitizedFunctionName, sanitizedAccountId); + expect(hasuraClient.trackTables).toBeCalledWith(sanitizedFunctionName, tableNames, sanitizedAccountId); + expect(hasuraClient.addPermissionsToTables).toBeCalledWith( + sanitizedFunctionName, + sanitizedAccountId, + tableNames, + sanitizedAccountId, + [ + 'select', + 'insert', + 'update', + 'delete' + ] + ); + }); + + it('formats user input before executing the query', async () => { + const provisioner = new Provisioner(hasuraClient, pgPool, crypto); + + await provisioner.createUserDb('morgs_near', 'pass; DROP TABLE users;--', 'databaseName UNION SELECT * FROM users --'); + + expect(pgClient.query.mock.calls).toMatchSnapshot(); + }); + + it('throws an error when it fails to create a postgres db', async () => { + pgClient.query = jest.fn().mockRejectedValue(error); + + const provisioner = new Provisioner(hasuraClient, pgPool, crypto); + + await expect(provisioner.provisionUserApi(accountId, functionName, databaseSchema)).rejects.toThrow('Failed to provision endpoint: Failed to create user db: some error'); + }); + + it('throws an error when it fails to add the db to hasura', async () => { + hasuraClient.addDatasource = jest.fn().mockRejectedValue(error); + + const provisioner = new Provisioner(hasuraClient, pgPool, crypto); + + await expect(provisioner.provisionUserApi(accountId, functionName, databaseSchema)).rejects.toThrow('Failed to provision endpoint: Failed to add datasource: some error'); + }); + + it('throws an error when it fails to run migrations', async () => { + hasuraClient.runMigrations = jest.fn().mockRejectedValue(error); + + const provisioner = new Provisioner(hasuraClient, pgPool, crypto); + + await expect(provisioner.provisionUserApi(accountId, functionName, databaseSchema)).rejects.toThrow('Failed to provision endpoint: Failed to run migrations: some error'); + }); + + it('throws an error when it fails to fetch table names', async () => { + hasuraClient.getTableNames = jest.fn().mockRejectedValue(error); + + const provisioner = new Provisioner(hasuraClient, pgPool, crypto); + + await expect(provisioner.provisionUserApi(accountId, functionName, databaseSchema)).rejects.toThrow('Failed to provision endpoint: Failed to fetch table names: some error'); + }); + + it('throws an error when it fails to track tables', async () => { + hasuraClient.trackTables = jest.fn().mockRejectedValue(error); + + const provisioner = new Provisioner(hasuraClient, pgPool, crypto); + + await expect(provisioner.provisionUserApi(accountId, functionName, databaseSchema)).rejects.toThrow('Failed to provision endpoint: Failed to track tables: some error'); + }); + + it('throws an error when it fails to track foreign key relationships', async () => { + hasuraClient.trackForeignKeyRelationships = jest.fn().mockRejectedValue(error); + + const provisioner = new Provisioner(hasuraClient, pgPool, crypto); + + await expect(provisioner.provisionUserApi(accountId, functionName, databaseSchema)).rejects.toThrow('Failed to provision endpoint: Failed to track foreign key relationships: some error'); + }) + + it('throws an error when it fails to add permissions to tables', async () => { + hasuraClient.addPermissionsToTables = jest.fn().mockRejectedValue(error); + + const provisioner = new Provisioner(hasuraClient, pgPool, crypto); + + await expect(provisioner.provisionUserApi(accountId, functionName, databaseSchema)).rejects.toThrow('Failed to provision endpoint: Failed to add permissions to tables: some error'); + }); }); }) diff --git a/indexer-js-queue-handler/serverless.yml b/indexer-js-queue-handler/serverless.yml index 368400cc7..71048ebd0 100644 --- a/indexer-js-queue-handler/serverless.yml +++ b/indexer-js-queue-handler/serverless.yml @@ -7,12 +7,17 @@ provider: name: aws runtime: nodejs16.x region: eu-central-1 - timeout: 15 + timeout: 30 environment: REGION: ${self:provider.region} STAGE: ${opt:stage, 'dev'} HASURA_ENDPOINT: ${env:HASURA_ENDPOINT} HASURA_ADMIN_SECRET: ${env:HASURA_ADMIN_SECRET} + PG_ADMIN_USER: ${env:PG_ADMIN_USER} + PG_ADMIN_PASSWORD: ${env:PG_ADMIN_PASSWORD} + PG_ADMIN_DATABASE: ${env:PG_ADMIN_DATABASE} + PG_HOST: ${env:PG_HOST} + PG_PORT: ${env:PG_PORT} tracing: lambda: true #enable X-Ray tracing iamRoleStatements: