From b58a0fc494851b3a8aa846e95da9e85d8ead0643 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Wed, 6 Sep 2023 10:54:55 -0700 Subject: [PATCH] fix: Improve Scope of Table Name Regex to Resolve Errors Regex for getting table names was not matching against valid schemas. I improved the regex to cover a much larger array of inputs and also made failure in generating the db methods not block the overall execution of the indexer. Now, it will only fail if the indexing code uses db methods but the methods fail to generate. --- frontend/src/utils/indexerRunner.js | 86 +++++++++++------- runner/src/indexer/indexer.test.ts | 132 +++++++++++++++++++++++++++- runner/src/indexer/indexer.ts | 109 +++++++++++++++-------- 3 files changed, 251 insertions(+), 76 deletions(-) diff --git a/frontend/src/utils/indexerRunner.js b/frontend/src/utils/indexerRunner.js index 6df114c29..6ca8dfa88 100644 --- a/frontend/src/utils/indexerRunner.js +++ b/frontend/src/utils/indexerRunner.js @@ -50,23 +50,8 @@ export default class IndexerRunner { return new Promise((resolve) => setTimeout(resolve, ms)); } - validateTableNames(tableNames) { - if (!(Array.isArray(tableNames) && tableNames.length > 0)) { - throw new Error("Schema does not have any tables. There should be at least one table."); - } - const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/; - - tableNames.forEach(name => { - if (!correctTableNameFormat.test(name)) { - throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`); - } - }); - } - async executeIndexerFunction(height, blockDetails, indexingCode, schema, schemaName) { let innerCode = indexingCode.match(/getBlock\s*\([^)]*\)\s*{([\s\S]*)}/)[1]; - let tableNames = Array.from(schema.matchAll(/CREATE TABLE\s+"(\w+)"/g), match => match[1]); // Get first capturing group of each match - this.validateTableNames(tableNames); if (blockDetails) { const block = Block.fromStreamerMessage(blockDetails); @@ -75,7 +60,7 @@ export default class IndexerRunner { block.events() console.log(block) - await this.runFunction(blockDetails, height, innerCode, schemaName, tableNames); + await this.runFunction(blockDetails, height, innerCode, schemaName, schema); } } @@ -103,7 +88,7 @@ export default class IndexerRunner { console.groupEnd() } - async runFunction(streamerMessage, blockHeight, indexerCode, schemaName, tableNames) { + async runFunction(streamerMessage, blockHeight, indexerCode, schemaName, schema) { const innerCodeWithBlockHelper = ` const block = Block.fromStreamerMessage(streamerMessage); @@ -162,30 +147,65 @@ export default class IndexerRunner { log: async (message) => { this.handleLog(blockHeight, message); }, - db: this.buildDatabaseContext(blockHeight, schemaName, tableNames) + db: this.buildDatabaseContext(blockHeight, schemaName, schema) }; wrappedFunction(Block, streamerMessage, context); } - buildDatabaseContext (blockHeight, schemaName, tables) { + validateTableNames(tableNames) { + if (!(Array.isArray(tableNames) && tableNames.length > 0)) { + throw new Error("Schema does not have any tables. There should be at least one table."); + } + const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + + tableNames.forEach(name => { + if (!name.includes("\"") && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes + throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`); + } + }); + } + + getTableNames (schema) { + const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS\s+)?"?(.+?)"?\s*\(/g; + const tableNames = Array.from(schema.matchAll(tableRegex), match => { + let tableName; + if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName + tableName = match[1].split('.')[1]; + tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName; + } else { + tableName = match[1]; + } + return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes + }); + this.validateTableNames(tableNames); + console.log('Retrieved the following table names from schema: ', tableNames); + return tableNames; + } + + sanitizeTableName (tableName) { + tableName = tableName.startsWith('"') && tableName.endsWith('"') ? tableName.substring(1, tableName.length - 1) : tableName; + return tableName.replace(/[^a-zA-Z0-9_]/g, '_'); + } + + buildDatabaseContext (blockHeight, schemaName, schema) { try { - const result = tables.reduce((prev, tableName) => ({ - ...prev, - [`insert_${tableName}`]: async (objects) => await this.insert(blockHeight, schemaName, tableName, objects), - [`select_${tableName}`]: async (object, limit = 0) => await this.select(blockHeight, schemaName, tableName, object, limit), - }), {}); + const tables = this.getTableNames(schema); + const result = tables.reduce((prev, tableName) => { + const sanitizedTableName = this.sanitizeTableName(tableName); + const funcForTable = { + [`insert_${sanitizedTableName}`]: async (objects) => await this.insert(blockHeight, schemaName, tableName, objects), + [`select_${sanitizedTableName}`]: async (object, limit = 0) => await this.select(blockHeight, schemaName, tableName, object, limit) + }; + + return { + ...prev, + ...funcForTable + }; + }, {}); return result; } catch (error) { - console.error('Caught error when generating DB methods. Falling back to generic methods.', error); - return { - insert: async (tableName, objects) => { - this.insert(blockHeight, schemaName, tableName, objects); - }, - select: async (tableName, object, limit = 0) => { - this.select(blockHeight, schemaName, tableName, object, limit); - } - }; + console.warn('Caught error when generating context.db methods. Building no functions. You can still use other context object methods.\n', error); } } diff --git a/runner/src/indexer/indexer.test.ts b/runner/src/indexer/indexer.test.ts index 2ace38cc2..f2c16b9e8 100644 --- a/runner/src/indexer/indexer.test.ts +++ b/runner/src/indexer/indexer.test.ts @@ -62,7 +62,98 @@ describe('Indexer unit tests', () => { "block_timestamp" DECIMAL(20, 0) NOT NULL, "receipt_id" VARCHAR NOT NULL, CONSTRAINT "post_likes_pkey" PRIMARY KEY ("post_id", "account_id") - );'`; + );`; + + const STRESS_TEST_SCHEMA = ` +CREATE TABLE creator_quest ( + account_id VARCHAR PRIMARY KEY, + num_components_created INTEGER NOT NULL DEFAULT 0, + completed BOOLEAN NOT NULL DEFAULT FALSE + ); + +CREATE TABLE + composer_quest ( + account_id VARCHAR PRIMARY KEY, + num_widgets_composed INTEGER NOT NULL DEFAULT 0, + completed BOOLEAN NOT NULL DEFAULT FALSE + ); + +CREATE TABLE + "contractor - quest" ( + account_id VARCHAR PRIMARY KEY, + num_contracts_deployed INTEGER NOT NULL DEFAULT 0, + completed BOOLEAN NOT NULL DEFAULT FALSE + ); + +CREATE TABLE + "posts" ( + "id" SERIAL NOT NULL, + "account_id" VARCHAR NOT NULL, + "block_height" DECIMAL(58, 0) NOT NULL, + "receipt_id" VARCHAR NOT NULL, + "content" TEXT NOT NULL, + "block_timestamp" DECIMAL(20, 0) NOT NULL, + "accounts_liked" JSONB NOT NULL DEFAULT '[]', + "last_comment_timestamp" DECIMAL(20, 0), + CONSTRAINT "posts_pkey" PRIMARY KEY ("id") + ); + +CREATE TABLE + "comments" ( + "id" SERIAL NOT NULL, + "post_id" SERIAL NOT NULL, + "account_id" VARCHAR NOT NULL, + "block_height" DECIMAL(58, 0) NOT NULL, + "content" TEXT NOT NULL, + "block_timestamp" DECIMAL(20, 0) NOT NULL, + "receipt_id" VARCHAR NOT NULL, + CONSTRAINT "comments_pkey" PRIMARY KEY ("id") + ); + +CREATE TABLE + "post_likes" ( + "post_id" SERIAL NOT NULL, + "account_id" VARCHAR NOT NULL, + "block_height" DECIMAL(58, 0), + "block_timestamp" DECIMAL(20, 0) NOT NULL, + "receipt_id" VARCHAR NOT NULL, + CONSTRAINT "post_likes_pkey" PRIMARY KEY ("post_id", "account_id") + ); + +CREATE UNIQUE INDEX "posts_account_id_block_height_key" ON "posts" ("account_id" ASC, "block_height" ASC); + +CREATE UNIQUE INDEX "comments_post_id_account_id_block_height_key" ON "comments" ( + "post_id" ASC, + "account_id" ASC, + "block_height" ASC +); + +CREATE INDEX + "posts_last_comment_timestamp_idx" ON "posts" ("last_comment_timestamp" DESC); + +ALTER TABLE + "comments" +ADD + CONSTRAINT "comments_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE + "post_likes" +ADD + CONSTRAINT "post_likes_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts" ("id") ON DELETE CASCADE ON UPDATE NO ACTION; + +CREATE TABLE IF NOT EXISTS + "public"."My Table1" (id serial PRIMARY KEY); + +CREATE TABLE + "Another-Table" (id serial PRIMARY KEY); + +CREATE TABLE +IF NOT EXISTS + "Third-Table" (id serial PRIMARY KEY); + +CREATE TABLE + yet_another_table (id serial PRIMARY KEY); +`; beforeEach(() => { process.env = { @@ -425,10 +516,43 @@ describe('Indexer unit tests', () => { }); const indexer = new Indexer('mainnet', { DmlHandler: mockDmlHandler }); - const context = indexer.buildContext(SOCIAL_SCHEMA, 'morgs.near/social_feed1', 1, 'postgres'); + const context = indexer.buildContext(STRESS_TEST_SCHEMA, 'morgs.near/social_feed1', 1, 'postgres'); + + expect(Object.keys(context.db)).toStrictEqual( + ['insert_creator_quest', + 'select_creator_quest', + 'insert_composer_quest', + 'select_composer_quest', + 'insert_contractor___quest', + 'select_contractor___quest', + 'insert_posts', + 'select_posts', + 'insert_comments', + 'select_comments', + 'insert_post_likes', + 'select_post_likes', + 'insert_My_Table1', + 'select_My_Table1', + 'insert_Another_Table', + 'select_Another_Table', + 'insert_Third_Table', + 'select_Third_Table', + 'insert_yet_another_table', + 'select_yet_another_table']); + }); + + test('indexer builds context and returns empty array if failed to generate db methods', async () => { + const mockDmlHandler: any = jest.fn().mockImplementation(() => { + return { + insert: jest.fn().mockReturnValue(true), + select: jest.fn().mockReturnValue(true) + }; + }); + + const indexer = new Indexer('mainnet', { DmlHandler: mockDmlHandler }); + const context = indexer.buildContext('', 'morgs.near/social_feed1', 1, 'postgres'); - // These calls would fail on a real database, but we are merely checking to ensure they exist - expect(Object.keys(context.db)).toStrictEqual(['insert_posts', 'select_posts', 'insert_comments', 'select_comments', 'insert_post_likes', 'select_post_likes']); + expect(Object.keys(context.db)).toStrictEqual([]); }); test('Indexer.runFunctions() allows imperative execution of GraphQL operations', async () => { diff --git a/runner/src/indexer/indexer.ts b/runner/src/indexer/indexer.ts index 4cae6a0ca..fe825bcd3 100644 --- a/runner/src/indexer/indexer.ts +++ b/runner/src/indexer/indexer.ts @@ -186,29 +186,7 @@ export default class Indexer { ].reduce((acc, val) => val(acc), indexerFunction); } - validateTableNames (tableNames: string[]): void { - if (!(tableNames.length > 0)) { - throw new Error('Schema does not have any tables. There should be at least one table.'); - } - const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/; - - tableNames.forEach((name: string) => { - if (!correctTableNameFormat.test(name)) { - throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`); - } - }); - } - - getTableNames (schema: string): string[] { - const tableNameMatcher = /CREATE TABLE\s+"(\w+)"/g; - const tableNames = Array.from(schema.matchAll(tableNameMatcher), match => match[1]); // Get first capturing group of each match - this.validateTableNames(tableNames); - console.log('Retrieved the following table names from schema: ', tableNames); - return tableNames; - } - buildContext (schema: string, functionName: string, blockHeight: number, hasuraRoleName: string): Context { - const tables = this.getTableNames(schema); const account = functionName.split('/')[0].replace(/[.-]/g, '_'); const functionNameWithoutAccount = functionName.split('/')[1].replace(/[.-]/g, '_'); const schemaName = functionName.replace(/[^a-zA-Z0-9]/g, '_'); @@ -237,26 +215,79 @@ export default class Indexer { fetchFromSocialApi: async (path, options) => { return await this.deps.fetch(`https://api.near.social${path}`, options); }, - db: this.buildDatabaseContext(account, schemaName, tables, blockHeight) + db: this.buildDatabaseContext(account, schemaName, schema, blockHeight) }; } - buildDatabaseContext (account: string, schemaName: string, tables: string[], blockHeight: number): Record any> { - let dmlHandler: DmlHandler | null = null; - const result = tables.reduce((prev, tableName) => ({ - ...prev, - [`insert_${tableName}`]: async (objects: any) => { - await this.writeLog(`context.db.insert_${tableName}`, blockHeight, `Calling context.db.insert_${tableName}.`, `Inserting object ${JSON.stringify(objects)} into table ${tableName} on schema ${schemaName}`); - dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account); - return await dmlHandler.insert(schemaName, tableName, Array.isArray(objects) ? objects : [objects]); - }, - [`select_${tableName}`]: async (object: any, limit = null) => { - await this.writeLog(`context.db.select_${tableName}`, blockHeight, `Calling context.db.select_${tableName}.`, `Selecting objects with values ${JSON.stringify(object)} from table ${tableName} on schema ${schemaName} with limit ${limit === null ? 'no' : limit}`); - dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account); - return await dmlHandler.select(schemaName, tableName, object, limit); - }, - }), {}); - return result; + validateTableNames (tableNames: string[]): void { + if (!(Array.isArray(tableNames) && tableNames.length > 0)) { + throw new Error('Schema does not have any tables. There should be at least one table.'); + } + const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + + tableNames.forEach(name => { + if (!name.includes('"') && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes + throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`); + } + }); + } + + getTableNames (schema: string): string[] { + const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS\s+)?"?(.+?)"?\s*\(/g; + const tableNames = Array.from(schema.matchAll(tableRegex), match => { + let tableName; + if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName + tableName = match[1].split('.')[1]; + tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName; + } else { + tableName = match[1]; + } + return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes + }); + this.validateTableNames(tableNames); + console.log('Retrieved the following table names from schema: ', tableNames); + return tableNames; + } + + sanitizeTableName (tableName: string): string { + tableName = tableName.startsWith('"') && tableName.endsWith('"') ? tableName.substring(1, tableName.length - 1) : tableName; + return tableName.replace(/[^a-zA-Z0-9_]/g, '_'); + } + + buildDatabaseContext (account: string, schemaName: string, schema: string, blockHeight: number): Record any> { + try { + const tables = this.getTableNames(schema); + let dmlHandler: DmlHandler | null = null; + const result = tables.reduce((prev, tableName) => { + const sanitizedTableName = this.sanitizeTableName(tableName); + const funcForTable = { + [`insert_${sanitizedTableName}`]: async (objects: any) => { + await this.writeLog(`context.db.insert_${sanitizedTableName}`, blockHeight, + `Calling context.db.insert_${sanitizedTableName}.`, + `Inserting object ${JSON.stringify(objects)} into table ${tableName} on schema ${schemaName}`); + dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account); + return await dmlHandler.insert(schemaName, tableName, Array.isArray(objects) ? objects : [objects]); + }, + [`select_${sanitizedTableName}`]: async (object: any, limit = null) => { + await this.writeLog(`context.db.select_${sanitizedTableName}`, blockHeight, + `Calling context.db.select_${sanitizedTableName}.`, + `Selecting objects with values ${JSON.stringify(object)} from table ${tableName} on schema ${schemaName} with limit ${limit === null ? 'no' : limit}`); + dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account); + return await dmlHandler.select(schemaName, tableName, object, limit); + } + }; + + return { + ...prev, + ...funcForTable + }; + }, {}); + return result; + } catch (error) { + console.warn('Caught error when generating context.db methods. Building no functions. You can still use other context object methods.\n', error); + } + + return {}; // Default to empty object if error } async setStatus (functionName: string, blockHeight: number, status: string): Promise {