diff --git a/frontend/src/utils/indexerRunner.js b/frontend/src/utils/indexerRunner.js index 6df114c29..6ca8dfa88 100644 --- a/frontend/src/utils/indexerRunner.js +++ b/frontend/src/utils/indexerRunner.js @@ -50,23 +50,8 @@ export default class IndexerRunner { return new Promise((resolve) => setTimeout(resolve, ms)); } - validateTableNames(tableNames) { - if (!(Array.isArray(tableNames) && tableNames.length > 0)) { - throw new Error("Schema does not have any tables. There should be at least one table."); - } - const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/; - - tableNames.forEach(name => { - if (!correctTableNameFormat.test(name)) { - throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`); - } - }); - } - async executeIndexerFunction(height, blockDetails, indexingCode, schema, schemaName) { let innerCode = indexingCode.match(/getBlock\s*\([^)]*\)\s*{([\s\S]*)}/)[1]; - let tableNames = Array.from(schema.matchAll(/CREATE TABLE\s+"(\w+)"/g), match => match[1]); // Get first capturing group of each match - this.validateTableNames(tableNames); if (blockDetails) { const block = Block.fromStreamerMessage(blockDetails); @@ -75,7 +60,7 @@ export default class IndexerRunner { block.events() console.log(block) - await this.runFunction(blockDetails, height, innerCode, schemaName, tableNames); + await this.runFunction(blockDetails, height, innerCode, schemaName, schema); } } @@ -103,7 +88,7 @@ export default class IndexerRunner { console.groupEnd() } - async runFunction(streamerMessage, blockHeight, indexerCode, schemaName, tableNames) { + async runFunction(streamerMessage, blockHeight, indexerCode, schemaName, schema) { const innerCodeWithBlockHelper = ` const block = Block.fromStreamerMessage(streamerMessage); @@ -162,30 +147,65 @@ export default class IndexerRunner { log: async (message) => { this.handleLog(blockHeight, message); }, - db: this.buildDatabaseContext(blockHeight, schemaName, tableNames) + db: this.buildDatabaseContext(blockHeight, schemaName, schema) }; wrappedFunction(Block, streamerMessage, context); } - buildDatabaseContext (blockHeight, schemaName, tables) { + validateTableNames(tableNames) { + if (!(Array.isArray(tableNames) && tableNames.length > 0)) { + throw new Error("Schema does not have any tables. There should be at least one table."); + } + const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + + tableNames.forEach(name => { + if (!name.includes("\"") && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes + throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`); + } + }); + } + + getTableNames (schema) { + const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS\s+)?"?(.+?)"?\s*\(/g; + const tableNames = Array.from(schema.matchAll(tableRegex), match => { + let tableName; + if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName + tableName = match[1].split('.')[1]; + tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName; + } else { + tableName = match[1]; + } + return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes + }); + this.validateTableNames(tableNames); + console.log('Retrieved the following table names from schema: ', tableNames); + return tableNames; + } + + sanitizeTableName (tableName) { + tableName = tableName.startsWith('"') && tableName.endsWith('"') ? tableName.substring(1, tableName.length - 1) : tableName; + return tableName.replace(/[^a-zA-Z0-9_]/g, '_'); + } + + buildDatabaseContext (blockHeight, schemaName, schema) { try { - const result = tables.reduce((prev, tableName) => ({ - ...prev, - [`insert_${tableName}`]: async (objects) => await this.insert(blockHeight, schemaName, tableName, objects), - [`select_${tableName}`]: async (object, limit = 0) => await this.select(blockHeight, schemaName, tableName, object, limit), - }), {}); + const tables = this.getTableNames(schema); + const result = tables.reduce((prev, tableName) => { + const sanitizedTableName = this.sanitizeTableName(tableName); + const funcForTable = { + [`insert_${sanitizedTableName}`]: async (objects) => await this.insert(blockHeight, schemaName, tableName, objects), + [`select_${sanitizedTableName}`]: async (object, limit = 0) => await this.select(blockHeight, schemaName, tableName, object, limit) + }; + + return { + ...prev, + ...funcForTable + }; + }, {}); return result; } catch (error) { - console.error('Caught error when generating DB methods. Falling back to generic methods.', error); - return { - insert: async (tableName, objects) => { - this.insert(blockHeight, schemaName, tableName, objects); - }, - select: async (tableName, object, limit = 0) => { - this.select(blockHeight, schemaName, tableName, object, limit); - } - }; + console.warn('Caught error when generating context.db methods. Building no functions. You can still use other context object methods.\n', error); } } diff --git a/runner/src/indexer/indexer.test.ts b/runner/src/indexer/indexer.test.ts index 2ace38cc2..f2c16b9e8 100644 --- a/runner/src/indexer/indexer.test.ts +++ b/runner/src/indexer/indexer.test.ts @@ -62,7 +62,98 @@ describe('Indexer unit tests', () => { "block_timestamp" DECIMAL(20, 0) NOT NULL, "receipt_id" VARCHAR NOT NULL, CONSTRAINT "post_likes_pkey" PRIMARY KEY ("post_id", "account_id") - );'`; + );`; + + const STRESS_TEST_SCHEMA = ` +CREATE TABLE creator_quest ( + account_id VARCHAR PRIMARY KEY, + num_components_created INTEGER NOT NULL DEFAULT 0, + completed BOOLEAN NOT NULL DEFAULT FALSE + ); + +CREATE TABLE + composer_quest ( + account_id VARCHAR PRIMARY KEY, + num_widgets_composed INTEGER NOT NULL DEFAULT 0, + completed BOOLEAN NOT NULL DEFAULT FALSE + ); + +CREATE TABLE + "contractor - quest" ( + account_id VARCHAR PRIMARY KEY, + num_contracts_deployed INTEGER NOT NULL DEFAULT 0, + completed BOOLEAN NOT NULL DEFAULT FALSE + ); + +CREATE TABLE + "posts" ( + "id" SERIAL NOT NULL, + "account_id" VARCHAR NOT NULL, + "block_height" DECIMAL(58, 0) NOT NULL, + "receipt_id" VARCHAR NOT NULL, + "content" TEXT NOT NULL, + "block_timestamp" DECIMAL(20, 0) NOT NULL, + "accounts_liked" JSONB NOT NULL DEFAULT '[]', + "last_comment_timestamp" DECIMAL(20, 0), + CONSTRAINT "posts_pkey" PRIMARY KEY ("id") + ); + +CREATE TABLE + "comments" ( + "id" SERIAL NOT NULL, + "post_id" SERIAL NOT NULL, + "account_id" VARCHAR NOT NULL, + "block_height" DECIMAL(58, 0) NOT NULL, + "content" TEXT NOT NULL, + "block_timestamp" DECIMAL(20, 0) NOT NULL, + "receipt_id" VARCHAR NOT NULL, + CONSTRAINT "comments_pkey" PRIMARY KEY ("id") + ); + +CREATE TABLE + "post_likes" ( + "post_id" SERIAL NOT NULL, + "account_id" VARCHAR NOT NULL, + "block_height" DECIMAL(58, 0), + "block_timestamp" DECIMAL(20, 0) NOT NULL, + "receipt_id" VARCHAR NOT NULL, + CONSTRAINT "post_likes_pkey" PRIMARY KEY ("post_id", "account_id") + ); + +CREATE UNIQUE INDEX "posts_account_id_block_height_key" ON "posts" ("account_id" ASC, "block_height" ASC); + +CREATE UNIQUE INDEX "comments_post_id_account_id_block_height_key" ON "comments" ( + "post_id" ASC, + "account_id" ASC, + "block_height" ASC +); + +CREATE INDEX + "posts_last_comment_timestamp_idx" ON "posts" ("last_comment_timestamp" DESC); + +ALTER TABLE + "comments" +ADD + CONSTRAINT "comments_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE + "post_likes" +ADD + CONSTRAINT "post_likes_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts" ("id") ON DELETE CASCADE ON UPDATE NO ACTION; + +CREATE TABLE IF NOT EXISTS + "public"."My Table1" (id serial PRIMARY KEY); + +CREATE TABLE + "Another-Table" (id serial PRIMARY KEY); + +CREATE TABLE +IF NOT EXISTS + "Third-Table" (id serial PRIMARY KEY); + +CREATE TABLE + yet_another_table (id serial PRIMARY KEY); +`; beforeEach(() => { process.env = { @@ -425,10 +516,43 @@ describe('Indexer unit tests', () => { }); const indexer = new Indexer('mainnet', { DmlHandler: mockDmlHandler }); - const context = indexer.buildContext(SOCIAL_SCHEMA, 'morgs.near/social_feed1', 1, 'postgres'); + const context = indexer.buildContext(STRESS_TEST_SCHEMA, 'morgs.near/social_feed1', 1, 'postgres'); + + expect(Object.keys(context.db)).toStrictEqual( + ['insert_creator_quest', + 'select_creator_quest', + 'insert_composer_quest', + 'select_composer_quest', + 'insert_contractor___quest', + 'select_contractor___quest', + 'insert_posts', + 'select_posts', + 'insert_comments', + 'select_comments', + 'insert_post_likes', + 'select_post_likes', + 'insert_My_Table1', + 'select_My_Table1', + 'insert_Another_Table', + 'select_Another_Table', + 'insert_Third_Table', + 'select_Third_Table', + 'insert_yet_another_table', + 'select_yet_another_table']); + }); + + test('indexer builds context and returns empty array if failed to generate db methods', async () => { + const mockDmlHandler: any = jest.fn().mockImplementation(() => { + return { + insert: jest.fn().mockReturnValue(true), + select: jest.fn().mockReturnValue(true) + }; + }); + + const indexer = new Indexer('mainnet', { DmlHandler: mockDmlHandler }); + const context = indexer.buildContext('', 'morgs.near/social_feed1', 1, 'postgres'); - // These calls would fail on a real database, but we are merely checking to ensure they exist - expect(Object.keys(context.db)).toStrictEqual(['insert_posts', 'select_posts', 'insert_comments', 'select_comments', 'insert_post_likes', 'select_post_likes']); + expect(Object.keys(context.db)).toStrictEqual([]); }); test('Indexer.runFunctions() allows imperative execution of GraphQL operations', async () => { diff --git a/runner/src/indexer/indexer.ts b/runner/src/indexer/indexer.ts index 4cae6a0ca..fe825bcd3 100644 --- a/runner/src/indexer/indexer.ts +++ b/runner/src/indexer/indexer.ts @@ -186,29 +186,7 @@ export default class Indexer { ].reduce((acc, val) => val(acc), indexerFunction); } - validateTableNames (tableNames: string[]): void { - if (!(tableNames.length > 0)) { - throw new Error('Schema does not have any tables. There should be at least one table.'); - } - const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/; - - tableNames.forEach((name: string) => { - if (!correctTableNameFormat.test(name)) { - throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`); - } - }); - } - - getTableNames (schema: string): string[] { - const tableNameMatcher = /CREATE TABLE\s+"(\w+)"/g; - const tableNames = Array.from(schema.matchAll(tableNameMatcher), match => match[1]); // Get first capturing group of each match - this.validateTableNames(tableNames); - console.log('Retrieved the following table names from schema: ', tableNames); - return tableNames; - } - buildContext (schema: string, functionName: string, blockHeight: number, hasuraRoleName: string): Context { - const tables = this.getTableNames(schema); const account = functionName.split('/')[0].replace(/[.-]/g, '_'); const functionNameWithoutAccount = functionName.split('/')[1].replace(/[.-]/g, '_'); const schemaName = functionName.replace(/[^a-zA-Z0-9]/g, '_'); @@ -237,26 +215,79 @@ export default class Indexer { fetchFromSocialApi: async (path, options) => { return await this.deps.fetch(`https://api.near.social${path}`, options); }, - db: this.buildDatabaseContext(account, schemaName, tables, blockHeight) + db: this.buildDatabaseContext(account, schemaName, schema, blockHeight) }; } - buildDatabaseContext (account: string, schemaName: string, tables: string[], blockHeight: number): Record any> { - let dmlHandler: DmlHandler | null = null; - const result = tables.reduce((prev, tableName) => ({ - ...prev, - [`insert_${tableName}`]: async (objects: any) => { - await this.writeLog(`context.db.insert_${tableName}`, blockHeight, `Calling context.db.insert_${tableName}.`, `Inserting object ${JSON.stringify(objects)} into table ${tableName} on schema ${schemaName}`); - dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account); - return await dmlHandler.insert(schemaName, tableName, Array.isArray(objects) ? objects : [objects]); - }, - [`select_${tableName}`]: async (object: any, limit = null) => { - await this.writeLog(`context.db.select_${tableName}`, blockHeight, `Calling context.db.select_${tableName}.`, `Selecting objects with values ${JSON.stringify(object)} from table ${tableName} on schema ${schemaName} with limit ${limit === null ? 'no' : limit}`); - dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account); - return await dmlHandler.select(schemaName, tableName, object, limit); - }, - }), {}); - return result; + validateTableNames (tableNames: string[]): void { + if (!(Array.isArray(tableNames) && tableNames.length > 0)) { + throw new Error('Schema does not have any tables. There should be at least one table.'); + } + const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + + tableNames.forEach(name => { + if (!name.includes('"') && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes + throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`); + } + }); + } + + getTableNames (schema: string): string[] { + const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS\s+)?"?(.+?)"?\s*\(/g; + const tableNames = Array.from(schema.matchAll(tableRegex), match => { + let tableName; + if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName + tableName = match[1].split('.')[1]; + tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName; + } else { + tableName = match[1]; + } + return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes + }); + this.validateTableNames(tableNames); + console.log('Retrieved the following table names from schema: ', tableNames); + return tableNames; + } + + sanitizeTableName (tableName: string): string { + tableName = tableName.startsWith('"') && tableName.endsWith('"') ? tableName.substring(1, tableName.length - 1) : tableName; + return tableName.replace(/[^a-zA-Z0-9_]/g, '_'); + } + + buildDatabaseContext (account: string, schemaName: string, schema: string, blockHeight: number): Record any> { + try { + const tables = this.getTableNames(schema); + let dmlHandler: DmlHandler | null = null; + const result = tables.reduce((prev, tableName) => { + const sanitizedTableName = this.sanitizeTableName(tableName); + const funcForTable = { + [`insert_${sanitizedTableName}`]: async (objects: any) => { + await this.writeLog(`context.db.insert_${sanitizedTableName}`, blockHeight, + `Calling context.db.insert_${sanitizedTableName}.`, + `Inserting object ${JSON.stringify(objects)} into table ${tableName} on schema ${schemaName}`); + dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account); + return await dmlHandler.insert(schemaName, tableName, Array.isArray(objects) ? objects : [objects]); + }, + [`select_${sanitizedTableName}`]: async (object: any, limit = null) => { + await this.writeLog(`context.db.select_${sanitizedTableName}`, blockHeight, + `Calling context.db.select_${sanitizedTableName}.`, + `Selecting objects with values ${JSON.stringify(object)} from table ${tableName} on schema ${schemaName} with limit ${limit === null ? 'no' : limit}`); + dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account); + return await dmlHandler.select(schemaName, tableName, object, limit); + } + }; + + return { + ...prev, + ...funcForTable + }; + }, {}); + return result; + } catch (error) { + console.warn('Caught error when generating context.db methods. Building no functions. You can still use other context object methods.\n', error); + } + + return {}; // Default to empty object if error } async setStatus (functionName: string, blockHeight: number, status: string): Promise {