Skip to content

Commit

Permalink
fix: Improve Scope of Table Name Regex to Resolve Errors
Browse files Browse the repository at this point in the history 
Regex for getting table names was not matching against valid schemas. I improved the regex to cover a much larger array of inputs and also made failure in generating the db methods not block the overall execution of the indexer. Now, it will only fail if the indexing code uses db methods but the methods fail to generate.
  • Loading branch information
@darunrs
darunrs authored Sep 6, 2023
1 parent a138bfa commit b58a0fc
Show file tree
Hide file tree
Showing 3 changed files with 251 additions and 76 deletions.
86 changes: 53 additions & 33 deletions frontend/src/utils/indexerRunner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,23 +50,8 @@ export default class IndexerRunner {
return new Promise((resolve) => setTimeout(resolve, ms));
}

validateTableNames(tableNames) {
if (!(Array.isArray(tableNames) && tableNames.length > 0)) {
throw new Error("Schema does not have any tables. There should be at least one table.");
}
const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/;

tableNames.forEach(name => {
if (!correctTableNameFormat.test(name)) {
throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`);
}
});
}

async executeIndexerFunction(height, blockDetails, indexingCode, schema, schemaName) {
let innerCode = indexingCode.match(/getBlock\s*\([^)]*\)\s*{([\s\S]*)}/)[1];
let tableNames = Array.from(schema.matchAll(/CREATE TABLE\s+"(\w+)"/g), match => match[1]); // Get first capturing group of each match
this.validateTableNames(tableNames);

if (blockDetails) {
const block = Block.fromStreamerMessage(blockDetails);
Expand All @@ -75,7 +60,7 @@ export default class IndexerRunner {
block.events()

console.log(block)
await this.runFunction(blockDetails, height, innerCode, schemaName, tableNames);
await this.runFunction(blockDetails, height, innerCode, schemaName, schema);
}
}

Expand Down Expand Up @@ -103,7 +88,7 @@ export default class IndexerRunner {
console.groupEnd()
}

async runFunction(streamerMessage, blockHeight, indexerCode, schemaName, tableNames) {
async runFunction(streamerMessage, blockHeight, indexerCode, schemaName, schema) {
const innerCodeWithBlockHelper =
`
const block = Block.fromStreamerMessage(streamerMessage);
Expand Down Expand Up @@ -162,30 +147,65 @@ export default class IndexerRunner {
log: async (message) => {
this.handleLog(blockHeight, message);
},
db: this.buildDatabaseContext(blockHeight, schemaName, tableNames)
db: this.buildDatabaseContext(blockHeight, schemaName, schema)
};

wrappedFunction(Block, streamerMessage, context);
}

buildDatabaseContext (blockHeight, schemaName, tables) {
validateTableNames(tableNames) {
if (!(Array.isArray(tableNames) && tableNames.length > 0)) {
throw new Error("Schema does not have any tables. There should be at least one table.");
}
const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/;

tableNames.forEach(name => {
if (!name.includes("\"") && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes
throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`);
}
});
}

getTableNames (schema) {
const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS\s+)?"?(.+?)"?\s*\(/g;
const tableNames = Array.from(schema.matchAll(tableRegex), match => {
let tableName;
if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName
tableName = match[1].split('.')[1];
tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName;
} else {
tableName = match[1];
}
return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes
});
this.validateTableNames(tableNames);
console.log('Retrieved the following table names from schema: ', tableNames);
return tableNames;
}

sanitizeTableName (tableName) {
tableName = tableName.startsWith('"') && tableName.endsWith('"') ? tableName.substring(1, tableName.length - 1) : tableName;
return tableName.replace(/[^a-zA-Z0-9_]/g, '_');
}

buildDatabaseContext (blockHeight, schemaName, schema) {
try {
const result = tables.reduce((prev, tableName) => ({
...prev,
[`insert_${tableName}`]: async (objects) => await this.insert(blockHeight, schemaName, tableName, objects),
[`select_${tableName}`]: async (object, limit = 0) => await this.select(blockHeight, schemaName, tableName, object, limit),
}), {});
const tables = this.getTableNames(schema);
const result = tables.reduce((prev, tableName) => {
const sanitizedTableName = this.sanitizeTableName(tableName);
const funcForTable = {
[`insert_${sanitizedTableName}`]: async (objects) => await this.insert(blockHeight, schemaName, tableName, objects),
[`select_${sanitizedTableName}`]: async (object, limit = 0) => await this.select(blockHeight, schemaName, tableName, object, limit)
};

return {
...prev,
...funcForTable
};
}, {});
return result;
} catch (error) {
console.error('Caught error when generating DB methods. Falling back to generic methods.', error);
return {
insert: async (tableName, objects) => {
this.insert(blockHeight, schemaName, tableName, objects);
},
select: async (tableName, object, limit = 0) => {
this.select(blockHeight, schemaName, tableName, object, limit);
}
};
console.warn('Caught error when generating context.db methods. Building no functions. You can still use other context object methods.\n', error);
}
}

Expand Down
132 changes: 128 additions & 4 deletions runner/src/indexer/indexer.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,98 @@ describe('Indexer unit tests', () => {
"block_timestamp" DECIMAL(20, 0) NOT NULL,
"receipt_id" VARCHAR NOT NULL,
CONSTRAINT "post_likes_pkey" PRIMARY KEY ("post_id", "account_id")
);'`;
);`;

const STRESS_TEST_SCHEMA = `
CREATE TABLE creator_quest (
account_id VARCHAR PRIMARY KEY,
num_components_created INTEGER NOT NULL DEFAULT 0,
completed BOOLEAN NOT NULL DEFAULT FALSE
);
CREATE TABLE
composer_quest (
account_id VARCHAR PRIMARY KEY,
num_widgets_composed INTEGER NOT NULL DEFAULT 0,
completed BOOLEAN NOT NULL DEFAULT FALSE
);
CREATE TABLE
"contractor - quest" (
account_id VARCHAR PRIMARY KEY,
num_contracts_deployed INTEGER NOT NULL DEFAULT 0,
completed BOOLEAN NOT NULL DEFAULT FALSE
);
CREATE TABLE
"posts" (
"id" SERIAL NOT NULL,
"account_id" VARCHAR NOT NULL,
"block_height" DECIMAL(58, 0) NOT NULL,
"receipt_id" VARCHAR NOT NULL,
"content" TEXT NOT NULL,
"block_timestamp" DECIMAL(20, 0) NOT NULL,
"accounts_liked" JSONB NOT NULL DEFAULT '[]',
"last_comment_timestamp" DECIMAL(20, 0),
CONSTRAINT "posts_pkey" PRIMARY KEY ("id")
);
CREATE TABLE
"comments" (
"id" SERIAL NOT NULL,
"post_id" SERIAL NOT NULL,
"account_id" VARCHAR NOT NULL,
"block_height" DECIMAL(58, 0) NOT NULL,
"content" TEXT NOT NULL,
"block_timestamp" DECIMAL(20, 0) NOT NULL,
"receipt_id" VARCHAR NOT NULL,
CONSTRAINT "comments_pkey" PRIMARY KEY ("id")
);
CREATE TABLE
"post_likes" (
"post_id" SERIAL NOT NULL,
"account_id" VARCHAR NOT NULL,
"block_height" DECIMAL(58, 0),
"block_timestamp" DECIMAL(20, 0) NOT NULL,
"receipt_id" VARCHAR NOT NULL,
CONSTRAINT "post_likes_pkey" PRIMARY KEY ("post_id", "account_id")
);
CREATE UNIQUE INDEX "posts_account_id_block_height_key" ON "posts" ("account_id" ASC, "block_height" ASC);
CREATE UNIQUE INDEX "comments_post_id_account_id_block_height_key" ON "comments" (
"post_id" ASC,
"account_id" ASC,
"block_height" ASC
);
CREATE INDEX
"posts_last_comment_timestamp_idx" ON "posts" ("last_comment_timestamp" DESC);
ALTER TABLE
"comments"
ADD
CONSTRAINT "comments_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION;
ALTER TABLE
"post_likes"
ADD
CONSTRAINT "post_likes_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts" ("id") ON DELETE CASCADE ON UPDATE NO ACTION;
CREATE TABLE IF NOT EXISTS
"public"."My Table1" (id serial PRIMARY KEY);
CREATE TABLE
"Another-Table" (id serial PRIMARY KEY);
CREATE TABLE
IF NOT EXISTS
"Third-Table" (id serial PRIMARY KEY);
CREATE TABLE
yet_another_table (id serial PRIMARY KEY);
`;

beforeEach(() => {
process.env = {
Expand Down Expand Up @@ -425,10 +516,43 @@ describe('Indexer unit tests', () => {
});

const indexer = new Indexer('mainnet', { DmlHandler: mockDmlHandler });
const context = indexer.buildContext(SOCIAL_SCHEMA, 'morgs.near/social_feed1', 1, 'postgres');
const context = indexer.buildContext(STRESS_TEST_SCHEMA, 'morgs.near/social_feed1', 1, 'postgres');

expect(Object.keys(context.db)).toStrictEqual(
['insert_creator_quest',
'select_creator_quest',
'insert_composer_quest',
'select_composer_quest',
'insert_contractor___quest',
'select_contractor___quest',
'insert_posts',
'select_posts',
'insert_comments',
'select_comments',
'insert_post_likes',
'select_post_likes',
'insert_My_Table1',
'select_My_Table1',
'insert_Another_Table',
'select_Another_Table',
'insert_Third_Table',
'select_Third_Table',
'insert_yet_another_table',
'select_yet_another_table']);
});

test('indexer builds context and returns empty array if failed to generate db methods', async () => {
const mockDmlHandler: any = jest.fn().mockImplementation(() => {
return {
insert: jest.fn().mockReturnValue(true),
select: jest.fn().mockReturnValue(true)
};
});

const indexer = new Indexer('mainnet', { DmlHandler: mockDmlHandler });
const context = indexer.buildContext('', 'morgs.near/social_feed1', 1, 'postgres');

// These calls would fail on a real database, but we are merely checking to ensure they exist
expect(Object.keys(context.db)).toStrictEqual(['insert_posts', 'select_posts', 'insert_comments', 'select_comments', 'insert_post_likes', 'select_post_likes']);
expect(Object.keys(context.db)).toStrictEqual([]);
});

test('Indexer.runFunctions() allows imperative execution of GraphQL operations', async () => {
Expand Down
109 changes: 70 additions & 39 deletions runner/src/indexer/indexer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,29 +186,7 @@ export default class Indexer {
].reduce((acc, val) => val(acc), indexerFunction);
}

validateTableNames (tableNames: string[]): void {
if (!(tableNames.length > 0)) {
throw new Error('Schema does not have any tables. There should be at least one table.');
}
const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/;

tableNames.forEach((name: string) => {
if (!correctTableNameFormat.test(name)) {
throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`);
}
});
}

getTableNames (schema: string): string[] {
const tableNameMatcher = /CREATE TABLE\s+"(\w+)"/g;
const tableNames = Array.from(schema.matchAll(tableNameMatcher), match => match[1]); // Get first capturing group of each match
this.validateTableNames(tableNames);
console.log('Retrieved the following table names from schema: ', tableNames);
return tableNames;
}

buildContext (schema: string, functionName: string, blockHeight: number, hasuraRoleName: string): Context {
const tables = this.getTableNames(schema);
const account = functionName.split('/')[0].replace(/[.-]/g, '_');
const functionNameWithoutAccount = functionName.split('/')[1].replace(/[.-]/g, '_');
const schemaName = functionName.replace(/[^a-zA-Z0-9]/g, '_');
Expand Down Expand Up @@ -237,26 +215,79 @@ export default class Indexer {
fetchFromSocialApi: async (path, options) => {
return await this.deps.fetch(`https://api.near.social${path}`, options);
},
db: this.buildDatabaseContext(account, schemaName, tables, blockHeight)
db: this.buildDatabaseContext(account, schemaName, schema, blockHeight)
};
}

buildDatabaseContext (account: string, schemaName: string, tables: string[], blockHeight: number): Record<string, (...args: any[]) => any> {
let dmlHandler: DmlHandler | null = null;
const result = tables.reduce((prev, tableName) => ({
...prev,
[`insert_${tableName}`]: async (objects: any) => {
await this.writeLog(`context.db.insert_${tableName}`, blockHeight, `Calling context.db.insert_${tableName}.`, `Inserting object ${JSON.stringify(objects)} into table ${tableName} on schema ${schemaName}`);
dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account);
return await dmlHandler.insert(schemaName, tableName, Array.isArray(objects) ? objects : [objects]);
},
[`select_${tableName}`]: async (object: any, limit = null) => {
await this.writeLog(`context.db.select_${tableName}`, blockHeight, `Calling context.db.select_${tableName}.`, `Selecting objects with values ${JSON.stringify(object)} from table ${tableName} on schema ${schemaName} with limit ${limit === null ? 'no' : limit}`);
dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account);
return await dmlHandler.select(schemaName, tableName, object, limit);
},
}), {});
return result;
validateTableNames (tableNames: string[]): void {
if (!(Array.isArray(tableNames) && tableNames.length > 0)) {
throw new Error('Schema does not have any tables. There should be at least one table.');
}
const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/;

tableNames.forEach(name => {
if (!name.includes('"') && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes
throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`);
}
});
}

getTableNames (schema: string): string[] {
const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS\s+)?"?(.+?)"?\s*\(/g;
const tableNames = Array.from(schema.matchAll(tableRegex), match => {
let tableName;
if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName
tableName = match[1].split('.')[1];
tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName;
} else {
tableName = match[1];
}
return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes
});
this.validateTableNames(tableNames);
console.log('Retrieved the following table names from schema: ', tableNames);
return tableNames;
}

sanitizeTableName (tableName: string): string {
tableName = tableName.startsWith('"') && tableName.endsWith('"') ? tableName.substring(1, tableName.length - 1) : tableName;
return tableName.replace(/[^a-zA-Z0-9_]/g, '_');
}

buildDatabaseContext (account: string, schemaName: string, schema: string, blockHeight: number): Record<string, (...args: any[]) => any> {
try {
const tables = this.getTableNames(schema);
let dmlHandler: DmlHandler | null = null;
const result = tables.reduce((prev, tableName) => {
const sanitizedTableName = this.sanitizeTableName(tableName);
const funcForTable = {
[`insert_${sanitizedTableName}`]: async (objects: any) => {
await this.writeLog(`context.db.insert_${sanitizedTableName}`, blockHeight,
`Calling context.db.insert_${sanitizedTableName}.`,
`Inserting object ${JSON.stringify(objects)} into table ${tableName} on schema ${schemaName}`);
dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account);
return await dmlHandler.insert(schemaName, tableName, Array.isArray(objects) ? objects : [objects]);
},
[`select_${sanitizedTableName}`]: async (object: any, limit = null) => {
await this.writeLog(`context.db.select_${sanitizedTableName}`, blockHeight,
`Calling context.db.select_${sanitizedTableName}.`,
`Selecting objects with values ${JSON.stringify(object)} from table ${tableName} on schema ${schemaName} with limit ${limit === null ? 'no' : limit}`);
dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account);
return await dmlHandler.select(schemaName, tableName, object, limit);
}
};

return {
...prev,
...funcForTable
};
}, {});
return result;
} catch (error) {
console.warn('Caught error when generating context.db methods. Building no functions. You can still use other context object methods.\n', error);
}

return {}; // Default to empty object if error
}

async setStatus (functionName: string, blockHeight: number, status: string): Promise<any> {
Expand Down

0 comments on commit b58a0fc

Please sign in to comment.