Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Improve Scope of Table Name Regex to Resolve Errors #188

Merged
merged 4 commits into from
Sep 6, 2023
Merged

fix: Improve Scope of Table Name Regex to Resolve Errors #188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
71eba24
fix: Improve table name regex coverage
darunrs Aug 29, 2023
f3232df
fix: Sanitize table names for function generation
darunrs Aug 29, 2023
6ec6489
Make frontend and runner implementation the same
darunrs Aug 29, 2023
67908d3
Address comments in PR
darunrs Aug 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 53 additions & 33 deletions frontend/src/utils/indexerRunner.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,23 +50,8 @@ export default class IndexerRunner {
return new Promise((resolve) => setTimeout(resolve, ms));
}

validateTableNames(tableNames) {
if (!(Array.isArray(tableNames) && tableNames.length > 0)) {
throw new Error("Schema does not have any tables. There should be at least one table.");
}
const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/;

tableNames.forEach(name => {
if (!correctTableNameFormat.test(name)) {
throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`);
}
});
}

async executeIndexerFunction(height, blockDetails, indexingCode, schema, schemaName) {
let innerCode = indexingCode.match(/getBlock\s*\([^)]*\)\s*{([\s\S]*)}/)[1];
let tableNames = Array.from(schema.matchAll(/CREATE TABLE\s+"(\w+)"/g), match => match[1]); // Get first capturing group of each match
this.validateTableNames(tableNames);

if (blockDetails) {
const block = Block.fromStreamerMessage(blockDetails);
Expand All @@ -75,7 +60,7 @@ export default class IndexerRunner {
block.events()

console.log(block)
await this.runFunction(blockDetails, height, innerCode, schemaName, tableNames);
await this.runFunction(blockDetails, height, innerCode, schemaName, schema);
}
}

Expand Down Expand Up @@ -103,7 +88,7 @@ export default class IndexerRunner {
console.groupEnd()
}

async runFunction(streamerMessage, blockHeight, indexerCode, schemaName, tableNames) {
async runFunction(streamerMessage, blockHeight, indexerCode, schemaName, schema) {
const innerCodeWithBlockHelper =
`
const block = Block.fromStreamerMessage(streamerMessage);
Expand Down Expand Up @@ -162,30 +147,65 @@ export default class IndexerRunner {
log: async (message) => {
this.handleLog(blockHeight, message);
},
db: this.buildDatabaseContext(blockHeight, schemaName, tableNames)
db: this.buildDatabaseContext(blockHeight, schemaName, schema)
};

wrappedFunction(Block, streamerMessage, context);
}

buildDatabaseContext (blockHeight, schemaName, tables) {
validateTableNames(tableNames) {
if (!(Array.isArray(tableNames) && tableNames.length > 0)) {
throw new Error("Schema does not have any tables. There should be at least one table.");
}
const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/;

tableNames.forEach(name => {
if (!name.includes("\"") && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes
throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`);
}
});
}

getTableNames (schema) {
const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS\s+)?"?(.+?)"?\s*\(/g;
const tableNames = Array.from(schema.matchAll(tableRegex), match => {
let tableName;
if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName
tableName = match[1].split('.')[1];
tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName;
} else {
tableName = match[1];
}
return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes
});
this.validateTableNames(tableNames);
console.log('Retrieved the following table names from schema: ', tableNames);
return tableNames;
}

sanitizeTableName (tableName) {
tableName = tableName.startsWith('"') && tableName.endsWith('"') ? tableName.substring(1, tableName.length - 1) : tableName;
return tableName.replace(/[^a-zA-Z0-9_]/g, '_');
}

buildDatabaseContext (blockHeight, schemaName, schema) {
try {
const result = tables.reduce((prev, tableName) => ({
...prev,
[`insert_${tableName}`]: async (objects) => await this.insert(blockHeight, schemaName, tableName, objects),
[`select_${tableName}`]: async (object, limit = 0) => await this.select(blockHeight, schemaName, tableName, object, limit),
}), {});
const tables = this.getTableNames(schema);
const result = tables.reduce((prev, tableName) => {
const sanitizedTableName = this.sanitizeTableName(tableName);
const funcForTable = {
[`insert_${sanitizedTableName}`]: async (objects) => await this.insert(blockHeight, schemaName, tableName, objects),
[`select_${sanitizedTableName}`]: async (object, limit = 0) => await this.select(blockHeight, schemaName, tableName, object, limit)
};

return {
...prev,
...funcForTable
};
}, {});
return result;
} catch (error) {
console.error('Caught error when generating DB methods. Falling back to generic methods.', error);
return {
insert: async (tableName, objects) => {
this.insert(blockHeight, schemaName, tableName, objects);
},
select: async (tableName, object, limit = 0) => {
this.select(blockHeight, schemaName, tableName, object, limit);
}
};
console.warn('Caught error when generating context.db methods. Building no functions. You can still use other context object methods.\n', error);
}
}

Expand Down
132 changes: 128 additions & 4 deletions runner/src/indexer/indexer.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,98 @@ describe('Indexer unit tests', () => {
"block_timestamp" DECIMAL(20, 0) NOT NULL,
"receipt_id" VARCHAR NOT NULL,
CONSTRAINT "post_likes_pkey" PRIMARY KEY ("post_id", "account_id")
);'`;
);`;

const STRESS_TEST_SCHEMA = `
CREATE TABLE creator_quest (
account_id VARCHAR PRIMARY KEY,
num_components_created INTEGER NOT NULL DEFAULT 0,
completed BOOLEAN NOT NULL DEFAULT FALSE
);

CREATE TABLE
composer_quest (
account_id VARCHAR PRIMARY KEY,
num_widgets_composed INTEGER NOT NULL DEFAULT 0,
completed BOOLEAN NOT NULL DEFAULT FALSE
);

CREATE TABLE
"contractor - quest" (
account_id VARCHAR PRIMARY KEY,
num_contracts_deployed INTEGER NOT NULL DEFAULT 0,
completed BOOLEAN NOT NULL DEFAULT FALSE
);

CREATE TABLE
"posts" (
"id" SERIAL NOT NULL,
"account_id" VARCHAR NOT NULL,
"block_height" DECIMAL(58, 0) NOT NULL,
"receipt_id" VARCHAR NOT NULL,
"content" TEXT NOT NULL,
"block_timestamp" DECIMAL(20, 0) NOT NULL,
"accounts_liked" JSONB NOT NULL DEFAULT '[]',
"last_comment_timestamp" DECIMAL(20, 0),
CONSTRAINT "posts_pkey" PRIMARY KEY ("id")
);

CREATE TABLE
"comments" (
"id" SERIAL NOT NULL,
"post_id" SERIAL NOT NULL,
"account_id" VARCHAR NOT NULL,
"block_height" DECIMAL(58, 0) NOT NULL,
"content" TEXT NOT NULL,
"block_timestamp" DECIMAL(20, 0) NOT NULL,
"receipt_id" VARCHAR NOT NULL,
CONSTRAINT "comments_pkey" PRIMARY KEY ("id")
);

CREATE TABLE
"post_likes" (
"post_id" SERIAL NOT NULL,
"account_id" VARCHAR NOT NULL,
"block_height" DECIMAL(58, 0),
"block_timestamp" DECIMAL(20, 0) NOT NULL,
"receipt_id" VARCHAR NOT NULL,
CONSTRAINT "post_likes_pkey" PRIMARY KEY ("post_id", "account_id")
);

CREATE UNIQUE INDEX "posts_account_id_block_height_key" ON "posts" ("account_id" ASC, "block_height" ASC);

CREATE UNIQUE INDEX "comments_post_id_account_id_block_height_key" ON "comments" (
"post_id" ASC,
"account_id" ASC,
"block_height" ASC
);

CREATE INDEX
"posts_last_comment_timestamp_idx" ON "posts" ("last_comment_timestamp" DESC);

ALTER TABLE
"comments"
ADD
CONSTRAINT "comments_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION;

ALTER TABLE
"post_likes"
ADD
CONSTRAINT "post_likes_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts" ("id") ON DELETE CASCADE ON UPDATE NO ACTION;

CREATE TABLE IF NOT EXISTS
"public"."My Table1" (id serial PRIMARY KEY);

CREATE TABLE
"Another-Table" (id serial PRIMARY KEY);

CREATE TABLE
IF NOT EXISTS
"Third-Table" (id serial PRIMARY KEY);

CREATE TABLE
yet_another_table (id serial PRIMARY KEY);
`;

beforeEach(() => {
process.env = {
Expand Down Expand Up @@ -425,10 +516,43 @@ describe('Indexer unit tests', () => {
});

const indexer = new Indexer('mainnet', { DmlHandler: mockDmlHandler });
const context = indexer.buildContext(SOCIAL_SCHEMA, 'morgs.near/social_feed1', 1, 'postgres');
const context = indexer.buildContext(STRESS_TEST_SCHEMA, 'morgs.near/social_feed1', 1, 'postgres');

expect(Object.keys(context.db)).toStrictEqual(
['insert_creator_quest',
'select_creator_quest',
'insert_composer_quest',
'select_composer_quest',
'insert_contractor___quest',
'select_contractor___quest',
'insert_posts',
'select_posts',
'insert_comments',
'select_comments',
'insert_post_likes',
'select_post_likes',
'insert_My_Table1',
'select_My_Table1',
'insert_Another_Table',
'select_Another_Table',
'insert_Third_Table',
'select_Third_Table',
'insert_yet_another_table',
'select_yet_another_table']);
});

test('indexer builds context and returns empty array if failed to generate db methods', async () => {
const mockDmlHandler: any = jest.fn().mockImplementation(() => {
return {
insert: jest.fn().mockReturnValue(true),
select: jest.fn().mockReturnValue(true)
};
});

const indexer = new Indexer('mainnet', { DmlHandler: mockDmlHandler });
const context = indexer.buildContext('', 'morgs.near/social_feed1', 1, 'postgres');

// These calls would fail on a real database, but we are merely checking to ensure they exist
expect(Object.keys(context.db)).toStrictEqual(['insert_posts', 'select_posts', 'insert_comments', 'select_comments', 'insert_post_likes', 'select_post_likes']);
expect(Object.keys(context.db)).toStrictEqual([]);
});

test('Indexer.runFunctions() allows imperative execution of GraphQL operations', async () => {
Expand Down
109 changes: 70 additions & 39 deletions runner/src/indexer/indexer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,29 +186,7 @@ export default class Indexer {
].reduce((acc, val) => val(acc), indexerFunction);
}

validateTableNames (tableNames: string[]): void {
if (!(tableNames.length > 0)) {
throw new Error('Schema does not have any tables. There should be at least one table.');
}
const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/;

tableNames.forEach((name: string) => {
if (!correctTableNameFormat.test(name)) {
throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`);
}
});
}

getTableNames (schema: string): string[] {
const tableNameMatcher = /CREATE TABLE\s+"(\w+)"/g;
const tableNames = Array.from(schema.matchAll(tableNameMatcher), match => match[1]); // Get first capturing group of each match
this.validateTableNames(tableNames);
console.log('Retrieved the following table names from schema: ', tableNames);
return tableNames;
}

buildContext (schema: string, functionName: string, blockHeight: number, hasuraRoleName: string): Context {
const tables = this.getTableNames(schema);
const account = functionName.split('/')[0].replace(/[.-]/g, '_');
const functionNameWithoutAccount = functionName.split('/')[1].replace(/[.-]/g, '_');
const schemaName = functionName.replace(/[^a-zA-Z0-9]/g, '_');
Expand Down Expand Up @@ -237,26 +215,79 @@ export default class Indexer {
fetchFromSocialApi: async (path, options) => {
return await this.deps.fetch(`https://api.near.social${path}`, options);
},
db: this.buildDatabaseContext(account, schemaName, tables, blockHeight)
db: this.buildDatabaseContext(account, schemaName, schema, blockHeight)
};
}

buildDatabaseContext (account: string, schemaName: string, tables: string[], blockHeight: number): Record<string, (...args: any[]) => any> {
let dmlHandler: DmlHandler | null = null;
const result = tables.reduce((prev, tableName) => ({
...prev,
[`insert_${tableName}`]: async (objects: any) => {
await this.writeLog(`context.db.insert_${tableName}`, blockHeight, `Calling context.db.insert_${tableName}.`, `Inserting object ${JSON.stringify(objects)} into table ${tableName} on schema ${schemaName}`);
dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account);
return await dmlHandler.insert(schemaName, tableName, Array.isArray(objects) ? objects : [objects]);
},
[`select_${tableName}`]: async (object: any, limit = null) => {
await this.writeLog(`context.db.select_${tableName}`, blockHeight, `Calling context.db.select_${tableName}.`, `Selecting objects with values ${JSON.stringify(object)} from table ${tableName} on schema ${schemaName} with limit ${limit === null ? 'no' : limit}`);
dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account);
return await dmlHandler.select(schemaName, tableName, object, limit);
},
}), {});
return result;
validateTableNames (tableNames: string[]): void {
if (!(Array.isArray(tableNames) && tableNames.length > 0)) {
throw new Error('Schema does not have any tables. There should be at least one table.');
}
const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/;

tableNames.forEach(name => {
if (!name.includes('"') && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes
throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`);
}
});
}

getTableNames (schema: string): string[] {
const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS\s+)?"?(.+?)"?\s*\(/g;
const tableNames = Array.from(schema.matchAll(tableRegex), match => {
let tableName;
if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName
tableName = match[1].split('.')[1];
tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName;
} else {
tableName = match[1];
}
return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes
});
this.validateTableNames(tableNames);
console.log('Retrieved the following table names from schema: ', tableNames);
return tableNames;
}

sanitizeTableName (tableName: string): string {
tableName = tableName.startsWith('"') && tableName.endsWith('"') ? tableName.substring(1, tableName.length - 1) : tableName;
return tableName.replace(/[^a-zA-Z0-9_]/g, '_');
}

buildDatabaseContext (account: string, schemaName: string, schema: string, blockHeight: number): Record<string, (...args: any[]) => any> {
try {
const tables = this.getTableNames(schema);
let dmlHandler: DmlHandler | null = null;
const result = tables.reduce((prev, tableName) => {
const sanitizedTableName = this.sanitizeTableName(tableName);
const funcForTable = {
[`insert_${sanitizedTableName}`]: async (objects: any) => {
await this.writeLog(`context.db.insert_${sanitizedTableName}`, blockHeight,
`Calling context.db.insert_${sanitizedTableName}.`,
`Inserting object ${JSON.stringify(objects)} into table ${tableName} on schema ${schemaName}`);
dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account);
return await dmlHandler.insert(schemaName, tableName, Array.isArray(objects) ? objects : [objects]);
},
[`select_${sanitizedTableName}`]: async (object: any, limit = null) => {
await this.writeLog(`context.db.select_${sanitizedTableName}`, blockHeight,
`Calling context.db.select_${sanitizedTableName}.`,
`Selecting objects with values ${JSON.stringify(object)} from table ${tableName} on schema ${schemaName} with limit ${limit === null ? 'no' : limit}`);
dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account);
return await dmlHandler.select(schemaName, tableName, object, limit);
}
};

return {
...prev,
...funcForTable
};
}, {});
return result;
} catch (error) {
console.warn('Caught error when generating context.db methods. Building no functions. You can still use other context object methods.\n', error);
}

return {}; // Default to empty object if error
}

async setStatus (functionName: string, blockHeight: number, status: string): Promise<any> {
Expand Down