diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/heavilyNormalizeViewDDL.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/heavilyNormalizeViewDDL.ts deleted file mode 100644 index 61a878a..0000000 --- a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/heavilyNormalizeViewDDL.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { - flattenSqlByReferencingAndTokenizingSubqueries, - hydrateSqlByReferencingAndReplacingSubqueryTokens, -} from '../../../../../__nonpublished_modules__/sql-subquery-tokenizer'; - -// TODO: find a better way of doing this rather than all this ad hock normalization... or atleast refactor this normalization to make it less dependent on previous transformation (maybe split parts into their own fns) -// TODO: support subqueries. right now we are not normalizing them. ideally we would flatten and run normalization on root + all subqueries, and then merge back -export const heavilyNormalizeViewDDL = ({ ddl }: { ddl: string }) => { - // 0. make all "as" words, except the first, lowercase (for some reason, SHOW CREATE lowercases all tokens EXCEPT `as`, which it explicitly upper cases...) - const casedAsDdl = ddl - .replace(/ AS /g, ' as ') // /g to apply to each - .replace(/ as /, ' AS '); // no /g so only apply to first case - - // 1. lowercase everything in the query, since show create does :upside_down_smiley: - const casedAsDdlParts = casedAsDdl.split(' AS '); - const casedDdl = [casedAsDdlParts[0], casedAsDdlParts[1].toLowerCase()].join(' AS '); - - // 2. strip out all of the back ticks. until we have an example where that breaks something, they're just too much to force users to have to maintain - const ddlWithoutBackticks = casedDdl.replace(/`/g, ''); - - // 3. strip out all of the parenthesis (since SHOW CREATE adds them everywhere and thats too much to force users to maintain); unideal since it modifies the meaning, but this is only relevant for diffing - const { flattenedSql: flattenedDdl, references } = flattenSqlByReferencingAndTokenizingSubqueries({ - sql: ddlWithoutBackticks, - }); - const whereCasing = !!flattenedDdl.match(/\sWHERE\s/) ? 'WHERE' : 'where'; // determine if user is using upper case or lower case "where" - const ddlSplitOnWhere = flattenedDdl.split(whereCasing); - if (ddlSplitOnWhere.length > 2) { - throw new Error('found more than two parts of DDL after splitting on where; unexpected'); // fail fast - } - const ddlWithoutParens = - ddlSplitOnWhere.length === 2 - ? ddlSplitOnWhere[0] + '\n' + whereCasing + '\n' + ddlSplitOnWhere[1].replace(/[\(\)]/g, '') // tslint:disable-line prefer-template - : ddlSplitOnWhere[0]; // if no where clause, then nothing to replace - const rehydratedDdl = hydrateSqlByReferencingAndReplacingSubqueryTokens({ - flattenedSql: ddlWithoutParens, - references, - }); - const ddlWithoutParensInWhereConditions = rehydratedDdl; - - // 4. strip out the first and last paren of the "FROM" section, IF it is defined; show create likes to wrap the whole "FROM" section w/ parens... - const ddlHasParenOpenRightAfterFromClause = !!ddlWithoutParensInWhereConditions.match(/\sfrom\s+\(/gi); - let ddlWithoutParenStartingAndEndingFromClause = ddlWithoutParensInWhereConditions; - if (ddlHasParenOpenRightAfterFromClause) { - // flatten the ddl so that we only have one FROM clause - const { flattenedSql: flattenedDdl, references } = flattenSqlByReferencingAndTokenizingSubqueries({ - sql: ddlWithoutParensInWhereConditions, - }); - - // split DDL on the from statement, since we know it exists - const flattenedDdlParts = flattenedDdl.split(/\sfrom\s/gi); - if (flattenedDdlParts.length !== 2) throw new Error('not exactly two parts after splitting on from; unexpected'); // fail fast - - // remove the parens from beginning and end - const fromClauseWithoutFirstOrLastParens = flattenedDdlParts[1].replace(/^\s*\(/, ' ').replace(/\)\s*$/, ' '); - - // join them back and the ddl w/o this mess - const flattenedDdlWithoutOpenCloseParenInFromClause = - flattenedDdlParts[0] + '\nfrom\n' + fromClauseWithoutFirstOrLastParens; // tslint:disable-line prefer-template - - // rehydrate them - const hydratedDdlWithoutOpenCloseParenInFromClause = hydrateSqlByReferencingAndReplacingSubqueryTokens({ - flattenedSql: flattenedDdlWithoutOpenCloseParenInFromClause, - references, - }); - - // and set the variable accessible to full fn scope - ddlWithoutParenStartingAndEndingFromClause = hydratedDdlWithoutOpenCloseParenInFromClause; - } - - // 5. strip out the "double parenthesis" that SHOW CREATE likes to put on the "join on" statements - const ddlWithoutDoubleParens = ddlWithoutParenStartingAndEndingFromClause.replace( - / on\(\((\w+\.\w+\s=\s\w+\.\w+)\)\)/g, - ' on $1 ', - ); - - // 6. strip out the final `;` if it exists - const ddlWithoutFinalSemicolon = ddlWithoutDoubleParens.replace(/;/g, ''); - - // 7. replace `,(` patterns w/ space in between, since our formatter downstream does not like that - const ddlWithoutTouchingCommaParen = ddlWithoutFinalSemicolon.replace(/,\(/g, ', ('); - - // 8. return that result - return ddlWithoutTouchingCommaParen; -}; diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/normalizeDDLToSupportLossyShowCreateStatements.test.integration.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/normalizeDDLToSupportLossyShowCreateStatements.test.integration.ts index 56bf213..7ec1c05 100644 --- a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/normalizeDDLToSupportLossyShowCreateStatements.test.integration.ts +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/normalizeDDLToSupportLossyShowCreateStatements.test.integration.ts @@ -137,4 +137,81 @@ CREATE VIEW view_spaceship_with_cargo AS }); expect(normalizedShowCreateDefSql).toEqual(normalizedUserSqlDef); }); + it('should find no change on this real world example where recursive support for views is required', async () => { + // define the view sql + const userDefSql = ` + CREATE VIEW \`view_contractor_current\` AS + SELECT + s.id, + s.uuid, + s.name, + ( + SELECT GROUP_CONCAT(contractor_version_to_contractor_license.contractor_license_id ORDER BY contractor_version_to_contractor_license.array_order_index asc separator ',') + FROM contractor_version_to_contractor_license WHERE contractor_version_to_contractor_license.contractor_version_id = v.id + ) as license_ids, + ( + SELECT GROUP_CONCAT(contractor_version_to_contact_method.contact_method_id ORDER BY contractor_version_to_contact_method.array_order_index asc separator ',') + FROM contractor_version_to_contact_method WHERE contractor_version_to_contact_method.contractor_version_id = v.id + ) as proposed_suggestion_change_ids, + s.created_at, + v.effective_at, + v.created_at as updated_at + FROM contractor s + JOIN contractor_cvp cvp ON s.id = cvp.contractor_id + JOIN contractor_version v ON v.id = cvp.contractor_version_id; +; + `; + + // apply the tables needed to apply the view + await connection.query({ sql: 'DROP TABLE IF EXISTS contractor' }); + await connection.query({ + sql: 'CREATE TABLE contractor ( id BIGINT, uuid VARCHAR(255), name VARCHAR(255), created_at DATETIME )', + }); + await connection.query({ sql: 'DROP TABLE IF EXISTS contractor_version' }); + await connection.query({ + sql: + 'CREATE TABLE contractor_version ( id BIGINT, contractor_id BIGINT, created_at DATETIME, effective_at DATETIME)', + }); + + await connection.query({ sql: 'DROP TABLE IF EXISTS contractor_cvp' }); + await connection.query({ + sql: 'CREATE TABLE contractor_cvp ( contractor_id BIGINT, contractor_version_id BIGINT)', + }); + await connection.query({ sql: 'DROP TABLE IF EXISTS contractor_version_to_contractor_license' }); + await connection.query({ + sql: + 'CREATE TABLE contractor_version_to_contractor_license ( contractor_version_id BIGINT, contractor_license_id BIGINT, array_order_index INT )', + }); + await connection.query({ sql: 'DROP TABLE IF EXISTS contractor_version_to_contact_method' }); + await connection.query({ + sql: + 'CREATE TABLE contractor_version_to_contact_method ( contractor_version_id BIGINT, contact_method_id BIGINT, array_order_index INT )', + }); + + // apply the sql + await connection.query({ sql: 'DROP VIEW IF EXISTS view_contractor_current;' }); // ensure possible previous state does not affect test + await connection.query({ sql: userDefSql }); + + // get get the SHOW CREATE sql + const liveResource = await getLiveResourceDefinitionFromDatabase({ + connection, + resourceName: 'view_contractor_current', + resourceType: ResourceType.VIEW, + }); + const showCreateDefSql = liveResource.sql; + + // check that we normalize to the same thing + const normalizedUserSqlDef = normalizeDDLToSupportLossyShowCreateStatements({ + ddl: userDefSql, + resourceType: ResourceType.VIEW, + }); + const normalizedShowCreateDefSql = stripIrrelevantContentFromResourceDDL({ + ddl: normalizeDDLToSupportLossyShowCreateStatements({ + ddl: showCreateDefSql, + resourceType: ResourceType.VIEW, + }), + resourceType: ResourceType.VIEW, + }); + expect(normalizedShowCreateDefSql).toEqual(normalizedUserSqlDef); + }); }); diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/normalizeDDLToSupportLossyShowCreateStatements.test.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/normalizeDDLToSupportLossyShowCreateStatements.test.ts index 6428de6..b4dbd59 100644 --- a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/normalizeDDLToSupportLossyShowCreateStatements.test.ts +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/normalizeDDLToSupportLossyShowCreateStatements.test.ts @@ -49,4 +49,12 @@ where 1=1 }); expect(cleanedUserDef).toEqual(cleanedShowCreateDef); }); + it('should be able to normalize this ddl so that show create def looks reasonable', () => { + const showCreateDdlExample = + "CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`%` SQL SECURITY DEFINER VIEW `view_contractor_current` AS select `s`.`id` AS `id`,`s`.`uuid` AS `uuid`,`s`.`name` AS `name`,(select group_concat(`contractor_version_to_contractor_license`.`contractor_license_id` order by `contractor_version_to_contractor_license`.`array_order_index` ASC separator ',') from `contractor_version_to_contractor_license` where (`contractor_version_to_contractor_license`.`contractor_version_id` = `v`.`id`)) AS `license_ids`,(select group_concat(`contractor_version_to_contact_method`.`contact_method_id` order by `contractor_version_to_contact_method`.`array_order_index` ASC separator ',') from `contractor_version_to_contact_method` where (`contractor_version_to_contact_method`.`contractor_version_id` = `v`.`id`)) AS `proposed_suggestion_change_ids`,`s`.`created_at` AS `created_at`,`v`.`effective_at` AS `effective_at`,`v`.`created_at` AS `updated_at` from ((`contractor` `s` join `contractor_cvp` `cvp` on((`s`.`id` = `cvp`.`contractor_id`))) join `contractor_version` `v` on((`v`.`id` = `cvp`.`contractor_version_id`)))"; + normalizeDDLToSupportLossyShowCreateStatements({ + ddl: showCreateDdlExample, + resourceType: ResourceType.VIEW, + }); + }); }); diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/normalizeDDLToSupportLossyShowCreateStatements.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/normalizeDDLToSupportLossyShowCreateStatements.ts index c54301f..036f247 100644 --- a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/normalizeDDLToSupportLossyShowCreateStatements.ts +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/normalizeDDLToSupportLossyShowCreateStatements.ts @@ -1,7 +1,8 @@ -import { ResourceType } from '../../../../../types'; -import strip from 'sql-strip-comments'; import sqlFormatter from 'sql-formatter'; -import { heavilyNormalizeViewDDL } from './heavilyNormalizeViewDDL'; +import strip from 'sql-strip-comments'; + +import { ResourceType } from '../../../../../types'; +import { recursivelyHeavilyNormalizeViewDdl } from './recursivelyHeavilyNormalizeViewDdl/recursivelyHeavilyNormalizeViewDdl'; const RESOURCES_WITH_LOSSY_SHOW_CREATE_STATEMENTS = [ResourceType.TABLE, ResourceType.VIEW]; @@ -28,7 +29,7 @@ export const normalizeDDLToSupportLossyShowCreateStatements = ({ // if view, fix a few fun "features" that the SHOW CREATE statement adds const extraNormalizedDDL = resourceType === ResourceType.VIEW - ? heavilyNormalizeViewDDL({ ddl: strippedDDL }) // SHOW CREATE for view has a lot of fun features, so extra normalization is needed + ? recursivelyHeavilyNormalizeViewDdl({ ddl: strippedDDL }) // SHOW CREATE for view has a lot of fun features, so extra normalization is needed : strippedDDL; // apply formatting, since views have bad formatting diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/__snapshots__/removeParenthesisSurroundingJoinsInFromClause.test.ts.snap b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/__snapshots__/removeParenthesisSurroundingJoinsInFromClause.test.ts.snap new file mode 100644 index 0000000..122b45b --- /dev/null +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/__snapshots__/removeParenthesisSurroundingJoinsInFromClause.test.ts.snap @@ -0,0 +1,19 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`removeParenthesisSurroundingJoinsInFromClause should remove them from every join 1`] = ` +"select + s.id as id, + s.uuid as uuid, + s.name as name, + s.created_at as created_at, + v.effective_at as effective_at, + v.created_at as updated_at +from + + + contractor s + join contractor_cvp cvp on s.id = cvp.contractor_id + + join contractor_version v on v.id = cvp.contractor_version_id + " +`; diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/__snapshots__/removeRedundantAliasDeclarations.test.ts.snap b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/__snapshots__/removeRedundantAliasDeclarations.test.ts.snap new file mode 100644 index 0000000..098309a --- /dev/null +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/__snapshots__/removeRedundantAliasDeclarations.test.ts.snap @@ -0,0 +1,25 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`removeRedundantAliasDeclarations redundant declarations to be removed and nonredundant to remain 1`] = ` +" +CREATE VIEW \`view_contractor_current\` AS +SELECT + s.id, + s.uuid, + s.name, + ( + SELECT GROUP_CONCAT(contractor_version_to_contractor_license.contractor_license_id ORDER BY contractor_version_to_contractor_license.array_order_index) + FROM contractor_version_to_contractor_license WHERE contractor_version_to_contractor_license.contractor_version_id = v.id + ) as license_ids, + ( + SELECT GROUP_CONCAT(contractor_version_to_contact_method.contact_method_id ORDER BY contractor_version_to_contact_method.array_order_index) + FROM contractor_version_to_contact_method WHERE contractor_version_to_contact_method.contractor_version_id = v.id + ) as proposed_suggestion_change_ids, + s.created_at, + v.effective_at, + v.created_at as updated_at +FROM contractor s +JOIN contractor_cvp cvp ON s.id = cvp.contractor_id +JOIN contractor_version v ON v.id = cvp.contractor_version_id; + " +`; diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/addSpacesBetweenCommaParenthesisOccurances.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/addSpacesBetweenCommaParenthesisOccurances.ts new file mode 100644 index 0000000..07dc593 --- /dev/null +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/addSpacesBetweenCommaParenthesisOccurances.ts @@ -0,0 +1,6 @@ +/** + * replace `,(` patterns w/ space in between, since our formatter downstream does not like that + */ +export const addSpacesBetweenCommaParenthesisOccurances = ({ ddl }: { ddl: string }) => { + return ddl.replace(/,\(/g, ', ('); +}; diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/lowercaseAllAsTokensExceptFirstOne.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/lowercaseAllAsTokensExceptFirstOne.ts new file mode 100644 index 0000000..8a5c1c1 --- /dev/null +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/lowercaseAllAsTokensExceptFirstOne.ts @@ -0,0 +1,10 @@ +/** + * make all "as" words, except the first, lowercase + * + * for some reason, SHOW CREATE lowercases all tokens EXCEPT `as`, which it explicitly upper cases...) + */ +export const lowercaseAllAsTokensExceptFirstOne = ({ ddl }: { ddl: string }) => { + return ddl + .replace(/ AS /g, ' as ') // /g to apply to each + .replace(/ as /, ' AS '); // no /g so only apply to first case +}; diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/lowercaseEverythingAfterFirstAs.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/lowercaseEverythingAfterFirstAs.ts new file mode 100644 index 0000000..864ba12 --- /dev/null +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/lowercaseEverythingAfterFirstAs.ts @@ -0,0 +1,8 @@ +/** + * lowercase everything in the query, since show create does :upside_down_smiley: + */ +export const lowercaseEverythingAfterFirstAs = ({ ddl }: { ddl: string }) => { + const casedAsDdlParts = ddl.split(' AS '); + const casedDdl = [casedAsDdlParts[0], casedAsDdlParts[1].toLowerCase()].join(' AS '); + return casedDdl; +}; diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeAllBackticks.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeAllBackticks.ts new file mode 100644 index 0000000..bfe802b --- /dev/null +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeAllBackticks.ts @@ -0,0 +1,8 @@ +/** + * strip out all of the back ticks. + * + * SHOW CREATE puts backticks __everywhere__ + * + * until we have an example where that breaks something, they're just too much to force users to have to maintain + */ +export const removeAllBackticks = ({ ddl }: { ddl: string }) => ddl.replace(/`/g, ''); diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeDoubleParenthesisInJoinOnConditions.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeDoubleParenthesisInJoinOnConditions.ts new file mode 100644 index 0000000..0b77f28 --- /dev/null +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeDoubleParenthesisInJoinOnConditions.ts @@ -0,0 +1,6 @@ +/** + * strip out the "double parenthesis" that SHOW CREATE likes to put on the "join on" statements + */ +export const removeDoubleParenthesisInJoinOnConditions = ({ sql }: { sql: string }) => { + return sql.replace(/ on\(\((\w+\.\w+\s=\s\w+\.\w+)\)\)/g, ' on $1 '); // note: strictly only do this if matching the SHOW CREATE weird format of ` on((tableA.column = tableB.column))` +}; diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeFinalSemicolon.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeFinalSemicolon.ts new file mode 100644 index 0000000..678ba6f --- /dev/null +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeFinalSemicolon.ts @@ -0,0 +1,6 @@ +/** + * show create drops the final semicolon. it really doesnt matter if the user still has theirs + */ +export const removeFinalSemicolon = ({ ddl }: { ddl: string }) => { + return ddl.replace(/;/g, ''); +}; diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeParenthesisFromWhereConditions.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeParenthesisFromWhereConditions.ts new file mode 100644 index 0000000..664bf50 --- /dev/null +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeParenthesisFromWhereConditions.ts @@ -0,0 +1,16 @@ +/** + * strip out all of the parenthesis (since SHOW CREATE adds them everywhere and thats too much to force users to maintain); unideal since it modifies the meaning, but this is only relevant for diffing + */ +export const removeParenthesisFromWhereConditions = ({ flattenedSql }: { flattenedSql: string }) => { + const whereCasing = !!flattenedSql.match(/\sWHERE\s/) ? 'WHERE' : 'where'; // determine if user is using upper case or lower case "where" + const sqlSplitOnWhere = flattenedSql.split(whereCasing); + if (sqlSplitOnWhere.length > 2) { + // should not occur because the sql should have been flattened already + throw new Error('found more than two parts of DDL after splitting on where; unexpected'); // fail fast + } + const sqlWithoutParens = + sqlSplitOnWhere.length === 2 + ? sqlSplitOnWhere[0] + '\n' + whereCasing + '\n' + sqlSplitOnWhere[1].replace(/[\(\)]/g, '') // tslint:disable-line prefer-template + : sqlSplitOnWhere[0]; // if no where clause, then nothing to replace + return sqlWithoutParens; +}; diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeParenthesisSurroundingJoinsInFromClause.test.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeParenthesisSurroundingJoinsInFromClause.test.ts new file mode 100644 index 0000000..e22211f --- /dev/null +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeParenthesisSurroundingJoinsInFromClause.test.ts @@ -0,0 +1,27 @@ +import { removeParenthesisSurroundingJoinsInFromClause } from './removeParenthesisSurroundingJoinsInFromClause'; + +describe('removeParenthesisSurroundingJoinsInFromClause', () => { + it('should remove them from every join', () => { + const exampleSql = ` +select + s.id as id, + s.uuid as uuid, + s.name as name, + s.created_at as created_at, + v.effective_at as effective_at, + v.created_at as updated_at +from + ( + ( + contractor s + join contractor_cvp cvp on s.id = cvp.contractor_id + ) + join contractor_version v on v.id = cvp.contractor_version_id + ) + `.trim(); + const normalizedSql = removeParenthesisSurroundingJoinsInFromClause({ flattenedSql: exampleSql }); + expect(normalizedSql).not.toContain('('); + expect(normalizedSql).not.toContain(')'); + expect(normalizedSql).toMatchSnapshot(); + }); +}); diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeParenthesisSurroundingJoinsInFromClause.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeParenthesisSurroundingJoinsInFromClause.ts new file mode 100644 index 0000000..8f2b1b5 --- /dev/null +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeParenthesisSurroundingJoinsInFromClause.ts @@ -0,0 +1,23 @@ +/** + * strip out the first and last paren of the "FROM" section, IF it is defined; + * + * show create likes to wrap the whole "FROM" section w/ parens... its unreasonable to ask people to do that too. + */ +export const removeParenthesisSurroundingJoinsInFromClause = ({ flattenedSql }: { flattenedSql: string }) => { + // check that this sql has this situation going on, if not, do nothing + const ddlHasParenOpenRightAfterFromClause = !!flattenedSql.match(/\sfrom\s+\(/gi); // note, we're not worried about subqueries because we expect flattenedSql as input + if (!ddlHasParenOpenRightAfterFromClause) return flattenedSql; // do nothing if the situation does not exist + + // split DDL on the from statement, since we know it exists + const flattenedSqlParts = flattenedSql.split(/\sfrom\s/gi); + if (flattenedSqlParts.length !== 2) throw new Error('not exactly two parts after splitting on from; unexpected'); // fail fast + + // remove the parens that encapsulate each join + const fromClauseWithoutParens = flattenedSqlParts[1].replace(/\(/g, '').replace(/\)/g, ''); // note: we replace all parens, since subqueries are taken care of + + // join them back and the ddl w/o this mess + const flattenedSqlWithoutOpenCloseParenInFromClause = flattenedSqlParts[0] + '\nfrom\n' + fromClauseWithoutParens; // tslint:disable-line prefer-template + + // return without the parens + return flattenedSqlWithoutOpenCloseParenInFromClause; +}; diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeRedundantAliasDeclarations.test.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeRedundantAliasDeclarations.test.ts new file mode 100644 index 0000000..e741bd0 --- /dev/null +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeRedundantAliasDeclarations.test.ts @@ -0,0 +1,31 @@ +import { removeRedundantAliasDeclarations } from './removeRedundantAliasDeclarations'; + +describe('removeRedundantAliasDeclarations', () => { + test('redundant declarations to be removed and nonredundant to remain', () => { + const exampleSql = ` +CREATE VIEW \`view_contractor_current\` AS +SELECT + s.id, + s.uuid as uuid, + s.name as name, + ( + SELECT GROUP_CONCAT(contractor_version_to_contractor_license.contractor_license_id ORDER BY contractor_version_to_contractor_license.array_order_index) + FROM contractor_version_to_contractor_license WHERE contractor_version_to_contractor_license.contractor_version_id = v.id + ) as license_ids, + ( + SELECT GROUP_CONCAT(contractor_version_to_contact_method.contact_method_id ORDER BY contractor_version_to_contact_method.array_order_index) + FROM contractor_version_to_contact_method WHERE contractor_version_to_contact_method.contractor_version_id = v.id + ) as proposed_suggestion_change_ids, + s.created_at, + v.effective_at, + v.created_at as updated_at +FROM contractor s +JOIN contractor_cvp cvp ON s.id = cvp.contractor_id +JOIN contractor_version v ON v.id = cvp.contractor_version_id; + `; + const normalizedSql = removeRedundantAliasDeclarations({ sql: exampleSql }); + expect(normalizedSql).not.toContain('as uuid'); + expect(normalizedSql).not.toContain('as name'); + expect(normalizedSql).toMatchSnapshot(); // for visual inspection to make sure no other defects occured + }); +}); diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeRedundantAliasDeclarations.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeRedundantAliasDeclarations.ts new file mode 100644 index 0000000..70113ac --- /dev/null +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/normalizations/removeRedundantAliasDeclarations.ts @@ -0,0 +1,8 @@ +/** + * SHOW CREATE adds `as column_name` to every `table.column_name` even when alias is implicit + * + * we dont want to force users to have to do that, so lets strip the redundant ones before comparing diff + */ +export const removeRedundantAliasDeclarations = ({ sql }: { sql: string }) => { + return sql.replace(/(\w+\.(\w+))\s+as\s+\2/g, '$1'); // i.e., if column name (second matching group) matches alias declaration, replace it with just the selector (first matching group) +}; diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/recursivelyHeavilyNormalizeViewDdl.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/recursivelyHeavilyNormalizeViewDdl.ts new file mode 100644 index 0000000..3d72e70 --- /dev/null +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/recursivelyHeavilyNormalizeViewDdl.ts @@ -0,0 +1,33 @@ +import { addSpacesBetweenCommaParenthesisOccurances } from './normalizations/addSpacesBetweenCommaParenthesisOccurances'; +import { lowercaseAllAsTokensExceptFirstOne } from './normalizations/lowercaseAllAsTokensExceptFirstOne'; +import { lowercaseEverythingAfterFirstAs } from './normalizations/lowercaseEverythingAfterFirstAs'; +import { removeAllBackticks } from './normalizations/removeAllBackticks'; +import { removeFinalSemicolon } from './normalizations/removeFinalSemicolon'; +import { recursivelyHeavilyNormalizeViewQuerySql } from './recursivelyHeavilyNormalizeViewQuerySql'; + +/** + * SHOW CREATE drops all user formatting and adds a ton of weird formatting that we dont want any user to have to enforce in order to use views as resources + * + * therefore, we heavily normalize it. + * + * we also do it recursively, because we support subqueries + */ +export const recursivelyHeavilyNormalizeViewDdl = ({ ddl }: { ddl: string }) => { + // 1. normalize the query as a baseline, setting casing on everything + let normalizedDdl = ddl; + normalizedDdl = lowercaseAllAsTokensExceptFirstOne({ ddl: normalizedDdl }); + normalizedDdl = lowercaseEverythingAfterFirstAs({ ddl: normalizedDdl }); + normalizedDdl = removeAllBackticks({ ddl: normalizedDdl }); + normalizedDdl = removeFinalSemicolon({ ddl: normalizedDdl }); + normalizedDdl = addSpacesBetweenCommaParenthesisOccurances({ ddl: normalizedDdl }); + + // 2. recursively normalize the query body; do so recursively to support the subqueries that may be present + const ddlParts = normalizedDdl.split(' AS '); // NOTE: there is only one upper case AS, so it splits the sql from the view initial create statement + const viewCreateHeader = ddlParts[0]; + const viewQuerySql = ddlParts[1]; + const normalizedViewQuerySql = recursivelyHeavilyNormalizeViewQuerySql({ sql: viewQuerySql }); + normalizedDdl = viewCreateHeader + ' AS ' + normalizedViewQuerySql; // tslint:disable-line prefer-template + + // 3. return the results + return normalizedDdl; +}; diff --git a/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/recursivelyHeavilyNormalizeViewQuerySql.ts b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/recursivelyHeavilyNormalizeViewQuerySql.ts new file mode 100644 index 0000000..5b19397 --- /dev/null +++ b/src/logic/schema/resourceDefinition/getDifferenceForResourceDefinition/normalizeDDLToSupportLossyShowCreateStatements/recursivelyHeavilyNormalizeViewDdl/recursivelyHeavilyNormalizeViewQuerySql.ts @@ -0,0 +1,45 @@ +import { + flattenSqlByReferencingAndTokenizingSubqueries, + hydrateSqlByReferencingAndReplacingSubqueryTokens, +} from '../../../../../../__nonpublished_modules__/sql-subquery-tokenizer'; +import { removeParenthesisFromWhereConditions } from './normalizations/removeParenthesisFromWhereConditions'; +import { removeParenthesisSurroundingJoinsInFromClause } from './normalizations/removeParenthesisSurroundingJoinsInFromClause'; +import { removeDoubleParenthesisInJoinOnConditions } from './normalizations/removeDoubleParenthesisInJoinOnConditions'; +import { SqlSubqueryReference } from '../../../../../../__nonpublished_modules__/sql-subquery-tokenizer/model/SqlSubqueryReference'; +import { removeRedundantAliasDeclarations } from './normalizations/removeRedundantAliasDeclarations'; + +/** + * SHOW CREATE drops all user formatting and adds a ton of weird formatting that we dont want any user to have to enforce in order to use views as resources + * + * therefore, we heavily normalize it. + * + * we also do it recursively, because we support subqueries + */ +export const recursivelyHeavilyNormalizeViewQuerySql = ({ sql }: { sql: string }) => { + // 1. flatten the ddl by extracting subqueries as referenced tokens + const { flattenedSql, references } = flattenSqlByReferencingAndTokenizingSubqueries({ sql }); + + // 2. normalize the top level flat ddl; hover over the fn's for jsdoc based intellisense explanations for each normalization + let normalizedFlattenedSql = flattenedSql; + normalizedFlattenedSql = removeParenthesisFromWhereConditions({ flattenedSql: normalizedFlattenedSql }); + normalizedFlattenedSql = removeDoubleParenthesisInJoinOnConditions({ sql: normalizedFlattenedSql }); + normalizedFlattenedSql = removeParenthesisSurroundingJoinsInFromClause({ flattenedSql: normalizedFlattenedSql }); + normalizedFlattenedSql = removeRedundantAliasDeclarations({ sql: normalizedFlattenedSql }); + + // 3. recursively apply this function to each referenced sql and replace the sql in the reference, ready to hydrate back + const normalizedReferences = references.map((reference) => { + const referencedSqlWithoutStartEndParen = reference.sql.slice(1, -1); // the flatten function puts subquery inside of parens + const normalizedReferencedSql = recursivelyHeavilyNormalizeViewQuerySql({ sql: referencedSqlWithoutStartEndParen }); + const normalizedReferencedSqlWithStartEndParens = `(${normalizedReferencedSql})`; // add those parens back + return new SqlSubqueryReference({ id: reference.id, sql: normalizedReferencedSqlWithStartEndParens }); + }); + + // 4. hydrate the references back up + const hydratedNormalizedSql = hydrateSqlByReferencingAndReplacingSubqueryTokens({ + flattenedSql: normalizedFlattenedSql, + references: normalizedReferences, + }); + + // 5. return the normalized sql + return hydratedNormalizedSql; +};