feat: define suggestColRefKeywords, suggestEngines, add typechecking …

…ci (#95)

* feat: add typechecking ci

* fix: review fixes

* fix: ts-ignore comments

---------

Co-authored-by: robhovsepyan <[email protected]>

.github/workflows/ci.yml

@@ -25,6 +25,24 @@ jobs:
       - name: Lint Files
         run: npm run lint
 
+  typecheck:
+    name: Typecheck files
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '18.x'
+          cache: 'npm'
+      - name: Install Packages
+        run: npm ci
+      - name: Typecheck Files
+        run: npm run typecheck
+
   generated:
     name: Check generated files
     runs-on: ubuntu-latest

.husky/pre-commit

@@ -1,4 +1,4 @@
 #!/bin/sh
 . "$(dirname "$0")/_/husky.sh"
 
-npx nano-staged
+npm run typecheck && npx nano-staged

package.json

@@ -26,6 +26,7 @@
     "lint": "npm run prettier -- --check",
     "fix": "npm run prettier -- --write",
     "prettier": "prettier \"**/*.{md,yaml,yml,json}\"",
+    "typecheck": "tsc --noEmit",
     "build": "rimraf dist && tsc -p tsconfig.build.json",
     "prepublishOnly": "npm run build"
   },

src/generator/lib/generator.ts

@@ -262,8 +262,8 @@ export async function generateParsers(parserNames: string[]): Promise<void> {
         throw new Error(`Could not find all requested parser definitions`);
     }
 
-    for (let i = 0; i < parserDefinitions.length; i++) {
-        console.log(`Generating ${parserDefinitions[i].parserName}`);
-        await generateParser(parserDefinitions[i]);
+    for (const parserDefinition of parserDefinitions) {
+        console.log(`Generating ${parserDefinition.parserName}`);
+        await generateParser(parserDefinition);
     }
 }

src/parsing/@types/expect.d.ts

@@ -0,0 +1,23 @@
+// Please note that the code below is the modified code distributed on the terms, mentioned below.
+// The copyright for the changes belongs to YANDEX LLC.
+//
+// Copyright 2023 YANDEX LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License")
+// You may not use this file except in compliance with the License.
+// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under
+// the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+// either express or implied. See the License for the specific language governing permissions
+// and limitations under the License.
+
+import type {Matchers} from 'expect';
+import type { TestCase } from '../test/testing';
+
+declare module 'expect' {
+  export interface Matchers<R extends void | Promise<void>, T = unknown> {
+    toEqualAutocompleteValues(values: string[]): R;
+    toEqualDefinition(testDefinition: TestCase): R;
+  }
+}

src/parsing/@types/jest.d.ts

@@ -0,0 +1,15 @@
+// Please note that the code below is the modified code distributed on the terms, mentioned below.
+// The copyright for the changes belongs to YANDEX LLC.
+//
+// Copyright 2023 YANDEX LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License")
+// You may not use this file except in compliance with the License.
+// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under
+// the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+// either express or implied. See the License for the specific language governing permissions
+// and limitations under the License.
+
+declare function fail(error?: any): never;

src/parsing/index.ts

@@ -16,7 +16,9 @@ export interface ParseResult {
     suggestColumns?: ColumnSuggestion;
     suggestAggregateFunctions?: AggregateFunctionsSuggestion;
     suggestAnalyticFunctions?: unknown;
-    suggestColRefKeywords?: unknown;
+    suggestColRefKeywords?: {
+        [type: string]: string[];
+    };
     suggestColumnAliases?: ColumnAliasSuggestion[];
     suggestCommonTableExpressions?: unknown;
     suggestDatabases?: DatabasesSuggestion;
@@ -26,8 +28,8 @@ export interface ParseResult {
     suggestIdentifiers?: IdentifierSuggestion[];
     suggestTemplates?: boolean;
     suggestEngines?: {
-        engines: string[],
-        functionalEngines: string[]
+        engines: Engines;
+        functionalEngines: Engines;
     };
 
     // Reasons for those fields are unknown
@@ -142,6 +144,8 @@ export interface ColumnAliasSuggestion {
     types: string[];
 }
 
+type Engines = string[];
+
 export function parseGenericSql(queryBeforeCursor: string, queryAfterCursor: string): ParseResult {
     let parser = genericAutocompleteParser as Parser;
     return parser.parseSql(queryBeforeCursor, queryAfterCursor);

src/parsing/lib/parsing-typed.ts

@@ -26,6 +26,8 @@
 // either express or implied. See the License for the specific language governing permissions
 // and limitations under the License.
 
+import {expect} from '@jest/globals'
+
 export interface CommonParser {
   identifyPartials(
     beforeCursor: string,

src/parsing/lib/sql-reference/types.ts

@@ -118,12 +118,16 @@ export function matchesType(
   const conversionTable = GENERIC_TYPE_CONVERSION;
   for (let i = 0; i < expectedTypes.length; i++) {
     for (let j = 0; j < actualTypes.length; j++) {
+      const expectedType = expectedTypes[i];
+      const actualType = actualTypes[j];
+      const convertedType = expectedType && conversionTable[expectedType];
+
       // To support future unknown types
-      if (typeof conversionTable[expectedTypes[i]] === 'undefined' || typeof conversionTable[expectedTypes[i]][actualTypes[j]] == 'undefined') {
+      if (!convertedType || !actualType || typeof convertedType === 'undefined' || typeof convertedType[actualType] == 'undefined') {
         return true;
       }
 
-      if (conversionTable[expectedTypes[i]] && conversionTable[expectedTypes[i]][actualTypes[j]]) {
+      if (convertedType && convertedType[actualType]) {
         return true;
       }
     }

src/parsing/test/jest.init.ts

@@ -27,6 +27,7 @@
 // the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 // either express or implied. See the License for the specific language governing permissions
 // and limitations under the License.
+import {expect} from '@jest/globals'
 
 import {getToEqualAutocompleteValues, toEqualDefinition} from './testing';
 

src/parsing/test/testing.ts

@@ -32,22 +32,34 @@
 import {existsSync, readFileSync} from 'fs';
 import {AutocompleteParser} from '../lib/types';
 import {describe, expect, it, beforeAll} from '@jest/globals';
+import type {ParseResult} from '..';
+
+interface ExpectedError {
+    text: string,
+    token: string,
+    loc: {
+        first_line: number,
+        last_line: number,
+        first_column: number,
+        last_column: number
+    },
+}
 
-interface TestCase {
+export interface TestCase {
     namePrefix: string; // ex. "should suggest keywords"
     beforeCursor: string;
     afterCursor: string;
     containsKeywords?: string[];
     doesNotContainKeywords?: string[];
-    containsColRefKeywords?: string[];
+    containsColRefKeywords?: boolean | string[];
     noErrors?: boolean;
     locationsOnly?: boolean;
     noLocations?: boolean;
     expectedDefinitions: unknown,
-    expectedLocations?: unknown;
+    expectedLocations?: ParseResult['locations'];
     expectedResult: {
         lowerCase?: boolean;
-        locations?: unknown;
+        locations?: ParseResult['locations'];
         suggestTables?: {
             identifierChain?: { name: string }[];
             onlyTables?: boolean;
@@ -57,30 +69,21 @@ interface TestCase {
         };
         suggestTemplates?: boolean;
     };
-    expectedErrors?: {
-        text: string,
-        token: string,
-        loc: {
-            first_line: number,
-            last_line: number,
-            first_column: number,
-            last_column: number
-        },
-    }[]
+    expectedErrors?: ExpectedError[]
 }
 
 interface GroupedTestCases {
     jisonFile: string;
     testCases: TestCase[];
 }
 
-export function getToEqualAutocompleteValues(actualItems, expectedValues) {
+export function getToEqualAutocompleteValues(actualItems: {value: string}[], expectedValues: string[]) {
     if (actualItems.length !== expectedValues.length) {
         return {pass: false, message: () => 'items length is not equal'};
     }
 
     for (let i = 0; i < expectedValues.length; i++) {
-        const stringValue = typeof actualItems[i] !== 'string' ? '' + actualItems[i].value : actualItems[i].value;
+        const stringValue = typeof actualItems[i] !== 'string' ? '' + actualItems[i]?.value : actualItems[i]?.value;
         if (stringValue !== expectedValues[i]) {
             return {
                 pass: false,
@@ -92,7 +95,7 @@ export function getToEqualAutocompleteValues(actualItems, expectedValues) {
     return {pass: true, message: () => 'test'};
 }
 
-export function toEqualDefinition(actualResponse, testDefinition: TestCase) {
+export function toEqualDefinition(actualResponse: Partial<ParseResult>, testDefinition: TestCase) {
     if (typeof testDefinition.noErrors === 'undefined' && actualResponse.errors && !testDefinition.expectedErrors) {
         let allRecoverable = true;
         actualResponse.errors.forEach(error => {
@@ -119,7 +122,7 @@ export function toEqualDefinition(actualResponse, testDefinition: TestCase) {
     ) {
         const expectedLoc =
             testDefinition.expectedLocations || testDefinition.expectedResult.locations;
-        const expectsType = expectedLoc.some(location => location.type === 'statementType');
+        const expectsType = expectedLoc?.some(location => location.type === 'statementType');
         if (!expectsType) {
             actualResponse.locations = actualResponse.locations.filter(
                 location => location.type !== 'statementType'
@@ -152,20 +155,24 @@ export function toEqualDefinition(actualResponse, testDefinition: TestCase) {
     }
 
     if (actualResponse.suggestKeywords) {
-        const weightFreeKeywords = [];
+        const weightFreeKeywords: ParseResult['suggestKeywords'] = [];
         actualResponse.suggestKeywords.forEach(keyword => {
-            weightFreeKeywords.push(keyword.value);
+            if (typeof keyword !== 'string') {
+                // This file is going to be obsolete in 2 weeks, when we rewrite tests
+                // @ts-ignore
+                weightFreeKeywords.push(keyword.value);
+            }
         });
         actualResponse.suggestKeywords = weightFreeKeywords;
     }
 
     if (!!testDefinition.noLocations) {
-        if (actualResponse.locations.length > 0) {
+        if (actualResponse.locations && actualResponse.locations.length > 0) {
             return {
                 pass: false,
                 message: constructTestCaseMessage(testDefinition, {
                     'Expected locations': 'none',
-                    'Found locations': actualResponse.locations.length,
+                    'Found locations': actualResponse.locations?.length,
                 }),
             };
         }
@@ -175,7 +182,9 @@ export function toEqualDefinition(actualResponse, testDefinition: TestCase) {
     }
     let deleteKeywords = false;
     if (testDefinition.containsColRefKeywords) {
-        if (typeof actualResponse.suggestColRefKeywords == 'undefined') {
+        const actualSuggestColRefKeywords = actualResponse.suggestColRefKeywords;
+
+        if (typeof actualSuggestColRefKeywords == 'undefined') {
             return {
                 pass: false,
                 message: constructTestCaseMessage(testDefinition, {
@@ -187,9 +196,9 @@ export function toEqualDefinition(actualResponse, testDefinition: TestCase) {
             testDefinition.containsColRefKeywords.forEach(keyword => {
                 contains =
                     contains &&
-                    (actualResponse.suggestColRefKeywords.BOOLEAN.indexOf(keyword) !== -1 ||
-                        actualResponse.suggestColRefKeywords.NUMBER.indexOf(keyword) !== -1 ||
-                        actualResponse.suggestColRefKeywords.STRING.indexOf(keyword) !== -1);
+                    (actualSuggestColRefKeywords.BOOLEAN?.indexOf(keyword) !== -1 ||
+                        actualSuggestColRefKeywords.NUMBER?.indexOf(keyword) !== -1 ||
+                        actualSuggestColRefKeywords.STRING?.indexOf(keyword) !== -1);
             });
             if (!contains) {
                 return {
@@ -207,7 +216,9 @@ export function toEqualDefinition(actualResponse, testDefinition: TestCase) {
     if (typeof testDefinition.containsKeywords !== 'undefined') {
         const keywords = actualResponse.suggestKeywords;
         let contains = true;
-        testDefinition.containsKeywords.forEach(keyword => {
+        testDefinition.containsKeywords.forEach((keyword): boolean | void => {
+            // This file is going to be obsolete in 2 weeks, when we rewrite tests
+            // @ts-ignore
             if (typeof keywords === 'undefined' || keywords.indexOf(keyword) === -1) {
                 contains = false;
                 return false;
@@ -227,7 +238,9 @@ export function toEqualDefinition(actualResponse, testDefinition: TestCase) {
     if (typeof testDefinition.doesNotContainKeywords !== 'undefined') {
         const keywords = actualResponse.suggestKeywords || [];
         let contains = false;
-        testDefinition.doesNotContainKeywords.forEach(keyword => {
+        testDefinition.doesNotContainKeywords.forEach((keyword): boolean | void => {
+            // This file is going to be obsolete in 2 weeks, when we rewrite tests
+            // @ts-ignore
             if (typeof keywords === 'undefined' || keywords.indexOf(keyword) !== -1) {
                 contains = true;
                 return false;
@@ -276,9 +289,15 @@ export function toEqualDefinition(actualResponse, testDefinition: TestCase) {
             }
         }
 
-        const filteredResponseErrors = actualResponse.errors.map((responseError, index) => {
+        const filteredResponseErrors = actualResponse.errors.map((responseError: Record<string, any>, index) => {
+            if (!testDefinition.expectedErrors) {
+                return {};
+            }
+
+            // This file is going to be obsolete in 2 weeks, when we rewrite tests
+            // @ts-ignore
             const expectedKeys = Object.keys(testDefinition.expectedErrors[index]);
-            return expectedKeys.reduce((acc, expectedKey) => {
+            return expectedKeys.reduce<Record<string, any>>((acc, expectedKey) => {
                 acc[expectedKey] = responseError[expectedKey];
                 return acc
             }, {});
@@ -340,7 +359,7 @@ function constructTestCaseMessage(testCase: TestCase, details: Record<string, an
     return () => message
 }
 
-function resultEquals(a, b): boolean {
+function resultEquals(a: any, b: any): boolean {
     if (typeof a !== typeof b) {
         return false;
     }
@@ -349,14 +368,20 @@ function resultEquals(a, b): boolean {
         return true;
     }
 
-    if (typeof a === 'object') {
+    if (typeof a === 'object' && a !== null) {
         const aKeys = Object.keys(a);
         if (aKeys.length !== Object.keys(b).length) {
             return false;
         }
-
+        
         for (let i = 0; i < aKeys.length; i++) {
-            if (!resultEquals(a[aKeys[i]], b[aKeys[i]])) {
+            const aKey = aKeys[i];
+
+            if (aKey === undefined) {
+                return false;
+            }
+
+            if (!resultEquals(a[aKey], b[aKey])) {
                 return false;
             }
         }
@@ -367,12 +392,12 @@ function resultEquals(a, b): boolean {
     return a == b;
 }
 
-function jsonStringToJsString(jsonString) {
+function jsonStringToJsString(jsonString: string): string {
     return jsonString
-        .replace(/'([a-zA-Z]+)':/g, (all, group) => {
+        .replace(/'([a-zA-Z]+)':/g, (_, group) => {
             return group + ':';
         })
-        .replace(/([:{,])/g, (all, group) => {
+        .replace(/([:{,])/g, (_, group) => {
             return group + ' ';
         })
         .replace(/[}]/g, ' }')