refactor: rename some folders, move some files around (#100)

* refactor: move and rename some files around

* refactor: move parsers/{dialect}/jison to parsers/{dialect}/grammar

* refactor: remove unused SqlSyntax grouping

* refactor: rename src/parsing to src/autocomplete

* refactor: add linter fix

.eslintrc

@@ -22,7 +22,7 @@
       }
     },
     {
-      "files": ["**/parser-extension.js", "src/parsing/lib/parsing.js"],
+      "files": ["**/parser-extension.js", "src/autocomplete/lib/parsing.js"],
       "rules": {
         // Parser is extended via reassignment, so this rule doesn't make sense
         "no-param-reassign": "off"

NOTICE.txt

@@ -22685,7 +22685,7 @@ Please note that changes have been made to the files:
 - /src/parsing/lib/parsing-typed.ts
 - /src/parsing/lib/string-distance.test.js
 - /src/parsing/lib/types.ts
-- /src/parsing/parsers/generic/jison/autocomplete_footer.jison
+- /src/parsing/parsers/generic/grammar/common/header_and_footer/footer.jison
 - /src/parsing/parsers/generic/genericAutocompleteParser.js
 - /src/parsing/parsers/generic/genericAutocompleteParser.test.ts
 - /src/parsing/parsers/generic/genericAutocompleteParser.locations.test.js

README.md

@@ -18,13 +18,13 @@ Resources to research:
 
 # How everything works
 
-1. Autocomplete parser source code for different `{dialect}` are defined in `src/parsing/parsers/{dialect}` directories.
-2. Lexer configuration is specified in `src/parsing/parsers/{dialect}/jison/sql.jisonlex` file.
-3. SQL grammar is specified in `src/parsing/parsers/{dialect}/jison/**/*.jison`. It is placed in multiple files for easier understanding. Test files `*.test.json` are placed nearby. Jison is basically bison, but for javascript.
-4. `src/parsing/parsers/{dialect}/jison/structure.json` specifies paths to a lexer, and to all the grammar files.
-5. Parser extension (basically the autocomplete features) is specified in `src/parsing/parsers/{dialect}/parser-extension.js`.
-6. `src/generator/main.js` concatenates all the jison files into a single big jison file, and runs the jison tool with the specified lexer, then wires everything up with the `parser-extension.js`, generating `src/parsing/parsers/{dialect}/{dialect}AutocompleteParser.js`.
-7. The generated file is in plain javascript, so we create a convenient typescript wrapper in `src/parsing/index.ts` with all the types and functions. Our users should include this file in their own projects.
+1. Autocomplete parser source code for different `{dialect}` are defined in `src/autocomplete/parsers/{dialect}` directories.
+2. Lexer configuration is specified in `src/autocomplete/parsers/{dialect}/grammar/sql.jisonlex` file.
+3. SQL grammar is specified in `src/autocomplete/parsers/{dialect}/grammar/**/*.jison`. It is placed in multiple files for easier understanding. Test files `*.test.json` are placed nearby. Jison is basically bison, but for javascript.
+4. `src/autocomplete/parsers/{dialect}/grammar/structure.json` specifies paths to a lexer, and to all the grammar files.
+5. Parser extension (basically the autocomplete features) is specified in `src/autocomplete/parsers/{dialect}/parser-extension.js`.
+6. `src/generator/main.js` concatenates all the jison files into a single big jison file, and runs the jison tool with the specified lexer, then wires everything up with the `parser-extension.js`, generating `src/autocomplete/parsers/{dialect}/{dialect}AutocompleteParser.js`.
+7. The generated file is in plain javascript, so we create a convenient typescript wrapper in `src/autocomplete/index.ts` with all the types and functions. Our users should include this file in their own projects.
 
 # Main scripts
 

jest.config.js

@@ -19,10 +19,10 @@ module.exports = {
   },
   moduleNameMapper: {
     '\\.(css|less|scss|sass)$': 'identity-obj-proxy',
-    '^\\./src/parsing/(.*)$': 'src/parsing/$1'
+    '^\\./src/autocomplete/(.*)$': 'src/parsing/$1'
   },
-  moduleDirectories: ['node_modules', 'src/parsing'],
-  testMatch: ['<rootDir>/src/parsing/**/*.test.(js|jsx|ts|tsx)'],
-  setupFilesAfterEnv: ['<rootDir>/src/parsing/test/jest.init.ts'],
-  collectCoverageFrom: ['<rootDir>/src/parsing/**/*.{js,jsx}']
+  moduleDirectories: ['node_modules', 'src/autocomplete'],
+  testMatch: ['<rootDir>/src/autocomplete/**/*.test.(js|jsx|ts|tsx)'],
+  setupFilesAfterEnv: ['<rootDir>/src/autocomplete/test/jest.init.ts'],
+  collectCoverageFrom: ['<rootDir>/src/autocomplete/**/*.{js,jsx}']
 };

package.json

@@ -19,7 +19,7 @@
   "scripts": {
     "prepare": "husky install",
     "test": "npm run generate && npm run test_without_generate",
-    "test_without_generate": "jest src/parsing",
+    "test_without_generate": "jest src/autocomplete",
     "build_generator": "cd src/generator && rimraf dist && tsc -p tsconfig.build.json",
     "launch_generator": "cd src/generator && node dist/main.js generic postgresql clickhouse",
     "generate": "npm run build_generator && npm run launch_generator",

src/parsing/parsers/clickhouse/clickhouseAutocompleteParser.test.ts → src/autocomplete/parsers/clickhouse/clickhouseAutocompleteParser.test.ts

@@ -16,10 +16,10 @@ import type {AutocompleteParser} from '../../lib/types';
 import {extractTestCases, runTestCases} from '../../test/testing';
 
 import {clickhouseAutocompleteParser} from './clickhouseAutocompleteParser';
-import structure from './jison/structure.json';
+import structure from './grammar/structure.json';
 
-const jisonFolder = 'src/parsing/parsers/clickhouse/jison';
-const groupedTestCases = extractTestCases(jisonFolder, structure.autocomplete);
+const grammarFolder = 'src/autocomplete/parsers/clickhouse/grammar';
+const groupedTestCases = extractTestCases(grammarFolder, structure.autocomplete);
 
 describe('clickhouseAutocompleteParser', () => {
     // TODO: Fix the types

src/parsing/parsers/clickhouse/jison/sql_main.jison → src/autocomplete/parsers/clickhouse/grammar/main.jison

@@ -14,11 +14,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-SqlSyntax
- : NewStatement SqlStatements EOF
- ;
+// Please note that the code below is the modified code distributed on the terms, mentioned below.
+// The copyright for the changes belongs to YANDEX LLC.
+//
+// Copyright 2023 YANDEX LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License")
+// You may not use this file except in compliance with the License.
+// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under
+// the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+// either express or implied. See the License for the specific language governing permissions
+// and limitations under the License.
 
-SqlAutocomplete
+SqlSyntax
  : NewStatement SqlStatements EOF
    {
      return parser.yy.result;

src/autocomplete/parsers/clickhouse/grammar/structure.json

@@ -0,0 +1,62 @@
+{
+  "lexer": "sql.jisonlex",
+  "autocomplete": [
+    "../../generic/grammar/common/header_and_footer/header.jison",
+    "../../generic/grammar/alter/alter_common.jison",
+    "../../generic/grammar/alter/alter_table.jison",
+    "../../generic/grammar/alter/alter_view.jison",
+    "../../generic/grammar/create/create_common.jison",
+    "../../generic/grammar/create/create_database_or_schema.jison",
+    "../../generic/grammar/create/create_role.jison",
+    "create/create_table.jison",
+    "../../generic/grammar/create/create_view.jison",
+    "../../generic/grammar/delete/delete.jison",
+    "../../generic/grammar/drop/drop_common.jison",
+    "../../generic/grammar/drop/drop_database.jison",
+    "../../generic/grammar/drop/drop_role.jison",
+    "../../generic/grammar/drop/drop_table.jison",
+    "../../generic/grammar/drop/drop_view.jison",
+    "explain/explain.jison",
+    "../../generic/grammar/insert/insert.jison",
+    "../../generic/grammar/select/cte_select_statement.jison",
+    "../../generic/grammar/select/from_clause.jison",
+    "../../generic/grammar/select/group_by_clause.jison",
+    "../../generic/grammar/select/having_clause.jison",
+    "../../generic/grammar/select/joins.jison",
+    "../../generic/grammar/select/limit_clause.jison",
+    "../../generic/grammar/select/order_by_clause.jison",
+    "select/select.jison",
+    "../../generic/grammar/select/select_conditions.jison",
+    "../../generic/grammar/select/union_clause.jison",
+    "../../generic/grammar/common/where/where_clause.jison",
+    "../../generic/grammar/set/set_common.jison",
+    "../../generic/grammar/set/set_all.jison",
+    "../../generic/grammar/set/set_option.jison",
+    "../../generic/grammar/truncate/truncate_table.jison",
+    "../../generic/grammar/udf/aggregate/aggregate_common.jison",
+    "../../generic/grammar/udf/aggregate/avg.jison",
+    "../../generic/grammar/udf/aggregate/count.jison",
+    "../../generic/grammar/udf/aggregate/max.jison",
+    "../../generic/grammar/udf/aggregate/min.jison",
+    "../../generic/grammar/udf/aggregate/stddev_pop.jison",
+    "../../generic/grammar/udf/aggregate/stddev_samp.jison",
+    "../../generic/grammar/udf/aggregate/sum.jison",
+    "../../generic/grammar/udf/aggregate/var_pop.jison",
+    "../../generic/grammar/udf/aggregate/var_samp.jison",
+    "../../generic/grammar/udf/aggregate/variance.jison",
+    "../../generic/grammar/udf/analytic/analytic.jison",
+    "../../generic/grammar/udf/function/array.jison",
+    "../../generic/grammar/udf/function/cast.jison",
+    "../../generic/grammar/udf/function/if.jison",
+    "../../generic/grammar/udf/function/map.jison",
+    "../../generic/grammar/udf/function/truncate.jison",
+    "../../generic/grammar/udf/udf_common.jison",
+    "../../generic/grammar/update/update_table.jison",
+    "../../generic/grammar/use/use.jison",
+    "../../generic/grammar/common/error/error.jison",
+    "main.jison",
+    "../../generic/grammar/common/table/table.jison",
+    "../../generic/grammar/common/value_expression/value_expression.jison",
+    "../../generic/grammar/common/header_and_footer/footer.jison"
+  ]
+}

src/parsing/parsers/generic/genericAutocompleteParser.test.ts → src/autocomplete/parsers/generic/genericAutocompleteParser.test.ts

@@ -35,10 +35,10 @@ import type {AutocompleteParser} from '../../lib/types';
 import {extractTestCases, runTestCases} from '../../test/testing';
 
 import {genericAutocompleteParser} from './genericAutocompleteParser';
-import structure from './jison/structure.json';
+import structure from './grammar/structure.json';
 
-const jisonFolder = 'src/parsing/parsers/generic/jison';
-const groupedTestCases = extractTestCases(jisonFolder, structure.autocomplete);
+const grammarFolder = 'src/autocomplete/parsers/generic/grammar';
+const groupedTestCases = extractTestCases(grammarFolder, structure.autocomplete);
 
 describe('genericAutocompleteParser', () => {
     // TODO: Fix the types

src/parsing/parsers/generic/jison/autocomplete_header.jison → src/autocomplete/parsers/generic/grammar/common/header_and_footer/header.jison

@@ -14,6 +14,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Please note that the code below is the modified code distributed on the terms, mentioned below.
+// The copyright for the changes belongs to YANDEX LLC.
+//
+// Copyright 2023 YANDEX LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License")
+// You may not use this file except in compliance with the License.
+// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under
+// the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+// either express or implied. See the License for the specific language governing permissions
+// and limitations under the License.
+
 %left 'AND' 'OR'
 %left 'BETWEEN'
 %left 'NOT' '!' '~'
@@ -24,6 +38,6 @@
 %nonassoc 'CURSOR' 'PARTIAL_CURSOR'
 %nonassoc 'IN' 'IS' 'LIKE' 'RLIKE' 'REGEXP' 'EXISTS' NEGATION
 
-%start SqlAutocomplete
+%start SqlSyntax
 
 %%

src/parsing/parsers/generic/jison/sql_main.jison → src/autocomplete/parsers/generic/grammar/main.jison

@@ -14,11 +14,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-SqlSyntax
- : NewStatement SqlStatements EOF
- ;
+// Please note that the code below is the modified code distributed on the terms, mentioned below.
+// The copyright for the changes belongs to YANDEX LLC.
+//
+// Copyright 2023 YANDEX LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License")
+// You may not use this file except in compliance with the License.
+// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under
+// the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+// either express or implied. See the License for the specific language governing permissions
+// and limitations under the License.
 
-SqlAutocomplete
+SqlSyntax
  : NewStatement SqlStatements EOF
    {
      return parser.yy.result;

src/parsing/parsers/generic/jison/structure.json → src/autocomplete/parsers/generic/grammar/structure.json

@@ -1,7 +1,7 @@
 {
   "lexer": "sql.jisonlex",
   "autocomplete": [
-    "autocomplete_header.jison",
+    "common/header_and_footer/header.jison",
     "alter/alter_common.jison",
     "alter/alter_table.jison",
     "alter/alter_view.jison",
@@ -28,7 +28,7 @@
     "select/select.jison",
     "select/select_conditions.jison",
     "select/union_clause.jison",
-    "select/where_clause.jison",
+    "common/where/where_clause.jison",
     "set/set_common.jison",
     "set/set_all.jison",
     "set/set_option.jison",
@@ -53,10 +53,10 @@
     "udf/udf_common.jison",
     "update/update_table.jison",
     "use/use.jison",
-    "sql_error.jison",
-    "sql_main.jison",
-    "sql_table.jison",
-    "sql_valueExpression.jison",
-    "autocomplete_footer.jison"
+    "common/error/error.jison",
+    "main.jison",
+    "common/table/table.jison",
+    "common/value_expression/value_expression.jison",
+    "common/header_and_footer/footer.jison"
   ]
 }

src/autocomplete/parsers/postgresql/grammar/structure.json

@@ -0,0 +1,63 @@
+{
+  "lexer": "sql.jisonlex",
+  "autocomplete": [
+    "../../generic/grammar/common/header_and_footer/header.jison",
+    "../../generic/grammar/alter/alter_common.jison",
+    "../../generic/grammar/alter/alter_table.jison",
+    "../../generic/grammar/alter/alter_view.jison",
+    "../../generic/grammar/create/create_common.jison",
+    "../../generic/grammar/create/create_database_or_schema.jison",
+    "../../generic/grammar/create/create_role.jison",
+    "../../generic/grammar/create/create_table.jison",
+    "../../generic/grammar/create/create_view.jison",
+    "../../generic/grammar/delete/delete.jison",
+    "../../generic/grammar/drop/drop_common.jison",
+    "../../generic/grammar/drop/drop_database.jison",
+    "../../generic/grammar/drop/drop_role.jison",
+    "../../generic/grammar/drop/drop_table.jison",
+    "../../generic/grammar/drop/drop_view.jison",
+    "../../generic/grammar/explain/explain.jison",
+    "../../generic/grammar/insert/insert.jison",
+    "../../generic/grammar/select/cte_select_statement.jison",
+    "../../generic/grammar/select/from_clause.jison",
+    "../../generic/grammar/select/group_by_clause.jison",
+    "../../generic/grammar/select/having_clause.jison",
+    "../../generic/grammar/select/joins.jison",
+    "select/limit_clause.jison",
+    "select/offset_clause.jison",
+    "select/order_by_clause.jison",
+    "select/select.jison",
+    "select/select_conditions.jison",
+    "../../generic/grammar/select/union_clause.jison",
+    "../../generic/grammar/common/where/where_clause.jison",
+    "../../generic/grammar/set/set_common.jison",
+    "../../generic/grammar/set/set_all.jison",
+    "../../generic/grammar/set/set_option.jison",
+    "../../generic/grammar/truncate/truncate_table.jison",
+    "../../generic/grammar/udf/aggregate/aggregate_common.jison",
+    "../../generic/grammar/udf/aggregate/avg.jison",
+    "../../generic/grammar/udf/aggregate/count.jison",
+    "../../generic/grammar/udf/aggregate/max.jison",
+    "../../generic/grammar/udf/aggregate/min.jison",
+    "../../generic/grammar/udf/aggregate/stddev_pop.jison",
+    "../../generic/grammar/udf/aggregate/stddev_samp.jison",
+    "../../generic/grammar/udf/aggregate/sum.jison",
+    "../../generic/grammar/udf/aggregate/var_pop.jison",
+    "../../generic/grammar/udf/aggregate/var_samp.jison",
+    "../../generic/grammar/udf/aggregate/variance.jison",
+    "../../generic/grammar/udf/analytic/analytic.jison",
+    "../../generic/grammar/udf/function/array.jison",
+    "../../generic/grammar/udf/function/cast.jison",
+    "../../generic/grammar/udf/function/if.jison",
+    "../../generic/grammar/udf/function/map.jison",
+    "../../generic/grammar/udf/function/truncate.jison",
+    "../../generic/grammar/udf/udf_common.jison",
+    "../../generic/grammar/update/update_table.jison",
+    "../../generic/grammar/use/use.jison",
+    "../../generic/grammar/common/error/error.jison",
+    "../../generic/grammar/main.jison",
+    "../../generic/grammar/common/table/table.jison",
+    "../../generic/grammar/common/value_expression/value_expression.jison",
+    "../../generic/grammar/common/header_and_footer/footer.jison"
+  ]
+}

src/parsing/parsers/postgresql/postgresqlAutocompleteParser.test.ts → src/autocomplete/parsers/postgresql/postgresqlAutocompleteParser.test.ts

@@ -4,12 +4,12 @@ import {CommonParser, assertPartials} from '../../lib/parsing-typed';
 import type {AutocompleteParser} from '../../lib/types';
 import {extractTestCases, runTestCases} from '../../test/testing';
 
-import structure from './jison/structure.json';
+import structure from './grammar/structure.json';
 import {postgresqlAutocompleteParser} from './postgresqlAutocompleteParser';
 
-const jisonFolder = 'src/parsing/parsers/postgresql/jison';
-const groupedTestCases = extractTestCases(jisonFolder, structure.autocomplete);
-// const groupedTestCases = extractTestCases(jisonFolder, structure.autocomplete.filter((item) => item.indexOf('offset_clause') > -1));
+const grammarFolder = 'src/autocomplete/parsers/postgresql/grammar';
+const groupedTestCases = extractTestCases(grammarFolder, structure.autocomplete);
+// const groupedTestCases = extractTestCases(grammarFolder, structure.autocomplete.filter((item) => item.indexOf('offset_clause') > -1));
 
 describe('postgresqlAutocompleteParser', () => {
     // TODO: Fix the types

src/parsing/test/testing.ts → src/autocomplete/test/testing.ts

@@ -431,13 +431,13 @@ For example, if alter_table.jison is part of the structure it will look for alte
 exists it'll parse it (TestCase[]). Test cases are grouped per found .jison file.
  */
 export function extractTestCases(
-    jisonFolder: string,
+    grammarFolder: string,
     structureFiles: string[],
 ): GroupedTestCases[] {
     const groupedTestCases: GroupedTestCases[] = [];
 
     structureFiles.forEach((jisonFile) => {
-        const testFile = `${jisonFolder}/${jisonFile.replace('.jison', '.test.json')}`;
+        const testFile = `${grammarFolder}/${jisonFile.replace('.jison', '.test.json')}`;
         if (!existsSync(testFile)) {
             return;
         }