tp: move tokenizer into its own folder

In prep for LALR based parsing of PerfettoSQL. Change-Id: I7113adb1e8eda9b66d12cba95e222a4cdf91d814
google · Sep 13, 2024 · 870964a · 870964a
1 parent 1e5f937
commit 870964a
Show file tree

Hide file tree

Showing 15 changed files with 102 additions and 16 deletions.
diff --git a/Android.bp b/Android.bp
@@ -2512,6 +2512,7 @@ cc_test {
         ":perfetto_src_trace_processor_perfetto_sql_parser_parser",
         ":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor",
+        ":perfetto_src_trace_processor_perfetto_sql_tokenizer_tokenizer",
         ":perfetto_src_trace_processor_sorter_sorter",
         ":perfetto_src_trace_processor_sqlite_bindings_bindings",
         ":perfetto_src_trace_processor_sqlite_sqlite",
@@ -13430,6 +13431,14 @@ filegroup {
     name: "perfetto_src_trace_processor_perfetto_sql_parser_test_utils",
 }
 
+// GN: //src/trace_processor/perfetto_sql/parser:unittests
+filegroup {
+    name: "perfetto_src_trace_processor_perfetto_sql_parser_unittests",
+    srcs: [
+        "src/trace_processor/perfetto_sql/parser/perfetto_sql_parser_unittest.cc",
+    ],
+}
+
 // GN: //src/trace_processor/perfetto_sql/preprocessor:grammar
 filegroup {
     name: "perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar",
@@ -13602,6 +13611,22 @@ genrule {
     ],
 }
 
+// GN: //src/trace_processor/perfetto_sql/tokenizer:tokenizer
+filegroup {
+    name: "perfetto_src_trace_processor_perfetto_sql_tokenizer_tokenizer",
+    srcs: [
+        "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.cc",
+    ],
+}
+
+// GN: //src/trace_processor/perfetto_sql/tokenizer:unittests
+filegroup {
+    name: "perfetto_src_trace_processor_perfetto_sql_tokenizer_unittests",
+    srcs: [
+        "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer_unittest.cc",
+    ],
+}
+
 // GN: //src/trace_processor/rpc:httpd
 filegroup {
     name: "perfetto_src_trace_processor_rpc_httpd",
@@ -13666,7 +13691,6 @@ filegroup {
         "src/trace_processor/sqlite/sql_source.cc",
         "src/trace_processor/sqlite/sql_stats_table.cc",
         "src/trace_processor/sqlite/sqlite_engine.cc",
-        "src/trace_processor/sqlite/sqlite_tokenizer.cc",
         "src/trace_processor/sqlite/sqlite_utils.cc",
         "src/trace_processor/sqlite/stats_table.cc",
     ],
@@ -13678,7 +13702,6 @@ filegroup {
     srcs: [
         "src/trace_processor/sqlite/db_sqlite_table_unittest.cc",
         "src/trace_processor/sqlite/sql_source_unittest.cc",
-        "src/trace_processor/sqlite/sqlite_tokenizer_unittest.cc",
         "src/trace_processor/sqlite/sqlite_utils_unittest.cc",
     ],
 }
@@ -15570,9 +15593,12 @@ cc_test {
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_types_types",
         ":perfetto_src_trace_processor_perfetto_sql_parser_parser",
         ":perfetto_src_trace_processor_perfetto_sql_parser_test_utils",
+        ":perfetto_src_trace_processor_perfetto_sql_parser_unittests",
         ":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor",
         ":perfetto_src_trace_processor_perfetto_sql_preprocessor_unittests",
+        ":perfetto_src_trace_processor_perfetto_sql_tokenizer_tokenizer",
+        ":perfetto_src_trace_processor_perfetto_sql_tokenizer_unittests",
         ":perfetto_src_trace_processor_rpc_rpc",
         ":perfetto_src_trace_processor_rpc_unittests",
         ":perfetto_src_trace_processor_sorter_sorter",
@@ -16621,6 +16647,7 @@ cc_binary {
         ":perfetto_src_trace_processor_perfetto_sql_parser_parser",
         ":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor",
+        ":perfetto_src_trace_processor_perfetto_sql_tokenizer_tokenizer",
         ":perfetto_src_trace_processor_rpc_httpd",
         ":perfetto_src_trace_processor_rpc_rpc",
         ":perfetto_src_trace_processor_rpc_stdiod",
@@ -17035,6 +17062,7 @@ cc_binary_host {
         ":perfetto_src_trace_processor_perfetto_sql_parser_parser",
         ":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor",
+        ":perfetto_src_trace_processor_perfetto_sql_tokenizer_tokenizer",
         ":perfetto_src_trace_processor_sorter_sorter",
         ":perfetto_src_trace_processor_sqlite_bindings_bindings",
         ":perfetto_src_trace_processor_sqlite_sqlite",

diff --git a/BUILD b/BUILD
@@ -270,6 +270,7 @@ perfetto_cc_library(
         ":src_trace_processor_perfetto_sql_parser_parser",
         ":src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":src_trace_processor_perfetto_sql_preprocessor_preprocessor",
+        ":src_trace_processor_perfetto_sql_tokenizer_tokenizer",
         ":src_trace_processor_rpc_rpc",
         ":src_trace_processor_sorter_sorter",
         ":src_trace_processor_sqlite_bindings_bindings",
@@ -3013,6 +3014,15 @@ perfetto_cc_amalgamated_sql(
     namespace = "stdlib",
 )
 
+# GN target: //src/trace_processor/perfetto_sql/tokenizer:tokenizer
+perfetto_filegroup(
+    name = "src_trace_processor_perfetto_sql_tokenizer_tokenizer",
+    srcs = [
+        "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.cc",
+        "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h",
+    ],
+)
+
 # GN target: //src/trace_processor/rpc:httpd
 perfetto_filegroup(
     name = "src_trace_processor_rpc_httpd",
@@ -3083,8 +3093,6 @@ perfetto_filegroup(
         "src/trace_processor/sqlite/sql_stats_table.h",
         "src/trace_processor/sqlite/sqlite_engine.cc",
         "src/trace_processor/sqlite/sqlite_engine.h",
-        "src/trace_processor/sqlite/sqlite_tokenizer.cc",
-        "src/trace_processor/sqlite/sqlite_tokenizer.h",
         "src/trace_processor/sqlite/sqlite_utils.cc",
         "src/trace_processor/sqlite/sqlite_utils.h",
         "src/trace_processor/sqlite/stats_table.cc",
@@ -6329,6 +6337,7 @@ perfetto_cc_library(
         ":src_trace_processor_perfetto_sql_parser_parser",
         ":src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":src_trace_processor_perfetto_sql_preprocessor_preprocessor",
+        ":src_trace_processor_perfetto_sql_tokenizer_tokenizer",
         ":src_trace_processor_sorter_sorter",
         ":src_trace_processor_sqlite_bindings_bindings",
         ":src_trace_processor_sqlite_sqlite",
@@ -6523,6 +6532,7 @@ perfetto_cc_binary(
         ":src_trace_processor_perfetto_sql_parser_parser",
         ":src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":src_trace_processor_perfetto_sql_preprocessor_preprocessor",
+        ":src_trace_processor_perfetto_sql_tokenizer_tokenizer",
         ":src_trace_processor_rpc_httpd",
         ":src_trace_processor_rpc_rpc",
         ":src_trace_processor_rpc_stdiod",
@@ -6774,6 +6784,7 @@ perfetto_cc_binary(
         ":src_trace_processor_perfetto_sql_parser_parser",
         ":src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":src_trace_processor_perfetto_sql_preprocessor_preprocessor",
+        ":src_trace_processor_perfetto_sql_tokenizer_tokenizer",
         ":src_trace_processor_sorter_sorter",
         ":src_trace_processor_sqlite_bindings_bindings",
         ":src_trace_processor_sqlite_sqlite",

diff --git a/src/trace_processor/BUILD.gn b/src/trace_processor/BUILD.gn
@@ -327,7 +327,9 @@ perfetto_unittest_source_set("unittests") {
       "perfetto_sql/intrinsics/functions:unittests",
       "perfetto_sql/intrinsics/operators:unittests",
       "perfetto_sql/intrinsics/table_functions:unittests",
+      "perfetto_sql/parser:unittests",
       "perfetto_sql/preprocessor:unittests",
+      "perfetto_sql/tokenizer:unittests",
       "sqlite:unittests",
     ]
   }

diff --git a/src/trace_processor/perfetto_sql/engine/BUILD.gn b/src/trace_processor/perfetto_sql/engine/BUILD.gn
@@ -46,6 +46,7 @@ source_set("engine") {
     "../../util:stdlib",
     "../parser",
     "../preprocessor",
+    "../tokenizer",
   ]
 }
 

diff --git a/src/trace_processor/perfetto_sql/parser/BUILD.gn b/src/trace_processor/perfetto_sql/parser/BUILD.gn
@@ -32,6 +32,7 @@ source_set("parser") {
     "../../util",
     "../../util:sql_argument",
     "../preprocessor",
+    "../tokenizer",
   ]
 }
 

diff --git a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc
@@ -30,8 +30,8 @@
 #include "perfetto/ext/base/string_utils.h"
 #include "src/trace_processor/perfetto_sql/parser/function_util.h"
 #include "src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h"
+#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h"
 #include "src/trace_processor/sqlite/sql_source.h"
-#include "src/trace_processor/sqlite/sqlite_tokenizer.h"
 #include "src/trace_processor/util/sql_argument.h"
 
 namespace perfetto {

diff --git a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.h b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.h
@@ -26,8 +26,8 @@
 #include "perfetto/ext/base/flat_hash_map.h"
 #include "src/trace_processor/perfetto_sql/parser/function_util.h"
 #include "src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h"
+#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h"
 #include "src/trace_processor/sqlite/sql_source.h"
-#include "src/trace_processor/sqlite/sqlite_tokenizer.h"
 #include "src/trace_processor/util/sql_argument.h"
 
 namespace perfetto {

diff --git a/src/trace_processor/perfetto_sql/preprocessor/BUILD.gn b/src/trace_processor/perfetto_sql/preprocessor/BUILD.gn
@@ -27,6 +27,7 @@ source_set("preprocessor") {
     "../../../base",
     "../../sqlite",
     "../../util",
+    "../tokenizer",
   ]
 }
 

diff --git a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.cc b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.cc
@@ -36,8 +36,8 @@
 #include "perfetto/ext/base/flat_hash_map.h"
 #include "perfetto/ext/base/string_utils.h"
 #include "src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar_interface.h"
+#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h"
 #include "src/trace_processor/sqlite/sql_source.h"
-#include "src/trace_processor/sqlite/sqlite_tokenizer.h"
 
 namespace perfetto::trace_processor {
 namespace {

diff --git a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h
@@ -24,8 +24,8 @@
 
 #include "perfetto/base/status.h"
 #include "perfetto/ext/base/flat_hash_map.h"
+#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h"
 #include "src/trace_processor/sqlite/sql_source.h"
-#include "src/trace_processor/sqlite/sqlite_tokenizer.h"
 
 namespace perfetto::trace_processor {
 

diff --git a/src/trace_processor/perfetto_sql/tokenizer/BUILD.gn b/src/trace_processor/perfetto_sql/tokenizer/BUILD.gn
@@ -0,0 +1,45 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("../../../../gn/test.gni")
+
+assert(enable_perfetto_trace_processor_sqlite)
+
+source_set("tokenizer") {
+  sources = [
+    "sqlite_tokenizer.cc",
+    "sqlite_tokenizer.h",
+  ]
+  deps = [
+    "../../../../gn:default_deps",
+    "../../../../gn:sqlite",
+    "../../../base",
+    "../../sqlite",
+  ]
+}
+
+perfetto_unittest_source_set("unittests") {
+  testonly = true
+  sources = [ "sqlite_tokenizer_unittest.cc" ]
+  deps = [
+    ":tokenizer",
+    "../../../../gn:default_deps",
+    "../../../../gn:gtest_and_gmock",
+    "../../../../gn:sqlite",
+    "../../../base",
+    "../../../base:test_support",
+    "../../sqlite",
+    "../../util",
+  ]
+}
diff --git a/...race_processor/sqlite/sqlite_tokenizer.cc → ...erfetto_sql/tokenizer/sqlite_tokenizer.cc b/...race_processor/sqlite/sqlite_tokenizer.cc → ...erfetto_sql/tokenizer/sqlite_tokenizer.cc
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-#include "src/trace_processor/sqlite/sqlite_tokenizer.h"
+#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h"
 
 #include <ctype.h>
 #include <sqlite3.h>

diff --git a/...trace_processor/sqlite/sqlite_tokenizer.h → ...perfetto_sql/tokenizer/sqlite_tokenizer.h b/...trace_processor/sqlite/sqlite_tokenizer.h → ...perfetto_sql/tokenizer/sqlite_tokenizer.h
@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-#ifndef SRC_TRACE_PROCESSOR_SQLITE_SQLITE_TOKENIZER_H_
-#define SRC_TRACE_PROCESSOR_SQLITE_SQLITE_TOKENIZER_H_
+#ifndef SRC_TRACE_PROCESSOR_PERFETTO_SQL_TOKENIZER_SQLITE_TOKENIZER_H_
+#define SRC_TRACE_PROCESSOR_PERFETTO_SQL_TOKENIZER_SQLITE_TOKENIZER_H_
 
 #include <optional>
 #include <string_view>
@@ -166,4 +166,4 @@ class SqliteTokenizer {
 }  // namespace trace_processor
 }  // namespace perfetto
 
-#endif  // SRC_TRACE_PROCESSOR_SQLITE_SQLITE_TOKENIZER_H_
+#endif  // SRC_TRACE_PROCESSOR_PERFETTO_SQL_TOKENIZER_SQLITE_TOKENIZER_H_
diff --git a/...essor/sqlite/sqlite_tokenizer_unittest.cc → ...ql/tokenizer/sqlite_tokenizer_unittest.cc b/...essor/sqlite/sqlite_tokenizer_unittest.cc → ...ql/tokenizer/sqlite_tokenizer_unittest.cc
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-#include "src/trace_processor/sqlite/sqlite_tokenizer.h"
+#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h"
 #include <vector>
 
 #include "perfetto/base/logging.h"

diff --git a/src/trace_processor/sqlite/BUILD.gn b/src/trace_processor/sqlite/BUILD.gn
@@ -28,8 +28,6 @@ source_set("sqlite") {
     "sql_stats_table.h",
     "sqlite_engine.cc",
     "sqlite_engine.h",
-    "sqlite_tokenizer.cc",
-    "sqlite_tokenizer.h",
     "sqlite_utils.cc",
     "sqlite_utils.h",
     "stats_table.cc",
@@ -64,7 +62,6 @@ perfetto_unittest_source_set("unittests") {
   sources = [
     "db_sqlite_table_unittest.cc",
     "sql_source_unittest.cc",
-    "sqlite_tokenizer_unittest.cc",
     "sqlite_utils_unittest.cc",
   ]
   deps = [