From 870964a365612d04aa4b325710f0ef9321ee4893 Mon Sep 17 00:00:00 2001 From: Lalit Maganti Date: Fri, 13 Sep 2024 17:27:26 +0100 Subject: [PATCH] tp: move tokenizer into its own folder In prep for LALR based parsing of PerfettoSQL. Change-Id: I7113adb1e8eda9b66d12cba95e222a4cdf91d814 --- Android.bp | 32 ++++++++++++- BUILD | 15 ++++++- src/trace_processor/BUILD.gn | 2 + .../perfetto_sql/engine/BUILD.gn | 1 + .../perfetto_sql/parser/BUILD.gn | 1 + .../parser/perfetto_sql_parser.cc | 2 +- .../perfetto_sql/parser/perfetto_sql_parser.h | 2 +- .../perfetto_sql/preprocessor/BUILD.gn | 1 + .../preprocessor/perfetto_sql_preprocessor.cc | 2 +- .../preprocessor/perfetto_sql_preprocessor.h | 2 +- .../perfetto_sql/tokenizer/BUILD.gn | 45 +++++++++++++++++++ .../tokenizer}/sqlite_tokenizer.cc | 2 +- .../tokenizer}/sqlite_tokenizer.h | 6 +-- .../tokenizer}/sqlite_tokenizer_unittest.cc | 2 +- src/trace_processor/sqlite/BUILD.gn | 3 -- 15 files changed, 102 insertions(+), 16 deletions(-) create mode 100644 src/trace_processor/perfetto_sql/tokenizer/BUILD.gn rename src/trace_processor/{sqlite => perfetto_sql/tokenizer}/sqlite_tokenizer.cc (99%) rename src/trace_processor/{sqlite => perfetto_sql/tokenizer}/sqlite_tokenizer.h (95%) rename src/trace_processor/{sqlite => perfetto_sql/tokenizer}/sqlite_tokenizer_unittest.cc (97%) diff --git a/Android.bp b/Android.bp index a45d4ff394..2d05823d2c 100644 --- a/Android.bp +++ b/Android.bp @@ -2512,6 +2512,7 @@ cc_test { ":perfetto_src_trace_processor_perfetto_sql_parser_parser", ":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar", ":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor", + ":perfetto_src_trace_processor_perfetto_sql_tokenizer_tokenizer", ":perfetto_src_trace_processor_sorter_sorter", ":perfetto_src_trace_processor_sqlite_bindings_bindings", ":perfetto_src_trace_processor_sqlite_sqlite", @@ -13430,6 +13431,14 @@ filegroup { name: "perfetto_src_trace_processor_perfetto_sql_parser_test_utils", } +// GN: //src/trace_processor/perfetto_sql/parser:unittests +filegroup { + name: "perfetto_src_trace_processor_perfetto_sql_parser_unittests", + srcs: [ + "src/trace_processor/perfetto_sql/parser/perfetto_sql_parser_unittest.cc", + ], +} + // GN: //src/trace_processor/perfetto_sql/preprocessor:grammar filegroup { name: "perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar", @@ -13602,6 +13611,22 @@ genrule { ], } +// GN: //src/trace_processor/perfetto_sql/tokenizer:tokenizer +filegroup { + name: "perfetto_src_trace_processor_perfetto_sql_tokenizer_tokenizer", + srcs: [ + "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.cc", + ], +} + +// GN: //src/trace_processor/perfetto_sql/tokenizer:unittests +filegroup { + name: "perfetto_src_trace_processor_perfetto_sql_tokenizer_unittests", + srcs: [ + "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer_unittest.cc", + ], +} + // GN: //src/trace_processor/rpc:httpd filegroup { name: "perfetto_src_trace_processor_rpc_httpd", @@ -13666,7 +13691,6 @@ filegroup { "src/trace_processor/sqlite/sql_source.cc", "src/trace_processor/sqlite/sql_stats_table.cc", "src/trace_processor/sqlite/sqlite_engine.cc", - "src/trace_processor/sqlite/sqlite_tokenizer.cc", "src/trace_processor/sqlite/sqlite_utils.cc", "src/trace_processor/sqlite/stats_table.cc", ], @@ -13678,7 +13702,6 @@ filegroup { srcs: [ "src/trace_processor/sqlite/db_sqlite_table_unittest.cc", "src/trace_processor/sqlite/sql_source_unittest.cc", - "src/trace_processor/sqlite/sqlite_tokenizer_unittest.cc", "src/trace_processor/sqlite/sqlite_utils_unittest.cc", ], } @@ -15570,9 +15593,12 @@ cc_test { ":perfetto_src_trace_processor_perfetto_sql_intrinsics_types_types", ":perfetto_src_trace_processor_perfetto_sql_parser_parser", ":perfetto_src_trace_processor_perfetto_sql_parser_test_utils", + ":perfetto_src_trace_processor_perfetto_sql_parser_unittests", ":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar", ":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor", ":perfetto_src_trace_processor_perfetto_sql_preprocessor_unittests", + ":perfetto_src_trace_processor_perfetto_sql_tokenizer_tokenizer", + ":perfetto_src_trace_processor_perfetto_sql_tokenizer_unittests", ":perfetto_src_trace_processor_rpc_rpc", ":perfetto_src_trace_processor_rpc_unittests", ":perfetto_src_trace_processor_sorter_sorter", @@ -16621,6 +16647,7 @@ cc_binary { ":perfetto_src_trace_processor_perfetto_sql_parser_parser", ":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar", ":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor", + ":perfetto_src_trace_processor_perfetto_sql_tokenizer_tokenizer", ":perfetto_src_trace_processor_rpc_httpd", ":perfetto_src_trace_processor_rpc_rpc", ":perfetto_src_trace_processor_rpc_stdiod", @@ -17035,6 +17062,7 @@ cc_binary_host { ":perfetto_src_trace_processor_perfetto_sql_parser_parser", ":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar", ":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor", + ":perfetto_src_trace_processor_perfetto_sql_tokenizer_tokenizer", ":perfetto_src_trace_processor_sorter_sorter", ":perfetto_src_trace_processor_sqlite_bindings_bindings", ":perfetto_src_trace_processor_sqlite_sqlite", diff --git a/BUILD b/BUILD index 6ce4ad980f..1d59aaedbb 100644 --- a/BUILD +++ b/BUILD @@ -270,6 +270,7 @@ perfetto_cc_library( ":src_trace_processor_perfetto_sql_parser_parser", ":src_trace_processor_perfetto_sql_preprocessor_grammar", ":src_trace_processor_perfetto_sql_preprocessor_preprocessor", + ":src_trace_processor_perfetto_sql_tokenizer_tokenizer", ":src_trace_processor_rpc_rpc", ":src_trace_processor_sorter_sorter", ":src_trace_processor_sqlite_bindings_bindings", @@ -3013,6 +3014,15 @@ perfetto_cc_amalgamated_sql( namespace = "stdlib", ) +# GN target: //src/trace_processor/perfetto_sql/tokenizer:tokenizer +perfetto_filegroup( + name = "src_trace_processor_perfetto_sql_tokenizer_tokenizer", + srcs = [ + "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.cc", + "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h", + ], +) + # GN target: //src/trace_processor/rpc:httpd perfetto_filegroup( name = "src_trace_processor_rpc_httpd", @@ -3083,8 +3093,6 @@ perfetto_filegroup( "src/trace_processor/sqlite/sql_stats_table.h", "src/trace_processor/sqlite/sqlite_engine.cc", "src/trace_processor/sqlite/sqlite_engine.h", - "src/trace_processor/sqlite/sqlite_tokenizer.cc", - "src/trace_processor/sqlite/sqlite_tokenizer.h", "src/trace_processor/sqlite/sqlite_utils.cc", "src/trace_processor/sqlite/sqlite_utils.h", "src/trace_processor/sqlite/stats_table.cc", @@ -6329,6 +6337,7 @@ perfetto_cc_library( ":src_trace_processor_perfetto_sql_parser_parser", ":src_trace_processor_perfetto_sql_preprocessor_grammar", ":src_trace_processor_perfetto_sql_preprocessor_preprocessor", + ":src_trace_processor_perfetto_sql_tokenizer_tokenizer", ":src_trace_processor_sorter_sorter", ":src_trace_processor_sqlite_bindings_bindings", ":src_trace_processor_sqlite_sqlite", @@ -6523,6 +6532,7 @@ perfetto_cc_binary( ":src_trace_processor_perfetto_sql_parser_parser", ":src_trace_processor_perfetto_sql_preprocessor_grammar", ":src_trace_processor_perfetto_sql_preprocessor_preprocessor", + ":src_trace_processor_perfetto_sql_tokenizer_tokenizer", ":src_trace_processor_rpc_httpd", ":src_trace_processor_rpc_rpc", ":src_trace_processor_rpc_stdiod", @@ -6774,6 +6784,7 @@ perfetto_cc_binary( ":src_trace_processor_perfetto_sql_parser_parser", ":src_trace_processor_perfetto_sql_preprocessor_grammar", ":src_trace_processor_perfetto_sql_preprocessor_preprocessor", + ":src_trace_processor_perfetto_sql_tokenizer_tokenizer", ":src_trace_processor_sorter_sorter", ":src_trace_processor_sqlite_bindings_bindings", ":src_trace_processor_sqlite_sqlite", diff --git a/src/trace_processor/BUILD.gn b/src/trace_processor/BUILD.gn index 4539b5adc7..3834b57ebb 100644 --- a/src/trace_processor/BUILD.gn +++ b/src/trace_processor/BUILD.gn @@ -327,7 +327,9 @@ perfetto_unittest_source_set("unittests") { "perfetto_sql/intrinsics/functions:unittests", "perfetto_sql/intrinsics/operators:unittests", "perfetto_sql/intrinsics/table_functions:unittests", + "perfetto_sql/parser:unittests", "perfetto_sql/preprocessor:unittests", + "perfetto_sql/tokenizer:unittests", "sqlite:unittests", ] } diff --git a/src/trace_processor/perfetto_sql/engine/BUILD.gn b/src/trace_processor/perfetto_sql/engine/BUILD.gn index b2576819cc..adb6b6d9f3 100644 --- a/src/trace_processor/perfetto_sql/engine/BUILD.gn +++ b/src/trace_processor/perfetto_sql/engine/BUILD.gn @@ -46,6 +46,7 @@ source_set("engine") { "../../util:stdlib", "../parser", "../preprocessor", + "../tokenizer", ] } diff --git a/src/trace_processor/perfetto_sql/parser/BUILD.gn b/src/trace_processor/perfetto_sql/parser/BUILD.gn index 831e3c80c8..f31cae205e 100644 --- a/src/trace_processor/perfetto_sql/parser/BUILD.gn +++ b/src/trace_processor/perfetto_sql/parser/BUILD.gn @@ -32,6 +32,7 @@ source_set("parser") { "../../util", "../../util:sql_argument", "../preprocessor", + "../tokenizer", ] } diff --git a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc index 8f30a949b6..4af83ea1e4 100644 --- a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc +++ b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc @@ -30,8 +30,8 @@ #include "perfetto/ext/base/string_utils.h" #include "src/trace_processor/perfetto_sql/parser/function_util.h" #include "src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h" +#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h" #include "src/trace_processor/sqlite/sql_source.h" -#include "src/trace_processor/sqlite/sqlite_tokenizer.h" #include "src/trace_processor/util/sql_argument.h" namespace perfetto { diff --git a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.h b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.h index 3252b9aed0..8321bb62a0 100644 --- a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.h +++ b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.h @@ -26,8 +26,8 @@ #include "perfetto/ext/base/flat_hash_map.h" #include "src/trace_processor/perfetto_sql/parser/function_util.h" #include "src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h" +#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h" #include "src/trace_processor/sqlite/sql_source.h" -#include "src/trace_processor/sqlite/sqlite_tokenizer.h" #include "src/trace_processor/util/sql_argument.h" namespace perfetto { diff --git a/src/trace_processor/perfetto_sql/preprocessor/BUILD.gn b/src/trace_processor/perfetto_sql/preprocessor/BUILD.gn index 7b3e7bc405..7314069323 100644 --- a/src/trace_processor/perfetto_sql/preprocessor/BUILD.gn +++ b/src/trace_processor/perfetto_sql/preprocessor/BUILD.gn @@ -27,6 +27,7 @@ source_set("preprocessor") { "../../../base", "../../sqlite", "../../util", + "../tokenizer", ] } diff --git a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.cc b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.cc index 31a2b0c850..e664c259c7 100644 --- a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.cc +++ b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.cc @@ -36,8 +36,8 @@ #include "perfetto/ext/base/flat_hash_map.h" #include "perfetto/ext/base/string_utils.h" #include "src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar_interface.h" +#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h" #include "src/trace_processor/sqlite/sql_source.h" -#include "src/trace_processor/sqlite/sqlite_tokenizer.h" namespace perfetto::trace_processor { namespace { diff --git a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h index 669d964107..fdbee3e12d 100644 --- a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h +++ b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h @@ -24,8 +24,8 @@ #include "perfetto/base/status.h" #include "perfetto/ext/base/flat_hash_map.h" +#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h" #include "src/trace_processor/sqlite/sql_source.h" -#include "src/trace_processor/sqlite/sqlite_tokenizer.h" namespace perfetto::trace_processor { diff --git a/src/trace_processor/perfetto_sql/tokenizer/BUILD.gn b/src/trace_processor/perfetto_sql/tokenizer/BUILD.gn new file mode 100644 index 0000000000..1fbbd8f279 --- /dev/null +++ b/src/trace_processor/perfetto_sql/tokenizer/BUILD.gn @@ -0,0 +1,45 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("../../../../gn/test.gni") + +assert(enable_perfetto_trace_processor_sqlite) + +source_set("tokenizer") { + sources = [ + "sqlite_tokenizer.cc", + "sqlite_tokenizer.h", + ] + deps = [ + "../../../../gn:default_deps", + "../../../../gn:sqlite", + "../../../base", + "../../sqlite", + ] +} + +perfetto_unittest_source_set("unittests") { + testonly = true + sources = [ "sqlite_tokenizer_unittest.cc" ] + deps = [ + ":tokenizer", + "../../../../gn:default_deps", + "../../../../gn:gtest_and_gmock", + "../../../../gn:sqlite", + "../../../base", + "../../../base:test_support", + "../../sqlite", + "../../util", + ] +} diff --git a/src/trace_processor/sqlite/sqlite_tokenizer.cc b/src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.cc similarity index 99% rename from src/trace_processor/sqlite/sqlite_tokenizer.cc rename to src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.cc index 972857b7a9..fb1b3f2f90 100644 --- a/src/trace_processor/sqlite/sqlite_tokenizer.cc +++ b/src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.cc @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "src/trace_processor/sqlite/sqlite_tokenizer.h" +#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h" #include #include diff --git a/src/trace_processor/sqlite/sqlite_tokenizer.h b/src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h similarity index 95% rename from src/trace_processor/sqlite/sqlite_tokenizer.h rename to src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h index c496b4cbe1..64019fb2a7 100644 --- a/src/trace_processor/sqlite/sqlite_tokenizer.h +++ b/src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef SRC_TRACE_PROCESSOR_SQLITE_SQLITE_TOKENIZER_H_ -#define SRC_TRACE_PROCESSOR_SQLITE_SQLITE_TOKENIZER_H_ +#ifndef SRC_TRACE_PROCESSOR_PERFETTO_SQL_TOKENIZER_SQLITE_TOKENIZER_H_ +#define SRC_TRACE_PROCESSOR_PERFETTO_SQL_TOKENIZER_SQLITE_TOKENIZER_H_ #include #include @@ -166,4 +166,4 @@ class SqliteTokenizer { } // namespace trace_processor } // namespace perfetto -#endif // SRC_TRACE_PROCESSOR_SQLITE_SQLITE_TOKENIZER_H_ +#endif // SRC_TRACE_PROCESSOR_PERFETTO_SQL_TOKENIZER_SQLITE_TOKENIZER_H_ diff --git a/src/trace_processor/sqlite/sqlite_tokenizer_unittest.cc b/src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer_unittest.cc similarity index 97% rename from src/trace_processor/sqlite/sqlite_tokenizer_unittest.cc rename to src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer_unittest.cc index e7aa4cff8e..9fc950beaa 100644 --- a/src/trace_processor/sqlite/sqlite_tokenizer_unittest.cc +++ b/src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer_unittest.cc @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "src/trace_processor/sqlite/sqlite_tokenizer.h" +#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h" #include #include "perfetto/base/logging.h" diff --git a/src/trace_processor/sqlite/BUILD.gn b/src/trace_processor/sqlite/BUILD.gn index fda30ba077..06150e0b1f 100644 --- a/src/trace_processor/sqlite/BUILD.gn +++ b/src/trace_processor/sqlite/BUILD.gn @@ -28,8 +28,6 @@ source_set("sqlite") { "sql_stats_table.h", "sqlite_engine.cc", "sqlite_engine.h", - "sqlite_tokenizer.cc", - "sqlite_tokenizer.h", "sqlite_utils.cc", "sqlite_utils.h", "stats_table.cc", @@ -64,7 +62,6 @@ perfetto_unittest_source_set("unittests") { sources = [ "db_sqlite_table_unittest.cc", "sql_source_unittest.cc", - "sqlite_tokenizer_unittest.cc", "sqlite_utils_unittest.cc", ] deps = [