added setting to disable new catalog fetch from info schema

hafenkran · Oct 26, 2024 · 674fa1e · 674fa1e
1 parent 3b0dc45
commit 674fa1e
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 22 deletions.
diff --git a/README.md b/README.md
@@ -221,11 +221,12 @@ D SELECT * FROM bigquery_scan('bigquery-public-data.geo_us_boundaries.cnecta', b
 
 ### Additional Extension Settings
 
-| Settings                        | Description                                                               | Default |
-| ------------------------------- | ------------------------------------------------------------------------- | ------- |
-| bq_debug_show_queries           | [DEBUG] - whether to print all queries sent to BigQuery to stdout         | `false` |
-| bq_experimental_filter_pushdown | [EXPERIMENTAL] - Whether or not to use filter pushdown                    | `true`  |
-| bq_curl_ca_bundle_path          | Path to the CA certificates used by cURL for SSL certificate verification |         |
+| Setting                         | Description                                                                                | Default |
+| ------------------------------- | ------------------------------------------------------------------------------------------ | ------- |
+| bq_debug_show_queries           | [DEBUG] - whether to print all queries sent to BigQuery to stdout                          | `false` |
+| bq_experimental_filter_pushdown | [EXPERIMENTAL] - Whether or not to use filter pushdown                                     | `true`  |
+| bq_experimental_use_info_schema | [EXPERIMENTAL] - Use information schema to fetch catalog info (often faster than REST API) | `true`  |
+| bq_curl_ca_bundle_path          | Path to the CA certificates used by cURL for SSL certificate verification                  |         |
 
 ## Limitations
 

diff --git a/src/bigquery_extension.cpp b/src/bigquery_extension.cpp
@@ -47,6 +47,11 @@ static void LoadInternal(DatabaseInstance &instance) {
                               LogicalType::BOOLEAN,
                               Value(bigquery::BigquerySettings::ExperimentalFilterPushdown()),
                               bigquery::BigquerySettings::SetExperimentalFilterPushdown);
+	config.AddExtensionOption("bq_experimental_use_info_schema",
+							  "Whether to fetch table infos from BQ information schema (currently experimental). Can be significantly faster than fetching from REST API.",
+							  LogicalType::BOOLEAN,
+							  Value(bigquery::BigquerySettings::ExperimentalFetchCatalogFromInformationSchema()),
+							  bigquery::BigquerySettings::SetExperimentalFetchCatalogFromInformationSchema);
     config.AddExtensionOption("bq_debug_show_queries",
                               "DEBUG SETTING: print all queries sent to BigQuery to stdout",
                               LogicalType::BOOLEAN,

diff --git a/src/include/bigquery_settings.hpp b/src/include/bigquery_settings.hpp
@@ -78,6 +78,15 @@ struct BigquerySettings {
 		}
 		CurlCaBundlePath() = path;
 	}
+
+	static bool& ExperimentalFetchCatalogFromInformationSchema() {
+		static bool bigquery_experimental_fetch_catalog_from_information_schema = true;
+		return bigquery_experimental_fetch_catalog_from_information_schema;
+	}
+
+	static void SetExperimentalFetchCatalogFromInformationSchema(ClientContext &context, SetScope scope, Value &parameter) {
+		ExperimentalFetchCatalogFromInformationSchema() = BooleanValue::Get(parameter);
+	}
 };
 
 

diff --git a/src/storage/bigquery_table_set.cpp b/src/storage/bigquery_table_set.cpp
@@ -5,6 +5,7 @@
 #include "bigquery_client.hpp"
 #include "bigquery_sql.hpp"
 #include "bigquery_utils.hpp"
+#include "bigquery_settings.hpp"
 #include "storage/bigquery_schema_entry.hpp"
 #include "storage/bigquery_table_set.hpp"
 #include "storage/bigquery_transaction.hpp"
@@ -71,24 +72,29 @@ void BigqueryTableSet::LoadEntries(ClientContext &context) {
     auto &transaction = BigqueryTransaction::Get(context, catalog);
     auto bqclient = transaction.GetBigqueryClient();
 
-	map<string, CreateTableInfo> table_infos;
-	bqclient->GetTableInfosFromDataset(schema.GetBigqueryDatasetRef(), table_infos);
-
-	for (auto &table_info : table_infos) {
-		auto table_entry = make_uniq<BigqueryTableEntry>(catalog, schema, table_info.second);
-		CreateEntry(std::move(table_entry));
+	if (BigquerySettings::ExperimentalFetchCatalogFromInformationSchema()) {
+		map<string, CreateTableInfo> table_infos;
+		bqclient->GetTableInfosFromDataset(schema.GetBigqueryDatasetRef(), table_infos);
+
+		for (auto &table_info : table_infos) {
+			auto table_entry = make_uniq<BigqueryTableEntry>(catalog, schema, table_info.second);
+			CreateEntry(std::move(table_entry));
+		}
+	} else {
+		unique_ptr<BigqueryTableInfo> info;
+		vector<unique_ptr<BigqueryTableInfo>> table_infos;
+
+		vector<BigqueryTableRef> table_refs = bqclient->GetTables(schema.name);
+		for (auto &table_ref : table_refs) {
+			info = make_uniq<BigqueryTableInfo>(table_ref);
+			bqclient->GetTableInfo(table_ref.dataset_id,
+								table_ref.table_id,
+								info->create_info->columns,
+								info->create_info->constraints);
+			auto table_entry = make_uniq<BigqueryTableEntry>(catalog, schema, *info);
+			CreateEntry(std::move(table_entry));
+		}
 	}
-
-	// vector<BigqueryTableRef> table_refs = bqclient->GetTables(schema.name);
-    // for (auto &table_ref : table_refs) {
-    //     info = make_uniq<BigqueryTableInfo>(table_ref);
-    //     bqclient->GetTableInfo(table_ref.dataset_id,
-    //                            table_ref.table_id,
-    //                            info->create_info->columns,
-    //                            info->create_info->constraints);
-    //     auto table_entry = make_uniq<BigqueryTableEntry>(catalog, schema, *info);
-    //     CreateEntry(std::move(table_entry));
-    // }
 }
 
 } // namespace bigquery