diff --git a/ibis/backends/bigquery/__init__.py b/ibis/backends/bigquery/__init__.py index d832eeb20acd3..5a0a13113158b 100644 --- a/ibis/backends/bigquery/__init__.py +++ b/ibis/backends/bigquery/__init__.py @@ -903,6 +903,10 @@ def list_tables( ::: schema [deprecated] The schema (dataset) inside `database` to perform the list against. + + TODO (mehmet): BigQuery API does not seem to allow for + retrieving only tables or only views: + https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/list#request-body """ table_loc = self._warn_and_create_table_loc(database, schema) diff --git a/ibis/backends/clickhouse/__init__.py b/ibis/backends/clickhouse/__init__.py index bc4a685ace116..07158b6f84cc6 100644 --- a/ibis/backends/clickhouse/__init__.py +++ b/ibis/backends/clickhouse/__init__.py @@ -197,17 +197,23 @@ def list_databases(self, like: str | None = None) -> list[str]: return self._filter_with_like(databases, like) def list_tables( - self, like: str | None = None, database: str | None = None + self, + like: str | None = None, + database: str | None = None, + no_views: bool = False, ) -> list[str]: - """List the tables in the database. + """List the tables and views in the database. Parameters ---------- like - A pattern to use for listing tables. + A pattern to use for listing tables/views. database - Database to list tables from. Default behavior is to show tables in - the current database. + Database to list tables/views from. Default behavior is to + show tables/views in the current database. + no_views + Whether to list only tables (True) or both tables and + views (False). """ query = sg.select(C.name).from_(sg.table("tables", db="system")) @@ -217,7 +223,10 @@ def list_tables( else: database = sge.convert(database) - query = query.where(C.database.eq(database).or_(C.is_temporary)) + where_exprs = [C.database.eq(database).or_(C.is_temporary)] + if no_views: + where_exprs.append(C.engine.neq("View")) + query = query.where(*where_exprs) with self._safe_raw_sql(query) as result: results = result.result_columns @@ -454,6 +463,7 @@ def raw_sql( external_data = self._normalize_external_tables(external_tables) with contextlib.suppress(AttributeError): query = query.sql(dialect=self.name, pretty=True) + self._log(query) return self.con.query(query, external_data=external_data, **kwargs) diff --git a/ibis/backends/clickhouse/tests/test_client.py b/ibis/backends/clickhouse/tests/test_client.py index 38ff49b8863d3..80deb7f93342b 100644 --- a/ibis/backends/clickhouse/tests/test_client.py +++ b/ibis/backends/clickhouse/tests/test_client.py @@ -190,6 +190,22 @@ def test_list_tables_database(con): assert set(tables) & set(tables2) +def test_list_tables_w_no_views(con, temp_table): + table = con.create_table(temp_table, obj={"a": [1]}) + view_name = f"{temp_table}_view" + con.create_view(name=view_name, obj=table) + + tables_and_views = con.list_tables() + assert temp_table in tables_and_views + assert view_name in tables_and_views + + tables = con.list_tables(no_views=True) + assert temp_table in tables + assert view_name not in tables + + con.drop_view(view_name, force=True) + + @pytest.fixture def temp_db(con, worker_id): dbname = f"clickhouse_create_database_{worker_id}" diff --git a/ibis/backends/flink/__init__.py b/ibis/backends/flink/__init__.py index 53d9205db7145..c9cea240f2f51 100644 --- a/ibis/backends/flink/__init__.py +++ b/ibis/backends/flink/__init__.py @@ -150,24 +150,32 @@ def list_tables( database: str | None = None, catalog: str | None = None, temp: bool = False, + no_views: bool = False, ) -> list[str]: """Return the list of table/view names. - Return the list of table/view names in the `database` and `catalog`. If - `database`/`catalog` are not specified, their default values will be - used. Temporary tables can only be listed for the default database and - catalog, hence `database` and `catalog` are ignored if `temp` is True. + Returns the list of all temporary or permanent table/view + names in the `database` and `catalog`. If `database` and + `catalog` are not specified, their default values will be + used. If `temp` is True, permanent tables/views get excluded, + and `database` and `catalog` arguments are ignored as + temporary tables/views can only be listed for the default + database and catalog. Parameters ---------- like : str, optional A pattern in Python's regex format. - temp : bool, optional - Whether to list temporary tables or permanent tables. database : str, optional The database to list tables of, if not the current one. catalog : str, optional The catalog to list tables of, if not the current one. + temp : bool, optional + Whether to list only temporary tables/views (True) or + both temporary and permanent tables/views (False). + no_views : bool, optional + Whether to list only tables (True) or both tables and + views (False). Returns ------- @@ -185,10 +193,17 @@ def list_tables( # the temporary tables in a given catalog and database. # Ref: https://nightlies.apache.org/flink/flink-docs-master/api/java/org/apache/flink/table/api/TableEnvironment.html tables = self._table_env.list_temporary_tables() + if no_views: + views = self._table_env.list_temporary_views() + tables = list(set(tables) - set(views)) + else: # Note (mehmet): `listTables` returns both tables and views. # Ref: Docstring for pyflink/table/table_environment.py:list_tables() tables = self._table_env._j_tenv.listTables(catalog, database) + if no_views: + views = self._table_env._j_tenv.listViews() + tables = list(set(tables) - set(views)) return self._filter_with_like(tables, like) @@ -199,14 +214,13 @@ def list_views( ) -> list[str]: """Return the list of view names. - Return the list of view names. - Parameters ---------- like : str, optional A pattern in Python's regex format. temp : bool, optional - Whether to list temporary views or permanent views. + Whether to list only the temporary views (True) or both + temporary and permanent views (False). Returns ------- diff --git a/ibis/backends/flink/tests/test_ddl.py b/ibis/backends/flink/tests/test_ddl.py index 44742bc7c8213..bfbdbb5f390ee 100644 --- a/ibis/backends/flink/tests/test_ddl.py +++ b/ibis/backends/flink/tests/test_ddl.py @@ -337,6 +337,28 @@ def test_create_view( assert temp_view not in con.list_tables() +def test_list_tables_w_no_views( + con, temp_table, awards_players_schema, csv_source_configs +): + table = con.create_table( + name=temp_table, + schema=awards_players_schema, + tbl_properties=csv_source_configs("awards_players"), + ) + view_name = "my_view" + con.create_view( + name=view_name, + obj=table, + force=False, + overwrite=False, + ) + + assert view_name in con.list_tables() + assert view_name not in con.list_tables(no_views=True) + + con.drop_view(view_name, force=True) + + def test_rename_table(con, awards_players_schema, temp_table, csv_source_configs): table_name = temp_table con.create_table( diff --git a/ibis/backends/pyspark/__init__.py b/ibis/backends/pyspark/__init__.py index ca346b6583f6b..fcec65d439093 100644 --- a/ibis/backends/pyspark/__init__.py +++ b/ibis/backends/pyspark/__init__.py @@ -198,7 +198,10 @@ def list_databases(self, like: str | None = None) -> list[str]: return self._filter_with_like(databases, like) def list_tables( - self, like: str | None = None, database: str | None = None + self, + like: str | None = None, + database: str | None = None, + no_views: bool = False, ) -> list[str]: """List the tables in the database. @@ -209,12 +212,25 @@ def list_tables( database Database to list tables from. Default behavior is to show tables in the current database. + no_views : bool, optional + Whether to list only tables (True) or both tables and + views (False). """ + view_set = set() + if no_views: + view_set = { + row.viewName + for row in self._session.sql( + f"SHOW VIEWS IN {database or self.current_database}" + ).collect() + } + tables = [ row.tableName for row in self._session.sql( f"SHOW TABLES IN {database or self.current_database}" ).collect() + if row.tableName not in view_set ] return self._filter_with_like(tables, like) diff --git a/ibis/backends/pyspark/tests/conftest.py b/ibis/backends/pyspark/tests/conftest.py index 0724fbe598c5b..f0b1574b8ff80 100644 --- a/ibis/backends/pyspark/tests/conftest.py +++ b/ibis/backends/pyspark/tests/conftest.py @@ -31,6 +31,8 @@ def _load_data(self, **_: Any) -> None: sort_cols = {"functional_alltypes": "id"} + # TODO (mehmet): Why are all created as views here? + # Why not use `self.connection.create_table()`? for name in TEST_TABLES: path = str(self.data_dir / "parquet" / f"{name}.parquet") t = s.read.parquet(path).repartition(num_partitions) diff --git a/ibis/backends/tests/test_api.py b/ibis/backends/tests/test_api.py index 482be7075abee..315712a8001bf 100644 --- a/ibis/backends/tests/test_api.py +++ b/ibis/backends/tests/test_api.py @@ -68,6 +68,22 @@ def test_list_tables(con): assert all(isinstance(table, str) for table in tables) +def test_list_tables_w_no_views(ddl_con, temp_view): + table_name = "functional_alltypes" + expr = ddl_con.table(table_name) + ddl_con.create_view(temp_view, expr) + + tables_and_views = ddl_con.list_tables() + assert table_name in tables_and_views + assert temp_view in tables_and_views + + tables = ddl_con.list_tables(no_views=True) + # Note: The following does not hold for backends that + # create the test tables as views, e.g. pyspark. + # assert table_name in tables + assert temp_view not in tables + + def test_tables_accessor_mapping(con): if con.name == "snowflake": pytest.skip("snowflake sometimes counts more tables than are around")