Skip to content

Commit

Permalink
feat(geospatial): update read_geo to support url
Browse files Browse the repository at this point in the history
In the upcoming update of the spatial extension, to be able to
read form a url, the httpfs extension is needed.
  • Loading branch information
ncclementi committed Dec 15, 2023
1 parent 2987f3e commit 8b331ef
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 1 deletion.
7 changes: 6 additions & 1 deletion ibis/backends/duckdb/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -528,8 +528,13 @@ def read_geo(
# load geospatial extension
self.load_extension("spatial")

source = util.normalize_filename(source)

if source.startswith(("http://", "https://", "s3://")):
self._load_extensions(["httpfs"])

source_expr = sa.select(sa.literal_column("*")).select_from(
sa.func.st_read(util.normalize_filename(source), _format_kwargs(kwargs))
sa.func.st_read(source, _format_kwargs(kwargs))
)

view = self._compile_temp_view(table_name, source_expr)
Expand Down
26 changes: 26 additions & 0 deletions ibis/backends/duckdb/tests/test_register.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,32 @@ def test_read_geo_to_geopandas(con, data_dir):
assert isinstance(gdf, gpd.GeoDataFrame)


@pytest.mark.xfail(
LINUX and SANDBOXED,
reason="nix on linux cannot download duckdb extensions or data due to sandboxing",
)
def test_read_geo_from_url():
# httpfs is needed to read from url on nightly spatial extension
gpd = pytest.importorskip("geopandas")

con = ibis.duckdb.connect()
# TODO(ncclementi): remove force install when spatial is updated
# see https://github.com/duckdb/duckdb_spatial/issues/210
con.raw_sql(
"""\
FORCE INSTALL spatial FROM 'http://nightly-extensions.duckdb.org';
LOAD spatial;
"""
)

url = "https://raw.githubusercontent.com/ibis-project/testing-data/master/geojson/zones.geojson"

t = con.read_geo(url)

gdf = t.head().to_pandas()
assert isinstance(gdf, gpd.GeoDataFrame)


@pytest.mark.xfail_version(
duckdb=["duckdb<0.7.0"], reason="read_json_auto doesn't exist", raises=exc.IbisError
)
Expand Down

0 comments on commit 8b331ef

Please sign in to comment.