From 8cec0c40467ae2f88f242facd5866fd430408e91 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Tue, 4 Jul 2023 10:02:29 +0800 Subject: [PATCH 1/9] geopandas: Mapping int/int64 to int32 for OGR_GMT format --- pygmt/helpers/tempfile.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pygmt/helpers/tempfile.py b/pygmt/helpers/tempfile.py index c184c5e73d3..b5eba162fdd 100644 --- a/pygmt/helpers/tempfile.py +++ b/pygmt/helpers/tempfile.py @@ -126,18 +126,27 @@ def tempfile_from_geojson(geojson): E.g. '1a2b3c4d5e6.gmt'. """ with GMTTempFile(suffix=".gmt") as tmpfile: + # pylint: disable=import-outside-toplevel + import geopandas as gpd + os.remove(tmpfile.name) # ensure file is deleted first ogrgmt_kwargs = {"filename": tmpfile.name, "driver": "OGR_GMT", "mode": "w"} try: + # Workaround to map int/int64 to int32 + # https://github.com/geopandas/geopandas/issues/967#issuecomment-842877704 + # https://github.com/GenericMappingTools/pygmt/issues/2497 + schema = gpd.io.file.infer_schema(geojson) + for col, dtype in schema["properties"].items(): + if dtype in ("int", "int64"): + schema["properties"][col] = "int32" + ogrgmt_kwargs["schema"] = schema # Using geopandas.to_file to directly export to OGR_GMT format geojson.to_file(**ogrgmt_kwargs) except AttributeError: - # pylint: disable=import-outside-toplevel # Other 'geo' formats which implement __geo_interface__ import json import fiona - import geopandas as gpd with fiona.Env(): jsontext = json.dumps(geojson.__geo_interface__) From f2f10cacc9e11a37136468fa62f3a47a1fa62025 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Thu, 3 Aug 2023 09:57:11 +0800 Subject: [PATCH 2/9] Update the comment --- pygmt/helpers/tempfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygmt/helpers/tempfile.py b/pygmt/helpers/tempfile.py index b5eba162fdd..99bf4615006 100644 --- a/pygmt/helpers/tempfile.py +++ b/pygmt/helpers/tempfile.py @@ -132,7 +132,7 @@ def tempfile_from_geojson(geojson): os.remove(tmpfile.name) # ensure file is deleted first ogrgmt_kwargs = {"filename": tmpfile.name, "driver": "OGR_GMT", "mode": "w"} try: - # Workaround to map int/int64 to int32 + # Map int/int64 to int32 because OGR_GMT only supports 32-bit integer # https://github.com/geopandas/geopandas/issues/967#issuecomment-842877704 # https://github.com/GenericMappingTools/pygmt/issues/2497 schema = gpd.io.file.infer_schema(geojson) From 84f87a5f5a4869dab153d61bb1c01dcc2c28da5a Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Wed, 9 Aug 2023 12:32:37 +0800 Subject: [PATCH 3/9] Fix a linting issue --- pygmt/helpers/tempfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygmt/helpers/tempfile.py b/pygmt/helpers/tempfile.py index 584f137b203..8771ce82f8d 100644 --- a/pygmt/helpers/tempfile.py +++ b/pygmt/helpers/tempfile.py @@ -132,7 +132,7 @@ def tempfile_from_geojson(geojson): os.remove(tmpfile.name) # ensure file is deleted first ogrgmt_kwargs = {"filename": tmpfile.name, "driver": "OGR_GMT", "mode": "w"} try: - # Map int/int64 to int32 because OGR_GMT only supports 32-bit integer + # Map int/int64 to int32 since OGR_GMT only supports 32-bit integer # https://github.com/geopandas/geopandas/issues/967#issuecomment-842877704 # https://github.com/GenericMappingTools/pygmt/issues/2497 schema = gpd.io.file.infer_schema(geojson) From c1b3b437b2cddc72861c0e6d9064c9c83cc67c1f Mon Sep 17 00:00:00 2001 From: Wei Ji <23487320+weiji14@users.noreply.github.com> Date: Fri, 11 Aug 2023 16:49:41 +1200 Subject: [PATCH 4/9] Add unit test for plotting GeoDataFrame with int32 and int64 columns Using the @RidgeTest.shp dataset that has been buffered, and colouring the polygons with a different colormap. --- .../test_geopandas_plot_int_dtypes.png.dvc | 4 ++ pygmt/tests/test_geopandas.py | 44 ++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 pygmt/tests/baseline/test_geopandas_plot_int_dtypes.png.dvc diff --git a/pygmt/tests/baseline/test_geopandas_plot_int_dtypes.png.dvc b/pygmt/tests/baseline/test_geopandas_plot_int_dtypes.png.dvc new file mode 100644 index 00000000000..89d18d4aef6 --- /dev/null +++ b/pygmt/tests/baseline/test_geopandas_plot_int_dtypes.png.dvc @@ -0,0 +1,4 @@ +outs: +- md5: 6ce1b73d25ac10900a2ce4c7148d2953 + size: 43434 + path: test_geopandas_plot_int_dtypes.png diff --git a/pygmt/tests/test_geopandas.py b/pygmt/tests/test_geopandas.py index 106ed798cf5..0029dfe6e3c 100644 --- a/pygmt/tests/test_geopandas.py +++ b/pygmt/tests/test_geopandas.py @@ -2,8 +2,9 @@ Test integration with geopandas. """ import numpy.testing as npt +import pandas as pd import pytest -from pygmt import Figure, info +from pygmt import Figure, info, which, makecpt gpd = pytest.importorskip("geopandas") shapely = pytest.importorskip("shapely") @@ -131,3 +132,44 @@ def test_geopandas_plot3d_non_default_circle(): style="c0.2c", ) return fig + + +@pytest.mark.parametrize("dtype", ["int32", "int64", pd.Int32Dtype(), pd.Int64Dtype()]) +@pytest.mark.mpl_image_compare(filename="test_geopandas_plot_int_dtypes.png") +def test_geopandas_plot_int_dtypes(dtype): + """ + Check that plotting a geopandas GeoDataFrame with integer columns works, + including int32 and int64 (non-nullable), Int32 and Int64 (nullable). + + This is a regression test for + https://github.com/GenericMappingTools/pygmt/issues/2497 + """ + # Read shapefile in geopandas.GeoDataFrame + shapefile = which( + fname="@RidgeTest.shp @RidgeTest.shx @RidgeTest.dbf @RidgeTest.prj", + download="c", + ) + gdf = gpd.read_file(shapefile[0]) + + # Reproject geometry and change dtype of NPOINTS column + gdf["geometry"] = ( + gdf.to_crs(crs="EPSG:3857") + .buffer(distance=100000) + .to_crs(crs="OGC:CRS84") # convert to lon/lat to prevent @null in PROJ CRS + ) + gdf["NPOINTS"] = gdf.NPOINTS.astype(dtype=dtype) + + # Plot figure with three polygons colored based on NPOINTS value + fig = Figure() + makecpt(cmap="lajolla", series=[10, 60, 10], continuous=True) + fig.plot( + data=gdf, + frame=True, + pen="1p,black", + close=True, + fill="+z", + cmap=True, + aspatial="Z=NPOINTS", + ) + fig.colorbar() + return fig From 20891c1b79bd56fca053460070e647d3b2cdf698 Mon Sep 17 00:00:00 2001 From: Wei Ji <23487320+weiji14@users.noreply.github.com> Date: Fri, 11 Aug 2023 16:51:41 +1200 Subject: [PATCH 5/9] Isort imports --- pygmt/tests/test_geopandas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygmt/tests/test_geopandas.py b/pygmt/tests/test_geopandas.py index 0029dfe6e3c..fb108ce1801 100644 --- a/pygmt/tests/test_geopandas.py +++ b/pygmt/tests/test_geopandas.py @@ -4,7 +4,7 @@ import numpy.testing as npt import pandas as pd import pytest -from pygmt import Figure, info, which, makecpt +from pygmt import Figure, info, makecpt, which gpd = pytest.importorskip("geopandas") shapely = pytest.importorskip("shapely") From 60ba52a84b54dce92840931f3e255a57383b52e6 Mon Sep 17 00:00:00 2001 From: Wei Ji <23487320+weiji14@users.noreply.github.com> Date: Sat, 19 Aug 2023 15:49:42 +1200 Subject: [PATCH 6/9] Change cmap from lajolla to lisbon The lajolla colormap is flipped from Scientific Colour Maps v7 to v8, causing differences in the baseline image. --- pygmt/tests/baseline/test_geopandas_plot_int_dtypes.png.dvc | 4 ++-- pygmt/tests/test_geopandas.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pygmt/tests/baseline/test_geopandas_plot_int_dtypes.png.dvc b/pygmt/tests/baseline/test_geopandas_plot_int_dtypes.png.dvc index 89d18d4aef6..a2e9a9a7975 100644 --- a/pygmt/tests/baseline/test_geopandas_plot_int_dtypes.png.dvc +++ b/pygmt/tests/baseline/test_geopandas_plot_int_dtypes.png.dvc @@ -1,4 +1,4 @@ outs: -- md5: 6ce1b73d25ac10900a2ce4c7148d2953 - size: 43434 +- md5: c1c6eda2a88adf4d96f18f4e0c5db4d5 + size: 43582 path: test_geopandas_plot_int_dtypes.png diff --git a/pygmt/tests/test_geopandas.py b/pygmt/tests/test_geopandas.py index fb108ce1801..b8de62414d3 100644 --- a/pygmt/tests/test_geopandas.py +++ b/pygmt/tests/test_geopandas.py @@ -161,7 +161,7 @@ def test_geopandas_plot_int_dtypes(dtype): # Plot figure with three polygons colored based on NPOINTS value fig = Figure() - makecpt(cmap="lajolla", series=[10, 60, 10], continuous=True) + makecpt(cmap="lisbon", series=[10, 60, 10], continuous=True) fig.plot( data=gdf, frame=True, From a6d431560c3af21ca1c4bc47bc717d93e1cd0d45 Mon Sep 17 00:00:00 2001 From: Wei Ji <23487320+weiji14@users.noreply.github.com> Date: Sat, 19 Aug 2023 15:53:51 +1200 Subject: [PATCH 7/9] Disable Int32 and Int64 dtype tests until geopandas 0.13.3 or greater Need patch from https://github.com/geopandas/geopandas/pull/2950 before we can use nullable dtypes pd.Int32Dtype() and pd.Int64Dtype(). --- pygmt/tests/test_geopandas.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pygmt/tests/test_geopandas.py b/pygmt/tests/test_geopandas.py index b8de62414d3..412f881a479 100644 --- a/pygmt/tests/test_geopandas.py +++ b/pygmt/tests/test_geopandas.py @@ -134,7 +134,17 @@ def test_geopandas_plot3d_non_default_circle(): return fig -@pytest.mark.parametrize("dtype", ["int32", "int64", pd.Int32Dtype(), pd.Int64Dtype()]) +@pytest.mark.parametrize( + "dtype", + [ + "int32", + "int64", + # Enable Int32/Int64 dtypes when geopandas>=0.13.3 is released with + # patch https://github.com/geopandas/geopandas/pull/2950 + # pd.Int32Dtype(), + # pd.Int64Dtype(), + ], +) @pytest.mark.mpl_image_compare(filename="test_geopandas_plot_int_dtypes.png") def test_geopandas_plot_int_dtypes(dtype): """ From 1e05a29f9a59d228d64b993ac9ebcc2f1e9d5833 Mon Sep 17 00:00:00 2001 From: Wei Ji <23487320+weiji14@users.noreply.github.com> Date: Sat, 19 Aug 2023 15:56:53 +1200 Subject: [PATCH 8/9] Remove unused pandas import --- pygmt/tests/test_geopandas.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pygmt/tests/test_geopandas.py b/pygmt/tests/test_geopandas.py index 412f881a479..44992df76da 100644 --- a/pygmt/tests/test_geopandas.py +++ b/pygmt/tests/test_geopandas.py @@ -2,7 +2,6 @@ Test integration with geopandas. """ import numpy.testing as npt -import pandas as pd import pytest from pygmt import Figure, info, makecpt, which From 2097add02f44949f6668aa63539666c341d2aa66 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Sun, 20 Aug 2023 10:55:46 +0800 Subject: [PATCH 9/9] Force an index name and reset index --- pygmt/helpers/tempfile.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pygmt/helpers/tempfile.py b/pygmt/helpers/tempfile.py index 8771ce82f8d..d9c132e696b 100644 --- a/pygmt/helpers/tempfile.py +++ b/pygmt/helpers/tempfile.py @@ -135,6 +135,9 @@ def tempfile_from_geojson(geojson): # Map int/int64 to int32 since OGR_GMT only supports 32-bit integer # https://github.com/geopandas/geopandas/issues/967#issuecomment-842877704 # https://github.com/GenericMappingTools/pygmt/issues/2497 + if geojson.index.name is None: + geojson.index.name = "index" + geojson = geojson.reset_index(drop=False) schema = gpd.io.file.infer_schema(geojson) for col, dtype in schema["properties"].items(): if dtype in ("int", "int64"):