Fix geometry issues

README.md

@@ -147,27 +147,6 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
 <!-- readme: collaborators,contributors -start -->
 <table>
 <tr>
-    <td align="center">
-        <a href="https://github.com/hazemakhalek">
-            <img src="https://avatars.githubusercontent.com/u/26235356?v=4" width="100;" alt="hazemakhalek"/>
-            <br />
-            <sub><b>Null</b></sub>
-        </a>
-    </td>
-    <td align="center">
-        <a href="https://github.com/jarry7">
-            <img src="https://avatars.githubusercontent.com/u/27745389?v=4" width="100;" alt="jarry7"/>
-            <br />
-            <sub><b>Jarrad Wright</b></sub>
-        </a>
-    </td>
-    <td align="center">
-        <a href="https://github.com/fneum">
-            <img src="https://avatars.githubusercontent.com/u/29101152?v=4" width="100;" alt="fneum"/>
-            <br />
-            <sub><b>Fabian Neumann</b></sub>
-        </a>
-    </td>
     <td align="center">
         <a href="https://github.com/ekatef">
             <img src="https://avatars.githubusercontent.com/u/30229437?v=4" width="100;" alt="ekatef"/>
@@ -176,70 +155,41 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
         </a>
     </td>
     <td align="center">
-        <a href="https://github.com/euronion">
-            <img src="https://avatars.githubusercontent.com/u/42553970?v=4" width="100;" alt="euronion"/>
-            <br />
-            <sub><b>Euronion</b></sub>
-        </a>
-    </td>
-    <td align="center">
-        <a href="https://github.com/Justus-coded">
-            <img src="https://avatars.githubusercontent.com/u/44394641?v=4" width="100;" alt="Justus-coded"/>
-            <br />
-            <sub><b>Justus Ilemobayo</b></sub>
-        </a>
-    </td></tr>
-<tr>
-    <td align="center">
-        <a href="https://github.com/mnm-matin">
-            <img src="https://avatars.githubusercontent.com/u/45293386?v=4" width="100;" alt="mnm-matin"/>
+        <a href="https://github.com/davide-f">
+            <img src="https://avatars.githubusercontent.com/u/67809479?v=4" width="100;" alt="davide-f"/>
             <br />
-            <sub><b>Mnm-matin</b></sub>
+            <sub><b>Davide-f</b></sub>
         </a>
     </td>
     <td align="center">
-        <a href="https://github.com/desenk">
-            <img src="https://avatars.githubusercontent.com/u/48335263?v=4" width="100;" alt="desenk"/>
+        <a href="https://github.com/pz-max">
+            <img src="https://avatars.githubusercontent.com/u/61968949?v=4" width="100;" alt="pz-max"/>
             <br />
-            <sub><b>Desen Kirli</b></sub>
+            <sub><b>Max Parzen</b></sub>
         </a>
     </td>
     <td align="center">
-        <a href="https://github.com/LukasFrankenQ">
-            <img src="https://avatars.githubusercontent.com/u/55196140?v=4" width="100;" alt="LukasFrankenQ"/>
+        <a href="https://github.com/restyled-commits">
+            <img src="https://avatars.githubusercontent.com/u/65077583?v=4" width="100;" alt="restyled-commits"/>
             <br />
-            <sub><b>Lukas Franken</b></sub>
+            <sub><b>Restyled Commits</b></sub>
         </a>
     </td>
     <td align="center">
-        <a href="https://github.com/pz-max">
-            <img src="https://avatars.githubusercontent.com/u/61968949?v=4" width="100;" alt="pz-max"/>
+        <a href="https://github.com/mnm-matin">
+            <img src="https://avatars.githubusercontent.com/u/45293386?v=4" width="100;" alt="mnm-matin"/>
             <br />
-            <sub><b>Max Parzen</b></sub>
+            <sub><b>Mnm-matin</b></sub>
         </a>
     </td>
     <td align="center">
-        <a href="https://github.com/Cesare-Caputo">
-            <img src="https://avatars.githubusercontent.com/u/62548290?v=4" width="100;" alt="Cesare-Caputo"/>
+        <a href="https://github.com/DeniseGiub">
+            <img src="https://avatars.githubusercontent.com/u/113139589?v=4" width="100;" alt="DeniseGiub"/>
             <br />
             <sub><b>Null</b></sub>
         </a>
-    </td>
-    <td align="center">
-        <a href="https://github.com/davide-f">
-            <img src="https://avatars.githubusercontent.com/u/67809479?v=4" width="100;" alt="davide-f"/>
-            <br />
-            <sub><b>Davide-f</b></sub>
-        </a>
     </td></tr>
 <tr>
-    <td align="center">
-        <a href="https://github.com/koen-vg">
-            <img src="https://avatars.githubusercontent.com/u/74298901?v=4" width="100;" alt="koen-vg"/>
-            <br />
-            <sub><b>Koen Van Greevenbroek</b></sub>
-        </a>
-    </td>
     <td align="center">
         <a href="https://github.com/Hazem-IEG">
             <img src="https://avatars.githubusercontent.com/u/87850910?v=4" width="100;" alt="Hazem-IEG"/>
@@ -255,46 +205,46 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
         </a>
     </td>
     <td align="center">
-        <a href="https://github.com/AnasAlgarei">
-            <img src="https://avatars.githubusercontent.com/u/101210563?v=4" width="100;" alt="AnasAlgarei"/>
+        <a href="https://github.com/Tomkourou">
+            <img src="https://avatars.githubusercontent.com/u/5240283?v=4" width="100;" alt="Tomkourou"/>
             <br />
-            <sub><b>AnasAlgarei</b></sub>
+            <sub><b>Thomas Kouroughli</b></sub>
         </a>
     </td>
     <td align="center">
-        <a href="https://github.com/restyled-commits">
-            <img src="https://avatars.githubusercontent.com/u/65077583?v=4" width="100;" alt="restyled-commits"/>
+        <a href="https://github.com/giacfalk">
+            <img src="https://avatars.githubusercontent.com/u/36954873?v=4" width="100;" alt="giacfalk"/>
             <br />
-            <sub><b>Restyled Commits</b></sub>
+            <sub><b>Giacomo Falchetta</b></sub>
         </a>
     </td>
     <td align="center">
-        <a href="https://github.com/DeniseGiub">
-            <img src="https://avatars.githubusercontent.com/u/113139589?v=4" width="100;" alt="DeniseGiub"/>
+        <a href="https://github.com/Ekaterina-Vo">
+            <img src="https://avatars.githubusercontent.com/u/99509555?v=4" width="100;" alt="Ekaterina-Vo"/>
             <br />
-            <sub><b>Null</b></sub>
+            <sub><b>Ekaterina-Vo</b></sub>
         </a>
-    </td></tr>
-<tr>
+    </td>
     <td align="center">
-        <a href="https://github.com/Tomkourou">
-            <img src="https://avatars.githubusercontent.com/u/5240283?v=4" width="100;" alt="Tomkourou"/>
+        <a href="https://github.com/euronion">
+            <img src="https://avatars.githubusercontent.com/u/42553970?v=4" width="100;" alt="euronion"/>
             <br />
-            <sub><b>Thomas Kouroughli</b></sub>
+            <sub><b>Euronion</b></sub>
         </a>
-    </td>
+    </td></tr>
+<tr>
     <td align="center">
-        <a href="https://github.com/giacfalk">
-            <img src="https://avatars.githubusercontent.com/u/36954873?v=4" width="100;" alt="giacfalk"/>
+        <a href="https://github.com/AnasAlgarei">
+            <img src="https://avatars.githubusercontent.com/u/101210563?v=4" width="100;" alt="AnasAlgarei"/>
             <br />
-            <sub><b>Giacomo Falchetta</b></sub>
+            <sub><b>AnasAlgarei</b></sub>
         </a>
     </td>
     <td align="center">
-        <a href="https://github.com/Ekaterina-Vo">
-            <img src="https://avatars.githubusercontent.com/u/99509555?v=4" width="100;" alt="Ekaterina-Vo"/>
+        <a href="https://github.com/LukasFrankenQ">
+            <img src="https://avatars.githubusercontent.com/u/55196140?v=4" width="100;" alt="LukasFrankenQ"/>
             <br />
-            <sub><b>Ekaterina-Vo</b></sub>
+            <sub><b>Lukas Franken</b></sub>
         </a>
     </td>
     <td align="center">
@@ -304,6 +254,13 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
             <sub><b>Jarrad Wright</b></sub>
         </a>
     </td>
+    <td align="center">
+        <a href="https://github.com/koen-vg">
+            <img src="https://avatars.githubusercontent.com/u/74298901?v=4" width="100;" alt="koen-vg"/>
+            <br />
+            <sub><b>Koen Van Greevenbroek</b></sub>
+        </a>
+    </td>
     <td align="center">
         <a href="https://github.com/EmreYorat">
             <img src="https://avatars.githubusercontent.com/u/93644024?v=4" width="100;" alt="EmreYorat"/>
@@ -319,6 +276,13 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
         </a>
     </td></tr>
 <tr>
+    <td align="center">
+        <a href="https://github.com/jarry7">
+            <img src="https://avatars.githubusercontent.com/u/27745389?v=4" width="100;" alt="jarry7"/>
+            <br />
+            <sub><b>Jarrad Wright</b></sub>
+        </a>
+    </td>
     <td align="center">
         <a href="https://github.com/squoilin">
             <img src="https://avatars.githubusercontent.com/u/4547840?v=4" width="100;" alt="squoilin"/>

scripts/build_bus_regions.py

@@ -76,6 +76,9 @@ def custom_voronoi_partition_pts(points, outline, add_bounds_shape=True, multipl
     import numpy as np
     from scipy.spatial import Voronoi
     from shapely.geometry import Point, Polygon
+    from shapely.ops import unary_union
+
+    outline = unary_union(outline)
 
     points = np.asarray(points)
 
@@ -267,13 +270,13 @@ def get_id(coords):
             offshore_regions.append(offshore_regions_c)
 
     # create geodataframe and remove nan shapes
-    onshore_regions = gpd.GeoDataFrame(
-        pd.concat(onshore_regions, ignore_index=True),
-        crs=country_shapes.crs,
-    ).dropna(axis="index", subset=["geometry"])
-
-    onshore_regions = pd.concat([onshore_regions], ignore_index=True).to_file(
-        snakemake.output.regions_onshore
+    onshore_regions = (
+        gpd.GeoDataFrame(
+            pd.concat(onshore_regions, ignore_index=True),
+            crs=country_shapes.crs,
+        )
+        .dropna(axis="index", subset=["geometry"])
+        .to_file(snakemake.output.regions_onshore)
     )
 
     if offshore_regions:

scripts/build_osm_network.py

@@ -830,8 +830,12 @@ def built_network(inputs, outputs, geo_crs, distance_crs):
     country_list = input
     bus_country_list = buses["country"].unique().tolist()
 
-    if len(bus_country_list) != len(country_list):
-        no_data_countries = set(country_list).difference(set(bus_country_list))
+    # it may happen that bus_country_list contains entries not relevant as a country name (e.g. "not found")
+    # difference can't give negative values; the following will return only releant country names
+    no_data_countries = set(country_list).difference(set(bus_country_list))
+
+    if len(no_data_countries) > 0:
+
         no_data_countries_shape = country_shapes[
             country_shapes.index.isin(no_data_countries) == True
         ].reset_index()

scripts/build_shapes.py

@@ -20,6 +20,7 @@
 import xarray as xr
 from _helpers import (
     configure_logging,
+    country_name_2_two_digits,
     sets_path_to_root,
     three_2_two_digits_country,
     two_2_three_digits_country,
@@ -80,6 +81,49 @@ def download_GADM(country_code, update=False, out_logging=False):
     return GADM_inputfile_gpkg, GADM_filename
 
 
+def restore_country_code_by_name(row, country_code):
+    if row["GID_0"] != country_code:
+        return country_name_2_two_digits(row["COUNTRY"])
+    else:
+        return row["GID_0"]
+
+
+#  file=file_gpkg, layer=layer_id, cc=country_code
+def build_gadm_df(file, layer, cc):
+    # read gpkg file
+    geodf = gpd.read_file(file, layer="ADM_ADM_" + str(layer)).to_crs(geo_crs)
+
+    # convert country name representation of the main country (GID_0 column)
+    geodf["GID_0"] = [three_2_two_digits_country(twoD_c) for twoD_c in geodf["GID_0"]]
+
+    # GID_0 may have some exotic values, "COUNTRY" column may be used instead
+    geodf["GID_0"] = geodf.apply(
+        lambda x: restore_country_code_by_name(x, country_code=cc), axis=1
+    )
+
+    # "not found" hardcoded according to country_converter conventions
+    geodf.drop(geodf[geodf["GID_0"] == "not found"].index, inplace=True)
+
+    # create a subindex column that is useful
+    # in the GADM processing of sub-national zones
+    geodf["GADM_ID"] = geodf[f"GID_{layer}"]
+
+    if layer >= 1:
+        available_gadm_codes = geodf["GADM_ID"].unique()
+        code_three_digits = two_2_three_digits_country(cc)
+
+        # normally the GADM code starts the ISO3
+        non_std_gadm_codes = [
+            w for w in available_gadm_codes if not w.startswith(code_three_digits)
+        ]
+
+        if len(non_std_gadm_codes) > 0:
+            df_filtered = geodf[geodf["GADM_ID"].isin(non_std_gadm_codes)]
+            df_filtered.to_csv("non_standard_gadm_" + cc + "_raw.csv", index=False)
+
+    return geodf
+
+
 def get_GADM_layer(country_list, layer_id, geo_crs, update=False, outlogging=False):
     """
     Function to retrive a specific layer id of a geopackage for a selection of countries
@@ -109,19 +153,7 @@ def get_GADM_layer(country_list, layer_id, geo_crs, update=False, outlogging=Fal
             # when layer id is negative or larger than the number of layers, select the last layer
             layer_id = len(list_layers) - 1
 
-        # read gpkg file
-        geodf_temp = gpd.read_file(file_gpkg, layer="ADM_ADM_" + str(layer_id)).to_crs(
-            geo_crs
-        )
-
-        # convert country name representation of the main country (GID_0 column)
-        geodf_temp["GID_0"] = [
-            three_2_two_digits_country(twoD_c) for twoD_c in geodf_temp["GID_0"]
-        ]
-
-        # create a subindex column that is useful
-        # in the GADM processing of sub-national zones
-        geodf_temp["GADM_ID"] = geodf_temp[f"GID_{layer_id}"]
+        geodf_temp = build_gadm_df(file=file_gpkg, layer=layer_id, cc=country_code)
 
         # append geodataframes
         geodf_list.append(geodf_temp)
@@ -267,7 +299,9 @@ def eez(countries, geo_crs, country_shapes, EEZ_gpkg, out_logging=False, distanc
     )
 
     ret_df = ret_df.apply(lambda x: make_valid(x))
-    country_shapes = country_shapes.apply(lambda x: make_valid(x))
+
+    # country_shapes may consist of different geometries which need to be united
+    country_shapes = country_shapes.apply(lambda x: make_valid(x)).unary_union
 
     country_shapes_with_buffer = country_shapes.buffer(distance)
     ret_df_new = ret_df.difference(country_shapes_with_buffer)
@@ -783,8 +817,19 @@ def gadm(
     # set index and simplify polygons
     df_gadm.set_index("GADM_ID", inplace=True)
     df_gadm["geometry"] = df_gadm["geometry"].map(_simplify_polys)
+    # df_gadm = df_gadm[df_gadm.geometry.is_valid & ~df_gadm.geometry.is_empty]
+    df_gadm.geometry = df_gadm.geometry.apply(
+        lambda r: make_valid(r) if not r.is_valid else r
+    )
     df_gadm = df_gadm[df_gadm.geometry.is_valid & ~df_gadm.geometry.is_empty]
 
+    # gadm_shapes.geometry = gadm_shapes.apply(
+    #     lambda row: make_valid(row.geometry)
+    #     if not row.geometry.is_valid
+    #     else row.geometry,
+    #     axis=1,
+    # )
+
     return df_gadm
 
 
@@ -813,7 +858,13 @@ def gadm(
 
     country_shapes = countries(countries_list, geo_crs, update, out_logging)
 
-    country_shapes.reset_index().to_file(snakemake.output.country_shapes)
+    # there may be "holes" in the countries geometry which cause troubles along the workflow
+    # e.g. that is the case for enclaves like Dahagram–Angarpota for IN/BD
+    country_shapes_valid = (
+        country_shapes.apply(lambda x: make_valid(x) if not x.is_valid else x)
+        .reset_index()
+        .to_file(snakemake.output.country_shapes)
+    )
 
     offshore_shapes = eez(
         countries_list, geo_crs, country_shapes, EEZ_gpkg, out_logging
@@ -837,4 +888,10 @@ def gadm(
         year,
         nprocesses=nprocesses,
     )
+    # gadm_shapes.geometry = gadm_shapes.apply(
+    #     lambda row: make_valid(row.geometry)
+    #     if not row.geometry.is_valid
+    #     else row.geometry,
+    #     axis=1,
+    # )
     save_to_geojson(gadm_shapes, out.gadm_shapes)