diff --git a/CHANGELOG.md b/CHANGELOG.md index 0837168..8cf6359 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # CHANGELOG +## 1.2.0 (2024-03-08) + +### Feat + +- refactor out geopandas entirely, use shapely + ## 1.1.2 (2024-02-15) ### Fix diff --git a/Makefile b/Makefile index 5ff6433..2602572 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ PACKAGE := org.osm_fieldwork.py NAME := fmtm-splitter -VERSION := 1.1.2 +VERSION := 1.2.0 # All python source files # MDS := $(wildcard ./docs/*.md) diff --git a/fmtm_splitter/__version__.py b/fmtm_splitter/__version__.py index 72f26f5..c68196d 100644 --- a/fmtm_splitter/__version__.py +++ b/fmtm_splitter/__version__.py @@ -1 +1 @@ -__version__ = "1.1.2" +__version__ = "1.2.0" diff --git a/fmtm_splitter/db.py b/fmtm_splitter/db.py index d8ba22c..c96e686 100644 --- a/fmtm_splitter/db.py +++ b/fmtm_splitter/db.py @@ -16,13 +16,12 @@ # """DB models for temporary tables in splitBySQL.""" import logging -import warnings from typing import Union -import geopandas as gpd import psycopg2 from psycopg2.extensions import register_adapter -from psycopg2.extras import Json, execute_values, register_uuid +from psycopg2.extras import Json, register_uuid +from shapely.geometry import Polygon try: import sqlalchemy @@ -89,13 +88,11 @@ def create_tables(conn: psycopg2.extensions.connection): create_cmd = """ CREATE TABLE project_aoi ( id UUID DEFAULT gen_random_uuid() PRIMARY KEY, - geom GEOMETRY(GEOMETRY, 4326), - tags JSONB + geom GEOMETRY(GEOMETRY, 4326) ); CREATE TABLE ways_poly ( id SERIAL PRIMARY KEY, - project_id VARCHAR, osm_id VARCHAR, geom GEOMETRY(GEOMETRY, 4326), tags JSONB @@ -103,7 +100,6 @@ def create_tables(conn: psycopg2.extensions.connection): CREATE TABLE ways_line ( id SERIAL PRIMARY KEY, - project_id VARCHAR, osm_id VARCHAR, geom GEOMETRY(GEOMETRY, 4326), tags JSONB @@ -137,42 +133,28 @@ def drop_tables(conn: psycopg2.extensions.connection): cur.execute(drop_cmd) -def gdf_to_postgis(gdf: gpd.GeoDataFrame, conn: psycopg2.extensions.connection, table_name: str, geom_name: str = "geom") -> None: +def aoi_to_postgis(conn: psycopg2.extensions.connection, geom: Polygon) -> None: """Export a GeoDataFrame to the project_aoi table in PostGIS. - Built-in geopandas to_wkb uses shapely underneath. - Uses a new cursor on existing connection, but not committed directly. Args: - gdf (gpd.GeoDataFrame): The GeoDataFrame to export. + geom (Polygon): The shapely geom to insert. conn (psycopg2.extensions.connection): The PostgreSQL connection. - table_name (str): The name of the table to insert data into. - geom_name (str, optional): The name of the geometry column. Defaults to "geom". Returns: None """ - # Only use dataframe copy, else the geom is transformed to WKBElement - gdf = gdf.copy() + log.debug("Adding AOI to project_aoi table") - # Rename existing geometry column if it doesn't match - if geom_name not in gdf.columns: - gdf = gdf.rename(columns={gdf.geometry.name: geom_name}).set_geometry(geom_name, crs=gdf.crs) - - log.debug("Converting geodataframe geom to wkb hex string") - # Ignore warning 'Geometry column does not contain geometry' - warnings.filterwarnings("ignore", category=UserWarning, module="fmtm_splitter.db") - gdf[geom_name] = gdf[geom_name].to_wkb(hex=True, include_srid=True) - warnings.filterwarnings("default", category=UserWarning, module="fmtm_splitter.db") - - # Build numpy array for db insert - tuples = [tuple(x) for x in gdf.to_numpy()] - cols = ",".join(list(gdf.columns)) - query = "INSERT INTO %s(%s) VALUES %%s" % (table_name, cols) + sql = """ + INSERT INTO project_aoi (geom) + VALUES (ST_SetSRID(CAST(%s AS GEOMETRY), 4326)); + """ cur = conn.cursor() - execute_values(cur, query, tuples) + cur.execute(sql, (geom.wkb_hex,)) + cur.close() def insert_geom(cur: psycopg2.extensions.cursor, table_name: str, **kwargs) -> None: @@ -188,5 +170,5 @@ def insert_geom(cur: psycopg2.extensions.cursor, table_name: str, **kwargs) -> N Returns: None """ - query = f"INSERT INTO {table_name}(project_id,geom,osm_id,tags) " "VALUES (%(project_id)s,%(geom)s,%(osm_id)s,%(tags)s)" + query = f"INSERT INTO {table_name}(geom,osm_id,tags) " "VALUES (%(geom)s,%(osm_id)s,%(tags)s)" cur.execute(query, kwargs) diff --git a/fmtm_splitter/splitter.py b/fmtm_splitter/splitter.py index 2b691bb..18ede23 100755 --- a/fmtm_splitter/splitter.py +++ b/fmtm_splitter/splitter.py @@ -25,21 +25,16 @@ from pathlib import Path from textwrap import dedent from typing import Optional, Union -from uuid import uuid4 import geojson - -# TODO refactor out geopandas -import geopandas as gpd import numpy as np from geojson import Feature, FeatureCollection, GeoJSON +from osm_rawdata.postgres import PostgresClient from psycopg2.extensions import connection -from shapely import to_geojson -from shapely.geometry import Polygon, shape -from shapely.ops import polygonize, unary_union +from shapely.geometry import Polygon, box, shape +from shapely.ops import unary_union -from fmtm_splitter.db import close_connection, create_connection, create_tables, drop_tables, gdf_to_postgis, insert_geom -from osm_rawdata.postgres import PostgresClient +from fmtm_splitter.db import aoi_to_postgis, close_connection, create_connection, create_tables, drop_tables, insert_geom # Instantiate logger log = logging.getLogger(__name__) @@ -63,12 +58,8 @@ def __init__( """ # Parse AOI, merge if multiple geometries if aoi_obj: - geojson = self.input_to_geojson(aoi_obj, merge=True) - self.aoi = self.geojson_to_gdf(geojson) - - # Rename fields to match schema & set id field - self.id = uuid4() - self.aoi["id"] = str(self.id) + geojson = self.input_to_geojson(aoi_obj) + self.aoi = self.geojson_to_shapely_polygon(geojson) # Init split features self.split_features = None @@ -98,21 +89,6 @@ def input_to_geojson(input_data: Union[str, FeatureCollection, dict], merge: boo log.error(err) raise ValueError(err) - if merge: - # If multiple geoms, combine / enclose via convex hull - - features = FMTMSplitter.geojson_to_featcol(parsed_geojson).get("features") - - # If only single geometry present, return - if len(features) == 1: - return parsed_geojson - - # Convert each feature into a Shapely geometry - # Extract from Feature type if necessary - geometries = [shape(feature.get("geometry", feature)) for feature in features] - merged_geom = unary_union(geometries) - return geojson.loads(to_geojson(merged_geom.convex_hull)) - return parsed_geojson @staticmethod @@ -121,9 +97,9 @@ def geojson_to_featcol(geojson: Union[FeatureCollection, Feature, dict]) -> Feat # Parse and unparse geojson to extract type if isinstance(geojson, FeatureCollection): # Handle FeatureCollection nesting - features = geojson.get("features") + features = geojson.get("features", []) elif isinstance(geojson, Feature): - # GeoPandas requests list of features + # Must be a list features = [geojson] else: # A standard geometry type. Has coordinates, no properties @@ -131,32 +107,25 @@ def geojson_to_featcol(geojson: Union[FeatureCollection, Feature, dict]) -> Feat return FeatureCollection(features) @staticmethod - def geojson_to_gdf(geojson: Union[FeatureCollection, Feature, dict]) -> gpd.GeoDataFrame: - """Parse GeoJSON and return GeoDataFrame. + def geojson_to_shapely_polygon(geojson: Union[FeatureCollection, Feature, dict]) -> Polygon: + """Parse GeoJSON and return shapely Polygon. - The GeoJSON may be of type FeatureCollection, Feature, or Geometry. + The GeoJSON may be of type FeatureCollection, Feature, or Polygon, + but should only contain one Polygon geometry in total. """ - features = FMTMSplitter.geojson_to_featcol(geojson).get("features") - log.debug(f"Parsed {len(features)} features") - log.debug("Converting to geodataframe") - data = gpd.GeoDataFrame(features, crs="EPSG:4326") - return FMTMSplitter.tidy_columns(data) + features = FMTMSplitter.geojson_to_featcol(geojson).get("features", []) + log.debug("Converting AOI to Shapely geometry") - @staticmethod - def tidy_columns(dataframe: gpd.GeoDataFrame) -> gpd.GeoDataFrame: - """Fix dataframe columns prior to geojson export or db insert. + if len(features) == 0: + msg = "The input AOI contains no geometries." + log.error(msg) + raise ValueError(msg) + elif len(features) > 1: + msg = "The input AOI cannot contain multiple geometries." + log.error(msg) + raise ValueError(msg) - Strips timestamps that are not json serializable. - Renames geometry column --> geom. - Removes 'type' field for insert into db. - """ - log.debug("Tidying up columns, renaming geometry to geom") - dataframe.rename(columns={"geometry": "geom", "properties": "tags"}, inplace=True) - dataframe.set_geometry("geom", inplace=True) - dataframe.drop(columns=["type"], inplace=True, errors="ignore") - # Drop any timestamps to prevent json parsing issues later - dataframe.drop(columns=["timestamp"], inplace=True, errors="ignore") - return dataframe + return shape(features[0].get("geometry")) def splitBySquare( # noqa: N802 self, @@ -172,32 +141,30 @@ def splitBySquare( # noqa: N802 """ log.debug("Splitting the AOI by squares") - xmin, ymin, xmax, ymax = self.aoi.total_bounds + xmin, ymin, xmax, ymax = self.aoi.bounds # 1 meters is this factor in degrees meter = 0.0000114 length = float(meters) * meter - wide = float(meters) * meter + width = float(meters) * meter - cols = list(np.arange(xmin, xmax + wide, wide)) + cols = list(np.arange(xmin, xmax + width, width)) rows = list(np.arange(ymin, ymax + length, length)) polygons = [] for x in cols[:-1]: for y in rows[:-1]: - polygons.append(Polygon([(x, y), (x + wide, y), (x + wide, y + length), (x, y + length)])) - grid = gpd.GeoDataFrame({"geometry": polygons}, crs="EPSG:4326") + polygons.append(box(x, y, x + width, y + length)) - clipped = gpd.clip(grid, self.aoi) - self.split_features = geojson.loads(clipped.to_json()) + self.split_features = FeatureCollection([Feature(geometry=poly) for poly in polygons]) return self.split_features def splitBySQL( # noqa: N802 self, sql: str, db: Union[str, connection], - buildings: int, - osm_extract: Union[dict, FeatureCollection] = None, + buildings: Optional[int] = None, + osm_extract: Optional[Union[dict, FeatureCollection]] = None, ) -> FeatureCollection: """Split the polygon by features in the database using an SQL query. @@ -226,6 +193,22 @@ def splitBySQL( # noqa: N802 log.error(msg) raise ValueError(msg) + # Run custom SQL + if not buildings or not osm_extract: + log.info("No `buildings` or `osm_extract` params passed, executing custom SQL") + # FIXME untested + conn = create_connection(db) + splitter_cursor = conn.cursor() + log.debug("Running custom splitting algorithm") + splitter_cursor.execute(sql) + features = splitter_cursor.fetchall()[0][0]["features"] + if features: + log.info(f"Query returned {len(features)} features") + else: + log.info("Query returned no features") + self.split_features = FeatureCollection(features) + return self.split_features + # Get existing db engine, or create new one conn = create_connection(db) @@ -234,9 +217,7 @@ def splitBySQL( # noqa: N802 create_tables(conn) # Add aoi to project_aoi table - log.debug(f"Adding AOI to project_aoi table: {self.aoi.to_dict()}") - self.aoi["tags"] = self.aoi["tags"].apply(json.dumps) - gdf_to_postgis(self.aoi, conn, "project_aoi", "geom") + aoi_to_postgis(conn, self.aoi) def json_str_to_dict(json_item: Union[str, dict]) -> dict: """Convert a JSON string to dict.""" @@ -270,7 +251,7 @@ def json_str_to_dict(json_item: Union[str, dict]) -> dict: osm_id = properties.get("osm_id") # Common attributes for db tables - common_args = dict(project_id=self.id, osm_id=osm_id, geom=wkb_element, tags=tags) + common_args = dict(osm_id=osm_id, geom=wkb_element, tags=tags) # Insert building polygons if tags.get("building") == "yes": @@ -281,26 +262,21 @@ def json_str_to_dict(json_item: Union[str, dict]) -> dict: insert_geom(cur, "ways_line", **common_args) # Use raw sql for view generation & remainder of script + # TODO get geom from project_aoi table instead of wkb string log.debug("Creating db view with intersecting polylines") - # Get aoi as geojson - aoi_geom = geojson.loads(self.aoi.to_json()).get("features", [{}])[0].get("geometry", {}) view = ( "DROP VIEW IF EXISTS lines_view;" "CREATE VIEW lines_view AS SELECT " "tags,geom FROM ways_line WHERE " - "ST_Intersects(ST_GeomFromGeoJson(%(geojson_str)s), geom)" + "ST_Intersects(ST_SetSRID(CAST(%s AS GEOMETRY), 4326), geom)" ) - cur.execute(view, {"geojson_str": aoi_geom}) + cur.execute(view, (self.aoi.wkb_hex,)) # Close current cursor cur.close() splitter_cursor = conn.cursor() - # Only insert buildings param is specified log.debug("Running task splitting algorithm") - if buildings: - splitter_cursor.execute(sql, {"num_buildings": buildings}) - else: - splitter_cursor.execute(sql) + splitter_cursor.execute(sql, {"num_buildings": buildings}) features = splitter_cursor.fetchall()[0][0]["features"] if features: @@ -323,25 +299,41 @@ def splitByFeature( # noqa: N802 """Split the polygon by features in the database. Args: - features(gpd.GeoSeries): GeoDataFrame of feautures to split by. + features(FeatureCollection): FeatureCollection of features + to polygonise and return. Returns: data (FeatureCollection): A multipolygon of all the task boundaries. """ - # gdf[(gdf['highway'] != 'turning_circle') | (gdf['highway'] != 'milestone')] - # gdf[(gdf.geom_type != 'Point')] - # gdf[['highway']] - log.debug("Splitting the AOI using a data extract") - gdf = gpd.GeoDataFrame.from_features(features, crs="EPSG:4326") - polygons = gpd.GeoSeries(polygonize(gdf.geometry)) - - self.split_features = geojson.loads(polygons.to_json()) + log.debug("Polygonising the FeatureCollection features") + # Extract all geometries from the input features + geometries = [] + for feature in features["features"]: + geom = feature["geometry"] + if geom["type"] == "Polygon": + geometries.append(shape(geom)) + elif geom["type"] == "LineString": + geometries.append(shape(geom)) + else: + log.warning(f"Ignoring unsupported geometry type: {geom['type']}") + + # Create a single MultiPolygon from all the polygons and linestrings + multi_polygon = unary_union(geometries) + + # Clip the multi_polygon by the AOI boundary + clipped_multi_polygon = multi_polygon.intersection(self.aoi) + + polygon_features = [Feature(geometry=polygon) for polygon in list(clipped_multi_polygon.geoms)] + + # Convert the Polygon Features into a FeatureCollection + self.split_features = FeatureCollection(features=polygon_features) + return self.split_features def outputGeojson( # noqa: N802 self, filename: str = "output.geojson", - ) -> FeatureCollection: + ) -> None: """Output a geojson file from split features.""" if not self.split_features: msg = "Feature splitting has not been executed. Do this first." @@ -356,7 +348,7 @@ def outputGeojson( # noqa: N802 def split_by_square( aoi: Union[str, FeatureCollection], meters: int = 100, - outfile: str = None, + outfile: Optional[str] = None, ) -> FeatureCollection: """Split an AOI by square, dividing into an even grid. @@ -370,25 +362,43 @@ def split_by_square( Returns: features (FeatureCollection): A multipolygon of all the task boundaries. """ - splitter = FMTMSplitter(aoi) - features = splitter.splitBySquare(meters) - if not features: - msg = "Failed to generate split features." - log.error(msg) - raise ValueError(msg) + # Parse AOI + parsed_aoi = FMTMSplitter.input_to_geojson(aoi) + aoi_featcol = FMTMSplitter.geojson_to_featcol(parsed_aoi) - if outfile: - splitter.outputGeojson(outfile) + # Handle multiple geometries passed + if len(feat_array := aoi_featcol.get("features", [])) > 1: + features = [] + for index, feat in enumerate(feat_array): + featcol = split_by_square( + FeatureCollection(features=[feat]), + meters, + f"{Path(outfile).stem}_{index}.geojson)" if outfile else None, + ) + feats = featcol.get("features", []) + if feats: + features += feats + # Parse FeatCols into single FeatCol + split_features = FeatureCollection(features) + else: + splitter = FMTMSplitter(aoi_featcol) + split_features = splitter.splitBySquare(meters) + if not split_features: + msg = "Failed to generate split features." + log.error(msg) + raise ValueError(msg) + if outfile: + splitter.outputGeojson(outfile) - return features + return split_features def split_by_sql( aoi: Union[str, FeatureCollection], db: Union[str, connection], - sql_file: str = None, - num_buildings: int = None, - outfile: str = None, + sql_file: Optional[Union[str, Path]] = None, + num_buildings: Optional[int] = None, + outfile: Optional[str] = None, osm_extract: Optional[Union[str, FeatureCollection]] = None, ) -> FeatureCollection: """Split an AOI with a custom SQL query or default FMTM query. @@ -431,7 +441,7 @@ def split_by_sql( raise ValueError(err) # Use FMTM splitter of num_buildings set, else use custom SQL - if num_buildings: + if not sql_file: sql_file = Path(__file__).parent / "fmtm_algorithm.sql" with open(sql_file, "r") as sql: @@ -439,13 +449,10 @@ def split_by_sql( # Parse AOI parsed_aoi = FMTMSplitter.input_to_geojson(aoi) + aoi_featcol = FMTMSplitter.geojson_to_featcol(parsed_aoi) # Extracts and parse extract geojson - extract_geojson = None if not osm_extract: - # For now we merge all geoms via convex hull - merged_aoi = FMTMSplitter.input_to_geojson(parsed_aoi) - # We want all polylines for splitting: # buildings, highways, waterways, railways config_data = dedent( @@ -471,54 +478,55 @@ def split_by_sql( "underpass", config_bytes, ) + # The total FeatureCollection area merged by osm-rawdata automatically extract_geojson = pg.execQuery( - merged_aoi, + aoi_featcol, extra_params={"fileName": "fmtm_splitter", "useStWithin": False}, ) - elif osm_extract: + else: extract_geojson = FMTMSplitter.input_to_geojson(osm_extract) + if not extract_geojson: err = "A valid data extract must be provided." log.error(err) raise ValueError(err) # Handle multiple geometries passed - if isinstance(parsed_aoi, FeatureCollection): - # FIXME why does only one geom split during test? - # FIXME other geoms return None during splitting - if len(feat_array := parsed_aoi.get("features", [])) > 1: - split_geoms = [] - for feat in feat_array: - splitter = FMTMSplitter(feat) - featcol = splitter.splitBySQL(query, db, num_buildings, osm_extract=extract_geojson) - features = featcol.get("features", []) - if features: - split_geoms += features - if outfile: - with open(outfile, "w") as jsonfile: - geojson.dump(split_geoms, jsonfile) - log.debug(f"Wrote split features to {outfile}") - # Parse FeatCols into single FeatCol - return FeatureCollection(split_geoms) - - splitter = FMTMSplitter(parsed_aoi) - split_geoms = splitter.splitBySQL(query, db, num_buildings, osm_extract=extract_geojson) - if not split_geoms: - msg = "Failed to generate split features." - log.error(msg) - raise ValueError(msg) - if outfile: - splitter.outputGeojson(outfile) + if len(feat_array := aoi_featcol.get("features", [])) > 1: + features = [] + for index, feat in enumerate(feat_array): + featcol = split_by_sql( + FeatureCollection(features=[feat]), + db, + sql_file, + num_buildings, + f"{Path(outfile).stem}_{index}.geojson)" if outfile else None, + osm_extract, + ) + feats = featcol.get("features", []) + if feats: + features += feats + # Parse FeatCols into single FeatCol + split_features = FeatureCollection(features) + else: + splitter = FMTMSplitter(aoi_featcol) + split_features = splitter.splitBySQL(query, db, num_buildings, osm_extract=extract_geojson) + if not split_features: + msg = "Failed to generate split features." + log.error(msg) + raise ValueError(msg) + if outfile: + splitter.outputGeojson(outfile) - return split_geoms + return split_features def split_by_features( aoi: Union[str, FeatureCollection], - db_table: str = None, + db_table: Optional[str] = None, geojson_input: Optional[Union[str, FeatureCollection]] = None, - outfile: str = None, + outfile: Optional[str] = None, ) -> FeatureCollection: """Split an AOI by geojson features or database features. @@ -544,20 +552,24 @@ def split_by_features( log.error(err) raise ValueError(err) - splitter = FMTMSplitter(aoi) - input_features = None + # Parse AOI + parsed_aoi = FMTMSplitter.input_to_geojson(aoi) + aoi_featcol = FMTMSplitter.geojson_to_featcol(parsed_aoi) # Features from database if db_table: # data = f"PG:{db_table}" # TODO get input_features from db + # input_features = + # featcol = FMTMSplitter.geojson_to_featcol(input_features) raise NotImplementedError("Splitting from db featurs it not implemented yet.") # Features from geojson if geojson_input: - input_features = FMTMSplitter.input_to_geojson(geojson_input) + input_parsed = FMTMSplitter.input_to_geojson(geojson_input) + input_featcol = FMTMSplitter.geojson_to_featcol(input_parsed) - if not isinstance(input_features, FeatureCollection): + if not isinstance(input_featcol, FeatureCollection): msg = ( f"Could not parse geojson data from {geojson_input}" if geojson_input @@ -566,16 +578,32 @@ def split_by_features( log.error(msg) raise ValueError(msg) - features = splitter.splitByFeature(input_features) - if not features: - msg = "Failed to generate split features." - log.error(msg) - raise ValueError(msg) - - if outfile: - splitter.outputGeojson(outfile) + # Handle multiple geometries passed + if len(feat_array := aoi_featcol.get("features", [])) > 1: + features = [] + for index, feat in enumerate(feat_array): + featcol = split_by_features( + FeatureCollection(features=[feat]), + db_table, + input_featcol, + f"{Path(outfile).stem}_{index}.geojson)" if outfile else None, + ) + feats = featcol.get("features", []) + if feats: + features += feats + # Parse FeatCols into single FeatCol + split_features = FeatureCollection(features) + else: + splitter = FMTMSplitter(aoi_featcol) + split_features = splitter.splitByFeature(input_featcol) + if not split_features: + msg = "Failed to generate split features." + log.error(msg) + raise ValueError(msg) + if outfile: + splitter.outputGeojson(outfile) - return features + return split_features def main(args_list: list[str] | None = None): diff --git a/pdm.lock b/pdm.lock index e945ec4..2a14b1a 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "docs", "test", "debug", "dev"] strategy = ["cross_platform"] lock_version = "4.4.1" -content_hash = "sha256:00975cfda055401eaf1559aa1b3442a595d3f29098cb9700755cc0cb7e532725" +content_hash = "sha256:75681344b3b0dab52574d55746732b280da07b915892bd39daad8ac40a583cd1" [[package]] name = "argcomplete" @@ -29,16 +29,6 @@ files = [ {file = "asttokens-2.4.1.tar.gz", hash = "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"}, ] -[[package]] -name = "attrs" -version = "23.2.0" -requires_python = ">=3.7" -summary = "Classes Without Boilerplate" -files = [ - {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, - {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, -] - [[package]] name = "babel" version = "2.14.0" @@ -168,31 +158,6 @@ files = [ {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] -[[package]] -name = "click-plugins" -version = "1.1.1" -summary = "An extension module for click to enable registering CLI commands via setuptools entry-points." -dependencies = [ - "click>=4.0", -] -files = [ - {file = "click-plugins-1.1.1.tar.gz", hash = "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b"}, - {file = "click_plugins-1.1.1-py2.py3-none-any.whl", hash = "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8"}, -] - -[[package]] -name = "cligj" -version = "0.7.2" -requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, <4" -summary = "Click params for commmand line interfaces to GeoJSON" -dependencies = [ - "click>=4.0", -] -files = [ - {file = "cligj-0.7.2-py3-none-any.whl", hash = "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df"}, - {file = "cligj-0.7.2.tar.gz", hash = "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27"}, -] - [[package]] name = "codetiming" version = "1.4.0" @@ -325,36 +290,6 @@ files = [ {file = "filelock-3.13.1.tar.gz", hash = "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e"}, ] -[[package]] -name = "fiona" -version = "1.9.5" -requires_python = ">=3.7" -summary = "Fiona reads and writes spatial data files" -dependencies = [ - "attrs>=19.2.0", - "certifi", - "click-plugins>=1.0", - "click~=8.0", - "cligj>=0.5", - "setuptools", - "six", -] -files = [ - {file = "fiona-1.9.5-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:5f40a40529ecfca5294260316cf987a0420c77a2f0cf0849f529d1afbccd093e"}, - {file = "fiona-1.9.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:374efe749143ecb5cfdd79b585d83917d2bf8ecfbfc6953c819586b336ce9c63"}, - {file = "fiona-1.9.5-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:35dae4b0308eb44617cdc4461ceb91f891d944fdebbcba5479efe524ec5db8de"}, - {file = "fiona-1.9.5-cp310-cp310-win_amd64.whl", hash = "sha256:5b4c6a3df53bee8f85bb46685562b21b43346be1fe96419f18f70fa1ab8c561c"}, - {file = "fiona-1.9.5-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:6ad04c1877b9fd742871b11965606c6a52f40706f56a48d66a87cc3073943828"}, - {file = "fiona-1.9.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9fb9a24a8046c724787719e20557141b33049466145fc3e665764ac7caf5748c"}, - {file = "fiona-1.9.5-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:d722d7f01a66f4ab6cd08d156df3fdb92f0669cf5f8708ddcb209352f416f241"}, - {file = "fiona-1.9.5-cp311-cp311-win_amd64.whl", hash = "sha256:7ede8ddc798f3d447536080c6db9a5fb73733ad8bdb190cb65eed4e289dd4c50"}, - {file = "fiona-1.9.5-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:8b098054a27c12afac4f819f98cb4d4bf2db9853f70b0c588d7d97d26e128c39"}, - {file = "fiona-1.9.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d9f29e9bcbb33232ff7fa98b4a3c2234db910c1dc6c4147fc36c0b8b930f2e0"}, - {file = "fiona-1.9.5-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:f1af08da4ecea5036cb81c9131946be4404245d1b434b5b24fd3871a1d4030d9"}, - {file = "fiona-1.9.5-cp312-cp312-win_amd64.whl", hash = "sha256:c521e1135c78dec0d7774303e5a1b4c62e0efb0e602bb8f167550ef95e0a2691"}, - {file = "fiona-1.9.5.tar.gz", hash = "sha256:99e2604332caa7692855c2ae6ed91e1fffdf9b59449aa8032dd18e070e59a2f7"}, -] - [[package]] name = "flatdict" version = "4.0.1" @@ -387,23 +322,6 @@ files = [ {file = "geojson-3.1.0.tar.gz", hash = "sha256:58a7fa40727ea058efc28b0e9ff0099eadf6d0965e04690830208d3ef571adac"}, ] -[[package]] -name = "geopandas" -version = "0.14.2" -requires_python = ">=3.9" -summary = "Geographic pandas extensions" -dependencies = [ - "fiona>=1.8.21", - "packaging", - "pandas>=1.4.0", - "pyproj>=3.3.0", - "shapely>=1.8.0", -] -files = [ - {file = "geopandas-0.14.2-py3-none-any.whl", hash = "sha256:0efa61235a68862c1c6be89fc3707cdeba67667d5676bb19e24f3c57a8c2f723"}, - {file = "geopandas-0.14.2.tar.gz", hash = "sha256:6e71d57b8376f9fdc9f1c3aa3170e7e420e91778de854f51013ae66fd371ccdb"}, -] - [[package]] name = "ghp-import" version = "2.1.0" @@ -861,44 +779,6 @@ files = [ {file = "paginate-0.5.6.tar.gz", hash = "sha256:5e6007b6a9398177a7e1648d04fdd9f8c9766a1a945bceac82f1929e8c78af2d"}, ] -[[package]] -name = "pandas" -version = "2.2.0" -requires_python = ">=3.9" -summary = "Powerful data structures for data analysis, time series, and statistics" -dependencies = [ - "numpy<2,>=1.22.4; python_version < \"3.11\"", - "numpy<2,>=1.23.2; python_version == \"3.11\"", - "numpy<2,>=1.26.0; python_version >= \"3.12\"", - "python-dateutil>=2.8.2", - "pytz>=2020.1", - "tzdata>=2022.7", -] -files = [ - {file = "pandas-2.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8108ee1712bb4fa2c16981fba7e68b3f6ea330277f5ca34fa8d557e986a11670"}, - {file = "pandas-2.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:736da9ad4033aeab51d067fc3bd69a0ba36f5a60f66a527b3d72e2030e63280a"}, - {file = "pandas-2.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38e0b4fc3ddceb56ec8a287313bc22abe17ab0eb184069f08fc6a9352a769b18"}, - {file = "pandas-2.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20404d2adefe92aed3b38da41d0847a143a09be982a31b85bc7dd565bdba0f4e"}, - {file = "pandas-2.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7ea3ee3f125032bfcade3a4cf85131ed064b4f8dd23e5ce6fa16473e48ebcaf5"}, - {file = "pandas-2.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9670b3ac00a387620489dfc1bca66db47a787f4e55911f1293063a78b108df1"}, - {file = "pandas-2.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:5a946f210383c7e6d16312d30b238fd508d80d927014f3b33fb5b15c2f895430"}, - {file = "pandas-2.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a1b438fa26b208005c997e78672f1aa8138f67002e833312e6230f3e57fa87d5"}, - {file = "pandas-2.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8ce2fbc8d9bf303ce54a476116165220a1fedf15985b09656b4b4275300e920b"}, - {file = "pandas-2.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2707514a7bec41a4ab81f2ccce8b382961a29fbe9492eab1305bb075b2b1ff4f"}, - {file = "pandas-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85793cbdc2d5bc32620dc8ffa715423f0c680dacacf55056ba13454a5be5de88"}, - {file = "pandas-2.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:cfd6c2491dc821b10c716ad6776e7ab311f7df5d16038d0b7458bc0b67dc10f3"}, - {file = "pandas-2.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a146b9dcacc3123aa2b399df1a284de5f46287a4ab4fbfc237eac98a92ebcb71"}, - {file = "pandas-2.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbc1b53c0e1fdf16388c33c3cca160f798d38aea2978004dd3f4d3dec56454c9"}, - {file = "pandas-2.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a41d06f308a024981dcaa6c41f2f2be46a6b186b902c94c2674e8cb5c42985bc"}, - {file = "pandas-2.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:159205c99d7a5ce89ecfc37cb08ed179de7783737cea403b295b5eda8e9c56d1"}, - {file = "pandas-2.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb1e1f3861ea9132b32f2133788f3b14911b68102d562715d71bd0013bc45440"}, - {file = "pandas-2.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:761cb99b42a69005dec2b08854fb1d4888fdf7b05db23a8c5a099e4b886a2106"}, - {file = "pandas-2.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a20628faaf444da122b2a64b1e5360cde100ee6283ae8effa0d8745153809a2e"}, - {file = "pandas-2.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f5be5d03ea2073627e7111f61b9f1f0d9625dc3c4d8dda72cc827b0c58a1d042"}, - {file = "pandas-2.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:a626795722d893ed6aacb64d2401d017ddc8a2341b49e0384ab9bf7112bdec30"}, - {file = "pandas-2.2.0.tar.gz", hash = "sha256:30b83f7c3eb217fb4d1b494a57a2fda5444f17834f5df2de6b2ffff68dc3c8e2"}, -] - [[package]] name = "parso" version = "0.8.3" @@ -1071,38 +951,6 @@ files = [ {file = "pymdown_extensions-10.7.tar.gz", hash = "sha256:c0d64d5cf62566f59e6b2b690a4095c931107c250a8c8e1351c1de5f6b036deb"}, ] -[[package]] -name = "pyproj" -version = "3.6.1" -requires_python = ">=3.9" -summary = "Python interface to PROJ (cartographic projections and coordinate transformations library)" -dependencies = [ - "certifi", -] -files = [ - {file = "pyproj-3.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ab7aa4d9ff3c3acf60d4b285ccec134167a948df02347585fdd934ebad8811b4"}, - {file = "pyproj-3.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4bc0472302919e59114aa140fd7213c2370d848a7249d09704f10f5b062031fe"}, - {file = "pyproj-3.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5279586013b8d6582e22b6f9e30c49796966770389a9d5b85e25a4223286cd3f"}, - {file = "pyproj-3.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80fafd1f3eb421694857f254a9bdbacd1eb22fc6c24ca74b136679f376f97d35"}, - {file = "pyproj-3.6.1-cp310-cp310-win32.whl", hash = "sha256:c41e80ddee130450dcb8829af7118f1ab69eaf8169c4bf0ee8d52b72f098dc2f"}, - {file = "pyproj-3.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:db3aedd458e7f7f21d8176f0a1d924f1ae06d725228302b872885a1c34f3119e"}, - {file = "pyproj-3.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ebfbdbd0936e178091309f6cd4fcb4decd9eab12aa513cdd9add89efa3ec2882"}, - {file = "pyproj-3.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:447db19c7efad70ff161e5e46a54ab9cc2399acebb656b6ccf63e4bc4a04b97a"}, - {file = "pyproj-3.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7e13c40183884ec7f94eb8e0f622f08f1d5716150b8d7a134de48c6110fee85"}, - {file = "pyproj-3.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65ad699e0c830e2b8565afe42bd58cc972b47d829b2e0e48ad9638386d994915"}, - {file = "pyproj-3.6.1-cp311-cp311-win32.whl", hash = "sha256:8b8acc31fb8702c54625f4d5a2a6543557bec3c28a0ef638778b7ab1d1772132"}, - {file = "pyproj-3.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:38a3361941eb72b82bd9a18f60c78b0df8408416f9340521df442cebfc4306e2"}, - {file = "pyproj-3.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1e9fbaf920f0f9b4ee62aab832be3ae3968f33f24e2e3f7fbb8c6728ef1d9746"}, - {file = "pyproj-3.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d227a865356f225591b6732430b1d1781e946893789a609bb34f59d09b8b0f8"}, - {file = "pyproj-3.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83039e5ae04e5afc974f7d25ee0870a80a6bd6b7957c3aca5613ccbe0d3e72bf"}, - {file = "pyproj-3.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb059ba3bced6f6725961ba758649261d85ed6ce670d3e3b0a26e81cf1aa8d"}, - {file = "pyproj-3.6.1-cp312-cp312-win32.whl", hash = "sha256:2d6ff73cc6dbbce3766b6c0bce70ce070193105d8de17aa2470009463682a8eb"}, - {file = "pyproj-3.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:7a27151ddad8e1439ba70c9b4b2b617b290c39395fa9ddb7411ebb0eb86d6fb0"}, - {file = "pyproj-3.6.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fd93c1a0c6c4aedc77c0fe275a9f2aba4d59b8acf88cebfc19fe3c430cfabf4f"}, - {file = "pyproj-3.6.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6420ea8e7d2a88cb148b124429fba8cd2e0fae700a2d96eab7083c0928a85110"}, - {file = "pyproj-3.6.1.tar.gz", hash = "sha256:44aa7c704c2b7d8fb3d483bbf75af6cb2350d30a63b144279a09b75fead501bf"}, -] - [[package]] name = "pytest" version = "7.4.4" @@ -1134,15 +982,6 @@ files = [ {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] -[[package]] -name = "pytz" -version = "2023.3.post1" -summary = "World timezone definitions, modern and historical" -files = [ - {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, - {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, -] - [[package]] name = "pyyaml" version = "6.0.1" @@ -1438,16 +1277,6 @@ files = [ {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, ] -[[package]] -name = "tzdata" -version = "2023.4" -requires_python = ">=2" -summary = "Provider of IANA time zone data" -files = [ - {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, - {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, -] - [[package]] name = "urllib3" version = "2.1.0" diff --git a/pyproject.toml b/pyproject.toml index 7cb0abf..6611a10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,6 @@ dependencies = [ "shapely>=1.8.1", "psycopg2>=2.9.1", "numpy>=1.21.0", - "geopandas>=0.11.0", "osm-rawdata>=0.2.2", ] requires-python = ">=3.10" @@ -44,7 +43,7 @@ pythonpath = "fmtm_splitter" [tool.commitizen] name = "cz_conventional_commits" -version = "1.1.2" +version = "1.2.0" version_files = [ "pyproject.toml:version", "fmtm_splitter/__version__.py", diff --git a/tests/conftest.py b/tests/conftest.py index f8bcbf8..50d604c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -81,9 +81,9 @@ def aoi_multi_json(): # Create Polygon for each square square_geojson = json.loads(to_geojson(box(square_minx, square_miny, square_maxx, square_maxy))) - squares.append(square_geojson) + squares.append(geojson.Feature(geometry=square_geojson)) - return geojson.FeatureCollection(squares) + return geojson.FeatureCollection(features=squares) @pytest.fixture(scope="session") diff --git a/tests/test_splitter.py b/tests/test_splitter.py index bb777fd..3a74189 100644 --- a/tests/test_splitter.py +++ b/tests/test_splitter.py @@ -24,6 +24,7 @@ from uuid import uuid4 import geojson +import pytest from fmtm_splitter.splitter import FMTMSplitter, main, split_by_features, split_by_sql, split_by_square @@ -49,8 +50,9 @@ def test_init_splitter_types(aoi_json): polygon = feature.get("geometry") FMTMSplitter(polygon) # FeatureCollection multiple geoms (4 polygons) - splitter = FMTMSplitter("tests/testdata/kathmandu_split.geojson") - assert len(splitter.aoi) == 1 + with pytest.raises(ValueError) as error: + FMTMSplitter("tests/testdata/kathmandu_split.geojson") + assert str(error.value) == "The input AOI cannot contain multiple geometries." def test_split_by_square_with_dict(aoi_json): @@ -109,11 +111,16 @@ def test_split_by_square_with_file_output(): def test_split_by_square_with_multigeom_input(aoi_multi_json): """Test divide by square from geojson dict types.""" + file_name = uuid4() + outfile = Path(__file__).parent.parent / f"{file_name}.geojson" features = split_by_square( aoi_multi_json, meters=50, + outfile=str(outfile), ) - assert len(features.get("features")) == 54 + assert len(features.get("features", [])) == 60 + for index in [0, 1, 2, 3]: + assert Path(f"{Path(outfile).stem}_{index}.geojson)").exists() def test_split_by_features_geojson(aoi_json): @@ -128,7 +135,7 @@ def test_split_by_features_geojson(aoi_json): assert len(features.get("features")) == 4 -def test_split_by_sql_fmtm(db, aoi_json, extract_json, output_json): +def test_split_by_sql_fmtm_with_extract(db, aoi_json, extract_json, output_json): """Test divide by square from geojson file.""" features = split_by_sql( aoi_json,