feat: add some QoL changes (#73)

* fix: change tqdm kwargs * feat: add progress bar for combining multiple results files * feat: add option to suppress errors if geometry isn't fully covered by OSM extracts * chore: add changelog entry * chore: modify changelog entry * fix: change docs css * chore: modify cli example * feat: add automatic CLI docs generation * fix: change docstrings for osm extracts * fix: replace the typo
kraina-ai · Apr 3, 2024 · d32aa50 · d32aa50
1 parent 88b3233
commit d32aa50
Show file tree

Hide file tree

Showing 16 changed files with 510 additions and 146 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Progress bars for final merge of multiple geoparquet files into a single file
+- Option to allow provided geometry to not be fully covered by existing OSM extracts [#68](https://github.com/kraina-ai/quackosm/issues/68)
+
+### Fixed
+
+- Changed tqdm's kwargs for parallel OSM extracts checks
+
 ## [0.5.1] - 2024-03-23
 
 ### Fixed

diff --git a/docs/api/CLI.md b/docs/api/CLI.md
@@ -0,0 +1,3 @@
+# CLI Reference
+
+Text below is a captured `--help` command output.
diff --git a/docs/assets/css/font.css b/docs/assets/css/font.css
@@ -1,5 +1,5 @@
 @import url("https://fonts.googleapis.com/css2?family=Noto+Sans+Mono&display=swap");
-@import url("https://fonts.googleapis.com/css2?family=Noto+Sans+Mono&text=┌─┬┐└┴┘│├┼┤&display=swap");
+@import url("https://fonts.googleapis.com/css2?family=Noto+Sans+Mono&text=┌─┬┐└┴┘│├┼┤╭╮╰╯&display=swap");
 @import url("https://fonts.googleapis.com/css2?family=Playpen+Sans:wght@500&display=swap");
 
 .md-header__topic:first-child {

diff --git a/docs/assets/css/jupyter.css b/docs/assets/css/jupyter.css
@@ -11,13 +11,14 @@
   right: 0;
 }
 
-.jp-CodeCell > .jp-Cell-outputWrapper {
+.jp-CodeCell>.jp-Cell-outputWrapper {
   margin-top: -10px;
   padding-top: 0;
   display: table-cell;
   text-align: left;
 }
-.jp-Cell-outputWrapper > .jp-Cell-outputCollapser {
+
+.jp-Cell-outputWrapper>.jp-Cell-outputCollapser {
   margin-top: -17px;
 }
 
@@ -26,6 +27,7 @@
 .jupyter-wrapper table.dataframe td {
   text-align: left;
 }
+
 .jupyter-wrapper table.dataframe {
   table-layout: auto;
 }
@@ -39,8 +41,8 @@
 }
 
 div.highlight pre code,
-div.jp-RenderedText.jp-OutputArea-output > pre,
-div.jp-RenderedText.jp-OutputArea-output.jp-OutputArea-executeResult > pre {
+div.jp-RenderedText.jp-OutputArea-output>pre,
+div.jp-RenderedText.jp-OutputArea-output.jp-OutputArea-executeResult>pre {
   font-family: "Noto Sans Mono", ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas,
     "Liberation Mono", "Courier New", monospace, SFMono-Regular, Consolas, Menlo,
     monospace;
@@ -57,3 +59,8 @@ div.admonition p {
   margin-top: 0.6rem;
   margin-bottom: 0.6rem;
 }
+
+.jupyter-wrapper .jp-RenderedText pre .ansi-bold {
+  font-weight: normal !important;
+  text-shadow: calc(-0.06ex) 0 0 currentColor, calc(0.06ex) 0 0 currentColor;
+}
diff --git a/docs/gen_cli_docs.py b/docs/gen_cli_docs.py
@@ -0,0 +1,75 @@
+"""Capture the CLI help page and save it to the docs."""
+
+from pathlib import Path
+from typing import cast
+
+import mkdocs_gen_files
+import typer
+from rich.console import Console
+from typer.rich_utils import (
+    COLOR_SYSTEM,
+    FORCE_TERMINAL,
+    STYLE_METAVAR,
+    STYLE_METAVAR_SEPARATOR,
+    STYLE_NEGATIVE_OPTION,
+    STYLE_NEGATIVE_SWITCH,
+    STYLE_OPTION,
+    STYLE_SWITCH,
+    STYLE_USAGE,
+    Theme,
+    highlighter,
+    rich_format_help,
+)
+
+from quackosm.cli import app
+
+API_DIRECTORY_PATH = Path("api")
+
+GLOBAL_CONSOLE = None
+
+
+def _get_rich_console_new(stderr: bool = False) -> Console:
+    global GLOBAL_CONSOLE # noqa: PLW0603
+    GLOBAL_CONSOLE = Console(
+        theme=Theme(
+            {
+                "option": STYLE_OPTION,
+                "switch": STYLE_SWITCH,
+                "negative_option": STYLE_NEGATIVE_OPTION,
+                "negative_switch": STYLE_NEGATIVE_SWITCH,
+                "metavar": STYLE_METAVAR,
+                "metavar_sep": STYLE_METAVAR_SEPARATOR,
+                "usage": STYLE_USAGE,
+            },
+        ),
+        record=True,
+        highlighter=highlighter,
+        color_system=COLOR_SYSTEM,
+        force_terminal=FORCE_TERMINAL,
+        width=240,
+        stderr=stderr,
+    )
+    return GLOBAL_CONSOLE
+
+
+typer.rich_utils._get_rich_console = _get_rich_console_new
+
+typer_obj = app
+
+click_obj = typer.main.get_command(typer_obj)
+ctx = typer.Context(command=click_obj, info_name="QuackOSM")
+rich_format_help(obj=click_obj, ctx=ctx, markup_mode="rich")
+html_text: str = cast(Console, GLOBAL_CONSOLE).export_html(
+    inline_styles=True,
+    code_format='<div class="highlight"><pre><code>{code}</code></pre></div>',
+)
+html_text = html_text.replace(
+    "font-weight: bold",
+    (
+        "font-weight: normal;"
+        " text-shadow: calc(-0.06ex) 0 0 currentColor, calc(0.06ex) 0 0 currentColor;"
+    ),
+)
+
+with mkdocs_gen_files.open(API_DIRECTORY_PATH / "CLI.md", "a") as fd:
+    print(html_text, file=fd)
diff --git a/examples/command_line_interface.ipynb b/examples/command_line_interface.ipynb
@@ -230,6 +230,13 @@
     "During first execution, QuackOSM will cache three PBF files sources locally. This operation takes some time."
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Geocoding"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -243,6 +250,13 @@
     "! QuackOSM --geom-filter-geocode 'Monaco-Ville, Monaco'"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### GeoJSON"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -256,6 +270,13 @@
     "! QuackOSM --geom-filter-geojson '{\"type\":\"Feature\",\"geometry\":{\"coordinates\":[[[7.416,43.734],[7.416,43.731],[7.421,43.731],[7.421,43.734],[7.416,43.734]]],\"type\":\"Polygon\"}}'"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Geohash"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -269,6 +290,13 @@
     "! QuackOSM --geom-filter-index-geohash spv2bcs"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### H3"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -282,6 +310,13 @@
     "! QuackOSM --geom-filter-index-h3 893969a4037ffff"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### S2"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -295,6 +330,13 @@
     "! QuackOSM --geom-filter-index-s2 12cdc28d"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### WKT"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -396,6 +438,13 @@
     "- `--compact-tags` (or `--compact`): will always keep tags together as a single column."
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Separated tags (`explode`)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -419,14 +468,14 @@
    },
    "outputs": [],
    "source": [
-    "! QuackOSM andorra.osm.pbf --osm-tags-filter '{ \"amenity\": \"parking\", \"building\": \"office\" }' --compact --output files/andorra_filtered_compact.geoparquet"
+    "! ./duckdb :memory: \"FROM read_parquet('files/andorra_filtered_exploded.geoparquet')\""
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Let's see the difference in the files structure."
+    "### Compact tags (`compact`)"
    ]
   },
   {
@@ -439,7 +488,7 @@
    },
    "outputs": [],
    "source": [
-    "! ./duckdb :memory: \"FROM read_parquet('files/andorra_filtered_exploded.geoparquet')\""
+    "! QuackOSM andorra.osm.pbf --osm-tags-filter '{ \"amenity\": \"parking\", \"building\": \"office\" }' --compact --output files/andorra_filtered_compact.geoparquet"
    ]
   },
   {
@@ -509,7 +558,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.7"
+   "version": "3.10.12"
   }
  },
  "nbformat": 4,

diff --git a/mkdocs.yml b/mkdocs.yml
@@ -89,6 +89,7 @@ plugins:
         - docs/copy_readme.py
         - docs/copy_examples.py
         - docs/gen_ref_pages.py
+        - docs/gen_cli_docs.py
   - search
   - mkdocstrings:
       custom_templates: docs/templates

diff --git a/quackosm/_exceptions.py b/quackosm/_exceptions.py
@@ -0,0 +1,7 @@
+class EmptyResultWarning(Warning): ...
+
+
+class GeometryNotCoveredWarning(Warning): ...
+
+
+class GeometryNotCoveredError(Exception): ...
diff --git a/quackosm/_rich_progress.py b/quackosm/_rich_progress.py
@@ -1,5 +1,6 @@
 # type: ignore
 """Wrapper over Rich progress bar."""
+
 import os
 from collections.abc import Iterable
 from contextlib import suppress
@@ -28,10 +29,13 @@ def show_total_elapsed_time(elapsed_seconds: float) -> None:
 
 
 class TaskProgressSpinner:
-    def __init__(self, step_name: str, step_number: str, silent_mode: bool):
+    def __init__(
+        self, step_name: str, step_number: str, silent_mode: bool, skip_step_number: bool = False
+    ):
         self.step_name = step_name
         self.step_number = step_number
         self.silent_mode = silent_mode
+        self.skip_step_number = skip_step_number
         self.progress = None
         self.force_terminal = os.getenv("FORCE_TERMINAL_MODE", "false").lower() == "true"
 
@@ -48,12 +52,19 @@ def __enter__(self):
                     TimeElapsedColumn,
                 )
 
-                self.progress = Progress(
+                columns = [
                     SpinnerColumn(),
                     TextColumn(f"[{self.step_number: >4}/{TOTAL_STEPS}]"),
                     TextColumn("[progress.description]{task.description}"),
                     TextColumn("•"),
                     TimeElapsedColumn(),
+                ]
+
+                if self.skip_step_number:
+                    columns.pop(1)
+
+                self.progress = Progress(
+                    *columns,
                     refresh_per_second=1,
                     transient=False,
                     console=Console(
@@ -77,15 +88,17 @@ def __exit__(self, exc_type, exc_value, exc_tb):
 
 
 class TaskProgressBar:
-    def __init__(self, step_name: str, step_number: str, silent_mode: bool):
+    def __init__(
+        self, step_name: str, step_number: str, silent_mode: bool, skip_step_number: bool = False
+    ):
         self.step_name = step_name
         self.step_number = step_number
         self.silent_mode = silent_mode
+        self.skip_step_number = skip_step_number
         self.progress = None
         self.force_terminal = os.getenv("FORCE_TERMINAL_MODE", "false").lower() == "true"
 
     def __enter__(self):
-
         try:  # pragma: no cover
             if self.silent_mode:
                 self.progress = None
@@ -113,7 +126,7 @@ def render(self, task: "Task") -> Text:
                         else:
                             return Text(f"{1/task.speed:.2f} s/it")  # noqa: FURB126
 
-                self.progress = Progress(
+                columns = [
                     SpinnerColumn(),
                     TextColumn(f"[{self.step_number: >4}/{TOTAL_STEPS}]"),
                     TextColumn(
@@ -128,6 +141,13 @@ def render(self, task: "Task") -> Text:
                     TimeRemainingColumn(),
                     TextColumn("•"),
                     SpeedColumn(),
+                ]
+
+                if self.skip_step_number:
+                    columns.pop(1)
+
+                self.progress = Progress(
+                    *columns,
                     refresh_per_second=1,
                     transient=False,
                     speed_estimate_period=1800,
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# CLI Reference

		Text below is a captured `--help` command output.