From 4dcaca7f4d15c6dbb131e71edc41348c1097ef4a Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 10 Mar 2024 09:08:25 +0100 Subject: [PATCH 1/8] add break condition for BaseHandlers Callbacks now return a boolean value indicating if processing should be stopped after the handler. --- lib/area.cc | 12 ++++++++---- lib/base_handler.h | 15 +++++++++------ lib/handler_chain.h | 20 +++++++++++++++----- lib/node_location_handler.cc | 6 ++++-- lib/python_handler.h | 15 ++++++++++----- lib/write_handler.cc | 20 ++++++++++++-------- 6 files changed, 58 insertions(+), 30 deletions(-) diff --git a/lib/area.cc b/lib/area.cc index 0690468e..1280b494 100644 --- a/lib/area.cc +++ b/lib/area.cc @@ -30,19 +30,22 @@ class AreaManagerSecondPassHandler : public BaseHandler { osmium::apply(ab, this->m_handlers); }); } - void node(osmium::Node const *n) override + bool node(osmium::Node const *n) override { m_mp_manager->handle_node(*n); + return false; } - void way(osmium::Way *w) override + bool way(osmium::Way *w) override { m_mp_manager->handle_way(*w); + return false; } - void relation(osmium::Relation const *r) override + bool relation(osmium::Relation const *r) override { m_mp_manager->handle_relation(*r); + return false; } void flush() override @@ -67,9 +70,10 @@ class AreaManager : public BaseHandler BaseHandler *first_pass_handler() { return this; } // first-pass-handler - void relation(osmium::Relation const *r) override + bool relation(osmium::Relation const *r) override { m_mp_manager.relation(*r); + return false; } AreaManagerSecondPassHandler *second_pass_handler(py::args args) diff --git a/lib/base_handler.h b/lib/base_handler.h index f7b59ae5..11359af3 100644 --- a/lib/base_handler.h +++ b/lib/base_handler.h @@ -23,12 +23,15 @@ class BaseHandler : public osmium::handler::Handler void changeset(const osmium::Changeset &o) { changeset(&o); } void area(const osmium::Area &o) { area(&o); } - // actual handler functions - virtual void node(const osmium::Node*) {} - virtual void way(osmium::Way *) {} - virtual void relation(const osmium::Relation*) {} - virtual void changeset(const osmium::Changeset*) {} - virtual void area(const osmium::Area*) {} + // Actual handler functions. + // All object handlers return a boolean which indicates if + // processing is finished (true) or should be continued with the next + // handler (false). + virtual bool node(const osmium::Node*) { return false; } + virtual bool way(osmium::Way *) { return false; } + virtual bool relation(const osmium::Relation*) { return false; } + virtual bool changeset(const osmium::Changeset*) { return false; } + virtual bool area(const osmium::Area*) { return false; } virtual void flush() {} }; diff --git a/lib/handler_chain.h b/lib/handler_chain.h index 268bd5f7..2af2599d 100644 --- a/lib/handler_chain.h +++ b/lib/handler_chain.h @@ -45,31 +45,41 @@ class HandlerChain : public osmium::handler::Handler void node(osmium::Node const &o) { for (auto const &handler : m_handlers) { - handler->node(&o); + if (handler->node(&o)) { + return; + } } } void way(osmium::Way &w) { for (auto const &handler : m_handlers) { - handler->way(&w); + if (handler->way(&w)) { + return; + } } } void relation(osmium::Relation const &o) { for (auto const &handler : m_handlers) { - handler->relation(&o); + if (handler->relation(&o)) { + return; + } } } void changeset(osmium::Changeset const &o) { for (auto const &handler : m_handlers) { - handler->changeset(&o); + if (handler->changeset(&o)) { + return; + } } } void area(osmium::Area const &o) { for (auto const &handler : m_handlers) { - handler->area(&o); + if (handler->area(&o)) { + return; + } } } diff --git a/lib/node_location_handler.cc b/lib/node_location_handler.cc index 98a3bbc5..e164e3c8 100644 --- a/lib/node_location_handler.cc +++ b/lib/node_location_handler.cc @@ -25,16 +25,18 @@ class NodeLocationsForWays : public BaseHandler : handler(idx) {} - void node(const osmium::Node *o) override + bool node(const osmium::Node *o) override { handler.node(*o); + return false; } - void way(osmium::Way *o) override + bool way(osmium::Way *o) override { if (apply_nodes_to_ways) { handler.way(*o); } + return false; } bool get_apply_nodes_to_ways() const { return apply_nodes_to_ways; } diff --git a/lib/python_handler.h b/lib/python_handler.h index 50e00320..85322223 100644 --- a/lib/python_handler.h +++ b/lib/python_handler.h @@ -56,7 +56,7 @@ class PythonHandler : public BaseHandler } - void node(osmium::Node const *n) override + bool node(osmium::Node const *n) override { if (m_enabled & osmium::osm_entity_bits::node) { pybind11::gil_scoped_acquire acquire; @@ -64,9 +64,10 @@ class PythonHandler : public BaseHandler ObjectGuard guard(obj); m_handler.attr("node")(obj); } + return false; } - void way(osmium::Way *w) override + bool way(osmium::Way *w) override { if (m_enabled & osmium::osm_entity_bits::way) { pybind11::gil_scoped_acquire acquire; @@ -74,9 +75,10 @@ class PythonHandler : public BaseHandler ObjectGuard guard(obj); m_handler.attr("way")(obj); } + return false; } - void relation(osmium::Relation const *r) override + bool relation(osmium::Relation const *r) override { if (m_enabled & osmium::osm_entity_bits::relation) { pybind11::gil_scoped_acquire acquire; @@ -84,9 +86,10 @@ class PythonHandler : public BaseHandler ObjectGuard guard(obj); m_handler.attr("relation")(obj); } + return false; } - void changeset(osmium::Changeset const *c) override + bool changeset(osmium::Changeset const *c) override { if (m_enabled & osmium::osm_entity_bits::changeset) { pybind11::gil_scoped_acquire acquire; @@ -94,9 +97,10 @@ class PythonHandler : public BaseHandler ObjectGuard guard(obj); m_handler.attr("changeset")(obj); } + return false; } - void area(osmium::Area const *a) override + bool area(osmium::Area const *a) override { if (m_enabled & osmium::osm_entity_bits::area) { pybind11::gil_scoped_acquire acquire; @@ -104,6 +108,7 @@ class PythonHandler : public BaseHandler ObjectGuard guard(obj); m_handler.attr("area")(obj); } + return false; } private: osmium::osm_entity_bits::type m_enabled; diff --git a/lib/write_handler.cc b/lib/write_handler.cc index e6bea85e..427b4e95 100644 --- a/lib/write_handler.cc +++ b/lib/write_handler.cc @@ -36,27 +36,31 @@ class WriteHandler : public BaseHandler virtual ~WriteHandler() { close(); } - void node(const osmium::Node* o) override + bool node(const osmium::Node* o) override { buffer.add_item(*o); flush_buffer(); + return false; } - void way(osmium::Way* o) override + bool way(osmium::Way* o) override { buffer.add_item(*o); flush_buffer(); + return false; } - void relation(const osmium::Relation* o) override + bool relation(const osmium::Relation* o) override { buffer.add_item(*o); flush_buffer(); + return false; } - void changeset(const osmium::Changeset*) override {} - - void area(const osmium::Area*) override {} + void flush() override + { + flush_buffer(true); + } void close() { @@ -68,11 +72,11 @@ class WriteHandler : public BaseHandler } private: - void flush_buffer() + void flush_buffer(bool force = false) { buffer.commit(); - if (buffer.committed() > buffer.capacity() - BUFFER_WRAP) { + if (force || buffer.committed() > buffer.capacity() - BUFFER_WRAP) { osmium::memory::Buffer new_buffer(buffer.capacity(), osmium::memory::Buffer::auto_grow::yes); using std::swap; swap(buffer, new_buffer); From 4c6a94bbf408919176f00c074eb0e9613a92d1a5 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 10 Mar 2024 09:17:38 +0100 Subject: [PATCH 2/8] add apply() versions that can take a file name --- lib/osmium.cc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/osmium.cc b/lib/osmium.cc index 322027b9..489e3c5b 100644 --- a/lib/osmium.cc +++ b/lib/osmium.cc @@ -45,6 +45,25 @@ PYBIND11_MODULE(_osmium, m) { }, py::arg("reader"), "Apply a chain of handlers."); + m.def("apply", [](std::string fn, BaseHandler &h) + { + py::gil_scoped_release release; + osmium::io::Reader rd{fn}; + osmium::apply(rd, h); + }, + py::arg("filename"), py::arg("handler"), + "Apply a single handler."); + m.def("apply", [](std::string fn, py::args args) + { + HandlerChain handler{args}; + { + py::gil_scoped_release release; + osmium::io::Reader rd{fn}; + osmium::apply(rd, handler); + } + }, + py::arg("filename"), + "Apply a chain of handlers."); py::class_(m, "BaseHandler"); From d49d2a87480866dbcc579ae5de5e3c0cdf70c506 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 10 Mar 2024 10:57:32 +0100 Subject: [PATCH 3/8] add basic EmptyTagFilter Filters all objects without any tags. --- CMakeLists.txt | 6 ++- lib/base_filter.h | 82 +++++++++++++++++++++++++++++++++++ lib/empty_tag_filter.cc | 37 ++++++++++++++++ lib/filter.cc | 26 +++++++++++ setup.py | 3 +- src/osmium/__init__.py | 1 + src/osmium/filter/__init__.py | 7 +++ 7 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 lib/base_filter.h create mode 100644 lib/empty_tag_filter.cc create mode 100644 lib/filter.cc create mode 100644 src/osmium/filter/__init__.py diff --git a/CMakeLists.txt b/CMakeLists.txt index ae663538..e5f8588f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,11 +84,15 @@ pybind11_add_module(_replication lib/replication.cc) set_module_output(_replication osmium/replication) pybind11_add_module(_area lib/area.cc) set_module_output(_area osmium/area) - +pybind11_add_module(_filter + lib/filter.cc + lib/empty_tag_filter.cc) +set_module_output(_filter osmium/filter) target_link_libraries(_osmium PRIVATE ${OSMIUM_LIBRARIES}) target_link_libraries(_replication PRIVATE ${OSMIUM_LIBRARIES}) target_link_libraries(_area PRIVATE ${OSMIUM_LIBRARIES}) +target_link_libraries(_filter PRIVATE ${OSMIUM_LIBRARIES}) # workaround for https://github.com/pybind/pybind11/issues/1272 if(APPLE) diff --git a/lib/base_filter.h b/lib/base_filter.h new file mode 100644 index 00000000..7678d9b4 --- /dev/null +++ b/lib/base_filter.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: BSD-2-Clause + * + * This file is part of pyosmium. (https://osmcode.org/pyosmium/) + * + * Copyright (C) 2024 Sarah Hoffmann and others. + * For a full list of authors see the git log. + */ +#ifndef PYOSMIUM_BASE_FILTER_HPP +#define PYOSMIUM_BASE_FILTER_HPP + +#include +#include + +#include "base_handler.h" + +namespace pyosmium { + +class BaseFilter : public BaseHandler { +public: + bool node(osmium::Node const *n) override + { + return (m_enabled_for & osmium::osm_entity_bits::node) + && (filter_node(n) != m_inverted); + } + + bool way(osmium::Way *n) override + { + return (m_enabled_for & osmium::osm_entity_bits::way) + && (filter_way(n) != m_inverted); + } + + bool relation(osmium::Relation const *n) override + { + return (m_enabled_for & osmium::osm_entity_bits::relation) + && (filter_relation(n) != m_inverted); + } + + bool area(osmium::Area const *n) override + { + return (m_enabled_for & osmium::osm_entity_bits::area) + && (filter_area(n) != m_inverted); + } + + bool changeset(osmium::Changeset const *n) override + { + return (m_enabled_for & osmium::osm_entity_bits::changeset) + && (filter_changeset(n) != m_inverted); + } + + BaseFilter *enable_for(osmium::osm_entity_bits::type entities) + { + m_enabled_for = entities; + return this; + } + + BaseFilter *invert(bool new_state) + { + m_inverted = new_state; + return this; + } + +protected: + virtual bool filter(osmium::OSMObject const *) { return false; } + virtual bool filter_node(osmium::Node const *n) { return filter(n); } + virtual bool filter_way(osmium::Way const *w) { return filter(w); } + virtual bool filter_relation(osmium::Relation const *r) { return filter(r); } + virtual bool filter_area(osmium::Area const *a) { return filter(a); } + virtual bool filter_changeset(osmium::Changeset const *) { return false; } + +private: + bool m_inverted = false; + osmium::osm_entity_bits::type m_enabled_for = osmium::osm_entity_bits::all; + +}; + + +void init_empty_tag_filter(pybind11::module &m); + +} // namespace + + +#endif //PYOSMIUM_BASE_FILTER_HPP diff --git a/lib/empty_tag_filter.cc b/lib/empty_tag_filter.cc new file mode 100644 index 00000000..e7b3d8d4 --- /dev/null +++ b/lib/empty_tag_filter.cc @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: BSD-2-Clause + * + * This file is part of pyosmium. (https://osmcode.org/pyosmium/) + * + * Copyright (C) 2024 Sarah Hoffmann and others. + * For a full list of authors see the git log. + */ +#include + +#include + +#include "base_filter.h" + +namespace py = pybind11; + +namespace { + +class EmptyTagFilter : public pyosmium::BaseFilter +{ + bool filter(osmium::OSMObject const *o) override + { + return o->tags().empty(); + } +}; + +} + +namespace pyosmium { + +void init_empty_tag_filter(pybind11::module &m) +{ + py::class_(m, "EmptyTagFilter") + .def(py::init<>()) + ; +} + +} // namespace diff --git a/lib/filter.cc b/lib/filter.cc new file mode 100644 index 00000000..015b10bd --- /dev/null +++ b/lib/filter.cc @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-2-Clause + * + * This file is part of pyosmium. (https://osmcode.org/pyosmium/) + * + * Copyright (C) 2024 Sarah Hoffmann and others. + * For a full list of authors see the git log. + */ +#include + +#include "base_filter.h" + + +namespace py = pybind11; + +PYBIND11_MODULE(_filter, m) { + py::class_(m, "BaseFilter") + .def("enable_for", &pyosmium::BaseFilter::enable_for, + py::arg("entities"), + "Set the OSM types this filter should be used for.") + .def("invert", &pyosmium::BaseFilter::invert, + py::arg("new_state")=true) + ; + + pyosmium::init_empty_tag_filter(m); +}; + diff --git a/setup.py b/setup.py index dc493505..aed27452 100644 --- a/setup.py +++ b/setup.py @@ -170,7 +170,8 @@ def build_extension(self, ext): ], ext_modules=[CMakeExtension('cmake_example')], - packages = ['osmium', 'osmium/osm', 'osmium/replication', 'osmium/area'], + packages = ['osmium', 'osmium/osm', 'osmium/replication', 'osmium/area', + 'osmium/filter'], package_dir = {'' : 'src'}, package_data = { 'osmium': ['py.typed', '*.pyi', 'replication/_replication.pyi', diff --git a/src/osmium/__init__.py b/src/osmium/__init__.py index c4863042..2b39a768 100644 --- a/src/osmium/__init__.py +++ b/src/osmium/__init__.py @@ -13,3 +13,4 @@ import osmium.index import osmium.geom import osmium.area +import osmium.filter diff --git a/src/osmium/filter/__init__.py b/src/osmium/filter/__init__.py new file mode 100644 index 00000000..38e70753 --- /dev/null +++ b/src/osmium/filter/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# This file is part of pyosmium. (https://osmcode.org/pyosmium/) +# +# Copyright (C) 2024 Sarah Hoffmann and others. +# For a full list of authors see the git log. +from ._filter import * From 6553dd7da7914f00e061d1f01e251559e340e977 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 10 Mar 2024 12:03:15 +0100 Subject: [PATCH 4/8] add tests for EmtpyTagFilter --- test/test_empty_tag_filter.py | 103 ++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 test/test_empty_tag_filter.py diff --git a/test/test_empty_tag_filter.py b/test/test_empty_tag_filter.py new file mode 100644 index 00000000..e5afd865 --- /dev/null +++ b/test/test_empty_tag_filter.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: BSD +# +# This file is part of Pyosmium. +# +# Copyright (C) 2024 Sarah Hoffmann. +import osmium as o + +import pytest + +class IDCollector: + + def __init__(self): + self.nodes = [] + self.ways = [] + self.relations = [] + + def node(self, n): + self.nodes.append(n.id) + + def way(self, w): + self.ways.append(w.id) + + def relation(self, r): + self.relations.append(r.id) + + +@pytest.fixture +def reader(opl_reader): + return opl_reader("""\ + n1 x1 y1 + n2 x1 y1 Tfoo=bar + w1 Nn1,n2 Thighway=road + w34 Nn2,n1 + r90 + r91 Tsome=thing + """) + +def test_filter_default_config(reader): + pre = IDCollector() + post = IDCollector() + o.apply(reader, pre, o.filter.EmptyTagFilter(), post) + + assert pre.nodes == [1, 2] + assert post.nodes == [2] + assert pre.ways == [1, 34] + assert post.ways == [1] + assert pre.relations == [90, 91] + assert post.relations == [91] + + +def test_filter_inverted(reader): + pre = IDCollector() + post = IDCollector() + o.apply(reader, pre, o.filter.EmptyTagFilter().invert(), post) + + assert pre.nodes == [1, 2] + assert post.nodes == [1] + assert pre.ways == [1, 34] + assert post.ways == [34] + assert pre.relations == [90, 91] + assert post.relations == [90] + + +def test_filter_restrict_entity(reader): + pre = IDCollector() + post = IDCollector() + o.apply(reader, pre, o.filter.EmptyTagFilter().enable_for(o.osm.WAY | o.osm.RELATION), post) + + assert pre.nodes == [1, 2] + assert post.nodes == [1, 2] + assert pre.ways == [1, 34] + assert post.ways == [1] + assert pre.relations == [90, 91] + assert post.relations == [91] + + +def test_filter_restrict_entity_invert(reader): + pre = IDCollector() + post = IDCollector() + o.apply(reader, pre, o.filter.EmptyTagFilter().enable_for(o.osm.NODE).invert(), post) + + assert pre.nodes == [1, 2] + assert post.nodes == [1] + assert pre.ways == [1, 34] + assert post.ways == [1, 34] + assert pre.relations == [90, 91] + assert post.relations == [90, 91] + + +def test_filter_chained(reader): + pre = IDCollector() + post = IDCollector() + o.apply(reader, pre, + o.filter.EmptyTagFilter().enable_for(o.osm.NODE).invert(False), + o.filter.EmptyTagFilter().enable_for(o.osm.WAY).invert(True), + post) + + assert pre.nodes == [1, 2] + assert post.nodes == [2] + assert pre.ways == [1, 34] + assert post.ways == [34] + assert pre.relations == [90, 91] + assert post.relations == [90, 91] From abf58b770d7c24e608b60df3dbfa023664d2032c Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 10 Mar 2024 15:26:37 +0100 Subject: [PATCH 5/8] add key filter --- CMakeLists.txt | 3 +- lib/base_filter.h | 1 + lib/filter.cc | 1 + lib/key_filter.cc | 64 +++++++++++++++++++++++++++++++++++ test/helpers.py | 17 ++++++++++ test/test_empty_tag_filter.py | 17 +--------- test/test_key_filter.py | 38 +++++++++++++++++++++ 7 files changed, 124 insertions(+), 17 deletions(-) create mode 100644 lib/key_filter.cc create mode 100644 test/test_key_filter.py diff --git a/CMakeLists.txt b/CMakeLists.txt index e5f8588f..399485cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -86,7 +86,8 @@ pybind11_add_module(_area lib/area.cc) set_module_output(_area osmium/area) pybind11_add_module(_filter lib/filter.cc - lib/empty_tag_filter.cc) + lib/empty_tag_filter.cc + lib/key_filter.cc) set_module_output(_filter osmium/filter) target_link_libraries(_osmium PRIVATE ${OSMIUM_LIBRARIES}) diff --git a/lib/base_filter.h b/lib/base_filter.h index 7678d9b4..02c81350 100644 --- a/lib/base_filter.h +++ b/lib/base_filter.h @@ -75,6 +75,7 @@ class BaseFilter : public BaseHandler { void init_empty_tag_filter(pybind11::module &m); +void init_key_filter(pybind11::module &m); } // namespace diff --git a/lib/filter.cc b/lib/filter.cc index 015b10bd..4eb90141 100644 --- a/lib/filter.cc +++ b/lib/filter.cc @@ -22,5 +22,6 @@ PYBIND11_MODULE(_filter, m) { ; pyosmium::init_empty_tag_filter(m); + pyosmium::init_key_filter(m); }; diff --git a/lib/key_filter.cc b/lib/key_filter.cc new file mode 100644 index 00000000..efcf5255 --- /dev/null +++ b/lib/key_filter.cc @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: BSD-2-Clause + * + * This file is part of pyosmium. (https://osmcode.org/pyosmium/) + * + * Copyright (C) 2024 Sarah Hoffmann and others. + * For a full list of authors see the git log. + */ +#include + +#include + +#include "base_filter.h" + +namespace py = pybind11; + +namespace { + +class KeyFilter : public pyosmium::BaseFilter +{ +public: + KeyFilter(py::args args) + { + if (args.empty()) { + throw py::type_error{"Need keys to filter on."}; + } + + m_keys.reserve(args.size()); + for (auto const &arg: args) { + if (!py::isinstance(arg)) { + throw py::type_error{"Arguments must be strings."}; + } + + m_keys.push_back(arg.cast()); + } + } + + bool filter(osmium::OSMObject const *o) override + { + auto const &tags = o->tags(); + for (auto const &key: m_keys) { + if (tags.has_key(key.c_str())) { + return false; + } + } + + return true; + } + +private: + std::vector m_keys; +}; + +} + +namespace pyosmium { + +void init_key_filter(pybind11::module &m) +{ + py::class_(m, "KeyFilter") + .def(py::init()) + ; +} + +} // namespace diff --git a/test/helpers.py b/test/helpers.py index a63b1868..5c042693 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -24,3 +24,20 @@ def relation(self, _): def area(self, _): self.counts[3] += 1 + + +class IDCollector: + + def __init__(self): + self.nodes = [] + self.ways = [] + self.relations = [] + + def node(self, n): + self.nodes.append(n.id) + + def way(self, w): + self.ways.append(w.id) + + def relation(self, r): + self.relations.append(r.id) diff --git a/test/test_empty_tag_filter.py b/test/test_empty_tag_filter.py index e5afd865..0b991ebb 100644 --- a/test/test_empty_tag_filter.py +++ b/test/test_empty_tag_filter.py @@ -7,22 +7,7 @@ import pytest -class IDCollector: - - def __init__(self): - self.nodes = [] - self.ways = [] - self.relations = [] - - def node(self, n): - self.nodes.append(n.id) - - def way(self, w): - self.ways.append(w.id) - - def relation(self, r): - self.relations.append(r.id) - +from helpers import IDCollector @pytest.fixture def reader(opl_reader): diff --git a/test/test_key_filter.py b/test/test_key_filter.py new file mode 100644 index 00000000..dd1ba762 --- /dev/null +++ b/test/test_key_filter.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: BSD +# +# This file is part of Pyosmium. +# +# Copyright (C) 2024 Sarah Hoffmann. +import osmium as o + +import pytest + +from helpers import IDCollector + +def test_filter_no_keys(): + with pytest.raises(TypeError, match="keys to filter"): + o.filter.KeyFilter() + + +@pytest.mark.parametrize('key', [None, 1, IDCollector(), 'a'.encode('utf-8')]) +def test_filter_bad_argument_types(key): + with pytest.raises(TypeError, match="must be strings"): + o.filter.KeyFilter("foo", key) + + +@pytest.mark.parametrize('key,result', [('foo', [1]), + ('name', [1, 2]), + ('kö*', [4])]) +def test_filter_simple(opl_reader, key, result): + data = """\ + n1 Tfoo=bar,name=loo + n2 Tname=else + n3 x9 y0 + n4 Tkö*=fr + """ + + post = IDCollector() + + o.apply(opl_reader(data), o.filter.KeyFilter(key), post) + + assert post.nodes == result From ee5bb1fb3105e052e89c731e4d074cb47f31e5eb Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 10 Mar 2024 15:52:31 +0100 Subject: [PATCH 6/8] allow filters in apply_* functions of SimpleHandler Changes amenity_list example to use the new filter function. --- examples/amenity_list.py | 14 ++++++-------- src/osmium/simple_handler.py | 19 +++++++++---------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/examples/amenity_list.py b/examples/amenity_list.py index f8ddcaf1..3f249e24 100644 --- a/examples/amenity_list.py +++ b/examples/amenity_list.py @@ -18,22 +18,20 @@ def print_amenity(self, tags, lon, lat): print("%f %f %-15s %s" % (lon, lat, tags['amenity'], name)) def node(self, n): - if 'amenity' in n.tags: - self.print_amenity(n.tags, n.location.lon, n.location.lat) + self.print_amenity(n.tags, n.location.lon, n.location.lat) def area(self, a): - if 'amenity' in a.tags: - wkb = wkbfab.create_multipolygon(a) - poly = wkblib.loads(wkb, hex=True) - centroid = poly.representative_point() - self.print_amenity(a.tags, centroid.x, centroid.y) + wkb = wkbfab.create_multipolygon(a) + poly = wkblib.loads(wkb, hex=True) + centroid = poly.representative_point() + self.print_amenity(a.tags, centroid.x, centroid.y) def main(osmfile): handler = AmenityListHandler() - handler.apply_file(osmfile) + handler.apply_file(osmfile, filters=[o.filter.KeyFilter('amenity')]) return 0 diff --git a/src/osmium/simple_handler.py b/src/osmium/simple_handler.py index d364070b..9ab49155 100644 --- a/src/osmium/simple_handler.py +++ b/src/osmium/simple_handler.py @@ -40,7 +40,7 @@ def enabled_for(self): return entities - def apply_file(self, filename, locations=False, idx='flex_mem'): + def apply_file(self, filename, locations=False, idx='flex_mem', filters=[]): """ Apply the handler to the given file. If locations is true, then a location handler will be applied before, which saves the node positions. In that case, the type of this position index can be @@ -49,38 +49,37 @@ def apply_file(self, filename, locations=False, idx='flex_mem'): handler for assembling multipolygons and areas from ways will be executed. """ - self._apply_object(str(filename), locations, idx) + self._apply_object(str(filename), locations, idx, filters) - def apply_buffer(self, buffer, format, locations=False, idx='flex_mem'): + def apply_buffer(self, buffer, format, locations=False, idx='flex_mem', filters=[]): """Apply the handler to a string buffer. The buffer must be a byte string. """ - self._apply_object(FileBuffer(buffer, format), locations, idx) + self._apply_object(FileBuffer(buffer, format), locations, idx, filters) - def _apply_object(self, obj, locations, idx): + def _apply_object(self, obj, locations, idx, filters): entities = self.enabled_for() - handlers = [self] if entities & osm_entity_bits.AREA: area = AreaManager() rd = Reader(obj, osm_entity_bits.RELATION) try: - apply(rd, area.first_pass_handler()) + apply(rd, *filters, area.first_pass_handler()) finally: rd.close() entities |= osm_entity_bits.OBJECT lh = NodeLocationsForWays(create_map(idx)) lh.ignore_errors() - handlers = [lh, self, area.second_pass_handler(self)] + handlers = [lh, area.second_pass_handler(*filters, self), *filters, self] elif locations: entities |= osm_entity_bits.NODE lh = NodeLocationsForWays(create_map(idx)) lh.ignore_errors() - handlers = [lh, self] + handlers = [lh, *filters, self] else: - handlers = [self] + handlers = [*filters, self] rd = Reader(obj, entities) try: From 36fdbdd1db0eb71361d7b394e76bd0cf15996749 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 10 Mar 2024 17:12:37 +0100 Subject: [PATCH 7/8] remove invert() function from filters It's too confusing to explain. --- lib/base_filter.h | 17 +++++------------ lib/filter.cc | 2 -- test/test_empty_tag_filter.py | 32 +++----------------------------- 3 files changed, 8 insertions(+), 43 deletions(-) diff --git a/lib/base_filter.h b/lib/base_filter.h index 02c81350..a055c37c 100644 --- a/lib/base_filter.h +++ b/lib/base_filter.h @@ -20,31 +20,31 @@ class BaseFilter : public BaseHandler { bool node(osmium::Node const *n) override { return (m_enabled_for & osmium::osm_entity_bits::node) - && (filter_node(n) != m_inverted); + && filter_node(n); } bool way(osmium::Way *n) override { return (m_enabled_for & osmium::osm_entity_bits::way) - && (filter_way(n) != m_inverted); + && filter_way(n); } bool relation(osmium::Relation const *n) override { return (m_enabled_for & osmium::osm_entity_bits::relation) - && (filter_relation(n) != m_inverted); + && filter_relation(n); } bool area(osmium::Area const *n) override { return (m_enabled_for & osmium::osm_entity_bits::area) - && (filter_area(n) != m_inverted); + && filter_area(n); } bool changeset(osmium::Changeset const *n) override { return (m_enabled_for & osmium::osm_entity_bits::changeset) - && (filter_changeset(n) != m_inverted); + && filter_changeset(n); } BaseFilter *enable_for(osmium::osm_entity_bits::type entities) @@ -53,12 +53,6 @@ class BaseFilter : public BaseHandler { return this; } - BaseFilter *invert(bool new_state) - { - m_inverted = new_state; - return this; - } - protected: virtual bool filter(osmium::OSMObject const *) { return false; } virtual bool filter_node(osmium::Node const *n) { return filter(n); } @@ -68,7 +62,6 @@ class BaseFilter : public BaseHandler { virtual bool filter_changeset(osmium::Changeset const *) { return false; } private: - bool m_inverted = false; osmium::osm_entity_bits::type m_enabled_for = osmium::osm_entity_bits::all; }; diff --git a/lib/filter.cc b/lib/filter.cc index 4eb90141..92d819e9 100644 --- a/lib/filter.cc +++ b/lib/filter.cc @@ -17,8 +17,6 @@ PYBIND11_MODULE(_filter, m) { .def("enable_for", &pyosmium::BaseFilter::enable_for, py::arg("entities"), "Set the OSM types this filter should be used for.") - .def("invert", &pyosmium::BaseFilter::invert, - py::arg("new_state")=true) ; pyosmium::init_empty_tag_filter(m); diff --git a/test/test_empty_tag_filter.py b/test/test_empty_tag_filter.py index 0b991ebb..ddd62913 100644 --- a/test/test_empty_tag_filter.py +++ b/test/test_empty_tag_filter.py @@ -33,19 +33,6 @@ def test_filter_default_config(reader): assert post.relations == [91] -def test_filter_inverted(reader): - pre = IDCollector() - post = IDCollector() - o.apply(reader, pre, o.filter.EmptyTagFilter().invert(), post) - - assert pre.nodes == [1, 2] - assert post.nodes == [1] - assert pre.ways == [1, 34] - assert post.ways == [34] - assert pre.relations == [90, 91] - assert post.relations == [90] - - def test_filter_restrict_entity(reader): pre = IDCollector() post = IDCollector() @@ -59,30 +46,17 @@ def test_filter_restrict_entity(reader): assert post.relations == [91] -def test_filter_restrict_entity_invert(reader): - pre = IDCollector() - post = IDCollector() - o.apply(reader, pre, o.filter.EmptyTagFilter().enable_for(o.osm.NODE).invert(), post) - - assert pre.nodes == [1, 2] - assert post.nodes == [1] - assert pre.ways == [1, 34] - assert post.ways == [1, 34] - assert pre.relations == [90, 91] - assert post.relations == [90, 91] - - def test_filter_chained(reader): pre = IDCollector() post = IDCollector() o.apply(reader, pre, - o.filter.EmptyTagFilter().enable_for(o.osm.NODE).invert(False), - o.filter.EmptyTagFilter().enable_for(o.osm.WAY).invert(True), + o.filter.EmptyTagFilter().enable_for(o.osm.NODE), + o.filter.EmptyTagFilter().enable_for(o.osm.WAY), post) assert pre.nodes == [1, 2] assert post.nodes == [2] assert pre.ways == [1, 34] - assert post.ways == [34] + assert post.ways == [1] assert pre.relations == [90, 91] assert post.relations == [90, 91] From 67e425467cbbabb40569750a03a99e0cd9d0bc00 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 10 Mar 2024 17:19:42 +0100 Subject: [PATCH 8/8] apply filters to changesets as well --- lib/empty_tag_filter.cc | 6 ++++++ lib/key_filter.cc | 12 ++++++++++++ test/helpers.py | 4 ++++ test/test_empty_tag_filter.py | 4 ++++ test/test_key_filter.py | 13 ++++++++----- 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/lib/empty_tag_filter.cc b/lib/empty_tag_filter.cc index e7b3d8d4..f1bc556b 100644 --- a/lib/empty_tag_filter.cc +++ b/lib/empty_tag_filter.cc @@ -21,6 +21,12 @@ class EmptyTagFilter : public pyosmium::BaseFilter { return o->tags().empty(); } + + bool filter_changeset(osmium::Changeset const *o) override + { + return o->tags().empty(); + } + }; } diff --git a/lib/key_filter.cc b/lib/key_filter.cc index efcf5255..d65819a2 100644 --- a/lib/key_filter.cc +++ b/lib/key_filter.cc @@ -46,6 +46,18 @@ class KeyFilter : public pyosmium::BaseFilter return true; } + bool filter_changeset(osmium::Changeset const *o) override + { + auto const &tags = o->tags(); + for (auto const &key: m_keys) { + if (tags.has_key(key.c_str())) { + return false; + } + } + + return true; + } + private: std::vector m_keys; }; diff --git a/test/helpers.py b/test/helpers.py index 5c042693..48395269 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -32,6 +32,7 @@ def __init__(self): self.nodes = [] self.ways = [] self.relations = [] + self.changesets = [] def node(self, n): self.nodes.append(n.id) @@ -41,3 +42,6 @@ def way(self, w): def relation(self, r): self.relations.append(r.id) + + def changeset(self, c): + self.changesets.append(c.id) diff --git a/test/test_empty_tag_filter.py b/test/test_empty_tag_filter.py index ddd62913..10a933a1 100644 --- a/test/test_empty_tag_filter.py +++ b/test/test_empty_tag_filter.py @@ -18,6 +18,8 @@ def reader(opl_reader): w34 Nn2,n1 r90 r91 Tsome=thing + c222 Ttodo=done + c223 """) def test_filter_default_config(reader): @@ -31,6 +33,8 @@ def test_filter_default_config(reader): assert post.ways == [1] assert pre.relations == [90, 91] assert post.relations == [91] + assert pre.changesets == [222, 223] + assert post.changesets == [222] def test_filter_restrict_entity(reader): diff --git a/test/test_key_filter.py b/test/test_key_filter.py index dd1ba762..06ad71dc 100644 --- a/test/test_key_filter.py +++ b/test/test_key_filter.py @@ -20,19 +20,22 @@ def test_filter_bad_argument_types(key): o.filter.KeyFilter("foo", key) -@pytest.mark.parametrize('key,result', [('foo', [1]), - ('name', [1, 2]), - ('kö*', [4])]) -def test_filter_simple(opl_reader, key, result): +@pytest.mark.parametrize('key,nodes,changesets', [('foo', [1], [10]), + ('name', [1, 2], [20]), + ('kö*', [4], [])]) +def test_filter_simple(opl_reader, key, nodes, changesets): data = """\ n1 Tfoo=bar,name=loo n2 Tname=else n3 x9 y0 n4 Tkö*=fr + c10 Tfoo=baz + c20 Tname=none """ post = IDCollector() o.apply(opl_reader(data), o.filter.KeyFilter(key), post) - assert post.nodes == result + assert post.nodes == nodes + assert post.changesets == changesets