From d40c138192903ebb292af8119bbc23ffd7408759 Mon Sep 17 00:00:00 2001 From: Rod S Date: Mon, 18 Apr 2022 23:04:29 -0700 Subject: [PATCH 1/5] Building blocks for tidier reuse. Define a "parts" file that contains reusable shapes. Define merge on parts files. Generate one such file per picosvg. --- src/nanoemoji/nanoemoji.py | 25 +++++- src/nanoemoji/parts.py | 138 +++++++++++++++++++++++++++++++ src/nanoemoji/write_part_file.py | 48 +++++++++++ tests/parts_test.py | 88 ++++++++++++++++++++ 4 files changed, 298 insertions(+), 1 deletion(-) create mode 100644 src/nanoemoji/parts.py create mode 100644 src/nanoemoji/write_part_file.py create mode 100644 tests/parts_test.py diff --git a/src/nanoemoji/nanoemoji.py b/src/nanoemoji/nanoemoji.py index 2efd54dd..41f7211e 100644 --- a/src/nanoemoji/nanoemoji.py +++ b/src/nanoemoji/nanoemoji.py @@ -249,6 +249,13 @@ def write_preamble(nw): ) nw.newline() + module_rule( + nw, + "write_part_file", + f"--reuse_tolerance $reuse_tolerance --output_file $out $in", + ) + nw.newline() + nw.newline() @@ -305,6 +312,10 @@ def picosvg_dest(clipped: bool, input_svg: Path) -> Path: return _dest_for_src(picosvg_dest, out_dir, input_svg, ".svg") +def part_file_dest(picosvg_file: Path) -> Path: + return picosvg_file.with_suffix(".parts.json") + + def bitmap_dest(input_svg: Path) -> Path: return _dest_for_src(bitmap_dest, bitmap_dir(), input_svg, ".png") @@ -337,6 +348,7 @@ def write_picosvg_builds( picosvg_builds: Set[Path], nw: NinjaWriter, clipped: bool, + reuse_tolerance: float, master: MasterConfig, ): rule_name = "picosvg_unclipped" @@ -350,6 +362,13 @@ def write_picosvg_builds( picosvg_builds.add(svg_file) nw.build(dest, rule_name, rel_build(svg_file)) + nw.build( + part_file_dest(dest), + "write_part_file", + dest, + variables={"reuse_tolerance": reuse_tolerance}, + ) + def write_bitmap_builds( bitmap_builds: Set[Path], @@ -623,7 +642,11 @@ def _run(argv): for master in font_config.masters: if font_config.has_picosvgs: write_picosvg_builds( - picosvg_builds, nw, font_config.clip_to_viewbox, master + picosvg_builds, + nw, + font_config.clip_to_viewbox, + font_config.reuse_tolerance, + master, ) nw.newline() diff --git a/src/nanoemoji/parts.py b/src/nanoemoji/parts.py new file mode 100644 index 00000000..edeae0bc --- /dev/null +++ b/src/nanoemoji/parts.py @@ -0,0 +1,138 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dataclasses +from functools import lru_cache +import json +from nanoemoji.config import FontConfig +from pathlib import Path +from picosvg.svg import SVG +from picosvg.svg_reuse import normalize +from picosvg.svg_types import SVGPath, SVGShape +from typing import Iterable, List, MutableMapping, Set, Tuple, Union + + +PathSource = Union[SVGShape, Iterable[SVGShape], "ReuseableParts", Path] + + +@lru_cache(maxsize=1) +def _default_tolerence() -> float: + return FontConfig().reuse_tolerance + + +def _is_iterable_of(thing, desired_type) -> bool: + try: + it = iter(thing) + except TypeError: + return False + + try: + val = next(it) + return isinstance(val, desired_type) + except StopIteration: + return True + + +@dataclasses.dataclass(frozen=True) +class Shape: + d: str # an SVG style path, e.g. the d attribute of + + +# A set of shapes that normalize to the same path +# Only the d attribute of the SVGPath is populated +@dataclasses.dataclass +class ShapeSet: + normalized: Shape + shapes: Set[Shape] = dataclasses.field(default_factory=set) + + +@dataclasses.dataclass +class ReuseableParts: + version: Tuple[int, int, int] = (1, 0, 0) + reuse_tolerance: float = dataclasses.field(default_factory=_default_tolerence) + shape_sets: MutableMapping[Shape, ShapeSet] = dataclasses.field( + default_factory=dict + ) + + def _add_norm_path(self, norm: Shape, shape: Shape): + if norm not in self.shape_sets: + self.shape_sets[norm] = ShapeSet(norm) + self.shape_sets[norm].shapes.add(shape) + + def _add(self, shape: Shape): + norm = shape + if self.reuse_tolerance != -1: + norm = Shape(normalize(SVGPath(d=shape.d), self.reuse_tolerance).d) + self._add_norm_path(norm, shape) + + def add(self, source: PathSource): + if isinstance(source, Path): + source = ReuseableParts.load(source) + + if isinstance(source, ReuseableParts): + for shape_set in source.shape_sets.values(): + for shape in shape_set.shapes: + self._add_norm_path(shape_set.normalized, shape) + else: + if not _is_iterable_of(source, SVGShape): + source = (source,) + for a_source in source: + if not isinstance(a_source, SVGShape): + raise ValueError(f"Illegal source {type(a_source)}") + svg_shape: SVGShape = a_source # pytype: disable=attribute-error + self._add(Shape(svg_shape.as_path().d)) + + def to_json(self): + json_dict = { + "version": ".".join(str(v) for v in self.version), + "reuse_tolerance": self.reuse_tolerance, + "shape_sets": [ + {"normalized": n.d, "shapes": [p.d for p in s.shapes]} + for n, s in self.shape_sets.items() + ], + } + return json.dumps(json_dict, indent=2) + + @classmethod + def fromstring(cls, string) -> "ReuseableParts": + first = string.strip()[0] + parts = cls() + if first == "<": + svg = SVG.fromstring(string).topicosvg() + for path in svg.xpath("//svg:path"): + parts.add(SVGPath(d=path.attrib["d"])) + elif first == "{": + json_dict = json.loads(string) + parts.version = tuple(int(v) for v in json_dict.pop("version").split(".")) + assert parts.version == (1, 0, 0), f"Bad version {parts.version}" + parts.reuse_tolerance = float(json_dict.pop("reuse_tolerance")) + for shape_set_json in json_dict.pop("shape_sets"): + norm = Shape(str(shape_set_json.pop("normalized"))) + shapes = {Shape(s) for s in shape_set_json.pop("shapes")} + if shape_set_json: + raise ValueError(f"Unconsumed input {shape_set_json}") + parts.shape_sets[norm] = ShapeSet(norm, shapes) + if json_dict: + raise ValueError(f"Unconsumed input {json_dict}") + + else: + raise ValueError(f"Unrecognized start sequence {string[:16]}") + return parts + + @classmethod + def load(cls, input_file: Path) -> "ReuseableParts": + ext = input_file.suffix.lower() + if ext not in {".svg", ".json"}: + raise ValueError(f"Unknown format {input_file}") + return cls.fromstring(input_file.read_text(encoding="utf-8")) diff --git a/src/nanoemoji/write_part_file.py b/src/nanoemoji/write_part_file.py new file mode 100644 index 00000000..c1cd9f86 --- /dev/null +++ b/src/nanoemoji/write_part_file.py @@ -0,0 +1,48 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generates a part file from 1..N input part sources. + +Part sources can be: + +1. Other part files +2. Svg files + +Or any mix thereof. +""" + + +from absl import app +from absl import flags +from functools import reduce +from nanoemoji.parts import ReuseableParts +from nanoemoji import util +from pathlib import Path + + +FLAGS = flags.FLAGS + + +def main(argv): + parts = [ReuseableParts.load(Path(part_file)) for part_file in argv[1:]] + if not parts: + raise ValueError("Specify at least one input") + parts = reduce(lambda a, c: a.add(c), parts[1:], parts[0]) + + with util.file_printer(FLAGS.output_file) as print: + print(parts.to_json()) + + +if __name__ == "__main__": + app.run(main) diff --git a/tests/parts_test.py b/tests/parts_test.py new file mode 100644 index 00000000..324ff45b --- /dev/null +++ b/tests/parts_test.py @@ -0,0 +1,88 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nanoemoji.parts import ReuseableParts +from picosvg.svg_types import SVGCircle, SVGRect +import pprint +import pytest +from test_helper import cleanup_temp_dirs, locate_test_file, mkdtemp + + +@pytest.fixture(scope="module", autouse=True) +def _cleanup_temporary_dirs(): + # The mkdtemp() docs say the user is responsible for deleting the directory + # and its contents when done with it. So we use an autouse fixture that + # automatically removes all the temp dirs at the end of the test module + yield + # teardown happens after the 'yield' + cleanup_temp_dirs() + + +# BUG? rect(2,1) and rect(1,2) do NOT normalize the same. +# TODO we get pointless precision, e.g. 1.2000000000000002 + + +def check_num_shapes(parts: ReuseableParts, expected_shape_sets: int): + assert len(parts.shape_sets) == expected_shape_sets, ",".join( + sorted(str(p) for p in parts.shape_sets.keys()) + ) + + +def test_collects_normalized_shapes(): + shapes = ( + SVGRect(width=2, height=1), + SVGRect(width=4, height=2), + SVGCircle(r=2), + ) + + parts = ReuseableParts() + parts.add(shapes) + + check_num_shapes(parts, 2) + + +def test_from_svg(): + parts = ReuseableParts.load(locate_test_file("rect.svg")) + check_num_shapes(parts, 1) + + +def test_merge(): + shapes1 = (SVGRect(width=2, height=1),) + shapes2 = ( + SVGRect(width=4, height=2), + SVGCircle(r=2), + ) + + p1 = ReuseableParts() + p1.add(shapes1) + check_num_shapes(p1, 1) + + p2 = ReuseableParts() + p2.add(shapes2) + check_num_shapes(p2, 2) + + p1.add(p2) + check_num_shapes(p1, 2) + + +def test_file_io(): + parts = ReuseableParts() + parts.add(locate_test_file("rect.svg")) + check_num_shapes(parts, 1) + + tmp_dir = mkdtemp() + tmp_file = tmp_dir / "rect.json" + tmp_file.write_text(parts.to_json()) + + assert parts == ReuseableParts.load(tmp_file) From 355b8704ff8b7c417f3bdae00d76a19a96332b29 Mon Sep 17 00:00:00 2001 From: Rod S Date: Thu, 28 Apr 2022 13:57:10 -0700 Subject: [PATCH 2/5] Use NewType instead of a dataclass for Shape --- src/nanoemoji/parts.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/nanoemoji/parts.py b/src/nanoemoji/parts.py index edeae0bc..18927648 100644 --- a/src/nanoemoji/parts.py +++ b/src/nanoemoji/parts.py @@ -20,7 +20,7 @@ from picosvg.svg import SVG from picosvg.svg_reuse import normalize from picosvg.svg_types import SVGPath, SVGShape -from typing import Iterable, List, MutableMapping, Set, Tuple, Union +from typing import Iterable, List, MutableMapping, NewType, Set, Tuple, Union PathSource = Union[SVGShape, Iterable[SVGShape], "ReuseableParts", Path] @@ -44,9 +44,8 @@ def _is_iterable_of(thing, desired_type) -> bool: return True -@dataclasses.dataclass(frozen=True) -class Shape: - d: str # an SVG style path, e.g. the d attribute of +# an SVG style path, e.g. the d attribute of +Shape = NewType("Shape", str) # A set of shapes that normalize to the same path @@ -73,7 +72,7 @@ def _add_norm_path(self, norm: Shape, shape: Shape): def _add(self, shape: Shape): norm = shape if self.reuse_tolerance != -1: - norm = Shape(normalize(SVGPath(d=shape.d), self.reuse_tolerance).d) + norm = Shape(normalize(SVGPath(d=shape), self.reuse_tolerance).d) self._add_norm_path(norm, shape) def add(self, source: PathSource): @@ -98,7 +97,7 @@ def to_json(self): "version": ".".join(str(v) for v in self.version), "reuse_tolerance": self.reuse_tolerance, "shape_sets": [ - {"normalized": n.d, "shapes": [p.d for p in s.shapes]} + {"normalized": n, "shapes": list(s.shapes)} for n, s in self.shape_sets.items() ], } From 0d6d5ed6b019e36bba2862e9f9d1d17934ed0ac4 Mon Sep 17 00:00:00 2001 From: Rod S Date: Thu, 28 Apr 2022 14:09:04 -0700 Subject: [PATCH 3/5] NewType for ShapeSet --- src/nanoemoji/parts.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/nanoemoji/parts.py b/src/nanoemoji/parts.py index 18927648..3ae3a162 100644 --- a/src/nanoemoji/parts.py +++ b/src/nanoemoji/parts.py @@ -48,31 +48,31 @@ def _is_iterable_of(thing, desired_type) -> bool: Shape = NewType("Shape", str) +# A normalized SVG style path +NormalizedShape = NewType("NormalizedShape", str) + + # A set of shapes that normalize to the same path -# Only the d attribute of the SVGPath is populated -@dataclasses.dataclass -class ShapeSet: - normalized: Shape - shapes: Set[Shape] = dataclasses.field(default_factory=set) +ShapeSet = NewType("ShapeSet", Set[Shape]) @dataclasses.dataclass class ReuseableParts: version: Tuple[int, int, int] = (1, 0, 0) reuse_tolerance: float = dataclasses.field(default_factory=_default_tolerence) - shape_sets: MutableMapping[Shape, ShapeSet] = dataclasses.field( + shape_sets: MutableMapping[NormalizedShape, ShapeSet] = dataclasses.field( default_factory=dict ) - def _add_norm_path(self, norm: Shape, shape: Shape): + def _add_norm_path(self, norm: NormalizedShape, shape: Shape): if norm not in self.shape_sets: - self.shape_sets[norm] = ShapeSet(norm) - self.shape_sets[norm].shapes.add(shape) + self.shape_sets[norm] = ShapeSet(set()) + self.shape_sets[norm].add(shape) def _add(self, shape: Shape): - norm = shape + norm = NormalizedShape(shape) if self.reuse_tolerance != -1: - norm = Shape(normalize(SVGPath(d=shape), self.reuse_tolerance).d) + norm = NormalizedShape(normalize(SVGPath(d=shape), self.reuse_tolerance).d) self._add_norm_path(norm, shape) def add(self, source: PathSource): @@ -80,9 +80,9 @@ def add(self, source: PathSource): source = ReuseableParts.load(source) if isinstance(source, ReuseableParts): - for shape_set in source.shape_sets.values(): - for shape in shape_set.shapes: - self._add_norm_path(shape_set.normalized, shape) + for normalized, shape_set in source.shape_sets.items(): + for shape in shape_set: + self._add_norm_path(normalized, shape) else: if not _is_iterable_of(source, SVGShape): source = (source,) @@ -97,8 +97,7 @@ def to_json(self): "version": ".".join(str(v) for v in self.version), "reuse_tolerance": self.reuse_tolerance, "shape_sets": [ - {"normalized": n, "shapes": list(s.shapes)} - for n, s in self.shape_sets.items() + {"normalized": n, "shapes": list(s)} for n, s in self.shape_sets.items() ], } return json.dumps(json_dict, indent=2) @@ -117,11 +116,11 @@ def fromstring(cls, string) -> "ReuseableParts": assert parts.version == (1, 0, 0), f"Bad version {parts.version}" parts.reuse_tolerance = float(json_dict.pop("reuse_tolerance")) for shape_set_json in json_dict.pop("shape_sets"): - norm = Shape(str(shape_set_json.pop("normalized"))) - shapes = {Shape(s) for s in shape_set_json.pop("shapes")} + norm = NormalizedShape(shape_set_json.pop("normalized")) + shapes = ShapeSet({Shape(s) for s in shape_set_json.pop("shapes")}) if shape_set_json: raise ValueError(f"Unconsumed input {shape_set_json}") - parts.shape_sets[norm] = ShapeSet(norm, shapes) + parts.shape_sets[norm] = shapes if json_dict: raise ValueError(f"Unconsumed input {json_dict}") From 3743e3f61d8141616baa947e34506094c7a86bb3 Mon Sep 17 00:00:00 2001 From: Rod S Date: Thu, 28 Apr 2022 14:13:34 -0700 Subject: [PATCH 4/5] Reduce flexibility of ReusableParts::load --- src/nanoemoji/parts.py | 5 +---- tests/parts_test.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/nanoemoji/parts.py b/src/nanoemoji/parts.py index 3ae3a162..a800e217 100644 --- a/src/nanoemoji/parts.py +++ b/src/nanoemoji/parts.py @@ -23,7 +23,7 @@ from typing import Iterable, List, MutableMapping, NewType, Set, Tuple, Union -PathSource = Union[SVGShape, Iterable[SVGShape], "ReuseableParts", Path] +PathSource = Union[SVGShape, Iterable[SVGShape], "ReuseableParts"] @lru_cache(maxsize=1) @@ -76,9 +76,6 @@ def _add(self, shape: Shape): self._add_norm_path(norm, shape) def add(self, source: PathSource): - if isinstance(source, Path): - source = ReuseableParts.load(source) - if isinstance(source, ReuseableParts): for normalized, shape_set in source.shape_sets.items(): for shape in shape_set: diff --git a/tests/parts_test.py b/tests/parts_test.py index 324ff45b..fa379390 100644 --- a/tests/parts_test.py +++ b/tests/parts_test.py @@ -78,7 +78,7 @@ def test_merge(): def test_file_io(): parts = ReuseableParts() - parts.add(locate_test_file("rect.svg")) + parts.add(ReuseableParts.load(locate_test_file("rect.svg"))) check_num_shapes(parts, 1) tmp_dir = mkdtemp() From 9dff1e9501b62026525a3f72382efdc54c565d7a Mon Sep 17 00:00:00 2001 From: Rod S Date: Thu, 28 Apr 2022 19:43:30 -0700 Subject: [PATCH 5/5] Don't reinvent wheels unless the new one is particularly awesome --- src/nanoemoji/parts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nanoemoji/parts.py b/src/nanoemoji/parts.py index a800e217..12f15561 100644 --- a/src/nanoemoji/parts.py +++ b/src/nanoemoji/parts.py @@ -105,8 +105,8 @@ def fromstring(cls, string) -> "ReuseableParts": parts = cls() if first == "<": svg = SVG.fromstring(string).topicosvg() - for path in svg.xpath("//svg:path"): - parts.add(SVGPath(d=path.attrib["d"])) + for shape in svg.shapes(): + parts.add(SVGPath(d=shape.as_path().d)) elif first == "{": json_dict = json.loads(string) parts.version = tuple(int(v) for v in json_dict.pop("version").split("."))