Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Support patternProperties in JSON schema helpers #1197

Merged
merged 7 commits into from
Nov 16, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,13 @@ repos:
cookiecutter/.*/meltano.yml
)$
- id: end-of-file-fixer
exclude: (cookiecutter/.*|docs/.*|samples/.*\.json)
exclude: |
(?x)^(
cookiecutter/.*|
docs/.*|
samples/.*\.json|
tests/snapshots/.*/.*\.json
)$
- id: trailing-whitespace
exclude: |
(?x)^(
Expand Down
12 changes: 12 additions & 0 deletions docs/CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,18 @@ def test_windows_only():

Supported platform markers are `windows`, `darwin`, and `linux`.

### Snapshot Testing

We use [pytest-snapshot](https://pypi.org/project/pytest-snapshot/) for snapshot testing.
To update snapshots, run:

```bash
nox -rs update_snapshots
```

This will run all tests with the `snapshot` marker and update any snapshots that have changed.
Commit the updated snapshots to your branch if they are expected to change.

## Testing Updates to Docs

Documentation runs on Sphinx, using ReadtheDocs style template, and hosting from
Expand Down
45 changes: 29 additions & 16 deletions noxfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,23 @@
"tests",
"doctest",
)
test_dependencies = [
"coverage[toml]",
"pytest",
"pytest-snapshot",
"freezegun",
"pandas",
"requests-mock",
# Cookiecutter tests
"black",
"cookiecutter",
"PyYAML",
"darglint",
"flake8",
"flake8-annotations",
"flake8-docstrings",
"mypy",
]


@session(python=python_versions)
Expand All @@ -53,22 +70,8 @@ def mypy(session: Session) -> None:
def tests(session: Session) -> None:
"""Execute pytest tests and compute coverage."""
session.install(".")
session.install(
"coverage[toml]",
"pytest",
"freezegun",
"pandas",
"requests-mock",
# Cookiecutter tests
"black",
"cookiecutter",
"PyYAML",
"darglint",
"flake8",
"flake8-annotations",
"flake8-docstrings",
"mypy",
)
session.install(*test_dependencies)

# temp fix until pyarrow is supported on python 3.11
if session.python != "3.11":
session.install(
Expand All @@ -91,6 +94,16 @@ def tests(session: Session) -> None:
session.notify("coverage", posargs=[])


@session(python=main_python_version)
def update_snapshots(session: Session) -> None:
"""Update pytest snapshots."""
args = session.posargs or ["-m", "snapshot"]

session.install(".")
session.install(*test_dependencies)
session.run("pytest", "--snapshot-update", *args)


@session(python=python_versions)
def doctest(session: Session) -> None:
"""Run examples with xdoctest."""
Expand Down
17 changes: 16 additions & 1 deletion poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,9 @@ flake8 = "^3.9.0"
flake8-annotations = "^2.9.1"
flake8-docstrings = "^1.6.0"

[tool.poetry.group.dev.dependencies]
pytest-snapshot = "^0.9.0"

[tool.black]
exclude = ".*simpleeval.*"

Expand All @@ -125,6 +128,7 @@ addopts = '-vvv --ignore=singer_sdk/helpers/_simpleeval.py -m "not external"'
markers = [
"external: Tests relying on external resources",
"windows: Tests that only run on Windows",
"snapshot: Tests that use pytest-snapshot",
]

[tool.commitizen]
Expand Down
19 changes: 19 additions & 0 deletions singer_sdk/_singerlib/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,22 @@
write_message,
)
from singer_sdk._singerlib.schema import Schema, resolve_schema_references

__all__ = [
"Catalog",
"CatalogEntry",
"Metadata",
"MetadataMapping",
"SelectionMask",
"StreamMetadata",
"ActivateVersionMessage",
"Message",
"RecordMessage",
"SchemaMessage",
"SingerMessageType",
"StateMessage",
"exclude_null_dict",
"write_message",
"Schema",
"resolve_schema_references",
]
21 changes: 21 additions & 0 deletions singer_sdk/typing.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@

from __future__ import annotations

import json
import sys
from typing import Any, Generic, Mapping, TypeVar, Union, cast

Expand Down Expand Up @@ -449,16 +450,20 @@ def __init__(
self,
*properties: Property,
additional_properties: W | type[W] | None = None,
pattern_properties: Mapping[str, W | type[W]] | None = None,
) -> None:
"""Initialize ObjectType from its list of properties.

Args:
properties: Zero or more attributes for this JSON object.
additional_properties: A schema to match against unnamed properties in
this object.
pattern_properties: A dictionary of regex patterns to match against
property names, and the schema to match against the values.
"""
self.wrapped: list[Property] = list(properties)
self.additional_properties = additional_properties
self.pattern_properties = pattern_properties

@property
def type_dict(self) -> dict: # type: ignore # OK: @classproperty vs @property
Expand All @@ -481,8 +486,24 @@ def type_dict(self) -> dict: # type: ignore # OK: @classproperty vs @property
if self.additional_properties:
result["additionalProperties"] = self.additional_properties.type_dict

if self.pattern_properties:
result["patternProperties"] = {
k: v.type_dict for k, v in self.pattern_properties.items()
}

return result

def to_json(self, **kwargs: Any) -> str:
"""Return a JSON string representation of the object.

Args:
**kwargs: Additional keyword arguments to pass to `json.dumps`.

Returns:
A JSON string.
"""
return json.dumps(self.type_dict, **kwargs)


class CustomType(JSONTypeHelper):
"""Accepts an arbitrary JSON Schema dictionary."""
Expand Down
6 changes: 6 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,9 @@ def outdir() -> str:

yield name
shutil.rmtree(name)


@pytest.fixture(scope="session")
def snapshot_dir() -> pathlib.Path:
"""Return the path to the snapshot directory."""
return pathlib.Path("tests/snapshots/")
Loading