From a8988145e2179e3300808656d59d10fe0dfade0a Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Fri, 3 Feb 2023 11:58:38 +0100 Subject: [PATCH 1/3] POC: Use securesystemslib base+json de/serializer securesystemslib now provides a generic de/serialization module, modelled after `tuf.api.serialization`: https://github.com/secure-systems-lab/securesystemslib/pull/487 This module provides abstract `BaseSerializer` and `BaseDeserializer` classes, which define serialize and deserialize methods respectively. They are similar to TUF's `MetadataSerializer` and `MetadataDeserializer`, but without restricting argument or return value type to `Metadata`. The module further provides `JSONSerializer` and `JSONDeserializer` classes, which implement serialize and deserialize methods respectively. They are similar to TUF's existing `JSONSerializer` and `JSONDeserializer`, but without `Metadata`-specific code. That is, `JSONSerializer` takes any object that implements `to_dict` (i.e. `JSONSerializable`), and `JSONDeserializer` just deserializes json bytes to dict, and leaves object specific deserialization (e.g. `Metadata.from_dict`) to a subclass implementation. This patch uses securesystemslib's base de/serializer classes in metadata method signatures (type-aliased for backwards compatibility) and factors out the json-specific de/serialization to securesystemslib. **IMPORTANT NOTE:** This is a POC to demonstrate how the `securesystemslib.serialization` can be used in general. It may not actually be that useful for python-tuf, given that python-tuf has a working serialization subpackage and below caveat. It can, however, be re-used in a couple of other classes, which don't have a proper de/serialization implementation yet. Generic DSSE - secureystemslib.Envelope.from_file - secureystemslib.Envelope.get_payload DSSE for in-toto payloads - in_toto.Envelope.from_file - in_toto.Envelope.get_payload in-toto Metadata pendant - in_toto.Metablock.from_file in-toto Metablock and DSSE abstraction - in_toto.AnyMetadata.from_file - in_toto.AnyMetadata.get_payload **CAVEAT:** Using securesystemslib's `Base[Des|S]erializer` instead of `Metadata[Des|S]erializer` (and `SignedSerializer`) weakens the interface, because it is more generic about the argument/return value type. Stricter typing and also raising the right TUF de/serialization errors's needs to be implemented by a tuf-specific de/serializer, see e.g. `tuf.api.serialization.json`. Signed-off-by: Lukas Puehringer --- tuf/api/metadata.py | 3 ++- tuf/api/serialization/__init__.py | 37 +++++-------------------------- tuf/api/serialization/json.py | 34 ++++++++++++---------------- 3 files changed, 22 insertions(+), 52 deletions(-) diff --git a/tuf/api/metadata.py b/tuf/api/metadata.py index 2f40499ce8..46fdaae52c 100644 --- a/tuf/api/metadata.py +++ b/tuf/api/metadata.py @@ -53,6 +53,7 @@ from securesystemslib import exceptions as sslib_exceptions from securesystemslib import hash as sslib_hash +from securesystemslib.serialization import JSONSerializable from securesystemslib.signer import Key, Signature, Signer from securesystemslib.storage import FilesystemBackend, StorageBackendInterface from securesystemslib.util import persist_temp_file @@ -82,7 +83,7 @@ T = TypeVar("T", "Root", "Timestamp", "Snapshot", "Targets") -class Metadata(Generic[T]): +class Metadata(Generic[T], JSONSerializable): """A container for signed TUF metadata. Provides methods to convert to and from dictionary, read and write to and diff --git a/tuf/api/serialization/__init__.py b/tuf/api/serialization/__init__.py index 7aef8b9884..cb2244ae8b 100644 --- a/tuf/api/serialization/__init__.py +++ b/tuf/api/serialization/__init__.py @@ -15,13 +15,15 @@ """ import abc -from typing import TYPE_CHECKING +from typing import TypeAlias + +from securesystemslib.serialization import BaseDeserializer, BaseSerializer from tuf.api.exceptions import RepositoryError -if TYPE_CHECKING: - # pylint: disable=cyclic-import - from tuf.api.metadata import Metadata, Signed +MetadataSerializer: TypeAlias = BaseSerializer +MetadataDeserializer: TypeAlias = BaseDeserializer +SignedSerializer: TypeAlias = BaseSerializer class SerializationError(RepositoryError): @@ -30,30 +32,3 @@ class SerializationError(RepositoryError): class DeserializationError(RepositoryError): """Error during deserialization.""" - - -class MetadataDeserializer(metaclass=abc.ABCMeta): - """Abstract base class for deserialization of Metadata objects.""" - - @abc.abstractmethod - def deserialize(self, raw_data: bytes) -> "Metadata": - """Deserialize bytes to Metadata object.""" - raise NotImplementedError - - -class MetadataSerializer(metaclass=abc.ABCMeta): - """Abstract base class for serialization of Metadata objects.""" - - @abc.abstractmethod - def serialize(self, metadata_obj: "Metadata") -> bytes: - """Serialize Metadata object to bytes.""" - raise NotImplementedError - - -class SignedSerializer(metaclass=abc.ABCMeta): - """Abstract base class for serialization of Signed objects.""" - - @abc.abstractmethod - def serialize(self, signed_obj: "Signed") -> bytes: - """Serialize Signed object to bytes.""" - raise NotImplementedError diff --git a/tuf/api/serialization/json.py b/tuf/api/serialization/json.py index 3355511a66..43333b5732 100644 --- a/tuf/api/serialization/json.py +++ b/tuf/api/serialization/json.py @@ -7,11 +7,13 @@ metadata to the OLPC Canonical JSON format for signature generation and verification. """ - -import json from typing import Optional from securesystemslib.formats import encode_canonical +from securesystemslib.serialization import ( + JSONDeserializer as BaseJSONDeserializer, +) +from securesystemslib.serialization import JSONSerializer as BaseJSONSerializer # pylint: disable=cyclic-import # ... to allow de/serializing Metadata and Signed objects here, while also @@ -20,20 +22,19 @@ from tuf.api.metadata import Metadata, Signed from tuf.api.serialization import ( DeserializationError, - MetadataDeserializer, - MetadataSerializer, SerializationError, SignedSerializer, ) -class JSONDeserializer(MetadataDeserializer): +class JSONDeserializer(BaseJSONDeserializer): """Provides JSON to Metadata deserialize method.""" def deserialize(self, raw_data: bytes) -> Metadata: """Deserialize utf-8 encoded JSON bytes into Metadata object.""" + try: - json_dict = json.loads(raw_data.decode("utf-8")) + json_dict = super().deserialize(raw_data) metadata_obj = Metadata.from_dict(json_dict) except Exception as e: @@ -42,7 +43,7 @@ def deserialize(self, raw_data: bytes) -> Metadata: return metadata_obj -class JSONSerializer(MetadataSerializer): +class JSONSerializer(BaseJSONSerializer): """Provides Metadata to JSON serialize method. Args: @@ -55,26 +56,19 @@ class JSONSerializer(MetadataSerializer): """ def __init__(self, compact: bool = False, validate: Optional[bool] = False): - self.compact = compact + super().__init__(compact) self.validate = validate - def serialize(self, metadata_obj: Metadata) -> bytes: + def serialize(self, obj: Metadata) -> bytes: """Serialize Metadata object into utf-8 encoded JSON bytes.""" try: - indent = None if self.compact else 1 - separators = (",", ":") if self.compact else (",", ": ") - json_bytes = json.dumps( - metadata_obj.to_dict(), - indent=indent, - separators=separators, - sort_keys=True, - ).encode("utf-8") + json_bytes = BaseJSONSerializer.serialize(self, obj) if self.validate: try: new_md_obj = JSONDeserializer().deserialize(json_bytes) - if metadata_obj != new_md_obj: + if obj != new_md_obj: raise ValueError( "Metadata changes if you serialize and deserialize." ) @@ -90,12 +84,12 @@ def serialize(self, metadata_obj: Metadata) -> bytes: class CanonicalJSONSerializer(SignedSerializer): """Provides Signed to OLPC Canonical JSON serialize method.""" - def serialize(self, signed_obj: Signed) -> bytes: + def serialize(self, obj: Signed) -> bytes: """Serialize Signed object into utf-8 encoded OLPC Canonical JSON bytes. """ try: - signed_dict = signed_obj.to_dict() + signed_dict = obj.to_dict() canonical_bytes = encode_canonical(signed_dict).encode("utf-8") except Exception as e: From 351da27307bac896f1a403b12674e477631cdb82 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Thu, 22 Dec 2022 12:33:13 +0100 Subject: [PATCH 2/3] POC: use securesystemslib SerializationMixin securesystemslib provides a mixin with generic methods to read and write to and from bytes and files, which are reusable by TUF's Metadata class, but also by the generic DSSE Envelope in securesystemslib, and the in-toto DSSE Envelope and in-toto Metablock. This patch demonstrates how the mixin can be used instead of TUF's own `Metadata.[to|from]_[bytes|file]` methods. **CAVEAT:** The default de/serializers can no longer be hardcoded into the method (as they were in `Metadata.[to|from]_[bytes|file]`). To still support defaults, users of the mixin must implement `_default_serializer` and `_default_deserializer` helper methods. This feels a bit intransparent and over-engineered. Signed-off-by: Lukas Puehringer --- tuf/api/metadata.py | 140 +++++--------------------------------------- 1 file changed, 14 insertions(+), 126 deletions(-) diff --git a/tuf/api/metadata.py b/tuf/api/metadata.py index 46fdaae52c..f2e525aed1 100644 --- a/tuf/api/metadata.py +++ b/tuf/api/metadata.py @@ -32,7 +32,6 @@ import fnmatch import io import logging -import tempfile from datetime import datetime from typing import ( IO, @@ -55,13 +54,12 @@ from securesystemslib import hash as sslib_hash from securesystemslib.serialization import JSONSerializable from securesystemslib.signer import Key, Signature, Signer -from securesystemslib.storage import FilesystemBackend, StorageBackendInterface -from securesystemslib.util import persist_temp_file from tuf.api.exceptions import LengthOrHashMismatchError, UnsignedMetadataError from tuf.api.serialization import ( MetadataDeserializer, MetadataSerializer, + SerializationMixin, SignedSerializer, ) @@ -83,7 +81,7 @@ T = TypeVar("T", "Root", "Timestamp", "Snapshot", "Targets") -class Metadata(Generic[T], JSONSerializable): +class Metadata(Generic[T], JSONSerializable, SerializationMixin): """A container for signed TUF metadata. Provides methods to convert to and from dictionary, read and write to and @@ -201,97 +199,21 @@ def from_dict(cls, metadata: Dict[str, Any]) -> "Metadata[T]": unrecognized_fields=metadata, ) - @classmethod - def from_file( - cls, - filename: str, - deserializer: Optional[MetadataDeserializer] = None, - storage_backend: Optional[StorageBackendInterface] = None, - ) -> "Metadata[T]": - """Load TUF metadata from file storage. - - Args: - filename: Path to read the file from. - deserializer: ``MetadataDeserializer`` subclass instance that - implements the desired wireline format deserialization. Per - default a ``JSONDeserializer`` is used. - storage_backend: Object that implements - ``securesystemslib.storage.StorageBackendInterface``. - Default is ``FilesystemBackend`` (i.e. a local file). - Raises: - StorageError: The file cannot be read. - tuf.api.serialization.DeserializationError: - The file cannot be deserialized. - - Returns: - TUF ``Metadata`` object. - """ - - if storage_backend is None: - storage_backend = FilesystemBackend() - - with storage_backend.get(filename) as file_obj: - return cls.from_bytes(file_obj.read(), deserializer) - - @classmethod - def from_bytes( - cls, - data: bytes, - deserializer: Optional[MetadataDeserializer] = None, - ) -> "Metadata[T]": - """Load TUF metadata from raw data. - - Args: - data: Metadata content. - deserializer: ``MetadataDeserializer`` implementation to use. - Default is ``JSONDeserializer``. - - Raises: - tuf.api.serialization.DeserializationError: - The file cannot be deserialized. - - Returns: - TUF ``Metadata`` object. - """ - - if deserializer is None: - # Use local scope import to avoid circular import errors - # pylint: disable=import-outside-toplevel - from tuf.api.serialization.json import JSONDeserializer - - deserializer = JSONDeserializer() - - return deserializer.deserialize(data) - - def to_bytes( - self, serializer: Optional[MetadataSerializer] = None - ) -> bytes: - """Return the serialized TUF file format as bytes. - - Note that if bytes are first deserialized into ``Metadata`` and then - serialized with ``to_bytes()``, the two are not required to be - identical even though the signatures are guaranteed to stay valid. If - byte-for-byte equivalence is required (which is the case when content - hashes are used in other metadata), the original content should be used - instead of re-serializing. - - Args: - serializer: ``MetadataSerializer`` instance that implements the - desired serialization format. Default is ``JSONSerializer``. - - Raises: - tuf.api.serialization.SerializationError: - The metadata object cannot be serialized. - """ + @staticmethod + def _default_deserializer() -> MetadataDeserializer: + """Default deserializer for ``Metadata.from_{bytes, file}``.""" + # pylint: disable=import-outside-toplevel + from tuf.api.serialization.json import JSONDeserializer - if serializer is None: - # Use local scope import to avoid circular import errors - # pylint: disable=import-outside-toplevel - from tuf.api.serialization.json import JSONSerializer + return JSONDeserializer() - serializer = JSONSerializer(compact=True) + @staticmethod + def _default_serializer() -> MetadataSerializer: + """Default serializer for ``Metadata.to_{bytes, file}``.""" + # pylint: disable=import-outside-toplevel + from tuf.api.serialization.json import JSONSerializer - return serializer.serialize(self) + return JSONSerializer(compact=True) def to_dict(self) -> Dict[str, Any]: """Return the dict representation of self.""" @@ -304,40 +226,6 @@ def to_dict(self) -> Dict[str, Any]: **self.unrecognized_fields, } - def to_file( - self, - filename: str, - serializer: Optional[MetadataSerializer] = None, - storage_backend: Optional[StorageBackendInterface] = None, - ) -> None: - """Write TUF metadata to file storage. - - Note that if a file is first deserialized into ``Metadata`` and then - serialized with ``to_file()``, the two files are not required to be - identical even though the signatures are guaranteed to stay valid. If - byte-for-byte equivalence is required (which is the case when file - hashes are used in other metadata), the original file should be used - instead of re-serializing. - - Args: - filename: Path to write the file to. - serializer: ``MetadataSerializer`` instance that implements the - desired serialization format. Default is ``JSONSerializer``. - storage_backend: ``StorageBackendInterface`` implementation. Default - is ``FilesystemBackend`` (i.e. a local file). - - Raises: - tuf.api.serialization.SerializationError: - The metadata object cannot be serialized. - StorageError: The file cannot be written. - """ - - bytes_data = self.to_bytes(serializer) - - with tempfile.TemporaryFile() as temp_file: - temp_file.write(bytes_data) - persist_temp_file(temp_file, filename, storage_backend) - # Signatures. def sign( self, From 2e1e2f187ebff49eafb2145c3b9d8313255d419f Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Fri, 3 Feb 2023 12:05:51 +0100 Subject: [PATCH 3/3] TMP: pin secure-systems-lab/securesystemslib#487 Signed-off-by: Lukas Puehringer --- pyproject.toml | 5 ++++- requirements-pinned.txt | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 72fa2a7738..5091c22100 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ classifiers = [ ] dependencies = [ "requests>=2.19.1", - "securesystemslib>=0.26.0", + "securesystemslib @ git+https://github.com/PradyumnaKrishna/securesystemslib@4c6be46" ] dynamic = ["version"] @@ -57,6 +57,9 @@ Source = "https://github.com/theupdateframework/python-tuf" [tool.hatch.version] path = "tuf/__init__.py" +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.build.targets.sdist] include = [ "/docs", diff --git a/requirements-pinned.txt b/requirements-pinned.txt index 190a020922..f4c07db03b 100644 --- a/requirements-pinned.txt +++ b/requirements-pinned.txt @@ -6,5 +6,5 @@ idna==3.4 # via requests pycparser==2.21 # via cffi pynacl==1.5.0 # via securesystemslib requests==2.28.1 -securesystemslib[crypto,pynacl]==0.26.0 +git+https://github.com/PradyumnaKrishna/securesystemslib@4c6be46 urllib3==1.26.14 # via requests