From ab7b15a0f684e3fcb7ec399a70b4a3f94d989439 Mon Sep 17 00:00:00 2001 From: Alexis Date: Thu, 18 Jul 2024 10:34:41 +0200 Subject: [PATCH 01/29] Store and retrieve attestations --- tests/common/db/packaging.py | 13 +++ tests/unit/forklift/test_legacy.py | 75 +++++++++++++++ tests/unit/packaging/test_utils.py | 40 +++++++- warehouse/attestations/__init__.py | 11 +++ warehouse/attestations/_core.py | 101 +++++++++++++++++++++ warehouse/forklift/legacy.py | 50 +++++++++- warehouse/packaging/models.py | 35 +++++++ warehouse/packaging/utils.py | 10 +- warehouse/templates/api/simple/detail.html | 2 +- 9 files changed, 328 insertions(+), 9 deletions(-) create mode 100644 warehouse/attestations/__init__.py create mode 100644 warehouse/attestations/_core.py diff --git a/tests/common/db/packaging.py b/tests/common/db/packaging.py index 2a12379da170..c67fdf6c9d70 100644 --- a/tests/common/db/packaging.py +++ b/tests/common/db/packaging.py @@ -30,6 +30,7 @@ Release, Role, RoleInvitation, + ReleaseFileAttestation, ) from warehouse.utils import readme @@ -99,6 +100,12 @@ class Meta: uploader = factory.SubFactory(UserFactory) description = factory.SubFactory(DescriptionFactory) +class ReleaseAttestationsFactory(WarehouseFactory): + class Meta: + model = ReleaseFileAttestation + + file = factory.SubFactory("tests.common.db.packaging.FileFactory") + attestation = "fake-attestation" class FileFactory(WarehouseFactory): class Meta: @@ -140,6 +147,12 @@ class Meta: ) ) + attestations = factory.RelatedFactoryList( + ReleaseAttestationsFactory, + factory_related_name="file", + size=1, + ) + class FileEventFactory(WarehouseFactory): class Meta: diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index be706f8c791d..b0953d2613fa 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -22,6 +22,7 @@ import pretend import pytest +from pydantic import TypeAdapter from pypi_attestations import ( Attestation, @@ -58,6 +59,7 @@ ProjectMacaroonWarningAssociation, Release, Role, + ReleaseFileAttestation, ) from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files @@ -3803,6 +3805,79 @@ def failing_verify(_self, _verifier, _policy, _dist): assert resp.status_code == 400 assert resp.status.startswith(expected_msg) + def test_upload_succeeds_upload_attestation(self, monkeypatch, + pyramid_config, + db_request, + metrics, + ): + + project = ProjectFactory.create() + version = "1.0" + publisher = GitHubPublisherFactory.create(projects=[project]) + claims = { + "sha": "somesha", + "repository": f"{publisher.repository_owner}/{publisher.repository_name}", + "workflow": "workflow_name", + } + identity = PublisherTokenContext(publisher, SignedClaims(claims)) + db_request.oidc_publisher = identity.publisher + db_request.oidc_claims = identity.claims + + db_request.db.add(Classifier(classifier="Environment :: Other Environment")) + db_request.db.add(Classifier(classifier="Programming Language :: Python")) + + filename = "{}-{}.tar.gz".format(project.name, "1.0") + attestation = Attestation( + version=1, + verification_material=VerificationMaterial( + certificate="somebase64string", transparency_entries=[dict()] + ), + envelope=Envelope( + statement="somebase64string", + signature="somebase64string", + ), + ) + + pyramid_config.testing_securitypolicy(identity=identity) + db_request.user = None + db_request.user_agent = "warehouse-tests/6.6.6" + db_request.POST = MultiDict( + { + "metadata_version": "1.2", + "name": project.name, + "attestations": f"[{attestation.model_dump_json()}]", + "version": version, + "summary": "This is my summary!", + "filetype": "sdist", + "md5_digest": _TAR_GZ_PKG_MD5, + "content": pretend.stub( + filename=filename, + file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), + type="application/tar", + ), + } + ) + + storage_service = pretend.stub(store=lambda path, filepath, meta: None) + db_request.find_service = lambda svc, name=None, context=None: { + IFileStorage: storage_service, + IMetricsService: metrics, + }.get(svc) + + process_attestations = pretend.call_recorder( + lambda request, artifact_path: [attestation] + ) + + monkeypatch.setattr(legacy, "_process_attestations", process_attestations) + + resp = legacy.file_upload(db_request) + + assert resp.status_code == 200 + + attestations_db = db_request.db.query(ReleaseFileAttestation).join(ReleaseFileAttestation.file).filter(File.filename == filename).all() + assert len(attestations_db) == 1 + assert TypeAdapter(Attestation).validate_json(attestations_db[0].attestation) + @pytest.mark.parametrize( "version, expected_version", [ diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index afa7bd2056fe..5f5f78553a1f 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -15,8 +15,9 @@ import pretend +import warehouse.packaging.models from warehouse.packaging.interfaces import ISimpleStorage -from warehouse.packaging.utils import _simple_detail, render_simple_detail +from warehouse.packaging.utils import _simple_detail, render_simple_detail, store_provenance_object from ...common.db.packaging import FileFactory, ProjectFactory, ReleaseFactory @@ -66,6 +67,43 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): + f".{project.normalized_name}.html" ) +def test_store_provenance_object_succeed(db_request, monkeypatch): + storage_service = pretend.stub( + store=pretend.call_recorder( + lambda path, file_path, *, meta=None: f"https://files/attestations/{path}.provenance" + ) + ) + + db_request.find_service = pretend.call_recorder( + lambda svc, name=None, context=None: { + ISimpleStorage: storage_service, + }.get(svc) + ) + + monkeypatch.setattr(warehouse.packaging.models.File, "publisher_url", "x-fake-publisher-url") + + file = FileFactory.create() + provenance_hash = store_provenance_object(db_request, file) + + assert provenance_hash is not None + + +def test_store_provenance_object_fails_no_attestations(db_request, monkeypatch): + file = FileFactory.create() + file.attestations = None + + provenance_hash = store_provenance_object(db_request, file) + assert provenance_hash is None + + +def test_store_provenance_object_fails_no_publisher_url(db_request, monkeypatch): + file = FileFactory.create() + + monkeypatch.setattr(warehouse.packaging.models.File, "publisher_url", None) + + provenance_hash = store_provenance_object(db_request, file) + assert provenance_hash is None + def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): project = ProjectFactory.create() diff --git a/warehouse/attestations/__init__.py b/warehouse/attestations/__init__.py new file mode 100644 index 000000000000..164f68b09175 --- /dev/null +++ b/warehouse/attestations/__init__.py @@ -0,0 +1,11 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/warehouse/attestations/_core.py b/warehouse/attestations/_core.py new file mode 100644 index 000000000000..c32d50533489 --- /dev/null +++ b/warehouse/attestations/_core.py @@ -0,0 +1,101 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib +import tempfile +from pathlib import Path + +from pydantic import BaseModel, TypeAdapter +from typing import Literal +from pypi_attestations import Attestation + +from warehouse.packaging import ISimpleStorage, File + + +class Publisher(BaseModel): + kind: str + """ + """ + + claims: object + """ + """ + +class AttestationBundle(BaseModel): + publisher: Publisher + """ + """ + + attestations: list[Attestation] + """ + Attestations are returned as an opaque + """ + +class Provenance(BaseModel): + version: Literal[1] + """ + The provenance object's version, set to 1 + """ + + attestation_bundles: list[AttestationBundle] + + + +def get_provenance_digest(request, file: File) -> str | None: + if not file.attestations: + return + + publisher_url = file.publisher_url + if not publisher_url: + return # # TODO(dm) + + provenance_file_path = generate_provenance_file(request, publisher_url, file) + + with Path(provenance_file_path).open("rb") as file_handler: + provenance_digest = hashlib.file_digest(file_handler, "sha256") + + return provenance_digest.hexdigest() + + +def generate_provenance_file(request, publisher_url: str, file: File) -> str: + + storage = request.find_service(ISimpleStorage) + publisher = Publisher( + kind=publisher_url, + claims={} # TODO(dm) + ) + + attestation_bundle = AttestationBundle( + publisher=publisher, + attestations=[ + TypeAdapter(Attestation).validate_json( + Path(release_attestation.attestation_path).read_text() + ) + for release_attestation in file.attestations + ] + ) + + provenance = Provenance( + version=1, + attestation_bundles=[attestation_bundle] + ) + + provenance_file_path = f"{file.path}.provenance" + with tempfile.NamedTemporaryFile() as f: + f.write(provenance.model_dump_json().encode("utf-8")) + f.flush() + + storage.store( + f"{file.path}.provenance", + f.name, + ) + + return provenance_file_path diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 9c290750845f..2eeba49f0194 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -70,6 +70,7 @@ Project, ProjectMacaroonWarningAssociation, Release, + ReleaseFileAttestation, ) from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files from warehouse.rate_limiting.interfaces import RateLimiterException @@ -366,7 +367,7 @@ def _is_duplicate_file(db_session, filename, hashes): return None -def _process_attestations(request, artifact_path: Path): +def _process_attestations(request, artifact_path: Path) -> list[Attestation]: """ Process any attestations included in a file upload request @@ -374,8 +375,7 @@ def _process_attestations(request, artifact_path: Path): artifact. Attestations are only allowed when uploading via a Trusted Publisher, because a Trusted Publisher provides the identity that will be used to verify the attestations. - Currently, only GitHub Actions Trusted Publishers are supported, and - attestations are discarded after verification. + Currently, only GitHub Actions Trusted Publishers are supported. """ metrics = request.find_service(IMetricsService, context=None) @@ -447,6 +447,41 @@ def _process_attestations(request, artifact_path: Path): # Log successful attestation upload metrics.increment("warehouse.upload.attestations.ok") + return attestations + + +def _store_attestations(request, file: File, attestations: list[Attestation]): + """Store the attestations along the release files. + + Attestations are living near the release file, like metadata files. They are named using + their filehash to allow storing more than 1 attestation by file. + + TODO(dm): Validate if the 8 hex chars are enough. + """ + storage = request.find_service(IFileStorage, name="archive") + + release_file_attestations = [] + for attestation in attestations: + + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write(attestation.model_dump_json().encoded("utf-8")) + + attestation_digest = hashlib.file_digest(tmp_file, "sha256").hexdigest() + attestation_path = f"{file.path}.{attestation_digest[:8]}.attestation" + + storage.store( + attestation_path, + tmp_file.name, + ) + + release_file_attestations.append( + ReleaseFileAttestation( + file=file, + attestation_file_sha256_digest=attestation_digest, + ) + ) + + request.db.add_all(release_file_attestations) @view_config( route_name="forklift.legacy.file_upload", @@ -1146,8 +1181,9 @@ def file_upload(request): k: h.hexdigest().lower() for k, h in metadata_file_hashes.items() } + attestations: list[Attestation] | None = None if "attestations" in request.POST: - _process_attestations( + attestations = _process_attestations( request=request, artifact_path=Path(temporary_filename) ) @@ -1247,6 +1283,11 @@ def file_upload(request): }, ) + # If the user provided attestations, store them along the release file + if attestations: + _store_attestations(request, file_, attestations) + # TODO(dm): Add some event? + # Check if the user has any 2FA methods enabled, and if not, email them. if request.user and not request.user.has_two_factor: warnings.append("Two factor authentication is not enabled for your account.") @@ -1304,6 +1345,7 @@ def file_upload(request): "path": file_data.path, "uploaded_via": file_data.uploaded_via, "upload_time": file_data.upload_time, + # TODO(dm): Add something here for attestation upload? } if request.registry.settings.get("warehouse.release_files_table") is not None: request.task(update_bigquery_release_files).delay(dist_metadata) diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index bd18aebd7fbf..9126e4037bef 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -801,6 +801,17 @@ def __table_args__(cls): # noqa comment="If True, the metadata for the file cannot be backfilled.", ) + # PEP 740 attestations + attestations: Mapped[list[ReleaseFileAttestation]] = orm.relationship( + cascade="all, delete-orphan", + lazy="dynamic", + passive_deletes=True, # TODO(dm) check-me + ) + + @property + def publisher_url(self) -> str | None : + return self.Event.additional["publisher_url"].as_string() # type: ignore[attr-defined] + @property def uploaded_via_trusted_publisher(self) -> bool: """Return True if the file was uploaded via a trusted publisher.""" @@ -959,3 +970,27 @@ class ProjectMacaroonWarningAssociation(db.Model): ForeignKey("projects.id", onupdate="CASCADE", ondelete="CASCADE"), primary_key=True, ) + +class ReleaseFileAttestation(db.Model): + """ + Association table between Release Files and Attestations. + + Attestations are stored as opaque blob because their implementation details are handled by the pypi_attestation package. + They are linked to release files as a one-to-many relationship. + """ + __tablename__ = "release_files_attestation" + + file_id: Mapped[UUID] = mapped_column( + ForeignKey("release_files.id", onupdate="CASCADE", ondelete="CASCADE"), + ) + file: Mapped[File] = orm.relationship(back_populates="attestations") + + attestation_file_sha256_digest: Mapped[str] = mapped_column(CITEXT) + + @hybrid_property + def attestation_path(self): + return self.file.path + self.attestation_file_sha256_digest[:8] + ".attestation" + + @attestation_path.expression # type: ignore + def attestation_path(self): + return func.concat(func.concat(self.file.path, self.attestation_file_sha256_digest[:8], ".attestation")) diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index a730a9cb4c6c..e2b927054f94 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -20,9 +20,11 @@ from sqlalchemy.orm import joinedload from warehouse.packaging.interfaces import ISimpleStorage -from warehouse.packaging.models import File, Project, Release +from warehouse.packaging.models import File, Project, Release, ReleaseFileAttestation +from warehouse.attestations._core import get_provenance_digest + +API_VERSION = "1.2" -API_VERSION = "1.1" def _simple_index(request, serial): @@ -43,8 +45,9 @@ def _simple_detail(project, request): # Get all of the files for this project. files = sorted( request.db.query(File) - .options(joinedload(File.release)) + .options(joinedload(File.release), joinedload(File.attestations)) .join(Release) + .join(ReleaseFileAttestation) .filter(Release.project == project) .all(), key=lambda f: (packaging_legacy.version.parse(f.release.version), f.filename), @@ -86,6 +89,7 @@ def _simple_detail(project, request): if file.metadata_file_sha256_digest else False ), + "provenance": get_provenance_digest(request, file), } for file in files ], diff --git a/warehouse/templates/api/simple/detail.html b/warehouse/templates/api/simple/detail.html index 24b0042c5863..05e0221a5612 100644 --- a/warehouse/templates/api/simple/detail.html +++ b/warehouse/templates/api/simple/detail.html @@ -20,7 +20,7 @@

Links for {{ name }}

{% for file in files -%} - {{ file.filename }}
+ {{ file.filename }}
{% endfor -%} From cf1359f0461361fb8c58a7c8fd79883e1e67b902 Mon Sep 17 00:00:00 2001 From: Alexis Date: Thu, 18 Jul 2024 11:24:47 +0200 Subject: [PATCH 02/29] Continue to work. --- tests/common/db/packaging.py | 4 +- tests/unit/forklift/test_legacy.py | 17 +- tests/unit/packaging/test_utils.py | 12 +- warehouse/attestations/_core.py | 25 ++- warehouse/attestations/models.py | 81 ++++++++++ warehouse/forklift/legacy.py | 245 ++++++++++++++--------------- warehouse/packaging/models.py | 27 +--- warehouse/packaging/utils.py | 63 ++++---- 8 files changed, 276 insertions(+), 198 deletions(-) create mode 100644 warehouse/attestations/models.py diff --git a/tests/common/db/packaging.py b/tests/common/db/packaging.py index c67fdf6c9d70..01421b0efe1a 100644 --- a/tests/common/db/packaging.py +++ b/tests/common/db/packaging.py @@ -28,9 +28,9 @@ ProhibitedProjectName, Project, Release, + ReleaseFileAttestation, Role, RoleInvitation, - ReleaseFileAttestation, ) from warehouse.utils import readme @@ -100,6 +100,7 @@ class Meta: uploader = factory.SubFactory(UserFactory) description = factory.SubFactory(DescriptionFactory) + class ReleaseAttestationsFactory(WarehouseFactory): class Meta: model = ReleaseFileAttestation @@ -107,6 +108,7 @@ class Meta: file = factory.SubFactory("tests.common.db.packaging.FileFactory") attestation = "fake-attestation" + class FileFactory(WarehouseFactory): class Meta: model = File diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 6141d05759e4..a620864c5464 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -22,8 +22,8 @@ import pretend import pytest -from pydantic import TypeAdapter +from pydantic import TypeAdapter from pypi_attestations import ( Attestation, Distribution, @@ -59,8 +59,8 @@ Project, ProjectMacaroonWarningAssociation, Release, - Role, ReleaseFileAttestation, + Role, ) from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files @@ -3810,11 +3810,13 @@ def failing_verify(_self, _verifier, _policy, _dist): assert resp.status_code == 400 assert resp.status.startswith(expected_msg) - def test_upload_succeeds_upload_attestation(self, monkeypatch, + def test_upload_succeeds_upload_attestation( + self, + monkeypatch, pyramid_config, db_request, metrics, - ): + ): project = ProjectFactory.create() version = "1.0" @@ -3879,7 +3881,12 @@ def test_upload_succeeds_upload_attestation(self, monkeypatch, assert resp.status_code == 200 - attestations_db = db_request.db.query(ReleaseFileAttestation).join(ReleaseFileAttestation.file).filter(File.filename == filename).all() + attestations_db = ( + db_request.db.query(ReleaseFileAttestation) + .join(ReleaseFileAttestation.file) + .filter(File.filename == filename) + .all() + ) assert len(attestations_db) == 1 assert TypeAdapter(Attestation).validate_json(attestations_db[0].attestation) diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index 5f5f78553a1f..cbe9458b3c93 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -16,8 +16,13 @@ import pretend import warehouse.packaging.models + from warehouse.packaging.interfaces import ISimpleStorage -from warehouse.packaging.utils import _simple_detail, render_simple_detail, store_provenance_object +from warehouse.packaging.utils import ( + _simple_detail, + render_simple_detail, + store_provenance_object, +) from ...common.db.packaging import FileFactory, ProjectFactory, ReleaseFactory @@ -67,6 +72,7 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): + f".{project.normalized_name}.html" ) + def test_store_provenance_object_succeed(db_request, monkeypatch): storage_service = pretend.stub( store=pretend.call_recorder( @@ -80,7 +86,9 @@ def test_store_provenance_object_succeed(db_request, monkeypatch): }.get(svc) ) - monkeypatch.setattr(warehouse.packaging.models.File, "publisher_url", "x-fake-publisher-url") + monkeypatch.setattr( + warehouse.packaging.models.File, "publisher_url", "x-fake-publisher-url" + ) file = FileFactory.create() provenance_hash = store_provenance_object(db_request, file) diff --git a/warehouse/attestations/_core.py b/warehouse/attestations/_core.py index c32d50533489..b52ddfb27560 100644 --- a/warehouse/attestations/_core.py +++ b/warehouse/attestations/_core.py @@ -11,13 +11,16 @@ # limitations under the License. import hashlib import tempfile + from pathlib import Path +from typing import Literal from pydantic import BaseModel, TypeAdapter -from typing import Literal -from pypi_attestations import Attestation +from pypi_attestations import ( + Attestation, +) -from warehouse.packaging import ISimpleStorage, File +from warehouse.packaging import File, ISimpleStorage class Publisher(BaseModel): @@ -29,6 +32,7 @@ class Publisher(BaseModel): """ """ + class AttestationBundle(BaseModel): publisher: Publisher """ @@ -39,6 +43,7 @@ class AttestationBundle(BaseModel): Attestations are returned as an opaque """ + class Provenance(BaseModel): version: Literal[1] """ @@ -48,7 +53,6 @@ class Provenance(BaseModel): attestation_bundles: list[AttestationBundle] - def get_provenance_digest(request, file: File) -> str | None: if not file.attestations: return @@ -68,10 +72,7 @@ def get_provenance_digest(request, file: File) -> str | None: def generate_provenance_file(request, publisher_url: str, file: File) -> str: storage = request.find_service(ISimpleStorage) - publisher = Publisher( - kind=publisher_url, - claims={} # TODO(dm) - ) + publisher = Publisher(kind=publisher_url, claims={}) # TODO(dm) attestation_bundle = AttestationBundle( publisher=publisher, @@ -80,13 +81,10 @@ def generate_provenance_file(request, publisher_url: str, file: File) -> str: Path(release_attestation.attestation_path).read_text() ) for release_attestation in file.attestations - ] + ], ) - provenance = Provenance( - version=1, - attestation_bundles=[attestation_bundle] - ) + provenance = Provenance(version=1, attestation_bundles=[attestation_bundle]) provenance_file_path = f"{file.path}.provenance" with tempfile.NamedTemporaryFile() as f: @@ -99,3 +97,4 @@ def generate_provenance_file(request, publisher_url: str, file: File) -> str: ) return provenance_file_path + diff --git a/warehouse/attestations/models.py b/warehouse/attestations/models.py new file mode 100644 index 000000000000..790a179533d4 --- /dev/null +++ b/warehouse/attestations/models.py @@ -0,0 +1,81 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import typing + +from uuid import UUID + +from sqlalchemy import ( + BigInteger, + CheckConstraint, + Column, + FetchedValue, + ForeignKey, + Index, + Integer, + String, + Text, + UniqueConstraint, + cast, + func, + or_, + orm, + select, + sql, +) +from sqlalchemy.dialects.postgresql import CITEXT +from sqlalchemy.exc import MultipleResultsFound, NoResultFound +from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import ( + Mapped, + attribute_keyed_dict, + declared_attr, + mapped_column, + validates, +) + +from warehouse import db + +if typing.TYPE_CHECKING: + from warehouse.packaging.models import File + + +class ReleaseFileAttestation(db.Model): + """ + Association table between Release Files and Attestations. + + Attestations are stored as opaque blob because their implementation details are handled by the pypi_attestation package. + They are linked to release files as a one-to-many relationship. + """ + + __tablename__ = "release_files_attestation" + + file_id: Mapped[UUID] = mapped_column( + ForeignKey("release_files.id", onupdate="CASCADE", ondelete="CASCADE"), + ) + file: Mapped[File] = orm.relationship(back_populates="attestations") + + attestation_file_sha256_digest: Mapped[str] = mapped_column(CITEXT) + + @hybrid_property + def attestation_path(self): + return self.file.path + self.attestation_file_sha256_digest[:8] + ".attestation" + + @attestation_path.expression # type: ignore + def attestation_path(self): + return func.concat( + func.concat( + self.file.path, self.attestation_file_sha256_digest[:8], ".attestation" + ) + ) diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index b12309c0ae75..0611e0490f19 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -28,14 +28,9 @@ import sentry_sdk import wtforms import wtforms.validators - from pydantic import TypeAdapter, ValidationError -from pypi_attestations import ( - Attestation, - AttestationType, - Distribution, - VerificationError, -) + +from pypi_attestations import Attestation, Distribution, VerificationError, AttestationType from pyramid.httpexceptions import ( HTTPBadRequest, HTTPException, @@ -51,6 +46,7 @@ from sqlalchemy.exc import MultipleResultsFound, NoResultFound from warehouse.admin.flags import AdminFlagValue +from warehouse.attestations.models import ReleaseFileAttestation from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB @@ -63,6 +59,7 @@ from warehouse.forklift.forms import UploadForm, _filetype_extension_mapping from warehouse.macaroons.models import Macaroon from warehouse.metrics import IMetricsService +from warehouse.packaging import File, IFileStorage from warehouse.packaging.interfaces import IFileStorage, IProjectService from warehouse.packaging.models import ( Dependency, @@ -74,7 +71,6 @@ Project, ProjectMacaroonWarningAssociation, Release, - ReleaseFileAttestation, ) from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files from warehouse.rate_limiting.interfaces import RateLimiterException @@ -371,122 +367,6 @@ def _is_duplicate_file(db_session, filename, hashes): return None -def _process_attestations(request, distribution: Distribution) -> list[Attestation]: - """ - Process any attestations included in a file upload request - - Attestations, if present, will be parsed and verified against the uploaded - artifact. Attestations are only allowed when uploading via a Trusted - Publisher, because a Trusted Publisher provides the identity that will be - used to verify the attestations. - Currently, only GitHub Actions Trusted Publishers are supported. - """ - - metrics = request.find_service(IMetricsService, context=None) - - publisher = request.oidc_publisher - if not publisher or not publisher.publisher_name == "GitHub": - raise _exc_with_message( - HTTPBadRequest, - "Attestations are currently only supported when using Trusted " - "Publishing with GitHub Actions.", - ) - try: - attestations = TypeAdapter(list[Attestation]).validate_json( - request.POST["attestations"] - ) - except ValidationError as e: - # Log invalid (malformed) attestation upload - metrics.increment("warehouse.upload.attestations.malformed") - raise _exc_with_message( - HTTPBadRequest, - f"Error while decoding the included attestation: {e}", - ) - - if len(attestations) > 1: - metrics.increment("warehouse.upload.attestations.failed_multiple_attestations") - raise _exc_with_message( - HTTPBadRequest, - "Only a single attestation per-file is supported at the moment.", - ) - - verification_policy = publisher.publisher_verification_policy(request.oidc_claims) - for attestation_model in attestations: - try: - # For now, attestations are not stored, just verified - predicate_type, _ = attestation_model.verify( - Verifier.production(), - verification_policy, - distribution, - ) - except VerificationError as e: - # Log invalid (failed verification) attestation upload - metrics.increment("warehouse.upload.attestations.failed_verify") - raise _exc_with_message( - HTTPBadRequest, - f"Could not verify the uploaded artifact using the included " - f"attestation: {e}", - ) - except Exception as e: - with sentry_sdk.push_scope() as scope: - scope.fingerprint = [e] - sentry_sdk.capture_message( - f"Unexpected error while verifying attestation: {e}" - ) - - raise _exc_with_message( - HTTPBadRequest, - f"Unknown error while trying to verify included attestations: {e}", - ) - - if predicate_type != AttestationType.PYPI_PUBLISH_V1: - metrics.increment( - "warehouse.upload.attestations.failed_unsupported_predicate_type" - ) - raise _exc_with_message( - HTTPBadRequest, - f"Attestation with unsupported predicate type: {predicate_type}", - ) - - # Log successful attestation upload - metrics.increment("warehouse.upload.attestations.ok") - - return attestations - - -def _store_attestations(request, file: File, attestations: list[Attestation]): - """Store the attestations along the release files. - - Attestations are living near the release file, like metadata files. They are named using - their filehash to allow storing more than 1 attestation by file. - - TODO(dm): Validate if the 8 hex chars are enough. - """ - storage = request.find_service(IFileStorage, name="archive") - - release_file_attestations = [] - for attestation in attestations: - - with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write(attestation.model_dump_json().encoded("utf-8")) - - attestation_digest = hashlib.file_digest(tmp_file, "sha256").hexdigest() - attestation_path = f"{file.path}.{attestation_digest[:8]}.attestation" - - storage.store( - attestation_path, - tmp_file.name, - ) - - release_file_attestations.append( - ReleaseFileAttestation( - file=file, - attestation_file_sha256_digest=attestation_digest, - ) - ) - - request.db.add_all(release_file_attestations) - @view_config( route_name="forklift.legacy.file_upload", uses_session=True, @@ -1412,3 +1292,120 @@ def missing_trailing_slash_redirect(request): "/legacy/ (with a trailing slash)", location=request.route_path("forklift.legacy.file_upload"), ) + + +def _process_attestations(request, distribution: Distribution) -> list[Attestation]: + """ + Process any attestations included in a file upload request + + Attestations, if present, will be parsed and verified against the uploaded + artifact. Attestations are only allowed when uploading via a Trusted + Publisher, because a Trusted Publisher provides the identity that will be + used to verify the attestations. + Currently, only GitHub Actions Trusted Publishers are supported. + """ + + metrics = request.find_service(IMetricsService, context=None) + + publisher = request.oidc_publisher + if not publisher or not publisher.publisher_name == "GitHub": + raise _exc_with_message( + HTTPBadRequest, + "Attestations are currently only supported when using Trusted " + "Publishing with GitHub Actions.", + ) + try: + attestations = TypeAdapter(list[Attestation]).validate_json( + request.POST["attestations"] + ) + except ValidationError as e: + # Log invalid (malformed) attestation upload + metrics.increment("warehouse.upload.attestations.malformed") + raise _exc_with_message( + HTTPBadRequest, + f"Error while decoding the included attestation: {e}", + ) + + if len(attestations) > 1: + metrics.increment("warehouse.upload.attestations.failed_multiple_attestations") + raise _exc_with_message( + HTTPBadRequest, + "Only a single attestation per-file is supported at the moment.", + ) + + verification_policy = publisher.publisher_verification_policy(request.oidc_claims) + for attestation_model in attestations: + try: + # For now, attestations are not stored, just verified + predicate_type, _ = attestation_model.verify( + Verifier.production(), + verification_policy, + distribution, + ) + except VerificationError as e: + # Log invalid (failed verification) attestation upload + metrics.increment("warehouse.upload.attestations.failed_verify") + raise _exc_with_message( + HTTPBadRequest, + f"Could not verify the uploaded artifact using the included " + f"attestation: {e}", + ) + except Exception as e: + with sentry_sdk.push_scope() as scope: + scope.fingerprint = [e] + sentry_sdk.capture_message( + f"Unexpected error while verifying attestation: {e}" + ) + + raise _exc_with_message( + HTTPBadRequest, + f"Unknown error while trying to verify included attestations: {e}", + ) + + if predicate_type != AttestationType.PYPI_PUBLISH_V1: + metrics.increment( + "warehouse.upload.attestations.failed_unsupported_predicate_type" + ) + raise _exc_with_message( + HTTPBadRequest, + f"Attestation with unsupported predicate type: {predicate_type}", + ) + + # Log successful attestation upload + metrics.increment("warehouse.upload.attestations.ok") + + return attestations + + +def _store_attestations(request, file: File, attestations: list[Attestation]): + """Store the attestations along the release files. + + Attestations are living near the release file, like metadata files. They are named using + their filehash to allow storing more than 1 attestation by file. + + TODO(dm): Validate if the 8 hex chars are enough. + """ + storage = request.find_service(IFileStorage, name="archive") + + release_file_attestations = [] + for attestation in attestations: + + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write(attestation.model_dump_json().encoded("utf-8")) + + attestation_digest = hashlib.file_digest(tmp_file, "sha256").hexdigest() + attestation_path = f"{file.path}.{attestation_digest[:8]}.attestation" + + storage.store( + attestation_path, + tmp_file.name, + ) + + release_file_attestations.append( + ReleaseFileAttestation( + file=file, + attestation_file_sha256_digest=attestation_digest, + ) + ) + + request.db.add_all(release_file_attestations) diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 9126e4037bef..79cb544b0fb3 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -62,6 +62,7 @@ from warehouse import db from warehouse.accounts.models import User +from warehouse.attestations.models import ReleaseFileAttestation from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.events.models import HasEvents @@ -809,7 +810,7 @@ def __table_args__(cls): # noqa ) @property - def publisher_url(self) -> str | None : + def publisher_url(self) -> str | None: return self.Event.additional["publisher_url"].as_string() # type: ignore[attr-defined] @property @@ -970,27 +971,3 @@ class ProjectMacaroonWarningAssociation(db.Model): ForeignKey("projects.id", onupdate="CASCADE", ondelete="CASCADE"), primary_key=True, ) - -class ReleaseFileAttestation(db.Model): - """ - Association table between Release Files and Attestations. - - Attestations are stored as opaque blob because their implementation details are handled by the pypi_attestation package. - They are linked to release files as a one-to-many relationship. - """ - __tablename__ = "release_files_attestation" - - file_id: Mapped[UUID] = mapped_column( - ForeignKey("release_files.id", onupdate="CASCADE", ondelete="CASCADE"), - ) - file: Mapped[File] = orm.relationship(back_populates="attestations") - - attestation_file_sha256_digest: Mapped[str] = mapped_column(CITEXT) - - @hybrid_property - def attestation_path(self): - return self.file.path + self.attestation_file_sha256_digest[:8] + ".attestation" - - @attestation_path.expression # type: ignore - def attestation_path(self): - return func.concat(func.concat(self.file.path, self.attestation_file_sha256_digest[:8], ".attestation")) diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index e2b927054f94..99a48578e27d 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -19,14 +19,13 @@ from pyramid_jinja2 import IJinja2Environment from sqlalchemy.orm import joinedload +from warehouse.attestations._core import get_provenance_digest from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.models import File, Project, Release, ReleaseFileAttestation -from warehouse.attestations._core import get_provenance_digest API_VERSION = "1.2" - def _simple_index(request, serial): # Fetch the name and normalized name for all of our projects projects = ( @@ -62,34 +61,42 @@ def _simple_detail(project, request): "versions": versions, "files": [ { - "filename": file.filename, - "url": request.route_url("packaging.file", path=file.path), - "hashes": { - "sha256": file.sha256_digest, + **{ + "filename": file.filename, + "url": request.route_url("packaging.file", path=file.path), + "hashes": { + "sha256": file.sha256_digest, + }, + "requires-python": ( + file.release.requires_python + if file.release.requires_python + else None + ), + "size": file.size, + "upload-time": file.upload_time.isoformat() + "Z", + "yanked": ( + file.release.yanked_reason + if file.release.yanked and file.release.yanked_reason + else file.release.yanked + ), + "data-dist-info-metadata": ( + {"sha256": file.metadata_file_sha256_digest} + if file.metadata_file_sha256_digest + else False + ), + "core-metadata": ( + {"sha256": file.metadata_file_sha256_digest} + if file.metadata_file_sha256_digest + else False + ), }, - "requires-python": ( - file.release.requires_python - if file.release.requires_python - else None - ), - "size": file.size, - "upload-time": file.upload_time.isoformat() + "Z", - "yanked": ( - file.release.yanked_reason - if file.release.yanked and file.release.yanked_reason - else file.release.yanked - ), - "data-dist-info-metadata": ( - {"sha256": file.metadata_file_sha256_digest} - if file.metadata_file_sha256_digest - else False - ), - "core-metadata": ( - {"sha256": file.metadata_file_sha256_digest} - if file.metadata_file_sha256_digest - else False + **( + { + "provenance": provenance_digest, + } + if (provenance_digest := get_provenance_digest(request, file)) + else {} ), - "provenance": get_provenance_digest(request, file), } for file in files ], From ae262d48780d7465a34a94daa1b7d8c71180f527 Mon Sep 17 00:00:00 2001 From: Alexis Date: Thu, 18 Jul 2024 15:17:46 +0200 Subject: [PATCH 03/29] Update attestation storage and retrieval --- tests/common/db/attestation.py | 28 +++++ tests/common/db/packaging.py | 10 +- tests/unit/attestations/__init__.py | 11 ++ tests/unit/attestations/test_core.py | 107 ++++++++++++++++++ tests/unit/forklift/test_legacy.py | 4 +- tests/unit/packaging/test_utils.py | 48 +------- warehouse/attestations/_core.py | 29 +++-- warehouse/attestations/models.py | 43 +------ warehouse/forklift/legacy.py | 18 ++- ...3f903_create_release_attestations_table.py | 73 ++++++++++++ warehouse/packaging/models.py | 15 ++- warehouse/packaging/utils.py | 2 +- 12 files changed, 267 insertions(+), 121 deletions(-) create mode 100644 tests/common/db/attestation.py create mode 100644 tests/unit/attestations/__init__.py create mode 100644 tests/unit/attestations/test_core.py create mode 100644 warehouse/migrations/versions/7a195fd3f903_create_release_attestations_table.py diff --git a/tests/common/db/attestation.py b/tests/common/db/attestation.py new file mode 100644 index 000000000000..816998719b2b --- /dev/null +++ b/tests/common/db/attestation.py @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib + +import factory + +from warehouse.attestations.models import ReleaseFileAttestation + +from .base import WarehouseFactory + + +class ReleaseAttestationsFactory(WarehouseFactory): + class Meta: + model = ReleaseFileAttestation + + file = factory.SubFactory("tests.common.db.packaging.FileFactory") + attestation_file_sha256_digest = factory.LazyAttribute( + lambda o: hashlib.sha256(o.file.filename.encode("utf8")).hexdigest() + ) diff --git a/tests/common/db/packaging.py b/tests/common/db/packaging.py index 01421b0efe1a..96af5ecbc7e3 100644 --- a/tests/common/db/packaging.py +++ b/tests/common/db/packaging.py @@ -28,13 +28,13 @@ ProhibitedProjectName, Project, Release, - ReleaseFileAttestation, Role, RoleInvitation, ) from warehouse.utils import readme from .accounts import UserFactory +from .attestation import ReleaseAttestationsFactory from .base import WarehouseFactory from .observations import ObserverFactory @@ -101,14 +101,6 @@ class Meta: description = factory.SubFactory(DescriptionFactory) -class ReleaseAttestationsFactory(WarehouseFactory): - class Meta: - model = ReleaseFileAttestation - - file = factory.SubFactory("tests.common.db.packaging.FileFactory") - attestation = "fake-attestation" - - class FileFactory(WarehouseFactory): class Meta: model = File diff --git a/tests/unit/attestations/__init__.py b/tests/unit/attestations/__init__.py new file mode 100644 index 000000000000..164f68b09175 --- /dev/null +++ b/tests/unit/attestations/__init__.py @@ -0,0 +1,11 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/unit/attestations/test_core.py b/tests/unit/attestations/test_core.py new file mode 100644 index 000000000000..a3005dcf36f2 --- /dev/null +++ b/tests/unit/attestations/test_core.py @@ -0,0 +1,107 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pathlib + +from pathlib import Path + +import pretend + +from pypi_attestations import Attestation, Envelope, VerificationMaterial + +import warehouse.packaging + +from tests.common.db.packaging import FileFactory +from warehouse.attestations._core import generate_provenance_file, get_provenance_digest +from warehouse.events.tags import EventTag +from warehouse.packaging import ISimpleStorage + +from ...common.db.packaging import FileEventFactory + + +def test_get_provenance_digest_succeed(db_request, monkeypatch): + file = FileFactory.create() + FileEventFactory.create( + source=file, + tag=EventTag.Project.ReleaseAdd, + additional={"publisher_url": "fake-publisher-url"}, + ) + + generate_provenance_file = pretend.call_recorder( + lambda request, publisher_url, file_: (Path("fake-path"), "deadbeef") + ) + monkeypatch.setattr( + warehouse.attestations._core, + "generate_provenance_file", + generate_provenance_file, + ) + + hex_digest = get_provenance_digest(db_request, file) + + assert hex_digest == "deadbeef" + + +def test_get_provenance_digest_fails_no_attestations(db_request, monkeypatch): + file = FileFactory.create() + monkeypatch.setattr(warehouse.packaging.models.File, "attestations", []) + + provenance_hash = get_provenance_digest(db_request, file) + assert provenance_hash is None + + +def test_get_provenance_digest_fails_no_publisher_url(db_request, monkeypatch): + file = FileFactory.create() + + provenance_hash = get_provenance_digest(db_request, file) + assert provenance_hash is None + + +def test_generate_provenance_file_succeed(db_request, monkeypatch): + + def store_function(path, file_path, *, meta=None): + return f"https://files/attestations/{path}.provenance" + + storage_service = pretend.stub(store=pretend.call_recorder(store_function)) + + db_request.find_service = pretend.call_recorder( + lambda svc, name=None, context=None: { + ISimpleStorage: storage_service, + }.get(svc) + ) + + publisher_url = "x-fake-publisher-url" + file = FileFactory.create() + FileEventFactory.create( + source=file, + tag=EventTag.Project.ReleaseAdd, + additional={"publisher_url": publisher_url}, + ) + + attestation = Attestation( + version=1, + verification_material=VerificationMaterial( + certificate="somebase64string", transparency_entries=[dict()] + ), + envelope=Envelope( + statement="somebase64string", + signature="somebase64string", + ), + ) + + read_text = pretend.call_recorder(lambda _: attestation.model_dump_json()) + + monkeypatch.setattr(pathlib.Path, "read_text", read_text) + + provenance_file_path, provenance_hash = generate_provenance_file( + db_request, publisher_url, file + ) + + assert provenance_hash is not None diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index a620864c5464..4bcbd6892ec0 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -23,7 +23,6 @@ import pretend import pytest -from pydantic import TypeAdapter from pypi_attestations import ( Attestation, Distribution, @@ -3872,7 +3871,7 @@ def test_upload_succeeds_upload_attestation( }.get(svc) process_attestations = pretend.call_recorder( - lambda request, artifact_path: [attestation] + lambda request, distribution: [attestation] ) monkeypatch.setattr(legacy, "_process_attestations", process_attestations) @@ -3888,7 +3887,6 @@ def test_upload_succeeds_upload_attestation( .all() ) assert len(attestations_db) == 1 - assert TypeAdapter(Attestation).validate_json(attestations_db[0].attestation) @pytest.mark.parametrize( "version, expected_version", diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index cbe9458b3c93..afa7bd2056fe 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -15,14 +15,8 @@ import pretend -import warehouse.packaging.models - from warehouse.packaging.interfaces import ISimpleStorage -from warehouse.packaging.utils import ( - _simple_detail, - render_simple_detail, - store_provenance_object, -) +from warehouse.packaging.utils import _simple_detail, render_simple_detail from ...common.db.packaging import FileFactory, ProjectFactory, ReleaseFactory @@ -73,46 +67,6 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): ) -def test_store_provenance_object_succeed(db_request, monkeypatch): - storage_service = pretend.stub( - store=pretend.call_recorder( - lambda path, file_path, *, meta=None: f"https://files/attestations/{path}.provenance" - ) - ) - - db_request.find_service = pretend.call_recorder( - lambda svc, name=None, context=None: { - ISimpleStorage: storage_service, - }.get(svc) - ) - - monkeypatch.setattr( - warehouse.packaging.models.File, "publisher_url", "x-fake-publisher-url" - ) - - file = FileFactory.create() - provenance_hash = store_provenance_object(db_request, file) - - assert provenance_hash is not None - - -def test_store_provenance_object_fails_no_attestations(db_request, monkeypatch): - file = FileFactory.create() - file.attestations = None - - provenance_hash = store_provenance_object(db_request, file) - assert provenance_hash is None - - -def test_store_provenance_object_fails_no_publisher_url(db_request, monkeypatch): - file = FileFactory.create() - - monkeypatch.setattr(warehouse.packaging.models.File, "publisher_url", None) - - provenance_hash = store_provenance_object(db_request, file) - assert provenance_hash is None - - def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): project = ProjectFactory.create() diff --git a/warehouse/attestations/_core.py b/warehouse/attestations/_core.py index b52ddfb27560..48d8fa3e84c0 100644 --- a/warehouse/attestations/_core.py +++ b/warehouse/attestations/_core.py @@ -16,9 +16,7 @@ from typing import Literal from pydantic import BaseModel, TypeAdapter -from pypi_attestations import ( - Attestation, -) +from pypi_attestations import Attestation from warehouse.packaging import File, ISimpleStorage @@ -55,21 +53,21 @@ class Provenance(BaseModel): def get_provenance_digest(request, file: File) -> str | None: if not file.attestations: - return + return None publisher_url = file.publisher_url if not publisher_url: - return # # TODO(dm) + return None # TODO(dm) - provenance_file_path = generate_provenance_file(request, publisher_url, file) - - with Path(provenance_file_path).open("rb") as file_handler: - provenance_digest = hashlib.file_digest(file_handler, "sha256") - - return provenance_digest.hexdigest() + provenance_file_path, provenance_digest = generate_provenance_file( + request, publisher_url, file + ) + return provenance_digest -def generate_provenance_file(request, publisher_url: str, file: File) -> str: +def generate_provenance_file( + request, publisher_url: str, file: File +) -> tuple[Path, str]: storage = request.find_service(ISimpleStorage) publisher = Publisher(kind=publisher_url, claims={}) # TODO(dm) @@ -86,15 +84,16 @@ def generate_provenance_file(request, publisher_url: str, file: File) -> str: provenance = Provenance(version=1, attestation_bundles=[attestation_bundle]) - provenance_file_path = f"{file.path}.provenance" + provenance_file_path = Path(f"{file.path}.provenance") with tempfile.NamedTemporaryFile() as f: f.write(provenance.model_dump_json().encode("utf-8")) f.flush() storage.store( - f"{file.path}.provenance", + provenance_file_path, f.name, ) - return provenance_file_path + file_digest = hashlib.file_digest(f, "sha256") + return provenance_file_path, file_digest.hexdigest() diff --git a/warehouse/attestations/models.py b/warehouse/attestations/models.py index 790a179533d4..8a93e2a8ca6f 100644 --- a/warehouse/attestations/models.py +++ b/warehouse/attestations/models.py @@ -15,35 +15,10 @@ from uuid import UUID -from sqlalchemy import ( - BigInteger, - CheckConstraint, - Column, - FetchedValue, - ForeignKey, - Index, - Integer, - String, - Text, - UniqueConstraint, - cast, - func, - or_, - orm, - select, - sql, -) +from sqlalchemy import ForeignKey, orm from sqlalchemy.dialects.postgresql import CITEXT -from sqlalchemy.exc import MultipleResultsFound, NoResultFound -from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy.orm import ( - Mapped, - attribute_keyed_dict, - declared_attr, - mapped_column, - validates, -) +from sqlalchemy.orm import Mapped, mapped_column from warehouse import db @@ -55,8 +30,8 @@ class ReleaseFileAttestation(db.Model): """ Association table between Release Files and Attestations. - Attestations are stored as opaque blob because their implementation details are handled by the pypi_attestation package. - They are linked to release files as a one-to-many relationship. + Attestations are stored on disk along the release files (and their + associated metadata). We keep in database only the attestation hash. """ __tablename__ = "release_files_attestation" @@ -70,12 +45,4 @@ class ReleaseFileAttestation(db.Model): @hybrid_property def attestation_path(self): - return self.file.path + self.attestation_file_sha256_digest[:8] + ".attestation" - - @attestation_path.expression # type: ignore - def attestation_path(self): - return func.concat( - func.concat( - self.file.path, self.attestation_file_sha256_digest[:8], ".attestation" - ) - ) + return f"{self.file.path}.{self.attestation_file_sha256_digest[:8]}.attestation" diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 0611e0490f19..318dd25f4094 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -28,9 +28,14 @@ import sentry_sdk import wtforms import wtforms.validators -from pydantic import TypeAdapter, ValidationError -from pypi_attestations import Attestation, Distribution, VerificationError, AttestationType +from pydantic import TypeAdapter, ValidationError +from pypi_attestations import ( + Attestation, + AttestationType, + Distribution, + VerificationError, +) from pyramid.httpexceptions import ( HTTPBadRequest, HTTPException, @@ -59,7 +64,6 @@ from warehouse.forklift.forms import UploadForm, _filetype_extension_mapping from warehouse.macaroons.models import Macaroon from warehouse.metrics import IMetricsService -from warehouse.packaging import File, IFileStorage from warehouse.packaging.interfaces import IFileStorage, IProjectService from warehouse.packaging.models import ( Dependency, @@ -1380,8 +1384,9 @@ def _process_attestations(request, distribution: Distribution) -> list[Attestati def _store_attestations(request, file: File, attestations: list[Attestation]): """Store the attestations along the release files. - Attestations are living near the release file, like metadata files. They are named using - their filehash to allow storing more than 1 attestation by file. + Attestations are living near the release file, like metadata files. + They are named using their filehash to allow storing more than 1 attestation + by file. TODO(dm): Validate if the 8 hex chars are enough. """ @@ -1391,7 +1396,7 @@ def _store_attestations(request, file: File, attestations: list[Attestation]): for attestation in attestations: with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write(attestation.model_dump_json().encoded("utf-8")) + tmp_file.write(attestation.model_dump_json().encode("utf-8")) attestation_digest = hashlib.file_digest(tmp_file, "sha256").hexdigest() attestation_path = f"{file.path}.{attestation_digest[:8]}.attestation" @@ -1399,6 +1404,7 @@ def _store_attestations(request, file: File, attestations: list[Attestation]): storage.store( attestation_path, tmp_file.name, + meta=None, ) release_file_attestations.append( diff --git a/warehouse/migrations/versions/7a195fd3f903_create_release_attestations_table.py b/warehouse/migrations/versions/7a195fd3f903_create_release_attestations_table.py new file mode 100644 index 000000000000..846001f0c98c --- /dev/null +++ b/warehouse/migrations/versions/7a195fd3f903_create_release_attestations_table.py @@ -0,0 +1,73 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +create Release Attestations table + +Revision ID: 7a195fd3f903 +Revises: bb6943882aa9 +Create Date: 2024-07-18 09:26:58.550457 +""" + +import sqlalchemy as sa + +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "7a195fd3f903" +down_revision = "bb6943882aa9" + +# Note: It is VERY important to ensure that a migration does not lock for a +# long period of time and to ensure that each individual migration does +# not break compatibility with the *previous* version of the code base. +# This is because the migrations will be ran automatically as part of the +# deployment process, but while the previous version of the code is still +# up and running. Thus backwards incompatible changes must be broken up +# over multiple migrations inside of multiple pull requests in order to +# phase them in over multiple deploys. +# +# By default, migrations cannot wait more than 4s on acquiring a lock +# and each individual statement cannot take more than 5s. This helps +# prevent situations where a slow migration takes the entire site down. +# +# If you need to increase this timeout for a migration, you can do so +# by adding: +# +# op.execute("SET statement_timeout = 5000") +# op.execute("SET lock_timeout = 4000") +# +# To whatever values are reasonable for this migration as part of your +# migration. + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "release_files_attestation", + sa.Column("file_id", sa.UUID(), nullable=False), + sa.Column( + "attestation_file_sha256_digest", postgresql.CITEXT(), nullable=False + ), + sa.Column( + "id", sa.UUID(), server_default=sa.text("gen_random_uuid()"), nullable=False + ), + sa.ForeignKeyConstraint( + ["file_id"], ["release_files.id"], onupdate="CASCADE", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("release_files_attestation") + # ### end Alembic commands ### diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 79cb544b0fb3..b1859a3ccc7e 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -66,6 +66,7 @@ from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.events.models import HasEvents +from warehouse.events.tags import EventTag from warehouse.integrations.vulnerabilities.models import VulnerabilityRecord from warehouse.observations.models import HasObservations from warehouse.organizations.models import ( @@ -805,13 +806,23 @@ def __table_args__(cls): # noqa # PEP 740 attestations attestations: Mapped[list[ReleaseFileAttestation]] = orm.relationship( cascade="all, delete-orphan", - lazy="dynamic", + lazy="joined", passive_deletes=True, # TODO(dm) check-me ) @property def publisher_url(self) -> str | None: - return self.Event.additional["publisher_url"].as_string() # type: ignore[attr-defined] + try: + release_event = self.events.where( + sql.and_( + self.Event.tag == EventTag.Project.ReleaseAdd, # type: ignore[attr-defined] + self.Event.additional["publisher_url"].as_string().is_not(None), # type: ignore[attr-defined] + ) + ).one() + except (NoResultFound, MultipleResultsFound): + return None + + return release_event.additional["publisher_url"] @property def uploaded_via_trusted_publisher(self) -> bool: diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 99a48578e27d..7d19299fb714 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -44,7 +44,7 @@ def _simple_detail(project, request): # Get all of the files for this project. files = sorted( request.db.query(File) - .options(joinedload(File.release), joinedload(File.attestations)) + .options(joinedload(File.release)) .join(Release) .join(ReleaseFileAttestation) .filter(Release.project == project) From e580a73d422f474aab396d686e3bd42645a84683 Mon Sep 17 00:00:00 2001 From: Alexis Date: Thu, 18 Jul 2024 15:19:59 +0200 Subject: [PATCH 04/29] Update comments --- warehouse/attestations/_core.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/warehouse/attestations/_core.py b/warehouse/attestations/_core.py index 48d8fa3e84c0..f4674438244c 100644 --- a/warehouse/attestations/_core.py +++ b/warehouse/attestations/_core.py @@ -24,31 +24,37 @@ class Publisher(BaseModel): kind: str """ + The kind of Trusted Publisher. """ claims: object """ + Claims specified by the publisher. """ class AttestationBundle(BaseModel): publisher: Publisher """ + The publisher associated with this set of attestations. """ attestations: list[Attestation] """ - Attestations are returned as an opaque + The list of attestations included in this bundle. """ class Provenance(BaseModel): version: Literal[1] """ - The provenance object's version, set to 1 + The provenance object's version, which is always 1. """ attestation_bundles: list[AttestationBundle] + """ + One or more attestation "bundles". + """ def get_provenance_digest(request, file: File) -> str | None: From fdcd78254c5aa5ab259ddb99fe25b635eed42c9f Mon Sep 17 00:00:00 2001 From: Alexis Date: Thu, 18 Jul 2024 15:21:16 +0200 Subject: [PATCH 05/29] Fix import --- warehouse/packaging/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 7d19299fb714..41d9aa7c7e4b 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -21,7 +21,8 @@ from warehouse.attestations._core import get_provenance_digest from warehouse.packaging.interfaces import ISimpleStorage -from warehouse.packaging.models import File, Project, Release, ReleaseFileAttestation +from warehouse.packaging.models import File, Project, Release +from warehouse.attestations.models import ReleaseFileAttestation API_VERSION = "1.2" From 226fd0d149a6f785b66cabeea2e5ed04fe54fafa Mon Sep 17 00:00:00 2001 From: Alexis Date: Thu, 18 Jul 2024 18:26:44 +0200 Subject: [PATCH 06/29] Please linter --- warehouse/packaging/models.py | 7 +++++-- warehouse/packaging/utils.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index b1859a3ccc7e..5f63e02a99d6 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -812,11 +812,14 @@ def __table_args__(cls): # noqa @property def publisher_url(self) -> str | None: + event_tag = self.Event.tag # type: ignore[attr-defined] + event_additional = self.Event.additional # type: ignore[attr-defined] + try: release_event = self.events.where( sql.and_( - self.Event.tag == EventTag.Project.ReleaseAdd, # type: ignore[attr-defined] - self.Event.additional["publisher_url"].as_string().is_not(None), # type: ignore[attr-defined] + event_tag == EventTag.Project.ReleaseAdd, + event_additional["publisher_url"].as_string().is_not(None), ) ).one() except (NoResultFound, MultipleResultsFound): diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 41d9aa7c7e4b..295b61ccc348 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -20,9 +20,9 @@ from sqlalchemy.orm import joinedload from warehouse.attestations._core import get_provenance_digest +from warehouse.attestations.models import ReleaseFileAttestation from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.models import File, Project, Release -from warehouse.attestations.models import ReleaseFileAttestation API_VERSION = "1.2" From 00ca53558ab1dfae29916376380e53c057124d78 Mon Sep 17 00:00:00 2001 From: Alexis Date: Thu, 18 Jul 2024 18:34:28 +0200 Subject: [PATCH 07/29] Use the correct event to attach a publisher_url to a `File` --- tests/unit/attestations/test_core.py | 4 ++-- warehouse/packaging/models.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/attestations/test_core.py b/tests/unit/attestations/test_core.py index a3005dcf36f2..5a2b0889034e 100644 --- a/tests/unit/attestations/test_core.py +++ b/tests/unit/attestations/test_core.py @@ -31,7 +31,7 @@ def test_get_provenance_digest_succeed(db_request, monkeypatch): file = FileFactory.create() FileEventFactory.create( source=file, - tag=EventTag.Project.ReleaseAdd, + tag=EventTag.File.FileAdd, additional={"publisher_url": "fake-publisher-url"}, ) @@ -81,7 +81,7 @@ def store_function(path, file_path, *, meta=None): file = FileFactory.create() FileEventFactory.create( source=file, - tag=EventTag.Project.ReleaseAdd, + tag=EventTag.File.FileAdd, additional={"publisher_url": publisher_url}, ) diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 5f63e02a99d6..8a4c65c07e54 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -818,7 +818,7 @@ def publisher_url(self) -> str | None: try: release_event = self.events.where( sql.and_( - event_tag == EventTag.Project.ReleaseAdd, + event_tag == EventTag.File.FileAdd, event_additional["publisher_url"].as_string().is_not(None), ) ).one() From 600b398666fe62c2004df501e7cb5eacf1b30dec Mon Sep 17 00:00:00 2001 From: Alexis Date: Tue, 23 Jul 2024 14:43:34 +0200 Subject: [PATCH 08/29] Rename ReleaseFileAttestation to Attestation --- tests/common/db/attestation.py | 4 ++-- tests/unit/forklift/test_legacy.py | 6 +++--- warehouse/attestations/models.py | 4 ++-- warehouse/forklift/legacy.py | 4 ++-- warehouse/packaging/models.py | 4 ++-- warehouse/packaging/utils.py | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/common/db/attestation.py b/tests/common/db/attestation.py index 816998719b2b..4a83982ba4a7 100644 --- a/tests/common/db/attestation.py +++ b/tests/common/db/attestation.py @@ -13,14 +13,14 @@ import factory -from warehouse.attestations.models import ReleaseFileAttestation +from warehouse.attestations.models import Attestation from .base import WarehouseFactory class ReleaseAttestationsFactory(WarehouseFactory): class Meta: - model = ReleaseFileAttestation + model = Attestation file = factory.SubFactory("tests.common.db.packaging.FileFactory") attestation_file_sha256_digest = factory.LazyAttribute( diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 4bcbd6892ec0..21b7460bf642 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -42,6 +42,7 @@ from warehouse.accounts.utils import UserContext from warehouse.admin.flags import AdminFlag, AdminFlagValue +from warehouse.attestations.models import Attestation as AttestationModel from warehouse.classifiers.models import Classifier from warehouse.forklift import legacy, metadata from warehouse.macaroons import IMacaroonService, caveats, security_policy @@ -58,7 +59,6 @@ Project, ProjectMacaroonWarningAssociation, Release, - ReleaseFileAttestation, Role, ) from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files @@ -3881,8 +3881,8 @@ def test_upload_succeeds_upload_attestation( assert resp.status_code == 200 attestations_db = ( - db_request.db.query(ReleaseFileAttestation) - .join(ReleaseFileAttestation.file) + db_request.db.query(AttestationModel) + .join(AttestationModel.file) .filter(File.filename == filename) .all() ) diff --git a/warehouse/attestations/models.py b/warehouse/attestations/models.py index 8a93e2a8ca6f..fd2ffaa000b2 100644 --- a/warehouse/attestations/models.py +++ b/warehouse/attestations/models.py @@ -26,9 +26,9 @@ from warehouse.packaging.models import File -class ReleaseFileAttestation(db.Model): +class Attestation(db.Model): """ - Association table between Release Files and Attestations. + Table used to store Attestations. Attestations are stored on disk along the release files (and their associated metadata). We keep in database only the attestation hash. diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 318dd25f4094..330ed18ca57a 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -51,7 +51,7 @@ from sqlalchemy.exc import MultipleResultsFound, NoResultFound from warehouse.admin.flags import AdminFlagValue -from warehouse.attestations.models import ReleaseFileAttestation +from warehouse.attestations.models import Attestation as AttestationModel from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB @@ -1408,7 +1408,7 @@ def _store_attestations(request, file: File, attestations: list[Attestation]): ) release_file_attestations.append( - ReleaseFileAttestation( + AttestationModel( file=file, attestation_file_sha256_digest=attestation_digest, ) diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 8a4c65c07e54..8b14f10ec17f 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -62,7 +62,7 @@ from warehouse import db from warehouse.accounts.models import User -from warehouse.attestations.models import ReleaseFileAttestation +from warehouse.attestations.models import Attestation from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.events.models import HasEvents @@ -804,7 +804,7 @@ def __table_args__(cls): # noqa ) # PEP 740 attestations - attestations: Mapped[list[ReleaseFileAttestation]] = orm.relationship( + attestations: Mapped[list[Attestation]] = orm.relationship( cascade="all, delete-orphan", lazy="joined", passive_deletes=True, # TODO(dm) check-me diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 295b61ccc348..b998532823ac 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -20,7 +20,7 @@ from sqlalchemy.orm import joinedload from warehouse.attestations._core import get_provenance_digest -from warehouse.attestations.models import ReleaseFileAttestation +from warehouse.attestations.models import Attestation from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.models import File, Project, Release @@ -47,7 +47,7 @@ def _simple_detail(project, request): request.db.query(File) .options(joinedload(File.release)) .join(Release) - .join(ReleaseFileAttestation) + .join(Attestation) .filter(Release.project == project) .all(), key=lambda f: (packaging_legacy.version.parse(f.release.version), f.filename), From 9dd8a91e1caed417b29deefb98c4f18982c14bfe Mon Sep 17 00:00:00 2001 From: Alexis Date: Thu, 25 Jul 2024 17:18:16 +0200 Subject: [PATCH 09/29] Update names --- tests/common/db/attestation.py | 2 +- tests/common/db/packaging.py | 4 ++-- tests/unit/forklift/test_legacy.py | 6 +++--- warehouse/attestations/_core.py | 2 +- warehouse/forklift/legacy.py | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/common/db/attestation.py b/tests/common/db/attestation.py index 4a83982ba4a7..9065f53a3e9f 100644 --- a/tests/common/db/attestation.py +++ b/tests/common/db/attestation.py @@ -18,7 +18,7 @@ from .base import WarehouseFactory -class ReleaseAttestationsFactory(WarehouseFactory): +class AttestationFactory(WarehouseFactory): class Meta: model = Attestation diff --git a/tests/common/db/packaging.py b/tests/common/db/packaging.py index 96af5ecbc7e3..369dc9f092d0 100644 --- a/tests/common/db/packaging.py +++ b/tests/common/db/packaging.py @@ -34,7 +34,7 @@ from warehouse.utils import readme from .accounts import UserFactory -from .attestation import ReleaseAttestationsFactory +from .attestation import AttestationFactory from .base import WarehouseFactory from .observations import ObserverFactory @@ -142,7 +142,7 @@ class Meta: ) attestations = factory.RelatedFactoryList( - ReleaseAttestationsFactory, + AttestationFactory, factory_related_name="file", size=1, ) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 21b7460bf642..1c63ec3b75d3 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -42,7 +42,7 @@ from warehouse.accounts.utils import UserContext from warehouse.admin.flags import AdminFlag, AdminFlagValue -from warehouse.attestations.models import Attestation as AttestationModel +from warehouse.attestations.models import Attestation as DatabaseAttestation from warehouse.classifiers.models import Classifier from warehouse.forklift import legacy, metadata from warehouse.macaroons import IMacaroonService, caveats, security_policy @@ -3881,8 +3881,8 @@ def test_upload_succeeds_upload_attestation( assert resp.status_code == 200 attestations_db = ( - db_request.db.query(AttestationModel) - .join(AttestationModel.file) + db_request.db.query(DatabaseAttestation) + .join(DatabaseAttestation.file) .filter(File.filename == filename) .all() ) diff --git a/warehouse/attestations/_core.py b/warehouse/attestations/_core.py index f4674438244c..e971bd7d4949 100644 --- a/warehouse/attestations/_core.py +++ b/warehouse/attestations/_core.py @@ -76,7 +76,7 @@ def generate_provenance_file( ) -> tuple[Path, str]: storage = request.find_service(ISimpleStorage) - publisher = Publisher(kind=publisher_url, claims={}) # TODO(dm) + publisher = Publisher(kind=publisher_url, claims=request.identity.claims) # TODO(dm) attestation_bundle = AttestationBundle( publisher=publisher, diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 330ed18ca57a..135ec308c3a2 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -51,7 +51,7 @@ from sqlalchemy.exc import MultipleResultsFound, NoResultFound from warehouse.admin.flags import AdminFlagValue -from warehouse.attestations.models import Attestation as AttestationModel +from warehouse.attestations.models import Attestation as DatabaseAttestation from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB @@ -1408,7 +1408,7 @@ def _store_attestations(request, file: File, attestations: list[Attestation]): ) release_file_attestations.append( - AttestationModel( + DatabaseAttestation( file=file, attestation_file_sha256_digest=attestation_digest, ) From bd042f22f0e0090a71bc411f6eff81763eef272c Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 26 Jul 2024 16:17:47 +0200 Subject: [PATCH 10/29] Update table migration --- warehouse/attestations/_core.py | 2 +- warehouse/attestations/models.py | 2 +- ....py => 7f0c9f105f44_create_attestations_table.py} | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) rename warehouse/migrations/versions/{7a195fd3f903_create_release_attestations_table.py => 7f0c9f105f44_create_attestations_table.py} (92%) diff --git a/warehouse/attestations/_core.py b/warehouse/attestations/_core.py index e971bd7d4949..28a31af2f5b6 100644 --- a/warehouse/attestations/_core.py +++ b/warehouse/attestations/_core.py @@ -76,7 +76,7 @@ def generate_provenance_file( ) -> tuple[Path, str]: storage = request.find_service(ISimpleStorage) - publisher = Publisher(kind=publisher_url, claims=request.identity.claims) # TODO(dm) + publisher = Publisher(kind=publisher_url, claims={}) attestation_bundle = AttestationBundle( publisher=publisher, diff --git a/warehouse/attestations/models.py b/warehouse/attestations/models.py index fd2ffaa000b2..c94502543f97 100644 --- a/warehouse/attestations/models.py +++ b/warehouse/attestations/models.py @@ -34,7 +34,7 @@ class Attestation(db.Model): associated metadata). We keep in database only the attestation hash. """ - __tablename__ = "release_files_attestation" + __tablename__ = "attestation" file_id: Mapped[UUID] = mapped_column( ForeignKey("release_files.id", onupdate="CASCADE", ondelete="CASCADE"), diff --git a/warehouse/migrations/versions/7a195fd3f903_create_release_attestations_table.py b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py similarity index 92% rename from warehouse/migrations/versions/7a195fd3f903_create_release_attestations_table.py rename to warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py index 846001f0c98c..70b485e17e6e 100644 --- a/warehouse/migrations/versions/7a195fd3f903_create_release_attestations_table.py +++ b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py @@ -10,11 +10,11 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -create Release Attestations table +create Attestations table -Revision ID: 7a195fd3f903 +Revision ID: 7f0c9f105f44 Revises: bb6943882aa9 -Create Date: 2024-07-18 09:26:58.550457 +Create Date: 2024-07-25 15:49:01.993869 """ import sqlalchemy as sa @@ -22,7 +22,7 @@ from alembic import op from sqlalchemy.dialects import postgresql -revision = "7a195fd3f903" +revision = "7f0c9f105f44" down_revision = "bb6943882aa9" # Note: It is VERY important to ensure that a migration does not lock for a @@ -51,7 +51,7 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.create_table( - "release_files_attestation", + "attestation", sa.Column("file_id", sa.UUID(), nullable=False), sa.Column( "attestation_file_sha256_digest", postgresql.CITEXT(), nullable=False @@ -69,5 +69,5 @@ def upgrade(): def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_table("release_files_attestation") + op.drop_table("attestation") # ### end Alembic commands ### From 1d055711e55eedb42850052150c9ef725a2a5ba5 Mon Sep 17 00:00:00 2001 From: Alexis Date: Mon, 29 Jul 2024 15:51:48 +0200 Subject: [PATCH 11/29] Update metrics --- tests/unit/forklift/test_legacy.py | 8 +++++++- warehouse/forklift/legacy.py | 20 +++++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 1c63ec3b75d3..4744f3df672c 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -3874,7 +3874,9 @@ def test_upload_succeeds_upload_attestation( lambda request, distribution: [attestation] ) - monkeypatch.setattr(legacy, "_process_attestations", process_attestations) + monkeypatch.setattr( + legacy, "_parse_and_verify_attestations", process_attestations + ) resp = legacy.file_upload(db_request) @@ -3888,6 +3890,10 @@ def test_upload_succeeds_upload_attestation( ) assert len(attestations_db) == 1 + assert ( + pretend.call("warehouse.upload.attestations.ok") in metrics.increments.calls + ) + @pytest.mark.parametrize( "version, expected_version", [ diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 135ec308c3a2..caaf317ab6be 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -1071,7 +1071,7 @@ def file_upload(request): attestations: list[Attestation] | None = None if "attestations" in request.POST: - attestations = _process_attestations( + attestations = _parse_and_verify_attestations( request=request, distribution=Distribution(name=filename, digest=file_hashes["sha256"]), ) @@ -1175,7 +1175,6 @@ def file_upload(request): # If the user provided attestations, store them along the release file if attestations: _store_attestations(request, file_, attestations) - # TODO(dm): Add some event? # Check if the user has any 2FA methods enabled, and if not, email them. if request.user and not request.user.has_two_factor: @@ -1298,7 +1297,9 @@ def missing_trailing_slash_redirect(request): ) -def _process_attestations(request, distribution: Distribution) -> list[Attestation]: +def _parse_and_verify_attestations( + request, distribution: Distribution +) -> list[Attestation]: """ Process any attestations included in a file upload request @@ -1375,9 +1376,6 @@ def _process_attestations(request, distribution: Distribution) -> list[Attestati f"Attestation with unsupported predicate type: {predicate_type}", ) - # Log successful attestation upload - metrics.increment("warehouse.upload.attestations.ok") - return attestations @@ -1391,8 +1389,9 @@ def _store_attestations(request, file: File, attestations: list[Attestation]): TODO(dm): Validate if the 8 hex chars are enough. """ storage = request.find_service(IFileStorage, name="archive") + metrics = request.find_service(IMetricsService, context=None) - release_file_attestations = [] + attestations_db_models = [] for attestation in attestations: with tempfile.NamedTemporaryFile() as tmp_file: @@ -1407,11 +1406,14 @@ def _store_attestations(request, file: File, attestations: list[Attestation]): meta=None, ) - release_file_attestations.append( + attestations_db_models.append( DatabaseAttestation( file=file, attestation_file_sha256_digest=attestation_digest, ) ) - request.db.add_all(release_file_attestations) + request.db.add_all(attestations_db_models) + + # Log successful attestation upload + metrics.increment("warehouse.upload.attestations.ok") From a55a00fa493532ba1d53c06c178c1f73e96e4b59 Mon Sep 17 00:00:00 2001 From: Alexis Date: Tue, 30 Jul 2024 14:01:49 +0200 Subject: [PATCH 12/29] Update attestations storage --- tests/unit/forklift/test_legacy.py | 2 +- warehouse/attestations/models.py | 13 ++++-- warehouse/forklift/legacy.py | 67 +++++++++++------------------- warehouse/packaging/models.py | 2 +- 4 files changed, 37 insertions(+), 47 deletions(-) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 4744f3df672c..869eb989043c 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -3891,7 +3891,7 @@ def test_upload_succeeds_upload_attestation( assert len(attestations_db) == 1 assert ( - pretend.call("warehouse.upload.attestations.ok") in metrics.increments.calls + pretend.call("warehouse.upload.attestations.ok") in metrics.increment.calls ) @pytest.mark.parametrize( diff --git a/warehouse/attestations/models.py b/warehouse/attestations/models.py index c94502543f97..dbde437498fb 100644 --- a/warehouse/attestations/models.py +++ b/warehouse/attestations/models.py @@ -13,6 +13,7 @@ import typing +from pathlib import Path from uuid import UUID from sqlalchemy import ForeignKey, orm @@ -30,8 +31,7 @@ class Attestation(db.Model): """ Table used to store Attestations. - Attestations are stored on disk along the release files (and their - associated metadata). We keep in database only the attestation hash. + Attestations are stored on disk. We keep in database only the attestation hash. """ __tablename__ = "attestation" @@ -45,4 +45,11 @@ class Attestation(db.Model): @hybrid_property def attestation_path(self): - return f"{self.file.path}.{self.attestation_file_sha256_digest[:8]}.attestation" + return "/".join( + [ + self.attestation_file_sha256_digest[:2], + self.attestation_file_sha256_digest[2:4], + self.attestation_file_sha256_digest[4:], + f"{Path(self.file.path).name}.attestation", + ] + ) diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index caaf317ab6be..d9ef5401fce0 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -1172,9 +1172,32 @@ def file_upload(request): }, ) - # If the user provided attestations, store them along the release file + # If the user provided attestations, store them if attestations: - _store_attestations(request, file_, attestations) + attestations_db_models = [] + for attestation in attestations: + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write(attestation.model_dump_json().encode("utf-8")) + + attestation_digest = hashlib.file_digest( + tmp_file, "sha256" + ).hexdigest() + database_attestation = DatabaseAttestation( + file=file_, attestation_file_sha256_digest=attestation_digest + ) + + storage.store( + database_attestation.attestation_path, + tmp_file.name, + meta=None, + ) + + attestations_db_models.append(database_attestation) + + request.db.add_all(attestations_db_models) + + # Log successful attestation upload + metrics.increment("warehouse.upload.attestations.ok") # Check if the user has any 2FA methods enabled, and if not, email them. if request.user and not request.user.has_two_factor: @@ -1377,43 +1400,3 @@ def _parse_and_verify_attestations( ) return attestations - - -def _store_attestations(request, file: File, attestations: list[Attestation]): - """Store the attestations along the release files. - - Attestations are living near the release file, like metadata files. - They are named using their filehash to allow storing more than 1 attestation - by file. - - TODO(dm): Validate if the 8 hex chars are enough. - """ - storage = request.find_service(IFileStorage, name="archive") - metrics = request.find_service(IMetricsService, context=None) - - attestations_db_models = [] - for attestation in attestations: - - with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write(attestation.model_dump_json().encode("utf-8")) - - attestation_digest = hashlib.file_digest(tmp_file, "sha256").hexdigest() - attestation_path = f"{file.path}.{attestation_digest[:8]}.attestation" - - storage.store( - attestation_path, - tmp_file.name, - meta=None, - ) - - attestations_db_models.append( - DatabaseAttestation( - file=file, - attestation_file_sha256_digest=attestation_digest, - ) - ) - - request.db.add_all(attestations_db_models) - - # Log successful attestation upload - metrics.increment("warehouse.upload.attestations.ok") diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 8b14f10ec17f..35defaffd839 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -807,7 +807,7 @@ def __table_args__(cls): # noqa attestations: Mapped[list[Attestation]] = orm.relationship( cascade="all, delete-orphan", lazy="joined", - passive_deletes=True, # TODO(dm) check-me + passive_deletes=True, ) @property From d9bd6a8fa615ed9992f9e34a760a5644dd55a9a6 Mon Sep 17 00:00:00 2001 From: Alexis Date: Wed, 7 Aug 2024 15:38:07 +0200 Subject: [PATCH 13/29] Update pypi-attestations and sigstore dependencies --- requirements/main.in | 4 ++-- requirements/main.txt | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/requirements/main.in b/requirements/main.in index 5e4305b22be1..90fb9ad65d6a 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -62,8 +62,8 @@ redis>=2.8.0,<6.0.0 rfc3986 sentry-sdk setuptools -sigstore~=3.0.0 -pypi-attestations==0.0.9 +sigstore~=3.1.0 +pypi-attestations==0.0.10 sqlalchemy[asyncio]>=2.0,<3.0 stdlib-list stripe diff --git a/requirements/main.txt b/requirements/main.txt index 7b9633aefa46..a261bef2ece5 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1750,9 +1750,9 @@ pyparsing==3.1.2 \ --hash=sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad \ --hash=sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742 # via linehaul -pypi-attestations==0.0.9 \ - --hash=sha256:3bfc07f64a8db0d6e2646720e70df7c7cb01a2936056c764a2cc3268969332f2 \ - --hash=sha256:4b38cce5d221c8145cac255bfafe650ec0028d924d2b3572394df8ba8f07a609 +pypi-attestations==0.0.10 \ + --hash=sha256:3671fd72c38f9ee539f48772894bccec1a73b93459ce64d9809c517e170ef2c5 \ + --hash=sha256:3e2df0bf8fbc612c825865f642f138cb7a16512c7215489f631c6b1869dbba26 # via -r requirements/main.in pyqrcode==1.2.1 \ --hash=sha256:1b2812775fa6ff5c527977c4cd2ccb07051ca7d0bc0aecf937a43864abe5eff6 \ @@ -2057,9 +2057,9 @@ sentry-sdk==2.11.0 \ --hash=sha256:4ca16e9f5c7c6bc2fb2d5c956219f4926b148e511fffdbbde711dc94f1e0468f \ --hash=sha256:d964710e2dbe015d9dc4ff0ad16225d68c3b36936b742a6fe0504565b760a3b7 # via -r requirements/main.in -sigstore==3.0.0 \ - --hash=sha256:6cc7dc92607c2fd481aada0f3c79e710e4c6086e3beab50b07daa9a50a79d109 \ - --hash=sha256:a6a9538a648e112a0c3d8092d3f73a351c7598164764f1e73a6b5ba406a3a0bd +sigstore==3.1.0 \ + --hash=sha256:3cfe2da19a053757a06bd9ecae322fa539fece7df3e8139d30e32172e41cb812 \ + --hash=sha256:cc0b52acff3ae25f7f1993e21dec4ebed44213c48e2ec095e8c06f69b3751fdf # via # -r requirements/main.in # pypi-attestations From efefd5031d91a18b1680609979e73b33d650ac65 Mon Sep 17 00:00:00 2001 From: Alexis Date: Wed, 7 Aug 2024 15:42:51 +0200 Subject: [PATCH 14/29] Fix wrong merge. --- warehouse/forklift/legacy.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 729a219a253b..75704b6a832b 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -1214,11 +1214,6 @@ def file_upload(request): # Log successful attestation upload metrics.increment("warehouse.upload.attestations.ok") - # Check if the user has any 2FA methods enabled, and if not, email them. - if request.user and not request.user.has_two_factor: - warnings.append("Two factor authentication is not enabled for your account.") - send_two_factor_not_yet_enabled_email(request, request.user) - request.db.flush() # flush db now so server default values are populated for celery # Push updates to BigQuery From 7f29774047ab93eca26c2babe5098b369dedb2ff Mon Sep 17 00:00:00 2001 From: Alexis Date: Thu, 8 Aug 2024 10:14:53 +0200 Subject: [PATCH 15/29] Generate Provenance file on upload. --- warehouse/attestations/_core.py | 109 +++++++++++++++++++------------ warehouse/attestations/models.py | 8 +-- warehouse/forklift/legacy.py | 8 +++ 3 files changed, 78 insertions(+), 47 deletions(-) diff --git a/warehouse/attestations/_core.py b/warehouse/attestations/_core.py index 28a31af2f5b6..408ff8c6b12d 100644 --- a/warehouse/attestations/_core.py +++ b/warehouse/attestations/_core.py @@ -11,50 +11,23 @@ # limitations under the License. import hashlib import tempfile +import typing from pathlib import Path -from typing import Literal -from pydantic import BaseModel, TypeAdapter -from pypi_attestations import Attestation - -from warehouse.packaging import File, ISimpleStorage - - -class Publisher(BaseModel): - kind: str - """ - The kind of Trusted Publisher. - """ - - claims: object - """ - Claims specified by the publisher. - """ +from pydantic import TypeAdapter +from pypi_attestations import Attestation, GitHubPublisher, GitLabPublisher, AttestationBundle, Provenance +from warehouse.oidc.models import OIDCPublisher +from warehouse.oidc.models import GitLabPublisher as GitLabOIDCPublisher +from warehouse.oidc.models import GitHubPublisher as GitHubOIDCPublisher -class AttestationBundle(BaseModel): - publisher: Publisher - """ - The publisher associated with this set of attestations. - """ - - attestations: list[Attestation] - """ - The list of attestations included in this bundle. - """ +from warehouse.packaging import File, ISimpleStorage -class Provenance(BaseModel): - version: Literal[1] - """ - The provenance object's version, which is always 1. - """ - attestation_bundles: list[AttestationBundle] - """ - One or more attestation "bundles". - """ +if typing.TYPE_CHECKING: + from pypi_attestations import Publisher def get_provenance_digest(request, file: File) -> str | None: @@ -71,12 +44,63 @@ def get_provenance_digest(request, file: File) -> str | None: return provenance_digest -def generate_provenance_file( - request, publisher_url: str, file: File -) -> tuple[Path, str]: +# def generate_provenance_file( +# request, publisher_url: str, file: File +# ) -> tuple[Path, str]: +# +# storage = request.find_service(ISimpleStorage) +# +# if publisher_url.startswith("https://github.com/"): +# publisher = GitHubPublisher() +# elif publisher_url.startswith("https://gitlab.com"): +# publisher = GitLabPublisher() +# else: +# raise Exception() # TODO(dm) +# +# attestation_bundle = AttestationBundle( +# publisher=publisher, +# attestations=[ +# TypeAdapter(Attestation).validate_json( +# Path(release_attestation.attestation_path).read_text() +# ) +# for release_attestation in file.attestations +# ], +# ) +# +# provenance = Provenance(attestation_bundles=[attestation_bundle]) +# +# provenance_file_path = Path(f"{file.path}.provenance") +# with tempfile.NamedTemporaryFile() as f: +# f.write(provenance.model_dump_json().encode("utf-8")) +# f.flush() +# +# storage.store( +# provenance_file_path, +# f.name, +# ) +# +# file_digest = hashlib.file_digest(f, "sha256") +# +# return provenance_file_path, file_digest.hexdigest() + + +def publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: + match publisher.publisher_name: + case "GitLab": + publisher = typing.cast(publisher, GitLabOIDCPublisher) + return GitLabPublisher(repository=publisher.project_path, environment="") # TODO(dm) + case "GitHub": + publisher = typing.cast(publisher, GitHubOIDCPublisher) + return GitHubPublisher(repository=publisher.repository, workflow="", environment="") + case _: + raise Exception() # TODO(dm) + +def generate_provenance_file2(request, file: File, oidc_publisher: OIDCPublisher, oidc_claims, attestations: list[Attestation]): storage = request.find_service(ISimpleStorage) - publisher = Publisher(kind=publisher_url, claims={}) + + # First + publisher: Publisher = publisher_from_oidc_publisher(oidc_publisher) attestation_bundle = AttestationBundle( publisher=publisher, @@ -88,7 +112,7 @@ def generate_provenance_file( ], ) - provenance = Provenance(version=1, attestation_bundles=[attestation_bundle]) + provenance = Provenance(attestation_bundles=[attestation_bundle]) provenance_file_path = Path(f"{file.path}.provenance") with tempfile.NamedTemporaryFile() as f: @@ -101,5 +125,4 @@ def generate_provenance_file( ) file_digest = hashlib.file_digest(f, "sha256") - - return provenance_file_path, file_digest.hexdigest() + pass diff --git a/warehouse/attestations/models.py b/warehouse/attestations/models.py index dbde437498fb..6bd7d1bfadf0 100644 --- a/warehouse/attestations/models.py +++ b/warehouse/attestations/models.py @@ -47,9 +47,9 @@ class Attestation(db.Model): def attestation_path(self): return "/".join( [ - self.attestation_file_sha256_digest[:2], - self.attestation_file_sha256_digest[2:4], - self.attestation_file_sha256_digest[4:], - f"{Path(self.file.path).name}.attestation", + self.attestation_file_sha256_digest[:2], + self.attestation_file_sha256_digest[2:4], + self.attestation_file_sha256_digest[4:], + f"{Path(self.file.path).name}.attestation", ] ) diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 75704b6a832b..7c93a3bd9971 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -51,6 +51,7 @@ from sqlalchemy.exc import MultipleResultsFound, NoResultFound from warehouse.admin.flags import AdminFlagValue +from warehouse.attestations._core import generate_provenance_file2 from warehouse.attestations.models import Attestation as DatabaseAttestation from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier @@ -1210,6 +1211,13 @@ def file_upload(request): attestations_db_models.append(database_attestation) request.db.add_all(attestations_db_models) + generate_provenance_file2( + request, + file_, + request.oidc_publisher, + request.oidc_claims, + attestations, + ) # Log successful attestation upload metrics.increment("warehouse.upload.attestations.ok") From 2f317492ed62de9f2652b1503a3e062427825324 Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 9 Aug 2024 17:02:04 +0200 Subject: [PATCH 16/29] Generate and store provenance file during upload. --- warehouse/attestations/_core.py | 108 ++++++++++---------------------- warehouse/forklift/legacy.py | 6 +- 2 files changed, 34 insertions(+), 80 deletions(-) diff --git a/warehouse/attestations/_core.py b/warehouse/attestations/_core.py index 408ff8c6b12d..401742e55418 100644 --- a/warehouse/attestations/_core.py +++ b/warehouse/attestations/_core.py @@ -15,101 +15,60 @@ from pathlib import Path -from pydantic import TypeAdapter -from pypi_attestations import Attestation, GitHubPublisher, GitLabPublisher, AttestationBundle, Provenance - -from warehouse.oidc.models import OIDCPublisher -from warehouse.oidc.models import GitLabPublisher as GitLabOIDCPublisher -from warehouse.oidc.models import GitHubPublisher as GitHubOIDCPublisher - - +from pypi_attestations import ( + Attestation, + AttestationBundle, + GitHubPublisher, + GitLabPublisher, + Provenance, + Publisher, +) + +from warehouse.oidc.models import ( + GitHubPublisher as GitHubOIDCPublisher, + GitLabPublisher as GitLabOIDCPublisher, + OIDCPublisher, +) from warehouse.packaging import File, ISimpleStorage -if typing.TYPE_CHECKING: - from pypi_attestations import Publisher - - def get_provenance_digest(request, file: File) -> str | None: - if not file.attestations: + """Returns the sha256 digest of the provenance file for the release.""" + if not file.attestations or not file.publisher_url: return None - publisher_url = file.publisher_url - if not publisher_url: - return None # TODO(dm) - - provenance_file_path, provenance_digest = generate_provenance_file( - request, publisher_url, file - ) - return provenance_digest - + storage = request.find_service(ISimpleStorage) + provenance_file = storage.get(f"{file.path}.provenance") -# def generate_provenance_file( -# request, publisher_url: str, file: File -# ) -> tuple[Path, str]: -# -# storage = request.find_service(ISimpleStorage) -# -# if publisher_url.startswith("https://github.com/"): -# publisher = GitHubPublisher() -# elif publisher_url.startswith("https://gitlab.com"): -# publisher = GitLabPublisher() -# else: -# raise Exception() # TODO(dm) -# -# attestation_bundle = AttestationBundle( -# publisher=publisher, -# attestations=[ -# TypeAdapter(Attestation).validate_json( -# Path(release_attestation.attestation_path).read_text() -# ) -# for release_attestation in file.attestations -# ], -# ) -# -# provenance = Provenance(attestation_bundles=[attestation_bundle]) -# -# provenance_file_path = Path(f"{file.path}.provenance") -# with tempfile.NamedTemporaryFile() as f: -# f.write(provenance.model_dump_json().encode("utf-8")) -# f.flush() -# -# storage.store( -# provenance_file_path, -# f.name, -# ) -# -# file_digest = hashlib.file_digest(f, "sha256") -# -# return provenance_file_path, file_digest.hexdigest() + return hashlib.file_digest(provenance_file, "sha256").hexdigest() def publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: match publisher.publisher_name: case "GitLab": - publisher = typing.cast(publisher, GitLabOIDCPublisher) - return GitLabPublisher(repository=publisher.project_path, environment="") # TODO(dm) + publisher = typing.cast(GitLabOIDCPublisher, publisher) + return GitLabPublisher( + repository=publisher.project_path, environment="" + ) # TODO(dm) case "GitHub": - publisher = typing.cast(publisher, GitHubOIDCPublisher) - return GitHubPublisher(repository=publisher.repository, workflow="", environment="") + publisher = typing.cast(GitHubOIDCPublisher, publisher) + return GitHubPublisher( + repository=publisher.repository, workflow="", environment="" + ) case _: raise Exception() # TODO(dm) -def generate_provenance_file2(request, file: File, oidc_publisher: OIDCPublisher, oidc_claims, attestations: list[Attestation]): +def generate_and_store_provenance_file( + request, file: File, attestations: list[Attestation] +): storage = request.find_service(ISimpleStorage) - # First - publisher: Publisher = publisher_from_oidc_publisher(oidc_publisher) + publisher: Publisher = publisher_from_oidc_publisher(request.oidc_publisher) attestation_bundle = AttestationBundle( publisher=publisher, - attestations=[ - TypeAdapter(Attestation).validate_json( - Path(release_attestation.attestation_path).read_text() - ) - for release_attestation in file.attestations - ], + attestations=attestations, ) provenance = Provenance(attestation_bundles=[attestation_bundle]) @@ -123,6 +82,3 @@ def generate_provenance_file2(request, file: File, oidc_publisher: OIDCPublisher provenance_file_path, f.name, ) - - file_digest = hashlib.file_digest(f, "sha256") - pass diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 7c93a3bd9971..577c56808629 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -51,7 +51,7 @@ from sqlalchemy.exc import MultipleResultsFound, NoResultFound from warehouse.admin.flags import AdminFlagValue -from warehouse.attestations._core import generate_provenance_file2 +from warehouse.attestations._core import generate_and_store_provenance_file from warehouse.attestations.models import Attestation as DatabaseAttestation from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier @@ -1211,11 +1211,9 @@ def file_upload(request): attestations_db_models.append(database_attestation) request.db.add_all(attestations_db_models) - generate_provenance_file2( + generate_and_store_provenance_file( request, file_, - request.oidc_publisher, - request.oidc_claims, attestations, ) From 0c967510178bacc6b8a6207d5fec257a241d8cd5 Mon Sep 17 00:00:00 2001 From: Alexis Date: Mon, 12 Aug 2024 12:14:20 +0200 Subject: [PATCH 17/29] Improve tests --- tests/unit/attestations/test_core.py | 159 +++++++++++++----- tests/unit/forklift/test_legacy.py | 4 +- warehouse/attestations/_core.py | 22 ++- warehouse/attestations/errors.py | 15 ++ .../7f0c9f105f44_create_attestations_table.py | 2 +- 5 files changed, 150 insertions(+), 52 deletions(-) create mode 100644 warehouse/attestations/errors.py diff --git a/tests/unit/attestations/test_core.py b/tests/unit/attestations/test_core.py index 5a2b0889034e..c8b9ad5990f9 100644 --- a/tests/unit/attestations/test_core.py +++ b/tests/unit/attestations/test_core.py @@ -9,25 +9,41 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import pathlib - -from pathlib import Path +import hashlib +import tempfile +import typing import pretend - -from pypi_attestations import Attestation, Envelope, VerificationMaterial +import pytest + +from pypi_attestations import ( + Attestation, + AttestationBundle, + Envelope, + GitHubPublisher, + GitLabPublisher, + Provenance, + VerificationMaterial, +) import warehouse.packaging -from tests.common.db.packaging import FileFactory -from warehouse.attestations._core import generate_provenance_file, get_provenance_digest +from tests.common.db.oidc import GitHubPublisherFactory, GitLabPublisherFactory +from tests.common.db.packaging import FileEventFactory, FileFactory +from warehouse.attestations._core import ( + generate_and_store_provenance_file, + get_provenance_digest, + publisher_from_oidc_publisher, +) +from warehouse.attestations.errors import UnknownPublisherError from warehouse.events.tags import EventTag from warehouse.packaging import ISimpleStorage -from ...common.db.packaging import FileEventFactory +if typing.TYPE_CHECKING: + from pathlib import Path -def test_get_provenance_digest_succeed(db_request, monkeypatch): +def test_get_provenance_digest(db_request): file = FileFactory.create() FileEventFactory.create( source=file, @@ -35,56 +51,81 @@ def test_get_provenance_digest_succeed(db_request, monkeypatch): additional={"publisher_url": "fake-publisher-url"}, ) - generate_provenance_file = pretend.call_recorder( - lambda request, publisher_url, file_: (Path("fake-path"), "deadbeef") - ) - monkeypatch.setattr( - warehouse.attestations._core, - "generate_provenance_file", - generate_provenance_file, - ) + with tempfile.NamedTemporaryFile() as f: + storage_service = pretend.stub(get=pretend.call_recorder(lambda p: f)) - hex_digest = get_provenance_digest(db_request, file) + db_request.find_service = pretend.call_recorder( + lambda svc, name=None, context=None: { + ISimpleStorage: storage_service, + }.get(svc) + ) - assert hex_digest == "deadbeef" + assert ( + get_provenance_digest(db_request, file) + == hashlib.file_digest(f, "sha256").hexdigest() + ) -def test_get_provenance_digest_fails_no_attestations(db_request, monkeypatch): +def test_get_provenance_digest_fails_no_publisher_url(db_request): file = FileFactory.create() - monkeypatch.setattr(warehouse.packaging.models.File, "attestations", []) - provenance_hash = get_provenance_digest(db_request, file) - assert provenance_hash is None + # If the publisher_url is missing, there is no provenance file + assert get_provenance_digest(db_request, file) is None -def test_get_provenance_digest_fails_no_publisher_url(db_request, monkeypatch): +def test_get_provenance_digest_fails_no_attestations(db_request): + # If the attestations are missing, there is no provenance file file = FileFactory.create() + file.attestations = [] + FileEventFactory.create( + source=file, + tag=EventTag.File.FileAdd, + additional={"publisher_url": "fake-publisher-url"}, + ) + assert get_provenance_digest(db_request, file) is None + + +def test_publisher_from_oidc_publisher_github(db_request): + publisher = GitHubPublisherFactory.create() + + attestation_publisher = publisher_from_oidc_publisher(publisher) + assert isinstance(attestation_publisher, GitHubPublisher) + assert attestation_publisher.repository == publisher.repository + assert attestation_publisher.workflow == publisher.workflow_filename + assert attestation_publisher.environment == publisher.environment + - provenance_hash = get_provenance_digest(db_request, file) - assert provenance_hash is None +def test_publisher_from_oidc_publisher_gitlab(db_request): + publisher = GitLabPublisherFactory.create() + attestation_publisher = publisher_from_oidc_publisher(publisher) + assert isinstance(attestation_publisher, GitLabPublisher) + assert attestation_publisher.repository == publisher.project_path + assert attestation_publisher.environment == publisher.environment -def test_generate_provenance_file_succeed(db_request, monkeypatch): - def store_function(path, file_path, *, meta=None): - return f"https://files/attestations/{path}.provenance" +def test_publisher_from_oidc_publisher_fails(db_request, monkeypatch): - storage_service = pretend.stub(store=pretend.call_recorder(store_function)) + publisher = pretend.stub(publisher_name="not-existing") + with pytest.raises(UnknownPublisherError): + publisher_from_oidc_publisher(publisher) + + +def test_generate_and_store_provenance_file_no_publisher(db_request): db_request.find_service = pretend.call_recorder( - lambda svc, name=None, context=None: { - ISimpleStorage: storage_service, - }.get(svc) + lambda svc, name=None, context=None: None ) + db_request.oidc_publisher = pretend.stub(publisher_name="not-existing") - publisher_url = "x-fake-publisher-url" - file = FileFactory.create() - FileEventFactory.create( - source=file, - tag=EventTag.File.FileAdd, - additional={"publisher_url": publisher_url}, + assert ( + generate_and_store_provenance_file(db_request, pretend.stub(), pretend.stub()) + is None ) + +def test_generate_and_store_provenance_file(db_request, monkeypatch): + attestation = Attestation( version=1, verification_material=VerificationMaterial( @@ -95,13 +136,43 @@ def store_function(path, file_path, *, meta=None): signature="somebase64string", ), ) + publisher = GitHubPublisher( + repository="fake-repository", + workflow="fake-workflow", + ) + provenance = Provenance( + attestation_bundles=[ + AttestationBundle( + publisher=publisher, + attestations=[attestation], + ) + ] + ) - read_text = pretend.call_recorder(lambda _: attestation.model_dump_json()) + @pretend.call_recorder + def storage_service_store(path: Path, file_path, *_args, **_kwargs): + expected = provenance.model_dump_json().encode("utf-8") + with open(file_path, "rb") as fp: + assert fp.read() == expected - monkeypatch.setattr(pathlib.Path, "read_text", read_text) + assert path.suffix == ".provenance" - provenance_file_path, provenance_hash = generate_provenance_file( - db_request, publisher_url, file + storage_service = pretend.stub(store=storage_service_store) + db_request.find_service = pretend.call_recorder( + lambda svc, name=None, context=None: { + ISimpleStorage: storage_service, + }.get(svc) ) - assert provenance_hash is not None + monkeypatch.setattr( + warehouse.attestations._core, + "publisher_from_oidc_publisher", + lambda s: publisher, + ) + + assert ( + generate_and_store_provenance_file( + db_request, FileFactory.create(), [attestation] + ) + is None + ) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index eb7e390a0936..694d04c3b2e4 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -3872,12 +3872,12 @@ def test_upload_succeeds_upload_attestation( IMetricsService: metrics, }.get(svc) - process_attestations = pretend.call_recorder( + parse_and_verify_attestations = pretend.call_recorder( lambda request, distribution: [attestation] ) monkeypatch.setattr( - legacy, "_parse_and_verify_attestations", process_attestations + legacy, "_parse_and_verify_attestations", parse_and_verify_attestations ) resp = legacy.file_upload(db_request) diff --git a/warehouse/attestations/_core.py b/warehouse/attestations/_core.py index 401742e55418..7982ff7b7374 100644 --- a/warehouse/attestations/_core.py +++ b/warehouse/attestations/_core.py @@ -15,6 +15,8 @@ from pathlib import Path +import sentry_sdk + from pypi_attestations import ( Attestation, AttestationBundle, @@ -24,6 +26,7 @@ Publisher, ) +from warehouse.attestations.errors import UnknownPublisherError from warehouse.oidc.models import ( GitHubPublisher as GitHubOIDCPublisher, GitLabPublisher as GitLabOIDCPublisher, @@ -48,15 +51,17 @@ def publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: case "GitLab": publisher = typing.cast(GitLabOIDCPublisher, publisher) return GitLabPublisher( - repository=publisher.project_path, environment="" - ) # TODO(dm) + repository=publisher.project_path, environment=publisher.environment + ) case "GitHub": publisher = typing.cast(GitHubOIDCPublisher, publisher) return GitHubPublisher( - repository=publisher.repository, workflow="", environment="" + repository=publisher.repository, + workflow=publisher.workflow_filename, + environment=publisher.environment, ) case _: - raise Exception() # TODO(dm) + raise UnknownPublisherError() def generate_and_store_provenance_file( @@ -64,7 +69,14 @@ def generate_and_store_provenance_file( ): storage = request.find_service(ISimpleStorage) - publisher: Publisher = publisher_from_oidc_publisher(request.oidc_publisher) + try: + publisher: Publisher = publisher_from_oidc_publisher(request.oidc_publisher) + except UnknownPublisherError: + sentry_sdk.capture_message( + f"Unsupported OIDCPublisher found {request.oidc_publisher.publisher_name}" + ) + + return attestation_bundle = AttestationBundle( publisher=publisher, diff --git a/warehouse/attestations/errors.py b/warehouse/attestations/errors.py new file mode 100644 index 000000000000..d04df4e69cbb --- /dev/null +++ b/warehouse/attestations/errors.py @@ -0,0 +1,15 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class UnknownPublisherError(Exception): + pass diff --git a/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py index 70b485e17e6e..44e946e15fb2 100644 --- a/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py +++ b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py @@ -23,7 +23,7 @@ from sqlalchemy.dialects import postgresql revision = "7f0c9f105f44" -down_revision = "bb6943882aa9" +down_revision = "208d494aac68" # Note: It is VERY important to ensure that a migration does not lock for a # long period of time and to ensure that each individual migration does From ff338da21da8cd1516b8492719d59d549030b9bb Mon Sep 17 00:00:00 2001 From: Alexis Date: Mon, 12 Aug 2024 14:23:12 +0200 Subject: [PATCH 18/29] Fix test error --- tests/unit/attestations/test_core.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/unit/attestations/test_core.py b/tests/unit/attestations/test_core.py index c8b9ad5990f9..0095248ee8c8 100644 --- a/tests/unit/attestations/test_core.py +++ b/tests/unit/attestations/test_core.py @@ -11,7 +11,8 @@ # limitations under the License. import hashlib import tempfile -import typing + +from pathlib import Path import pretend import pytest @@ -39,9 +40,6 @@ from warehouse.events.tags import EventTag from warehouse.packaging import ISimpleStorage -if typing.TYPE_CHECKING: - from pathlib import Path - def test_get_provenance_digest(db_request): file = FileFactory.create() From 284c488fd6f631d759ace96f4f2470eed860a835 Mon Sep 17 00:00:00 2001 From: Alexis Date: Mon, 12 Aug 2024 17:21:40 +0200 Subject: [PATCH 19/29] Fix test error --- tests/unit/attestations/test_core.py | 6 +++--- tests/unit/forklift/test_legacy.py | 4 ++-- warehouse/attestations/_core.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/unit/attestations/test_core.py b/tests/unit/attestations/test_core.py index 0095248ee8c8..af4d805c4085 100644 --- a/tests/unit/attestations/test_core.py +++ b/tests/unit/attestations/test_core.py @@ -38,7 +38,7 @@ ) from warehouse.attestations.errors import UnknownPublisherError from warehouse.events.tags import EventTag -from warehouse.packaging import ISimpleStorage +from warehouse.packaging import IFileStorage def test_get_provenance_digest(db_request): @@ -54,7 +54,7 @@ def test_get_provenance_digest(db_request): db_request.find_service = pretend.call_recorder( lambda svc, name=None, context=None: { - ISimpleStorage: storage_service, + IFileStorage: storage_service, }.get(svc) ) @@ -158,7 +158,7 @@ def storage_service_store(path: Path, file_path, *_args, **_kwargs): storage_service = pretend.stub(store=storage_service_store) db_request.find_service = pretend.call_recorder( lambda svc, name=None, context=None: { - ISimpleStorage: storage_service, + IFileStorage: storage_service, }.get(svc) ) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 694d04c3b2e4..341dd5310697 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -3446,7 +3446,7 @@ def test_upload_with_valid_attestation_succeeds( } ) - storage_service = pretend.stub(store=lambda path, filepath, meta: None) + storage_service = pretend.stub(store=lambda path, filepath, *, meta=None: None) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, @@ -3866,7 +3866,7 @@ def test_upload_succeeds_upload_attestation( } ) - storage_service = pretend.stub(store=lambda path, filepath, meta: None) + storage_service = pretend.stub(store=lambda path, filepath, *, meta=None: None) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, diff --git a/warehouse/attestations/_core.py b/warehouse/attestations/_core.py index 7982ff7b7374..4b8ac101db78 100644 --- a/warehouse/attestations/_core.py +++ b/warehouse/attestations/_core.py @@ -32,7 +32,7 @@ GitLabPublisher as GitLabOIDCPublisher, OIDCPublisher, ) -from warehouse.packaging import File, ISimpleStorage +from warehouse.packaging import File, IFileStorage def get_provenance_digest(request, file: File) -> str | None: @@ -40,7 +40,7 @@ def get_provenance_digest(request, file: File) -> str | None: if not file.attestations or not file.publisher_url: return None - storage = request.find_service(ISimpleStorage) + storage = request.find_service(IFileStorage) provenance_file = storage.get(f"{file.path}.provenance") return hashlib.file_digest(provenance_file, "sha256").hexdigest() @@ -67,7 +67,7 @@ def publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: def generate_and_store_provenance_file( request, file: File, attestations: list[Attestation] ): - storage = request.find_service(ISimpleStorage) + storage = request.find_service(IFileStorage) try: publisher: Publisher = publisher_from_oidc_publisher(request.oidc_publisher) From ba6752d6f5254db4c4216c306bb6e35e8c2035b3 Mon Sep 17 00:00:00 2001 From: Alexis Date: Wed, 14 Aug 2024 18:49:23 +0200 Subject: [PATCH 20/29] Fix merge error --- warehouse/forklift/legacy.py | 77 +++++++++++++++++++ .../7f0c9f105f44_create_attestations_table.py | 2 +- 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index f1955984852e..ce530d75ced5 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -377,6 +377,83 @@ def _is_duplicate_file(db_session, filename, hashes): return None +def _verify_url(url: str, publisher_url: str | None) -> bool: + """ + Verify a given URL against a Trusted Publisher URL + + A URL is considered "verified" iff it matches the Trusted Publisher URL + such that, when both URLs are normalized: + - The scheme component is the same (e.g: both use `https`) + - The authority component is the same (e.g.: `github.com`) + - The path component is the same, or a sub-path of the Trusted Publisher URL + (e.g.: `org/project` and `org/project/issues.html` will pass verification + against an `org/project` Trusted Publisher path component) + - The path component of the Trusted Publisher URL is not empty + Note: We compare the authority component instead of the host component because + the authority includes the host, and in practice neither URL should have user + nor port information. + """ + if not publisher_url: + return False + + publisher_uri = rfc3986.api.uri_reference(publisher_url).normalize() + user_uri = rfc3986.api.uri_reference(url).normalize() + if publisher_uri.path is None: + # Currently no Trusted Publishers have an empty path component, + # so we defensively fail verification. + return False + elif user_uri.path and publisher_uri.path: + is_subpath = publisher_uri.path == user_uri.path or user_uri.path.startswith( + publisher_uri.path + "/" + ) + else: + is_subpath = publisher_uri.path == user_uri.path + + return ( + publisher_uri.scheme == user_uri.scheme + and publisher_uri.authority == user_uri.authority + and is_subpath + ) + + +def _sort_releases(request: Request, project: Project): + releases = ( + request.db.query(Release) + .filter(Release.project == project) + .options( + orm.load_only( + Release.project_id, + Release.version, + Release._pypi_ordering, + ) + ) + .all() + ) + for i, r in enumerate( + sorted(releases, key=lambda x: packaging_legacy.version.parse(x.version)) + ): + # NOTE: If we set r._pypi_ordering, even to the same value it was + # previously, then SQLAlchemy will decide that it needs to load + # Release.description (with N+1 queries) for some reason that I + # can't possibly fathom why. The SQLAlchemy docs say that + # raisedload doesn't prevent SQLAlchemy for doing queries that it + # needs to do for the Unit of Work pattern, so I guess since each + # Release object was modified it feels the need to load each of + # them... but I haven no idea why that means it feels the need to + # load the Release.description relationship as well. + # + # Technically, we can still execute a query for every release if + # someone goes back and releases a version "0" or something that + # would cause most or all of the releases to need to "shift" and + # get updated. + # + # We maybe want to convert this away from using the ORM and build up + # a mapping of Release.id -> new _pypi_ordering and do a single bulk + # update query to eliminate the possibility we trigger this again. + if r._pypi_ordering != i: + r._pypi_ordering = i + + @view_config( route_name="forklift.legacy.file_upload", uses_session=True, diff --git a/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py index 44e946e15fb2..5c3d144dc09a 100644 --- a/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py +++ b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py @@ -23,7 +23,7 @@ from sqlalchemy.dialects import postgresql revision = "7f0c9f105f44" -down_revision = "208d494aac68" +down_revision = "26455e3712a2" # Note: It is VERY important to ensure that a migration does not lock for a # long period of time and to ensure that each individual migration does From a26047d7b4400b3704a6b0297469cd331efcdebc Mon Sep 17 00:00:00 2001 From: Alexis Date: Thu, 15 Aug 2024 10:04:45 +0200 Subject: [PATCH 21/29] Simplify legacy answer --- warehouse/packaging/utils.py | 60 ++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 34 deletions(-) diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 910713ffacaf..aa4c1173aa0c 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -73,42 +73,34 @@ def _simple_detail(project, request): "versions": versions, "files": [ { - **{ - "filename": file.filename, - "url": request.route_url("packaging.file", path=file.path), - "hashes": { - "sha256": file.sha256_digest, - }, - "requires-python": ( - file.release.requires_python - if file.release.requires_python - else None - ), - "size": file.size, - "upload-time": file.upload_time.isoformat() + "Z", - "yanked": ( - file.release.yanked_reason - if file.release.yanked and file.release.yanked_reason - else file.release.yanked - ), - "data-dist-info-metadata": ( - {"sha256": file.metadata_file_sha256_digest} - if file.metadata_file_sha256_digest - else False - ), - "core-metadata": ( - {"sha256": file.metadata_file_sha256_digest} - if file.metadata_file_sha256_digest - else False - ), + "filename": file.filename, + "url": request.route_url("packaging.file", path=file.path), + "hashes": { + "sha256": file.sha256_digest, }, - **( - { - "provenance": provenance_digest, - } - if (provenance_digest := get_provenance_digest(request, file)) - else {} + "requires-python": ( + file.release.requires_python + if file.release.requires_python + else None ), + "size": file.size, + "upload-time": file.upload_time.isoformat() + "Z", + "yanked": ( + file.release.yanked_reason + if file.release.yanked and file.release.yanked_reason + else file.release.yanked + ), + "data-dist-info-metadata": ( + {"sha256": file.metadata_file_sha256_digest} + if file.metadata_file_sha256_digest + else False + ), + "core-metadata": ( + {"sha256": file.metadata_file_sha256_digest} + if file.metadata_file_sha256_digest + else False + ), + "provenance": get_provenance_digest(request, file) } for file in files ], From 8b99fb76781ba4b95c584b4c915b761726ea7078 Mon Sep 17 00:00:00 2001 From: Alexis Date: Thu, 15 Aug 2024 14:52:34 +0200 Subject: [PATCH 22/29] Introduce AttestationsService --- tests/common/db/attestation.py | 4 +- tests/unit/api/test_simple.py | 3 + tests/unit/attestations/test_core.py | 4 +- tests/unit/attestations/test_services.py | 252 +++++++++ tests/unit/forklift/test_legacy.py | 492 ++---------------- warehouse/attestations/_core.py | 6 +- warehouse/attestations/errors.py | 6 +- warehouse/attestations/interfaces.py | 37 ++ warehouse/attestations/models.py | 8 +- warehouse/attestations/services.py | 159 ++++++ warehouse/forklift/legacy.py | 141 +---- .../7f0c9f105f44_create_attestations_table.py | 30 +- warehouse/packaging/utils.py | 2 +- 13 files changed, 531 insertions(+), 613 deletions(-) create mode 100644 tests/unit/attestations/test_services.py create mode 100644 warehouse/attestations/interfaces.py create mode 100644 warehouse/attestations/services.py diff --git a/tests/common/db/attestation.py b/tests/common/db/attestation.py index 9065f53a3e9f..2080519a806f 100644 --- a/tests/common/db/attestation.py +++ b/tests/common/db/attestation.py @@ -23,6 +23,6 @@ class Meta: model = Attestation file = factory.SubFactory("tests.common.db.packaging.FileFactory") - attestation_file_sha256_digest = factory.LazyAttribute( - lambda o: hashlib.sha256(o.file.filename.encode("utf8")).hexdigest() + attestation_file_blake2_digest = factory.LazyAttribute( + lambda o: hashlib.blake2b(o.file.filename.encode("utf8")).hexdigest() ) diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index f1df2a0743d8..dfbba3920b9b 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -286,6 +286,7 @@ def test_with_files_no_serial(self, db_request, content_type, renderer_override) "upload-time": f.upload_time.isoformat() + "Z", "data-dist-info-metadata": False, "core-metadata": False, + "provenance": None, } for f in files ], @@ -334,6 +335,7 @@ def test_with_files_with_serial(self, db_request, content_type, renderer_overrid "upload-time": f.upload_time.isoformat() + "Z", "data-dist-info-metadata": False, "core-metadata": False, + "provenance": None, } for f in files ], @@ -427,6 +429,7 @@ def test_with_files_with_version_multi_digit( if f.metadata_file_sha256_digest is not None else False ), + "provenance": None, } for f in files ], diff --git a/tests/unit/attestations/test_core.py b/tests/unit/attestations/test_core.py index af4d805c4085..49c15500328c 100644 --- a/tests/unit/attestations/test_core.py +++ b/tests/unit/attestations/test_core.py @@ -36,7 +36,7 @@ get_provenance_digest, publisher_from_oidc_publisher, ) -from warehouse.attestations.errors import UnknownPublisherError +from warehouse.attestations.errors import UnsupportedPublisherError from warehouse.events.tags import EventTag from warehouse.packaging import IFileStorage @@ -106,7 +106,7 @@ def test_publisher_from_oidc_publisher_fails(db_request, monkeypatch): publisher = pretend.stub(publisher_name="not-existing") - with pytest.raises(UnknownPublisherError): + with pytest.raises(UnsupportedPublisherError): publisher_from_oidc_publisher(publisher) diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py new file mode 100644 index 000000000000..01a2c54d8cf5 --- /dev/null +++ b/tests/unit/attestations/test_services.py @@ -0,0 +1,252 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pretend +import pytest + +from pydantic import TypeAdapter +from pypi_attestations import ( + Attestation, + AttestationType, + Envelope, + VerificationError, + VerificationMaterial, +) +from sigstore.verify import Verifier +from zope.interface.verify import verifyClass + +from tests.common.db.packaging import FileFactory +from warehouse.attestations.errors import AttestationUploadError +from warehouse.attestations.interfaces import IAttestationsService +from warehouse.attestations.services import AttestationsService +from warehouse.metrics import IMetricsService +from warehouse.packaging import IFileStorage + +VALID_ATTESTATION = Attestation( + version=1, + verification_material=VerificationMaterial( + certificate="somebase64string", transparency_entries=[dict()] + ), + envelope=Envelope( + statement="somebase64string", + signature="somebase64string", + ), +) + + +class TestAttestationsService: + def test_interface_matches(self): + assert verifyClass(IAttestationsService, AttestationsService) + + def test_create_service(self): + claims = {"claim": "fake"} + request = pretend.stub( + find_service=pretend.call_recorder( + lambda svc, context=None, name=None: { + # IFileStorage: user_service, + # IMetricsService: sender, + }.get(svc) + ), + oidc_claims=claims, + oidc_publisher=pretend.stub(), + ) + + service = AttestationsService.create_service(None, request) + assert not set(request.find_service.calls) ^ { + pretend.call(IFileStorage), + pretend.call(IMetricsService), + } + + assert service.claims == claims + + def test_persist(self, db_request): + @pretend.call_recorder + def storage_service_store(path: str, file_path, *_args, **_kwargs): + expected = VALID_ATTESTATION.model_dump_json().encode("utf-8") + with open(file_path, "rb") as fp: + assert fp.read() == expected + + assert path.endswith(".attestation") + + attestations_service = AttestationsService( + storage=pretend.stub( + store=storage_service_store, + ), + metrics=pretend.stub(), + publisher=pretend.stub(), + claims=pretend.stub(), + ) + + attestations_service.persist([VALID_ATTESTATION], FileFactory.create()) + + def test_parse_no_publisher(self): + attestations_service = AttestationsService( + storage=pretend.stub(), + metrics=pretend.stub(), + publisher=None, + claims=pretend.stub(), + ) + + with pytest.raises( + AttestationUploadError, + match="Attestations are only supported when using Trusted", + ): + attestations_service.parse(pretend.stub(), pretend.stub()) + + def test_parse_unsupported_publisher(self): + attestations_service = AttestationsService( + storage=pretend.stub(), + metrics=pretend.stub(), + publisher=pretend.stub(publisher_name="fake-publisher"), + claims=pretend.stub(), + ) + + with pytest.raises( + AttestationUploadError, + match="Attestations are only supported when using Trusted", + ): + attestations_service.parse(pretend.stub(), pretend.stub()) + + def test_parse_malformed_attestation(self, metrics): + attestations_service = AttestationsService( + storage=pretend.stub(), + metrics=metrics, + publisher=pretend.stub(publisher_name="GitHub"), + claims=pretend.stub(), + ) + + with pytest.raises( + AttestationUploadError, + match="Error while decoding the included attestation", + ): + attestations_service.parse({"malformed-attestation"}, pretend.stub()) + + assert ( + pretend.call("warehouse.upload.attestations.malformed") + in metrics.increment.calls + ) + + def test_parse_multiple_attestations(self, metrics): + attestations_service = AttestationsService( + storage=pretend.stub(), + metrics=metrics, + publisher=pretend.stub( + publisher_name="GitHub", + ), + claims=pretend.stub(), + ) + + with pytest.raises( + AttestationUploadError, match="Only a single attestation per file" + ): + attestations_service.parse( + TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION, VALID_ATTESTATION] + ), + pretend.stub(), + ) + + assert ( + pretend.call("warehouse.upload.attestations.failed_multiple_attestations") + in metrics.increment.calls + ) + + @pytest.mark.parametrize( + "verify_exception, expected_message", + [ + ( + VerificationError, + "Could not verify the uploaded", + ), + ( + ValueError, + "Unknown error while", + ), + ], + ) + def test_parse_failed_verification( + self, metrics, monkeypatch, verify_exception, expected_message + ): + attestations_service = AttestationsService( + storage=pretend.stub(), + metrics=metrics, + publisher=pretend.stub( + publisher_name="GitHub", + publisher_verification_policy=pretend.call_recorder(lambda c: None), + ), + claims=pretend.stub(), + ) + + def failing_verify(_self, _verifier, _policy, _dist): + raise verify_exception("error") + + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) + monkeypatch.setattr(Attestation, "verify", failing_verify) + + with pytest.raises(AttestationUploadError, match=expected_message): + attestations_service.parse( + TypeAdapter(list[Attestation]).dump_json([VALID_ATTESTATION]), + pretend.stub(), + ) + + def test_parse_wrong_predicate(self, metrics, monkeypatch): + attestations_service = AttestationsService( + storage=pretend.stub(), + metrics=metrics, + publisher=pretend.stub( + publisher_name="GitHub", + publisher_verification_policy=pretend.call_recorder(lambda c: None), + ), + claims=pretend.stub(), + ) + + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) + monkeypatch.setattr( + Attestation, "verify", lambda *args: ("wrong-predicate", {}) + ) + + with pytest.raises( + AttestationUploadError, match="Attestation with unsupported predicate" + ): + attestations_service.parse( + TypeAdapter(list[Attestation]).dump_json([VALID_ATTESTATION]), + pretend.stub(), + ) + + assert ( + pretend.call( + "warehouse.upload.attestations.failed_unsupported_predicate_type" + ) + in metrics.increment.calls + ) + + def test_parse_succeed(self, metrics, monkeypatch): + attestations_service = AttestationsService( + storage=pretend.stub(), + metrics=metrics, + publisher=pretend.stub( + publisher_name="GitHub", + publisher_verification_policy=pretend.call_recorder(lambda c: None), + ), + claims=pretend.stub(), + ) + + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) + monkeypatch.setattr( + Attestation, "verify", lambda *args: (AttestationType.PYPI_PUBLISH_V1, {}) + ) + + attestations = attestations_service.parse( + TypeAdapter(list[Attestation]).dump_json([VALID_ATTESTATION]), + pretend.stub(), + ) + assert attestations == [VALID_ATTESTATION] diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index ae146b6bb281..ad3fb078eaee 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -23,15 +23,8 @@ import pretend import pytest -from pypi_attestations import ( - Attestation, - Distribution, - Envelope, - VerificationError, - VerificationMaterial, -) +from pypi_attestations import Attestation, Envelope, VerificationMaterial from pyramid.httpexceptions import HTTPBadRequest, HTTPForbidden, HTTPTooManyRequests -from sigstore.verify import Verifier from sqlalchemy import and_, exists from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import joinedload @@ -42,6 +35,8 @@ from warehouse.accounts.utils import UserContext from warehouse.admin.flags import AdminFlag, AdminFlagValue +from warehouse.attestations.errors import AttestationUploadError +from warehouse.attestations.interfaces import IAttestationsService from warehouse.attestations.models import Attestation as DatabaseAttestation from warehouse.classifiers.models import Classifier from warehouse.forklift import legacy, metadata @@ -65,6 +60,7 @@ from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files from ...common.db.accounts import EmailFactory, UserFactory +from ...common.db.attestation import AttestationFactory from ...common.db.classifiers import ClassifierFactory from ...common.db.oidc import GitHubPublisherFactory from ...common.db.packaging import ( @@ -2425,85 +2421,6 @@ def test_upload_fails_without_oidc_publisher_permission( "See /the/help/url/ for more information." ).format(project.name) - def test_upload_attestation_fails_without_oidc_publisher( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - project_service, - macaroon_service, - ): - project = ProjectFactory.create() - owner = UserFactory.create() - maintainer = UserFactory.create() - RoleFactory.create(user=owner, project=project, role_name="Owner") - RoleFactory.create(user=maintainer, project=project, role_name="Maintainer") - - EmailFactory.create(user=maintainer) - db_request.user = maintainer - raw_macaroon, macaroon = macaroon_service.create_macaroon( - "fake location", - "fake description", - [caveats.RequestUser(user_id=str(maintainer.id))], - user_id=maintainer.id, - ) - identity = UserContext(maintainer, macaroon) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="some_cert", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", - "version": "1.0", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - extract_http_macaroon = pretend.call_recorder(lambda r, _: raw_macaroon) - monkeypatch.setattr( - security_policy, "_extract_http_macaroon", extract_http_macaroon - ) - - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMacaroonService: macaroon_service, - IMetricsService: metrics, - IProjectService: project_service, - }.get(svc) - db_request.user_agent = "warehouse-tests/6.6.6" - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status == ( - "400 Attestations are currently only supported when using Trusted " - "Publishing with GitHub Actions." - ) - @pytest.mark.parametrize( "plat", [ @@ -3412,7 +3329,7 @@ def test_upload_succeeds_creates_release( ), ] - def test_upload_with_valid_attestation_succeeds( + def test_upload_succeeds_with_valid_attestation( self, monkeypatch, pyramid_config, @@ -3469,295 +3386,55 @@ def test_upload_with_valid_attestation_succeeds( ) storage_service = pretend.stub(store=lambda path, filepath, *, meta=None: None) + attestations_service = pretend.stub( + parse=lambda *args, **kwargs: [attestation], + persist_attestations=lambda attestations, file: [ + AttestationFactory.create(file=file) + ], + ) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, + IAttestationsService: attestations_service, }.get(svc) record_event = pretend.call_recorder( lambda self, *, tag, request=None, additional: None ) monkeypatch.setattr(HasEvents, "record_event", record_event) - - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: ( - "https://docs.pypi.org/attestations/publish/v1", - None, - ) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - resp = legacy.file_upload(db_request) assert resp.status_code == 200 - assert len(verify.calls) == 1 - verified_distribution = verify.calls[0].args[3] - assert verified_distribution == Distribution( - name=filename, digest=_TAR_GZ_PKG_SHA256 - ) - - def test_upload_with_invalid_attestation_predicate_type_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None - ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - invalid_predicate_type = "Unsupported predicate type" - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: (invalid_predicate_type, None) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - f"400 Attestation with unsupported predicate type: {invalid_predicate_type}" - ) - - def test_upload_with_multiple_attestations_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}," - f" {attestation.model_dump_json()}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None - ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: ( - "https://docs.pypi.org/attestations/publish/v1", - None, - ) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - "400 Only a single attestation per-file is supported at the moment." - ) - - def test_upload_with_malformed_attestation_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": "[{'a_malformed_attestation': 3}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None + assert ( + pretend.call("warehouse.upload.attestations.ok") in metrics.increment.calls ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - "400 Error while decoding the included attestation:" + attestations_db = ( + db_request.db.query(DatabaseAttestation) + .join(DatabaseAttestation.file) + .filter(File.filename == filename) + .all() ) + assert len(attestations_db) == 1 @pytest.mark.parametrize( - "verify_exception, expected_msg", + "expected_message", [ - ( - VerificationError, - "400 Could not verify the uploaded artifact using the included " - "attestation", - ), - ( - ValueError, - "400 Unknown error while trying to verify included attestations", - ), + ("Attestations are only supported when using",), + ("Error while decoding the included attestation",), + ("Only a single attestation",), + ("Could not verify the uploaded",), + ("Unknown error while trying",), + ("Attestation with unsupported predicate",), ], ) - def test_upload_with_failing_attestation_verification( + def test_upload_fails_attestation_error( self, monkeypatch, pyramid_config, db_request, metrics, - verify_exception, - expected_msg, + expected_message, ): from warehouse.events.models import HasEvents @@ -3777,16 +3454,6 @@ def test_upload_with_failing_attestation_verification( db_request.db.add(Classifier(classifier="Programming Language :: Python")) filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) pyramid_config.testing_securitypolicy(identity=identity) db_request.user = None @@ -3795,7 +3462,7 @@ def test_upload_with_failing_attestation_verification( { "metadata_version": "1.2", "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", + "attestations": "", "version": version, "summary": "This is my summary!", "filetype": "sdist", @@ -3809,9 +3476,15 @@ def test_upload_with_failing_attestation_verification( ) storage_service = pretend.stub(store=lambda path, filepath, meta: None) + + def stub_parse(*_args, **_kwargs): + raise AttestationUploadError(expected_message) + + attestations_service = pretend.stub(parse=stub_parse) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, + IAttestationsService: attestations_service, }.get(svc) record_event = pretend.call_recorder( @@ -3819,104 +3492,13 @@ def test_upload_with_failing_attestation_verification( ) monkeypatch.setattr(HasEvents, "record_event", record_event) - def failing_verify(_self, _verifier, _policy, _dist): - raise verify_exception("error") - - monkeypatch.setattr(Attestation, "verify", failing_verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - with pytest.raises(HTTPBadRequest) as excinfo: legacy.file_upload(db_request) resp = excinfo.value assert resp.status_code == 400 - assert resp.status.startswith(expected_msg) - - def test_upload_succeeds_upload_attestation( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, *, meta=None: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - parse_and_verify_attestations = pretend.call_recorder( - lambda request, distribution: [attestation] - ) - - monkeypatch.setattr( - legacy, "_parse_and_verify_attestations", parse_and_verify_attestations - ) - - resp = legacy.file_upload(db_request) - - assert resp.status_code == 200 - - attestations_db = ( - db_request.db.query(DatabaseAttestation) - .join(DatabaseAttestation.file) - .filter(File.filename == filename) - .all() - ) - assert len(attestations_db) == 1 - - assert ( - pretend.call("warehouse.upload.attestations.ok") in metrics.increment.calls - ) + assert resp.status.startswith(f"400 {expected_message}") @pytest.mark.parametrize( "url, expected", diff --git a/warehouse/attestations/_core.py b/warehouse/attestations/_core.py index 4b8ac101db78..73d3db3463c4 100644 --- a/warehouse/attestations/_core.py +++ b/warehouse/attestations/_core.py @@ -26,7 +26,7 @@ Publisher, ) -from warehouse.attestations.errors import UnknownPublisherError +from warehouse.attestations.errors import UnsupportedPublisherError from warehouse.oidc.models import ( GitHubPublisher as GitHubOIDCPublisher, GitLabPublisher as GitLabOIDCPublisher, @@ -61,7 +61,7 @@ def publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: environment=publisher.environment, ) case _: - raise UnknownPublisherError() + raise UnsupportedPublisherError def generate_and_store_provenance_file( @@ -71,7 +71,7 @@ def generate_and_store_provenance_file( try: publisher: Publisher = publisher_from_oidc_publisher(request.oidc_publisher) - except UnknownPublisherError: + except UnsupportedPublisherError: sentry_sdk.capture_message( f"Unsupported OIDCPublisher found {request.oidc_publisher.publisher_name}" ) diff --git a/warehouse/attestations/errors.py b/warehouse/attestations/errors.py index d04df4e69cbb..463a34a4da69 100644 --- a/warehouse/attestations/errors.py +++ b/warehouse/attestations/errors.py @@ -11,5 +11,9 @@ # limitations under the License. -class UnknownPublisherError(Exception): +class UnsupportedPublisherError(Exception): + pass + + +class AttestationUploadError(Exception): pass diff --git a/warehouse/attestations/interfaces.py b/warehouse/attestations/interfaces.py new file mode 100644 index 000000000000..9c248fc4a835 --- /dev/null +++ b/warehouse/attestations/interfaces.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from pypi_attestations import Attestation, Distribution +from zope.interface import Interface + +from warehouse.packaging.models import File + + +class IAttestationsService(Interface): + + def create_service(context, request): + """ + Create the service, given the context and request for which it is being + created for. + """ + + def persist(attestations: list[Attestation], file: File): + """ + ̀¦Persist attestations in storage. + """ + pass + + def parse(attestation_data, distribution: Distribution) -> list[Attestation]: + """ + Process any attestations included in a file upload request + """ diff --git a/warehouse/attestations/models.py b/warehouse/attestations/models.py index 6bd7d1bfadf0..9b95bfb0d7ad 100644 --- a/warehouse/attestations/models.py +++ b/warehouse/attestations/models.py @@ -41,15 +41,15 @@ class Attestation(db.Model): ) file: Mapped[File] = orm.relationship(back_populates="attestations") - attestation_file_sha256_digest: Mapped[str] = mapped_column(CITEXT) + attestation_file_blake2_digest: Mapped[str] = mapped_column(CITEXT) @hybrid_property def attestation_path(self): return "/".join( [ - self.attestation_file_sha256_digest[:2], - self.attestation_file_sha256_digest[2:4], - self.attestation_file_sha256_digest[4:], + self.attestation_file_blake2_digest[:2], + self.attestation_file_blake2_digest[2:4], + self.attestation_file_blake2_digest[4:], f"{Path(self.file.path).name}.attestation", ] ) diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py new file mode 100644 index 000000000000..28d8d4df5ff6 --- /dev/null +++ b/warehouse/attestations/services.py @@ -0,0 +1,159 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib +import tempfile + +import sentry_sdk + +from pydantic import TypeAdapter, ValidationError +from pypi_attestations import ( + Attestation, + AttestationType, + Distribution, + VerificationError, +) +from sigstore.verify import Verifier +from zope.interface import implementer + +from warehouse.attestations.errors import AttestationUploadError +from warehouse.attestations.interfaces import IAttestationsService +from warehouse.attestations.models import Attestation as DatabaseAttestation +from warehouse.metrics.interfaces import IMetricsService +from warehouse.oidc.interfaces import SignedClaims +from warehouse.oidc.models import OIDCPublisher +from warehouse.packaging.interfaces import IFileStorage +from warehouse.packaging.models import File + + +@implementer(IAttestationsService) +class AttestationsService: + + def __init__( + self, + storage: IFileStorage, + metrics: IMetricsService, + publisher: OIDCPublisher, + claims: SignedClaims, + ): + self.storage: IFileStorage = storage + self.metrics: IMetricsService = metrics + + self.publisher: OIDCPublisher = publisher + self.claims: SignedClaims = claims + + @classmethod + def create_service(cls, _context, request): + return cls( + storage=request.find_service(IFileStorage), + metrics=request.find_service(IMetricsService), + claims=request.oidc_claims, + publisher=request.oidc_publisher, + ) + + def persist( + self, attestations: list[Attestation], file: File + ) -> list[DatabaseAttestation]: + attestations_db_models = [] + for attestation in attestations: + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write(attestation.model_dump_json().encode("utf-8")) + + attestation_digest = hashlib.file_digest( + tmp_file, "blake2b" + ).hexdigest() + database_attestation = DatabaseAttestation( + file=file, attestation_file_blake2_digest=attestation_digest + ) + + self.storage.store( + database_attestation.attestation_path, + tmp_file.name, + meta=None, + ) + + attestations_db_models.append(database_attestation) + + return attestations_db_models + + def parse(self, attestation_data, distribution: Distribution) -> list[Attestation]: + """ + Process any attestations included in a file upload request + + Attestations, if present, will be parsed and verified against the uploaded + artifact. Attestations are only allowed when uploading via a Trusted + Publisher, because a Trusted Publisher provides the identity that will be + used to verify the attestations. + Only GitHub Actions Trusted Publishers are supported. + + Warning, attestation data at the beginning of this function is untrusted. + """ + if not self.publisher or not self.publisher.publisher_name == "GitHub": + raise AttestationUploadError( + "Attestations are only supported when using Trusted " + "Publishing with GitHub Actions.", + ) + + try: + attestations = TypeAdapter(list[Attestation]).validate_json( + attestation_data + ) + except ValidationError as e: + # Log invalid (malformed) attestation upload + self.metrics.increment("warehouse.upload.attestations.malformed") + raise AttestationUploadError( + f"Error while decoding the included attestation: {e}", + ) + + if len(attestations) > 1: + self.metrics.increment( + "warehouse.upload.attestations.failed_multiple_attestations" + ) + + raise AttestationUploadError( + "Only a single attestation per file is supported.", + ) + + verification_policy = self.publisher.publisher_verification_policy(self.claims) + for attestation_model in attestations: + try: + predicate_type, _ = attestation_model.verify( + Verifier.production(), + verification_policy, + distribution, + ) + except VerificationError as e: + # Log invalid (failed verification) attestation upload + self.metrics.increment("warehouse.upload.attestations.failed_verify") + raise AttestationUploadError( + f"Could not verify the uploaded artifact using the included " + f"attestation: {e}", + ) + except Exception as e: + with sentry_sdk.push_scope() as scope: + scope.fingerprint = [e] + sentry_sdk.capture_message( + f"Unexpected error while verifying attestation: {e}" + ) + + raise AttestationUploadError( + f"Unknown error while trying to verify included attestations: {e}", + ) + + if predicate_type != AttestationType.PYPI_PUBLISH_V1: + self.metrics.increment( + "warehouse.upload.attestations.failed_unsupported_predicate_type" + ) + raise AttestationUploadError( + f"Attestation with unsupported predicate type: {predicate_type}", + ) + + return attestations diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index ce530d75ced5..9b3b70c2d889 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -30,13 +30,7 @@ import wtforms import wtforms.validators -from pydantic import TypeAdapter, ValidationError -from pypi_attestations import ( - Attestation, - AttestationType, - Distribution, - VerificationError, -) +from pypi_attestations import Attestation, Distribution from pyramid.httpexceptions import ( HTTPBadRequest, HTTPException, @@ -48,13 +42,13 @@ ) from pyramid.request import Request from pyramid.view import view_config -from sigstore.verify import Verifier from sqlalchemy import and_, exists, func, orm from sqlalchemy.exc import MultipleResultsFound, NoResultFound from warehouse.admin.flags import AdminFlagValue from warehouse.attestations._core import generate_and_store_provenance_file -from warehouse.attestations.models import Attestation as DatabaseAttestation +from warehouse.attestations.errors import AttestationUploadError +from warehouse.attestations.interfaces import IAttestationsService from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB @@ -1172,13 +1166,6 @@ def file_upload(request): k: h.hexdigest().lower() for k, h in metadata_file_hashes.items() } - attestations: list[Attestation] | None = None - if "attestations" in request.POST: - attestations = _parse_and_verify_attestations( - request=request, - distribution=Distribution(name=filename, digest=file_hashes["sha256"]), - ) - # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. @@ -1275,27 +1262,29 @@ def file_upload(request): }, ) - # If the user provided attestations, store them - if attestations: - attestations_db_models = [] - for attestation in attestations: - with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write(attestation.model_dump_json().encode("utf-8")) - - attestation_digest = hashlib.file_digest( - tmp_file, "sha256" - ).hexdigest() - database_attestation = DatabaseAttestation( - file=file_, attestation_file_sha256_digest=attestation_digest - ) + # If the user provided attestations, verify and store them + if "attestations" in request.POST: + attestation_service = request.find_service( + IAttestationsService, context=None + ) - storage.store( - database_attestation.attestation_path, - tmp_file.name, - meta=None, - ) + try: + attestations: list[Attestation] = attestation_service.parse( + attestation_data=request.POST["attestations"], + distribution=Distribution( + name=filename, digest=file_hashes["sha256"] + ), + ) + except AttestationUploadError as e: + raise _exc_with_message( + HTTPBadRequest, + str(e), + ) - attestations_db_models.append(database_attestation) + attestations_db_models = attestation_service.persist_attestations( + attestations=attestations, + file=file_, + ) request.db.add_all(attestations_db_models) generate_and_store_provenance_file( @@ -1437,85 +1426,3 @@ def missing_trailing_slash_redirect(request): "/legacy/ (with a trailing slash)", location=request.route_path("forklift.legacy.file_upload"), ) - - -def _parse_and_verify_attestations( - request, distribution: Distribution -) -> list[Attestation]: - """ - Process any attestations included in a file upload request - - Attestations, if present, will be parsed and verified against the uploaded - artifact. Attestations are only allowed when uploading via a Trusted - Publisher, because a Trusted Publisher provides the identity that will be - used to verify the attestations. - Currently, only GitHub Actions Trusted Publishers are supported. - """ - - metrics = request.find_service(IMetricsService, context=None) - - publisher = request.oidc_publisher - if not publisher or not publisher.publisher_name == "GitHub": - raise _exc_with_message( - HTTPBadRequest, - "Attestations are currently only supported when using Trusted " - "Publishing with GitHub Actions.", - ) - try: - attestations = TypeAdapter(list[Attestation]).validate_json( - request.POST["attestations"] - ) - except ValidationError as e: - # Log invalid (malformed) attestation upload - metrics.increment("warehouse.upload.attestations.malformed") - raise _exc_with_message( - HTTPBadRequest, - f"Error while decoding the included attestation: {e}", - ) - - if len(attestations) > 1: - metrics.increment("warehouse.upload.attestations.failed_multiple_attestations") - raise _exc_with_message( - HTTPBadRequest, - "Only a single attestation per-file is supported at the moment.", - ) - - verification_policy = publisher.publisher_verification_policy(request.oidc_claims) - for attestation_model in attestations: - try: - # For now, attestations are not stored, just verified - predicate_type, _ = attestation_model.verify( - Verifier.production(), - verification_policy, - distribution, - ) - except VerificationError as e: - # Log invalid (failed verification) attestation upload - metrics.increment("warehouse.upload.attestations.failed_verify") - raise _exc_with_message( - HTTPBadRequest, - f"Could not verify the uploaded artifact using the included " - f"attestation: {e}", - ) - except Exception as e: - with sentry_sdk.push_scope() as scope: - scope.fingerprint = [e] - sentry_sdk.capture_message( - f"Unexpected error while verifying attestation: {e}" - ) - - raise _exc_with_message( - HTTPBadRequest, - f"Unknown error while trying to verify included attestations: {e}", - ) - - if predicate_type != AttestationType.PYPI_PUBLISH_V1: - metrics.increment( - "warehouse.upload.attestations.failed_unsupported_predicate_type" - ) - raise _exc_with_message( - HTTPBadRequest, - f"Attestation with unsupported predicate type: {predicate_type}", - ) - - return attestations diff --git a/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py index 5c3d144dc09a..2b15277127f6 100644 --- a/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py +++ b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py @@ -13,7 +13,7 @@ create Attestations table Revision ID: 7f0c9f105f44 -Revises: bb6943882aa9 +Revises: 26455e3712a2 Create Date: 2024-07-25 15:49:01.993869 """ @@ -25,36 +25,13 @@ revision = "7f0c9f105f44" down_revision = "26455e3712a2" -# Note: It is VERY important to ensure that a migration does not lock for a -# long period of time and to ensure that each individual migration does -# not break compatibility with the *previous* version of the code base. -# This is because the migrations will be ran automatically as part of the -# deployment process, but while the previous version of the code is still -# up and running. Thus backwards incompatible changes must be broken up -# over multiple migrations inside of multiple pull requests in order to -# phase them in over multiple deploys. -# -# By default, migrations cannot wait more than 4s on acquiring a lock -# and each individual statement cannot take more than 5s. This helps -# prevent situations where a slow migration takes the entire site down. -# -# If you need to increase this timeout for a migration, you can do so -# by adding: -# -# op.execute("SET statement_timeout = 5000") -# op.execute("SET lock_timeout = 4000") -# -# To whatever values are reasonable for this migration as part of your -# migration. - def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.create_table( "attestation", sa.Column("file_id", sa.UUID(), nullable=False), sa.Column( - "attestation_file_sha256_digest", postgresql.CITEXT(), nullable=False + "attestation_file_blake2_digest", postgresql.CITEXT(), nullable=False ), sa.Column( "id", sa.UUID(), server_default=sa.text("gen_random_uuid()"), nullable=False @@ -64,10 +41,7 @@ def upgrade(): ), sa.PrimaryKeyConstraint("id"), ) - # ### end Alembic commands ### def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.drop_table("attestation") - # ### end Alembic commands ### diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index aa4c1173aa0c..8bc6fbce43de 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -100,7 +100,7 @@ def _simple_detail(project, request): if file.metadata_file_sha256_digest else False ), - "provenance": get_provenance_digest(request, file) + "provenance": get_provenance_digest(request, file), } for file in files ], From 342b8bf42e7fe19bc84cb1b39a78531095ae1ec3 Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 16 Aug 2024 11:18:27 +0200 Subject: [PATCH 23/29] Rename AttestationsService to ReleaseVerification service and integrate Provenance related objects --- tests/unit/api/test_simple.py | 22 ++ tests/unit/attestations/test_core.py | 176 -------------- tests/unit/attestations/test_services.py | 293 +++++++++++++++++------ tests/unit/forklift/test_legacy.py | 26 +- tests/unit/packaging/test_utils.py | 36 +++ warehouse/attestations/__init__.py | 16 ++ warehouse/attestations/_core.py | 96 -------- warehouse/attestations/interfaces.py | 24 +- warehouse/attestations/services.py | 116 +++++++-- warehouse/forklift/legacy.py | 25 +- warehouse/packaging/models.py | 2 +- warehouse/packaging/utils.py | 8 +- 12 files changed, 436 insertions(+), 404 deletions(-) delete mode 100644 tests/unit/attestations/test_core.py delete mode 100644 warehouse/attestations/_core.py diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index dfbba3920b9b..fd0f8c273779 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -18,6 +18,7 @@ from pyramid.testing import DummyRequest from warehouse.api import simple +from warehouse.attestations import IReleaseVerificationService from warehouse.packaging.utils import API_VERSION from ...common.db.accounts import UserFactory @@ -87,6 +88,16 @@ def test_selects(self, header, expected): class TestSimpleIndex: + + @pytest.fixture + def db_request(self, db_request): + """Override db_request to add the Release Verification service""" + db_request.find_service = lambda svc, name=None, context=None: { + IReleaseVerificationService: pretend.stub(), + }.get(svc) + + return db_request + @pytest.mark.parametrize( "content_type,renderer_override", CONTENT_TYPE_PARAMS, @@ -185,6 +196,17 @@ def test_quarantined_project_omitted_from_index(self, db_request): class TestSimpleDetail: + @pytest.fixture + def db_request(self, db_request): + """Override db_request to add the Release Verification service""" + db_request.find_service = lambda svc, name=None, context=None: { + IReleaseVerificationService: pretend.stub( + get_provenance_digest=lambda *args, **kwargs: None, + ), + }.get(svc) + + return db_request + def test_redirects(self, pyramid_request): project = pretend.stub(normalized_name="foo") diff --git a/tests/unit/attestations/test_core.py b/tests/unit/attestations/test_core.py deleted file mode 100644 index 49c15500328c..000000000000 --- a/tests/unit/attestations/test_core.py +++ /dev/null @@ -1,176 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import hashlib -import tempfile - -from pathlib import Path - -import pretend -import pytest - -from pypi_attestations import ( - Attestation, - AttestationBundle, - Envelope, - GitHubPublisher, - GitLabPublisher, - Provenance, - VerificationMaterial, -) - -import warehouse.packaging - -from tests.common.db.oidc import GitHubPublisherFactory, GitLabPublisherFactory -from tests.common.db.packaging import FileEventFactory, FileFactory -from warehouse.attestations._core import ( - generate_and_store_provenance_file, - get_provenance_digest, - publisher_from_oidc_publisher, -) -from warehouse.attestations.errors import UnsupportedPublisherError -from warehouse.events.tags import EventTag -from warehouse.packaging import IFileStorage - - -def test_get_provenance_digest(db_request): - file = FileFactory.create() - FileEventFactory.create( - source=file, - tag=EventTag.File.FileAdd, - additional={"publisher_url": "fake-publisher-url"}, - ) - - with tempfile.NamedTemporaryFile() as f: - storage_service = pretend.stub(get=pretend.call_recorder(lambda p: f)) - - db_request.find_service = pretend.call_recorder( - lambda svc, name=None, context=None: { - IFileStorage: storage_service, - }.get(svc) - ) - - assert ( - get_provenance_digest(db_request, file) - == hashlib.file_digest(f, "sha256").hexdigest() - ) - - -def test_get_provenance_digest_fails_no_publisher_url(db_request): - file = FileFactory.create() - - # If the publisher_url is missing, there is no provenance file - assert get_provenance_digest(db_request, file) is None - - -def test_get_provenance_digest_fails_no_attestations(db_request): - # If the attestations are missing, there is no provenance file - file = FileFactory.create() - file.attestations = [] - FileEventFactory.create( - source=file, - tag=EventTag.File.FileAdd, - additional={"publisher_url": "fake-publisher-url"}, - ) - assert get_provenance_digest(db_request, file) is None - - -def test_publisher_from_oidc_publisher_github(db_request): - publisher = GitHubPublisherFactory.create() - - attestation_publisher = publisher_from_oidc_publisher(publisher) - assert isinstance(attestation_publisher, GitHubPublisher) - assert attestation_publisher.repository == publisher.repository - assert attestation_publisher.workflow == publisher.workflow_filename - assert attestation_publisher.environment == publisher.environment - - -def test_publisher_from_oidc_publisher_gitlab(db_request): - publisher = GitLabPublisherFactory.create() - - attestation_publisher = publisher_from_oidc_publisher(publisher) - assert isinstance(attestation_publisher, GitLabPublisher) - assert attestation_publisher.repository == publisher.project_path - assert attestation_publisher.environment == publisher.environment - - -def test_publisher_from_oidc_publisher_fails(db_request, monkeypatch): - - publisher = pretend.stub(publisher_name="not-existing") - - with pytest.raises(UnsupportedPublisherError): - publisher_from_oidc_publisher(publisher) - - -def test_generate_and_store_provenance_file_no_publisher(db_request): - db_request.find_service = pretend.call_recorder( - lambda svc, name=None, context=None: None - ) - db_request.oidc_publisher = pretend.stub(publisher_name="not-existing") - - assert ( - generate_and_store_provenance_file(db_request, pretend.stub(), pretend.stub()) - is None - ) - - -def test_generate_and_store_provenance_file(db_request, monkeypatch): - - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - publisher = GitHubPublisher( - repository="fake-repository", - workflow="fake-workflow", - ) - provenance = Provenance( - attestation_bundles=[ - AttestationBundle( - publisher=publisher, - attestations=[attestation], - ) - ] - ) - - @pretend.call_recorder - def storage_service_store(path: Path, file_path, *_args, **_kwargs): - expected = provenance.model_dump_json().encode("utf-8") - with open(file_path, "rb") as fp: - assert fp.read() == expected - - assert path.suffix == ".provenance" - - storage_service = pretend.stub(store=storage_service_store) - db_request.find_service = pretend.call_recorder( - lambda svc, name=None, context=None: { - IFileStorage: storage_service, - }.get(svc) - ) - - monkeypatch.setattr( - warehouse.attestations._core, - "publisher_from_oidc_publisher", - lambda s: publisher, - ) - - assert ( - generate_and_store_provenance_file( - db_request, FileFactory.create(), [attestation] - ) - is None - ) diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index 01a2c54d8cf5..5c37d92fd5bb 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -9,6 +9,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import hashlib +import tempfile import pretend import pytest @@ -16,20 +18,31 @@ from pydantic import TypeAdapter from pypi_attestations import ( Attestation, + AttestationBundle, AttestationType, Envelope, + GitHubPublisher, + GitLabPublisher, + Provenance, VerificationError, VerificationMaterial, ) from sigstore.verify import Verifier from zope.interface.verify import verifyClass -from tests.common.db.packaging import FileFactory -from warehouse.attestations.errors import AttestationUploadError -from warehouse.attestations.interfaces import IAttestationsService -from warehouse.attestations.services import AttestationsService +from tests.common.db.oidc import GitHubPublisherFactory, GitLabPublisherFactory +from tests.common.db.packaging import FileEventFactory, FileFactory +from warehouse.attestations import ( + Attestation as DatabaseAttestation, + AttestationUploadError, + IReleaseVerificationService, + ReleaseVerificationService, + UnsupportedPublisherError, + services, +) +from warehouse.events.tags import EventTag from warehouse.metrics import IMetricsService -from warehouse.packaging import IFileStorage +from warehouse.packaging import File, IFileStorage VALID_ATTESTATION = Attestation( version=1, @@ -45,29 +58,21 @@ class TestAttestationsService: def test_interface_matches(self): - assert verifyClass(IAttestationsService, AttestationsService) + assert verifyClass(IReleaseVerificationService, ReleaseVerificationService) def test_create_service(self): - claims = {"claim": "fake"} request = pretend.stub( find_service=pretend.call_recorder( - lambda svc, context=None, name=None: { - # IFileStorage: user_service, - # IMetricsService: sender, - }.get(svc) + lambda svc, context=None, name=None: None ), - oidc_claims=claims, - oidc_publisher=pretend.stub(), ) - service = AttestationsService.create_service(None, request) + assert ReleaseVerificationService.create_service(None, request) is not None assert not set(request.find_service.calls) ^ { pretend.call(IFileStorage), pretend.call(IMetricsService), } - assert service.claims == claims - def test_persist(self, db_request): @pretend.call_recorder def storage_service_store(path: str, file_path, *_args, **_kwargs): @@ -77,81 +82,86 @@ def storage_service_store(path: str, file_path, *_args, **_kwargs): assert path.endswith(".attestation") - attestations_service = AttestationsService( + release_verification = ReleaseVerificationService( storage=pretend.stub( store=storage_service_store, ), metrics=pretend.stub(), - publisher=pretend.stub(), - claims=pretend.stub(), ) + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION] + ) + file = FileFactory.create(attestations=[]) + db_request.oidc_publisher = pretend.stub(publisher_name="GitHub") + release_verification.persist_attestations([VALID_ATTESTATION], file) + + attestations_db = ( + db_request.db.query(DatabaseAttestation) + .join(DatabaseAttestation.file) + .filter(File.filename == file.filename) + .all() + ) + assert len(attestations_db) == 1 - attestations_service.persist([VALID_ATTESTATION], FileFactory.create()) - - def test_parse_no_publisher(self): - attestations_service = AttestationsService( + def test_parse_no_publisher(self, db_request): + release_verification = ReleaseVerificationService( storage=pretend.stub(), metrics=pretend.stub(), - publisher=None, - claims=pretend.stub(), ) + db_request.oidc_publisher = None with pytest.raises( AttestationUploadError, match="Attestations are only supported when using Trusted", ): - attestations_service.parse(pretend.stub(), pretend.stub()) + release_verification.parse_attestations(db_request, pretend.stub()) - def test_parse_unsupported_publisher(self): - attestations_service = AttestationsService( + def test_parse_unsupported_publisher(self, db_request): + release_verification = ReleaseVerificationService( storage=pretend.stub(), metrics=pretend.stub(), - publisher=pretend.stub(publisher_name="fake-publisher"), - claims=pretend.stub(), ) - + db_request.oidc_publisher = pretend.stub(publisher_name="not-existing") with pytest.raises( AttestationUploadError, match="Attestations are only supported when using Trusted", ): - attestations_service.parse(pretend.stub(), pretend.stub()) + release_verification.parse_attestations(db_request, pretend.stub()) - def test_parse_malformed_attestation(self, metrics): - attestations_service = AttestationsService( + def test_parse_malformed_attestation(self, metrics, db_request): + release_verification = ReleaseVerificationService( storage=pretend.stub(), metrics=metrics, - publisher=pretend.stub(publisher_name="GitHub"), - claims=pretend.stub(), ) + db_request.oidc_publisher = pretend.stub(publisher_name="GitHub") + db_request.POST["attestations"] = "{'malformed-attestation'}" with pytest.raises( AttestationUploadError, match="Error while decoding the included attestation", ): - attestations_service.parse({"malformed-attestation"}, pretend.stub()) + release_verification.parse_attestations(db_request, pretend.stub()) assert ( pretend.call("warehouse.upload.attestations.malformed") in metrics.increment.calls ) - def test_parse_multiple_attestations(self, metrics): - attestations_service = AttestationsService( + def test_parse_multiple_attestations(self, metrics, db_request): + release_verification = ReleaseVerificationService( storage=pretend.stub(), metrics=metrics, - publisher=pretend.stub( - publisher_name="GitHub", - ), - claims=pretend.stub(), ) + db_request.oidc_publisher = pretend.stub(publisher_name="GitHub") + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION, VALID_ATTESTATION] + ) with pytest.raises( AttestationUploadError, match="Only a single attestation per file" ): - attestations_service.parse( - TypeAdapter(list[Attestation]).dump_json( - [VALID_ATTESTATION, VALID_ATTESTATION] - ), + release_verification.parse_attestations( + db_request, pretend.stub(), ) @@ -174,16 +184,20 @@ def test_parse_multiple_attestations(self, metrics): ], ) def test_parse_failed_verification( - self, metrics, monkeypatch, verify_exception, expected_message + self, metrics, monkeypatch, db_request, verify_exception, expected_message ): - attestations_service = AttestationsService( + release_verification = ReleaseVerificationService( storage=pretend.stub(), metrics=metrics, - publisher=pretend.stub( - publisher_name="GitHub", - publisher_verification_policy=pretend.call_recorder(lambda c: None), - ), - claims=pretend.stub(), + ) + + db_request.oidc_publisher = pretend.stub( + publisher_name="GitHub", + publisher_verification_policy=pretend.call_recorder(lambda c: None), + ) + db_request.oidc_claims = {"sha": "somesha"} + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION] ) def failing_verify(_self, _verifier, _policy, _dist): @@ -193,20 +207,23 @@ def failing_verify(_self, _verifier, _policy, _dist): monkeypatch.setattr(Attestation, "verify", failing_verify) with pytest.raises(AttestationUploadError, match=expected_message): - attestations_service.parse( - TypeAdapter(list[Attestation]).dump_json([VALID_ATTESTATION]), + release_verification.parse_attestations( + db_request, pretend.stub(), ) - def test_parse_wrong_predicate(self, metrics, monkeypatch): - attestations_service = AttestationsService( + def test_parse_wrong_predicate(self, metrics, monkeypatch, db_request): + release_verification = ReleaseVerificationService( storage=pretend.stub(), metrics=metrics, - publisher=pretend.stub( - publisher_name="GitHub", - publisher_verification_policy=pretend.call_recorder(lambda c: None), - ), - claims=pretend.stub(), + ) + db_request.oidc_publisher = pretend.stub( + publisher_name="GitHub", + publisher_verification_policy=pretend.call_recorder(lambda c: None), + ) + db_request.oidc_claims = {"sha": "somesha"} + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION] ) monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) @@ -217,8 +234,8 @@ def test_parse_wrong_predicate(self, metrics, monkeypatch): with pytest.raises( AttestationUploadError, match="Attestation with unsupported predicate" ): - attestations_service.parse( - TypeAdapter(list[Attestation]).dump_json([VALID_ATTESTATION]), + release_verification.parse_attestations( + db_request, pretend.stub(), ) @@ -229,15 +246,18 @@ def test_parse_wrong_predicate(self, metrics, monkeypatch): in metrics.increment.calls ) - def test_parse_succeed(self, metrics, monkeypatch): - attestations_service = AttestationsService( + def test_parse_succeed(self, metrics, monkeypatch, db_request): + release_verification = ReleaseVerificationService( storage=pretend.stub(), metrics=metrics, - publisher=pretend.stub( - publisher_name="GitHub", - publisher_verification_policy=pretend.call_recorder(lambda c: None), - ), - claims=pretend.stub(), + ) + db_request.oidc_publisher = pretend.stub( + publisher_name="GitHub", + publisher_verification_policy=pretend.call_recorder(lambda c: None), + ) + db_request.oidc_claims = {"sha": "somesha"} + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION] ) monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) @@ -245,8 +265,133 @@ def test_parse_succeed(self, metrics, monkeypatch): Attestation, "verify", lambda *args: (AttestationType.PYPI_PUBLISH_V1, {}) ) - attestations = attestations_service.parse( - TypeAdapter(list[Attestation]).dump_json([VALID_ATTESTATION]), + attestations = release_verification.parse_attestations( + db_request, pretend.stub(), ) assert attestations == [VALID_ATTESTATION] + + def test_get_provenance_digest(self, db_request): + file = FileFactory.create() + FileEventFactory.create( + source=file, + tag=EventTag.File.FileAdd, + additional={"publisher_url": "fake-publisher-url"}, + ) + + with tempfile.NamedTemporaryFile() as f: + release_verification = ReleaseVerificationService( + storage=pretend.stub(get=pretend.call_recorder(lambda p: f)), + metrics=pretend.stub(), + ) + + assert ( + release_verification.get_provenance_digest(file) + == hashlib.file_digest(f, "sha256").hexdigest() + ) + + def test_get_provenance_digest_fails_no_publisher_url(self, db_request): + release_verification = ReleaseVerificationService( + storage=pretend.stub(), + metrics=pretend.stub(), + ) + + # If the publisher_url is missing, there is no provenance file + assert release_verification.get_provenance_digest(FileFactory.create()) is None + + def test_get_provenance_digest_fails_no_attestations(self, db_request): + # If the attestations are missing, there is no provenance file + file = FileFactory.create() + file.attestations = [] + FileEventFactory.create( + source=file, + tag=EventTag.File.FileAdd, + additional={"publisher_url": "fake-publisher-url"}, + ) + release_verification = ReleaseVerificationService( + storage=pretend.stub(), + metrics=pretend.stub(), + ) + + assert release_verification.get_provenance_digest(file) is None + + def test_generate_and_store_provenance_file_no_publisher(self): + release_verification = ReleaseVerificationService( + storage=pretend.stub(), + metrics=pretend.stub(), + ) + + oidc_publisher = pretend.stub(publisher_name="not-existing") + + assert ( + release_verification.generate_and_store_provenance_file( + oidc_publisher, pretend.stub(), pretend.stub() + ) + is None + ) + + def test_generate_and_store_provenance_file(self, db_request, monkeypatch): + + publisher = GitHubPublisher( + repository="fake-repository", + workflow="fake-workflow", + ) + provenance = Provenance( + attestation_bundles=[ + AttestationBundle( + publisher=publisher, + attestations=[VALID_ATTESTATION], + ) + ] + ) + + @pretend.call_recorder + def storage_service_store(path, file_path, *_args, **_kwargs): + expected = provenance.model_dump_json().encode("utf-8") + with open(file_path, "rb") as fp: + assert fp.read() == expected + + assert path.suffix == ".provenance" + + monkeypatch.setattr( + services, + "_publisher_from_oidc_publisher", + lambda s: publisher, + ) + + release_verification = ReleaseVerificationService( + storage=pretend.stub(store=storage_service_store), + metrics=pretend.stub(), + ) + assert ( + release_verification.generate_and_store_provenance_file( + pretend.stub(), FileFactory.create(), [VALID_ATTESTATION] + ) + is None + ) + + +def test_publisher_from_oidc_publisher_github(db_request): + publisher = GitHubPublisherFactory.create() + + attestation_publisher = services._publisher_from_oidc_publisher(publisher) + assert isinstance(attestation_publisher, GitHubPublisher) + assert attestation_publisher.repository == publisher.repository + assert attestation_publisher.workflow == publisher.workflow_filename + assert attestation_publisher.environment == publisher.environment + + +def test_publisher_from_oidc_publisher_gitlab(db_request): + publisher = GitLabPublisherFactory.create() + + attestation_publisher = services._publisher_from_oidc_publisher(publisher) + assert isinstance(attestation_publisher, GitLabPublisher) + assert attestation_publisher.repository == publisher.project_path + assert attestation_publisher.environment == publisher.environment + + +def test_publisher_from_oidc_publisher_fails(): + publisher = pretend.stub(publisher_name="not-existing") + + with pytest.raises(UnsupportedPublisherError): + services._publisher_from_oidc_publisher(publisher) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index ad3fb078eaee..127f81214fc2 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -35,9 +35,11 @@ from warehouse.accounts.utils import UserContext from warehouse.admin.flags import AdminFlag, AdminFlagValue -from warehouse.attestations.errors import AttestationUploadError -from warehouse.attestations.interfaces import IAttestationsService -from warehouse.attestations.models import Attestation as DatabaseAttestation +from warehouse.attestations import ( + Attestation as DatabaseAttestation, + AttestationUploadError, + IReleaseVerificationService, +) from warehouse.classifiers.models import Classifier from warehouse.forklift import legacy, metadata from warehouse.macaroons import IMacaroonService, caveats, security_policy @@ -3385,17 +3387,19 @@ def test_upload_succeeds_with_valid_attestation( } ) + def persist_attestations(attestations, file): + file.attestations.append(AttestationFactory.create(file=file)) + storage_service = pretend.stub(store=lambda path, filepath, *, meta=None: None) - attestations_service = pretend.stub( - parse=lambda *args, **kwargs: [attestation], - persist_attestations=lambda attestations, file: [ - AttestationFactory.create(file=file) - ], + release_verification = pretend.stub( + parse_attestations=lambda *args, **kwargs: [attestation], + persist_attestations=persist_attestations, + generate_and_store_provenance_file=lambda p, f, a: None, ) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, - IAttestationsService: attestations_service, + IReleaseVerificationService: release_verification, }.get(svc) record_event = pretend.call_recorder( @@ -3480,11 +3484,11 @@ def test_upload_fails_attestation_error( def stub_parse(*_args, **_kwargs): raise AttestationUploadError(expected_message) - attestations_service = pretend.stub(parse=stub_parse) + release_verification = pretend.stub(parse_attestations=stub_parse) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, - IAttestationsService: attestations_service, + IReleaseVerificationService: release_verification, }.get(svc) record_event = pretend.call_recorder( diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index afa7bd2056fe..f953d66be1da 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -15,6 +15,7 @@ import pretend +from warehouse.attestations import IReleaseVerificationService from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.utils import _simple_detail, render_simple_detail @@ -27,11 +28,37 @@ def test_simple_detail_empty_string(db_request): FileFactory.create(release=release) db_request.route_url = lambda *a, **kw: "the-url" + db_request.find_service = lambda svc, name=None, context=None: { + IReleaseVerificationService: pretend.stub( + get_provenance_digest=pretend.call_recorder(lambda f: None), + ), + }.get(svc) + expected_content = _simple_detail(project, db_request) assert expected_content["files"][0]["requires-python"] is None +def test_simple_detail_with_provenance(db_request): + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0") + FileFactory.create(release=release) + + hash_digest = "deadbeefdeadbeefdeadbeefdeadbeef" + + db_request.route_url = lambda *a, **kw: "the-url" + db_request.find_service = pretend.call_recorder( + lambda svc, name=None, context=None: { + IReleaseVerificationService: pretend.stub( + get_provenance_digest=pretend.call_recorder(lambda f: hash_digest), + ), + }.get(svc) + ) + + expected_content = _simple_detail(project, db_request) + assert expected_content["files"][0]["provenance"] == hash_digest + + def test_render_simple_detail(db_request, monkeypatch, jinja): project = ProjectFactory.create() release1 = ReleaseFactory.create(project=project, version="1.0") @@ -49,6 +76,12 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): monkeypatch.setattr(hashlib, "blake2b", fakeblake2b) db_request.route_url = lambda *a, **kw: "the-url" + db_request.find_service = lambda svc, name=None, context=None: { + IReleaseVerificationService: pretend.stub( + get_provenance_digest=pretend.call_recorder(lambda f: None), + ), + }.get(svc) + template = jinja.get_template("templates/api/simple/detail.html") expected_content = template.render( **_simple_detail(project, db_request), request=db_request @@ -78,6 +111,9 @@ def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): db_request.find_service = pretend.call_recorder( lambda svc, name=None, context=None: { ISimpleStorage: storage_service, + IReleaseVerificationService: pretend.stub( + get_provenance_digest=pretend.call_recorder(lambda f: None), + ), }.get(svc) ) diff --git a/warehouse/attestations/__init__.py b/warehouse/attestations/__init__.py index 164f68b09175..ca8a54fde2de 100644 --- a/warehouse/attestations/__init__.py +++ b/warehouse/attestations/__init__.py @@ -9,3 +9,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from warehouse.attestations.errors import ( + AttestationUploadError, + UnsupportedPublisherError, +) +from warehouse.attestations.interfaces import IReleaseVerificationService +from warehouse.attestations.models import Attestation +from warehouse.attestations.services import ReleaseVerificationService + +__all__ = [ + "Attestation", + "AttestationUploadError", + "IReleaseVerificationService", + "ReleaseVerificationService", + "UnsupportedPublisherError", +] diff --git a/warehouse/attestations/_core.py b/warehouse/attestations/_core.py deleted file mode 100644 index 73d3db3463c4..000000000000 --- a/warehouse/attestations/_core.py +++ /dev/null @@ -1,96 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import hashlib -import tempfile -import typing - -from pathlib import Path - -import sentry_sdk - -from pypi_attestations import ( - Attestation, - AttestationBundle, - GitHubPublisher, - GitLabPublisher, - Provenance, - Publisher, -) - -from warehouse.attestations.errors import UnsupportedPublisherError -from warehouse.oidc.models import ( - GitHubPublisher as GitHubOIDCPublisher, - GitLabPublisher as GitLabOIDCPublisher, - OIDCPublisher, -) -from warehouse.packaging import File, IFileStorage - - -def get_provenance_digest(request, file: File) -> str | None: - """Returns the sha256 digest of the provenance file for the release.""" - if not file.attestations or not file.publisher_url: - return None - - storage = request.find_service(IFileStorage) - provenance_file = storage.get(f"{file.path}.provenance") - - return hashlib.file_digest(provenance_file, "sha256").hexdigest() - - -def publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: - match publisher.publisher_name: - case "GitLab": - publisher = typing.cast(GitLabOIDCPublisher, publisher) - return GitLabPublisher( - repository=publisher.project_path, environment=publisher.environment - ) - case "GitHub": - publisher = typing.cast(GitHubOIDCPublisher, publisher) - return GitHubPublisher( - repository=publisher.repository, - workflow=publisher.workflow_filename, - environment=publisher.environment, - ) - case _: - raise UnsupportedPublisherError - - -def generate_and_store_provenance_file( - request, file: File, attestations: list[Attestation] -): - storage = request.find_service(IFileStorage) - - try: - publisher: Publisher = publisher_from_oidc_publisher(request.oidc_publisher) - except UnsupportedPublisherError: - sentry_sdk.capture_message( - f"Unsupported OIDCPublisher found {request.oidc_publisher.publisher_name}" - ) - - return - - attestation_bundle = AttestationBundle( - publisher=publisher, - attestations=attestations, - ) - - provenance = Provenance(attestation_bundles=[attestation_bundle]) - - provenance_file_path = Path(f"{file.path}.provenance") - with tempfile.NamedTemporaryFile() as f: - f.write(provenance.model_dump_json().encode("utf-8")) - f.flush() - - storage.store( - provenance_file_path, - f.name, - ) diff --git a/warehouse/attestations/interfaces.py b/warehouse/attestations/interfaces.py index 9c248fc4a835..5992243b5352 100644 --- a/warehouse/attestations/interfaces.py +++ b/warehouse/attestations/interfaces.py @@ -9,15 +9,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import TYPE_CHECKING from pypi_attestations import Attestation, Distribution +from pyramid.request import Request from zope.interface import Interface -from warehouse.packaging.models import File - -class IAttestationsService(Interface): +class IReleaseVerificationService(Interface): def create_service(context, request): """ @@ -25,13 +23,27 @@ def create_service(context, request): created for. """ - def persist(attestations: list[Attestation], file: File): + def persist_attestations(attestations: list[Attestation], file): """ ̀¦Persist attestations in storage. """ pass - def parse(attestation_data, distribution: Distribution) -> list[Attestation]: + def parse_attestations( + request: Request, distribution: Distribution + ) -> list[Attestation]: """ Process any attestations included in a file upload request """ + + def generate_and_store_provenance_file( + oidc_publisher, file, attestations: list[Attestation] + ) -> None: + """ + Generate and store a provenance file for a release, its attestations and an OIDCPublisher. + """ + + def get_provenance_digest(file) -> str | None: + """ + Compute a provenance file digest for a `File` if it exists. + """ diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py index 28d8d4df5ff6..b1dee8943f46 100644 --- a/warehouse/attestations/services.py +++ b/warehouse/attestations/services.py @@ -11,58 +11,84 @@ # limitations under the License. import hashlib import tempfile +import typing + +from pathlib import Path import sentry_sdk from pydantic import TypeAdapter, ValidationError from pypi_attestations import ( Attestation, + AttestationBundle, AttestationType, Distribution, + GitHubPublisher, + GitLabPublisher, + Provenance, + Publisher, VerificationError, ) +from pyramid.request import Request from sigstore.verify import Verifier from zope.interface import implementer -from warehouse.attestations.errors import AttestationUploadError -from warehouse.attestations.interfaces import IAttestationsService +from warehouse.attestations.errors import ( + AttestationUploadError, + UnsupportedPublisherError, +) +from warehouse.attestations.interfaces import IReleaseVerificationService from warehouse.attestations.models import Attestation as DatabaseAttestation from warehouse.metrics.interfaces import IMetricsService -from warehouse.oidc.interfaces import SignedClaims -from warehouse.oidc.models import OIDCPublisher +from warehouse.oidc.models import ( + GitHubPublisher as GitHubOIDCPublisher, + GitLabPublisher as GitLabOIDCPublisher, + OIDCPublisher, +) from warehouse.packaging.interfaces import IFileStorage from warehouse.packaging.models import File -@implementer(IAttestationsService) -class AttestationsService: +def _publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: + """ + Convert an OIDCPublisher object in a pypy-attestations Publisher. + """ + match publisher.publisher_name: + case "GitLab": + publisher = typing.cast(GitLabOIDCPublisher, publisher) + return GitLabPublisher( + repository=publisher.project_path, environment=publisher.environment + ) + case "GitHub": + publisher = typing.cast(GitHubOIDCPublisher, publisher) + return GitHubPublisher( + repository=publisher.repository, + workflow=publisher.workflow_filename, + environment=publisher.environment, + ) + case _: + raise UnsupportedPublisherError + + +@implementer(IReleaseVerificationService) +class ReleaseVerificationService: def __init__( self, storage: IFileStorage, metrics: IMetricsService, - publisher: OIDCPublisher, - claims: SignedClaims, ): self.storage: IFileStorage = storage self.metrics: IMetricsService = metrics - self.publisher: OIDCPublisher = publisher - self.claims: SignedClaims = claims - @classmethod - def create_service(cls, _context, request): + def create_service(cls, _context, request: Request): return cls( storage=request.find_service(IFileStorage), metrics=request.find_service(IMetricsService), - claims=request.oidc_claims, - publisher=request.oidc_publisher, ) - def persist( - self, attestations: list[Attestation], file: File - ) -> list[DatabaseAttestation]: - attestations_db_models = [] + def persist_attestations(self, attestations: list[Attestation], file: File) -> None: for attestation in attestations: with tempfile.NamedTemporaryFile() as tmp_file: tmp_file.write(attestation.model_dump_json().encode("utf-8")) @@ -80,11 +106,11 @@ def persist( meta=None, ) - attestations_db_models.append(database_attestation) - - return attestations_db_models + file.attestations.append(database_attestation) - def parse(self, attestation_data, distribution: Distribution) -> list[Attestation]: + def parse_attestations( + self, request: Request, distribution: Distribution + ) -> list[Attestation]: """ Process any attestations included in a file upload request @@ -96,7 +122,8 @@ def parse(self, attestation_data, distribution: Distribution) -> list[Attestatio Warning, attestation data at the beginning of this function is untrusted. """ - if not self.publisher or not self.publisher.publisher_name == "GitHub": + publisher: OIDCPublisher | None = request.oidc_publisher + if not publisher or not publisher.publisher_name == "GitHub": raise AttestationUploadError( "Attestations are only supported when using Trusted " "Publishing with GitHub Actions.", @@ -104,7 +131,7 @@ def parse(self, attestation_data, distribution: Distribution) -> list[Attestatio try: attestations = TypeAdapter(list[Attestation]).validate_json( - attestation_data + request.POST["attestations"] ) except ValidationError as e: # Log invalid (malformed) attestation upload @@ -122,7 +149,9 @@ def parse(self, attestation_data, distribution: Distribution) -> list[Attestatio "Only a single attestation per file is supported.", ) - verification_policy = self.publisher.publisher_verification_policy(self.claims) + verification_policy = publisher.publisher_verification_policy( + request.oidc_claims + ) for attestation_model in attestations: try: predicate_type, _ = attestation_model.verify( @@ -157,3 +186,40 @@ def parse(self, attestation_data, distribution: Distribution) -> list[Attestatio ) return attestations + + def generate_and_store_provenance_file( + self, oidc_publisher: OIDCPublisher, file: File, attestations: list[Attestation] + ) -> None: + try: + publisher: Publisher = _publisher_from_oidc_publisher(oidc_publisher) + except UnsupportedPublisherError: + sentry_sdk.capture_message( + f"Unsupported OIDCPublisher found {oidc_publisher.publisher_name}" + ) + + return + + attestation_bundle = AttestationBundle( + publisher=publisher, + attestations=attestations, + ) + + provenance = Provenance(attestation_bundles=[attestation_bundle]) + + provenance_file_path = Path(f"{file.path}.provenance") + with tempfile.NamedTemporaryFile() as f: + f.write(provenance.model_dump_json().encode("utf-8")) + f.flush() + + self.storage.store( + provenance_file_path, + f.name, + ) + + def get_provenance_digest(self, file: File) -> str | None: + """Returns the sha256 digest of the provenance file for the release.""" + if not file.attestations or not file.publisher_url: + return None + + provenance_file = self.storage.get(f"{file.path}.provenance") + return hashlib.file_digest(provenance_file, "sha256").hexdigest() diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 9b3b70c2d889..1f36de8c2971 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -46,9 +46,7 @@ from sqlalchemy.exc import MultipleResultsFound, NoResultFound from warehouse.admin.flags import AdminFlagValue -from warehouse.attestations._core import generate_and_store_provenance_file -from warehouse.attestations.errors import AttestationUploadError -from warehouse.attestations.interfaces import IAttestationsService +from warehouse.attestations import AttestationUploadError, IReleaseVerificationService from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB @@ -1265,15 +1263,17 @@ def file_upload(request): # If the user provided attestations, verify and store them if "attestations" in request.POST: attestation_service = request.find_service( - IAttestationsService, context=None + IReleaseVerificationService, context=None ) try: - attestations: list[Attestation] = attestation_service.parse( - attestation_data=request.POST["attestations"], - distribution=Distribution( - name=filename, digest=file_hashes["sha256"] - ), + attestations: list[Attestation] = ( + attestation_service.parse_attestations( + request, + Distribution( + name=filename, digest=file_hashes["sha256"] + ), + ) ) except AttestationUploadError as e: raise _exc_with_message( @@ -1281,14 +1281,13 @@ def file_upload(request): str(e), ) - attestations_db_models = attestation_service.persist_attestations( + attestation_service.persist_attestations( attestations=attestations, file=file_, ) - request.db.add_all(attestations_db_models) - generate_and_store_provenance_file( - request, + attestation_service.generate_and_store_provenance_file( + request.oidc_publisher, file_, attestations, ) diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 46e198ae0eed..caffc7648dc5 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -62,7 +62,6 @@ from warehouse import db from warehouse.accounts.models import User -from warehouse.attestations.models import Attestation from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.events.models import HasEvents @@ -83,6 +82,7 @@ from warehouse.utils.db.types import bool_false, datetime_now if typing.TYPE_CHECKING: + from warehouse.attestations.models import Attestation from warehouse.oidc.models import OIDCPublisher _MONOTONIC_SEQUENCE = 42 diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 8bc6fbce43de..e08d68f64eec 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -19,7 +19,7 @@ from pyramid_jinja2 import IJinja2Environment from sqlalchemy.orm import joinedload -from warehouse.attestations._core import get_provenance_digest +from warehouse.attestations import IReleaseVerificationService from warehouse.attestations.models import Attestation from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.models import File, LifecycleStatus, Project, Release @@ -67,6 +67,10 @@ def _simple_detail(project, request): {f.release.version for f in files}, key=packaging_legacy.version.parse ) + release_verification = request.find_service( + IReleaseVerificationService, context=None + ) + return { "meta": {"api-version": API_VERSION, "_last-serial": project.last_serial}, "name": project.normalized_name, @@ -100,7 +104,7 @@ def _simple_detail(project, request): if file.metadata_file_sha256_digest else False ), - "provenance": get_provenance_digest(request, file), + "provenance": release_verification.get_provenance_digest(file), } for file in files ], From 6e4c91e5fc22898ef1987f7aaf5996f828c9ce7b Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 16 Aug 2024 11:37:54 +0200 Subject: [PATCH 24/29] Remove useless check --- tests/unit/attestations/test_services.py | 9 --------- warehouse/attestations/services.py | 4 +--- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index 5c37d92fd5bb..84c6ba5243e5 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -290,15 +290,6 @@ def test_get_provenance_digest(self, db_request): == hashlib.file_digest(f, "sha256").hexdigest() ) - def test_get_provenance_digest_fails_no_publisher_url(self, db_request): - release_verification = ReleaseVerificationService( - storage=pretend.stub(), - metrics=pretend.stub(), - ) - - # If the publisher_url is missing, there is no provenance file - assert release_verification.get_provenance_digest(FileFactory.create()) is None - def test_get_provenance_digest_fails_no_attestations(self, db_request): # If the attestations are missing, there is no provenance file file = FileFactory.create() diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py index b1dee8943f46..69d577350865 100644 --- a/warehouse/attestations/services.py +++ b/warehouse/attestations/services.py @@ -119,8 +119,6 @@ def parse_attestations( Publisher, because a Trusted Publisher provides the identity that will be used to verify the attestations. Only GitHub Actions Trusted Publishers are supported. - - Warning, attestation data at the beginning of this function is untrusted. """ publisher: OIDCPublisher | None = request.oidc_publisher if not publisher or not publisher.publisher_name == "GitHub": @@ -218,7 +216,7 @@ def generate_and_store_provenance_file( def get_provenance_digest(self, file: File) -> str | None: """Returns the sha256 digest of the provenance file for the release.""" - if not file.attestations or not file.publisher_url: + if not file.attestations: return None provenance_file = self.storage.get(f"{file.path}.provenance") From 80988d018f1d62a330b327fba417548e098c5562 Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 16 Aug 2024 12:01:08 +0200 Subject: [PATCH 25/29] Integrate generate_and_store_provenance within persist_attestations --- tests/unit/attestations/test_services.py | 18 ++++++++++++++---- tests/unit/forklift/test_legacy.py | 4 ++-- warehouse/attestations/interfaces.py | 4 ++-- warehouse/attestations/services.py | 8 ++++++-- warehouse/forklift/legacy.py | 13 ++----------- 5 files changed, 26 insertions(+), 21 deletions(-) diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index 84c6ba5243e5..32fad2131e72 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -30,6 +30,8 @@ from sigstore.verify import Verifier from zope.interface.verify import verifyClass +import warehouse.attestations.services + from tests.common.db.oidc import GitHubPublisherFactory, GitLabPublisherFactory from tests.common.db.packaging import FileEventFactory, FileFactory from warehouse.attestations import ( @@ -73,7 +75,7 @@ def test_create_service(self): pretend.call(IMetricsService), } - def test_persist(self, db_request): + def test_persist_attestations(self, db_request, monkeypatch): @pretend.call_recorder def storage_service_store(path: str, file_path, *_args, **_kwargs): expected = VALID_ATTESTATION.model_dump_json().encode("utf-8") @@ -92,8 +94,16 @@ def storage_service_store(path: str, file_path, *_args, **_kwargs): [VALID_ATTESTATION] ) file = FileFactory.create(attestations=[]) - db_request.oidc_publisher = pretend.stub(publisher_name="GitHub") - release_verification.persist_attestations([VALID_ATTESTATION], file) + + monkeypatch.setattr( + warehouse.attestations.services.ReleaseVerificationService, + "generate_and_store_provenance_file", + lambda *args, **kwargs: None, + ) + + release_verification.persist_attestations( + pretend.stub(), [VALID_ATTESTATION], file + ) attestations_db = ( db_request.db.query(DatabaseAttestation) @@ -356,7 +366,7 @@ def storage_service_store(path, file_path, *_args, **_kwargs): ) assert ( release_verification.generate_and_store_provenance_file( - pretend.stub(), FileFactory.create(), [VALID_ATTESTATION] + pretend.stub(), [VALID_ATTESTATION], FileFactory.create() ) is None ) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 127f81214fc2..43b7ce416ec7 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -3387,14 +3387,14 @@ def test_upload_succeeds_with_valid_attestation( } ) - def persist_attestations(attestations, file): + def persist_attestations(oidc_publisher, attestations, file): file.attestations.append(AttestationFactory.create(file=file)) storage_service = pretend.stub(store=lambda path, filepath, *, meta=None: None) release_verification = pretend.stub( parse_attestations=lambda *args, **kwargs: [attestation], persist_attestations=persist_attestations, - generate_and_store_provenance_file=lambda p, f, a: None, + generate_and_store_provenance_file=lambda p, a, f: None, ) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, diff --git a/warehouse/attestations/interfaces.py b/warehouse/attestations/interfaces.py index 5992243b5352..7973592a606c 100644 --- a/warehouse/attestations/interfaces.py +++ b/warehouse/attestations/interfaces.py @@ -23,7 +23,7 @@ def create_service(context, request): created for. """ - def persist_attestations(attestations: list[Attestation], file): + def persist_attestations(oidc_publisher, attestations: list[Attestation], file): """ ̀¦Persist attestations in storage. """ @@ -37,7 +37,7 @@ def parse_attestations( """ def generate_and_store_provenance_file( - oidc_publisher, file, attestations: list[Attestation] + oidc_publisher, attestations: list[Attestation], file ) -> None: """ Generate and store a provenance file for a release, its attestations and an OIDCPublisher. diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py index 69d577350865..d221b281a0b5 100644 --- a/warehouse/attestations/services.py +++ b/warehouse/attestations/services.py @@ -88,7 +88,9 @@ def create_service(cls, _context, request: Request): metrics=request.find_service(IMetricsService), ) - def persist_attestations(self, attestations: list[Attestation], file: File) -> None: + def persist_attestations( + self, oidc_publisher: OIDCPublisher, attestations: list[Attestation], file: File + ) -> None: for attestation in attestations: with tempfile.NamedTemporaryFile() as tmp_file: tmp_file.write(attestation.model_dump_json().encode("utf-8")) @@ -108,6 +110,8 @@ def persist_attestations(self, attestations: list[Attestation], file: File) -> N file.attestations.append(database_attestation) + self.generate_and_store_provenance_file(oidc_publisher, attestations, file) + def parse_attestations( self, request: Request, distribution: Distribution ) -> list[Attestation]: @@ -186,7 +190,7 @@ def parse_attestations( return attestations def generate_and_store_provenance_file( - self, oidc_publisher: OIDCPublisher, file: File, attestations: list[Attestation] + self, oidc_publisher: OIDCPublisher, attestations: list[Attestation], file: File ) -> None: try: publisher: Publisher = _publisher_from_oidc_publisher(oidc_publisher) diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 1f36de8c2971..0cad399e5ab5 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -1270,9 +1270,7 @@ def file_upload(request): attestations: list[Attestation] = ( attestation_service.parse_attestations( request, - Distribution( - name=filename, digest=file_hashes["sha256"] - ), + Distribution(name=filename, digest=file_hashes["sha256"]), ) ) except AttestationUploadError as e: @@ -1282,14 +1280,7 @@ def file_upload(request): ) attestation_service.persist_attestations( - attestations=attestations, - file=file_, - ) - - attestation_service.generate_and_store_provenance_file( - request.oidc_publisher, - file_, - attestations, + request.oidc_publisher, attestations, file_ ) # Log successful attestation upload From eec0b46f3bf62a6b2ef74f45e926d2b05472bd80 Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 16 Aug 2024 12:14:10 +0200 Subject: [PATCH 26/29] Remove file.publisher_url which is no longer used. --- warehouse/forklift/legacy.py | 2 ++ warehouse/packaging/models.py | 18 ------------------ 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 03eebfea9372..51a4f28bde8f 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -368,6 +368,7 @@ def _is_duplicate_file(db_session, filename, hashes): return None + _pypi_project_urls = [ "https://pypi.org/project/", "https://pypi.org/p/", @@ -391,6 +392,7 @@ def _verify_url_pypi(url: str, project_name: str, project_normalized_name: str) for candidate_url in candidate_urls ) + def _verify_url_with_trusted_publisher(url: str, publisher_url: str) -> bool: """ Verify a given URL against a Trusted Publisher URL diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index d70d4c6b7f4b..4bd999825886 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -65,7 +65,6 @@ from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.events.models import HasEvents -from warehouse.events.tags import EventTag from warehouse.integrations.vulnerabilities.models import VulnerabilityRecord from warehouse.observations.models import HasObservations from warehouse.organizations.models import ( @@ -842,23 +841,6 @@ def __table_args__(cls): # noqa passive_deletes=True, ) - @property - def publisher_url(self) -> str | None: - event_tag = self.Event.tag # type: ignore[attr-defined] - event_additional = self.Event.additional # type: ignore[attr-defined] - - try: - release_event = self.events.where( - sql.and_( - event_tag == EventTag.File.FileAdd, - event_additional["publisher_url"].as_string().is_not(None), - ) - ).one() - except (NoResultFound, MultipleResultsFound): - return None - - return release_event.additional["publisher_url"] - @property def uploaded_via_trusted_publisher(self) -> bool: """Return True if the file was uploaded via a trusted publisher.""" From 98d39c4f5019d233d423a4b5c3a514cebb7dbfb2 Mon Sep 17 00:00:00 2001 From: Alexis Date: Tue, 20 Aug 2024 16:21:52 +0200 Subject: [PATCH 27/29] Rename ReleaseAttestationService to IntegrityService --- tests/unit/api/test_simple.py | 6 +- tests/unit/attestations/test_services.py | 193 +++++++++++++---------- tests/unit/forklift/test_legacy.py | 26 ++- tests/unit/packaging/test_utils.py | 10 +- warehouse/attestations/__init__.py | 8 +- warehouse/attestations/interfaces.py | 19 ++- warehouse/attestations/services.py | 32 ++-- warehouse/forklift/legacy.py | 22 +-- warehouse/packaging/utils.py | 8 +- 9 files changed, 182 insertions(+), 142 deletions(-) diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index fd0f8c273779..27d28b7f7b05 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -18,7 +18,7 @@ from pyramid.testing import DummyRequest from warehouse.api import simple -from warehouse.attestations import IReleaseVerificationService +from warehouse.attestations import IIntegrityService from warehouse.packaging.utils import API_VERSION from ...common.db.accounts import UserFactory @@ -93,7 +93,7 @@ class TestSimpleIndex: def db_request(self, db_request): """Override db_request to add the Release Verification service""" db_request.find_service = lambda svc, name=None, context=None: { - IReleaseVerificationService: pretend.stub(), + IIntegrityService: pretend.stub(), }.get(svc) return db_request @@ -200,7 +200,7 @@ class TestSimpleDetail: def db_request(self, db_request): """Override db_request to add the Release Verification service""" db_request.find_service = lambda svc, name=None, context=None: { - IReleaseVerificationService: pretend.stub( + IIntegrityService: pretend.stub( get_provenance_digest=lambda *args, **kwargs: None, ), }.get(svc) diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index 32fad2131e72..f3b0b8b38d0a 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -30,15 +30,13 @@ from sigstore.verify import Verifier from zope.interface.verify import verifyClass -import warehouse.attestations.services - from tests.common.db.oidc import GitHubPublisherFactory, GitLabPublisherFactory from tests.common.db.packaging import FileEventFactory, FileFactory from warehouse.attestations import ( Attestation as DatabaseAttestation, AttestationUploadError, - IReleaseVerificationService, - ReleaseVerificationService, + IIntegrityService, + IntegrityService, UnsupportedPublisherError, services, ) @@ -60,7 +58,7 @@ class TestAttestationsService: def test_interface_matches(self): - assert verifyClass(IReleaseVerificationService, ReleaseVerificationService) + assert verifyClass(IIntegrityService, IntegrityService) def test_create_service(self): request = pretend.stub( @@ -69,7 +67,7 @@ def test_create_service(self): ), ) - assert ReleaseVerificationService.create_service(None, request) is not None + assert IntegrityService.create_service(None, request) is not None assert not set(request.find_service.calls) ^ { pretend.call(IFileStorage), pretend.call(IMetricsService), @@ -84,26 +82,16 @@ def storage_service_store(path: str, file_path, *_args, **_kwargs): assert path.endswith(".attestation") - release_verification = ReleaseVerificationService( + integrity_service = IntegrityService( storage=pretend.stub( store=storage_service_store, ), metrics=pretend.stub(), ) - db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( - [VALID_ATTESTATION] - ) - file = FileFactory.create(attestations=[]) - monkeypatch.setattr( - warehouse.attestations.services.ReleaseVerificationService, - "generate_and_store_provenance_file", - lambda *args, **kwargs: None, - ) + file = FileFactory.create(attestations=[]) - release_verification.persist_attestations( - pretend.stub(), [VALID_ATTESTATION], file - ) + integrity_service.persist_attestations([VALID_ATTESTATION], file) attestations_db = ( db_request.db.query(DatabaseAttestation) @@ -112,9 +100,10 @@ def storage_service_store(path: str, file_path, *_args, **_kwargs): .all() ) assert len(attestations_db) == 1 + assert len(file.attestations) == 1 def test_parse_no_publisher(self, db_request): - release_verification = ReleaseVerificationService( + integrity_service = IntegrityService( storage=pretend.stub(), metrics=pretend.stub(), ) @@ -124,10 +113,10 @@ def test_parse_no_publisher(self, db_request): AttestationUploadError, match="Attestations are only supported when using Trusted", ): - release_verification.parse_attestations(db_request, pretend.stub()) + integrity_service.parse_attestations(db_request, pretend.stub()) def test_parse_unsupported_publisher(self, db_request): - release_verification = ReleaseVerificationService( + integrity_service = IntegrityService( storage=pretend.stub(), metrics=pretend.stub(), ) @@ -136,10 +125,10 @@ def test_parse_unsupported_publisher(self, db_request): AttestationUploadError, match="Attestations are only supported when using Trusted", ): - release_verification.parse_attestations(db_request, pretend.stub()) + integrity_service.parse_attestations(db_request, pretend.stub()) def test_parse_malformed_attestation(self, metrics, db_request): - release_verification = ReleaseVerificationService( + integrity_service = IntegrityService( storage=pretend.stub(), metrics=metrics, ) @@ -150,7 +139,7 @@ def test_parse_malformed_attestation(self, metrics, db_request): AttestationUploadError, match="Error while decoding the included attestation", ): - release_verification.parse_attestations(db_request, pretend.stub()) + integrity_service.parse_attestations(db_request, pretend.stub()) assert ( pretend.call("warehouse.upload.attestations.malformed") @@ -158,7 +147,7 @@ def test_parse_malformed_attestation(self, metrics, db_request): ) def test_parse_multiple_attestations(self, metrics, db_request): - release_verification = ReleaseVerificationService( + integrity_service = IntegrityService( storage=pretend.stub(), metrics=metrics, ) @@ -170,7 +159,7 @@ def test_parse_multiple_attestations(self, metrics, db_request): with pytest.raises( AttestationUploadError, match="Only a single attestation per file" ): - release_verification.parse_attestations( + integrity_service.parse_attestations( db_request, pretend.stub(), ) @@ -196,7 +185,7 @@ def test_parse_multiple_attestations(self, metrics, db_request): def test_parse_failed_verification( self, metrics, monkeypatch, db_request, verify_exception, expected_message ): - release_verification = ReleaseVerificationService( + integrity_service = IntegrityService( storage=pretend.stub(), metrics=metrics, ) @@ -217,13 +206,13 @@ def failing_verify(_self, _verifier, _policy, _dist): monkeypatch.setattr(Attestation, "verify", failing_verify) with pytest.raises(AttestationUploadError, match=expected_message): - release_verification.parse_attestations( + integrity_service.parse_attestations( db_request, pretend.stub(), ) def test_parse_wrong_predicate(self, metrics, monkeypatch, db_request): - release_verification = ReleaseVerificationService( + integrity_service = IntegrityService( storage=pretend.stub(), metrics=metrics, ) @@ -244,7 +233,7 @@ def test_parse_wrong_predicate(self, metrics, monkeypatch, db_request): with pytest.raises( AttestationUploadError, match="Attestation with unsupported predicate" ): - release_verification.parse_attestations( + integrity_service.parse_attestations( db_request, pretend.stub(), ) @@ -257,7 +246,7 @@ def test_parse_wrong_predicate(self, metrics, monkeypatch, db_request): ) def test_parse_succeed(self, metrics, monkeypatch, db_request): - release_verification = ReleaseVerificationService( + integrity_service = IntegrityService( storage=pretend.stub(), metrics=metrics, ) @@ -275,72 +264,77 @@ def test_parse_succeed(self, metrics, monkeypatch, db_request): Attestation, "verify", lambda *args: (AttestationType.PYPI_PUBLISH_V1, {}) ) - attestations = release_verification.parse_attestations( + attestations = integrity_service.parse_attestations( db_request, pretend.stub(), ) assert attestations == [VALID_ATTESTATION] - def test_get_provenance_digest(self, db_request): - file = FileFactory.create() - FileEventFactory.create( - source=file, - tag=EventTag.File.FileAdd, - additional={"publisher_url": "fake-publisher-url"}, - ) - - with tempfile.NamedTemporaryFile() as f: - release_verification = ReleaseVerificationService( - storage=pretend.stub(get=pretend.call_recorder(lambda p: f)), - metrics=pretend.stub(), - ) - - assert ( - release_verification.get_provenance_digest(file) - == hashlib.file_digest(f, "sha256").hexdigest() - ) - - def test_get_provenance_digest_fails_no_attestations(self, db_request): - # If the attestations are missing, there is no provenance file - file = FileFactory.create() - file.attestations = [] - FileEventFactory.create( - source=file, - tag=EventTag.File.FileAdd, - additional={"publisher_url": "fake-publisher-url"}, - ) - release_verification = ReleaseVerificationService( + def test_generate_provenance_unsupported_publisher(self): + integrity_service = IntegrityService( storage=pretend.stub(), metrics=pretend.stub(), ) - assert release_verification.get_provenance_digest(file) is None + oidc_publisher = pretend.stub(publisher_name="not-existing") - def test_generate_and_store_provenance_file_no_publisher(self): - release_verification = ReleaseVerificationService( + assert ( + integrity_service.generate_provenance(oidc_publisher, pretend.stub()) + is None + ) + + @pytest.mark.parametrize( + "publisher_name", + [ + "github", + "gitlab", + ], + ) + def test_generate_provenance_succeeds(self, publisher_name: str, monkeypatch): + integrity_service = IntegrityService( storage=pretend.stub(), metrics=pretend.stub(), ) - oidc_publisher = pretend.stub(publisher_name="not-existing") - - assert ( - release_verification.generate_and_store_provenance_file( - oidc_publisher, pretend.stub(), pretend.stub() + if publisher_name == "github": + publisher = GitHubPublisher( + repository="fake-repository", + workflow="fake-workflow", ) - is None + else: + publisher = GitLabPublisher( + repository="fake-repository", + environment="fake-env", + ) + + monkeypatch.setattr( + services, + "_publisher_from_oidc_publisher", + lambda s: publisher, ) - def test_generate_and_store_provenance_file(self, db_request, monkeypatch): + provenance = integrity_service.generate_provenance( + pretend.stub(), + [VALID_ATTESTATION], + ) - publisher = GitHubPublisher( - repository="fake-repository", - workflow="fake-workflow", + assert provenance == Provenance( + attestation_bundles=[ + AttestationBundle( + publisher=publisher, + attestations=[VALID_ATTESTATION], + ) + ] ) + + def test_persist_provenance_succeeds(self, db_request): provenance = Provenance( attestation_bundles=[ AttestationBundle( - publisher=publisher, + publisher=GitHubPublisher( + repository="fake-repository", + workflow="fake-workflow", + ), attestations=[VALID_ATTESTATION], ) ] @@ -354,23 +348,50 @@ def storage_service_store(path, file_path, *_args, **_kwargs): assert path.suffix == ".provenance" - monkeypatch.setattr( - services, - "_publisher_from_oidc_publisher", - lambda s: publisher, - ) - - release_verification = ReleaseVerificationService( + integrity_service = IntegrityService( storage=pretend.stub(store=storage_service_store), metrics=pretend.stub(), ) assert ( - release_verification.generate_and_store_provenance_file( - pretend.stub(), [VALID_ATTESTATION], FileFactory.create() - ) + integrity_service.persist_provenance(provenance, FileFactory.create()) is None ) + def test_get_provenance_digest(self, db_request): + file = FileFactory.create() + FileEventFactory.create( + source=file, + tag=EventTag.File.FileAdd, + additional={"publisher_url": "fake-publisher-url"}, + ) + + with tempfile.NamedTemporaryFile() as f: + integrity_service = IntegrityService( + storage=pretend.stub(get=pretend.call_recorder(lambda p: f)), + metrics=pretend.stub(), + ) + + assert ( + integrity_service.get_provenance_digest(file) + == hashlib.file_digest(f, "sha256").hexdigest() + ) + + def test_get_provenance_digest_fails_no_attestations(self, db_request): + # If the attestations are missing, there is no provenance file + file = FileFactory.create() + file.attestations = [] + FileEventFactory.create( + source=file, + tag=EventTag.File.FileAdd, + additional={"publisher_url": "fake-publisher-url"}, + ) + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=pretend.stub(), + ) + + assert integrity_service.get_provenance_digest(file) is None + def test_publisher_from_oidc_publisher_github(db_request): publisher = GitHubPublisherFactory.create() diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index ecbc5a4d5f87..0e48620bc95a 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -38,7 +38,7 @@ from warehouse.attestations import ( Attestation as DatabaseAttestation, AttestationUploadError, - IReleaseVerificationService, + IIntegrityService, ) from warehouse.classifiers.models import Classifier from warehouse.forklift import legacy, metadata @@ -3338,8 +3338,10 @@ def test_upload_succeeds_creates_release( ), ] + @pytest.mark.parametrize("provenance_rv", [None, "fake-provenance-object"]) def test_upload_succeeds_with_valid_attestation( self, + provenance_rv, monkeypatch, pyramid_config, db_request, @@ -3394,19 +3396,25 @@ def test_upload_succeeds_with_valid_attestation( } ) - def persist_attestations(oidc_publisher, attestations, file): + def persist_attestations(attestations, file): file.attestations.append(AttestationFactory.create(file=file)) + def persist_provenance(provenance_object, file): + assert provenance_object == provenance_rv + storage_service = pretend.stub(store=lambda path, filepath, *, meta=None: None) - release_verification = pretend.stub( + integrity_service = pretend.stub( parse_attestations=lambda *args, **kwargs: [attestation], persist_attestations=persist_attestations, - generate_and_store_provenance_file=lambda p, a, f: None, + generate_provenance=pretend.call_recorder( + lambda oidc_publisher, attestations: provenance_rv + ), + persist_provenance=persist_provenance, ) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, - IReleaseVerificationService: release_verification, + IIntegrityService: integrity_service, }.get(svc) record_event = pretend.call_recorder( @@ -3428,6 +3436,10 @@ def persist_attestations(oidc_publisher, attestations, file): ) assert len(attestations_db) == 1 + assert integrity_service.generate_provenance.calls == [ + pretend.call(db_request.oidc_publisher, [attestation]) + ] + @pytest.mark.parametrize( "expected_message", [ @@ -3491,11 +3503,11 @@ def test_upload_fails_attestation_error( def stub_parse(*_args, **_kwargs): raise AttestationUploadError(expected_message) - release_verification = pretend.stub(parse_attestations=stub_parse) + integrity_service = pretend.stub(parse_attestations=stub_parse) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, - IReleaseVerificationService: release_verification, + IIntegrityService: integrity_service, }.get(svc) record_event = pretend.call_recorder( diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index f953d66be1da..8065c8b9ad8c 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -15,7 +15,7 @@ import pretend -from warehouse.attestations import IReleaseVerificationService +from warehouse.attestations import IIntegrityService from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.utils import _simple_detail, render_simple_detail @@ -29,7 +29,7 @@ def test_simple_detail_empty_string(db_request): db_request.route_url = lambda *a, **kw: "the-url" db_request.find_service = lambda svc, name=None, context=None: { - IReleaseVerificationService: pretend.stub( + IIntegrityService: pretend.stub( get_provenance_digest=pretend.call_recorder(lambda f: None), ), }.get(svc) @@ -49,7 +49,7 @@ def test_simple_detail_with_provenance(db_request): db_request.route_url = lambda *a, **kw: "the-url" db_request.find_service = pretend.call_recorder( lambda svc, name=None, context=None: { - IReleaseVerificationService: pretend.stub( + IIntegrityService: pretend.stub( get_provenance_digest=pretend.call_recorder(lambda f: hash_digest), ), }.get(svc) @@ -77,7 +77,7 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): db_request.route_url = lambda *a, **kw: "the-url" db_request.find_service = lambda svc, name=None, context=None: { - IReleaseVerificationService: pretend.stub( + IIntegrityService: pretend.stub( get_provenance_digest=pretend.call_recorder(lambda f: None), ), }.get(svc) @@ -111,7 +111,7 @@ def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): db_request.find_service = pretend.call_recorder( lambda svc, name=None, context=None: { ISimpleStorage: storage_service, - IReleaseVerificationService: pretend.stub( + IIntegrityService: pretend.stub( get_provenance_digest=pretend.call_recorder(lambda f: None), ), }.get(svc) diff --git a/warehouse/attestations/__init__.py b/warehouse/attestations/__init__.py index ca8a54fde2de..76c816d7fef3 100644 --- a/warehouse/attestations/__init__.py +++ b/warehouse/attestations/__init__.py @@ -14,14 +14,14 @@ AttestationUploadError, UnsupportedPublisherError, ) -from warehouse.attestations.interfaces import IReleaseVerificationService +from warehouse.attestations.interfaces import IIntegrityService from warehouse.attestations.models import Attestation -from warehouse.attestations.services import ReleaseVerificationService +from warehouse.attestations.services import IntegrityService __all__ = [ "Attestation", "AttestationUploadError", - "IReleaseVerificationService", - "ReleaseVerificationService", + "IIntegrityService", + "IntegrityService", "UnsupportedPublisherError", ] diff --git a/warehouse/attestations/interfaces.py b/warehouse/attestations/interfaces.py index 7973592a606c..4055e92055a5 100644 --- a/warehouse/attestations/interfaces.py +++ b/warehouse/attestations/interfaces.py @@ -10,12 +10,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pypi_attestations import Attestation, Distribution +from pypi_attestations import Attestation, Distribution, Provenance from pyramid.request import Request from zope.interface import Interface -class IReleaseVerificationService(Interface): +class IIntegrityService(Interface): def create_service(context, request): """ @@ -23,7 +23,7 @@ def create_service(context, request): created for. """ - def persist_attestations(oidc_publisher, attestations: list[Attestation], file): + def persist_attestations(attestations: list[Attestation], file): """ ̀¦Persist attestations in storage. """ @@ -36,11 +36,16 @@ def parse_attestations( Process any attestations included in a file upload request """ - def generate_and_store_provenance_file( - oidc_publisher, attestations: list[Attestation], file - ) -> None: + def generate_provenance( + oidc_publisher, attestations: list[Attestation] + ) -> Provenance | None: """ - Generate and store a provenance file for a release, its attestations and an OIDCPublisher. + Generate a Provenance object from an OIDCPublisher and its attestations. + """ + + def persist_provenance(provenance: Provenance, file) -> None: + """ + Persist a Provenance object in storage. """ def get_provenance_digest(file) -> str | None: diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py index d221b281a0b5..056c47c4894e 100644 --- a/warehouse/attestations/services.py +++ b/warehouse/attestations/services.py @@ -37,7 +37,7 @@ AttestationUploadError, UnsupportedPublisherError, ) -from warehouse.attestations.interfaces import IReleaseVerificationService +from warehouse.attestations.interfaces import IIntegrityService from warehouse.attestations.models import Attestation as DatabaseAttestation from warehouse.metrics.interfaces import IMetricsService from warehouse.oidc.models import ( @@ -51,7 +51,7 @@ def _publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: """ - Convert an OIDCPublisher object in a pypy-attestations Publisher. + Convert an OIDCPublisher object in a pypi-attestations Publisher. """ match publisher.publisher_name: case "GitLab": @@ -70,8 +70,8 @@ def _publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: raise UnsupportedPublisherError -@implementer(IReleaseVerificationService) -class ReleaseVerificationService: +@implementer(IIntegrityService) +class IntegrityService: def __init__( self, @@ -88,9 +88,7 @@ def create_service(cls, _context, request: Request): metrics=request.find_service(IMetricsService), ) - def persist_attestations( - self, oidc_publisher: OIDCPublisher, attestations: list[Attestation], file: File - ) -> None: + def persist_attestations(self, attestations: list[Attestation], file: File) -> None: for attestation in attestations: with tempfile.NamedTemporaryFile() as tmp_file: tmp_file.write(attestation.model_dump_json().encode("utf-8")) @@ -110,8 +108,6 @@ def persist_attestations( file.attestations.append(database_attestation) - self.generate_and_store_provenance_file(oidc_publisher, attestations, file) - def parse_attestations( self, request: Request, distribution: Distribution ) -> list[Attestation]: @@ -189,9 +185,9 @@ def parse_attestations( return attestations - def generate_and_store_provenance_file( - self, oidc_publisher: OIDCPublisher, attestations: list[Attestation], file: File - ) -> None: + def generate_provenance( + self, oidc_publisher: OIDCPublisher, attestations: list[Attestation] + ) -> Provenance | None: try: publisher: Publisher = _publisher_from_oidc_publisher(oidc_publisher) except UnsupportedPublisherError: @@ -199,15 +195,23 @@ def generate_and_store_provenance_file( f"Unsupported OIDCPublisher found {oidc_publisher.publisher_name}" ) - return + return None attestation_bundle = AttestationBundle( publisher=publisher, attestations=attestations, ) - provenance = Provenance(attestation_bundles=[attestation_bundle]) + return Provenance(attestation_bundles=[attestation_bundle]) + def persist_provenance( + self, + provenance: Provenance, + file: File, + ) -> None: + """ + Persist a Provenance object in storage. + """ provenance_file_path = Path(f"{file.path}.provenance") with tempfile.NamedTemporaryFile() as f: f.write(provenance.model_dump_json().encode("utf-8")) diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 51a4f28bde8f..be2aee5986d6 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -46,7 +46,7 @@ from sqlalchemy.exc import MultipleResultsFound, NoResultFound from warehouse.admin.flags import AdminFlagValue -from warehouse.attestations import AttestationUploadError, IReleaseVerificationService +from warehouse.attestations import AttestationUploadError, IIntegrityService from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB @@ -1306,16 +1306,12 @@ def file_upload(request): # If the user provided attestations, verify and store them if "attestations" in request.POST: - attestation_service = request.find_service( - IReleaseVerificationService, context=None - ) + integrity_service = request.find_service(IIntegrityService, context=None) try: - attestations: list[Attestation] = ( - attestation_service.parse_attestations( - request, - Distribution(name=filename, digest=file_hashes["sha256"]), - ) + attestations: list[Attestation] = integrity_service.parse_attestations( + request, + Distribution(name=filename, digest=file_hashes["sha256"]), ) except AttestationUploadError as e: raise _exc_with_message( @@ -1323,9 +1319,13 @@ def file_upload(request): str(e), ) - attestation_service.persist_attestations( - request.oidc_publisher, attestations, file_ + integrity_service.persist_attestations(attestations, file_) + + provenance = integrity_service.generate_provenance( + request.oidc_publisher, attestations ) + if provenance: + integrity_service.persist_provenance(provenance, file_) # Log successful attestation upload metrics.increment("warehouse.upload.attestations.ok") diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index e08d68f64eec..5fc67bc29761 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -19,7 +19,7 @@ from pyramid_jinja2 import IJinja2Environment from sqlalchemy.orm import joinedload -from warehouse.attestations import IReleaseVerificationService +from warehouse.attestations import IIntegrityService from warehouse.attestations.models import Attestation from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.models import File, LifecycleStatus, Project, Release @@ -67,9 +67,7 @@ def _simple_detail(project, request): {f.release.version for f in files}, key=packaging_legacy.version.parse ) - release_verification = request.find_service( - IReleaseVerificationService, context=None - ) + integrity_service = request.find_service(IIntegrityService, context=None) return { "meta": {"api-version": API_VERSION, "_last-serial": project.last_serial}, @@ -104,7 +102,7 @@ def _simple_detail(project, request): if file.metadata_file_sha256_digest else False ), - "provenance": release_verification.get_provenance_digest(file), + "provenance": integrity_service.get_provenance_digest(file), } for file in files ], From 8d00c87646d9e6152bfb658608e5cd472972a01e Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Wed, 21 Aug 2024 18:36:35 +0000 Subject: [PATCH 28/29] Linting --- tests/unit/attestations/test_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index f3b0b8b38d0a..a83a0c302b00 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -170,7 +170,7 @@ def test_parse_multiple_attestations(self, metrics, db_request): ) @pytest.mark.parametrize( - "verify_exception, expected_message", + ("verify_exception", "expected_message"), [ ( VerificationError, From 2ca191807ea89d1721bf295ee7f45271379534cf Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 21 Aug 2024 14:37:44 -0400 Subject: [PATCH 29/29] requirements: bump sigstore, pypi-attestations Signed-off-by: William Woodruff --- requirements/main.in | 4 ++-- requirements/main.txt | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/requirements/main.in b/requirements/main.in index 545e72d0e651..6fa06b782f43 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -62,8 +62,8 @@ redis>=2.8.0,<6.0.0 rfc3986 sentry-sdk setuptools -sigstore~=3.1.0 -pypi-attestations==0.0.10 +sigstore~=3.2.0 +pypi-attestations==0.0.11 sqlalchemy[asyncio]>=2.0,<3.0 stdlib-list stripe diff --git a/requirements/main.txt b/requirements/main.txt index c36a9d08e192..e1ff17e074af 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1770,9 +1770,9 @@ pyparsing==3.1.2 \ --hash=sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad \ --hash=sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742 # via linehaul -pypi-attestations==0.0.10 \ - --hash=sha256:3671fd72c38f9ee539f48772894bccec1a73b93459ce64d9809c517e170ef2c5 \ - --hash=sha256:3e2df0bf8fbc612c825865f642f138cb7a16512c7215489f631c6b1869dbba26 +pypi-attestations==0.0.11 \ + --hash=sha256:b730e6b23874d94da0f3817b1f9dd3ecb6a80d685f62a18ad96e5b0396149ded \ + --hash=sha256:e74329074f049568591e300373e12fcd46a35e21723110856546e33bf2949efa # via -r requirements/main.in pyqrcode==1.2.1 \ --hash=sha256:1b2812775fa6ff5c527977c4cd2ccb07051ca7d0bc0aecf937a43864abe5eff6 \ @@ -2079,9 +2079,9 @@ sentry-sdk==2.13.0 \ --hash=sha256:6beede8fc2ab4043da7f69d95534e320944690680dd9a963178a49de71d726c6 \ --hash=sha256:8d4a576f7a98eb2fdb40e13106e41f330e5c79d72a68be1316e7852cf4995260 # via -r requirements/main.in -sigstore==3.1.0 \ - --hash=sha256:3cfe2da19a053757a06bd9ecae322fa539fece7df3e8139d30e32172e41cb812 \ - --hash=sha256:cc0b52acff3ae25f7f1993e21dec4ebed44213c48e2ec095e8c06f69b3751fdf +sigstore==3.2.0 \ + --hash=sha256:25c8a871a3a6adf959c0cde598ea8bef8794f1a29277d067111eb4ded4ba7f65 \ + --hash=sha256:d18508f34febb7775065855e92557fa1c2c16580df88f8e8903b9514438bad44 # via # -r requirements/main.in # pypi-attestations