diff --git a/requirements/main.in b/requirements/main.in index 068b7ff21a4c..6fa06b782f43 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -62,8 +62,8 @@ redis>=2.8.0,<6.0.0 rfc3986 sentry-sdk setuptools -sigstore~=3.0.0 -pypi-attestations==0.0.9 +sigstore~=3.2.0 +pypi-attestations==0.0.11 sqlalchemy[asyncio]>=2.0,<3.0 stdlib-list stripe diff --git a/requirements/main.txt b/requirements/main.txt index 922852b57c3e..e1ff17e074af 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1770,9 +1770,9 @@ pyparsing==3.1.2 \ --hash=sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad \ --hash=sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742 # via linehaul -pypi-attestations==0.0.9 \ - --hash=sha256:3bfc07f64a8db0d6e2646720e70df7c7cb01a2936056c764a2cc3268969332f2 \ - --hash=sha256:4b38cce5d221c8145cac255bfafe650ec0028d924d2b3572394df8ba8f07a609 +pypi-attestations==0.0.11 \ + --hash=sha256:b730e6b23874d94da0f3817b1f9dd3ecb6a80d685f62a18ad96e5b0396149ded \ + --hash=sha256:e74329074f049568591e300373e12fcd46a35e21723110856546e33bf2949efa # via -r requirements/main.in pyqrcode==1.2.1 \ --hash=sha256:1b2812775fa6ff5c527977c4cd2ccb07051ca7d0bc0aecf937a43864abe5eff6 \ @@ -2079,9 +2079,9 @@ sentry-sdk==2.13.0 \ --hash=sha256:6beede8fc2ab4043da7f69d95534e320944690680dd9a963178a49de71d726c6 \ --hash=sha256:8d4a576f7a98eb2fdb40e13106e41f330e5c79d72a68be1316e7852cf4995260 # via -r requirements/main.in -sigstore==3.0.0 \ - --hash=sha256:6cc7dc92607c2fd481aada0f3c79e710e4c6086e3beab50b07daa9a50a79d109 \ - --hash=sha256:a6a9538a648e112a0c3d8092d3f73a351c7598164764f1e73a6b5ba406a3a0bd +sigstore==3.2.0 \ + --hash=sha256:25c8a871a3a6adf959c0cde598ea8bef8794f1a29277d067111eb4ded4ba7f65 \ + --hash=sha256:d18508f34febb7775065855e92557fa1c2c16580df88f8e8903b9514438bad44 # via # -r requirements/main.in # pypi-attestations diff --git a/tests/common/db/attestation.py b/tests/common/db/attestation.py new file mode 100644 index 000000000000..2080519a806f --- /dev/null +++ b/tests/common/db/attestation.py @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib + +import factory + +from warehouse.attestations.models import Attestation + +from .base import WarehouseFactory + + +class AttestationFactory(WarehouseFactory): + class Meta: + model = Attestation + + file = factory.SubFactory("tests.common.db.packaging.FileFactory") + attestation_file_blake2_digest = factory.LazyAttribute( + lambda o: hashlib.blake2b(o.file.filename.encode("utf8")).hexdigest() + ) diff --git a/tests/common/db/packaging.py b/tests/common/db/packaging.py index 2a12379da170..369dc9f092d0 100644 --- a/tests/common/db/packaging.py +++ b/tests/common/db/packaging.py @@ -34,6 +34,7 @@ from warehouse.utils import readme from .accounts import UserFactory +from .attestation import AttestationFactory from .base import WarehouseFactory from .observations import ObserverFactory @@ -140,6 +141,12 @@ class Meta: ) ) + attestations = factory.RelatedFactoryList( + AttestationFactory, + factory_related_name="file", + size=1, + ) + class FileEventFactory(WarehouseFactory): class Meta: diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index 9937038dd145..5f8003d50eb5 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -18,6 +18,7 @@ from pyramid.testing import DummyRequest from warehouse.api import simple +from warehouse.attestations import IIntegrityService from warehouse.packaging.utils import API_VERSION from ...common.db.accounts import UserFactory @@ -87,6 +88,16 @@ def test_selects(self, header, expected): class TestSimpleIndex: + + @pytest.fixture + def db_request(self, db_request): + """Override db_request to add the Release Verification service""" + db_request.find_service = lambda svc, name=None, context=None: { + IIntegrityService: pretend.stub(), + }.get(svc) + + return db_request + @pytest.mark.parametrize( ("content_type", "renderer_override"), CONTENT_TYPE_PARAMS, @@ -185,6 +196,17 @@ def test_quarantined_project_omitted_from_index(self, db_request): class TestSimpleDetail: + @pytest.fixture + def db_request(self, db_request): + """Override db_request to add the Release Verification service""" + db_request.find_service = lambda svc, name=None, context=None: { + IIntegrityService: pretend.stub( + get_provenance_digest=lambda *args, **kwargs: None, + ), + }.get(svc) + + return db_request + def test_redirects(self, pyramid_request): project = pretend.stub(normalized_name="foo") @@ -286,6 +308,7 @@ def test_with_files_no_serial(self, db_request, content_type, renderer_override) "upload-time": f.upload_time.isoformat() + "Z", "data-dist-info-metadata": False, "core-metadata": False, + "provenance": None, } for f in files ], @@ -334,6 +357,7 @@ def test_with_files_with_serial(self, db_request, content_type, renderer_overrid "upload-time": f.upload_time.isoformat() + "Z", "data-dist-info-metadata": False, "core-metadata": False, + "provenance": None, } for f in files ], @@ -427,6 +451,7 @@ def test_with_files_with_version_multi_digit( if f.metadata_file_sha256_digest is not None else False ), + "provenance": None, } for f in files ], diff --git a/tests/unit/attestations/__init__.py b/tests/unit/attestations/__init__.py new file mode 100644 index 000000000000..164f68b09175 --- /dev/null +++ b/tests/unit/attestations/__init__.py @@ -0,0 +1,11 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py new file mode 100644 index 000000000000..a83a0c302b00 --- /dev/null +++ b/tests/unit/attestations/test_services.py @@ -0,0 +1,419 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib +import tempfile + +import pretend +import pytest + +from pydantic import TypeAdapter +from pypi_attestations import ( + Attestation, + AttestationBundle, + AttestationType, + Envelope, + GitHubPublisher, + GitLabPublisher, + Provenance, + VerificationError, + VerificationMaterial, +) +from sigstore.verify import Verifier +from zope.interface.verify import verifyClass + +from tests.common.db.oidc import GitHubPublisherFactory, GitLabPublisherFactory +from tests.common.db.packaging import FileEventFactory, FileFactory +from warehouse.attestations import ( + Attestation as DatabaseAttestation, + AttestationUploadError, + IIntegrityService, + IntegrityService, + UnsupportedPublisherError, + services, +) +from warehouse.events.tags import EventTag +from warehouse.metrics import IMetricsService +from warehouse.packaging import File, IFileStorage + +VALID_ATTESTATION = Attestation( + version=1, + verification_material=VerificationMaterial( + certificate="somebase64string", transparency_entries=[dict()] + ), + envelope=Envelope( + statement="somebase64string", + signature="somebase64string", + ), +) + + +class TestAttestationsService: + def test_interface_matches(self): + assert verifyClass(IIntegrityService, IntegrityService) + + def test_create_service(self): + request = pretend.stub( + find_service=pretend.call_recorder( + lambda svc, context=None, name=None: None + ), + ) + + assert IntegrityService.create_service(None, request) is not None + assert not set(request.find_service.calls) ^ { + pretend.call(IFileStorage), + pretend.call(IMetricsService), + } + + def test_persist_attestations(self, db_request, monkeypatch): + @pretend.call_recorder + def storage_service_store(path: str, file_path, *_args, **_kwargs): + expected = VALID_ATTESTATION.model_dump_json().encode("utf-8") + with open(file_path, "rb") as fp: + assert fp.read() == expected + + assert path.endswith(".attestation") + + integrity_service = IntegrityService( + storage=pretend.stub( + store=storage_service_store, + ), + metrics=pretend.stub(), + ) + + file = FileFactory.create(attestations=[]) + + integrity_service.persist_attestations([VALID_ATTESTATION], file) + + attestations_db = ( + db_request.db.query(DatabaseAttestation) + .join(DatabaseAttestation.file) + .filter(File.filename == file.filename) + .all() + ) + assert len(attestations_db) == 1 + assert len(file.attestations) == 1 + + def test_parse_no_publisher(self, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=pretend.stub(), + ) + + db_request.oidc_publisher = None + with pytest.raises( + AttestationUploadError, + match="Attestations are only supported when using Trusted", + ): + integrity_service.parse_attestations(db_request, pretend.stub()) + + def test_parse_unsupported_publisher(self, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=pretend.stub(), + ) + db_request.oidc_publisher = pretend.stub(publisher_name="not-existing") + with pytest.raises( + AttestationUploadError, + match="Attestations are only supported when using Trusted", + ): + integrity_service.parse_attestations(db_request, pretend.stub()) + + def test_parse_malformed_attestation(self, metrics, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=metrics, + ) + + db_request.oidc_publisher = pretend.stub(publisher_name="GitHub") + db_request.POST["attestations"] = "{'malformed-attestation'}" + with pytest.raises( + AttestationUploadError, + match="Error while decoding the included attestation", + ): + integrity_service.parse_attestations(db_request, pretend.stub()) + + assert ( + pretend.call("warehouse.upload.attestations.malformed") + in metrics.increment.calls + ) + + def test_parse_multiple_attestations(self, metrics, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=metrics, + ) + + db_request.oidc_publisher = pretend.stub(publisher_name="GitHub") + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION, VALID_ATTESTATION] + ) + with pytest.raises( + AttestationUploadError, match="Only a single attestation per file" + ): + integrity_service.parse_attestations( + db_request, + pretend.stub(), + ) + + assert ( + pretend.call("warehouse.upload.attestations.failed_multiple_attestations") + in metrics.increment.calls + ) + + @pytest.mark.parametrize( + ("verify_exception", "expected_message"), + [ + ( + VerificationError, + "Could not verify the uploaded", + ), + ( + ValueError, + "Unknown error while", + ), + ], + ) + def test_parse_failed_verification( + self, metrics, monkeypatch, db_request, verify_exception, expected_message + ): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=metrics, + ) + + db_request.oidc_publisher = pretend.stub( + publisher_name="GitHub", + publisher_verification_policy=pretend.call_recorder(lambda c: None), + ) + db_request.oidc_claims = {"sha": "somesha"} + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION] + ) + + def failing_verify(_self, _verifier, _policy, _dist): + raise verify_exception("error") + + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) + monkeypatch.setattr(Attestation, "verify", failing_verify) + + with pytest.raises(AttestationUploadError, match=expected_message): + integrity_service.parse_attestations( + db_request, + pretend.stub(), + ) + + def test_parse_wrong_predicate(self, metrics, monkeypatch, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=metrics, + ) + db_request.oidc_publisher = pretend.stub( + publisher_name="GitHub", + publisher_verification_policy=pretend.call_recorder(lambda c: None), + ) + db_request.oidc_claims = {"sha": "somesha"} + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION] + ) + + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) + monkeypatch.setattr( + Attestation, "verify", lambda *args: ("wrong-predicate", {}) + ) + + with pytest.raises( + AttestationUploadError, match="Attestation with unsupported predicate" + ): + integrity_service.parse_attestations( + db_request, + pretend.stub(), + ) + + assert ( + pretend.call( + "warehouse.upload.attestations.failed_unsupported_predicate_type" + ) + in metrics.increment.calls + ) + + def test_parse_succeed(self, metrics, monkeypatch, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=metrics, + ) + db_request.oidc_publisher = pretend.stub( + publisher_name="GitHub", + publisher_verification_policy=pretend.call_recorder(lambda c: None), + ) + db_request.oidc_claims = {"sha": "somesha"} + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION] + ) + + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) + monkeypatch.setattr( + Attestation, "verify", lambda *args: (AttestationType.PYPI_PUBLISH_V1, {}) + ) + + attestations = integrity_service.parse_attestations( + db_request, + pretend.stub(), + ) + assert attestations == [VALID_ATTESTATION] + + def test_generate_provenance_unsupported_publisher(self): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=pretend.stub(), + ) + + oidc_publisher = pretend.stub(publisher_name="not-existing") + + assert ( + integrity_service.generate_provenance(oidc_publisher, pretend.stub()) + is None + ) + + @pytest.mark.parametrize( + "publisher_name", + [ + "github", + "gitlab", + ], + ) + def test_generate_provenance_succeeds(self, publisher_name: str, monkeypatch): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=pretend.stub(), + ) + + if publisher_name == "github": + publisher = GitHubPublisher( + repository="fake-repository", + workflow="fake-workflow", + ) + else: + publisher = GitLabPublisher( + repository="fake-repository", + environment="fake-env", + ) + + monkeypatch.setattr( + services, + "_publisher_from_oidc_publisher", + lambda s: publisher, + ) + + provenance = integrity_service.generate_provenance( + pretend.stub(), + [VALID_ATTESTATION], + ) + + assert provenance == Provenance( + attestation_bundles=[ + AttestationBundle( + publisher=publisher, + attestations=[VALID_ATTESTATION], + ) + ] + ) + + def test_persist_provenance_succeeds(self, db_request): + provenance = Provenance( + attestation_bundles=[ + AttestationBundle( + publisher=GitHubPublisher( + repository="fake-repository", + workflow="fake-workflow", + ), + attestations=[VALID_ATTESTATION], + ) + ] + ) + + @pretend.call_recorder + def storage_service_store(path, file_path, *_args, **_kwargs): + expected = provenance.model_dump_json().encode("utf-8") + with open(file_path, "rb") as fp: + assert fp.read() == expected + + assert path.suffix == ".provenance" + + integrity_service = IntegrityService( + storage=pretend.stub(store=storage_service_store), + metrics=pretend.stub(), + ) + assert ( + integrity_service.persist_provenance(provenance, FileFactory.create()) + is None + ) + + def test_get_provenance_digest(self, db_request): + file = FileFactory.create() + FileEventFactory.create( + source=file, + tag=EventTag.File.FileAdd, + additional={"publisher_url": "fake-publisher-url"}, + ) + + with tempfile.NamedTemporaryFile() as f: + integrity_service = IntegrityService( + storage=pretend.stub(get=pretend.call_recorder(lambda p: f)), + metrics=pretend.stub(), + ) + + assert ( + integrity_service.get_provenance_digest(file) + == hashlib.file_digest(f, "sha256").hexdigest() + ) + + def test_get_provenance_digest_fails_no_attestations(self, db_request): + # If the attestations are missing, there is no provenance file + file = FileFactory.create() + file.attestations = [] + FileEventFactory.create( + source=file, + tag=EventTag.File.FileAdd, + additional={"publisher_url": "fake-publisher-url"}, + ) + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=pretend.stub(), + ) + + assert integrity_service.get_provenance_digest(file) is None + + +def test_publisher_from_oidc_publisher_github(db_request): + publisher = GitHubPublisherFactory.create() + + attestation_publisher = services._publisher_from_oidc_publisher(publisher) + assert isinstance(attestation_publisher, GitHubPublisher) + assert attestation_publisher.repository == publisher.repository + assert attestation_publisher.workflow == publisher.workflow_filename + assert attestation_publisher.environment == publisher.environment + + +def test_publisher_from_oidc_publisher_gitlab(db_request): + publisher = GitLabPublisherFactory.create() + + attestation_publisher = services._publisher_from_oidc_publisher(publisher) + assert isinstance(attestation_publisher, GitLabPublisher) + assert attestation_publisher.repository == publisher.project_path + assert attestation_publisher.environment == publisher.environment + + +def test_publisher_from_oidc_publisher_fails(): + publisher = pretend.stub(publisher_name="not-existing") + + with pytest.raises(UnsupportedPublisherError): + services._publisher_from_oidc_publisher(publisher) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index e127299ca204..6e062ef1f0f2 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -23,15 +23,8 @@ import pretend import pytest -from pypi_attestations import ( - Attestation, - Distribution, - Envelope, - VerificationError, - VerificationMaterial, -) +from pypi_attestations import Attestation, Envelope, VerificationMaterial from pyramid.httpexceptions import HTTPBadRequest, HTTPForbidden, HTTPTooManyRequests -from sigstore.verify import Verifier from sqlalchemy import and_, exists from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import joinedload @@ -42,6 +35,11 @@ from warehouse.accounts.utils import UserContext from warehouse.admin.flags import AdminFlag, AdminFlagValue +from warehouse.attestations import ( + Attestation as DatabaseAttestation, + AttestationUploadError, + IIntegrityService, +) from warehouse.classifiers.models import Classifier from warehouse.forklift import legacy, metadata from warehouse.macaroons import IMacaroonService, caveats, security_policy @@ -63,6 +61,7 @@ from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files from ...common.db.accounts import EmailFactory, UserFactory +from ...common.db.attestation import AttestationFactory from ...common.db.classifiers import ClassifierFactory from ...common.db.oidc import GitHubPublisherFactory from ...common.db.packaging import ( @@ -2420,85 +2419,6 @@ def test_upload_fails_without_oidc_publisher_permission( "See /the/help/url/ for more information." ).format(project.name) - def test_upload_attestation_fails_without_oidc_publisher( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - project_service, - macaroon_service, - ): - project = ProjectFactory.create() - owner = UserFactory.create() - maintainer = UserFactory.create() - RoleFactory.create(user=owner, project=project, role_name="Owner") - RoleFactory.create(user=maintainer, project=project, role_name="Maintainer") - - EmailFactory.create(user=maintainer) - db_request.user = maintainer - raw_macaroon, macaroon = macaroon_service.create_macaroon( - "fake location", - "fake description", - [caveats.RequestUser(user_id=str(maintainer.id))], - user_id=maintainer.id, - ) - identity = UserContext(maintainer, macaroon) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="some_cert", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", - "version": "1.0", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - extract_http_macaroon = pretend.call_recorder(lambda r, _: raw_macaroon) - monkeypatch.setattr( - security_policy, "_extract_http_macaroon", extract_http_macaroon - ) - - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMacaroonService: macaroon_service, - IMetricsService: metrics, - IProjectService: project_service, - }.get(svc) - db_request.user_agent = "warehouse-tests/6.6.6" - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status == ( - "400 Attestations are currently only supported when using Trusted " - "Publishing with GitHub Actions." - ) - @pytest.mark.parametrize( "plat", [ @@ -3408,8 +3328,10 @@ def test_upload_succeeds_creates_release( ), ] - def test_upload_with_valid_attestation_succeeds( + @pytest.mark.parametrize("provenance_rv", [None, "fake-provenance-object"]) + def test_upload_succeeds_with_valid_attestation( self, + provenance_rv, monkeypatch, pyramid_config, db_request, @@ -3464,296 +3386,68 @@ def test_upload_with_valid_attestation_succeeds( } ) - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None - ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: ( - "https://docs.pypi.org/attestations/publish/v1", - None, - ) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - - resp = legacy.file_upload(db_request) - - assert resp.status_code == 200 - - assert len(verify.calls) == 1 - verified_distribution = verify.calls[0].args[3] - assert verified_distribution == Distribution( - name=filename, digest=_TAR_GZ_PKG_SHA256 - ) + def persist_attestations(attestations, file): + file.attestations.append(AttestationFactory.create(file=file)) - def test_upload_with_invalid_attestation_predicate_type_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents + def persist_provenance(provenance_object, file): + assert provenance_object == provenance_rv - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", + storage_service = pretend.stub(store=lambda path, filepath, *, meta=None: None) + integrity_service = pretend.stub( + parse_attestations=lambda *args, **kwargs: [attestation], + persist_attestations=persist_attestations, + generate_provenance=pretend.call_recorder( + lambda oidc_publisher, attestations: provenance_rv ), + persist_provenance=persist_provenance, ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, + IIntegrityService: integrity_service, }.get(svc) record_event = pretend.call_recorder( lambda self, *, tag, request=None, additional: None ) monkeypatch.setattr(HasEvents, "record_event", record_event) + resp = legacy.file_upload(db_request) - invalid_predicate_type = "Unsupported predicate type" - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: (invalid_predicate_type, None) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - f"400 Attestation with unsupported predicate type: {invalid_predicate_type}" - ) - - def test_upload_with_multiple_attestations_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}," - f" {attestation.model_dump_json()}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None - ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: ( - "https://docs.pypi.org/attestations/publish/v1", - None, - ) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - "400 Only a single attestation per-file is supported at the moment." - ) - - def test_upload_with_malformed_attestation_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") + assert resp.status_code == 200 - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": "[{'a_malformed_attestation': 3}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } + assert ( + pretend.call("warehouse.upload.attestations.ok") in metrics.increment.calls ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None + attestations_db = ( + db_request.db.query(DatabaseAttestation) + .join(DatabaseAttestation.file) + .filter(File.filename == filename) + .all() ) - monkeypatch.setattr(HasEvents, "record_event", record_event) + assert len(attestations_db) == 1 - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - "400 Error while decoding the included attestation:" - ) + assert integrity_service.generate_provenance.calls == [ + pretend.call(db_request.oidc_publisher, [attestation]) + ] @pytest.mark.parametrize( - ("verify_exception", "expected_msg"), + "expected_message", [ - ( - VerificationError, - "400 Could not verify the uploaded artifact using the included " - "attestation", - ), - ( - ValueError, - "400 Unknown error while trying to verify included attestations", - ), + "Attestations are only supported when using", + "Error while decoding the included attestation", + "Only a single attestation", + "Could not verify the uploaded", + "Unknown error while trying", + "Attestation with unsupported predicate", ], ) - def test_upload_with_failing_attestation_verification( + def test_upload_fails_attestation_error( self, monkeypatch, pyramid_config, db_request, metrics, - verify_exception, - expected_msg, + expected_message, ): from warehouse.events.models import HasEvents @@ -3773,16 +3467,6 @@ def test_upload_with_failing_attestation_verification( db_request.db.add(Classifier(classifier="Programming Language :: Python")) filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) pyramid_config.testing_securitypolicy(identity=identity) db_request.user = None @@ -3791,7 +3475,7 @@ def test_upload_with_failing_attestation_verification( { "metadata_version": "1.2", "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", + "attestations": "", "version": version, "summary": "This is my summary!", "filetype": "sdist", @@ -3805,9 +3489,15 @@ def test_upload_with_failing_attestation_verification( ) storage_service = pretend.stub(store=lambda path, filepath, meta: None) + + def stub_parse(*_args, **_kwargs): + raise AttestationUploadError(expected_message) + + integrity_service = pretend.stub(parse_attestations=stub_parse) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, + IIntegrityService: integrity_service, }.get(svc) record_event = pretend.call_recorder( @@ -3815,19 +3505,13 @@ def test_upload_with_failing_attestation_verification( ) monkeypatch.setattr(HasEvents, "record_event", record_event) - def failing_verify(_self, _verifier, _policy, _dist): - raise verify_exception("error") - - monkeypatch.setattr(Attestation, "verify", failing_verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - with pytest.raises(HTTPBadRequest) as excinfo: legacy.file_upload(db_request) resp = excinfo.value assert resp.status_code == 400 - assert resp.status.startswith(expected_msg) + assert resp.status.startswith(f"400 {expected_message}") @pytest.mark.parametrize( ("url", "expected"), diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index afa7bd2056fe..8065c8b9ad8c 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -15,6 +15,7 @@ import pretend +from warehouse.attestations import IIntegrityService from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.utils import _simple_detail, render_simple_detail @@ -27,11 +28,37 @@ def test_simple_detail_empty_string(db_request): FileFactory.create(release=release) db_request.route_url = lambda *a, **kw: "the-url" + db_request.find_service = lambda svc, name=None, context=None: { + IIntegrityService: pretend.stub( + get_provenance_digest=pretend.call_recorder(lambda f: None), + ), + }.get(svc) + expected_content = _simple_detail(project, db_request) assert expected_content["files"][0]["requires-python"] is None +def test_simple_detail_with_provenance(db_request): + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0") + FileFactory.create(release=release) + + hash_digest = "deadbeefdeadbeefdeadbeefdeadbeef" + + db_request.route_url = lambda *a, **kw: "the-url" + db_request.find_service = pretend.call_recorder( + lambda svc, name=None, context=None: { + IIntegrityService: pretend.stub( + get_provenance_digest=pretend.call_recorder(lambda f: hash_digest), + ), + }.get(svc) + ) + + expected_content = _simple_detail(project, db_request) + assert expected_content["files"][0]["provenance"] == hash_digest + + def test_render_simple_detail(db_request, monkeypatch, jinja): project = ProjectFactory.create() release1 = ReleaseFactory.create(project=project, version="1.0") @@ -49,6 +76,12 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): monkeypatch.setattr(hashlib, "blake2b", fakeblake2b) db_request.route_url = lambda *a, **kw: "the-url" + db_request.find_service = lambda svc, name=None, context=None: { + IIntegrityService: pretend.stub( + get_provenance_digest=pretend.call_recorder(lambda f: None), + ), + }.get(svc) + template = jinja.get_template("templates/api/simple/detail.html") expected_content = template.render( **_simple_detail(project, db_request), request=db_request @@ -78,6 +111,9 @@ def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): db_request.find_service = pretend.call_recorder( lambda svc, name=None, context=None: { ISimpleStorage: storage_service, + IIntegrityService: pretend.stub( + get_provenance_digest=pretend.call_recorder(lambda f: None), + ), }.get(svc) ) diff --git a/warehouse/attestations/__init__.py b/warehouse/attestations/__init__.py new file mode 100644 index 000000000000..76c816d7fef3 --- /dev/null +++ b/warehouse/attestations/__init__.py @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from warehouse.attestations.errors import ( + AttestationUploadError, + UnsupportedPublisherError, +) +from warehouse.attestations.interfaces import IIntegrityService +from warehouse.attestations.models import Attestation +from warehouse.attestations.services import IntegrityService + +__all__ = [ + "Attestation", + "AttestationUploadError", + "IIntegrityService", + "IntegrityService", + "UnsupportedPublisherError", +] diff --git a/warehouse/attestations/errors.py b/warehouse/attestations/errors.py new file mode 100644 index 000000000000..463a34a4da69 --- /dev/null +++ b/warehouse/attestations/errors.py @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class UnsupportedPublisherError(Exception): + pass + + +class AttestationUploadError(Exception): + pass diff --git a/warehouse/attestations/interfaces.py b/warehouse/attestations/interfaces.py new file mode 100644 index 000000000000..4055e92055a5 --- /dev/null +++ b/warehouse/attestations/interfaces.py @@ -0,0 +1,54 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pypi_attestations import Attestation, Distribution, Provenance +from pyramid.request import Request +from zope.interface import Interface + + +class IIntegrityService(Interface): + + def create_service(context, request): + """ + Create the service, given the context and request for which it is being + created for. + """ + + def persist_attestations(attestations: list[Attestation], file): + """ + ̀¦Persist attestations in storage. + """ + pass + + def parse_attestations( + request: Request, distribution: Distribution + ) -> list[Attestation]: + """ + Process any attestations included in a file upload request + """ + + def generate_provenance( + oidc_publisher, attestations: list[Attestation] + ) -> Provenance | None: + """ + Generate a Provenance object from an OIDCPublisher and its attestations. + """ + + def persist_provenance(provenance: Provenance, file) -> None: + """ + Persist a Provenance object in storage. + """ + + def get_provenance_digest(file) -> str | None: + """ + Compute a provenance file digest for a `File` if it exists. + """ diff --git a/warehouse/attestations/models.py b/warehouse/attestations/models.py new file mode 100644 index 000000000000..9b95bfb0d7ad --- /dev/null +++ b/warehouse/attestations/models.py @@ -0,0 +1,55 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import typing + +from pathlib import Path +from uuid import UUID + +from sqlalchemy import ForeignKey, orm +from sqlalchemy.dialects.postgresql import CITEXT +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, mapped_column + +from warehouse import db + +if typing.TYPE_CHECKING: + from warehouse.packaging.models import File + + +class Attestation(db.Model): + """ + Table used to store Attestations. + + Attestations are stored on disk. We keep in database only the attestation hash. + """ + + __tablename__ = "attestation" + + file_id: Mapped[UUID] = mapped_column( + ForeignKey("release_files.id", onupdate="CASCADE", ondelete="CASCADE"), + ) + file: Mapped[File] = orm.relationship(back_populates="attestations") + + attestation_file_blake2_digest: Mapped[str] = mapped_column(CITEXT) + + @hybrid_property + def attestation_path(self): + return "/".join( + [ + self.attestation_file_blake2_digest[:2], + self.attestation_file_blake2_digest[2:4], + self.attestation_file_blake2_digest[4:], + f"{Path(self.file.path).name}.attestation", + ] + ) diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py new file mode 100644 index 000000000000..056c47c4894e --- /dev/null +++ b/warehouse/attestations/services.py @@ -0,0 +1,231 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib +import tempfile +import typing + +from pathlib import Path + +import sentry_sdk + +from pydantic import TypeAdapter, ValidationError +from pypi_attestations import ( + Attestation, + AttestationBundle, + AttestationType, + Distribution, + GitHubPublisher, + GitLabPublisher, + Provenance, + Publisher, + VerificationError, +) +from pyramid.request import Request +from sigstore.verify import Verifier +from zope.interface import implementer + +from warehouse.attestations.errors import ( + AttestationUploadError, + UnsupportedPublisherError, +) +from warehouse.attestations.interfaces import IIntegrityService +from warehouse.attestations.models import Attestation as DatabaseAttestation +from warehouse.metrics.interfaces import IMetricsService +from warehouse.oidc.models import ( + GitHubPublisher as GitHubOIDCPublisher, + GitLabPublisher as GitLabOIDCPublisher, + OIDCPublisher, +) +from warehouse.packaging.interfaces import IFileStorage +from warehouse.packaging.models import File + + +def _publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: + """ + Convert an OIDCPublisher object in a pypi-attestations Publisher. + """ + match publisher.publisher_name: + case "GitLab": + publisher = typing.cast(GitLabOIDCPublisher, publisher) + return GitLabPublisher( + repository=publisher.project_path, environment=publisher.environment + ) + case "GitHub": + publisher = typing.cast(GitHubOIDCPublisher, publisher) + return GitHubPublisher( + repository=publisher.repository, + workflow=publisher.workflow_filename, + environment=publisher.environment, + ) + case _: + raise UnsupportedPublisherError + + +@implementer(IIntegrityService) +class IntegrityService: + + def __init__( + self, + storage: IFileStorage, + metrics: IMetricsService, + ): + self.storage: IFileStorage = storage + self.metrics: IMetricsService = metrics + + @classmethod + def create_service(cls, _context, request: Request): + return cls( + storage=request.find_service(IFileStorage), + metrics=request.find_service(IMetricsService), + ) + + def persist_attestations(self, attestations: list[Attestation], file: File) -> None: + for attestation in attestations: + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write(attestation.model_dump_json().encode("utf-8")) + + attestation_digest = hashlib.file_digest( + tmp_file, "blake2b" + ).hexdigest() + database_attestation = DatabaseAttestation( + file=file, attestation_file_blake2_digest=attestation_digest + ) + + self.storage.store( + database_attestation.attestation_path, + tmp_file.name, + meta=None, + ) + + file.attestations.append(database_attestation) + + def parse_attestations( + self, request: Request, distribution: Distribution + ) -> list[Attestation]: + """ + Process any attestations included in a file upload request + + Attestations, if present, will be parsed and verified against the uploaded + artifact. Attestations are only allowed when uploading via a Trusted + Publisher, because a Trusted Publisher provides the identity that will be + used to verify the attestations. + Only GitHub Actions Trusted Publishers are supported. + """ + publisher: OIDCPublisher | None = request.oidc_publisher + if not publisher or not publisher.publisher_name == "GitHub": + raise AttestationUploadError( + "Attestations are only supported when using Trusted " + "Publishing with GitHub Actions.", + ) + + try: + attestations = TypeAdapter(list[Attestation]).validate_json( + request.POST["attestations"] + ) + except ValidationError as e: + # Log invalid (malformed) attestation upload + self.metrics.increment("warehouse.upload.attestations.malformed") + raise AttestationUploadError( + f"Error while decoding the included attestation: {e}", + ) + + if len(attestations) > 1: + self.metrics.increment( + "warehouse.upload.attestations.failed_multiple_attestations" + ) + + raise AttestationUploadError( + "Only a single attestation per file is supported.", + ) + + verification_policy = publisher.publisher_verification_policy( + request.oidc_claims + ) + for attestation_model in attestations: + try: + predicate_type, _ = attestation_model.verify( + Verifier.production(), + verification_policy, + distribution, + ) + except VerificationError as e: + # Log invalid (failed verification) attestation upload + self.metrics.increment("warehouse.upload.attestations.failed_verify") + raise AttestationUploadError( + f"Could not verify the uploaded artifact using the included " + f"attestation: {e}", + ) + except Exception as e: + with sentry_sdk.push_scope() as scope: + scope.fingerprint = [e] + sentry_sdk.capture_message( + f"Unexpected error while verifying attestation: {e}" + ) + + raise AttestationUploadError( + f"Unknown error while trying to verify included attestations: {e}", + ) + + if predicate_type != AttestationType.PYPI_PUBLISH_V1: + self.metrics.increment( + "warehouse.upload.attestations.failed_unsupported_predicate_type" + ) + raise AttestationUploadError( + f"Attestation with unsupported predicate type: {predicate_type}", + ) + + return attestations + + def generate_provenance( + self, oidc_publisher: OIDCPublisher, attestations: list[Attestation] + ) -> Provenance | None: + try: + publisher: Publisher = _publisher_from_oidc_publisher(oidc_publisher) + except UnsupportedPublisherError: + sentry_sdk.capture_message( + f"Unsupported OIDCPublisher found {oidc_publisher.publisher_name}" + ) + + return None + + attestation_bundle = AttestationBundle( + publisher=publisher, + attestations=attestations, + ) + + return Provenance(attestation_bundles=[attestation_bundle]) + + def persist_provenance( + self, + provenance: Provenance, + file: File, + ) -> None: + """ + Persist a Provenance object in storage. + """ + provenance_file_path = Path(f"{file.path}.provenance") + with tempfile.NamedTemporaryFile() as f: + f.write(provenance.model_dump_json().encode("utf-8")) + f.flush() + + self.storage.store( + provenance_file_path, + f.name, + ) + + def get_provenance_digest(self, file: File) -> str | None: + """Returns the sha256 digest of the provenance file for the release.""" + if not file.attestations: + return None + + provenance_file = self.storage.get(f"{file.path}.provenance") + return hashlib.file_digest(provenance_file, "sha256").hexdigest() diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 89010bbaaee0..baae8da64e84 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -30,13 +30,7 @@ import wtforms import wtforms.validators -from pydantic import TypeAdapter, ValidationError -from pypi_attestations import ( - Attestation, - AttestationType, - Distribution, - VerificationError, -) +from pypi_attestations import Attestation, Distribution from pyramid.httpexceptions import ( HTTPBadRequest, HTTPException, @@ -48,11 +42,11 @@ ) from pyramid.request import Request from pyramid.view import view_config -from sigstore.verify import Verifier from sqlalchemy import and_, exists, func, orm from sqlalchemy.exc import MultipleResultsFound, NoResultFound from warehouse.admin.flags import AdminFlagValue +from warehouse.attestations import AttestationUploadError, IIntegrityService from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB @@ -377,88 +371,6 @@ def _is_duplicate_file(db_session, filename, hashes): return None -def _process_attestations(request, distribution: Distribution): - """ - Process any attestations included in a file upload request - - Attestations, if present, will be parsed and verified against the uploaded - artifact. Attestations are only allowed when uploading via a Trusted - Publisher, because a Trusted Publisher provides the identity that will be - used to verify the attestations. - Currently, only GitHub Actions Trusted Publishers are supported, and - attestations are discarded after verification. - """ - - metrics = request.find_service(IMetricsService, context=None) - - publisher = request.oidc_publisher - if not publisher or not publisher.publisher_name == "GitHub": - raise _exc_with_message( - HTTPBadRequest, - "Attestations are currently only supported when using Trusted " - "Publishing with GitHub Actions.", - ) - try: - attestations = TypeAdapter(list[Attestation]).validate_json( - request.POST["attestations"] - ) - except ValidationError as e: - # Log invalid (malformed) attestation upload - metrics.increment("warehouse.upload.attestations.malformed") - raise _exc_with_message( - HTTPBadRequest, - f"Error while decoding the included attestation: {e}", - ) - - if len(attestations) > 1: - metrics.increment("warehouse.upload.attestations.failed_multiple_attestations") - raise _exc_with_message( - HTTPBadRequest, - "Only a single attestation per-file is supported at the moment.", - ) - - verification_policy = publisher.publisher_verification_policy(request.oidc_claims) - for attestation_model in attestations: - try: - # For now, attestations are not stored, just verified - predicate_type, _ = attestation_model.verify( - Verifier.production(), - verification_policy, - distribution, - ) - except VerificationError as e: - # Log invalid (failed verification) attestation upload - metrics.increment("warehouse.upload.attestations.failed_verify") - raise _exc_with_message( - HTTPBadRequest, - f"Could not verify the uploaded artifact using the included " - f"attestation: {e}", - ) - except Exception as e: - with sentry_sdk.push_scope() as scope: - scope.fingerprint = [e] - sentry_sdk.capture_message( - f"Unexpected error while verifying attestation: {e}" - ) - - raise _exc_with_message( - HTTPBadRequest, - f"Unknown error while trying to verify included attestations: {e}", - ) - - if predicate_type != AttestationType.PYPI_PUBLISH_V1: - metrics.increment( - "warehouse.upload.attestations.failed_unsupported_predicate_type" - ) - raise _exc_with_message( - HTTPBadRequest, - f"Attestation with unsupported predicate type: {predicate_type}", - ) - - # Log successful attestation upload - metrics.increment("warehouse.upload.attestations.ok") - - _pypi_project_urls = [ "https://pypi.org/project/", "https://pypi.org/p/", @@ -1269,12 +1181,6 @@ def file_upload(request): k: h.hexdigest().lower() for k, h in metadata_file_hashes.items() } - if "attestations" in request.POST: - _process_attestations( - request=request, - distribution=Distribution(name=filename, digest=file_hashes["sha256"]), - ) - # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. @@ -1371,6 +1277,32 @@ def file_upload(request): }, ) + # If the user provided attestations, verify and store them + if "attestations" in request.POST: + integrity_service = request.find_service(IIntegrityService, context=None) + + try: + attestations: list[Attestation] = integrity_service.parse_attestations( + request, + Distribution(name=filename, digest=file_hashes["sha256"]), + ) + except AttestationUploadError as e: + raise _exc_with_message( + HTTPBadRequest, + str(e), + ) + + integrity_service.persist_attestations(attestations, file_) + + provenance = integrity_service.generate_provenance( + request.oidc_publisher, attestations + ) + if provenance: + integrity_service.persist_provenance(provenance, file_) + + # Log successful attestation upload + metrics.increment("warehouse.upload.attestations.ok") + # For existing releases, we check if any of the existing project URLs are unverified # and have been verified in the current upload. In that case, we mark them as # verified. diff --git a/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py new file mode 100644 index 000000000000..2b15277127f6 --- /dev/null +++ b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py @@ -0,0 +1,47 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +create Attestations table + +Revision ID: 7f0c9f105f44 +Revises: 26455e3712a2 +Create Date: 2024-07-25 15:49:01.993869 +""" + +import sqlalchemy as sa + +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "7f0c9f105f44" +down_revision = "26455e3712a2" + + +def upgrade(): + op.create_table( + "attestation", + sa.Column("file_id", sa.UUID(), nullable=False), + sa.Column( + "attestation_file_blake2_digest", postgresql.CITEXT(), nullable=False + ), + sa.Column( + "id", sa.UUID(), server_default=sa.text("gen_random_uuid()"), nullable=False + ), + sa.ForeignKeyConstraint( + ["file_id"], ["release_files.id"], onupdate="CASCADE", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade(): + op.drop_table("attestation") diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 344689b36d0b..615739f3a1b8 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -81,6 +81,7 @@ from warehouse.utils.db.types import bool_false, datetime_now if typing.TYPE_CHECKING: + from warehouse.attestations.models import Attestation from warehouse.oidc.models import OIDCPublisher _MONOTONIC_SEQUENCE = 42 @@ -852,6 +853,13 @@ def __table_args__(cls): # noqa comment="If True, the metadata for the file cannot be backfilled.", ) + # PEP 740 attestations + attestations: Mapped[list[Attestation]] = orm.relationship( + cascade="all, delete-orphan", + lazy="joined", + passive_deletes=True, + ) + @property def uploaded_via_trusted_publisher(self) -> bool: """Return True if the file was uploaded via a trusted publisher.""" diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 30c85b3feb50..5fc67bc29761 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -19,10 +19,12 @@ from pyramid_jinja2 import IJinja2Environment from sqlalchemy.orm import joinedload +from warehouse.attestations import IIntegrityService +from warehouse.attestations.models import Attestation from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.models import File, LifecycleStatus, Project, Release -API_VERSION = "1.1" +API_VERSION = "1.2" def _simple_index(request, serial): @@ -51,6 +53,7 @@ def _simple_detail(project, request): request.db.query(File) .options(joinedload(File.release)) .join(Release) + .join(Attestation) .filter(Release.project == project) # Exclude projects that are in the `quarantine-enter` lifecycle status. .join(Project) @@ -64,6 +67,8 @@ def _simple_detail(project, request): {f.release.version for f in files}, key=packaging_legacy.version.parse ) + integrity_service = request.find_service(IIntegrityService, context=None) + return { "meta": {"api-version": API_VERSION, "_last-serial": project.last_serial}, "name": project.normalized_name, @@ -97,6 +102,7 @@ def _simple_detail(project, request): if file.metadata_file_sha256_digest else False ), + "provenance": integrity_service.get_provenance_digest(file), } for file in files ], diff --git a/warehouse/templates/api/simple/detail.html b/warehouse/templates/api/simple/detail.html index 24b0042c5863..05e0221a5612 100644 --- a/warehouse/templates/api/simple/detail.html +++ b/warehouse/templates/api/simple/detail.html @@ -20,7 +20,7 @@

Links for {{ name }}

{% for file in files -%} - {{ file.filename }}
+ {{ file.filename }}
{% endfor -%}