diff --git a/dev/environment b/dev/environment index 2a1713df28d4..9388cb593bb0 100644 --- a/dev/environment +++ b/dev/environment @@ -46,6 +46,7 @@ BREACHED_EMAILS=warehouse.accounts.NullEmailBreachedService BREACHED_PASSWORDS=warehouse.accounts.NullPasswordBreachedService OIDC_BACKEND=warehouse.oidc.services.NullOIDCPublisherService +ATTESTATIONS_BACKEND=warehouse.attestations.services.NullIntegrityService METRICS_BACKEND=warehouse.metrics.DataDogMetrics host=notdatadog diff --git a/requirements/main.in b/requirements/main.in index b8d563c00c36..d1332cda631d 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -62,8 +62,8 @@ redis>=2.8.0,<6.0.0 rfc3986 sentry-sdk setuptools -sigstore~=3.0.0 -pypi-attestations==0.0.9 +sigstore~=3.2.0 +pypi-attestations==0.0.11 sqlalchemy[asyncio]>=2.0,<3.0 stdlib-list stripe diff --git a/requirements/main.txt b/requirements/main.txt index b1d6bae915bb..890504f9a98d 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1776,9 +1776,9 @@ pyparsing==3.1.4 \ --hash=sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c \ --hash=sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032 # via linehaul -pypi-attestations==0.0.9 \ - --hash=sha256:3bfc07f64a8db0d6e2646720e70df7c7cb01a2936056c764a2cc3268969332f2 \ - --hash=sha256:4b38cce5d221c8145cac255bfafe650ec0028d924d2b3572394df8ba8f07a609 +pypi-attestations==0.0.11 \ + --hash=sha256:b730e6b23874d94da0f3817b1f9dd3ecb6a80d685f62a18ad96e5b0396149ded \ + --hash=sha256:e74329074f049568591e300373e12fcd46a35e21723110856546e33bf2949efa # via -r requirements/main.in pyqrcode==1.2.1 \ --hash=sha256:1b2812775fa6ff5c527977c4cd2ccb07051ca7d0bc0aecf937a43864abe5eff6 \ @@ -2091,9 +2091,9 @@ sentry-sdk==2.13.0 \ --hash=sha256:6beede8fc2ab4043da7f69d95534e320944690680dd9a963178a49de71d726c6 \ --hash=sha256:8d4a576f7a98eb2fdb40e13106e41f330e5c79d72a68be1316e7852cf4995260 # via -r requirements/main.in -sigstore==3.0.0 \ - --hash=sha256:6cc7dc92607c2fd481aada0f3c79e710e4c6086e3beab50b07daa9a50a79d109 \ - --hash=sha256:a6a9538a648e112a0c3d8092d3f73a351c7598164764f1e73a6b5ba406a3a0bd +sigstore==3.2.0 \ + --hash=sha256:25c8a871a3a6adf959c0cde598ea8bef8794f1a29277d067111eb4ded4ba7f65 \ + --hash=sha256:d18508f34febb7775065855e92557fa1c2c16580df88f8e8903b9514438bad44 # via # -r requirements/main.in # pypi-attestations diff --git a/tests/common/db/attestation.py b/tests/common/db/attestation.py new file mode 100644 index 000000000000..2080519a806f --- /dev/null +++ b/tests/common/db/attestation.py @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib + +import factory + +from warehouse.attestations.models import Attestation + +from .base import WarehouseFactory + + +class AttestationFactory(WarehouseFactory): + class Meta: + model = Attestation + + file = factory.SubFactory("tests.common.db.packaging.FileFactory") + attestation_file_blake2_digest = factory.LazyAttribute( + lambda o: hashlib.blake2b(o.file.filename.encode("utf8")).hexdigest() + ) diff --git a/tests/common/db/packaging.py b/tests/common/db/packaging.py index 2a12379da170..4085ee617dac 100644 --- a/tests/common/db/packaging.py +++ b/tests/common/db/packaging.py @@ -34,6 +34,7 @@ from warehouse.utils import readme from .accounts import UserFactory +from .attestation import AttestationFactory from .base import WarehouseFactory from .observations import ObserverFactory @@ -140,6 +141,13 @@ class Meta: ) ) + # Empty attestations by default. + attestations = factory.RelatedFactoryList( + AttestationFactory, + factory_related_name="file", + size=0, + ) + class FileEventFactory(WarehouseFactory): class Meta: diff --git a/tests/conftest.py b/tests/conftest.py index d63b445c933b..ebee90a5f93a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,6 +44,8 @@ from warehouse.accounts import services as account_services from warehouse.accounts.interfaces import ITokenService, IUserService from warehouse.admin.flags import AdminFlag, AdminFlagValue +from warehouse.attestations import services as attestations_services +from warehouse.attestations.interfaces import IIntegrityService from warehouse.email import services as email_services from warehouse.email.interfaces import IEmailSender from warehouse.helpdesk import services as helpdesk_services @@ -57,7 +59,7 @@ from warehouse.organizations import services as organization_services from warehouse.organizations.interfaces import IOrganizationService from warehouse.packaging import services as packaging_services -from warehouse.packaging.interfaces import IProjectService +from warehouse.packaging.interfaces import IFileStorage, IProjectService from warehouse.subscriptions import services as subscription_services from warehouse.subscriptions.interfaces import IBillingService, ISubscriptionService @@ -112,6 +114,15 @@ def metrics(): ) +@pytest.fixture +def storage_service(tmp_path): + """ + A good-enough local file storage service. + """ + + return packaging_services.LocalArchiveFileStorage(tmp_path) + + @pytest.fixture def remote_addr(): return "1.2.3.4" @@ -173,6 +184,8 @@ def pyramid_services( project_service, github_oidc_service, activestate_oidc_service, + integrity_service, + storage_service, macaroon_service, helpdesk_service, ): @@ -194,7 +207,9 @@ def pyramid_services( services.register_service( activestate_oidc_service, IOIDCPublisherService, None, name="activestate" ) + services.register_service(integrity_service, IIntegrityService, None, name="") services.register_service(macaroon_service, IMacaroonService, None, name="") + services.register_service(storage_service, IFileStorage, None, name="archive") services.register_service(helpdesk_service, IHelpDeskService, None) return services @@ -324,6 +339,7 @@ def get_app_config(database, nondefaults=None): "docs.backend": "warehouse.packaging.services.LocalDocsStorage", "sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage", "billing.backend": "warehouse.subscriptions.services.MockStripeBillingService", + "attestations.backend": "warehouse.attestations.services.NullIntegrityService", "billing.api_base": "http://stripe:12111", "billing.api_version": "2020-08-27", "mail.backend": "warehouse.email.services.SMTPEmailSender", @@ -387,13 +403,11 @@ def get_db_session_for_app_config(app_config): @pytest.fixture(scope="session") def app_config(database): - return get_app_config(database) @pytest.fixture(scope="session") def app_config_dbsession_from_env(database): - nondefaults = { "warehouse.db_create_session": lambda r: r.environ.get("warehouse.db_session") } @@ -539,6 +553,11 @@ def activestate_oidc_service(db_session): ) +@pytest.fixture +def integrity_service(db_session): + return attestations_services.NullIntegrityService(db_session) + + @pytest.fixture def macaroon_service(db_session): return macaroon_services.DatabaseMacaroonService(db_session) diff --git a/tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation b/tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation new file mode 100644 index 000000000000..9f7a4eb28f53 --- /dev/null +++ b/tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation @@ -0,0 +1 @@ +{"version":1,"verification_material":{"certificate":"MIIC6zCCAnGgAwIBAgIUFgmhIYx8gvBGePCTacG/4kbBdRwwCgYIKoZIzj0EAwMwNzEVMBMGA1UE\nChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjQwODI5\nMTcwOTM5WhcNMjQwODI5MTcxOTM5WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEtGrMPml4\nOtsRJ3Z6qRahs0kHCZxP4n9fvrJE957WVxgAGg4k6a1PbRJY9nT9wKpRrZmKV++AgA9ndhdruXXa\nAKOCAZAwggGMMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAdBgNVHQ4EFgQU\nosNvhYEuTPfgyU/dZfu93lFGRNswHwYDVR0jBBgwFoAU39Ppz1YkEZb5qNjpKFWixi4YZD8wQAYD\nVR0RAQH/BDYwNIEyOTE5NDM2MTU4MjM2LWNvbXB1dGVAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3Vu\ndC5jb20wKQYKKwYBBAGDvzABAQQbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMCsGCisGAQQB\ng78wAQgEHQwbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMIGKBgorBgEEAdZ5AgQCBHwEegB4\nAHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGRnx0/aQAABAMARzBFAiBogvcK\nHIIR9FcX1vQgDhGtAl0XQoMRiEB3OdUWO94P1gIhANdJlyISdtvVrHes25dWKTLepy+IzQmzfQU/\nS7cxWHmOMAoGCCqGSM49BAMDA2gAMGUCMGe2xTiuenbjdt1d2e4IaCiwRh2G4KAtyujRESSSUbpu\nGme/o9ouiApeONBv2CvvGAIxAOEkAGFO3aALE3IPNosxqaz9MbqJOdmYhB1Cz1D7xbFc/m243VxJ\nWxaC/uOFEpyiYQ==\n","transparency_entries":[{"logIndex":"125970014","logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="},"kindVersion":{"kind":"dsse","version":"0.0.1"},"integratedTime":"1724951379","inclusionPromise":{"signedEntryTimestamp":"MEUCIQCHrKFTeXNY432S0bUSBS69S8d5JnNcDXa41q6OEvxEwgIgaZstc5Jpm0IgwFC7RDTXYEAKk+3aG/MkRkaPdJdyn8U="},"inclusionProof":{"logIndex":"4065752","rootHash":"7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=","treeSize":"4065754","hashes":["NwJgWJoxjearbnEIT9bnWXpzo0LGNrR1cpWId0g66rE=","kLjpW3Eh7pQJNOvyntghzF57tcfqk2IzX7cqiBDgGf8=","FW8y9LQ1i3q+MnbeGJipKGl4VfX1zRBOD7TmhbEw7uI=","mKcbGJDJ/+buNbXy9Eyv94nVoAyUauuIlN3cJg3qSBY=","5VytqqAHhfRkRWMrY43UXWCnRBb7JwElMlKpY5JueBc=","mZJnD39LTKdis2wUTz1OOMx3r7HwgJh9rnb2VwiPzts=","MXZOQFJFiOjREF0xwMOCXu29HwTchjTtl/BeFoI51wY=","g8zCkHnLwO3LojK7g5AnqE8ezSNRnCSz9nCL5GD3a8A=","RrZsD/RSxNoujlvq/MsCEvLSkKZfv0jmQM9Kp7qbJec=","QxmVWsbTp4cClxuAkuT51UH2EY7peHMVGKq7+b+cGwQ=","Q2LAtNzOUh+3PfwfMyNxYb06fTQmF3VeTT6Fr6Upvfc=","ftwAu6v62WFDoDmcZ1JKfrRPrvuiIw5v3BvRsgQj7N8="],"checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n4065754\n7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=\n\n— rekor.sigstore.dev wNI9ajBGAiEAhMomhZHOTNB5CVPO98CMXCv01ZlIF+C+CgzraAB01r8CIQCEuXbv6aqguUpB/ig5eXRIbarvxLXkg3nX48DzambktQ==\n"}},"canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiZHNzZSIsInNwZWMiOnsiZW52ZWxvcGVIYXNoIjp7ImFsZ29yaXRobSI6InNoYTI1NiIsInZhbHVlIjoiOWRiNGJjMzE3MTgyZWI3NzljNDIyY2Q0NGI2ZDdlYTk5ZWM1M2Q3M2JiY2ZjZWVmZTIyNWVlYjQ3NTQyMjc4OCJ9LCJwYXlsb2FkSGFzaCI6eyJhbGdvcml0aG0iOiJzaGEyNTYiLCJ2YWx1ZSI6IjlkYjY0MjlhOTkzZGFiYTI4NzAwODk2ZTY2MzNjNzkxYWE0MDM3ODQ4NjJiYzY2MDBkM2E4NjYwMGQzYjA1NjMifSwic2lnbmF0dXJlcyI6W3sic2lnbmF0dXJlIjoiTUVVQ0lCaGlOL25NR0w3aHpZQk9QQjlUTGtuaEdTZEtuQ0Q0ekI3TDV5ZXc0QmJ3QWlFQXJzOHl6MCtCT2NnSEtzS0JzTXVOeVlhREdaRTBVV0JuMEdwNVpGMzUvU2M9IiwidmVyaWZpZXIiOiJMUzB0TFMxQ1JVZEpUaUJEUlZKVVNVWkpRMEZVUlMwdExTMHRDazFKU1VNMmVrTkRRVzVIWjBGM1NVSkJaMGxWUm1kdGFFbFplRGhuZGtKSFpWQkRWR0ZqUnk4MGEySkNaRkozZDBObldVbExiMXBKZW1vd1JVRjNUWGNLVG5wRlZrMUNUVWRCTVZWRlEyaE5UV015Ykc1ak0xSjJZMjFWZFZwSFZqSk5ValIzU0VGWlJGWlJVVVJGZUZaNllWZGtlbVJIT1hsYVV6RndZbTVTYkFwamJURnNXa2RzYUdSSFZYZElhR05PVFdwUmQwOUVTVFZOVkdOM1QxUk5OVmRvWTA1TmFsRjNUMFJKTlUxVVkzaFBWRTAxVjJwQlFVMUdhM2RGZDFsSUNrdHZXa2w2YWpCRFFWRlpTVXR2V2tsNmFqQkVRVkZqUkZGblFVVjBSM0pOVUcxc05FOTBjMUpLTTFvMmNWSmhhSE13YTBoRFduaFFORzQ1Wm5aeVNrVUtPVFUzVjFaNFowRkhaelJyTm1FeFVHSlNTbGs1YmxRNWQwdHdVbkphYlV0V0t5dEJaMEU1Ym1Sb1pISjFXRmhoUVV0UFEwRmFRWGRuWjBkTlRVRTBSd3BCTVZWa1JIZEZRaTkzVVVWQmQwbElaMFJCVkVKblRsWklVMVZGUkVSQlMwSm5aM0pDWjBWR1FsRmpSRUY2UVdSQ1owNVdTRkUwUlVablVWVnZjMDUyQ21oWlJYVlVVR1puZVZVdlpGcG1kVGt6YkVaSFVrNXpkMGgzV1VSV1VqQnFRa0puZDBadlFWVXpPVkJ3ZWpGWmEwVmFZalZ4VG1wd1MwWlhhWGhwTkZrS1drUTRkMUZCV1VSV1VqQlNRVkZJTDBKRVdYZE9TVVY1VDFSRk5VNUVUVEpOVkZVMFRXcE5Na3hYVG5aaVdFSXhaRWRXUVZwSFZqSmFWM2gyWTBkV2VRcE1iV1I2V2xoS01tRlhUbXhaVjA1cVlqTldkV1JETldwaU1qQjNTMUZaUzB0M1dVSkNRVWRFZG5wQlFrRlJVV0poU0ZJd1kwaE5Oa3g1T1doWk1rNTJDbVJYTlRCamVUVnVZakk1Ym1KSFZYVlpNamwwVFVOelIwTnBjMGRCVVZGQ1p6YzRkMEZSWjBWSVVYZGlZVWhTTUdOSVRUWk1lVGxvV1RKT2RtUlhOVEFLWTNrMWJtSXlPVzVpUjFWMVdUSTVkRTFKUjB0Q1oyOXlRbWRGUlVGa1dqVkJaMUZEUWtoM1JXVm5RalJCU0ZsQk0xUXdkMkZ6WWtoRlZFcHFSMUkwWXdwdFYyTXpRWEZLUzFoeWFtVlFTek12YURSd2VXZERPSEEzYnpSQlFVRkhVbTU0TUM5aFVVRkJRa0ZOUVZKNlFrWkJhVUp2WjNaalMwaEpTVkk1Um1OWUNqRjJVV2RFYUVkMFFXd3dXRkZ2VFZKcFJVSXpUMlJWVjA4NU5GQXhaMGxvUVU1a1NteDVTVk5rZEhaV2NraGxjekkxWkZkTFZFeGxjSGtyU1hwUmJYb0tabEZWTDFNM1kzaFhTRzFQVFVGdlIwTkRjVWRUVFRRNVFrRk5SRUV5WjBGTlIxVkRUVWRsTW5oVWFYVmxibUpxWkhReFpESmxORWxoUTJsM1VtZ3lSd28wUzBGMGVYVnFVa1ZUVTFOVlluQjFSMjFsTDI4NWIzVnBRWEJsVDA1Q2RqSkRkblpIUVVsNFFVOUZhMEZIUms4ellVRk1SVE5KVUU1dmMzaHhZWG81Q2sxaWNVcFBaRzFaYUVJeFEzb3hSRGQ0WWtaakwyMHlORE5XZUVwWGVHRkRMM1ZQUmtWd2VXbFpVVDA5Q2kwdExTMHRSVTVFSUVORlVsUkpSa2xEUVZSRkxTMHRMUzBLIn1dfX0="}]},"envelope":{"statement":"eyJfdHlwZSI6Imh0dHBzOi8vaW4tdG90by5pby9TdGF0ZW1lbnQvdjEiLCJzdWJqZWN0IjpbeyJu\nYW1lIjoic2FtcGxlcHJvamVjdC0zLjAuMC50YXIuZ3oiLCJkaWdlc3QiOnsic2hhMjU2IjoiMTE3\nZWQ4OGU1ZGIwNzNiYjkyOTY5YTc1NDU3NDVmZDk3N2VlODViNzAxOTcwNmRkMjU2YTY0MDU4Zjcw\nOTYzZCJ9fV0sInByZWRpY2F0ZVR5cGUiOiJodHRwczovL2RvY3MucHlwaS5vcmcvYXR0ZXN0YXRp\nb25zL3B1Ymxpc2gvdjEiLCJwcmVkaWNhdGUiOm51bGx9\n","signature":"MEUCIBhiN/nMGL7hzYBOPB9TLknhGSdKnCD4zB7L5yew4BbwAiEArs8yz0+BOcgHKsKBsMuNyYaD\nGZE0UWBn0Gp5ZF35/Sc=\n"}} \ No newline at end of file diff --git a/tests/functional/api/test_simple.py b/tests/functional/api/test_simple.py index 636530b0233b..377563c7dd02 100644 --- a/tests/functional/api/test_simple.py +++ b/tests/functional/api/test_simple.py @@ -10,9 +10,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +import hashlib + from http import HTTPStatus +from pathlib import Path + +import pymacaroons + +from warehouse.macaroons import caveats -from ...common.db.packaging import ProjectFactory, ReleaseFactory +from ...common.db.accounts import EmailFactory, UserFactory +from ...common.db.macaroons import MacaroonFactory +from ...common.db.oidc import GitHubPublisherFactory +from ...common.db.packaging import ProjectFactory, ReleaseFactory, RoleFactory + +_HERE = Path(__file__).parent +_ASSETS = _HERE.parent / "_fixtures" def test_simple_api_html(webtest): @@ -31,3 +44,88 @@ def test_simple_api_detail(webtest): assert resp.content_type == "text/html" assert "X-PyPI-Last-Serial" in resp.headers assert f"Links for {project.normalized_name}" in resp.text + + +def test_simple_attestations_from_upload(webtest): + user = UserFactory.create( + password=( # 'password' + "$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ" + "HOJaqfBroT0JCieHug281c" + ) + ) + EmailFactory.create(user=user, verified=True) + project = ProjectFactory.create(name="sampleproject") + RoleFactory.create(user=user, project=project, role_name="Owner") + publisher = GitHubPublisherFactory.create(projects=[project]) + + # Construct the macaroon. This needs to be based on a Trusted Publisher, which is + # required to upload attestations + dm = MacaroonFactory.create( + oidc_publisher_id=publisher.id, + caveats=[ + caveats.OIDCPublisher(oidc_publisher_id=str(publisher.id)), + caveats.ProjectID(project_ids=[str(p.id) for p in publisher.projects]), + ], + additional={"oidc": {"ref": "someref", "sha": "somesha"}}, + ) + + m = pymacaroons.Macaroon( + location="localhost", + identifier=str(dm.id), + key=dm.key, + version=pymacaroons.MACAROON_V2, + ) + for caveat in dm.caveats: + m.add_first_party_caveat(caveats.serialize(caveat)) + serialized_macaroon = f"pypi-{m.serialize()}" + + with open(_ASSETS / "sampleproject-3.0.0.tar.gz", "rb") as f: + content = f.read() + + with open( + _ASSETS / "sampleproject-3.0.0.tar.gz.publish.attestation", + ) as f: + attestation = f.read() + + webtest.set_authorization(("Basic", ("__token__", serialized_macaroon))) + webtest.post( + "/legacy/?:action=file_upload", + params={ + "name": "sampleproject", + "sha256_digest": ( + "117ed88e5db073bb92969a7545745fd977ee85b7019706dd256a64058f70963d" + ), + "filetype": "sdist", + "metadata_version": "2.1", + "version": "3.0.0", + "attestations": f"[{attestation}]", + }, + upload_files=[("content", "sampleproject-3.0.0.tar.gz", content)], + status=HTTPStatus.OK, + ) + + assert len(project.releases) == 1 + assert project.releases[0].files.count() == 1 + assert len(project.releases[0].files[0].attestations) == 1 + + expected_provenance = hashlib.sha256(b"sampleproject-3.0.0.tar.gz:1").hexdigest() + expected_filename = "sampleproject-3.0.0.tar.gz" + + response = webtest.get("/simple/sampleproject/", status=HTTPStatus.OK) + link = response.html.find("a", text=expected_filename) + + assert "data-provenance" in link.attrs + assert link.get("data-provenance") == expected_provenance + + response = webtest.get( + "/simple/sampleproject/", + headers={"Accept": "application/vnd.pypi.simple.v1+json"}, + status=HTTPStatus.OK, + ) + + assert response.content_type == "application/vnd.pypi.simple.v1+json" + + json_content = response.json + assert len(json_content["files"]) == 1 + assert json_content["files"][0]["filename"] == expected_filename + assert json_content["files"][0]["provenance"] == expected_provenance diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index 9937038dd145..ba8980762c21 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -17,6 +17,7 @@ from pyramid.httpexceptions import HTTPMovedPermanently from pyramid.testing import DummyRequest +from tests.common.db.attestation import AttestationFactory from warehouse.api import simple from warehouse.packaging.utils import API_VERSION @@ -286,6 +287,7 @@ def test_with_files_no_serial(self, db_request, content_type, renderer_override) "upload-time": f.upload_time.isoformat() + "Z", "data-dist-info-metadata": False, "core-metadata": False, + "provenance": None, } for f in files ], @@ -334,6 +336,7 @@ def test_with_files_with_serial(self, db_request, content_type, renderer_overrid "upload-time": f.upload_time.isoformat() + "Z", "data-dist-info-metadata": False, "core-metadata": False, + "provenance": None, } for f in files ], @@ -427,6 +430,7 @@ def test_with_files_with_version_multi_digit( if f.metadata_file_sha256_digest is not None else False ), + "provenance": None, } for f in files ], @@ -439,6 +443,65 @@ def test_with_files_with_version_multi_digit( if renderer_override is not None: assert db_request.override_renderer == renderer_override + def test_with_files_varying_provenance(self, db_request, integrity_service): + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0.0") + + # wheel with provenance, sdist with no provenance + wheel = FileFactory.create( + release=release, + filename=f"{project.name}-1.0.0.whl", + packagetype="bdist_wheel", + metadata_file_sha256_digest="deadbeefdeadbeefdeadbeefdeadbeef", + ) + AttestationFactory.create(file=wheel) + sdist = FileFactory.create( + release=release, + filename=f"{project.name}-1.0.0.tar.gz", + packagetype="sdist", + ) + + files = [sdist, wheel] + + urls_iter = (f"/file/{f.filename}" for f in files) + db_request.matchdict["name"] = project.normalized_name + db_request.route_url = lambda *a, **kw: next(urls_iter) + user = UserFactory.create() + je = JournalEntryFactory.create(name=project.name, submitted_by=user) + + assert simple.simple_detail(project, db_request) == { + "meta": {"_last-serial": je.id, "api-version": API_VERSION}, + "name": project.normalized_name, + "versions": ["1.0.0"], + "files": [ + { + "filename": f.filename, + "url": f"/file/{f.filename}", + "hashes": {"sha256": f.sha256_digest}, + "requires-python": f.requires_python, + "yanked": False, + "size": f.size, + "upload-time": f.upload_time.isoformat() + "Z", + "data-dist-info-metadata": ( + {"sha256": "deadbeefdeadbeefdeadbeefdeadbeef"} + if f.metadata_file_sha256_digest is not None + else False + ), + "core-metadata": ( + {"sha256": "deadbeefdeadbeefdeadbeefdeadbeef"} + if f.metadata_file_sha256_digest is not None + else False + ), + "provenance": integrity_service.get_provenance_digest(f), + } + for f in files + ], + } + + # Backstop: assert that we're testing at least provenance above + # by confirming that the wheel has one. + assert integrity_service.get_provenance_digest(wheel) is not None + def test_with_files_quarantined_omitted_from_index(self, db_request): db_request.accept = "text/html" project = ProjectFactory.create(lifecycle_status="quarantine-enter") diff --git a/tests/unit/attestations/__init__.py b/tests/unit/attestations/__init__.py new file mode 100644 index 000000000000..164f68b09175 --- /dev/null +++ b/tests/unit/attestations/__init__.py @@ -0,0 +1,11 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/unit/attestations/test_init.py b/tests/unit/attestations/test_init.py new file mode 100644 index 000000000000..6c5e26a8776f --- /dev/null +++ b/tests/unit/attestations/test_init.py @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pretend + +from warehouse import attestations +from warehouse.attestations.interfaces import IIntegrityService + + +def test_includeme(): + fake_service_klass = pretend.stub(create_service=pretend.stub()) + config = pretend.stub( + registry=pretend.stub( + settings={"attestations.backend": "fake.path.to.backend"} + ), + maybe_dotted=pretend.call_recorder( + lambda attr: fake_service_klass, + ), + register_service_factory=pretend.call_recorder( + lambda factory, iface, name=None: None + ), + ) + + attestations.includeme(config) + + assert config.maybe_dotted.calls == [pretend.call("fake.path.to.backend")] + assert config.register_service_factory.calls == [ + pretend.call(fake_service_klass.create_service, IIntegrityService), + ] diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py new file mode 100644 index 000000000000..3750a669adb0 --- /dev/null +++ b/tests/unit/attestations/test_services.py @@ -0,0 +1,440 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib + +import pretend +import pytest + +from pydantic import TypeAdapter +from pypi_attestations import ( + Attestation, + AttestationBundle, + AttestationType, + Envelope, + GitHubPublisher, + GitLabPublisher, + Provenance, + VerificationError, + VerificationMaterial, +) +from sigstore.verify import Verifier +from zope.interface.verify import verifyClass + +from tests.common.db.oidc import GitHubPublisherFactory, GitLabPublisherFactory +from tests.common.db.packaging import FileFactory +from warehouse.attestations import ( + Attestation as DatabaseAttestation, + AttestationUploadError, + IIntegrityService, + IntegrityService, + UnsupportedPublisherError, + services, +) +from warehouse.packaging import File + +VALID_ATTESTATION = Attestation( + version=1, + verification_material=VerificationMaterial( + certificate="somebase64string", transparency_entries=[dict()] + ), + envelope=Envelope( + statement="somebase64string", + signature="somebase64string", + ), +) + + +class TestNullIntegrityService: + def test_interface_matches(self): + assert verifyClass(IIntegrityService, services.NullIntegrityService) + + def test_get_provenance_digest(self, db_request): + db_request.oidc_publisher = pretend.stub( + publisher_name="GitHub", + repository="fake/fake", + workflow_filename="fake.yml", + environment="fake", + ) + + file = FileFactory.create() + service = services.NullIntegrityService(session=db_request.db) + + provenance = service.generate_provenance(db_request, file, [VALID_ATTESTATION]) + assert isinstance(provenance, Provenance) + + provenance_digest = service.get_provenance_digest(file) + assert isinstance(provenance_digest, str) + + +class TestIntegrityService: + def test_interface_matches(self): + assert verifyClass(IIntegrityService, IntegrityService) + + def test_create_service(self, db_request): + service = IntegrityService.create_service(None, db_request) + assert isinstance(service, IntegrityService) + + def test_persist_attestations_succeeds(self, db_request, storage_service): + integrity_service = IntegrityService( + storage=storage_service, + metrics=pretend.stub(), + session=db_request.db, + ) + + file = FileFactory.create() + integrity_service._persist_attestations([VALID_ATTESTATION], file) + + attestations_db = ( + db_request.db.query(DatabaseAttestation) + .join(DatabaseAttestation.file) + .filter(File.filename == file.filename) + .all() + ) + assert len(attestations_db) == 1 + assert len(file.attestations) == 1 + + attestation_path = attestations_db[0].attestation_path + + assert attestation_path.endswith(".attestation") + assert ( + storage_service.get(attestation_path).read() + == VALID_ATTESTATION.model_dump_json().encode() + ) + + def test_parse_attestations_fails_no_publisher(self, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=pretend.stub(), + session=db_request.db, + ) + + db_request.oidc_publisher = None + with pytest.raises( + AttestationUploadError, + match="Attestations are only supported when using Trusted", + ): + integrity_service.parse_attestations(db_request, pretend.stub()) + + def test_parse_attestations_fails_unsupported_publisher(self, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=pretend.stub(), + session=db_request.db, + ) + db_request.oidc_publisher = pretend.stub(publisher_name="not-existing") + with pytest.raises( + AttestationUploadError, + match="Attestations are only supported when using Trusted", + ): + integrity_service.parse_attestations(db_request, pretend.stub()) + + def test_parse_attestations_fails_malformed_attestation(self, metrics, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=metrics, + session=db_request.db, + ) + + db_request.oidc_publisher = pretend.stub(publisher_name="GitHub") + db_request.POST["attestations"] = "{'malformed-attestation'}" + with pytest.raises( + AttestationUploadError, + match="Malformed attestations", + ): + integrity_service.parse_attestations(db_request, pretend.stub()) + + assert ( + pretend.call("warehouse.upload.attestations.malformed") + in metrics.increment.calls + ) + + def test_parse_attestations_fails_multiple_attestations(self, metrics, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=metrics, + session=db_request.db, + ) + + db_request.oidc_publisher = pretend.stub(publisher_name="GitHub") + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION, VALID_ATTESTATION] + ) + with pytest.raises( + AttestationUploadError, match="Only a single attestation per file" + ): + integrity_service.parse_attestations( + db_request, + pretend.stub(), + ) + + assert ( + pretend.call("warehouse.upload.attestations.failed_multiple_attestations") + in metrics.increment.calls + ) + + @pytest.mark.parametrize( + ("verify_exception", "expected_message"), + [ + ( + VerificationError, + "Could not verify the uploaded", + ), + ( + ValueError, + "Unknown error while", + ), + ], + ) + def test_parse_attestations_fails_verification( + self, metrics, monkeypatch, db_request, verify_exception, expected_message + ): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=metrics, + session=db_request.db, + ) + + db_request.oidc_publisher = pretend.stub( + publisher_name="GitHub", + publisher_verification_policy=pretend.call_recorder(lambda c: None), + ) + db_request.oidc_claims = {"sha": "somesha"} + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION] + ) + + def failing_verify(_self, _verifier, _policy, _dist): + raise verify_exception("error") + + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) + monkeypatch.setattr(Attestation, "verify", failing_verify) + + with pytest.raises(AttestationUploadError, match=expected_message): + integrity_service.parse_attestations( + db_request, + pretend.stub(), + ) + + def test_parse_attestations_fails_wrong_predicate( + self, metrics, monkeypatch, db_request + ): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=metrics, + session=db_request.db, + ) + db_request.oidc_publisher = pretend.stub( + publisher_name="GitHub", + publisher_verification_policy=pretend.call_recorder(lambda c: None), + ) + db_request.oidc_claims = {"sha": "somesha"} + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION] + ) + + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) + monkeypatch.setattr( + Attestation, "verify", lambda *args: ("wrong-predicate", {}) + ) + + with pytest.raises( + AttestationUploadError, match="Attestation with unsupported predicate" + ): + integrity_service.parse_attestations( + db_request, + pretend.stub(), + ) + + assert ( + pretend.call( + "warehouse.upload.attestations.failed_unsupported_predicate_type" + ) + in metrics.increment.calls + ) + + def test_parse_attestations_succeeds(self, metrics, monkeypatch, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=metrics, + session=db_request.db, + ) + db_request.oidc_publisher = pretend.stub( + publisher_name="GitHub", + publisher_verification_policy=pretend.call_recorder(lambda c: None), + ) + db_request.oidc_claims = {"sha": "somesha"} + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION] + ) + + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) + monkeypatch.setattr( + Attestation, "verify", lambda *args: (AttestationType.PYPI_PUBLISH_V1, {}) + ) + + attestations = integrity_service.parse_attestations( + db_request, + pretend.stub(), + ) + assert attestations == [VALID_ATTESTATION] + + def test_generate_provenance_fails_unsupported_publisher(self, db_request, metrics): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=pretend.stub(), + session=db_request.db, + ) + + db_request.oidc_publisher = pretend.stub(publisher_name="not-existing") + + file = FileFactory.create() + assert ( + integrity_service.generate_provenance(db_request, file, [VALID_ATTESTATION]) + is None + ) + + # If the generate provenance fails, verify that no attestations are stored + assert not file.attestations + + @pytest.mark.parametrize( + "publisher_factory", + [ + GitHubPublisherFactory, + GitLabPublisherFactory, + ], + ) + def test_generate_provenance_succeeds( + self, db_request, metrics, storage_service, publisher_factory + ): + integrity_service = IntegrityService( + storage=storage_service, + metrics=metrics, + session=db_request.db, + ) + + file = FileFactory.create() + + db_request.oidc_publisher = publisher_factory.create() + provenance = integrity_service.generate_provenance( + db_request, + file, + [VALID_ATTESTATION], + ) + + expected_provenance = Provenance( + attestation_bundles=[ + AttestationBundle( + publisher=services._publisher_from_oidc_publisher( + db_request.oidc_publisher + ), + attestations=[VALID_ATTESTATION], + ) + ] + ) + + assert provenance == expected_provenance + + # We can round-trip the provenance object out of storage. + provenance_from_store = Provenance.model_validate_json( + storage_service.get(f"{file.path}.provenance").read() + ) + assert provenance_from_store == expected_provenance == provenance + + # Generate provenance also persist attestations + assert ( + storage_service.get(file.attestations[0].attestation_path).read() + == VALID_ATTESTATION.model_dump_json().encode() + ) + + def test_persist_provenance_succeeds(self, db_request, storage_service, metrics): + provenance = Provenance( + attestation_bundles=[ + AttestationBundle( + publisher=GitHubPublisher( + repository="fake-repository", + workflow="fake-workflow", + ), + attestations=[VALID_ATTESTATION], + ) + ] + ) + + integrity_service = IntegrityService( + storage=storage_service, + metrics=metrics, + session=db_request.db, + ) + file = FileFactory.create() + assert integrity_service._persist_provenance(provenance, file) is None + + assert ( + storage_service.get(f"{file.path}.provenance").read() + == provenance.model_dump_json().encode() + ) + + def test_get_provenance_digest_succeeds(self, db_request, metrics, storage_service): + file = FileFactory.create() + + integrity_service = IntegrityService( + storage=storage_service, + metrics=metrics, + session=db_request.db, + ) + + db_request.oidc_publisher = GitHubPublisherFactory.create() + + integrity_service.generate_provenance(db_request, file, [VALID_ATTESTATION]) + provenance_file = f"{file.path}.provenance" + + assert ( + integrity_service.get_provenance_digest(file) + == hashlib.file_digest( + storage_service.get(provenance_file), "sha256" + ).hexdigest() + ) + + def test_get_provenance_digest_fails_no_attestations(self, db_request): + # If the attestations are missing, there is no provenance file + file = FileFactory.create() + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=pretend.stub(), + session=db_request.db, + ) + + assert integrity_service.get_provenance_digest(file) is None + + +def test_publisher_from_oidc_publisher_succeeds_github(db_request): + publisher = GitHubPublisherFactory.create() + + attestation_publisher = services._publisher_from_oidc_publisher(publisher) + assert isinstance(attestation_publisher, GitHubPublisher) + assert attestation_publisher.repository == publisher.repository + assert attestation_publisher.workflow == publisher.workflow_filename + assert attestation_publisher.environment == publisher.environment + + +def test_publisher_from_oidc_publisher_succeeds_gitlab(db_request): + publisher = GitLabPublisherFactory.create() + + attestation_publisher = services._publisher_from_oidc_publisher(publisher) + assert isinstance(attestation_publisher, GitLabPublisher) + assert attestation_publisher.repository == publisher.project_path + assert attestation_publisher.environment == publisher.environment + + +def test_publisher_from_oidc_publisher_fails_unsupported(): + publisher = pretend.stub(publisher_name="not-existing") + + with pytest.raises(UnsupportedPublisherError): + services._publisher_from_oidc_publisher(publisher) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 739dd4081efc..dff82a1210db 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -12,6 +12,7 @@ import hashlib import io +import json import re import tarfile import tempfile @@ -23,15 +24,8 @@ import pretend import pytest -from pypi_attestations import ( - Attestation, - Distribution, - Envelope, - VerificationError, - VerificationMaterial, -) +from pypi_attestations import Attestation, Envelope, VerificationMaterial from pyramid.httpexceptions import HTTPBadRequest, HTTPForbidden, HTTPTooManyRequests -from sigstore.verify import Verifier from sqlalchemy import and_, exists from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import joinedload @@ -42,6 +36,7 @@ from warehouse.accounts.utils import UserContext from warehouse.admin.flags import AdminFlag, AdminFlagValue +from warehouse.attestations import Attestation as DatabaseAttestation from warehouse.classifiers.models import Classifier from warehouse.forklift import legacy, metadata from warehouse.macaroons import IMacaroonService, caveats, security_policy @@ -2420,85 +2415,6 @@ def test_upload_fails_without_oidc_publisher_permission( "See /the/help/url/ for more information." ).format(project.name) - def test_upload_attestation_fails_without_oidc_publisher( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - project_service, - macaroon_service, - ): - project = ProjectFactory.create() - owner = UserFactory.create() - maintainer = UserFactory.create() - RoleFactory.create(user=owner, project=project, role_name="Owner") - RoleFactory.create(user=maintainer, project=project, role_name="Maintainer") - - EmailFactory.create(user=maintainer) - db_request.user = maintainer - raw_macaroon, macaroon = macaroon_service.create_macaroon( - "fake location", - "fake description", - [caveats.RequestUser(user_id=str(maintainer.id))], - user_id=maintainer.id, - ) - identity = UserContext(maintainer, macaroon) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="some_cert", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", - "version": "1.0", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - extract_http_macaroon = pretend.call_recorder(lambda r, _: raw_macaroon) - monkeypatch.setattr( - security_policy, "_extract_http_macaroon", extract_http_macaroon - ) - - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMacaroonService: macaroon_service, - IMetricsService: metrics, - IProjectService: project_service, - }.get(svc) - db_request.user_agent = "warehouse-tests/6.6.6" - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status == ( - "400 Attestations are currently only supported when using Trusted " - "Publishing with GitHub Actions." - ) - @pytest.mark.parametrize( "plat", [ @@ -3408,7 +3324,7 @@ def test_upload_succeeds_creates_release( ), ] - def test_upload_with_valid_attestation_succeeds( + def test_upload_succeeds_with_valid_attestation( self, monkeypatch, pyramid_config, @@ -3464,296 +3380,56 @@ def test_upload_with_valid_attestation_succeeds( } ) - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - record_event = pretend.call_recorder( lambda self, *, tag, request=None, additional: None ) monkeypatch.setattr(HasEvents, "record_event", record_event) - - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: ( - "https://docs.pypi.org/attestations/publish/v1", - None, - ) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - resp = legacy.file_upload(db_request) assert resp.status_code == 200 - assert len(verify.calls) == 1 - verified_distribution = verify.calls[0].args[3] - assert verified_distribution == Distribution( - name=filename, digest=_TAR_GZ_PKG_SHA256 - ) - - def test_upload_with_invalid_attestation_predicate_type_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None - ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - invalid_predicate_type = "Unsupported predicate type" - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: (invalid_predicate_type, None) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - f"400 Attestation with unsupported predicate type: {invalid_predicate_type}" - ) - - def test_upload_with_multiple_attestations_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}," - f" {attestation.model_dump_json()}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None - ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: ( - "https://docs.pypi.org/attestations/publish/v1", - None, - ) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - "400 Only a single attestation per-file is supported at the moment." - ) - - def test_upload_with_malformed_attestation_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": "[{'a_malformed_attestation': 3}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None + assert ( + pretend.call("warehouse.upload.attestations.ok") in metrics.increment.calls ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - "400 Error while decoding the included attestation:" + attestations_db = ( + db_request.db.query(DatabaseAttestation) + .join(DatabaseAttestation.file) + .filter(File.filename == filename) + .all() ) + assert len(attestations_db) == 1 @pytest.mark.parametrize( - ("verify_exception", "expected_msg"), + "invalid_attestations", [ - ( - VerificationError, - "400 Could not verify the uploaded artifact using the included " - "attestation", - ), - ( - ValueError, - "400 Unknown error while trying to verify included attestations", - ), + # Bad top-level types. + "", + {}, + 1, + # Empty attestation sets not permitted. + [], + # Wrong version number. + [ + { + "version": 2, + "verification_material": { + "certificate": "somebase64string", + "transparency_entries": [{}], + }, + "envelope": { + "statement": "somebase64string", + "signature": "somebase64string", + }, + }, + ], ], ) - def test_upload_with_failing_attestation_verification( + def test_upload_fails_attestation_error( self, monkeypatch, pyramid_config, db_request, - metrics, - verify_exception, - expected_msg, + invalid_attestations, ): from warehouse.events.models import HasEvents @@ -3773,16 +3449,6 @@ def test_upload_with_failing_attestation_verification( db_request.db.add(Classifier(classifier="Programming Language :: Python")) filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) pyramid_config.testing_securitypolicy(identity=identity) db_request.user = None @@ -3791,7 +3457,7 @@ def test_upload_with_failing_attestation_verification( { "metadata_version": "1.2", "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", + "attestations": json.dumps(invalid_attestations), "version": version, "summary": "This is my summary!", "filetype": "sdist", @@ -3804,30 +3470,18 @@ def test_upload_with_failing_attestation_verification( } ) - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - record_event = pretend.call_recorder( lambda self, *, tag, request=None, additional: None ) monkeypatch.setattr(HasEvents, "record_event", record_event) - def failing_verify(_self, _verifier, _policy, _dist): - raise verify_exception("error") - - monkeypatch.setattr(Attestation, "verify", failing_verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - with pytest.raises(HTTPBadRequest) as excinfo: legacy.file_upload(db_request) resp = excinfo.value assert resp.status_code == 400 - assert resp.status.startswith(expected_msg) + assert resp.status.startswith("400 Malformed attestations") @pytest.mark.parametrize( ("url", "expected"), diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index afa7bd2056fe..92edf774d92d 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -15,6 +15,8 @@ import pretend +from tests.common.db.attestation import AttestationFactory +from warehouse.attestations import IIntegrityService from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.utils import _simple_detail, render_simple_detail @@ -32,6 +34,20 @@ def test_simple_detail_empty_string(db_request): assert expected_content["files"][0]["requires-python"] is None +def test_simple_detail_with_provenance(db_request, integrity_service): + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0") + file = FileFactory.create(release=release) + AttestationFactory.create(file=file) + + db_request.route_url = lambda *a, **kw: "the-url" + + expected_content = _simple_detail(project, db_request) + assert expected_content["files"][0][ + "provenance" + ] == integrity_service.get_provenance_digest(file) + + def test_render_simple_detail(db_request, monkeypatch, jinja): project = ProjectFactory.create() release1 = ReleaseFactory.create(project=project, version="1.0") @@ -49,6 +65,7 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): monkeypatch.setattr(hashlib, "blake2b", fakeblake2b) db_request.route_url = lambda *a, **kw: "the-url" + template = jinja.get_template("templates/api/simple/detail.html") expected_content = template.render( **_simple_detail(project, db_request), request=db_request @@ -67,7 +84,9 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): ) -def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): +def test_render_simple_detail_with_store( + db_request, monkeypatch, jinja, integrity_service +): project = ProjectFactory.create() storage_service = pretend.stub( @@ -78,6 +97,7 @@ def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): db_request.find_service = pretend.call_recorder( lambda svc, name=None, context=None: { ISimpleStorage: storage_service, + IIntegrityService: integrity_service, }.get(svc) ) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index d391253d61cb..5808d94c408a 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -250,6 +250,7 @@ def __init__(self): "warehouse.packaging.project_create_user_ratelimit_string": "20 per hour", "warehouse.packaging.project_create_ip_ratelimit_string": "40 per hour", "oidc.backend": "warehouse.oidc.services.OIDCPublisherService", + "attestations.backend": "warehouse.attestations.services.IntegrityService", "warehouse.organizations.max_undecided_organization_applications": 3, "reconcile_file_storages.batch_size": 100, "metadata_backfill.batch_size": 500, @@ -348,6 +349,7 @@ def __init__(self): pretend.call(".accounts"), pretend.call(".macaroons"), pretend.call(".oidc"), + pretend.call(".attestations"), pretend.call(".manage"), pretend.call(".organizations"), pretend.call(".subscriptions"), diff --git a/warehouse/attestations/__init__.py b/warehouse/attestations/__init__.py new file mode 100644 index 000000000000..783b03e02fcd --- /dev/null +++ b/warehouse/attestations/__init__.py @@ -0,0 +1,36 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from warehouse.attestations.errors import ( + AttestationUploadError, + UnsupportedPublisherError, +) +from warehouse.attestations.interfaces import IIntegrityService +from warehouse.attestations.models import Attestation +from warehouse.attestations.services import IntegrityService + +__all__ = [ + "Attestation", + "AttestationUploadError", + "IIntegrityService", + "IntegrityService", + "UnsupportedPublisherError", +] + + +def includeme(config): + integrity_service_class = config.maybe_dotted( + config.registry.settings["attestations.backend"] + ) + config.register_service_factory( + integrity_service_class.create_service, IIntegrityService + ) diff --git a/warehouse/attestations/errors.py b/warehouse/attestations/errors.py new file mode 100644 index 000000000000..463a34a4da69 --- /dev/null +++ b/warehouse/attestations/errors.py @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class UnsupportedPublisherError(Exception): + pass + + +class AttestationUploadError(Exception): + pass diff --git a/warehouse/attestations/interfaces.py b/warehouse/attestations/interfaces.py new file mode 100644 index 000000000000..c6fa9ef4c86b --- /dev/null +++ b/warehouse/attestations/interfaces.py @@ -0,0 +1,42 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pypi_attestations import Attestation, Distribution, Provenance +from pyramid.request import Request +from zope.interface import Interface + + +class IIntegrityService(Interface): + def create_service(context, request): + """ + Create the service for the given context and request. + """ + + def parse_attestations( + request: Request, distribution: Distribution + ) -> list[Attestation]: + """ + Process any attestations included in a file upload request + """ + + def generate_provenance( + request, file, attestations: list[Attestation] + ) -> Provenance | None: + """ + Generate and persist a Provenance object for the given file and list of + associated attestations. + """ + + def get_provenance_digest(file) -> str | None: + """ + Compute a provenance file digest for a `File` if it exists. + """ diff --git a/warehouse/attestations/models.py b/warehouse/attestations/models.py new file mode 100644 index 000000000000..9b95bfb0d7ad --- /dev/null +++ b/warehouse/attestations/models.py @@ -0,0 +1,55 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import typing + +from pathlib import Path +from uuid import UUID + +from sqlalchemy import ForeignKey, orm +from sqlalchemy.dialects.postgresql import CITEXT +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, mapped_column + +from warehouse import db + +if typing.TYPE_CHECKING: + from warehouse.packaging.models import File + + +class Attestation(db.Model): + """ + Table used to store Attestations. + + Attestations are stored on disk. We keep in database only the attestation hash. + """ + + __tablename__ = "attestation" + + file_id: Mapped[UUID] = mapped_column( + ForeignKey("release_files.id", onupdate="CASCADE", ondelete="CASCADE"), + ) + file: Mapped[File] = orm.relationship(back_populates="attestations") + + attestation_file_blake2_digest: Mapped[str] = mapped_column(CITEXT) + + @hybrid_property + def attestation_path(self): + return "/".join( + [ + self.attestation_file_blake2_digest[:2], + self.attestation_file_blake2_digest[2:4], + self.attestation_file_blake2_digest[4:], + f"{Path(self.file.path).name}.attestation", + ] + ) diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py new file mode 100644 index 000000000000..31dc8c1cbf60 --- /dev/null +++ b/warehouse/attestations/services.py @@ -0,0 +1,323 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib +import tempfile +import typing +import warnings + +from pathlib import Path + +import sentry_sdk + +from pydantic import TypeAdapter, ValidationError +from pypi_attestations import ( + Attestation, + AttestationBundle, + AttestationType, + Distribution, + GitHubPublisher, + GitLabPublisher, + Provenance, + Publisher, + VerificationError, +) +from pyramid.request import Request +from sigstore.verify import Verifier +from zope.interface import implementer + +from warehouse.attestations.errors import ( + AttestationUploadError, + UnsupportedPublisherError, +) +from warehouse.attestations.interfaces import IIntegrityService +from warehouse.attestations.models import Attestation as DatabaseAttestation +from warehouse.metrics.interfaces import IMetricsService +from warehouse.oidc.models import ( + GitHubPublisher as GitHubOIDCPublisher, + GitLabPublisher as GitLabOIDCPublisher, + OIDCPublisher, +) +from warehouse.packaging.interfaces import IFileStorage +from warehouse.packaging.models import File +from warehouse.utils.exceptions import InsecureIntegrityServiceWarning + + +def _publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: + """ + Convert an OIDCPublisher object in a pypi-attestations Publisher. + """ + match publisher.publisher_name: + case "GitLab": + publisher = typing.cast(GitLabOIDCPublisher, publisher) + return GitLabPublisher( + repository=publisher.project_path, environment=publisher.environment + ) + case "GitHub": + publisher = typing.cast(GitHubOIDCPublisher, publisher) + return GitHubPublisher( + repository=publisher.repository, + workflow=publisher.workflow_filename, + environment=publisher.environment, + ) + case _: + raise UnsupportedPublisherError + + +def _extract_attestations_from_request(request: Request) -> list[Attestation]: + """ + Extract well-formed attestation objects from the given request's payload. + """ + + metrics = request.find_service(IMetricsService, context=None) + + try: + attestations = TypeAdapter(list[Attestation]).validate_json( + request.POST["attestations"] + ) + except ValidationError as e: + # Log invalid (malformed) attestation upload + metrics.increment("warehouse.upload.attestations.malformed") + raise AttestationUploadError( + f"Malformed attestations: {e}", + ) + + # Empty attestation sets are not permitted; users should omit `attestations` + # entirely to upload without attestations. + if not attestations: + raise AttestationUploadError( + "Malformed attestations: an empty attestation set is not permitted" + ) + + # This is a temporary constraint; multiple attestations per file will + # be supported in the future. + if len(attestations) > 1: + metrics.increment("warehouse.upload.attestations.failed_multiple_attestations") + + raise AttestationUploadError( + "Only a single attestation per file is supported", + ) + + return attestations + + +@implementer(IIntegrityService) +class NullIntegrityService: + def __init__(self, session): + warnings.warn( + "NullIntegrityService is intended only for use in development, " + "you should not use it in production due to the lack of actual " + "attestation verification.", + InsecureIntegrityServiceWarning, + ) + self.db = session + + @classmethod + def create_service(cls, _context, request): + return cls(session=request.db) + + def parse_attestations( + self, request: Request, _distribution: Distribution + ) -> list[Attestation]: + return _extract_attestations_from_request(request) + + def generate_provenance( + self, request: Request, file: File, attestations: list[Attestation] + ) -> Provenance | None: + publisher = _publisher_from_oidc_publisher(request.oidc_publisher) + attestation_bundle = AttestationBundle( + publisher=publisher, + attestations=attestations, + ) + provenance = Provenance(attestation_bundles=[attestation_bundle]) + + for attestation in attestations: + self.db.add( + DatabaseAttestation( + file=file, + attestation_file_blake2_digest=hashlib.blake2b( + attestation.model_dump_json().encode("utf-8") + ).hexdigest(), + ) + ) + + return provenance + + def get_provenance_digest(self, file: File) -> str | None: + if not file.attestations: + return None + + # For the null service, our "provenance digest" is just the digest + # of the release file's name merged with the number of attestations. + # We do this because there's no verification involved; we just need + # a unique value to preserve invariants. + return hashlib.sha256( + f"{file.filename}:{len(file.attestations)}".encode() + ).hexdigest() + + +@implementer(IIntegrityService) +class IntegrityService: + def __init__(self, storage: IFileStorage, metrics: IMetricsService, session): + self.storage: IFileStorage = storage + self.metrics: IMetricsService = metrics + self.db = session + + @classmethod + def create_service(cls, _context, request): + return cls( + storage=request.find_service(IFileStorage, name="archive"), + metrics=request.find_service(IMetricsService), + session=request.db, + ) + + def parse_attestations( + self, request: Request, distribution: Distribution + ) -> list[Attestation]: + """ + Process any attestations included in a file upload request + + Attestations, if present, will be parsed and verified against the uploaded + artifact. Attestations are only allowed when uploading via a Trusted + Publisher, because a Trusted Publisher provides the identity that will be + used to verify the attestations. + Only GitHub Actions Trusted Publishers are supported. + """ + publisher: OIDCPublisher | None = request.oidc_publisher + if not publisher or not publisher.publisher_name == "GitHub": + raise AttestationUploadError( + "Attestations are only supported when using Trusted " + "Publishing with GitHub Actions.", + ) + + attestations = _extract_attestations_from_request(request) + + verification_policy = publisher.publisher_verification_policy( + request.oidc_claims + ) + for attestation_model in attestations: + try: + predicate_type, _ = attestation_model.verify( + Verifier.production(), + verification_policy, + distribution, + ) + except VerificationError as e: + # Log invalid (failed verification) attestation upload + self.metrics.increment("warehouse.upload.attestations.failed_verify") + raise AttestationUploadError( + f"Could not verify the uploaded artifact using the included " + f"attestation: {e}", + ) + except Exception as e: + with sentry_sdk.push_scope() as scope: + scope.fingerprint = [e] + sentry_sdk.capture_message( + f"Unexpected error while verifying attestation: {e}" + ) + + raise AttestationUploadError( + f"Unknown error while trying to verify included attestations: {e}", + ) + + if predicate_type != AttestationType.PYPI_PUBLISH_V1: + self.metrics.increment( + "warehouse.upload.attestations.failed_unsupported_predicate_type" + ) + raise AttestationUploadError( + f"Attestation with unsupported predicate type: {predicate_type}", + ) + + return attestations + + def generate_provenance( + self, request: Request, file: File, attestations: list[Attestation] + ) -> Provenance | None: + + # Generate the provenance object. + provenance = self._build_provenance_object(request.oidc_publisher, attestations) + + if not provenance: + return None + + # Persist the attestations and provenance objects. We only do this + # after generating the provenance above, to prevent orphaned artifacts + # on any generation failures. + self._persist_attestations(attestations, file) + self._persist_provenance(provenance, file) + + return provenance + + def get_provenance_digest(self, file: File) -> str | None: + """Returns the sha256 digest of the provenance file for the release.""" + if not file.attestations: + return None + + provenance_file = self.storage.get(f"{file.path}.provenance") + return hashlib.file_digest(provenance_file, "sha256").hexdigest() + + def _persist_attestations( + self, attestations: list[Attestation], file: File + ) -> None: + for attestation in attestations: + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write(attestation.model_dump_json().encode("utf-8")) + + attestation_digest = hashlib.file_digest( + tmp_file, "blake2b" + ).hexdigest() + database_attestation = DatabaseAttestation( + file=file, attestation_file_blake2_digest=attestation_digest + ) + self.db.add(database_attestation) + + self.storage.store( + database_attestation.attestation_path, + tmp_file.name, + meta=None, + ) + + def _build_provenance_object( + self, oidc_publisher: OIDCPublisher, attestations: list[Attestation] + ) -> Provenance | None: + try: + publisher: Publisher = _publisher_from_oidc_publisher(oidc_publisher) + except UnsupportedPublisherError: + sentry_sdk.capture_message( + f"Unsupported OIDCPublisher found {oidc_publisher.publisher_name}" + ) + + return None + + attestation_bundle = AttestationBundle( + publisher=publisher, + attestations=attestations, + ) + + return Provenance(attestation_bundles=[attestation_bundle]) + + def _persist_provenance( + self, + provenance: Provenance, + file: File, + ) -> None: + """ + Persist a Provenance object in storage. + """ + provenance_file_path = Path(f"{file.path}.provenance") + with tempfile.NamedTemporaryFile() as f: + f.write(provenance.model_dump_json().encode("utf-8")) + f.flush() + + self.storage.store( + provenance_file_path, + f.name, + ) diff --git a/warehouse/config.py b/warehouse/config.py index 3c11a156a607..be8430d1d075 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -405,6 +405,12 @@ def configure(settings=None): "OIDC_BACKEND", default="warehouse.oidc.services.OIDCPublisherService", ) + maybe_set( + settings, + "attestations.backend", + "ATTESTATIONS_BACKEND", + default="warehouse.attestations.services.IntegrityService", + ) # Pythondotorg integration settings maybe_set( @@ -460,7 +466,7 @@ def configure(settings=None): "warehouse.account.accounts_search_ratelimit_string", "ACCOUNTS_SEARCH_RATELIMIT_STRING", default="100 per hour", - ), + ) maybe_set( settings, "warehouse.account.password_reset_ratelimit_string", @@ -740,6 +746,9 @@ def configure(settings=None): # Register support for OIDC based authentication config.include(".oidc") + # Register support for attestations + config.include(".attestations") + # Register logged-in views config.include(".manage") diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index d8e9ca7f2c7a..233a0b50177f 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -29,13 +29,7 @@ import wtforms import wtforms.validators -from pydantic import TypeAdapter, ValidationError -from pypi_attestations import ( - Attestation, - AttestationType, - Distribution, - VerificationError, -) +from pypi_attestations import Attestation, Distribution from pyramid.httpexceptions import ( HTTPBadRequest, HTTPException, @@ -47,11 +41,11 @@ ) from pyramid.request import Request from pyramid.view import view_config -from sigstore.verify import Verifier from sqlalchemy import and_, exists, func, orm from sqlalchemy.exc import MultipleResultsFound, NoResultFound from warehouse.admin.flags import AdminFlagValue +from warehouse.attestations import AttestationUploadError, IIntegrityService from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB @@ -376,88 +370,6 @@ def _is_duplicate_file(db_session, filename, hashes): return None -def _process_attestations(request, distribution: Distribution): - """ - Process any attestations included in a file upload request - - Attestations, if present, will be parsed and verified against the uploaded - artifact. Attestations are only allowed when uploading via a Trusted - Publisher, because a Trusted Publisher provides the identity that will be - used to verify the attestations. - Currently, only GitHub Actions Trusted Publishers are supported, and - attestations are discarded after verification. - """ - - metrics = request.find_service(IMetricsService, context=None) - - publisher = request.oidc_publisher - if not publisher or not publisher.publisher_name == "GitHub": - raise _exc_with_message( - HTTPBadRequest, - "Attestations are currently only supported when using Trusted " - "Publishing with GitHub Actions.", - ) - try: - attestations = TypeAdapter(list[Attestation]).validate_json( - request.POST["attestations"] - ) - except ValidationError as e: - # Log invalid (malformed) attestation upload - metrics.increment("warehouse.upload.attestations.malformed") - raise _exc_with_message( - HTTPBadRequest, - f"Error while decoding the included attestation: {e}", - ) - - if len(attestations) > 1: - metrics.increment("warehouse.upload.attestations.failed_multiple_attestations") - raise _exc_with_message( - HTTPBadRequest, - "Only a single attestation per-file is supported at the moment.", - ) - - verification_policy = publisher.publisher_verification_policy(request.oidc_claims) - for attestation_model in attestations: - try: - # For now, attestations are not stored, just verified - predicate_type, _ = attestation_model.verify( - Verifier.production(), - verification_policy, - distribution, - ) - except VerificationError as e: - # Log invalid (failed verification) attestation upload - metrics.increment("warehouse.upload.attestations.failed_verify") - raise _exc_with_message( - HTTPBadRequest, - f"Could not verify the uploaded artifact using the included " - f"attestation: {e}", - ) - except Exception as e: - with sentry_sdk.push_scope() as scope: - scope.fingerprint = [e] - sentry_sdk.capture_message( - f"Unexpected error while verifying attestation: {e}" - ) - - raise _exc_with_message( - HTTPBadRequest, - f"Unknown error while trying to verify included attestations: {e}", - ) - - if predicate_type != AttestationType.PYPI_PUBLISH_V1: - metrics.increment( - "warehouse.upload.attestations.failed_unsupported_predicate_type" - ) - raise _exc_with_message( - HTTPBadRequest, - f"Attestation with unsupported predicate type: {predicate_type}", - ) - - # Log successful attestation upload - metrics.increment("warehouse.upload.attestations.ok") - - def _sort_releases(request: Request, project: Project): releases = ( request.db.query(Release) @@ -1251,12 +1163,6 @@ def file_upload(request): k: h.hexdigest().lower() for k, h in metadata_file_hashes.items() } - if "attestations" in request.POST: - _process_attestations( - request=request, - distribution=Distribution(name=filename, digest=file_hashes["sha256"]), - ) - # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. @@ -1353,6 +1259,26 @@ def file_upload(request): }, ) + # If the user provided attestations, verify and store them + if "attestations" in request.POST: + integrity_service = request.find_service(IIntegrityService, context=None) + + try: + attestations: list[Attestation] = integrity_service.parse_attestations( + request, + Distribution(name=filename, digest=file_hashes["sha256"]), + ) + except AttestationUploadError as e: + raise _exc_with_message( + HTTPBadRequest, + str(e), + ) + + integrity_service.generate_provenance(request, file_, attestations) + + # Log successful attestation upload + metrics.increment("warehouse.upload.attestations.ok") + # For existing releases, we check if any of the existing project URLs are unverified # and have been verified in the current upload. In that case, we mark them as # verified. diff --git a/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py b/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py new file mode 100644 index 000000000000..959bb011ef32 --- /dev/null +++ b/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py @@ -0,0 +1,47 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +recreate attestations table + +Revision ID: 4037669366ca +Revises: 606abd3b8e7f +Create Date: 2024-08-21 20:33:53.489489 +""" + +import sqlalchemy as sa + +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "4037669366ca" +down_revision = "606abd3b8e7f" + + +def upgrade(): + op.create_table( + "attestation", + sa.Column("file_id", sa.UUID(), nullable=False), + sa.Column( + "attestation_file_blake2_digest", postgresql.CITEXT(), nullable=False + ), + sa.Column( + "id", sa.UUID(), server_default=sa.text("gen_random_uuid()"), nullable=False + ), + sa.ForeignKeyConstraint( + ["file_id"], ["release_files.id"], onupdate="CASCADE", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade(): + op.drop_table("attestation") diff --git a/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py b/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py index 31a1b5636bf4..eb36da1dfdd9 100644 --- a/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py +++ b/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py @@ -17,7 +17,6 @@ Create Date: 2024-08-21 19:52:40.084048 """ - from alembic import op revision = "7ca0f1f5e7b3" diff --git a/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py index c86d454ddc5a..2b15277127f6 100644 --- a/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py +++ b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py @@ -11,6 +11,7 @@ # limitations under the License. """ create Attestations table + Revision ID: 7f0c9f105f44 Revises: 26455e3712a2 Create Date: 2024-07-25 15:49:01.993869 diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index f171bd437db2..73fabbdbb26f 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -81,6 +81,7 @@ from warehouse.utils.db.types import bool_false, datetime_now if typing.TYPE_CHECKING: + from warehouse.attestations.models import Attestation from warehouse.oidc.models import OIDCPublisher _MONOTONIC_SEQUENCE = 42 @@ -839,6 +840,13 @@ def __table_args__(cls): # noqa comment="If True, the metadata for the file cannot be backfilled.", ) + # PEP 740 attestations + attestations: Mapped[list[Attestation]] = orm.relationship( + cascade="all, delete-orphan", + lazy="joined", + passive_deletes=True, + ) + @property def uploaded_via_trusted_publisher(self) -> bool: """Return True if the file was uploaded via a trusted publisher.""" diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 30c85b3feb50..03c33a94a844 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -19,10 +19,11 @@ from pyramid_jinja2 import IJinja2Environment from sqlalchemy.orm import joinedload +from warehouse.attestations import IIntegrityService from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.models import File, LifecycleStatus, Project, Release -API_VERSION = "1.1" +API_VERSION = "1.2" def _simple_index(request, serial): @@ -64,6 +65,8 @@ def _simple_detail(project, request): {f.release.version for f in files}, key=packaging_legacy.version.parse ) + integrity_service = request.find_service(IIntegrityService) + return { "meta": {"api-version": API_VERSION, "_last-serial": project.last_serial}, "name": project.normalized_name, @@ -97,6 +100,7 @@ def _simple_detail(project, request): if file.metadata_file_sha256_digest else False ), + "provenance": integrity_service.get_provenance_digest(file), } for file in files ], diff --git a/warehouse/templates/api/simple/detail.html b/warehouse/templates/api/simple/detail.html index 24b0042c5863..05e0221a5612 100644 --- a/warehouse/templates/api/simple/detail.html +++ b/warehouse/templates/api/simple/detail.html @@ -20,7 +20,7 @@

Links for {{ name }}

{% for file in files -%} - {{ file.filename }}
+ {{ file.filename }}
{% endfor -%} diff --git a/warehouse/utils/exceptions.py b/warehouse/utils/exceptions.py index 238c9222de1b..2b4b3c674e5d 100644 --- a/warehouse/utils/exceptions.py +++ b/warehouse/utils/exceptions.py @@ -13,3 +13,7 @@ class InsecureOIDCPublisherWarning(UserWarning): pass + + +class InsecureIntegrityServiceWarning(UserWarning): + pass