Reapply "Store attestations for PEP740 (#16302)" (#16545)

dev/environment

@@ -46,6 +46,7 @@ BREACHED_EMAILS=warehouse.accounts.NullEmailBreachedService
 BREACHED_PASSWORDS=warehouse.accounts.NullPasswordBreachedService
 
 OIDC_BACKEND=warehouse.oidc.services.NullOIDCPublisherService
+ATTESTATIONS_BACKEND=warehouse.attestations.services.NullIntegrityService
 
 METRICS_BACKEND=warehouse.metrics.DataDogMetrics host=notdatadog
 

requirements/main.in

@@ -62,8 +62,8 @@ redis>=2.8.0,<6.0.0
 rfc3986
 sentry-sdk
 setuptools
-sigstore~=3.0.0
-pypi-attestations==0.0.9
+sigstore~=3.2.0
+pypi-attestations==0.0.11
 sqlalchemy[asyncio]>=2.0,<3.0
 stdlib-list
 stripe

requirements/main.txt

@@ -1776,9 +1776,9 @@ pyparsing==3.1.4 \
     --hash=sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c \
     --hash=sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032
     # via linehaul
-pypi-attestations==0.0.9 \
-    --hash=sha256:3bfc07f64a8db0d6e2646720e70df7c7cb01a2936056c764a2cc3268969332f2 \
-    --hash=sha256:4b38cce5d221c8145cac255bfafe650ec0028d924d2b3572394df8ba8f07a609
+pypi-attestations==0.0.11 \
+    --hash=sha256:b730e6b23874d94da0f3817b1f9dd3ecb6a80d685f62a18ad96e5b0396149ded \
+    --hash=sha256:e74329074f049568591e300373e12fcd46a35e21723110856546e33bf2949efa
     # via -r requirements/main.in
 pyqrcode==1.2.1 \
     --hash=sha256:1b2812775fa6ff5c527977c4cd2ccb07051ca7d0bc0aecf937a43864abe5eff6 \
@@ -2091,9 +2091,9 @@ sentry-sdk==2.13.0 \
     --hash=sha256:6beede8fc2ab4043da7f69d95534e320944690680dd9a963178a49de71d726c6 \
     --hash=sha256:8d4a576f7a98eb2fdb40e13106e41f330e5c79d72a68be1316e7852cf4995260
     # via -r requirements/main.in
-sigstore==3.0.0 \
-    --hash=sha256:6cc7dc92607c2fd481aada0f3c79e710e4c6086e3beab50b07daa9a50a79d109 \
-    --hash=sha256:a6a9538a648e112a0c3d8092d3f73a351c7598164764f1e73a6b5ba406a3a0bd
+sigstore==3.2.0 \
+    --hash=sha256:25c8a871a3a6adf959c0cde598ea8bef8794f1a29277d067111eb4ded4ba7f65 \
+    --hash=sha256:d18508f34febb7775065855e92557fa1c2c16580df88f8e8903b9514438bad44
     # via
     #   -r requirements/main.in
     #   pypi-attestations

tests/common/db/attestation.py

@@ -0,0 +1,28 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import hashlib
+
+import factory
+
+from warehouse.attestations.models import Attestation
+
+from .base import WarehouseFactory
+
+
+class AttestationFactory(WarehouseFactory):
+    class Meta:
+        model = Attestation
+
+    file = factory.SubFactory("tests.common.db.packaging.FileFactory")
+    attestation_file_blake2_digest = factory.LazyAttribute(
+        lambda o: hashlib.blake2b(o.file.filename.encode("utf8")).hexdigest()
+    )

tests/common/db/packaging.py

@@ -34,6 +34,7 @@
 from warehouse.utils import readme
 
 from .accounts import UserFactory
+from .attestation import AttestationFactory
 from .base import WarehouseFactory
 from .observations import ObserverFactory
 
@@ -140,6 +141,13 @@ class Meta:
         )
     )
 
+    # Empty attestations by default.
+    attestations = factory.RelatedFactoryList(
+        AttestationFactory,
+        factory_related_name="file",
+        size=0,
+    )
+
 
 class FileEventFactory(WarehouseFactory):
     class Meta:

tests/conftest.py

@@ -44,6 +44,8 @@
 from warehouse.accounts import services as account_services
 from warehouse.accounts.interfaces import ITokenService, IUserService
 from warehouse.admin.flags import AdminFlag, AdminFlagValue
+from warehouse.attestations import services as attestations_services
+from warehouse.attestations.interfaces import IIntegrityService
 from warehouse.email import services as email_services
 from warehouse.email.interfaces import IEmailSender
 from warehouse.helpdesk import services as helpdesk_services
@@ -57,7 +59,7 @@
 from warehouse.organizations import services as organization_services
 from warehouse.organizations.interfaces import IOrganizationService
 from warehouse.packaging import services as packaging_services
-from warehouse.packaging.interfaces import IProjectService
+from warehouse.packaging.interfaces import IFileStorage, IProjectService
 from warehouse.subscriptions import services as subscription_services
 from warehouse.subscriptions.interfaces import IBillingService, ISubscriptionService
 
@@ -112,6 +114,15 @@ def metrics():
     )
 
 
+@pytest.fixture
+def storage_service(tmp_path):
+    """
+    A good-enough local file storage service.
+    """
+
+    return packaging_services.LocalArchiveFileStorage(tmp_path)
+
+
 @pytest.fixture
 def remote_addr():
     return "1.2.3.4"
@@ -173,6 +184,8 @@ def pyramid_services(
     project_service,
     github_oidc_service,
     activestate_oidc_service,
+    integrity_service,
+    storage_service,
     macaroon_service,
     helpdesk_service,
 ):
@@ -194,7 +207,9 @@ def pyramid_services(
     services.register_service(
         activestate_oidc_service, IOIDCPublisherService, None, name="activestate"
     )
+    services.register_service(integrity_service, IIntegrityService, None, name="")
     services.register_service(macaroon_service, IMacaroonService, None, name="")
+    services.register_service(storage_service, IFileStorage, None, name="archive")
     services.register_service(helpdesk_service, IHelpDeskService, None)
 
     return services
@@ -324,6 +339,7 @@ def get_app_config(database, nondefaults=None):
         "docs.backend": "warehouse.packaging.services.LocalDocsStorage",
         "sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage",
         "billing.backend": "warehouse.subscriptions.services.MockStripeBillingService",
+        "attestations.backend": "warehouse.attestations.services.NullIntegrityService",
         "billing.api_base": "http://stripe:12111",
         "billing.api_version": "2020-08-27",
         "mail.backend": "warehouse.email.services.SMTPEmailSender",
@@ -387,13 +403,11 @@ def get_db_session_for_app_config(app_config):
 
 @pytest.fixture(scope="session")
 def app_config(database):
-
     return get_app_config(database)
 
 
 @pytest.fixture(scope="session")
 def app_config_dbsession_from_env(database):
-
     nondefaults = {
         "warehouse.db_create_session": lambda r: r.environ.get("warehouse.db_session")
     }
@@ -539,6 +553,11 @@ def activestate_oidc_service(db_session):
     )
 
 
+@pytest.fixture
+def integrity_service(db_session):
+    return attestations_services.NullIntegrityService(db_session)
+
+
 @pytest.fixture
 def macaroon_service(db_session):
     return macaroon_services.DatabaseMacaroonService(db_session)

tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation

@@ -0,0 +1 @@
+{"version":1,"verification_material":{"certificate":"MIIC6zCCAnGgAwIBAgIUFgmhIYx8gvBGePCTacG/4kbBdRwwCgYIKoZIzj0EAwMwNzEVMBMGA1UE\nChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjQwODI5\nMTcwOTM5WhcNMjQwODI5MTcxOTM5WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEtGrMPml4\nOtsRJ3Z6qRahs0kHCZxP4n9fvrJE957WVxgAGg4k6a1PbRJY9nT9wKpRrZmKV++AgA9ndhdruXXa\nAKOCAZAwggGMMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAdBgNVHQ4EFgQU\nosNvhYEuTPfgyU/dZfu93lFGRNswHwYDVR0jBBgwFoAU39Ppz1YkEZb5qNjpKFWixi4YZD8wQAYD\nVR0RAQH/BDYwNIEyOTE5NDM2MTU4MjM2LWNvbXB1dGVAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3Vu\ndC5jb20wKQYKKwYBBAGDvzABAQQbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMCsGCisGAQQB\ng78wAQgEHQwbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMIGKBgorBgEEAdZ5AgQCBHwEegB4\nAHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGRnx0/aQAABAMARzBFAiBogvcK\nHIIR9FcX1vQgDhGtAl0XQoMRiEB3OdUWO94P1gIhANdJlyISdtvVrHes25dWKTLepy+IzQmzfQU/\nS7cxWHmOMAoGCCqGSM49BAMDA2gAMGUCMGe2xTiuenbjdt1d2e4IaCiwRh2G4KAtyujRESSSUbpu\nGme/o9ouiApeONBv2CvvGAIxAOEkAGFO3aALE3IPNosxqaz9MbqJOdmYhB1Cz1D7xbFc/m243VxJ\nWxaC/uOFEpyiYQ==\n","transparency_entries":[{"logIndex":"125970014","logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="},"kindVersion":{"kind":"dsse","version":"0.0.1"},"integratedTime":"1724951379","inclusionPromise":{"signedEntryTimestamp":"MEUCIQCHrKFTeXNY432S0bUSBS69S8d5JnNcDXa41q6OEvxEwgIgaZstc5Jpm0IgwFC7RDTXYEAKk+3aG/MkRkaPdJdyn8U="},"inclusionProof":{"logIndex":"4065752","rootHash":"7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=","treeSize":"4065754","hashes":["NwJgWJoxjearbnEIT9bnWXpzo0LGNrR1cpWId0g66rE=","kLjpW3Eh7pQJNOvyntghzF57tcfqk2IzX7cqiBDgGf8=","FW8y9LQ1i3q+MnbeGJipKGl4VfX1zRBOD7TmhbEw7uI=","mKcbGJDJ/+buNbXy9Eyv94nVoAyUauuIlN3cJg3qSBY=","5VytqqAHhfRkRWMrY43UXWCnRBb7JwElMlKpY5JueBc=","mZJnD39LTKdis2wUTz1OOMx3r7HwgJh9rnb2VwiPzts=","MXZOQFJFiOjREF0xwMOCXu29HwTchjTtl/BeFoI51wY=","g8zCkHnLwO3LojK7g5AnqE8ezSNRnCSz9nCL5GD3a8A=","RrZsD/RSxNoujlvq/MsCEvLSkKZfv0jmQM9Kp7qbJec=","QxmVWsbTp4cClxuAkuT51UH2EY7peHMVGKq7+b+cGwQ=","Q2LAtNzOUh+3PfwfMyNxYb06fTQmF3VeTT6Fr6Upvfc=","ftwAu6v62WFDoDmcZ1JKfrRPrvuiIw5v3BvRsgQj7N8="],"checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n4065754\n7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=\n\n— rekor.sigstore.dev wNI9ajBGAiEAhMomhZHOTNB5CVPO98CMXCv01ZlIF+C+CgzraAB01r8CIQCEuXbv6aqguUpB/ig5eXRIbarvxLXkg3nX48DzambktQ==\n"}},"canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiZHNzZSIsInNwZWMiOnsiZW52ZWxvcGVIYXNoIjp7ImFsZ29yaXRobSI6InNoYTI1NiIsInZhbHVlIjoiOWRiNGJjMzE3MTgyZWI3NzljNDIyY2Q0NGI2ZDdlYTk5ZWM1M2Q3M2JiY2ZjZWVmZTIyNWVlYjQ3NTQyMjc4OCJ9LCJwYXlsb2FkSGFzaCI6eyJhbGdvcml0aG0iOiJzaGEyNTYiLCJ2YWx1ZSI6IjlkYjY0MjlhOTkzZGFiYTI4NzAwODk2ZTY2MzNjNzkxYWE0MDM3ODQ4NjJiYzY2MDBkM2E4NjYwMGQzYjA1NjMifSwic2lnbmF0dXJlcyI6W3sic2lnbmF0dXJlIjoiTUVVQ0lCaGlOL25NR0w3aHpZQk9QQjlUTGtuaEdTZEtuQ0Q0ekI3TDV5ZXc0QmJ3QWlFQXJzOHl6MCtCT2NnSEtzS0JzTXVOeVlhREdaRTBVV0JuMEdwNVpGMzUvU2M9IiwidmVyaWZpZXIiOiJMUzB0TFMxQ1JVZEpUaUJEUlZKVVNVWkpRMEZVUlMwdExTMHRDazFKU1VNMmVrTkRRVzVIWjBGM1NVSkJaMGxWUm1kdGFFbFplRGhuZGtKSFpWQkRWR0ZqUnk4MGEySkNaRkozZDBObldVbExiMXBKZW1vd1JVRjNUWGNLVG5wRlZrMUNUVWRCTVZWRlEyaE5UV015Ykc1ak0xSjJZMjFWZFZwSFZqSk5ValIzU0VGWlJGWlJVVVJGZUZaNllWZGtlbVJIT1hsYVV6RndZbTVTYkFwamJURnNXa2RzYUdSSFZYZElhR05PVFdwUmQwOUVTVFZOVkdOM1QxUk5OVmRvWTA1TmFsRjNUMFJKTlUxVVkzaFBWRTAxVjJwQlFVMUdhM2RGZDFsSUNrdHZXa2w2YWpCRFFWRlpTVXR2V2tsNmFqQkVRVkZqUkZGblFVVjBSM0pOVUcxc05FOTBjMUpLTTFvMmNWSmhhSE13YTBoRFduaFFORzQ1Wm5aeVNrVUtPVFUzVjFaNFowRkhaelJyTm1FeFVHSlNTbGs1YmxRNWQwdHdVbkphYlV0V0t5dEJaMEU1Ym1Sb1pISjFXRmhoUVV0UFEwRmFRWGRuWjBkTlRVRTBSd3BCTVZWa1JIZEZRaTkzVVVWQmQwbElaMFJCVkVKblRsWklVMVZGUkVSQlMwSm5aM0pDWjBWR1FsRmpSRUY2UVdSQ1owNVdTRkUwUlVablVWVnZjMDUyQ21oWlJYVlVVR1puZVZVdlpGcG1kVGt6YkVaSFVrNXpkMGgzV1VSV1VqQnFRa0puZDBadlFWVXpPVkJ3ZWpGWmEwVmFZalZ4VG1wd1MwWlhhWGhwTkZrS1drUTRkMUZCV1VSV1VqQlNRVkZJTDBKRVdYZE9TVVY1VDFSRk5VNUVUVEpOVkZVMFRXcE5Na3hYVG5aaVdFSXhaRWRXUVZwSFZqSmFWM2gyWTBkV2VRcE1iV1I2V2xoS01tRlhUbXhaVjA1cVlqTldkV1JETldwaU1qQjNTMUZaUzB0M1dVSkNRVWRFZG5wQlFrRlJVV0poU0ZJd1kwaE5Oa3g1T1doWk1rNTJDbVJYTlRCamVUVnVZakk1Ym1KSFZYVlpNamwwVFVOelIwTnBjMGRCVVZGQ1p6YzRkMEZSWjBWSVVYZGlZVWhTTUdOSVRUWk1lVGxvV1RKT2RtUlhOVEFLWTNrMWJtSXlPVzVpUjFWMVdUSTVkRTFKUjB0Q1oyOXlRbWRGUlVGa1dqVkJaMUZEUWtoM1JXVm5RalJCU0ZsQk0xUXdkMkZ6WWtoRlZFcHFSMUkwWXdwdFYyTXpRWEZLUzFoeWFtVlFTek12YURSd2VXZERPSEEzYnpSQlFVRkhVbTU0TUM5aFVVRkJRa0ZOUVZKNlFrWkJhVUp2WjNaalMwaEpTVkk1Um1OWUNqRjJVV2RFYUVkMFFXd3dXRkZ2VFZKcFJVSXpUMlJWVjA4NU5GQXhaMGxvUVU1a1NteDVTVk5rZEhaV2NraGxjekkxWkZkTFZFeGxjSGtyU1hwUmJYb0tabEZWTDFNM1kzaFhTRzFQVFVGdlIwTkRjVWRUVFRRNVFrRk5SRUV5WjBGTlIxVkRUVWRsTW5oVWFYVmxibUpxWkhReFpESmxORWxoUTJsM1VtZ3lSd28wUzBGMGVYVnFVa1ZUVTFOVlluQjFSMjFsTDI4NWIzVnBRWEJsVDA1Q2RqSkRkblpIUVVsNFFVOUZhMEZIUms4ellVRk1SVE5KVUU1dmMzaHhZWG81Q2sxaWNVcFBaRzFaYUVJeFEzb3hSRGQ0WWtaakwyMHlORE5XZUVwWGVHRkRMM1ZQUmtWd2VXbFpVVDA5Q2kwdExTMHRSVTVFSUVORlVsUkpSa2xEUVZSRkxTMHRMUzBLIn1dfX0="}]},"envelope":{"statement":"eyJfdHlwZSI6Imh0dHBzOi8vaW4tdG90by5pby9TdGF0ZW1lbnQvdjEiLCJzdWJqZWN0IjpbeyJu\nYW1lIjoic2FtcGxlcHJvamVjdC0zLjAuMC50YXIuZ3oiLCJkaWdlc3QiOnsic2hhMjU2IjoiMTE3\nZWQ4OGU1ZGIwNzNiYjkyOTY5YTc1NDU3NDVmZDk3N2VlODViNzAxOTcwNmRkMjU2YTY0MDU4Zjcw\nOTYzZCJ9fV0sInByZWRpY2F0ZVR5cGUiOiJodHRwczovL2RvY3MucHlwaS5vcmcvYXR0ZXN0YXRp\nb25zL3B1Ymxpc2gvdjEiLCJwcmVkaWNhdGUiOm51bGx9\n","signature":"MEUCIBhiN/nMGL7hzYBOPB9TLknhGSdKnCD4zB7L5yew4BbwAiEArs8yz0+BOcgHKsKBsMuNyYaD\nGZE0UWBn0Gp5ZF35/Sc=\n"}}

tests/functional/api/test_simple.py

@@ -10,9 +10,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import hashlib
+
 from http import HTTPStatus
+from pathlib import Path
+
+import pymacaroons
+
+from warehouse.macaroons import caveats
 
-from ...common.db.packaging import ProjectFactory, ReleaseFactory
+from ...common.db.accounts import EmailFactory, UserFactory
+from ...common.db.macaroons import MacaroonFactory
+from ...common.db.oidc import GitHubPublisherFactory
+from ...common.db.packaging import ProjectFactory, ReleaseFactory, RoleFactory
+
+_HERE = Path(__file__).parent
+_ASSETS = _HERE.parent / "_fixtures"
 
 
 def test_simple_api_html(webtest):
@@ -31,3 +44,88 @@ def test_simple_api_detail(webtest):
     assert resp.content_type == "text/html"
     assert "X-PyPI-Last-Serial" in resp.headers
     assert f"Links for {project.normalized_name}" in resp.text
+
+
+def test_simple_attestations_from_upload(webtest):
+    user = UserFactory.create(
+        password=(  # 'password'
+            "$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ"
+            "HOJaqfBroT0JCieHug281c"
+        )
+    )
+    EmailFactory.create(user=user, verified=True)
+    project = ProjectFactory.create(name="sampleproject")
+    RoleFactory.create(user=user, project=project, role_name="Owner")
+    publisher = GitHubPublisherFactory.create(projects=[project])
+
+    # Construct the macaroon. This needs to be based on a Trusted Publisher, which is
+    # required to upload attestations
+    dm = MacaroonFactory.create(
+        oidc_publisher_id=publisher.id,
+        caveats=[
+            caveats.OIDCPublisher(oidc_publisher_id=str(publisher.id)),
+            caveats.ProjectID(project_ids=[str(p.id) for p in publisher.projects]),
+        ],
+        additional={"oidc": {"ref": "someref", "sha": "somesha"}},
+    )
+
+    m = pymacaroons.Macaroon(
+        location="localhost",
+        identifier=str(dm.id),
+        key=dm.key,
+        version=pymacaroons.MACAROON_V2,
+    )
+    for caveat in dm.caveats:
+        m.add_first_party_caveat(caveats.serialize(caveat))
+    serialized_macaroon = f"pypi-{m.serialize()}"
+
+    with open(_ASSETS / "sampleproject-3.0.0.tar.gz", "rb") as f:
+        content = f.read()
+
+    with open(
+        _ASSETS / "sampleproject-3.0.0.tar.gz.publish.attestation",
+    ) as f:
+        attestation = f.read()
+
+    webtest.set_authorization(("Basic", ("__token__", serialized_macaroon)))
+    webtest.post(
+        "/legacy/?:action=file_upload",
+        params={
+            "name": "sampleproject",
+            "sha256_digest": (
+                "117ed88e5db073bb92969a7545745fd977ee85b7019706dd256a64058f70963d"
+            ),
+            "filetype": "sdist",
+            "metadata_version": "2.1",
+            "version": "3.0.0",
+            "attestations": f"[{attestation}]",
+        },
+        upload_files=[("content", "sampleproject-3.0.0.tar.gz", content)],
+        status=HTTPStatus.OK,
+    )
+
+    assert len(project.releases) == 1
+    assert project.releases[0].files.count() == 1
+    assert len(project.releases[0].files[0].attestations) == 1
+
+    expected_provenance = hashlib.sha256(b"sampleproject-3.0.0.tar.gz:1").hexdigest()
+    expected_filename = "sampleproject-3.0.0.tar.gz"
+
+    response = webtest.get("/simple/sampleproject/", status=HTTPStatus.OK)
+    link = response.html.find("a", text=expected_filename)
+
+    assert "data-provenance" in link.attrs
+    assert link.get("data-provenance") == expected_provenance
+
+    response = webtest.get(
+        "/simple/sampleproject/",
+        headers={"Accept": "application/vnd.pypi.simple.v1+json"},
+        status=HTTPStatus.OK,
+    )
+
+    assert response.content_type == "application/vnd.pypi.simple.v1+json"
+
+    json_content = response.json
+    assert len(json_content["files"]) == 1
+    assert json_content["files"][0]["filename"] == expected_filename
+    assert json_content["files"][0]["provenance"] == expected_provenance