[WIP] warehouse: TUF initialization

.gitignore

@@ -19,6 +19,7 @@ docker-compose.override.yaml
 
 node_modules/
 
+dev/tuf.*
 dev/example.sql
 dev/prod.sql
 dev/prod.sql.xz
@@ -28,6 +29,7 @@ warehouse/.commit
 warehouse/static/components
 warehouse/static/dist
 warehouse/admin/static/dist
+warehouse/tuf/dist
 
 tags
 *.sw*

Makefile

@@ -5,6 +5,7 @@ BRANCH := $(shell echo "$${TRAVIS_BRANCH:-master}")
 DB := example
 IPYTHON := no
 LOCALES := $(shell .state/env/bin/python -c "from warehouse.i18n import KNOWN_LOCALES; print(' '.join(set(KNOWN_LOCALES)-{'en'}))")
+WAREHOUSE_CLI := docker-compose run --rm web python -m warehouse
 
 # set environment variable WAREHOUSE_IPYTHON_SHELL=1 if IPython
 # needed in development environment
@@ -154,6 +155,16 @@ initdb:
 	docker-compose run --rm web python -m warehouse db upgrade head
 	$(MAKE) reindex
 
+inittuf:
+	$(WAREHOUSE_CLI) tuf keypair --name root --path /opt/warehouse/src/dev/tuf.root
+	$(WAREHOUSE_CLI) tuf keypair --name snapshot --path /opt/warehouse/src/dev/tuf.snapshot
+	$(WAREHOUSE_CLI) tuf keypair --name targets --path /opt/warehouse/src/dev/tuf.targets
+	$(WAREHOUSE_CLI) tuf keypair --name timestamp --path /opt/warehouse/src/dev/tuf.timestamp
+	$(WAREHOUSE_CLI) tuf keypair --name bins --path /opt/warehouse/src/dev/tuf.bins
+	$(WAREHOUSE_CLI) tuf keypair --name bin-n --path /opt/warehouse/src/dev/tuf.bin-n
+	$(WAREHOUSE_CLI) tuf new-repo
+	$(WAREHOUSE_CLI) tuf build-targets
+
 reindex:
 	docker-compose run --rm web python -m warehouse search reindex
 

Procfile

@@ -4,3 +4,4 @@ web-uploads: bin/start-web python -m gunicorn.app.wsgiapp -c gunicorn-uploads.co
 worker: bin/start-worker celery -A warehouse worker -Q default -l info --max-tasks-per-child 32
 worker-malware: bin/start-worker celery -A warehouse worker -Q malware -l info --max-tasks-per-child 32
 worker-beat: bin/start-worker celery -A warehouse beat -S redbeat.RedBeatScheduler -l info
+worker-tuf: bin/start-worker celery -A warehouse worker -Q tuf -l info --max-tasks-per-child 32

dev/environment

@@ -40,3 +40,12 @@ TOKEN_EMAIL_SECRET="an insecure email verification secret key"
 TOKEN_TWO_FACTOR_SECRET="an insecure two-factor auth secret key"
 
 WAREHOUSE_LEGACY_DOMAIN=pypi.python.org
+
+TUF_KEY_BACKEND=warehouse.tuf.services.LocalKeyService key.path=/opt/warehouse/src/dev
+TUF_REPO_BACKEND=warehouse.tuf.services.LocalRepositoryService repo.path=/opt/warehouse/src/warehouse/tuf/dist
+TUF_ROOT_SECRET="an insecure private key password"
+TUF_SNAPSHOT_SECRET="an insecure private key password"
+TUF_TARGETS_SECRET="an insecure private key password"
+TUF_TIMESTAMP_SECRET="an insecure private key password"
+TUF_BINS_SECRET="an insecure private key password"
+TUF_BIN_N_SECRET="an insecure private key password"

docker-compose.yml

@@ -87,6 +87,7 @@ services:
         DEVEL: "yes"
     command: hupper -m celery -A warehouse worker -B -S redbeat.RedBeatScheduler -l info
     volumes:
+      - ./dev:/opt/warehouse/src/dev:z
       - ./warehouse:/opt/warehouse/src/warehouse:z
     env_file: dev/environment
     environment:

requirements/main.in

@@ -54,6 +54,7 @@ stdlib-list
 structlog
 transaction
 trove-classifiers
+https://github.com/theupdateframework/tuf/archive/5d16f91ca7251d573106307f8269db1f680f77a4.zip
 typeguard
 webauthn
 whitenoise

requirements/main.txt

@@ -340,6 +340,11 @@ idna==2.10 \
     --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \
     --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 \
     # via email-validator, requests
+iso8601==0.1.12 \
+    --hash=sha256:210e0134677cc0d02f6028087fee1df1e1d76d372ee1db0bf30bf66c5c1c89a3 \
+    --hash=sha256:49c4b20e1f38aa5cf109ddcd39647ac419f928512c869dc01d5c7098eddede82 \
+    --hash=sha256:bbbae5fb4a7abfe71d4688fd64bff70b91bbd74ef6a99d964bab18f7fdf286dd \
+    # via tuf
 itsdangerous==1.1.0 \
     --hash=sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19 \
     --hash=sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749 \
@@ -645,7 +650,7 @@ pyramid==1.10.4 \
 python-dateutil==2.8.1 \
     --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
     --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \
-    # via alembic, botocore, celery-redbeat, elasticsearch-dsl
+    # via alembic, botocore, celery-redbeat, elasticsearch-dsl, securesystemslib
 python-editor==1.0.4 \
     --hash=sha256:1bf6e860a8ad52a14c3ee1252d5dc25b2030618ed80c022598f00176adc8367d \
     --hash=sha256:51fda6bcc5ddbbb7063b2af7509e43bd84bfc32a4ff71349ec7847713882327b \
@@ -676,7 +681,7 @@ requests-aws4auth==1.0 \
 requests==2.24.0 \
     --hash=sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b \
     --hash=sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898 \
-    # via -r requirements/main.in, datadog, google-api-core, premailer, requests-aws4auth
+    # via -r requirements/main.in, datadog, google-api-core, premailer, requests-aws4auth, tuf
 rfc3986==1.4.0 \
     --hash=sha256:112398da31a3344dc25dbf477d8df6cb34f9278a94fee2625d89e4514be8bb9d \
     --hash=sha256:af9147e9aceda37c91a05f4deb128d4b4b49d6b199775fd2d2927768abdc8f50 \
@@ -689,6 +694,10 @@ s3transfer==0.3.3 \
     --hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \
     --hash=sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db \
     # via boto3
+securesystemslib==0.15.0 \
+    --hash=sha256:456459fa16893869b2a23444179f742e774bdbf24ec1156549cca03cb338dd13 \
+    --hash=sha256:faf04a10682c34f589fde12cb27ce51ba61768a6f9c2455bab99332b8e90d180 \
+    # via tuf
 sentry-sdk==0.16.2 \
     --hash=sha256:2de15b13836fa3522815a933bd9c887c77f4868071043349f94f1b896c1bcfb8 \
     --hash=sha256:38bb09d0277117f76507c8728d9a5156f09a47ac5175bb8072513859d19a593b \
@@ -754,6 +763,9 @@ trove-classifiers==2020.8.17 \
     --hash=sha256:a38bf56e5e5db7e9a5c5bc65b3aae1e4058ad0092e4fbcaefe824bd0c370ecbf \
     --hash=sha256:ee9e1c3f525bd02b1eebedbb6b8e023e3612bc3065309b017387786adac54e61 \
     # via -r requirements/main.in
+https://github.com/theupdateframework/tuf/archive/00e15c4714f33dedd9cb5cc5602e12a94261c254.zip \
+    --hash=sha256:6cb5fccd9da807fdf19da5f63dc05f95f0904ab3c55c68aa9bc6dab37371ca37 \
+    # via -r requirements/main.in
 typeguard==2.9.1 \
     --hash=sha256:529ef3d88189cc457f4340388028412f71be8091c2c943465146d4170fb67288 \
     --hash=sha256:e258567e62d28f9a51d4f7c71f491154e9ef0889286ad2f37e3e22e4f668b21b \

tests/conftest.py

@@ -178,6 +178,8 @@ def app_config(database):
         "files.backend": "warehouse.packaging.services.LocalFileStorage",
         "docs.backend": "warehouse.packaging.services.LocalFileStorage",
         "mail.backend": "warehouse.email.services.SMTPEmailSender",
+        "tuf.key_backend": "warehouse.tuf.services.LocalKeyService",
+        "tuf.repo_backend": "warehouse.tuf.services.LocalRepositoryService",
         "malware_check.backend": (
             "warehouse.malware.services.PrinterMalwareCheckService"
         ),

tests/unit/test_config.py

@@ -320,6 +320,7 @@ def __init__(self):
             pretend.call(".malware"),
             pretend.call(".manage"),
             pretend.call(".packaging"),
+            pretend.call(".tuf"),
             pretend.call(".redirects"),
             pretend.call(".routes"),
             pretend.call(".admin"),
@@ -370,7 +371,8 @@ def __init__(self):
         ),
     ]
     assert configurator_obj.add_static_view.calls == [
-        pretend.call("static", "warehouse:static/dist/", cache_max_age=315360000)
+        pretend.call("tuf", "warehouse:tuf/dist/metadata.staged/"),
+        pretend.call("static", "warehouse:static/dist/", cache_max_age=315360000),
     ]
     assert configurator_obj.add_cache_buster.calls == [
         pretend.call("warehouse:static/dist/", cachebuster_obj)

tests/unit/test_tasks.py

@@ -504,8 +504,12 @@ def test_includeme(env, ssl, broker_url, expected_url, transport_options):
         "task_queues": (
             Queue("default", routing_key="task.#"),
             Queue("malware", routing_key="malware.#"),
+            Queue("tuf", routing_key="tuf.#"),
         ),
-        "task_routes": {"warehouse.malware.tasks.*": {"queue": "malware"}},
+        "task_routes": {
+            "warehouse.malware.tasks.*": {"queue": "malware"},
+            "warehouse.tuf.tasks.*": {"queue": "tuf"},
+        },
         "REDBEAT_REDIS_URL": (config.registry.settings["celery.scheduler_url"]),
     }.items():
         assert app.conf[key] == value

warehouse/cli/tuf.py

@@ -0,0 +1,172 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+
+import click
+
+from tuf import repository_tool
+
+from warehouse.cli import warehouse
+from warehouse.config import Environment
+from warehouse.tuf import BIN_N_ROLE, BINS_ROLE, TOPLEVEL_ROLES, utils
+
+# TUF_REPO = "warehouse/tuf/dist"
+
+
+def _make_backsigned_fileinfo_from_file(file):
+    return utils.make_fileinfo(file, custom={"backsigned": True})
+
+
+def _key_service(config):
+    key_service_class = config.maybe_dotted(config.registry.settings["tuf.key_backend"])
+    return key_service_class.create_service(None, config)
+
+
+def _repository_service(config):
+    repo_service_class = config.maybe_dotted(
+        config.registry.settings["tuf.repo_backend"]
+    )
+    return repo_service_class.create_service(None, config)
+
+
+@warehouse.group()  # pragma: no-branch
+def tuf():
+    """
+    Manage Warehouse's TUF state.
+    """
+
+
+# TODO: Need subcommands for:
+# 1. creating the world (totally new TUF repo, including root)
+# 2. updating the root metadata (including revocations?)
+# 3. removing stale metadata
+
+
+@tuf.command()
+@click.pass_obj
+@click.option("--name", "name_", help="The name of the TUF role for this keypair")
+@click.option("--path", "path_", help="The basename of the Ed25519 keypair to generate")
+def keypair(config, name_, path_):
+    repository_tool.generate_and_write_ed25519_keypair(
+        path_, password=config.registry.settings[f"tuf.{name_}.secret"]
+    )
+
+
+@tuf.command()
+@click.pass_obj
+def new_repo(config):
+    """
+    Initialize the TUF repository from scratch, including a brand new root.
+    """
+
+    repository = repository_tool.create_new_repository(
+        config.registry.settings["tuf.repo.path"]
+    )
+
+    key_service = _key_service(config)
+    for role in TOPLEVEL_ROLES:
+        role_obj = getattr(repository, role)
+        role_obj.threshold = config.registry.settings[f"tuf.{role}.threshold"]
+
+        if config.registry.settings["warehouse.env"] == Environment.development:
+            role_obj.expiration = datetime.datetime.now() + datetime.timedelta(
+                seconds=config.registry.settings["tuf.development_metadata_expiry"]
+            )
+
+        pubkeys = key_service.pubkeys_for_role(role)
+        privkeys = key_service.privkeys_for_role(role)
+        if len(pubkeys) < role_obj.threshold or len(privkeys) < role_obj.threshold:
+            raise click.ClickException(
+                f"Unable to initialize TUF repo ({role} needs {role_obj.threshold} keys"
+            )
+
+        for pubkey in pubkeys:
+            role_obj.add_verification_key(pubkey)
+
+        for privkey in privkeys:
+            role_obj.load_signing_key(privkey)
+
+    repository.mark_dirty(TOPLEVEL_ROLES)
+    repository.writeall(consistent_snapshot=True,)
+
+
+@tuf.command()
+@click.pass_obj
+def build_targets(config):
+    """
+    Given an initialized (but empty) TUF repository, create the delegated
+    targets role (bins) and its hashed bin delegations (each bin-n).
+    """
+
+    repo_service = _repository_service(config)
+    repository = repo_service.load_repository()
+
+    # Load signing keys. We do this upfront for the top-level roles.
+    key_service = _key_service(config)
+    for role in ["snapshot", "targets", "timestamp"]:
+        role_obj = getattr(repository, role)
+
+        [role_obj.load_signing_key(k) for k in key_service.privkeys_for_role(role)]
+
+    # NOTE: TUF normally does delegations by path patterns (i.e., globs), but PyPI
+    # doesn't store its uploads on the same logical host as the TUF repository.
+    # The last parameter to `delegate` is a special sentinel for this.
+    repository.targets.delegate(
+        BINS_ROLE, key_service.pubkeys_for_role(BINS_ROLE), ["*"]
+    )
+    for privkey in key_service.privkeys_for_role(BINS_ROLE):
+        repository.targets(BINS_ROLE).load_signing_key(privkey)
+
+    repository.targets(BINS_ROLE).delegate_hashed_bins(
+        [],
+        key_service.pubkeys_for_role(BIN_N_ROLE),
+        config.registry.settings["tuf.bin-n.count"],
+    )
+    for privkey in key_service.privkeys_for_role(BIN_N_ROLE):
+        for delegation in repository.targets(BINS_ROLE).delegations:
+            delegation.load_signing_key(privkey)
+
+    dirty_roles = ["snapshot", "targets", "timestamp", BINS_ROLE]
+    for idx in range(1, 2 ** 16, 4):
+        low = f"{idx - 1:04x}"
+        high = f"{idx + 2:04x}"
+        dirty_roles.append(f"{low}-{high}")
+
+    # Collect the "paths" for every PyPI package. These are packages already in
+    # existence, so we'll add some additional data to their targets to
+    # indicate that we're back-signing them.
+    from warehouse.db import Session
+    from warehouse.packaging.models import File
+
+    db = Session(bind=config.registry["sqlalchemy.engine"])
+    for file in db.query(File).all():
+        fileinfo = _make_backsigned_fileinfo_from_file(file)
+        repository.targets(BINS_ROLE).add_target_to_bin(
+            file.path,
+            number_of_bins=config.registry.settings["tuf.bin-n.count"],
+            fileinfo=fileinfo,
+        )
+
+    repository.mark_dirty(dirty_roles)
+    repository.writeall(
+        consistent_snapshot=True, use_existing_fileinfo=True,
+    )
+
+
+@tuf.command()
+@click.pass_obj
+def new_root(config):
+    """
+    Create a new
+    """
+    pass

warehouse/config.py

@@ -200,13 +200,21 @@ def configure(settings=None):
         coercer=int,
         default=21600,  # 6 hours
     )
+    maybe_set(settings, "tuf.root.secret", "TUF_ROOT_SECRET")
+    maybe_set(settings, "tuf.snapshot.secret", "TUF_SNAPSHOT_SECRET")
+    maybe_set(settings, "tuf.targets.secret", "TUF_TARGETS_SECRET")
+    maybe_set(settings, "tuf.timestamp.secret", "TUF_TIMESTAMP_SECRET")
+    maybe_set(settings, "tuf.bins.secret", "TUF_BINS_SECRET")
+    maybe_set(settings, "tuf.bin-n.secret", "TUF_BIN_N_SECRET")
     maybe_set_compound(settings, "files", "backend", "FILES_BACKEND")
     maybe_set_compound(settings, "docs", "backend", "DOCS_BACKEND")
     maybe_set_compound(settings, "origin_cache", "backend", "ORIGIN_CACHE")
     maybe_set_compound(settings, "mail", "backend", "MAIL_BACKEND")
     maybe_set_compound(settings, "metrics", "backend", "METRICS_BACKEND")
     maybe_set_compound(settings, "breached_passwords", "backend", "BREACHED_PASSWORDS")
     maybe_set_compound(settings, "malware_check", "backend", "MALWARE_CHECK_BACKEND")
+    maybe_set_compound(settings, "tuf", "key_backend", "TUF_KEY_BACKEND")
+    maybe_set_compound(settings, "tuf", "repo_backend", "TUF_REPO_BACKEND")
 
     # Add the settings we use when the environment is set to development.
     if settings["warehouse.env"] == Environment.development:
@@ -235,6 +243,10 @@ def configure(settings=None):
             ],
         )
 
+        # For development only: this artificially prolongs the expirations of any
+        # Warehouse-generated TUF metadata by approximately one year.
+        settings.setdefault("tuf.development_metadata_expiry", 31536000)
+
     # Actually setup our Pyramid Configurator with the values pulled in from
     # the environment as well as the ones passed in to the configure function.
     config = Configurator(settings=settings)
@@ -402,6 +414,12 @@ def configure(settings=None):
     # Allow the packaging app to register any services it has.
     config.include(".packaging")
 
+    # Register TUF support for package integrity
+    config.include(".tuf")
+
+    # Serve the TUF metadata files.
+    config.add_static_view("tuf", "warehouse:tuf/dist/metadata.staged/")
+
     # Configure redirection support
     config.include(".redirects")