forked from pypi/warehouse
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adds CLI and task support for initializing a TUF repository. Adds a service interface (IKeyService) with a filesystem-based implementation (LocalKeyService).
- Loading branch information
1 parent
a5db57d
commit eb24643
Showing
31 changed files
with
1,046 additions
and
54 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
2f0570511d84b133d6e1f875920587a9@@@@100000@@@@dfacbe8d4966935d16b5c9b7910e7b01189f83e0db2d5169eb4d7a2edab91924@@@@f3f67a27b03a24c8b767f5721bc345b5@@@@e9855ff4f41d1ec10877b3476eabd99c86f2162ef16172aaa17200af3ec97a0e7b05a389e3084a8c79aa2756fd999eaa67f5462e37a024b66feba784a3e70850d8d2466165ba86d745cac2a526d44439892f46a3342d1c890589c833f652942283b59441b214625b275de95d16ec199cb4fbbd7fa8b1442153fbac65db18d9c9e1345b37a7fa850d2ffe0d13035f28d68a3b47ddb310750002e8e96e751b633f06e4d9c70fa117de12d848cab845f830e082c51e0ad42342f2c24869b091e3c7dae70410578fbb6877a975983aa3dde6aa699e4e45ba7114c4e373109d2128c5211b3471390cb0ef09da3cd5e552813a906f604038218f7891213cefa10fbb7ba6c01e98c821ba45b69504d5ac2d41feeafccc974d82b89c33d200b6ceb04a7af47f08f8258cfea9a6855d3cbe91bc0892ddaabd0f508a6ed85f811dcff66e0fed066f3607f5a5f1ccecc80ad1ede362 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "ca2694d781367f94974c5176cae6c7290fe3b65f03a5c6331fe500b8c700f3aa"}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
c4509bc89725ce7b1467b4d596536564@@@@100000@@@@47c863bddaad67e2482760e0da329f3f9170dee5de9e9265cd23873947b4a373@@@@8333670a562d5b2fa37559bb4372de32@@@@d567e40c7d544b4d41f023d208cacdb76009991fb1dc4681e8b7ebae42d5756c21434937061297dfe264150d3ab9c00985d4217ca18c70904d8c1becc9e172a1677221d09d7f9de5b6778340bf26d305dcf34556066e744abcf96bd489e612d6ea9c483838f21bf06646d250b5d7f380cccf57eb990387baf28c3fd00815b1db3a34418d879718607fb54ccfb1fe20edc06bdb765dce6d3617dba5cc5aff41f5eab62a6e0f8e095a97dda3784dcb95f97af5a1c4de7fb0424d659469305cafb092a209ab78932444ffb497286cb44e4ba360e0025f71fc108e8f3b2fac794a05242ef5167bdcd5e4a1a3573805cb91b8359bcb317eb026c2ee102db096b50b5f6fc478af5146dbcd9e2ada046d6a392854e779ffa4b2bdbb0ea90d41eb6030fd0d98f0aa34510a2a6e9292d5095140864658d8db63701b661ed8ee3b6a428dfb9163e7918942ffb0506f290a76fc4703 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "a4361da485b8ae5cb10915d5254d96f05ed3d8d4c87979c5a203504b4b65ecf5"}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
83589fb72074fcb8533c8c4414015c30@@@@100000@@@@be3d99c519272cc8b8c1150ca03450c0aac21d084a0dc79966faca4d05787ff1@@@@dccf47ed06f92da01e9de35113477e88@@@@72e37b2acdb2115888975e66990009e6f6a4cdd94ff77473e056d16ef33e9ef6ea450308f1b4962f544a0796d7b7f788b35a6ae33d6cc83e56cc416440716418102d39573692e87350935088bc08b4b82d910731a0f536bebded8f80e49c3df052da2bf657ed6a1f64463da477905bc8a1bc8fb65215653a6efa3e0a74b2fabcb65d96fc1f1c31196878e9eb39eca1cfb709b69a400f159f5e8f8dc9b765590fb5cac705170149b71c0f54951d239eafaa9ea9b8c3ac0897e57e5c8f027f8d160d91e3995da00535ced37ff41bccd31e7ed5076c22b448f9a7e701a2024de56b839fd7be4689c23f8b81102fe4711585300862b01f499408295723f995775de33ce69e666883bb25302adb1ce6b21ac710ceeacefacc316aa80432a7d8e9819d78913707b8daef35d03667ab11727e6ea6bc44b9356449f6d6aed53825f2fd1397c68d2669b2b1895874678234a42afd |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "5cb1a1622f72ae901bc2ed25546503bae70b53308833385a11b6630fb52c9bc9"}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
97074b6eccb9d5920192f231249c33dc@@@@100000@@@@3965a3500a3de4f6a052f9f26efd2714affae878daec6df53c50fe20fb3b5732@@@@dbc2d164f88ea8d77b093db20475db0c@@@@d6db7159767d46f6f255a2184b2615df892f067ca0861dba18ccc5b6b7dce95d5d2b377d0d8424d39fb0b337e9bd086a6405804a55e7c8d591f090dc55d38c0820a6bc2ddd27078e1bc482fa4f19a66941cf023680d032da70a8ac6bb5c9fc7454a273d014ef569f7a79f89145c29ade6a5f56752a9e4b5f3a4ee0426680b3d1921a962a6edee5c77661439f7bc95674fbc009ecf903c20fbac786e94164bc43df1fb8536bc6666ce098cfad411c852154e4dd7dda0b9046cd4310d7ae9b822be3cb7b62e66e73718424743e2297b091d2cd8cfce108f1f6a2e17ad382655c517d8131da3a72ac1f2ae8cd760705bd17ec5ff6834c86e8f08fa804152774146a3bda063ae29a01be00723b96e3370d9f046a1287ad4ccaa007cc016d59bf935268cb8b7946bb5b5480962137ed6cb449df43aa83d3d086e4b10e7c33110847c9bc1ee2c24ebbd66e989c6e28c52ac28d |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "c19d2c8532667d7784f5b33f5bdccd529b52c5d55754b4ef989a261bf1695431"}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
e25efe7833ec8682ff0d09366d264fe5@@@@100000@@@@ecb082633529fc0d392b3d0c8f36972423c91060bbfc1903a8e390deeb650817@@@@3f116f9b022d8e1e3959e99ee800f5cd@@@@45defac0e6ef65003645fd34e3a39a47579e6da578529fe425bd9798e6007cb901d4003e866068a853e17faeba365367f63ead34fac88bca0938f308e2556913c8f9ed56892262704b0a7dcb25bedf61fcd8c72af6eaa4b8630936b704f2aae364313211a32d3bac84d88273fd2e2ee48f1c3d7bef8de74a800124f469bacad916ed3d1242b2c691548b68d3e890a8ef19d3e56817a94e1e508136a8f6cd4d5af4012a355e874bfe446504b75991187f76f521dc3b589d1eae126d978e1e7b268688e1992729cb820f5057352e6c916af4ce448bf0a26d1cf52968a5398c6c516cae831f7f75a7885489fae79fb3a9f8faae0dc32ceaf17106df7834ffc4e28589df0d886a08f51751d2c15999d538197a1e392647ffc81a314e87f936e2087762f0d6004a862789e27021e11e489818320da3bd8801f208deb0b4aa4d1f6319712729bb716c71837ac229d216180ef4 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "715ea559e2fa80a2bad3c50836c55ce33d256fa7ad5468931cc76a8a10e86d37"}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
7a59984187faa4893ceba030f5c5b4a7@@@@100000@@@@7ec97a52365194434c8294e50ceb454222194670104eeacce30846222cdd2756@@@@20d9afcaebb214eb40b3827d6730f318@@@@7763c7a7fd2d9897ad9951149bbd00ced57d9a68339ce0b1a1dbfc7cbb1defdd6d2a87178c140a9de2180e58616f6004a53f384356743b3aad3d54d073e70d8a6c9420ee9e6880405d969f59ddac072235740259868af52a9c0d4582ce3cd0435e9dc0138f0571b02629721f11308e5a6fb26f75d2bd9d298da3bfaad4ec6aaa80d0d81a2e22690d8416b188e132b87bc5ef89d1460e4f7b2c12dac555ad99212b30c73d5f3ef30a77e0eb2f0f82e88c869ef585a8707267be76619c7a3e430563be1d7d1073ed1706bea7d5232e784674c104920d4f44382beeb8d07cfe0d9f5ec0812497d48d2f8b71634a2ae29ca9bf2732d4e0c3216a62b318d11dfc1608f0c9f9e8eff142b45cce4ac7bc91c4967fcd9b3b5484bf016375100391321c54e6a3da85cfb99c6e429da2a586ea15d2f09e2f7acb3e5358bef10f0116b9f3da2aa2c49f9fb07ffcaea4677144bc1969 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "5e12fa7b474cb886cdc0cd14c5017b09b3759125ab7e6985bd9c5953063de333"}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,176 @@ | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
import datetime | ||
|
||
import click | ||
|
||
from tuf import repository_tool | ||
|
||
from warehouse.cli import warehouse | ||
from warehouse.config import Environment | ||
from warehouse.tuf import utils | ||
from warehouse.tuf.constants import BIN_N_COUNT, TOPLEVEL_ROLES, Role | ||
|
||
|
||
def _make_backsigned_fileinfo_from_file(file): | ||
return utils.make_fileinfo(file, custom={"backsigned": True}) | ||
|
||
|
||
def _key_service(config): | ||
key_service_class = config.maybe_dotted(config.registry.settings["tuf.key_backend"]) | ||
return key_service_class.create_service(None, config) | ||
|
||
|
||
def _repository_service(config): | ||
repo_service_class = config.maybe_dotted( | ||
config.registry.settings["tuf.repo_backend"] | ||
) | ||
return repo_service_class.create_service(None, config) | ||
|
||
|
||
def _set_expiration_for_role(config, role_obj, role_name): | ||
# If we're initializing TUF for development purposes, give | ||
# every role a long expiration time so that developers don't have to | ||
# continually re-initialize it. | ||
if config.registry.settings["warehouse.env"] == Environment.development: | ||
role_obj.expiration = datetime.datetime.now() + datetime.timedelta( | ||
seconds=config.registry.settings["tuf.development_metadata_expiry"] | ||
) | ||
else: | ||
role_obj.expiration = datetime.datetime.now() + datetime.timedelta( | ||
seconds=config.registry.settings[f"tuf.{role_name}.expiry"] | ||
) | ||
|
||
|
||
@warehouse.group() # pragma: no-branch | ||
def tuf(): | ||
""" | ||
Manage Warehouse's TUF state. | ||
""" | ||
|
||
|
||
@tuf.command() | ||
@click.pass_obj | ||
@click.option("--name", "name_", help="The name of the TUF role for this keypair") | ||
@click.option("--path", "path_", help="The basename of the Ed25519 keypair to generate") | ||
def keypair(config, name_, path_): | ||
""" | ||
Generate a new TUF keypair, for development purposes. | ||
""" | ||
|
||
repository_tool.generate_and_write_ed25519_keypair( | ||
path_, password=config.registry.settings[f"tuf.{name_}.secret"] | ||
) | ||
|
||
|
||
@tuf.command() | ||
@click.pass_obj | ||
def new_repo(config): | ||
""" | ||
Initialize the TUF repository from scratch, including a brand new root. | ||
""" | ||
|
||
repository = repository_tool.create_new_repository( | ||
config.registry.settings["tuf.repo.path"] | ||
) | ||
|
||
key_service = _key_service(config) | ||
for role in TOPLEVEL_ROLES: | ||
role_obj = getattr(repository, role) | ||
role_obj.threshold = config.registry.settings[f"tuf.{role}.threshold"] | ||
_set_expiration_for_role(config, role_obj, role) | ||
|
||
pubkeys = key_service.pubkeys_for_role(role) | ||
privkeys = key_service.privkeys_for_role(role) | ||
if len(pubkeys) < role_obj.threshold or len(privkeys) < role_obj.threshold: | ||
raise click.ClickException( | ||
f"Unable to initialize TUF repo ({role} needs {role_obj.threshold} keys" | ||
) | ||
|
||
for pubkey in pubkeys: | ||
role_obj.add_verification_key(pubkey) | ||
|
||
for privkey in privkeys: | ||
role_obj.load_signing_key(privkey) | ||
|
||
repository.mark_dirty(TOPLEVEL_ROLES) | ||
repository.writeall( | ||
consistent_snapshot=True, | ||
) | ||
|
||
|
||
@tuf.command() | ||
@click.pass_obj | ||
def build_targets(config): | ||
""" | ||
Given an initialized (but empty) TUF repository, create the delegated | ||
targets role (bins) and its hashed bin delegations (each bin-n). | ||
""" | ||
|
||
repo_service = _repository_service(config) | ||
repository = repo_service.load_repository() | ||
|
||
# Load signing keys. We do this upfront for the top-level roles. | ||
key_service = _key_service(config) | ||
for role in ["snapshot", "targets", "timestamp"]: | ||
role_obj = getattr(repository, role) | ||
|
||
[role_obj.load_signing_key(k) for k in key_service.privkeys_for_role(role)] | ||
|
||
# NOTE: TUF normally does delegations by path patterns (i.e., globs), but PyPI | ||
# doesn't store its uploads on the same logical host as the TUF repository. | ||
# The last parameter to `delegate` is a special sentinel for this. | ||
repository.targets.delegate( | ||
Role.BINS.value, key_service.pubkeys_for_role(Role.BINS.value), ["*"] | ||
) | ||
bins_role = repository.targets(Role.BINS.value) | ||
_set_expiration_for_role(config, bins_role, Role.BINS.value) | ||
|
||
for privkey in key_service.privkeys_for_role(Role.BINS.value): | ||
bins_role.load_signing_key(privkey) | ||
|
||
bins_role.delegate_hashed_bins( | ||
[], | ||
key_service.pubkeys_for_role(Role.BIN_N.value), | ||
BIN_N_COUNT, | ||
) | ||
|
||
dirty_roles = ["snapshot", "targets", "timestamp", Role.BINS.value] | ||
for bin_n_role in bins_role.delegations: | ||
_set_expiration_for_role(config, bin_n_role, Role.BIN_N.value) | ||
dirty_roles.append(bin_n_role.rolename) | ||
|
||
for privkey in key_service.privkeys_for_role(Role.BIN_N.value): | ||
for bin_n_role in bins_role.delegations: | ||
bin_n_role.load_signing_key(privkey) | ||
|
||
# Collect the "paths" for every PyPI package. These are packages already in | ||
# existence, so we'll add some additional data to their targets to | ||
# indicate that we're back-signing them. | ||
from warehouse.db import Session | ||
from warehouse.packaging.models import File | ||
|
||
db = Session(bind=config.registry["sqlalchemy.engine"]) | ||
for file in db.query(File).all(): | ||
fileinfo = _make_backsigned_fileinfo_from_file(file) | ||
bins_role.add_target_to_bin( | ||
file.path, | ||
number_of_bins=BIN_N_COUNT, | ||
fileinfo=fileinfo, | ||
) | ||
|
||
repository.mark_dirty(dirty_roles) | ||
repository.writeall( | ||
consistent_snapshot=True, | ||
use_existing_fileinfo=True, | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.