diff --git a/ci-requirements.txt b/ci-requirements.txt index e0ef0d81d6..45f441a6b3 100644 --- a/ci-requirements.txt +++ b/ci-requirements.txt @@ -7,3 +7,4 @@ bandit # Pin to versions supported by `coveralls` (see .travis.yml) # https://github.com/coveralls-clients/coveralls-python/releases/tag/1.8.1 coverage<5.0 +mock; python_version < "3" diff --git a/dev-requirements.txt b/dev-requirements.txt index f1c8515fc9..4c3fd36b2e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -15,7 +15,6 @@ # Be sure to leave these comments at the top of the new file. # -e . -<<<<<<< HEAD astroid==2.3.2; python_version >= "3.0" astroid==1.6.5 ; python_version < "3.0" # pyup: ignore @@ -41,6 +40,7 @@ iso8601==0.1.12 isort==4.3.21 lazy-object-proxy==1.4.3 mccabe==0.6.1 +mock==3.0.5; python_version < "3.3" more-itertools==7.2.0 ; python_version >= "3.0" # via zipp more-itertools==5.0.0 ; python_version < "3.0" # pyup: ignore packaging==19.2 # via tox @@ -53,7 +53,7 @@ pylint==2.4.3; python_version >= "3.0" pylint==1.9.3 ; python_version < "3.0" # pyup: ignore pynacl==1.3.0 pyparsing==2.4.2 # via packaging -python-dateutil==2.8.1 # via securesystemslib +python-dateutil==2.8.0 # via securesystemslib pyyaml==5.1.2 requests==2.22.0 scandir==1.10.0 ; python_version < "3.0" # via pathlib2 diff --git a/docs/TUTORIAL.md b/docs/TUTORIAL.md index 0b38b76840..6bb3190730 100644 --- a/docs/TUTORIAL.md +++ b/docs/TUTORIAL.md @@ -125,7 +125,7 @@ The following four key files should now exist: If a filepath is not given, the KEYID of the generated key is used as the filename. The key files are written to the current working directory. -``` +```python >>> generate_and_write_rsa_keypair() Enter a password for the encrypted RSA key (/path/to/b5b8de8aeda674bce948fbe82cab07e309d6775fc0ec299199d16746dc2bd54c): Confirm: @@ -242,11 +242,14 @@ top-level roles, including itself. >>> repository.dirty_roles() Dirty roles: ['root'] -# The status() function also prints the next role that needs editing. In this -# example, the 'targets' role needs editing next, since the root role is now -# fully valid. +# Status does a dry run of writeall(), that is it actually signs metadata +# TODO: exlain this >>> repository.status() 'targets' role contains 0 / 1 public keys. +'snapshot' role contains 0 / 1 public keys. +'timestamp' role contains 0 / 1 public keys. +'root' role contains 2 / 2 signatures. +'targets' role contains 0 / 1 signatures. # In the next section, update the other top-level roles and create a repository # with valid metadata. @@ -363,7 +366,7 @@ the target filepaths to metadata. # Note: Since we set the 'recursive_walk' argument to false, the 'myproject' # sub-directory is excluded from 'list_of_targets'. >>> list_of_targets -['repository/targets/file2.txt', 'repository/targets/file1.txt', 'repository/targets/file3.txt'] +['/path/to/repository/targets/file2.txt', '/path/to/repository/targets/file1.txt', '/path/to/repository/targets/file3.txt'] # Add the list of target paths to the metadata of the top-level Targets role. # Any target file paths that might already exist are NOT replaced, and @@ -374,17 +377,12 @@ the target filepaths to metadata. # these targets can be included in Targets metadata. >>> repository.targets.add_targets(list_of_targets) -# Note that you can also add targets to existing delegated targets roles, -# accessing them this way: ->>> repository.targets('').add_target(...) ->>> repository.targets('').add_targets(...) - # Individual target files may also be added to roles, including custom data # about the target. In the example below, file permissions of the target # (octal number specifying file access for owner, group, others (e.g., 0755) is # added alongside the default fileinfo. All target objects in metadata include # the target's filepath, hash, and length. ->>> target4_filepath = "repository/targets/myproject/file4.txt" +>>> target4_filepath = os.path.abspath("repository/targets/myproject/file4.txt") >>> octal_file_permissions = oct(os.stat(target4_filepath).st_mode)[4:] >>> custom_file_permissions = {'file_permissions': octal_file_permissions} >>> repository.targets.add_target(target4_filepath, custom_file_permissions) @@ -416,7 +414,7 @@ Enter a password for the encrypted RSA key (/path/to/timestamp_key): # Which roles are dirty? >>> repository.dirty_roles() -Dirty roles: ['timestamp', 'snapshot', 'targets'] +Dirty roles: ['snapshot', 'targets', 'timestamp'] # Generate new versions of the modified top-level metadata (targets, snapshot, # and timestamp). @@ -433,12 +431,15 @@ new metadata to disk. # Remove a target file listed in the "targets" metadata. The target file is # not actually deleted from the file system. ->>> repository.targets.remove_target('file3.txt') +>>> repository.targets.remove_target('myproject/file4.txt') # repository.writeall() writes any required metadata files (e.g., if # targets.json is updated, snapshot.json and timestamp.json are also written # to disk), updates those that have changed, and any that need updating to make # a new "snapshot" (new snapshot.json and timestamp.json). +# TODO: Explain why we need to mark as dirty + +>>> repository.mark_dirty(['snapshot', 'timestamp']) >>> repository.writeall() ``` @@ -449,16 +450,24 @@ sign metadata. Repository maintainers can dump the portion of metadata that is normally signed, sign it with an external signing tool, and append the signature to already existing metadata. -First, the signable portion of metadata can be generated -as follows: +First, the signable portion of metadata can be generated as follows: ```Python ->>> signable_content = dump_signable_metadata('targets.json') +>>> signable_content = dump_signable_metadata('repository/metadata.staged/timestamp.json') ``` -The externally generated signature can then be appended to metadata: +Then, use a tool like securesystemslib to create a signature over the signable +portion: +```python +>>> from securesystemslib.formats import encode_canonical +>>> from securesystemslib.keys import create_signature +>>> signature = create_signature( + private_timestamp_key, encode_canonical(signable_content).encode()) +``` + +Finally, append the signature to the metadata ```Python ->>> append_signature(signature, 'targets.json') +>>> append_signature(signature, 'repository/metadata.staged/timestamp.json') ``` Note that the format of the signature is the format expected in metadata, which @@ -490,13 +499,11 @@ targets and generate signed metadata. # Make a delegation (delegate trust of 'foo*.tgz' files) from "targets" to # "unclaimed", where 'unclaimed' initially contains zero targets. -# delegate(rolename, list_of_public_keys, paths, threshold=1, -# list_of_targets=None, path_hash_prefixes=None) ->>> repository.targets.delegate('unclaimed', [public_unclaimed_key], ['foo*.tgz']) - -# Thereafter, we can access a delegated role this way: ->>> repository.targets(">> repository.targets.delegate('unclaimed', [public_unclaimed_key], ['myproject/*.txt']) +# Thereafter, we can access a delegated role by add name to and e.g. add +# targets just like to the top-level target role. +>>> repository.targets("unclaimed").add_target("myproject/file4.txt") # Load the private key of "unclaimed" so that unclaimed's metadata can be # signed, and valid metadata created. @@ -505,14 +512,8 @@ Enter a password for the encrypted RSA key (/path/to/unclaimed_key): >>> repository.targets("unclaimed").load_signing_key(private_unclaimed_key) -# Update an attribute of the unclaimed role. Note: writeall() will -# automatically increment this version number automatically, so the written -# unclaimed will be version 3. ->>> repository.targets("unclaimed").version = 2 - -# Dirty roles? -$ repository.dirty_roles() -Dirty roles: ['timestamp', 'snapshot', 'targets', 'unclaimed'] +>>>> repository.dirty_roles() +Dirty roles: ['targets', 'unclaimed'] # Write the metadata of "unclaimed", "targets", "snapshot, # and "timestamp". @@ -638,7 +639,7 @@ saved on the client side. ```python >>> from tuf.repository_tool import * ->>> create_tuf_client_directory("repository/", "client/") +>>> create_tuf_client_directory("repository/", "client/tufrepo/") ``` `create_tuf_client_directory()` moves metadata from `repository/metadata` to diff --git a/requirements.txt b/requirements.txt index ae15bc5910..2f9d9009eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -126,9 +126,10 @@ pynacl==1.3.0 \ --hash=sha256:bd4ecb473a96ad0f90c20acba4f0bf0df91a4e03a1f4dd6a4bdc9ca75aa3a715 \ --hash=sha256:e2da3c13307eac601f3de04887624939aca8ee3c9488a0bb0eca4fb9401fc6b1 \ --hash=sha256:f67814c38162f4deb31f68d590771a29d5ae3b1bd64b75cf232308e5c74777e0 -python-dateutil==2.8.1 \ - --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \ - --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a # via securesystemslib +python-dateutil==2.8.0 \ + --hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \ + --hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e \ + # via securesystemslib requests==2.22.0 \ --hash=sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 \ --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31 diff --git a/setup.py b/setup.py index 78c1582b14..8c56c5794e 100755 --- a/setup.py +++ b/setup.py @@ -116,6 +116,9 @@ 'six>=1.11.0', 'securesystemslib>=0.12.0' ], + tests_require = [ + 'mock; python_version < "3.3"' + ], packages = find_packages(exclude=['tests']), scripts = [ 'tuf/scripts/repo.py', diff --git a/tests/test_tutorial.py b/tests/test_tutorial.py index be9cac7514..d234b4d2ce 100644 --- a/tests/test_tutorial.py +++ b/tests/test_tutorial.py @@ -33,6 +33,14 @@ import datetime # part of TUTORIAL.md import os # part of TUTORIAL.md, but also needed separately import shutil +import sys +import tempfile + +if sys.version_info >= (3, 3): + import unittest.mock as mock + +else: + import mock from tuf.repository_tool import * # part of TUTORIAL.md @@ -41,14 +49,13 @@ class TestTutorial(unittest.TestCase): def setUp(self): - clean_test_environment() - - + self.working_dir = os.getcwd() + self.test_dir = os.path.realpath(tempfile.mkdtemp()) + os.chdir(self.test_dir) def tearDown(self): - clean_test_environment() - - + os.chdir(self.working_dir) + shutil.rmtree(self.test_dir) def test_tutorial(self): """ @@ -125,24 +132,29 @@ def test_tutorial(self): private_root_key2 = import_rsa_privatekey_from_file( 'root_key2', password='password') + repository.root.load_signing_key(private_root_key) repository.root.load_signing_key(private_root_key2) - - # TODO: dirty_roles() doesn't return the list of dirty roles; it just - # prints the list. It should probably it should return it as well. - # If that's not changed, perhaps we should test the print output from the - # dirty_roles() statement here. - repository.dirty_roles() - # self.assertEqual(repository.dirty_roles(), ['root']) - - - # TODO: status() should return some sort of value that indicates what - # it prints. It's currently just printing status information. - # If that's not changed, perhaps we should test the print output from the - # status() statement here. - repository.status() - + # Patch logger to assert that it accurately logs dirty roles + with mock.patch("tuf.repository_tool.logger") as mock_logger: + repository.dirty_roles() + mock_logger.info.assert_called_with("Dirty roles: ['root']") + + # Patch logger to assert that it accurately logs the repo's status. Since + # the logger is called multiple times, we have to assert for the accurate + # sequence of calls or rather its call arguments. + with mock.patch("tuf.repository_lib.logger") as mock_logger: + repository.status() + self.assertListEqual( + [ + "'targets' role contains 0 / 1 public keys.", + "'snapshot' role contains 0 / 1 public keys.", + "'timestamp' role contains 0 / 1 public keys.", + "'root' role contains 2 / 2 signatures.", + "'targets' role contains 0 / 1 signatures." + ], + [args[0] for args, _ in mock_logger.info.call_args_list]) generate_and_write_rsa_keypair('targets_key', password='password') generate_and_write_rsa_keypair('snapshot_key', password='password') @@ -262,13 +274,11 @@ def test_tutorial(self): 'timestamp_key', 'password') repository.timestamp.load_signing_key(private_timestamp_key) - # TODO: dirty_roles() doesn't return the list of dirty roles; it just - # prints the list. It should probably it should return it as well. - # If that's not changed, perhaps we should test the print output from the - # dirty_roles() statement here. - repository.dirty_roles() - # self.assertEqual( - # repository.dirty_roles(), ['timestamp', 'snapshot', 'targets']) + # Patch logger to assert that it accurately logs dirty roles + with mock.patch("tuf.repository_tool.logger") as mock_logger: + repository.dirty_roles() + mock_logger.info.assert_called_with( + "Dirty roles: ['snapshot', 'targets', 'timestamp']") repository.writeall() @@ -276,6 +286,7 @@ def test_tutorial(self): self.assertTrue(os.path.exists(os.path.join( 'repository','targets', 'file3.txt'))) + repository.mark_dirty(['snapshot', 'timestamp']) repository.writeall() signable_content = dump_signable_metadata( @@ -301,13 +312,10 @@ def test_tutorial(self): repository.targets("unclaimed").version = 2 - # TODO: dirty_roles() doesn't return the list of dirty roles; it just - # prints the list. It should probably it should return it as well. - # If that's not changed, perhaps we should test the print output from the - # dirty_roles() statement here. - repository.dirty_roles() - # self.assertEqual(repository.dirty_roles(), - # ['timestamp', 'snapshot', 'targets', 'unclaimed']) + with mock.patch("tuf.repository_tool.logger") as mock_logger: + repository.dirty_roles() + mock_logger.info.assert_called_with( + "Dirty roles: ['targets', 'unclaimed']") repository.writeall() @@ -337,6 +345,9 @@ def test_tutorial(self): targets = repository.get_filepaths_in_directory( os.path.join('repository', 'targets', 'myproject'), recursive_walk=True) + repository.targets('unclaimed').delegate_hashed_bins( + targets, [public_unclaimed_key], 32) + for delegation in repository.targets('unclaimed').delegations: delegation.load_signing_key(private_unclaimed_key) @@ -373,41 +384,6 @@ def test_tutorial(self): - - -def clean_test_environment(): - """ - Delete temporary files and directories from this test (or with the same name - as those created by this test...). - """ - for directory in ['repository', 'my_repo', 'client', - 'repository/targets/my_project']: - if os.path.exists(directory): - shutil.rmtree(directory) - - for fname in [ - os.path.join('repository', 'targets', 'file1.txt'), - os.path.join('repository', 'targets', 'file2.txt'), - os.path.join('repository', 'targets', 'file3.txt'), - 'root_key', - 'root_key.pub', - 'root_key2', - 'root_key2.pub', - 'ed25519_key', - 'ed25519_key.pub', - 'targets_key', - 'targets_key.pub', - 'snapshot_key', - 'snapshot_key.pub', - 'timestamp_key', - 'timestamp_key.pub', - 'unclaimed_key', - 'unclaimed_key.pub']: - if os.path.exists(fname): - os.remove(fname) - - - # Run unit test. if __name__ == '__main__': unittest.main() diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index ef20c19ec1..8a79321079 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -446,7 +446,8 @@ def dirty_roles(self): None. """ - logger.info('Dirty roles: ' + str(tuf.roledb.get_dirty_roles(self._repository_name))) + logger.info('Dirty roles: ' + + str(sorted(tuf.roledb.get_dirty_roles(self._repository_name))))