Enhancements for hashed bin delegation

tests/test_repository_lib.py

@@ -335,7 +335,7 @@ def test_generate_targets_metadata(self):
     datetime_object = datetime.datetime(2030, 1, 1, 12, 0)
     expiration_date = datetime_object.isoformat() + 'Z'
     file_permissions = oct(os.stat(file1_path).st_mode)[4:]
-    target_files = {'file.txt': {'file_permission': file_permissions}}
+    target_files = {'file.txt': {'custom': {'file_permission': file_permissions}}}
 
     delegations = {"keys": {
       "a394c28384648328b16731f81440d72243c77bb44c07c040be99347f0df7d7bf": {

tests/test_repository_tool.py

@@ -369,6 +369,108 @@ def test_writeall(self):
     self.assertRaises(securesystemslib.exceptions.FormatError, repository.writeall, 3)
 
 
+  def test_writeall_no_files(self):
+    # Test writeall() when using pre-supplied fileinfo
+
+    repository_name = 'test_repository'
+    temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory)
+    repository_directory = os.path.join(temporary_directory, 'repository')
+    targets_directory = os.path.join(repository_directory,
+                                     repo_tool.TARGETS_DIRECTORY_NAME)
+
+    repository = repo_tool.create_new_repository(repository_directory, repository_name)
+
+    # (1) Load the public and private keys of the top-level roles, and one
+    # delegated role.
+    keystore_directory = os.path.join('repository_data', 'keystore')
+
+    # Load the public keys.
+    root_pubkey_path = os.path.join(keystore_directory, 'root_key.pub')
+    targets_pubkey_path = os.path.join(keystore_directory, 'targets_key.pub')
+    snapshot_pubkey_path = os.path.join(keystore_directory, 'snapshot_key.pub')
+    timestamp_pubkey_path = os.path.join(keystore_directory, 'timestamp_key.pub')
+
+    root_pubkey = repo_tool.import_rsa_publickey_from_file(root_pubkey_path)
+    targets_pubkey = \
+      repo_tool.import_ed25519_publickey_from_file(targets_pubkey_path)
+    snapshot_pubkey = \
+      repo_tool.import_ed25519_publickey_from_file(snapshot_pubkey_path)
+    timestamp_pubkey = \
+      repo_tool.import_ed25519_publickey_from_file(timestamp_pubkey_path)
+
+    # Load the private keys.
+    root_privkey_path = os.path.join(keystore_directory, 'root_key')
+    targets_privkey_path = os.path.join(keystore_directory, 'targets_key')
+    snapshot_privkey_path = os.path.join(keystore_directory, 'snapshot_key')
+    timestamp_privkey_path = os.path.join(keystore_directory, 'timestamp_key')
+
+    root_privkey = \
+      repo_tool.import_rsa_privatekey_from_file(root_privkey_path, 'password')
+    targets_privkey = \
+      repo_tool.import_ed25519_privatekey_from_file(targets_privkey_path,
+                                                'password')
+    snapshot_privkey = \
+      repo_tool.import_ed25519_privatekey_from_file(snapshot_privkey_path,
+                                                'password')
+    timestamp_privkey = \
+      repo_tool.import_ed25519_privatekey_from_file(timestamp_privkey_path,
+                                                'password')
+
+
+    # (2) Add top-level verification keys.
+    repository.root.add_verification_key(root_pubkey)
+    repository.targets.add_verification_key(targets_pubkey)
+    repository.snapshot.add_verification_key(snapshot_pubkey)
+
+    # Verify that repository.writeall() fails for insufficient threshold
+    # of signatures (default threshold = 1).
+    self.assertRaises(tuf.exceptions.UnsignedMetadataError, repository.writeall)
+
+    repository.timestamp.add_verification_key(timestamp_pubkey)
+
+
+    # (3) Load top-level signing keys.
+    repository.status()
+    repository.root.load_signing_key(root_privkey)
+    repository.status()
+    repository.targets.load_signing_key(targets_privkey)
+    repository.status()
+    repository.snapshot.load_signing_key(snapshot_privkey)
+    repository.status()
+
+    # Verify that repository.writeall() fails for insufficient threshold
+    # of signatures (default threshold = 1).
+    self.assertRaises(tuf.exceptions.UnsignedMetadataError, repository.writeall)
+
+    repository.timestamp.load_signing_key(timestamp_privkey)
+
+    # Add target fileinfo
+    target1_hashes = {'sha256': 'c2986576f5fdfd43944e2b19e775453b96748ec4fe2638a6d2f32f1310967095'}
+    target2_hashes = {'sha256': '517c0ce943e7274a2431fa5751e17cfd5225accd23e479bfaad13007751e87ef'}
+    target1_fileinfo = tuf.formats.make_fileinfo(555, target1_hashes)
+    target2_fileinfo = tuf.formats.make_fileinfo(37, target2_hashes)
+    target1 = os.path.join(targets_directory, 'file1.txt')
+    target2 = os.path.join(targets_directory, 'file2.txt')
+    repository.targets.add_target(target1, fileinfo=target1_fileinfo)
+    repository.targets.add_target(target2, fileinfo=target2_fileinfo)
+
+    repository.writeall(use_existing_fileinfo=True)
+
+    # Verify that the expected metadata is written.
+    metadata_directory = os.path.join(repository_directory,
+                                      repo_tool.METADATA_STAGED_DIRECTORY_NAME)
+
+    for role in ['root.json', 'targets.json', 'snapshot.json', 'timestamp.json']:
+      role_filepath = os.path.join(metadata_directory, role)
+      role_signable = securesystemslib.util.load_json_file(role_filepath)
+
+      # Raise 'securesystemslib.exceptions.FormatError' if 'role_signable' is
+      # an invalid signable.
+      tuf.formats.check_signable_object_format(role_signable)
+
+      self.assertTrue(os.path.exists(role_filepath))
+
+
 
   def test_get_filepaths_in_directory(self):
     # Test normal case.
@@ -1076,14 +1178,14 @@ def test_add_target(self):
 
     self.assertEqual(len(self.targets_object.target_files), 2)
     self.assertTrue('file2.txt' in self.targets_object.target_files)
-    self.assertEqual(self.targets_object.target_files['file2.txt'],
+    self.assertEqual(self.targets_object.target_files['file2.txt']['custom'],
                      custom_file_permissions)
 
     # Attempt to replace target that has already been added.
     octal_file_permissions2 = oct(os.stat(target_filepath).st_mode)[4:]
     custom_file_permissions2 = {'file_permissions': octal_file_permissions}
     self.targets_object.add_target(target2_filepath, custom_file_permissions2)
-    self.assertEqual(self.targets_object.target_files['file2.txt'],
+    self.assertEqual(self.targets_object.target_files['file2.txt']['custom'],
     custom_file_permissions2)
 
     # Test improperly formatted arguments.
@@ -1094,18 +1196,6 @@ def test_add_target(self):
                       target_filepath, 3)
 
 
-    # Test invalid filepath argument (i.e., non-existent or invalid file.)
-    self.assertRaises(securesystemslib.exceptions.Error, self.targets_object.add_target,
-                      'non-existent.txt')
-
-    # Not under the repository's targets directory.
-    self.assertRaises(securesystemslib.exceptions.Error, self.targets_object.add_target,
-                      self.temporary_directory)
-
-    # Not a file (i.e., a valid path, but a directory.)
-    test_directory = os.path.join(self.targets_directory, 'test_directory')
-    os.mkdir(test_directory)
-    self.assertRaises(securesystemslib.exceptions.Error, self.targets_object.add_target, test_directory)
 
 
 
@@ -1134,17 +1224,6 @@ def test_add_targets(self):
     # Test improperly formatted arguments.
     self.assertRaises(securesystemslib.exceptions.FormatError, self.targets_object.add_targets, 3)
 
-    # Test invalid filepath argument (i.e., non-existent or invalid file.)
-    self.assertRaises(securesystemslib.exceptions.Error, self.targets_object.add_targets,
-                      ['non-existent.txt'])
-    self.assertRaises(securesystemslib.exceptions.Error, self.targets_object.add_targets,
-                      [target1_filepath, target2_filepath, 'non-existent.txt'])
-    self.assertRaises(securesystemslib.exceptions.Error, self.targets_object.add_targets,
-                      [self.temporary_directory])
-    temp_directory = os.path.join(self.targets_directory, 'temp')
-    os.mkdir(temp_directory)
-    self.assertRaises(securesystemslib.exceptions.Error, self.targets_object.add_targets,
-                      [temp_directory])
 
 
 
@@ -1288,6 +1367,32 @@ def test_delegate_hashed_bins(self):
     public_keys = [public_key]
     list_of_targets = [target1_filepath]
 
+
+    # A helper function to check that the range of prefixes the role is
+    # delegated for, specified in path_hash_prefixes, matches the range
+    # implied by the bin, or delegation role, name.
+    def check_prefixes_match_range():
+      roleinfo = tuf.roledb.get_roleinfo(self.targets_object.rolename,
+          'test_repository')
+      have_prefixes = False
+
+      for delegated_role in roleinfo['delegations']['roles']:
+        if len(delegated_role['path_hash_prefixes']) > 0:
+          rolename = delegated_role['name']
+          prefixes = delegated_role['path_hash_prefixes']
+          have_prefixes = True
+
+          if len(prefixes) > 1:
+            prefix_range = "{}-{}".format(prefixes[0], prefixes[-1])
+          else:
+            prefix_range = prefixes[0]
+
+          self.assertEqual(rolename, prefix_range)
+
+      # We expect at least one delegation with some path_hash_prefixes
+      self.assertTrue(have_prefixes)
+
+
     # Test delegate_hashed_bins() and verify that 16 hashed bins have
     # been delegated in the parent's roleinfo.
     self.targets_object.delegate_hashed_bins(list_of_targets, public_keys,
@@ -1299,11 +1404,14 @@ def test_delegate_hashed_bins(self):
 
     self.assertEqual(sorted(self.targets_object.get_delegated_rolenames()),
                      sorted(delegated_rolenames))
+    check_prefixes_match_range()
 
     # For testing / coverage purposes, try to create delegated bins that
     # hold a range of hash prefixes (e.g., bin name: 000-003).
     self.targets_object.delegate_hashed_bins(list_of_targets, public_keys,
                                              number_of_bins=512)
+    check_prefixes_match_range()
+
     # Test improperly formatted arguments.
     self.assertRaises(securesystemslib.exceptions.FormatError,
                       self.targets_object.delegate_hashed_bins, 3, public_keys,
@@ -1341,21 +1449,20 @@ def test_add_target_to_bin(self):
 
     # Set needed arguments by delegate_hashed_bins().
     public_keys = [public_key]
-    list_of_targets = [target1_filepath]
 
     # Delegate to hashed bins.  The target filepath to be tested is expected
     # to contain a hash prefix of 'e', and should be available at:
     # repository.targets('e').
-    self.targets_object.delegate_hashed_bins(list_of_targets, public_keys,
+    self.targets_object.delegate_hashed_bins([], public_keys,
         number_of_bins=16)
 
     # Ensure each hashed bin initially contains zero targets.
     for delegation in self.targets_object.delegations:
-      self.assertTrue(target1_filepath not in delegation.target_files)
+      self.assertEqual(delegation.target_files, {})
 
     # Add 'target1_filepath' and verify that the relative path of
     # 'target1_filepath' is added to the correct bin.
-    self.targets_object.add_target_to_bin(os.path.basename(target1_filepath))
+    self.targets_object.add_target_to_bin(os.path.basename(target1_filepath), 16)
 
     for delegation in self.targets_object.delegations:
       if delegation.rolename == '5':
@@ -1364,55 +1471,37 @@ def test_add_target_to_bin(self):
       else:
         self.assertFalse('file1.txt' in delegation.target_files)
 
-    # Verify that 'path_hash_prefixes' must exist for hashed bin delegations.
-
-    roleinfo = tuf.roledb.get_roleinfo(self.targets_object.rolename,
-        repository_name)
-
-    for delegated_role in roleinfo['delegations']['roles']:
-      delegated_role['path_hash_prefixes'] = []
-
-    tuf.roledb.update_roleinfo(self.targets_object.rolename, roleinfo,
-        repository_name=repository_name)
-    self.assertRaises(securesystemslib.exceptions.Error,
-                      self.targets_object.add_target_to_bin, target1_filepath)
-
-    # Verify that an exception is raised if a target does not match with
-    # any of the 'path_hash_prefixes'.
-    roleinfo = tuf.roledb.get_roleinfo(self.targets_object.rolename,
-        repository_name)
-    delegated_role = roleinfo['delegations']['roles'][0]
-    delegated_role['path_hash_prefixes'] = ['faac']
-    delegated_roles = list()
-    delegated_roles.append(delegated_role)
-    roleinfo['delegations']['roles'] = delegated_roles
-    tuf.roledb.update_roleinfo(self.targets_object.rolename, roleinfo,
-        repository_name=repository_name)
-
-    self.assertRaises(securesystemslib.exceptions.Error,
-                      self.targets_object.add_target_to_bin, target1_filepath)
-
     # Test for non-existent delegations and hashed bins.
     empty_targets_role = repo_tool.Targets(self.targets_directory, 'empty',
         repository_name=repository_name)
 
     self.assertRaises(securesystemslib.exceptions.Error,
                       empty_targets_role.add_target_to_bin,
-                      target1_filepath)
+                      os.path.basename(target1_filepath), 16)
 
     # Test for a required hashed bin that does not exist.
-    self.targets_object.revoke('e')
+    self.targets_object.revoke('5')
     self.assertRaises(securesystemslib.exceptions.Error,
                       self.targets_object.add_target_to_bin,
-                      target1_filepath)
+                      os.path.basename(target1_filepath), 16)
+
+    # Test adding a target with fileinfo
+    target2_hashes = {'sha256': '517c0ce943e7274a2431fa5751e17cfd5225accd23e479bfaad13007751e87ef'}
+    target2_fileinfo = tuf.formats.make_fileinfo(37, target2_hashes)
+    target2_filepath = os.path.join(self.targets_directory, 'file2.txt')
+
+    self.targets_object.add_target_to_bin(os.path.basename(target2_filepath), 16, fileinfo=target2_fileinfo)
+
+    for delegation in self.targets_object.delegations:
+      if delegation.rolename == '0':
+        self.assertTrue('file2.txt' in delegation.target_files)
+
+      else:
+        self.assertFalse('file2.txt' in delegation.target_files)
 
     # Test improperly formatted argument.
     self.assertRaises(securesystemslib.exceptions.FormatError,
-                      self.targets_object.add_target_to_bin, 3)
-
-    # Invalid target file path argument.
-    self.assertRaises(securesystemslib.exceptions.Error,
-                      self.targets_object.add_target_to_bin, '/non-existent')
+                      self.targets_object.add_target_to_bin, 3, 'foo')
 
 
 
@@ -1426,48 +1515,43 @@ def test_remove_target_from_bin(self):
 
     # Set needed arguments by delegate_hashed_bins().
     public_keys = [public_key]
-    list_of_targets = [os.path.basename(target1_filepath)]
 
     # Delegate to hashed bins.  The target filepath to be tested is expected
     # to contain a hash prefix of 'e', and can be accessed as:
     # repository.targets('e').
-    self.targets_object.delegate_hashed_bins(list_of_targets, public_keys,
+    self.targets_object.delegate_hashed_bins([], public_keys,
                                              number_of_bins=16)
 
     # Ensure each hashed bin initially contains zero targets.
     for delegation in self.targets_object.delegations:
-      self.assertTrue(os.path.basename(target1_filepath) not in delegation.target_files)
+      self.assertEqual(delegation.target_files, {})
 
     # Add 'target1_filepath' and verify that the relative path of
     # 'target1_filepath' is added to the correct bin.
-    self.targets_object.add_target_to_bin(os.path.basename(target1_filepath))
+    self.targets_object.add_target_to_bin(os.path.basename(target1_filepath), 16)
 
     for delegation in self.targets_object.delegations:
       if delegation.rolename == '5':
         self.assertTrue('file1.txt' in delegation.target_files)
-
+        self.assertTrue(len(delegation.target_files) == 1)
       else:
         self.assertTrue('file1.txt' not in delegation.target_files)
 
     # Test the remove_target_from_bin() method.  Verify that 'target1_filepath'
     # has been removed.
-    self.targets_object.remove_target_from_bin(os.path.basename(target1_filepath))
+    self.targets_object.remove_target_from_bin(os.path.basename(target1_filepath), 16)
 
     for delegation in self.targets_object.delegations:
-      if delegation.rolename == 'e':
-        self.assertTrue('file1.txt' not in delegation.target_files)
-
-      else:
-        self.assertTrue('file1.txt' not in delegation.target_files)
+      self.assertTrue('file1.txt' not in delegation.target_files)
 
 
     # Test improperly formatted argument.
     self.assertRaises(securesystemslib.exceptions.FormatError,
-                      self.targets_object.remove_target_from_bin, 3)
+        self.targets_object.remove_target_from_bin, 3, 'foo')
 
     # Invalid target file path argument.
     self.assertRaises(securesystemslib.exceptions.Error,
-        self.targets_object.remove_target_from_bin, '/non-existent')
+        self.targets_object.remove_target_from_bin, '/non-existent', 16)
 
 
 

tests/test_tutorial.py

@@ -358,10 +358,10 @@ def test_tutorial(self):
           targets, [public_unclaimed_key], 32)
 
       self.assertListEqual([
-            "Creating hashed bin delegations.",
-            "1 total targets.",
-            "32 hashed bins.",
-            "256 total hash prefixes.",
+            "Creating hashed bin delegations.\n"
+            "1 total targets.\n"
+            "32 hashed bins.\n"
+            "256 total hash prefixes.\n"
             "Each bin ranges over 8 hash prefixes."
           ] + ["Adding a verification key that has already been used."] * 32,
           [