repository_lib.py

#!/usr/bin/env python

# Copyright 2014 - 2017, New York University and the TUF contributors
# SPDX-License-Identifier: MIT OR Apache-2.0

"""
<Program Name>
  repository_lib.py

<Author>
  Vladimir Diaz <vladimir.v.diaz@gmail.com>

<Started>
  June 1, 2014.

<Copyright>
  See LICENSE-MIT OR LICENSE for licensing information.

<Purpose>
  Provide a library for the repository tool that can create a TUF repository.
  The repository tool can be used with the Python interpreter in interactive
  mode, or imported directly into a Python module.  See 'tuf/README' for the
  complete guide to using 'tuf.repository_tool.py'.
"""

import os
import errno
import time
import logging
import shutil
import json
import tempfile

import securesystemslib # pylint: disable=unused-import
from securesystemslib import exceptions as sslib_exceptions
from securesystemslib import formats as sslib_formats
from securesystemslib import hash as sslib_hash
from securesystemslib import interface as sslib_interface
from securesystemslib import keys as sslib_keys
from securesystemslib import util as sslib_util
from securesystemslib import storage as sslib_storage

from tuf import exceptions
from tuf import formats
from tuf import keydb
from tuf import log
from tuf import roledb
from tuf import settings
from tuf import sig


# See 'log.py' to learn how logging is handled in TUF.
logger = logging.getLogger(__name__)

# The extension of TUF metadata.
METADATA_EXTENSION = '.json'

# The targets and metadata directory names.  Metadata files are written
# to the staged metadata directory instead of the "live" one.
METADATA_STAGED_DIRECTORY_NAME = 'metadata.staged'
METADATA_DIRECTORY_NAME = 'metadata'
TARGETS_DIRECTORY_NAME = 'targets'

# The metadata filenames of the top-level roles.
ROOT_FILENAME = 'root' + METADATA_EXTENSION
TARGETS_FILENAME = 'targets' + METADATA_EXTENSION
SNAPSHOT_FILENAME = 'snapshot' + METADATA_EXTENSION
TIMESTAMP_FILENAME = 'timestamp' + METADATA_EXTENSION

# Log warning when metadata expires in n days, or less.
# root = 1 month, snapshot = 1 day, targets = 10 days, timestamp = 1 day.
ROOT_EXPIRES_WARN_SECONDS = 2630000
SNAPSHOT_EXPIRES_WARN_SECONDS = 86400
TARGETS_EXPIRES_WARN_SECONDS = 864000
TIMESTAMP_EXPIRES_WARN_SECONDS = 86400

# Supported key types.
SUPPORTED_KEY_TYPES = ['rsa', 'ed25519', 'ecdsa', 'ecdsa-sha2-nistp256']

# The algorithm used by the repository to generate the path hash prefixes
# of hashed bin delegations.  Please see delegate_hashed_bins()
HASH_FUNCTION = settings.DEFAULT_HASH_ALGORITHM




def _generate_and_write_metadata(rolename, metadata_filename,
  targets_directory, metadata_directory, storage_backend,
  consistent_snapshot=False, filenames=None, allow_partially_signed=False,
  increment_version_number=True, repository_name='default',
  use_existing_fileinfo=False, use_timestamp_length=True,
  use_timestamp_hashes=True, use_snapshot_length=False,
  use_snapshot_hashes=False):
  """
  Non-public function that can generate and write the metadata for the
  specified 'rolename'.  It also increments the version number of 'rolename' if
  the 'increment_version_number' argument is True.
  """

  metadata = None

  # Retrieve the roleinfo of 'rolename' to extract the needed metadata
  # attributes, such as version number, expiration, etc.
  roleinfo = roledb.get_roleinfo(rolename, repository_name)
  previous_keyids = roleinfo.get('previous_keyids', [])
  previous_threshold = roleinfo.get('previous_threshold', 1)
  signing_keyids = sorted(set(roleinfo['signing_keyids']))

  # Generate the appropriate role metadata for 'rolename'.
  if rolename == 'root':
    metadata = generate_root_metadata(roleinfo['version'], roleinfo['expires'],
        consistent_snapshot, repository_name)

    _log_warning_if_expires_soon(ROOT_FILENAME, roleinfo['expires'],
                                 ROOT_EXPIRES_WARN_SECONDS)



  elif rolename == 'snapshot':
    metadata = generate_snapshot_metadata(metadata_directory,
        roleinfo['version'], roleinfo['expires'],
        storage_backend, consistent_snapshot, repository_name,
        use_length=use_snapshot_length, use_hashes=use_snapshot_hashes)


    _log_warning_if_expires_soon(SNAPSHOT_FILENAME, roleinfo['expires'],
        SNAPSHOT_EXPIRES_WARN_SECONDS)

  elif rolename == 'timestamp':
    # If filenames don't have "snapshot_filename" key, defaults to "snapshot.json"
    snapshot_file_path = (filenames and filenames['snapshot']) \
        or SNAPSHOT_FILENAME

    metadata = generate_timestamp_metadata(snapshot_file_path, roleinfo['version'],
        roleinfo['expires'], storage_backend, repository_name,
        use_length=use_timestamp_length, use_hashes=use_timestamp_hashes)

    _log_warning_if_expires_soon(TIMESTAMP_FILENAME, roleinfo['expires'],
        TIMESTAMP_EXPIRES_WARN_SECONDS)

  # All other roles are either the top-level 'targets' role, or
  # a delegated role.
  else:
    # Only print a warning if the top-level 'targets' role expires soon.
    if rolename == 'targets':
      _log_warning_if_expires_soon(TARGETS_FILENAME, roleinfo['expires'],
          TARGETS_EXPIRES_WARN_SECONDS)

    # Don't hash-prefix consistent target files if they are handled out of band
    consistent_targets = consistent_snapshot and not use_existing_fileinfo

    metadata = generate_targets_metadata(targets_directory,
        roleinfo['paths'], roleinfo['version'], roleinfo['expires'],
        roleinfo['delegations'], consistent_targets, use_existing_fileinfo,
        storage_backend, repository_name)

    # Update roledb with the latest delegations info collected during
    # generate_targets_metadata()
    roledb.update_roleinfo(rolename, roleinfo,
        repository_name=repository_name)


  # Before writing 'rolename' to disk, automatically increment its version
  # number (if 'increment_version_number' is True) so that the caller does not
  # have to manually perform this action.  The version number should be
  # incremented in both the metadata file and roledb (required so that Snapshot
  # references the latest version).

  # Store the 'current_version' in case the version number must be restored
  # (e.g., if 'rolename' cannot be written to disk because its metadata is not
  # properly signed).
  current_version = metadata['version']
  if increment_version_number:
    roleinfo = roledb.get_roleinfo(rolename, repository_name)
    metadata['version'] = metadata['version'] + 1
    roleinfo['version'] = roleinfo['version'] + 1
    roledb.update_roleinfo(rolename, roleinfo,
        repository_name=repository_name)

  else:
    logger.debug('Not incrementing ' + repr(rolename) + '\'s version number.')

  if rolename in roledb.TOP_LEVEL_ROLES and not allow_partially_signed:
    # Verify that the top-level 'rolename' is fully signed.  Only a delegated
    # role should not be written to disk without full verification of its
    # signature(s), since it can only be considered fully signed depending on
    # the delegating role.
    signable = sign_metadata(metadata, signing_keyids, metadata_filename,
        repository_name)


    def should_write():
      # Root must be signed by its previous keys and threshold.
      if rolename == 'root' and len(previous_keyids) > 0:
        if not sig.verify(signable, rolename, repository_name,
            previous_threshold, previous_keyids):
          return False

        else:
          logger.debug('Root is signed by a threshold of its previous keyids.')

      # In the normal case, we should write metadata if the threshold is met.
      return sig.verify(signable, rolename, repository_name,
          roleinfo['threshold'], roleinfo['signing_keyids'])


    if should_write():
      _remove_invalid_and_duplicate_signatures(signable, repository_name)

      # Root should always be written as if consistent_snapshot is True (i.e.,
      # write <version>.root.json and root.json to disk).
      if rolename == 'root':
        consistent_snapshot = True
      filename = write_metadata_file(signable, metadata_filename,
          metadata['version'], consistent_snapshot, storage_backend)

    # 'signable' contains an invalid threshold of signatures.
    else:
      # Since new metadata cannot be successfully written, restore the current
      # version number.
      roleinfo = roledb.get_roleinfo(rolename, repository_name)
      roleinfo['version'] = current_version
      roledb.update_roleinfo(rolename, roleinfo,
          repository_name=repository_name)

      # Note that 'signable' is an argument to tuf.UnsignedMetadataError().
      raise exceptions.UnsignedMetadataError('Not enough'
          ' signatures for ' + repr(metadata_filename), signable)

  # 'rolename' is a delegated role or a top-level role that is partially
  # signed, and thus its signatures should not be verified.
  else:
    signable = sign_metadata(metadata, signing_keyids, metadata_filename,
        repository_name)
    _remove_invalid_and_duplicate_signatures(signable, repository_name)

    # Root should always be written as if consistent_snapshot is True (i.e.,
    # <version>.root.json and root.json).
    if rolename == 'root':
      filename = write_metadata_file(signable, metadata_filename,
          metadata['version'], consistent_snapshot=True,
          storage_backend=storage_backend)

    else:
      filename = write_metadata_file(signable, metadata_filename,
          metadata['version'], consistent_snapshot, storage_backend)

  return signable, filename





def _metadata_is_partially_loaded(rolename, signable, repository_name):
  """
  Non-public function that determines whether 'rolename' is loaded with
  at least zero good signatures, but an insufficient threshold (which means
  'rolename' was written to disk with repository.write_partial()).  A repository
  maintainer may write partial metadata without including a valid signature.
  However, the final repository.write() must include a threshold number of
  signatures.

  If 'rolename' is found to be partially loaded, mark it as partially loaded in
  its 'roledb' roleinfo.  This function exists to assist in deciding whether
  a role's version number should be incremented when write() or write_parital()
  is called.  Return True if 'rolename' was partially loaded, False otherwise.
  """

  # The signature status lists the number of good signatures, including
  # bad, untrusted, unknown, etc.
  status = sig.get_signature_status(signable, rolename, repository_name)

  if len(status['good_sigs']) < status['threshold'] and \
                                                  len(status['good_sigs']) >= 0:
    return True

  else:
    return False





def _check_role_keys(rolename, repository_name):
  """
  Non-public function that verifies the public and signing keys of 'rolename'.
  If either contain an invalid threshold of keys, raise an exception.
  """

  # Extract the total number of public and private keys of 'rolename' from its
  # roleinfo in 'roledb'.
  roleinfo = roledb.get_roleinfo(rolename, repository_name)
  total_keyids = len(roleinfo['keyids'])
  threshold = roleinfo['threshold']
  total_signatures = len(roleinfo['signatures'])
  total_signing_keys = len(roleinfo['signing_keyids'])

  # Raise an exception for an invalid threshold of public keys.
  if total_keyids < threshold:
    raise exceptions.InsufficientKeysError(repr(rolename) + ' role contains'
      ' ' + repr(total_keyids) + ' / ' + repr(threshold) + ' public keys.')

  # Raise an exception for an invalid threshold of signing keys.
  if total_signatures == 0 and total_signing_keys < threshold:
    raise exceptions.InsufficientKeysError(repr(rolename) + ' role contains'
      ' ' + repr(total_signing_keys) + ' / ' + repr(threshold) + ' signing keys.')





def _remove_invalid_and_duplicate_signatures(signable, repository_name):
  """
    Non-public function that removes invalid or duplicate signatures from
    'signable'.  'signable' may contain signatures (invalid) from previous
    versions of the metadata that were loaded with load_repository().  Invalid,
    or duplicate signatures, are removed from 'signable'.
  """

  # Store the keyids of valid signatures.  'signature_keyids' is checked for
  # duplicates rather than comparing signature objects because PSS may generate
  # duplicate valid signatures for the same data, yet contain different
  # signatures.
  signature_keyids = []

  for signature in signable['signatures']:
    signed = sslib_formats.encode_canonical(signable['signed']).encode('utf-8')
    keyid = signature['keyid']
    key = None

    # Remove 'signature' from 'signable' if the listed keyid does not exist
    # in 'keydb'.
    try:
      key = keydb.get_key(keyid, repository_name=repository_name)

    except exceptions.UnknownKeyError:
      signable['signatures'].remove(signature)
      continue

    # Remove 'signature' from 'signable' if it is an invalid signature.
    if not sslib_keys.verify_signature(key, signature, signed):
      logger.debug('Removing invalid signature for ' + repr(keyid))
      signable['signatures'].remove(signature)

    # Although valid, it may still need removal if it is a duplicate.  Check
    # the keyid, rather than the signature, to remove duplicate PSS signatures.
    # PSS may generate multiple different signatures for the same keyid.
    else:
      if keyid in signature_keyids:
        signable['signatures'].remove(signature)

      # 'keyid' is valid and not a duplicate, so add it to 'signature_keyids'.
      else:
        signature_keyids.append(keyid)





def _delete_obsolete_metadata(metadata_directory, snapshot_metadata,
    consistent_snapshot, repository_name, storage_backend):
  """
  Non-public function that deletes metadata files marked as removed by
  'repository_tool.py'.  Revoked metadata files are not actually deleted until
  this function is called.  Obsolete metadata should *not* be retained in
  "metadata.staged", otherwise they may be re-loaded by 'load_repository()'.

  Note: Obsolete metadata may not always be easily detected (by inspecting
  top-level metadata during loading) due to partial metadata and top-level
  metadata that have not been written yet.
  """

  # Walk the repository's metadata sub-directory, which is where all metadata
  # is stored (including delegated roles).  The 'django.json' role (e.g.,
  # delegated by Targets) would be located in the
  # '{repository_directory}/metadata/' directory.
  metadata_files = sorted(storage_backend.list_folder(metadata_directory))
  for metadata_role in metadata_files:
    if metadata_role.endswith('root.json'):
      continue

    metadata_path = os.path.join(metadata_directory, metadata_role)

    # Strip the version number if 'consistent_snapshot' is True.  Example:
    # '10.django.json' --> 'django.json'.  Consistent and non-consistent
    # metadata might co-exist if write() and
    # write(consistent_snapshot=True) are mixed, so ensure only
    # '<version_number>.filename' metadata is stripped.

    # Should we check if 'consistent_snapshot' is True? It might have been
    # set previously, but 'consistent_snapshot' can potentially be False
    # now.  We'll proceed with the understanding that 'metadata_name' can
    # have a prepended version number even though the repository is now
    # a non-consistent one.
    if metadata_role not in snapshot_metadata['meta']:
      metadata_role, junk = _strip_version_number(metadata_role,
          consistent_snapshot)

    else:
      logger.debug(repr(metadata_role) + ' found in the snapshot role.')

    # Strip metadata extension from filename.  The role database does not
    # include the metadata extension.
    if metadata_role.endswith(METADATA_EXTENSION):
      metadata_role = metadata_role[:-len(METADATA_EXTENSION)]
    else:
      logger.debug(repr(metadata_role) + ' does not match'
          ' supported extension ' + repr(METADATA_EXTENSION))

    if metadata_role in roledb.TOP_LEVEL_ROLES:
      logger.debug('Not removing top-level metadata ' + repr(metadata_role))
      return

    # Delete the metadata file if it does not exist in 'roledb'.
    # 'repository_tool.py' might have removed 'metadata_name,'
    # but its metadata file is not actually deleted yet.  Do it now.
    if not roledb.role_exists(metadata_role, repository_name):
      logger.info('Removing outdated metadata: ' + repr(metadata_path))
      storage_backend.remove(metadata_path)

    else:
      logger.debug('Not removing metadata: ' + repr(metadata_path))

      # TODO: Should we delete outdated consistent snapshots, or does it make
      # more sense for integrators to remove outdated consistent snapshots?




def _get_written_metadata(metadata_signable):
  """
  Non-public function that returns the actual content of written metadata.
  """

  # Explicitly specify the JSON separators for Python 2 + 3 consistency.
  written_metadata_content = json.dumps(metadata_signable, indent=1,
      separators=(',', ': '), sort_keys=True).encode('utf-8')

  return written_metadata_content





def _strip_version_number(metadata_filename, consistent_snapshot):
  """
  Strip from 'metadata_filename' any version number (in the
  expected '{dirname}/<version_number>.rolename.<ext>' format) that
  it may contain, and return the stripped filename and version number,
  as a tuple.  'consistent_snapshot' is a boolean indicating if a version
  number is prepended to 'metadata_filename'.
  """

  # Strip the version number if 'consistent_snapshot' is True.
  # Example: '10.django.json' --> 'django.json'
  if consistent_snapshot:
    dirname, basename = os.path.split(metadata_filename)
    version_number, basename = basename.split('.', 1)
    stripped_metadata_filename = os.path.join(dirname, basename)

    if not version_number.isdigit():
      return metadata_filename, ''

    else:
      return stripped_metadata_filename, version_number

  else:
    return metadata_filename, ''




def _load_top_level_metadata(repository, top_level_filenames, repository_name):
  """
  Load the metadata of the Root, Timestamp, Targets, and Snapshot roles.  At a
  minimum, the Root role must exist and load successfully.
  """

  root_filename = top_level_filenames[ROOT_FILENAME]
  targets_filename = top_level_filenames[TARGETS_FILENAME]
  snapshot_filename = top_level_filenames[SNAPSHOT_FILENAME]
  timestamp_filename = top_level_filenames[TIMESTAMP_FILENAME]

  root_metadata = None
  targets_metadata = None
  snapshot_metadata = None
  timestamp_metadata = None

  # Load 'root.json'.  A Root role file without a version number is always
  # written.
  try:
    # Initialize the key and role metadata of the top-level roles.
    signable = sslib_util.load_json_file(root_filename)
    try:
      formats.check_signable_object_format(signable)
    except exceptions.UnsignedMetadataError:
      # Downgrade the error to a warning because a use case exists where
      # metadata may be generated unsigned on one machine and signed on another.
      logger.warning('Unsigned metadata object: ' + repr(signable))

    root_metadata = signable['signed']
    keydb.create_keydb_from_root_metadata(root_metadata, repository_name)
    roledb.create_roledb_from_root_metadata(root_metadata, repository_name)

    # Load Root's roleinfo and update 'roledb'.
    roleinfo = roledb.get_roleinfo('root', repository_name)
    roleinfo['consistent_snapshot'] = root_metadata['consistent_snapshot']
    roleinfo['signatures'] = []
    for signature in signable['signatures']:
      if signature not in roleinfo['signatures']:
        roleinfo['signatures'].append(signature)

      else:
        logger.debug('Found a Root signature that is already loaded:'
          ' ' + repr(signature))

    # By default, roleinfo['partial_loaded'] of top-level roles should be set
    # to False in 'create_roledb_from_root_metadata()'.  Update this field, if
    # necessary, now that we have its signable object.
    if _metadata_is_partially_loaded('root', signable, repository_name):
      roleinfo['partial_loaded'] = True

    else:
      logger.debug('Root was not partially loaded.')

    _log_warning_if_expires_soon(ROOT_FILENAME, roleinfo['expires'],
                                 ROOT_EXPIRES_WARN_SECONDS)

    roledb.update_roleinfo('root', roleinfo, mark_role_as_dirty=False,
        repository_name=repository_name)

    # Ensure the 'consistent_snapshot' field is extracted.
    consistent_snapshot = root_metadata['consistent_snapshot']

  except sslib_exceptions.StorageError as error:
    raise exceptions.RepositoryError('Cannot load the required'
        ' root file: ' + repr(root_filename)) from error

  # Load 'timestamp.json'.  A Timestamp role file without a version number is
  # always written.
  try:
    signable = sslib_util.load_json_file(timestamp_filename)
    timestamp_metadata = signable['signed']
    for signature in signable['signatures']:
      repository.timestamp.add_signature(signature, mark_role_as_dirty=False)

    # Load Timestamp's roleinfo and update 'roledb'.
    roleinfo = roledb.get_roleinfo('timestamp', repository_name)
    roleinfo['expires'] = timestamp_metadata['expires']
    roleinfo['version'] = timestamp_metadata['version']

    if _metadata_is_partially_loaded('timestamp', signable, repository_name):
      roleinfo['partial_loaded'] = True

    else:
      logger.debug('The Timestamp role was not partially loaded.')

    _log_warning_if_expires_soon(TIMESTAMP_FILENAME, roleinfo['expires'],
                                 TIMESTAMP_EXPIRES_WARN_SECONDS)

    roledb.update_roleinfo('timestamp', roleinfo, mark_role_as_dirty=False,
        repository_name=repository_name)

  except sslib_exceptions.StorageError as error:
    raise exceptions.RepositoryError('Cannot load the Timestamp '
        'file: ' + repr(timestamp_filename)) from error

  # Load 'snapshot.json'.  A consistent snapshot.json must be calculated if
  # 'consistent_snapshot' is True.
  # The Snapshot and Root roles are both accessed by their hashes.
  if consistent_snapshot:
    snapshot_version = timestamp_metadata['meta'][SNAPSHOT_FILENAME]['version']

    dirname, basename = os.path.split(snapshot_filename)
    basename = basename.split(METADATA_EXTENSION, 1)[0]
    snapshot_filename = os.path.join(dirname,
        str(snapshot_version) + '.' + basename + METADATA_EXTENSION)

  try:
    signable = sslib_util.load_json_file(snapshot_filename)
    try:
      formats.check_signable_object_format(signable)
    except exceptions.UnsignedMetadataError:
      # Downgrade the error to a warning because a use case exists where
      # metadata may be generated unsigned on one machine and signed on another.
      logger.warning('Unsigned metadata object: ' + repr(signable))

    snapshot_metadata = signable['signed']

    for signature in signable['signatures']:
      repository.snapshot.add_signature(signature, mark_role_as_dirty=False)

    # Load Snapshot's roleinfo and update 'roledb'.
    roleinfo = roledb.get_roleinfo('snapshot', repository_name)
    roleinfo['expires'] = snapshot_metadata['expires']
    roleinfo['version'] = snapshot_metadata['version']

    if _metadata_is_partially_loaded('snapshot', signable, repository_name):
      roleinfo['partial_loaded'] = True

    else:
      logger.debug('Snapshot was not partially loaded.')

    _log_warning_if_expires_soon(SNAPSHOT_FILENAME, roleinfo['expires'],
                                 SNAPSHOT_EXPIRES_WARN_SECONDS)

    roledb.update_roleinfo('snapshot', roleinfo, mark_role_as_dirty=False,
        repository_name=repository_name)

  except sslib_exceptions.StorageError as error:
    raise exceptions.RepositoryError('The Snapshot file '
        'cannot be loaded: '+ repr(snapshot_filename)) from error

  # Load 'targets.json'.  A consistent snapshot of the Targets role must be
  # calculated if 'consistent_snapshot' is True.
  if consistent_snapshot:
    targets_version = snapshot_metadata['meta'][TARGETS_FILENAME]['version']
    dirname, basename = os.path.split(targets_filename)
    targets_filename = os.path.join(dirname, str(targets_version) + '.' + basename)

  try:
    signable = sslib_util.load_json_file(targets_filename)
    try:
      formats.check_signable_object_format(signable)
    except exceptions.UnsignedMetadataError:
      # Downgrade the error to a warning because a use case exists where
      # metadata may be generated unsigned on one machine and signed on another.
      logger.warning('Unsigned metadata object: ' + repr(signable))

    targets_metadata = signable['signed']

    for signature in signable['signatures']:
      repository.targets.add_signature(signature, mark_role_as_dirty=False)

    # Update 'targets.json' in 'roledb'
    roleinfo = roledb.get_roleinfo('targets', repository_name)
    roleinfo['paths'] = targets_metadata['targets']
    roleinfo['version'] = targets_metadata['version']
    roleinfo['expires'] = targets_metadata['expires']
    roleinfo['delegations'] = targets_metadata['delegations']

    if _metadata_is_partially_loaded('targets', signable, repository_name):
      roleinfo['partial_loaded'] = True

    else:
      logger.debug('Targets file was not partially loaded.')

    _log_warning_if_expires_soon(TARGETS_FILENAME, roleinfo['expires'],
                                 TARGETS_EXPIRES_WARN_SECONDS)

    roledb.update_roleinfo('targets', roleinfo, mark_role_as_dirty=False,
        repository_name=repository_name)

    # Add the keys specified in the delegations field of the Targets role.
    for keyid, key_metadata in targets_metadata['delegations']['keys'].items():

      # Use the keyid found in the delegation
      key_object, _ = sslib_keys.format_metadata_to_key(key_metadata,
          keyid)

      # Add 'key_object' to the list of recognized keys.  Keys may be shared,
      # so do not raise an exception if 'key_object' has already been loaded.
      # In contrast to the methods that may add duplicate keys, do not log
      # a warning as there may be many such duplicate key warnings.  The
      # repository maintainer should have also been made aware of the duplicate
      # key when it was added.
      try:
        keydb.add_key(key_object, keyid=None, repository_name=repository_name)

      except exceptions.KeyAlreadyExistsError:
        pass

  except sslib_exceptions.StorageError as error:
    raise exceptions.RepositoryError('The Targets file '
        'can not be loaded: ' + repr(targets_filename)) from error

  return repository, consistent_snapshot




def _log_warning_if_expires_soon(rolename, expires_iso8601_timestamp,
    seconds_remaining_to_warn):
  """
  Non-public function that logs a warning if 'rolename' expires in
  'seconds_remaining_to_warn' seconds, or less.
  """

  # Metadata stores expiration datetimes in ISO8601 format.  Convert to
  # unix timestamp, subtract from current time.time() (also in POSIX time)
  # and compare against 'seconds_remaining_to_warn'.  Log a warning message
  # to console if 'rolename' expires soon.
  datetime_object = formats.expiry_string_to_datetime(
      expires_iso8601_timestamp)
  expires_unix_timestamp = \
    formats.datetime_to_unix_timestamp(datetime_object)
  seconds_until_expires = expires_unix_timestamp - int(time.time())

  if seconds_until_expires <= seconds_remaining_to_warn:
    if seconds_until_expires <= 0:
      logger.warning(
          repr(rolename) + ' expired ' + repr(datetime_object.ctime() + ' (UTC).'))

    else:
      days_until_expires = seconds_until_expires / 86400
      logger.warning(repr(rolename) + ' expires ' + datetime_object.ctime() + ''
        ' (UTC).  ' + repr(days_until_expires) + ' day(s) until it expires.')

  else:
    pass





def import_rsa_privatekey_from_file(filepath, password=None):
  """
  <Purpose>
    Import the encrypted PEM file in 'filepath', decrypt it, and return the key
    object in 'securesystemslib.RSAKEY_SCHEMA' format.

  <Arguments>
    filepath:
      <filepath> file, an RSA encrypted PEM file.  Unlike the public RSA PEM
      key file, 'filepath' does not have an extension.

    password:
      The passphrase to decrypt 'filepath'.

  <Exceptions>
    securesystemslib.exceptions.FormatError, if the arguments are improperly
    formatted.

    securesystemslib.exceptions.CryptoError, if 'filepath' is not a valid
    encrypted key file.

  <Side Effects>
    The contents of 'filepath' is read, decrypted, and the key stored.

  <Returns>
    An RSA key object, conformant to 'securesystemslib.RSAKEY_SCHEMA'.
  """

  # Note: securesystemslib.interface.import_rsa_privatekey_from_file() does not
  # allow both 'password' and 'prompt' to be True, nor does it automatically
  # prompt for a password if the key file is encrypted and a password isn't
  # given.
  try:
    private_key = sslib_interface.import_rsa_privatekey_from_file(
        filepath, password)

  # The user might not have given a password for an encrypted private key.
  # Prompt for a password for convenience.
  except sslib_exceptions.CryptoError:
    if password is None:
      private_key = sslib_interface.import_rsa_privatekey_from_file(
          filepath, password, prompt=True)

    else:
      raise

  return private_key







def import_ed25519_privatekey_from_file(filepath, password=None):
  """
  <Purpose>
    Import the encrypted ed25519 TUF key file in 'filepath', decrypt it, and
    return the key object in 'securesystemslib.ED25519KEY_SCHEMA' format.

    The TUF private key (may also contain the public part) is encrypted with
    AES 256 and CTR the mode of operation.  The password is strengthened with
    PBKDF2-HMAC-SHA256.

  <Arguments>
    filepath:
      <filepath> file, an RSA encrypted TUF key file.

    password:
      The password, or passphrase, to import the private key (i.e., the
      encrypted key file 'filepath' must be decrypted before the ed25519 key
      object can be returned.

  <Exceptions>
    securesystemslib.exceptions.FormatError, if the arguments are improperly
    formatted or the imported key object contains an invalid key type (i.e.,
    not 'ed25519').

    securesystemslib.exceptions.CryptoError, if 'filepath' cannot be decrypted.

    securesystemslib.exceptions.UnsupportedLibraryError, if 'filepath' cannot be
    decrypted due to an invalid configuration setting (i.e., invalid
    'tuf.settings' setting).

  <Side Effects>
    'password' is used to decrypt the 'filepath' key file.

  <Returns>
    An ed25519 key object of the form: 'securesystemslib.ED25519KEY_SCHEMA'.
  """

  # Note: securesystemslib.interface.import_ed25519_privatekey_from_file() does
  # not allow both 'password' and 'prompt' to be True, nor does it
  # automatically prompt for a password if the key file is encrypted and a
  # password isn't given.
  try:
    private_key = sslib_interface.import_ed25519_privatekey_from_file(
        filepath, password)

  # The user might not have given a password for an encrypted private key.
  # Prompt for a password for convenience.
  except sslib_exceptions.CryptoError:
    if password is None:
      private_key = sslib_interface.import_ed25519_privatekey_from_file(
          filepath, password, prompt=True)

    else:
      raise

  return private_key



def get_delegated_roles_metadata_filenames(metadata_directory,
    consistent_snapshot, storage_backend=None):
  """
  Return a dictionary containing all filenames in 'metadata_directory'
  except the top-level roles.
  If multiple versions of a file exist because of a consistent snapshot,
  only the file with biggest version prefix is included.
  """

  filenames = {}
  metadata_files = sorted(storage_backend.list_folder(metadata_directory),
      reverse=True)

  # Iterate over role metadata files, sorted by their version-number prefix, with
  # more recent versions first, and only add the most recent version of any
  # (non top-level) metadata to the list of returned filenames. Note that there
  # should only be one version of each file, if consistent_snapshot is False.
  for metadata_role in metadata_files:
    metadata_path = os.path.join(metadata_directory, metadata_role)

    # Strip the version number if 'consistent_snapshot' is True,
    # or if 'metadata_role' is Root.
    # Example:  '10.django.json' --> 'django.json'
    consistent = \
      metadata_role.endswith('root.json') or consistent_snapshot == True
    metadata_name, junk = _strip_version_number(metadata_role,
      consistent)

    if metadata_name.endswith(METADATA_EXTENSION):
      extension_length = len(METADATA_EXTENSION)
      metadata_name = metadata_name[:-extension_length]

    else:
      logger.debug('Skipping file with unsupported metadata'
          ' extension: ' + repr(metadata_path))
      continue

    # Skip top-level roles, only interested in delegated roles.
    if metadata_name in roledb.TOP_LEVEL_ROLES:
      continue

    # Prevent reloading duplicate versions if consistent_snapshot is True
    if metadata_name not in filenames:
      filenames[metadata_name] = metadata_path

  return filenames



def get_top_level_metadata_filenames(metadata_directory):
  """
  <Purpose>
    Return a dictionary containing the filenames of the top-level roles.
    If 'metadata_directory' is set to 'metadata', the dictionary
    returned would contain:

    filenames = {'root.json': 'metadata/root.json',
                 'targets.json': 'metadata/targets.json',
                 'snapshot.json': 'metadata/snapshot.json',
                 'timestamp.json': 'metadata/timestamp.json'}

    If 'metadata_directory' is not set by the caller, the current directory is
    used.

  <Arguments>
    metadata_directory:
      The directory containing the metadata files.

  <Exceptions>
    securesystemslib.exceptions.FormatError, if 'metadata_directory' is
    improperly formatted.

  <Side Effects>
    None.

  <Returns>
    A dictionary containing the expected filenames of the top-level
    metadata files, such as 'root.json' and 'snapshot.json'.
  """

  # Does 'metadata_directory' have the correct format?
  # Ensure the arguments have the appropriate number of objects and object
  # types, and that all dict keys are properly named.
  # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch.
  sslib_formats.PATH_SCHEMA.check_match(metadata_directory)

  # Store the filepaths of the top-level roles, including the
  # 'metadata_directory' for each one.
  filenames = {}

  filenames[ROOT_FILENAME] = \
    os.path.join(metadata_directory, ROOT_FILENAME)

  filenames[TARGETS_FILENAME] = \
    os.path.join(metadata_directory, TARGETS_FILENAME)

  filenames[SNAPSHOT_FILENAME] = \
    os.path.join(metadata_directory, SNAPSHOT_FILENAME)

  filenames[TIMESTAMP_FILENAME] = \
    os.path.join(metadata_directory, TIMESTAMP_FILENAME)

  return filenames





def get_targets_metadata_fileinfo(filename, storage_backend, custom=None):
  """
  <Purpose>
    Retrieve the file information of 'filename'.  The object returned
    conforms to 'tuf.formats.TARGETS_FILEINFO_SCHEMA'.  The information
    generated for 'filename' is stored in metadata files like 'targets.json'.
    The fileinfo object returned has the form:

    fileinfo = {'length': 1024,
                'hashes': {'sha256': 1233dfba312, ...},
                'custom': {...}}

  <Arguments>
    filename:
      The metadata file whose file information is needed.  It must exist.

    custom:
      An optional object providing additional information about the file.

    storage_backend:
      An object which implements
      securesystemslib.storage.StorageBackendInterface.

  <Exceptions>
    securesystemslib.exceptions.FormatError, if 'filename' is improperly
    formatted.

  <Side Effects>
    The file is opened and information about the file is generated,
    such as file size and its hash.

  <Returns>
    A dictionary conformant to 'tuf.formats.TARGETS_FILEINFO_SCHEMA'.  This
    dictionary contains the length, hashes, and custom data about the
    'filename' metadata file.  SHA256 hashes are generated by default.
  """

  # Does 'filename' and 'custom' have the correct format?
  # Ensure the arguments have the appropriate number of objects and object
  # types, and that all dict keys are properly named.
  # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch.
  sslib_formats.PATH_SCHEMA.check_match(filename)
  if custom is not None:
    formats.CUSTOM_SCHEMA.check_match(custom)

  # Note: 'filehashes' is a dictionary of the form
  # {'sha256': 1233dfba312, ...}.  'custom' is an optional
  # dictionary that a client might define to include additional
  # file information, such as the file's author, version/revision
  # numbers, etc.
  filesize, filehashes = sslib_util.get_file_details(filename,
      settings.FILE_HASH_ALGORITHMS, storage_backend)

  return formats.make_targets_fileinfo(filesize, filehashes, custom=custom)





def get_metadata_versioninfo(rolename, repository_name):
  """
  <Purpose>
    Retrieve the version information of 'rolename'.  The object returned
    conforms to 'tuf.formats.VERSIONINFO_SCHEMA'.  The information
    generated for 'rolename' is stored in 'snapshot.json'.
    The versioninfo object returned has the form:

    versioninfo = {'version': 14}

  <Arguments>
    rolename:
      The metadata role whose versioninfo is needed.  It must exist, otherwise
      a 'tuf.exceptions.UnknownRoleError' exception is raised.

    repository_name:
      The name of the repository.  If not supplied, 'rolename' is added to the
      'default' repository.

  <Exceptions>
    securesystemslib.exceptions.FormatError, if 'rolename' is improperly
    formatted.

    tuf.exceptions.UnknownRoleError, if 'rolename' does not exist.

  <Side Effects>
    None.

  <Returns>
    A dictionary conformant to 'tuf.formats.VERSIONINFO_SCHEMA'.
    This dictionary contains the version  number of 'rolename'.
  """

  # Does 'rolename' have the correct format?
  # Ensure the arguments have the appropriate number of objects and object
  # types, and that all dict keys are properly named.
  formats.ROLENAME_SCHEMA.check_match(rolename)

  roleinfo = roledb.get_roleinfo(rolename, repository_name)
  versioninfo = {'version': roleinfo['version']}

  return versioninfo





def create_bin_name(low, high, prefix_len):
  """
  <Purpose>
    Create a string name of a delegated hash bin, where name will be a range of
    zero-padded (up to prefix_len) strings i.e. for low=00, high=07,
    prefix_len=3 the returned name would be '000-007'.

  <Arguments>
    low:
      The low end of the prefix range to be binned

    high:
      The high end of the prefix range to be binned

    prefix_len:
      The length of the prefix range components

  <Returns>
    A string bin name, with each end of the range zero-padded up to prefix_len
  """
  if low == high:
    return "{low:0{len}x}".format(low=low, len=prefix_len)

  return "{low:0{len}x}-{high:0{len}x}".format(low=low, high=high,
      len=prefix_len)





def get_bin_numbers(number_of_bins):
  """
  <Purpose>
    Given the desired number of bins (number_of_bins) calculate the prefix
    length (prefix_length), total number of prefixes (prefix_count) and the
    number of prefixes to be stored in each bin (bin_size).
    Example: number_of_bins = 32
      prefix_length = 2
      prefix_count = 256
      bin_size = 8
    That is, each of the 32 hashed bins are responsible for 8 hash prefixes,
    i.e. 00-07, 08-0f, ..., f8-ff.

  <Arguments>
    number_of_bins:
      The number of hashed bins in use

  <Returns>
    A tuple of three values:
      1. prefix_length: the length of each prefix
      2. prefix_count: the total number of prefixes in use
      3. bin_size: the number of hash prefixes to be stored in each bin
  """
  # Convert 'number_of_bins' to hexadecimal and determine the number of
  # hexadecimal digits needed by each hash prefix
  prefix_length = len("{:x}".format(number_of_bins - 1))
  # Calculate the total number of hash prefixes (e.g., 000 - FFF total values)
  prefix_count = 16 ** prefix_length
  # Determine how many prefixes to assign to each bin
  bin_size = prefix_count // number_of_bins

  # For simplicity, ensure that 'prefix_count' (16 ^ n) can be evenly
  # distributed over 'number_of_bins' (must be 2 ^ n).  Each bin will contain
  # (prefix_count / number_of_bins) hash prefixes.
  if prefix_count % number_of_bins != 0:
    # Note: x % y != 0 does not guarantee that y is not a power of 2 for
    # arbitrary x and y values. However, due to the relationship between
    # number_of_bins and prefix_count, it is true for them.
    raise sslib_exceptions.Error('The "number_of_bins" argument'
        ' must be a power of 2.')

  return prefix_length, prefix_count, bin_size





def find_bin_for_target_hash(target_hash, number_of_bins):
  """
  <Purpose>
    For a given hashed filename, target_hash, calculate the name of a hashed bin
    into which this file would be delegated given number_of_bins bins are in
    use.

  <Arguments>
    target_hash:
      The hash of the target file's path

    number_of_bins:
      The number of hashed_bins in use

  <Returns>
    The name of the hashed bin target_hash would be binned into
  """

  prefix_length, _, bin_size = get_bin_numbers(number_of_bins)

  prefix = int(target_hash[:prefix_length], 16)

  low = prefix - (prefix % bin_size)
  high = (low + bin_size - 1)

  return create_bin_name(low, high, prefix_length)





def get_target_hash(target_filepath):
  """
  <Purpose>
    Compute the hash of 'target_filepath'. This is useful in conjunction with
    the "path_hash_prefixes" attribute in a delegated targets role, which
    tells us which paths a role is implicitly responsible for.

    The repository may optionally organize targets into hashed bins to ease
    target delegations and role metadata management.  The use of consistent
    hashing allows for a uniform distribution of targets into bins.

  <Arguments>
    target_filepath:
      The path to the target file on the repository. This will be relative to
      the 'targets' (or equivalent) directory on a given mirror.

  <Exceptions>
    None.

  <Side Effects>
    None.

  <Returns>
    The hash of 'target_filepath'.

  """
  formats.RELPATH_SCHEMA.check_match(target_filepath)

  digest_object = sslib_hash.digest(algorithm=HASH_FUNCTION)
  digest_object.update(target_filepath.encode('utf-8'))
  return digest_object.hexdigest()




def generate_root_metadata(version, expiration_date, consistent_snapshot,
  repository_name='default'):
  """
  <Purpose>
    Create the root metadata.  'roledb' and 'keydb'
    are read and the information returned by these modules is used to generate
    the root metadata object.

  <Arguments>
    version:
      The metadata version number.  Clients use the version number to
      determine if the downloaded version is newer than the one currently
      trusted.

    expiration_date:
      The expiration date of the metadata file.  Conformant to
      'securesystemslib.formats.ISO8601_DATETIME_SCHEMA'.

    consistent_snapshot:
      Boolean.  If True, a file digest is expected to be prepended to the
      filename of any target file located in the targets directory.  Each digest
      is stripped from the target filename and listed in the snapshot metadata.

    repository_name:
      The name of the repository.  If not supplied, 'rolename' is added to the
      'default' repository.

  <Exceptions>
    securesystemslib.exceptions.FormatError, if the generated root metadata
    object could not be generated with the correct format.

    securesystemslib.exceptions.Error, if an error is encountered while
    generating the root metadata object (e.g., a required top-level role not
    found in 'roledb'.)

  <Side Effects>
    The contents of 'keydb' and 'roledb' are read.

  <Returns>
    A root metadata object, conformant to 'tuf.formats.ROOT_SCHEMA'.
  """

  # Do the arguments have the correct format?
  # Ensure the arguments have the appropriate number of objects and object
  # types, and that all dict keys are properly named.  Raise
  # 'securesystemslib.exceptions.FormatError' if any of the arguments are
  # improperly formatted.
  formats.METADATAVERSION_SCHEMA.check_match(version)
  sslib_formats.ISO8601_DATETIME_SCHEMA.check_match(expiration_date)
  sslib_formats.BOOLEAN_SCHEMA.check_match(consistent_snapshot)
  sslib_formats.NAME_SCHEMA.check_match(repository_name)

  # The role and key dictionaries to be saved in the root metadata object.
  # Conformant to 'ROLEDICT_SCHEMA' and 'KEYDICT_SCHEMA', respectively.
  roledict = {}
  keydict = {}
  keylist = []

  # Extract the role, threshold, and keyid information of the top-level roles,
  # which Root stores in its metadata.  The necessary role metadata is generated
  # from this information.
  for rolename in roledb.TOP_LEVEL_ROLES:

    # If a top-level role is missing from 'roledb', raise an exception.
    if not roledb.role_exists(rolename, repository_name):
      raise sslib_exceptions.Error(repr(rolename) + ' not in'
          ' "roledb".')

    # Collect keys from all roles in a list
    keyids = roledb.get_role_keyids(rolename, repository_name)
    for keyid in keyids:
      key = keydb.get_key(keyid, repository_name=repository_name)
      keylist.append(key)

    # Generate the authentication information Root establishes for each
    # top-level role.
    role_threshold = roledb.get_role_threshold(rolename, repository_name)
    role_metadata = formats.build_dict_conforming_to_schema(
        formats.ROLE_SCHEMA,
        keyids=keyids,
        threshold=role_threshold)
    roledict[rolename] = role_metadata

  # Create the root metadata 'keys' dictionary
  _, keydict = keys_to_keydict(keylist)

  # Use generalized build_dict_conforming_to_schema func to produce a dict that
  # contains all the appropriate information for this type of metadata,
  # checking that the result conforms to the appropriate schema.
  # TODO: Later, probably after the rewrite for TUF Issue #660, generalize
  #       further, upward, by replacing generate_targets_metadata,
  #       generate_root_metadata, etc. with one function that generates
  #       metadata, possibly rolling that upwards into the calling function.
  #       There are very few things that really need to be done differently.
  return formats.build_dict_conforming_to_schema(
      formats.ROOT_SCHEMA,
      version=version,
      expires=expiration_date,
      keys=keydict,
      roles=roledict,
      consistent_snapshot=consistent_snapshot)





def generate_targets_metadata(targets_directory, target_files, version,
    expiration_date, delegations=None, write_consistent_targets=False,
    use_existing_fileinfo=False, storage_backend=None,
    repository_name='default'):
  """
  <Purpose>
    Generate the targets metadata object. The targets in 'target_files' must
    exist at the same path they should on the repo.  'target_files' is a list
    of targets.  The 'custom' field of the targets metadata is not currently
    supported.

  <Arguments>
    targets_directory:
      The absolute path to a directory containing the target files and
      directories of the repository.

    target_files:
      The target files tracked by 'targets.json'.  'target_files' is a
      dictionary mapping target paths (relative to the targets directory) to
      a dict matching tuf.formats.LOOSE_FILEINFO_SCHEMA.  LOOSE_FILEINFO_SCHEMA
      can support multiple different value patterns:
      1) an empty dictionary - for when fileinfo should be generated
      2) a dictionary matching tuf.formats.CUSTOM_SCHEMA - for when fileinfo
         should be generated, with the supplied custom metadata attached
      3) a dictionary matching tuf.formats.FILEINFO_SCHEMA - for when full
         fileinfo is provided in conjunction with use_existing_fileinfo

    version:
      The metadata version number.  Clients use the version number to
      determine if the downloaded version is newer than the one currently
      trusted.

    expiration_date:
      The expiration date of the metadata file.  Conformant to
      'securesystemslib.formats.ISO8601_DATETIME_SCHEMA'.

    delegations:
      The delegations made by the targets role to be generated.  'delegations'
      must match 'tuf.formats.DELEGATIONS_SCHEMA'.

    write_consistent_targets:
      Boolean that indicates whether file digests should be prepended to the
      target files.
      NOTE: it is an error for write_consistent_targets to be True when
      use_existing_fileinfo is also True. We can not create consistent targets
      for a target file where the fileinfo isn't generated by tuf.

    use_existing_fileinfo:
      Boolean that indicates whether to use the complete fileinfo, including
      hashes, as already exists in the roledb (True) or whether to generate
      hashes (False).

    storage_backend:
      An object which implements
      securesystemslib.storage.StorageBackendInterface.

    repository_name:
      The name of the repository.  If not supplied, 'default' repository
      is used.

  <Exceptions>
    securesystemslib.exceptions.FormatError, if an error occurred trying to
    generate the targets metadata object.

    securesystemslib.exceptions.Error, if use_existing_fileinfo is False and
    any of the target files cannot be read.

    securesystemslib.exceptions.Error, if use_existing_fileinfo is True and
    some of the target files do not have corresponding hashes in the roledb.

    securesystemslib.exceptions.Error, if both of use_existing_fileinfo and
    write_consistent_targets are True.

  <Side Effects>
    If use_existing_fileinfo is False, the target files are read from storage
    and file information about them is generated.
    If 'write_consistent_targets' is True, each target in 'target_files' will be
    copied to a file with a digest prepended to its filename. For example, if
    'some_file.txt' is one of the targets of 'target_files', consistent targets
    <sha-2 hash>.some_file.txt, <sha-3 hash>.some_file.txt, etc., are created
    and the content of 'some_file.txt' will be copied into them.

  <Returns>
    A targets metadata object, conformant to
    'tuf.formats.TARGETS_SCHEMA'.
  """

  # Do the arguments have the correct format?
  # Ensure the arguments have the appropriate number of objects and object
  # types, and that all dict keys are properly named.
  # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch.
  sslib_formats.PATH_SCHEMA.check_match(targets_directory)
  formats.PATH_FILEINFO_SCHEMA.check_match(target_files)
  formats.METADATAVERSION_SCHEMA.check_match(version)
  sslib_formats.ISO8601_DATETIME_SCHEMA.check_match(expiration_date)
  sslib_formats.BOOLEAN_SCHEMA.check_match(write_consistent_targets)
  sslib_formats.BOOLEAN_SCHEMA.check_match(use_existing_fileinfo)

  if write_consistent_targets and use_existing_fileinfo:
    raise sslib_exceptions.Error('Cannot support writing consistent'
        ' targets and using existing fileinfo.')

  if delegations is not None:
    formats.DELEGATIONS_SCHEMA.check_match(delegations)
    # If targets role has delegations, collect the up-to-date 'keyids' and
    # 'threshold' for each role. Update the delegations keys dictionary.
    delegations_keys = []
    # Update 'keyids' and 'threshold' for each delegated role
    for role in delegations['roles']:
      role['keyids'] = roledb.get_role_keyids(role['name'],
          repository_name)
      role['threshold'] = roledb.get_role_threshold(role['name'],
          repository_name)

      # Collect all delegations keys for generating the delegations keydict
      for keyid in role['keyids']:
        key = keydb.get_key(keyid, repository_name=repository_name)
        delegations_keys.append(key)

    _, delegations['keys'] = keys_to_keydict(delegations_keys)


  # Store the file attributes of targets in 'target_files'.  'filedict',
  # conformant to 'tuf.formats.FILEDICT_SCHEMA', is added to the
  # targets metadata object returned.
  filedict = {}

  if use_existing_fileinfo:
    # Use the provided fileinfo dicts, conforming to FILEINFO_SCHEMA, rather than
    # generating fileinfo
    for target, fileinfo in target_files.items():

      # Ensure all fileinfo entries in target_files have a non-empty hashes dict
      if not fileinfo.get('hashes', None):
        raise sslib_exceptions.Error('use_existing_fileinfo option'
            ' set but no hashes exist in fileinfo for ' + repr(target))

      # and a non-empty length
      if fileinfo.get('length', -1) < 0:
        raise sslib_exceptions.Error('use_existing_fileinfo option'
            ' set but no length exists in fileinfo for ' + repr(target))

      filedict[target] = fileinfo

  else:
    # Generate the fileinfo dicts by accessing the target files on storage.
    # Default to accessing files on local storage.
    if storage_backend is None:
      storage_backend = sslib_storage.FilesystemBackend()

    filedict = _generate_targets_fileinfo(target_files, targets_directory,
        write_consistent_targets, storage_backend)

  # Generate the targets metadata object.
  # Use generalized build_dict_conforming_to_schema func to produce a dict that
  # contains all the appropriate information for targets metadata,
  # checking that the result conforms to the appropriate schema.
  # TODO: Later, probably after the rewrite for TUF Issue #660, generalize
  #       further, upward, by replacing generate_targets_metadata,
  #       generate_root_metadata, etc. with one function that generates
  #       metadata, possibly rolling that upwards into the calling function.
  #       There are very few things that really need to be done differently.
  if delegations is not None:
    return formats.build_dict_conforming_to_schema(
        formats.TARGETS_SCHEMA,
        version=version,
        expires=expiration_date,
        targets=filedict,
        delegations=delegations)
  else:
    return formats.build_dict_conforming_to_schema(
        formats.TARGETS_SCHEMA,
        version=version,
        expires=expiration_date,
        targets=filedict)
  # TODO: As an alternative to the odd if/else above where we decide whether or
  #       not to include the delegations argument based on whether or not it is
  #       None, consider instead adding a check in
  #       build_dict_conforming_to_schema that skips a keyword if that keyword
  #       is optional in the schema and the value passed in is set to None....





def _generate_targets_fileinfo(target_files, targets_directory,
    write_consistent_targets, storage_backend):
  """
  Iterate over target_files and:
    * ensure they exist in the targets_directory
    * generate a fileinfo dict for the target file, including hashes
    * copy 'target_path' to 'digest_target' if write_consistent_targets
  add all generated fileinfo dicts to a dictionary mapping
  targetpath: fileinfo and return the dict.
  """

  filedict = {}

  # Generate the fileinfo of all the target files listed in 'target_files'.
  for target, fileinfo in target_files.items():

    # The root-most folder of the targets directory should not be included in
    # target paths listed in targets metadata.
    # (e.g., 'targets/more_targets/somefile.txt' -> 'more_targets/somefile.txt')
    relative_targetpath = target

    # Note: join() discards 'targets_directory' if 'target' contains a leading
    # path separator (i.e., is treated as an absolute path).
    target_path = os.path.join(targets_directory, target.lstrip(os.sep))

    # Add 'custom' if it has been provided.  Custom data about the target is
    # optional and will only be included in metadata (i.e., a 'custom' field in
    # the target's fileinfo dictionary) if specified here.
    custom_data = fileinfo.get('custom', None)

    filedict[relative_targetpath] = \
        get_targets_metadata_fileinfo(target_path, storage_backend, custom_data)

    # Copy 'target_path' to 'digest_target' if consistent hashing is enabled.
    if write_consistent_targets:
      for target_digest in filedict[relative_targetpath]['hashes'].values():
        dirname, basename = os.path.split(target_path)
        digest_filename = target_digest + '.' + basename
        digest_target = os.path.join(dirname, digest_filename)
        shutil.copyfile(target_path, digest_target)

  return filedict



def _get_hashes_and_length_if_needed(use_length, use_hashes, full_file_path,
    storage_backend):
  """
  Calculate length and hashes only if they are required,
  otherwise, for adopters of tuf with lots of delegations,
  this will cause unnecessary overhead.
  """

  length = None
  hashes = None
  if use_length:
    length = sslib_util.get_file_length(full_file_path,
        storage_backend)

  if use_hashes:
    hashes = sslib_util.get_file_hashes(full_file_path,
        settings.FILE_HASH_ALGORITHMS, storage_backend)

  return length, hashes



def generate_snapshot_metadata(metadata_directory, version, expiration_date,
    storage_backend, consistent_snapshot=False,
    repository_name='default', use_length=False, use_hashes=False):
  """
  <Purpose>
    Create the snapshot metadata.  The minimum metadata must exist (i.e.,
    'root.json' and 'targets.json'). This function searches
    'metadata_directory' and the resulting snapshot file will list all the
    delegated roles found there.

  <Arguments>
    metadata_directory:
      The directory containing the 'root.json' and 'targets.json' metadata
      files.

    version:
      The metadata version number.  Clients use the version number to
      determine if the downloaded version is newer than the one currently
      trusted.

    expiration_date:
      The expiration date of the metadata file.
      Conformant to 'securesystemslib.formats.ISO8601_DATETIME_SCHEMA'.

    storage_backend:
      An object which implements
      securesystemslib.storage.StorageBackendInterface.

    consistent_snapshot:
      Boolean.  If True, a file digest is expected to be prepended to the
      filename of any target file located in the targets directory.  Each digest
      is stripped from the target filename and listed in the snapshot metadata.

    repository_name:
      The name of the repository.  If not supplied, 'rolename' is added to the
      'default' repository.

    use_length:
      Whether to include the optional length attribute for targets
      metadata files in the snapshot metadata.
      Default is False to save bandwidth but without losing security
      from rollback attacks.
      Read more at section 5.6 from the Mercury paper:
      https://www.usenix.org/conference/atc17/technical-sessions/presentation/kuppusamy

    use_hashes:
      Whether to include the optional hashes attribute for targets
      metadata files in the snapshot metadata.
      Default is False to save bandwidth but without losing security
      from rollback attacks.
      Read more at section 5.6 from the Mercury paper:
      https://www.usenix.org/conference/atc17/technical-sessions/presentation/kuppusamy

  <Exceptions>
    securesystemslib.exceptions.FormatError, if the arguments are improperly
    formatted.

    securesystemslib.exceptions.Error, if an error occurred trying to generate
    the snapshot metadata object.

  <Side Effects>
    The 'root.json' and 'targets.json' files are read.

  <Returns>
    The snapshot metadata object, conformant to 'tuf.formats.SNAPSHOT_SCHEMA'.
  """

  # Do the arguments have the correct format?
  # This check ensures arguments have the appropriate number of objects and
  # object types, and that all dict keys are properly named.
  # Raise 'securesystemslib.exceptions.FormatError' if the check fails.
  sslib_formats.PATH_SCHEMA.check_match(metadata_directory)
  formats.METADATAVERSION_SCHEMA.check_match(version)
  sslib_formats.ISO8601_DATETIME_SCHEMA.check_match(expiration_date)
  sslib_formats.BOOLEAN_SCHEMA.check_match(consistent_snapshot)
  sslib_formats.NAME_SCHEMA.check_match(repository_name)
  sslib_formats.BOOLEAN_SCHEMA.check_match(use_length)
  sslib_formats.BOOLEAN_SCHEMA.check_match(use_hashes)

  # Snapshot's 'fileinfodict' shall contain the version number of Root,
  # Targets, and all delegated roles of the repository.
  fileinfodict = {}

  length, hashes = _get_hashes_and_length_if_needed(use_length, use_hashes,
      os.path.join(metadata_directory, TARGETS_FILENAME), storage_backend)

  targets_role = TARGETS_FILENAME[:-len(METADATA_EXTENSION)]

  targets_file_version = get_metadata_versioninfo(targets_role,
      repository_name)

  # Make file info dictionary with make_metadata_fileinfo because
  # in the tuf spec length and hashes are optional for all
  # METAFILES in snapshot.json including the top-level targets file.
  fileinfodict[TARGETS_FILENAME] = formats.make_metadata_fileinfo(
      targets_file_version['version'], length, hashes)

  # Search the metadata directory and generate the versioninfo of all the role
  # files found there.  This information is stored in the 'meta' field of
  # 'snapshot.json'.

  metadata_files = sorted(storage_backend.list_folder(metadata_directory),
      reverse=True)
  for metadata_filename in metadata_files:
    # Strip the version number if 'consistent_snapshot' is True.
    # Example:  '10.django.json'  --> 'django.json'
    metadata_name, junk = _strip_version_number(metadata_filename,
        consistent_snapshot)

    # All delegated roles are added to the snapshot file.
    if metadata_filename.endswith(METADATA_EXTENSION):
      rolename = metadata_filename[:-len(METADATA_EXTENSION)]

      # Obsolete role files may still be found.  Ensure only roles loaded
      # in the roledb are included in the Snapshot metadata.  Since the
      # snapshot and timestamp roles are not listed in snapshot.json, do not
      # list these roles found in the metadata directory.
      if roledb.role_exists(rolename, repository_name) and \
          rolename not in roledb.TOP_LEVEL_ROLES:

        length, hashes = _get_hashes_and_length_if_needed(use_length, use_hashes,
            os.path.join(metadata_directory, metadata_filename), storage_backend)

        file_version = get_metadata_versioninfo(rolename,
            repository_name)

        fileinfodict[metadata_name] = formats.make_metadata_fileinfo(
            file_version['version'], length, hashes)

    else:
      logger.debug('Metadata file has an unsupported file'
          ' extension: ' + metadata_filename)

  # Generate the Snapshot metadata object.
  # Use generalized build_dict_conforming_to_schema func to produce a dict that
  # contains all the appropriate information for snapshot metadata,
  # checking that the result conforms to the appropriate schema.
  # TODO: Later, probably after the rewrite for TUF Issue #660, generalize
  #       further, upward, by replacing generate_targets_metadata,
  #       generate_root_metadata, etc. with one function that generates
  #       metadata, possibly rolling that upwards into the calling function.
  #       There are very few things that really need to be done differently.
  return formats.build_dict_conforming_to_schema(
      formats.SNAPSHOT_SCHEMA,
      version=version,
      expires=expiration_date,
      meta=fileinfodict)






def generate_timestamp_metadata(snapshot_file_path, version, expiration_date,
    storage_backend, repository_name, use_length=True, use_hashes=True):
  """
  <Purpose>
    Generate the timestamp metadata object.  The 'snapshot.json' file must
    exist.

  <Arguments>
    snapshot_file_path:
      Path to the required snapshot metadata file.  The timestamp role
      needs to the calculate the file size and hash of this file.

    version:
      The timestamp's version number.  Clients use the version number to
      determine if the downloaded version is newer than the one currently
      trusted.

    expiration_date:
      The expiration date of the metadata file, conformant to
      'securesystemslib.formats.ISO8601_DATETIME_SCHEMA'.

    storage_backend:
      An object which implements
      securesystemslib.storage.StorageBackendInterface.

    repository_name:
      The name of the repository.  If not supplied, 'rolename' is added to the
      'default' repository.

    use_length:
      Whether to include the optional length attribute of the snapshot
      metadata file in the timestamp metadata.
      Default is True.

    use_hashes:
      Whether to include the optional hashes attribute of the snapshot
      metadata file in the timestamp metadata.
      Default is True.

  <Exceptions>
    securesystemslib.exceptions.FormatError, if the generated timestamp metadata
    object cannot be formatted correctly, or one of the arguments is improperly
    formatted.

  <Side Effects>
    None.

  <Returns>
    A timestamp metadata object, conformant to 'tuf.formats.TIMESTAMP_SCHEMA'.
  """

  # Do the arguments have the correct format?
  # This check ensures arguments have the appropriate number of objects and
  # object types, and that all dict keys are properly named.
  # Raise 'securesystemslib.exceptions.FormatError' if the check fails.
  sslib_formats.PATH_SCHEMA.check_match(snapshot_file_path)
  formats.METADATAVERSION_SCHEMA.check_match(version)
  sslib_formats.ISO8601_DATETIME_SCHEMA.check_match(expiration_date)
  sslib_formats.NAME_SCHEMA.check_match(repository_name)
  sslib_formats.BOOLEAN_SCHEMA.check_match(use_length)
  sslib_formats.BOOLEAN_SCHEMA.check_match(use_hashes)

  snapshot_fileinfo = {}

  length, hashes = _get_hashes_and_length_if_needed(use_length, use_hashes,
      snapshot_file_path, storage_backend)

  snapshot_filename = os.path.basename(snapshot_file_path)
  # Retrieve the versioninfo of the Snapshot metadata file.
  snapshot_version = get_metadata_versioninfo('snapshot', repository_name)
  snapshot_fileinfo[snapshot_filename] = \
      formats.make_metadata_fileinfo(snapshot_version['version'],
          length, hashes)

  # Generate the timestamp metadata object.
  # Use generalized build_dict_conforming_to_schema func to produce a dict that
  # contains all the appropriate information for timestamp metadata,
  # checking that the result conforms to the appropriate schema.
  # TODO: Later, probably after the rewrite for TUF Issue #660, generalize
  #       further, upward, by replacing generate_targets_metadata,
  #       generate_root_metadata, etc. with one function that generates
  #       metadata, possibly rolling that upwards into the calling function.
  #       There are very few things that really need to be done differently.
  return formats.build_dict_conforming_to_schema(
      formats.TIMESTAMP_SCHEMA,
      version=version,
      expires=expiration_date,
      meta=snapshot_fileinfo)





def sign_metadata(metadata_object, keyids, filename, repository_name):
  """
  <Purpose>
    Sign a metadata object. If any of the keyids have already signed the file,
    the old signature is replaced.  The keys in 'keyids' must already be
    loaded in 'keydb'.

  <Arguments>
    metadata_object:
      The metadata object to sign.  For example, 'metadata' might correspond to
      'tuf.formats.ROOT_SCHEMA' or
      'tuf.formats.TARGETS_SCHEMA'.

    keyids:
      The keyids list of the signing keys.

    filename:
      The intended filename of the signed metadata object.
      For example, 'root.json' or 'targets.json'.  This function
      does NOT save the signed metadata to this filename.

    repository_name:
      The name of the repository.  If not supplied, 'rolename' is added to the
      'default' repository.

  <Exceptions>
    securesystemslib.exceptions.FormatError, if a valid 'signable' object could
    not be generated or the arguments are improperly formatted.

    securesystemslib.exceptions.Error, if an invalid keytype was found in the
    keystore.

  <Side Effects>
    None.

  <Returns>
    A signable object conformant to 'tuf.formats.SIGNABLE_SCHEMA'.
  """

  # Do the arguments have the correct format?
  # This check ensures arguments have the appropriate number of objects and
  # object types, and that all dict keys are properly named.
  # Raise 'securesystemslib.exceptions.FormatError' if the check fails.
  formats.ANYROLE_SCHEMA.check_match(metadata_object)
  sslib_formats.KEYIDS_SCHEMA.check_match(keyids)
  sslib_formats.PATH_SCHEMA.check_match(filename)
  sslib_formats.NAME_SCHEMA.check_match(repository_name)

  # Make sure the metadata is in 'signable' format.  That is,
  # it contains a 'signatures' field containing the result
  # of signing the 'signed' field of 'metadata' with each
  # keyid of 'keyids'.
  signable = formats.make_signable(metadata_object)

  # Sign the metadata with each keyid in 'keyids'.  'signable' should have
  # zero signatures (metadata_object contained none).
  for keyid in keyids:

    # Load the signing key.
    key = keydb.get_key(keyid, repository_name=repository_name)
    # Generate the signature using the appropriate signing method.
    if key['keytype'] in SUPPORTED_KEY_TYPES:
      if 'private' in key['keyval']:
        signed = sslib_formats.encode_canonical(signable['signed']).encode('utf-8')
        try:
          signature = sslib_keys.create_signature(key, signed)
          signable['signatures'].append(signature)

        except Exception:
          logger.warning('Unable to create signature for keyid: ' + repr(keyid))

      else:
        logger.debug('Private key unset.  Skipping: ' + repr(keyid))

    else:
      raise sslib_exceptions.Error('The keydb contains a key with'
        ' an invalid key type.' + repr(key['keytype']))

  # Raise 'securesystemslib.exceptions.FormatError' if the resulting 'signable'
  # is not formatted correctly.
  try:
    formats.check_signable_object_format(signable)
  except exceptions.UnsignedMetadataError:
    # Downgrade the error to a warning because a use case exists where
    # metadata may be generated unsigned on one machine and signed on another.
    logger.warning('Unsigned metadata object: ' + repr(signable))


  return signable





def write_metadata_file(metadata, filename, version_number, consistent_snapshot,
    storage_backend):
  """
  <Purpose>
    If necessary, write the 'metadata' signable object to 'filename'.

  <Arguments>
    metadata:
      The object that will be saved to 'filename', conformant to
      'tuf.formats.SIGNABLE_SCHEMA'.

    filename:
      The filename of the metadata to be written (e.g., 'root.json').

    version_number:
      The version number of the metadata file to be written.  The version
      number is needed for consistent snapshots, which prepend the version
      number to 'filename'.

    consistent_snapshot:
      Boolean that determines whether the metadata file's digest should be
      prepended to the filename.

    storage_backend:
      An object which implements
      securesystemslib.storage.StorageBackendInterface.

  <Exceptions>
    securesystemslib.exceptions.FormatError, if the arguments are improperly
    formatted.

    securesystemslib.exceptions.Error, if the directory of 'filename' does not
    exist.

    Any other runtime (e.g., IO) exception.

  <Side Effects>
    The 'filename' file is created, or overwritten if it exists.

  <Returns>
    The filename of the written file.
  """

  # Do the arguments have the correct format?
  # This check ensures arguments have the appropriate number of objects and
  # object types, and that all dict keys are properly named.
  # Raise 'securesystemslib.exceptions.FormatError' if the check fails.
  formats.SIGNABLE_SCHEMA.check_match(metadata)
  sslib_formats.PATH_SCHEMA.check_match(filename)
  formats.METADATAVERSION_SCHEMA.check_match(version_number)
  sslib_formats.BOOLEAN_SCHEMA.check_match(consistent_snapshot)

  if storage_backend is None:
    storage_backend = sslib_storage.FilesystemBackend()

  # Generate the actual metadata file content of 'metadata'.  Metadata is
  # saved as JSON and includes formatting, such as indentation and sorted
  # objects.  The new digest of 'metadata' is also calculated to help determine
  # if re-saving is required.
  file_content = _get_written_metadata(metadata)

  # We previously verified whether new metadata needed to be written (i.e., has
  # not been previously written or has changed).  It is now assumed that the
  # caller intends to write changes that have been marked as dirty.

  # The 'metadata' object is written to 'file_object'.  To avoid partial
  # metadata from being written, 'metadata' is first written to a temporary
  # location (i.e., 'file_object') and then moved to 'filename'.
  file_object = tempfile.TemporaryFile()

  # Serialize 'metadata' to the file-like object and then write 'file_object'
  # to disk.  The dictionary keys of 'metadata' are sorted and indentation is
  # used.
  file_object.write(file_content)

  if consistent_snapshot:
    dirname, basename = os.path.split(filename)
    basename = basename.split(METADATA_EXTENSION, 1)[0]
    version_and_filename = str(version_number) + '.' + basename + METADATA_EXTENSION
    written_consistent_filename = os.path.join(dirname, version_and_filename)

    # If we were to point consistent snapshots to 'written_filename', they
    # would always point to the current version.  Example: 1.root.json and
    # 2.root.json -> root.json.  If consistent snapshot is True, we should save
    # the consistent snapshot and point 'written_filename' to it.
    logger.debug('Creating a consistent file for ' + repr(filename))
    logger.debug('Saving ' + repr(written_consistent_filename))
    sslib_util.persist_temp_file(file_object,
        written_consistent_filename, should_close=False)

  else:
    logger.debug('Not creating a consistent snapshot for ' + repr(filename))

  logger.debug('Saving ' + repr(filename))
  storage_backend.put(file_object, filename)

  file_object.close()

  return filename





def _log_status_of_top_level_roles(targets_directory, metadata_directory,
    repository_name, storage_backend):
  """
  Non-public function that logs whether any of the top-level roles contain an
  invalid number of public and private keys, or an insufficient threshold of
  signatures.  Considering that the top-level metadata have to be verified in
  the expected root -> targets -> snapshot -> timestamp order, this function
  logs the error message and returns as soon as a required metadata file is
  found to be invalid.  It is assumed here that the delegated roles have been
  written and verified.  Example output:

  'root' role contains 1 / 1 signatures.
  'targets' role contains 1 / 1 signatures.
  'snapshot' role contains 1 / 1 signatures.
  'timestamp' role contains 1 / 1 signatures.

  Note:  Temporary metadata is generated so that file hashes & sizes may be
  computed and verified against the attached signatures.  'metadata_directory'
  should be a directory in a temporary repository directory.
  """

  # The expected full filenames of the top-level roles needed to write them to
  # disk.
  filenames = get_top_level_metadata_filenames(metadata_directory)
  root_filename = filenames[ROOT_FILENAME]
  targets_filename = filenames[TARGETS_FILENAME]
  snapshot_filename = filenames[SNAPSHOT_FILENAME]
  timestamp_filename = filenames[TIMESTAMP_FILENAME]

  # Verify that the top-level roles contain a valid number of public keys and
  # that their corresponding private keys have been loaded.
  for rolename in ['root', 'targets', 'snapshot', 'timestamp']:
    try:
      _check_role_keys(rolename, repository_name)

    except exceptions.InsufficientKeysError as e:
      logger.info(str(e))

  # Do the top-level roles contain a valid threshold of signatures?  Top-level
  # metadata is verified in Root -> Targets -> Snapshot -> Timestamp order.
  # Verify the metadata of the Root role.
  dirty_rolenames = roledb.get_dirty_roles(repository_name)

  root_roleinfo = roledb.get_roleinfo('root', repository_name)
  root_is_dirty = None
  if 'root' in dirty_rolenames:
    root_is_dirty = True

  else:
    root_is_dirty = False

  try:
    signable, root_filename = \
      _generate_and_write_metadata('root', root_filename, targets_directory,
          metadata_directory, storage_backend, repository_name=repository_name)
    _log_status('root', signable, repository_name)

  # 'tuf.exceptions.UnsignedMetadataError' raised if metadata contains an
  # invalid threshold of signatures.  log the valid/threshold message, where
  # valid < threshold.
  except exceptions.UnsignedMetadataError as e:
    _log_status('root', e.signable, repository_name)
    return

  finally:
    roledb.unmark_dirty(['root'], repository_name)
    roledb.update_roleinfo('root', root_roleinfo,
        mark_role_as_dirty=root_is_dirty, repository_name=repository_name)

  # Verify the metadata of the Targets role.
  targets_roleinfo = roledb.get_roleinfo('targets', repository_name)
  targets_is_dirty = None
  if 'targets' in dirty_rolenames:
    targets_is_dirty = True

  else:
    targets_is_dirty = False

  try:
    signable, targets_filename = \
      _generate_and_write_metadata('targets', targets_filename,
          targets_directory, metadata_directory, storage_backend,
          repository_name=repository_name)
    _log_status('targets', signable, repository_name)

  except exceptions.UnsignedMetadataError as e:
    _log_status('targets', e.signable, repository_name)
    return

  finally:
    roledb.unmark_dirty(['targets'], repository_name)
    roledb.update_roleinfo('targets', targets_roleinfo,
        mark_role_as_dirty=targets_is_dirty, repository_name=repository_name)

  # Verify the metadata of the snapshot role.
  snapshot_roleinfo = roledb.get_roleinfo('snapshot', repository_name)
  snapshot_is_dirty = None
  if 'snapshot' in dirty_rolenames:
    snapshot_is_dirty = True

  else:
    snapshot_is_dirty = False

  filenames = {'root': root_filename, 'targets': targets_filename}
  try:
    signable, snapshot_filename = \
      _generate_and_write_metadata('snapshot', snapshot_filename,
          targets_directory, metadata_directory, storage_backend, False,
          filenames, repository_name=repository_name)
    _log_status('snapshot', signable, repository_name)

  except exceptions.UnsignedMetadataError as e:
    _log_status('snapshot', e.signable, repository_name)
    return

  finally:
    roledb.unmark_dirty(['snapshot'], repository_name)
    roledb.update_roleinfo('snapshot', snapshot_roleinfo,
        mark_role_as_dirty=snapshot_is_dirty, repository_name=repository_name)

  # Verify the metadata of the Timestamp role.
  timestamp_roleinfo = roledb.get_roleinfo('timestamp', repository_name)
  timestamp_is_dirty = None
  if 'timestamp' in dirty_rolenames:
    timestamp_is_dirty = True

  else:
    timestamp_is_dirty = False

  filenames = {'snapshot': snapshot_filename}
  try:
    signable, timestamp_filename = \
      _generate_and_write_metadata('timestamp', timestamp_filename,
          targets_directory, metadata_directory, storage_backend,
          False, filenames, repository_name=repository_name)
    _log_status('timestamp', signable, repository_name)

  except exceptions.UnsignedMetadataError as e:
    _log_status('timestamp', e.signable, repository_name)
    return

  finally:
    roledb.unmark_dirty(['timestamp'], repository_name)
    roledb.update_roleinfo('timestamp', timestamp_roleinfo,
        mark_role_as_dirty=timestamp_is_dirty, repository_name=repository_name)



def _log_status(rolename, signable, repository_name):
  """
  Non-public function logs the number of (good/threshold) signatures of
  'rolename'.
  """

  status = sig.get_signature_status(signable, rolename, repository_name)

  logger.info(repr(rolename) + ' role contains ' + \
    repr(len(status['good_sigs'])) + ' / ' + repr(status['threshold']) + \
    ' signatures.')





def create_tuf_client_directory(repository_directory, client_directory):
  """
  <Purpose>
    Create client directory structure as 'tuf.client.updater' expects it.
    Metadata files downloaded from a remote TUF repository are saved to
    'client_directory'.
    The Root file must initially exist before an update request can be
    satisfied.  create_tuf_client_directory() ensures the minimum metadata
    is copied and that required directories ('previous' and 'current') are
    created in 'client_directory'.  Software updaters integrating TUF may
    use the client directory created as an initial copy of the repository's
    metadata.

  <Arguments>
    repository_directory:
      The path of the root repository directory.  The 'metadata' and 'targets'
      sub-directories should be available in 'repository_directory'.  The
      metadata files of 'repository_directory' are copied to 'client_directory'.

    client_directory:
      The path of the root client directory.  The 'current' and 'previous'
      sub-directories are created and will store the metadata files copied
      from 'repository_directory'.  'client_directory' will store metadata
      and target files downloaded from a TUF repository.

  <Exceptions>
    securesystemslib.exceptions.FormatError, if the arguments are improperly
    formatted.

    tuf.exceptions.RepositoryError, if the metadata directory in
    'client_directory' already exists.

  <Side Effects>
    Copies metadata files and directories from 'repository_directory' to
    'client_directory'.  Parent directories are created if they do not exist.

  <Returns>
    None.
  """

  # Do the arguments have the correct format?
  # This check ensures arguments have the appropriate number of objects and
  # object types, and that all dict keys are properly named.
  # Raise 'securesystemslib.exceptions.FormatError' if the check fails.
  sslib_formats.PATH_SCHEMA.check_match(repository_directory)
  sslib_formats.PATH_SCHEMA.check_match(client_directory)

  # Set the absolute path of the Repository's metadata directory.  The metadata
  # directory should be the one served by the Live repository.  At a minimum,
  # the repository's root file must be copied.
  repository_directory = os.path.abspath(repository_directory)
  metadata_directory = os.path.join(repository_directory,
      METADATA_DIRECTORY_NAME)

  # Set the client's metadata directory, which will store the metadata copied
  # from the repository directory set above.
  client_directory = os.path.abspath(client_directory)
  client_metadata_directory = os.path.join(client_directory,
      METADATA_DIRECTORY_NAME)

  # If the client's metadata directory does not already exist, create it and
  # any of its parent directories, otherwise raise an exception.  An exception
  # is raised to avoid accidentally overwriting previous metadata.
  try:
    os.makedirs(client_metadata_directory)

  except OSError as e:
    if e.errno == errno.EEXIST:
      message = 'Cannot create a fresh client metadata directory: ' +\
        repr(client_metadata_directory) + '.  Already exists.'
      raise exceptions.RepositoryError(message)

    # Testing of non-errno.EEXIST exceptions have been verified on all
    # supported OSs.  An unexpected exception (the '/' directory exists, rather
    # than disallowed path) is possible on Travis, so the '#pragma: no branch'
    # below is included to prevent coverage failure.
    else: #pragma: no branch
      raise

  # Move all  metadata to the client's 'current' and 'previous' directories.
  # The root metadata file MUST exist in '{client_metadata_directory}/current'.
  # 'tuf.client.updater' expects the 'current' and 'previous' directories to
  # exist under 'metadata'.
  client_current = os.path.join(client_metadata_directory, 'current')
  client_previous = os.path.join(client_metadata_directory, 'previous')
  shutil.copytree(metadata_directory, client_current)
  shutil.copytree(metadata_directory, client_previous)



def disable_console_log_messages():
  """
  <Purpose>
    Disable logger messages printed to the console.  For example, repository
    maintainers may want to call this function if many roles will be sharing
    keys, otherwise detected duplicate keys will continually log a warning
    message.

  <Arguments>
    None.

  <Exceptions>
    None.

  <Side Effects>
    Removes the 'tuf.log' console handler, added by default when
    'tuf.repository_tool.py' is imported.

  <Returns>
    None.
  """

  log.remove_console_handler()



def keys_to_keydict(keys):
  """
  <Purpose>
    Iterate over a list of keys and return a list of keyids and a dict mapping
    keyid to key metadata

  <Arguments>
    keys:
      A list of key objects conforming to
      securesystemslib.formats.ANYKEYLIST_SCHEMA.

  <Returns>
    keyids:
      A list of keyids conforming to securesystemslib.formats.KEYID_SCHEMA
    keydict:
      A dictionary conforming to securesystemslib.formats.KEYDICT_SCHEMA
  """
  keyids = []
  keydict = {}

  for key in keys:
    keyid = key['keyid']
    key_metadata_format = sslib_keys.format_keyval_to_metadata(
        key['keytype'], key['scheme'], key['keyval'])

    new_keydict = {keyid: key_metadata_format}
    keydict.update(new_keydict)
    keyids.append(keyid)
  return keyids, keydict




if __name__ == '__main__':
  # The interactive sessions of the documentation strings can
  # be tested by running repository_lib.py as a standalone module:
  # $ python repository_lib.py.
  import doctest
  doctest.testmod()