This is a manual squash commit of detect_expiry_322, to avoid merge c…

…onflicts.

Comment lines from the individual commits include:
1. Fix theupdateframework#322 by detecting expiry of stale files. initial attempt
2. temp commit of files from Soma
3. removing freeze_attack_stale_expiry and leaving the test added to indefinite freeze attack
4. fixing indefinite freeze attack test: now incorporates old reject-freshly-downloaded-but-expired-timestamp test as well as reject-stale-already-present-but-expired-snapshot test
5. small refinements to indefinite freeze attack test
6. Pulled the recursion out of the except block in refresh() to avoid unprintable nested exceptions.
7. Added comments to the last commit (retry_once)
8. Merge pull request #1 from awwad/detect_expiry_322_temp (removing cruft in another branch)

tests/test_indefinite_freeze_attack.py

@@ -15,6 +15,14 @@
     than verifying text output), use pre-generated repository files, and
     discontinue use of the old repository tools. -vladimir.v.diaz
 
+  March 9, 2016.
+    Additional test added relating to issue:
+    https://github.com/theupdateframework/tuf/issues/322
+    If a metadata file is not downloaded (no indication of a new version
+    available), its expiration must still be detected. This add'l test
+    complains if such does not occur, and accompanies code to detect it.
+    -sebastien.awwad
+
 <Copyright>
   See LICENSE for licensing information.
 
@@ -171,6 +179,7 @@ def tearDown(self):
 
 
   def test_without_tuf(self):
+    # Test 1:
     # Scenario:
     # 'timestamp.json' specifies the latest version of the repository files.
     # A client should only accept the same version of this file up to a certain
@@ -215,9 +224,108 @@ def test_without_tuf(self):
     # Verify 'download_fileinfo' is equal to the current local file.
     self.assertEqual(download_fileinfo, fileinfo)
 
+    # Test 2:
+    # See description of scenario in Test 2 in the test_with_tuf method.
+    # Without TUF, Test 2 is tantamount to Test 1, and so it is not tested
+    # again here.
 
 
   def test_with_tuf(self):
+    # Two tests are conducted here.
+    # Test 1: If an expired timestamp is downloaded, is it recognized as such?
+    # Test 2: If we find that the timestamp acquired from a mirror indicates
+    #         that there is no new snapshot file, and our current snapshot
+    #         file is expired, is it recognized as such?
+    #
+    # Test 2 addresses this issue:
+    #   https://github.com/theupdateframework/tuf/issues/322
+    #
+    # If time has passed and our snapshot (or any targets role) is expired, and
+    # the mirror whose timestamp we fetched doesn't indicate the existence of a
+    # new snapshot version, we still need to check that it's expired and notify
+    # the software update system / application / user. This test creates that
+    # scenario. The correct behavior is to raise an exception.
+    #
+    # Background: Expiration checks were previously conducted
+    # (ensure_not_expired) when the metadata file was downloaded. If no new
+    # metadata file was downloaded, no expiry check would occurs. (Exception:
+    # root was checked for expiration at the beginning of each refresh() cycle,
+    # and timestamp was always checked because it was always fetched.) Snapshot
+    # and targets were never checked if the user does not have evidence that
+    # they have changed.
+
+    timestamp_path = os.path.join(self.repository_directory, 'metadata',
+                                  'timestamp.json')
+
+    # Modify the timestamp file on the remote repository.  'timestamp.json'
+    # must be properly updated and signed with 'repository_tool.py', otherwise
+    # the client will reject it as invalid metadata.  The resulting
+    # 'timestamp.json' should be valid metadata, but expired (as intended).
+    repository = repo_tool.load_repository(self.repository_directory)
+
+    key_file = os.path.join(self.keystore_directory, 'timestamp_key') 
+    timestamp_private = repo_tool.import_rsa_privatekey_from_file(key_file,
+                                                                  'password')
+
+    repository.timestamp.load_signing_key(timestamp_private)
+
+    # Load snapshot keys.
+    key_file = os.path.join(self.keystore_directory, 'snapshot_key') 
+    snapshot_private = repo_tool.import_rsa_privatekey_from_file(key_file,
+                                                                  'password')
+    repository.snapshot.load_signing_key(snapshot_private)
+
+    # Load root keys.
+    key_file = os.path.join(self.keystore_directory, 'root_key')
+    root_private = repo_tool.import_rsa_privatekey_from_file(key_file,
+                                                                  'password')
+    repository.root.load_signing_key(root_private)
+
+    time.sleep(0.1)
+
+    # Expire snapshot in 7s
+    datetime_object = tuf.formats.unix_timestamp_to_datetime(int(time.time() +
+                                                                 7))
+    repository.snapshot.expiration = datetime_object
+
+    # Now write to the repository
+    repository.write()
+    # And move the staged metadata to the "live" metadata.
+    time.sleep(0.1)
+    shutil.rmtree(os.path.join(self.repository_directory, 'metadata'))
+    shutil.copytree(os.path.join(self.repository_directory, 'metadata.staged'),
+                    os.path.join(self.repository_directory, 'metadata'))
+
+    # Refresh metadata on the client. For this refresh, all data is not expired.
+    time.sleep(0.1)
+    logger.info('Test: Refreshing #1 - Initial metadata refresh occurring.')
+    self.repository_updater.refresh()
+    logger.info("Test: Refreshed #1 - Initial metadata refresh occurred. Now "
+                "sleeping 6s.")
+
+    # Sleep for at least 7 seconds to ensure 'repository.snapshot.expiration'
+    # is reached.
+    time.sleep(7)
+    logger.info("Test: Refreshing #2 - Now trying to refresh again after "
+      " snapshot expiry.")
+    try:
+      self.repository_updater.refresh() # We expect this to fail!
+
+    except tuf.ExpiredMetadataError as e:
+      logger.info("Test: Refresh #2 - failed as expected. Stale-expired "
+                  "snapshot case generated a tuf.ExpiredMetadataError exception"
+                  ". Test pass.")
+    #I think that I only expect tuf.ExpiredMetadata error here. A
+    # NoWorkingMirrorError indicates something else in this case - unavailable
+    # repo, for example.
+    else:
+      self.fail("TUF failed to detect expired stale snapshot metadata. Freeze.")
+
+
+
+
+    # Part 1:
+
     # The same scenario outlined in test_without_tuf() is followed here, except
     # with a TUF client.  The TUF client performs a refresh of top-level
     # metadata, which also includes 'timestamp.json'.
@@ -237,8 +345,8 @@ def test_with_tuf(self):
 
     repository.timestamp.load_signing_key(timestamp_private)
 
-    # expire in 1 second.
-    datetime_object = tuf.formats.unix_timestamp_to_datetime(int(time.time() + 1))
+    # expire in 4 seconds.
+    datetime_object = tuf.formats.unix_timestamp_to_datetime(int(time.time() + 4))
     repository.timestamp.expiration = datetime_object
     repository.write()
 
@@ -248,15 +356,19 @@ def test_with_tuf(self):
                     os.path.join(self.repository_directory, 'metadata'))
 
     # Verify that the TUF client detects outdated metadata and refuses to
-    # continue the update process.  Sleep for at least 2 seconds to ensure
+    # continue the update process.  Sleep for at least 4 seconds to ensure
     # 'repository.timestamp.expiration' is reached.
-    time.sleep(2)
+    time.sleep(4)
     try:
       self.repository_updater.refresh()
 
     except tuf.NoWorkingMirrorError as e:
       for mirror_url, mirror_error in six.iteritems(e.mirror_errors):
         self.assertTrue(isinstance(mirror_error, tuf.ExpiredMetadataError))
+
+    else:
+      self.fail("TUF failed to detect expired stale timestamp metadata. Freeze "
+        "attack successful.")
 
 
 if __name__ == '__main__':

tuf/client/updater.py

@@ -643,6 +643,13 @@ def refresh(self, unsafely_update_root_if_necessary=True):
         logger.info('An expired Root metadata was loaded and must be updated.')
         raise
 
+    # If an exception is raised during the metadata update attempts, we will
+    # attempt to update root metadata once by recursing with a special argument
+    # to avoid further recursion. We use this bool and pull the recursion out
+    # of the except block so as to avoid unprintable nested NoWorkingMirrorError
+    # exceptions.
+    retry_once = False
+
     # Use default but sane information for timestamp metadata, and do not
     # require strict checks on its required length.
     try: 
@@ -652,19 +659,53 @@ def refresh(self, unsafely_update_root_if_necessary=True):
       self._update_metadata_if_changed('root')
       self._update_metadata_if_changed('targets')
 
+    # There are two distinct error scenarios that can rise from the
+    # _update_metadata_if_changed calls in the try block above:
+    #
+    #   - tuf.NoWorkingMirrorError:
+    #
+    #      If a change to a metadata file IS detected in an
+    #      _update_metadata_if_changed call, but we are unable to download a
+    #      valid (not expired, properly signed, valid) version of that metadata
+    #      file, and unexpired version of that metadata file, a
+    #      tuf.NoWorkingMirrorError rises to this point.
+    # 
+    #   - tuf.ExpiredMetadataError:
+    #
+    #      If, on the other hand, a change to a metadata file IS NOT detected in
+    #      a given _update_metadata_if_changed call, but we observe that the
+    #      version of the metadata file we have on hand is now expired, a
+    #      tuf.ExpiredMetadataError exception rises to this point.
+    #
     except tuf.NoWorkingMirrorError as e:
       if unsafely_update_root_if_necessary:
-        message = 'Valid top-level metadata cannot be downloaded.  Unsafely '+\
-          'update the Root metadata.'
-        logger.info(message)
-
-        self._update_metadata('root', DEFAULT_ROOT_UPPERLENGTH)
-        self.refresh(unsafely_update_root_if_necessary=False)
-
+        logger.info('Valid top-level metadata cannot be downloaded.  Unsafely '
+          'update the Root metadata.')
+        retry_once = True
+
       else:
         raise
 
+    except tuf.ExpiredMetadataError as e:
+      if unsafely_update_root_if_necessary:
+        logger.info('No changes were detected from the mirrors for a given role'
+          ', and that metadata that is available on disk has been found to be '
+          'expired. Trying to update root in case of foul play.')
+        retry_once = True
+
+      # The caller explicitly requested not to unsafely fetch an expired Root.
+      else:
+        logger.info('No changes were detected from the mirrors for a given role'
+          ', and that metadata that is available on disk has been found to be '
+          'expired. Your metadata is out of date.')
+        raise
 
+    # Update failed and we aren't already in a retry. Try once more after
+    # updating root.
+    if retry_once:
+        self._update_metadata('root', DEFAULT_ROOT_UPPERLENGTH)
+        self.refresh(unsafely_update_root_if_necessary=False)
+
 
 
 
@@ -1578,6 +1619,13 @@ def _update_metadata_if_changed(self, metadata_role,
                                          expected_versioninfo):
       logger.info(repr(uncompressed_metadata_filename) + ' up-to-date.')
 
+      # Since we have not downloaded a new version of this metadata, we
+      # should check to see if our version is stale and notify the user
+      # if so. This raises tuf.ExpiredMetadataError if the metadata we
+      # have is expired. Resolves issue #322.
+      self._ensure_not_expired(self.metadata['current'][metadata_role],
+                               metadata_role)
+
       return
 
     logger.debug('Metadata ' + repr(uncompressed_metadata_filename) + ' has changed.')