chore(logging): Refactored rendering interface to use navigators

Rather than using file paths, the API takes journal navigators.
Added StreamJournalNavigator to take the place of file paths.

citest/base/__init__.py

@@ -40,7 +40,8 @@
     unset_global_journal)
 
 from .journal_navigator import (
-    JournalNavigator)
+    JournalNavigator,
+    StreamJournalNavigator)
 
 from .journal_processor import (
     JournalProcessor,

citest/base/journal_navigator.py

@@ -15,22 +15,43 @@
 """Various journal iterators to facilitate navigating through journal JSON."""
 
 import json
+import os
+
+try:
+  from StringIO import StringIO
+except ImportError:
+  from io import StringIO
 
 from .record_stream import RecordInputStream
 
 
 class JournalNavigator(object):
   """Iterates over journal JSON."""
 
-  def __init__(self):
-    """Constructor"""
-    self.__input_stream = None
-    self.__decoder = json.JSONDecoder()
+  @property
+  def journal_id(self):
+    """Returns identifier specifying original location of content."""
+    raise NotImplementedError('{0}.journal_id not implemented'.format(
+        self.__class__.__name__))
 
-  def __iter__(self):
-    """Iterate over the contents of the journal."""
-    self.__check_open()
-    return self
+  @property
+  def journal_name(self):
+    """Provides the name of the journal, typically the file basename."""
+    raise NotImplementedError('{0}.journal_name not implemented'.format(
+        self.__class__.__name__))
+
+  def next(self):
+    """Return the next item in the journal.
+
+    Raises:
+      StopIteration when there are no more elements.
+    """
+    raise NotImplementedError('{0}.next() not implemented'.format(
+        self.__class__.__name__))
+
+
+class StreamJournalNavigator(JournalNavigator):
+  """Iterates over journal JSON from a stream."""
 
   def __next__(self):
     return self.next()
@@ -41,33 +62,62 @@ def open(self, path):
     Args:
       path: [string] The path to load the journal from.
     """
-    if self.__input_stream != None:
+    if self.__input_stream is not None:
       raise ValueError('Navigator is already open.')
     self.__input_stream = RecordInputStream(open(path, 'rb'))
 
-  def close(self):
-    """Close the journal."""
-    self.__check_open()
-    self.__input_stream.close()
-    self.__input_stream = None
+  @staticmethod
+  def new_from_path(path):
+    """Create a new navigator using the contents of a file.
+
+    Args:
+      path: [string] Path to journal file.
+    """
+    with open(path, 'rb') as stream:
+      return StreamJournalNavigator.new_from_bytes(path, stream.read())
+
+  @staticmethod
+  def new_from_bytes(journal_id, contents):
+    """Create a new navigator using the given record-encoded journal.
+
+    Args:
+      journal_id: [string] Identifies the source of the bytes.
+      contents: [string] Raw byte contents of a record-encoded journal file.
+    """
+    return StreamJournalNavigator(
+        journal_id, RecordInputStream(StringIO(contents)))
+
+  @property
+  def journal_id(self):
+    return self.__id
+
+  @property
+  def journal_name(self):
+    basename = os.path.basename(self.__id)
+    if basename.endswith('.journal'):
+      basename = os.path.splitext(basename)[0]
+    return basename
+
+  def __init__(self, journal_id, stream):
+    """Constructor"""
+    self.__id = journal_id
+    self.__input_stream = stream
+    self.__decoder = json.JSONDecoder()
+
+  def __iter__(self):
+    return self
 
   def next(self):
     """Return the next item in the journal.
 
     Raises:
       StopIteration when there are no more elements.
     """
-    self.__check_open()
     json_str = next(self.__input_stream)
 
     try:
       return self.__decoder.decode(json_str)
 
     except ValueError:
-      print('Invalid json record:\n{0}'.format(json_str))
+      logging.error('Invalid json record:\n%s', json_str)
       raise
-
-  def __check_open(self):
-    """Verify that the navigator is open (and thus valid to iterate)."""
-    if self.__input_stream is None:
-      raise ValueError('Navigator is not open.')

citest/base/journal_processor.py

@@ -15,7 +15,7 @@
 
 """Processes a journal by calling specialized handlers on each entry."""
 
-from .journal_navigator import JournalNavigator
+from .journal_navigator import StreamJournalNavigator
 
 
 class ProcessedEntityManager(object):
@@ -144,23 +144,17 @@ def terminate(self):
     """Terminate the processor (finished processing)."""
     pass
 
-  def process(self, input_path):
+  def process(self, navigator):
     """Process the contents of the journal indicatd by input_path.
 
     Args:
-      input_path: [string] The path to the journal.
+      navigator: [JournalNavigator] The journal to process.
     """
-    navigator = JournalNavigator()
-    navigator.open(input_path)
-    try:
-      for obj in navigator:
-        entry_type = obj.get('_type')
-        handler = (self.__handler_registry.get(entry_type)
-                   or self.__default_handler)
-        handler(obj)
-
-    finally:
-      navigator.close()
+    for obj in navigator:
+      entry_type = obj.get('_type')
+      handler = (self.__handler_registry.get(entry_type)
+                 or self.__default_handler)
+      handler(obj)
 
   def handle_unknown(self, obj):
     """The default handler for processing entries with unregistered _type.

citest/reporting/dump_renderer.py

@@ -216,7 +216,7 @@ def main(argv):
   options = parser.parse_args(argv[1:])
   for path in options.journals:
     processor = DumpRenderer(vars(options))
-    processor.process(path)
+    processor.process(JournalStreamNavigator.new_from_path(path))
     processor.terminate()
 
 

citest/reporting/generate_html_report.py

@@ -33,6 +33,7 @@
 import resource
 import sys
 
+from citest.base import StreamJournalNavigator
 from citest.reporting.html_renderer import HtmlRenderer
 from citest.reporting.html_document_manager import HtmlDocumentManager
 from citest.reporting.html_index_renderer import HtmlIndexRenderer
@@ -54,7 +55,7 @@ def journal_to_html(input_path, prune=False):
       title='Report for {0}'.format(os.path.basename(input_path)))
 
   processor = HtmlRenderer(document_manager, prune=prune)
-  processor.process(input_path)
+  processor.process(StreamJournalNavigator.new_from_path(input_path))
   processor.terminate()
   document_manager.wrap_tag(document_manager.new_tag('table'))
   document_manager.build_to_path(output_path)
@@ -106,7 +107,7 @@ def build_index(journal_list, output_dir):
 
   processor = HtmlIndexRenderer(document_manager)
   for journal in journal_list:
-    processor.process(journal)
+    processor.process(StreamJournalNavigator.new_from_path(journal))
   processor.terminate()
 
   tr_tag = document_manager.make_tag_container(

citest/reporting/html_index_renderer.py

@@ -129,30 +129,26 @@ def output_column_names(self):
     """Returns list of column names for the summary table."""
     return ['P', 'F', 'Test Module', 'Time']
 
-  def process(self, journal):
+  def process(self, navigator):
     """Overrides JournalProcessor.process() for an individual journal.
 
     When we process a journal, we're going to reduce it down to a summary
     line in our index that links to the journal output.
     We're also going to accumulate overall statistics for the index summary.
 
     Args:
-      journal: [string] The path to the journal file to process.
+      navigator: [JournalNavigator] The the journal we're going to process.
     """
     self.__reset_journal_counters()
-    super(HtmlIndexRenderer, self).process(journal)
+    super(HtmlIndexRenderer, self).process(navigator)
 
     if self.__passed_count == 0 and self.__failed_count == 0:
       sys.stderr.write(
           'No tests recorded in {0}. Assuming this is an error.\n'.format(
-              journal))
+              navigator.journal_id))
       self.__failed_count = 1
 
-    journal_basename = os.path.basename(journal)
-    if journal_basename.endswith('.journal'):
-      journal_basename = os.path.splitext(journal_basename)[0]
-    html_path = os.path.splitext(journal)[0] + '.html'
-
+    html_path = navigator.journal_name + '.html'
     self.__total_passed += self.__passed_count
     self.__total_failed += self.__failed_count
 

citest/reporting/html_index_table_renderer.py

@@ -18,7 +18,10 @@
 
 import os
 import sys
-from citest.base import (JournalProcessor, ProcessedEntityManager)
+from citest.base import (
+    JournalProcessor,
+    ProcessedEntityManager,
+    StreamJournalNavigator)
 from .html_document_manager import HtmlDocumentManager
 
 class TestStats(object):
@@ -257,7 +260,8 @@ def process_all(document_manager, journal_list, output_dir):
     # Process all the journals to get the stats that we're going
     # to render into the table cells.
     for journal in sorted(journal_list):
-      summary, stats, details = processor.process(journal)
+      summary, stats, details = processor.process(
+          StreamJournalNavigator.new_from_path(journal))
       journal_to_cell[journal] = summary
       journal_to_stats[journal] = stats
       journal_to_details[journal] = details
@@ -430,21 +434,18 @@ def __handle_generic(self, entry):
             self.__in_test = None
         return
 
-  def process(self, journal):
+  def process(self, navigator):
     self.__reset_journal_counters()
 
-    super(HtmlIndexTableRenderer, self).process(journal)
+    super(HtmlIndexTableRenderer, self).process(navigator)
 
     if self.__stats.count == 0:
       sys.stderr.write(
           'No tests recorded in {0}. Assuming this is an error.\n'.format(
-              journal))
+              navigator.journal_id))
       self.__stats.error = 1
 
-    journal_basename = os.path.basename(journal)
-    if journal_basename.endswith('.journal'):
-      journal_basename = os.path.splitext(journal_basename)[0]
-    html_path = os.path.splitext(journal)[0] + '.html'
+    html_path = navigator.journal_name + '.html'
     self.__total_stats.aggregate(self.__stats)
 
     _, css = self.__document_manager.determine_attribute_css_kwargs(

tests/base/journal_navigator_test.py

@@ -0,0 +1,84 @@
+# Copyright 2018 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Test citest.reporting.html_renderer module."""
+
+import os
+import shutil
+import tempfile
+import unittest
+
+from citest.base import (
+    Journal,
+    StreamJournalNavigator)
+
+
+class StreamNavigatorTest(unittest.TestCase):
+  # pylint: disable=missing-docstring
+
+  @classmethod
+  def setUpClass(cls):
+    cls.temp_dir = tempfile.mkdtemp(prefix='journal_nav_test')
+
+  @classmethod
+  def tearDownClass(cls):
+    shutil.rmtree(cls.temp_dir)
+
+  def test_iterator(self):
+    journal = Journal()
+    path = os.path.join(self.temp_dir, 'test_iterator.journal')
+    expect = []
+
+    journal.open_with_path(path, TestString='TestValue', TestNum=123)
+    expect.append({'_type': 'JournalMessage',
+                   '_value': 'Starting journal.',
+                   'TestString': 'TestValue',
+                   'TestNum': 123})
+
+    journal.write_message('Initial Message')
+    expect.append({'_type': 'JournalMessage', '_value': 'Initial Message'})
+
+    journal.begin_context('OUTER', TestProperty='BeginOuter')
+    expect.append({'_type': 'JournalContextControl',
+                   'control': 'BEGIN',
+                   '_title': 'OUTER',
+                   'TestProperty': 'BeginOuter'})
+
+    journal.write_message('Context Message', format='pre')
+    expect.append({'_type': 'JournalMessage', '_value': 'Context Message',
+                   'format': 'pre'})
+
+    journal.end_context(TestProperty='END OUTER')
+    expect.append({'_type': 'JournalContextControl',
+                   'control': 'END',
+                   'TestProperty': 'END OUTER'})
+
+    journal.terminate(EndProperty='xyz')
+    expect.append({'_type': 'JournalMessage',
+                   '_value': 'Finished journal.',
+                   'EndProperty': 'xyz'})
+
+    # We're going to pop off expect, so reverse it
+    # so that we can check in order.
+    expect.reverse()
+    navigator = StreamJournalNavigator.new_from_path(path)
+    for record in navigator:
+        del(record['_thread'])
+        del(record['_timestamp'])
+        self.assertEquals(record, expect.pop())
+    self.assertEquals([], expect)
+
+
+if __name__ == '__main__':
+  unittest.main()