Merge pull request #1578 from tseaver/logging-parse_protobuf_entries

Add support for parsing log entries w/ 'protoPayload' from resources

gcloud/logging/client.py

@@ -17,6 +17,7 @@
 
 from gcloud.client import JSONClient
 from gcloud.logging.connection import Connection
+from gcloud.logging.entries import ProtobufEntry
 from gcloud.logging.entries import StructEntry
 from gcloud.logging.entries import TextEntry
 from gcloud.logging.logger import Logger
@@ -71,13 +72,16 @@ def _entry_from_resource(self, resource, loggers):
         :rtype; One of:
                 :class:`gcloud.logging.entries.TextEntry`,
                 :class:`gcloud.logging.entries.StructEntry`,
+                :class:`gcloud.logging.entries.ProtobufEntry`
         :returns: the entry instance, constructed via the resource
         """
         if 'textPayload' in resource:
             return TextEntry.from_api_repr(resource, self, loggers)
         elif 'jsonPayload' in resource:
             return StructEntry.from_api_repr(resource, self, loggers)
-        raise ValueError('Cannot parse job resource')
+        elif 'protoPayload' in resource:
+            return ProtobufEntry.from_api_repr(resource, self, loggers)
+        raise ValueError('Cannot parse log entry resource')
 
     def list_entries(self, projects=None, filter_=None, order_by=None,
                      page_size=None, page_token=None):

gcloud/logging/entries.py

@@ -76,7 +76,7 @@ def from_api_repr(cls, resource, client, loggers=None):
 
 
 class TextEntry(_BaseEntry):
-    """Entry created via a write request with ``textPayload``.
+    """Entry created with ``textPayload``.
 
     See:
     https://cloud.google.com/logging/docs/api/ref_v2beta1/rest/v2beta1/LogEntry
@@ -85,9 +85,18 @@ class TextEntry(_BaseEntry):
 
 
 class StructEntry(_BaseEntry):
-    """Entry created via a write request with ``jsonPayload``.
+    """Entry created with ``jsonPayload``.
 
     See:
     https://cloud.google.com/logging/docs/api/ref_v2beta1/rest/v2beta1/LogEntry
     """
     _PAYLOAD_KEY = 'jsonPayload'
+
+
+class ProtobufEntry(_BaseEntry):
+    """Entry created with ``protoPayload``.
+
+    See:
+    https://cloud.google.com/logging/docs/api/ref_v2beta1/rest/v2beta1/LogEntry
+    """
+    _PAYLOAD_KEY = 'protoPayload'

gcloud/logging/test_client.py

@@ -104,9 +104,11 @@ def test_list_entries_defaults(self):
         self.assertEqual(req['data'], SENT)
 
     def test_list_entries_explicit(self):
+        # pylint: disable=too-many-statements
         from datetime import datetime
         from gcloud._helpers import UTC
         from gcloud.logging import DESCENDING
+        from gcloud.logging.entries import ProtobufEntry
         from gcloud.logging.entries import StructEntry
         from gcloud.logging.logger import Logger
         from gcloud.logging.test_entries import _datetime_to_rfc3339_w_nanos
@@ -116,7 +118,10 @@ def test_list_entries_explicit(self):
         NOW = datetime.utcnow().replace(tzinfo=UTC)
         TIMESTAMP = _datetime_to_rfc3339_w_nanos(NOW)
         IID1 = 'IID1'
+        IID2 = 'IID2'
         PAYLOAD = {'message': 'MESSAGE', 'weather': 'partly cloudy'}
+        PROTO_PAYLOAD = PAYLOAD.copy()
+        PROTO_PAYLOAD['@type'] = 'type.googleapis.com/testing.example'
         TOKEN = 'TOKEN'
         PAGE_SIZE = 42
         SENT = {
@@ -136,6 +141,15 @@ def test_list_entries_explicit(self):
                 'timestamp': TIMESTAMP,
                 'logName': 'projects/%s/logs/%s' % (
                     self.PROJECT, self.LOGGER_NAME),
+            }, {
+                'protoPayload': PROTO_PAYLOAD,
+                'insertId': IID2,
+                'resource': {
+                    'type': 'global',
+                },
+                'timestamp': TIMESTAMP,
+                'logName': 'projects/%s/logs/%s' % (
+                    self.PROJECT, self.LOGGER_NAME),
             }],
         }
         creds = _Credentials()
@@ -144,7 +158,8 @@ def test_list_entries_explicit(self):
         entries, token = client.list_entries(
             projects=[PROJECT1, PROJECT2], filter_=FILTER, order_by=DESCENDING,
             page_size=PAGE_SIZE, page_token=TOKEN)
-        self.assertEqual(len(entries), 1)
+        self.assertEqual(len(entries), 2)
+
         entry = entries[0]
         self.assertTrue(isinstance(entry, StructEntry))
         self.assertEqual(entry.insert_id, IID1)
@@ -155,6 +170,19 @@ def test_list_entries_explicit(self):
         self.assertEqual(logger.name, self.LOGGER_NAME)
         self.assertTrue(logger.client is client)
         self.assertEqual(logger.project, self.PROJECT)
+
+        entry = entries[1]
+        self.assertTrue(isinstance(entry, ProtobufEntry))
+        self.assertEqual(entry.insert_id, IID2)
+        self.assertEqual(entry.payload, PROTO_PAYLOAD)
+        self.assertEqual(entry.timestamp, NOW)
+        logger = entry.logger
+        self.assertEqual(logger.name, self.LOGGER_NAME)
+        self.assertTrue(logger.client is client)
+        self.assertEqual(logger.project, self.PROJECT)
+
+        self.assertTrue(entries[0].logger is entries[1].logger)
+
         self.assertEqual(token, None)
         self.assertEqual(len(conn._requested), 1)
         req = conn._requested[0]

gcloud/logging/test_entries.py

@@ -15,14 +15,18 @@
 import unittest2
 
 
-class TestTextEntry(unittest2.TestCase):
+class Test_BaseEntry(unittest2.TestCase):
 
     PROJECT = 'PROJECT'
     LOGGER_NAME = 'LOGGER_NAME'
 
     def _getTargetClass(self):
-        from gcloud.logging.entries import TextEntry
-        return TextEntry
+        from gcloud.logging.entries import _BaseEntry
+
+        class _Dummy(_BaseEntry):
+            _PAYLOAD_KEY = 'dummyPayload'
+
+        return _Dummy
 
     def _makeOne(self, *args, **kw):
         return self._getTargetClass()(*args, **kw)
@@ -53,7 +57,7 @@ def test_from_api_repr_missing_data_no_loggers(self):
         PAYLOAD = 'PAYLOAD'
         LOG_NAME = 'projects/%s/logs/%s' % (self.PROJECT, self.LOGGER_NAME)
         API_REPR = {
-            'textPayload': PAYLOAD,
+            'dummyPayload': PAYLOAD,
             'logName': LOG_NAME,
         }
         klass = self._getTargetClass()
@@ -76,7 +80,7 @@ def test_from_api_repr_w_loggers_no_logger_match(self):
         TIMESTAMP = _datetime_to_rfc3339_w_nanos(NOW)
         LOG_NAME = 'projects/%s/logs/%s' % (self.PROJECT, self.LOGGER_NAME)
         API_REPR = {
-            'textPayload': PAYLOAD,
+            'dummyPayload': PAYLOAD,
             'logName': LOG_NAME,
             'insertId': IID,
             'timestamp': TIMESTAMP,
@@ -103,110 +107,7 @@ def test_from_api_repr_w_loggers_w_logger_match(self):
         TIMESTAMP = _datetime_to_rfc3339_w_nanos(NOW)
         LOG_NAME = 'projects/%s/logs/%s' % (self.PROJECT, self.LOGGER_NAME)
         API_REPR = {
-            'textPayload': PAYLOAD,
-            'logName': LOG_NAME,
-            'insertId': IID,
-            'timestamp': TIMESTAMP,
-        }
-        LOGGER = object()
-        loggers = {LOG_NAME: LOGGER}
-        klass = self._getTargetClass()
-        entry = klass.from_api_repr(API_REPR, client, loggers=loggers)
-        self.assertEqual(entry.payload, PAYLOAD)
-        self.assertEqual(entry.insert_id, IID)
-        self.assertEqual(entry.timestamp, NOW)
-        self.assertTrue(entry.logger is LOGGER)
-
-
-class TestStructEntry(unittest2.TestCase):
-
-    PROJECT = 'PROJECT'
-    LOGGER_NAME = 'LOGGER_NAME'
-
-    def _getTargetClass(self):
-        from gcloud.logging.entries import StructEntry
-        return StructEntry
-
-    def _makeOne(self, *args, **kw):
-        return self._getTargetClass()(*args, **kw)
-
-    def test_ctor_defaults(self):
-        PAYLOAD = {'message': 'MESSAGE', 'weather': 'partly cloudy'}
-        logger = _Logger(self.LOGGER_NAME, self.PROJECT)
-        entry = self._makeOne(PAYLOAD, logger)
-        self.assertEqual(entry.payload, PAYLOAD)
-        self.assertTrue(entry.logger is logger)
-        self.assertTrue(entry.insert_id is None)
-        self.assertTrue(entry.timestamp is None)
-
-    def test_ctor_explicit(self):
-        import datetime
-        PAYLOAD = {'message': 'MESSAGE', 'weather': 'partly cloudy'}
-        IID = 'IID'
-        TIMESTAMP = datetime.datetime.now()
-        logger = _Logger(self.LOGGER_NAME, self.PROJECT)
-        entry = self._makeOne(PAYLOAD, logger, IID, TIMESTAMP)
-        self.assertEqual(entry.payload, PAYLOAD)
-        self.assertTrue(entry.logger is logger)
-        self.assertEqual(entry.insert_id, IID)
-        self.assertEqual(entry.timestamp, TIMESTAMP)
-
-    def test_from_api_repr_missing_data_no_loggers(self):
-        client = _Client(self.PROJECT)
-        PAYLOAD = {'message': 'MESSAGE', 'weather': 'partly cloudy'}
-        LOG_NAME = 'projects/%s/logs/%s' % (self.PROJECT, self.LOGGER_NAME)
-        API_REPR = {
-            'jsonPayload': PAYLOAD,
-            'logName': LOG_NAME,
-        }
-        klass = self._getTargetClass()
-        entry = klass.from_api_repr(API_REPR, client)
-        self.assertEqual(entry.payload, PAYLOAD)
-        self.assertTrue(entry.insert_id is None)
-        self.assertTrue(entry.timestamp is None)
-        logger = entry.logger
-        self.assertTrue(isinstance(logger, _Logger))
-        self.assertTrue(logger.client is client)
-        self.assertEqual(logger.name, self.LOGGER_NAME)
-
-    def test_from_api_repr_w_loggers_no_logger_match(self):
-        from datetime import datetime
-        from gcloud._helpers import UTC
-        client = _Client(self.PROJECT)
-        PAYLOAD = {'message': 'MESSAGE', 'weather': 'partly cloudy'}
-        IID = 'IID'
-        NOW = datetime.utcnow().replace(tzinfo=UTC)
-        TIMESTAMP = _datetime_to_rfc3339_w_nanos(NOW)
-        LOG_NAME = 'projects/%s/logs/%s' % (self.PROJECT, self.LOGGER_NAME)
-        API_REPR = {
-            'jsonPayload': PAYLOAD,
-            'logName': LOG_NAME,
-            'insertId': IID,
-            'timestamp': TIMESTAMP,
-        }
-        loggers = {}
-        klass = self._getTargetClass()
-        entry = klass.from_api_repr(API_REPR, client, loggers=loggers)
-        self.assertEqual(entry.payload, PAYLOAD)
-        self.assertEqual(entry.insert_id, IID)
-        self.assertEqual(entry.timestamp, NOW)
-        logger = entry.logger
-        self.assertTrue(isinstance(logger, _Logger))
-        self.assertTrue(logger.client is client)
-        self.assertEqual(logger.name, self.LOGGER_NAME)
-        self.assertEqual(loggers, {LOG_NAME: logger})
-
-    def test_from_api_repr_w_loggers_w_logger_match(self):
-        from datetime import datetime
-        from gcloud._helpers import UTC
-        client = _Client(self.PROJECT)
-        PAYLOAD = {'message': 'MESSAGE', 'weather': 'partly cloudy'}
-        IID = 'IID'
-        NOW = datetime.utcnow().replace(tzinfo=UTC)
-        TIMESTAMP = _datetime_to_rfc3339_w_nanos(NOW)
-        LOG_NAME = 'projects/%s/logs/%s' % (self.PROJECT, self.LOGGER_NAME)
-        API_REPR = {
-            'jsonPayload': PAYLOAD,
+            'dummyPayload': PAYLOAD,
             'logName': LOG_NAME,
             'insertId': IID,
             'timestamp': TIMESTAMP,