From 015f75148dd37c758c87175c9f0c34158227f348 Mon Sep 17 00:00:00 2001 From: Kornelijus Survila Date: Mon, 3 Dec 2018 14:49:21 -0700 Subject: [PATCH 1/2] Fix span batching to stackdriver --- .../trace/exporters/stackdriver_exporter.py | 13 ++- .../exporters/test_stackdriver_exporter.py | 90 ++++++++----------- 2 files changed, 44 insertions(+), 59 deletions(-) diff --git a/opencensus/trace/exporters/stackdriver_exporter.py b/opencensus/trace/exporters/stackdriver_exporter.py index 95ae9f7ab..4aa28c660 100644 --- a/opencensus/trace/exporters/stackdriver_exporter.py +++ b/opencensus/trace/exporters/stackdriver_exporter.py @@ -215,13 +215,16 @@ def emit(self, span_datas): for sd in span_datas: trace_span_map[sd.context.trace_id] += [sd] + stackdriver_spans = [] # Write spans to Stackdriver for _, sds in trace_span_map.items(): # convert to the legacy trace json for easier refactoring # TODO: refactor this to use the span data directly trace = span_data.format_legacy_trace_json(sds) - stackdriver_spans = self.translate_to_stackdriver(trace) - self.client.batch_write_spans(project, stackdriver_spans) + stackdriver_spans.extend(self.translate_to_stackdriver(trace)) + + # FIXME: batch up to quota (25000 spans) + self.client.batch_write_spans(project, {'spans': stackdriver_spans}) def export(self, span_datas): """ @@ -247,7 +250,6 @@ def translate_to_stackdriver(self, trace): set_attributes(trace) spans_json = trace.get('spans') trace_id = trace.get('traceId') - spans_list = [] for span in spans_json: span_name = 'projects/{}/traces/{}/spans/{}'.format( @@ -272,10 +274,7 @@ def translate_to_stackdriver(self, trace): parent_span_id = str(span.get('parentSpanId')) span_json['parentSpanId'] = parent_span_id - spans_list.append(span_json) - - spans = {'spans': spans_list} - return spans + yield span_json def map_attributes(self, attribute_map): if attribute_map is None: diff --git a/tests/unit/trace/exporters/test_stackdriver_exporter.py b/tests/unit/trace/exporters/test_stackdriver_exporter.py index e02c36765..335d51c2c 100644 --- a/tests/unit/trace/exporters/test_stackdriver_exporter.py +++ b/tests/unit/trace/exporters/test_stackdriver_exporter.py @@ -200,63 +200,49 @@ def test_translate_to_stackdriver(self, mr_mock): exporter = stackdriver_exporter.StackdriverExporter( client=client, project_id=project_id) - spans = exporter.translate_to_stackdriver(trace) + spans = list(exporter.translate_to_stackdriver(trace)) - expected_traces = { - 'spans': [{ - 'name': - 'projects/{}/traces/{}/spans/{}'.format( + expected_traces = [{ + 'name': 'projects/{}/traces/{}/spans/{}'.format( project_id, trace_id, span_id), - 'displayName': { - 'value': span_name, - 'truncated_byte_count': 0 - }, - 'attributes': { - 'attributeMap': { - 'g.co/agent': { - 'string_value': { - 'truncated_byte_count': - 0, - 'value': + 'displayName': { + 'value': span_name, + 'truncated_byte_count': 0 + }, + 'attributes': { + 'attributeMap': { + 'g.co/agent': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'opencensus-python [{}]'.format(__version__) - } - }, - 'key': { - 'string_value': { - 'truncated_byte_count': 0, - 'value': 'value' - } - }, - '/http/host': { - 'string_value': { - 'truncated_byte_count': 0, - 'value': 'host' - } + } + }, + 'key': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'value' + } + }, + '/http/host': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'host' } } - }, - 'spanId': - str(span_id), - 'startTime': - start_time, - 'endTime': - end_time, - 'parentSpanId': - str(parent_span_id), - 'status': - None, - 'links': - None, - 'stackTrace': - None, - 'timeEvents': - None, - 'childSpanCount': - 0, - 'sameProcessAsParentSpan': - None - }] - } + } + }, + 'spanId': str(span_id), + 'startTime': start_time, + 'endTime': end_time, + 'parentSpanId': str(parent_span_id), + 'status': None, + 'links': None, + 'stackTrace': None, + 'timeEvents': None, + 'childSpanCount': 0, + 'sameProcessAsParentSpan': None + }] self.assertEqual(spans, expected_traces) From eee0c033ea623609fdde9ff5586d7b95cec628fd Mon Sep 17 00:00:00 2001 From: Chris Kleinknecht Date: Wed, 5 Dec 2018 13:28:54 -0800 Subject: [PATCH 2/2] Lose FIXME comment --- opencensus/trace/exporters/stackdriver_exporter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/opencensus/trace/exporters/stackdriver_exporter.py b/opencensus/trace/exporters/stackdriver_exporter.py index 4aa28c660..b4a269c33 100644 --- a/opencensus/trace/exporters/stackdriver_exporter.py +++ b/opencensus/trace/exporters/stackdriver_exporter.py @@ -223,7 +223,6 @@ def emit(self, span_datas): trace = span_data.format_legacy_trace_json(sds) stackdriver_spans.extend(self.translate_to_stackdriver(trace)) - # FIXME: batch up to quota (25000 spans) self.client.batch_write_spans(project, {'spans': stackdriver_spans}) def export(self, span_datas):