From 9fb194bfee50d4199e8c595d2dec8629e20f1b54 Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Sat, 8 May 2021 12:59:01 -0700 Subject: [PATCH 01/23] Initial implementation of autoinstrumentation. Works with the flask example. --- examples/server/Makefile | 4 +- examples/server/server.py | 7 +- setup.py | 7 +- src/hypertrace/agent/__init__.py | 2 +- .../agent/autoinstrumentation/__init__.py | 0 .../hypertrace_instrument.py | 56 +++++++ .../autoinstrumentation/sitecustomize.py | 10 ++ src/hypertrace/agent/config/config_pb2.py | 146 ++++++++---------- src/hypertrace/agent/init/__init__.py | 3 +- .../agent/instrumentation/flask/__init__.py | 34 +++- 10 files changed, 178 insertions(+), 91 deletions(-) create mode 100644 src/hypertrace/agent/autoinstrumentation/__init__.py create mode 100644 src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py create mode 100644 src/hypertrace/agent/autoinstrumentation/sitecustomize.py diff --git a/examples/server/Makefile b/examples/server/Makefile index 6352c658..a2e633b7 100644 --- a/examples/server/Makefile +++ b/examples/server/Makefile @@ -3,10 +3,10 @@ install-deps: run: @rm -rf __pycache__ || true - HT_CONFIG_FILE=./config.yaml FLASK_APP="server:create_app()" FLASK_ENV=development flask run -p 9000 + HT_ENABLE_CONSOLE_SPAN_EXPORTER=True HT_CONFIG_FILE=./config.yaml FLASK_APP="server:create_app()" FLASK_ENV=development hypertrace-instrument flask run -p 9000 run-hypertrace: docker-compose -f docker-compose-hypertrace.yml up --renew-anon-volumes -d run-mysql: - docker-compose -f ./mysql/docker-compose.yml up --renew-anon-volumes -d \ No newline at end of file + docker-compose -f ./mysql/docker-compose.yml up --renew-anon-volumes -d diff --git a/examples/server/server.py b/examples/server/server.py index 1baf37fe..f016d71b 100644 --- a/examples/server/server.py +++ b/examples/server/server.py @@ -1,5 +1,4 @@ from flask import Flask, request, jsonify, make_response, Response -from hypertrace.agent import Agent import time import json import logging @@ -37,9 +36,9 @@ def insert_user(name: str) -> int: def create_app(): app = Flask(__name__) - agent = Agent() - agent.register_flask_app(app) - agent.register_mysql() +# agent = Agent() +# agent.register_flask_app(app) +# agent.register_mysql() @app.route('/', methods=['POST']) def hello(): diff --git a/setup.py b/setup.py index 62c4c6ea..6f34b5c1 100644 --- a/setup.py +++ b/setup.py @@ -43,5 +43,10 @@ "google==3.0.0", "pyyaml", "protobuf==3.15.8" - ] + ], + entry_points = { + 'console_scripts': [ + 'hypertrace-instrument = hypertrace.agent.autoinstrumentation.hypertrace_instrument:run', + ], + } ) diff --git a/src/hypertrace/agent/__init__.py b/src/hypertrace/agent/__init__.py index 691efe56..30e792d1 100644 --- a/src/hypertrace/agent/__init__.py +++ b/src/hypertrace/agent/__init__.py @@ -72,7 +72,7 @@ def __init__(self): err, traceback.format_exc()) - def register_flask_app(self, app: flask.Flask) -> None: + def register_flask_app(self, app: flask.Flask = None) -> None: '''Register the flask instrumentation module wrapper''' logger.debug('Calling Agent.register_flask_app.') if not self.is_enabled(): diff --git a/src/hypertrace/agent/autoinstrumentation/__init__.py b/src/hypertrace/agent/autoinstrumentation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py b/src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py new file mode 100644 index 00000000..e7d10351 --- /dev/null +++ b/src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py @@ -0,0 +1,56 @@ +# Based upon the OTel autoinstrumentation feature +'''This module implements a CLI command that be used to +autoinstrument existing pythong programs that use supported +modules.''' +import argparse +from logging import getLogger +from os import environ, execl, getcwd +from os.path import abspath, dirname, pathsep +from shutil import which + +logger = getLogger(__file__) + +def parse_args(): + parser = argparse.ArgumentParser( + description=""" + hypertrace-instrument automatically instruments a Python + program and runs the program + """ + ) + + parser.add_argument("command", help="Your Python application.") + parser.add_argument( + "command_args", + help="Arguments for your application.", + nargs=argparse.REMAINDER, + ) + return parser.parse_args() + +def run() -> None: + args = parse_args() + + python_path = environ.get("PYTHONPATH") + + if not python_path: + python_path = [] + + else: + python_path = python_path.split(pathsep) + + cwd_path = getcwd() + + if cwd_path not in python_path: + python_path.insert(0, cwd_path) + + filedir_path = dirname(abspath(__file__)) + + python_path = [path for path in python_path if path != filedir_path] + + python_path.insert(0, filedir_path) + environ["PYTHONPATH"] = pathsep.join(python_path) + from flask import Flask + + executable = which(args.command) + execl(executable, executable, *args.command_args) + +run() diff --git a/src/hypertrace/agent/autoinstrumentation/sitecustomize.py b/src/hypertrace/agent/autoinstrumentation/sitecustomize.py new file mode 100644 index 00000000..47624f78 --- /dev/null +++ b/src/hypertrace/agent/autoinstrumentation/sitecustomize.py @@ -0,0 +1,10 @@ +'''Enable instrumentationon all supported modules.''' +from hypertrace.agent import Agent +agent = Agent() +agent.register_flask_app() +agent.register_grpc_server() +agent.register_grpc_client() +agent.register_mysql() +agent.register_postgresql() +agent.register_requests() +agent.register_aiohttp_client() diff --git a/src/hypertrace/agent/config/config_pb2.py b/src/hypertrace/agent/config/config_pb2.py index acefb025..8d80422c 100644 --- a/src/hypertrace/agent/config/config_pb2.py +++ b/src/hypertrace/agent/config/config_pb2.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: config.proto -"""Generated protocol buffer code.""" + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf.internal import enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message @@ -19,9 +20,8 @@ name='config.proto', package='org.hypertrace.agent.config', syntax='proto3', - serialized_options=b'\n\033org.hypertrace.agent.configZ$github.com/hypertrace/goagent/config', - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\x0c\x63onfig.proto\x12\x1borg.hypertrace.agent.config\x1a\x1egoogle/protobuf/wrappers.proto\"\x8b\x04\n\x0b\x41gentConfig\x12\x32\n\x0cservice_name\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x39\n\treporting\x18\x02 \x01(\x0b\x32&.org.hypertrace.agent.config.Reporting\x12>\n\x0c\x64\x61ta_capture\x18\x03 \x01(\x0b\x32(.org.hypertrace.agent.config.DataCapture\x12K\n\x13propagation_formats\x18\x04 \x03(\x0e\x32..org.hypertrace.agent.config.PropagationFormat\x12+\n\x07\x65nabled\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x39\n\tjavaagent\x18\x06 \x01(\x0b\x32&.org.hypertrace.agent.config.JavaAgent\x12]\n\x13resource_attributes\x18\x07 \x03(\x0b\x32@.org.hypertrace.agent.config.AgentConfig.ResourceAttributesEntry\x1a\x39\n\x17ResourceAttributesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x90\x02\n\tReporting\x12.\n\x08\x65ndpoint\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12*\n\x06secure\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12+\n\x05token\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x03opa\x18\x04 \x01(\x0b\x32 .org.hypertrace.agent.config.Opa\x12K\n\x13trace_reporter_type\x18\x05 \x01(\x0e\x32..org.hypertrace.agent.config.TraceReporterType\"\x9c\x01\n\x03Opa\x12.\n\x08\x65ndpoint\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x38\n\x13poll_period_seconds\x18\x02 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12+\n\x07\x65nabled\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"d\n\x07Message\x12+\n\x07request\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12,\n\x08response\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"\xb0\x02\n\x0b\x44\x61taCapture\x12:\n\x0chttp_headers\x18\x01 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12\x37\n\thttp_body\x18\x02 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12:\n\x0crpc_metadata\x18\x03 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12\x36\n\x08rpc_body\x18\x04 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12\x38\n\x13\x62ody_max_size_bytes\x18\x05 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\"C\n\tJavaAgent\x12\x36\n\x10\x66ilter_jar_paths\x18\x01 \x03(\x0b\x32\x1c.google.protobuf.StringValue*-\n\x11PropagationFormat\x12\x06\n\x02\x42\x33\x10\x00\x12\x10\n\x0cTRACECONTEXT\x10\x01*:\n\x11TraceReporterType\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\n\n\x06ZIPKIN\x10\x01\x12\x08\n\x04OTLP\x10\x02\x42\x43\n\x1borg.hypertrace.agent.configZ$github.com/hypertrace/goagent/configb\x06proto3' + serialized_options=_b('\n\033org.hypertrace.agent.configZ$github.com/hypertrace/goagent/config'), + serialized_pb=_b('\n\x0c\x63onfig.proto\x12\x1borg.hypertrace.agent.config\x1a\x1egoogle/protobuf/wrappers.proto\"\x8b\x04\n\x0b\x41gentConfig\x12\x32\n\x0cservice_name\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x39\n\treporting\x18\x02 \x01(\x0b\x32&.org.hypertrace.agent.config.Reporting\x12>\n\x0c\x64\x61ta_capture\x18\x03 \x01(\x0b\x32(.org.hypertrace.agent.config.DataCapture\x12K\n\x13propagation_formats\x18\x04 \x03(\x0e\x32..org.hypertrace.agent.config.PropagationFormat\x12+\n\x07\x65nabled\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x39\n\tjavaagent\x18\x06 \x01(\x0b\x32&.org.hypertrace.agent.config.JavaAgent\x12]\n\x13resource_attributes\x18\x07 \x03(\x0b\x32@.org.hypertrace.agent.config.AgentConfig.ResourceAttributesEntry\x1a\x39\n\x17ResourceAttributesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x90\x02\n\tReporting\x12.\n\x08\x65ndpoint\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12*\n\x06secure\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12+\n\x05token\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x03opa\x18\x04 \x01(\x0b\x32 .org.hypertrace.agent.config.Opa\x12K\n\x13trace_reporter_type\x18\x05 \x01(\x0e\x32..org.hypertrace.agent.config.TraceReporterType\"\x9c\x01\n\x03Opa\x12.\n\x08\x65ndpoint\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x38\n\x13poll_period_seconds\x18\x02 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12+\n\x07\x65nabled\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"d\n\x07Message\x12+\n\x07request\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12,\n\x08response\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"\xb0\x02\n\x0b\x44\x61taCapture\x12:\n\x0chttp_headers\x18\x01 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12\x37\n\thttp_body\x18\x02 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12:\n\x0crpc_metadata\x18\x03 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12\x36\n\x08rpc_body\x18\x04 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12\x38\n\x13\x62ody_max_size_bytes\x18\x05 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\"C\n\tJavaAgent\x12\x36\n\x10\x66ilter_jar_paths\x18\x01 \x03(\x0b\x32\x1c.google.protobuf.StringValue*-\n\x11PropagationFormat\x12\x06\n\x02\x42\x33\x10\x00\x12\x10\n\x0cTRACECONTEXT\x10\x01*:\n\x11TraceReporterType\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\n\n\x06ZIPKIN\x10\x01\x12\x08\n\x04OTLP\x10\x02\x42\x43\n\x1borg.hypertrace.agent.configZ$github.com/hypertrace/goagent/configb\x06proto3') , dependencies=[google_dot_protobuf_dot_wrappers__pb2.DESCRIPTOR,]) @@ -30,18 +30,15 @@ full_name='org.hypertrace.agent.config.PropagationFormat', filename=None, file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, values=[ _descriptor.EnumValueDescriptor( name='B3', index=0, number=0, serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), + type=None), _descriptor.EnumValueDescriptor( name='TRACECONTEXT', index=1, number=1, serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), + type=None), ], containing_type=None, serialized_options=None, @@ -56,23 +53,19 @@ full_name='org.hypertrace.agent.config.TraceReporterType', filename=None, file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, values=[ _descriptor.EnumValueDescriptor( name='UNSPECIFIED', index=0, number=0, serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), + type=None), _descriptor.EnumValueDescriptor( name='ZIPKIN', index=1, number=1, serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), + type=None), _descriptor.EnumValueDescriptor( name='OTLP', index=2, number=2, serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), + type=None), ], containing_type=None, serialized_options=None, @@ -96,29 +89,28 @@ filename=None, file=DESCRIPTOR, containing_type=None, - create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='key', full_name='org.hypertrace.agent.config.AgentConfig.ResourceAttributesEntry.key', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), + has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='value', full_name='org.hypertrace.agent.config.AgentConfig.ResourceAttributesEntry.value', index=1, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), + has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - serialized_options=b'8\001', + serialized_options=_b('8\001'), is_extendable=False, syntax='proto3', extension_ranges=[], @@ -134,7 +126,6 @@ filename=None, file=DESCRIPTOR, containing_type=None, - create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='service_name', full_name='org.hypertrace.agent.config.AgentConfig.service_name', index=0, @@ -142,49 +133,49 @@ has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='reporting', full_name='org.hypertrace.agent.config.AgentConfig.reporting', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='data_capture', full_name='org.hypertrace.agent.config.AgentConfig.data_capture', index=2, number=3, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='propagation_formats', full_name='org.hypertrace.agent.config.AgentConfig.propagation_formats', index=3, number=4, type=14, cpp_type=8, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='enabled', full_name='org.hypertrace.agent.config.AgentConfig.enabled', index=4, number=5, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='javaagent', full_name='org.hypertrace.agent.config.AgentConfig.javaagent', index=5, number=6, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='resource_attributes', full_name='org.hypertrace.agent.config.AgentConfig.resource_attributes', index=6, number=7, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -208,7 +199,6 @@ filename=None, file=DESCRIPTOR, containing_type=None, - create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='endpoint', full_name='org.hypertrace.agent.config.Reporting.endpoint', index=0, @@ -216,35 +206,35 @@ has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='secure', full_name='org.hypertrace.agent.config.Reporting.secure', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='token', full_name='org.hypertrace.agent.config.Reporting.token', index=2, number=3, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='opa', full_name='org.hypertrace.agent.config.Reporting.opa', index=3, number=4, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='trace_reporter_type', full_name='org.hypertrace.agent.config.Reporting.trace_reporter_type', index=4, number=5, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -268,7 +258,6 @@ filename=None, file=DESCRIPTOR, containing_type=None, - create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='endpoint', full_name='org.hypertrace.agent.config.Opa.endpoint', index=0, @@ -276,21 +265,21 @@ has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='poll_period_seconds', full_name='org.hypertrace.agent.config.Opa.poll_period_seconds', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='enabled', full_name='org.hypertrace.agent.config.Opa.enabled', index=2, number=3, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -314,7 +303,6 @@ filename=None, file=DESCRIPTOR, containing_type=None, - create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='request', full_name='org.hypertrace.agent.config.Message.request', index=0, @@ -322,14 +310,14 @@ has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='response', full_name='org.hypertrace.agent.config.Message.response', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -353,7 +341,6 @@ filename=None, file=DESCRIPTOR, containing_type=None, - create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='http_headers', full_name='org.hypertrace.agent.config.DataCapture.http_headers', index=0, @@ -361,35 +348,35 @@ has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='http_body', full_name='org.hypertrace.agent.config.DataCapture.http_body', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='rpc_metadata', full_name='org.hypertrace.agent.config.DataCapture.rpc_metadata', index=2, number=3, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='rpc_body', full_name='org.hypertrace.agent.config.DataCapture.rpc_body', index=3, number=4, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='body_max_size_bytes', full_name='org.hypertrace.agent.config.DataCapture.body_max_size_bytes', index=4, number=5, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -413,7 +400,6 @@ filename=None, file=DESCRIPTOR, containing_type=None, - create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='filter_jar_paths', full_name='org.hypertrace.agent.config.JavaAgent.filter_jar_paths', index=0, @@ -421,7 +407,7 @@ has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -472,54 +458,54 @@ DESCRIPTOR.enum_types_by_name['TraceReporterType'] = _TRACEREPORTERTYPE _sym_db.RegisterFileDescriptor(DESCRIPTOR) -AgentConfig = _reflection.GeneratedProtocolMessageType('AgentConfig', (_message.Message,), { +AgentConfig = _reflection.GeneratedProtocolMessageType('AgentConfig', (_message.Message,), dict( - 'ResourceAttributesEntry' : _reflection.GeneratedProtocolMessageType('ResourceAttributesEntry', (_message.Message,), { - 'DESCRIPTOR' : _AGENTCONFIG_RESOURCEATTRIBUTESENTRY, - '__module__' : 'config_pb2' + ResourceAttributesEntry = _reflection.GeneratedProtocolMessageType('ResourceAttributesEntry', (_message.Message,), dict( + DESCRIPTOR = _AGENTCONFIG_RESOURCEATTRIBUTESENTRY, + __module__ = 'config_pb2' # @@protoc_insertion_point(class_scope:org.hypertrace.agent.config.AgentConfig.ResourceAttributesEntry) - }) + )) , - 'DESCRIPTOR' : _AGENTCONFIG, - '__module__' : 'config_pb2' + DESCRIPTOR = _AGENTCONFIG, + __module__ = 'config_pb2' # @@protoc_insertion_point(class_scope:org.hypertrace.agent.config.AgentConfig) - }) + )) _sym_db.RegisterMessage(AgentConfig) _sym_db.RegisterMessage(AgentConfig.ResourceAttributesEntry) -Reporting = _reflection.GeneratedProtocolMessageType('Reporting', (_message.Message,), { - 'DESCRIPTOR' : _REPORTING, - '__module__' : 'config_pb2' +Reporting = _reflection.GeneratedProtocolMessageType('Reporting', (_message.Message,), dict( + DESCRIPTOR = _REPORTING, + __module__ = 'config_pb2' # @@protoc_insertion_point(class_scope:org.hypertrace.agent.config.Reporting) - }) + )) _sym_db.RegisterMessage(Reporting) -Opa = _reflection.GeneratedProtocolMessageType('Opa', (_message.Message,), { - 'DESCRIPTOR' : _OPA, - '__module__' : 'config_pb2' +Opa = _reflection.GeneratedProtocolMessageType('Opa', (_message.Message,), dict( + DESCRIPTOR = _OPA, + __module__ = 'config_pb2' # @@protoc_insertion_point(class_scope:org.hypertrace.agent.config.Opa) - }) + )) _sym_db.RegisterMessage(Opa) -Message = _reflection.GeneratedProtocolMessageType('Message', (_message.Message,), { - 'DESCRIPTOR' : _MESSAGE, - '__module__' : 'config_pb2' +Message = _reflection.GeneratedProtocolMessageType('Message', (_message.Message,), dict( + DESCRIPTOR = _MESSAGE, + __module__ = 'config_pb2' # @@protoc_insertion_point(class_scope:org.hypertrace.agent.config.Message) - }) + )) _sym_db.RegisterMessage(Message) -DataCapture = _reflection.GeneratedProtocolMessageType('DataCapture', (_message.Message,), { - 'DESCRIPTOR' : _DATACAPTURE, - '__module__' : 'config_pb2' +DataCapture = _reflection.GeneratedProtocolMessageType('DataCapture', (_message.Message,), dict( + DESCRIPTOR = _DATACAPTURE, + __module__ = 'config_pb2' # @@protoc_insertion_point(class_scope:org.hypertrace.agent.config.DataCapture) - }) + )) _sym_db.RegisterMessage(DataCapture) -JavaAgent = _reflection.GeneratedProtocolMessageType('JavaAgent', (_message.Message,), { - 'DESCRIPTOR' : _JAVAAGENT, - '__module__' : 'config_pb2' +JavaAgent = _reflection.GeneratedProtocolMessageType('JavaAgent', (_message.Message,), dict( + DESCRIPTOR = _JAVAAGENT, + __module__ = 'config_pb2' # @@protoc_insertion_point(class_scope:org.hypertrace.agent.config.JavaAgent) - }) + )) _sym_db.RegisterMessage(JavaAgent) diff --git a/src/hypertrace/agent/init/__init__.py b/src/hypertrace/agent/init/__init__.py index fca9365a..7701dca3 100644 --- a/src/hypertrace/agent/init/__init__.py +++ b/src/hypertrace/agent/init/__init__.py @@ -109,7 +109,8 @@ def flask_init(self, app) -> None: from hypertrace.agent.instrumentation.flask import FlaskInstrumentorWrapper # pylint: disable=C0415 self._modules_initialized['flask'] = True self._flask_instrumentor_wrapper = FlaskInstrumentorWrapper() - self._flask_instrumentor_wrapper.instrument_app(app) + if app: + self._flask_instrumentor_wrapper.instrument_app(app) self.init_instrumentor_wrapper_base_for_http( self._flask_instrumentor_wrapper) self.init_propagation() diff --git a/src/hypertrace/agent/instrumentation/flask/__init__.py b/src/hypertrace/agent/instrumentation/flask/__init__.py index b76fe90d..2eb99296 100644 --- a/src/hypertrace/agent/instrumentation/flask/__init__.py +++ b/src/hypertrace/agent/instrumentation/flask/__init__.py @@ -7,6 +7,7 @@ import json import flask from opentelemetry.instrumentation.flask import ( + _InstrumentedFlask, FlaskInstrumentor, get_default_span_name, _teardown_request, @@ -14,6 +15,8 @@ ) from hypertrace.agent import constants # pylint: disable=R0801 from hypertrace.agent.instrumentation import BaseInstrumentorWrapper +from hypertrace.agent.init import AgentInit +from hypertrace.agent.config import AgentConfig # Initialize logger logger = logging.getLogger(__name__) # pylint: disable=C0103 @@ -66,8 +69,6 @@ def hypertrace_before_request() -> None: return hypertrace_before_request # Per request post-handler - - def _hypertrace_after_request(flask_wrapper) -> flask.wrappers.Response: '''This function is invoked by flask to set the handler''' def hypertrace_after_request(response): @@ -97,6 +98,26 @@ def hypertrace_after_request(response): return hypertrace_after_request +class _HypertraceInstrumentedFlask(_InstrumentedFlask, BaseInstrumentorWrapper): + + def __init__(self, *args, **kwargs): + _InstrumentedFlask.__init__(self,*args, **kwargs) + BaseInstrumentorWrapper.__init__(self) + self.before_request(_hypertrace_before_request(self)) + self.after_request(_hypertrace_after_request(self)) + config: AgentConfig = AgentConfig() + self.set_process_request_headers( + config.agent_config.data_capture.http_headers.request) + self.set_process_request_body( + config.agent_config.data_capture.http_body.request) + self.set_process_response_headers( + config.agent_config.data_capture.http_headers.response) + self.set_process_response_body( + config.agent_config.data_capture.http_body.response) + self.set_body_max_size( + config.agent_config.data_capture.body_max_size_bytes) + + # Main Flask Instrumentor Wrapper class. class FlaskInstrumentorWrapper(FlaskInstrumentor, BaseInstrumentorWrapper): '''Hypertrace wrapper around OTel Flask instrumentor class''' @@ -106,6 +127,15 @@ def __init__(self): super().__init__() self._app = None + def _instrument(self, **kwargs): + self._original_flask = flask.Flask + name_callback = kwargs.get("name_callback") + tracer_provider = kwargs.get("tracer_provider") + if callable(name_callback): + _HypertraceInstrumentedFlask.name_callback = name_callback + _HypertraceInstrumentedFlask._tracer_provider = tracer_provider + flask.Flask = _HypertraceInstrumentedFlask + # Initialize instrumentation wrapper def instrument_app(self, app, name_callback=get_default_span_name) -> None: '''Initialize instrumentation''' From f86287e2fb264e0cace1d032e6fa8692bbd30a9d Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Sat, 8 May 2021 18:20:51 -0700 Subject: [PATCH 02/23] Adding ability to selectively enable modules to autoinstrumentation command. --- examples/server/Makefile | 2 +- .../hypertrace_instrument.py | 5 ++- .../autoinstrumentation/sitecustomize.py | 41 +++++++++++++++---- .../agent/instrumentation/flask/__init__.py | 4 +- 4 files changed, 39 insertions(+), 13 deletions(-) diff --git a/examples/server/Makefile b/examples/server/Makefile index a2e633b7..eb8f28ff 100644 --- a/examples/server/Makefile +++ b/examples/server/Makefile @@ -3,7 +3,7 @@ install-deps: run: @rm -rf __pycache__ || true - HT_ENABLE_CONSOLE_SPAN_EXPORTER=True HT_CONFIG_FILE=./config.yaml FLASK_APP="server:create_app()" FLASK_ENV=development hypertrace-instrument flask run -p 9000 + HT_INSTRUMENTED_MODULES="flask,mysql" HT_CONFIG_FILE=./config.yaml FLASK_APP="server:create_app()" FLASK_ENV=development hypertrace-instrument flask run -p 9000 run-hypertrace: docker-compose -f docker-compose-hypertrace.yml up --renew-anon-volumes -d diff --git a/src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py b/src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py index e7d10351..b0d6c8ff 100644 --- a/src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py +++ b/src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py @@ -11,6 +11,7 @@ logger = getLogger(__file__) def parse_args(): + '''Parse CLI arguments.''' parser = argparse.ArgumentParser( description=""" hypertrace-instrument automatically instruments a Python @@ -19,14 +20,17 @@ def parse_args(): ) parser.add_argument("command", help="Your Python application.") + parser.add_argument( "command_args", help="Arguments for your application.", nargs=argparse.REMAINDER, ) + return parser.parse_args() def run() -> None: + '''hypertrace-instrument Entry point''' args = parse_args() python_path = environ.get("PYTHONPATH") @@ -48,7 +52,6 @@ def run() -> None: python_path.insert(0, filedir_path) environ["PYTHONPATH"] = pathsep.join(python_path) - from flask import Flask executable = which(args.command) execl(executable, executable, *args.command_args) diff --git a/src/hypertrace/agent/autoinstrumentation/sitecustomize.py b/src/hypertrace/agent/autoinstrumentation/sitecustomize.py index 47624f78..d7345557 100644 --- a/src/hypertrace/agent/autoinstrumentation/sitecustomize.py +++ b/src/hypertrace/agent/autoinstrumentation/sitecustomize.py @@ -1,10 +1,33 @@ -'''Enable instrumentationon all supported modules.''' -from hypertrace.agent import Agent +'''Enable instrumentationon all supported modules.''' # pylint: disable=R0401 +import os # pylint: disable=R0401 +import logging +from hypertrace.agent import Agent # pylint: disable=R0401 + +# Initialize logger +logger = logging.getLogger(__name__) # pylint: disable=C0103 + +if 'HT_INSTRUMENTED_MODULES' in os.environ: + logger.debug("[env] Loaded HT_INSTRUMENTED_MODULES from env") + modules_ = os.environ['HT_INSTRUMENTED_MODULES'] + +modules_array = modules_.split(',') + agent = Agent() -agent.register_flask_app() -agent.register_grpc_server() -agent.register_grpc_client() -agent.register_mysql() -agent.register_postgresql() -agent.register_requests() -agent.register_aiohttp_client() +for mod in modules_array: + if mod is None or len(mod) == 0: + continue + + if mod == 'flask': + agent.register_flask_app() + if mod == 'grpc:server': + agent.register_grpc_server() + if mod == 'grpc:client': + agent.register_grpc_client() + if mod == 'mysql': + agent.register_mysql() + if mod == 'postgresql': + agent.register_postgresql() + if mod == 'requests': + agent.register_requests() + if mod == 'aiohttp-client': + agent.register_aiohttp_client() diff --git a/src/hypertrace/agent/instrumentation/flask/__init__.py b/src/hypertrace/agent/instrumentation/flask/__init__.py index 0630c62b..19a07b70 100644 --- a/src/hypertrace/agent/instrumentation/flask/__init__.py +++ b/src/hypertrace/agent/instrumentation/flask/__init__.py @@ -133,12 +133,12 @@ def __init__(self): def _instrument(self, **kwargs): '''Override OTel method that sets up global flask instrumentation''' - self._original_flask = flask.Flask + self._original_flask = flask.Flask # pylint: disable = W0201 name_callback = kwargs.get("name_callback") tracer_provider = kwargs.get("tracer_provider") if callable(name_callback): _HypertraceInstrumentedFlask.name_callback = name_callback - _HypertraceInstrumentedFlask._tracer_provider = tracer_provider + _HypertraceInstrumentedFlask._tracer_provider = tracer_provider # pylint: disable=W0212 flask.Flask = _HypertraceInstrumentedFlask # Initialize instrumentation wrapper From 7472e8fe35d4f359b9828910291700ffcaf0d2ce Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Sat, 8 May 2021 18:33:50 -0700 Subject: [PATCH 03/23] By default, enable instrumentation on all supported modules. --- examples/server/Makefile | 2 +- .../agent/autoinstrumentation/sitecustomize.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/examples/server/Makefile b/examples/server/Makefile index eb8f28ff..63101e09 100644 --- a/examples/server/Makefile +++ b/examples/server/Makefile @@ -3,7 +3,7 @@ install-deps: run: @rm -rf __pycache__ || true - HT_INSTRUMENTED_MODULES="flask,mysql" HT_CONFIG_FILE=./config.yaml FLASK_APP="server:create_app()" FLASK_ENV=development hypertrace-instrument flask run -p 9000 + HT_CONFIG_FILE=./config.yaml FLASK_APP="server:create_app()" FLASK_ENV=development hypertrace-instrument flask run -p 9000 run-hypertrace: docker-compose -f docker-compose-hypertrace.yml up --renew-anon-volumes -d diff --git a/src/hypertrace/agent/autoinstrumentation/sitecustomize.py b/src/hypertrace/agent/autoinstrumentation/sitecustomize.py index d7345557..80b1db25 100644 --- a/src/hypertrace/agent/autoinstrumentation/sitecustomize.py +++ b/src/hypertrace/agent/autoinstrumentation/sitecustomize.py @@ -3,16 +3,31 @@ import logging from hypertrace.agent import Agent # pylint: disable=R0401 +DEFAULTS = [ + 'flask', + 'mysql', + 'postgresql', + 'grpc:server', + 'grpc:client', + 'requests', + 'aiohttp' +] + # Initialize logger logger = logging.getLogger(__name__) # pylint: disable=C0103 +modules_ = '' if 'HT_INSTRUMENTED_MODULES' in os.environ: logger.debug("[env] Loaded HT_INSTRUMENTED_MODULES from env") modules_ = os.environ['HT_INSTRUMENTED_MODULES'] modules_array = modules_.split(',') +if len(modules_array) == 1 and modules_array[0] == '': + modules_array = DEFAULTS + agent = Agent() + for mod in modules_array: if mod is None or len(mod) == 0: continue From 879d67db4628b4d09db0a1cbd0788186ddc891a9 Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Sun, 9 May 2021 18:12:15 -0700 Subject: [PATCH 04/23] Adding hypertrace-instrument tests to standard test suite. --- Makefile | 1 - .../hypertrace_instrument.py | 3 +- tests/autoinstrumentation/docker-compose.yml | 21 ++++ tests/autoinstrumentation/docker-healthcheck | 32 +++++ tests/autoinstrumentation/requirements.txt | 17 +++ .../sql/docker-compose.yml | 15 +++ tests/autoinstrumentation/sql/init.sql | 4 + tests/autoinstrumentation/test_flask_1.py | 96 +++++++++++++++ tests/autoinstrumentation/test_flask_1.sh | 61 +++++++++ tests/autoinstrumentation/test_flask_2.py | 116 ++++++++++++++++++ tests/autoinstrumentation/test_flask_2.sh | 34 +++++ tests/autoinstrumentation/tox.ini | 38 ++++++ 12 files changed, 436 insertions(+), 2 deletions(-) create mode 100644 tests/autoinstrumentation/docker-compose.yml create mode 100755 tests/autoinstrumentation/docker-healthcheck create mode 100644 tests/autoinstrumentation/requirements.txt create mode 100644 tests/autoinstrumentation/sql/docker-compose.yml create mode 100644 tests/autoinstrumentation/sql/init.sql create mode 100644 tests/autoinstrumentation/test_flask_1.py create mode 100755 tests/autoinstrumentation/test_flask_1.sh create mode 100644 tests/autoinstrumentation/test_flask_2.py create mode 100755 tests/autoinstrumentation/test_flask_2.sh create mode 100644 tests/autoinstrumentation/tox.ini diff --git a/Makefile b/Makefile index 0e895965..723842c2 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,6 @@ test-integration: cd ${TEST_DIR}/gunicorn; HT_LOG_LEVEL=${LOG_LEVEL} tox -e ${PY_TARGET} cd ${TEST_DIR}/requests; HT_LOG_LEVEL=${LOG_LEVEL} tox -e ${PY_TARGET} cd ${TEST_DIR}/aiohttp; HT_LOG_LEVEL=${LOG_LEVEL} tox -e ${PY_TARGET} - cd ${TEST_DIR}/docker; HT_LOG_LEVEL=${LOG_LEVEL} tox -e ${PY_TARGET} .PHONY: test test: test-unit test-integration diff --git a/src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py b/src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py index b0d6c8ff..c17878cd 100644 --- a/src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py +++ b/src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py @@ -56,4 +56,5 @@ def run() -> None: executable = which(args.command) execl(executable, executable, *args.command_args) -run() +if __name__ == '__main__': + run() diff --git a/tests/autoinstrumentation/docker-compose.yml b/tests/autoinstrumentation/docker-compose.yml new file mode 100644 index 00000000..b08dfea9 --- /dev/null +++ b/tests/autoinstrumentation/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3.1' + +services: + mysqldb: + image: mysql + container_name: mysqldb + command: --default-authentication-plugin=mysql_native_password + restart: always + ports: + - "3306:3306" + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: hypertrace + volumes: + - ./sql:/docker-entrypoint-initdb.d + - ./docker-healthcheck:/docker-healthcheck + healthcheck: + test: [ "CMD", "/docker-healthcheck" ] + timeout: 40s + interval: 3s + retries: 20 diff --git a/tests/autoinstrumentation/docker-healthcheck b/tests/autoinstrumentation/docker-healthcheck new file mode 100755 index 00000000..6c22eedc --- /dev/null +++ b/tests/autoinstrumentation/docker-healthcheck @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copied from https://github.com/docker-library/healthcheck/blob/master/mysql/docker-healthcheck + +set -eo pipefail + +if [ "$MYSQL_RANDOM_ROOT_PASSWORD" ] && [ -z "$MYSQL_USER" ] && [ -z "$MYSQL_PASSWORD" ]; then + # there's no way we can guess what the random MySQL password was + echo >&2 'healthcheck error: cannot determine random root password (and MYSQL_USER and MYSQL_PASSWORD were not set)' + exit 0 +fi + +host="$(hostname --ip-address || echo '127.0.0.1')" +user="${MYSQL_USER:-root}" +export MYSQL_PWD="${MYSQL_PASSWORD:-$MYSQL_ROOT_PASSWORD}" + +args=( + # force mysql to not use the local "mysqld.sock" (test "external" connectibility) + -h"$host" + -u"$user" + --silent +) + +if command -v mysqladmin &> /dev/null; then + if mysqladmin "${args[@]}" ping > /dev/null; then + if select="$(echo 'SELECT 1' | mysql "${args[@]}")" && [ "$select" = '1' ]; then + exit 0 + fi + fi +fi + +exit 1 \ No newline at end of file diff --git a/tests/autoinstrumentation/requirements.txt b/tests/autoinstrumentation/requirements.txt new file mode 100644 index 00000000..d63e7fb5 --- /dev/null +++ b/tests/autoinstrumentation/requirements.txt @@ -0,0 +1,17 @@ +opentelemetry-api==1.1.0 +opentelemetry-instrumentation +opentelemetry-exporter-zipkin +opentelemetry-exporter-otlp==1.1.0 +opentelemetry-instrumentation-flask +opentelemetry-instrumentation-mysql +opentelemetry-instrumentation-requests +opentelemetry-instrumentation-wsgi +opentelemetry-util-http +google +protobuf +pyyaml +flask +pytest +pytest-xdist +mysql-connector-python==8.0.23 + diff --git a/tests/autoinstrumentation/sql/docker-compose.yml b/tests/autoinstrumentation/sql/docker-compose.yml new file mode 100644 index 00000000..148c3250 --- /dev/null +++ b/tests/autoinstrumentation/sql/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3.1' + +services: + + mysqldb: + image: mysql + command: --default-authentication-plugin=mysql_native_password + restart: always + ports: + - "3306:3306" + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: test + volumes: + - ./init.sql:/docker-entrypoint-initdb.d/init.sql diff --git a/tests/autoinstrumentation/sql/init.sql b/tests/autoinstrumentation/sql/init.sql new file mode 100644 index 00000000..7ec704d3 --- /dev/null +++ b/tests/autoinstrumentation/sql/init.sql @@ -0,0 +1,4 @@ +create table hypertrace_data( + col1 INT NOT NULL, + col2 VARCHAR(100) NOT NULL +); diff --git a/tests/autoinstrumentation/test_flask_1.py b/tests/autoinstrumentation/test_flask_1.py new file mode 100644 index 00000000..077e038e --- /dev/null +++ b/tests/autoinstrumentation/test_flask_1.py @@ -0,0 +1,96 @@ +import sys +import os +import logging +import flask +import pytest +import traceback +import json +from werkzeug.serving import make_server +from flask import request +import time +import atexit +import threading +from flask import Flask +#from opentelemetry.exporter.jaeger.thrift import JaegerExporter +from opentelemetry import trace as trace_api +from opentelemetry.sdk.trace import TracerProvider, export +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.sdk.trace.export import BatchSpanProcessor, SimpleSpanProcessor + +def setup_custom_logger(name): + try: + formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S') + handler = logging.FileHandler('agent.log', mode='a') + handler.setFormatter(formatter) + screen_handler = logging.StreamHandler(stream=sys.stdout) + screen_handler.setFormatter(formatter) + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.addHandler(handler) + logger.addHandler(screen_handler) + return logger + except: + print('Failed to customize logger: exception=%s, stacktrace=%s', + sys.exc_info()[0], + traceback.format_exc()) + +# Run the flask web server in a separate thread +class FlaskServer(threading.Thread): + def __init__(self, app): + super().__init__() + self.daemon = True + threading.Thread.__init__(self) + self.srv = make_server('localhost', 5000, app) + self.ctx = app.app_context() + self.ctx.push() + + def run(self): + logger.info('starting server.') + self.srv.serve_forever() + self.start() + +def test_run(): + logger = setup_custom_logger(__name__) + logger.info('Initializing flask app.') + # Create Flask app + app = Flask(__name__) + @app.before_first_request + def before_first_request(): + logger.debug("test_program: before_first_request() called") + + @app.route("/route1") + def testAPI1(): + logger.info('Serving request for /route1.') + response = flask.Response(mimetype='application/json') + response.headers['tester3'] = 'tester3' + response.data = str('{ "a": "a", "xyz": "xyz" }') + return response + + logger.info('Flask app initialized.') + + server = FlaskServer(app) + + logger.info('Running test calls.') + with app.test_client() as c: + try: + logger.info('Making test call to /route1') + r1 = app.test_client().get( + 'http://localhost:5000/route1', + headers={ 'tester1': 'tester1', + 'tester2':'tester2' + } + ) + logger.info('Reading /route1 response.') + a1 = r1.get_json()['a'] + assert a1 == 'a' + logger.info('r1 result: ' + str(a1)) + return 0 + except: + logger.error('Failed to initialize flask instrumentation wrapper: exception=%s, stacktrace=%s', + sys.exc_info()[0], + traceback.format_exc()) + raise sys.exc_info()[0] + +if __name__ == '__main__': + test_run() diff --git a/tests/autoinstrumentation/test_flask_1.sh b/tests/autoinstrumentation/test_flask_1.sh new file mode 100755 index 00000000..dad9c29f --- /dev/null +++ b/tests/autoinstrumentation/test_flask_1.sh @@ -0,0 +1,61 @@ +#!/bin/bash +PYTHON_PATH=../../src:$PYTHON_PATH +SPAN=`HT_INSTRUMENTED_MODULES="flask" HT_ENABLE_CONSOLE_SPAN_EXPORTER=True python ../../src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py python test_flask_1.py | egrep -v "INFO|DEBUG"` +echo SPAN=${SPAN} +TRACE_ID=`echo ${SPAN} | jq .context.trace_id | sed 's/\"//g'` +echo TRACE_ID=$TRACE_ID +if [ -z "${TRACE_ID}" ]; +then + echo "Didn't find TRACE_ID." + exit 1 +fi +METHOD=`echo ${SPAN} | jq '.attributes."http.method"' | sed 's/\"//g'` +echo METHOD=${METHOD} +if [ "${METHOD}" != "GET" ]; +then + echo "Didn't find METHOD." + exit 1 +fi +TARGET=`echo ${SPAN} | jq '.attributes."http.target"' | sed 's/\"//g'` +echo TARGET=${TARGET} +if [ "${TARGET}" != "/route1" ]; +then + echo "Didn't find TARGET." + exit 1 +fi +RESPONSE_CONTENT_TYPE=`echo ${SPAN} | jq '.attributes."http.response.header.content-type"' | sed 's/\"//g'` +echo RESPONSE_CONTENT_TYPE=${RESPONSE_CONTENT_TYPE} +if [ "${RESPONSE_CONTENT_TYPE}" != "application/json" ]; +then + echo "Didn't find RESPONSE_CONTENT_TYPE." + exit 1 +fi +CUSTOM_REQUEST_HEADER=`echo ${SPAN} | jq '.attributes."http.request.header.tester1"' | sed 's/\"//g'` +echo CUSTOM_REQUEST_HEADER=${CUSTOM_REQUEST_HEADER} +if [ "${CUSTOM_REQUEST_HEADER}" != "tester1" ]; +then + echo "Didn't find CUSTOM_REQUEST_HEADER." + exit 1 +fi +CUSTOM_RESPONSE_HEADER=`echo ${SPAN} | jq '.attributes."http.response.header.tester3"' | sed 's/\"//g'` +echo CUSTOM_RESPONSE_HEADER=${CUSTOM_RESPONSE_HEADER} +if [ "${CUSTOM_RESPONSE_HEADER}" != "tester3" ]; +then + echo "Didn't find CUSTOM_RESPONSE_HEADER." + exit 1 +fi +RESPONSE_BODY=`echo ${SPAN} | jq '.attributes."http.response.body"' | sed 's/\"//g'` +echo RESPONSE_BODY=${RESPONSE_BODY} +if [ "${RESPONSE_BODY}" != "{ \\"a\\": \\"a\\", \\"xyz\\": \\"xyz\\" }" ]; +then + echo "Didn't find RESPONSE_BODY." + exit 1 +fi +HTTP_STATUS_CODE=`echo ${SPAN} | jq '.attributes."http.status_code"' | sed 's/\"//g'` +echo HTTP_STATUS_CODE=${HTTP_STATUS_CODE} +if [ "${HTTP_STATUS_CODE}" != "200" ]; +then + echo "Didn't find HTTP_STATUS_CODE." + exit 1 +fi +exit 0 diff --git a/tests/autoinstrumentation/test_flask_2.py b/tests/autoinstrumentation/test_flask_2.py new file mode 100644 index 00000000..cb0c303a --- /dev/null +++ b/tests/autoinstrumentation/test_flask_2.py @@ -0,0 +1,116 @@ +import sys +import os +import logging +import flask +import pytest +import traceback +import json +from werkzeug.serving import make_server +from flask import request +import time +import atexit +import threading +from flask import Flask +import mysql.connector + +def setup_custom_logger(name): + try: + formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S') + handler = logging.FileHandler('agent.log', mode='a') + handler.setFormatter(formatter) + screen_handler = logging.StreamHandler(stream=sys.stdout) + screen_handler.setFormatter(formatter) + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.addHandler(handler) + logger.addHandler(screen_handler) + return logger + except: + print('Failed to customize logger: exception=%s, stacktrace=%s', + sys.exc_info()[0], + traceback.format_exc()) + +# Run the flask web server in a separate thread +class FlaskServer(threading.Thread): + def __init__(self, app): + super().__init__() + self.daemon = True + threading.Thread.__init__(self) + self.srv = make_server('localhost', 5000, app) + self.ctx = app.app_context() + self.ctx.push() + + def run(self): + logger.info('starting server.') + self.srv.serve_forever() + self.start() + +def test_run(): + logger = setup_custom_logger(__name__) + logger.info('Initializing flask app.') + # Create Flask app + app = Flask(__name__) + @app.before_first_request + def before_first_request(): + logger.debug("test_program: before_first_request() called") + + @app.route("/route1") + def testAPI1(): + logger.info('Serving request for /route1.') + try: + logger.info('Making connection to mysql.') + cnx = mysql.connector.connect(database='hypertrace', + username='root', + password='root', + host='localhost', + port=3306) + logger.info('Connect successfully.') + cursor = cnx.cursor() + logger.info('Running INSERT statement.') + cursor.execute( + "INSERT INTO hypertrace_data (col1, col2) VALUES (123, 'abcdefghijklmnopqrstuvwxyz')") + logger.info('Statement ran successfully') + logger.info('Closing cursor.') + cursor.close() + logger.info('Closing connection.') + cnx.close() + logger.info('Connection closed.') + + response = flask.Response(mimetype='application/json') + response.headers['tester3'] = 'tester3' + response.data = str('{ "a": "a", "xyz": "xyz" }') + return response + except: + logger.error('Failed to initialize mysql instrumentation wrapper: exception=%s, stacktrace=%s', + sys.exc_info()[0], + traceback.format_exc()) + raise sys.exc_info()[0] + + logger.info('Flask app initialized.') + + server = FlaskServer(app) + + logger.info('Running test calls.') + with app.test_client() as c: + try: + logger.info('Making test call to /route1') + r1 = app.test_client().get( + 'http://localhost:5000/route1', + headers={ 'tester1': 'tester1', + 'tester2':'tester2' + } + ) + logger.info('Reading /route1 response.') + a1 = r1.get_json()['a'] + assert a1 == 'a' + logger.info('r1 result: ' + str(a1)) + return 0 + except: + logger.error('Failed to initialize flask instrumentation wrapper: exception=%s, stacktrace=%s', + sys.exc_info()[0], + traceback.format_exc()) + raise sys.exc_info()[0] + +if __name__ == '__main__': + test_run() diff --git a/tests/autoinstrumentation/test_flask_2.sh b/tests/autoinstrumentation/test_flask_2.sh new file mode 100755 index 00000000..888dfcc5 --- /dev/null +++ b/tests/autoinstrumentation/test_flask_2.sh @@ -0,0 +1,34 @@ +#!/bin/bash +PYTHON_PATH=../../src:$PYTHON_PATH +SPAN=`HT_INSTRUMENTED_MODULES="mysql" HT_ENABLE_CONSOLE_SPAN_EXPORTER=True python ../../src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py python test_flask_2.py | egrep -v "INFO|DEBUG"` +echo SPAN=${SPAN} +TRACE_ID=`echo ${SPAN} | jq .context.trace_id | sed 's/\"//g'` +echo TRACE_ID=$TRACE_ID +if [ -z "${TRACE_ID}" ]; +then + echo "Didn't find TRACE_ID." + exit 1 +fi +SQL_STATEMENT=`echo ${SPAN} | jq '.attributes."db.statement"' | sed 's/\"//g'` +echo SQL_STATEMENT=${SQL_STATEMENT} +if [ "${SQL_STATEMENT}" != "INSERT INTO hypertrace_data (col1, col2) VALUES (123, 'abcdefghijklmnopqrstuvwxyz')" ]; +then + echo "Didn't find HTTP_STATUS_CODE." + exit 1 +fi +DB_USER=`echo ${SPAN} | jq '.attributes."db.user"' | sed 's/\"//g'` +echo DB_USER=${DB_USER} +if [ "${DB_USER}" != "root" ]; +then + echo "Didn't find DB_USER." + exit 1 +fi +STATUS_CODE=`echo ${SPAN} | jq '.status."status_code"' | sed 's/\"//g'` +echo STATUS_CODE=${STATUS_CODE} +if [ "${STATUS_CODE}" != "UNSET" ]; +then + echo "Didn't find STATUS_CODE." + exit 1 +fi + +exit 0 diff --git a/tests/autoinstrumentation/tox.ini b/tests/autoinstrumentation/tox.ini new file mode 100644 index 00000000..40927640 --- /dev/null +++ b/tests/autoinstrumentation/tox.ini @@ -0,0 +1,38 @@ +[tox] +skipsdist = true +#envlist = py3{7,8,9} +envlist = py37 + +[testenv] +sitepackages = True +install_command = pip install {opts} {packages} + +deps = + -rrequirements.txt + +whitelist_externals = + pytest + sleep + docker-compose + test_flask_1.sh + bash + test_flask_2.sh + +setenv = + PYTHONPATH=../../src + HT_SERVICE_NAME=python_agent_test1 + HT_LOG_LEVEL={env:HT_LOG_LEVEL:} + HT_ENABLE_CONSOLE_SPAN_EXPORTER=True + +commands = + docker-compose stop mysqldb + bash -ec "if [ `uname` = 'Linux' ] && [ "{env:GITHUB_ACTIONS:false}" != "true" ]; then chcon -h system_u:object_r:bin_t:s0 sql; chcon -Rt svirt_sandbox_file_t sql; fi" + bash -ec "if [ `uname` = 'Linux' ] && [ "{env:GITHUB_ACTIONS:false}" != "true" ]; then chcon -h system_u:object_r:bin_t:s0 docker-healthcheck; chcon -Rt svirt_sandbox_file_t docker-healthcheck; fi" + docker-compose up -d --remove-orphans --force-recreate -V mysqldb + bash -ec "set -x; while [ `docker inspect -f '\{\{ .State.Health.Status \}\}' mysqldb` != 'healthy' ]; do echo 'Waiting for MySQL to be up'; sleep 2; done" +# ./test_flask_1.sh + ./test_flask_2.sh +# docker-compose stop mysqldb +# docker-compose down --rmi all + +recreate = True From e888128f1ed92f81e1d9ce9ed8400305dd1cb813 Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Sun, 9 May 2021 18:51:59 -0700 Subject: [PATCH 05/23] Regenerating pydocs. Added autoinstrumentation example. --- Makefile | 1 + .../hypertrace_instrument.html | 198 ++++++++++++++++++ .../agent/autoinstrumentation/index.html | 72 +++++++ .../autoinstrumentation/sitecustomize.html | 107 ++++++++++ docs/hypertrace/agent/config/config_pb2.html | 151 ++++++------- docs/hypertrace/agent/config/index.html | 2 +- docs/hypertrace/agent/index.html | 13 +- docs/hypertrace/agent/init/index.html | 9 +- .../agent/instrumentation/aiohttp/index.html | 2 +- .../agent/instrumentation/flask/index.html | 51 ++++- .../agent/instrumentation/grpc/index.html | 9 +- .../agent/instrumentation/index.html | 1 + docs/hypertrace/version.html | 2 +- examples/autoinstrumentation/Makefile | 12 ++ examples/autoinstrumentation/config.yaml | 6 + .../docker-compose-hypertrace.yml | 115 ++++++++++ .../mysql/docker-compose.yml | 15 ++ examples/autoinstrumentation/mysql/init.sql | 4 + examples/autoinstrumentation/requirements.txt | 3 + examples/autoinstrumentation/server.py | 58 +++++ examples/server/Makefile | 2 +- examples/server/server.py | 7 +- .../autoinstrumentation/sitecustomize.py | 10 +- .../agent/instrumentation/flask/__init__.py | 2 +- tests/autoinstrumentation/tox.ini | 9 +- 25 files changed, 738 insertions(+), 123 deletions(-) create mode 100644 docs/hypertrace/agent/autoinstrumentation/hypertrace_instrument.html create mode 100644 docs/hypertrace/agent/autoinstrumentation/index.html create mode 100644 docs/hypertrace/agent/autoinstrumentation/sitecustomize.html create mode 100644 examples/autoinstrumentation/Makefile create mode 100644 examples/autoinstrumentation/config.yaml create mode 100644 examples/autoinstrumentation/docker-compose-hypertrace.yml create mode 100644 examples/autoinstrumentation/mysql/docker-compose.yml create mode 100644 examples/autoinstrumentation/mysql/init.sql create mode 100644 examples/autoinstrumentation/requirements.txt create mode 100644 examples/autoinstrumentation/server.py diff --git a/Makefile b/Makefile index 723842c2..f2ace2a3 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ test-integration: cd ${TEST_DIR}/gunicorn; HT_LOG_LEVEL=${LOG_LEVEL} tox -e ${PY_TARGET} cd ${TEST_DIR}/requests; HT_LOG_LEVEL=${LOG_LEVEL} tox -e ${PY_TARGET} cd ${TEST_DIR}/aiohttp; HT_LOG_LEVEL=${LOG_LEVEL} tox -e ${PY_TARGET} + cd ${TEST_DIR}/autoinstrumentation; HT_LOG_LEVEL=${LOG_LEVEL} tox -e ${PY_TARGET} .PHONY: test test: test-unit test-integration diff --git a/docs/hypertrace/agent/autoinstrumentation/hypertrace_instrument.html b/docs/hypertrace/agent/autoinstrumentation/hypertrace_instrument.html new file mode 100644 index 00000000..cf71d302 --- /dev/null +++ b/docs/hypertrace/agent/autoinstrumentation/hypertrace_instrument.html @@ -0,0 +1,198 @@ + + + + + + +hypertrace.agent.autoinstrumentation.hypertrace_instrument API documentation + + + + + + + + + + + +
+
+
+

Module hypertrace.agent.autoinstrumentation.hypertrace_instrument

+
+
+

This module implements a CLI command that be used to +autoinstrument existing pythong programs that use supported +modules.

+
+ +Expand source code + +
# Based upon the OTel autoinstrumentation feature
+'''This module implements a CLI command that be used to
+autoinstrument existing pythong programs that use supported
+modules.'''
+import argparse
+from logging import getLogger
+from os import environ, execl, getcwd
+from os.path import abspath, dirname, pathsep
+from shutil import which
+
+logger = getLogger(__file__)
+
+def parse_args():
+    '''Parse CLI arguments.'''
+    parser = argparse.ArgumentParser(
+        description="""
+        hypertrace-instrument automatically instruments a Python
+        program and runs the program
+        """
+    )
+
+    parser.add_argument("command", help="Your Python application.")
+
+    parser.add_argument(
+        "command_args",
+        help="Arguments for your application.",
+        nargs=argparse.REMAINDER,
+    )
+
+    return parser.parse_args()
+
+def run() -> None:
+    '''hypertrace-instrument Entry point'''
+    args = parse_args()
+
+    python_path = environ.get("PYTHONPATH")
+
+    if not python_path:
+        python_path = []
+
+    else:
+        python_path = python_path.split(pathsep)
+
+    cwd_path = getcwd()
+
+    if cwd_path not in python_path:
+        python_path.insert(0, cwd_path)
+
+    filedir_path = dirname(abspath(__file__))
+
+    python_path = [path for path in python_path if path != filedir_path]
+
+    python_path.insert(0, filedir_path)
+    environ["PYTHONPATH"] = pathsep.join(python_path)
+
+    executable = which(args.command)
+    execl(executable, executable, *args.command_args)
+
+if __name__ == '__main__':
+    run()
+
+
+
+
+
+
+
+

Functions

+
+
+def parse_args() +
+
+

Parse CLI arguments.

+
+ +Expand source code + +
def parse_args():
+    '''Parse CLI arguments.'''
+    parser = argparse.ArgumentParser(
+        description="""
+        hypertrace-instrument automatically instruments a Python
+        program and runs the program
+        """
+    )
+
+    parser.add_argument("command", help="Your Python application.")
+
+    parser.add_argument(
+        "command_args",
+        help="Arguments for your application.",
+        nargs=argparse.REMAINDER,
+    )
+
+    return parser.parse_args()
+
+
+
+def run() ‑> NoneType +
+
+

hypertrace-instrument Entry point

+
+ +Expand source code + +
def run() -> None:
+    '''hypertrace-instrument Entry point'''
+    args = parse_args()
+
+    python_path = environ.get("PYTHONPATH")
+
+    if not python_path:
+        python_path = []
+
+    else:
+        python_path = python_path.split(pathsep)
+
+    cwd_path = getcwd()
+
+    if cwd_path not in python_path:
+        python_path.insert(0, cwd_path)
+
+    filedir_path = dirname(abspath(__file__))
+
+    python_path = [path for path in python_path if path != filedir_path]
+
+    python_path.insert(0, filedir_path)
+    environ["PYTHONPATH"] = pathsep.join(python_path)
+
+    executable = which(args.command)
+    execl(executable, executable, *args.command_args)
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/docs/hypertrace/agent/autoinstrumentation/index.html b/docs/hypertrace/agent/autoinstrumentation/index.html new file mode 100644 index 00000000..4d64a44a --- /dev/null +++ b/docs/hypertrace/agent/autoinstrumentation/index.html @@ -0,0 +1,72 @@ + + + + + + +hypertrace.agent.autoinstrumentation API documentation + + + + + + + + + + + +
+ + +
+ + + \ No newline at end of file diff --git a/docs/hypertrace/agent/autoinstrumentation/sitecustomize.html b/docs/hypertrace/agent/autoinstrumentation/sitecustomize.html new file mode 100644 index 00000000..bbc8121f --- /dev/null +++ b/docs/hypertrace/agent/autoinstrumentation/sitecustomize.html @@ -0,0 +1,107 @@ + + + + + + +hypertrace.agent.autoinstrumentation.sitecustomize API documentation + + + + + + + + + + + +
+
+
+

Module hypertrace.agent.autoinstrumentation.sitecustomize

+
+
+

Enable instrumentationon all supported modules.

+
+ +Expand source code + +
'''Enable instrumentationon all supported modules.''' # pylint: disable=R0401
+import os
+import logging
+from hypertrace.agent import Agent
+
+DEFAULTS = [
+    'flask',
+    'mysql',
+    'postgresql',
+    'grpc:server',
+    'grpc:client',
+    'requests',
+    'aiohttp'
+]
+
+# Initialize logger
+logger = logging.getLogger(__name__)  # pylint: disable=C0103
+
+MODULES = ''
+if 'HT_INSTRUMENTED_MODULES' in os.environ:
+    logger.debug("[env] Loaded HT_INSTRUMENTED_MODULES from env")
+    MODULES = os.environ['HT_INSTRUMENTED_MODULES']
+
+modules_array = MODULES.split(',')
+
+if len(modules_array) == 1 and modules_array[0] == '':
+    modules_array = DEFAULTS
+
+agent = Agent()
+
+for mod in modules_array:
+    if mod is None or len(mod) == 0:
+        continue
+
+    if mod == 'flask':
+        agent.register_flask_app()
+    if mod == 'grpc:server':
+        agent.register_grpc_server()
+    if mod == 'grpc:client':
+        agent.register_grpc_client()
+    if mod == 'mysql':
+        agent.register_mysql()
+    if mod == 'postgresql':
+        agent.register_postgresql()
+    if mod == 'requests':
+        agent.register_requests()
+    if mod == 'aiohttp-client':
+        agent.register_aiohttp_client()
+
+
+
+
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/docs/hypertrace/agent/config/config_pb2.html b/docs/hypertrace/agent/config/config_pb2.html index 5dbed03b..34c053a1 100644 --- a/docs/hypertrace/agent/config/config_pb2.html +++ b/docs/hypertrace/agent/config/config_pb2.html @@ -5,7 +5,7 @@ hypertrace.agent.config.config_pb2 API documentation - + @@ -22,15 +22,15 @@

Module hypertrace.agent.config.config_pb2

-

Generated protocol buffer code.

Expand source code -
# -*- coding: utf-8 -*-
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
+
# Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: config.proto
-"""Generated protocol buffer code."""
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
 from google.protobuf.internal import enum_type_wrapper
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import message as _message
@@ -48,9 +48,8 @@ 
Module hypertrace.agent.config.config_pb2

   name='config.proto',
   package='org.hypertrace.agent.config',
   syntax='proto3',
-  serialized_options=b'\n\033org.hypertrace.agent.configZ$github.com/hypertrace/goagent/config',
-  create_key=_descriptor._internal_create_key,
-  serialized_pb=b'\n\x0c\x63onfig.proto\x12\x1borg.hypertrace.agent.config\x1a\x1egoogle/protobuf/wrappers.proto\"\x8b\x04\n\x0b\x41gentConfig\x12\x32\n\x0cservice_name\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x39\n\treporting\x18\x02 \x01(\x0b\x32&.org.hypertrace.agent.config.Reporting\x12>\n\x0c\x64\x61ta_capture\x18\x03 \x01(\x0b\x32(.org.hypertrace.agent.config.DataCapture\x12K\n\x13propagation_formats\x18\x04 \x03(\x0e\x32..org.hypertrace.agent.config.PropagationFormat\x12+\n\x07\x65nabled\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x39\n\tjavaagent\x18\x06 \x01(\x0b\x32&.org.hypertrace.agent.config.JavaAgent\x12]\n\x13resource_attributes\x18\x07 \x03(\x0b\x32@.org.hypertrace.agent.config.AgentConfig.ResourceAttributesEntry\x1a\x39\n\x17ResourceAttributesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x90\x02\n\tReporting\x12.\n\x08\x65ndpoint\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12*\n\x06secure\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12+\n\x05token\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x03opa\x18\x04 \x01(\x0b\x32 .org.hypertrace.agent.config.Opa\x12K\n\x13trace_reporter_type\x18\x05 \x01(\x0e\x32..org.hypertrace.agent.config.TraceReporterType\"\x9c\x01\n\x03Opa\x12.\n\x08\x65ndpoint\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x38\n\x13poll_period_seconds\x18\x02 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12+\n\x07\x65nabled\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"d\n\x07Message\x12+\n\x07request\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12,\n\x08response\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"\xb0\x02\n\x0b\x44\x61taCapture\x12:\n\x0chttp_headers\x18\x01 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12\x37\n\thttp_body\x18\x02 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12:\n\x0crpc_metadata\x18\x03 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12\x36\n\x08rpc_body\x18\x04 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12\x38\n\x13\x62ody_max_size_bytes\x18\x05 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\"C\n\tJavaAgent\x12\x36\n\x10\x66ilter_jar_paths\x18\x01 \x03(\x0b\x32\x1c.google.protobuf.StringValue*-\n\x11PropagationFormat\x12\x06\n\x02\x42\x33\x10\x00\x12\x10\n\x0cTRACECONTEXT\x10\x01*:\n\x11TraceReporterType\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\n\n\x06ZIPKIN\x10\x01\x12\x08\n\x04OTLP\x10\x02\x42\x43\n\x1borg.hypertrace.agent.configZ$github.com/hypertrace/goagent/configb\x06proto3'
+  serialized_options=_b('\n\033org.hypertrace.agent.configZ$github.com/hypertrace/goagent/config'),
+  serialized_pb=_b('\n\x0c\x63onfig.proto\x12\x1borg.hypertrace.agent.config\x1a\x1egoogle/protobuf/wrappers.proto\"\x8b\x04\n\x0b\x41gentConfig\x12\x32\n\x0cservice_name\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x39\n\treporting\x18\x02 \x01(\x0b\x32&.org.hypertrace.agent.config.Reporting\x12>\n\x0c\x64\x61ta_capture\x18\x03 \x01(\x0b\x32(.org.hypertrace.agent.config.DataCapture\x12K\n\x13propagation_formats\x18\x04 \x03(\x0e\x32..org.hypertrace.agent.config.PropagationFormat\x12+\n\x07\x65nabled\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x39\n\tjavaagent\x18\x06 \x01(\x0b\x32&.org.hypertrace.agent.config.JavaAgent\x12]\n\x13resource_attributes\x18\x07 \x03(\x0b\x32@.org.hypertrace.agent.config.AgentConfig.ResourceAttributesEntry\x1a\x39\n\x17ResourceAttributesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x90\x02\n\tReporting\x12.\n\x08\x65ndpoint\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12*\n\x06secure\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12+\n\x05token\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x03opa\x18\x04 \x01(\x0b\x32 .org.hypertrace.agent.config.Opa\x12K\n\x13trace_reporter_type\x18\x05 \x01(\x0e\x32..org.hypertrace.agent.config.TraceReporterType\"\x9c\x01\n\x03Opa\x12.\n\x08\x65ndpoint\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x38\n\x13poll_period_seconds\x18\x02 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12+\n\x07\x65nabled\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"d\n\x07Message\x12+\n\x07request\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12,\n\x08response\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"\xb0\x02\n\x0b\x44\x61taCapture\x12:\n\x0chttp_headers\x18\x01 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12\x37\n\thttp_body\x18\x02 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12:\n\x0crpc_metadata\x18\x03 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12\x36\n\x08rpc_body\x18\x04 \x01(\x0b\x32$.org.hypertrace.agent.config.Message\x12\x38\n\x13\x62ody_max_size_bytes\x18\x05 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\"C\n\tJavaAgent\x12\x36\n\x10\x66ilter_jar_paths\x18\x01 \x03(\x0b\x32\x1c.google.protobuf.StringValue*-\n\x11PropagationFormat\x12\x06\n\x02\x42\x33\x10\x00\x12\x10\n\x0cTRACECONTEXT\x10\x01*:\n\x11TraceReporterType\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\n\n\x06ZIPKIN\x10\x01\x12\x08\n\x04OTLP\x10\x02\x42\x43\n\x1borg.hypertrace.agent.configZ$github.com/hypertrace/goagent/configb\x06proto3')
   ,
   dependencies=[google_dot_protobuf_dot_wrappers__pb2.DESCRIPTOR,])
 
@@ -59,18 +58,15 @@ 
Module hypertrace.agent.config.config_pb2

   full_name='org.hypertrace.agent.config.PropagationFormat',
   filename=None,
   file=DESCRIPTOR,
-  create_key=_descriptor._internal_create_key,
   values=[
     _descriptor.EnumValueDescriptor(
       name='B3', index=0, number=0,
       serialized_options=None,
-      type=None,
-      create_key=_descriptor._internal_create_key),
+      type=None),
     _descriptor.EnumValueDescriptor(
       name='TRACECONTEXT', index=1, number=1,
       serialized_options=None,
-      type=None,
-      create_key=_descriptor._internal_create_key),
+      type=None),
   ],
   containing_type=None,
   serialized_options=None,
@@ -85,23 +81,19 @@ 
Module hypertrace.agent.config.config_pb2

   full_name='org.hypertrace.agent.config.TraceReporterType',
   filename=None,
   file=DESCRIPTOR,
-  create_key=_descriptor._internal_create_key,
   values=[
     _descriptor.EnumValueDescriptor(
       name='UNSPECIFIED', index=0, number=0,
       serialized_options=None,
-      type=None,
-      create_key=_descriptor._internal_create_key),
+      type=None),
     _descriptor.EnumValueDescriptor(
       name='ZIPKIN', index=1, number=1,
       serialized_options=None,
-      type=None,
-      create_key=_descriptor._internal_create_key),
+      type=None),
     _descriptor.EnumValueDescriptor(
       name='OTLP', index=2, number=2,
       serialized_options=None,
-      type=None,
-      create_key=_descriptor._internal_create_key),
+      type=None),
   ],
   containing_type=None,
   serialized_options=None,
@@ -125,29 +117,28 @@ 
Module hypertrace.agent.config.config_pb2

   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
-  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='key', full_name='org.hypertrace.agent.config.AgentConfig.ResourceAttributesEntry.key', index=0,
       number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=b"".decode('utf-8'),
+      has_default_value=False, default_value=_b("").decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='value', full_name='org.hypertrace.agent.config.AgentConfig.ResourceAttributesEntry.value', index=1,
       number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=b"".decode('utf-8'),
+      has_default_value=False, default_value=_b("").decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
   nested_types=[],
   enum_types=[
   ],
-  serialized_options=b'8\001',
+  serialized_options=_b('8\001'),
   is_extendable=False,
   syntax='proto3',
   extension_ranges=[],
@@ -163,7 +154,6 @@ 
Module hypertrace.agent.config.config_pb2

   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
-  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='service_name', full_name='org.hypertrace.agent.config.AgentConfig.service_name', index=0,
@@ -171,49 +161,49 @@ 
Module hypertrace.agent.config.config_pb2

       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='reporting', full_name='org.hypertrace.agent.config.AgentConfig.reporting', index=1,
       number=2, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='data_capture', full_name='org.hypertrace.agent.config.AgentConfig.data_capture', index=2,
       number=3, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='propagation_formats', full_name='org.hypertrace.agent.config.AgentConfig.propagation_formats', index=3,
       number=4, type=14, cpp_type=8, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='enabled', full_name='org.hypertrace.agent.config.AgentConfig.enabled', index=4,
       number=5, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='javaagent', full_name='org.hypertrace.agent.config.AgentConfig.javaagent', index=5,
       number=6, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='resource_attributes', full_name='org.hypertrace.agent.config.AgentConfig.resource_attributes', index=6,
       number=7, type=11, cpp_type=10, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
@@ -237,7 +227,6 @@ 
Module hypertrace.agent.config.config_pb2

   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
-  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='endpoint', full_name='org.hypertrace.agent.config.Reporting.endpoint', index=0,
@@ -245,35 +234,35 @@ 
Module hypertrace.agent.config.config_pb2

       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='secure', full_name='org.hypertrace.agent.config.Reporting.secure', index=1,
       number=2, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='token', full_name='org.hypertrace.agent.config.Reporting.token', index=2,
       number=3, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='opa', full_name='org.hypertrace.agent.config.Reporting.opa', index=3,
       number=4, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='trace_reporter_type', full_name='org.hypertrace.agent.config.Reporting.trace_reporter_type', index=4,
       number=5, type=14, cpp_type=8, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
@@ -297,7 +286,6 @@ 
Module hypertrace.agent.config.config_pb2

   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
-  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='endpoint', full_name='org.hypertrace.agent.config.Opa.endpoint', index=0,
@@ -305,21 +293,21 @@ 
Module hypertrace.agent.config.config_pb2

       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='poll_period_seconds', full_name='org.hypertrace.agent.config.Opa.poll_period_seconds', index=1,
       number=2, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='enabled', full_name='org.hypertrace.agent.config.Opa.enabled', index=2,
       number=3, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
@@ -343,7 +331,6 @@ 
Module hypertrace.agent.config.config_pb2

   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
-  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='request', full_name='org.hypertrace.agent.config.Message.request', index=0,
@@ -351,14 +338,14 @@ 
Module hypertrace.agent.config.config_pb2

       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='response', full_name='org.hypertrace.agent.config.Message.response', index=1,
       number=2, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
@@ -382,7 +369,6 @@ 
Module hypertrace.agent.config.config_pb2

   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
-  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='http_headers', full_name='org.hypertrace.agent.config.DataCapture.http_headers', index=0,
@@ -390,35 +376,35 @@ 
Module hypertrace.agent.config.config_pb2

       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='http_body', full_name='org.hypertrace.agent.config.DataCapture.http_body', index=1,
       number=2, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='rpc_metadata', full_name='org.hypertrace.agent.config.DataCapture.rpc_metadata', index=2,
       number=3, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='rpc_body', full_name='org.hypertrace.agent.config.DataCapture.rpc_body', index=3,
       number=4, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='body_max_size_bytes', full_name='org.hypertrace.agent.config.DataCapture.body_max_size_bytes', index=4,
       number=5, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
@@ -442,7 +428,6 @@ 
Module hypertrace.agent.config.config_pb2

   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
-  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='filter_jar_paths', full_name='org.hypertrace.agent.config.JavaAgent.filter_jar_paths', index=0,
@@ -450,7 +435,7 @@ 
Module hypertrace.agent.config.config_pb2

       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+      serialized_options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
@@ -501,54 +486,54 @@ 
Module hypertrace.agent.config.config_pb2

 DESCRIPTOR.enum_types_by_name['TraceReporterType'] = _TRACEREPORTERTYPE
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
-AgentConfig = _reflection.GeneratedProtocolMessageType('AgentConfig', (_message.Message,), {
+AgentConfig = _reflection.GeneratedProtocolMessageType('AgentConfig', (_message.Message,), dict(
 
-  'ResourceAttributesEntry' : _reflection.GeneratedProtocolMessageType('ResourceAttributesEntry', (_message.Message,), {
-    'DESCRIPTOR' : _AGENTCONFIG_RESOURCEATTRIBUTESENTRY,
-    '__module__' : 'config_pb2'
+  ResourceAttributesEntry = _reflection.GeneratedProtocolMessageType('ResourceAttributesEntry', (_message.Message,), dict(
+    DESCRIPTOR = _AGENTCONFIG_RESOURCEATTRIBUTESENTRY,
+    __module__ = 'config_pb2'
     # @@protoc_insertion_point(class_scope:org.hypertrace.agent.config.AgentConfig.ResourceAttributesEntry)
-    })
+    ))
   ,
-  'DESCRIPTOR' : _AGENTCONFIG,
-  '__module__' : 'config_pb2'
+  DESCRIPTOR = _AGENTCONFIG,
+  __module__ = 'config_pb2'
   # @@protoc_insertion_point(class_scope:org.hypertrace.agent.config.AgentConfig)
-  })
+  ))
 _sym_db.RegisterMessage(AgentConfig)
 _sym_db.RegisterMessage(AgentConfig.ResourceAttributesEntry)
 
-Reporting = _reflection.GeneratedProtocolMessageType('Reporting', (_message.Message,), {
-  'DESCRIPTOR' : _REPORTING,
-  '__module__' : 'config_pb2'
+Reporting = _reflection.GeneratedProtocolMessageType('Reporting', (_message.Message,), dict(
+  DESCRIPTOR = _REPORTING,
+  __module__ = 'config_pb2'
   # @@protoc_insertion_point(class_scope:org.hypertrace.agent.config.Reporting)
-  })
+  ))
 _sym_db.RegisterMessage(Reporting)
 
-Opa = _reflection.GeneratedProtocolMessageType('Opa', (_message.Message,), {
-  'DESCRIPTOR' : _OPA,
-  '__module__' : 'config_pb2'
+Opa = _reflection.GeneratedProtocolMessageType('Opa', (_message.Message,), dict(
+  DESCRIPTOR = _OPA,
+  __module__ = 'config_pb2'
   # @@protoc_insertion_point(class_scope:org.hypertrace.agent.config.Opa)
-  })
+  ))
 _sym_db.RegisterMessage(Opa)
 
-Message = _reflection.GeneratedProtocolMessageType('Message', (_message.Message,), {
-  'DESCRIPTOR' : _MESSAGE,
-  '__module__' : 'config_pb2'
+Message = _reflection.GeneratedProtocolMessageType('Message', (_message.Message,), dict(
+  DESCRIPTOR = _MESSAGE,
+  __module__ = 'config_pb2'
   # @@protoc_insertion_point(class_scope:org.hypertrace.agent.config.Message)
-  })
+  ))
 _sym_db.RegisterMessage(Message)
 
-DataCapture = _reflection.GeneratedProtocolMessageType('DataCapture', (_message.Message,), {
-  'DESCRIPTOR' : _DATACAPTURE,
-  '__module__' : 'config_pb2'
+DataCapture = _reflection.GeneratedProtocolMessageType('DataCapture', (_message.Message,), dict(
+  DESCRIPTOR = _DATACAPTURE,
+  __module__ = 'config_pb2'
   # @@protoc_insertion_point(class_scope:org.hypertrace.agent.config.DataCapture)
-  })
+  ))
 _sym_db.RegisterMessage(DataCapture)
 
-JavaAgent = _reflection.GeneratedProtocolMessageType('JavaAgent', (_message.Message,), {
-  'DESCRIPTOR' : _JAVAAGENT,
-  '__module__' : 'config_pb2'
+JavaAgent = _reflection.GeneratedProtocolMessageType('JavaAgent', (_message.Message,), dict(
+  DESCRIPTOR = _JAVAAGENT,
+  __module__ = 'config_pb2'
   # @@protoc_insertion_point(class_scope:org.hypertrace.agent.config.JavaAgent)
-  })
+  ))
 _sym_db.RegisterMessage(JavaAgent)
 
 
diff --git a/docs/hypertrace/agent/config/index.html b/docs/hypertrace/agent/config/index.html
index b5325cfb..9503ad73 100644
--- a/docs/hypertrace/agent/config/index.html
+++ b/docs/hypertrace/agent/config/index.html
@@ -252,7 +252,7 @@ 
Sub-modules

 

 
hypertrace.agent.config.config_pb2

 

-
Generated protocol buffer code.

+

 

 
hypertrace.agent.config.default

 

diff --git a/docs/hypertrace/agent/index.html b/docs/hypertrace/agent/index.html
index fea47d67..551ff5a9 100644
--- a/docs/hypertrace/agent/index.html
+++ b/docs/hypertrace/agent/index.html
@@ -101,7 +101,7 @@ 
Module hypertrace.agent

                          err,
                          traceback.format_exc())
 
-    def register_flask_app(self, app: flask.Flask) -> None:
+    def register_flask_app(self, app: flask.Flask = None) -> None:
         '''Register the flask instrumentation module wrapper'''
         logger.debug('Calling Agent.register_flask_app.')
         if not self.is_enabled():
@@ -219,6 +219,10 @@ 
Module hypertrace.agent

 

 
Sub-modules

 

+
hypertrace.agent.autoinstrumentation

+

+

+

 
hypertrace.agent.config

 

 
Agent configuration logic that pull in values from a defaults list,
@@ -321,7 +325,7 @@ 
Classes

                          err,
                          traceback.format_exc())
 
-    def register_flask_app(self, app: flask.Flask) -> None:
+    def register_flask_app(self, app: flask.Flask = None) -> None:
         '''Register the flask instrumentation module wrapper'''
         logger.debug('Calling Agent.register_flask_app.')
         if not self.is_enabled():
@@ -480,7 +484,7 @@ 
Methods
-def register_flask_app(self, app: flask.app.Flask) ‑> NoneType +def register_flask_app(self, app: flask.app.Flask = None) ‑> NoneType

Register the flask instrumentation module wrapper

@@ -488,7 +492,7 @@

Methods

Expand source code -
def register_flask_app(self, app: flask.Flask) -> None:
+
def register_flask_app(self, app: flask.Flask = None) -> None:
     '''Register the flask instrumentation module wrapper'''
     logger.debug('Calling Agent.register_flask_app.')
     if not self.is_enabled():
@@ -659,6 +663,7 @@ 
Index

 
 
  • Sub-modules
    
 
      
+
    • hypertrace.agent.autoinstrumentation
      • 
 
    • hypertrace.agent.config
      • 
 
    • hypertrace.agent.constants
      • 
 
    • hypertrace.agent.init
      • 
diff --git a/docs/hypertrace/agent/init/index.html b/docs/hypertrace/agent/init/index.html
index f4fa1c7b..f203e59b 100644
--- a/docs/hypertrace/agent/init/index.html
+++ b/docs/hypertrace/agent/init/index.html
@@ -155,7 +155,8 @@ 
      Module hypertrace.agent.init
      
             from hypertrace.agent.instrumentation.flask import FlaskInstrumentorWrapper  # pylint: disable=C0415
             self._modules_initialized['flask'] = True
             self._flask_instrumentor_wrapper = FlaskInstrumentorWrapper()
-            self._flask_instrumentor_wrapper.instrument_app(app)
+            if app:
+                self._flask_instrumentor_wrapper.instrument_app(app)
             self.init_instrumentor_wrapper_base_for_http(
                 self._flask_instrumentor_wrapper)
         except Exception as err:  # pylint: disable=W0703
@@ -483,7 +484,8 @@ 
      Classes
      
             from hypertrace.agent.instrumentation.flask import FlaskInstrumentorWrapper  # pylint: disable=C0415
             self._modules_initialized['flask'] = True
             self._flask_instrumentor_wrapper = FlaskInstrumentorWrapper()
-            self._flask_instrumentor_wrapper.instrument_app(app)
+            if app:
+                self._flask_instrumentor_wrapper.instrument_app(app)
             self.init_instrumentor_wrapper_base_for_http(
                 self._flask_instrumentor_wrapper)
         except Exception as err:  # pylint: disable=W0703
@@ -763,7 +765,8 @@ 
      Methods
      
         from hypertrace.agent.instrumentation.flask import FlaskInstrumentorWrapper  # pylint: disable=C0415
         self._modules_initialized['flask'] = True
         self._flask_instrumentor_wrapper = FlaskInstrumentorWrapper()
-        self._flask_instrumentor_wrapper.instrument_app(app)
+        if app:
+            self._flask_instrumentor_wrapper.instrument_app(app)
         self.init_instrumentor_wrapper_base_for_http(
             self._flask_instrumentor_wrapper)
     except Exception as err:  # pylint: disable=W0703
diff --git a/docs/hypertrace/agent/instrumentation/aiohttp/index.html b/docs/hypertrace/agent/instrumentation/aiohttp/index.html
index d309dd76..6c612c29 100644
--- a/docs/hypertrace/agent/instrumentation/aiohttp/index.html
+++ b/docs/hypertrace/agent/instrumentation/aiohttp/index.html
@@ -253,7 +253,7 @@ 
      Module hypertrace.agent.instrumentation.aiohttpFunctions
      
 
      
 
      
-def create_trace_config(url_filter: Optional[Callable[[str], str]] = None, span_name: Union[Callable[[aiohttp.tracing.TraceRequestStartParams], str], str, NoneType] = None, tracer_provider: opentelemetry.trace.TracerProvider = None, aiohttp_client_wrapper: AioHttpClientInstrumentorWrapper = None) ‑> aiohttp.tracing.TraceConfig
+def create_trace_config(url_filter: Union[Callable[[str], str], NoneType] = None, span_name: Union[Callable[[aiohttp.tracing.TraceRequestStartParams], str], str, NoneType] = None, tracer_provider: opentelemetry.trace.TracerProvider = None, aiohttp_client_wrapper: AioHttpClientInstrumentorWrapper = None) ‑> aiohttp.tracing.TraceConfig
 
      
 
      
 
      Build an aiohttp-client trace config for use with Hypertrace
      
diff --git a/docs/hypertrace/agent/instrumentation/flask/index.html b/docs/hypertrace/agent/instrumentation/flask/index.html
index d4e0c45b..da1acbba 100644
--- a/docs/hypertrace/agent/instrumentation/flask/index.html
+++ b/docs/hypertrace/agent/instrumentation/flask/index.html
@@ -27,7 +27,7 @@ 
      Module hypertrace.agent.instrumentation.flask
      
 Expand source code
 
-
      '''Hypertrace flask instrumentor module wrapper.'''
+
      '''Hypertrace flask instrumentor module wrapper.''' # pylint: disable=R0401
 import sys
 import os.path
 import logging
@@ -36,6 +36,7 @@ 
      Module hypertrace.agent.instrumentation.flask
      Module hypertrace.agent.instrumentation.flaskModule hypertrace.agent.instrumentation.flaskModule hypertrace.agent.instrumentation.flaskModule hypertrace.agent.instrumentation.flaskClasses
         super().__init__()
         self._app = None
 
-
-
+    def _instrument(self, **kwargs):
+        '''Override OTel method that sets up global flask instrumentation'''
+        self._original_flask = flask.Flask # pylint: disable = W0201
+        name_callback = kwargs.get("name_callback")
+        tracer_provider = kwargs.get("tracer_provider")
+        if callable(name_callback):
+            _HypertraceInstrumentedFlask.name_callback = name_callback
+        _HypertraceInstrumentedFlask._tracer_provider = tracer_provider # pylint: disable=W0212
+        flask.Flask = _HypertraceInstrumentedFlask
 
     # Initialize instrumentation wrapper
     def instrument_app(self, app, name_callback=get_default_span_name) -> None:
diff --git a/docs/hypertrace/agent/instrumentation/grpc/index.html b/docs/hypertrace/agent/instrumentation/grpc/index.html
index 102b18ab..fbef6e82 100644
--- a/docs/hypertrace/agent/instrumentation/grpc/index.html
+++ b/docs/hypertrace/agent/instrumentation/grpc/index.html
@@ -639,8 +639,7 @@ 
      Inherited members
      
     ) -> None:
         '''process streaming request for hypertrace'''
         logger.debug(
-            'Entering OpenTelemetryClientInterceptorWrapper.intercept_stream().')
-        # COME_BACK -- need to implement this
      
+            'Entering OpenTelemetryClientInterceptorWrapper.intercept_stream().')
      
 
 
      Ancestors
      
 
        
@@ -669,8 +668,7 @@ 
        Methods
        
 ) -> None:
     '''process streaming request for hypertrace'''
     logger.debug(
-        'Entering OpenTelemetryClientInterceptorWrapper.intercept_stream().')
-    # COME_BACK -- need to implement this
    • 
+        'Entering OpenTelemetryClientInterceptorWrapper.intercept_stream().')
    @@ -783,8 +781,7 @@

    Methods

    context) -> None: '''Setup interceptor helper for streaming requests.''' logger.debug( - 'Entering OpenTelemetryServerInterceptorWrapper.intercept_server_stream().') - # COME_BACK -- need to implement this     + 'Entering OpenTelemetryServerInterceptorWrapper.intercept_server_stream().')

    Ancestors

      diff --git a/docs/hypertrace/agent/instrumentation/index.html b/docs/hypertrace/agent/instrumentation/index.html index c19487f9..1c58a116 100644 --- a/docs/hypertrace/agent/instrumentation/index.html +++ b/docs/hypertrace/agent/instrumentation/index.html @@ -811,6 +811,7 @@

      Subclasses

    diff --git a/examples/autoinstrumentation/Makefile b/examples/autoinstrumentation/Makefile new file mode 100644 index 00000000..63101e09 --- /dev/null +++ b/examples/autoinstrumentation/Makefile @@ -0,0 +1,12 @@ +install-deps: + pip3 install -r requirements.txt + +run: + @rm -rf __pycache__ || true + HT_CONFIG_FILE=./config.yaml FLASK_APP="server:create_app()" FLASK_ENV=development hypertrace-instrument flask run -p 9000 + +run-hypertrace: + docker-compose -f docker-compose-hypertrace.yml up --renew-anon-volumes -d + +run-mysql: + docker-compose -f ./mysql/docker-compose.yml up --renew-anon-volumes -d diff --git a/examples/autoinstrumentation/config.yaml b/examples/autoinstrumentation/config.yaml new file mode 100644 index 00000000..320f1699 --- /dev/null +++ b/examples/autoinstrumentation/config.yaml @@ -0,0 +1,6 @@ +service_name: "server" +propagation_formats: + - "B3" +reporting: + endpoint: http://localhost:9411/api/v2/spans + trace_reporter_type: ZIPKIN diff --git a/examples/autoinstrumentation/docker-compose-hypertrace.yml b/examples/autoinstrumentation/docker-compose-hypertrace.yml new file mode 100644 index 00000000..502c5183 --- /dev/null +++ b/examples/autoinstrumentation/docker-compose-hypertrace.yml @@ -0,0 +1,115 @@ +## Copied from https://github.com/hypertrace/hypertrace/blob/main/docker/docker-compose.yml + +## This does everything you need to get a hypertracing system started. +## You can connect to the UI at port 2020 and send data to it on any supported tracing solution. +## Note: Our stack is dependent on pinot and it is a cpu heavy during startup. +## The depends_on has a max wait time of 1 min, so if you don't have enough resources, you may have to re-run the same command. +## we are looking at improving this. +version: "2.4" +services: + + +# This container includes the UI and APIs it needs for storage queries. + hypertrace: + image: hypertrace/hypertrace:main + container_name: hypertrace + environment: + - MONGO_HOST=mongo + - ZK_CONNECT_STR=zookeeper:2181/hypertrace-views + ports: + - 2020:2020 + healthcheck: + start_period: 20s + depends_on: + mongo: + condition: service_healthy + kafka-zookeeper: + condition: service_healthy + pinot: + condition: service_started + +# Ingestion pipeline + + # Collects spans in different trace formats like Jaeger, Zipkin, etc + hypertrace-collector: + image: hypertrace/hypertrace-collector:main + container_name: hypertrace-collector + ports: + - 4317:4317 # grpc-otel + - 55681:55681 # http-otel + - 14268:14268 # Jaeger http + - 9411:9411 # Zipkin HTTP + environment: + - EXPORTER_KAFKA_BROKER=kafka-zookeeper:9092 + - EXPORTER_KAFKA_TOPIC=jaeger-spans + networks: + default: + # Allows sample apps to connect with platform-specific hostnames + aliases: + - jaeger + - jaeger-collector + - zipkin + depends_on: + kafka-zookeeper: + condition: service_healthy + # all-in-one ingestion pipeline for hypertrace + hypertrace-ingester: + image: hypertrace/hypertrace-ingester + container_name: hypertrace-ingester + environment: + - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 + - DEFAULT_TENANT_ID=__default + - SPAN_GROUPBY_SESSION_WINDOW_INTERVAL=2 + - REPLICATION_FACTOR=1 + - ENTITY_SERVICE_HOST_CONFIG=hypertrace + - ENTITY_SERVICE_PORT_CONFIG=9001 + - ATTRIBUTE_SERVICE_HOST_CONFIG=hypertrace + - ATTRIBUTE_SERVICE_PORT_CONFIG=9001 + - CONFIG_SERVICE_HOST_CONFIG=hypertrace + - CONFIG_SERVICE_PORT_CONFIG=9001 + - NUM_STREAM_THREADS=1 + - PRE_CREATE_TOPICS=true + - PRODUCER_VALUE_SERDE=org.hypertrace.core.kafkastreams.framework.serdes.GenericAvroSerde + volumes: + - ../docker/configs/log4j2.properties:/app/resources/log4j2.properties:ro + depends_on: + kafka-zookeeper: + condition: service_healthy + hypertrace: + # service_started, not service_healthy as pinot and deps can take longer than 60s to start + condition: service_started + +# Third-party data services: + + # Kafka is used for streaming functionality. + # ZooKeeper is required by Kafka and Pinot + kafka-zookeeper: + image: hypertrace/kafka-zookeeper:main + container_name: kafka-zookeeper + networks: + default: + # prevents apps from having to use the hostname kafka-zookeeper + aliases: + - kafka + - zookeeper + # Stores entities like API, service and backend + mongo: + image: hypertrace/mongodb:main + container_name: mongo + # Stores spans and traces and provides aggregation functions + pinot: + image: hypertrace/pinot-servicemanager:main + container_name: pinot + environment: + - LOG_LEVEL=error + networks: + default: + # Usually, Pinot is distributed, and clients connect to the controller + aliases: + - pinot-controller + - pinot-server + - pinot-broker + cpu_shares: 2048 + depends_on: + kafka-zookeeper: + condition: service_healthy \ No newline at end of file diff --git a/examples/autoinstrumentation/mysql/docker-compose.yml b/examples/autoinstrumentation/mysql/docker-compose.yml new file mode 100644 index 00000000..148c3250 --- /dev/null +++ b/examples/autoinstrumentation/mysql/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3.1' + +services: + + mysqldb: + image: mysql + command: --default-authentication-plugin=mysql_native_password + restart: always + ports: + - "3306:3306" + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: test + volumes: + - ./init.sql:/docker-entrypoint-initdb.d/init.sql diff --git a/examples/autoinstrumentation/mysql/init.sql b/examples/autoinstrumentation/mysql/init.sql new file mode 100644 index 00000000..b5c561d1 --- /dev/null +++ b/examples/autoinstrumentation/mysql/init.sql @@ -0,0 +1,4 @@ +CREATE TABLE `users`( + id INT AUTO_INCREMENT primary key NOT NULL, + `name` VARCHAR(100) NOT NULL +); diff --git a/examples/autoinstrumentation/requirements.txt b/examples/autoinstrumentation/requirements.txt new file mode 100644 index 00000000..9c6b95e2 --- /dev/null +++ b/examples/autoinstrumentation/requirements.txt @@ -0,0 +1,3 @@ +../../ +flask +mysql-connector-python==8.0.23 diff --git a/examples/autoinstrumentation/server.py b/examples/autoinstrumentation/server.py new file mode 100644 index 00000000..b2b34550 --- /dev/null +++ b/examples/autoinstrumentation/server.py @@ -0,0 +1,58 @@ +from flask import Flask, request, jsonify, make_response, Response +import time +import json +import logging +import mysql.connector + + +logging.basicConfig() +logging.getLogger().setLevel(logging.DEBUG) + + +def stream_body(res_body: str): + new_line_chars = ['{', '}', ',', '[', ']'] + for c in res_body: + time.sleep(0.3) + logging.debug("Sending response chunk") + if c in new_line_chars: + yield c + "\n" + else: + yield c + + +def insert_user(name: str) -> int: + cnx = mysql.connector.connect(database='test', + username='root', + password='root', + host='localhost', + port=3306) + cursor = cnx.cursor() + query = "INSERT INTO users (`name`) VALUES (%s)" + cursor.execute(query, (name,)) + id = cursor.lastrowid + cnx.close() + return id + + +def create_app(): + app = Flask(__name__) + + @app.route('/', methods=['POST']) + def hello(): + request_data = request.get_json() + name = request_data['name'] + + user_id = insert_user(name) + + res_body = {'id': user_id, 'message': f'Hello {name}'} + + stream = request.args.get('stream') + if stream == 'true': + # Send the response as stream + return Response(stream_body(json.dumps(res_body)), mimetype='application/json') + else: + # Send the entire response + response = make_response(jsonify(res_body)) + response.headers['Content-Type'] = 'application/json' + return response + return app diff --git a/examples/server/Makefile b/examples/server/Makefile index 63101e09..21b5f7bc 100644 --- a/examples/server/Makefile +++ b/examples/server/Makefile @@ -3,7 +3,7 @@ install-deps: run: @rm -rf __pycache__ || true - HT_CONFIG_FILE=./config.yaml FLASK_APP="server:create_app()" FLASK_ENV=development hypertrace-instrument flask run -p 9000 + HT_CONFIG_FILE=./config.yaml FLASK_APP="server:create_app()" FLASK_ENV=development flask run -p 9000 run-hypertrace: docker-compose -f docker-compose-hypertrace.yml up --renew-anon-volumes -d diff --git a/examples/server/server.py b/examples/server/server.py index f016d71b..1baf37fe 100644 --- a/examples/server/server.py +++ b/examples/server/server.py @@ -1,4 +1,5 @@ from flask import Flask, request, jsonify, make_response, Response +from hypertrace.agent import Agent import time import json import logging @@ -36,9 +37,9 @@ def insert_user(name: str) -> int: def create_app(): app = Flask(__name__) -# agent = Agent() -# agent.register_flask_app(app) -# agent.register_mysql() + agent = Agent() + agent.register_flask_app(app) + agent.register_mysql() @app.route('/', methods=['POST']) def hello(): diff --git a/src/hypertrace/agent/autoinstrumentation/sitecustomize.py b/src/hypertrace/agent/autoinstrumentation/sitecustomize.py index 80b1db25..60885988 100644 --- a/src/hypertrace/agent/autoinstrumentation/sitecustomize.py +++ b/src/hypertrace/agent/autoinstrumentation/sitecustomize.py @@ -1,7 +1,7 @@ '''Enable instrumentationon all supported modules.''' # pylint: disable=R0401 -import os # pylint: disable=R0401 +import os import logging -from hypertrace.agent import Agent # pylint: disable=R0401 +from hypertrace.agent import Agent DEFAULTS = [ 'flask', @@ -16,12 +16,12 @@ # Initialize logger logger = logging.getLogger(__name__) # pylint: disable=C0103 -modules_ = '' +MODULES = '' if 'HT_INSTRUMENTED_MODULES' in os.environ: logger.debug("[env] Loaded HT_INSTRUMENTED_MODULES from env") - modules_ = os.environ['HT_INSTRUMENTED_MODULES'] + MODULES = os.environ['HT_INSTRUMENTED_MODULES'] -modules_array = modules_.split(',') +modules_array = MODULES.split(',') if len(modules_array) == 1 and modules_array[0] == '': modules_array = DEFAULTS diff --git a/src/hypertrace/agent/instrumentation/flask/__init__.py b/src/hypertrace/agent/instrumentation/flask/__init__.py index 19a07b70..d95cd9aa 100644 --- a/src/hypertrace/agent/instrumentation/flask/__init__.py +++ b/src/hypertrace/agent/instrumentation/flask/__init__.py @@ -1,4 +1,4 @@ -'''Hypertrace flask instrumentor module wrapper.''' +'''Hypertrace flask instrumentor module wrapper.''' # pylint: disable=R0401 import sys import os.path import logging diff --git a/tests/autoinstrumentation/tox.ini b/tests/autoinstrumentation/tox.ini index 40927640..f5f9c4c0 100644 --- a/tests/autoinstrumentation/tox.ini +++ b/tests/autoinstrumentation/tox.ini @@ -1,7 +1,6 @@ [tox] skipsdist = true -#envlist = py3{7,8,9} -envlist = py37 +envlist = py3{7,8,9} [testenv] sitepackages = True @@ -30,9 +29,9 @@ commands = bash -ec "if [ `uname` = 'Linux' ] && [ "{env:GITHUB_ACTIONS:false}" != "true" ]; then chcon -h system_u:object_r:bin_t:s0 docker-healthcheck; chcon -Rt svirt_sandbox_file_t docker-healthcheck; fi" docker-compose up -d --remove-orphans --force-recreate -V mysqldb bash -ec "set -x; while [ `docker inspect -f '\{\{ .State.Health.Status \}\}' mysqldb` != 'healthy' ]; do echo 'Waiting for MySQL to be up'; sleep 2; done" -# ./test_flask_1.sh + ./test_flask_1.sh ./test_flask_2.sh -# docker-compose stop mysqldb -# docker-compose down --rmi all + docker-compose stop mysqldb + docker-compose down --rmi all recreate = True From 35a0fcbdfdb24bb317fb6d0934c62ccf43874ba0 Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Sun, 9 May 2021 18:54:57 -0700 Subject: [PATCH 06/23] Updating readme with autoinstrumentation instructions. --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index e3d2b67f..0b0aec9d 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,15 @@ agent.registerMySQL() # instrument the MySQL client ... ``` +or + +- Use the autoinstrumentation CLI +``` +HT_INSTRUMENTED_MODULES=flask,mysql +hypertrace-instrument python app.py +``` +By default, all supported modules are instrumented. + For further examples, check our [examples section](./examples) ### Configuration From f6a649abd02867813879dfb5a362fb0927500065 Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Mon, 10 May 2021 18:00:29 -0700 Subject: [PATCH 07/23] Updating README with clarification regarding not adding instrumentation code to an application using autoinstrumentation. --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0b0aec9d..9050a5cf 100644 --- a/README.md +++ b/README.md @@ -36,20 +36,22 @@ from hypertrace.agent import Agent ... agent = Agent() # initialize the agent -agent.registerFlaskApp(app) # instrument a flask application -agent.registerMySQL() # instrument the MySQL client +agent.register_flask_app(app) # instrument a flask application +agent.register_mysql() # instrument the MySQL client ... ``` or -- Use the autoinstrumentation CLI +- Use the autoinstrumentation CLI (without any modifiation to application code) ``` HT_INSTRUMENTED_MODULES=flask,mysql hypertrace-instrument python app.py ``` By default, all supported modules are instrumented. +Note, do not attempt to instantiate a hypertrace.agent.Agent object while using hypertrace-instrument. + For further examples, check our [examples section](./examples) ### Configuration From 6ebb408e2ccfaab1cd8830368f7abed4931fc173 Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Mon, 10 May 2021 22:13:43 -0700 Subject: [PATCH 08/23] Implement singleton for Agent class to ensure instrumentation classes are only initialized once. --- src/hypertrace/agent/__init__.py | 35 ++++++++++++++++++--------- src/hypertrace/agent/init/__init__.py | 29 +++++++++++++++++++--- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/src/hypertrace/agent/__init__.py b/src/hypertrace/agent/__init__.py index 36f12110..f8b7ef6a 100644 --- a/src/hypertrace/agent/__init__.py +++ b/src/hypertrace/agent/__init__.py @@ -58,19 +58,32 @@ def setup_custom_logger(name: str) -> logging.Logger: class Agent: '''Top-level entry point for Hypertrace agent.''' + _instance = None + + def __new__(cls): + '''constructor''' + if cls._instance is None: + logger.debug('Creating Agent') + cls._instance = super(Agent, cls).__new__(cls) + cls._instance._initialized = False + else: + logger.debug('Using existing Agent.') + return cls._instance def __init__(self): - '''Constructor''' - logger.debug('Initializing Agent.') - if not self.is_enabled(): - return - try: - self._config = AgentConfig() - self._init = AgentInit(self._config) - except Exception as err: # pylint: disable=W0703 - logger.error('Failed to initialize Agent: exception=%s, stacktrace=%s', - err, - traceback.format_exc()) + '''Initializer''' + if not self._initialized: # pylint: disable=E0203: + logger.debug('Initializing Agent.') + if not self.is_enabled(): + return + try: + self._config = AgentConfig() + self._init = AgentInit(self._config) + self._initialized = True + except Exception as err: # pylint: disable=W0703 + logger.error('Failed to initialize Agent: exception=%s, stacktrace=%s', + err, + traceback.format_exc()) def register_flask_app(self, app: flask.Flask = None) -> None: '''Register the flask instrumentation module wrapper''' diff --git a/src/hypertrace/agent/init/__init__.py b/src/hypertrace/agent/init/__init__.py index 17bf3f1a..c070c69e 100644 --- a/src/hypertrace/agent/init/__init__.py +++ b/src/hypertrace/agent/init/__init__.py @@ -34,7 +34,6 @@ def __init__(self, agent_config: AgentConfig): "requests": False, "aiohttp_client": False } - self._tracer_provider = None try: @@ -123,6 +122,8 @@ def init_instrumentation_flask(self, app) -> None: '''Creates a flask instrumentation wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.flaskInit().') try: + if self.is_registered('flask'): + return from hypertrace.agent.instrumentation.flask import FlaskInstrumentorWrapper # pylint: disable=C0415 self._modules_initialized['flask'] = True self._flask_instrumentor_wrapper = FlaskInstrumentorWrapper() @@ -140,6 +141,8 @@ def init_instrumentation_grpc_server(self) -> None: '''Creates a grpc server wrapper based on hypertrace config''' logger.debug('Calling AgentInit.grpcServerInit') try: + if self.is_registered('grpc:server'): + return from hypertrace.agent.instrumentation.grpc import ( # pylint: disable=C0415 GrpcInstrumentorServerWrapper ) @@ -167,6 +170,8 @@ def init_instrumentation_grpc_client(self) -> None: '''Creates a grpc client wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.grpcClientInit') try: + if self.is_registered('grpc:client'): + return from hypertrace.agent.instrumentation.grpc import ( # pylint: disable=C0415 GrpcInstrumentorClientWrapper ) @@ -195,6 +200,8 @@ def init_instrumentation_mysql(self) -> None: '''Creates a mysql server wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.mysqlInit()') try: + if self.is_registered('mysql'): + return from hypertrace.agent.instrumentation.mysql import ( # pylint: disable=C0415 MySQLInstrumentorWrapper ) @@ -213,6 +220,8 @@ def init_instrumentation_postgresql(self) -> None: '''Creates a postgresql client wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.postgreSQLInit()') try: + if self.is_registered('postgresql'): + return from hypertrace.agent.instrumentation.postgresql import ( # pylint: disable=C0415 PostgreSQLInstrumentorWrapper ) @@ -231,6 +240,8 @@ def init_instrumentation_requests(self) -> None: '''Creates a requests client wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.requestsInit()') try: + if self.is_registered('requests'): + return from hypertrace.agent.instrumentation.requests import ( # pylint: disable=C0415 RequestsInstrumentorWrapper ) @@ -249,6 +260,8 @@ def aiohttp_client_init(self) -> None: '''Creates an aiohttp-client wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.aioHttpClientInit()') try: + if self.is_registered('aiohttp_client'): + return from hypertrace.agent.instrumentation.aiohttp import ( # pylint: disable=C0415 AioHttpClientInstrumentorWrapper ) @@ -263,10 +276,13 @@ def aiohttp_client_init(self) -> None: traceback.format_exc()) # Common wrapper initialization logic - def init_instrumentor_wrapper_base_for_http(self, instrumentor) -> None: + def init_instrumentor_wrapper_base_for_http(self, + instrumentor, + call_instrument: bool = True) -> None: '''Common wrapper initialization logic''' logger.debug('Calling AgentInit.initInstrumentorWrapperBaseForHTTP().') - instrumentor.instrument() + if call_instrument: + instrumentor.instrument() instrumentor.set_process_request_headers( self._config.agent_config.data_capture.http_headers.request) instrumentor.set_process_request_body( @@ -325,3 +341,10 @@ def _init_otlp_exporter(self) -> None: logger.error('Failed to initialize OTLP exporter: exception=%s, stacktrace=%s', err, traceback.format_exc()) + + def is_registered(self, module: str) -> bool: + '''Is an instrumentation module already registered?''' + try: + return self._modules_initialized[module] + except Exception as err: # pylint: disable=W0703,W0612 + return False From 41d52f70198e302d0e5002856c72726826aa0d9f Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Tue, 11 May 2021 20:12:22 -0700 Subject: [PATCH 09/23] Update package dependencies to set minimum package versions. --- dev-requirements.txt | 40 ++++++++++++++++++++-------------------- requirements.txt | 32 ++++++++++++++++---------------- setup.py | 32 ++++++++++++++++---------------- 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index fa765703..6d8fd014 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,21 +1,21 @@ -opentelemetry-api==1.1.0 -opentelemetry-exporter-otlp==1.1.0 -opentelemetry-exporter-zipkin==1.1.0 -opentelemetry-instrumentation==0.20b0 -opentelemetry-instrumentation-aiohttp-client==0.20b0 -opentelemetry-instrumentation-wsgi==0.20b0 -opentelemetry-instrumentation-flask==0.20b0 -opentelemetry-instrumentation-mysql==0.20b0 -opentelemetry-instrumentation-psycopg2==0.20b0 -opentelemetry-instrumentation-requests==0.20b0 -opentelemetry-instrumentation-grpc==0.20b0 -opentelemetry-propagator-b3==1.1.0 -opentelemetry-sdk==1.1.0 -opentelemetry-util-http==0.20b0 -google==3.0.0 +opentelemetry-api>=1.1.0 +opentelemetry-exporter-otlp>=1.1.0 +opentelemetry-exporter-zipkin>=1.1.0 +opentelemetry-instrumentation>=0.20b0 +opentelemetry-instrumentation-aiohttp-client>=0.20b0 +opentelemetry-instrumentation-wsgi>=0.20b0 +opentelemetry-instrumentation-flask>=0.20b0 +opentelemetry-instrumentation-mysql>=0.20b0 +opentelemetry-instrumentation-psycopg2>=0.20b0 +opentelemetry-instrumentation-requests>=0.20b0 +opentelemetry-instrumentation-grpc>=0.20b0 +opentelemetry-propagator-b3>=1.1.0 +opentelemetry-sdk>=1.1.0 +opentelemetry-util-http>=0.20b0 +google>=3.0.0 pyyaml -pytest==6.2.3 -protobuf==3.15.8 -tox==3.23.0 -pylint==2.7.4 -pdoc3==0.9.2 +pytest>=6.2.3 +protobuf>=3.15.8 +tox>=3.23.0 +pylint>=2.7.4 +pdoc3>=0.9.2 diff --git a/requirements.txt b/requirements.txt index 4ab30ad5..a727a719 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,17 @@ -opentelemetry-api==1.1.0 -opentelemetry-exporter-otlp==1.1.0 -opentelemetry-exporter-zipkin==1.1.0 -opentelemetry-instrumentation==0.20b0 -opentelemetry-instrumentation-aiohttp-client==0.20b0 -opentelemetry-instrumentation-wsgi==0.20b0 -opentelemetry-instrumentation-flask==0.20b0 -opentelemetry-instrumentation-mysql==0.20b0 -opentelemetry-instrumentation-psycopg2==0.20b0 -opentelemetry-instrumentation-requests==0.20b0 -opentelemetry-instrumentation-grpc==0.20b0 -opentelemetry-propagator-b3==1.1.0 -opentelemetry-sdk==1.1.0 -opentelemetry-util-http==0.20b0 -google==3.0.0 +opentelemetry-api>=1.1.0 +opentelemetry-exporter-otlp>=1.1.0 +opentelemetry-exporter-zipkin>=1.1.0 +opentelemetry-instrumentation>=0.20b0 +opentelemetry-instrumentation-aiohttp-client>=0.20b0 +opentelemetry-instrumentation-wsgi>=0.20b0 +opentelemetry-instrumentation-flask>=0.20b0 +opentelemetry-instrumentation-mysql>=0.20b0 +opentelemetry-instrumentation-psycopg2>=0.20b0 +opentelemetry-instrumentation-requests>=0.20b0 +opentelemetry-instrumentation-grpc>=0.20b0 +opentelemetry-propagator-b3>=1.1.0 +opentelemetry-sdk>=1.1.0 +opentelemetry-util-http>=0.20b0 +google>=3.0.0 pyyaml -protobuf==3.15.8 +protobuf>=3.15.8 diff --git a/setup.py b/setup.py index 6f34b5c1..e7f899ef 100644 --- a/setup.py +++ b/setup.py @@ -26,23 +26,23 @@ packages=find_packages(where="src"), python_requires=">=3.7", install_requires=[ - "opentelemetry-api==1.1.0", - "opentelemetry-exporter-otlp==1.1.0", - "opentelemetry-exporter-zipkin==1.1.0", - "opentelemetry-instrumentation==0.20b0", - "opentelemetry-instrumentation-aiohttp-client==0.20b0", - "opentelemetry-instrumentation-wsgi==0.20b0", - "opentelemetry-instrumentation-flask==0.20b0", - "opentelemetry-instrumentation-mysql==0.20b0", - "opentelemetry-instrumentation-psycopg2==0.20b0", - "opentelemetry-instrumentation-requests==0.20b0", - "opentelemetry-instrumentation-grpc==0.20b0", - "opentelemetry-propagator-b3==1.1.0", - "opentelemetry-sdk==1.1.0", - "opentelemetry-util-http==0.20b0", - "google==3.0.0", + "opentelemetry-api>=1.1.0", + "opentelemetry-exporter-otlp>=1.1.0", + "opentelemetry-exporter-zipkin>=1.1.0", + "opentelemetry-instrumentation>=0.20b0", + "opentelemetry-instrumentation-aiohttp-client>=0.20b0", + "opentelemetry-instrumentation-wsgi>=0.20b0", + "opentelemetry-instrumentation-flask>=0.20b0", + "opentelemetry-instrumentation-mysql>=0.20b0", + "opentelemetry-instrumentation-psycopg2>=0.20b0", + "opentelemetry-instrumentation-requests>=0.20b0", + "opentelemetry-instrumentation-grpc>=0.20b0", + "opentelemetry-propagator-b3>=1.1.0", + "opentelemetry-sdk>=1.1.0", + "opentelemetry-util-http>=0.20b0", + "google>=3.0.0", "pyyaml", - "protobuf==3.15.8" + "protobuf>=3.15.8" ], entry_points = { 'console_scripts': [ From 4bac59d30da2be0d7472f01acd8bdb0fc65e0eff Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Tue, 11 May 2021 20:19:57 -0700 Subject: [PATCH 10/23] Updating code to fix new lint issue being reported. --- src/hypertrace/agent/config/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hypertrace/agent/config/file.py b/src/hypertrace/agent/config/file.py index ac5faf08..eee60a1d 100644 --- a/src/hypertrace/agent/config/file.py +++ b/src/hypertrace/agent/config/file.py @@ -18,7 +18,7 @@ def load_config_from_file(filepath): try: path = os.path.abspath(filepath) - file = open(path, 'r') + file = open(path, 'r') # pytest: disable=R1732 from_file_config = yaml.load(file, Loader=yaml.FullLoader) file.close() From 08f6c1a1c2eb50094c6ff03de67aba40044862d5 Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Tue, 11 May 2021 20:30:40 -0700 Subject: [PATCH 11/23] Correcting typo. --- src/hypertrace/agent/config/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hypertrace/agent/config/file.py b/src/hypertrace/agent/config/file.py index eee60a1d..f381fe5f 100644 --- a/src/hypertrace/agent/config/file.py +++ b/src/hypertrace/agent/config/file.py @@ -18,7 +18,7 @@ def load_config_from_file(filepath): try: path = os.path.abspath(filepath) - file = open(path, 'r') # pytest: disable=R1732 + file = open(path, 'r') # pylint: disable=R1732 from_file_config = yaml.load(file, Loader=yaml.FullLoader) file.close() From 9af39144c997d8b6af305a7a325f4a43a184b156 Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Tue, 11 May 2021 21:32:02 -0700 Subject: [PATCH 12/23] A thread safe singleton implementation. --- src/hypertrace/agent/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/hypertrace/agent/__init__.py b/src/hypertrace/agent/__init__.py index f8b7ef6a..a8efdea2 100644 --- a/src/hypertrace/agent/__init__.py +++ b/src/hypertrace/agent/__init__.py @@ -2,6 +2,7 @@ import os import os.path import sys +import threading import logging import traceback import flask @@ -59,13 +60,15 @@ def setup_custom_logger(name: str) -> logging.Logger: class Agent: '''Top-level entry point for Hypertrace agent.''' _instance = None + _singleton_lock = threading.Lock() def __new__(cls): '''constructor''' if cls._instance is None: - logger.debug('Creating Agent') - cls._instance = super(Agent, cls).__new__(cls) - cls._instance._initialized = False + with cls._singleton_lock: + logger.debug('Creating Agent') + cls._instance = super(Agent, cls).__new__(cls) + cls._instance._initialized = False else: logger.debug('Using existing Agent.') return cls._instance From afaa6f66fdb6f0fdde0cd5da9073d235006ec5c8 Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Wed, 12 May 2021 17:02:59 -0700 Subject: [PATCH 13/23] Updating hypertrace_instrument run() function with comments. Broke PYTHONPATH update logic into a separate function. --- .../hypertrace_instrument.py | 29 +++++++++++++++---- .../agent/instrumentation/flask/__init__.py | 7 +++-- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py b/src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py index c17878cd..636da868 100644 --- a/src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py +++ b/src/hypertrace/agent/autoinstrumentation/hypertrace_instrument.py @@ -29,31 +29,50 @@ def parse_args(): return parser.parse_args() -def run() -> None: - '''hypertrace-instrument Entry point''' - args = parse_args() - +def update_python_path() -> None: + '''Retrieve existing PYTHONPATH''' python_path = environ.get("PYTHONPATH") + # Split the paths if not python_path: python_path = [] - else: python_path = python_path.split(pathsep) + # Get the current working directory cwd_path = getcwd() + # If this directory is already in python_path, remove it. + python_path = [path for path in python_path if path != cwd_path] + + # If cwd is not in the PYTHONPATH, add it to the front. if cwd_path not in python_path: python_path.insert(0, cwd_path) + # What is the directory containing this python file? filedir_path = dirname(abspath(__file__)) + # If this directory is already in python_path, remove it. python_path = [path for path in python_path if path != filedir_path] + # If this diretory is not in python_path, add it to the front python_path.insert(0, filedir_path) + + # Reset PYTHONPATH environment variable environ["PYTHONPATH"] = pathsep.join(python_path) +def run() -> None: + '''hypertrace-instrument Entry point''' + args = parse_args() + + # update PYTHONPATH env var + update_python_path() + + # Get full path to the command that was passed in as an + # argument executable = which(args.command) + + # Execute the app execl(executable, executable, *args.command_args) if __name__ == '__main__': diff --git a/src/hypertrace/agent/instrumentation/flask/__init__.py b/src/hypertrace/agent/instrumentation/flask/__init__.py index d95cd9aa..2f3f4565 100644 --- a/src/hypertrace/agent/instrumentation/flask/__init__.py +++ b/src/hypertrace/agent/instrumentation/flask/__init__.py @@ -142,13 +142,16 @@ def _instrument(self, **kwargs): flask.Flask = _HypertraceInstrumentedFlask # Initialize instrumentation wrapper - def instrument_app(self, app, name_callback=get_default_span_name) -> None: + def instrument_app(self, + app, + name_callback=get_default_span_name, + tracer_provider=None) -> None: '''Initialize instrumentation''' logger.debug('Entering FlaskInstrumentorWrapper.instument_app().') try: # Call parent class's initialization - super().instrument_app(app, name_callback) + super().instrument_app(app, name_callback, tracer_provider) self._app = app # Set pre-request handler From 0c0be16f0bbafa843b7512cb201d61233a6f549b Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Wed, 12 May 2021 17:18:29 -0700 Subject: [PATCH 14/23] sitecustomize.py cleanup. --- .../autoinstrumentation/sitecustomize.py | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/hypertrace/agent/autoinstrumentation/sitecustomize.py b/src/hypertrace/agent/autoinstrumentation/sitecustomize.py index 60885988..36ec4a5d 100644 --- a/src/hypertrace/agent/autoinstrumentation/sitecustomize.py +++ b/src/hypertrace/agent/autoinstrumentation/sitecustomize.py @@ -4,13 +4,13 @@ from hypertrace.agent import Agent DEFAULTS = [ - 'flask', - 'mysql', - 'postgresql', - 'grpc:server', - 'grpc:client', - 'requests', - 'aiohttp' + 'flask', + 'mysql', + 'postgresql', + 'grpc:server', + 'grpc:client', + 'requests', + 'aiohttp' ] # Initialize logger @@ -20,29 +20,36 @@ if 'HT_INSTRUMENTED_MODULES' in os.environ: logger.debug("[env] Loaded HT_INSTRUMENTED_MODULES from env") MODULES = os.environ['HT_INSTRUMENTED_MODULES'] + if len(MODULES) > 0: + MODULES = MODULES.replace(' ', '') modules_array = MODULES.split(',') -if len(modules_array) == 1 and modules_array[0] == '': +if len(modules_array) == 1 \ + and modules_array[0] == '': modules_array = DEFAULTS +# Create Hypertrace agent agent = Agent() +# Initialize desired instrumentation wrappers for mod in modules_array: if mod is None or len(mod) == 0: continue if mod == 'flask': agent.register_flask_app() - if mod == 'grpc:server': + elif mod == 'grpc:server': agent.register_grpc_server() - if mod == 'grpc:client': + elif mod == 'grpc:client': agent.register_grpc_client() - if mod == 'mysql': + elif mod == 'mysql': agent.register_mysql() - if mod == 'postgresql': + elif mod == 'postgresql': agent.register_postgresql() - if mod == 'requests': + elif mod == 'requests': agent.register_requests() - if mod == 'aiohttp-client': + elif mod == 'aiohttp-client': agent.register_aiohttp_client() + else: + logger.error('Unknown module name: %s', mod) From a58419abc83434fe7d61a67ebbb2c13bf5624c1e Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Wed, 12 May 2021 17:24:33 -0700 Subject: [PATCH 15/23] Adding pydoc comment back into generated file. --- src/hypertrace/agent/config/config_pb2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hypertrace/agent/config/config_pb2.py b/src/hypertrace/agent/config/config_pb2.py index 8d80422c..912da743 100644 --- a/src/hypertrace/agent/config/config_pb2.py +++ b/src/hypertrace/agent/config/config_pb2.py @@ -1,5 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: config.proto +"""Generated protocol buffer code.""" import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) From ac084d692c30dc1631828dbbc8d85754362f36dd Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Wed, 12 May 2021 17:39:12 -0700 Subject: [PATCH 16/23] Updating flask initialization logic to correctly call instrument_app() vs instrument(). Added detailed explanation of how it is supposed to owrik in comments. --- src/hypertrace/agent/init/__init__.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/hypertrace/agent/init/__init__.py b/src/hypertrace/agent/init/__init__.py index c070c69e..7d648756 100644 --- a/src/hypertrace/agent/init/__init__.py +++ b/src/hypertrace/agent/init/__init__.py @@ -127,10 +127,25 @@ def init_instrumentation_flask(self, app) -> None: from hypertrace.agent.instrumentation.flask import FlaskInstrumentorWrapper # pylint: disable=C0415 self._modules_initialized['flask'] = True self._flask_instrumentor_wrapper = FlaskInstrumentorWrapper() + call_default_instrumentor = True + # There are two ways to initialize the flask instrumenation + # wrapper. The first (and original way) instruments the specific + # Flask object that is passed in). The second way is to globally + # replace the Flask class definition with the hypertrace instrumentation + # wrapper class. + # + # If an app object is provided, then the flask wrapper is initialized + # by calling the instrument_app method. Then, there is no need to call + # instrument() (so, we pass False as the second argument to + # self.init_instrumentor_wrapper_base_for_http(). + # + # If no app object was provided, then instrument() is called. if app: self._flask_instrumentor_wrapper.instrument_app(app) + call_default_instrumentor = False self.init_instrumentor_wrapper_base_for_http( - self._flask_instrumentor_wrapper) + self._flask_instrumentor_wrapper, + call_default_instrumentor) except Exception as err: # pylint: disable=W0703 logger.error(constants.INST_WRAP_EXCEPTION_MSSG, 'flask', From bc1149cfc544ce77543745a02ec210b23f862b79 Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Wed, 12 May 2021 17:44:29 -0700 Subject: [PATCH 17/23] Adding explanation of selinux commands. --- tests/autoinstrumentation/tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/autoinstrumentation/tox.ini b/tests/autoinstrumentation/tox.ini index f5f9c4c0..f7871518 100644 --- a/tests/autoinstrumentation/tox.ini +++ b/tests/autoinstrumentation/tox.ini @@ -25,6 +25,7 @@ setenv = commands = docker-compose stop mysqldb + # Set selinux permissions on volumes being mounted into docker containers bash -ec "if [ `uname` = 'Linux' ] && [ "{env:GITHUB_ACTIONS:false}" != "true" ]; then chcon -h system_u:object_r:bin_t:s0 sql; chcon -Rt svirt_sandbox_file_t sql; fi" bash -ec "if [ `uname` = 'Linux' ] && [ "{env:GITHUB_ACTIONS:false}" != "true" ]; then chcon -h system_u:object_r:bin_t:s0 docker-healthcheck; chcon -Rt svirt_sandbox_file_t docker-healthcheck; fi" docker-compose up -d --remove-orphans --force-recreate -V mysqldb From a32f5d0b2fff8d73d13d89c3cf4791349ef6a6ea Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Wed, 12 May 2021 17:55:45 -0700 Subject: [PATCH 18/23] Adding comment back into generated file. Making argument parsing logic more readable. --- examples/autoinstrumentation/Makefile | 2 +- .../agent/autoinstrumentation/sitecustomize.py | 9 ++++----- src/hypertrace/agent/config/config_pb2.py | 1 - 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/examples/autoinstrumentation/Makefile b/examples/autoinstrumentation/Makefile index 63101e09..efd9e16c 100644 --- a/examples/autoinstrumentation/Makefile +++ b/examples/autoinstrumentation/Makefile @@ -3,7 +3,7 @@ install-deps: run: @rm -rf __pycache__ || true - HT_CONFIG_FILE=./config.yaml FLASK_APP="server:create_app()" FLASK_ENV=development hypertrace-instrument flask run -p 9000 + HT_INSTRUMENTED_MODULES=flask HT_ENABLE_CONSOLE_SPAN_EXPORTER=True HT_CONFIG_FILE=./config.yaml FLASK_APP="server:create_app()" FLASK_ENV=development hypertrace-instrument flask run -p 9000 run-hypertrace: docker-compose -f docker-compose-hypertrace.yml up --renew-anon-volumes -d diff --git a/src/hypertrace/agent/autoinstrumentation/sitecustomize.py b/src/hypertrace/agent/autoinstrumentation/sitecustomize.py index 36ec4a5d..fe5f0bda 100644 --- a/src/hypertrace/agent/autoinstrumentation/sitecustomize.py +++ b/src/hypertrace/agent/autoinstrumentation/sitecustomize.py @@ -10,7 +10,7 @@ 'grpc:server', 'grpc:client', 'requests', - 'aiohttp' + 'aiohttp-client' ] # Initialize logger @@ -23,10 +23,9 @@ if len(MODULES) > 0: MODULES = MODULES.replace(' ', '') -modules_array = MODULES.split(',') - -if len(modules_array) == 1 \ - and modules_array[0] == '': +if len(MODULES) > 0: + modules_array = MODULES.split(',') +else: modules_array = DEFAULTS # Create Hypertrace agent diff --git a/src/hypertrace/agent/config/config_pb2.py b/src/hypertrace/agent/config/config_pb2.py index 912da743..8d80422c 100644 --- a/src/hypertrace/agent/config/config_pb2.py +++ b/src/hypertrace/agent/config/config_pb2.py @@ -1,6 +1,5 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: config.proto -"""Generated protocol buffer code.""" import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) From f4dcca51e2aa8d3ded495b777a016838ccfda914 Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Wed, 12 May 2021 17:56:24 -0700 Subject: [PATCH 19/23] Restoring Makefile to original configuration. --- examples/autoinstrumentation/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/autoinstrumentation/Makefile b/examples/autoinstrumentation/Makefile index efd9e16c..63101e09 100644 --- a/examples/autoinstrumentation/Makefile +++ b/examples/autoinstrumentation/Makefile @@ -3,7 +3,7 @@ install-deps: run: @rm -rf __pycache__ || true - HT_INSTRUMENTED_MODULES=flask HT_ENABLE_CONSOLE_SPAN_EXPORTER=True HT_CONFIG_FILE=./config.yaml FLASK_APP="server:create_app()" FLASK_ENV=development hypertrace-instrument flask run -p 9000 + HT_CONFIG_FILE=./config.yaml FLASK_APP="server:create_app()" FLASK_ENV=development hypertrace-instrument flask run -p 9000 run-hypertrace: docker-compose -f docker-compose-hypertrace.yml up --renew-anon-volumes -d From d9b2ae7d71a48f16c0893d9df00fc435e2d25aeb Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Wed, 12 May 2021 18:13:12 -0700 Subject: [PATCH 20/23] Removing agent.log creation. --- src/hypertrace/agent/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/hypertrace/agent/__init__.py b/src/hypertrace/agent/__init__.py index a8efdea2..199b256b 100644 --- a/src/hypertrace/agent/__init__.py +++ b/src/hypertrace/agent/__init__.py @@ -18,8 +18,6 @@ def setup_custom_logger(name: str) -> logging.Logger: try: formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') - handler = logging.FileHandler('agent.log', mode='a') - handler.setFormatter(formatter) screen_handler = logging.StreamHandler(stream=sys.stdout) screen_handler.setFormatter(formatter) log_level = logging.INFO @@ -41,7 +39,6 @@ def setup_custom_logger(name: str) -> logging.Logger: if ht_log_level == 'NOTSET': log_level = logging.NOTSET logger_.setLevel(log_level) - logger_.addHandler(handler) logger_.addHandler(screen_handler) return logger_ except Exception as err: # pylint: disable=W0703 From 4050b2c35a3f109cb4e5524c9ed07a48bf97c4eb Mon Sep 17 00:00:00 2001 From: rcbjBlueMars Date: Wed, 12 May 2021 19:36:16 -0700 Subject: [PATCH 21/23] Updating pydoc documentation. --- .../hypertrace_instrument.html | 67 +++++++- .../autoinstrumentation/sitecustomize.html | 40 +++-- docs/hypertrace/agent/config/file.html | 4 +- docs/hypertrace/agent/index.html | 83 ++++++---- docs/hypertrace/agent/init/index.html | 154 ++++++++++++++++-- .../agent/instrumentation/flask/index.html | 23 ++- 6 files changed, 293 insertions(+), 78 deletions(-) diff --git a/docs/hypertrace/agent/autoinstrumentation/hypertrace_instrument.html b/docs/hypertrace/agent/autoinstrumentation/hypertrace_instrument.html index cf71d302..eec8a47f 100644 --- a/docs/hypertrace/agent/autoinstrumentation/hypertrace_instrument.html +++ b/docs/hypertrace/agent/autoinstrumentation/hypertrace_instrument.html @@ -62,31 +62,50 @@

    Module hypertrace.agent.autoinstrumentation.hypertrace_i return parser.parse_args() -def run() -> None: - '''hypertrace-instrument Entry point''' - args = parse_args() - +def update_python_path() -> None: + '''Retrieve existing PYTHONPATH''' python_path = environ.get("PYTHONPATH") + # Split the paths if not python_path: python_path = [] - else: python_path = python_path.split(pathsep) + # Get the current working directory cwd_path = getcwd() + # If this directory is already in python_path, remove it. + python_path = [path for path in python_path if path != cwd_path] + + # If cwd is not in the PYTHONPATH, add it to the front. if cwd_path not in python_path: python_path.insert(0, cwd_path) + # What is the directory containing this python file? filedir_path = dirname(abspath(__file__)) + # If this directory is already in python_path, remove it. python_path = [path for path in python_path if path != filedir_path] + # If this diretory is not in python_path, add it to the front python_path.insert(0, filedir_path) + + # Reset PYTHONPATH environment variable environ["PYTHONPATH"] = pathsep.join(python_path) +def run() -> None: + '''hypertrace-instrument Entry point''' + args = parse_args() + + # update PYTHONPATH env var + update_python_path() + + # Get full path to the command that was passed in as an + # argument executable = which(args.command) + + # Execute the app execl(executable, executable, *args.command_args) if __name__ == '__main__': @@ -142,28 +161,57 @@

    Functions

    '''hypertrace-instrument Entry point''' args = parse_args() + # update PYTHONPATH env var + update_python_path() + + # Get full path to the command that was passed in as an + # argument + executable = which(args.command) + + # Execute the app + execl(executable, executable, *args.command_args)     + + +
    +def update_python_path() ‑> NoneType +
    +
    +

    Retrieve existing PYTHONPATH

    +
    + +Expand source code + +
    def update_python_path() -> None:
+    '''Retrieve existing PYTHONPATH'''
     python_path = environ.get("PYTHONPATH")
 
+    # Split the paths
     if not python_path:
         python_path = []
-
     else:
         python_path = python_path.split(pathsep)
 
+    # Get the current working directory
     cwd_path = getcwd()
 
+    # If this directory is already in python_path, remove it.
+    python_path = [path for path in python_path if path != cwd_path]
+
+    # If cwd is not in the PYTHONPATH, add it to the front.
     if cwd_path not in python_path:
         python_path.insert(0, cwd_path)
 
+    # What is the directory containing this python file?
     filedir_path = dirname(abspath(__file__))
 
+    # If this directory is already in python_path, remove it.
     python_path = [path for path in python_path if path != filedir_path]
 
+    # If this diretory is not in python_path, add it to the front
     python_path.insert(0, filedir_path)
-    environ["PYTHONPATH"] = pathsep.join(python_path)
 
-    executable = which(args.command)
-    execl(executable, executable, *args.command_args)
    + # Reset PYTHONPATH environment variable + environ["PYTHONPATH"] = pathsep.join(python_path)
    @@ -186,6 +234,7 @@

    Index

    diff --git a/docs/hypertrace/agent/autoinstrumentation/sitecustomize.html b/docs/hypertrace/agent/autoinstrumentation/sitecustomize.html index bbc8121f..89d1906d 100644 --- a/docs/hypertrace/agent/autoinstrumentation/sitecustomize.html +++ b/docs/hypertrace/agent/autoinstrumentation/sitecustomize.html @@ -33,13 +33,13 @@

    Module hypertrace.agent.autoinstrumentation.sitecustomiz from hypertrace.agent import Agent DEFAULTS = [ - 'flask', - 'mysql', - 'postgresql', - 'grpc:server', - 'grpc:client', - 'requests', - 'aiohttp' + 'flask', + 'mysql', + 'postgresql', + 'grpc:server', + 'grpc:client', + 'requests', + 'aiohttp-client' ] # Initialize logger @@ -49,32 +49,38 @@

    Module hypertrace.agent.autoinstrumentation.sitecustomiz if 'HT_INSTRUMENTED_MODULES' in os.environ: logger.debug("[env] Loaded HT_INSTRUMENTED_MODULES from env") MODULES = os.environ['HT_INSTRUMENTED_MODULES'] + if len(MODULES) > 0: + MODULES = MODULES.replace(' ', '') -modules_array = MODULES.split(',') - -if len(modules_array) == 1 and modules_array[0] == '': +if len(MODULES) > 0: + modules_array = MODULES.split(',') +else: modules_array = DEFAULTS +# Create Hypertrace agent agent = Agent() +# Initialize desired instrumentation wrappers for mod in modules_array: if mod is None or len(mod) == 0: continue if mod == 'flask': agent.register_flask_app() - if mod == 'grpc:server': + elif mod == 'grpc:server': agent.register_grpc_server() - if mod == 'grpc:client': + elif mod == 'grpc:client': agent.register_grpc_client() - if mod == 'mysql': + elif mod == 'mysql': agent.register_mysql() - if mod == 'postgresql': + elif mod == 'postgresql': agent.register_postgresql() - if mod == 'requests': + elif mod == 'requests': agent.register_requests() - if mod == 'aiohttp-client': - agent.register_aiohttp_client() + elif mod == 'aiohttp-client': + agent.register_aiohttp_client() + else: + logger.error('Unknown module name: %s', mod)

    diff --git a/docs/hypertrace/agent/config/file.html b/docs/hypertrace/agent/config/file.html index f9904e16..7a5c5579 100644 --- a/docs/hypertrace/agent/config/file.html +++ b/docs/hypertrace/agent/config/file.html @@ -47,7 +47,7 @@

    Module hypertrace.agent.config.file

    try: path = os.path.abspath(filepath) - file = open(path, 'r') + file = open(path, 'r') # pylint: disable=R1732 from_file_config = yaml.load(file, Loader=yaml.FullLoader) file.close() @@ -87,7 +87,7 @@

    Functions

    try: path = os.path.abspath(filepath) - file = open(path, 'r') + file = open(path, 'r') # pylint: disable=R1732 from_file_config = yaml.load(file, Loader=yaml.FullLoader) file.close() diff --git a/docs/hypertrace/agent/index.html b/docs/hypertrace/agent/index.html index 551ff5a9..3ddf65fe 100644 --- a/docs/hypertrace/agent/index.html +++ b/docs/hypertrace/agent/index.html @@ -31,6 +31,7 @@

    Module hypertrace.agent

    import os import os.path import sys +import threading import logging import traceback import flask @@ -46,8 +47,6 @@

    Module hypertrace.agent

    try: formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') - handler = logging.FileHandler('agent.log', mode='a') - handler.setFormatter(formatter) screen_handler = logging.StreamHandler(stream=sys.stdout) screen_handler.setFormatter(formatter) log_level = logging.INFO @@ -69,7 +68,6 @@

    Module hypertrace.agent

    if ht_log_level == 'NOTSET': log_level = logging.NOTSET logger_.setLevel(log_level) - logger_.addHandler(handler) logger_.addHandler(screen_handler) return logger_ except Exception as err: # pylint: disable=W0703 @@ -87,19 +85,34 @@

    Module hypertrace.agent

    class Agent: '''Top-level entry point for Hypertrace agent.''' + _instance = None + _singleton_lock = threading.Lock() + + def __new__(cls): + '''constructor''' + if cls._instance is None: + with cls._singleton_lock: + logger.debug('Creating Agent') + cls._instance = super(Agent, cls).__new__(cls) + cls._instance._initialized = False + else: + logger.debug('Using existing Agent.') + return cls._instance def __init__(self): - '''Constructor''' - logger.debug('Initializing Agent.') - if not self.is_enabled(): - return - try: - self._config = AgentConfig() - self._init = AgentInit(self._config) - except Exception as err: # pylint: disable=W0703 - logger.error('Failed to initialize Agent: exception=%s, stacktrace=%s', - err, - traceback.format_exc()) + '''Initializer''' + if not self._initialized: # pylint: disable=E0203: + logger.debug('Initializing Agent.') + if not self.is_enabled(): + return + try: + self._config = AgentConfig() + self._init = AgentInit(self._config) + self._initialized = True + except Exception as err: # pylint: disable=W0703 + logger.error('Failed to initialize Agent: exception=%s, stacktrace=%s', + err, + traceback.format_exc()) def register_flask_app(self, app: flask.Flask = None) -> None: '''Register the flask instrumentation module wrapper''' @@ -261,8 +274,6 @@

    Functions

    try: formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') - handler = logging.FileHandler('agent.log', mode='a') - handler.setFormatter(formatter) screen_handler = logging.StreamHandler(stream=sys.stdout) screen_handler.setFormatter(formatter) log_level = logging.INFO @@ -284,7 +295,6 @@

    Functions

    if ht_log_level == 'NOTSET': log_level = logging.NOTSET logger_.setLevel(log_level) - logger_.addHandler(handler) logger_.addHandler(screen_handler) return logger_ except Exception as err: # pylint: disable=W0703 @@ -304,26 +314,41 @@

    Classes

    Top-level entry point for Hypertrace agent.

    -

    Constructor

    +

    Initializer

    Expand source code 
    class Agent:
     '''Top-level entry point for Hypertrace agent.'''
+    _instance = None
+    _singleton_lock = threading.Lock()
+
+    def __new__(cls):
+        '''constructor'''
+        if cls._instance is None:
+            with cls._singleton_lock:
+                logger.debug('Creating Agent')
+                cls._instance = super(Agent, cls).__new__(cls)
+                cls._instance._initialized = False
+        else:
+            logger.debug('Using existing Agent.')
+        return cls._instance
 
     def __init__(self):
-        '''Constructor'''
-        logger.debug('Initializing Agent.')
-        if not self.is_enabled():
-            return
-        try:
-            self._config = AgentConfig()
-            self._init = AgentInit(self._config)
-        except Exception as err:  # pylint: disable=W0703
-            logger.error('Failed to initialize Agent: exception=%s, stacktrace=%s',
-                         err,
-                         traceback.format_exc())
+        '''Initializer'''
+        if not self._initialized: # pylint: disable=E0203:
+            logger.debug('Initializing Agent.')
+            if not self.is_enabled():
+                return
+            try:
+                self._config = AgentConfig()
+                self._init = AgentInit(self._config)
+                self._initialized = True
+            except Exception as err:  # pylint: disable=W0703
+                logger.error('Failed to initialize Agent: exception=%s, stacktrace=%s',
+                             err,
+                             traceback.format_exc())
 
     def register_flask_app(self, app: flask.Flask = None) -> None:
         '''Register the flask instrumentation module wrapper'''
diff --git a/docs/hypertrace/agent/init/index.html b/docs/hypertrace/agent/init/index.html
index f203e59b..4fc4ce3f 100644
--- a/docs/hypertrace/agent/init/index.html
+++ b/docs/hypertrace/agent/init/index.html
@@ -63,7 +63,6 @@ 
    Module hypertrace.agent.init
    
             "requests": False,
             "aiohttp_client": False
         }
-
         self._tracer_provider = None
 
         try:
@@ -152,13 +151,30 @@ 
    Module hypertrace.agent.init
    
         '''Creates a flask instrumentation wrapper using the config defined in hypertraceconfig'''
         logger.debug('Calling AgentInit.flaskInit().')
         try:
+            if self.is_registered('flask'):
+                return
             from hypertrace.agent.instrumentation.flask import FlaskInstrumentorWrapper  # pylint: disable=C0415
             self._modules_initialized['flask'] = True
             self._flask_instrumentor_wrapper = FlaskInstrumentorWrapper()
+            call_default_instrumentor = True
+            # There are two ways to initialize the flask instrumenation
+            # wrapper. The first (and original way) instruments the specific
+            # Flask object that is passed in). The second way is to globally
+            # replace the Flask class definition with the hypertrace instrumentation
+            # wrapper class.
+            #
+            # If an app object is provided, then the flask wrapper is initialized
+            # by calling the instrument_app method. Then, there is no need to call
+            # instrument() (so, we pass False as the second argument to
+            # self.init_instrumentor_wrapper_base_for_http().
+            #
+            # If no app object was provided, then instrument() is called.
             if app:
                 self._flask_instrumentor_wrapper.instrument_app(app)
+                call_default_instrumentor = False
             self.init_instrumentor_wrapper_base_for_http(
-                self._flask_instrumentor_wrapper)
+                self._flask_instrumentor_wrapper,
+                call_default_instrumentor)
         except Exception as err:  # pylint: disable=W0703
             logger.error(constants.INST_WRAP_EXCEPTION_MSSG,
                          'flask',
@@ -169,6 +185,8 @@ 
    Module hypertrace.agent.init
    
         '''Creates a grpc server wrapper based on hypertrace config'''
         logger.debug('Calling AgentInit.grpcServerInit')
         try:
+            if self.is_registered('grpc:server'):
+                return
             from hypertrace.agent.instrumentation.grpc import (  # pylint: disable=C0415
                 GrpcInstrumentorServerWrapper
             )
@@ -196,6 +214,8 @@ 
    Module hypertrace.agent.init
    
         '''Creates a grpc client wrapper using the config defined in hypertraceconfig'''
         logger.debug('Calling AgentInit.grpcClientInit')
         try:
+            if self.is_registered('grpc:client'):
+                return
             from hypertrace.agent.instrumentation.grpc import (  # pylint: disable=C0415
                 GrpcInstrumentorClientWrapper
             )
@@ -224,6 +244,8 @@ 
    Module hypertrace.agent.init
    
         '''Creates a mysql server wrapper using the config defined in hypertraceconfig'''
         logger.debug('Calling AgentInit.mysqlInit()')
         try:
+            if self.is_registered('mysql'):
+                return
             from hypertrace.agent.instrumentation.mysql import (  # pylint: disable=C0415
                 MySQLInstrumentorWrapper
             )
@@ -242,6 +264,8 @@ 
    Module hypertrace.agent.init
    
         '''Creates a postgresql client wrapper using the config defined in hypertraceconfig'''
         logger.debug('Calling AgentInit.postgreSQLInit()')
         try:
+            if self.is_registered('postgresql'):
+                return
             from hypertrace.agent.instrumentation.postgresql import (  # pylint: disable=C0415
                 PostgreSQLInstrumentorWrapper
             )
@@ -260,6 +284,8 @@ 
    Module hypertrace.agent.init
    
         '''Creates a requests client wrapper using the config defined in hypertraceconfig'''
         logger.debug('Calling AgentInit.requestsInit()')
         try:
+            if self.is_registered('requests'):
+                return
             from hypertrace.agent.instrumentation.requests import (  # pylint: disable=C0415
                 RequestsInstrumentorWrapper
             )
@@ -278,6 +304,8 @@ 
    Module hypertrace.agent.init
    
         '''Creates an aiohttp-client wrapper using the config defined in hypertraceconfig'''
         logger.debug('Calling AgentInit.aioHttpClientInit()')
         try:
+            if self.is_registered('aiohttp_client'):
+                return
             from hypertrace.agent.instrumentation.aiohttp import (  # pylint: disable=C0415
                 AioHttpClientInstrumentorWrapper
             )
@@ -292,10 +320,13 @@ 
    Module hypertrace.agent.init
    
                          traceback.format_exc())
 
     # Common wrapper initialization logic
-    def init_instrumentor_wrapper_base_for_http(self, instrumentor) -> None:
+    def init_instrumentor_wrapper_base_for_http(self,
+                                                instrumentor,
+                                                call_instrument: bool = True) -> None:
         '''Common wrapper initialization logic'''
         logger.debug('Calling AgentInit.initInstrumentorWrapperBaseForHTTP().')
-        instrumentor.instrument()
+        if call_instrument:
+            instrumentor.instrument()
         instrumentor.set_process_request_headers(
             self._config.agent_config.data_capture.http_headers.request)
         instrumentor.set_process_request_body(
@@ -353,7 +384,14 @@ 
    Module hypertrace.agent.init
    
         except Exception as err:  # pylint: disable=W0703
             logger.error('Failed to initialize OTLP exporter: exception=%s, stacktrace=%s',
                          err,
-                         traceback.format_exc())
    + traceback.format_exc()) + + def is_registered(self, module: str) -> bool: + '''Is an instrumentation module already registered?''' + try: + return self._modules_initialized[module] + except Exception as err: # pylint: disable=W0703,W0612 + return False
    @@ -392,7 +430,6 @@

    Classes

    "requests": False, "aiohttp_client": False } - self._tracer_provider = None try: @@ -481,13 +518,30 @@

    Classes

    '''Creates a flask instrumentation wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.flaskInit().') try: + if self.is_registered('flask'): + return from hypertrace.agent.instrumentation.flask import FlaskInstrumentorWrapper # pylint: disable=C0415 self._modules_initialized['flask'] = True self._flask_instrumentor_wrapper = FlaskInstrumentorWrapper() + call_default_instrumentor = True + # There are two ways to initialize the flask instrumenation + # wrapper. The first (and original way) instruments the specific + # Flask object that is passed in). The second way is to globally + # replace the Flask class definition with the hypertrace instrumentation + # wrapper class. + # + # If an app object is provided, then the flask wrapper is initialized + # by calling the instrument_app method. Then, there is no need to call + # instrument() (so, we pass False as the second argument to + # self.init_instrumentor_wrapper_base_for_http(). + # + # If no app object was provided, then instrument() is called. if app: self._flask_instrumentor_wrapper.instrument_app(app) + call_default_instrumentor = False self.init_instrumentor_wrapper_base_for_http( - self._flask_instrumentor_wrapper) + self._flask_instrumentor_wrapper, + call_default_instrumentor) except Exception as err: # pylint: disable=W0703 logger.error(constants.INST_WRAP_EXCEPTION_MSSG, 'flask', @@ -498,6 +552,8 @@

    Classes

    '''Creates a grpc server wrapper based on hypertrace config''' logger.debug('Calling AgentInit.grpcServerInit') try: + if self.is_registered('grpc:server'): + return from hypertrace.agent.instrumentation.grpc import ( # pylint: disable=C0415 GrpcInstrumentorServerWrapper ) @@ -525,6 +581,8 @@

    Classes

    '''Creates a grpc client wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.grpcClientInit') try: + if self.is_registered('grpc:client'): + return from hypertrace.agent.instrumentation.grpc import ( # pylint: disable=C0415 GrpcInstrumentorClientWrapper ) @@ -553,6 +611,8 @@

    Classes

    '''Creates a mysql server wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.mysqlInit()') try: + if self.is_registered('mysql'): + return from hypertrace.agent.instrumentation.mysql import ( # pylint: disable=C0415 MySQLInstrumentorWrapper ) @@ -571,6 +631,8 @@

    Classes

    '''Creates a postgresql client wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.postgreSQLInit()') try: + if self.is_registered('postgresql'): + return from hypertrace.agent.instrumentation.postgresql import ( # pylint: disable=C0415 PostgreSQLInstrumentorWrapper ) @@ -589,6 +651,8 @@

    Classes

    '''Creates a requests client wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.requestsInit()') try: + if self.is_registered('requests'): + return from hypertrace.agent.instrumentation.requests import ( # pylint: disable=C0415 RequestsInstrumentorWrapper ) @@ -607,6 +671,8 @@

    Classes

    '''Creates an aiohttp-client wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.aioHttpClientInit()') try: + if self.is_registered('aiohttp_client'): + return from hypertrace.agent.instrumentation.aiohttp import ( # pylint: disable=C0415 AioHttpClientInstrumentorWrapper ) @@ -621,10 +687,13 @@

    Classes

    traceback.format_exc()) # Common wrapper initialization logic - def init_instrumentor_wrapper_base_for_http(self, instrumentor) -> None: + def init_instrumentor_wrapper_base_for_http(self, + instrumentor, + call_instrument: bool = True) -> None: '''Common wrapper initialization logic''' logger.debug('Calling AgentInit.initInstrumentorWrapperBaseForHTTP().') - instrumentor.instrument() + if call_instrument: + instrumentor.instrument() instrumentor.set_process_request_headers( self._config.agent_config.data_capture.http_headers.request) instrumentor.set_process_request_body( @@ -682,7 +751,14 @@

    Classes

    except Exception as err: # pylint: disable=W0703 logger.error('Failed to initialize OTLP exporter: exception=%s, stacktrace=%s', err, - traceback.format_exc()) + traceback.format_exc()) + + def is_registered(self, module: str) -> bool: + '''Is an instrumentation module already registered?''' + try: + return self._modules_initialized[module] + except Exception as err: # pylint: disable=W0703,W0612 + return False

    Methods

    @@ -699,6 +775,8 @@

    Methods

    '''Creates an aiohttp-client wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.aioHttpClientInit()') try: + if self.is_registered('aiohttp_client'): + return from hypertrace.agent.instrumentation.aiohttp import ( # pylint: disable=C0415 AioHttpClientInstrumentorWrapper ) @@ -762,13 +840,30 @@

    Methods

    '''Creates a flask instrumentation wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.flaskInit().') try: + if self.is_registered('flask'): + return from hypertrace.agent.instrumentation.flask import FlaskInstrumentorWrapper # pylint: disable=C0415 self._modules_initialized['flask'] = True self._flask_instrumentor_wrapper = FlaskInstrumentorWrapper() + call_default_instrumentor = True + # There are two ways to initialize the flask instrumenation + # wrapper. The first (and original way) instruments the specific + # Flask object that is passed in). The second way is to globally + # replace the Flask class definition with the hypertrace instrumentation + # wrapper class. + # + # If an app object is provided, then the flask wrapper is initialized + # by calling the instrument_app method. Then, there is no need to call + # instrument() (so, we pass False as the second argument to + # self.init_instrumentor_wrapper_base_for_http(). + # + # If no app object was provided, then instrument() is called. if app: self._flask_instrumentor_wrapper.instrument_app(app) + call_default_instrumentor = False self.init_instrumentor_wrapper_base_for_http( - self._flask_instrumentor_wrapper) + self._flask_instrumentor_wrapper, + call_default_instrumentor) except Exception as err: # pylint: disable=W0703 logger.error(constants.INST_WRAP_EXCEPTION_MSSG, 'flask', @@ -789,6 +884,8 @@

    Methods

    '''Creates a grpc client wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.grpcClientInit') try: + if self.is_registered('grpc:client'): + return from hypertrace.agent.instrumentation.grpc import ( # pylint: disable=C0415 GrpcInstrumentorClientWrapper ) @@ -826,6 +923,8 @@

    Methods

    '''Creates a grpc server wrapper based on hypertrace config''' logger.debug('Calling AgentInit.grpcServerInit') try: + if self.is_registered('grpc:server'): + return from hypertrace.agent.instrumentation.grpc import ( # pylint: disable=C0415 GrpcInstrumentorServerWrapper ) @@ -862,6 +961,8 @@

    Methods

    '''Creates a mysql server wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.mysqlInit()') try: + if self.is_registered('mysql'): + return from hypertrace.agent.instrumentation.mysql import ( # pylint: disable=C0415 MySQLInstrumentorWrapper ) @@ -889,6 +990,8 @@

    Methods

    '''Creates a postgresql client wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.postgreSQLInit()') try: + if self.is_registered('postgresql'): + return from hypertrace.agent.instrumentation.postgresql import ( # pylint: disable=C0415 PostgreSQLInstrumentorWrapper ) @@ -916,6 +1019,8 @@

    Methods

    '''Creates a requests client wrapper using the config defined in hypertraceconfig''' logger.debug('Calling AgentInit.requestsInit()') try: + if self.is_registered('requests'): + return from hypertrace.agent.instrumentation.requests import ( # pylint: disable=C0415 RequestsInstrumentorWrapper ) @@ -931,7 +1036,7 @@

    Methods

    -def init_instrumentor_wrapper_base_for_http(self, instrumentor) ‑> NoneType +def init_instrumentor_wrapper_base_for_http(self, instrumentor, call_instrument: bool = True) ‑> NoneType

    Common wrapper initialization logic

    @@ -939,10 +1044,13 @@

    Methods

    Expand source code -
    def init_instrumentor_wrapper_base_for_http(self, instrumentor) -> None:
+
    def init_instrumentor_wrapper_base_for_http(self,
+                                            instrumentor,
+                                            call_instrument: bool = True) -> None:
     '''Common wrapper initialization logic'''
     logger.debug('Calling AgentInit.initInstrumentorWrapperBaseForHTTP().')
-    instrumentor.instrument()
+    if call_instrument:
+        instrumentor.instrument()
     instrumentor.set_process_request_headers(
         self._config.agent_config.data_capture.http_headers.request)
     instrumentor.set_process_request_body(
@@ -1017,6 +1125,23 @@ 
    Methods
    
     trace.set_tracer_provider(self._tracer_provider)
    +
    +def is_registered(self, module: str) ‑> bool +
    +
    +

    Is an instrumentation module already registered?

    +
    + +Expand source code + +
    def is_registered(self, module: str) -> bool:
+    '''Is an instrumentation module already registered?'''
+    try:
+        return self._modules_initialized[module]
+    except Exception as err: # pylint: disable=W0703,W0612
+        return False
    +
    +
    def register_processor(self, processor) ‑> NoneType
    @@ -1084,6 +1209,7 @@

    init_instrumentor_wrapper_base_for_http
  • init_propagation
  • init_trace_provider
    • +
  • is_registered
  • register_processor
  • set_console_span_processor
    • diff --git a/docs/hypertrace/agent/instrumentation/flask/index.html b/docs/hypertrace/agent/instrumentation/flask/index.html index da1acbba..58633a19 100644 --- a/docs/hypertrace/agent/instrumentation/flask/index.html +++ b/docs/hypertrace/agent/instrumentation/flask/index.html @@ -171,13 +171,16 @@

    Module hypertrace.agent.instrumentation.flask

    Classes flask.Flask = _HypertraceInstrumentedFlask # Initialize instrumentation wrapper - def instrument_app(self, app, name_callback=get_default_span_name) -> None: + def instrument_app(self, + app, + name_callback=get_default_span_name, + tracer_provider=None) -> None: '''Initialize instrumentation''' logger.debug('Entering FlaskInstrumentorWrapper.instument_app().') try: # Call parent class's initialization - super().instrument_app(app, name_callback) + super().instrument_app(app, name_callback, tracer_provider) self._app = app # Set pre-request handler @@ -339,7 +345,7 @@

    Methods

    -def instrument_app(self, app, name_callback=<function get_default_span_name>) ‑> NoneType +def instrument_app(self, app, name_callback=<function get_default_span_name>, tracer_provider=None) ‑> NoneType

    Initialize instrumentation

    @@ -347,13 +353,16 @@

    Methods

    Expand source code -
    def instrument_app(self, app, name_callback=get_default_span_name) -> None:
+
    def instrument_app(self,
+                   app,
+                   name_callback=get_default_span_name,
+                   tracer_provider=None) -> None:
     '''Initialize instrumentation'''
     logger.debug('Entering FlaskInstrumentorWrapper.instument_app().')
     try:
 
         # Call parent class's initialization
-        super().instrument_app(app, name_callback)
+        super().instrument_app(app, name_callback, tracer_provider)
 
         self._app = app
         # Set pre-request handler

From 340973580b0f4fabde9b4ffbc6267326e3d11c9d Mon Sep 17 00:00:00 2001
From: rcbjBlueMars 
Date: Wed, 12 May 2021 19:36:58 -0700
Subject: [PATCH 22/23] Removing extra argument in
 FlaskInstrumentor.instrument_app().

---
 src/hypertrace/agent/instrumentation/flask/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hypertrace/agent/instrumentation/flask/__init__.py b/src/hypertrace/agent/instrumentation/flask/__init__.py
index 2f3f4565..4e50106f 100644
--- a/src/hypertrace/agent/instrumentation/flask/__init__.py
+++ b/src/hypertrace/agent/instrumentation/flask/__init__.py
@@ -151,7 +151,7 @@ def instrument_app(self,
         try:
 
             # Call parent class's initialization
-            super().instrument_app(app, name_callback, tracer_provider)
+            super().instrument_app(app, name_callback)
 
             self._app = app
             # Set pre-request handler

From 98e8f365edfbf575f8d104d4d16c5eee8a4e547f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Carlos=20Ch=C3=A1vez?= 
Date: Thu, 13 May 2021 07:03:38 +0200
Subject: [PATCH 23/23] docs: fixes readme.

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 9050a5cf..6bcbe1f9 100644
--- a/README.md
+++ b/README.md
@@ -43,14 +43,14 @@ agent.register_mysql() # instrument the MySQL client
 
 or
 
-- Use the autoinstrumentation CLI (without any modifiation to application code)
+- Use the autoinstrumentation CLI (without any modification to application code)
 ```
 HT_INSTRUMENTED_MODULES=flask,mysql
 hypertrace-instrument python app.py
 ```
 By default, all supported modules are instrumented.
 
-Note, do not attempt to instantiate a hypertrace.agent.Agent object while using hypertrace-instrument.
+*Important:* do not attempt to instantiate a hypertrace.agent.Agent object while using hypertrace-instrument.
 
 For further examples, check our [examples section](./examples)