Initial implementation (#1)

* intial implementation

* integration test

* Update CHANGELOG.md

* Update README.md

* idl submodule

* make headers generic

* fix test

* fixed type in examples

* refactored HttpRecorder into Async and Sync versions

* refactored redundant headers out of http_recorders

* refactored constants to separate module. addressed flake8 formatting. throw NotImplemented() from abstract classes

* clean up extra lines

.gitignore

@@ -97,6 +97,8 @@ venv.bak/
 # Rope project settings
 .ropeproject
 
+.idea/
+
 # mkdocs documentation
 /site
 

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "haystack-idl"]
+	path = haystack-idl
+	url = https://github.com/ExpediaDotCom/haystack-idl.git

CHANGELOG.md

@@ -0,0 +1,4 @@
+# Changes by Version
+
+## 1.0.0 (2019-02-25)
+Haystack OpenTracing compliant library for Python

Makefile

@@ -0,0 +1,52 @@
+.PHONY: build
+build:
+	echo "nothing to compile"
+
+.PHONY: bootstrap
+bootstrap:
+	python setup.py install
+
+.PHONY: test
+test:
+	python setup.py test
+
+.PHONY: example
+example: bootstrap
+	python examples/main.py
+
+.PHONY: lint
+lint:
+	pip install flake8
+	python -m flake8 ./haystack --exclude *_pb2*
+
+.PHONY: integration_tests
+integration_tests:
+	docker-compose -f tests/integration/docker-compose.yml -p sandbox up -d
+	sleep 15
+	docker run -it
+	    --rm \
+		--network=sandbox_default \
+		-v $(pwd):/ws \
+		-w /ws \
+		python:3.6 \
+		/bin/sh -c 'python setup.py install && pip install kafka-python && python tests/integration/integration.py'
+	docker-compose -f tests/integration/docker-compose.yml -p sandbox stop
+
+.PHONY: dist
+dist: bootstrap lint test integration_tests
+	pip install wheel
+	python setup.py sdist
+	python setup.py bdist_wheel
+
+.PHONY: publish
+publish:
+	pip install twine
+	python -m twine upload dist/*
+
+.PHONY: proto_compile
+proto_compile:
+	git submodule init -- ./haystack-idl
+	git submodule update
+	pip install grpcio-tools
+	python -m grpc_tools.protoc -I haystack-idl/ --python_out=./haystack  haystack-idl/proto/span.proto
+	python -m grpc_tools.protoc -I haystack-idl/proto --python_out=./haystack/proto --grpc_python_out=./haystack/proto agent/spanAgent.proto

README.md

@@ -1,2 +1,41 @@
-# haystack-client-python
-Haystack bindings for OpenTracing in Python
+# Haystack bindings for Python OpenTracing API
+This is Haystack's client library for Python that implements [OpenTracing](https://github.com/opentracing/opentracing-python/)
+
+Further information can be found on [opentracing.io](https://opentracing.io/) 
+
+## Using this library
+See examples in /examples directory. 
+
+**If there is a Scope, it will act as the parent to any newly started Span** unless the programmer passes 
+`ignore_active_span=True` at `start_span()/start_active_span()` time or specified parent context explicitly using 
+`childOf=parent_context`
+
+As demonstrated in the examples, starting a span can be done as a managed resource using `start_active_span()`
+```python
+with opentracing.tracer.start_active_span("span-name") as scope:
+    do_stuff()
+```
+
+or finish the span on your own terms with
+```python
+span = opentracing.tracer.start_span("span-name")
+do_stuff()
+span.finish()
+```
+
+See opentracing [usage](https://github.com/opentracing/opentracing-python/#usage) for additional information.
+
+#### Logging
+All modules define their logger via `logging.getLogger(__name__)`
+
+So in order to define specific logging format or level for this library use `getLogger('haystack')` or configure the
+root logger.
+
+## How to configure build environment
+Create a python3 virtual environment, activate it and then `make bootstrap`
+
+## Running the example code
+`make example`
+
+## How to Release this library
+TBD

examples/main.py

@@ -0,0 +1,93 @@
+import opentracing
+import time
+import logging
+from opentracing.ext import tags
+from haystack import HaystackTracer
+from haystack import AsyncHttpRecorder
+from haystack import LoggerRecorder
+
+recorder = LoggerRecorder()
+logging.basicConfig(level=logging.DEBUG)
+
+
+def act_as_remote_service(headers):
+    # remote service would have it"s own tracer
+    with HaystackTracer("Service-B", recorder,) as tracer:
+        opentracing.tracer = tracer
+
+        # simulate network transfer delay
+        time.sleep(.25)
+
+        # now as-if this was executing on the remote service, extract the parent span ctx from headers
+        upstream_span_ctx = opentracing.tracer.extract(opentracing.Format.HTTP_HEADERS, headers)
+        with opentracing.tracer.start_active_span("controller_method", child_of=upstream_span_ctx) as parent_scope:
+            parent_scope.span.set_tag(tags.SPAN_KIND, "server")
+            # simulate downstream service doing some work before replying
+            time.sleep(1)
+
+
+def make_a_downstream_request():
+    # create a child span representing the downstream request from current span.
+    # Behind the scenes this uses the scope_manger to access the current active
+    # span and create a child of it.
+    with opentracing.tracer.start_active_span("downstream_req") as child_scope:
+
+        child_scope.span.set_tag(tags.SPAN_KIND, "client")
+
+        # add some baggage (i.e. something that should propagate across
+        # process boundaries)
+        child_scope.span.set_baggage_item("baggage-item", "baggage-item-value")
+
+        # carrier here represents http headers
+        carrier = {}
+        opentracing.tracer.inject(child_scope.span.context, opentracing.Format.HTTP_HEADERS, carrier)
+        act_as_remote_service(carrier)
+
+        # process the response from downstream
+        time.sleep(.15)
+
+
+def use_http_recorder():
+    endpoint = "http://<replace_me>"
+    global recorder
+    recorder = AsyncHttpRecorder(collector_url=endpoint)
+
+
+def main():
+    """
+    Represents an application/service
+    """
+    # instantiate a tracer with app version common tag and set it
+    # to opentracing.tracer property
+    opentracing.tracer = HaystackTracer("Service-A",
+                                        recorder,
+                                        common_tags={"app.version": "1234"})
+
+    logging.info("mock request received")
+    with opentracing.tracer.start_active_span("a_controller_method") as parent_scope:
+
+        # add a tag, tags are part of a span and do not propagate
+        # (tags have semantic conventions, see https://opentracing.io/specification/conventions/)
+        parent_scope.span.set_tag(tags.HTTP_URL, "http://localhost/mocksvc")
+        parent_scope.span.set_tag(tags.HTTP_METHOD, "GET")
+        parent_scope.span.set_tag(tags.SPAN_KIND, "server")
+
+        # doing some work.. validation, processing, etc
+        time.sleep(.25)
+
+        # tag the span with some information about the processing
+        parent_scope.span.log_kv(
+            {"string": "foobar", "int": 42, "float": 4.2, "bool": True, "obj": {"ok": "hmm", "blah": 4324}})
+
+        make_a_downstream_request()
+
+        # uncomment this line to tag the span with an error
+        # parent_scope.span.set_tag(tags.ERROR, True)
+
+    logging.info("done in main")
+
+
+if __name__ == "__main__":
+    # uncomment line below to send traces to haystack collector using http recorder
+    # use_http_recorder()
+    main()

examples/serverless/handler.py

@@ -0,0 +1,83 @@
+# AWS Lambda Example
+import opentracing
+import os
+from requests import RequestException
+from opentracing.ext import tags
+from haystack import HaystackTracer
+from haystack import SyncHttpRecorder
+
+"""
+Note: Recorder implementation in serverless applications requires careful 
+consideration. For Example, in AWS, due to the way AWS lambda freezes execution 
+context, it's not reliable to send requests via the AsyncHttpRecorder. If the 
+function is not time-sensitive in reply or is async, SyncHttpRecorder is a good 
+fit as shown below. If the function cannot afford to dispatch the span 
+in-process, then it is recommended to either setup a haystack agent in the 
+network and utilize HaystackAgentRecorder or offload the span record dispatching
+via Queue -> Worker model. In AWS this could mean implementing a SQSRecorder 
+which puts the finished span onto a SQS queue. The queue could then notify a 
+lambda implementing SyncHttpRecorder to dispatch the records. 
+"""
+
+recorder = SyncHttpRecorder(os.env["COLLECTOR_URL"])
+
+# suppose it is desired to tag all traces with the application version
+common_tags = {
+    "svc_ver": os["APP_VERSION"]
+}
+opentracing.tracer = HaystackTracer("example-service",
+                                    recorder, common_tags=common_tags)
+
+
+def invoke_downstream(headers):
+    return "done"
+
+
+def process_downstream_response(response):
+    return "done"
+
+
+def handler(event, context):
+
+    # extract the span context from headers if this is a downstream service
+    parent_ctx = opentracing.tracer.extract(opentracing.Format.HTTP_HEADERS,
+                                            event)
+
+    # now create a span representing the work of this entire function
+    with opentracing.tracer.start_active_span("example-operation",
+                                              child_of=parent_ctx) as request_scope:
+
+        # log any important tags/baggage to the handler's span
+        span = request_scope.span
+        span.set_tag(tags.HTTP_URL, event["url"])
+        span.set_tag(tags.SPAN_KIND, "server")
+
+        # do some work (for ex. validation)
+        # then log an event to the span which will timestamp the time taken up until this point
+        span.log_kv({"validation_result": "success"})
+
+        # an example of invoking a downstream service
+        # Behind the scenes this uses the scope_manger to access the current active span and create a child of it.
+        with opentracing.tracer.start_active_span("example-downstream-call") as child_scope:
+            # span kind tags help haystack stitch and merge spans
+            child_scope.span.set_tag(tags.SPAN_KIND, "client")
+
+            headers = {
+                "Content-Type": "application/json"
+            }
+
+            # inject the child span into the headers of downstream request
+            opentracing.tracer.inject(child_scope.span.context, opentracing.Format.HTTP_HEADERS, headers)
+
+            try:
+                span.set_tag(tags.HTTP_URL, "https://downstream.url")
+                response = invoke_downstream(headers)
+            except RequestException:
+                # when there's an issue a span can be tagged with error to flag it in haystack trends
+                child_scope.span.set_tag(tags.ERROR, True)
+
+        # be cognitive of which scope is handling the response processing. here we're back into the request_scope, thus
+        # child_scope's span will only represent total time interacting with the downstream service
+        lambda_response = process_downstream_response(response)
+
+    return lambda_response

haystack/__init__.py

@@ -0,0 +1,5 @@
+from .tracer import HaystackTracer  # noqa
+from .http_recorder import AsyncHttpRecorder  # noqa
+from .http_recorder import SyncHttpRecorder  # noqa
+from .agent_recorder import HaystackAgentRecorder  # noqa
+from .recorder import LoggerRecorder  # noqa