omnibus: update datadog-trace-agent config and add invoke task

* Merge datadog-agent datadog-trace-agent software definitions together
* Add trace-agent makefile for agent5 omnibus build
* Add DataDog/apm code owners to paths /cmd/trace-agent/ and /pkg/trace/
* pkg/trace/info: remove git_version
* gitignore: added trace-agent-msg.h to ignore list

.github/CODEOWNERS

@@ -10,6 +10,7 @@
 # and another the rest of the directory.
 
 /cmd/                                   @DataDog/agent-core
+/cmd/trace-agent/                       @DataDog/apm
 /omnibus/                               @DataDog/agent-core
 /pkg/                                   @DataDog/agent-core
 /pkg/aggregator/                        @DataDog/agent-core
@@ -20,6 +21,7 @@
 /pkg/serializer/                        @DataDog/agent-core
 /pkg/status/                            @DataDog/agent-core
 /pkg/version/                           @DataDog/agent-core
+/pkg/trace/                             @DataDog/apm
 
 /cmd/cluster-agent/                     @DataDog/container-integrations
 /Dockerfiles/                           @DataDog/container-integrations

.gitignore

@@ -72,3 +72,9 @@ cmd/agent/android/app/build
 
 # local copy of device signing cert
 platform.pk8
+
+# trace agent windows artifacts
+cmd/trace-agent/windows_resources/trace-agent-msg.rc
+cmd/trace-agent/windows_resources/*.bin
+pkg/trace/info/git_version.go
+trace-agent-msg.h

Makefile.trace

@@ -0,0 +1,47 @@
+# This Makefile is used within the release process of the main Datadog Agent to pre-package datadog-trace-agent:
+# https://github.com/DataDog/datadog-agent/blob/2b7055c/omnibus/config/software/datadog-trace-agent.rb
+
+# if the TRACE_AGENT_VERSION environment variable isn't set, default to 0.99.0
+TRACE_AGENT_VERSION := $(if $(TRACE_AGENT_VERSION),$(TRACE_AGENT_VERSION), 0.99.0)
+
+# break up the version
+SPLAT = $(subst ., ,$(TRACE_AGENT_VERSION))
+VERSION_MAJOR = $(shell echo $(word 1, $(SPLAT)) | sed 's/[^0-9]*//g')
+VERSION_MINOR = $(shell echo $(word 2, $(SPLAT)) | sed 's/[^0-9]*//g')
+VERSION_PATCH = $(shell echo $(word 3, $(SPLAT)) | sed 's/[^0-9]*//g')
+
+# account for some defaults
+VERSION_MAJOR := $(if $(VERSION_MAJOR),$(VERSION_MAJOR), 0)
+VERSION_MINOR := $(if $(VERSION_MINOR),$(VERSION_MINOR), 0)
+VERSION_PATCH := $(if $(VERSION_PATCH),$(VERSION_PATCH), 0)
+
+install:
+	# generate versioning information and installing the binary.
+	go generate ./pkg/trace/info
+	go install ./cmd/trace-agent
+
+binaries:
+	test -n "$(V)" # $$V must be set to the release version tag, e.g. "make binaries V=1.2.3"
+
+	# compiling release binaries for tag $(V)
+	git checkout $(V)
+	mkdir -p ./bin
+	TRACE_AGENT_VERSION=$(V) go generate ./internal/info
+	go get -u github.com/karalabe/xgo
+	xgo -dest=bin -go=1.10 -out=trace-agent-$(V) -targets=windows-6.1/amd64,linux/amd64,darwin-10.11/amd64 ./cmd/trace-agent
+	mv ./bin/trace-agent-$(V)-windows-6.1-amd64.exe ./bin/trace-agent-$(V)-windows-amd64.exe
+	mv ./bin/trace-agent-$(V)-darwin-10.11-amd64 ./bin/trace-agent-$(V)-darwin-amd64 
+	git reset --hard head && git checkout -
+
+ci:
+	# task used by CI
+	go get -u golang.org/x/lint/golint
+	golint -set_exit_status=1 ./cmd/trace-agent ./internal/filters ./internal/api ./internal/test ./internal/info ./internal/quantile ./internal/obfuscate ./internal/sampler ./internal/metrics ./internal/watchdog ./internal/writer ./internal/flags ./internal/osutil
+	go install ./cmd/trace-agent
+	go test -v -race ./...
+
+windows:
+	# pre-packages resources needed for the windows release
+	windmc --target pe-x86-64 -r cmd/trace-agent/windows_resources cmd/trace-agent/windows_resources/trace-agent-msg.mc
+	windres --define MAJ_VER=$(VERSION_MAJOR) --define MIN_VER=$(VERSION_MINOR) --define PATCH_VER=$(VERSION_PATCH) -i cmd/trace-agent/windows_resources/trace-agent.rc --target=pe-x86-64 -O coff -o cmd/trace-agent/rsrc.syso
+

omnibus/config/projects/agent.rb

@@ -126,7 +126,6 @@
 dependency 'jmxfetch'
 
 # External agents
-dependency 'datadog-trace-agent'
 dependency 'datadog-process-agent' # Includes network-tracer
 
 if osx?

omnibus/config/software/datadog-agent.rb

@@ -64,6 +64,15 @@
 
   copy 'bin', install_dir
 
+
+  command "invoke trace-agent.build", :env => env
+
+  if windows?
+    copy 'bin/trace-agent/trace-agent.exe', "#{Omnibus::Config.source_dir()}/datadog-agent/src/github.com/DataDog/datadog-agent/bin/agent"
+  else
+    copy 'bin/trace-agent/trace-agent', "#{install_dir}/embedded/bin"
+  end
+
   if linux?
     if debian?
       erb source: "upstart_debian.conf.erb",

pkg/trace/info/make.go

@@ -43,8 +43,7 @@ func main() {
 	}
 }
 
-const template = `
-// Code generated by 'go run make.go'. DO NOT EDIT.
+const template = `// Code generated by 'go run make.go'. DO NOT EDIT.
 
 package info
 

pkg/trace/writer/backoff/exponential_nix_test.go

@@ -0,0 +1,19 @@
+// +build !windows
+
+package backoff
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDefaultRandomSeed(t *testing.T) {
+	assert := assert.New(t)
+
+	delayProvider1 := DefaultExponentialDelayProvider()
+	delayProvider2 := DefaultExponentialDelayProvider()
+
+	// Ensure different timers are not synchronized in their backoffing (use different seeds)
+	assert.NotEqual(delayProvider1(0, nil), delayProvider2(0, nil))
+}

tasks/__init__.py

@@ -4,7 +4,7 @@
 import os
 from invoke import Collection
 
-from . import agent, android, benchmarks, customaction, docker, dogstatsd, pylauncher, cluster_agent, systray, release
+from . import agent, trace, android, benchmarks, customaction, docker, dogstatsd, pylauncher, cluster_agent, systray, release
 
 from .go import fmt, lint, vet, cyclo, ineffassign, misspell, deps, lint_licenses, reset
 from .test import test, integration_tests, version, lint_teamassignment, lint_releasenote, lint_milestone, lint_filenames, e2e_tests
@@ -39,6 +39,7 @@
 ns.add_collection(cluster_agent)
 ns.add_collection(customaction)
 ns.add_collection(benchmarks, name="bench")
+ns.add_collection(trace, name="trace-agent")
 ns.add_collection(docker)
 ns.add_collection(dogstatsd)
 ns.add_collection(pylauncher)

tasks/agent.py

@@ -253,14 +253,13 @@ def integration_tests(ctx, install_deps=False, race=False, remote_docker=False):
     if remote_docker:
         test_args["exec_opts"] = "-exec \"inv docker.dockerize-test\""
 
-    go_cmd = 'INTEGRATION=yes go test {race_opt} -tags "{go_build_tags}" {exec_opts}'.format(**test_args)
+    go_cmd = 'go test {race_opt} -tags "{go_build_tags}" {exec_opts}'.format(**test_args)
 
     prefixes = [
         "./test/integration/config_providers/...",
         "./test/integration/corechecks/...",
         "./test/integration/listeners/...",
         "./test/integration/util/kubelet/...",
-        "./pkg/trace/test/testsuite/...",
     ]
 
     for prefix in prefixes:

tasks/test.py

@@ -18,6 +18,7 @@
 from .build_tags import get_default_build_tags, get_build_tags
 from .agent import integration_tests as agent_integration_tests
 from .dogstatsd import integration_tests as dsd_integration_tests
+from .trace import integration_tests as trace_integration_tests
 from .cluster_agent import integration_tests as dca_integration_tests
 
 PROFILE_COV = "profile.cov"
@@ -279,6 +280,7 @@ def integration_tests(ctx, install_deps=False, race=False, remote_docker=False):
     agent_integration_tests(ctx, install_deps, race, remote_docker)
     dsd_integration_tests(ctx, install_deps, race, remote_docker)
     dca_integration_tests(ctx, install_deps, race, remote_docker)
+    trace_integration_tests(ctx, install_deps, race, remote_docker)
 
 
 @task

tasks/trace.py

@@ -0,0 +1,85 @@
+import os
+import sys
+import shutil
+
+import invoke
+from invoke import task
+
+from .utils import bin_name, get_build_flags, get_version_numeric_only, load_release_versions
+from .utils import REPO_PATH
+from .build_tags import get_build_tags, get_default_build_tags, LINUX_ONLY_TAGS, REDHAT_AND_DEBIAN_ONLY_TAGS, REDHAT_AND_DEBIAN_DIST
+from .go import deps
+
+BIN_PATH = os.path.join(".", "bin", "trace-agent")
+DEFAULT_BUILD_TAGS = ["netcgo"]
+
+@task
+def build(ctx, rebuild=False, race=False, precompile_only=False, use_embedded_libs=False,
+          build_include=None, build_exclude=None, puppy=False, use_venv=False):
+    """
+    Build the trace agent.
+    """
+
+    # generate windows resources
+    if sys.platform == 'win32':
+        ver = get_version_numeric_only(ctx)
+        maj_ver, min_ver, patch_ver = ver.split(".")
+
+        ctx.run("windmc --target pe-x86-64 -r cmd/trace-agent/windows_resources cmd/trace-agent/windows_resources/trace-agent-msg.mc")
+        ctx.run("windres --define MAJ_VER={maj_ver} --define MIN_VER={min_ver} --define PATCH_VER={patch_ver} -i cmd/trace-agent/windows_resources/trace-agent.rc --target=pe-x86-64 -O coff -o cmd/trace-agent/rsrc.syso".format(
+            maj_ver=maj_ver,
+            min_ver=min_ver,
+            patch_ver=patch_ver
+        ))
+
+    ldflags, gcflags, env = get_build_flags(ctx, use_embedded_libs=use_embedded_libs, use_venv=use_venv)
+    build_include = DEFAULT_BUILD_TAGS if build_include is None else build_include.split(",")
+    build_exclude = [] if build_exclude is None else build_exclude.split(",")
+
+    if puppy:
+        # Puppy mode overrides whatever passed through `--build-exclude` and `--build-include`
+        build_tags = get_default_build_tags(puppy=True)
+    else:
+        build_tags = get_build_tags(build_include, build_exclude)
+
+    cmd = "go build {race_opt} {build_type} -tags \"{go_build_tags}\" "
+    cmd += "-o {agent_bin} -gcflags=\"{gcflags}\" -ldflags=\"{ldflags}\" {REPO_PATH}/cmd/trace-agent"
+
+    args = {
+        "race_opt": "-race" if race else "",
+        "build_type": "-a" if rebuild else ("-i" if precompile_only else ""),
+        "go_build_tags": " ".join(build_tags),
+        "agent_bin": os.path.join(BIN_PATH, bin_name("trace-agent", android=False)),
+        "gcflags": gcflags,
+        "ldflags": ldflags,
+        "REPO_PATH": REPO_PATH,
+    }
+
+    ctx.run("go generate {REPO_PATH}/pkg/trace/info".format(**args), env=env)
+    ctx.run(cmd.format(**args), env=env)
+
+@task
+def integration_tests(ctx, install_deps=False, race=False, remote_docker=False):
+    """
+    Run integration tests for trace agent
+    """
+    if install_deps:
+        deps(ctx)
+
+    test_args = {
+        "go_build_tags": " ".join(get_default_build_tags()),
+        "race_opt": "-race" if race else "",
+        "exec_opts": "",
+    }
+
+    if remote_docker:
+        test_args["exec_opts"] = "-exec \"inv docker.dockerize-test\""
+
+    go_cmd = 'INTEGRATION=yes go test {race_opt} -tags "{go_build_tags}" {exec_opts}'.format(**test_args)
+
+    prefixes = [
+        "./pkg/trace/test/testsuite/...",
+    ]
+
+    for prefix in prefixes:
+        ctx.run("{} {}".format(go_cmd, prefix))