From d37d79d7d73f0057480db28be051f55b20b0726a Mon Sep 17 00:00:00 2001 From: Gabriel Aszalos Date: Wed, 16 Jan 2019 10:17:52 -0500 Subject: [PATCH] omnibus: update datadog-trace-agent config and add invoke task * Merge datadog-agent datadog-trace-agent software definitions together * Add trace-agent makefile for agent5 omnibus build * Add DataDog/apm code owners to paths /cmd/trace-agent/ and /pkg/trace/ * pkg/trace/info: remove git_version * gitignore: added trace-agent-msg.h to ignore list --- .github/CODEOWNERS | 2 + .gitignore | 6 ++ Makefile.trace | 47 ++++++++++ omnibus/config/projects/agent.rb | 1 - omnibus/config/software/datadog-agent.rb | 9 ++ .../config/software/datadog-trace-agent.rb | 69 --------------- pkg/trace/info/git_version.go | 16 ---- pkg/trace/info/make.go | 3 +- .../writer/backoff/exponential_nix_test.go | 19 +++++ tasks/__init__.py | 3 +- tasks/agent.py | 3 +- tasks/test.py | 2 + tasks/trace.py | 85 +++++++++++++++++++ 13 files changed, 174 insertions(+), 91 deletions(-) create mode 100644 Makefile.trace delete mode 100644 omnibus/config/software/datadog-trace-agent.rb delete mode 100644 pkg/trace/info/git_version.go create mode 100644 pkg/trace/writer/backoff/exponential_nix_test.go create mode 100644 tasks/trace.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d5bafad4924c0..ce60ec4b3339a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,6 +10,7 @@ # and another the rest of the directory. /cmd/ @DataDog/agent-core +/cmd/trace-agent/ @DataDog/apm /omnibus/ @DataDog/agent-core /pkg/ @DataDog/agent-core /pkg/aggregator/ @DataDog/agent-core @@ -20,6 +21,7 @@ /pkg/serializer/ @DataDog/agent-core /pkg/status/ @DataDog/agent-core /pkg/version/ @DataDog/agent-core +/pkg/trace/ @DataDog/apm /cmd/cluster-agent/ @DataDog/container-integrations /Dockerfiles/ @DataDog/container-integrations diff --git a/.gitignore b/.gitignore index 2702a0feeb217..993c929efb3b4 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,9 @@ cmd/agent/android/app/build # local copy of device signing cert platform.pk8 + +# trace agent windows artifacts +cmd/trace-agent/windows_resources/trace-agent-msg.rc +cmd/trace-agent/windows_resources/*.bin +pkg/trace/info/git_version.go +trace-agent-msg.h diff --git a/Makefile.trace b/Makefile.trace new file mode 100644 index 0000000000000..7c5c1e752cf78 --- /dev/null +++ b/Makefile.trace @@ -0,0 +1,47 @@ +# This Makefile is used within the release process of the main Datadog Agent to pre-package datadog-trace-agent: +# https://github.com/DataDog/datadog-agent/blob/2b7055c/omnibus/config/software/datadog-trace-agent.rb + +# if the TRACE_AGENT_VERSION environment variable isn't set, default to 0.99.0 +TRACE_AGENT_VERSION := $(if $(TRACE_AGENT_VERSION),$(TRACE_AGENT_VERSION), 0.99.0) + +# break up the version +SPLAT = $(subst ., ,$(TRACE_AGENT_VERSION)) +VERSION_MAJOR = $(shell echo $(word 1, $(SPLAT)) | sed 's/[^0-9]*//g') +VERSION_MINOR = $(shell echo $(word 2, $(SPLAT)) | sed 's/[^0-9]*//g') +VERSION_PATCH = $(shell echo $(word 3, $(SPLAT)) | sed 's/[^0-9]*//g') + +# account for some defaults +VERSION_MAJOR := $(if $(VERSION_MAJOR),$(VERSION_MAJOR), 0) +VERSION_MINOR := $(if $(VERSION_MINOR),$(VERSION_MINOR), 0) +VERSION_PATCH := $(if $(VERSION_PATCH),$(VERSION_PATCH), 0) + +install: + # generate versioning information and installing the binary. + go generate ./pkg/trace/info + go install ./cmd/trace-agent + +binaries: + test -n "$(V)" # $$V must be set to the release version tag, e.g. "make binaries V=1.2.3" + + # compiling release binaries for tag $(V) + git checkout $(V) + mkdir -p ./bin + TRACE_AGENT_VERSION=$(V) go generate ./internal/info + go get -u github.com/karalabe/xgo + xgo -dest=bin -go=1.10 -out=trace-agent-$(V) -targets=windows-6.1/amd64,linux/amd64,darwin-10.11/amd64 ./cmd/trace-agent + mv ./bin/trace-agent-$(V)-windows-6.1-amd64.exe ./bin/trace-agent-$(V)-windows-amd64.exe + mv ./bin/trace-agent-$(V)-darwin-10.11-amd64 ./bin/trace-agent-$(V)-darwin-amd64 + git reset --hard head && git checkout - + +ci: + # task used by CI + go get -u golang.org/x/lint/golint + golint -set_exit_status=1 ./cmd/trace-agent ./internal/filters ./internal/api ./internal/test ./internal/info ./internal/quantile ./internal/obfuscate ./internal/sampler ./internal/metrics ./internal/watchdog ./internal/writer ./internal/flags ./internal/osutil + go install ./cmd/trace-agent + go test -v -race ./... + +windows: + # pre-packages resources needed for the windows release + windmc --target pe-x86-64 -r cmd/trace-agent/windows_resources cmd/trace-agent/windows_resources/trace-agent-msg.mc + windres --define MAJ_VER=$(VERSION_MAJOR) --define MIN_VER=$(VERSION_MINOR) --define PATCH_VER=$(VERSION_PATCH) -i cmd/trace-agent/windows_resources/trace-agent.rc --target=pe-x86-64 -O coff -o cmd/trace-agent/rsrc.syso + diff --git a/omnibus/config/projects/agent.rb b/omnibus/config/projects/agent.rb index aae28c4978e5c..5558bb93f2677 100644 --- a/omnibus/config/projects/agent.rb +++ b/omnibus/config/projects/agent.rb @@ -126,7 +126,6 @@ dependency 'jmxfetch' # External agents -dependency 'datadog-trace-agent' dependency 'datadog-process-agent' # Includes network-tracer if osx? diff --git a/omnibus/config/software/datadog-agent.rb b/omnibus/config/software/datadog-agent.rb index 0f1975f5eb805..c87dc239d0eb4 100644 --- a/omnibus/config/software/datadog-agent.rb +++ b/omnibus/config/software/datadog-agent.rb @@ -64,6 +64,15 @@ copy 'bin', install_dir + + command "invoke trace-agent.build", :env => env + + if windows? + copy 'bin/trace-agent/trace-agent.exe', "#{Omnibus::Config.source_dir()}/datadog-agent/src/github.com/DataDog/datadog-agent/bin/agent" + else + copy 'bin/trace-agent/trace-agent', "#{install_dir}/embedded/bin" + end + if linux? if debian? erb source: "upstart_debian.conf.erb", diff --git a/omnibus/config/software/datadog-trace-agent.rb b/omnibus/config/software/datadog-trace-agent.rb deleted file mode 100644 index a32428869c78a..0000000000000 --- a/omnibus/config/software/datadog-trace-agent.rb +++ /dev/null @@ -1,69 +0,0 @@ -# Unless explicitly stated otherwise all files in this repository are licensed -# under the Apache License Version 2.0. -# This product includes software developed at Datadog (https:#www.datadoghq.com/). -# Copyright 2016-2019 Datadog, Inc. - -require "./lib/ostools.rb" -require 'pathname' - -name "datadog-trace-agent" - -dependency "datadog-agent" - -trace_agent_version = ENV['TRACE_AGENT_VERSION'] -if trace_agent_version.nil? || trace_agent_version.empty? - trace_agent_version = 'master' -end -default_version trace_agent_version - -source git: 'https://github.com/DataDog/datadog-trace-agent.git' -relative_path 'src/github.com/DataDog/datadog-trace-agent' - -if windows? - trace_agent_binary = "trace-agent.exe" -else - trace_agent_binary = "trace-agent" -end - -build do - ship_license "https://raw.githubusercontent.com/DataDog/datadog-trace-agent/#{version}/LICENSE" - # set GOPATH on the omnibus source dir for this software - gopath = Pathname.new(project_dir) + '../../../..' - if windows? - env = { - # Trace agent uses GNU make to build. Some of the input to gnu make - # needs the path with `\` as separators, some needs `/`. Provide both, - # and let the makefile sort it out (ugh) - - # also on windows don't modify the path. Modifying the path here mixes - # `/` with `\` in the PATH variable, which confuses the make (and sub-processes) - # below. When properly configured the path on the windows box is sufficient. - 'GOPATH' => "#{windows_safe_path(gopath.to_path)}", - } - else - env = { - 'GOPATH' => gopath.to_path, - 'PATH' => "#{gopath.to_path}/bin:#{ENV['PATH']}", - } - end - - block do - # defer compilation step in a block to allow getting the project's build version, which is populated - # only once the software that the project takes its version from (i.e. `datadog-agent`) has finished building - env['TRACE_AGENT_VERSION'] = project.build_version.gsub(/[^0-9\.]/, '') # used by gorake.rb in the trace-agent, only keep digits and dots - - # build trace-agent - if windows? - command "make windows", :env => env - end - command "make install", :env => env - - # copy binary - if windows? - #copy "#{gopath.to_path}/bin/#{trace_agent_binary}", "#{install_dir}/bin/agent" - copy "#{gopath.to_path}/bin/#{trace_agent_binary}", "#{Omnibus::Config.source_dir()}/datadog-agent/src/github.com/DataDog/datadog-agent/bin/agent" - else - copy "#{gopath.to_path}/bin/#{trace_agent_binary}", "#{install_dir}/embedded/bin" - end - end -end diff --git a/pkg/trace/info/git_version.go b/pkg/trace/info/git_version.go deleted file mode 100644 index aee8feeee33bd..0000000000000 --- a/pkg/trace/info/git_version.go +++ /dev/null @@ -1,16 +0,0 @@ -// Code generated by 'go run make.go'. DO NOT EDIT. - -package info - -import ( - "runtime" - "strings" -) - -func init() { - Version = "0.99.0" - GitCommit = "90dcab81" - GitBranch = "HEAD" - BuildDate = "2019-01-08 10:02:22.993077 +0100 CET m=+0.007560824" - GoVersion = strings.TrimPrefix(runtime.Version(), "go") -} diff --git a/pkg/trace/info/make.go b/pkg/trace/info/make.go index c8addcdb3d7df..51168a218d68d 100644 --- a/pkg/trace/info/make.go +++ b/pkg/trace/info/make.go @@ -43,8 +43,7 @@ func main() { } } -const template = ` -// Code generated by 'go run make.go'. DO NOT EDIT. +const template = `// Code generated by 'go run make.go'. DO NOT EDIT. package info diff --git a/pkg/trace/writer/backoff/exponential_nix_test.go b/pkg/trace/writer/backoff/exponential_nix_test.go new file mode 100644 index 0000000000000..1d1aa854b79db --- /dev/null +++ b/pkg/trace/writer/backoff/exponential_nix_test.go @@ -0,0 +1,19 @@ +// +build !windows + +package backoff + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDefaultRandomSeed(t *testing.T) { + assert := assert.New(t) + + delayProvider1 := DefaultExponentialDelayProvider() + delayProvider2 := DefaultExponentialDelayProvider() + + // Ensure different timers are not synchronized in their backoffing (use different seeds) + assert.NotEqual(delayProvider1(0, nil), delayProvider2(0, nil)) +} diff --git a/tasks/__init__.py b/tasks/__init__.py index c368684145640..5aebf8d1a353c 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -4,7 +4,7 @@ import os from invoke import Collection -from . import agent, android, benchmarks, customaction, docker, dogstatsd, pylauncher, cluster_agent, systray, release +from . import agent, trace, android, benchmarks, customaction, docker, dogstatsd, pylauncher, cluster_agent, systray, release from .go import fmt, lint, vet, cyclo, ineffassign, misspell, deps, lint_licenses, reset from .test import test, integration_tests, version, lint_teamassignment, lint_releasenote, lint_milestone, lint_filenames, e2e_tests @@ -39,6 +39,7 @@ ns.add_collection(cluster_agent) ns.add_collection(customaction) ns.add_collection(benchmarks, name="bench") +ns.add_collection(trace, name="trace-agent") ns.add_collection(docker) ns.add_collection(dogstatsd) ns.add_collection(pylauncher) diff --git a/tasks/agent.py b/tasks/agent.py index b60e982f3b626..aafe88e2ab542 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -253,14 +253,13 @@ def integration_tests(ctx, install_deps=False, race=False, remote_docker=False): if remote_docker: test_args["exec_opts"] = "-exec \"inv docker.dockerize-test\"" - go_cmd = 'INTEGRATION=yes go test {race_opt} -tags "{go_build_tags}" {exec_opts}'.format(**test_args) + go_cmd = 'go test {race_opt} -tags "{go_build_tags}" {exec_opts}'.format(**test_args) prefixes = [ "./test/integration/config_providers/...", "./test/integration/corechecks/...", "./test/integration/listeners/...", "./test/integration/util/kubelet/...", - "./pkg/trace/test/testsuite/...", ] for prefix in prefixes: diff --git a/tasks/test.py b/tasks/test.py index 95f0e6129daed..9bb91b95989a2 100644 --- a/tasks/test.py +++ b/tasks/test.py @@ -18,6 +18,7 @@ from .build_tags import get_default_build_tags, get_build_tags from .agent import integration_tests as agent_integration_tests from .dogstatsd import integration_tests as dsd_integration_tests +from .trace import integration_tests as trace_integration_tests from .cluster_agent import integration_tests as dca_integration_tests PROFILE_COV = "profile.cov" @@ -279,6 +280,7 @@ def integration_tests(ctx, install_deps=False, race=False, remote_docker=False): agent_integration_tests(ctx, install_deps, race, remote_docker) dsd_integration_tests(ctx, install_deps, race, remote_docker) dca_integration_tests(ctx, install_deps, race, remote_docker) + trace_integration_tests(ctx, install_deps, race, remote_docker) @task diff --git a/tasks/trace.py b/tasks/trace.py new file mode 100644 index 0000000000000..8c719525717b5 --- /dev/null +++ b/tasks/trace.py @@ -0,0 +1,85 @@ +import os +import sys +import shutil + +import invoke +from invoke import task + +from .utils import bin_name, get_build_flags, get_version_numeric_only, load_release_versions +from .utils import REPO_PATH +from .build_tags import get_build_tags, get_default_build_tags, LINUX_ONLY_TAGS, REDHAT_AND_DEBIAN_ONLY_TAGS, REDHAT_AND_DEBIAN_DIST +from .go import deps + +BIN_PATH = os.path.join(".", "bin", "trace-agent") +DEFAULT_BUILD_TAGS = ["netcgo"] + +@task +def build(ctx, rebuild=False, race=False, precompile_only=False, use_embedded_libs=False, + build_include=None, build_exclude=None, puppy=False, use_venv=False): + """ + Build the trace agent. + """ + + # generate windows resources + if sys.platform == 'win32': + ver = get_version_numeric_only(ctx) + maj_ver, min_ver, patch_ver = ver.split(".") + + ctx.run("windmc --target pe-x86-64 -r cmd/trace-agent/windows_resources cmd/trace-agent/windows_resources/trace-agent-msg.mc") + ctx.run("windres --define MAJ_VER={maj_ver} --define MIN_VER={min_ver} --define PATCH_VER={patch_ver} -i cmd/trace-agent/windows_resources/trace-agent.rc --target=pe-x86-64 -O coff -o cmd/trace-agent/rsrc.syso".format( + maj_ver=maj_ver, + min_ver=min_ver, + patch_ver=patch_ver + )) + + ldflags, gcflags, env = get_build_flags(ctx, use_embedded_libs=use_embedded_libs, use_venv=use_venv) + build_include = DEFAULT_BUILD_TAGS if build_include is None else build_include.split(",") + build_exclude = [] if build_exclude is None else build_exclude.split(",") + + if puppy: + # Puppy mode overrides whatever passed through `--build-exclude` and `--build-include` + build_tags = get_default_build_tags(puppy=True) + else: + build_tags = get_build_tags(build_include, build_exclude) + + cmd = "go build {race_opt} {build_type} -tags \"{go_build_tags}\" " + cmd += "-o {agent_bin} -gcflags=\"{gcflags}\" -ldflags=\"{ldflags}\" {REPO_PATH}/cmd/trace-agent" + + args = { + "race_opt": "-race" if race else "", + "build_type": "-a" if rebuild else ("-i" if precompile_only else ""), + "go_build_tags": " ".join(build_tags), + "agent_bin": os.path.join(BIN_PATH, bin_name("trace-agent", android=False)), + "gcflags": gcflags, + "ldflags": ldflags, + "REPO_PATH": REPO_PATH, + } + + ctx.run("go generate {REPO_PATH}/pkg/trace/info".format(**args), env=env) + ctx.run(cmd.format(**args), env=env) + +@task +def integration_tests(ctx, install_deps=False, race=False, remote_docker=False): + """ + Run integration tests for trace agent + """ + if install_deps: + deps(ctx) + + test_args = { + "go_build_tags": " ".join(get_default_build_tags()), + "race_opt": "-race" if race else "", + "exec_opts": "", + } + + if remote_docker: + test_args["exec_opts"] = "-exec \"inv docker.dockerize-test\"" + + go_cmd = 'INTEGRATION=yes go test {race_opt} -tags "{go_build_tags}" {exec_opts}'.format(**test_args) + + prefixes = [ + "./pkg/trace/test/testsuite/...", + ] + + for prefix in prefixes: + ctx.run("{} {}".format(go_cmd, prefix))