internal/appsec: add new AppSec beta functionality and support for contrib/net/http

.circleci/config.yml

@@ -1,4 +1,4 @@
-version: 2
+version: 2.1
 
 plain-go114: &plain-go114
   working_directory: /home/circleci/dd-trace-go.v1
@@ -8,7 +8,7 @@ plain-go114: &plain-go114
         GOPATH: "/home/circleci/go"
 
 jobs:
-  go1.12-build:
+  go1_12-build:
     # Validate that the core builds with go1.12
     docker:
       - image: circleci/golang:1.12
@@ -21,7 +21,7 @@ jobs:
     - run:
         name: build
         command: |
-          go build ./ddtrace/... ./profiler/...
+          go build ./ddtrace/... ./profiler/... ./internal/appsec
 
   metadata:
     <<: *plain-go114
@@ -79,9 +79,15 @@ jobs:
 
 
   test-core:
+    parameters:
+      appsec:
+        description: "compile and enable appsec"
+        default: false
+        type: boolean
     resource_class: xlarge
     environment: # environment variables for the build itself
       TEST_RESULTS: /tmp/test-results # path to where test results will be saved
+      DD_APPSEC_ENABLED: << parameters.appsec >>
     <<: *plain-go114
 
     steps:
@@ -94,7 +100,7 @@ jobs:
           name: Testing
           command: |
             PACKAGE_NAMES=$(go list ./... | grep -v /contrib/ | circleci tests split --split-by=timings --timings-type=classname)
-            gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v -race -coverprofile=coverage.txt -covermode=atomic
+            gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v <<# parameters.appsec >>-tags appsec<</ parameters.appsec >> -race -coverprofile=coverage.txt -covermode=atomic
 
       - save_cache:
           key: go-mod-v4-{{ checksum "go.sum" }}
@@ -282,8 +288,11 @@ workflows:
   version: 2
   build-and-test:
     jobs:
-      - go1.12-build
+      - go1_12-build
       - metadata
       - lint
-      - test-core
+      - test-core:
+          matrix:
+            parameters:
+              appsec: [ true, false ]
       - test-contrib

LICENSE-3rdparty.csv

@@ -1,2 +1,3 @@
 Component,Origin,License,Copyright
-import,io.opentracing,Apache-2.0,Copyright 2016-2017 The OpenTracing Authors
+import,io.opentracing,Apache-2.0,Copyright 2016-2017 The OpenTracing Authors
+appsec,https://github.com/DataDog/libddwaf,Apache-2.0 OR BSD-3-Clause,Copyright (c) 2021 Datadog <[email protected]>

contrib/internal/httputil/trace.go

@@ -15,6 +15,7 @@ import (
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
+	"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo/instrumentation/httpinstr"
 )
 
 // TraceConfig defines the configuration for request tracing.
@@ -53,8 +54,7 @@ func TraceAndServe(h http.Handler, cfg *TraceConfig) {
 	defer span.Finish(cfg.FinishOpts...)
 
 	cfg.ResponseWriter = wrapResponseWriter(cfg.ResponseWriter, span)
-
-	h.ServeHTTP(cfg.ResponseWriter, cfg.Request.WithContext(ctx))
+	httpinstr.WrapHandler(h, span).ServeHTTP(cfg.ResponseWriter, cfg.Request.WithContext(ctx))
 }
 
 // responseWriter is a small wrapper around an http response writer that will
@@ -69,8 +69,13 @@ func newResponseWriter(w http.ResponseWriter, span ddtrace.Span) *responseWriter
 	return &responseWriter{w, span, 0}
 }
 
+// Status returns the status code that was monitored.
+func (w *responseWriter) Status() int {
+	return w.status
+}
+
 // Write writes the data to the connection as part of an HTTP reply.
-// We explicitely call WriteHeader with the 200 status code
+// We explicitly call WriteHeader with the 200 status code
 // in order to get it reported into the span.
 func (w *responseWriter) Write(b []byte) (int, error) {
 	if w.status == 0 {

ddtrace/tracer/option.go

@@ -109,6 +109,14 @@ func (c *config) HasFeature(f string) bool {
 	return ok
 }
 
+// client returns the HTTP client to use.
+func (c *config) client() *http.Client {
+	if c.httpClient == nil {
+		return defaultClient
+	}
+	return c.httpClient
+}
+
 // StartOption represents a function that can be provided as a parameter to Start.
 type StartOption func(*config)
 
@@ -215,7 +223,7 @@ func newConfig(opts ...StartOption) *config {
 		}
 	}
 	if c.transport == nil {
-		c.transport = newHTTPTransport(c.agentAddr, c.httpClient)
+		c.transport = newHTTPTransport(c.agentAddr, c.client())
 	}
 	if c.propagator == nil {
 		c.propagator = NewPropagator(nil)

ddtrace/tracer/option_test.go

@@ -7,6 +7,7 @@ package tracer
 
 import (
 	"math"
+	"net/http"
 	"os"
 	"path/filepath"
 	"testing"
@@ -38,6 +39,15 @@ func TestTracerOptionsDefaults(t *testing.T) {
 		assert.Equal("localhost:8126", c.agentAddr)
 		assert.Equal("localhost:8125", c.dogstatsdAddr)
 		assert.Nil(nil, c.httpClient)
+		assert.Equal(defaultClient, c.client())
+	})
+
+	t.Run("http-client", func(t *testing.T) {
+		c := newConfig()
+		assert.Equal(t, defaultClient, c.client())
+		client := &http.Client{}
+		WithHTTPClient(client)(c)
+		assert.Equal(t, client, c.client())
 	})
 
 	t.Run("analytics", func(t *testing.T) {

ddtrace/tracer/tracer.go

@@ -17,7 +17,9 @@ import (
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/internal"
+	"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec"
 	"gopkg.in/DataDog/dd-trace-go.v1/internal/log"
+	"gopkg.in/DataDog/dd-trace-go.v1/internal/version"
 )
 
 var _ ddtrace.Tracer = (*tracer)(nil)
@@ -213,6 +215,18 @@ func newTracer(opts ...StartOption) *tracer {
 		t.reportHealthMetrics(statsInterval)
 	}()
 	t.stats.Start()
+	appsec.Start(&appsec.Config{
+		Client:   c.client(),
+		Version:  version.Tag,
+		AgentURL: fmt.Sprintf("http://%s/", resolveAddr(c.agentAddr)),
+		Hostname: c.hostname,
+		Service: appsec.ServiceConfig{
+			Name:        c.serviceName,
+			Version:     c.version,
+			Environment: c.env,
+		},
+		Tags: c.globalTags,
+	})
 	return t
 }
 
@@ -464,6 +478,7 @@ func (t *tracer) Stop() {
 	t.wg.Wait()
 	t.traceWriter.stop()
 	t.config.statsd.Close()
+	appsec.Stop()
 }
 
 // Inject uses the configured or default TextMap Propagator.

ddtrace/tracer/tracer_test.go

@@ -74,6 +74,12 @@ func TestTracerCleanStop(t *testing.T) {
 	if testing.Short() {
 		return
 	}
+	// Avoid CI timeouts due to AppSec slowing down this test due to its init
+	// time.
+	if old := os.Getenv("DD_APPSEC_ENABLED"); old != "" {
+		os.Unsetenv("DD_APPSEC_ENABLED")
+		defer os.Setenv("DD_APPSEC_ENABLED", old)
+	}
 	os.Setenv("DD_TRACE_STARTUP_LOGS", "0")
 	defer os.Unsetenv("DD_TRACE_STARTUP_LOGS")
 

ddtrace/tracer/transport.go

@@ -87,9 +87,6 @@ type httpTransport struct {
 // otherwise needing to customize the transport layer, for instance when using
 // a unix domain socket.
 func newHTTPTransport(addr string, client *http.Client) *httpTransport {
-	if client == nil {
-		client = defaultClient
-	}
 	// initialize the default EncoderPool with Encoder headers
 	defaultHeaders := map[string]string{
 		"Datadog-Meta-Lang":             "go",