internal/appsec: add new AppSec beta functionality and support for contrib/net/http

.circleci/config.yml

@@ -1,4 +1,4 @@
-version: 2
+version: 2.1
 
 plain-go114: &plain-go114
   working_directory: /home/circleci/dd-trace-go.v1
@@ -8,8 +8,13 @@ plain-go114: &plain-go114
         GOPATH: "/home/circleci/go"
 
 jobs:
-  go1.12-build:
+  go1_12-build:
     # Validate that the core builds with go1.12
+    parameters:
+      build_tags:
+        description: "go build tags used to compile"
+        default: ""
+        type: string
     docker:
       - image: circleci/golang:1.12
         environment:
@@ -25,7 +30,7 @@ jobs:
           # See https://github.com/DataDog/dd-trace-go/pull/1029
           sudo apt update && sudo apt install ca-certificates libgnutls30 -y
 
-          go build ./ddtrace/... ./profiler/...
+          go build -v -tags "<< parameters.build_tags >>" ./ddtrace/... ./profiler/... ./internal/appsec/...
 
   metadata:
     <<: *plain-go114
@@ -83,6 +88,11 @@ jobs:
 
 
   test-core:
+    parameters:
+      build_tags:
+        description: "go build tags to use to compile the tests"
+        default: ""
+        type: string
     resource_class: xlarge
     environment: # environment variables for the build itself
       TEST_RESULTS: /tmp/test-results # path to where test results will be saved
@@ -98,7 +108,7 @@ jobs:
           name: Testing
           command: |
             PACKAGE_NAMES=$(go list ./... | grep -v /contrib/ | circleci tests split --split-by=timings --timings-type=classname)
-            gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v -race -coverprofile=coverage.txt -covermode=atomic
+            env DD_APPSEC_ENABLED=$(test "<< parameters.build_tags >>" = appsec && echo -n true) gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v -race -coverprofile=coverage.txt -covermode=atomic -tags "<< parameters.build_tags >>"
 
       - save_cache:
           key: go-mod-v4-{{ checksum "go.sum" }}
@@ -118,6 +128,11 @@ jobs:
 
 
   test-contrib:
+    parameters:
+      build_tags:
+        description: "go build tags to use to compile the tests"
+        default: ""
+        type: string
     resource_class: xlarge
     environment: # environment variables for the build itself
       TEST_RESULTS: /tmp/test-results # path to where test results will be saved
@@ -241,7 +256,7 @@ jobs:
           name: Testing integrations
           command: |
             PACKAGE_NAMES=$(go list ./contrib/... | grep -v -e grpc.v12 -e google.golang.org/api | circleci tests split --split-by=timings --timings-type=classname)
-            gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v -race -coverprofile=coverage.txt -covermode=atomic
+            env DD_APPSEC_ENABLED=$(test "<< parameters.build_tags >>" = appsec && echo -n true) gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v -race -coverprofile=coverage.txt -covermode=atomic -tags "<< parameters.build_tags >>"
 
       - save_cache:
           key: go-mod-v4-{{ checksum "go.sum" }}
@@ -288,8 +303,17 @@ workflows:
   version: 2
   build-and-test:
     jobs:
-      - go1.12-build
+      - go1_12-build:
+          matrix:
+            parameters:
+              build_tags: [ "", "appsec" ]
       - metadata
       - lint
-      - test-core
-      - test-contrib
+      - test-core:
+          matrix:
+            parameters:
+              build_tags: [ "", "appsec" ]
+      - test-contrib:
+          matrix:
+            parameters:
+              build_tags: [ "", "appsec" ]

LICENSE-3rdparty.csv

@@ -1,2 +1,3 @@
 Component,Origin,License,Copyright
-import,io.opentracing,Apache-2.0,Copyright 2016-2017 The OpenTracing Authors
+import,io.opentracing,Apache-2.0,Copyright 2016-2017 The OpenTracing Authors
+appsec,https://github.com/DataDog/libddwaf,Apache-2.0 OR BSD-3-Clause,Copyright (c) 2021 Datadog <[email protected]>

contrib/internal/httputil/trace.go

@@ -15,6 +15,7 @@ import (
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
+	"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo/instrumentation/httpinstr"
 )
 
 // TraceConfig defines the configuration for request tracing.
@@ -53,8 +54,7 @@ func TraceAndServe(h http.Handler, cfg *TraceConfig) {
 	defer span.Finish(cfg.FinishOpts...)
 
 	cfg.ResponseWriter = wrapResponseWriter(cfg.ResponseWriter, span)
-
-	h.ServeHTTP(cfg.ResponseWriter, cfg.Request.WithContext(ctx))
+	httpinstr.WrapHandler(h, span).ServeHTTP(cfg.ResponseWriter, cfg.Request.WithContext(ctx))
 }
 
 // responseWriter is a small wrapper around an http response writer that will
@@ -69,8 +69,13 @@ func newResponseWriter(w http.ResponseWriter, span ddtrace.Span) *responseWriter
 	return &responseWriter{w, span, 0}
 }
 
+// Status returns the status code that was monitored.
+func (w *responseWriter) Status() int {
+	return w.status
+}
+
 // Write writes the data to the connection as part of an HTTP reply.
-// We explicitely call WriteHeader with the 200 status code
+// We explicitly call WriteHeader with the 200 status code
 // in order to get it reported into the span.
 func (w *responseWriter) Write(b []byte) (int, error) {
 	if w.status == 0 {

ddtrace/tracer/log_test.go

@@ -84,6 +84,11 @@ func TestStartupLog(t *testing.T) {
 }
 
 func TestLogSamplingRules(t *testing.T) {
+	// Disable AppSec to avoid their logs
+	if old := os.Getenv("DD_APPSEC_ENABLED"); old != "" {
+		os.Unsetenv("DD_APPSEC_ENABLED")
+		defer os.Setenv("DD_APPSEC_ENABLED", old)
+	}
 	assert := assert.New(t)
 	tp := new(testLogger)
 	os.Setenv("DD_TRACE_SAMPLING_RULES", `[{"service": "some.service", "sample_rate": 0.234}, {"service": "other.service"}, {"service": "last.service", "sample_rate": 0.56}, {"odd": "pairs"}, {"sample_rate": 9.10}]`)

ddtrace/tracer/option.go

@@ -109,6 +109,14 @@ func (c *config) HasFeature(f string) bool {
 	return ok
 }
 
+// client returns the HTTP client to use.
+func (c *config) client() *http.Client {
+	if c.httpClient == nil {
+		return defaultClient
+	}
+	return c.httpClient
+}
+
 // StartOption represents a function that can be provided as a parameter to Start.
 type StartOption func(*config)
 
@@ -215,7 +223,7 @@ func newConfig(opts ...StartOption) *config {
 		}
 	}
 	if c.transport == nil {
-		c.transport = newHTTPTransport(c.agentAddr, c.httpClient)
+		c.transport = newHTTPTransport(c.agentAddr, c.client())
 	}
 	if c.propagator == nil {
 		c.propagator = NewPropagator(nil)

ddtrace/tracer/option_test.go

@@ -7,6 +7,7 @@ package tracer
 
 import (
 	"math"
+	"net/http"
 	"os"
 	"path/filepath"
 	"testing"
@@ -38,6 +39,15 @@ func TestTracerOptionsDefaults(t *testing.T) {
 		assert.Equal("localhost:8126", c.agentAddr)
 		assert.Equal("localhost:8125", c.dogstatsdAddr)
 		assert.Nil(nil, c.httpClient)
+		assert.Equal(defaultClient, c.client())
+	})
+
+	t.Run("http-client", func(t *testing.T) {
+		c := newConfig()
+		assert.Equal(t, defaultClient, c.client())
+		client := &http.Client{}
+		WithHTTPClient(client)(c)
+		assert.Equal(t, client, c.client())
 	})
 
 	t.Run("analytics", func(t *testing.T) {