internal/appsec: add new AppSec beta functionality and support for co…

…ntrib/net/http (#1007)

Beta functionality for AppSec has been added to internal/appsec. It is also integrated into contrib/internal/httputil and can be enabled for all HTTP integrations using this (all do at this point).

The appsec package also has C bindings, which are disabled by default for the beta, but will likely be enabled later on. For this reason, to use AppSec, one must build their program using the "appsec" tag and enable it using DD_APPSEC_ENABLED=true.

.circleci/config.yml

@@ -1,4 +1,4 @@
-version: 2
+version: 2.1
 
 plain-go114: &plain-go114
   working_directory: /home/circleci/dd-trace-go.v1
@@ -8,8 +8,13 @@ plain-go114: &plain-go114
         GOPATH: "/home/circleci/go"
 
 jobs:
-  go1.12-build:
+  go1_12-build:
     # Validate that the core builds with go1.12
+    parameters:
+      build_tags:
+        description: "go build tags used to compile"
+        default: ""
+        type: string
     docker:
       - image: circleci/golang:1.12
         environment:
@@ -25,7 +30,7 @@ jobs:
           # See https://github.com/DataDog/dd-trace-go/pull/1029
           sudo apt update && sudo apt install ca-certificates libgnutls30 -y
 
-          go build ./ddtrace/... ./profiler/...
+          go build -v -tags "<< parameters.build_tags >>" ./ddtrace/... ./profiler/... ./internal/appsec/...
 
   metadata:
     <<: *plain-go114
@@ -83,6 +88,11 @@ jobs:
 
 
   test-core:
+    parameters:
+      build_tags:
+        description: "go build tags to use to compile the tests"
+        default: ""
+        type: string
     resource_class: xlarge
     environment: # environment variables for the build itself
       TEST_RESULTS: /tmp/test-results # path to where test results will be saved
@@ -98,7 +108,7 @@ jobs:
           name: Testing
           command: |
             PACKAGE_NAMES=$(go list ./... | grep -v /contrib/ | circleci tests split --split-by=timings --timings-type=classname)
-            gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v -race -coverprofile=coverage.txt -covermode=atomic
+            env DD_APPSEC_ENABLED=$(test "<< parameters.build_tags >>" = appsec && echo -n true) gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v -race -coverprofile=coverage.txt -covermode=atomic -tags "<< parameters.build_tags >>"
 
       - save_cache:
           key: go-mod-v4-{{ checksum "go.sum" }}
@@ -118,6 +128,11 @@ jobs:
 
 
   test-contrib:
+    parameters:
+      build_tags:
+        description: "go build tags to use to compile the tests"
+        default: ""
+        type: string
     resource_class: xlarge
     environment: # environment variables for the build itself
       TEST_RESULTS: /tmp/test-results # path to where test results will be saved
@@ -241,7 +256,7 @@ jobs:
           name: Testing integrations
           command: |
             PACKAGE_NAMES=$(go list ./contrib/... | grep -v -e grpc.v12 -e google.golang.org/api | circleci tests split --split-by=timings --timings-type=classname)
-            gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v -race -coverprofile=coverage.txt -covermode=atomic
+            env DD_APPSEC_ENABLED=$(test "<< parameters.build_tags >>" = appsec && echo -n true) gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v -race -coverprofile=coverage.txt -covermode=atomic -tags "<< parameters.build_tags >>"
 
       - save_cache:
           key: go-mod-v4-{{ checksum "go.sum" }}
@@ -288,8 +303,17 @@ workflows:
   version: 2
   build-and-test:
     jobs:
-      - go1.12-build
+      - go1_12-build:
+          matrix:
+            parameters:
+              build_tags: [ "", "appsec" ]
       - metadata
       - lint
-      - test-core
-      - test-contrib
+      - test-core:
+          matrix:
+            parameters:
+              build_tags: [ "", "appsec" ]
+      - test-contrib:
+          matrix:
+            parameters:
+              build_tags: [ "", "appsec" ]

LICENSE-3rdparty.csv

@@ -1,2 +1,3 @@
 Component,Origin,License,Copyright
-import,io.opentracing,Apache-2.0,Copyright 2016-2017 The OpenTracing Authors
+import,io.opentracing,Apache-2.0,Copyright 2016-2017 The OpenTracing Authors
+appsec,https://github.com/DataDog/libddwaf,Apache-2.0 OR BSD-3-Clause,Copyright (c) 2021 Datadog <[email protected]>

contrib/internal/httputil/trace.go

@@ -15,6 +15,7 @@ import (
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
+	"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo/instrumentation/httpinstr"
 )
 
 // TraceConfig defines the configuration for request tracing.
@@ -53,8 +54,7 @@ func TraceAndServe(h http.Handler, cfg *TraceConfig) {
 	defer span.Finish(cfg.FinishOpts...)
 
 	cfg.ResponseWriter = wrapResponseWriter(cfg.ResponseWriter, span)
-
-	h.ServeHTTP(cfg.ResponseWriter, cfg.Request.WithContext(ctx))
+	httpinstr.WrapHandler(h, span).ServeHTTP(cfg.ResponseWriter, cfg.Request.WithContext(ctx))
 }
 
 // responseWriter is a small wrapper around an http response writer that will
@@ -69,8 +69,13 @@ func newResponseWriter(w http.ResponseWriter, span ddtrace.Span) *responseWriter
 	return &responseWriter{w, span, 0}
 }
 
+// Status returns the status code that was monitored.
+func (w *responseWriter) Status() int {
+	return w.status
+}
+
 // Write writes the data to the connection as part of an HTTP reply.
-// We explicitely call WriteHeader with the 200 status code
+// We explicitly call WriteHeader with the 200 status code
 // in order to get it reported into the span.
 func (w *responseWriter) Write(b []byte) (int, error) {
 	if w.status == 0 {

ddtrace/tracer/log_test.go

@@ -84,6 +84,11 @@ func TestStartupLog(t *testing.T) {
 }
 
 func TestLogSamplingRules(t *testing.T) {
+	// Disable AppSec to avoid their logs
+	if old := os.Getenv("DD_APPSEC_ENABLED"); old != "" {
+		os.Unsetenv("DD_APPSEC_ENABLED")
+		defer os.Setenv("DD_APPSEC_ENABLED", old)
+	}
 	assert := assert.New(t)
 	tp := new(testLogger)
 	os.Setenv("DD_TRACE_SAMPLING_RULES", `[{"service": "some.service", "sample_rate": 0.234}, {"service": "other.service"}, {"service": "last.service", "sample_rate": 0.56}, {"odd": "pairs"}, {"sample_rate": 9.10}]`)

ddtrace/tracer/option.go

@@ -109,6 +109,14 @@ func (c *config) HasFeature(f string) bool {
 	return ok
 }
 
+// client returns the HTTP client to use.
+func (c *config) client() *http.Client {
+	if c.httpClient == nil {
+		return defaultClient
+	}
+	return c.httpClient
+}
+
 // StartOption represents a function that can be provided as a parameter to Start.
 type StartOption func(*config)
 
@@ -215,7 +223,7 @@ func newConfig(opts ...StartOption) *config {
 		}
 	}
 	if c.transport == nil {
-		c.transport = newHTTPTransport(c.agentAddr, c.httpClient)
+		c.transport = newHTTPTransport(c.agentAddr, c.client())
 	}
 	if c.propagator == nil {
 		c.propagator = NewPropagator(nil)

ddtrace/tracer/option_test.go

@@ -7,6 +7,7 @@ package tracer
 
 import (
 	"math"
+	"net/http"
 	"os"
 	"path/filepath"
 	"testing"
@@ -38,6 +39,15 @@ func TestTracerOptionsDefaults(t *testing.T) {
 		assert.Equal("localhost:8126", c.agentAddr)
 		assert.Equal("localhost:8125", c.dogstatsdAddr)
 		assert.Nil(nil, c.httpClient)
+		assert.Equal(defaultClient, c.client())
+	})
+
+	t.Run("http-client", func(t *testing.T) {
+		c := newConfig()
+		assert.Equal(t, defaultClient, c.client())
+		client := &http.Client{}
+		WithHTTPClient(client)(c)
+		assert.Equal(t, client, c.client())
 	})
 
 	t.Run("analytics", func(t *testing.T) {