Skip to content

Commit

Permalink
serverless/appsec: use appsec-internal-go configs
Browse files Browse the repository at this point in the history
  • Loading branch information
@Hellzy
Hellzy committed Nov 23, 2023
1 parent ef05e34 commit c57e966
Show file tree
Hide file tree
Showing 4 changed files with 87 additions and 497 deletions.
10 changes: 8 additions & 2 deletions pkg/serverless/appsec/appsec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ func (a *AppSec) Monitor(addresses map[string]any) (res waf.Result) {
timeout := a.cfg.WafTimeout

// Ask the WAF for schema reporting if API security is enabled
if config.APISecurityEnabled() && config.APISecuritySampleRate() >= rand.Float64() {
if a.canExtractSchemas() {
addresses["waf.context.processor"] = map[string]any{"extract-schema": true}
}

Expand All @@ -140,7 +140,7 @@ func (a *AppSec) Monitor(addresses map[string]any) (res waf.Result) {
}
if !a.eventsRateLimiter.Allow() {
log.Debugf("appsec: security events discarded: the rate limit of %d events/s is reached", a.cfg.TraceRateLimit)
return waf.Result{}
res = waf.Result{}
}
return res
}
Expand All @@ -157,3 +157,9 @@ func wafHealth() error {
}
return nil
}

// canExtractSchemas checks that API Security is enabled
// and that sampling rate allows schema extraction for a specific monitoring instance
func (a *AppSec) canExtractSchemas() bool {
return a.cfg.APISec.Enabled && a.cfg.APISec.SampleRate >= rand.Float64()
}
19 changes: 11 additions & 8 deletions pkg/serverless/appsec/appsec_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"strconv"
"testing"

"github.com/DataDog/datadog-agent/pkg/serverless/appsec/config"
waf "github.com/DataDog/go-libddwaf/v2"

"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -49,13 +48,13 @@ func TestMonitor(t *testing.T) {
t.Skip("host not supported by appsec", err)
}

t.Setenv("DD_SERVERLESS_APPSEC_ENABLED", "true")
t.Setenv("DD_APPSEC_WAF_TIMEOUT", "2s")
asm, err := newAppSec()
require.NoError(t, err)
require.Nil(t, err)

t.Run("events-detection", func(t *testing.T) {
t.Setenv("DD_SERVERLESS_APPSEC_ENABLED", "true")
t.Setenv("DD_APPSEC_WAF_TIMEOUT", "2s")
asm, err := newAppSec()
require.NoError(t, err)
defer asm.Close()

uri := "/path/to/resource/../../../../../database.yml.sqlite3"
addresses := map[string]interface{}{
"server.request.uri.raw": uri,
Expand All @@ -77,7 +76,11 @@ func TestMonitor(t *testing.T) {
t.Run("api-security", func(t *testing.T) {
t.Setenv("DD_API_SECURITY_REQUEST_SAMPLE_RATE", "1.0")
t.Setenv("DD_EXPERIMENTAL_API_SECURITY_ENABLED", "true")
config.Refresh()
t.Setenv("DD_SERVERLESS_APPSEC_ENABLED", "true")
t.Setenv("DD_APPSEC_WAF_TIMEOUT", "2s")
asm, err := newAppSec()
require.NoError(t, err)
defer asm.Close()
for _, tc := range []struct {
name string
pathParams map[string]any
Expand Down
185 changes: 17 additions & 168 deletions pkg/serverless/appsec/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,69 +9,35 @@ package config
import (
"fmt"
"os"
"regexp"
"strconv"
"time"
"unicode"
"unicode/utf8"

"github.com/DataDog/appsec-internal-go/appsec"
"github.com/DataDog/datadog-agent/pkg/util/log"
)

const (
enabledEnvVar = "DD_SERVERLESS_APPSEC_ENABLED"
rulesEnvVar = "DD_APPSEC_RULES"
wafTimeoutEnvVar = "DD_APPSEC_WAF_TIMEOUT"
traceRateLimitEnvVar = "DD_APPSEC_TRACE_RATE_LIMIT"
obfuscatorKeyEnvVar = "DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP"
obfuscatorValueEnvVar = "DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP"
tracingEnabledEnvVar = "DD_APM_TRACING_ENABLED"
apiSecurityEnabledEnvVar = "DD_EXPERIMENTAL_API_SECURITY_ENABLED"
apiSecuritySampleRateEnvVar = "DD_API_SECURITY_REQUEST_SAMPLE_RATE"
enabledEnvVar = "DD_SERVERLESS_APPSEC_ENABLED"
tracingEnabledEnvVar = "DD_APM_TRACING_ENABLED"
)

const (
defaultAPISecSampleRate = 10. / 100
defaultWAFTimeout = 4 * time.Millisecond
defaultTraceRate = 100 // up to 100 appsec traces/s
defaultObfuscatorKeyRegex = `(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization`
defaultObfuscatorValueRegex = `(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}`
)
var standalone bool

// StartOption is used to customize the AppSec configuration when invoked with appsec.Start()
type StartOption func(c *Config)

// Config is the AppSec configuration.
type Config struct {
// rules loaded via the env var DD_APPSEC_RULES. When not set, the builtin rules will be used.
// Rules is the security rules loaded via the env var DD_APPSEC_RULES.
// When not set, the builtin rules will be used.
Rules []byte
// Maximum WAF execution time
// WafTimeout is the maximum WAF execution time
WafTimeout time.Duration
// AppSec trace rate limit (traces per second).
// TraceRateLimit is the rate limit of AppSec traces (per second).
TraceRateLimit uint
// Obfuscator configuration parameters
Obfuscator ObfuscatorConfig
}

// ObfuscatorConfig wraps the key and value regexp to be passed to the WAF to perform obfuscation.
type ObfuscatorConfig struct {
KeyRegex string
ValueRegex string
}

// Config variables assigned at init by reading the environment
var (
apiSecEnabled bool
apiSecSampleRate float64
standalone bool
)

// Refresh updates the configuration by reading from the environment
func Refresh() {
standalone = isStandalone()
apiSecEnabled = apiSecurityEnabled()
apiSecSampleRate = apiSecuritySampleRate()
// Obfuscator is the configuration for sensitive data obfuscation (in-WAF)
Obfuscator appsec.ObfuscatorConfig
// APISec is the configuration for API Security schema collection
APISec appsec.APISecConfig
}

// IsEnabled returns true when appsec is enabled when the environment variable
Expand All @@ -93,120 +59,21 @@ func IsStandalone() bool {
return standalone
}

// APISecurityEnabled returns whether API security is enabled through the environment
func APISecurityEnabled() bool {
return apiSecEnabled
}

// APISecuritySampleRate returns the sample rate for API Security schema collection, read from the env
func APISecuritySampleRate() float64 {
return apiSecSampleRate
}

// NewConfig returns a new appsec configuration read from the environment
func NewConfig() (*Config, error) {
rules, err := readRulesConfig()
rules, err := appsec.RulesFromEnv()
if err != nil {
return nil, err
}
return &Config{
Rules: rules,
WafTimeout: readWAFTimeoutConfig(),
TraceRateLimit: readRateLimitConfig(),
Obfuscator: readObfuscatorConfig(),
WafTimeout: appsec.WAFTimeoutFromEnv(),
TraceRateLimit: appsec.RateLimitFromEnv(),
Obfuscator: appsec.NewObfuscatorConfig(),
APISec: appsec.NewAPISecConfig(),
}, nil
}

func readWAFTimeoutConfig() (timeout time.Duration) {
timeout = defaultWAFTimeout
value := os.Getenv(wafTimeoutEnvVar)
if value == "" {
return
}

// Check if the value ends with a letter, which means the user has
// specified their own time duration unit(s) such as 1s200ms.
// Otherwise, default to microseconds.
if lastRune, _ := utf8.DecodeLastRuneInString(value); !unicode.IsLetter(lastRune) {
value += "us" // Add the default microsecond time-duration suffix
}

parsed, err := time.ParseDuration(value)
if err != nil {
logEnvVarParsingError(wafTimeoutEnvVar, value, err, timeout)
return
}
if parsed <= 0 {
logUnexpectedEnvVarValue(wafTimeoutEnvVar, parsed, "expecting a strictly positive duration", timeout)
return
}
return parsed
}

func readRateLimitConfig() (rate uint) {
rate = defaultTraceRate
value := os.Getenv(traceRateLimitEnvVar)
if value == "" {
return rate
}
parsed, err := strconv.ParseUint(value, 10, 0)
if err != nil {
logEnvVarParsingError(traceRateLimitEnvVar, value, err, rate)
return
}
if rate == 0 {
logUnexpectedEnvVarValue(traceRateLimitEnvVar, parsed, "expecting a value strictly greater than 0", rate)
return
}
return uint(parsed)
}

func readObfuscatorConfig() ObfuscatorConfig {
keyRE := readObfuscatorConfigRegexp(obfuscatorKeyEnvVar, defaultObfuscatorKeyRegex)
valueRE := readObfuscatorConfigRegexp(obfuscatorValueEnvVar, defaultObfuscatorValueRegex)
return ObfuscatorConfig{KeyRegex: keyRE, ValueRegex: valueRE}
}

func readObfuscatorConfigRegexp(name, defaultValue string) string {
val, present := os.LookupEnv(name)
if !present {
log.Debugf("appsec: %s not defined, starting with the default obfuscator regular expression", name)
return defaultValue
}
if _, err := regexp.Compile(val); err != nil {
log.Errorf("appsec: could not compile the configured obfuscator regular expression `%s=%s`. Using the default value instead", name, val)
return defaultValue
}
log.Debugf("appsec: starting with the configured obfuscator regular expression %s", name)
return val
}

func readRulesConfig() (rules []byte, err error) {
rules = []byte(appsec.StaticRecommendedRules)
filepath := os.Getenv(rulesEnvVar)
if filepath == "" {
log.Info("appsec: starting with the default recommended security rules")
return rules, nil
}
buf, err := os.ReadFile(filepath)
if err != nil {
if os.IsNotExist(err) {
log.Errorf("appsec: could not find the rules file in path %s: %v.", filepath, err)
}
return nil, err
}
log.Info("appsec: starting with the security rules from file %s", filepath)
return buf, nil
}

func logEnvVarParsingError(name, value string, err error, defaultValue interface{}) {
log.Errorf("appsec: could not parse the env var %s=%s as a duration: %v. Using default value %v.", name, value, err, defaultValue)
}

func logUnexpectedEnvVarValue(name string, value interface{}, reason string, defaultValue interface{}) {
log.Errorf("appsec: unexpected configuration value of %s=%v: %s. Using default value %v.", name, value, reason, defaultValue)
}

// isStandalone is reads the env and reports whether appsec runs in standalone mode
// Used at init and for testing
func isStandalone() bool {
Expand All @@ -215,24 +82,6 @@ func isStandalone() bool {
return set && !enabled
}

func apiSecurityEnabled() bool {
enabled, _ := strconv.ParseBool(os.Getenv(apiSecurityEnabledEnvVar))
return enabled
}

func apiSecuritySampleRate() float64 {
rate, err := strconv.ParseFloat(os.Getenv(apiSecuritySampleRateEnvVar), 64)
if err != nil {
log.Debugf("appsec: could not parse %s. Defaulting to %f", apiSecuritySampleRateEnvVar, defaultAPISecSampleRate)
return defaultAPISecSampleRate
}
if rate < 0. || rate > 1. {
log.Debugf("appsec: %s value must be between 0 and 1. Defaulting to %f", apiSecuritySampleRateEnvVar, defaultAPISecSampleRate)
return defaultAPISecSampleRate
}
return rate
}

func init() {
Refresh()
standalone = isStandalone()
}
Loading

0 comments on commit c57e966

Please sign in to comment.