diff --git a/adapters/adform/adform_test.go b/adapters/adform/adform_test.go index 7023ef0eb61..f677f347c0c 100644 --- a/adapters/adform/adform_test.go +++ b/adapters/adform/adform_test.go @@ -185,7 +185,7 @@ func preparePrebidRequest(serverUrl string, t *testing.T) *pbs.PBSRequest { pbsCookie := usersync.ParsePBSCookieFromRequest(prebidHttpRequest, &config.HostCookie{}) pbsCookie.TrySync("adform", adformTestData.buyerUID) fakeWriter := httptest.NewRecorder() - pbsCookie.SetCookieOnResponse(fakeWriter, "", time.Minute) + pbsCookie.SetCookieOnResponse(fakeWriter, &config.HostCookie{Domain: ""}, time.Minute) prebidHttpRequest.Header.Add("Cookie", fakeWriter.Header().Get("Set-Cookie")) cacheClient, _ := dummycache.New() diff --git a/adapters/appnexus/appnexus_test.go b/adapters/appnexus/appnexus_test.go index cf849adaf33..2c54a37fb29 100644 --- a/adapters/appnexus/appnexus_test.go +++ b/adapters/appnexus/appnexus_test.go @@ -367,7 +367,7 @@ func TestAppNexusBasicResponse(t *testing.T) { pc := usersync.ParsePBSCookieFromRequest(req, &config.HostCookie{}) pc.TrySync("adnxs", andata.buyerUID) fakewriter := httptest.NewRecorder() - pc.SetCookieOnResponse(fakewriter, "", 90*24*time.Hour) + pc.SetCookieOnResponse(fakewriter, &config.HostCookie{Domain: ""}, 90*24*time.Hour) req.Header.Add("Cookie", fakewriter.Header().Get("Set-Cookie")) cacheClient, _ := dummycache.New() diff --git a/adapters/audienceNetwork/facebook_test.go b/adapters/audienceNetwork/facebook_test.go index 78b7c868e69..304df4cdbea 100644 --- a/adapters/audienceNetwork/facebook_test.go +++ b/adapters/audienceNetwork/facebook_test.go @@ -209,7 +209,7 @@ func GenerateBidRequestForTestData(fbdata bidInfo, url string) (*pbs.PBSRequest, pc := usersync.ParsePBSCookieFromRequest(req, &config.HostCookie{}) pc.TrySync("audienceNetwork", fbdata.buyerUID) fakewriter := httptest.NewRecorder() - pc.SetCookieOnResponse(fakewriter, "", 90*24*time.Hour) + pc.SetCookieOnResponse(fakewriter, &config.HostCookie{Domain: ""}, 90*24*time.Hour) req.Header.Add("Cookie", fakewriter.Header().Get("Set-Cookie")) cacheClient, _ := dummycache.New() diff --git a/adapters/lifestreet/lifestreet_test.go b/adapters/lifestreet/lifestreet_test.go index 60b6aed3656..14dda25d2a9 100644 --- a/adapters/lifestreet/lifestreet_test.go +++ b/adapters/lifestreet/lifestreet_test.go @@ -227,7 +227,7 @@ func TestLifestreetBasicResponse(t *testing.T) { pc := usersync.ParsePBSCookieFromRequest(req, &config.HostCookie{}) fakewriter := httptest.NewRecorder() - pc.SetCookieOnResponse(fakewriter, "", 90*24*time.Hour) + pc.SetCookieOnResponse(fakewriter, &config.HostCookie{Domain: ""}, 90*24*time.Hour) req.Header.Add("Cookie", fakewriter.Header().Get("Set-Cookie")) cacheClient, _ := dummycache.New() diff --git a/adapters/pubmatic/pubmatic_test.go b/adapters/pubmatic/pubmatic_test.go index e278e439e2b..8bf22eb0639 100644 --- a/adapters/pubmatic/pubmatic_test.go +++ b/adapters/pubmatic/pubmatic_test.go @@ -656,7 +656,7 @@ func TestPubmaticSampleRequest(t *testing.T) { pc := usersync.ParsePBSCookieFromRequest(httpReq, &config.HostCookie{}) pc.TrySync("pubmatic", "12345") fakewriter := httptest.NewRecorder() - pc.SetCookieOnResponse(fakewriter, "", 90*24*time.Hour) + pc.SetCookieOnResponse(fakewriter, &config.HostCookie{Domain: ""}, 90*24*time.Hour) httpReq.Header.Add("Cookie", fakewriter.Header().Get("Set-Cookie")) cacheClient, _ := dummycache.New() diff --git a/adapters/pulsepoint/pulsepoint_test.go b/adapters/pulsepoint/pulsepoint_test.go index 71c0406a7ae..f2b86dc630d 100644 --- a/adapters/pulsepoint/pulsepoint_test.go +++ b/adapters/pulsepoint/pulsepoint_test.go @@ -226,7 +226,7 @@ func SampleRequest(numberOfImpressions int, t *testing.T) *pbs.PBSRequest { pc := usersync.ParsePBSCookieFromRequest(httpReq, &config.HostCookie{}) pc.TrySync("pulsepoint", "pulsepointUser123") fakewriter := httptest.NewRecorder() - pc.SetCookieOnResponse(fakewriter, "", 90*24*time.Hour) + pc.SetCookieOnResponse(fakewriter, &config.HostCookie{Domain: ""}, 90*24*time.Hour) httpReq.Header.Add("Cookie", fakewriter.Header().Get("Set-Cookie")) // parse the http request cacheClient, _ := dummycache.New() diff --git a/adapters/rubicon/rubicon_test.go b/adapters/rubicon/rubicon_test.go index c8b7ecadbe8..a263f6bef69 100644 --- a/adapters/rubicon/rubicon_test.go +++ b/adapters/rubicon/rubicon_test.go @@ -945,7 +945,7 @@ func CreatePrebidRequest(server *httptest.Server, t *testing.T) (an *RubiconAdap pc := usersync.ParsePBSCookieFromRequest(req, &config.HostCookie{}) pc.TrySync("rubicon", rubidata.buyerUID) fakewriter := httptest.NewRecorder() - pc.SetCookieOnResponse(fakewriter, "", 90*24*time.Hour) + pc.SetCookieOnResponse(fakewriter, &config.HostCookie{Domain: ""}, 90*24*time.Hour) req.Header.Add("Cookie", fakewriter.Header().Get("Set-Cookie")) cacheClient, _ := dummycache.New() diff --git a/adapters/sovrn/sovrn_test.go b/adapters/sovrn/sovrn_test.go index c84b0dbca7a..67d49e62ab8 100644 --- a/adapters/sovrn/sovrn_test.go +++ b/adapters/sovrn/sovrn_test.go @@ -188,7 +188,7 @@ func SampleSovrnRequest(numberOfImpressions int, t *testing.T) *pbs.PBSRequest { pc := usersync.ParsePBSCookieFromRequest(httpReq, &config.HostCookie{}) pc.TrySync("sovrn", testSovrnUserId) fakewriter := httptest.NewRecorder() - pc.SetCookieOnResponse(fakewriter, "", 90*24*time.Hour) + pc.SetCookieOnResponse(fakewriter, &config.HostCookie{Domain: ""}, 90*24*time.Hour) httpReq.Header.Add("Cookie", fakewriter.Header().Get("Set-Cookie")) // parse the http request cacheClient, _ := dummycache.New() diff --git a/config/config.go b/config/config.go index 6eff84dac36..f60c8c86e59 100644 --- a/config/config.go +++ b/config/config.go @@ -59,6 +59,8 @@ type Configuration struct { BlacklistedAcctMap map[string]bool } +const MIN_COOKIE_SIZE_BYTES = 500 + type HTTPClient struct { MaxIdleConns int `mapstructure:"max_idle_connections"` MaxIdleConnsPerHost int `mapstructure:"max_idle_connections_per_host"` @@ -175,12 +177,13 @@ type FileLogs struct { } type HostCookie struct { - Domain string `mapstructure:"domain"` - Family string `mapstructure:"family"` - CookieName string `mapstructure:"cookie_name"` - OptOutURL string `mapstructure:"opt_out_url"` - OptInURL string `mapstructure:"opt_in_url"` - OptOutCookie Cookie `mapstructure:"optout_cookie"` + Domain string `mapstructure:"domain"` + Family string `mapstructure:"family"` + CookieName string `mapstructure:"cookie_name"` + OptOutURL string `mapstructure:"opt_out_url"` + OptInURL string `mapstructure:"opt_in_url"` + MaxCookieSizeBytes int `mapstructure:"max_cookie_size_bytes"` + OptOutCookie Cookie `mapstructure:"optout_cookie"` // Cookie timeout in days TTL int64 `mapstructure:"ttl_days"` } @@ -389,7 +392,7 @@ func New(v *viper.Viper) (*Configuration, error) { } c.setDerivedDefaults() - // To look for a request's publisher_id into the NonStandardPublishers in + // To look for a request's publisher_id in the NonStandardPublishers list in // O(1) time, we fill this hash table located in the NonStandardPublisherMap field of GDPR c.GDPR.NonStandardPublisherMap = make(map[string]int) for i := 0; i < len(c.GDPR.NonStandardPublishers); i++ { @@ -410,6 +413,11 @@ func New(v *viper.Viper) (*Configuration, error) { c.BlacklistedAcctMap[c.BlacklistedAccts[i]] = true } + if err := isValidCookieSize(c.HostCookie.MaxCookieSizeBytes); err != nil { + glog.Fatal(fmt.Printf("Max cookie size %d cannot be less than %d \n", c.HostCookie.MaxCookieSizeBytes, MIN_COOKIE_SIZE_BYTES)) + return nil, err + } + glog.Info("Logging the resolved configuration:") logGeneral(reflect.ValueOf(c), " \t") if errs := c.validate(); len(errs) > 0 { @@ -525,6 +533,7 @@ func SetupViper(v *viper.Viper, filename string) { v.SetDefault("host_cookie.optout_cookie.name", "") v.SetDefault("host_cookie.value", "") v.SetDefault("host_cookie.ttl_days", 90) + v.SetDefault("host_cookie.max_cookie_size_bytes", 0) v.SetDefault("http_client.max_idle_connections", 400) v.SetDefault("http_client.max_idle_connections_per_host", 10) v.SetDefault("http_client.idle_connection_timeout_seconds", 60) @@ -683,3 +692,12 @@ func setBidderDefaults(v *viper.Viper, bidder string) { v.SetDefault(adapterCfgPrefix+bidder+".disabled", false) v.SetDefault(adapterCfgPrefix+bidder+".partner_id", "") } + +func isValidCookieSize(maxCookieSize int) error { + // If a non-zero-less-than-500-byte "host_cookie.max_cookie_size_bytes" value was specified in the + // environment configuration of prebid-server, default to 500 bytes + if maxCookieSize != 0 && maxCookieSize < MIN_COOKIE_SIZE_BYTES { + return fmt.Errorf("Configured cookie size is less than allowed minimum size of %d \n", MIN_COOKIE_SIZE_BYTES) + } + return nil +} diff --git a/config/config_test.go b/config/config_test.go index 10ba3258c19..80d278daf08 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -2,6 +2,7 @@ package config import ( "bytes" + "fmt" "strings" "testing" "time" @@ -22,6 +23,7 @@ func TestDefaults(t *testing.T) { cmpInts(t, "auction_timeouts_ms.max", int(cfg.AuctionTimeouts.Max), 0) cmpInts(t, "max_request_size", int(cfg.MaxRequestSize), 1024*256) cmpInts(t, "host_cookie.ttl_days", int(cfg.HostCookie.TTL), 90) + cmpInts(t, "host_cookie.max_cookie_size_bytes", cfg.HostCookie.MaxCookieSizeBytes, 0) cmpStrings(t, "datacache.type", cfg.DataCache.Type, "dummy") cmpStrings(t, "adapters.pubmatic.endpoint", cfg.Adapters[string(openrtb_ext.BidderPubmatic)].Endpoint, "http://hbopenbid.pubmatic.com/translator?source=prebid-server") cmpInts(t, "currency_converter.fetch_interval_seconds", cfg.CurrencyConverter.FetchIntervalSeconds, 1800) @@ -39,6 +41,7 @@ host_cookie: domain: cookies.prebid.org opt_out_url: http://prebid.org/optout opt_in_url: http://prebid.org/optin + max_cookie_size_bytes: 32768 external_url: http://prebid-server.prebid.org/ host: prebid-server.prebid.org port: 1234 @@ -300,6 +303,28 @@ func TestLimitTimeout(t *testing.T) { doTimeoutTest(t, 15, 0, 20, 15) } +func TestCookieSizeError(t *testing.T) { + type aTest struct { + cookieHost *HostCookie + expectError bool + } + testCases := []aTest{ + {cookieHost: &HostCookie{MaxCookieSizeBytes: 1 << 15}, expectError: false}, //32 KB, no error + {cookieHost: &HostCookie{MaxCookieSizeBytes: 800}, expectError: false}, + {cookieHost: &HostCookie{MaxCookieSizeBytes: 500}, expectError: false}, + {cookieHost: &HostCookie{MaxCookieSizeBytes: 0}, expectError: false}, + {cookieHost: &HostCookie{MaxCookieSizeBytes: 200}, expectError: true}, + {cookieHost: &HostCookie{MaxCookieSizeBytes: -100}, expectError: true}, + } + for i := range testCases { + if testCases[i].expectError { + assert.Error(t, isValidCookieSize(testCases[i].cookieHost.MaxCookieSizeBytes), fmt.Sprintf("Configuration.HostCooki.MaxCookieSizeBytes less than MIN_COOKIE_SIZE_BYTES = %d and not equal to zero should return an error", MIN_COOKIE_SIZE_BYTES)) + } else { + assert.NoError(t, isValidCookieSize(testCases[i].cookieHost.MaxCookieSizeBytes), fmt.Sprintf("Configuration.HostCooki.MaxCookieSizeBytes greater than MIN_COOKIE_SIZE_BYTES = %d or equal to zero should not return an error", MIN_COOKIE_SIZE_BYTES)) + } + } +} + func newDefaultConfig(t *testing.T) *Configuration { v := viper.New() SetupViper(v, "") diff --git a/endpoints/setuid.go b/endpoints/setuid.go index 6a17592bd09..0275e354664 100644 --- a/endpoints/setuid.go +++ b/endpoints/setuid.go @@ -76,7 +76,7 @@ func NewSetUIDEndpoint(cfg config.HostCookie, perms gdpr.Permissions, pbsanalyti so.Success = true } - pc.SetCookieOnResponse(w, cfg.Domain, cookieTTL) + pc.SetCookieOnResponse(w, &cfg, cookieTTL) }) } diff --git a/pbs/usersync.go b/pbs/usersync.go index 8194e7f4ee2..b388d47d1fc 100644 --- a/pbs/usersync.go +++ b/pbs/usersync.go @@ -95,7 +95,7 @@ func (deps *UserSyncDeps) OptOut(w http.ResponseWriter, r *http.Request, _ httpr pc := usersync.ParsePBSCookieFromRequest(r, deps.HostCookieConfig) pc.SetPreference(optout == "") - pc.SetCookieOnResponse(w, deps.HostCookieConfig.Domain, deps.HostCookieConfig.TTLDuration()) + pc.SetCookieOnResponse(w, deps.HostCookieConfig, deps.HostCookieConfig.TTLDuration()) if optout == "" { http.Redirect(w, r, deps.HostCookieConfig.OptInURL, 301) } else { diff --git a/usersync/cookie.go b/usersync/cookie.go index a2b773aa2da..7334d15f358 100644 --- a/usersync/cookie.go +++ b/usersync/cookie.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "encoding/json" "errors" + "math" "net/http" "time" @@ -160,11 +161,32 @@ func (cookie *PBSCookie) GetId(bidderName openrtb_ext.BidderName) (id string, ex } // SetCookieOnResponse is a shortcut for "ToHTTPCookie(); cookie.setDomain(domain); setCookie(w, cookie)" -func (cookie *PBSCookie) SetCookieOnResponse(w http.ResponseWriter, domain string, ttl time.Duration) { +func (cookie *PBSCookie) SetCookieOnResponse(w http.ResponseWriter, cfg *config.HostCookie, ttl time.Duration) { httpCookie := cookie.ToHTTPCookie(ttl) + var domain string = cfg.Domain + if domain != "" { httpCookie.Domain = domain } + + var currSize int = len([]byte(httpCookie.String())) + for cfg.MaxCookieSizeBytes > 0 && currSize > cfg.MaxCookieSizeBytes && len(cookie.uids) > 0 { + var oldestElem string = "" + var oldestDate int64 = math.MaxInt64 + for key, value := range cookie.uids { + timeUntilExpiration := time.Until(value.Expires) + if timeUntilExpiration < time.Duration(oldestDate) { + oldestElem = key + oldestDate = int64(timeUntilExpiration) + } + } + delete(cookie.uids, oldestElem) + httpCookie = cookie.ToHTTPCookie(ttl) + if domain != "" { + httpCookie.Domain = domain + } + currSize = len([]byte(httpCookie.String())) + } http.SetCookie(w, httpCookie) } diff --git a/usersync/cookie_test.go b/usersync/cookie_test.go index 82cc4f32b9f..2c28956bd87 100644 --- a/usersync/cookie_test.go +++ b/usersync/cookie_test.go @@ -67,7 +67,7 @@ func TestBidderNameGets(t *testing.T) { func TestRejectAudienceNetworkCookie(t *testing.T) { raw := &PBSCookie{ uids: map[string]uidWithExpiry{ - "audienceNetwork": newTempId("0"), + "audienceNetwork": newTempId("0", 10), }, optOut: false, birthday: timestamp(), @@ -162,7 +162,7 @@ func TestParseOtherCookie(t *testing.T) { func TestCookieReadWrite(t *testing.T) { cookie := newSampleCookie() - received := writeThenRead(cookie) + received := writeThenRead(cookie, 0) uid, exists, isLive := received.GetUID("adnxs") if !exists || !isLive || uid != "123" { t.Errorf("Received cookie should have the adnxs ID=123. Got %s", uid) @@ -262,6 +262,37 @@ func TestGetUIDsWithNilCookie(t *testing.T) { assert.Len(t, uids, 0, "GetUIDs shouldn't return any user syncs for a nil cookie") } +func TestTrimCookiesClosestExpirationDates(t *testing.T) { + cookieToSend, cookieToSendLen := newTestCookie() + closestToExpirationDate := "key7" + + type aTest struct { + maxCookieSize int + expAction string + } + testCases := []aTest{ + {maxCookieSize: 2000, expAction: "equal"}, //1 don't trim, set + {maxCookieSize: 0, expAction: "equal"}, //2 unlimited size: don't trim, set + {maxCookieSize: 800, expAction: "trim"}, //3 trim to size and set + {maxCookieSize: 500, expAction: "trim"}, //4 trim to size and set + {maxCookieSize: 200, expAction: "empty"}, //5 insufficient size, trim to zero lenght and set + {maxCookieSize: -100, expAction: "empty"}, //6 invalid size, trim to zero lenght and set + } + for i := range testCases { + processedCookie := writeThenRead(cookieToSend, testCases[i].maxCookieSize) + switch testCases[i].expAction { + case "equal": + assert.Equal(t, cookieToSendLen, len(processedCookie.uids), "[Test %d] MaxCookieSizeBytes equal to zero or bigger than %d bytes should be enough to set and remain cookie unchanged \n", i+1, len(processedCookie.uids)) + assert.Containsf(t, processedCookie.uids, closestToExpirationDate, "[Test %d] Oldest entry in cookie should not have been eliminated", i+1) + case "trim": + assert.Equal(t, cookieToSendLen > len(processedCookie.uids), true, "[Test %d] MaxCookieSizeBytes of %d is smaller than %d bytes and cookie entries should have been removed\n", i+1, testCases[i].maxCookieSize, cookieToSendLen) + assert.NotContainsf(t, processedCookie.uids, closestToExpirationDate, "[Test %d] Oldest entry in cookie should not have been eliminated", i+1) + case "empty": + assert.Equal(t, len(processedCookie.uids), 0, "[Test %d] MaxCookieSizeBytes of %d is too small, processedCookie.uids should be empty\n", i+1) + } + } +} + func ensureEmptyMap(t *testing.T, cookie *PBSCookie) { if !cookie.AllowSyncs() { t.Error("Empty cookies should allow user syncs.") @@ -333,31 +364,49 @@ func ensureConsistency(t *testing.T, cookie *PBSCookie) { } } -func newTempId(uid string) uidWithExpiry { +func newTempId(uid string, offset int) uidWithExpiry { return uidWithExpiry{ UID: uid, - Expires: time.Now().Add(10 * time.Minute), + Expires: time.Now().Add(time.Duration(offset) * time.Minute), } } func newSampleCookie() *PBSCookie { return &PBSCookie{ uids: map[string]uidWithExpiry{ - "adnxs": newTempId("123"), - "rubicon": newTempId("456"), + "adnxs": newTempId("123", 10), + "rubicon": newTempId("456", 10), + }, + optOut: false, + birthday: timestamp(), + } +} + +func newTestCookie() (*PBSCookie, int) { + var mediumSizeCookie *PBSCookie = &PBSCookie{ + uids: map[string]uidWithExpiry{ + "key1": newTempId("12345678901234567890123456789012345678901234567890", 7), + "key2": newTempId("abcdefghijklmnopqrstuvwxyz", 6), + "key3": newTempId("ABCDEFGHIJKLMNOPQRSTUVWXYZ", 6), + "key4": newTempId("12345678901234567890123456789612345678901234567890", 5), + "key5": newTempId("aAbBcCdDeEfFgGhHiIjJkKlLmMnNoOpPqQrRsStTuUvVwWxXyYzZ", 4), + "key6": newTempId("12345678901234567890123456789012345678901234567890", 3), + "key7": newTempId("abcdefghijklmnopqrstuvwxyz", 2), }, optOut: false, birthday: timestamp(), } + return mediumSizeCookie, len(mediumSizeCookie.uids) } -func writeThenRead(cookie *PBSCookie) *PBSCookie { +func writeThenRead(cookie *PBSCookie, maxCookieSize int) *PBSCookie { w := httptest.NewRecorder() - cookie.SetCookieOnResponse(w, "mock-domain", 90*24*time.Hour) + hostCookie := &config.HostCookie{Domain: "mock-domain", MaxCookieSizeBytes: maxCookieSize} + cookie.SetCookieOnResponse(w, hostCookie, 90*24*time.Hour) writtenCookie := w.HeaderMap.Get("Set-Cookie") header := http.Header{} header.Add("Cookie", writtenCookie) request := http.Request{Header: header} - return ParsePBSCookieFromRequest(&request, &config.HostCookie{}) + return ParsePBSCookieFromRequest(&request, hostCookie) }