diff --git a/config/config.go b/config/config.go index dc50c8a062c..b1516337d9e 100644 --- a/config/config.go +++ b/config/config.go @@ -500,6 +500,7 @@ func SetupViper(v *viper.Viper, filename string) { v.SetDefault("datacache.ttl_seconds", 0) v.SetDefault("category_mapping.filesystem.enabled", true) v.SetDefault("category_mapping.filesystem.directorypath", "./static/category-mapping") + v.SetDefault("category_mapping.http.endpoint", "") v.SetDefault("stored_requests.filesystem", false) v.SetDefault("stored_requests.directorypath", "./stored_requests/data/by_id") v.SetDefault("stored_requests.postgres.connection.dbname", "") diff --git a/exchange/exchange.go b/exchange/exchange.go index da454563faf..5fd1bb6a29f 100644 --- a/exchange/exchange.go +++ b/exchange/exchange.go @@ -139,7 +139,7 @@ func (e *exchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidReque conversions := e.currencyConverter.Rates() adapterBids, adapterExtra := e.getAllBids(auctionCtx, cleanRequests, aliases, bidAdjustmentFactors, blabels, conversions) - bidCategory, adapterBids, err := applyCategoryMapping(requestExt, adapterBids, *categoriesFetcher, targData) + bidCategory, adapterBids, err := applyCategoryMapping(ctx, requestExt, adapterBids, *categoriesFetcher, targData) auc := newAuction(adapterBids, len(bidRequest.Imp)) if err != nil { return nil, fmt.Errorf("Error in category mapping : %s", err.Error()) @@ -321,7 +321,7 @@ func (e *exchange) buildBidResponse(ctx context.Context, liveAdapters []openrtb_ return bidResponse, err } -func applyCategoryMapping(requestExt openrtb_ext.ExtRequest, seatBids map[openrtb_ext.BidderName]*pbsOrtbSeatBid, categoriesFetcher stored_requests.CategoryFetcher, targData *targetData) (map[string]string, map[openrtb_ext.BidderName]*pbsOrtbSeatBid, error) { +func applyCategoryMapping(ctx context.Context, requestExt openrtb_ext.ExtRequest, seatBids map[openrtb_ext.BidderName]*pbsOrtbSeatBid, categoriesFetcher stored_requests.CategoryFetcher, targData *targetData) (map[string]string, map[openrtb_ext.BidderName]*pbsOrtbSeatBid, error) { res := make(map[string]string) type bidDedupe struct { @@ -375,7 +375,7 @@ func applyCategoryMapping(requestExt openrtb_ext.ExtRequest, seatBids map[openrt continue } else { //if unique IAB category is present then translate it to the adserver category based on mapping file - category, err = categoriesFetcher.FetchCategories(primaryAdServer, publisher, bidIabCat[0]) + category, err = categoriesFetcher.FetchCategories(ctx, primaryAdServer, publisher, bidIabCat[0]) if err != nil || category == "" { //TODO: add metrics //if mapping required but no mapping file is found then discard the bid diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index 66823055207..868ca097d27 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -587,7 +587,7 @@ func TestCategoryMapping(t *testing.T) { adapterBids[bidderName1] = &seatBid - bidCategory, adapterBids, err := applyCategoryMapping(requestExt, adapterBids, categoriesFetcher, targData) + bidCategory, adapterBids, err := applyCategoryMapping(nil, requestExt, adapterBids, categoriesFetcher, targData) assert.Equal(t, nil, err, "Category mapping error should be empty") assert.Equal(t, "10.00_Electronics_30s", bidCategory["bid_id1"], "Category mapping doesn't match") @@ -652,7 +652,7 @@ func TestCategoryDedupe(t *testing.T) { adapterBids[bidderName1] = &seatBid - bidCategory, adapterBids, err := applyCategoryMapping(requestExt, adapterBids, categoriesFetcher, targData) + bidCategory, adapterBids, err := applyCategoryMapping(nil, requestExt, adapterBids, categoriesFetcher, targData) assert.Equal(t, nil, err, "Category mapping error should be empty") assert.Equal(t, 2, len(adapterBids[bidderName1].bids), "Bidders number doesn't match") diff --git a/stored_requests/backends/db_fetcher/fetcher.go b/stored_requests/backends/db_fetcher/fetcher.go index abb58731775..f12166eafec 100644 --- a/stored_requests/backends/db_fetcher/fetcher.go +++ b/stored_requests/backends/db_fetcher/fetcher.go @@ -93,7 +93,7 @@ func (fetcher *dbFetcher) FetchRequests(ctx context.Context, requestIDs []string return storedRequestData, storedImpData, errs } -func (fetcher *dbFetcher) FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) { +func (fetcher *dbFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) { return "", nil } diff --git a/stored_requests/backends/empty_fetcher/fetcher.go b/stored_requests/backends/empty_fetcher/fetcher.go index 298449c59f0..25e8ead434b 100644 --- a/stored_requests/backends/empty_fetcher/fetcher.go +++ b/stored_requests/backends/empty_fetcher/fetcher.go @@ -27,6 +27,6 @@ func (fetcher EmptyFetcher) FetchRequests(ctx context.Context, requestIDs []stri return } -func (fetcher EmptyFetcher) FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) { +func (fetcher EmptyFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) { return "", nil } diff --git a/stored_requests/backends/file_fetcher/fetcher.go b/stored_requests/backends/file_fetcher/fetcher.go index 61c5265ca3c..60853f65da7 100644 --- a/stored_requests/backends/file_fetcher/fetcher.go +++ b/stored_requests/backends/file_fetcher/fetcher.go @@ -22,12 +22,7 @@ func NewFileFetcher(directory string) (stored_requests.AllFetcher, error) { type eagerFetcher struct { FileSystem FileSystem - Categories map[string]map[string]Category -} - -type Category struct { - Id string - Name string + Categories map[string]map[string]stored_requests.Category } func (fetcher *eagerFetcher) FetchRequests(ctx context.Context, requestIDs []string, impIDs []string) (map[string]json.RawMessage, map[string]json.RawMessage, []error) { @@ -38,7 +33,7 @@ func (fetcher *eagerFetcher) FetchRequests(ctx context.Context, requestIDs []str return storedRequests, storedImpressions, errs } -func (fetcher *eagerFetcher) FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) { +func (fetcher *eagerFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) { fileName := primaryAdServer if len(publisherId) != 0 { @@ -46,7 +41,7 @@ func (fetcher *eagerFetcher) FetchCategories(primaryAdServer, publisherId, iabCa } if fetcher.Categories == nil { - fetcher.Categories = make(map[string]map[string]Category) + fetcher.Categories = make(map[string]map[string]stored_requests.Category) } if data, ok := fetcher.Categories[fileName]; ok { return data[iabCategory].Id, nil @@ -56,7 +51,7 @@ func (fetcher *eagerFetcher) FetchCategories(primaryAdServer, publisherId, iabCa if file, ok := primaryAdServerDir.Files[fileName]; ok { - tmp := make(map[string]Category) + tmp := make(map[string]stored_requests.Category) if err := json.Unmarshal(file, &tmp); err != nil { return "", fmt.Errorf("Unable to unmarshal categories for adserver: '%s', publisherId: '%s'", primaryAdServer, publisherId) diff --git a/stored_requests/backends/file_fetcher/fetcher_test.go b/stored_requests/backends/file_fetcher/fetcher_test.go index 11628352f1f..2429a77cd25 100644 --- a/stored_requests/backends/file_fetcher/fetcher_test.go +++ b/stored_requests/backends/file_fetcher/fetcher_test.go @@ -114,7 +114,7 @@ func TestCategoriesFetcherWithPublisher(t *testing.T) { if err != nil { t.Errorf("Failed to create a category Fetcher: %v", err) } - category, err := fetcher.FetchCategories("test", "categories", "IAB1-1") + category, err := fetcher.FetchCategories(nil, "test", "categories", "IAB1-1") assert.Equal(t, nil, err, "Categories were loaded incorrectly") assert.Equal(t, "Beverages", category, "Categories were loaded incorrectly") } @@ -124,7 +124,7 @@ func TestCategoriesFetcherWithoutPublisher(t *testing.T) { if err != nil { t.Errorf("Failed to create a category Fetcher: %v", err) } - category, err := fetcher.FetchCategories("test", "", "IAB1-1") + category, err := fetcher.FetchCategories(nil, "test", "", "IAB1-1") assert.Equal(t, nil, err, "Categories were loaded incorrectly") assert.Equal(t, "VideoGames", category, "Categories were loaded incorrectly") } @@ -134,7 +134,7 @@ func TestCategoriesFetcherNoCategory(t *testing.T) { if err != nil { t.Errorf("Failed to create a category Fetcher: %v", err) } - _, fetchingErr := fetcher.FetchCategories("test", "", "IAB1-100") + _, fetchingErr := fetcher.FetchCategories(nil, "test", "", "IAB1-100") assert.Equal(t, fmt.Errorf("Unable to find category for adserver 'test', publisherId: '', iab category: 'IAB1-100'"), fetchingErr, "Categories were loaded incorrectly") } @@ -144,7 +144,7 @@ func TestCategoriesFetcherBrokenJson(t *testing.T) { if err != nil { t.Errorf("Failed to create a category Fetcher: %v", err) } - _, fetchingErr := fetcher.FetchCategories("test", "broken", "IAB1-100") + _, fetchingErr := fetcher.FetchCategories(nil, "test", "broken", "IAB1-100") assert.Equal(t, fmt.Errorf("Unable to unmarshal categories for adserver: 'test', publisherId: 'broken'"), fetchingErr, "Categories were loaded incorrectly") } @@ -154,7 +154,7 @@ func TestCategoriesFetcherNoCategoriesFile(t *testing.T) { if err != nil { t.Errorf("Failed to create a category Fetcher: %v", err) } - _, fetchingErr := fetcher.FetchCategories("test", "not_exists", "IAB1-100") + _, fetchingErr := fetcher.FetchCategories(nil, "test", "not_exists", "IAB1-100") assert.Equal(t, fmt.Errorf("Unable to find mapping file for adserver: 'test', publisherId: 'not_exists'"), fetchingErr, "Categories were loaded incorrectly") } diff --git a/stored_requests/backends/http_fetcher/fetcher.go b/stored_requests/backends/http_fetcher/fetcher.go index 34175c38ee6..b7e42c9e6cf 100644 --- a/stored_requests/backends/http_fetcher/fetcher.go +++ b/stored_requests/backends/http_fetcher/fetcher.go @@ -56,9 +56,10 @@ func NewFetcher(client *http.Client, endpoint string) *HttpFetcher { } type HttpFetcher struct { - client *http.Client - Endpoint string - hasQuery bool + client *http.Client + Endpoint string + hasQuery bool + Categories map[string]map[string]stored_requests.Category } func (fetcher *HttpFetcher) FetchRequests(ctx context.Context, requestIDs []string, impIDs []string) (requestData map[string]json.RawMessage, impData map[string]json.RawMessage, errs []error) { @@ -80,8 +81,54 @@ func (fetcher *HttpFetcher) FetchRequests(ctx context.Context, requestIDs []stri return } -func (fetcher *HttpFetcher) FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) { - return "", nil +func (fetcher *HttpFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) { + if fetcher.Categories == nil { + fetcher.Categories = make(map[string]map[string]stored_requests.Category) + } + + //in NewFetcher function there is a code to add "?" at the end of url + //in case of categories we don't expect to have any parameters, that's why we need to remove "?" + var dataName, url string + if publisherId != "" { + dataName = fmt.Sprintf("%s_%s", primaryAdServer, publisherId) + url = fmt.Sprintf("%s/%s/%s.json", strings.TrimSuffix(fetcher.Endpoint, "?"), primaryAdServer, publisherId) + } else { + dataName = primaryAdServer + url = fmt.Sprintf("%s/%s.json", strings.TrimSuffix(fetcher.Endpoint, "?"), primaryAdServer) + } + + if data, ok := fetcher.Categories[dataName]; ok { + if val, ok := data[iabCategory]; ok { + return val.Id, nil + } else { + return "", fmt.Errorf("Unable to find category mapping for adserver: '%s', publisherId: '%s'", primaryAdServer, publisherId) + } + } + + httpReq, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", err + } + + httpResp, err := ctxhttp.Do(ctx, fetcher.client, httpReq) + if err != nil { + return "", err + } + defer httpResp.Body.Close() + + respBytes, err := ioutil.ReadAll(httpResp.Body) + tmp := make(map[string]stored_requests.Category) + + if err := json.Unmarshal(respBytes, &tmp); err != nil { + return "", fmt.Errorf("Unable to unmarshal categories for adserver: '%s', publisherId: '%s'", primaryAdServer, publisherId) + } + fetcher.Categories[dataName] = tmp + + if val, ok := tmp[iabCategory]; ok { + return val.Id, nil + } else { + return "", fmt.Errorf("Unable to find category mapping for adserver: '%s', publisherId: '%s'", primaryAdServer, publisherId) + } } func buildRequest(endpoint string, requestIDs []string, impIDs []string) (*http.Request, error) { diff --git a/stored_requests/fetcher.go b/stored_requests/fetcher.go index f42fb985eb4..808495e4584 100644 --- a/stored_requests/fetcher.go +++ b/stored_requests/fetcher.go @@ -27,13 +27,13 @@ type Fetcher interface { type CategoryFetcher interface { // FetchCategories fetches the ad-server/publisher specific category for the given IAB category - FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) + FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) } // AllFetcher is an iterface that encapsulates both the original Fetcher and the CategoryFetcher type AllFetcher interface { FetchRequests(ctx context.Context, requestIDs []string, impIDs []string) (requestData map[string]json.RawMessage, impData map[string]json.RawMessage, errs []error) - FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) + FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) } // NotFoundError is an error type to flag that an ID was not found by the Fetcher. @@ -44,6 +44,11 @@ type NotFoundError struct { DataType string } +type Category struct { + Id string + Name string +} + func (e NotFoundError) Error() string { return fmt.Sprintf(`Stored %s with ID="%s" not found.`, e.DataType, e.ID) } @@ -176,7 +181,7 @@ func (f *fetcherWithCache) FetchRequests(ctx context.Context, requestIDs []strin return } -func (f *fetcherWithCache) FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) { +func (f *fetcherWithCache) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) { return "", nil } diff --git a/stored_requests/fetcher_test.go b/stored_requests/fetcher_test.go index 13a003637c0..c1040acdb90 100644 --- a/stored_requests/fetcher_test.go +++ b/stored_requests/fetcher_test.go @@ -215,7 +215,7 @@ func (f *mockFetcher) FetchRequests(ctx context.Context, requestIDs []string, im return args.Get(0).(map[string]json.RawMessage), args.Get(1).(map[string]json.RawMessage), args.Get(2).([]error) } -func (f *mockFetcher) FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) { +func (f *mockFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) { return "", nil } diff --git a/stored_requests/multifetcher.go b/stored_requests/multifetcher.go index 01cbb8098e6..24cf848448c 100644 --- a/stored_requests/multifetcher.go +++ b/stored_requests/multifetcher.go @@ -36,10 +36,10 @@ func (mf MultiFetcher) FetchRequests(ctx context.Context, requestIDs []string, i return } -func (mf MultiFetcher) FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) { +func (mf MultiFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) { for _, f := range mf { if cf, ok := f.(CategoryFetcher); ok { - iabCategory, _ := cf.FetchCategories(primaryAdServer, publisherId, iabCategory) + iabCategory, _ := cf.FetchCategories(ctx, primaryAdServer, publisherId, iabCategory) if iabCategory != "" { return iabCategory, nil }