Implementation of Categories Http fetcher (prebid#882)

* Implementation of Categories Http fetcher -Added code to fetch data using http -Added previously loaded categories to run http request just at the first time * Moved common Category struct to stored_request * Added comments * Minor refactoring * Minor refactoring -Added verification if category exist in map * Minor refactoring * Minor refactoring -strings.Replace changed to strings.TrimSuffix
flux-dev-team · May 20, 2019 · cc8a978 · cc8a978
1 parent 0dc7ad1
commit cc8a978
Show file tree

Hide file tree

Showing 11 changed files with 80 additions and 32 deletions.
diff --git a/config/config.go b/config/config.go
@@ -500,6 +500,7 @@ func SetupViper(v *viper.Viper, filename string) {
 	v.SetDefault("datacache.ttl_seconds", 0)
 	v.SetDefault("category_mapping.filesystem.enabled", true)
 	v.SetDefault("category_mapping.filesystem.directorypath", "./static/category-mapping")
+	v.SetDefault("category_mapping.http.endpoint", "")
 	v.SetDefault("stored_requests.filesystem", false)
 	v.SetDefault("stored_requests.directorypath", "./stored_requests/data/by_id")
 	v.SetDefault("stored_requests.postgres.connection.dbname", "")

diff --git a/exchange/exchange.go b/exchange/exchange.go
@@ -139,7 +139,7 @@ func (e *exchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidReque
 	conversions := e.currencyConverter.Rates()
 
 	adapterBids, adapterExtra := e.getAllBids(auctionCtx, cleanRequests, aliases, bidAdjustmentFactors, blabels, conversions)
-	bidCategory, adapterBids, err := applyCategoryMapping(requestExt, adapterBids, *categoriesFetcher, targData)
+	bidCategory, adapterBids, err := applyCategoryMapping(ctx, requestExt, adapterBids, *categoriesFetcher, targData)
 	auc := newAuction(adapterBids, len(bidRequest.Imp))
 	if err != nil {
 		return nil, fmt.Errorf("Error in category mapping : %s", err.Error())
@@ -321,7 +321,7 @@ func (e *exchange) buildBidResponse(ctx context.Context, liveAdapters []openrtb_
 	return bidResponse, err
 }
 
-func applyCategoryMapping(requestExt openrtb_ext.ExtRequest, seatBids map[openrtb_ext.BidderName]*pbsOrtbSeatBid, categoriesFetcher stored_requests.CategoryFetcher, targData *targetData) (map[string]string, map[openrtb_ext.BidderName]*pbsOrtbSeatBid, error) {
+func applyCategoryMapping(ctx context.Context, requestExt openrtb_ext.ExtRequest, seatBids map[openrtb_ext.BidderName]*pbsOrtbSeatBid, categoriesFetcher stored_requests.CategoryFetcher, targData *targetData) (map[string]string, map[openrtb_ext.BidderName]*pbsOrtbSeatBid, error) {
 	res := make(map[string]string)
 
 	type bidDedupe struct {
@@ -375,7 +375,7 @@ func applyCategoryMapping(requestExt openrtb_ext.ExtRequest, seatBids map[openrt
 					continue
 				} else {
 					//if unique IAB category is present then translate it to the adserver category based on mapping file
-					category, err = categoriesFetcher.FetchCategories(primaryAdServer, publisher, bidIabCat[0])
+					category, err = categoriesFetcher.FetchCategories(ctx, primaryAdServer, publisher, bidIabCat[0])
 					if err != nil || category == "" {
 						//TODO: add metrics
 						//if mapping required but no mapping file is found then discard the bid

diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go
@@ -587,7 +587,7 @@ func TestCategoryMapping(t *testing.T) {
 
 	adapterBids[bidderName1] = &seatBid
 
-	bidCategory, adapterBids, err := applyCategoryMapping(requestExt, adapterBids, categoriesFetcher, targData)
+	bidCategory, adapterBids, err := applyCategoryMapping(nil, requestExt, adapterBids, categoriesFetcher, targData)
 
 	assert.Equal(t, nil, err, "Category mapping error should be empty")
 	assert.Equal(t, "10.00_Electronics_30s", bidCategory["bid_id1"], "Category mapping doesn't match")
@@ -652,7 +652,7 @@ func TestCategoryDedupe(t *testing.T) {
 
 		adapterBids[bidderName1] = &seatBid
 
-		bidCategory, adapterBids, err := applyCategoryMapping(requestExt, adapterBids, categoriesFetcher, targData)
+		bidCategory, adapterBids, err := applyCategoryMapping(nil, requestExt, adapterBids, categoriesFetcher, targData)
 
 		assert.Equal(t, nil, err, "Category mapping error should be empty")
 		assert.Equal(t, 2, len(adapterBids[bidderName1].bids), "Bidders number doesn't match")

diff --git a/stored_requests/backends/db_fetcher/fetcher.go b/stored_requests/backends/db_fetcher/fetcher.go
@@ -93,7 +93,7 @@ func (fetcher *dbFetcher) FetchRequests(ctx context.Context, requestIDs []string
 	return storedRequestData, storedImpData, errs
 }
 
-func (fetcher *dbFetcher) FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) {
+func (fetcher *dbFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) {
 	return "", nil
 }
 

diff --git a/stored_requests/backends/empty_fetcher/fetcher.go b/stored_requests/backends/empty_fetcher/fetcher.go
@@ -27,6 +27,6 @@ func (fetcher EmptyFetcher) FetchRequests(ctx context.Context, requestIDs []stri
 	return
 }
 
-func (fetcher EmptyFetcher) FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) {
+func (fetcher EmptyFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) {
 	return "", nil
 }
diff --git a/stored_requests/backends/file_fetcher/fetcher.go b/stored_requests/backends/file_fetcher/fetcher.go
@@ -22,12 +22,7 @@ func NewFileFetcher(directory string) (stored_requests.AllFetcher, error) {
 
 type eagerFetcher struct {
 	FileSystem FileSystem
-	Categories map[string]map[string]Category
-}
-
-type Category struct {
-	Id   string
-	Name string
+	Categories map[string]map[string]stored_requests.Category
 }
 
 func (fetcher *eagerFetcher) FetchRequests(ctx context.Context, requestIDs []string, impIDs []string) (map[string]json.RawMessage, map[string]json.RawMessage, []error) {
@@ -38,15 +33,15 @@ func (fetcher *eagerFetcher) FetchRequests(ctx context.Context, requestIDs []str
 	return storedRequests, storedImpressions, errs
 }
 
-func (fetcher *eagerFetcher) FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) {
+func (fetcher *eagerFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) {
 	fileName := primaryAdServer
 
 	if len(publisherId) != 0 {
 		fileName = primaryAdServer + "_" + publisherId
 	}
 
 	if fetcher.Categories == nil {
-		fetcher.Categories = make(map[string]map[string]Category)
+		fetcher.Categories = make(map[string]map[string]stored_requests.Category)
 	}
 	if data, ok := fetcher.Categories[fileName]; ok {
 		return data[iabCategory].Id, nil
@@ -56,7 +51,7 @@ func (fetcher *eagerFetcher) FetchCategories(primaryAdServer, publisherId, iabCa
 
 		if file, ok := primaryAdServerDir.Files[fileName]; ok {
 
-			tmp := make(map[string]Category)
+			tmp := make(map[string]stored_requests.Category)
 
 			if err := json.Unmarshal(file, &tmp); err != nil {
 				return "", fmt.Errorf("Unable to unmarshal categories for adserver: '%s', publisherId: '%s'", primaryAdServer, publisherId)

diff --git a/stored_requests/backends/file_fetcher/fetcher_test.go b/stored_requests/backends/file_fetcher/fetcher_test.go
@@ -114,7 +114,7 @@ func TestCategoriesFetcherWithPublisher(t *testing.T) {
 	if err != nil {
 		t.Errorf("Failed to create a category Fetcher: %v", err)
 	}
-	category, err := fetcher.FetchCategories("test", "categories", "IAB1-1")
+	category, err := fetcher.FetchCategories(nil, "test", "categories", "IAB1-1")
 	assert.Equal(t, nil, err, "Categories were loaded incorrectly")
 	assert.Equal(t, "Beverages", category, "Categories were loaded incorrectly")
 }
@@ -124,7 +124,7 @@ func TestCategoriesFetcherWithoutPublisher(t *testing.T) {
 	if err != nil {
 		t.Errorf("Failed to create a category Fetcher: %v", err)
 	}
-	category, err := fetcher.FetchCategories("test", "", "IAB1-1")
+	category, err := fetcher.FetchCategories(nil, "test", "", "IAB1-1")
 	assert.Equal(t, nil, err, "Categories were loaded incorrectly")
 	assert.Equal(t, "VideoGames", category, "Categories were loaded incorrectly")
 }
@@ -134,7 +134,7 @@ func TestCategoriesFetcherNoCategory(t *testing.T) {
 	if err != nil {
 		t.Errorf("Failed to create a category Fetcher: %v", err)
 	}
-	_, fetchingErr := fetcher.FetchCategories("test", "", "IAB1-100")
+	_, fetchingErr := fetcher.FetchCategories(nil, "test", "", "IAB1-100")
 	assert.Equal(t, fmt.Errorf("Unable to find category for adserver 'test', publisherId: '', iab category: 'IAB1-100'"),
 		fetchingErr, "Categories were loaded incorrectly")
 }
@@ -144,7 +144,7 @@ func TestCategoriesFetcherBrokenJson(t *testing.T) {
 	if err != nil {
 		t.Errorf("Failed to create a category Fetcher: %v", err)
 	}
-	_, fetchingErr := fetcher.FetchCategories("test", "broken", "IAB1-100")
+	_, fetchingErr := fetcher.FetchCategories(nil, "test", "broken", "IAB1-100")
 	assert.Equal(t, fmt.Errorf("Unable to unmarshal categories for adserver: 'test', publisherId: 'broken'"),
 		fetchingErr, "Categories were loaded incorrectly")
 }
@@ -154,7 +154,7 @@ func TestCategoriesFetcherNoCategoriesFile(t *testing.T) {
 	if err != nil {
 		t.Errorf("Failed to create a category Fetcher: %v", err)
 	}
-	_, fetchingErr := fetcher.FetchCategories("test", "not_exists", "IAB1-100")
+	_, fetchingErr := fetcher.FetchCategories(nil, "test", "not_exists", "IAB1-100")
 	assert.Equal(t, fmt.Errorf("Unable to find mapping file for adserver: 'test', publisherId: 'not_exists'"),
 		fetchingErr, "Categories were loaded incorrectly")
 }
diff --git a/stored_requests/backends/http_fetcher/fetcher.go b/stored_requests/backends/http_fetcher/fetcher.go
@@ -56,9 +56,10 @@ func NewFetcher(client *http.Client, endpoint string) *HttpFetcher {
 }
 
 type HttpFetcher struct {
-	client   *http.Client
-	Endpoint string
-	hasQuery bool
+	client     *http.Client
+	Endpoint   string
+	hasQuery   bool
+	Categories map[string]map[string]stored_requests.Category
 }
 
 func (fetcher *HttpFetcher) FetchRequests(ctx context.Context, requestIDs []string, impIDs []string) (requestData map[string]json.RawMessage, impData map[string]json.RawMessage, errs []error) {
@@ -80,8 +81,54 @@ func (fetcher *HttpFetcher) FetchRequests(ctx context.Context, requestIDs []stri
 	return
 }
 
-func (fetcher *HttpFetcher) FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) {
-	return "", nil
+func (fetcher *HttpFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) {
+	if fetcher.Categories == nil {
+		fetcher.Categories = make(map[string]map[string]stored_requests.Category)
+	}
+
+	//in NewFetcher function there is a code to add "?" at the end of url
+	//in case of categories we don't expect to have any parameters, that's why we need to remove "?"
+	var dataName, url string
+	if publisherId != "" {
+		dataName = fmt.Sprintf("%s_%s", primaryAdServer, publisherId)
+		url = fmt.Sprintf("%s/%s/%s.json", strings.TrimSuffix(fetcher.Endpoint, "?"), primaryAdServer, publisherId)
+	} else {
+		dataName = primaryAdServer
+		url = fmt.Sprintf("%s/%s.json", strings.TrimSuffix(fetcher.Endpoint, "?"), primaryAdServer)
+	}
+
+	if data, ok := fetcher.Categories[dataName]; ok {
+		if val, ok := data[iabCategory]; ok {
+			return val.Id, nil
+		} else {
+			return "", fmt.Errorf("Unable to find category mapping for adserver: '%s', publisherId: '%s'", primaryAdServer, publisherId)
+		}
+	}
+
+	httpReq, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return "", err
+	}
+
+	httpResp, err := ctxhttp.Do(ctx, fetcher.client, httpReq)
+	if err != nil {
+		return "", err
+	}
+	defer httpResp.Body.Close()
+
+	respBytes, err := ioutil.ReadAll(httpResp.Body)
+	tmp := make(map[string]stored_requests.Category)
+
+	if err := json.Unmarshal(respBytes, &tmp); err != nil {
+		return "", fmt.Errorf("Unable to unmarshal categories for adserver: '%s', publisherId: '%s'", primaryAdServer, publisherId)
+	}
+	fetcher.Categories[dataName] = tmp
+
+	if val, ok := tmp[iabCategory]; ok {
+		return val.Id, nil
+	} else {
+		return "", fmt.Errorf("Unable to find category mapping for adserver: '%s', publisherId: '%s'", primaryAdServer, publisherId)
+	}
 }
 
 func buildRequest(endpoint string, requestIDs []string, impIDs []string) (*http.Request, error) {

diff --git a/stored_requests/fetcher.go b/stored_requests/fetcher.go
@@ -27,13 +27,13 @@ type Fetcher interface {
 
 type CategoryFetcher interface {
 	// FetchCategories fetches the ad-server/publisher specific category for the given IAB category
-	FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error)
+	FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error)
 }
 
 // AllFetcher is an iterface that encapsulates both the original Fetcher and the CategoryFetcher
 type AllFetcher interface {
 	FetchRequests(ctx context.Context, requestIDs []string, impIDs []string) (requestData map[string]json.RawMessage, impData map[string]json.RawMessage, errs []error)
-	FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error)
+	FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error)
 }
 
 // NotFoundError is an error type to flag that an ID was not found by the Fetcher.
@@ -44,6 +44,11 @@ type NotFoundError struct {
 	DataType string
 }
 
+type Category struct {
+	Id   string
+	Name string
+}
+
 func (e NotFoundError) Error() string {
 	return fmt.Sprintf(`Stored %s with ID="%s" not found.`, e.DataType, e.ID)
 }
@@ -176,7 +181,7 @@ func (f *fetcherWithCache) FetchRequests(ctx context.Context, requestIDs []strin
 	return
 }
 
-func (f *fetcherWithCache) FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) {
+func (f *fetcherWithCache) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) {
 	return "", nil
 }
 

diff --git a/stored_requests/fetcher_test.go b/stored_requests/fetcher_test.go
@@ -215,7 +215,7 @@ func (f *mockFetcher) FetchRequests(ctx context.Context, requestIDs []string, im
 	return args.Get(0).(map[string]json.RawMessage), args.Get(1).(map[string]json.RawMessage), args.Get(2).([]error)
 }
 
-func (f *mockFetcher) FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) {
+func (f *mockFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) {
 	return "", nil
 }
 

diff --git a/stored_requests/multifetcher.go b/stored_requests/multifetcher.go
@@ -36,10 +36,10 @@ func (mf MultiFetcher) FetchRequests(ctx context.Context, requestIDs []string, i
 	return
 }
 
-func (mf MultiFetcher) FetchCategories(primaryAdServer, publisherId, iabCategory string) (string, error) {
+func (mf MultiFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) {
 	for _, f := range mf {
 		if cf, ok := f.(CategoryFetcher); ok {
-			iabCategory, _ := cf.FetchCategories(primaryAdServer, publisherId, iabCategory)
+			iabCategory, _ := cf.FetchCategories(ctx, primaryAdServer, publisherId, iabCategory)
 			if iabCategory != "" {
 				return iabCategory, nil
 			}