Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a BidderRequest struct to hold bidder specific request info #1611

Merged
merged 9 commits into from
Dec 10, 2020
Merged

Add a BidderRequest struct to hold bidder specific request info #1611

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
c8bfa89
Work In Progress
SyntaxNode Nov 23, 2020
339b569
Refactor splitBidRequest and cleanOpenRTBRequests args
mansinahar Nov 30, 2020
789e480
Refactor recoverSafely args
mansinahar Nov 30, 2020
5821cdf
Change name from Bidder to AuctionRequestBidder
mansinahar Dec 1, 2020
85ffc00
Go back to BidderRequest from AuctionRequestBidder
mansinahar Dec 1, 2020
f8bbf0f
Update tests
mansinahar Dec 1, 2020
8f45c6d
Merge confict mistake
mansinahar Dec 4, 2020
2868d19
Fix build error
mansinahar Dec 4, 2020
3a398d6
Remove unused blabels
mansinahar Dec 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 51 additions & 36 deletions exchange/exchange.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@ func NewExchange(adapters map[openrtb_ext.BidderName]adaptedBidder, cache prebid
}
}

// AuctionRequest holds the bid request for the auction
// and all other information needed to process that request
type AuctionRequest struct {
BidRequest *openrtb.BidRequest
Account config.Account
Expand All @@ -107,6 +109,15 @@ type AuctionRequest struct {
LegacyLabels pbsmetrics.Labels
}

// BidderRequest holds the bidder specific request and all other
// information needed to process that bidder request.
type BidderRequest struct {
SyntaxNode marked this conversation as resolved.
Show resolved Hide resolved
BidRequest *openrtb.BidRequest
BidderName openrtb_ext.BidderName
BidderCoreName openrtb_ext.BidderName
BidderLabels pbsmetrics.AdapterLabels
}

func (e *exchange) HoldAuction(ctx context.Context, r AuctionRequest, debugLog *DebugLog) (*openrtb.BidResponse, error) {
var err error
requestExt, err := extractBidRequestExt(r.BidRequest)
Expand Down Expand Up @@ -134,12 +145,12 @@ func (e *exchange) HoldAuction(ctx context.Context, r AuctionRequest, debugLog *

// Slice of BidRequests, each a copy of the original cleaned to only contain bidder data for the named bidder
blabels := make(map[openrtb_ext.BidderName]*pbsmetrics.AdapterLabels)
cleanRequests, aliases, privacyLabels, errs := cleanOpenRTBRequests(ctx, r.BidRequest, requestExt, r.UserSyncs, blabels, r.LegacyLabels, e.gDPR, usersyncIfAmbiguous, e.privacyConfig, &r.Account)
bidderRequests, privacyLabels, errs := cleanOpenRTBRequests(ctx, r, requestExt, blabels, e.gDPR, usersyncIfAmbiguous, e.privacyConfig)
SyntaxNode marked this conversation as resolved.
Show resolved Hide resolved

e.me.RecordRequestPrivacy(privacyLabels)

// List of bidders we have requests for.
liveAdapters := listBiddersWithRequests(cleanRequests)
liveAdapters := listBiddersWithRequests(bidderRequests)

// If we need to cache bids, then it will take some time to call prebid cache.
// We should reduce the amount of time the bidders have, to compensate.
Expand All @@ -149,7 +160,7 @@ func (e *exchange) HoldAuction(ctx context.Context, r AuctionRequest, debugLog *
// Get currency rates conversions for the auction
conversions := e.currencyConverter.Rates()

adapterBids, adapterExtra, anyBidsReturned := e.getAllBids(auctionCtx, cleanRequests, aliases, bidAdjustmentFactors, blabels, conversions)
adapterBids, adapterExtra, anyBidsReturned := e.getAllBids(auctionCtx, bidderRequests, bidAdjustmentFactors, blabels, conversions)

var auc *auction
var cacheErrs []error
Expand Down Expand Up @@ -322,37 +333,39 @@ func (e *exchange) makeAuctionContext(ctx context.Context, needsCache bool) (auc
}

// This piece sends all the requests to the bidder adapters and gathers the results.
func (e *exchange) getAllBids(ctx context.Context, cleanRequests map[openrtb_ext.BidderName]*openrtb.BidRequest, aliases map[string]string, bidAdjustments map[string]float64, blabels map[openrtb_ext.BidderName]*pbsmetrics.AdapterLabels, conversions currencies.Conversions) (map[openrtb_ext.BidderName]*pbsOrtbSeatBid, map[openrtb_ext.BidderName]*seatResponseExtra, bool) {
func (e *exchange) getAllBids(ctx context.Context, bidderRequests []BidderRequest,
bidAdjustments map[string]float64, blabels map[openrtb_ext.BidderName]*pbsmetrics.AdapterLabels,
SyntaxNode marked this conversation as resolved.
Show resolved Hide resolved
conversions currencies.Conversions) (map[openrtb_ext.BidderName]*pbsOrtbSeatBid,
map[openrtb_ext.BidderName]*seatResponseExtra, bool) {
// Set up pointers to the bid results
adapterBids := make(map[openrtb_ext.BidderName]*pbsOrtbSeatBid, len(cleanRequests))
adapterExtra := make(map[openrtb_ext.BidderName]*seatResponseExtra, len(cleanRequests))
chBids := make(chan *bidResponseWrapper, len(cleanRequests))
adapterBids := make(map[openrtb_ext.BidderName]*pbsOrtbSeatBid, len(bidderRequests))
adapterExtra := make(map[openrtb_ext.BidderName]*seatResponseExtra, len(bidderRequests))
chBids := make(chan *bidResponseWrapper, len(bidderRequests))
bidsFound := false

for bidderName, req := range cleanRequests {
for _, bidder := range bidderRequests {
// Here we actually call the adapters and collect the bids.
coreBidder := resolveBidder(string(bidderName), aliases)
bidderRunner := e.recoverSafely(cleanRequests, func(aName openrtb_ext.BidderName, coreBidder openrtb_ext.BidderName, request *openrtb.BidRequest, bidlabels *pbsmetrics.AdapterLabels, conversions currencies.Conversions) {
bidderRunner := e.recoverSafely(bidderRequests, func(bidderRequest BidderRequest, conversions currencies.Conversions) {
// Passing in aName so a doesn't change out from under the go routine
if bidlabels.Adapter == "" {
glog.Errorf("Exchange: bidlables for %s (%s) missing adapter string", aName, coreBidder)
bidlabels.Adapter = coreBidder
if bidderRequest.BidderLabels.Adapter == "" {
glog.Errorf("Exchange: bidlables for %s (%s) missing adapter string", bidderRequest.BidderName, bidderRequest.BidderCoreName)
bidderRequest.BidderLabels.Adapter = bidderRequest.BidderCoreName
}
brw := new(bidResponseWrapper)
brw.bidder = aName
brw.bidder = bidderRequest.BidderName
// Defer basic metrics to insure we capture them after all the values have been set
defer func() {
e.me.RecordAdapterRequest(*bidlabels)
e.me.RecordAdapterRequest(bidderRequest.BidderLabels)
}()
start := time.Now()

adjustmentFactor := 1.0
if givenAdjustment, ok := bidAdjustments[string(aName)]; ok {
if givenAdjustment, ok := bidAdjustments[string(bidderRequest.BidderName)]; ok {
adjustmentFactor = givenAdjustment
}
var reqInfo adapters.ExtraRequestInfo
reqInfo.PbsEntryPoint = bidlabels.RType
bids, err := e.adapterMap[coreBidder].requestBid(ctx, request, aName, adjustmentFactor, conversions, &reqInfo)
reqInfo.PbsEntryPoint = bidderRequest.BidderLabels.RType
bids, err := e.adapterMap[bidderRequest.BidderCoreName].requestBid(ctx, bidderRequest.BidRequest, bidderRequest.BidderName, adjustmentFactor, conversions, &reqInfo)

// Add in time reporting
elapsed := time.Since(start)
Expand All @@ -365,26 +378,26 @@ func (e *exchange) getAllBids(ctx context.Context, cleanRequests map[openrtb_ext
}

// Timing statistics
e.me.RecordAdapterTime(*bidlabels, time.Since(start))
e.me.RecordAdapterTime(bidderRequest.BidderLabels, time.Since(start))
serr := errsToBidderErrors(err)
bidlabels.AdapterBids = bidsToMetric(brw.adapterBids)
bidlabels.AdapterErrors = errorsToMetric(err)
bidderRequest.BidderLabels.AdapterBids = bidsToMetric(brw.adapterBids)
bidderRequest.BidderLabels.AdapterErrors = errorsToMetric(err)
// Append any bid validation errors to the error list
ae.Errors = serr
brw.adapterExtra = ae
if bids != nil {
for _, bid := range bids.bids {
var cpm = float64(bid.bid.Price * 1000)
e.me.RecordAdapterPrice(*bidlabels, cpm)
e.me.RecordAdapterBidReceived(*bidlabels, bid.bidType, bid.bid.AdM != "")
e.me.RecordAdapterPrice(bidderRequest.BidderLabels, cpm)
e.me.RecordAdapterBidReceived(bidderRequest.BidderLabels, bid.bidType, bid.bid.AdM != "")
}
}
chBids <- brw
}, chBids)
go bidderRunner(bidderName, coreBidder, req, blabels[coreBidder], conversions)
go bidderRunner(bidder, conversions)
}
// Wait for the bidders to do their thing
for i := 0; i < len(cleanRequests); i++ {
for i := 0; i < len(bidderRequests); i++ {
brw := <-chBids

//if bidder returned no bids back - remove bidder from further processing
Expand All @@ -402,15 +415,17 @@ func (e *exchange) getAllBids(ctx context.Context, cleanRequests map[openrtb_ext
return adapterBids, adapterExtra, bidsFound
}

func (e *exchange) recoverSafely(cleanRequests map[openrtb_ext.BidderName]*openrtb.BidRequest, inner func(openrtb_ext.BidderName, openrtb_ext.BidderName, *openrtb.BidRequest, *pbsmetrics.AdapterLabels, currencies.Conversions), chBids chan *bidResponseWrapper) func(openrtb_ext.BidderName, openrtb_ext.BidderName, *openrtb.BidRequest, *pbsmetrics.AdapterLabels, currencies.Conversions) {
return func(aName openrtb_ext.BidderName, coreBidder openrtb_ext.BidderName, request *openrtb.BidRequest, bidlabels *pbsmetrics.AdapterLabels, conversions currencies.Conversions) {
func (e *exchange) recoverSafely(bidderRequests []BidderRequest,
inner func(BidderRequest, currencies.Conversions),
chBids chan *bidResponseWrapper) func(BidderRequest, currencies.Conversions) {
return func(bidderRequest BidderRequest, conversions currencies.Conversions) {
defer func() {
if r := recover(); r != nil {

allBidders := ""
sb := strings.Builder{}
for k := range cleanRequests {
sb.WriteString(string(k))
for _, bidder := range bidderRequests {
sb.WriteString(bidder.BidderName.String())
sb.WriteString(",")
}
if sb.Len() > 0 {
Expand All @@ -419,15 +434,15 @@ func (e *exchange) recoverSafely(cleanRequests map[openrtb_ext.BidderName]*openr

glog.Errorf("OpenRTB auction recovered panic from Bidder %s: %v. "+
"Account id: %s, All Bidders: %s, Stack trace is: %v",
coreBidder, r, bidlabels.PubID, allBidders, string(debug.Stack()))
e.me.RecordAdapterPanic(*bidlabels)
bidderRequest.BidderCoreName, r, bidderRequest.BidderLabels.PubID, allBidders, string(debug.Stack()))
e.me.RecordAdapterPanic(bidderRequest.BidderLabels)
// Let the master request know that there is no data here
brw := new(bidResponseWrapper)
brw.adapterExtra = new(seatResponseExtra)
chBids <- brw
}
}()
inner(aName, coreBidder, request, bidlabels, conversions)
inner(bidderRequest, conversions)
}
}

Expand Down Expand Up @@ -867,11 +882,11 @@ func buildCacheURL(cache prebid_cache_client.Client, uuid string) string {
return strings.TrimPrefix(cacheURL.String(), "//")
}

func listBiddersWithRequests(cleanRequests map[openrtb_ext.BidderName]*openrtb.BidRequest) []openrtb_ext.BidderName {
liveAdapters := make([]openrtb_ext.BidderName, len(cleanRequests))
func listBiddersWithRequests(bidderRequests []BidderRequest) []openrtb_ext.BidderName {
liveAdapters := make([]openrtb_ext.BidderName, len(bidders))
i := 0
for a := range cleanRequests {
liveAdapters[i] = a
for _, bidderRequest := range bidderRequests {
liveAdapters[i] = bidderRequest.BidderName
i++
}
// Randomize the list of adapters to make the auction more fair
Expand Down
34 changes: 23 additions & 11 deletions exchange/exchange_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1152,18 +1152,10 @@ func TestPanicRecovery(t *testing.T) {
e := NewExchange(adapters, nil, cfg, &metricsConf.DummyMetricsEngine{}, gdpr.AlwaysAllow{}, currencyConverter, nilCategoryFetcher{}).(*exchange)

chBids := make(chan *bidResponseWrapper, 1)
panicker := func(aName openrtb_ext.BidderName, coreBidder openrtb_ext.BidderName, request *openrtb.BidRequest, bidlabels *pbsmetrics.AdapterLabels, conversions currencies.Conversions) {
panicker := func(bidderRequest BidderRequest, conversions currencies.Conversions) {
panic("panic!")
}
cleanReqs := map[openrtb_ext.BidderName]*openrtb.BidRequest{
"bidder1": {
ID: "b-1",
},
"bidder2": {
ID: "b-2",
},
}
recovered := e.recoverSafely(cleanReqs, panicker, chBids)

apnLabels := pbsmetrics.AdapterLabels{
Source: pbsmetrics.DemandWeb,
RType: pbsmetrics.ReqTypeORTB2Web,
Expand All @@ -1172,7 +1164,27 @@ func TestPanicRecovery(t *testing.T) {
CookieFlag: pbsmetrics.CookieFlagYes,
AdapterBids: pbsmetrics.AdapterBidNone,
}
recovered(openrtb_ext.BidderAppnexus, openrtb_ext.BidderAppnexus, nil, &apnLabels, nil)

bidderRequests := []BidderRequest{
{
BidderName: "bidder1",
BidderCoreName: "appnexus",
BidderLabels: apnLabels,
BidRequest: &openrtb.BidRequest{
ID: "b-1",
},
},
{
BidderName: "bidder2",
BidderCoreName: "bidder2",
BidRequest: &openrtb.BidRequest{
ID: "b-2",
},
},
}

recovered := e.recoverSafely(bidderRequests, panicker, chBids)
recovered(bidderRequests[0], nil)
}

func buildImpExt(t *testing.T, jsonFilename string) json.RawMessage {
Expand Down
Loading