Core: Update TCF policy version validation (#3498)

src/main/java/org/prebid/server/privacy/gdpr/TcfDefinerService.java

@@ -66,14 +66,16 @@ public class TcfDefinerService {
     private final BidderCatalog bidderCatalog;
     private final IpAddressHelper ipAddressHelper;
     private final Metrics metrics;
+    private final double samplingRate;
 
     public TcfDefinerService(GdprConfig gdprConfig,
                              Set<String> eeaCountries,
                              Tcf2Service tcf2Service,
                              GeoLocationServiceWrapper geoLocationServiceWrapper,
                              BidderCatalog bidderCatalog,
                              IpAddressHelper ipAddressHelper,
-                             Metrics metrics) {
+                             Metrics metrics,
+                             double samplingRate) {
 
         this.gdprEnabled = gdprConfig != null && BooleanUtils.isNotFalse(gdprConfig.getEnabled());
         this.gdprDefaultValue = gdprConfig != null ? gdprConfig.getDefaultValue() : null;
@@ -85,6 +87,7 @@ public TcfDefinerService(GdprConfig gdprConfig,
         this.bidderCatalog = Objects.requireNonNull(bidderCatalog);
         this.ipAddressHelper = Objects.requireNonNull(ipAddressHelper);
         this.metrics = Objects.requireNonNull(metrics);
+        this.samplingRate = samplingRate;
     }
 
     /**
@@ -360,11 +363,14 @@ private TCStringParsingResult toValidResult(String consentString, TCStringParsin
         }
 
         final int tcfPolicyVersion = tcString.getTcfPolicyVersion();
-        // disable support for tcf policy version > 5
+        // support for tcf policy version > 5
         if (tcfPolicyVersion > 5) {
-            warnings.add("Parsing consent string: %s failed. TCF policy version %d is not supported".formatted(
-                    consentString, tcfPolicyVersion));
-            return TCStringParsingResult.of(TCStringEmpty.create(), warnings);
+            metrics.updateAlertsMetrics(MetricName.general);
+
+            final String message = "Unknown tcfPolicyVersion %s, defaulting to gvlSpecificationVersion=3"
+                    .formatted(tcfPolicyVersion);
+            UNDEFINED_CORRUPT_CONSENT_LOGGER.warn(message, samplingRate);
+            warnings.add(message);
         }
 
         return TCStringParsingResult.of(tcString, warnings);

src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java

@@ -2,7 +2,6 @@
 
 import com.iabtcf.decoder.TCString;
 import io.vertx.core.Future;
-import org.prebid.server.exception.PreBidException;
 import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor;
 
 import java.util.Map;
@@ -21,10 +20,6 @@ public VersionedVendorListService(VendorListService vendorListServiceV2, VendorL
     public Future<Map<Integer, Vendor>> forConsent(TCString consent) {
         final int tcfPolicyVersion = consent.getTcfPolicyVersion();
         final int vendorListVersion = consent.getVendorListVersion();
-        if (tcfPolicyVersion > 5) {
-            return Future.failedFuture(new PreBidException(
-                    "Invalid tcf policy version: %d".formatted(tcfPolicyVersion)));
-        }
 
         return tcfPolicyVersion < 4
                 ? vendorListServiceV2.forVersion(vendorListVersion)

src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java

@@ -164,7 +164,8 @@ TcfDefinerService tcfDefinerService(
             GeoLocationServiceWrapper geoLocationServiceWrapper,
             BidderCatalog bidderCatalog,
             IpAddressHelper ipAddressHelper,
-            Metrics metrics) {
+            Metrics metrics,
+            @Value("${logging.sampling-rate:0.01}") double samplingRate) {
 
         final Set<String> eeaCountries = new HashSet<>(Arrays.asList(eeaCountriesAsString.trim().split(",")));
 
@@ -175,7 +176,8 @@ TcfDefinerService tcfDefinerService(
                 geoLocationServiceWrapper,
                 bidderCatalog,
                 ipAddressHelper,
-                metrics);
+                metrics,
+                samplingRate);
     }
 
     @Bean

src/test/groovy/org/prebid/server/functional/testcontainers/scaffolding/VendorList.groovy

@@ -10,6 +10,8 @@ import org.testcontainers.containers.MockServerContainer
 import static org.mockserver.model.HttpRequest.request
 import static org.mockserver.model.HttpResponse.response
 import static org.mockserver.model.HttpStatusCode.OK_200
+import static org.prebid.server.functional.model.mock.services.vendorlist.GvlSpecificationVersion.V2
+import static org.prebid.server.functional.model.mock.services.vendorlist.GvlSpecificationVersion.V3
 import static org.prebid.server.functional.model.mock.services.vendorlist.VendorListResponse.Vendor
 import static org.prebid.server.functional.model.mock.services.vendorlist.VendorListResponse.getDefaultVendorListResponse
 import static org.prebid.server.functional.util.privacy.TcfConsent.GENERIC_VENDOR_ID
@@ -46,6 +48,7 @@ class VendorList extends NetworkScaffolding {
         def prepareEncodeResponseBody = encode(defaultVendorListResponse.tap {
             it.tcfPolicyVersion = tcfPolicyVersion.vendorListVersion
             it.vendors = vendors
+            it.gvlSpecificationVersion = tcfPolicyVersion >= TcfPolicyVersion.TCF_POLICY_V4 ? V3 : V2
         })
 
         mockServerClient.when(request().withPath(prepareEndpoint), Times.unlimited(), TimeToLive.unlimited(), -10)

src/test/groovy/org/prebid/server/functional/tests/privacy/GdprAmpSpec.groovy

@@ -26,6 +26,7 @@ import static org.prebid.server.functional.model.config.Purpose.P2
 import static org.prebid.server.functional.model.config.Purpose.P4
 import static org.prebid.server.functional.model.config.PurposeEnforcement.BASIC
 import static org.prebid.server.functional.model.config.PurposeEnforcement.NO
+import static org.prebid.server.functional.model.mock.services.vendorlist.GvlSpecificationVersion.V3
 import static org.prebid.server.functional.model.privacy.Metric.TEMPLATE_ADAPTER_DISALLOWED_COUNT
 import static org.prebid.server.functional.model.privacy.Metric.TEMPLATE_REQUEST_DISALLOWED_COUNT
 import static org.prebid.server.functional.model.request.amp.ConsentType.BOGUS
@@ -360,11 +361,12 @@ class GdprAmpSpec extends PrivacyBaseSpec {
         tcfPolicyVersion << [TCF_POLICY_V2, TCF_POLICY_V4, TCF_POLICY_V5]
     }
 
-    def "PBS amp with invalid consent.tcfPolicyVersion parameter should reject request and include proper warning"() {
-        given: "Tcf consent string"
-        def invalidTcfPolicyVersion = PBSUtils.getRandomNumber(5, 63)
+    def "PBS amp shouldn't reject request with proper warning and metrics when incoming consent.tcfPolicyVersion have invalid parameter"() {
+        given: "Tcf consent string with invalid tcf policy version"
         def tcfConsent = new TcfConsent.Builder()
+                .setPurposesLITransparency(BASIC_ADS)
                 .setTcfPolicyVersion(invalidTcfPolicyVersion)
+                .setVendorLegitimateInterest([GENERIC_VENDOR_ID])
                 .build()
 
         and: "AMP request"
@@ -377,13 +379,28 @@ class GdprAmpSpec extends PrivacyBaseSpec {
         def storedRequest = StoredRequest.getStoredRequest(ampRequest, ampStoredRequest)
         storedRequestDao.save(storedRequest)
 
+        and: "Flush metrics"
+        flushMetrics(privacyPbsService)
+
         when: "PBS processes amp request"
         def response = privacyPbsService.sendAmpRequest(ampRequest)
 
         then: "Bid response should contain warning"
         assert response.ext?.warnings[PREBID]*.code == [999]
         assert response.ext?.warnings[PREBID]*.message ==
-                ["Parsing consent string: ${tcfConsent} failed. TCF policy version ${invalidTcfPolicyVersion} is not supported" as String]
+                ["Unknown tcfPolicyVersion ${invalidTcfPolicyVersion}, defaulting to gvlSpecificationVersion=3" as String]
+
+        and: "Alerts.general metrics should be populated"
+        def metrics = privacyPbsService.sendCollectedMetricsRequest()
+        assert metrics["alerts.general"] == 1
+
+        and: "Bidder should be called"
+        assert bidder.getBidderRequest(ampStoredRequest.id)
+
+        where:
+        invalidTcfPolicyVersion << [MIN_INVALID_TCF_POLICY_VERSION,
+                                    PBSUtils.getRandomNumber(MIN_INVALID_TCF_POLICY_VERSION, MAX_INVALID_TCF_POLICY_VERSION),
+                                    MAX_INVALID_TCF_POLICY_VERSION]
     }
 
     def "PBS amp should emit the same error without a second GVL list request if a retry is too soon for the exponential-backoff"() {
@@ -634,4 +651,39 @@ class GdprAmpSpec extends PrivacyBaseSpec {
         assert !metrics[TEMPLATE_REQUEST_DISALLOWED_COUNT.getValue(ampStoredRequest, TRANSMIT_EIDS)]
         assert !metrics[TEMPLATE_REQUEST_DISALLOWED_COUNT.getValue(ampStoredRequest, TRANSMIT_PRECISE_GEO)]
     }
+
+    def "PBS amp should set 3 for tcfPolicyVersion when tcfPolicyVersion is #tcfPolicyVersion"() {
+        given: "Tcf consent setup"
+        def tcfConsent = new TcfConsent.Builder()
+                .setPurposesLITransparency(BASIC_ADS)
+                .setTcfPolicyVersion(tcfPolicyVersion.value)
+                .setVendorListVersion(tcfPolicyVersion.vendorListVersion)
+                .setVendorLegitimateInterest([GENERIC_VENDOR_ID])
+                .build()
+
+        and: "AMP request"
+        def ampRequest = getGdprAmpRequest(tcfConsent)
+
+        and: "Default stored request"
+        def ampStoredRequest = BidRequest.defaultStoredRequest
+
+        and: "Stored request in DB"
+        def storedRequest = StoredRequest.getStoredRequest(ampRequest, ampStoredRequest)
+        storedRequestDao.save(storedRequest)
+
+        and: "Set vendor list response"
+        vendorListResponse.setResponse(tcfPolicyVersion)
+
+        when: "PBS processes amp request"
+        privacyPbsService.sendAmpRequest(ampRequest)
+
+        then: "Used vendor list have proper specification version of GVL"
+        def properVendorListPath = VENDOR_LIST_PATH.replace("{VendorVersion}", tcfPolicyVersion.vendorListVersion.toString())
+        PBSUtils.waitUntil { privacyPbsService.isFileExist(properVendorListPath) }
+        def vendorList = privacyPbsService.getValueFromContainer(properVendorListPath, VendorListConsent.class)
+        assert vendorList.gvlSpecificationVersion == V3
+
+        where:
+        tcfPolicyVersion << [TCF_POLICY_V4, TCF_POLICY_V5]
+    }
 }

src/test/groovy/org/prebid/server/functional/tests/privacy/GdprAuctionSpec.groovy

@@ -6,7 +6,6 @@ import org.prebid.server.functional.model.config.AccountGdprConfig
 import org.prebid.server.functional.model.config.PurposeConfig
 import org.prebid.server.functional.model.config.PurposeEnforcement
 import org.prebid.server.functional.model.request.auction.DistributionChannel
-import org.prebid.server.functional.model.request.auction.RegsExt
 import org.prebid.server.functional.model.response.auction.ErrorType
 import org.prebid.server.functional.service.PrebidServerService
 import org.prebid.server.functional.testcontainers.container.PrebidServerContainer
@@ -25,6 +24,7 @@ import static org.prebid.server.functional.model.config.Purpose.P1
 import static org.prebid.server.functional.model.config.Purpose.P2
 import static org.prebid.server.functional.model.config.Purpose.P4
 import static org.prebid.server.functional.model.config.PurposeEnforcement.NO
+import static org.prebid.server.functional.model.mock.services.vendorlist.GvlSpecificationVersion.V3
 import static org.prebid.server.functional.model.pricefloors.Country.BULGARIA
 import static org.prebid.server.functional.model.pricefloors.Country.CAN
 import static org.prebid.server.functional.model.pricefloors.Country.USA
@@ -314,11 +314,12 @@ class GdprAuctionSpec extends PrivacyBaseSpec {
         tcfPolicyVersion << [TCF_POLICY_V2, TCF_POLICY_V4, TCF_POLICY_V5]
     }
 
-    def "PBS auction should reject request with proper warning when incoming consent.tcfPolicyVersion have invalid parameter"() {
-        given: "Tcf consent string"
-        def invalidTcfPolicyVersion = PBSUtils.getRandomNumber(5, 63)
+    def "PBS auction shouldn't reject request with proper warning and metrics when incoming consent.tcfPolicyVersion have invalid parameter"() {
+        given: "Tcf consent string with invalid tcf policy version"
         def tcfConsent = new TcfConsent.Builder()
+                .setPurposesLITransparency(BASIC_ADS)
                 .setTcfPolicyVersion(invalidTcfPolicyVersion)
+                .setVendorLegitimateInterest([GENERIC_VENDOR_ID])
                 .build()
 
         and: "Bid request"
@@ -333,7 +334,22 @@ class GdprAuctionSpec extends PrivacyBaseSpec {
         then: "Bid response should contain warning"
         assert response.ext?.warnings[ErrorType.PREBID]*.code == [999]
         assert response.ext?.warnings[ErrorType.PREBID]*.message ==
-                ["Parsing consent string: ${tcfConsent} failed. TCF policy version ${invalidTcfPolicyVersion} is not supported" as String]
+                ["Unknown tcfPolicyVersion ${invalidTcfPolicyVersion}, defaulting to gvlSpecificationVersion=3" as String]
+
+        and: "Alerts.general metrics should be populated"
+        def metrics = privacyPbsService.sendCollectedMetricsRequest()
+        assert metrics["alerts.general"] == 1
+
+        and: "Bid response should contain seatBid"
+        assert response.seatbid.size() == 1
+
+        and: "Bidder should be called"
+        assert bidder.getBidderRequest(bidRequest.id)
+
+        where:
+        invalidTcfPolicyVersion << [MIN_INVALID_TCF_POLICY_VERSION,
+                                    PBSUtils.getRandomNumber(MIN_INVALID_TCF_POLICY_VERSION, MAX_INVALID_TCF_POLICY_VERSION),
+                                    MAX_INVALID_TCF_POLICY_VERSION]
     }
 
     def "PBS auction should emit the same error without a second GVL list request if a retry is too soon for the exponential-backoff"() {
@@ -761,4 +777,32 @@ class GdprAuctionSpec extends PrivacyBaseSpec {
         assert !metrics[TEMPLATE_REQUEST_DISALLOWED_COUNT.getValue(bidRequest, TRANSMIT_EIDS)]
         assert !metrics[TEMPLATE_REQUEST_DISALLOWED_COUNT.getValue(bidRequest, TRANSMIT_PRECISE_GEO)]
     }
+
+    def "PBS auction should set 3 for tcfPolicyVersion when tcfPolicyVersion is #tcfPolicyVersion"() {
+        given: "Tcf consent setup"
+        def tcfConsent = new TcfConsent.Builder()
+                .setPurposesLITransparency(BASIC_ADS)
+                .setTcfPolicyVersion(tcfPolicyVersion.value)
+                .setVendorListVersion(tcfPolicyVersion.vendorListVersion)
+                .setVendorLegitimateInterest([GENERIC_VENDOR_ID])
+                .build()
+
+        and: "Bid request"
+        def bidRequest = getGdprBidRequest(tcfConsent)
+
+        and: "Set vendor list response"
+        vendorListResponse.setResponse(tcfPolicyVersion)
+
+        when: "PBS processes auction request"
+        privacyPbsService.sendAuctionRequest(bidRequest)
+
+        then: "Used vendor list have proper specification version of GVL"
+        def properVendorListPath = VENDOR_LIST_PATH.replace("{VendorVersion}", tcfPolicyVersion.vendorListVersion.toString())
+        PBSUtils.waitUntil { privacyPbsService.isFileExist(properVendorListPath) }
+        def vendorList = privacyPbsService.getValueFromContainer(properVendorListPath, VendorListConsent.class)
+        assert vendorList.gvlSpecificationVersion == V3
+
+        where:
+        tcfPolicyVersion << [TCF_POLICY_V4, TCF_POLICY_V5]
+    }
 }

src/test/groovy/org/prebid/server/functional/tests/privacy/PrivacyBaseSpec.groovy

@@ -103,6 +103,8 @@ abstract class PrivacyBaseSpec extends BaseSpec {
     protected static final String VALID_VALUE_FOR_GPC_HEADER = "1"
     protected static final GppConsent SIMPLE_GPC_DISALLOW_LOGIC = new UsNatV1Consent.Builder().setGpc(true).build()
     protected static final VendorList vendorListResponse = new VendorList(networkServiceContainer)
+    protected static final Integer MAX_INVALID_TCF_POLICY_VERSION = 63
+    protected static final Integer MIN_INVALID_TCF_POLICY_VERSION = 6
 
     @Shared
     protected final PrebidServerService privacyPbsService = pbsServiceFactory.getService(GDPR_VENDOR_LIST_CONFIG +

src/test/groovy/org/prebid/server/functional/util/privacy/VendorListConsent.groovy

@@ -1,11 +1,12 @@
 package org.prebid.server.functional.util.privacy
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import org.prebid.server.functional.model.mock.services.vendorlist.GvlSpecificationVersion
 
 @JsonIgnoreProperties(ignoreUnknown = true)
 class VendorListConsent {
 
     Integer vendorListVersion
     Integer tcfPolicyVersion
-    Integer gvlSpecificationVersion
+    GvlSpecificationVersion gvlSpecificationVersion
 }