From 2606e7529f36aca8e645fc38814ba9924baba8a9 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 16 Aug 2024 14:20:22 +0100 Subject: [PATCH] New Module: 51Degrees (#3650) Co-authored-by: James Rosewell Co-authored-by: Marin Miletic Co-authored-by: Sarana-Anna Co-authored-by: Eugene Dorfman Co-authored-by: Krasilchuk Yaroslav --- go.mod | 5 + go.sum | 11 + modules/builder.go | 4 + .../fiftyonedegrees/devicedetection/README.md | 255 +++++++ .../devicedetection/account_info_extractor.go | 37 + .../account_info_extractor_test.go | 74 ++ .../devicedetection/account_validator.go | 28 + .../devicedetection/account_validator_test.go | 71 ++ .../fiftyonedegrees/devicedetection/config.go | 80 ++ .../devicedetection/config_test.go | 119 +++ .../devicedetection/context.go | 8 + .../devicedetection/device_detector.go | 157 ++++ .../devicedetection/device_detector_test.go | 190 +++++ .../devicedetection/device_info_extractor.go | 121 +++ .../device_info_extractor_test.go | 130 ++++ .../devicedetection/evidence_extractor.go | 118 +++ .../evidence_extractor_test.go | 256 +++++++ .../devicedetection/fiftyone_device_types.go | 77 ++ .../fiftyone_device_types_test.go | 90 +++ .../hook_auction_entrypoint.go | 27 + .../hook_raw_auction_request.go | 173 +++++ .../fiftyonedegrees/devicedetection/models.go | 66 ++ .../devicedetection/models_test.go | 63 ++ .../fiftyonedegrees/devicedetection/module.go | 107 +++ .../devicedetection/module_test.go | 703 ++++++++++++++++++ .../request_headers_extractor.go | 47 ++ .../request_headers_extractor_test.go | 118 +++ .../devicedetection/sample/pbs.json | 84 +++ .../devicedetection/sample/request_data.json | 114 +++ .../devicedetection/sua_payload_extractor.go | 144 ++++ 30 files changed, 3477 insertions(+) create mode 100644 modules/fiftyonedegrees/devicedetection/README.md create mode 100644 modules/fiftyonedegrees/devicedetection/account_info_extractor.go create mode 100644 modules/fiftyonedegrees/devicedetection/account_info_extractor_test.go create mode 100644 modules/fiftyonedegrees/devicedetection/account_validator.go create mode 100644 modules/fiftyonedegrees/devicedetection/account_validator_test.go create mode 100644 modules/fiftyonedegrees/devicedetection/config.go create mode 100644 modules/fiftyonedegrees/devicedetection/config_test.go create mode 100644 modules/fiftyonedegrees/devicedetection/context.go create mode 100644 modules/fiftyonedegrees/devicedetection/device_detector.go create mode 100644 modules/fiftyonedegrees/devicedetection/device_detector_test.go create mode 100644 modules/fiftyonedegrees/devicedetection/device_info_extractor.go create mode 100644 modules/fiftyonedegrees/devicedetection/device_info_extractor_test.go create mode 100644 modules/fiftyonedegrees/devicedetection/evidence_extractor.go create mode 100644 modules/fiftyonedegrees/devicedetection/evidence_extractor_test.go create mode 100644 modules/fiftyonedegrees/devicedetection/fiftyone_device_types.go create mode 100644 modules/fiftyonedegrees/devicedetection/fiftyone_device_types_test.go create mode 100644 modules/fiftyonedegrees/devicedetection/hook_auction_entrypoint.go create mode 100644 modules/fiftyonedegrees/devicedetection/hook_raw_auction_request.go create mode 100644 modules/fiftyonedegrees/devicedetection/models.go create mode 100644 modules/fiftyonedegrees/devicedetection/models_test.go create mode 100644 modules/fiftyonedegrees/devicedetection/module.go create mode 100644 modules/fiftyonedegrees/devicedetection/module_test.go create mode 100644 modules/fiftyonedegrees/devicedetection/request_headers_extractor.go create mode 100644 modules/fiftyonedegrees/devicedetection/request_headers_extractor_test.go create mode 100644 modules/fiftyonedegrees/devicedetection/sample/pbs.json create mode 100644 modules/fiftyonedegrees/devicedetection/sample/request_data.json create mode 100644 modules/fiftyonedegrees/devicedetection/sua_payload_extractor.go diff --git a/go.mod b/go.mod index c16acc331df..423abe2438e 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( ) require ( + github.com/51Degrees/device-detection-go/v4 v4.4.35 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -67,6 +68,10 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.3.0 // indirect + github.com/tidwall/gjson v1.17.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/sjson v1.2.5 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect diff --git a/go.sum b/go.sum index 00827d6fb6c..d62ceefeefb 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/51Degrees/device-detection-go/v4 v4.4.35 h1:qhP2tzoXhGE1aYY3NftMJ+ccxz0+2kM8aF4SH7fTyuA= +github.com/51Degrees/device-detection-go/v4 v4.4.35/go.mod h1:dbdG1fySqdY+a5pUnZ0/G0eD03G6H3Vh8kRC+1f9qSc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= @@ -483,6 +485,15 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/vrischmann/go-metrics-influxdb v0.1.1 h1:xneKFRjsS4BiVYvAKaM/rOlXYd1pGHksnES0ECCJLgo= github.com/vrischmann/go-metrics-influxdb v0.1.1/go.mod h1:q7YC8bFETCYopXRMtUvQQdLaoVhpsEwvQS2zZEYCqg8= diff --git a/modules/builder.go b/modules/builder.go index e5d04e149af..36ac5589add 100644 --- a/modules/builder.go +++ b/modules/builder.go @@ -1,6 +1,7 @@ package modules import ( + fiftyonedegreesDevicedetection "github.com/prebid/prebid-server/v2/modules/fiftyonedegrees/devicedetection" prebidOrtb2blocking "github.com/prebid/prebid-server/v2/modules/prebid/ortb2blocking" ) @@ -8,6 +9,9 @@ import ( // vendor and module names are chosen based on the module directory name func builders() ModuleBuilders { return ModuleBuilders{ + "fiftyonedegrees": { + "devicedetection": fiftyonedegreesDevicedetection.Builder, + }, "prebid": { "ortb2blocking": prebidOrtb2blocking.Builder, }, diff --git a/modules/fiftyonedegrees/devicedetection/README.md b/modules/fiftyonedegrees/devicedetection/README.md new file mode 100644 index 00000000000..645fb407fe5 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/README.md @@ -0,0 +1,255 @@ +## Overview + +The 51Degrees module enriches an incoming OpenRTB request with [51Degrees Device Data](https://51degrees.com/documentation/_device_detection__overview.html). + +The module sets the following fields of the device object: `make`, `model`, `os`, `osv`, `h`, `w`, `ppi`, `pxratio` - interested bidder adapters may use these fields as needed. In addition the module sets `device.ext.fiftyonedegrees_deviceId` to a permanent device ID which can be rapidly looked up in on premise data exposing over 250 properties including the device age, chip set, codec support, and price, operating system and app/browser versions, age, and embedded features. + +## Operation Details + +### Evidence + +The module uses `device.ua` (User Agent) and `device.sua` (Structured User Agent) provided in the oRTB request payload as input (or 'evidence' in 51Degrees terminology). There is a fallback to the corresponding HTTP request headers if any of these are not present in the oRTB payload - in particular: `User-Agent` and `Sec-CH-UA-*` (aka User-Agent Client Hints). To make sure Prebid.js sends Structured User Agent in the oRTB payload - we strongly advice publishers to enable [First Party Data Enrichment module](dev-docs/modules/enrichmentFpdModule.html) for their wrappers and specify + +```js +pbjs.setConfig({ + firstPartyData: { + uaHints: [ + 'architecture', + 'model', + 'platform', + 'platformVersion', + 'fullVersionList', + ] + } +}) +``` + +### Data File Updates + +The module operates **fully autonomously and does not make any requests to any cloud services in real time to do device detection**. This is an [on-premise data](https://51degrees.com/developers/deployment-options/on-premise-data) deployment in 51Degrees terminology. The module operates using a local data file that is loaded into memory fully or partially during operation. The data file is occasionally updated to accomodate new devices, so it is recommended to enable automatic data updates in the module configuration. Alternatively `watch_file_system` option can be used and the file may be downloaded and replaced on disk manually. See the configuration options below. + +## Setup + +The 51Degrees module operates using a data file. You can get started with a free Lite data file that can be downloaded here: [51Degrees-LiteV4.1.hash](https://github.com/51Degrees/device-detection-data/blob/main/51Degrees-LiteV4.1.hash). The Lite file is capable of detecting limited device information, so if you need in-depth device data, please contact 51Degrees to obtain a license: [https://51degrees.com/contact-us](https://51degrees.com/contact-us?ContactReason=Free%20Trial). + +Put the data file in a file system location writable by the system account that is running the Prebid Server module and specify that directory location in the configuration parameters. The location needs to be writable if you would like to enable [automatic data file updates](https://51degrees.com/documentation/_features__automatic_datafile_updates.html). + +### Execution Plan + +This module supports running at two stages: + +* entrypoint: this is where incoming requests are parsed and device detection evidences are extracted. +* raw-auction-request: this is where outgoing auction requests to each bidder are enriched with the device detection data + +We recommend defining the execution plan right in the account config +so the module is only invoked for specific accounts. See below for an example. + +### Global Config + +There is no host-company level config for this module. + +### Account-Level Config + +To start using current module in PBS-Go you have to enable module and add `fiftyone-devicedetection-entrypoint-hook` and `fiftyone-devicedetection-raw-auction-request-hook` into hooks execution plan inside your config file: +Here's a general template for the account config used in PBS-Go: + +```json +{ + "hooks": { + "enabled":true, + "modules": { + "fiftyonedegrees": { + "devicedetection": { + "enabled": true, + "make_temp_copy": true, + "data_file": { + "path": "path/to/51Degrees-LiteV4.1.hash", + "update": { + "auto": true, + "url": "", + "polling_interval": 1800, + "license_key": "", + "product": "V4Enterprise", + "watch_file_system": "true", + "on_startup": true + } + } + } + }, + "host_execution_plan": { + "endpoints": { + "/openrtb2/auction": { + "stages": { + "entrypoint": { + "groups": [ + { + "timeout": 10, + "hook_sequence": [ + { + "module_code": "fiftyonedegrees.devicedetection", + "hook_impl_code": "fiftyone-devicedetection-entrypoint-hook" + } + ] + } + ] + }, + "raw_auction_request": { + "groups": [ + { + "timeout": 10, + "hook_sequence": [ + { + "module_code": "fiftyonedegrees.devicedetection", + "hook_impl_code": "fiftyone-devicedetection-raw-auction-request-hook" + } + ] + } + ] + } + } + } + } + } + } + } +} +``` + +The same config in YAML format: +```yaml +hooks: + enabled: true + modules: + fiftyonedegrees: + devicedetection: + enabled: true + make_temp_copy: true + data_file: + path: path/to/51Degrees-LiteV4.1.hash + update: + auto: true + url: "" + polling_interval: 1800 + license_key: "" + product: V4Enterprise + watch_file_system: 'true' + host_execution_plan: + endpoints: + "/openrtb2/auction": + stages: + entrypoint: + groups: + - timeout: 10 + hook_sequence: + - module_code: fiftyonedegrees.devicedetection + hook_impl_code: fiftyone-devicedetection-entrypoint-hook + raw_auction_request: + groups: + - timeout: 10 + hook_sequence: + - module_code: fiftyonedegrees.devicedetection + hook_impl_code: fiftyone-devicedetection-raw-auction-request-hook +``` + +Note that at a minimum (besides adding to the host_execution_plan) you need to enable the module and specify a path to the data file in the configuration. +Sample module enablement configuration in JSON and YAML formats: + +```json +{ + "modules": { + "fiftyonedegrees": { + "devicedetection": { + "enabled": true, + "data_file": { + "path": "path/to/51Degrees-LiteV4.1.hash" + } + } + } + } +} +``` + +```yaml + modules: + fiftyonedegrees: + devicedetection: + enabled: true + data_file: + path: "/path/to/51Degrees-LiteV4.1.hash" +``` + +## Module Configuration Parameters + +The parameter names are specified with full path using dot-notation. F.e. `section_name` .`sub_section` .`param_name` would result in this nesting in the JSON configuration: + +```json +{ + "section_name": { + "sub_section": { + "param_name": "param-value" + } + } +} +``` + +| Param Name | Required| Type | Default value | Description | +|:-------|:------|:------|:------|:---------------------------------------| +| `account_filter` .`allow_list` | No | list of strings | [] (empty list) | A list of account IDs that are allowed to use this module - only relevant if enabled globally for the host. If empty, all accounts are allowed. Full-string match is performed (whitespaces and capitalization matter). | +| `data_file` .`path` | **Yes** | string | null |The full path to the device detection data file. Sample file can be downloaded from [data repo on GitHub](https://github.com/51Degrees/device-detection-data/blob/main/51Degrees-LiteV4.1.hash), or get an Enterprise data file [here](https://51degrees.com/pricing). | +| `data_file` .`make_temp_copy` | No | boolean | true | If true, the engine will create a temporary copy of the data file rather than using the data file directly. | +| `data_file` .`update` .`auto` | No | boolean | true | If enabled, the engine will periodically (at predefined time intervals - see `polling-interval` parameter) check if new data file is available. When the new data file is available engine downloads it and switches to it for device detection. If custom `url` is not specified `license_key` param is required. | +| `data_file` .`update` .`on_startup` | No | boolean | false | If enabled, engine will check for the updated data file right away without waiting for the defined time interval. | +| `data_file` .`update` .`url` | No | string | null | Configure the engine to check the specified URL for the availability of the updated data file. If not specified the [51Degrees distributor service](https://51degrees.com/documentation/4.4/_info__distributor.html) URL will be used, which requires a License Key. | +| `data_file` .`update` .`license_key` | No | string | null | Required if `auto` is true and custom `url` is not specified. Allows to download the data file from the [51Degrees distributor service](https://51degrees.com/documentation/4.4/_info__distributor.html). | +| `data_file` .`update` .`watch_file_system` | No | boolean | true | If enabled the engine will watch the data file path for any changes, and automatically reload the data file from disk once it is updated. | +| `data_file` .`update` .`polling_interval` | No | int | 1800 | The time interval in seconds between consequent attempts to download an updated data file. Default = 1800 seconds = 30 minutes. | +| `data_file` .`update` .`product`| No | string | `V4Enterprise` | Set the Product used when checking for new device detection data files. A Product is exclusive to the 51Degrees paid service. Please see options [here](https://51degrees.com/documentation/_info__distributor.html). | +| `performance` .`profile` | No | string | `Balanced` | `performance.*` parameters are related to the tradeoffs between speed of device detection and RAM consumption or accuracy. `profile` dictates the proportion between the use of the RAM (the more RAM used - the faster is the device detection) and reads from disk (less RAM but slower device detection). Must be one of: `LowMemory`, `MaxPerformance`, `HighPerformance`, `Balanced`, `BalancedTemp`, `InMemory`. Defaults to `Balanced`. | +| `performance` .`concurrency` | No | int | 10 | Specify the expected number of concurrent operations that engine does. This sets the concurrency of the internal caches to avoid excessive locking. Default: 10. | +| `performance` .`difference` | No | int | 0 | Set the maximum difference to allow when processing evidence (HTTP headers). The meaning is the difference in hash value between the hash that was found, and the hash that is being searched for. By default this is 0. For more information see [51Degrees documentation](https://51degrees.com/documentation/_device_detection__hash.html). | +| `performance` .`drift` | No | int | 0 | Set the maximum drift to allow when matching hashes. If the drift is exceeded, the result is considered invalid and values will not be returned. By default this is 0. For more information see [51Degrees documentation](https://51degrees.com/documentation/_device_detection__hash.html). | +| `performance` .`allow_unmatched` | No | boolean | false | If set to false, a non-matching evidence will result in properties with no values set. If set to true, a non-matching evidence will cause the 'default profiles' to be returned. This means that properties will always have values (i.e. no need to check .hasValue) but some may be inaccurate. By default, this is false. | + +## Running the demo + +1. Download dependencies: +```bash +go mod download +``` + +2. Replace the original config file `pbs.json` (placed in the repository root or in `/etc/config`) with the sample [config file](sample/pbs.json): +``` +cp modules/fiftyonedegrees/devicedetection/sample/pbs.json pbs.json +``` + +3. Download `51Degrees-LiteV4.1.hash` from [[GitHub](https://github.com/51Degrees/device-detection-data/blob/main/51Degrees-LiteV4.1.hash)] and put it in the project root directory. + +```bash +curl -o 51Degrees-LiteV4.1.hash -L https://github.com/51Degrees/device-detection-data/raw/main/51Degrees-LiteV4.1.hash +``` + +4. Create a directory for sample stored requests (needed for the server to run): +```bash +mkdir -p sample/stored +``` + +5. Start the server: +```bash +go run main.go +``` + +6. Run sample request: +```bash +curl \ +--header "Content-Type: application/json" \ +http://localhost:8000/openrtb2/auction \ +--data @modules/fiftyonedegrees/devicedetection/sample/request_data.json +``` + +7. Observe the `device` object get enriched with `devicetype`, `os`, `osv`, `w`, `h` and `ext.fiftyonedegrees_deviceId`. + +## Maintainer contacts + +Any suggestions or questions can be directed to [support@51degrees.com](support@51degrees.com) e-mail. + +Or just open new [issue](https://github.com/prebid/prebid-server/issues/new) or [pull request](https://github.com/prebid/prebid-server/pulls) in this repository. diff --git a/modules/fiftyonedegrees/devicedetection/account_info_extractor.go b/modules/fiftyonedegrees/devicedetection/account_info_extractor.go new file mode 100644 index 00000000000..2a5168cfe0c --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/account_info_extractor.go @@ -0,0 +1,37 @@ +package devicedetection + +import ( + "github.com/tidwall/gjson" +) + +type accountInfo struct { + Id string +} + +type accountInfoExtractor struct{} + +func newAccountInfoExtractor() accountInfoExtractor { + return accountInfoExtractor{} +} + +// extract extracts the account information from the payload +// The account information is extracted from the publisher id or site publisher id +func (x accountInfoExtractor) extract(payload []byte) *accountInfo { + if payload == nil { + return nil + } + + publisherResult := gjson.GetBytes(payload, "app.publisher.id") + if publisherResult.Exists() { + return &accountInfo{ + Id: publisherResult.String(), + } + } + publisherResult = gjson.GetBytes(payload, "site.publisher.id") + if publisherResult.Exists() { + return &accountInfo{ + Id: publisherResult.String(), + } + } + return nil +} diff --git a/modules/fiftyonedegrees/devicedetection/account_info_extractor_test.go b/modules/fiftyonedegrees/devicedetection/account_info_extractor_test.go new file mode 100644 index 00000000000..2d32f7915b5 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/account_info_extractor_test.go @@ -0,0 +1,74 @@ +package devicedetection + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var ( + siteRequestPayload = []byte(` + { + "site": { + "publisher": { + "id": "p-bid-config-test-005" + } + } + } + `) + + mobileRequestPayload = []byte(` + { + "app": { + "publisher": { + "id": "p-bid-config-test-005" + } + } + } + `) + + emptyPayload = []byte(`{}`) +) + +func TestPublisherIdExtraction(t *testing.T) { + tests := []struct { + name string + payload []byte + expected string + expectNil bool + }{ + { + name: "SiteRequest", + payload: siteRequestPayload, + expected: "p-bid-config-test-005", + }, + { + name: "MobileRequest", + payload: mobileRequestPayload, + expected: "p-bid-config-test-005", + }, + { + name: "EmptyPublisherId", + payload: emptyPayload, + expectNil: true, + }, + { + name: "EmptyPayload", + payload: nil, + expectNil: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + extractor := newAccountInfoExtractor() + accountInfo := extractor.extract(tt.payload) + + if tt.expectNil { + assert.Nil(t, accountInfo) + } else { + assert.Equal(t, tt.expected, accountInfo.Id) + } + }) + } +} diff --git a/modules/fiftyonedegrees/devicedetection/account_validator.go b/modules/fiftyonedegrees/devicedetection/account_validator.go new file mode 100644 index 00000000000..fdff92531a7 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/account_validator.go @@ -0,0 +1,28 @@ +package devicedetection + +import "slices" + +// defaultAccountValidator is a struct that contains an accountInfoExtractor +// and is used to validate if an account is allowed +type defaultAccountValidator struct { + AccountExtractor accountInfoExtractor +} + +func newAccountValidator() *defaultAccountValidator { + return &defaultAccountValidator{ + AccountExtractor: newAccountInfoExtractor(), + } +} + +func (x defaultAccountValidator) isAllowed(cfg config, req []byte) bool { + if len(cfg.AccountFilter.AllowList) == 0 { + return true + } + + accountInfo := x.AccountExtractor.extract(req) + if accountInfo != nil && slices.Contains(cfg.AccountFilter.AllowList, accountInfo.Id) { + return true + } + + return false +} diff --git a/modules/fiftyonedegrees/devicedetection/account_validator_test.go b/modules/fiftyonedegrees/devicedetection/account_validator_test.go new file mode 100644 index 00000000000..25f99e3b796 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/account_validator_test.go @@ -0,0 +1,71 @@ +package devicedetection + +import ( + "encoding/json" + "testing" + + "github.com/prebid/openrtb/v20/openrtb2" + + "github.com/stretchr/testify/assert" +) + +func TestIsAllowed(t *testing.T) { + tests := []struct { + name string + allowList []string + expectedResult bool + }{ + { + name: "allowed", + allowList: []string{"1001"}, + expectedResult: true, + }, + { + name: "empty", + allowList: []string{}, + expectedResult: true, + }, + { + name: "disallowed", + allowList: []string{"1002"}, + expectedResult: false, + }, + { + name: "allow_list_is_nil", + allowList: nil, + expectedResult: true, + }, + { + name: "allow_list_contains_multiple", + allowList: []string{"1000", "1001", "1002"}, + expectedResult: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + validator := newAccountValidator() + cfg := config{ + AccountFilter: accountFilter{AllowList: test.allowList}, + } + + res := validator.isAllowed( + cfg, toBytes( + &openrtb2.BidRequest{ + App: &openrtb2.App{ + Publisher: &openrtb2.Publisher{ + ID: "1001", + }, + }, + }, + ), + ) + assert.Equal(t, test.expectedResult, res) + }) + } +} + +func toBytes(v interface{}) []byte { + res, _ := json.Marshal(v) + return res +} diff --git a/modules/fiftyonedegrees/devicedetection/config.go b/modules/fiftyonedegrees/devicedetection/config.go new file mode 100644 index 00000000000..a5519026791 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/config.go @@ -0,0 +1,80 @@ +package devicedetection + +import ( + "encoding/json" + "os" + + "github.com/51Degrees/device-detection-go/v4/dd" + "github.com/pkg/errors" + + "github.com/prebid/prebid-server/v2/util/jsonutil" +) + +type config struct { + DataFile dataFile `json:"data_file"` + AccountFilter accountFilter `json:"account_filter"` + Performance performance `json:"performance"` +} + +type dataFile struct { + Path string `json:"path"` + Update dataFileUpdate `json:"update"` + MakeTempCopy *bool `json:"make_temp_copy"` +} + +type dataFileUpdate struct { + Auto bool `json:"auto"` + Url string `json:"url"` + License string `json:"license_key"` + PollingInterval int `json:"polling_interval"` + Product string `json:"product"` + WatchFileSystem *bool `json:"watch_file_system"` + OnStartup bool `json:"on_startup"` +} + +type accountFilter struct { + AllowList []string `json:"allow_list"` +} + +type performance struct { + Profile string `json:"profile"` + Concurrency *int `json:"concurrency"` + Difference *int `json:"difference"` + AllowUnmatched *bool `json:"allow_unmatched"` + Drift *int `json:"drift"` +} + +var performanceProfileMap = map[string]dd.PerformanceProfile{ + "Default": dd.Default, + "LowMemory": dd.LowMemory, + "BalancedTemp": dd.BalancedTemp, + "Balanced": dd.Balanced, + "HighPerformance": dd.HighPerformance, + "InMemory": dd.InMemory, +} + +func (c *config) getPerformanceProfile() dd.PerformanceProfile { + mappedResult, ok := performanceProfileMap[c.Performance.Profile] + if !ok { + return dd.Default + } + + return mappedResult +} + +func parseConfig(data json.RawMessage) (config, error) { + var cfg config + if err := jsonutil.UnmarshalValid(data, &cfg); err != nil { + return cfg, errors.Wrap(err, "failed to parse config") + } + return cfg, nil +} + +func validateConfig(cfg config) error { + _, err := os.Stat(cfg.DataFile.Path) + if err != nil { + return errors.Wrap(err, "error opening hash file path") + } + + return nil +} diff --git a/modules/fiftyonedegrees/devicedetection/config_test.go b/modules/fiftyonedegrees/devicedetection/config_test.go new file mode 100644 index 00000000000..e2478d82b7d --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/config_test.go @@ -0,0 +1,119 @@ +package devicedetection + +import ( + "os" + "testing" + + "github.com/51Degrees/device-detection-go/v4/dd" + "github.com/stretchr/testify/assert" +) + +func TestParseConfig(t *testing.T) { + cfgRaw := []byte(`{ + "enabled": true, + "data_file": { + "path": "path/to/51Degrees-LiteV4.1.hash", + "update": { + "auto": true, + "url": "https://my.datafile.com/datafile.gz", + "polling_interval": 3600, + "license_key": "your_license_key", + "product": "V4Enterprise", + "on_startup": true + } + }, + "account_filter": {"allow_list": ["123"]}, + "performance": { + "profile": "default", + "concurrency": 1, + "difference": 1, + "allow_unmatched": true, + "drift": 1 + } + }`) + + cfg, err := parseConfig(cfgRaw) + + assert.NoError(t, err) + + assert.Equal(t, cfg.DataFile.Path, "path/to/51Degrees-LiteV4.1.hash") + assert.True(t, cfg.DataFile.Update.Auto) + assert.Equal(t, cfg.DataFile.Update.Url, "https://my.datafile.com/datafile.gz") + assert.Equal(t, cfg.DataFile.Update.PollingInterval, 3600) + assert.Equal(t, cfg.DataFile.Update.License, "your_license_key") + assert.Equal(t, cfg.DataFile.Update.Product, "V4Enterprise") + assert.True(t, cfg.DataFile.Update.OnStartup) + assert.Equal(t, cfg.AccountFilter.AllowList, []string{"123"}) + assert.Equal(t, cfg.Performance.Profile, "default") + assert.Equal(t, *cfg.Performance.Concurrency, 1) + assert.Equal(t, *cfg.Performance.Difference, 1) + assert.True(t, *cfg.Performance.AllowUnmatched) + assert.Equal(t, *cfg.Performance.Drift, 1) + assert.Equal(t, cfg.getPerformanceProfile(), dd.Default) +} + +func TestValidateConfig(t *testing.T) { + file, err := os.Create("test-validate-config.hash") + if err != nil { + t.Errorf("Failed to create file: %v", err) + } + defer file.Close() + defer os.Remove("test-validate-config.hash") + + cfgRaw := []byte(`{ + "enabled": true, + "data_file": { + "path": "test-validate-config.hash", + "update": { + "auto": true, + "url": "https://my.datafile.com/datafile.gz", + "polling_interval": 3600, + "licence_key": "your_licence_key", + "product": "V4Enterprise" + } + }, + "account_filter": {"allow_list": ["123"]}, + "performance": { + "profile": "default", + "concurrency": 1, + "difference": 1, + "allow_unmatched": true, + "drift": 1 + } + }`) + + cfg, err := parseConfig(cfgRaw) + assert.NoError(t, err) + + err = validateConfig(cfg) + assert.NoError(t, err) + +} + +func TestInvalidPerformanceProfile(t *testing.T) { + cfgRaw := []byte(`{ + "enabled": true, + "data_file": { + "path": "test-validate-config.hash", + "update": { + "auto": true, + "url": "https://my.datafile.com/datafile.gz", + "polling_interval": 3600, + "licence_key": "your_licence_key", + "product": "V4Enterprise" + } + }, + "account_filter": {"allow_list": ["123"]}, + "performance": { + "profile": "123", + "concurrency": 1, + "difference": 1, + "allow_unmatched": true, + "drift": 1 + } + }`) + cfg, err := parseConfig(cfgRaw) + assert.NoError(t, err) + + assert.Equal(t, cfg.getPerformanceProfile(), dd.Default) +} diff --git a/modules/fiftyonedegrees/devicedetection/context.go b/modules/fiftyonedegrees/devicedetection/context.go new file mode 100644 index 00000000000..3c10dd2f393 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/context.go @@ -0,0 +1,8 @@ +package devicedetection + +// Context keys for device detection +const ( + evidenceFromHeadersCtxKey = "evidence_from_headers" + evidenceFromSuaCtxKey = "evidence_from_sua" + ddEnabledCtxKey = "dd_enabled" +) diff --git a/modules/fiftyonedegrees/devicedetection/device_detector.go b/modules/fiftyonedegrees/devicedetection/device_detector.go new file mode 100644 index 00000000000..8369d343d34 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/device_detector.go @@ -0,0 +1,157 @@ +package devicedetection + +import ( + "github.com/51Degrees/device-detection-go/v4/dd" + "github.com/51Degrees/device-detection-go/v4/onpremise" + "github.com/pkg/errors" +) + +type engine interface { + Process(evidences []onpremise.Evidence) (*dd.ResultsHash, error) + GetHttpHeaderKeys() []dd.EvidenceKey +} + +type extractor interface { + extract(results Results, ua string) (*deviceInfo, error) +} + +type defaultDeviceDetector struct { + cfg *dd.ConfigHash + deviceInfoExtractor extractor + engine engine +} + +func newDeviceDetector(cfg *dd.ConfigHash, moduleConfig *config) (*defaultDeviceDetector, error) { + engineOptions := buildEngineOptions(moduleConfig, cfg) + + ddEngine, err := onpremise.New( + engineOptions..., + ) + if err != nil { + return nil, errors.Wrap(err, "Failed to create onpremise engine.") + } + + deviceDetector := &defaultDeviceDetector{ + engine: ddEngine, + cfg: cfg, + deviceInfoExtractor: newDeviceInfoExtractor(), + } + + return deviceDetector, nil +} + +func buildEngineOptions(moduleConfig *config, configHash *dd.ConfigHash) []onpremise.EngineOptions { + options := []onpremise.EngineOptions{ + onpremise.WithDataFile(moduleConfig.DataFile.Path), + } + + options = append( + options, + onpremise.WithProperties([]string{ + "HardwareVendor", + "HardwareName", + "DeviceType", + "PlatformVendor", + "PlatformName", + "PlatformVersion", + "BrowserVendor", + "BrowserName", + "BrowserVersion", + "ScreenPixelsWidth", + "ScreenPixelsHeight", + "PixelRatio", + "Javascript", + "GeoLocation", + "HardwareModel", + "HardwareFamily", + "HardwareModelVariants", + "ScreenInchesHeight", + "IsCrawler", + }), + ) + + options = append( + options, + onpremise.WithConfigHash(configHash), + ) + + if moduleConfig.DataFile.MakeTempCopy != nil { + options = append( + options, + onpremise.WithTempDataCopy(*moduleConfig.DataFile.MakeTempCopy), + ) + } + + dataUpdateOptions := []onpremise.EngineOptions{ + onpremise.WithAutoUpdate(moduleConfig.DataFile.Update.Auto), + } + + if moduleConfig.DataFile.Update.Url != "" { + dataUpdateOptions = append( + dataUpdateOptions, + onpremise.WithDataUpdateUrl( + moduleConfig.DataFile.Update.Url, + ), + ) + } + + if moduleConfig.DataFile.Update.PollingInterval > 0 { + dataUpdateOptions = append( + dataUpdateOptions, + onpremise.WithPollingInterval( + moduleConfig.DataFile.Update.PollingInterval, + ), + ) + } + + if moduleConfig.DataFile.Update.License != "" { + dataUpdateOptions = append( + dataUpdateOptions, + onpremise.WithLicenseKey(moduleConfig.DataFile.Update.License), + ) + } + + if moduleConfig.DataFile.Update.Product != "" { + dataUpdateOptions = append( + dataUpdateOptions, + onpremise.WithProduct(moduleConfig.DataFile.Update.Product), + ) + } + + if moduleConfig.DataFile.Update.WatchFileSystem != nil { + dataUpdateOptions = append( + dataUpdateOptions, + onpremise.WithFileWatch( + *moduleConfig.DataFile.Update.WatchFileSystem, + ), + ) + } + + dataUpdateOptions = append( + dataUpdateOptions, + onpremise.WithUpdateOnStart(moduleConfig.DataFile.Update.OnStartup), + ) + + options = append( + options, + dataUpdateOptions..., + ) + + return options +} + +func (x defaultDeviceDetector) getSupportedHeaders() []dd.EvidenceKey { + return x.engine.GetHttpHeaderKeys() +} + +func (x defaultDeviceDetector) getDeviceInfo(evidence []onpremise.Evidence, ua string) (*deviceInfo, error) { + results, err := x.engine.Process(evidence) + if err != nil { + return nil, errors.Wrap(err, "Failed to process evidence") + } + defer results.Free() + + deviceInfo, err := x.deviceInfoExtractor.extract(results, ua) + + return deviceInfo, err +} diff --git a/modules/fiftyonedegrees/devicedetection/device_detector_test.go b/modules/fiftyonedegrees/devicedetection/device_detector_test.go new file mode 100644 index 00000000000..84d6ab28cc0 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/device_detector_test.go @@ -0,0 +1,190 @@ +package devicedetection + +import ( + "fmt" + "testing" + + "github.com/51Degrees/device-detection-go/v4/dd" + "github.com/51Degrees/device-detection-go/v4/onpremise" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestBuildEngineOptions(t *testing.T) { + cases := []struct { + cfgRaw []byte + length int + }{ + { + cfgRaw: []byte(`{ + "enabled": true, + "data_file": { + "path": "path/to/51Degrees-LiteV4.1.hash", + "update": { + "auto": true, + "url": "https://my.datafile.com/datafile.gz", + "polling_interval": 3600, + "license_key": "your_license_key", + "product": "V4Enterprise", + "watch_file_system": true, + "on_startup": true + }, + "make_temp_copy": true + }, + "account_filter": {"allow_list": ["123"]}, + "performance": { + "profile": "default", + "concurrency": 1, + "difference": 1, + "allow_unmatched": true, + "drift": 1 + } + }`), + length: 11, + // data_file.path, data_file.update.auto:true, url, polling_interval, license_key, product, confighash, properties + // data_file.update.on_startup:true, data_file.update.watch_file_system:true, data_file.make_temp_copy:true + }, + { + cfgRaw: []byte(`{ + "enabled": true, + "data_file": { + "path": "path/to/51Degrees-LiteV4.1.hash" + }, + "account_filter": {"allow_list": ["123"]}, + "performance": { + "profile": "default", + "concurrency": 1, + "difference": 1, + "allow_unmatched": true, + "drift": 1 + } + }`), + length: 5, // data_file.update.auto:false, data_file.path, confighash, properties, data_file.update.on_startup:false + }, + } + + for _, c := range cases { + cfg, err := parseConfig(c.cfgRaw) + assert.NoError(t, err) + configHash := configHashFromConfig(&cfg) + options := buildEngineOptions(&cfg, configHash) + assert.Equal(t, c.length, len(options)) + } +} + +type engineMock struct { + mock.Mock +} + +func (e *engineMock) Process(evidences []onpremise.Evidence) (*dd.ResultsHash, error) { + args := e.Called(evidences) + res := args.Get(0) + if res == nil { + return nil, args.Error(1) + } + + return res.(*dd.ResultsHash), args.Error(1) +} + +func (e *engineMock) GetHttpHeaderKeys() []dd.EvidenceKey { + args := e.Called() + return args.Get(0).([]dd.EvidenceKey) +} + +type extractorMock struct { + mock.Mock +} + +func (e *extractorMock) extract(results Results, ua string) (*deviceInfo, error) { + args := e.Called(results, ua) + return args.Get(0).(*deviceInfo), args.Error(1) +} + +func TestGetDeviceInfo(t *testing.T) { + tests := []struct { + name string + engineResponse *dd.ResultsHash + engineError error + expectedResult *deviceInfo + expectedError string + }{ + { + name: "Success_path", + engineResponse: &dd.ResultsHash{}, + engineError: nil, + expectedResult: &deviceInfo{ + DeviceId: "123", + }, + expectedError: "", + }, + { + name: "Error_path", + engineResponse: nil, + engineError: fmt.Errorf("error"), + expectedResult: nil, + expectedError: "Failed to process evidence: error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + extractorM := &extractorMock{} + extractorM.On("extract", mock.Anything, mock.Anything).Return( + &deviceInfo{ + DeviceId: "123", + }, nil, + ) + + engineM := &engineMock{} + engineM.On("Process", mock.Anything).Return( + tt.engineResponse, tt.engineError, + ) + + deviceDetector := defaultDeviceDetector{ + cfg: nil, + deviceInfoExtractor: extractorM, + engine: engineM, + } + + result, err := deviceDetector.getDeviceInfo( + []onpremise.Evidence{{ + Prefix: dd.HttpEvidenceQuery, + Key: "key", + Value: "val", + }}, "ua", + ) + + if tt.expectedError == "" { + assert.NoError(t, err) + assert.NotNil(t, result) + assert.Equal(t, tt.expectedResult.DeviceId, result.DeviceId) + } else { + assert.Errorf(t, err, tt.expectedError) + assert.Nil(t, result) + } + }) + } +} + +func TestGetSupportedHeaders(t *testing.T) { + engineM := &engineMock{} + + engineM.On("GetHttpHeaderKeys").Return( + []dd.EvidenceKey{{ + Key: "key", + Prefix: dd.HttpEvidenceQuery, + }}, + ) + + deviceDetector := defaultDeviceDetector{ + cfg: nil, + deviceInfoExtractor: nil, + engine: engineM, + } + + result := deviceDetector.getSupportedHeaders() + assert.NotNil(t, result) + assert.Equal(t, len(result), 1) + assert.Equal(t, result[0].Key, "key") + +} diff --git a/modules/fiftyonedegrees/devicedetection/device_info_extractor.go b/modules/fiftyonedegrees/devicedetection/device_info_extractor.go new file mode 100644 index 00000000000..1c913e21696 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/device_info_extractor.go @@ -0,0 +1,121 @@ +package devicedetection + +import ( + "strconv" + + "github.com/golang/glog" + "github.com/pkg/errors" +) + +// deviceInfoExtractor is a struct that contains the methods to extract device information +// from the results of the device detection +type deviceInfoExtractor struct{} + +func newDeviceInfoExtractor() deviceInfoExtractor { + return deviceInfoExtractor{} +} + +type Results interface { + ValuesString(string, string) (string, error) + HasValues(string) (bool, error) + DeviceId() (string, error) +} + +type deviceInfoProperty string + +const ( + deviceInfoHardwareVendor deviceInfoProperty = "HardwareVendor" + deviceInfoHardwareName deviceInfoProperty = "HardwareName" + deviceInfoDeviceType deviceInfoProperty = "DeviceType" + deviceInfoPlatformVendor deviceInfoProperty = "PlatformVendor" + deviceInfoPlatformName deviceInfoProperty = "PlatformName" + deviceInfoPlatformVersion deviceInfoProperty = "PlatformVersion" + deviceInfoBrowserVendor deviceInfoProperty = "BrowserVendor" + deviceInfoBrowserName deviceInfoProperty = "BrowserName" + deviceInfoBrowserVersion deviceInfoProperty = "BrowserVersion" + deviceInfoScreenPixelsWidth deviceInfoProperty = "ScreenPixelsWidth" + deviceInfoScreenPixelsHeight deviceInfoProperty = "ScreenPixelsHeight" + deviceInfoPixelRatio deviceInfoProperty = "PixelRatio" + deviceInfoJavascript deviceInfoProperty = "Javascript" + deviceInfoGeoLocation deviceInfoProperty = "GeoLocation" + deviceInfoHardwareModel deviceInfoProperty = "HardwareModel" + deviceInfoHardwareFamily deviceInfoProperty = "HardwareFamily" + deviceInfoHardwareModelVariants deviceInfoProperty = "HardwareModelVariants" + deviceInfoScreenInchesHeight deviceInfoProperty = "ScreenInchesHeight" +) + +func (x deviceInfoExtractor) extract(results Results, ua string) (*deviceInfo, error) { + hardwareVendor := x.getValue(results, deviceInfoHardwareVendor) + hardwareName := x.getValue(results, deviceInfoHardwareName) + deviceType := x.getValue(results, deviceInfoDeviceType) + platformVendor := x.getValue(results, deviceInfoPlatformVendor) + platformName := x.getValue(results, deviceInfoPlatformName) + platformVersion := x.getValue(results, deviceInfoPlatformVersion) + browserVendor := x.getValue(results, deviceInfoBrowserVendor) + browserName := x.getValue(results, deviceInfoBrowserName) + browserVersion := x.getValue(results, deviceInfoBrowserVersion) + screenPixelsWidth, _ := strconv.ParseInt(x.getValue(results, deviceInfoScreenPixelsWidth), 10, 64) + screenPixelsHeight, _ := strconv.ParseInt(x.getValue(results, deviceInfoScreenPixelsHeight), 10, 64) + pixelRatio, _ := strconv.ParseFloat(x.getValue(results, deviceInfoPixelRatio), 10) + javascript, _ := strconv.ParseBool(x.getValue(results, deviceInfoJavascript)) + geoLocation, _ := strconv.ParseBool(x.getValue(results, deviceInfoGeoLocation)) + deviceId, err := results.DeviceId() + if err != nil { + return nil, errors.Wrap(err, "Failed to get device id.") + } + hardwareModel := x.getValue(results, deviceInfoHardwareModel) + hardwareFamily := x.getValue(results, deviceInfoHardwareFamily) + hardwareModelVariants := x.getValue(results, deviceInfoHardwareModelVariants) + screenInchedHeight, _ := strconv.ParseFloat(x.getValue(results, deviceInfoScreenInchesHeight), 10) + + p := &deviceInfo{ + HardwareVendor: hardwareVendor, + HardwareName: hardwareName, + DeviceType: deviceType, + PlatformVendor: platformVendor, + PlatformName: platformName, + PlatformVersion: platformVersion, + BrowserVendor: browserVendor, + BrowserName: browserName, + BrowserVersion: browserVersion, + ScreenPixelsWidth: screenPixelsWidth, + ScreenPixelsHeight: screenPixelsHeight, + PixelRatio: pixelRatio, + Javascript: javascript, + GeoLocation: geoLocation, + UserAgent: ua, + DeviceId: deviceId, + HardwareModel: hardwareModel, + HardwareFamily: hardwareFamily, + HardwareModelVariants: hardwareModelVariants, + ScreenInchesHeight: screenInchedHeight, + } + + return p, nil +} + +// function getValue return a value results for a property +func (x deviceInfoExtractor) getValue(results Results, propertyName deviceInfoProperty) string { + // Get the values in string + value, err := results.ValuesString( + string(propertyName), + ",", + ) + if err != nil { + glog.Errorf("Failed to get results values string.") + return "" + } + + hasValues, err := results.HasValues(string(propertyName)) + if err != nil { + glog.Errorf("Failed to check if a matched value exists for property %s.\n", propertyName) + return "" + } + + if !hasValues { + glog.Warningf("Property %s does not have a matched value.\n", propertyName) + return "Unknown" + } + + return value +} diff --git a/modules/fiftyonedegrees/devicedetection/device_info_extractor_test.go b/modules/fiftyonedegrees/devicedetection/device_info_extractor_test.go new file mode 100644 index 00000000000..197e3928602 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/device_info_extractor_test.go @@ -0,0 +1,130 @@ +package devicedetection + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +type ResultsHashMock struct { + mock.Mock +} + +func (m *ResultsHashMock) DeviceId() (string, error) { + return "", nil +} + +func (m *ResultsHashMock) ValuesString(prop1 string, prop2 string) (string, error) { + args := m.Called(prop1, prop2) + return args.String(0), args.Error(1) +} + +func (m *ResultsHashMock) HasValues(prop1 string) (bool, error) { + args := m.Called(prop1) + return args.Bool(0), args.Error(1) +} + +func TestDeviceInfoExtraction(t *testing.T) { + results := &ResultsHashMock{} + + extractor := newDeviceInfoExtractor() + mockValue(results, "HardwareName", "Macbook") + mockValues(results) + + deviceInfo, _ := extractor.extract(results, "ua") + assert.NotNil(t, deviceInfo) + + assert.Equal(t, deviceInfo.HardwareName, "Macbook") + assertDeviceInfo(t, deviceInfo) +} + +func TestDeviceInfoExtractionNoProperty(t *testing.T) { + results := &ResultsHashMock{} + + extractor := newDeviceInfoExtractor() + results.Mock.On("ValuesString", "HardwareName", ",").Return("", errors.New("Error")) + results.Mock.On("HasValues", "HardwareName").Return(true, nil) + mockValues(results) + + deviceInfo, _ := extractor.extract(results, "ua") + assert.NotNil(t, deviceInfo) + + assertDeviceInfo(t, deviceInfo) + assert.Equal(t, deviceInfo.HardwareName, "") +} + +func TestDeviceInfoExtractionNoValue(t *testing.T) { + results := &ResultsHashMock{} + + extractor := newDeviceInfoExtractor() + mockValues(results) + mockValue(results, "HardwareVendor", "Apple") + + results.Mock.On("ValuesString", "HardwareName", ",").Return("Macbook", nil) + results.Mock.On("HasValues", "HardwareName").Return(false, nil) + + deviceInfo, _ := extractor.extract(results, "ua") + assert.NotNil(t, deviceInfo) + assertDeviceInfo(t, deviceInfo) + assert.Equal(t, deviceInfo.HardwareName, "Unknown") +} + +func TestDeviceInfoExtractionHasValueError(t *testing.T) { + results := &ResultsHashMock{} + + extractor := newDeviceInfoExtractor() + mockValue(results, "HardwareVendor", "Apple") + + results.Mock.On("ValuesString", "HardwareName", ",").Return("Macbook", nil) + results.Mock.On("HasValues", "HardwareName").Return(true, errors.New("error")) + + mockValues(results) + + deviceInfo, _ := extractor.extract(results, "ua") + assert.NotNil(t, deviceInfo) + assertDeviceInfo(t, deviceInfo) + assert.Equal(t, deviceInfo.HardwareName, "") +} + +func mockValues(results *ResultsHashMock) { + mockValue(results, "HardwareVendor", "Apple") + mockValue(results, "DeviceType", "Desctop") + mockValue(results, "PlatformVendor", "Apple") + mockValue(results, "PlatformName", "MacOs") + mockValue(results, "PlatformVersion", "14") + mockValue(results, "BrowserVendor", "Google") + mockValue(results, "BrowserName", "Crome") + mockValue(results, "BrowserVersion", "12") + mockValue(results, "ScreenPixelsWidth", "1024") + mockValue(results, "ScreenPixelsHeight", "1080") + mockValue(results, "PixelRatio", "223") + mockValue(results, "Javascript", "true") + mockValue(results, "GeoLocation", "true") + mockValue(results, "HardwareModel", "Macbook") + mockValue(results, "HardwareFamily", "Macbook") + mockValue(results, "HardwareModelVariants", "Macbook") + mockValue(results, "ScreenInchesHeight", "12") +} + +func assertDeviceInfo(t *testing.T, deviceInfo *deviceInfo) { + assert.Equal(t, deviceInfo.HardwareVendor, "Apple") + assert.Equal(t, deviceInfo.DeviceType, "Desctop") + assert.Equal(t, deviceInfo.PlatformVendor, "Apple") + assert.Equal(t, deviceInfo.PlatformName, "MacOs") + assert.Equal(t, deviceInfo.PlatformVersion, "14") + assert.Equal(t, deviceInfo.BrowserVendor, "Google") + assert.Equal(t, deviceInfo.BrowserName, "Crome") + assert.Equal(t, deviceInfo.BrowserVersion, "12") + assert.Equal(t, deviceInfo.ScreenPixelsWidth, int64(1024)) + assert.Equal(t, deviceInfo.ScreenPixelsHeight, int64(1080)) + assert.Equal(t, deviceInfo.PixelRatio, float64(223)) + assert.Equal(t, deviceInfo.Javascript, true) + assert.Equal(t, deviceInfo.GeoLocation, true) +} + +func mockValue(results *ResultsHashMock, name string, value string) { + results.Mock.On("ValuesString", name, ",").Return(value, nil) + results.Mock.On("HasValues", name).Return(true, nil) +} diff --git a/modules/fiftyonedegrees/devicedetection/evidence_extractor.go b/modules/fiftyonedegrees/devicedetection/evidence_extractor.go new file mode 100644 index 00000000000..1d67e1cdeed --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/evidence_extractor.go @@ -0,0 +1,118 @@ +package devicedetection + +import ( + "net/http" + + "github.com/51Degrees/device-detection-go/v4/onpremise" + "github.com/pkg/errors" + + "github.com/51Degrees/device-detection-go/v4/dd" + "github.com/prebid/prebid-server/v2/hooks/hookstage" +) + +type defaultEvidenceExtractor struct { + valFromHeaders evidenceFromRequestHeadersExtractor + valFromSUA evidenceFromSUAPayloadExtractor +} + +func newEvidenceExtractor() *defaultEvidenceExtractor { + evidenceExtractor := &defaultEvidenceExtractor{ + valFromHeaders: newEvidenceFromRequestHeadersExtractor(), + valFromSUA: newEvidenceFromSUAPayloadExtractor(), + } + + return evidenceExtractor +} + +func (x *defaultEvidenceExtractor) fromHeaders(request *http.Request, httpHeaderKeys []dd.EvidenceKey) []stringEvidence { + return x.valFromHeaders.extract(request, httpHeaderKeys) +} + +func (x *defaultEvidenceExtractor) fromSuaPayload(payload []byte) []stringEvidence { + return x.valFromSUA.extract(payload) +} + +// merge merges two slices of stringEvidence into one slice of stringEvidence +func merge(val1, val2 []stringEvidence) []stringEvidence { + evidenceMap := make(map[string]stringEvidence) + for _, e := range val1 { + evidenceMap[e.Key] = e + } + + for _, e := range val2 { + _, exists := evidenceMap[e.Key] + if !exists { + evidenceMap[e.Key] = e + } + } + + evidence := make([]stringEvidence, 0) + + for _, e := range evidenceMap { + evidence = append(evidence, e) + } + + return evidence +} + +func (x *defaultEvidenceExtractor) extract(ctx hookstage.ModuleContext) ([]onpremise.Evidence, string, error) { + if ctx == nil { + return nil, "", errors.New("context is nil") + } + + suaStrings, err := x.getEvidenceStrings(ctx[evidenceFromSuaCtxKey]) + if err != nil { + return nil, "", errors.Wrap(err, "error extracting sua evidence") + } + headerString, err := x.getEvidenceStrings(ctx[evidenceFromHeadersCtxKey]) + if err != nil { + return nil, "", errors.Wrap(err, "error extracting header evidence") + } + + // Merge evidence from headers and SUA, sua has higher priority + evidenceStrings := merge(suaStrings, headerString) + + if len(evidenceStrings) > 0 { + userAgentE, exists := getEvidenceByKey(evidenceStrings, userAgentHeader) + if !exists { + return nil, "", errors.New("User-Agent not found") + } + + evidence := x.extractEvidenceFromStrings(evidenceStrings) + + return evidence, userAgentE.Value, nil + } + + return nil, "", nil +} + +func (x *defaultEvidenceExtractor) getEvidenceStrings(source interface{}) ([]stringEvidence, error) { + if source == nil { + return []stringEvidence{}, nil + } + + evidenceStrings, ok := source.([]stringEvidence) + if !ok { + return nil, errors.New("bad cast to []stringEvidence") + } + + return evidenceStrings, nil +} + +func (x *defaultEvidenceExtractor) extractEvidenceFromStrings(strEvidence []stringEvidence) []onpremise.Evidence { + evidenceResult := make([]onpremise.Evidence, len(strEvidence)) + for i, e := range strEvidence { + prefix := dd.HttpHeaderString + if e.Prefix == queryPrefix { + prefix = dd.HttpEvidenceQuery + } + + evidenceResult[i] = onpremise.Evidence{ + Prefix: prefix, + Key: e.Key, + Value: e.Value, + } + } + + return evidenceResult +} diff --git a/modules/fiftyonedegrees/devicedetection/evidence_extractor_test.go b/modules/fiftyonedegrees/devicedetection/evidence_extractor_test.go new file mode 100644 index 00000000000..9abdf799643 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/evidence_extractor_test.go @@ -0,0 +1,256 @@ +package devicedetection + +import ( + "net/http" + "testing" + + "github.com/51Degrees/device-detection-go/v4/dd" + "github.com/prebid/prebid-server/v2/hooks/hookstage" + "github.com/stretchr/testify/assert" +) + +func TestFromHeaders(t *testing.T) { + extractor := newEvidenceExtractor() + + req := http.Request{ + Header: make(map[string][]string), + } + req.Header.Add("header", "Value") + req.Header.Add("Sec-CH-UA-Full-Version-List", "Chrome;12") + evidenceKeys := []dd.EvidenceKey{ + { + Prefix: dd.EvidencePrefix(10), + Key: "header", + }, + { + Prefix: dd.EvidencePrefix(10), + Key: "Sec-CH-UA-Full-Version-List", + }, + } + + evidence := extractor.fromHeaders(&req, evidenceKeys) + + assert.NotNil(t, evidence) + assert.NotEmpty(t, evidence) + assert.Equal(t, evidence[0].Value, "Value") + assert.Equal(t, evidence[0].Key, "header") + assert.Equal(t, evidence[1].Value, "Chrome;12") + assert.Equal(t, evidence[1].Key, "Sec-CH-UA-Full-Version-List") +} + +func TestFromSuaPayload(t *testing.T) { + tests := []struct { + name string + payload []byte + evidenceSize int + evidenceKeyOrder int + expectedKey string + expectedValue string + }{ + { + name: "from_SUA_tag", + payload: []byte(`{ + "device": { + "sua": { + "browsers": [ + { + "brand": "Google Chrome", + "version": ["121", "0", "6167", "184"] + } + ], + "platform": { + "brand": "macOS", + "version": ["14", "0", "0"] + }, + "architecture": "arm" + } + } + }`), + evidenceSize: 4, + evidenceKeyOrder: 0, + expectedKey: "Sec-Ch-Ua-Arch", + expectedValue: "arm", + }, + { + name: "from_UA_headers", + payload: []byte(`{ + "device": { + "ua": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148", + "sua": { + "architecture": "arm" + } + } + }`), + evidenceSize: 2, + evidenceKeyOrder: 1, + expectedKey: "Sec-Ch-Ua-Arch", + expectedValue: "arm", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + extractor := newEvidenceExtractor() + + evidence := extractor.fromSuaPayload(tt.payload) + + assert.NotNil(t, evidence) + assert.NotEmpty(t, evidence) + assert.Equal(t, len(evidence), tt.evidenceSize) + assert.Equal(t, evidence[tt.evidenceKeyOrder].Key, tt.expectedKey) + assert.Equal(t, evidence[tt.evidenceKeyOrder].Value, tt.expectedValue) + }) + } +} + +func TestExtract(t *testing.T) { + uaEvidence1 := stringEvidence{ + Prefix: "ua1", + Key: userAgentHeader, + Value: "uav1", + } + uaEvidence2 := stringEvidence{ + Prefix: "ua2", + Key: userAgentHeader, + Value: "uav2", + } + evidence1 := stringEvidence{ + Prefix: "e1", + Key: "k1", + Value: "v1", + } + emptyEvidence := stringEvidence{ + Prefix: "empty", + Key: "e1", + Value: "", + } + + tests := []struct { + name string + ctx hookstage.ModuleContext + wantEvidenceCount int + wantUserAgent string + wantError bool + }{ + { + name: "nil", + ctx: nil, + wantError: true, + }, + { + name: "empty", + ctx: hookstage.ModuleContext{ + evidenceFromSuaCtxKey: []stringEvidence{}, + evidenceFromHeadersCtxKey: []stringEvidence{}, + }, + wantEvidenceCount: 0, + wantUserAgent: "", + }, + { + name: "from_headers", + ctx: hookstage.ModuleContext{ + evidenceFromHeadersCtxKey: []stringEvidence{uaEvidence1}, + }, + wantEvidenceCount: 1, + wantUserAgent: "uav1", + }, + { + name: "from_headers_no_user_agent", + ctx: hookstage.ModuleContext{ + evidenceFromHeadersCtxKey: []stringEvidence{evidence1}, + }, + wantError: true, + }, + { + name: "from_sua", + ctx: hookstage.ModuleContext{ + evidenceFromSuaCtxKey: []stringEvidence{uaEvidence1}, + }, + wantEvidenceCount: 1, + wantUserAgent: "uav1", + }, + { + name: "from_sua_no_user_agent", + ctx: hookstage.ModuleContext{ + evidenceFromSuaCtxKey: []stringEvidence{evidence1}, + }, + wantError: true, + }, + { + name: "from_headers_error", + ctx: hookstage.ModuleContext{ + evidenceFromHeadersCtxKey: "bad value", + }, + wantError: true, + }, + { + name: "from_sua_error", + ctx: hookstage.ModuleContext{ + evidenceFromHeadersCtxKey: []stringEvidence{}, + evidenceFromSuaCtxKey: "bad value", + }, + wantError: true, + }, + { + name: "from_sua_and_headers", + ctx: hookstage.ModuleContext{ + evidenceFromHeadersCtxKey: []stringEvidence{uaEvidence1}, + evidenceFromSuaCtxKey: []stringEvidence{evidence1}, + }, + wantEvidenceCount: 2, + wantUserAgent: "uav1", + }, + { + name: "from_sua_and_headers_sua_can_overwrite_if_ua_present", + ctx: hookstage.ModuleContext{ + evidenceFromHeadersCtxKey: []stringEvidence{uaEvidence1}, + evidenceFromSuaCtxKey: []stringEvidence{uaEvidence2}, + }, + wantEvidenceCount: 1, + wantUserAgent: "uav2", + }, + { + name: "empty_string_values", + ctx: hookstage.ModuleContext{ + evidenceFromHeadersCtxKey: []stringEvidence{emptyEvidence}, + }, + wantError: true, + }, + { + name: "empty_sua_values", + ctx: hookstage.ModuleContext{ + evidenceFromSuaCtxKey: []stringEvidence{emptyEvidence}, + }, + wantError: true, + }, + { + name: "mixed_valid_and_invalid", + ctx: hookstage.ModuleContext{ + evidenceFromHeadersCtxKey: []stringEvidence{uaEvidence1}, + evidenceFromSuaCtxKey: "bad value", + }, + wantError: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + extractor := newEvidenceExtractor() + evidence, userAgent, err := extractor.extract(test.ctx) + + if test.wantError { + assert.Error(t, err) + assert.Nil(t, evidence) + assert.Equal(t, userAgent, "") + } else if test.wantEvidenceCount == 0 { + assert.NoError(t, err) + assert.Nil(t, evidence) + assert.Equal(t, userAgent, "") + } else { + assert.NoError(t, err) + assert.Equal(t, len(evidence), test.wantEvidenceCount) + assert.Equal(t, userAgent, test.wantUserAgent) + } + }) + } +} diff --git a/modules/fiftyonedegrees/devicedetection/fiftyone_device_types.go b/modules/fiftyonedegrees/devicedetection/fiftyone_device_types.go new file mode 100644 index 00000000000..7237698117d --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/fiftyone_device_types.go @@ -0,0 +1,77 @@ +package devicedetection + +import ( + "github.com/prebid/openrtb/v20/adcom1" +) + +type deviceTypeMap = map[deviceType]adcom1.DeviceType + +var mobileOrTabletDeviceTypes = []deviceType{ + deviceTypeMobile, + deviceTypeSmartPhone, +} + +var personalComputerDeviceTypes = []deviceType{ + deviceTypeDesktop, + deviceTypeEReader, + deviceTypeVehicleDisplay, +} + +var tvDeviceTypes = []deviceType{ + deviceTypeTv, +} + +var phoneDeviceTypes = []deviceType{ + deviceTypePhone, +} + +var tabletDeviceTypes = []deviceType{ + deviceTypeTablet, +} + +var connectedDeviceTypes = []deviceType{ + deviceTypeIoT, + deviceTypeRouter, + deviceTypeSmallScreen, + deviceTypeSmartSpeaker, + deviceTypeSmartWatch, +} + +var setTopBoxDeviceTypes = []deviceType{ + deviceTypeMediaHub, + deviceTypeConsole, +} + +var oohDeviceTypes = []deviceType{ + deviceTypeKiosk, +} + +func applyCollection(items []deviceType, value adcom1.DeviceType, mappedCollection deviceTypeMap) { + for _, item := range items { + mappedCollection[item] = value + } +} + +var deviceTypeMapCollection = deviceTypeMap{} + +func init() { + applyCollection(mobileOrTabletDeviceTypes, adcom1.DeviceMobile, deviceTypeMapCollection) + applyCollection(personalComputerDeviceTypes, adcom1.DevicePC, deviceTypeMapCollection) + applyCollection(tvDeviceTypes, adcom1.DeviceTV, deviceTypeMapCollection) + applyCollection(phoneDeviceTypes, adcom1.DevicePhone, deviceTypeMapCollection) + applyCollection(tabletDeviceTypes, adcom1.DeviceTablet, deviceTypeMapCollection) + applyCollection(connectedDeviceTypes, adcom1.DeviceConnected, deviceTypeMapCollection) + applyCollection(setTopBoxDeviceTypes, adcom1.DeviceSetTopBox, deviceTypeMapCollection) + applyCollection(oohDeviceTypes, adcom1.DeviceOOH, deviceTypeMapCollection) +} + +// fiftyOneDtToRTB converts a 51Degrees device type to an OpenRTB device type. +// If the device type is not recognized, it defaults to PC. +func fiftyOneDtToRTB(val string) adcom1.DeviceType { + id, ok := deviceTypeMapCollection[deviceType(val)] + if ok { + return id + } + + return adcom1.DevicePC +} diff --git a/modules/fiftyonedegrees/devicedetection/fiftyone_device_types_test.go b/modules/fiftyonedegrees/devicedetection/fiftyone_device_types_test.go new file mode 100644 index 00000000000..5fd0203bac8 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/fiftyone_device_types_test.go @@ -0,0 +1,90 @@ +package devicedetection + +import ( + "testing" + + "github.com/prebid/openrtb/v20/adcom1" + "github.com/stretchr/testify/assert" +) + +func TestFiftyOneDtToRTB(t *testing.T) { + cases := []struct { + fiftyOneDt string + rtbDt adcom1.DeviceType + }{ + { + fiftyOneDt: "Phone", + rtbDt: adcom1.DevicePhone, + }, + { + fiftyOneDt: "Console", + rtbDt: adcom1.DeviceSetTopBox, + }, + { + fiftyOneDt: "Desktop", + rtbDt: adcom1.DevicePC, + }, + { + fiftyOneDt: "EReader", + rtbDt: adcom1.DevicePC, + }, + { + fiftyOneDt: "IoT", + rtbDt: adcom1.DeviceConnected, + }, + { + fiftyOneDt: "Kiosk", + rtbDt: adcom1.DeviceOOH, + }, + { + fiftyOneDt: "MediaHub", + rtbDt: adcom1.DeviceSetTopBox, + }, + { + fiftyOneDt: "Mobile", + rtbDt: adcom1.DeviceMobile, + }, + { + fiftyOneDt: "Router", + rtbDt: adcom1.DeviceConnected, + }, + { + fiftyOneDt: "SmallScreen", + rtbDt: adcom1.DeviceConnected, + }, + { + fiftyOneDt: "SmartPhone", + rtbDt: adcom1.DeviceMobile, + }, + { + fiftyOneDt: "SmartSpeaker", + rtbDt: adcom1.DeviceConnected, + }, + { + fiftyOneDt: "SmartWatch", + rtbDt: adcom1.DeviceConnected, + }, + { + fiftyOneDt: "Tablet", + rtbDt: adcom1.DeviceTablet, + }, + { + fiftyOneDt: "Tv", + rtbDt: adcom1.DeviceTV, + }, + { + fiftyOneDt: "Vehicle Display", + rtbDt: adcom1.DevicePC, + }, + { + fiftyOneDt: "Unknown", + rtbDt: adcom1.DevicePC, + }, + } + + for _, c := range cases { + t.Run(c.fiftyOneDt, func(t *testing.T) { + assert.Equal(t, c.rtbDt, fiftyOneDtToRTB(c.fiftyOneDt)) + }) + } +} diff --git a/modules/fiftyonedegrees/devicedetection/hook_auction_entrypoint.go b/modules/fiftyonedegrees/devicedetection/hook_auction_entrypoint.go new file mode 100644 index 00000000000..911f20e1840 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/hook_auction_entrypoint.go @@ -0,0 +1,27 @@ +package devicedetection + +import ( + "github.com/prebid/prebid-server/v2/hooks/hookexecution" + "github.com/prebid/prebid-server/v2/hooks/hookstage" +) + +// handleAuctionEntryPointRequestHook is a hookstage.HookFunc that is used to handle the auction entrypoint request hook. +func handleAuctionEntryPointRequestHook(cfg config, payload hookstage.EntrypointPayload, deviceDetector deviceDetector, evidenceExtractor evidenceExtractor, accountValidator accountValidator) (result hookstage.HookResult[hookstage.EntrypointPayload], err error) { + // if account/domain is not allowed, return failure + if !accountValidator.isAllowed(cfg, payload.Body) { + return hookstage.HookResult[hookstage.EntrypointPayload]{}, hookexecution.NewFailure("account not allowed") + } + // fetch evidence from headers and sua + evidenceFromHeaders := evidenceExtractor.fromHeaders(payload.Request, deviceDetector.getSupportedHeaders()) + evidenceFromSua := evidenceExtractor.fromSuaPayload(payload.Body) + + // create a Module context and set the evidence from headers, evidence from sua and dd enabled flag + moduleContext := make(hookstage.ModuleContext) + moduleContext[evidenceFromHeadersCtxKey] = evidenceFromHeaders + moduleContext[evidenceFromSuaCtxKey] = evidenceFromSua + moduleContext[ddEnabledCtxKey] = true + + return hookstage.HookResult[hookstage.EntrypointPayload]{ + ModuleContext: moduleContext, + }, nil +} diff --git a/modules/fiftyonedegrees/devicedetection/hook_raw_auction_request.go b/modules/fiftyonedegrees/devicedetection/hook_raw_auction_request.go new file mode 100644 index 00000000000..1146c3cc639 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/hook_raw_auction_request.go @@ -0,0 +1,173 @@ +package devicedetection + +import ( + "fmt" + "math" + + "github.com/prebid/prebid-server/v2/hooks/hookexecution" + "github.com/prebid/prebid-server/v2/hooks/hookstage" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" +) + +func handleAuctionRequestHook(ctx hookstage.ModuleInvocationContext, deviceDetector deviceDetector, evidenceExtractor evidenceExtractor) (hookstage.HookResult[hookstage.RawAuctionRequestPayload], error) { + var result hookstage.HookResult[hookstage.RawAuctionRequestPayload] + + // If the entrypoint hook was not configured, return the result without any changes + if ctx.ModuleContext == nil { + return result, hookexecution.NewFailure("entrypoint hook was not configured") + } + + result.ChangeSet.AddMutation( + func(rawPayload hookstage.RawAuctionRequestPayload) (hookstage.RawAuctionRequestPayload, error) { + evidence, ua, err := evidenceExtractor.extract(ctx.ModuleContext) + if err != nil { + return rawPayload, hookexecution.NewFailure("error extracting evidence %s", err) + } + if evidence == nil { + return rawPayload, hookexecution.NewFailure("error extracting evidence") + } + + deviceInfo, err := deviceDetector.getDeviceInfo(evidence, ua) + if err != nil { + return rawPayload, hookexecution.NewFailure("error getting device info %s", err) + } + + result, err := hydrateFields(deviceInfo, rawPayload) + if err != nil { + return rawPayload, hookexecution.NewFailure(fmt.Sprintf("error hydrating fields %s", err)) + } + + return result, nil + }, hookstage.MutationUpdate, + ) + + return result, nil +} + +// hydrateFields hydrates the fields in the raw auction request payload with the device information +func hydrateFields(fiftyOneDd *deviceInfo, payload hookstage.RawAuctionRequestPayload) (hookstage.RawAuctionRequestPayload, error) { + devicePayload := gjson.GetBytes(payload, "device") + dPV := devicePayload.Value() + if dPV == nil { + return payload, nil + } + + deviceObject := dPV.(map[string]any) + deviceObject = setMissingFields(deviceObject, fiftyOneDd) + deviceObject = signDeviceData(deviceObject, fiftyOneDd) + + return mergeDeviceIntoPayload(payload, deviceObject) +} + +// setMissingFields sets fields such as ["devicetype", "ua", "make", "os", "osv", "h", "w", "pxratio", "js", "geoFetch", "model", "ppi"] +// if they are not already present in the device object +func setMissingFields(deviceObj map[string]any, fiftyOneDd *deviceInfo) map[string]any { + optionalFields := map[string]func() any{ + "devicetype": func() any { + return fiftyOneDtToRTB(fiftyOneDd.DeviceType) + }, + "ua": func() any { + if fiftyOneDd.UserAgent != ddUnknown { + return fiftyOneDd.UserAgent + } + return nil + }, + "make": func() any { + if fiftyOneDd.HardwareVendor != ddUnknown { + return fiftyOneDd.HardwareVendor + } + return nil + }, + "os": func() any { + if fiftyOneDd.PlatformName != ddUnknown { + return fiftyOneDd.PlatformName + } + return nil + }, + "osv": func() any { + if fiftyOneDd.PlatformVersion != ddUnknown { + return fiftyOneDd.PlatformVersion + } + return nil + }, + "h": func() any { + return fiftyOneDd.ScreenPixelsHeight + }, + "w": func() any { + return fiftyOneDd.ScreenPixelsWidth + }, + "pxratio": func() any { + return fiftyOneDd.PixelRatio + }, + "js": func() any { + val := 0 + if fiftyOneDd.Javascript { + val = 1 + } + return val + }, + "geoFetch": func() any { + val := 0 + if fiftyOneDd.GeoLocation { + val = 1 + } + return val + }, + "model": func() any { + newVal := fiftyOneDd.HardwareModel + if newVal == ddUnknown { + newVal = fiftyOneDd.HardwareName + } + if newVal != ddUnknown { + return newVal + } + return nil + }, + "ppi": func() any { + if fiftyOneDd.ScreenPixelsHeight > 0 && fiftyOneDd.ScreenInchesHeight > 0 { + ppi := float64(fiftyOneDd.ScreenPixelsHeight) / fiftyOneDd.ScreenInchesHeight + return int(math.Round(ppi)) + } + return nil + }, + } + + for field, valFunc := range optionalFields { + _, ok := deviceObj[field] + if !ok { + val := valFunc() + if val != nil { + deviceObj[field] = val + } + } + } + + return deviceObj +} + +// signDeviceData signs the device data with the device information in the ext map of the device object +func signDeviceData(deviceObj map[string]any, fiftyOneDd *deviceInfo) map[string]any { + extObj, ok := deviceObj["ext"] + var ext map[string]any + if ok { + ext = extObj.(map[string]any) + } else { + ext = make(map[string]any) + } + + ext["fiftyonedegrees_deviceId"] = fiftyOneDd.DeviceId + deviceObj["ext"] = ext + + return deviceObj +} + +// mergeDeviceIntoPayload merges the modified device object back into the RawAuctionRequestPayload +func mergeDeviceIntoPayload(payload hookstage.RawAuctionRequestPayload, deviceObject map[string]any) (hookstage.RawAuctionRequestPayload, error) { + newPayload, err := sjson.SetBytes(payload, "device", deviceObject) + if err != nil { + return payload, err + } + + return newPayload, nil +} diff --git a/modules/fiftyonedegrees/devicedetection/models.go b/modules/fiftyonedegrees/devicedetection/models.go new file mode 100644 index 00000000000..c58daa211fd --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/models.go @@ -0,0 +1,66 @@ +package devicedetection + +// Prefixes in literal format +const queryPrefix = "query." +const headerPrefix = "header." +const ddUnknown = "Unknown" + +// Evidence where all fields are in string format +type stringEvidence struct { + Prefix string + Key string + Value string +} + +func getEvidenceByKey(e []stringEvidence, key string) (stringEvidence, bool) { + for _, evidence := range e { + if evidence.Key == key { + return evidence, true + } + } + return stringEvidence{}, false +} + +type deviceType string + +const ( + deviceTypePhone = "Phone" + deviceTypeConsole = "Console" + deviceTypeDesktop = "Desktop" + deviceTypeEReader = "EReader" + deviceTypeIoT = "IoT" + deviceTypeKiosk = "Kiosk" + deviceTypeMediaHub = "MediaHub" + deviceTypeMobile = "Mobile" + deviceTypeRouter = "Router" + deviceTypeSmallScreen = "SmallScreen" + deviceTypeSmartPhone = "SmartPhone" + deviceTypeSmartSpeaker = "SmartSpeaker" + deviceTypeSmartWatch = "SmartWatch" + deviceTypeTablet = "Tablet" + deviceTypeTv = "Tv" + deviceTypeVehicleDisplay = "Vehicle Display" +) + +type deviceInfo struct { + HardwareVendor string + HardwareName string + DeviceType string + PlatformVendor string + PlatformName string + PlatformVersion string + BrowserVendor string + BrowserName string + BrowserVersion string + ScreenPixelsWidth int64 + ScreenPixelsHeight int64 + PixelRatio float64 + Javascript bool + GeoLocation bool + HardwareFamily string + HardwareModel string + HardwareModelVariants string + UserAgent string + DeviceId string + ScreenInchesHeight float64 +} diff --git a/modules/fiftyonedegrees/devicedetection/models_test.go b/modules/fiftyonedegrees/devicedetection/models_test.go new file mode 100644 index 00000000000..898f25f4144 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/models_test.go @@ -0,0 +1,63 @@ +package devicedetection + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetEvidenceByKey(t *testing.T) { + populatedEvidence := []stringEvidence{ + {Key: "key1", Value: "value1"}, + {Key: "key2", Value: "value2"}, + {Key: "key3", Value: "value3"}, + } + + tests := []struct { + name string + evidence []stringEvidence + key string + expectEvidence stringEvidence + expectFound bool + }{ + { + name: "nil_evidence", + evidence: nil, + key: "key2", + expectEvidence: stringEvidence{}, + expectFound: false, + }, + { + name: "empty_evidence", + evidence: []stringEvidence{}, + key: "key2", + expectEvidence: stringEvidence{}, + expectFound: false, + }, + { + name: "key_found", + evidence: populatedEvidence, + key: "key2", + expectEvidence: stringEvidence{ + Key: "key2", + Value: "value2", + }, + expectFound: true, + }, + { + name: "key_not_found", + evidence: populatedEvidence, + key: "key4", + expectEvidence: stringEvidence{}, + expectFound: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, exists := getEvidenceByKey(test.evidence, test.key) + assert.Equal(t, test.expectFound, exists) + assert.Equal(t, test.expectEvidence, result) + }) + } +} diff --git a/modules/fiftyonedegrees/devicedetection/module.go b/modules/fiftyonedegrees/devicedetection/module.go new file mode 100644 index 00000000000..df72e6338a5 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/module.go @@ -0,0 +1,107 @@ +package devicedetection + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/51Degrees/device-detection-go/v4/dd" + "github.com/51Degrees/device-detection-go/v4/onpremise" + "github.com/pkg/errors" + "github.com/prebid/prebid-server/v2/hooks/hookstage" + "github.com/prebid/prebid-server/v2/modules/moduledeps" +) + +func configHashFromConfig(cfg *config) *dd.ConfigHash { + configHash := dd.NewConfigHash(cfg.getPerformanceProfile()) + if cfg.Performance.Concurrency != nil { + configHash.SetConcurrency(uint16(*cfg.Performance.Concurrency)) + } + + if cfg.Performance.Difference != nil { + configHash.SetDifference(int32(*cfg.Performance.Difference)) + } + + if cfg.Performance.AllowUnmatched != nil { + configHash.SetAllowUnmatched(*cfg.Performance.AllowUnmatched) + } + + if cfg.Performance.Drift != nil { + configHash.SetDrift(int32(*cfg.Performance.Drift)) + } + return configHash +} + +func Builder(rawConfig json.RawMessage, _ moduledeps.ModuleDeps) (interface{}, error) { + cfg, err := parseConfig(rawConfig) + if err != nil { + return Module{}, errors.Wrap(err, "failed to parse config") + } + + err = validateConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "invalid config") + } + + configHash := configHashFromConfig(&cfg) + + deviceDetectorImpl, err := newDeviceDetector( + configHash, + &cfg, + ) + if err != nil { + return nil, errors.Wrap(err, "failed to create device detector") + } + + return Module{ + cfg, + deviceDetectorImpl, + newEvidenceExtractor(), + newAccountValidator(), + }, + nil +} + +type Module struct { + config config + deviceDetector deviceDetector + evidenceExtractor evidenceExtractor + accountValidator accountValidator +} + +type deviceDetector interface { + getSupportedHeaders() []dd.EvidenceKey + getDeviceInfo(evidence []onpremise.Evidence, ua string) (*deviceInfo, error) +} + +type accountValidator interface { + isAllowed(cfg config, req []byte) bool +} + +type evidenceExtractor interface { + fromHeaders(request *http.Request, httpHeaderKeys []dd.EvidenceKey) []stringEvidence + fromSuaPayload(payload []byte) []stringEvidence + extract(ctx hookstage.ModuleContext) ([]onpremise.Evidence, string, error) +} + +func (m Module) HandleEntrypointHook( + _ context.Context, + _ hookstage.ModuleInvocationContext, + payload hookstage.EntrypointPayload, +) (hookstage.HookResult[hookstage.EntrypointPayload], error) { + return handleAuctionEntryPointRequestHook( + m.config, + payload, + m.deviceDetector, + m.evidenceExtractor, + m.accountValidator, + ) +} + +func (m Module) HandleRawAuctionHook( + _ context.Context, + mCtx hookstage.ModuleInvocationContext, + _ hookstage.RawAuctionRequestPayload, +) (hookstage.HookResult[hookstage.RawAuctionRequestPayload], error) { + return handleAuctionRequestHook(mCtx, m.deviceDetector, m.evidenceExtractor) +} diff --git a/modules/fiftyonedegrees/devicedetection/module_test.go b/modules/fiftyonedegrees/devicedetection/module_test.go new file mode 100644 index 00000000000..7b8095ac431 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/module_test.go @@ -0,0 +1,703 @@ +package devicedetection + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "os" + "testing" + + "github.com/51Degrees/device-detection-go/v4/dd" + "github.com/51Degrees/device-detection-go/v4/onpremise" + "github.com/prebid/prebid-server/v2/hooks/hookstage" + "github.com/prebid/prebid-server/v2/modules/moduledeps" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +type mockAccValidator struct { + mock.Mock +} + +func (m *mockAccValidator) isAllowed(cfg config, req []byte) bool { + args := m.Called(cfg, req) + return args.Bool(0) +} + +type mockEvidenceExtractor struct { + mock.Mock +} + +func (m *mockEvidenceExtractor) fromHeaders(request *http.Request, httpHeaderKeys []dd.EvidenceKey) []stringEvidence { + args := m.Called(request, httpHeaderKeys) + + return args.Get(0).([]stringEvidence) +} + +func (m *mockEvidenceExtractor) fromSuaPayload(payload []byte) []stringEvidence { + args := m.Called(payload) + + return args.Get(0).([]stringEvidence) +} + +func (m *mockEvidenceExtractor) extract(ctx hookstage.ModuleContext) ([]onpremise.Evidence, string, error) { + args := m.Called(ctx) + + res := args.Get(0) + if res == nil { + return nil, args.String(1), args.Error(2) + } + + return res.([]onpremise.Evidence), args.String(1), args.Error(2) +} + +type mockDeviceDetector struct { + mock.Mock +} + +func (m *mockDeviceDetector) getSupportedHeaders() []dd.EvidenceKey { + args := m.Called() + return args.Get(0).([]dd.EvidenceKey) +} + +func (m *mockDeviceDetector) getDeviceInfo(evidence []onpremise.Evidence, ua string) (*deviceInfo, error) { + + args := m.Called(evidence, ua) + + res := args.Get(0) + + if res == nil { + return nil, args.Error(1) + } + + return res.(*deviceInfo), args.Error(1) +} + +func TestHandleEntrypointHookAccountNotAllowed(t *testing.T) { + var mockValidator mockAccValidator + + mockValidator.On("isAllowed", mock.Anything, mock.Anything).Return(false) + + module := Module{ + accountValidator: &mockValidator, + } + + _, err := module.HandleEntrypointHook(nil, hookstage.ModuleInvocationContext{}, hookstage.EntrypointPayload{}) + assert.Error(t, err) + assert.Equal(t, "hook execution failed: account not allowed", err.Error()) +} + +func TestHandleEntrypointHookAccountAllowed(t *testing.T) { + var mockValidator mockAccValidator + + mockValidator.On("isAllowed", mock.Anything, mock.Anything).Return(true) + + var mockEvidenceExtractor mockEvidenceExtractor + mockEvidenceExtractor.On("fromHeaders", mock.Anything, mock.Anything).Return( + []stringEvidence{{ + Prefix: "123", + Key: "key", + Value: "val", + }}, + ) + + mockEvidenceExtractor.On("fromSuaPayload", mock.Anything, mock.Anything).Return( + []stringEvidence{{ + Prefix: "123", + Key: "User-Agent", + Value: "ua", + }}, + ) + + var mockDeviceDetector mockDeviceDetector + + mockDeviceDetector.On("getSupportedHeaders").Return( + []dd.EvidenceKey{{ + Prefix: dd.HttpEvidenceQuery, + Key: "key", + }}, + ) + + module := Module{ + deviceDetector: &mockDeviceDetector, + evidenceExtractor: &mockEvidenceExtractor, + accountValidator: &mockValidator, + } + + result, err := module.HandleEntrypointHook(nil, hookstage.ModuleInvocationContext{}, hookstage.EntrypointPayload{}) + assert.NoError(t, err) + + assert.Equal( + t, result.ModuleContext[evidenceFromHeadersCtxKey], []stringEvidence{{ + Prefix: "123", + Key: "key", + Value: "val", + }}, + ) + + assert.Equal( + t, result.ModuleContext[evidenceFromSuaCtxKey], []stringEvidence{{ + Prefix: "123", + Key: "User-Agent", + Value: "ua", + }}, + ) +} + +func TestHandleRawAuctionHookNoCtx(t *testing.T) { + module := Module{} + + _, err := module.HandleRawAuctionHook( + nil, + hookstage.ModuleInvocationContext{}, + hookstage.RawAuctionRequestPayload{}, + ) + assert.Errorf(t, err, "entrypoint hook was not configured") +} + +func TestHandleRawAuctionHookExtractError(t *testing.T) { + var mockValidator mockAccValidator + + mockValidator.On("isAllowed", mock.Anything, mock.Anything).Return(true) + + var evidenceExtractorM mockEvidenceExtractor + evidenceExtractorM.On("extract", mock.Anything).Return( + nil, + "ua", + nil, + ) + + var mockDeviceDetector mockDeviceDetector + + module := Module{ + deviceDetector: &mockDeviceDetector, + evidenceExtractor: &evidenceExtractorM, + accountValidator: &mockValidator, + } + + mctx := make(hookstage.ModuleContext) + + mctx[ddEnabledCtxKey] = true + + result, err := module.HandleRawAuctionHook( + context.TODO(), hookstage.ModuleInvocationContext{ + ModuleContext: mctx, + }, + hookstage.RawAuctionRequestPayload{}, + ) + + assert.NoError(t, err) + assert.Equal(t, len(result.ChangeSet.Mutations()), 1) + assert.Equal(t, result.ChangeSet.Mutations()[0].Type(), hookstage.MutationUpdate) + + mutation := result.ChangeSet.Mutations()[0] + + body := []byte(`{}`) + + _, err = mutation.Apply(body) + assert.Errorf(t, err, "error extracting evidence") + + var mockEvidenceErrExtractor mockEvidenceExtractor + mockEvidenceErrExtractor.On("extract", mock.Anything).Return( + nil, + "", + errors.New("error"), + ) + + module.evidenceExtractor = &mockEvidenceErrExtractor + + result, err = module.HandleRawAuctionHook( + context.TODO(), hookstage.ModuleInvocationContext{ + ModuleContext: mctx, + }, + hookstage.RawAuctionRequestPayload{}, + ) + + assert.NoError(t, err) + + assert.Equal(t, len(result.ChangeSet.Mutations()), 1) + + assert.Equal(t, result.ChangeSet.Mutations()[0].Type(), hookstage.MutationUpdate) + + mutation = result.ChangeSet.Mutations()[0] + + _, err = mutation.Apply(body) + assert.Errorf(t, err, "error extracting evidence error") + +} + +func TestHandleRawAuctionHookEnrichment(t *testing.T) { + var mockValidator mockAccValidator + + mockValidator.On("isAllowed", mock.Anything, mock.Anything).Return(true) + + var mockEvidenceExtractor mockEvidenceExtractor + mockEvidenceExtractor.On("extract", mock.Anything).Return( + []onpremise.Evidence{ + { + Key: "key", + Value: "val", + }, + }, + "ua", + nil, + ) + + var deviceDetectorM mockDeviceDetector + + deviceDetectorM.On("getDeviceInfo", mock.Anything, mock.Anything).Return( + &deviceInfo{ + HardwareVendor: "Apple", + HardwareName: "Macbook", + DeviceType: "device", + PlatformVendor: "Apple", + PlatformName: "MacOs", + PlatformVersion: "14", + BrowserVendor: "Google", + BrowserName: "Crome", + BrowserVersion: "12", + ScreenPixelsWidth: 1024, + ScreenPixelsHeight: 1080, + PixelRatio: 223, + Javascript: true, + GeoLocation: true, + HardwareFamily: "Macbook", + HardwareModel: "Macbook", + HardwareModelVariants: "Macbook", + UserAgent: "ua", + DeviceId: "", + }, + nil, + ) + + module := Module{ + deviceDetector: &deviceDetectorM, + evidenceExtractor: &mockEvidenceExtractor, + accountValidator: &mockValidator, + } + + mctx := make(hookstage.ModuleContext) + mctx[ddEnabledCtxKey] = true + + result, err := module.HandleRawAuctionHook( + nil, hookstage.ModuleInvocationContext{ + ModuleContext: mctx, + }, + []byte{}, + ) + assert.NoError(t, err) + assert.Equal(t, len(result.ChangeSet.Mutations()), 1) + assert.Equal(t, result.ChangeSet.Mutations()[0].Type(), hookstage.MutationUpdate) + + mutation := result.ChangeSet.Mutations()[0] + + body := []byte(`{ + "device": { + "connectiontype": 2, + "ext": { + "atts": 0, + "ifv": "1B8EFA09-FF8F-4123-B07F-7283B50B3870" + }, + "sua": { + "source": 2, + "browsers": [ + { + "brand": "Not A(Brand", + "version": [ + "99", + "0", + "0", + "0" + ] + }, + { + "brand": "Google Chrome", + "version": [ + "121", + "0", + "6167", + "184" + ] + }, + { + "brand": "Chromium", + "version": [ + "121", + "0", + "6167", + "184" + ] + } + ], + "platform": { + "brand": "macOS", + "version": [ + "14", + "0", + "0" + ] + }, + "mobile": 0, + "architecture": "arm", + "model": "" + } + } + }`) + + mutationResult, err := mutation.Apply(body) + + require.JSONEq(t, string(mutationResult), `{ + "device": { + "connectiontype": 2, + "ext": { + "atts": 0, + "ifv": "1B8EFA09-FF8F-4123-B07F-7283B50B3870", + "fiftyonedegrees_deviceId":"" + }, + "sua": { + "source": 2, + "browsers": [ + { + "brand": "Not A(Brand", + "version": [ + "99", + "0", + "0", + "0" + ] + }, + { + "brand": "Google Chrome", + "version": [ + "121", + "0", + "6167", + "184" + ] + }, + { + "brand": "Chromium", + "version": [ + "121", + "0", + "6167", + "184" + ] + } + ], + "platform": { + "brand": "macOS", + "version": [ + "14", + "0", + "0" + ] + }, + "mobile": 0, + "architecture": "arm", + "model": "" + } + ,"devicetype":2,"ua":"ua","make":"Apple","model":"Macbook","os":"MacOs","osv":"14","h":1080,"w":1024,"pxratio":223,"js":1,"geoFetch":1} + }`) + + var deviceDetectorErrM mockDeviceDetector + + deviceDetectorErrM.On("getDeviceInfo", mock.Anything, mock.Anything).Return( + nil, + errors.New("error"), + ) + + module.deviceDetector = &deviceDetectorErrM + + result, err = module.HandleRawAuctionHook( + nil, hookstage.ModuleInvocationContext{ + ModuleContext: mctx, + }, + []byte{}, + ) + + assert.NoError(t, err) + + assert.Equal(t, len(result.ChangeSet.Mutations()), 1) + + assert.Equal(t, result.ChangeSet.Mutations()[0].Type(), hookstage.MutationUpdate) + + mutation = result.ChangeSet.Mutations()[0] + + _, err = mutation.Apply(body) + assert.Errorf(t, err, "error getting device info") +} + +func TestHandleRawAuctionHookEnrichmentWithErrors(t *testing.T) { + var mockValidator mockAccValidator + + mockValidator.On("isAllowed", mock.Anything, mock.Anything).Return(true) + + var mockEvidenceExtractor mockEvidenceExtractor + mockEvidenceExtractor.On("extract", mock.Anything).Return( + []onpremise.Evidence{ + { + Key: "key", + Value: "val", + }, + }, + "ua", + nil, + ) + + var mockDeviceDetector mockDeviceDetector + + mockDeviceDetector.On("getDeviceInfo", mock.Anything, mock.Anything).Return( + &deviceInfo{ + HardwareVendor: "Apple", + HardwareName: "Macbook", + DeviceType: "device", + PlatformVendor: "Apple", + PlatformName: "MacOs", + PlatformVersion: "14", + BrowserVendor: "Google", + BrowserName: "Crome", + BrowserVersion: "12", + ScreenPixelsWidth: 1024, + ScreenPixelsHeight: 1080, + PixelRatio: 223, + Javascript: true, + GeoLocation: true, + HardwareFamily: "Macbook", + HardwareModel: "Macbook", + HardwareModelVariants: "Macbook", + UserAgent: "ua", + DeviceId: "", + ScreenInchesHeight: 7, + }, + nil, + ) + + module := Module{ + deviceDetector: &mockDeviceDetector, + evidenceExtractor: &mockEvidenceExtractor, + accountValidator: &mockValidator, + } + + mctx := make(hookstage.ModuleContext) + mctx[ddEnabledCtxKey] = true + + result, err := module.HandleRawAuctionHook( + nil, hookstage.ModuleInvocationContext{ + ModuleContext: mctx, + }, + []byte{}, + ) + assert.NoError(t, err) + assert.Equal(t, len(result.ChangeSet.Mutations()), 1) + assert.Equal(t, result.ChangeSet.Mutations()[0].Type(), hookstage.MutationUpdate) + + mutation := result.ChangeSet.Mutations()[0] + + mutationResult, err := mutation.Apply(hookstage.RawAuctionRequestPayload(`{"device":{}}`)) + assert.NoError(t, err) + require.JSONEq(t, string(mutationResult), `{"device":{"devicetype":2,"ua":"ua","make":"Apple","model":"Macbook","os":"MacOs","osv":"14","h":1080,"w":1024,"pxratio":223,"js":1,"geoFetch":1,"ppi":154,"ext":{"fiftyonedegrees_deviceId":""}}}`) +} + +func TestConfigHashFromConfig(t *testing.T) { + cfg := config{ + Performance: performance{ + Profile: "", + Concurrency: nil, + Difference: nil, + AllowUnmatched: nil, + Drift: nil, + }, + } + + result := configHashFromConfig(&cfg) + assert.Equal(t, result.PerformanceProfile(), dd.Default) + assert.Equal(t, result.Concurrency(), uint16(0xa)) + assert.Equal(t, result.Difference(), int32(0)) + assert.Equal(t, result.AllowUnmatched(), false) + assert.Equal(t, result.Drift(), int32(0)) + + concurrency := 1 + difference := 1 + allowUnmatched := true + drift := 1 + + cfg = config{ + Performance: performance{ + Profile: "Balanced", + Concurrency: &concurrency, + Difference: &difference, + AllowUnmatched: &allowUnmatched, + Drift: &drift, + }, + } + + result = configHashFromConfig(&cfg) + assert.Equal(t, result.PerformanceProfile(), dd.Balanced) + assert.Equal(t, result.Concurrency(), uint16(1)) + assert.Equal(t, result.Difference(), int32(1)) + assert.Equal(t, result.AllowUnmatched(), true) + assert.Equal(t, result.Drift(), int32(1)) + + cfg = config{ + Performance: performance{ + Profile: "InMemory", + }, + } + result = configHashFromConfig(&cfg) + assert.Equal(t, result.PerformanceProfile(), dd.InMemory) + + cfg = config{ + Performance: performance{ + Profile: "HighPerformance", + }, + } + result = configHashFromConfig(&cfg) + assert.Equal(t, result.PerformanceProfile(), dd.HighPerformance) +} + +func TestSignDeviceData(t *testing.T) { + devicePld := map[string]any{ + "ext": map[string]any{ + "my-key": "my-value", + }, + } + + deviceInfo := deviceInfo{ + DeviceId: "test-device-id", + } + + result := signDeviceData(devicePld, &deviceInfo) + r, err := json.Marshal(result) + if err != nil { + t.Fatalf("unexpected error: %s", err.Error()) + } + + require.JSONEq( + t, + `{"ext":{"fiftyonedegrees_deviceId":"test-device-id","my-key":"my-value"}}`, + string(r), + ) +} + +func TestBuilderWithInvalidJson(t *testing.T) { + _, err := Builder([]byte(`{`), moduledeps.ModuleDeps{}) + assert.Error(t, err) + assert.Errorf(t, err, "failed to parse config") +} + +func TestBuilderWithInvalidConfig(t *testing.T) { + _, err := Builder([]byte(`{"data_file":{}}`), moduledeps.ModuleDeps{}) + assert.Error(t, err) + assert.Errorf(t, err, "invalid config") +} + +func TestBuilderHandleDeviceDetectorError(t *testing.T) { + var mockConfig config + mockConfig.Performance.Profile = "default" + testFile, _ := os.Create("test-builder-config.hash") + defer testFile.Close() + defer os.Remove("test-builder-config.hash") + + _, err := Builder( + []byte(`{ + "enabled": true, + "data_file": { + "path": "test-builder-config.hash", + "update": { + "auto": true, + "url": "https://my.datafile.com/datafile.gz", + "polling_interval": 3600, + "licence_key": "your_licence_key", + "product": "V4Enterprise" + } + }, + "account_filter": {"allow_list": ["123"]}, + "performance": { + "profile": "123", + "concurrency": 1, + "difference": 1, + "allow_unmatched": true, + "drift": 1 + } + }`), moduledeps.ModuleDeps{}, + ) + assert.Error(t, err) + assert.Errorf(t, err, "failed to create device detector") +} + +func TestHydrateFields(t *testing.T) { + deviceInfo := &deviceInfo{ + HardwareVendor: "Apple", + HardwareName: "Macbook", + DeviceType: "device", + PlatformVendor: "Apple", + PlatformName: "MacOs", + PlatformVersion: "14", + BrowserVendor: "Google", + BrowserName: "Crome", + BrowserVersion: "12", + ScreenPixelsWidth: 1024, + ScreenPixelsHeight: 1080, + PixelRatio: 223, + Javascript: true, + GeoLocation: true, + HardwareFamily: "Macbook", + HardwareModel: "Macbook", + HardwareModelVariants: "Macbook", + UserAgent: "ua", + DeviceId: "dev-ide", + } + + rawPld := `{ + "imp": [{ + "id": "", + "banner": { + "topframe": 1, + "format": [ + { + "w": 728, + "h": 90 + } + ], + "pos": 1 + }, + "bidfloor": 0.01, + "bidfloorcur": "USD" + }], + "device": { + "model": "Macintosh", + "w": 843, + "h": 901, + "dnt": 0, + "ua": "Mozilla/5.0 (Linux; Android 13; SAMSUNG SM-A037U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/23.0 Chrome/115.0.0.0 Mobile Safari/537.36", + "language": "en", + "sua": {"browsers":[{"brand":"Not/A)Brand","version":["99","0","0","0"]},{"brand":"Samsung Internet","version":["23","0","1","1"]},{"brand":"Chromium","version":["115","0","5790","168"]}],"platform":{"brand":"Android","version":["13","0","0"]},"mobile":1,"model":"SM-A037U","source":2}, + "ext": {"h":"901","w":843} + }, + "cur": [ + "USD" + ], + "tmax": 1700 + }` + + payload, err := hydrateFields(deviceInfo, []byte(rawPld)) + assert.NoError(t, err) + + var deviceHolder struct { + Device json.RawMessage `json:"device"` + } + + err = json.Unmarshal(payload, &deviceHolder) + if err != nil { + t.Fatalf("unexpected error: %s", err.Error()) + } + + require.JSONEq( + t, + `{"devicetype":2,"dnt":0,"ext":{"fiftyonedegrees_deviceId":"dev-ide","h":"901","w":843},"geoFetch":1,"h":901,"js":1,"language":"en","make":"Apple","model":"Macintosh","os":"MacOs","osv":"14","pxratio":223,"sua":{"browsers":[{"brand":"Not/A)Brand","version":["99","0","0","0"]},{"brand":"Samsung Internet","version":["23","0","1","1"]},{"brand":"Chromium","version":["115","0","5790","168"]}],"mobile":1,"model":"SM-A037U","platform":{"brand":"Android","version":["13","0","0"]},"source":2},"ua":"Mozilla/5.0 (Linux; Android 13; SAMSUNG SM-A037U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/23.0 Chrome/115.0.0.0 Mobile Safari/537.36","w":843}`, + string(deviceHolder.Device), + ) +} diff --git a/modules/fiftyonedegrees/devicedetection/request_headers_extractor.go b/modules/fiftyonedegrees/devicedetection/request_headers_extractor.go new file mode 100644 index 00000000000..8440886b353 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/request_headers_extractor.go @@ -0,0 +1,47 @@ +package devicedetection + +import ( + "net/http" + "strings" + + "github.com/51Degrees/device-detection-go/v4/dd" +) + +// evidenceFromRequestHeadersExtractor is a struct that extracts evidence from http request headers +type evidenceFromRequestHeadersExtractor struct{} + +func newEvidenceFromRequestHeadersExtractor() evidenceFromRequestHeadersExtractor { + return evidenceFromRequestHeadersExtractor{} +} + +func (x evidenceFromRequestHeadersExtractor) extract(request *http.Request, httpHeaderKeys []dd.EvidenceKey) []stringEvidence { + return x.extractEvidenceStrings(request, httpHeaderKeys) +} + +func (x evidenceFromRequestHeadersExtractor) extractEvidenceStrings(r *http.Request, keys []dd.EvidenceKey) []stringEvidence { + evidence := make([]stringEvidence, 0) + for _, e := range keys { + if e.Prefix == dd.HttpEvidenceQuery { + continue + } + + // Get evidence from headers + headerVal := r.Header.Get(e.Key) + if headerVal == "" { + continue + } + + if e.Key != secUaFullVersionList && e.Key != secChUa { + headerVal = strings.Replace(headerVal, "\"", "", -1) + } + + if headerVal != "" { + evidence = append(evidence, stringEvidence{ + Prefix: headerPrefix, + Key: e.Key, + Value: headerVal, + }) + } + } + return evidence +} diff --git a/modules/fiftyonedegrees/devicedetection/request_headers_extractor_test.go b/modules/fiftyonedegrees/devicedetection/request_headers_extractor_test.go new file mode 100644 index 00000000000..77fbed3a42f --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/request_headers_extractor_test.go @@ -0,0 +1,118 @@ +package devicedetection + +import ( + "net/http" + "testing" + + "github.com/51Degrees/device-detection-go/v4/dd" + "github.com/stretchr/testify/assert" +) + +func TestExtractEvidenceStrings(t *testing.T) { + tests := []struct { + name string + headers map[string]string + keys []dd.EvidenceKey + expectedEvidence []stringEvidence + }{ + { + name: "Ignored_query_evidence", + headers: map[string]string{ + "User-Agent": "Mozilla/5.0", + }, + keys: []dd.EvidenceKey{ + {Prefix: dd.HttpEvidenceQuery, Key: "User-Agent"}, + }, + expectedEvidence: []stringEvidence{}, + }, + { + name: "Empty_headers", + headers: map[string]string{}, + keys: []dd.EvidenceKey{ + {Prefix: dd.HttpHeaderString, Key: "User-Agent"}, + }, + expectedEvidence: []stringEvidence{}, + }, + { + name: "Single_header", + headers: map[string]string{ + "User-Agent": "Mozilla/5.0", + }, + keys: []dd.EvidenceKey{ + {Prefix: dd.HttpHeaderString, Key: "User-Agent"}, + }, + expectedEvidence: []stringEvidence{ + {Prefix: headerPrefix, Key: "User-Agent", Value: "Mozilla/5.0"}, + }, + }, + { + name: "Multiple_headers", + headers: map[string]string{ + "User-Agent": "Mozilla/5.0", + "Accept": "text/html", + }, + keys: []dd.EvidenceKey{ + {Prefix: dd.HttpHeaderString, Key: "User-Agent"}, + {Prefix: dd.HttpEvidenceQuery, Key: "Query"}, + {Prefix: dd.HttpHeaderString, Key: "Accept"}, + }, + expectedEvidence: []stringEvidence{ + {Prefix: headerPrefix, Key: "User-Agent", Value: "Mozilla/5.0"}, + {Prefix: headerPrefix, Key: "Accept", Value: "text/html"}, + }, + }, + { + name: "Header_with_quotes_removed", + headers: map[string]string{ + "IP-List": "\"92.0.4515.159\"", + }, + keys: []dd.EvidenceKey{ + {Prefix: dd.HttpHeaderString, Key: "IP-List"}, + }, + expectedEvidence: []stringEvidence{ + {Prefix: headerPrefix, Key: "IP-List", Value: "92.0.4515.159"}, + }, + }, + { + name: "Sec-CH-UA_headers_with_quotes_left", + headers: map[string]string{ + "Sec-CH-UA": "\"Chromium\";v=\"92\", \"Google Chrome\";v=\"92\"", + }, + keys: []dd.EvidenceKey{ + {Prefix: dd.HttpHeaderString, Key: secChUa}, + }, + expectedEvidence: []stringEvidence{ + {Prefix: headerPrefix, Key: secChUa, Value: "\"Chromium\";v=\"92\", \"Google Chrome\";v=\"92\""}, + }, + }, + { + name: "Sec-CH-UA-Full-Version-List_headers_with_quotes_left", + headers: map[string]string{ + "Sec-CH-UA-Full-Version-List": "\"92.0.4515.159\"", + }, + keys: []dd.EvidenceKey{ + {Prefix: dd.HttpHeaderString, Key: secUaFullVersionList}, + }, + expectedEvidence: []stringEvidence{ + {Prefix: headerPrefix, Key: secUaFullVersionList, Value: "\"92.0.4515.159\""}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req := http.Request{ + Header: make(map[string][]string), + } + + for key, value := range test.headers { + req.Header.Set(key, value) + } + + extractor := newEvidenceFromRequestHeadersExtractor() + evidence := extractor.extractEvidenceStrings(&req, test.keys) + + assert.Equal(t, test.expectedEvidence, evidence) + }) + } +} diff --git a/modules/fiftyonedegrees/devicedetection/sample/pbs.json b/modules/fiftyonedegrees/devicedetection/sample/pbs.json new file mode 100644 index 00000000000..43fd28610f1 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/sample/pbs.json @@ -0,0 +1,84 @@ +{ + "adapters": [ + { + "appnexus": { + "enabled": true + } + } + ], + "gdpr": { + "enabled": true, + "default_value": 0, + "timeouts_ms": { + "active_vendorlist_fetch": 900000 + } + }, + "stored_requests": { + "filesystem": { + "enabled": true, + "directorypath": "sample/stored" + } + }, + "stored_responses": { + "filesystem": { + "enabled": true, + "directorypath": "sample/stored" + } + }, + "hooks": { + "enabled": true, + "modules": { + "fiftyonedegrees": { + "devicedetection": { + "enabled": true, + "data_file": { + "path": "TAC-HashV41.hash", + "update": { + "auto": false, + "polling_interval": 3600, + "license_key": "YOUR_LICENSE_KEY", + "product": "V4Enterprise" + } + }, + "performance": { + "profile": "InMemory" + } + } + } + }, + "host_execution_plan": { + "endpoints": { + "/openrtb2/auction": { + "stages": { + "entrypoint": { + "groups": [ + { + "timeout": 10, + "hook_sequence": [ + { + "module_code": "fiftyonedegrees.devicedetection", + "hook_impl_code": "fiftyone-devicedetection-entrypoint-hook" + } + ] + } + ] + }, + "raw_auction_request": { + "groups": [ + { + "timeout": 10, + "hook_sequence": [ + { + "module_code": "fiftyonedegrees.devicedetection", + "hook_impl_code": "fiftyone-devicedetection-raw-auction-request-hook" + } + ] + } + ] + } + } + } + } + } + } +} \ No newline at end of file diff --git a/modules/fiftyonedegrees/devicedetection/sample/request_data.json b/modules/fiftyonedegrees/devicedetection/sample/request_data.json new file mode 100644 index 00000000000..1f6bc8900f8 --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/sample/request_data.json @@ -0,0 +1,114 @@ +{ + "imp": [{ + "ext": { + "data": { + "adserver": { + "name": "gam", + "adslot": "test" + }, + "pbadslot": "test", + "gpid": "test" + }, + "gpid": "test", + "prebid": { + "bidder": { + "appnexus": { + "placement_id": 1, + "use_pmt_rule": false + } + }, + "adunitcode": "25e8ad9f-13a4-4404-ba74-f9eebff0e86c", + "floors": { + "floorMin": 0.01 + } + } + }, + "id": "2529eeea-813e-4da6-838f-f91c28d64867", + "banner": { + "topframe": 1, + "format": [ + { + "w": 728, + "h": 90 + } + ], + "pos": 1 + }, + "bidfloor": 0.01, + "bidfloorcur": "USD" + }], + "site": { + "domain": "test.com", + "publisher": { + "domain": "test.com", + "id": "1" + }, + "page": "https://www.test.com/" + }, + "device": { + "ua": "Mozilla/5.0 (Linux; Android 11; SM-G998W) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Mobile Safari/537.36" + }, + "id": "fc4670ce-4985-4316-a245-b43c885dc37a", + "test": 1, + "cur": [ + "USD" + ], + "source": { + "ext": { + "schain": { + "ver": "1.0", + "complete": 1, + "nodes": [ + { + "asi": "example.com", + "sid": "1234", + "hp": 1 + } + ] + } + } + }, + "ext": { + "prebid": { + "cache": { + "bids": { + "returnCreative": true + }, + "vastxml": { + "returnCreative": true + } + }, + "auctiontimestamp": 1698390609882, + "targeting": { + "includewinners": true, + "includebidderkeys": false + }, + "schains": [ + { + "bidders": [ + "appnexus" + ], + "schain": { + "ver": "1.0", + "complete": 1, + "nodes": [ + { + "asi": "example.com", + "sid": "1234", + "hp": 1 + } + ] + } + } + ], + "floors": { + "enabled": false, + "floorMin": 0.01, + "floorMinCur": "USD" + }, + "createtids": false + } + }, + "user": {}, + "tmax": 1700 +} \ No newline at end of file diff --git a/modules/fiftyonedegrees/devicedetection/sua_payload_extractor.go b/modules/fiftyonedegrees/devicedetection/sua_payload_extractor.go new file mode 100644 index 00000000000..ab69210449f --- /dev/null +++ b/modules/fiftyonedegrees/devicedetection/sua_payload_extractor.go @@ -0,0 +1,144 @@ +package devicedetection + +import ( + "fmt" + "strings" + + "github.com/spf13/cast" + "github.com/tidwall/gjson" +) + +const ( + secChUaArch = "Sec-Ch-Ua-Arch" + secChUaMobile = "Sec-Ch-Ua-Mobile" + secChUaModel = "Sec-Ch-Ua-Model" + secChUaPlatform = "Sec-Ch-Ua-Platform" + secUaFullVersionList = "Sec-Ch-Ua-Full-Version-List" + secChUaPlatformVersion = "Sec-Ch-Ua-Platform-Version" + secChUa = "Sec-Ch-Ua" + + userAgentHeader = "User-Agent" +) + +// evidenceFromSUAPayloadExtractor extracts evidence from the SUA payload of device +type evidenceFromSUAPayloadExtractor struct{} + +func newEvidenceFromSUAPayloadExtractor() evidenceFromSUAPayloadExtractor { + return evidenceFromSUAPayloadExtractor{} +} + +// Extract extracts evidence from the SUA payload +func (x evidenceFromSUAPayloadExtractor) extract(payload []byte) []stringEvidence { + if payload != nil { + return x.extractEvidenceStrings(payload) + } + + return nil +} + +var ( + uaPath = "device.ua" + archPath = "device.sua.architecture" + mobilePath = "device.sua.mobile" + modelPath = "device.sua.model" + platformBrandPath = "device.sua.platform.brand" + platformVersionPath = "device.sua.platform.version" + browsersPath = "device.sua.browsers" +) + +// extractEvidenceStrings extracts evidence from the SUA payload +func (x evidenceFromSUAPayloadExtractor) extractEvidenceStrings(payload []byte) []stringEvidence { + res := make([]stringEvidence, 0, 10) + + uaResult := gjson.GetBytes(payload, uaPath) + if uaResult.Exists() { + res = append( + res, + stringEvidence{Prefix: headerPrefix, Key: userAgentHeader, Value: uaResult.String()}, + ) + } + + archResult := gjson.GetBytes(payload, archPath) + if archResult.Exists() { + res = x.appendEvidenceIfExists(res, secChUaArch, archResult.String()) + } + + mobileResult := gjson.GetBytes(payload, mobilePath) + if mobileResult.Exists() { + res = x.appendEvidenceIfExists(res, secChUaMobile, mobileResult.String()) + } + + modelResult := gjson.GetBytes(payload, modelPath) + if modelResult.Exists() { + res = x.appendEvidenceIfExists(res, secChUaModel, modelResult.String()) + } + + platformBrandResult := gjson.GetBytes(payload, platformBrandPath) + if platformBrandResult.Exists() { + res = x.appendEvidenceIfExists(res, secChUaPlatform, platformBrandResult.String()) + } + + platformVersionResult := gjson.GetBytes(payload, platformVersionPath) + if platformVersionResult.Exists() { + res = x.appendEvidenceIfExists( + res, + secChUaPlatformVersion, + strings.Join(resultToStringArray(platformVersionResult.Array()), "."), + ) + } + + browsersResult := gjson.GetBytes(payload, browsersPath) + if browsersResult.Exists() { + res = x.appendEvidenceIfExists(res, secUaFullVersionList, x.extractBrowsers(browsersResult)) + + } + + return res +} + +func resultToStringArray(array []gjson.Result) []string { + strArray := make([]string, len(array)) + for i, result := range array { + strArray[i] = result.String() + } + + return strArray +} + +// appendEvidenceIfExists appends evidence to the destination if the value is not nil +func (x evidenceFromSUAPayloadExtractor) appendEvidenceIfExists(destination []stringEvidence, name string, value interface{}) []stringEvidence { + if value != nil { + valStr := cast.ToString(value) + if len(valStr) == 0 { + return destination + } + + return append( + destination, + stringEvidence{Prefix: headerPrefix, Key: name, Value: valStr}, + ) + } + + return destination +} + +// extractBrowsers extracts browsers from the SUA payload +func (x evidenceFromSUAPayloadExtractor) extractBrowsers(browsers gjson.Result) string { + if !browsers.IsArray() { + return "" + } + + browsersRaw := make([]string, len(browsers.Array())) + + for i, result := range browsers.Array() { + brand := result.Get("brand").String() + versionsRaw := result.Get("version").Array() + versions := resultToStringArray(versionsRaw) + + browsersRaw[i] = fmt.Sprintf(`"%s";v="%s"`, brand, strings.Join(versions, ".")) + } + + res := strings.Join(browsersRaw, ",") + + return res +}