Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support MySql driver #2356

Merged
merged 31 commits into from
Nov 17, 2022
Merged

Support MySql driver #2356

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
4192762
Add support of MySql database driver
victorshevtsov Aug 17, 2022
db51dcf
Merge branch 'prebid:master' into master
victorshevtsov Aug 19, 2022
6343c15
Fix params for fetchDelta query for MySql driver
victorshevtsov Aug 20, 2022
578c709
Fixed DatabaseUpdatePolling.validate method
victorshevtsov Sep 13, 2022
3a8c16f
Merge branch 'prebid:master' into master
victorshevtsov Oct 5, 2022
71e91fb
Migrate Database Connection Config
victorshevtsov Oct 5, 2022
2e5a7a8
Merge branch 'prebid:master' into master
victorshevtsov Oct 5, 2022
71801ff
Refactor dbFetcher
victorshevtsov Oct 12, 2022
257b172
fix typo - storedRequestsTests
victorshevtsov Oct 12, 2022
c2c68ff
Call v.GetString and v.GetInt instead of v.Get
victorshevtsov Oct 12, 2022
81da8b3
Reduce noise on startup if no dbname in the config
victorshevtsov Oct 12, 2022
c35b29d
remove connStringMySql & connStringPostgres funcs
victorshevtsov Nov 9, 2022
ae396d2
Update query example with `$LAST_UPDATED`
victorshevtsov Nov 9, 2022
1a2b592
Remove `occurrence` struct
victorshevtsov Nov 9, 2022
bc16ac3
consolidated `if err...` clause
victorshevtsov Nov 9, 2022
aa4c904
Fix typo LAST_UPDATE -> LAST_UPDATED
victorshevtsov Nov 9, 2022
17b7068
Update migration
victorshevtsov Nov 9, 2022
9f46445
Eliminate the need for `relfect`
victorshevtsov Nov 11, 2022
0a2c686
Simplify func signatures of db providers
victorshevtsov Nov 11, 2022
bf836b6
Update comments & docs explaining query parameters
victorshevtsov Nov 11, 2022
5afcc5e
Migrate query parameters
victorshevtsov Nov 11, 2022
5351540
Merge branch 'master' into master
victorshevtsov Nov 11, 2022
98758bc
Update check for wildcards in SQL query
victorshevtsov Nov 15, 2022
42e2f5c
Update SQL queries migration
victorshevtsov Nov 15, 2022
3a63b20
Delete unused constants
victorshevtsov Nov 16, 2022
a686ab8
Update doc/stored-requests.md
victorshevtsov Nov 16, 2022
a72deca
Shorten the test using the testify package
victorshevtsov Nov 16, 2022
9b22905
Add check for a nil pointer
victorshevtsov Nov 16, 2022
2f63fd7
Add migration for ID_LIST parameter of SQL queries
victorshevtsov Nov 16, 2022
f0de6a6
Test conn strings for MySql and Postgres providers
victorshevtsov Nov 16, 2022
8fa2287
Merge branch 'prebid:master' into master
victorshevtsov Nov 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
201 changes: 165 additions & 36 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -850,20 +850,6 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) {
v.SetDefault("stored_requests.filesystem.enabled", false)
v.SetDefault("stored_requests.filesystem.directorypath", "./stored_requests/data/by_id")
v.SetDefault("stored_requests.directorypath", "./stored_requests/data/by_id")
v.SetDefault("stored_requests.postgres.connection.dbname", "")
v.SetDefault("stored_requests.postgres.connection.host", "")
v.SetDefault("stored_requests.postgres.connection.port", 0)
v.SetDefault("stored_requests.postgres.connection.user", "")
v.SetDefault("stored_requests.postgres.connection.password", "")
v.SetDefault("stored_requests.postgres.fetcher.query", "")
v.SetDefault("stored_requests.postgres.fetcher.amp_query", "")
v.SetDefault("stored_requests.postgres.initialize_caches.timeout_ms", 0)
v.SetDefault("stored_requests.postgres.initialize_caches.query", "")
v.SetDefault("stored_requests.postgres.initialize_caches.amp_query", "")
v.SetDefault("stored_requests.postgres.poll_for_updates.refresh_rate_seconds", 0)
v.SetDefault("stored_requests.postgres.poll_for_updates.timeout_ms", 0)
v.SetDefault("stored_requests.postgres.poll_for_updates.query", "")
v.SetDefault("stored_requests.postgres.poll_for_updates.amp_query", "")
v.SetDefault("stored_requests.http.endpoint", "")
v.SetDefault("stored_requests.http.amp_endpoint", "")
v.SetDefault("stored_requests.in_memory_cache.type", "none")
Expand All @@ -880,17 +866,6 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) {
// PBS is not in the business of storing video content beyond the normal prebid cache system.
v.SetDefault("stored_video_req.filesystem.enabled", false)
v.SetDefault("stored_video_req.filesystem.directorypath", "")
v.SetDefault("stored_video_req.postgres.connection.dbname", "")
v.SetDefault("stored_video_req.postgres.connection.host", "")
v.SetDefault("stored_video_req.postgres.connection.port", 0)
v.SetDefault("stored_video_req.postgres.connection.user", "")
v.SetDefault("stored_video_req.postgres.connection.password", "")
v.SetDefault("stored_video_req.postgres.fetcher.query", "")
v.SetDefault("stored_video_req.postgres.initialize_caches.timeout_ms", 0)
v.SetDefault("stored_video_req.postgres.initialize_caches.query", "")
v.SetDefault("stored_video_req.postgres.poll_for_updates.refresh_rate_seconds", 0)
v.SetDefault("stored_video_req.postgres.poll_for_updates.timeout_ms", 0)
v.SetDefault("stored_video_req.postgres.poll_for_updates.query", "")
v.SetDefault("stored_video_req.http.endpoint", "")
v.SetDefault("stored_video_req.in_memory_cache.type", "none")
v.SetDefault("stored_video_req.in_memory_cache.ttl_seconds", 0)
Expand All @@ -904,17 +879,6 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) {
v.SetDefault("stored_video_req.http_events.timeout_ms", 0)
v.SetDefault("stored_responses.filesystem.enabled", false)
v.SetDefault("stored_responses.filesystem.directorypath", "")
v.SetDefault("stored_responses.postgres.connection.dbname", "")
v.SetDefault("stored_responses.postgres.connection.host", "")
v.SetDefault("stored_responses.postgres.connection.port", 0)
v.SetDefault("stored_responses.postgres.connection.user", "")
v.SetDefault("stored_responses.postgres.connection.password", "")
v.SetDefault("stored_responses.postgres.fetcher.query", "")
v.SetDefault("stored_responses.postgres.initialize_caches.timeout_ms", 0)
v.SetDefault("stored_responses.postgres.initialize_caches.query", "")
v.SetDefault("stored_responses.postgres.poll_for_updates.refresh_rate_seconds", 0)
v.SetDefault("stored_responses.postgres.poll_for_updates.timeout_ms", 0)
v.SetDefault("stored_responses.postgres.poll_for_updates.query", "")
v.SetDefault("stored_responses.http.endpoint", "")
v.SetDefault("stored_responses.in_memory_cache.type", "none")
v.SetDefault("stored_responses.in_memory_cache.ttl_seconds", 0)
Expand Down Expand Up @@ -1039,6 +1003,7 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) {
migrateConfigPurposeOneTreatment(v)
migrateConfigSpecialFeature1(v)
migrateConfigTCF2PurposeFlags(v)
migrateConfigDatabaseConnection(v)

// These defaults must be set after the migrate functions because those functions look for the presence of these
// config fields and there isn't a way to detect presence of a config field using the viper package if a default
Expand Down Expand Up @@ -1190,6 +1155,170 @@ func migrateConfigTCF2PurposeEnabledFlags(v *viper.Viper) {
}
}

func migrateConfigDatabaseConnection(v *viper.Viper) {

type QueryParamMigration struct {
old string
new string
}

type QueryMigration struct {
name string
params []QueryParamMigration
}

type Migration struct {
old string
new string
fields []string
queryMigrations []QueryMigration
}

queryParamMigrations := struct {
RequestIdList QueryParamMigration
ImpIdList QueryParamMigration
IdList QueryParamMigration
LastUpdated QueryParamMigration
}{
RequestIdList: QueryParamMigration{
old: "%REQUEST_ID_LIST%",
new: "$REQUEST_ID_LIST",
},
ImpIdList: QueryParamMigration{
old: "%IMP_ID_LIST%",
new: "$IMP_ID_LIST",
},
IdList: QueryParamMigration{
old: "%ID_LIST%",
new: "$ID_LIST",
},
LastUpdated: QueryParamMigration{
old: "$1",
new: "$LAST_UPDATED",
},
}

queryMigrations := []QueryMigration{
{
name: "fetcher.query",
params: []QueryParamMigration{queryParamMigrations.RequestIdList, queryParamMigrations.ImpIdList, queryParamMigrations.IdList},
},
{
name: "fetcher.amp_query",
params: []QueryParamMigration{queryParamMigrations.RequestIdList, queryParamMigrations.ImpIdList, queryParamMigrations.IdList},
},
{
name: "poll_for_updates.query",
params: []QueryParamMigration{queryParamMigrations.LastUpdated},
},
{
name: "poll_for_updates.amp_query",
params: []QueryParamMigration{queryParamMigrations.LastUpdated},
},
}

migrations := []Migration{
{
old: "stored_requests.postgres",
new: "stored_requests.database",
fields: []string{
"connection.dbname",
"connection.host",
"connection.port",
"connection.user",
"connection.password",
"fetcher.query",
"fetcher.amp_query",
"initialize_caches.timeout_ms",
"initialize_caches.query",
"initialize_caches.amp_query",
"poll_for_updates.refresh_rate_seconds",
"poll_for_updates.timeout_ms",
"poll_for_updates.query",
"poll_for_updates.amp_query",
},
queryMigrations: queryMigrations,
},
{
old: "stored_video_req.postgres",
new: "stored_video_req.database",
fields: []string{
"connection.dbname",
"connection.host",
"connection.port",
"connection.user",
"connection.password",
"fetcher.query",
"initialize_caches.timeout_ms",
"initialize_caches.query",
"poll_for_updates.refresh_rate_seconds",
"poll_for_updates.timeout_ms",
"poll_for_updates.query",
},
queryMigrations: queryMigrations,
},
{
old: "stored_responses.postgres",
new: "stored_responses.database",
fields: []string{
"connection.dbname",
"connection.host",
"connection.port",
"connection.user",
"connection.password",
"fetcher.query",
"initialize_caches.timeout_ms",
"initialize_caches.query",
"poll_for_updates.refresh_rate_seconds",
"poll_for_updates.timeout_ms",
"poll_for_updates.query",
},
queryMigrations: queryMigrations,
},
}

for _, migration := range migrations {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, I'm wondering if this function should instead only choose .database or .postgres instead of mixing the contents of the two if both are present. I think the behavior will be less confusing if someone has both set. Maybe something like:

for _, migration := range migrations {
	driverField := migration.new + ".connection.driver"
	if !v.IsSet(migration.new) && v.IsSet(migration.old) {
		glog.Warning(fmt.Sprintf("%s is deprecated and should be changed to %s", migration.old, migration.new))
		glog.Warning(fmt.Sprintf("%s is not set, using default (postgres)", driverField))
		v.Set(driverField, "postgres")

		for _, field := range migration.fields {
			oldField := migration.old + "." + field
			newField := migration.new + "." + field
			if v.IsSet(oldField) {
				glog.Warning(fmt.Sprintf("%s is deprecated and should be changed to %s", oldField, newField))
				v.Set(newField, v.Get(oldField))
			}
		}
	} else if v.IsSet(migration.new) && v.IsSet(migration.old) {
		glog.Warning(fmt.Sprintf("using %s and ignoring deprecated %s", migration.new, migration.old))

		for _, field := range migration.fields {
			oldField := migration.old + "." + field
			newField := migration.new + "." + field
			if v.IsSet(oldField) {
				glog.Warning(fmt.Sprintf("using %s and ignoring deprecated %s", newField, oldField))
			}
		}
	}
}

@VeronikaSolovei9 thoughts?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point! Yes, "new" config should be used if it's specified.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated.

driverField := migration.new + ".connection.driver"
if !v.IsSet(migration.new) && v.IsSet(migration.old) {
glog.Warning(fmt.Sprintf("%s is deprecated and should be changed to %s", migration.old, migration.new))
glog.Warning(fmt.Sprintf("%s is not set, using default (postgres)", driverField))
v.Set(driverField, "postgres")

for _, field := range migration.fields {
oldField := migration.old + "." + field
newField := migration.new + "." + field
if v.IsSet(oldField) {
glog.Warning(fmt.Sprintf("%s is deprecated and should be changed to %s", oldField, newField))
v.Set(newField, v.Get(oldField))
}
}

for _, queryMigration := range migration.queryMigrations {
oldQueryField := migration.old + "." + queryMigration.name
newQueryField := migration.new + "." + queryMigration.name
queryString := v.GetString(oldQueryField)
for _, queryParam := range queryMigration.params {
if strings.Contains(queryString, queryParam.old) {
glog.Warning(fmt.Sprintf("Query param %s for %s is deprecated and should be changed to %s", queryParam.old, oldQueryField, queryParam.new))
queryString = strings.ReplaceAll(queryString, queryParam.old, queryParam.new)
v.Set(newQueryField, queryString)
}
}
}
} else if v.IsSet(migration.new) && v.IsSet(migration.old) {
glog.Warning(fmt.Sprintf("using %s and ignoring deprecated %s", migration.new, migration.old))

for _, field := range migration.fields {
oldField := migration.old + "." + field
newField := migration.new + "." + field
if v.IsSet(oldField) {
glog.Warning(fmt.Sprintf("using %s and ignoring deprecated %s", newField, oldField))
}
}
}
}
}

func setBidderDefaults(v *viper.Viper, bidder string) {
adapterCfgPrefix := "adapters." + bidder
v.BindEnv(adapterCfgPrefix+".disabled", "")
Expand Down
Loading