From 87d706107330cde71ff4ab29b0321edebad796f7 Mon Sep 17 00:00:00 2001 From: Kehan Pan Date: Sat, 3 Aug 2024 16:36:16 +0800 Subject: [PATCH] ip address geolocation lookup Signed-off-by: Kehan Pan --- config/account.go | 16 +- config/account_test.go | 24 + config/config.go | 43 +- config/config_test.go | 53 + config/countrycode/countrycode.go | 63 ++ config/countrycode/countrycode_test.go | 60 ++ country-codes.csv | 244 +++++ endpoints/openrtb2/auction_benchmark_test.go | 1 + endpoints/openrtb2/test_utils.go | 1 + errortypes/severity.go | 10 + errortypes/severity_test.go | 28 + exchange/exchange.go | 137 ++- exchange/exchange_test.go | 557 ++++++++++- exchange/geolocation.go | 168 ++++ exchange/geolocation_test.go | 643 ++++++++++++ exchange/utils.go | 62 +- exchange/utils_test.go | 65 +- geolocation/geoinfo.go | 36 + geolocation/geolocation.go | 43 + geolocation/geolocation_test.go | 38 + .../geolocationtest/geolocationtest.go | 44 + geolocation/maxmind/maxmind.go | 97 ++ geolocation/maxmind/maxmind_test.go | 147 +++ .../test-data/GeoLite2-City-Bad-Data.tar.gz | Bin 0 -> 136 bytes .../maxmind/test-data/GeoLite2-City.tar.gz | Bin 0 -> 11087 bytes geolocation/maxmind/test-data/nothing.mmdb | 0 geolocation/maxmind/test-data/nothing.tar.gz | Bin 0 -> 111 bytes .../remotefilesyncer/remotefilesyncer.go | 284 ++++++ .../remotefilesyncer/remotefilesyncer_test.go | 915 ++++++++++++++++++ go.mod | 16 +- go.sum | 18 +- main.go | 10 + main_test.go | 4 + metrics/config/metrics.go | 9 + metrics/go_metrics.go | 18 + metrics/go_metrics_test.go | 35 + metrics/metrics.go | 1 + metrics/metrics_mock.go | 4 + metrics/prometheus/preload.go | 4 + metrics/prometheus/prometheus.go | 18 + metrics/prometheus/prometheus_test.go | 29 + privacy/scrubber.go | 6 +- privacy/scrubber_test.go | 2 +- router/router.go | 41 +- util/task/func_runner.go | 10 +- util/task/func_runner_test.go | 14 + util/task/ticker_task.go | 36 +- util/task/ticker_task_test.go | 40 + util/timeutil/time.go | 47 + util/timeutil/time_test.go | 64 ++ 50 files changed, 4084 insertions(+), 121 deletions(-) create mode 100644 config/countrycode/countrycode.go create mode 100644 config/countrycode/countrycode_test.go create mode 100644 country-codes.csv create mode 100644 exchange/geolocation.go create mode 100644 exchange/geolocation_test.go create mode 100644 geolocation/geoinfo.go create mode 100644 geolocation/geolocation.go create mode 100644 geolocation/geolocation_test.go create mode 100644 geolocation/geolocationtest/geolocationtest.go create mode 100644 geolocation/maxmind/maxmind.go create mode 100644 geolocation/maxmind/maxmind_test.go create mode 100644 geolocation/maxmind/test-data/GeoLite2-City-Bad-Data.tar.gz create mode 100644 geolocation/maxmind/test-data/GeoLite2-City.tar.gz create mode 100644 geolocation/maxmind/test-data/nothing.mmdb create mode 100644 geolocation/maxmind/test-data/nothing.tar.gz create mode 100644 geolocation/remotefilesyncer/remotefilesyncer.go create mode 100644 geolocation/remotefilesyncer/remotefilesyncer_test.go create mode 100644 util/timeutil/time_test.go diff --git a/config/account.go b/config/account.go index 80ce6d41dea..f348360cb0a 100644 --- a/config/account.go +++ b/config/account.go @@ -42,6 +42,7 @@ type Account struct { DefaultBidLimit int `mapstructure:"default_bid_limit" json:"default_bid_limit"` BidAdjustments *openrtb_ext.ExtRequestPrebidBidAdjustments `mapstructure:"bidadjustments" json:"bidadjustments"` Privacy AccountPrivacy `mapstructure:"privacy" json:"privacy"` + GeoLocation AccountGeoLocation `mapstructure:"geolocation" json:"geolocation"` } // CookieSync represents the account-level defaults for the cookie sync endpoint. @@ -156,9 +157,10 @@ type AccountGDPR struct { Purpose9 AccountGDPRPurpose `mapstructure:"purpose9" json:"purpose9"` Purpose10 AccountGDPRPurpose `mapstructure:"purpose10" json:"purpose10"` // Hash table of purpose configs for convenient purpose config lookup - PurposeConfigs map[consentconstants.Purpose]*AccountGDPRPurpose - PurposeOneTreatment AccountGDPRPurposeOneTreatment `mapstructure:"purpose_one_treatment" json:"purpose_one_treatment"` - SpecialFeature1 AccountGDPRSpecialFeature `mapstructure:"special_feature1" json:"special_feature1"` + PurposeConfigs map[consentconstants.Purpose]*AccountGDPRPurpose + PurposeOneTreatment AccountGDPRPurposeOneTreatment `mapstructure:"purpose_one_treatment" json:"purpose_one_treatment"` + SpecialFeature1 AccountGDPRSpecialFeature `mapstructure:"special_feature1" json:"special_feature1"` + ConsentStringMeansInScope *bool `mapstructure:"consent_string_means_in_scope" json:"consent_string_means_in_scope"` } // EnabledForChannelType indicates whether GDPR is turned on at the account level for the specified channel type @@ -351,6 +353,14 @@ type CookieDeprecation struct { TTLSec int `mapstructure:"ttl_sec"` } +type AccountGeoLocation struct { + Enabled bool `mapstructure:"enabled" json:"enabled,omitempty"` +} + +func (g *AccountGeoLocation) IsGeoLocationEnabled() bool { + return g.Enabled +} + // AccountDSA represents DSA configuration type AccountDSA struct { Default string `mapstructure:"default" json:"default"` diff --git a/config/account_test.go b/config/account_test.go index c529af09f15..5eb4df76352 100644 --- a/config/account_test.go +++ b/config/account_test.go @@ -1014,3 +1014,27 @@ func TestIPMaskingValidate(t *testing.T) { }) } } + +func TestGeoLocation(t *testing.T) { + tests := []struct { + geoloc *AccountGeoLocation + expected bool + }{ + { + geoloc: &AccountGeoLocation{ + Enabled: true, + }, + expected: true, + }, + { + geoloc: &AccountGeoLocation{ + Enabled: false, + }, + expected: false, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expected, test.geoloc.IsGeoLocationEnabled()) + } +} diff --git a/config/config.go b/config/config.go index bc1e17fbc29..d3da9df285d 100644 --- a/config/config.go +++ b/config/config.go @@ -103,6 +103,7 @@ type Configuration struct { Hooks Hooks `mapstructure:"hooks"` Validations Validations `mapstructure:"validations"` PriceFloors PriceFloors `mapstructure:"price_floors"` + GeoLocation GeoLocation `mapstructure:"geolocation"` } type Admin struct { @@ -253,8 +254,9 @@ type GDPR struct { // If the gdpr flag is unset in a request, but geo.country is set, we will assume GDPR applies if and only // if the country matches one on this list. If both the GDPR flag and country are not set, we default // to DefaultValue - EEACountries []string `mapstructure:"eea_countries"` - EEACountriesMap map[string]struct{} + EEACountries []string `mapstructure:"eea_countries"` + EEACountriesMap map[string]struct{} + ConsentStringMeansInScope bool `mapstructure:"consent_string_means_in_scope"` } func (cfg *GDPR) validate(v *viper.Viper, errs []error) []error { @@ -659,6 +661,27 @@ type DefReqFiles struct { FileName string `mapstructure:"name"` } +type GeoLocation struct { + Enabled bool `mapstructure:"enabled"` + Type string `mapstructure:"type"` + Maxmind GeoLocationMaxmind `mapstructure:"maxmind"` +} + +type GeoLocationMaxmind struct { + RemoteFileSyncer MaxmindRemoteFileSyncer `mapstructure:"remote_file_syncer"` +} + +type MaxmindRemoteFileSyncer struct { + HttpClient HTTPClient `mapstructure:"http_client"` + DownloadURL string `mapstructure:"download_url"` + SaveFilePath string `mapstructure:"save_filepath"` + TmpFilePath string `mapstructure:"tmp_filepath"` + RetryCount int `mapstructure:"retry_count"` + RetryIntervalMillis int `mapstructure:"retry_interval_ms"` + TimeoutMillis int `mapstructure:"timeout_ms"` + UpdateIntervalMillis int `mapstructure:"update_interval_ms"` +} + type Debug struct { TimeoutNotification TimeoutNotification `mapstructure:"timeout_notification"` OverrideToken string `mapstructure:"override_token"` @@ -1140,6 +1163,7 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) { "FIN", "FRA", "GUF", "DEU", "GIB", "GRC", "GLP", "GGY", "HUN", "ISL", "IRL", "IMN", "ITA", "JEY", "LVA", "LIE", "LTU", "LUX", "MLT", "MTQ", "MYT", "NLD", "NOR", "POL", "PRT", "REU", "ROU", "BLM", "MAF", "SPM", "SVK", "SVN", "ESP", "SWE", "GBR"}) + v.SetDefault("gdpr.consent_string_means_in_scope", false) v.SetDefault("ccpa.enforce", false) v.SetDefault("lmt.enforce", true) v.SetDefault("currency_converter.fetch_url", "https://cdn.jsdelivr.net/gh/prebid/currency-file@1/latest.json") @@ -1171,6 +1195,7 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) { v.SetDefault("account_defaults.privacy.privacysandbox.topicsdomain", "") v.SetDefault("account_defaults.privacy.privacysandbox.cookiedeprecation.enabled", false) v.SetDefault("account_defaults.privacy.privacysandbox.cookiedeprecation.ttl_sec", 604800) + v.SetDefault("account_defaults.geolocation.enabled", false) v.SetDefault("account_defaults.events_enabled", false) v.BindEnv("account_defaults.privacy.dsa.default") @@ -1188,6 +1213,20 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) { v.SetDefault("price_floors.fetcher.http_client.idle_connection_timeout_seconds", 60) v.SetDefault("price_floors.fetcher.max_retries", 10) + v.SetDefault("geolocation.enabled", false) + v.SetDefault("geolocation.type", "maxmind") + v.SetDefault("geolocation.maxmind.remote_file_syncer.download_url", "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz") + v.SetDefault("geolocation.maxmind.remote_file_syncer.save_filepath", "/var/tmp/prebid/GeoLite2-City.tar.gz") + v.SetDefault("geolocation.maxmind.remote_file_syncer.tmp_filepath", "/var/tmp/prebid/tmp/GeoLite2-City.tar.gz") + v.SetDefault("geolocation.maxmind.remote_file_syncer.retry_count", 3) + v.SetDefault("geolocation.maxmind.remote_file_syncer.retry_interval_ms", 3000) + v.SetDefault("geolocation.maxmind.remote_file_syncer.timeout_ms", 300000) + v.SetDefault("geolocation.maxmind.remote_file_syncer.update_interval_ms", 0) + v.SetDefault("geolocation.maxmind.remote_file_syncer.http_client.max_connections_per_host", 0) + v.SetDefault("geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections", 40) + v.SetDefault("geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections_per_host", 2) + v.SetDefault("geolocation.maxmind.remote_file_syncer.http_client.idle_connection_timeout_seconds", 60) + v.SetDefault("account_defaults.events_enabled", false) v.SetDefault("compression.response.enable_gzip", false) v.SetDefault("compression.request.enable_gzip", false) diff --git a/config/config_test.go b/config/config_test.go index 414f2635fd5..0fb5b4fb87e 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -209,9 +209,24 @@ func TestDefaults(t *testing.T) { cmpStrings(t, "account_defaults.privacy.topicsdomain", "", cfg.AccountDefaults.Privacy.PrivacySandbox.TopicsDomain) cmpBools(t, "account_defaults.privacy.privacysandbox.cookiedeprecation.enabled", false, cfg.AccountDefaults.Privacy.PrivacySandbox.CookieDeprecation.Enabled) cmpInts(t, "account_defaults.privacy.privacysandbox.cookiedeprecation.ttl_sec", 604800, cfg.AccountDefaults.Privacy.PrivacySandbox.CookieDeprecation.TTLSec) + cmpBools(t, "account_defaults.geolocation.enabled", false, cfg.AccountDefaults.GeoLocation.Enabled) cmpBools(t, "account_defaults.events.enabled", false, cfg.AccountDefaults.Events.Enabled) + cmpBools(t, "geolocation.enabled", false, cfg.GeoLocation.Enabled) + cmpStrings(t, "geolocation.type", "maxmind", cfg.GeoLocation.Type) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_connections_per_host", 0, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxConnsPerHost) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections", 40, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxIdleConns) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections_per_host", 2, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxIdleConnsPerHost) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.idle_connection_timeout_seconds", 60, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.IdleConnTimeout) + cmpStrings(t, "geolocation.maxmind.remote_file_syncer.download_url", "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.DownloadURL) + cmpStrings(t, "geolocation.maxmind.remote_file_syncer.save_filepath", "/var/tmp/prebid/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.SaveFilePath) + cmpStrings(t, "geolocation.maxmind.remote_file_syncer.tmp_filepath", "/var/tmp/prebid/tmp/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.TmpFilePath) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.retry_count", 3, cfg.GeoLocation.Maxmind.RemoteFileSyncer.RetryCount) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.retry_interval_ms", 3000, cfg.GeoLocation.Maxmind.RemoteFileSyncer.RetryIntervalMillis) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.timeout_ms", 300000, cfg.GeoLocation.Maxmind.RemoteFileSyncer.TimeoutMillis) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.update_interval_ms", 0, cfg.GeoLocation.Maxmind.RemoteFileSyncer.UpdateIntervalMillis) + cmpBools(t, "hooks.enabled", false, cfg.Hooks.Enabled) cmpStrings(t, "validations.banner_creative_max_size", "skip", cfg.Validations.BannerCreativeMaxSize) cmpStrings(t, "validations.secure_markup", "skip", cfg.Validations.SecureMarkup) @@ -227,6 +242,8 @@ func TestDefaults(t *testing.T) { cmpInts(t, "account_defaults.privacy.ipv6.anon_keep_bits", 56, cfg.AccountDefaults.Privacy.IPv6Config.AnonKeepBits) cmpInts(t, "account_defaults.privacy.ipv4.anon_keep_bits", 24, cfg.AccountDefaults.Privacy.IPv4Config.AnonKeepBits) + cmpBools(t, "gdpr.consent_string_means_in_scope", false, cfg.GDPR.ConsentStringMeansInScope) + //Assert purpose VendorExceptionMap hash tables were built correctly cmpBools(t, "analytics.agma.enabled", false, cfg.Analytics.Agma.Enabled) cmpStrings(t, "analytics.agma.endpoint.timeout", "2s", cfg.Analytics.Agma.Endpoint.Timeout) @@ -350,6 +367,7 @@ gdpr: default_value: "1" non_standard_publishers: ["pub1", "pub2"] eea_countries: ["eea1", "eea2"] + consent_string_means_in_scope: false tcf2: purpose1: enforce_vendors: false @@ -494,6 +512,23 @@ price_floors: max_idle_connections_per_host: 2 idle_connection_timeout_seconds: 10 max_retries: 5 +geolocation: + enabled: false + type: maxmind + maxmind: + remote_file_syncer: + http_client: + max_connections_per_host: 0 + max_idle_connections: 40 + max_idle_connections_per_host: 2 + idle_connection_timeout_seconds: 60 + download_url: "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz" + save_filepath: "/var/tmp/prebid/GeoLite2-City.tar.gz" + tmp_filepath: "/var/tmp/prebid/tmp/GeoLite2-City.tar.gz" + retry_count: 3 + retry_interval_ms: 3000 + timeout_ms: 300000 + update_interval_ms: 0 account_defaults: events: enabled: true @@ -541,6 +576,8 @@ account_defaults: cookiedeprecation: enabled: true ttl_sec: 86400 + geolocation: + enabled: false tmax_adjustments: enabled: true bidder_response_duration_min_ms: 700 @@ -631,6 +668,7 @@ func TestFullConfig(t *testing.T) { cmpInts(t, "http_client_cache.idle_connection_timeout_seconds", 3, cfg.CacheClient.IdleConnTimeout) cmpInts(t, "gdpr.host_vendor_id", 15, cfg.GDPR.HostVendorID) cmpStrings(t, "gdpr.default_value", "1", cfg.GDPR.DefaultValue) + cmpBools(t, "gdpr.consent_string_means_in_scope", false, cfg.GDPR.ConsentStringMeansInScope) cmpStrings(t, "host_schain_node.asi", "pbshostcompany.com", cfg.HostSChainNode.ASI) cmpStrings(t, "host_schain_node.sid", "00001", cfg.HostSChainNode.SID) cmpStrings(t, "host_schain_node.rid", "BidRequest", cfg.HostSChainNode.RID) @@ -890,6 +928,21 @@ func TestFullConfig(t *testing.T) { cmpStrings(t, "analytics.agma.accounts.0.publisher_id", "publisher-id", cfg.Analytics.Agma.Accounts[0].PublisherId) cmpStrings(t, "analytics.agma.accounts.0.code", "agma-code", cfg.Analytics.Agma.Accounts[0].Code) cmpStrings(t, "analytics.agma.accounts.0.site_app_id", "site-or-app-id", cfg.Analytics.Agma.Accounts[0].SiteAppId) + + cmpBools(t, "account_defaults.geolocation.enabled", false, cfg.GeoLocation.Enabled) + cmpBools(t, "geolocation.enabled", false, cfg.GeoLocation.Enabled) + cmpStrings(t, "geolocation.type", "maxmind", cfg.GeoLocation.Type) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_connections_per_host", 0, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxConnsPerHost) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections", 40, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxIdleConns) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections_per_host", 2, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxIdleConnsPerHost) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.idle_connection_timeout_seconds", 60, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.IdleConnTimeout) + cmpStrings(t, "geolocation.maxmind.remote_file_syncer.download_url", "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.DownloadURL) + cmpStrings(t, "geolocation.maxmind.remote_file_syncer.save_filepath", "/var/tmp/prebid/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.SaveFilePath) + cmpStrings(t, "geolocation.maxmind.remote_file_syncer.tmp_filepath", "/var/tmp/prebid/tmp/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.TmpFilePath) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.retry_count", 3, cfg.GeoLocation.Maxmind.RemoteFileSyncer.RetryCount) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.retry_interval_ms", 3000, cfg.GeoLocation.Maxmind.RemoteFileSyncer.RetryIntervalMillis) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.timeout_ms", 300000, cfg.GeoLocation.Maxmind.RemoteFileSyncer.TimeoutMillis) + cmpInts(t, "geolocation.maxmind.remote_file_syncer.update_interval_ms", 0, cfg.GeoLocation.Maxmind.RemoteFileSyncer.UpdateIntervalMillis) } func TestValidateConfig(t *testing.T) { diff --git a/config/countrycode/countrycode.go b/config/countrycode/countrycode.go new file mode 100644 index 00000000000..0292f3ba72b --- /dev/null +++ b/config/countrycode/countrycode.go @@ -0,0 +1,63 @@ +package countrycode + +import ( + "strings" +) + +type CountryCode struct { + map2To3 map[string]string + map3To2 map[string]string +} + +func New() *CountryCode { + return &CountryCode{ + map2To3: make(map[string]string), + map3To2: make(map[string]string), + } +} + +// Load loads country code mapping data +func (c *CountryCode) Load(data string) { + toAlpha2 := make(map[string]string) + toAlpha3 := make(map[string]string) + for _, line := range strings.Split(data, "\n") { + if line == "" { + continue + } + fields := strings.Split(line, ",") + if len(fields) < 2 { + continue + } + alpha2 := strings.TrimSpace(fields[0]) + alpha3 := strings.TrimSpace(fields[1]) + toAlpha2[alpha3] = alpha2 + toAlpha3[alpha2] = alpha3 + } + + c.map2To3 = toAlpha3 + c.map3To2 = toAlpha2 +} + +// ToAlpha3 converts country code alpha2 to alpha3 +func (c *CountryCode) ToAlpha3(alpha2 string) string { + return c.map2To3[alpha2] +} + +// ToAlpha2 converts country code alpha3 to alpha2 +func (c *CountryCode) ToAlpha2(alpha3 string) string { + return c.map3To2[alpha3] +} + +var defaultCountryCode = New() + +func Load(data string) { + defaultCountryCode.Load(data) +} + +func ToAlpha3(alpha2 string) string { + return defaultCountryCode.ToAlpha3(alpha2) +} + +func ToAlpha2(alpha3 string) string { + return defaultCountryCode.ToAlpha2(alpha3) +} diff --git a/config/countrycode/countrycode_test.go b/config/countrycode/countrycode_test.go new file mode 100644 index 00000000000..196b936d56c --- /dev/null +++ b/config/countrycode/countrycode_test.go @@ -0,0 +1,60 @@ +package countrycode + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCountryCode(t *testing.T) { + Load(` +AD,AND +AE,ARE +AF,AFG +`) + assert.Equal(t, "AND", ToAlpha3("AD"), "map AD to AND") + assert.Equal(t, "AE", ToAlpha2("ARE"), "map ARE to AE") +} + +func TestCountryCodeToAlpha3(t *testing.T) { + c := New() + c.Load(` +AD,AND +AE,ARE +AF,AFG +`) + tests := []struct { + input string + expected string + }{ + {"AD", "AND"}, + {"AE", "ARE"}, + {"XX", ""}, + {"", ""}, + } + + for _, test := range tests { + assert.Equal(t, test.expected, c.ToAlpha3(test.input), "map %s to alpha3", test.input) + } +} + +func TestCountryCodeToAlpha2(t *testing.T) { + c := New() + c.Load(` +AD,AND +AE,ARE +AF,AFG +`) + tests := []struct { + input string + expected string + }{ + {"AND", "AD"}, + {"ARE", "AE"}, + {"", ""}, + } + + for _, test := range tests { + assert.Equal(t, test.expected, c.ToAlpha2(test.input), "map %s to alpha2", test.input) + } +} diff --git a/country-codes.csv b/country-codes.csv new file mode 100644 index 00000000000..2895f157982 --- /dev/null +++ b/country-codes.csv @@ -0,0 +1,244 @@ +AF,AFG +AL,ALB +DZ,DZA +AS,ASM +AD,AND +AO,AGO +AI,AIA +AQ,ATA +AG,ATG +AR,ARG +AM,ARM +AW,ABW +AU,AUS +AT,AUT +AZ,AZE +BS,BHS +BH,BHR +BD,BGD +BB,BRB +BY,BLR +BE,BEL +BZ,BLZ +BJ,BEN +BM,BMU +BT,BTN +BO,BOL +BA,BIH +BW,BWA +BV,BVT +BR,BRA +IO,IOT +BN,BRN +BG,BGR +BF,BFA +BI,BDI +KH,KHM +CM,CMR +CA,CAN +CV,CPV +KY,CYM +CF,CAF +TD,TCD +CL,CHL +CN,CHN +CX,CXR +CC,CCK +CO,COL +KM,COM +CG,COG +CD,COD +CK,COK +CR,CRI +CI,CIV +HR,HRV +CU,CUB +CY,CYP +CZ,CZE +DK,DNK +DJ,DJI +DM,DMA +DO,DOM +EC,ECU +EG,EGY +SV,SLV +GQ,GNQ +ER,ERI +EE,EST +ET,ETH +FK,FLK +FO,FRO +FJ,FJI +FI,FIN +FR,FRA +GF,GUF +PF,PYF +TF,ATF +GA,GAB +GM,GMB +GE,GEO +DE,DEU +GH,GHA +GI,GIB +GR,GRC +GL,GRL +GD,GRD +GP,GLP +GU,GUM +GT,GTM +GG,GGY +GN,GIN +GW,GNB +GY,GUY +HT,HTI +HM,HMD +VA,VAT +HN,HND +HK,HKG +HU,HUN +IS,ISL +IN,IND +ID,IDN +IR,IRN +IQ,IRQ +IE,IRL +IM,IMN +IL,ISR +IT,ITA +JM,JAM +JP,JPN +JE,JEY +JO,JOR +KZ,KAZ +KE,KEN +KI,KIR +KP,PRK +KR,KOR +KW,KWT +KG,KGZ +LA,LAO +LV,LVA +LB,LBN +LS,LSO +LR,LBR +LY,LBY +LI,LIE +LT,LTU +LU,LUX +MO,MAC +MK,MKD +MG,MDG +MW,MWI +MY,MYS +MV,MDV +ML,MLI +MT,MLT +MH,MHL +MQ,MTQ +MR,MRT +MU,MUS +YT,MYT +MX,MEX +FM,FSM +MD,MDA +MC,MCO +MN,MNG +ME,MNE +MS,MSR +MA,MAR +MZ,MOZ +MM,MMR +NA,NAM +NR,NRU +NP,NPL +NL,NLD +AN,ANT +NC,NCL +NZ,NZL +NI,NIC +NE,NER +NG,NGA +NU,NIU +NF,NFK +MP,MNP +NO,NOR +OM,OMN +PK,PAK +PW,PLW +PS,PSE +PA,PAN +PG,PNG +PY,PRY +PE,PER +PH,PHL +PN,PCN +PL,POL +PT,PRT +PR,PRI +QA,QAT +RE,REU +RO,ROU +RU,RUS +RW,RWA +SH,SHN +KN,KNA +LC,LCA +PM,SPM +VC,VCT +WS,WSM +SM,SMR +ST,STP +SA,SAU +SN,SEN +RS,SRB +SC,SYC +SL,SLE +SG,SGP +SK,SVK +SI,SVN +SB,SLB +SO,SOM +ZA,ZAF +GS,SGS +ES,ESP +LK,LKA +SD,SDN +SR,SUR +SJ,SJM +SZ,SWZ +SE,SWE +CH,CHE +SY,SYR +TW,TWN +TJ,TJK +TZ,TZA +TH,THA +TL,TLS +TG,TGO +TK,TKL +TO,TON +TT,TTO +TN,TUN +TR,TUR +TM,TKM +TC,TCA +TV,TUV +UG,UGA +UA,UKR +AE,ARE +GB,GBR +UK,GBR +US,USA +UM,UMI +UY,URY +UZ,UZB +VU,VUT +VE,VEN +VN,VNM +VG,VGB +VI,VIR +WF,WLF +EH,ESH +YE,YEM +ZM,ZMB +ZW,ZWE diff --git a/endpoints/openrtb2/auction_benchmark_test.go b/endpoints/openrtb2/auction_benchmark_test.go index d3dbc518bd3..9780c6c92e3 100644 --- a/endpoints/openrtb2/auction_benchmark_test.go +++ b/endpoints/openrtb2/auction_benchmark_test.go @@ -99,6 +99,7 @@ func BenchmarkOpenrtbEndpoint(b *testing.B) { &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil, + &exchange.NilGeoLocationResolver{}, ) endpoint, _ := NewEndpoint( diff --git a/endpoints/openrtb2/test_utils.go b/endpoints/openrtb2/test_utils.go index b2f2826739f..52c8d36cdbf 100644 --- a/endpoints/openrtb2/test_utils.go +++ b/endpoints/openrtb2/test_utils.go @@ -1230,6 +1230,7 @@ func buildTestExchange(testCfg *testConfigValues, adapterMap map[openrtb_ext.Bid &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil, + &exchange.NilGeoLocationResolver{}, ) testExchange = &exchangeTestWrapper{ diff --git a/errortypes/severity.go b/errortypes/severity.go index 5f9cd80dd28..b589e92a6f8 100644 --- a/errortypes/severity.go +++ b/errortypes/severity.go @@ -39,6 +39,16 @@ func ContainsFatalError(errors []error) bool { return false } +// FirstFatalError returns the first Fatal error found +func FirstFatalError(errors []error) error { + for _, err := range errors { + if isFatal(err) { + return err + } + } + return nil +} + // FatalOnly returns a new error list with only the fatal severity errors. func FatalOnly(errs []error) []error { errsFatal := make([]error, 0, len(errs)) diff --git a/errortypes/severity_test.go b/errortypes/severity_test.go index 8330316a8d2..f6fa04de2dc 100644 --- a/errortypes/severity_test.go +++ b/errortypes/severity_test.go @@ -56,6 +56,34 @@ func TestContainsFatalError(t *testing.T) { } } +func TestFirstFatalErrors(t *testing.T) { + fatalError := &stubError{severity: SeverityFatal} + fatalError2 := &stubError{severity: SeverityFatal} + notFatalError := &stubError{severity: SeverityWarning} + unknownSeverityError := errors.New("anyError") + + tests := []struct { + errors []error + first error + }{ + {[]error{}, nil}, + {[]error{fatalError}, fatalError}, + {[]error{fatalError2}, fatalError2}, + {[]error{notFatalError}, nil}, + {[]error{unknownSeverityError}, unknownSeverityError}, + {[]error{notFatalError, unknownSeverityError}, unknownSeverityError}, + {[]error{fatalError, fatalError2}, fatalError}, + {[]error{fatalError2, fatalError}, fatalError2}, + {[]error{notFatalError, fatalError, fatalError2}, fatalError}, + {[]error{fatalError2, unknownSeverityError, fatalError}, fatalError}, + {[]error{notFatalError, fatalError2, unknownSeverityError, fatalError}, fatalError2}, + } + + for _, test := range tests { + assert.Equal(t, test.first, FirstFatalError(test.errors), "FirstFatalError(%v)", test.errors) + } +} + func TestFatalOnly(t *testing.T) { fatalError := &stubError{severity: SeverityFatal} notFatalError := &stubError{severity: SeverityWarning} diff --git a/exchange/exchange.go b/exchange/exchange.go index 5c27b0d3c5a..2d42b6c2786 100644 --- a/exchange/exchange.go +++ b/exchange/exchange.go @@ -17,6 +17,9 @@ import ( "github.com/prebid/prebid-server/v2/ortb" "github.com/prebid/prebid-server/v2/privacy" + gdprAPI "github.com/prebid/go-gdpr/api" + "github.com/prebid/go-gdpr/vendorconsent" + gpplib "github.com/prebid/go-gpp" "github.com/prebid/prebid-server/v2/adapters" "github.com/prebid/prebid-server/v2/adservertargeting" "github.com/prebid/prebid-server/v2/bidadjustment" @@ -85,6 +88,8 @@ type exchange struct { macroReplacer macros.Replacer priceFloorEnabled bool priceFloorFetcher floors.FloorFetcher + geoLocationEnabled bool + geoLocationResolver GeoLocationResolver } // Container to pass out response ext data from the GetAllBids goroutines back into the main thread @@ -133,7 +138,7 @@ func (randomDeduplicateBidBooleanGenerator) Generate() bool { return rand.Intn(100) < 50 } -func NewExchange(adapters map[openrtb_ext.BidderName]AdaptedBidder, cache prebid_cache_client.Client, cfg *config.Configuration, requestValidator ortb.RequestValidator, syncersByBidder map[string]usersync.Syncer, metricsEngine metrics.MetricsEngine, infos config.BidderInfos, gdprPermsBuilder gdpr.PermissionsBuilder, currencyConverter *currency.RateConverter, categoriesFetcher stored_requests.CategoryFetcher, adsCertSigner adscert.Signer, macroReplacer macros.Replacer, priceFloorFetcher floors.FloorFetcher) Exchange { +func NewExchange(adapters map[openrtb_ext.BidderName]AdaptedBidder, cache prebid_cache_client.Client, cfg *config.Configuration, requestValidator ortb.RequestValidator, syncersByBidder map[string]usersync.Syncer, metricsEngine metrics.MetricsEngine, infos config.BidderInfos, gdprPermsBuilder gdpr.PermissionsBuilder, currencyConverter *currency.RateConverter, categoriesFetcher stored_requests.CategoryFetcher, adsCertSigner adscert.Signer, macroReplacer macros.Replacer, priceFloorFetcher floors.FloorFetcher, geoLocationResolver GeoLocationResolver) Exchange { bidderToSyncerKey := map[string]string{} for bidder, syncer := range syncersByBidder { bidderToSyncerKey[bidder] = syncer.Key() @@ -181,6 +186,8 @@ func NewExchange(adapters map[openrtb_ext.BidderName]AdaptedBidder, cache prebid macroReplacer: macroReplacer, priceFloorEnabled: cfg.PriceFloors.Enabled, priceFloorFetcher: priceFloorFetcher, + geoLocationEnabled: cfg.GeoLocation.Enabled, + geoLocationResolver: geoLocationResolver, } } @@ -221,6 +228,46 @@ type AuctionRequest struct { TmaxAdjustments *TmaxAdjustmentsPreprocessed } +// RequestPrivacy holds privacies of request +type RequestPrivacy struct { + // GDPR + Consent string + ParsedConsent gdprAPI.VendorConsents + GDPRDefaultValue gdpr.Signal + GDPRSignal gdpr.Signal + GDPRChannelEnabled bool + GDPREnforced bool + + // LMT + LMTEnforced bool + + // CCPA + CCPAProvided bool + CCPAEnforced bool + + // COPPA + COPPAEnforced bool + + // GPP + ParsedGPP gpplib.GppContainer +} + +func (p *RequestPrivacy) MakePrivacyLabels() (labels metrics.PrivacyLabels) { + if p == nil { + return + } + labels.CCPAProvided = p.CCPAProvided + labels.CCPAEnforced = p.CCPAEnforced + labels.COPPAEnforced = p.COPPAEnforced + labels.LMTEnforced = p.LMTEnforced + labels.GDPREnforced = p.GDPREnforced + if p.GDPREnforced && p.ParsedConsent != nil { + version := int(p.ParsedConsent.Version()) + labels.GDPRTCFVersion = metrics.TCFVersionToValue(version) + } + return +} + // BidderRequest holds the bidder specific request and all other // information needed to process that bidder request. type BidderRequest struct { @@ -238,6 +285,8 @@ func (e *exchange) HoldAuction(ctx context.Context, r *AuctionRequest, debugLog return nil, nil } + errs := EnrichGeoLocation(ctx, r.BidRequestWrapper, r.Account, e.geoLocationResolver) + err := r.HookExecutor.ExecuteProcessedAuctionStage(r.BidRequestWrapper) if err != nil { return nil, err @@ -269,12 +318,22 @@ func (e *exchange) HoldAuction(ctx context.Context, r *AuctionRequest, debugLog _, targData.cacheHost, targData.cachePath = e.cache.GetExtCacheData() } + requestPrivacy, privacyErrs := e.extractRequestPrivacy(r) + if errf := errortypes.FirstFatalError(privacyErrs); errf != nil { + return nil, errf + } + errs = append(errs, privacyErrs...) + + geoPrivacyErrs := EnrichGeoLocationWithPrivacy(ctx, r.BidRequestWrapper, r.Account, e.geoLocationResolver, requestPrivacy, r.TCF2Config) + errs = append(errs, geoPrivacyErrs...) + // Get currency rates conversions for the auction conversions := currency.GetAuctionCurrencyRates(e.currencyConverter, requestExtPrebid.CurrencyConversions) var floorErrs []error if e.priceFloorEnabled { floorErrs = floors.EnrichWithPriceFloors(r.BidRequestWrapper, r.Account, conversions, e.priceFloorFetcher) + errs = append(errs, floorErrs...) } responseDebugAllow, accountDebugAllow, debugLog := getDebugInfo(r.BidRequestWrapper.Test, requestExtPrebid, r.Account.DebugAllow, debugLog) @@ -314,16 +373,9 @@ func (e *exchange) HoldAuction(ctx context.Context, r *AuctionRequest, debugLog recordImpMetrics(r.BidRequestWrapper, e.me) // Make our best guess if GDPR applies - gdprDefaultValue := e.parseGDPRDefaultValue(r.BidRequestWrapper) - gdprSignal, err := getGDPR(r.BidRequestWrapper) - if err != nil { - return nil, err - } - channelEnabled := r.TCF2Config.ChannelEnabled(channelTypeMap[r.LegacyLabels.RType]) - gdprEnforced := enforceGDPR(gdprSignal, gdprDefaultValue, channelEnabled) dsaWriter := dsa.Writer{ Config: r.Account.Privacy.DSA, - GDPRInScope: gdprEnforced, + GDPRInScope: requestPrivacy.GDPREnforced, } if err := dsaWriter.Write(r.BidRequestWrapper); err != nil { return nil, err @@ -339,13 +391,13 @@ func (e *exchange) HoldAuction(ctx context.Context, r *AuctionRequest, debugLog Prebid: *requestExtPrebid, SChain: requestExt.GetSChain(), } - bidderRequests, privacyLabels, errs := e.requestSplitter.cleanOpenRTBRequests(ctx, *r, requestExtLegacy, gdprSignal, gdprEnforced, bidAdjustmentFactors) - for _, err := range errs { + bidderRequests, cleanErrs := e.requestSplitter.cleanOpenRTBRequests(ctx, *r, requestExtLegacy, requestPrivacy, bidAdjustmentFactors) + for _, err := range cleanErrs { if errortypes.ReadCode(err) == errortypes.InvalidImpFirstPartyDataErrorCode { return nil, err } } - errs = append(errs, floorErrs...) + errs = append(errs, cleanErrs...) mergedBidAdj, err := bidadjustment.Merge(r.BidRequestWrapper, r.Account.BidAdjustments) if err != nil { @@ -356,7 +408,7 @@ func (e *exchange) HoldAuction(ctx context.Context, r *AuctionRequest, debugLog } bidAdjustmentRules := bidadjustment.BuildRules(mergedBidAdj) - e.me.RecordRequestPrivacy(privacyLabels) + e.me.RecordRequestPrivacy(requestPrivacy.MakePrivacyLabels()) if len(r.StoredAuctionResponses) > 0 || len(r.StoredBidResponses) > 0 { e.me.RecordStoredResponse(r.PubID) @@ -565,9 +617,20 @@ func buildMultiBidMap(prebid *openrtb_ext.ExtRequestPrebid) map[string]openrtb_e return multiBidMap } -func (e *exchange) parseGDPRDefaultValue(r *openrtb_ext.RequestWrapper) gdpr.Signal { +func (e *exchange) parseGDPRDefaultValue(r *openrtb_ext.RequestWrapper, account config.Account, parsedConsent gdprAPI.VendorConsents) gdpr.Signal { gdprDefaultValue := e.gdprDefaultValue + // requests may have consent without gdpr signal. check if setting is enabled to assume gdpr applies + if parsedConsent != nil && parsedConsent.Version() > 0 { + if account.GDPR.ConsentStringMeansInScope != nil { + if *account.GDPR.ConsentStringMeansInScope { + gdprDefaultValue = gdpr.SignalYes + } + } else if e.privacyConfig.GDPR.ConsentStringMeansInScope { + gdprDefaultValue = gdpr.SignalYes + } + } + var geo *openrtb2.Geo if r.User != nil && r.User.Geo != nil { geo = r.User.Geo @@ -589,6 +652,52 @@ func (e *exchange) parseGDPRDefaultValue(r *openrtb_ext.RequestWrapper) gdpr.Sig return gdprDefaultValue } +func (e *exchange) extractRequestPrivacy(r *AuctionRequest) (p *RequestPrivacy, errs []error) { + req := r.BidRequestWrapper + + var gpp gpplib.GppContainer + if req.BidRequest.Regs != nil && len(req.BidRequest.Regs.GPP) > 0 { + var gppErrs []error + gpp, gppErrs = gpplib.Parse(req.BidRequest.Regs.GPP) + if len(gppErrs) > 0 { + errs = append(errs, gppErrs[0]) + } + } + + consent, err := getConsent(req, gpp) + if err != nil { + errs = append(errs, err) + } + parsedConsent, err := vendorconsent.ParseString(consent) + if err != nil { + parsedConsent = nil + } + + gdprDefaultValue := e.parseGDPRDefaultValue(req, r.Account, parsedConsent) + gdprSignal, err := getGDPR(req) + if err != nil { + errs = append(errs, err) + return + } + channelEnabled := r.TCF2Config.ChannelEnabled(channelTypeMap[r.LegacyLabels.RType]) + gdprEnforced := enforceGDPR(gdprSignal, gdprDefaultValue, channelEnabled) + + lmtEnforcer := extractLMT(req.BidRequest, e.privacyConfig) + + p = &RequestPrivacy{ + Consent: consent, + ParsedConsent: parsedConsent, + GDPRDefaultValue: gdprDefaultValue, + GDPRSignal: gdprSignal, + GDPRChannelEnabled: channelEnabled, + GDPREnforced: gdprEnforced, + COPPAEnforced: req.BidRequest.Regs != nil && req.BidRequest.Regs.COPPA == 1, + LMTEnforced: lmtEnforcer.ShouldEnforce(unknownBidder), + ParsedGPP: gpp, + } + return +} + func recordImpMetrics(r *openrtb_ext.RequestWrapper, metricsEngine metrics.MetricsEngine) { for _, impInRequest := range r.GetImp() { var impLabels metrics.ImpLabels = metrics.ImpLabels{ diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index bab8dded1c5..7a26fddb58c 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -19,6 +19,9 @@ import ( "time" "github.com/buger/jsonparser" + "github.com/prebid/go-gdpr/vendorconsent" + gpplib "github.com/prebid/go-gpp" + gppConstants "github.com/prebid/go-gpp/constants" "github.com/prebid/openrtb/v20/openrtb2" "github.com/prebid/prebid-server/v2/adapters" "github.com/prebid/prebid-server/v2/config" @@ -84,7 +87,7 @@ func TestNewExchange(t *testing.T) { }, }.Builder - e := NewExchange(adapters, nil, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, nilCategoryFetcher{}, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil).(*exchange) + e := NewExchange(adapters, nil, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, nilCategoryFetcher{}, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil, nil).(*exchange) for _, bidderName := range knownAdapters { if _, ok := e.adapterMap[bidderName]; !ok { if biddersInfo[string(bidderName)].IsEnabled() { @@ -134,7 +137,7 @@ func TestCharacterEscape(t *testing.T) { }, }.Builder - e := NewExchange(adapters, nil, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, nilCategoryFetcher{}, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil).(*exchange) + e := NewExchange(adapters, nil, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, nilCategoryFetcher{}, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil, nil).(*exchange) // 3) Build all the parameters e.buildBidResponse(ctx.Background(), liveA... ) needs //liveAdapters []openrtb_ext.BidderName, @@ -1237,7 +1240,7 @@ func TestGetBidCacheInfoEndToEnd(t *testing.T) { }, }.Builder - e := NewExchange(adapters, pbc, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, nilCategoryFetcher{}, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil).(*exchange) + e := NewExchange(adapters, pbc, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, nilCategoryFetcher{}, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil, nil).(*exchange) // 3) Build all the parameters e.buildBidResponse(ctx.Background(), liveA... ) needs liveAdapters := []openrtb_ext.BidderName{bidderName} @@ -1596,7 +1599,7 @@ func TestBidResponseCurrency(t *testing.T) { }, }.Builder - e := NewExchange(adapters, nil, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, nilCategoryFetcher{}, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil).(*exchange) + e := NewExchange(adapters, nil, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, nilCategoryFetcher{}, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil, nil).(*exchange) liveAdapters := make([]openrtb_ext.BidderName, 1) liveAdapters[0] = "appnexus" @@ -1744,7 +1747,7 @@ func TestBidResponseImpExtInfo(t *testing.T) { t.Fatalf("Error intializing adapters: %v", adaptersErr) } - e := NewExchange(adapters, nil, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, nil, gdprPermsBuilder, nil, nilCategoryFetcher{}, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil).(*exchange) + e := NewExchange(adapters, nil, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, nil, gdprPermsBuilder, nil, nilCategoryFetcher{}, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil, nil).(*exchange) liveAdapters := make([]openrtb_ext.BidderName, 1) liveAdapters[0] = "appnexus" @@ -1838,7 +1841,7 @@ func TestRaceIntegration(t *testing.T) { }, }.Builder - ex := NewExchange(adapters, &wellBehavedCache{}, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, &nilCategoryFetcher{}, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil).(*exchange) + ex := NewExchange(adapters, &wellBehavedCache{}, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, &nilCategoryFetcher{}, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil, nil).(*exchange) _, err = ex.HoldAuction(context.Background(), auctionRequest, &debugLog) if err != nil { t.Errorf("HoldAuction returned unexpected error: %v", err) @@ -1936,7 +1939,7 @@ func TestPanicRecovery(t *testing.T) { }, }.Builder - e := NewExchange(adapters, nil, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, nilCategoryFetcher{}, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil).(*exchange) + e := NewExchange(adapters, nil, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, nilCategoryFetcher{}, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil, nil).(*exchange) chBids := make(chan *bidResponseWrapper, 1) panicker := func(bidderRequest BidderRequest, conversions currency.Conversions) { @@ -2006,7 +2009,7 @@ func TestPanicRecoveryHighLevel(t *testing.T) { allowAllBidders: true, }, }.Builder - e := NewExchange(adapters, &mockCache{}, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, categoriesFetcher, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil).(*exchange) + e := NewExchange(adapters, &mockCache{}, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, categoriesFetcher, &adscert.NilSigner{}, macros.NewStringIndexBasedReplacer(), nil, nil).(*exchange) e.adapterMap[openrtb_ext.BidderBeachfront] = panicingAdapter{} e.adapterMap[openrtb_ext.BidderAppnexus] = panicingAdapter{} @@ -4577,7 +4580,7 @@ func TestPassExperimentConfigsToHoldAuction(t *testing.T) { }, }.Builder - e := NewExchange(adapters, nil, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, nilCategoryFetcher{}, &signer, macros.NewStringIndexBasedReplacer(), nil).(*exchange) + e := NewExchange(adapters, nil, cfg, &mockRequestValidator{}, map[string]usersync.Syncer{}, &metricsConf.NilMetricsEngine{}, biddersInfo, gdprPermsBuilder, currencyConverter, nilCategoryFetcher{}, &signer, macros.NewStringIndexBasedReplacer(), nil, nil).(*exchange) // Define mock incoming bid requeset mockBidRequest := &openrtb2.BidRequest{ @@ -6352,3 +6355,539 @@ type mockRequestValidator struct { func (mrv *mockRequestValidator) ValidateImp(imp *openrtb_ext.ImpWrapper, cfg ortb.ValidationConfig, index int, aliases map[string]string, hasStoredResponses bool, storedBidResponses stored_responses.ImpBidderStoredResp) []error { return mrv.errors } + +func TestParseGDPRDefaultValue(t *testing.T) { + var ( + boolTrue = true + boolFalse = false + ) + + tests := []struct { + name string + defaultValue gdpr.Signal + privacyConfig config.Privacy + req *openrtb2.BidRequest + account config.Account + consent string + output gdpr.Signal + }{ + { + "Exchange default value is SignalYes, no other settings", + gdpr.SignalYes, + config.Privacy{}, + &openrtb2.BidRequest{}, + config.Account{}, + "", + gdpr.SignalYes, + }, + { + "Exchange default value is SignalNo, no other settings", + gdpr.SignalNo, + config.Privacy{}, + &openrtb2.BidRequest{}, + config.Account{}, + "", + gdpr.SignalNo, + }, + { + "Exchange default value is SignalNo, User is in EEA, Device is not in EEA", + gdpr.SignalNo, + config.Privacy{ + GDPR: config.GDPR{ + EEACountriesMap: map[string]struct{}{"ALA": {}}, + }, + }, + &openrtb2.BidRequest{ + Device: &openrtb2.Device{Geo: &openrtb2.Geo{Country: "USA"}}, + User: &openrtb2.User{Geo: &openrtb2.Geo{Country: "ALA"}}, + }, + config.Account{ + GDPR: config.AccountGDPR{ + ConsentStringMeansInScope: nil, + }, + }, + "", + gdpr.SignalYes, + }, + { + "Exchange default value is SignalNo, User is not in EEA, Device is not in EEA", + gdpr.SignalNo, + config.Privacy{ + GDPR: config.GDPR{ + EEACountriesMap: map[string]struct{}{"ALA": {}}, + }, + }, + &openrtb2.BidRequest{ + Device: &openrtb2.Device{Geo: &openrtb2.Geo{Country: "USA"}}, + User: &openrtb2.User{Geo: &openrtb2.Geo{Country: "USA"}}, + }, + config.Account{ + GDPR: config.AccountGDPR{ + ConsentStringMeansInScope: nil, + }, + }, + "", + gdpr.SignalNo, + }, + { + "Exchange default value is SignalNo, Device is in EEA", + gdpr.SignalNo, + config.Privacy{ + GDPR: config.GDPR{ + EEACountriesMap: map[string]struct{}{"ALA": {}}, + }, + }, + &openrtb2.BidRequest{ + Device: &openrtb2.Device{Geo: &openrtb2.Geo{Country: "ALA"}}, + }, + config.Account{ + GDPR: config.AccountGDPR{ + ConsentStringMeansInScope: nil, + }, + }, + "", + gdpr.SignalYes, + }, + { + "Exchange default value is SignalNo, Device is not in EEA", + gdpr.SignalNo, + config.Privacy{ + GDPR: config.GDPR{ + EEACountriesMap: map[string]struct{}{"ALA": {}}, + }, + }, + &openrtb2.BidRequest{ + Device: &openrtb2.Device{Geo: &openrtb2.Geo{Country: "USA"}}, + }, + config.Account{ + GDPR: config.AccountGDPR{ + ConsentStringMeansInScope: nil, + }, + }, + "", + gdpr.SignalNo, + }, + { + "Exchange default value is SignalNo, with consent, and means in scope", + gdpr.SignalNo, + config.Privacy{}, + &openrtb2.BidRequest{}, + config.Account{ + GDPR: config.AccountGDPR{ + ConsentStringMeansInScope: &boolTrue, + }, + }, + "CPuKGCPPuKGCPNEAAAENCZCAAAAAAAAAAAAAAAAAAAAA", + gdpr.SignalYes, + }, + { + "Exchange default value is SignalNo, with consent, and not means in scope", + gdpr.SignalNo, + config.Privacy{}, + &openrtb2.BidRequest{}, + config.Account{ + GDPR: config.AccountGDPR{ + ConsentStringMeansInScope: &boolFalse, + }, + }, + "CPuKGCPPuKGCPNEAAAENCZCAAAAAAAAAAAAAAAAAAAAA", + gdpr.SignalNo, + }, + { + "Exchange default value is SignalNo, without consent, and means in scope", + gdpr.SignalNo, + config.Privacy{}, + &openrtb2.BidRequest{}, + config.Account{ + GDPR: config.AccountGDPR{ + ConsentStringMeansInScope: &boolTrue, + }, + }, + "", + gdpr.SignalNo, + }, + { + "Exchange default value is SignalNo, with invalid consent, and means in scope", + gdpr.SignalNo, + config.Privacy{}, + &openrtb2.BidRequest{}, + config.Account{ + GDPR: config.AccountGDPR{ + ConsentStringMeansInScope: &boolTrue, + }, + }, + "invalid consent", + gdpr.SignalNo, + }, + { + "Exchange default value is SignalNo, with consent, default means in scope", + gdpr.SignalNo, + config.Privacy{ + GDPR: config.GDPR{ + ConsentStringMeansInScope: true, + }, + }, + &openrtb2.BidRequest{}, + config.Account{ + GDPR: config.AccountGDPR{ + ConsentStringMeansInScope: nil, + }, + }, + "CPuKGCPPuKGCPNEAAAENCZCAAAAAAAAAAAAAAAAAAAAA", + gdpr.SignalYes, + }, + { + "Exchange default value is SignalNo, with consent, and means in scope, in EEA", + gdpr.SignalNo, + config.Privacy{ + GDPR: config.GDPR{ + EEACountriesMap: map[string]struct{}{"ALA": {}}, + ConsentStringMeansInScope: true, + }, + }, + &openrtb2.BidRequest{ + Device: &openrtb2.Device{Geo: &openrtb2.Geo{Country: "USA"}}, + }, + config.Account{}, + "CPuKGCPPuKGCPNEAAAENCZCAAAAAAAAAAAAAAAAAAAAA", + gdpr.SignalNo, + }, + { + "Exchange default value is SignalNo, with consent, and means in scope, in EEA unknown", + gdpr.SignalNo, + config.Privacy{ + GDPR: config.GDPR{ + EEACountriesMap: map[string]struct{}{"ALA": {}}, + ConsentStringMeansInScope: true, + }, + }, + &openrtb2.BidRequest{ + Device: &openrtb2.Device{Geo: &openrtb2.Geo{Country: ""}}, + }, + config.Account{}, + "CPuKGCPPuKGCPNEAAAENCZCAAAAAAAAAAAAAAAAAAAAA", + gdpr.SignalYes, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + e := new(exchange) + e.gdprDefaultValue = test.defaultValue + e.privacyConfig = test.privacyConfig + req := &openrtb_ext.RequestWrapper{BidRequest: test.req} + parsedConsent, _ := vendorconsent.ParseString(test.consent) + + assert.Equal(t, test.output, e.parseGDPRDefaultValue(req, test.account, parsedConsent)) + }) + } +} + +func TestExtractRequestPrivacyGDPR(t *testing.T) { + var ( + SignalNo int8 = 0 + SignalYes int8 = 1 + BoolTrue = true + //BoolFalse = false + ) + + tests := []struct { + name string + req *openrtb2.BidRequest + account config.Account + tcf2config gdpr.TCF2ConfigReader + legacyLabels metrics.Labels + requestPrivacy *RequestPrivacy + errsCount int + errsHaveFatal bool + }{ + { + "Request without consent, default no, signal ambiguous, channel disabled", + &openrtb2.BidRequest{}, + config.Account{}, + gdpr.NewTCF2Config(config.TCF2{Enabled: false}, config.AccountGDPR{}), + metrics.Labels{}, + &RequestPrivacy{ + Consent: "", + GDPRDefaultValue: gdpr.SignalNo, + GDPRSignal: gdpr.SignalAmbiguous, + GDPRChannelEnabled: false, + GDPREnforced: false, + }, + 0, + false, + }, + { + "Request without consent, default no, signal ambiguous, channel enabled", + &openrtb2.BidRequest{}, + config.Account{}, + gdpr.NewTCF2Config(config.TCF2{Enabled: true}, config.AccountGDPR{}), + metrics.Labels{}, + &RequestPrivacy{ + Consent: "", + GDPRDefaultValue: gdpr.SignalNo, + GDPRSignal: gdpr.SignalAmbiguous, + GDPRChannelEnabled: true, + GDPREnforced: false, + }, + 0, + false, + }, + { + "Request with consent, default no, signal yes", + &openrtb2.BidRequest{ + Regs: &openrtb2.Regs{GDPR: &SignalYes, Ext: json.RawMessage(`{"gdpr":1}`)}, + User: &openrtb2.User{Ext: json.RawMessage(`{"consent":"CPuKGCPPuKGCPNEAAAENCZCAAAAAAAAAAAAAAAAAAAAA"}`)}, + }, + config.Account{}, + gdpr.NewTCF2Config(config.TCF2{Enabled: true}, config.AccountGDPR{}), + metrics.Labels{}, + &RequestPrivacy{ + Consent: "CPuKGCPPuKGCPNEAAAENCZCAAAAAAAAAAAAAAAAAAAAA", + GDPRDefaultValue: gdpr.SignalNo, + GDPRSignal: gdpr.SignalYes, + GDPRChannelEnabled: true, + GDPREnforced: true, + }, + 0, + false, + }, + { + "Request with consent, default no, signal no", + &openrtb2.BidRequest{ + Regs: &openrtb2.Regs{GDPR: &SignalNo, Ext: json.RawMessage(`{"gdpr":0}`)}, + User: &openrtb2.User{Ext: json.RawMessage(`{"consent":"CPuKGCPPuKGCPNEAAAENCZCAAAAAAAAAAAAAAAAAAAAA"}`)}, + }, + config.Account{}, + gdpr.NewTCF2Config(config.TCF2{Enabled: true}, config.AccountGDPR{}), + metrics.Labels{}, + &RequestPrivacy{ + Consent: "CPuKGCPPuKGCPNEAAAENCZCAAAAAAAAAAAAAAAAAAAAA", + GDPRDefaultValue: gdpr.SignalNo, + GDPRSignal: gdpr.SignalNo, + GDPRChannelEnabled: true, + GDPREnforced: false, + }, + 0, + false, + }, + { + "Request with consent, default yes, signal yes", + &openrtb2.BidRequest{ + Regs: &openrtb2.Regs{GDPR: &SignalYes, Ext: json.RawMessage(`{"gdpr":1}`)}, + User: &openrtb2.User{Ext: json.RawMessage(`{"consent":"CPuKGCPPuKGCPNEAAAENCZCAAAAAAAAAAAAAAAAAAAAA"}`)}, + }, + config.Account{GDPR: config.AccountGDPR{ConsentStringMeansInScope: &BoolTrue}}, + gdpr.NewTCF2Config(config.TCF2{Enabled: true}, config.AccountGDPR{}), + metrics.Labels{}, + &RequestPrivacy{ + Consent: "CPuKGCPPuKGCPNEAAAENCZCAAAAAAAAAAAAAAAAAAAAA", + GDPRDefaultValue: gdpr.SignalYes, + GDPRSignal: gdpr.SignalYes, + GDPRChannelEnabled: true, + GDPREnforced: true, + }, + 0, + false, + }, + { + "Request with invalid user ext, default no, signal Yes", + &openrtb2.BidRequest{ + Regs: &openrtb2.Regs{GDPR: &SignalYes, Ext: json.RawMessage(`{"gdpr":1}`)}, + User: &openrtb2.User{Ext: json.RawMessage(`{"CPuKGCPPuKGCPNEAAAENCZCAAAAAAAAAAAAAAAAAAAAA"}`)}, + }, + config.Account{}, + gdpr.NewTCF2Config(config.TCF2{Enabled: true}, config.AccountGDPR{}), + metrics.Labels{}, + &RequestPrivacy{ + Consent: "", + GDPRDefaultValue: gdpr.SignalNo, + GDPRSignal: gdpr.SignalYes, + GDPRChannelEnabled: true, + GDPREnforced: true, + }, + 1, + false, + }, + { + "Request with consent, default no, signal invalid, should return fatal error", + &openrtb2.BidRequest{ + Regs: &openrtb2.Regs{GDPR: &SignalYes, Ext: json.RawMessage(`{"gdpr1"}`)}, + User: &openrtb2.User{Ext: json.RawMessage(`{"consent":"CPuKGCPPuKGCPNEAAAENCZCAAAAAAAAAAAAAAAAAAAAA"}`)}, + }, + config.Account{}, + gdpr.NewTCF2Config(config.TCF2{Enabled: true}, config.AccountGDPR{}), + metrics.Labels{}, + nil, + 1, + true, + }, + { + "Request without consent, default no, signal no, but gpp exists", + &openrtb2.BidRequest{ + Regs: &openrtb2.Regs{GPP: "DBACNYA~CPXxRfAPXxRfAAfKABENB-CgAAAAAAAAAAYgAAAAAAAA~1NYN", GPPSID: []int8{2, 6}}, + User: &openrtb2.User{}, + }, + config.Account{}, + gdpr.NewTCF2Config(config.TCF2{Enabled: true}, config.AccountGDPR{}), + metrics.Labels{}, + &RequestPrivacy{ + Consent: "CPXxRfAPXxRfAAfKABENB-CgAAAAAAAAAAYgAAAAAAAA", + GDPRDefaultValue: gdpr.SignalNo, + GDPRSignal: gdpr.SignalYes, + GDPRChannelEnabled: true, + GDPREnforced: true, + }, + 0, + false, + }, + { + "Request without consent, default no, signal no, but invalid gpp exists", + &openrtb2.BidRequest{ + Regs: &openrtb2.Regs{GPP: "CPXxRfAPXxRfAAfKABENB-CgAAAAAAAAAAYgAAAAAAAA~1NYN", GPPSID: []int8{2, 6}}, + User: &openrtb2.User{}, + }, + config.Account{}, + gdpr.NewTCF2Config(config.TCF2{Enabled: true}, config.AccountGDPR{}), + metrics.Labels{}, + &RequestPrivacy{ + Consent: "", + GDPRDefaultValue: gdpr.SignalNo, + GDPRSignal: gdpr.SignalYes, + GDPRChannelEnabled: true, + GDPREnforced: true, + }, + 1, + false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req := test.req + if req.User != nil && len(req.User.Ext) > 0 { + userExt := make(map[string]any) + _ = json.Unmarshal(req.User.Ext, &userExt) + if val, ok := userExt["consent"]; ok && test.requestPrivacy != nil { + consent, _ := val.(string) + test.requestPrivacy.ParsedConsent, _ = vendorconsent.ParseString(consent) + } + } + + if req.Regs != nil && req.Regs.GPP != "" && test.requestPrivacy != nil { + test.requestPrivacy.ParsedGPP, _ = gpplib.Parse(req.Regs.GPP) + for _, section := range test.requestPrivacy.ParsedGPP.Sections { + if section.GetID() == gppConstants.SectionTCFEU2 { + test.requestPrivacy.ParsedConsent, _ = vendorconsent.ParseString(section.GetValue()) + } + } + } + + e := new(exchange) + auctionRequest := new(AuctionRequest) + auctionRequest.BidRequestWrapper = &openrtb_ext.RequestWrapper{BidRequest: req} + auctionRequest.Account = test.account + auctionRequest.TCF2Config = test.tcf2config + auctionRequest.LegacyLabels = test.legacyLabels + + output, errs := e.extractRequestPrivacy(auctionRequest) + assert.True(t, reflect.DeepEqual(test.requestPrivacy, output), "expected output to match. Expected: %+v, Got: %+v", test.requestPrivacy, output) + assert.Equal(t, test.errsCount, len(errs)) + if test.errsHaveFatal { + assert.True(t, errortypes.FirstFatalError(errs) != nil) + } + }) + } +} + +func TestExtractRequestPrivacyLMT(t *testing.T) { + var ( + Lmt0 int8 = 0 + Lmt1 int8 = 1 + ) + tests := []struct { + name string + req *openrtb2.BidRequest + privacyConfig config.Privacy + expected bool + }{ + { + "Request device lmt is 0", + &openrtb2.BidRequest{ + Device: &openrtb2.Device{Lmt: &Lmt0}, + }, + config.Privacy{ + LMT: config.LMT{Enforce: true}, + }, + false, + }, + { + "Request device lmt is 1", + &openrtb2.BidRequest{ + Device: &openrtb2.Device{Lmt: &Lmt1}, + }, + config.Privacy{ + LMT: config.LMT{Enforce: true}, + }, + true, + }, + { + "Request device lmt is nil", + &openrtb2.BidRequest{ + Device: &openrtb2.Device{Lmt: nil}, + }, + config.Privacy{ + LMT: config.LMT{Enforce: true}, + }, + false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + e := new(exchange) + e.privacyConfig = test.privacyConfig + auctionRequest := new(AuctionRequest) + auctionRequest.BidRequestWrapper = &openrtb_ext.RequestWrapper{BidRequest: test.req} + auctionRequest.TCF2Config = gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}) + + output, _ := e.extractRequestPrivacy(auctionRequest) + assert.Equal(t, test.expected, output.LMTEnforced) + }) + } +} + +func TestExtractRequestPrivacyCOPPA(t *testing.T) { + tests := []struct { + name string + req *openrtb2.BidRequest + expected bool + }{ + { + "Request COPPA is 0", + &openrtb2.BidRequest{ + Regs: &openrtb2.Regs{COPPA: 0}, + }, + false, + }, + { + "Request COPPA is 1", + &openrtb2.BidRequest{ + Regs: &openrtb2.Regs{COPPA: 1}, + }, + true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + e := new(exchange) + auctionRequest := new(AuctionRequest) + auctionRequest.BidRequestWrapper = &openrtb_ext.RequestWrapper{BidRequest: test.req} + auctionRequest.TCF2Config = gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}) + + output, _ := e.extractRequestPrivacy(auctionRequest) + assert.Equal(t, test.expected, output.COPPAEnforced) + }) + } +} diff --git a/exchange/geolocation.go b/exchange/geolocation.go new file mode 100644 index 00000000000..e3bb2eaa532 --- /dev/null +++ b/exchange/geolocation.go @@ -0,0 +1,168 @@ +package exchange + +import ( + "context" + "errors" + + tcf2 "github.com/prebid/go-gdpr/vendorconsent/tcf2" + "github.com/prebid/openrtb/v20/openrtb2" + "github.com/prebid/prebid-server/v2/config" + "github.com/prebid/prebid-server/v2/config/countrycode" + "github.com/prebid/prebid-server/v2/gdpr" + "github.com/prebid/prebid-server/v2/geolocation" + "github.com/prebid/prebid-server/v2/metrics" + "github.com/prebid/prebid-server/v2/openrtb_ext" + "github.com/prebid/prebid-server/v2/privacy" + "github.com/prebid/prebid-server/v2/util/iputil" +) + +type GeoLocationResolver interface { + Lookup(ctx context.Context, ip string, country string) (*geolocation.GeoInfo, error) +} + +type geoLocationResolver struct { + geoloc geolocation.GeoLocation + me metrics.MetricsEngine +} + +func (g *geoLocationResolver) Lookup(ctx context.Context, ip string, country string) (*geolocation.GeoInfo, error) { + if g.geoloc == nil || ip == "" || country != "" { + return nil, errors.New("geolocation lookup skipped") + } + geoinfo, err := g.geoloc.Lookup(ctx, ip) + g.me.RecordGeoLocationRequest(err == nil) + return geoinfo, err +} + +func NewGeoLocationResolver(geoloc geolocation.GeoLocation, me metrics.MetricsEngine) *geoLocationResolver { + return &geoLocationResolver{ + geoloc: geoloc, + me: me, + } +} + +func countryFromDevice(device *openrtb2.Device) string { + if device == nil || device.Geo == nil { + return "" + } + return device.Geo.Country +} + +func EnrichGeoLocation(ctx context.Context, req *openrtb_ext.RequestWrapper, account config.Account, geoResolver GeoLocationResolver) (errs []error) { + if !account.GeoLocation.IsGeoLocationEnabled() { + return nil + } + + device := req.BidRequest.Device + if device == nil { + return []error{errors.New("device is nil")} + } + + ip := device.IP + if ip == "" { + ip = device.IPv6 + } + country := countryFromDevice(device) + geoinfo, err := geoResolver.Lookup(ctx, ip, country) + if err != nil { + return []error{err} + } + + updateDeviceGeo(req.BidRequest, geoinfo) + + return +} + +func EnrichGeoLocationWithPrivacy( + ctx context.Context, + req *openrtb_ext.RequestWrapper, + account config.Account, + geoResolver GeoLocationResolver, + requestPrivacy *RequestPrivacy, + tcf2Config gdpr.TCF2ConfigReader, +) (errs []error) { + if !account.GeoLocation.IsGeoLocationEnabled() { + return nil + } + + device := req.BidRequest.Device + if device == nil { + return []error{errors.New("device is nil")} + } + + if requestPrivacy.GDPREnforced { + return + } + + country := countryFromDevice(device) + ip := maybeMaskIP(device, account.Privacy, requestPrivacy, tcf2Config) + geoinfo, err := geoResolver.Lookup(ctx, ip, country) + if err != nil { + return []error{err} + } + + updateDeviceGeo(req.BidRequest, geoinfo) + + return +} + +func maybeMaskIP(device *openrtb2.Device, accountPrivacy config.AccountPrivacy, requestPrivacy *RequestPrivacy, tcf2Config gdpr.TCF2ConfigReader) string { + if device == nil { + return "" + } + + shouldBeMasked := shouldMaskIP(requestPrivacy, tcf2Config) + if device.IP != "" { + if shouldBeMasked { + return privacy.ScrubIP(device.IP, accountPrivacy.IPv4Config.AnonKeepBits, iputil.IPv4BitSize) + } + return device.IP + } else if device.IPv6 != "" { + if shouldBeMasked { + return privacy.ScrubIP(device.IPv6, accountPrivacy.IPv6Config.AnonKeepBits, iputil.IPv6BitSize) + } + return device.IPv6 + } + return "" +} + +func shouldMaskIP(requestPrivacy *RequestPrivacy, tcf2Config gdpr.TCF2ConfigReader) bool { + if requestPrivacy.COPPAEnforced || requestPrivacy.LMTEnforced { + return true + } + if requestPrivacy.ParsedConsent != nil { + cm, ok := requestPrivacy.ParsedConsent.(tcf2.ConsentMetadata) + return ok && !tcf2Config.FeatureOneEnforced() && !cm.SpecialFeatureOptIn(1) + } + return false +} + +func updateDeviceGeo(req *openrtb2.BidRequest, geoinfo *geolocation.GeoInfo) { + if req.Device == nil || geoinfo == nil { + return + } + + device := *req.Device + if device.Geo == nil { + device.Geo = &openrtb2.Geo{} + } + + geo := device.Geo + if alpha3 := countrycode.ToAlpha3(geoinfo.Country); alpha3 != "" { + geo.Country = alpha3 + } + if geoinfo.Region != "" { + geo.Region = geoinfo.Region + } + if offset, err := geolocation.TimezoneToUTCOffset(geoinfo.TimeZone); err == nil { + geo.UTCOffset = int64(offset) + } + + req.Device = &device +} + +type NilGeoLocationResolver struct{} + +func (g *NilGeoLocationResolver) Lookup(ctx context.Context, ip string, country string) (*geolocation.GeoInfo, error) { + return &geolocation.GeoInfo{}, nil +} diff --git a/exchange/geolocation_test.go b/exchange/geolocation_test.go new file mode 100644 index 00000000000..ca60de0c674 --- /dev/null +++ b/exchange/geolocation_test.go @@ -0,0 +1,643 @@ +package exchange + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/prebid/go-gdpr/vendorconsent" + "github.com/prebid/openrtb/v20/openrtb2" + "github.com/prebid/prebid-server/v2/config" + "github.com/prebid/prebid-server/v2/config/countrycode" + "github.com/prebid/prebid-server/v2/gdpr" + "github.com/prebid/prebid-server/v2/geolocation" + "github.com/prebid/prebid-server/v2/geolocation/geolocationtest" + "github.com/prebid/prebid-server/v2/metrics" + "github.com/prebid/prebid-server/v2/openrtb_ext" + "github.com/prebid/prebid-server/v2/util/jsonutil" + "github.com/stretchr/testify/assert" +) + +type mockMetrics struct { + metrics.MetricsEngineMock + success int64 + fail int64 +} + +func (m *mockMetrics) RecordGeoLocationRequest(success bool) { + if success { + atomic.AddInt64(&m.success, 1) + } else { + atomic.AddInt64(&m.fail, 1) + } +} + +type mockGeoLocationResolverResult struct { + geo *geolocation.GeoInfo + err error +} + +type mockGeoLocationResolver struct { + data map[string]mockGeoLocationResolverResult +} + +func (g *mockGeoLocationResolver) Lookup(ctx context.Context, ip string, country string) (*geolocation.GeoInfo, error) { + if ip == "" || country != "" { + return &geolocation.GeoInfo{}, assert.AnError + } + + if g.data == nil { + return &geolocation.GeoInfo{}, nil + } + if result, ok := g.data[ip]; ok { + return result.geo, result.err + } + return &geolocation.GeoInfo{}, nil +} + +func makeMockGeoLocationResolver(data map[string]mockGeoLocationResolverResult) GeoLocationResolver { + return &mockGeoLocationResolver{data: data} +} + +func TestGeoLocationResolver(t *testing.T) { + me := &mockMetrics{} + geoservice := geolocationtest.NewMockGeoLocation(map[string]*geolocation.GeoInfo{ + "1.1.1.1": {Country: "CN"}, + "1.1.1.2": {Country: "US"}, + }) + tests := []struct { + name string + geoloc geolocation.GeoLocation + ip string + country string + geoCountry string + geoErr bool + }{ + { + "Resolver is nil", + nil, "1.1.1.1", "", "", true, + }, + { + "Lookup empty IP", + geoservice, "", "", "", true, + }, + { + "Lookup valid IP, country has value", + geoservice, "1.1.1.1", "CN", "", true, + }, + { + "Lookup unknown IP", + geoservice, "2.2.2.2", "", "", true, + }, + { + "Lookup successful, response country is CN", + geoservice, "1.1.1.1", "", "CN", false, + }, + { + "Lookup successful, response country is US", + geoservice, "1.1.1.2", "", "US", false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resolver := NewGeoLocationResolver(test.geoloc, me) + geo, err := resolver.Lookup(context.Background(), test.ip, test.country) + if test.geoErr { + assert.Error(t, err, "geolocation should return error") + } else { + assert.NoError(t, err, "geolocation should not return error. Error: %v", err) + assert.Equal(t, test.geoCountry, geo.Country) + } + }) + } + + assert.Equal(t, int64(2), me.success, "metrics success count should be 2") + assert.Equal(t, int64(1), me.fail, "metrics fail count should be 1") +} + +func TestEnrichGeoLocation(t *testing.T) { + countrycode.Load("CN,CHN\n") + + resolver := makeMockGeoLocationResolver(map[string]mockGeoLocationResolverResult{ + "1.1.1.1": { + geo: &geolocation.GeoInfo{Country: "CN", Region: "Shanghai", TimeZone: "Asia/Shanghai"}, + err: nil, + }, + "1111:2222:3333:4400::": { + geo: &geolocation.GeoInfo{Country: "CN", Region: "Sichuan", TimeZone: "UTC"}, + err: nil, + }, + "2.2.2.2": { + geo: nil, + err: assert.AnError, + }, + }) + tests := []struct { + name string + req *openrtb2.BidRequest + account config.Account + resolver GeoLocationResolver + expectedCountry string + expectedRegion string + expectedUTCOffset int64 + errsCount int + }{ + { + "Enrich device. geoLocation is disabled", + &openrtb2.BidRequest{ + Device: &openrtb2.Device{IP: "1.1.1.1"}, + }, + config.Account{GeoLocation: config.AccountGeoLocation{Enabled: false}}, + resolver, + "", + "", + 0, + 0, + }, + { + "Enrich device. GeoLocation is enabled, IPv4 is used", + &openrtb2.BidRequest{ + Device: &openrtb2.Device{IP: "1.1.1.1", IPv6: "1111:2222:3333:4400::"}, + }, + config.Account{GeoLocation: config.AccountGeoLocation{Enabled: true}}, + resolver, + "CHN", + "Shanghai", + 480, + 0, + }, + { + "Enrich device. GeoLocation is enabled, IPv6 is used", + &openrtb2.BidRequest{ + Device: &openrtb2.Device{IPv6: "1111:2222:3333:4400::"}, + }, + config.Account{GeoLocation: config.AccountGeoLocation{Enabled: true}}, + resolver, + "CHN", + "Sichuan", + 0, + 0, + }, + { + "Enrich device. device is nil", + &openrtb2.BidRequest{}, + config.Account{GeoLocation: config.AccountGeoLocation{Enabled: true}}, + resolver, + "", + "", + 0, + 1, + }, + { + "Enrich device. country exists", + &openrtb2.BidRequest{ + Device: &openrtb2.Device{IP: "1.1.1.1", Geo: &openrtb2.Geo{Country: "USA"}}, + }, + config.Account{GeoLocation: config.AccountGeoLocation{Enabled: true}}, + resolver, + "USA", + "", + 0, + 1, + }, + { + "Enrich device. resolver returns error", + &openrtb2.BidRequest{ + Device: &openrtb2.Device{IP: "2.2.2.2"}, + }, + config.Account{GeoLocation: config.AccountGeoLocation{Enabled: true}}, + resolver, + "", + "", + 0, + 1, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req := &openrtb_ext.RequestWrapper{BidRequest: test.req} + errs := EnrichGeoLocation(context.Background(), req, test.account, test.resolver) + assert.Equal(t, test.errsCount, len(errs), "errors count should be %d", test.errsCount) + + var ( + country string + region string + utcoffset int64 + ) + if req.BidRequest.Device != nil && req.BidRequest.Device.Geo != nil { + country = req.BidRequest.Device.Geo.Country + region = req.BidRequest.Device.Geo.Region + utcoffset = req.BidRequest.Device.Geo.UTCOffset + } + assert.Equal(t, test.expectedCountry, country, "country should be %s", test.expectedCountry) + assert.Equal(t, test.expectedRegion, region, "region should be %s", test.expectedRegion) + assert.Equal(t, test.expectedUTCOffset, utcoffset, "utc offset should be %d", test.expectedUTCOffset) + }) + } +} + +func TestEnrichGeoLocationWithPrivacy(t *testing.T) { + countrycode.Load("CN,CHN\n") + + resolver := makeMockGeoLocationResolver(map[string]mockGeoLocationResolverResult{ + "1.1.1.0": { + geo: &geolocation.GeoInfo{Country: "CN", Region: "", TimeZone: "UTC"}, + err: nil, + }, + "1.1.1.1": { + geo: &geolocation.GeoInfo{Country: "CN", Region: "Shanghai", TimeZone: "Asia/Shanghai"}, + err: nil, + }, + "1111:2222:3333:4400::": { + geo: &geolocation.GeoInfo{Country: "CN", Region: "Sichuan", TimeZone: "UTC"}, + err: nil, + }, + "2.2.2.2": { + geo: nil, + err: assert.AnError, + }, + }) + tests := []struct { + name string + req *openrtb2.BidRequest + account config.Account + resolver GeoLocationResolver + requestPrivacy *RequestPrivacy + tcf2config gdpr.TCF2ConfigReader + expectedCountry string + expectedRegion string + expectedUTCOffset int64 + errsCount int + }{ + { + "Enrich device. geoLocation is disabled", + &openrtb2.BidRequest{ + Device: &openrtb2.Device{IP: "1.1.1.1"}, + }, + config.Account{ + Privacy: config.AccountPrivacy{IPv4Config: config.IPv4{AnonKeepBits: 24}, IPv6Config: config.IPv6{AnonKeepBits: 56}}, + GeoLocation: config.AccountGeoLocation{Enabled: false}, + }, + resolver, + &RequestPrivacy{GDPREnforced: false, LMTEnforced: false}, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + "", + "", + 0, + 0, + }, + { + "Enrich device. GDPR not enforced, LMT not enforced, device is nil", + &openrtb2.BidRequest{}, + config.Account{ + Privacy: config.AccountPrivacy{IPv4Config: config.IPv4{AnonKeepBits: 24}, IPv6Config: config.IPv6{AnonKeepBits: 56}}, + GeoLocation: config.AccountGeoLocation{Enabled: true}, + }, + resolver, + &RequestPrivacy{GDPREnforced: false, LMTEnforced: false}, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + "", + "", + 0, + 1, + }, + { + "Enrich device. GDPR not enforced, LMT not enforced, resolver returns error", + &openrtb2.BidRequest{ + Device: &openrtb2.Device{IP: "2.2.2.2"}, + }, + config.Account{ + Privacy: config.AccountPrivacy{IPv4Config: config.IPv4{AnonKeepBits: 24}, IPv6Config: config.IPv6{AnonKeepBits: 56}}, + GeoLocation: config.AccountGeoLocation{Enabled: true}, + }, + resolver, + &RequestPrivacy{GDPREnforced: false, LMTEnforced: false}, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + "", + "", + 0, + 1, + }, + { + "Enrich device. GDPR enforced, LMT not enforced, should not enrich", + &openrtb2.BidRequest{ + Device: &openrtb2.Device{IP: "1.1.1.1"}, + }, + config.Account{ + Privacy: config.AccountPrivacy{IPv4Config: config.IPv4{AnonKeepBits: 24}, IPv6Config: config.IPv6{AnonKeepBits: 56}}, + GeoLocation: config.AccountGeoLocation{Enabled: true}, + }, + resolver, + &RequestPrivacy{GDPREnforced: true, LMTEnforced: false}, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + "", + "", + 0, + 0, + }, + { + "Enrich device. GDPR not enforced, LMT not enforced, country exists", + &openrtb2.BidRequest{ + Device: &openrtb2.Device{IP: "1.1.1.1", Geo: &openrtb2.Geo{Country: "USA"}}, + }, + config.Account{ + Privacy: config.AccountPrivacy{IPv4Config: config.IPv4{AnonKeepBits: 24}, IPv6Config: config.IPv6{AnonKeepBits: 56}}, + GeoLocation: config.AccountGeoLocation{Enabled: true}, + }, + resolver, + &RequestPrivacy{GDPREnforced: false, LMTEnforced: false}, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + "USA", + "", + 0, + 1, + }, + { + "Enrich device. GDPR not enforced, LMT not enforced, IPv4 is used", + &openrtb2.BidRequest{ + Device: &openrtb2.Device{IP: "1.1.1.1", IPv6: "1111:2222:3333:4400::"}, + }, + config.Account{ + Privacy: config.AccountPrivacy{IPv4Config: config.IPv4{AnonKeepBits: 24}, IPv6Config: config.IPv6{AnonKeepBits: 56}}, + GeoLocation: config.AccountGeoLocation{Enabled: true}, + }, + resolver, + &RequestPrivacy{GDPREnforced: false, LMTEnforced: false}, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + "CHN", + "Shanghai", + 480, + 0, + }, + { + "Enrich device. GDPR not enforced, LMT enforced, IPv4 is used", + &openrtb2.BidRequest{ + Device: &openrtb2.Device{IP: "1.1.1.1", IPv6: "1111:2222:3333:4400::"}, + }, + config.Account{ + Privacy: config.AccountPrivacy{IPv4Config: config.IPv4{AnonKeepBits: 24}, IPv6Config: config.IPv6{AnonKeepBits: 56}}, + GeoLocation: config.AccountGeoLocation{Enabled: true}, + }, + resolver, + &RequestPrivacy{GDPREnforced: false, LMTEnforced: true}, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + "CHN", + "", + 0, + 0, + }, + { + "Enrich device. GDPR not enforced, LMT enforced, IPv6 is used", + &openrtb2.BidRequest{ + Device: &openrtb2.Device{IPv6: "1111:2222:3333:4400::"}, + }, + config.Account{ + Privacy: config.AccountPrivacy{IPv4Config: config.IPv4{AnonKeepBits: 24}, IPv6Config: config.IPv6{AnonKeepBits: 56}}, + GeoLocation: config.AccountGeoLocation{Enabled: true}, + }, + resolver, + &RequestPrivacy{GDPREnforced: false, LMTEnforced: true}, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + "CHN", + "Sichuan", + 0, + 0, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req := &openrtb_ext.RequestWrapper{BidRequest: test.req} + errs := EnrichGeoLocationWithPrivacy(context.Background(), req, test.account, test.resolver, test.requestPrivacy, test.tcf2config) + assert.Equal(t, test.errsCount, len(errs), "errors count should be %d", test.errsCount) + + var ( + country string + region string + utcoffset int64 + ) + if req.BidRequest.Device != nil && req.BidRequest.Device.Geo != nil { + country = req.BidRequest.Device.Geo.Country + region = req.BidRequest.Device.Geo.Region + utcoffset = req.BidRequest.Device.Geo.UTCOffset + } + assert.Equal(t, test.expectedCountry, country, "country should be %s", test.expectedCountry) + assert.Equal(t, test.expectedRegion, region, "region should be %s", test.expectedRegion) + assert.Equal(t, test.expectedUTCOffset, utcoffset, "utc offset should be %d", test.expectedUTCOffset) + }) + } +} + +func TestCountryFromDevice(t *testing.T) { + tests := []struct { + device *openrtb2.Device + country string + }{ + {nil, ""}, + {&openrtb2.Device{}, ""}, + {&openrtb2.Device{Geo: &openrtb2.Geo{}}, ""}, + {&openrtb2.Device{Geo: &openrtb2.Geo{Country: "US"}}, "US"}, + } + + for _, test := range tests { + assert.Equal(t, test.country, countryFromDevice(test.device)) + } +} + +func TestMaybeMaskIP(t *testing.T) { + tests := []struct { + name string + device *openrtb2.Device + accountPrivacy config.AccountPrivacy + reqPrivacy *RequestPrivacy + tcf2Config gdpr.TCF2ConfigReader + output string + }{ + { + "Device is nil, ip should be empty", + nil, + config.AccountPrivacy{}, + &RequestPrivacy{LMTEnforced: false}, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + "", + }, + { + "IPv4 and IPv6 both empty, ip should be empty", + &openrtb2.Device{IP: "", IPv6: ""}, + config.AccountPrivacy{}, + &RequestPrivacy{LMTEnforced: false}, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + "", + }, + { + "IPv4 with no privacy", + &openrtb2.Device{IP: "1.1.1.1", IPv6: "1111:2222:3333:4444:5555:6666:7777:8888"}, + config.AccountPrivacy{IPv4Config: config.IPv4{AnonKeepBits: 24}}, + &RequestPrivacy{LMTEnforced: false}, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + "1.1.1.1", + }, + { + "IPv6 with no privacy", + &openrtb2.Device{IPv6: "1111:2222:3333:4444:5555:6666:7777:8888"}, + config.AccountPrivacy{IPv6Config: config.IPv6{AnonKeepBits: 56}}, + &RequestPrivacy{LMTEnforced: false}, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + "1111:2222:3333:4444:5555:6666:7777:8888", + }, + { + "IPv4 and IPv6 with privacy, IPv4 is preferred", + &openrtb2.Device{IP: "1.1.1.1", IPv6: "1111:2222:3333:4444:5555:6666:7777:8888"}, + config.AccountPrivacy{IPv4Config: config.IPv4{AnonKeepBits: 24}}, + &RequestPrivacy{LMTEnforced: true}, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + "1.1.1.0", + }, + { + "IPv6 with privacy", + &openrtb2.Device{IPv6: "1111:2222:3333:4444:5555:6666:7777:8888"}, + config.AccountPrivacy{IPv6Config: config.IPv6{AnonKeepBits: 56}}, + &RequestPrivacy{LMTEnforced: true}, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + "1111:2222:3333:4400::", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ip := maybeMaskIP(test.device, test.accountPrivacy, test.reqPrivacy, test.tcf2Config) + assert.Equal(t, test.output, ip) + }) + } +} + +func TestShouldMaskIP(t *testing.T) { + tests := []struct { + desc string + reqPrivacy *RequestPrivacy + tcf2Config gdpr.TCF2ConfigReader + output bool + }{ + { + "Nothing enforced", + &RequestPrivacy{ + COPPAEnforced: false, + LMTEnforced: false, + Consent: "", + }, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + false, + }, + { + "COPPA enforced", + &RequestPrivacy{ + COPPAEnforced: true, + LMTEnforced: false, + Consent: "", + }, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + true, + }, + { + "LMT enforced", + &RequestPrivacy{ + COPPAEnforced: false, + LMTEnforced: true, + Consent: "", + }, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + true, + }, + { + "TCF2 without SP1 consent enforced", + &RequestPrivacy{ + COPPAEnforced: false, + LMTEnforced: false, + Consent: "CPuKGCPPuKGCPNEAAAENCZCAAAAAAAAAAAAAAAAAAAAA", + }, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + true, + }, + { + "TCF2 with SP1 consent enforced", + &RequestPrivacy{ + COPPAEnforced: false, + LMTEnforced: false, + Consent: "CQDkxqbQDkxqbHcAAAENCZCIAAAAAAAAAAAAAAAAAAAA.II7Nd_X__bX9n-_7_6ft0eY1f9_r37uQzDhfNs-8F3L_W_LwX32E7NF36tq4KmR4ku1bBIQNtHMnUDUmxaolVrzHsak2cpyNKJ_JkknsZe2dYGF9Pn9lD-YKZ7_5_9_f52T_9_9_-39z3_9f___dv_-__-vjf_599n_v9fV_78_Kf9______-____________8A", + }, + gdpr.NewTCF2Config(config.TCF2{}, config.AccountGDPR{}), + false, + }, + { + "TCF2 with SP1 host enforced", + &RequestPrivacy{ + COPPAEnforced: false, + LMTEnforced: false, + Consent: "", + }, + gdpr.NewTCF2Config( + config.TCF2{SpecialFeature1: config.TCF2SpecialFeature{Enforce: true}}, + config.AccountGDPR{}, + ), + false, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + if test.reqPrivacy.Consent != "" { + parsedConsent, err := vendorconsent.ParseString(test.reqPrivacy.Consent) + assert.NoError(t, err, "Failed to parse consent string") + test.reqPrivacy.ParsedConsent = parsedConsent + } + assert.Equal(t, test.output, shouldMaskIP(test.reqPrivacy, test.tcf2Config)) + }) + } +} + +func TestUpdateDeviceGeo(t *testing.T) { + countrycode.Load("CN,CHN\n") + + tests := []struct { + device *openrtb2.Device + geoinfo *geolocation.GeoInfo + expectedDevice *openrtb2.Device + }{ + { + nil, + &geolocation.GeoInfo{Country: "CN"}, + nil, + }, + { + &openrtb2.Device{}, + nil, + &openrtb2.Device{}, + }, + { + &openrtb2.Device{Geo: &openrtb2.Geo{}}, + nil, + &openrtb2.Device{Geo: &openrtb2.Geo{}}, + }, + { + &openrtb2.Device{}, + &geolocation.GeoInfo{Country: "CN", Region: "Shanghai", TimeZone: "Asia/Shanghai"}, + &openrtb2.Device{Geo: &openrtb2.Geo{Country: "CHN", Region: "Shanghai", UTCOffset: 480}}, + }, + // bad geo info + { + &openrtb2.Device{Geo: &openrtb2.Geo{Country: "CN", Region: "Chongqing", UTCOffset: 420}}, + &geolocation.GeoInfo{Country: "", Region: "", TimeZone: "UNKNOWN"}, + &openrtb2.Device{Geo: &openrtb2.Geo{Country: "CN", Region: "Chongqing", UTCOffset: 420}}, + }, + } + + for _, test := range tests { + req := &openrtb2.BidRequest{Device: test.device} + updateDeviceGeo(req, test.geoinfo) + expected, _ := jsonutil.Marshal(test.expectedDevice) + updated, _ := jsonutil.Marshal(req.Device) + assert.Equal(t, string(expected), string(updated), "device should be %s", string(expected)) + } +} diff --git a/exchange/utils.go b/exchange/utils.go index 9b4ffd21f3e..f8512ba86f4 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -9,11 +9,9 @@ import ( "math/rand" "strings" - "github.com/prebid/go-gdpr/vendorconsent" gpplib "github.com/prebid/go-gpp" gppConstants "github.com/prebid/go-gpp/constants" "github.com/prebid/openrtb/v20/openrtb2" - "github.com/prebid/prebid-server/v2/config" "github.com/prebid/prebid-server/v2/errortypes" "github.com/prebid/prebid-server/v2/firstpartydata" @@ -60,10 +58,9 @@ type requestSplitter struct { func (rs *requestSplitter) cleanOpenRTBRequests(ctx context.Context, auctionReq AuctionRequest, requestExt *openrtb_ext.ExtRequest, - gdprSignal gdpr.Signal, - gdprEnforced bool, + requestPrivacy *RequestPrivacy, bidAdjustmentFactors map[string]float64, -) (allowedBidderRequests []BidderRequest, privacyLabels metrics.PrivacyLabels, errs []error) { +) (allowedBidderRequests []BidderRequest, errs []error) { req := auctionReq.BidRequestWrapper requestAliases, requestAliasesGVLIDs, errs := getRequestAliases(req) @@ -94,55 +91,26 @@ func (rs *requestSplitter) cleanOpenRTBRequests(ctx context.Context, //this function should be executed after getAuctionBidderRequests allBidderRequests = mergeBidderRequests(allBidderRequests, bidderNameToBidderReq) - var gpp gpplib.GppContainer - if req.BidRequest.Regs != nil && len(req.BidRequest.Regs.GPP) > 0 { - var gppErrs []error - gpp, gppErrs = gpplib.Parse(req.BidRequest.Regs.GPP) - if len(gppErrs) > 0 { - errs = append(errs, gppErrs[0]) - } - } - if auctionReq.Account.PriceFloors.IsAdjustForBidAdjustmentEnabled() { //Apply BidAdjustmentFactor to imp.BidFloor applyBidAdjustmentToFloor(allBidderRequests, bidAdjustmentFactors) } - consent, err := getConsent(req, gpp) - if err != nil { - errs = append(errs, err) - } - - ccpaEnforcer, err := extractCCPA(req.BidRequest, rs.privacyConfig, &auctionReq.Account, requestAliases, channelTypeMap[auctionReq.LegacyLabels.RType], gpp) + ccpaEnforcer, err := extractCCPA(req.BidRequest, rs.privacyConfig, &auctionReq.Account, requestAliases, channelTypeMap[auctionReq.LegacyLabels.RType], requestPrivacy.ParsedGPP) if err != nil { errs = append(errs, err) } - lmtEnforcer := extractLMT(req.BidRequest, rs.privacyConfig) - - // request level privacy policies - coppa := req.BidRequest.Regs != nil && req.BidRequest.Regs.COPPA == 1 - lmt := lmtEnforcer.ShouldEnforce(unknownBidder) - - privacyLabels.CCPAProvided = ccpaEnforcer.CanEnforce() - privacyLabels.CCPAEnforced = ccpaEnforcer.ShouldEnforce(unknownBidder) - privacyLabels.COPPAEnforced = coppa - privacyLabels.LMTEnforced = lmt + requestPrivacy.CCPAProvided = ccpaEnforcer.CanEnforce() + requestPrivacy.CCPAEnforced = ccpaEnforcer.ShouldEnforce(unknownBidder) var gdprPerms gdpr.Permissions = &gdpr.AlwaysAllow{} - if gdprEnforced { - privacyLabels.GDPREnforced = true - parsedConsent, err := vendorconsent.ParseString(consent) - if err == nil { - version := int(parsedConsent.Version()) - privacyLabels.GDPRTCFVersion = metrics.TCFVersionToValue(version) - } - + if requestPrivacy.GDPREnforced { gdprRequestInfo := gdpr.RequestInfo{ AliasGVLIDs: requestAliasesGVLIDs, - Consent: consent, - GDPRSignal: gdprSignal, + Consent: requestPrivacy.Consent, + GDPRSignal: requestPrivacy.GDPRSignal, PublisherID: auctionReq.LegacyLabels.PubID, } gdprPerms = rs.gdprPermsBuilder(auctionReq.TCF2Config, gdprRequestInfo) @@ -162,7 +130,7 @@ func (rs *requestSplitter) cleanOpenRTBRequests(ctx context.Context, var auctionPermissions gdpr.AuctionPermissions var gdprErr error - if gdprEnforced { + if requestPrivacy.GDPREnforced { auctionPermissions, gdprErr = gdprPerms.AuctionActivitiesAllowed(ctx, bidderRequest.BidderCoreName, bidderRequest.BidderName) if !auctionPermissions.AllowBidRequest { // auction request is not permitted by GDPR @@ -190,7 +158,7 @@ func (rs *requestSplitter) cleanOpenRTBRequests(ctx context.Context, } else { // run existing policies (GDPR, CCPA, COPPA, LMT) // potentially block passing IDs based on GDPR - if gdprEnforced && (gdprErr != nil || !auctionPermissions.PassID) { + if requestPrivacy.GDPREnforced && (gdprErr != nil || !auctionPermissions.PassID) { privacy.ScrubGdprID(reqWrapper) buyerUIDRemoved = true } @@ -210,7 +178,7 @@ func (rs *requestSplitter) cleanOpenRTBRequests(ctx context.Context, } else { // run existing policies (GDPR, CCPA, COPPA, LMT) // potentially block passing geo based on GDPR - if gdprEnforced && (gdprErr != nil || !auctionPermissions.PassGeo) { + if requestPrivacy.GDPREnforced && (gdprErr != nil || !auctionPermissions.PassGeo) { privacy.ScrubGeoAndDeviceIP(reqWrapper, ipConf) } // potentially block passing geo based on CCPA @@ -219,8 +187,8 @@ func (rs *requestSplitter) cleanOpenRTBRequests(ctx context.Context, } } - if lmt || coppa { - privacy.ScrubDeviceIDsIPsUserDemoExt(reqWrapper, ipConf, "eids", coppa) + if requestPrivacy.LMTEnforced || requestPrivacy.COPPAEnforced { + privacy.ScrubDeviceIDsIPsUserDemoExt(reqWrapper, ipConf, "eids", requestPrivacy.COPPAEnforced) } passTIDAllowed := auctionReq.Activities.Allow(privacy.ActivityTransmitTIDs, scopedName, privacy.NewRequestFromBidRequest(*req)) @@ -238,8 +206,8 @@ func (rs *requestSplitter) cleanOpenRTBRequests(ctx context.Context, // GPP downgrade: always downgrade unless we can confirm GPP is supported if shouldSetLegacyPrivacy(rs.bidderInfo, string(bidderRequest.BidderCoreName)) { - setLegacyGDPRFromGPP(bidderRequest.BidRequest, gpp) - setLegacyUSPFromGPP(bidderRequest.BidRequest, gpp) + setLegacyGDPRFromGPP(bidderRequest.BidRequest, requestPrivacy.ParsedGPP) + setLegacyUSPFromGPP(bidderRequest.BidRequest, requestPrivacy.ParsedGPP) } } diff --git a/exchange/utils_test.go b/exchange/utils_test.go index 69cfdf70abf..999752c9273 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -8,6 +8,7 @@ import ( "sort" "testing" + "github.com/prebid/go-gdpr/vendorconsent" "github.com/prebid/prebid-server/v2/stored_responses" gpplib "github.com/prebid/go-gpp" @@ -767,7 +768,7 @@ func TestCleanOpenRTBRequests(t *testing.T) { hostSChainNode: nil, bidderInfo: config.BidderInfos{}, } - bidderRequests, _, err := reqSplitter.cleanOpenRTBRequests(context.Background(), test.req, nil, gdpr.SignalNo, false, map[string]float64{}) + bidderRequests, err := reqSplitter.cleanOpenRTBRequests(context.Background(), test.req, nil, &RequestPrivacy{COPPAEnforced: test.applyCOPPA}, map[string]float64{}) if test.hasError { assert.NotNil(t, err, "Error shouldn't be nil") } else { @@ -833,7 +834,7 @@ func TestCleanOpenRTBRequestsWithFPD(t *testing.T) { bidderInfo: config.BidderInfos{}, } - bidderRequests, _, err := reqSplitter.cleanOpenRTBRequests(context.Background(), test.req, nil, gdpr.SignalNo, false, map[string]float64{}) + bidderRequests, err := reqSplitter.cleanOpenRTBRequests(context.Background(), test.req, nil, &RequestPrivacy{}, map[string]float64{}) assert.Empty(t, err, "No errors should be returned") for _, bidderRequest := range bidderRequests { bidderName := bidderRequest.BidderName @@ -1148,7 +1149,7 @@ func TestCleanOpenRTBRequestsWithBidResponses(t *testing.T) { bidderInfo: config.BidderInfos{}, } - actualBidderRequests, _, err := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, gdpr.SignalNo, false, map[string]float64{}) + actualBidderRequests, err := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, &RequestPrivacy{}, map[string]float64{}) assert.Empty(t, err, "No errors should be returned") assert.Len(t, actualBidderRequests, len(test.expectedBidderRequests), "result len doesn't match for testCase %s", test.description) for _, actualBidderRequest := range actualBidderRequests { @@ -1320,7 +1321,8 @@ func TestCleanOpenRTBRequestsCCPA(t *testing.T) { bidderInfo: config.BidderInfos{}, } - bidderRequests, privacyLabels, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, gdpr.SignalNo, false, map[string]float64{}) + requestPrivacy := &RequestPrivacy{} + bidderRequests, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, requestPrivacy, map[string]float64{}) result := bidderRequests[0] assert.Nil(t, errs) @@ -1333,7 +1335,7 @@ func TestCleanOpenRTBRequestsCCPA(t *testing.T) { assert.NotEqual(t, result.BidRequest.Device.DIDMD5, "", test.description+":Device.DIDMD5") metricsMock.AssertNotCalled(t, "RecordAdapterBuyerUIDScrubbed", openrtb_ext.BidderAppnexus) } - assert.Equal(t, test.expectPrivacyLabels, privacyLabels, test.description+":PrivacyLabels") + assert.Equal(t, test.expectPrivacyLabels, requestPrivacy.MakePrivacyLabels(), test.description+":PrivacyLabels") } } @@ -1399,7 +1401,7 @@ func TestCleanOpenRTBRequestsCCPAErrors(t *testing.T) { bidderInfo: config.BidderInfos{}, } - _, _, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, &reqExtStruct, gdpr.SignalNo, false, map[string]float64{}) + _, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, &reqExtStruct, &RequestPrivacy{}, map[string]float64{}) assert.ElementsMatch(t, []error{test.expectError}, errs, test.description) } @@ -1458,7 +1460,8 @@ func TestCleanOpenRTBRequestsCOPPA(t *testing.T) { bidderInfo: config.BidderInfos{}, } - bidderRequests, privacyLabels, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, gdpr.SignalNo, false, map[string]float64{}) + requestPrivacy := &RequestPrivacy{COPPAEnforced: test.coppa == 1} + bidderRequests, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, requestPrivacy, map[string]float64{}) result := bidderRequests[0] assert.Nil(t, errs) @@ -1469,7 +1472,7 @@ func TestCleanOpenRTBRequestsCOPPA(t *testing.T) { assert.NotEqual(t, result.BidRequest.User.BuyerUID, "", test.description+":User.BuyerUID") assert.NotEqual(t, result.BidRequest.User.Yob, int64(0), test.description+":User.Yob") } - assert.Equal(t, test.expectPrivacyLabels, privacyLabels, test.description+":PrivacyLabels") + assert.Equal(t, test.expectPrivacyLabels, requestPrivacy.MakePrivacyLabels(), test.description+":PrivacyLabels") } } @@ -1551,7 +1554,7 @@ func TestCleanOpenRTBRequestsSChain(t *testing.T) { bidderInfo: config.BidderInfos{}, } - bidderRequests, _, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, extRequest, gdpr.SignalNo, false, map[string]float64{}) + bidderRequests, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, extRequest, &RequestPrivacy{}, map[string]float64{}) if test.hasError == true { assert.NotNil(t, errs) assert.Len(t, bidderRequests, 0) @@ -1622,7 +1625,7 @@ func TestCleanOpenRTBRequestsBidderParams(t *testing.T) { bidderInfo: config.BidderInfos{}, } - bidderRequests, _, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, extRequest, gdpr.SignalNo, false, map[string]float64{}) + bidderRequests, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, extRequest, &RequestPrivacy{}, map[string]float64{}) if test.hasError == true { assert.NotNil(t, errs) assert.Len(t, bidderRequests, 0) @@ -2214,7 +2217,9 @@ func TestCleanOpenRTBRequestsLMT(t *testing.T) { bidderInfo: config.BidderInfos{}, } - results, privacyLabels, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, gdpr.SignalNo, false, map[string]float64{}) + lmtEnforcer := extractLMT(req, privacyConfig) + requestPrivacy := &RequestPrivacy{LMTEnforced: lmtEnforcer.ShouldEnforce(unknownBidder)} + results, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, requestPrivacy, map[string]float64{}) result := results[0] assert.Nil(t, errs) @@ -2225,7 +2230,7 @@ func TestCleanOpenRTBRequestsLMT(t *testing.T) { assert.NotEqual(t, result.BidRequest.User.BuyerUID, "", test.description+":User.BuyerUID") assert.NotEqual(t, result.BidRequest.Device.DIDMD5, "", test.description+":Device.DIDMD5") } - assert.Equal(t, test.expectPrivacyLabels, privacyLabels, test.description+":PrivacyLabels") + assert.Equal(t, test.expectPrivacyLabels, requestPrivacy.MakePrivacyLabels(), test.description+":PrivacyLabels") } } @@ -2327,7 +2332,15 @@ func TestCleanOpenRTBRequestsGDPR(t *testing.T) { bidderInfo: config.BidderInfos{}, } - results, privacyLabels, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, test.gdprSignal, test.gdprEnforced, map[string]float64{}) + parsedConsent, _ := vendorconsent.ParseString(test.gdprConsent) + + requestPrivacy := &RequestPrivacy{ + GDPRSignal: test.gdprSignal, + GDPREnforced: test.gdprEnforced, + Consent: test.gdprConsent, + ParsedConsent: parsedConsent, + } + results, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, requestPrivacy, map[string]float64{}) result := results[0] if test.expectError { @@ -2345,7 +2358,7 @@ func TestCleanOpenRTBRequestsGDPR(t *testing.T) { assert.NotEqual(t, result.BidRequest.Device.DIDMD5, "", test.description+":Device.DIDMD5") metricsMock.AssertNotCalled(t, "RecordAdapterBuyerUIDScrubbed", openrtb_ext.BidderAppnexus) } - assert.Equal(t, test.expectPrivacyLabels, privacyLabels, test.description+":PrivacyLabels") + assert.Equal(t, test.expectPrivacyLabels, requestPrivacy.MakePrivacyLabels(), test.description+":PrivacyLabels") } } @@ -2422,7 +2435,11 @@ func TestCleanOpenRTBRequestsGDPRBlockBidRequest(t *testing.T) { bidderInfo: config.BidderInfos{}, } - results, _, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, gdpr.SignalYes, test.gdprEnforced, map[string]float64{}) + requestPrivacy := &RequestPrivacy{ + GDPRSignal: gdpr.SignalYes, + GDPREnforced: test.gdprEnforced, + } + results, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, requestPrivacy, map[string]float64{}) // extract bidder name from each request in the results bidders := []openrtb_ext.BidderName{} @@ -2502,6 +2519,11 @@ func TestCleanOpenRTBRequestsWithOpenRTBDowngrade(t *testing.T) { }, }.Builder + var gpp gpplib.GppContainer + if test.req.BidRequestWrapper.BidRequest.Regs != nil && len(test.req.BidRequestWrapper.BidRequest.Regs.GPP) > 0 { + gpp, _ = gpplib.Parse(test.req.BidRequestWrapper.BidRequest.Regs.GPP) + } + reqSplitter := &requestSplitter{ bidderToSyncerKey: map[string]string{}, me: &metrics.MetricsEngineMock{}, @@ -2510,7 +2532,10 @@ func TestCleanOpenRTBRequestsWithOpenRTBDowngrade(t *testing.T) { hostSChainNode: nil, bidderInfo: test.bidderInfos, } - bidderRequests, _, err := reqSplitter.cleanOpenRTBRequests(context.Background(), test.req, nil, gdpr.SignalNo, false, map[string]float64{}) + requstPrivacy := &RequestPrivacy{ + ParsedGPP: gpp, + } + bidderRequests, err := reqSplitter.cleanOpenRTBRequests(context.Background(), test.req, nil, requstPrivacy, map[string]float64{}) assert.Nil(t, err, "Err should be nil") bidRequest := bidderRequests[0] assert.Equal(t, test.expectRegs, bidRequest.BidRequest.Regs) @@ -3286,7 +3311,7 @@ func TestCleanOpenRTBRequestsSChainMultipleBidders(t *testing.T) { hostSChainNode: nil, bidderInfo: config.BidderInfos{}, } - bidderRequests, _, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, extRequest, gdpr.SignalNo, false, map[string]float64{}) + bidderRequests, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, extRequest, &RequestPrivacy{}, map[string]float64{}) assert.Nil(t, errs) assert.Len(t, bidderRequests, 2, "Bid request count is not 2") @@ -3407,7 +3432,7 @@ func TestCleanOpenRTBRequestsBidAdjustment(t *testing.T) { hostSChainNode: nil, bidderInfo: config.BidderInfos{}, } - results, _, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, gdpr.SignalNo, false, test.bidAdjustmentFactor) + results, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, &RequestPrivacy{}, test.bidAdjustmentFactor) result := results[0] assert.Nil(t, errs) assert.Equal(t, test.expectedImp, result.BidRequest.Imp, test.description) @@ -3850,7 +3875,7 @@ func TestCleanOpenRTBRequestsFilterBidderRequestExt(t *testing.T) { bidderInfo: config.BidderInfos{}, } - bidderRequests, _, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, extRequest, gdpr.SignalNo, false, map[string]float64{}) + bidderRequests, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, extRequest, &RequestPrivacy{}, map[string]float64{}) assert.Equal(t, test.wantError, len(errs) != 0, test.desc) sort.Slice(bidderRequests, func(i, j int) bool { return bidderRequests[i].BidderCoreName < bidderRequests[j].BidderCoreName @@ -4871,7 +4896,7 @@ func TestCleanOpenRTBRequestsActivities(t *testing.T) { bidderInfo: config.BidderInfos{}, } - bidderRequests, _, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, gdpr.SignalNo, false, map[string]float64{}) + bidderRequests, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, &RequestPrivacy{}, map[string]float64{}) assert.Empty(t, errs) assert.Len(t, bidderRequests, test.expectedReqNumber) diff --git a/geolocation/geoinfo.go b/geolocation/geoinfo.go new file mode 100644 index 00000000000..5740bb7093f --- /dev/null +++ b/geolocation/geoinfo.go @@ -0,0 +1,36 @@ +package geolocation + +type GeoInfo struct { + // Name of the geo location data provider. + Vendor string + + // Continent code in two-letter format. + Continent string + + // Country code in ISO-3166-1-alpha-2 format. + Country string + + // Region code in ISO-3166-2 format. + Region string + + // Numeric region code. + RegionCode int + + City string + + // Google Metro code. + MetroGoogle string + + // Nielsen Designated Market Areas (DMA's). + MetroNielsen int + + Zip string + + ConnectionSpeed string + + Lat float64 + + Lon float64 + + TimeZone string +} diff --git a/geolocation/geolocation.go b/geolocation/geolocation.go new file mode 100644 index 00000000000..b4b185c762d --- /dev/null +++ b/geolocation/geolocation.go @@ -0,0 +1,43 @@ +package geolocation + +import ( + "context" + "errors" + "time" + + "github.com/prebid/prebid-server/v2/util/timeutil" +) + +var ( + ErrDatabaseUnavailable = errors.New("database is unavailable") + ErrLookupIPInvalid = errors.New("lookup IP is invalid") + ErrLookupTimeout = errors.New("lookup timeout") +) + +// Retrieves geolocation information by IP address. +// +// Provided default implementation - MaxMind +// Each vendor (host company) might provide its own implementation. +type GeoLocation interface { + Lookup(ctx context.Context, ip string) (*GeoInfo, error) +} + +type NilGeoLocation struct{} + +func (g *NilGeoLocation) Lookup(ctx context.Context, ip string) (*GeoInfo, error) { + return &GeoInfo{}, nil +} + +func NewNilGeoLocation() *NilGeoLocation { + return &NilGeoLocation{} +} + +// TimezoneToUTCOffset returns UTC offset of timezone in minutes. +func TimezoneToUTCOffset(name string) (int, error) { + loc, err := timeutil.LoadLocation(name) + if err != nil { + return 0, err + } + _, offset := time.Now().In(loc).Zone() + return offset / 60, nil +} diff --git a/geolocation/geolocation_test.go b/geolocation/geolocation_test.go new file mode 100644 index 00000000000..5cda272de2a --- /dev/null +++ b/geolocation/geolocation_test.go @@ -0,0 +1,38 @@ +package geolocation + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTimezoneToUTCOffset(t *testing.T) { + tests := []struct { + timezone string + offset int + failed bool + }{ + {"Asia/Shanghai", 8 * 60, false}, + {"Asia/Tokyo", 9 * 60, false}, + {"UTC", 0, false}, + {"Unknown", 0, true}, + } + + for _, test := range tests { + offset, err := TimezoneToUTCOffset(test.timezone) + if test.failed { + assert.Error(t, err, "timezone %s should be invalid", test.timezone) + } else { + assert.NoError(t, err, "timezone %s should be valid", test.timezone) + assert.Equal(t, test.offset, offset, "timezone %s should have offset minutes %d", test.timezone, test.offset) + } + } +} + +func TestNilGeoLocation(t *testing.T) { + loc := NewNilGeoLocation() + geo, err := loc.Lookup(context.Background(), "") + assert.NoError(t, err, "nil geolocation should not return error") + assert.NotNil(t, geo, "nil geolocation should return empty geo info") +} diff --git a/geolocation/geolocationtest/geolocationtest.go b/geolocation/geolocationtest/geolocationtest.go new file mode 100644 index 00000000000..8de927a9d09 --- /dev/null +++ b/geolocation/geolocationtest/geolocationtest.go @@ -0,0 +1,44 @@ +package geolocationtest + +import ( + "context" + "errors" + "sync" + + "github.com/prebid/prebid-server/v2/geolocation" +) + +type MockGeoLocation struct { + mu sync.RWMutex + data map[string]*geolocation.GeoInfo +} + +func NewMockGeoLocation(data map[string]*geolocation.GeoInfo) *MockGeoLocation { + if data == nil { + data = make(map[string]*geolocation.GeoInfo) + } + return &MockGeoLocation{ + data: data, + } +} + +func (m *MockGeoLocation) Add(ip string, info *geolocation.GeoInfo) { + m.mu.Lock() + defer m.mu.Unlock() + m.data[ip] = info +} + +func (m *MockGeoLocation) Remove(ip string, info *geolocation.GeoInfo) { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.data, ip) +} + +func (m *MockGeoLocation) Lookup(ctx context.Context, ip string) (*geolocation.GeoInfo, error) { + m.mu.RLock() + defer m.mu.RUnlock() + if info, ok := m.data[ip]; ok { + return info, nil + } + return nil, errors.New("not found") +} diff --git a/geolocation/maxmind/maxmind.go b/geolocation/maxmind/maxmind.go new file mode 100644 index 00000000000..21344239f51 --- /dev/null +++ b/geolocation/maxmind/maxmind.go @@ -0,0 +1,97 @@ +package maxmind + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "net" + "os" + "sync/atomic" + + "github.com/prebid/prebid-server/v2/geolocation" + + geoip2 "github.com/oschwald/geoip2-golang" +) + +const Vendor = "maxmind" + +const DatabaseFileName = "GeoLite2-City.mmdb" + +// GeoLocation implementations geolocation.GeoLocation interface. +type GeoLocation struct { + reader atomic.Pointer[geoip2.Reader] +} + +func (g *GeoLocation) Lookup(_ context.Context, ipAddress string) (*geolocation.GeoInfo, error) { + ip := net.ParseIP(ipAddress) + if len(ip) == 0 { + return nil, geolocation.ErrLookupIPInvalid + } + + reader := g.reader.Load() + if reader == nil { + return nil, geolocation.ErrDatabaseUnavailable + } + + record, err := reader.City(ip) + if err != nil { + return nil, err + } + + info := &geolocation.GeoInfo{ + Vendor: Vendor, + Continent: record.Continent.Code, + Country: record.Country.IsoCode, + Zip: record.Postal.Code, + Lat: record.Location.Latitude, + Lon: record.Location.Longitude, + TimeZone: record.Location.TimeZone, + } + if len(record.Subdivisions) > 0 { + info.Region = record.Subdivisions[0].IsoCode + } + if len(record.City.Names) > 0 { + info.City = record.City.Names["en"] + } + return info, nil +} + +// SetDataPath loads data and updates the reader. +func (g *GeoLocation) SetDataPath(filepath string) error { + file, err := os.Open(filepath) + if err != nil { + return err + } + defer file.Close() + + gzipReader, err := gzip.NewReader(file) + if err != nil { + return err + } + defer gzipReader.Close() + + tarReader := tar.NewReader(gzipReader) + for { + header, err := tarReader.Next() + // io.EOF and other errors + if err != nil { + return errors.New("failed to read tar file: " + err.Error()) + } + + if header.Name == DatabaseFileName { + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, tarReader); err != nil { + return err + } + reader, err := geoip2.FromBytes(buf.Bytes()) + if err != nil { + return err + } + g.reader.Store(reader) + return nil + } + } +} diff --git a/geolocation/maxmind/maxmind_test.go b/geolocation/maxmind/maxmind_test.go new file mode 100644 index 00000000000..835ff30dcbf --- /dev/null +++ b/geolocation/maxmind/maxmind_test.go @@ -0,0 +1,147 @@ +package maxmind + +import ( + "context" + "testing" + + "github.com/prebid/prebid-server/v2/geolocation" + + "github.com/stretchr/testify/assert" +) + +// File is only for testing purposes, never used in the production environment. +// File is taken from the official MaxMind repository. +// https://github.com/maxmind/MaxMind-DB/blob/main/test-data/GeoLite2-City-Test.mmdb +const testDataPath = "./test-data/GeoLite2-City.tar.gz" + +const ( + testIP = "2.125.160.216" + testIPv6 = "2001:480::" +) + +func TestGeoLocationNoReader(t *testing.T) { + geo := &GeoLocation{} + _, err := geo.Lookup(context.Background(), testIP) + assert.Error(t, err, "should return error if data path is not set") +} + +func TestGeoLocationSetDataPath(t *testing.T) { + geo := &GeoLocation{} + tests := []struct { + name string + path string + failed bool + }{ + { + "File not exists", + "no_file", + true, + }, + { + "File is not a tar.gz archive", + "./test-data/nothing.mmdb", + true, + }, + { + "Archive does not contain GeoLite2-City.mmdb", + "./test-data/nothing.tar.gz", + true, + }, + { + "Archive contains GeoLite2-City.mmdb, but GeoLite2-City.mmdb has bad data", + "./test-data/GeoLite2-City-Bad-Data.tar.gz", + true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := geo.SetDataPath(test.path) + if test.failed { + assert.Error(t, err, "data path %s should return error", test.path) + } else { + assert.NoError(t, err, "data path %s should not return error", test.path) + } + }) + } +} + +func TestGeoLocationLookup(t *testing.T) { + geo := &GeoLocation{} + err := geo.SetDataPath(testDataPath) + assert.NoError(t, err, "geolocation should load data from %s", testDataPath) + + tests := []struct { + name string + ip string + expected *geolocation.GeoInfo + failed bool + }{ + { + "Lookup empty IP", + "", + nil, + true, + }, + { + "Lookup incorrect IP", + "bad ip", + nil, + true, + }, + { + "Lookup valid IPv4", + testIP, + &geolocation.GeoInfo{ + Vendor: Vendor, + Continent: "EU", + Country: "GB", + Region: "ENG", + RegionCode: 0, + City: "Boxford", + Zip: "OX1", + Lat: 51.75, + Lon: -1.25, + TimeZone: "Europe/London", + }, + false, + }, + { + "Lookup valid IPv6", + testIPv6, + &geolocation.GeoInfo{ + Vendor: Vendor, + Continent: "NA", + Country: "US", + Region: "CA", + RegionCode: 0, + City: "San Diego", + Zip: "92101", + Lat: 32.7203, + Lon: -117.1552, + TimeZone: "America/Los_Angeles", + }, + false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + geoInfo, err := geo.Lookup(context.Background(), test.ip) + if test.failed { + assert.Error(t, err, "geolocation lookup should return error. IP: %s", test.ip) + } else { + assert.NoError(t, err, "geolocation lookup should not return error. IP: %s", test.ip) + assert.Equal(t, test.expected, geoInfo, "geolocation should be equal. IP: %s", test.ip) + } + }) + } +} + +func TestGeoLocationReaderClosed(t *testing.T) { + geo := &GeoLocation{} + geo.SetDataPath(testDataPath) + geo.reader.Load().Close() + _, err := geo.Lookup(context.Background(), testIP) + assert.Error(t, err, "should return error if reader is closed") +} diff --git a/geolocation/maxmind/test-data/GeoLite2-City-Bad-Data.tar.gz b/geolocation/maxmind/test-data/GeoLite2-City-Bad-Data.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..72831ae2d04627c2b1df5dcb3c01b39fa2fc6638 GIT binary patch literal 136 zcmV;30C)c%iwFQh_{C-b19MN!_sJ|tHPUs?EUDDX%}q&SpgJ%xFfcPQQ2^2AW~N}; zzzD*J07F9sLlZMILt}Fj15;xK149#IBNGM%1L`{k6O6DTyVCTm$WpQ7{Td!6+C7bOiw0NE@^O2mk<4S}mXe literal 0 HcmV?d00001 diff --git a/geolocation/maxmind/test-data/GeoLite2-City.tar.gz b/geolocation/maxmind/test-data/GeoLite2-City.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..f229e56d949dddcfde4a1366a7b4c75fd4de425d GIT binary patch literal 11087 zcmZ8{cRZVa^lw`&RW({QYwuCiDy4+lyS5soW=rkbB1P?0wP$K?YHx}(HdT8@?M2gazGq=GpIi^e8#z^}26p85ss1rebBX#IQb-C5u)^EfbT) z)0sI`^~UYG%{nPZM?ZUqLr4W;Z;!@a3Uz+IaMcXm-AK5Giq=jI8RO2(afi50TL`tj z;o4WVU_nkl>vzu^T=dk`dQ8trvDP*=8q{JVI&Ldpk;b+*dF3GT8V(~XxCWD;prqfP zJg>u06LOVPh~Bc(#kS z52QObGkU~=qp>wK!lf!Yv))^fgXEW!lS$%#`C7J}2k+^B_3KEj)|v`4QJk%)7#=C6 zZ0(@gcPgwhg@^i7z7%Zjik)p75*Q?fWhSR{aIQR2?B>(f2g#d5E%!=4U;C|k!bpAx zF^8GKjyuM9dUV}upv6Y+){euehB#^cVMj+2MXR@OvlJH#3`KrhR*ab_#yLHNr?0px zjRh;A7?)$`T%H~gIaw@9O7J@TP30!%n25^~cRFt4_R$czWjBDB&Vuw6b-#PFH$$vx zeF}jxq6MaKse9E{{6;u(xUGi8R1iduKzi>4cX7#>VN(wa;u~?a;;@YyOcx0-oq}o4 zCiL3I4302}IsJk?D--#ap_^t%m#Y8VK9D|2K@t4tacjjhke%sU50Tckg7S(sM>tyY zAYy&~_xM4EiLm?17i2;OPmA34KBH$CsVOxcJURpYUJ5<0`LAJKu36j9)5qX#z&qWM zauwkRH5oOuk?!+_1e}XrN}le?X0LdS=5zuO3#BT7ZE!l2?zfC_6ycixuOio9se6JD z2XV%~SvW94&|+w&nz59~ExU33#m0uf!}TXa!vn*74i%Y#5GR+Z@`jZlQrZE;3C(j3 z2MmI)ediZwV44K&M|_z#iu=3S3A((h!r;(?RvfXN&#_M>1BR542wWQJOFH+ebFmF= z`cxVfIGZX5Oo8m0=~6d|)5E1$6_1HO6fF+pWa1A*v%w0;Qe;g@T<^h>$P8s4er=*% z(IpFJ!k06}Lw*crCYye3O7LFRTE>(VNfVw!*2CoVg_gLu4nqIiOMQteyX_qFe8 zKYa8h6<_;~_5*EvZ4yEH+&h-|8aUQy{an3!+Qfq7mXz|uuutLh%m-hZ$a5(zNnmDV z9%No}GuC;Cg5?a@WB871&F$`-1apf|45~4D_Cbq(&B<06_$D4@kd8eZyYndMzVK9f zrH_F8*JSXeb#&&v10P1%H5m~TWuZfZtbca3!gd1gYc@EmTA)yi|6tciGU=UFc`(cu z&hV?BU|f_j$UC*Vpy^!sa!v@IT%^WJ?I&VpOj!NEQq;=f52pHaQZIA%7o))$32+5& zyjaB{i{APZp6b)_{cOvjisOecj&M8X7jmBqpT40)(`hCd5d!ZUMqd+XVrf^z$T~F) zeW+HvqOTdIvA2-WZ63&1Z>=aTGD~bU&tbwY3+R~Yu}`Gm5OoQjQ;Y2vNazZJc!&v0 z1GU6Q+kJrm;WX?P?%~ns{}+H1Lt#9O@GsDR!D=}~LYI@^EGfE+Z2I~CoL8z%*D(pD z-p0}_Lj^_5Y>W`(p+#D@LRRuEpIL#@)eRy)+vxwxR1D4|EEY<>Nyb*CYgq&2>YC8D zdzKG`fiblK8#{Ch7hSUL@N%f{jGGq(uJ}R&1LZv~U=)zn7x3>3J_=*@|8P2@h-R}Vuk;21S7+_iX z+u+-{=pumUfBN{}y9Sy6EPe|9Ka+^2CtHSA6RjwOFOmt!?gh>eRg+qR0CRL- z|H}}6=z6zX^rsb&_5a-e&pSeZ6a4^4{sMtdGu>=vIPgLU+^&*q`)6`{87iB&X03(` z`kO(92t@P4Ovn%baWtb0hAar(`zVzLWDD1FIW}ir3Pp3nB=v;5qxm1Zr;ce5W3yIvvk)PlI8V8&kx0)8sid*dgd)>yNW2Dc>aOe;4cpOr@10O(H^?SAs!B7IZ z{wx40bh+#f7TMpCkcdN$3Xx!u^ZulFaA-Y*ihb`OO%Wze%PJGy8%3>z?Ii*?Dq?%_ zmJ$X=ya`wPaNcQ$Q$W?*XkBb?88tL8p$}L2b^ITIx$Q@Wj6-w*8l6#QcsTEhL$+hk zyOe{!i_yUlfBw72pjb?tB-t_}0nVHJF1oi1B+ZyhlY8$D22ezRL$@27p3Q#d%#AL( z=v^g42?jte*Y1RekU0Y7ud;9jDNxq^>$G%_vz%VBEdxWSR_d>w)AMOXkJop2x@65@IT z8*IiG2%*EY!p%rwAgl(w3lv*${_*SPf2Cxf;I2ACkSIVBBmwEX%7*shJ0<||4=-&) z(7lj8Kp|wgwWzF*ah3nzE*b8w9RLsM0pOulrpT%=B){IaA|A?TnGv*`A-=+h69Wja z?oer9aXN1Vcb;++T`=P6t8OFXI(vaST%haMV!8{2hY=-k8*)kv@Zh=u9t3Jf;R^(K zw2)|j`%!DjM4r#!Do73kjX?>Iv`$YvRepW=eG4JQjUzXFE{h>g5emd;`Tw0~zm1I; z1dH`x2#-d)H8Yyh58U{GDpz9iWhnKY0|7RIdC7w<9J1oe@e36HFU#dN9=RO2odZi4 zMz{jyvSvwu$#aB5SfYR{>i)ab{NHdkO!E{#@xB=(2+au>47}D7obn>ZkdwC&X_4Vz zo2g9L2$Qs5*rL>xdVrn~z4lN>n+zwM}HS`zaq%YvK3Z3NKeqKzCCQis?ve2w9SZ=G+jv zZG3;;OY2BW1g-fP2MbmQB!Gb+p9;W!u)TvJKLHc}0zEn|2}5({0;nu24!5uoP1E!D zSD@%Smp&q1rMO%?V|RfaGRH{O00A4vU~U8IGi~(4133vP5R()|T86 z>i~phTGhsb`+Ja<^2cSxT~_MliXUUDx>m`S5pSm1%k%I@pUb-Qz7PUhPu$^g*N#2= z{No#BRRSR_(z4e1Pah8E-qA{iTO;A%!X?zfiwn^-Kp&8!>Tj@|u{XoJx@lob>Mb11 z32}T6clUL5&OtCRBQn7J7tb(ek;j!t`g0%R6{vRz;8MK*E}Byl4ZnpYT!Zkd-$QD( z4FI3}_#;snEN92gF3mb*CG4gz88?9j80U%y$2VMr^93c$L#LGx9#1eAQt#V8{nS7a!7h z`^LeN6|fcogFq360MLt0a%TPlSrO|H1h^S*JRO3QA7Udwm!CB-2|*E{T`^x^ND$%0A6fI15rwS4b3$zY`*-`Q zKyJKB8!myhP0b_-a%@|*tp*&0sjNVsz5jj;^5F%lUPb_ybfavh-{>~|$k3dem%cF= z0NyR~kQS-6A*_N)czGkV52t9Mt$L)jat&2`3>^NxIrPSD`YJ?Vv@}nQfj=W!OZ{v} zErXj9;Z)ca=ALFc}fugn)l{HAs}>$d3s>U`>|0VEOtwwhD>1(w&Sz=MRCgnemuR&0M*nmm@1@%U|=!{st`mxO&$ z-RY*sj5jh2g_|Ew8_0V6Xt;}8v+4+Zgj@5y5#sTVk);dN+-1iFAcg?^{>dtPkO7Sj zzaU7>xED|fwF3++Y&@-#Y@7!ZPhdBxIU*p(RDj;Kr#FFDT?flaDhRaXD(xk8k6HW`LgTUB;u;s^a z47wnPZgArg>~0vHfJK>IuQfZ;I={@;#3X?1fH8qiKHj;49wWe-GRQ->8y5mM+Ht=C z9UX%JPBqSl-1HmJphIvY2euz12(Y32uRl`}qBjeD3m=^G+ZL`stvS$k52_xh|L(*N z-Ng+BEN!M81#jFk$2-FwCJX^vL4JW4J}3je|FjLYHUVk7X(fedT3a40I5O?vsU~@VEhR0E1<*cD88EQX5xRVG3NRFgZrJ8n><32@u5|jl`Fr#r`Xde`jvN zf)lRwC)UGy2?$es;z=t(r}U~P;T8t~n*JLwl@5|1$i@@IbA7B>QrbcG!e-lsEl}hR z)-t>>CluDl8V;!+jXpcBBmh{tKwg|ZdRNf#v8x5BX`RK;bv`EPQklqDiGp*#f^WM;B1a9%L_Kp|!S7&_XqY}(v)Xh;K zhJG(HD5O*nY8=D4vH1fOkFuT@; z1GpDL0P$`bAvR4Ko3;#X?wZz0!Z})U9D`11%+0^y(mF7sdSdfkK+B{?)ln#cdH?-| z^ksxHn|R?0!s$k_|5y>>9wZ%j7&QB8)Abyh4_%WHFp<2b3-V4&3mYenm7%kt@*mll zCa@wxq)lyomGg|!rTV07?fK+M@#>bAd)6CmC-1H{mwe_oj8mXhlHV=IyFJ93PF?C{ z=DGZCwW3F+PQkiHWN}|j93AeHWSVl?Q>>3f8BsK&X2x438oWF*w{~V0C&$Jns|$aK zz7T5sUa7QF&|lwwZAZze#jeGPqO22*8R8Y#10zG0g0G@>dXR6P)%=WNNO*~zU9k`` zEn+lHREw)ScCdZF9q0MtiP(6OQCru? zbXsqdvX_78NJLjAwh^ySZl9H5S{hvXxPwO&Q$pp4|GjzdAWmDG0kMx6&XD#~w}E|p zsAwfJ6)IuKx|koYjh9_TqPQEBBcaq-FXw?I^MG&JIgav`Ptc|tP*=;Hm=!!Jecf&~ z`D8ozhvTGeX6;^@8NEZRgy&}ux*z{gubQeht0_cY@D@6-zx!FCu-;#hf1CC?Eo=da zVlYzkP|ZU&6L(z&Y5UqIs)=fN%$wIXY*Hjry~S&-U{<)71Gkl&u$WXQDrTjX`K3SQ!6u|mq@1HvPb-#4EWknVI*rA(b85d-YnrWL+^;Hjtcx7CZq6X8Z9(U4}OQLeD{q)u1m-i}exUi-Dk_SxRRx^Z3$-2J`7SlwE>F#U5b zk1N(VPsW$&g3Xew&P0rKDj8)p2j~pveDCjTx z(4LjAQs~S(KG5*OD9czli&#}KP#}vNdR#%6pE@4aDSQ}{X>J}rmW-5v+dBG=XsM73 zm(_6`+ZwDJ$qkr_y`iX0iYn$7_+c|i7TZnwMojf_+@FHtV?6Ep$9*Ic67{SzleNCv z%_Cu_ZN%4U^~KL;(-yK#HO4*8CsZZ;uC3+XnX3oWOY|VL;sxEeVx!?X1Y@;15xbnE z8R*+*_?Q@9RUg*M_FmcU5}gb5dKXsP&9uSu%ESwiIGt&2V53ksVzoZ)ut~ zezdh%hsEeh`o%K*s!fcymOkllVAH>6Ec#pjOamoJJfQfX%;k-)5rtQs+>c<`Ilhuq z!Sx5jtCJF?S9^l1ETy$_lZD%J*$Y~O&(|qODrYB7mLEk|=?xpsqTdd8w)4x;9B*+~ zJsD{INxzNbpVZ0}*`V;IxOejSXejFCFIw zGNo!$`g5mp2i6RNs_z_sy!4V6{4#XCP}U{N|f2Z zpBPfI)%g+W#b1q*O2oW(P%d;W)6eU$pQK{`asAEHnY1jh-uciTq#T^X8)f@VAZL~# z%|A%djmwA0=Rp(C=vWgi;aQSyp8~THn>f_AC0ezbgI6hOx4&q&#;TFcbIWZ%nhxyk zyiiTMl5DTa&|&x6w@Rtj(uvPrM*CYwu%ZK}hh^tGO3nnok1AxBnp?RKAc-b9PvfW6 z1fLhs?pWJM+|kucZYw`N1^u+|vnk9hySHh)bEx{RYlzNa?C1Nb$oF|d9H?K+FEp|w zcIVa1owHuoq6RWo+AD*$7S}x-bF}=&S&2$Y zI#zvEdvR9XDod!i!a&2ucO=u+>u=w$clCSft9}DN?rWyOucji^v)?;Xr+*Rhpw_Ot zfD^wh;@2K%+l}~_H0gCXw3g8jUQOeehG93fOMu5Rrc}@Fm>kx|O~((|yCvd16{>(2 z-g-7sd}*4%O<+>5@2gqDo4olw(({~)f!o2n)IeZ6B6-isT0?EpQiH*zw>h_Nb1caY z>bv8=`LT37>zoei`L3W>v+RpshHYbMF^_H~vxqjc>lgn*i82O*nbrW4!*Kx}@q)@? zmr{}Du*~j~!~@}VhPc{6pWUr_uTJxFOklgko8L1I&ix-ULXCJ{LEfkCDe#|r%CEgq zQYDj4F_fr4HV=1Q7^sM^{1JLq@}T!4O|G79R?*rp<)kWIp1p5u`J9kM)=UnSrGDnr z`;VhjR3o4(@53e0Iq2};lwhhlR^9Y8f9?x6dqSs)sn5#ue=qF3-l&bHx{V{>FV0uS z^`w?&y0oRitpkRH3uUXxMGALlVdayaM*$UPCsa@3OTNb%Z*x^Enf)a4GuYpk@{Rly zW&f?jN&7xCK{eHffwnCjEkh2Ug?&9}Yk_2=gzRJrW6_n6e_1^kt-1Kfa^jmLPo|4Q zt?(=R=Jd+O?vrPKDr_5+KotL`yld%Q1tv*wH8!L;d~q=h-*QEnGkkomxo7|H#!yrc zLx|~#Jv(=P(_Q2n!F8LH9v;|(V!3p)%hLWmu`g3J;MtH(DB_QwHR*P+YJOaAeF^nRn_h_$=v>l(=WyGJhfb~vTSh8*^_laSMKRGG~?rk^*(~gezS!` zk)1bM^m9+7DaK?MKI-+~qF+pzS=!p7b167>EBPWov14<76%v`2JqB+wuN~01Ww^ID z+W`|k*#0OHwWHWxWmWr=Qt z*>HNEiT-m{ZXVe>D=LX8b0rNe4$|Ery(^tN`Y5F1Cc`YE_H%oY9aJqtZe-DE^f~Kz zrt&L$-A~F{yO63fSTtUCgTl6U^+d7iqPG4_yoHFBJN$Osxv*g)?=sUZp1*;fKQ)?T zu7ayya0Zz+{orBvBv(!!t?0p1Xz3`b{I&9awb$-K>tklmW3GF%erMA1Jbwb5QX2E3 zhDT#FCEt2X>=+(o&iOyff|&w{i>X_IIdiA8jK0+zFi zlM%&Vr|ESn7v-Cw#|(H*xo=D<`cvYs{)A~-YAM?#mwEg>%EcB=XzCZv8q2YIzwr@u z5^FnA>4mwa%v!H?QA_4z6N<5I$MbHrrim9N~~s9vZE9qn4Ll241o@^QSA zAraos@7)$$Ya5B5!+i*wluCPk9|(F}jOZD-EeG4$rI)78wPB99&!jiXkXg@2->5ONpljbai-8KmJ3)WaAB#$`2o-SvwOyJ z11ED^pClNHOOC$Y_L+FSem!}n5kZfFPolZbO>=)t9t_e5_#_KuL~)f#f41eW*s;1_ z`a;)LE=mEtkbo44#x@X6c8?M`8^pePx!t zm6%?~>)C!OEd34o^=RswKe)Q^1(|lBVWFM39%yX#wGwOj+)FxCvHIR0K7D(t3Dmes zws3dNrgHng+k>yN;nuyjaXj_{O}J!AFsO6pJ$=|h*@q4P^$z+Kc!5ZPg77ZuOqaGo(kJbO zY9;b)cc?Gg??!95k}lvf*nvH6;6t#Orq$Q!3u3!-#G(|opLCJNMeTR|32t6?2( zlPKIPZ3Ux^QWOgQF5H5AWard37WK}ZuYua1JCnw~j@90wUc54P;$5Odr&?-*y0gsv6B6K`N+J)?9iIFcx$(&O=M(HhOlb+Vp&Gp3*ZG|_z z)72w$tS7&kxHTEiVNlXg8{=COesCGIWR~|uy`t)L$#{98VcgDeztGjEkgGFJugY|e z;O}6-=JxdOJj&MV0lsdyr&C*+NTr{_9z$8%^|2Jsu}js*;i~cXdr=St8gQ7r9ozb5 zc)^pQa|OME#UiLS_tjte#oiabt_kPv2?D3?`u}`nGg<;JR2z4pFB3`1q+f({>_@)e z-XB@;1}D&u)t;xxWmhku*YirMDVqe7#}~r+{J~G0wF4=ePZit8b<4GQ7=M@|@TC?~ z+L*5>MwzE1@^eG&1fkAXUHT%tdX}x_C2@nV(x1VnI*Zy>ah?JW2hPk(&omub34F5y&KE38clI5Y zdg7!}>x0QVT+$7G)3w4-Rg@@M%xq4x#9_PJSHt=-O~d+^_jOf&n(cOw@(x5OS^do8 z#mPhtNA&9bC~rn|=#C#3AE56Q>SY?OykL{6(mdeQX(%#8HaJRY9kyFg9PW*u#FV76 z-%EHaGR|zHU>MN5`br?cD{V(Kw{InLqxqopS$EFPv3{#^&jH2$>&6eTDo?Lc|5k+i zmFbqf&=q+5?Bz8kp~&xCS~gI8Y0q0C>(E`Q1hLx_%c2b4US5W4z8aZtfi|>dfi2M4&RoT}y%L&;di~5qvSnFqa z_Cm(un?3c4awK)j%`%NYty6efdQC(aH+#CsikCLvL36ShCt8bo>$6Eqvv#`OdEiGO zyW8NPGF=CObsjh4#H1H9JOcFS2E$RW($gB)Vjp(|!kNJ8>qHr8g47>FM{N~!z(AL! zM8U$U54bk*i5kU+0>}Jif(vWOlJh+x2d)^VAr>HVu=_$JW zveLKt?w=I>S!C#PUuL+J;nPng{3b2x1||m#TeX5ZN1rF^ay^Z*y5=PF{Q~4OM7S}o z63X0}JDd{jUPdBHrHh|wlYH%(K=iv6Q#suM(<6Q;=O5bN>rwoW{405h@A~}d_{L2$ zVt)pfU$@HI3_;UZ-X?k+CY4$XE2we?F;nia*0T|SHQCQD+d|h|+gTWuD&Ho}JZdIg zs?^Qs1s*k0Ss5P)v!w$5bYdT5bbzPhh95~lqafSv{niP+FUjw)cm^AX1&8_;zHTdg zpa+hE0?GsMri%e6Dw zMm;@VC8m>JlTEkku|X%Ur$!k3I=rq=mKIY#r>AGs_}8X*FQ40G^C;OSs=_sQs|9ls1H z@-kzLn`A?RZSNI}%{=|UlOf>JxXR)b zNfR!v{LStX)ub(3BE8)5W}UHdE8NA3fWE=6Y}Mg@jCn zw31!$;OW3!hoP@FmT-%<3My*}HZQe}eLR244yXEdC5#siqPU0zT#1Oz47|8iBczCT zay9dccR5W1N=0^*4SU@K_zMnP;sREmtKlT>mKiz5GOBkxG)m(6;pwzvhCQA2Aocl} z?uk-3a8fh*wK$v3SQH4p{5CxrhAqbroh?3Vvr>C%D@zogI`Xi@6SDe)zhLBXkG+es zdXP#1^}?QX@GBgq<-}o<7dhd?b})G~jWQP5OkXQp>kuBi&qd&2 zT&P2Pc9ki|P0*(0x5l~yJ6Iz;)1`MC-H(i@y;8qe_|EBTARvWXK0a~Ky4pjtoUypr zM+Y@s)KDGbauo@yisI*TAzqt z@_AZIdx#8N9emYHHzypjj&_J1Ro!e#Vd}ZHd>Snc%xW;!? z_g%QceyG;7Mzo_%-;p21 z7PS$!Os25B%TB)@==jh%TS7Ei#yfsl&wqC6yCvpcGOcjCEnVQjNX*PI@LNmf?`M>6 zZ-FOE1=h=AxAe4~qe`Sy?h&=r6boI5)9gw+w)9TguBH86LOG?FT^=k~9qKQ(*k@z& zgBv=EN;JV9(~SXdZ3Vc9LTM);*}mA3W|6%35Ra4YcuGlwX-foo?8M@x{Di=!#6^f< zaL(!EFuVI|*G8yBk*Fzx?V@cY1N){Bdo|naIZ>k9sm^tBeVENotXY@6&~~Yxd%3@n znW3A)=P_Gq04g8!nm0*1*}I5U5WPb&o7}QHOnKJjMmfV*A(BvmRCX{@X#Dr)r2El$ zPzBhmgGRB! RU=$2;000Q4S= s.retryCount { + return fmt.Errorf("sync file max retries exceeded (%d)", s.retryCount) + } + + select { + case <-time.After(s.retryInterval): + continue + case <-s.done: + return errors.New("sync file stopped") + } + } + + return nil +} + +func (s *RemoteFileSyncer) sync() error { + err := downloadFileFromURL(s.client, s.downloadURL, s.tmpFilePath, s.timeout) + if err != nil { + return err + } + + err = os.Rename(s.tmpFilePath, s.saveFilePath) + if err != nil { + _ = os.Remove(s.tmpFilePath) + return err + } + return nil +} + +func (s *RemoteFileSyncer) processSavedFile() error { + if err := s.processor.SetDataPath(s.saveFilePath); err != nil { + _ = os.Remove(s.saveFilePath) + return err + } + return nil +} + +func (s *RemoteFileSyncer) updateIfNeeded() error { + fileinfo, err := os.Stat(s.saveFilePath) + if errors.Is(err, os.ErrNotExist) { + return s.run() + } + + remoteSize, err := remoteFileSize(s.client, s.downloadURL, s.timeout) + if err != nil { + return err + } + if remoteSize != fileinfo.Size() { + return s.run() + } + return nil +} + +func createAndCheckWritePermissionsFor(datapath string) error { + dir := filepath.Dir(datapath) + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return err + } + + temp, err := os.CreateTemp(dir, "permission_test_") + if err != nil { + return fmt.Errorf("no write permission in directory: %v", err) + } + defer os.Remove(temp.Name()) + defer temp.Close() + + r, err := os.Open(temp.Name()) + if err != nil { + return fmt.Errorf("no read permission in directory: %v", err) + } + defer r.Close() + + return nil +} + +// downloadFileFromURL downloads a file to the datapath. overwrite datapath if it exists. +func downloadFileFromURL(client HTTPClient, url string, datapath string, timeout time.Duration) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + resp, err := client.Do(req) + if err != nil { + return err + } + defer func() { + if resp.Body != nil { + _, _ = io.ReadAll(resp.Body) + resp.Body.Close() + } + }() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("bad status code: %d", resp.StatusCode) + } + + output, err := os.Create(datapath) + if err != nil { + return err + } + defer output.Close() + + if _, err := io.Copy(output, resp.Body); err != nil { + return err + } + return nil +} + +func remoteFileSize(client HTTPClient, url string, timeout time.Duration) (int64, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) + if err != nil { + return 0, err + } + resp, err := client.Do(req) + if err != nil { + return 0, err + } + return resp.ContentLength, nil +} diff --git a/geolocation/remotefilesyncer/remotefilesyncer_test.go b/geolocation/remotefilesyncer/remotefilesyncer_test.go new file mode 100644 index 00000000000..bbffb04e465 --- /dev/null +++ b/geolocation/remotefilesyncer/remotefilesyncer_test.go @@ -0,0 +1,915 @@ +package remotefilesyncer + +import ( + "bytes" + "io" + "net/http" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +type testRemoteFileProcessor struct { + setDataPath func(datapath string) error +} + +func (p *testRemoteFileProcessor) SetDataPath(datapath string) error { + return p.setDataPath(datapath) +} + +func makeTestRemoteFileProcessor(setDataPath func(datapath string) error) RemoteFileProcessor { + return &testRemoteFileProcessor{setDataPath: setDataPath} +} + +func makeTestRemoteFileProcessorOK() RemoteFileProcessor { + return makeTestRemoteFileProcessor(func(datapath string) error { + return nil + }) +} + +type testHTTPClient struct { + do func(req *http.Request) (*http.Response, error) +} + +func (c *testHTTPClient) Do(req *http.Request) (*http.Response, error) { + return c.do(req) +} + +func makeTestHTTPClient(do func(req *http.Request) (*http.Response, error)) HTTPClient { + return &testHTTPClient{do: do} +} + +func makeTestHTTPClient200() HTTPClient { + return makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader([]byte("imdata"))), + }, nil + }) +} + +func createFile(datapath string, content string) { + file, _ := os.Create(datapath) + file.Write([]byte(content)) + file.Close() +} + +func createTempFile(dir string, content string) (string, error) { + file, err := os.CreateTemp(dir, "") + if err != nil { + return "", err + } + defer file.Close() + file.Write([]byte(content)) + return file.Name(), nil +} + +func assertFileContent(t *testing.T, datapath string, content string) { + file, err := os.Open(datapath) + assert.NoError(t, err, "File should exist") + defer file.Close() + buf := new(bytes.Buffer) + io.Copy(buf, file) + assert.Equal(t, content, buf.String(), "File content should be correct") +} + +func TestOptionsValidate(t *testing.T) { + dir := t.TempDir() + + tests := []struct { + processor RemoteFileProcessor + client HTTPClient + downloadURL string + saveFilePath string + tmpFilePath string + retryCount int + retryInterval time.Duration + timeout time.Duration + updateInterval time.Duration + hasError bool + }{ + { + makeTestRemoteFileProcessorOK(), + makeTestHTTPClient200(), + "http://example.com", + filepath.Join(dir, "foo"), + filepath.Join(dir, "tmp"), + 0, + 10 * time.Second, + 10 * time.Second, + 0, + false, + }, + { + nil, + makeTestHTTPClient200(), + "http://example.com", + filepath.Join(dir, "foo"), + filepath.Join(dir, "tmp"), + 0, + 10 * time.Second, + 10 * time.Second, + 0, + true, + }, + { + makeTestRemoteFileProcessorOK(), + nil, + "http://example.com", + filepath.Join(dir, "foo"), + filepath.Join(dir, "tmp"), + 0, + 10 * time.Second, + 10 * time.Second, + 0, + true, + }, + { + makeTestRemoteFileProcessorOK(), + makeTestHTTPClient200(), + "", + filepath.Join(dir, "foo"), + filepath.Join(dir, "tmp"), + 0, + 10 * time.Second, + 10 * time.Second, + 0, + true, + }, + { + makeTestRemoteFileProcessorOK(), + makeTestHTTPClient200(), + "http://example.com", + "", + filepath.Join(dir, "tmp"), + 0, + 10 * time.Second, + 10 * time.Second, + 0, + true, + }, + { + makeTestRemoteFileProcessorOK(), + makeTestHTTPClient200(), + "http://example.com", + filepath.Join(dir, "foo"), + "", + 0, + 10 * time.Second, + 10 * time.Second, + 0, + true, + }, + { + makeTestRemoteFileProcessorOK(), + makeTestHTTPClient200(), + "http://example.com", + filepath.Join(dir, "foo"), + filepath.Join(dir, "tmp"), + -1, + 10 * time.Second, + 10 * time.Second, + 0, + true, + }, + { + makeTestRemoteFileProcessorOK(), + makeTestHTTPClient200(), + "http://example.com", + filepath.Join(dir, "foo"), + filepath.Join(dir, "tmp"), + 0, + -10 * time.Second, + 10 * time.Second, + 0, + true, + }, + { + makeTestRemoteFileProcessorOK(), + makeTestHTTPClient200(), + "http://example.com", + filepath.Join(dir, "foo"), + filepath.Join(dir, "tmp"), + 0, + 10 * time.Second, + -10 * time.Second, + 0, + true, + }, + { + makeTestRemoteFileProcessorOK(), + makeTestHTTPClient200(), + "http://example.com", + filepath.Join(dir, "foo"), + filepath.Join(dir, "tmp"), + 0, + 10 * time.Second, + 10 * time.Second, + -1, + true, + }, + } + + for _, test := range tests { + t.Run("OptionsValidate", func(t *testing.T) { + opts := Options{ + Processor: test.processor, + Client: test.client, + DownloadURL: test.downloadURL, + SaveFilePath: test.saveFilePath, + TmpFilePath: test.tmpFilePath, + RetryCount: test.retryCount, + RetryInterval: test.retryInterval, + Timeout: test.timeout, + UpdateInterval: test.updateInterval, + } + err := opts.Validate() + if test.hasError { + assert.Error(t, err, "Options.Validate() should return error") + } else { + assert.NoError(t, err, "Options.Validate() should not return error. Error: %v", err) + } + }) + } +} + +func TestNewRemoteFileSyncer(t *testing.T) { + dir := t.TempDir() + readdir := filepath.Join(dir, "readonly") + os.MkdirAll(readdir, 0555) + + tests := []struct { + name string + processor RemoteFileProcessor + client HTTPClient + downloadURL string + saveFilePath string + tmpFilePath string + retryCount int + retryInterval time.Duration + timeout time.Duration + updateInterval time.Duration + hasError bool + }{ + { + "New syncer, successful", + makeTestRemoteFileProcessorOK(), + makeTestHTTPClient200(), + "http://example.com", + filepath.Join(dir, "foo"), + filepath.Join(dir, "tmp"), + 0, + 10 * time.Second, + 10 * time.Second, + 0, + false, + }, + { + "New syncer, invalid options", + makeTestRemoteFileProcessorOK(), + makeTestHTTPClient200(), + "http://example.com", + filepath.Join(dir, "foo"), + filepath.Join(dir, "tmp"), + 0, + 10 * time.Second, + 10 * time.Second, + -1, + true, + }, + { + "New syncer, read-only save file path", + makeTestRemoteFileProcessorOK(), + makeTestHTTPClient200(), + "http://example.com", + filepath.Join(readdir, "foo"), + filepath.Join(dir, "tmp"), + 0, + 10 * time.Second, + 10 * time.Second, + 0, + true, + }, + { + "New syncer, read-only tmp file path", + makeTestRemoteFileProcessorOK(), + makeTestHTTPClient200(), + "http://example.com", + filepath.Join(dir, "foo"), + filepath.Join(readdir, "tmp"), + 0, + 10 * time.Second, + 10 * time.Second, + 0, + true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + opts := Options{ + Processor: test.processor, + Client: test.client, + DownloadURL: test.downloadURL, + SaveFilePath: test.saveFilePath, + TmpFilePath: test.tmpFilePath, + RetryCount: test.retryCount, + RetryInterval: test.retryInterval, + Timeout: test.timeout, + UpdateInterval: test.updateInterval, + } + syncer, err := NewRemoteFileSyncer(opts) + if test.hasError { + assert.Error(t, err, "NewRemoteFileSyncer should return error") + } else { + assert.NoError(t, err, "NewRemoteFileSyncer should not return error. Error: %v", err) + assert.NotNil(t, syncer.ttask) + } + }) + } +} + +func TestRemoteFileSyncerStart(t *testing.T) { + const filecontent = "imdata" + var ( + processorCalled int64 = 0 + clientHeadCalled int64 = 0 + clientGetCalled int64 = 0 + ) + syncer, _ := NewRemoteFileSyncer(Options{ + Processor: makeTestRemoteFileProcessor(func(datapath string) error { + atomic.AddInt64(&processorCalled, 1) + return nil + }), + Client: makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + switch req.Method { + case http.MethodHead: + atomic.AddInt64(&clientHeadCalled, 1) + return &http.Response{ + StatusCode: 200, + ContentLength: int64(len(filecontent)), + }, nil + case http.MethodGet: + atomic.AddInt64(&clientGetCalled, 1) + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader([]byte(filecontent))), + }, nil + } + return nil, assert.AnError + }), + DownloadURL: "http://example.com", + SaveFilePath: filepath.Join(t.TempDir(), "foo"), + TmpFilePath: filepath.Join(t.TempDir(), "tmp"), + RetryCount: 0, + RetryInterval: 10 * time.Second, + Timeout: 10 * time.Second, + UpdateInterval: 10 * time.Millisecond, + }) + err := syncer.Start() + defer syncer.Stop() + + // wait for updating to be checked + <-time.After(20 * time.Millisecond) + + assert.NoError(t, err) + assert.Equal(t, int64(1), atomic.LoadInt64(&processorCalled), "Processor should be called once") + assert.Equal(t, int64(1), atomic.LoadInt64(&clientGetCalled), "HTTPClient should be called once for GET") + assert.True(t, atomic.LoadInt64(&clientHeadCalled) >= 1, "HTTPClient should be called at least once for HEAD") + assertFileContent(t, syncer.saveFilePath, filecontent) +} + +func TestRemoteFileSyncerStartIsSyncing(t *testing.T) { + syncer, _ := NewRemoteFileSyncer(Options{ + Processor: makeTestRemoteFileProcessorOK(), + Client: makeTestHTTPClient200(), + DownloadURL: "http://example.com", + SaveFilePath: filepath.Join(t.TempDir(), "foo"), + TmpFilePath: filepath.Join(t.TempDir(), "tmp"), + RetryCount: 0, + RetryInterval: 10 * time.Second, + Timeout: 10 * time.Second, + UpdateInterval: 0, + }) + syncer.syncing.Store(true) // Set syncing to true to know run called + err := syncer.Start() + defer syncer.Stop() + + assert.Equal(t, ErrSyncInProgress, err) + assert.NoFileExists(t, syncer.saveFilePath) +} + +func TestRemoteFileSyncerStartFileExists(t *testing.T) { + dir := t.TempDir() + datapath, _ := createTempFile(dir, "imdata") + + var ( + processorCalled int64 = 0 + clientCalled int64 = 0 + ) + syncer, _ := NewRemoteFileSyncer(Options{ + Processor: makeTestRemoteFileProcessor(func(datapath string) error { + atomic.AddInt64(&processorCalled, 1) + return nil + }), + Client: makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + atomic.AddInt64(&clientCalled, 1) + return nil, nil + }), + DownloadURL: "http://example.com", + SaveFilePath: datapath, + TmpFilePath: filepath.Join(t.TempDir(), "tmp"), + RetryCount: 0, + RetryInterval: 10 * time.Second, + Timeout: 10 * time.Second, + UpdateInterval: 0, + }) + err := syncer.Start() + defer syncer.Stop() + + assert.NoError(t, err) + assert.Equal(t, int64(1), atomic.LoadInt64(&processorCalled), "Processor should be called once") + assert.Equal(t, int64(0), atomic.LoadInt64(&clientCalled), "HTTPClient should not be called") +} + +func TestRemoteFileSyncerStartFileExistsInvalid(t *testing.T) { + dir := t.TempDir() + datapath, _ := createTempFile(dir, "imdata") + + var ( + processorCalled int64 = 0 + clientCalled int64 = 0 + ) + syncer, _ := NewRemoteFileSyncer(Options{ + Processor: makeTestRemoteFileProcessor(func(datapath string) error { + atomic.AddInt64(&processorCalled, 1) + return assert.AnError + }), + Client: makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + atomic.AddInt64(&clientCalled, 1) + return nil, nil + }), + DownloadURL: "http://example.com", + SaveFilePath: datapath, + TmpFilePath: filepath.Join(t.TempDir(), "tmp"), + RetryCount: 0, + RetryInterval: 10 * time.Second, + Timeout: 10 * time.Second, + UpdateInterval: 0, + }) + err := syncer.Start() + defer syncer.Stop() + + assert.Error(t, err) + assert.Equal(t, int64(1), atomic.LoadInt64(&processorCalled), "Processor should be called once") + assert.Equal(t, int64(0), atomic.LoadInt64(&clientCalled), "HTTPClient should not be called") +} + +func TestRemoteFileSyncerStop(t *testing.T) { + syncer, _ := NewRemoteFileSyncer(Options{ + Processor: makeTestRemoteFileProcessorOK(), + Client: makeTestHTTPClient200(), + DownloadURL: "http://example.com", + SaveFilePath: filepath.Join(t.TempDir(), "foo"), + TmpFilePath: filepath.Join(t.TempDir(), "tmp"), + RetryCount: 0, + RetryInterval: 10 * time.Second, + Timeout: 10 * time.Second, + UpdateInterval: 0, + }) + syncer.Stop() + + _, ok := <-syncer.done + assert.False(t, ok, "done channel should be closed") + _, ok = <-syncer.ttask.Done() + assert.False(t, ok, "TickerTask should be closed") +} + +func TestRemoteFileSyncerRunRetryWhenSyncErr(t *testing.T) { + dir := t.TempDir() + + var ( + clientCalled int64 = 0 + ) + syncer, _ := NewRemoteFileSyncer(Options{ + Processor: makeTestRemoteFileProcessorOK(), + Client: makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + atomic.AddInt64(&clientCalled, 1) + return nil, assert.AnError + }), + DownloadURL: "http://example.com", + SaveFilePath: filepath.Join(dir, "foo"), + TmpFilePath: filepath.Join(dir, "tmp"), + RetryCount: 2, + RetryInterval: 10 * time.Millisecond, + Timeout: 10 * time.Millisecond, + UpdateInterval: 0, + }) + err := syncer.run() + assert.Error(t, err) + assert.Equal(t, int64(3), clientCalled, "HTTPClient should be called 1 + 2 times") +} + +func TestRemoteFileSyncerRunRetryWhenProcessSavedFileErr(t *testing.T) { + dir := t.TempDir() + + var ( + processorCalled int64 = 0 + ) + syncer, _ := NewRemoteFileSyncer(Options{ + Processor: makeTestRemoteFileProcessor(func(datapath string) error { + atomic.AddInt64(&processorCalled, 1) + return assert.AnError + }), + Client: makeTestHTTPClient200(), + DownloadURL: "http://example.com", + SaveFilePath: filepath.Join(dir, "foo"), + TmpFilePath: filepath.Join(dir, "tmp"), + RetryCount: 2, + RetryInterval: 10 * time.Millisecond, + Timeout: 10 * time.Millisecond, + UpdateInterval: 0, + }) + err := syncer.run() + assert.Error(t, err) + assert.Equal(t, int64(3), processorCalled, "processor should be called 1 + 2 times") +} + +func TestRemoteFileSyncerRunRetryWhenTaskStop(t *testing.T) { + dir := t.TempDir() + + var ( + clientCalled int64 = 0 + ) + syncer, _ := NewRemoteFileSyncer(Options{ + Processor: makeTestRemoteFileProcessorOK(), + Client: makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + atomic.AddInt64(&clientCalled, 1) + return nil, assert.AnError + }), + DownloadURL: "http://example.com", + SaveFilePath: filepath.Join(dir, "foo"), + TmpFilePath: filepath.Join(dir, "tmp"), + RetryCount: 2, + RetryInterval: 10 * time.Millisecond, + Timeout: 10 * time.Millisecond, + UpdateInterval: 0, + }) + syncer.Stop() + err := syncer.run() + assert.Error(t, err) + assert.Equal(t, int64(1), clientCalled, "HTTPClient should be called 1 times because syncer is stopped") +} + +func TestRemoteFileSyncerSync(t *testing.T) { + dir := t.TempDir() + readdir := filepath.Join(dir, "readonly") + os.MkdirAll(readdir, 0555) + + tests := []struct { + name string + client HTTPClient + saveFilePath string + hasErr bool + }{ + { + "Sync successful", + makeTestHTTPClient200(), + filepath.Join(dir, "foo"), + false, + }, + { + "Sync failed, client returns error", + makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + return nil, assert.AnError + }), + filepath.Join(dir, "foo"), + true, + }, + { + "Sync failed, save file path is read-only", + makeTestHTTPClient200(), + readdir, + true, + }, + } + + syncer, _ := NewRemoteFileSyncer(Options{ + Processor: makeTestRemoteFileProcessorOK(), + Client: makeTestHTTPClient(nil), + DownloadURL: "http://example.com", + SaveFilePath: filepath.Join(dir, "foo"), + TmpFilePath: filepath.Join(dir, "tmp"), + RetryCount: 0, + RetryInterval: 10 * time.Second, + Timeout: 10 * time.Second, + UpdateInterval: 0, + }) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + syncer.client = test.client + syncer.saveFilePath = test.saveFilePath + err := syncer.sync() + if test.hasErr { + assert.Error(t, err, "RemoteFileSyncer.sync should return error") + } else { + assert.NoError(t, err, "RemoteFileSyncer.sync should not return error. Error %v", err) + } + }) + } +} + +func TestRemoteFileSyncerProcessSavedFile(t *testing.T) { + dir := t.TempDir() + datapath, _ := createTempFile(dir, "imdata") + syncer := &RemoteFileSyncer{ + processor: makeTestRemoteFileProcessor(func(datapath string) error { + return assert.AnError + }), + saveFilePath: datapath, + } + err := syncer.processSavedFile() + assert.Error(t, err) + assert.NoFileExists(t, datapath, "should remove file if process failed") +} + +func TestRemoteFileSyncerUpdateIfNeeded(t *testing.T) { + dir := t.TempDir() + syncer, _ := NewRemoteFileSyncer(Options{ + Processor: makeTestRemoteFileProcessorOK(), + Client: makeTestHTTPClient(nil), + DownloadURL: "http://example.com", + SaveFilePath: filepath.Join(dir, "foo"), + TmpFilePath: filepath.Join(dir, "tmp"), + RetryCount: 0, + RetryInterval: 10 * time.Second, + Timeout: 10 * time.Second, + UpdateInterval: 0, + }) + + tests := []struct { + name string + client HTTPClient + saveFilePath string + saveFileData string + needed bool + hasError bool + }{ + { + "File not exists", + makeTestHTTPClient200(), + filepath.Join(dir, "foo"), + "", + true, + true, + }, + { + "File exists, get content length error", + makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + return nil, assert.AnError + }), + filepath.Join(dir, "foo"), + "imdata", + false, + true, + }, + { + "File exists, content length is different", + makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + return &http.Response{ContentLength: 1}, nil + }), + filepath.Join(dir, "foo"), + "imdata", + true, + true, + }, + { + "File exists, content length is the same", + makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + return &http.Response{ContentLength: 6}, nil + }), + filepath.Join(dir, "foo"), + "imdata", + false, + false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.saveFileData != "" { + createFile(test.saveFilePath, test.saveFileData) + } + syncer.client = test.client + syncer.saveFilePath = test.saveFilePath + syncer.syncing.Store(true) // let run return a known error for checking run is called + + err := syncer.updateIfNeeded() + if test.hasError { + assert.Error(t, err, "RemoteFileSyncer.updateIfNeeded should return error") + } else { + assert.NoError(t, err, "RemoteFileSyncer.updateIfNeeded should not return error. Error: %v", err) + } + if test.needed { + assert.Equal(t, ErrSyncInProgress, err) + } + }) + } +} + +func TestCreateAndCheckWritePermissionsFor(t *testing.T) { + dir := t.TempDir() + readdir := filepath.Join(dir, "readonly") + os.MkdirAll(readdir, 0555) + + tests := []struct { + name string + datapath string + hasError bool + }{ + { + "Directory write permissions are granted", + filepath.Join(dir, "foo"), + false, + }, + { + "Directory write permissions are granted to a nested directory", + filepath.Join(dir, "foo/bar"), + false, + }, + { + "Directory is read-only, can not create file", + filepath.Join(readdir, "foo"), + true, + }, + { + "Directory is read-only, can not create directory", + filepath.Join(readdir, "foo/bar"), + true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := createAndCheckWritePermissionsFor(test.datapath) + if test.hasError { + assert.Error(t, err, "should return error") + assert.NoFileExists(t, test.datapath, "should not create file") + } else { + assert.NoError(t, err, "should not return error. Error: %v", err) + } + }) + } +} + +func TestDownloadFileFromURL(t *testing.T) { + tests := []struct { + name string + client HTTPClient + url string + datapath string + timeout time.Duration + data string + hasError bool + }{ + { + "Succesful. Downloaded data is 'imdata'", + makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader([]byte("imdata"))), + }, nil + }), + "http://example.com", + filepath.Join(t.TempDir(), "foo"), + 1 * time.Second, + "imdata", + false, + }, + { + "Download from a invalid URL", + makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader([]byte("imdata"))), + }, nil + }), + "!@#$%^&*()", + filepath.Join(t.TempDir(), "foo"), + 1 * time.Second, + "", + true, + }, + { + "Download from a valid URL, returns error", + makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + return nil, assert.AnError + }), + "http://example.com", + filepath.Join(t.TempDir(), "foo"), + 1 * time.Second, + "", + true, + }, + { + "Download from a valid URL, returns status code 404", + makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 404}, nil + }), + "http://example.com", + filepath.Join(t.TempDir(), "foo"), + 1 * time.Second, + "", + true, + }, + { + "Download from a valid URL, save to a directory", + makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader([]byte("imdata"))), + }, nil + }), + "http://example.com", + t.TempDir(), + 1 * time.Second, + "", + true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := downloadFileFromURL(test.client, test.url, test.datapath, test.timeout) + if test.hasError { + assert.Error(t, err, "DownloadFileFromURL should return error") + } else { + assert.NoError(t, err, "DownloadFileFromURL should not return error. Error: %v", err) + assertFileContent(t, test.datapath, test.data) + } + }) + } +} + +func TestRemoteFileSize(t *testing.T) { + tests := []struct { + name string + client HTTPClient + url string + timeout time.Duration + length int64 + hasError bool + }{ + { + "Successful. ContentLength is 100", + makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + ContentLength: 100, + }, nil + }), + "http://example.com", + 1 * time.Second, + 100, + false, + }, + { + "Request a invalid URL", + makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + ContentLength: 100, + }, nil + }), + "!@#$%^&*()", + 1 * time.Second, + 0, + true, + }, + { + "Request a valid URL, returns error", + makeTestHTTPClient(func(req *http.Request) (*http.Response, error) { + return nil, assert.AnError + }), + "http://example.com", + 1 * time.Second, + 0, + true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + length, err := remoteFileSize(test.client, test.url, test.timeout) + if test.hasError { + assert.Error(t, err, "RemoteFileSize should return error") + } else { + assert.NoError(t, err, "RemoteFileSize should not return error. Error: %v", err) + assert.Equal(t, test.length, length, "RemoteFileSize should return correct file size") + } + }) + } +} diff --git a/go.mod b/go.mod index 423abe2438e..e8faab12c4e 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/prebid/prebid-server/v2 go 1.21 require ( + github.com/51Degrees/device-detection-go/v4 v4.4.35 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/IABTechLab/adscert v0.34.0 github.com/NYTimes/gziphandler v1.1.1 @@ -22,6 +23,7 @@ require ( github.com/lib/pq v1.10.4 github.com/mitchellh/copystructure v1.2.0 github.com/modern-go/reflect2 v1.0.2 + github.com/oschwald/geoip2-golang v1.11.0 github.com/pkg/errors v0.9.1 github.com/prebid/go-gdpr v1.12.0 github.com/prebid/go-gpp v0.2.0 @@ -30,8 +32,11 @@ require ( github.com/prometheus/client_model v0.2.0 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/rs/cors v1.8.2 + github.com/spf13/cast v1.5.0 github.com/spf13/viper v1.12.0 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.9.0 + github.com/tidwall/gjson v1.17.1 + github.com/tidwall/sjson v1.2.5 github.com/vrischmann/go-metrics-influxdb v0.1.1 github.com/xeipuuv/gojsonschema v1.2.0 github.com/yudai/gojsondiff v1.0.0 @@ -43,7 +48,6 @@ require ( ) require ( - github.com/51Degrees/device-detection-go/v4 v4.4.35 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -56,6 +60,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/oschwald/maxminddb-golang v1.13.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -63,21 +68,18 @@ require ( github.com/prometheus/procfs v0.7.3 // indirect github.com/sergi/go-diff v1.2.0 // indirect github.com/spf13/afero v1.8.2 // indirect - github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/objx v0.5.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.3.0 // indirect - github.com/tidwall/gjson v1.17.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect - github.com/tidwall/sjson v1.2.5 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect github.com/yudai/pp v2.0.1+incompatible // indirect golang.org/x/crypto v0.21.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/sys v0.20.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.66.4 // indirect diff --git a/go.sum b/go.sum index d62ceefeefb..e52ed18fe25 100644 --- a/go.sum +++ b/go.sum @@ -137,6 +137,7 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= @@ -318,9 +319,11 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= @@ -383,6 +386,10 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.11.0 h1:+CqWgvj0OZycCaqclBD1pxKHAU+tOkHmQIWvDHq2aug= github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg= +github.com/oschwald/geoip2-golang v1.11.0 h1:hNENhCn1Uyzhf9PTmquXENiWS6AlxAEnBII6r8krA3w= +github.com/oschwald/geoip2-golang v1.11.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo= +github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU= +github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -438,6 +445,7 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqn github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -470,8 +478,9 @@ github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiu github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -480,8 +489,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= @@ -740,8 +750,8 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/main.go b/main.go index 4ef96b34724..5d305a59d7f 100644 --- a/main.go +++ b/main.go @@ -1,14 +1,17 @@ package main import ( + _ "embed" "flag" "net/http" "path/filepath" "runtime" "time" + _ "time/tzdata" jsoniter "github.com/json-iterator/go" "github.com/prebid/prebid-server/v2/config" + "github.com/prebid/prebid-server/v2/config/countrycode" "github.com/prebid/prebid-server/v2/currency" "github.com/prebid/prebid-server/v2/openrtb_ext" "github.com/prebid/prebid-server/v2/router" @@ -55,10 +58,17 @@ func main() { } } +// CountryCodeData will be packaged into the binary file at compile time. +// +//go:embed country-codes.csv +var CountryCodeData string + const configFileName = "pbs" const infoDirectory = "./static/bidder-info" func loadConfig(bidderInfos config.BidderInfos) (*config.Configuration, error) { + countrycode.Load(CountryCodeData) + v := viper.New() config.SetupViper(v, configFileName, bidderInfos) return config.New(v, bidderInfos, openrtb_ext.NormalizeBidderName) diff --git a/main_test.go b/main_test.go index 79ae373d473..b33119c2b7a 100644 --- a/main_test.go +++ b/main_test.go @@ -65,3 +65,7 @@ func TestViperEnv(t *testing.T) { assert.Equal(t, 60, v.Get("host_cookie.ttl_days"), "Config With Underscores") assert.ElementsMatch(t, []string{"1.1.1.1/24", "2.2.2.2/24"}, v.Get("request_validation.ipv4_private_networks"), "Arrays") } + +func TestCountryCodeData(t *testing.T) { + assert.NotEqual(t, "", CountryCodeData, "CountryCodeData should not be empty") +} diff --git a/metrics/config/metrics.go b/metrics/config/metrics.go index 8b544bd966b..ad4af787f50 100644 --- a/metrics/config/metrics.go +++ b/metrics/config/metrics.go @@ -371,6 +371,12 @@ func (me *MultiMetricsEngine) RecordModuleTimeout(labels metrics.ModuleLabels) { } } +func (me *MultiMetricsEngine) RecordGeoLocationRequest(success bool) { + for _, thisME := range *me { + thisME.RecordGeoLocationRequest(success) + } +} + // NilMetricsEngine implements the MetricsEngine interface where no metrics are actually captured. This is // used if no metric backend is configured and also for tests. type NilMetricsEngine struct{} @@ -546,3 +552,6 @@ func (me *NilMetricsEngine) RecordModuleExecutionError(labels metrics.ModuleLabe func (me *NilMetricsEngine) RecordModuleTimeout(labels metrics.ModuleLabels) { } + +func (me *NilMetricsEngine) RecordGeoLocationRequest(success bool) { +} diff --git a/metrics/go_metrics.go b/metrics/go_metrics.go index ac28a18c8e1..65a79dc60c9 100644 --- a/metrics/go_metrics.go +++ b/metrics/go_metrics.go @@ -82,6 +82,10 @@ type Metrics struct { // Module metrics ModuleMetrics map[string]map[string]*ModuleMetrics + // GeoLocation metrics + GeoLocationRequestsSuccess metrics.Meter + GeoLocationRequestsFailure metrics.Meter + OverheadTimer map[OverheadType]metrics.Timer } @@ -206,6 +210,9 @@ func NewBlankMetrics(registry metrics.Registry, exchanges []string, disabledMetr ModuleMetrics: make(map[string]map[string]*ModuleMetrics), + GeoLocationRequestsSuccess: blankMeter, + GeoLocationRequestsFailure: blankMeter, + exchanges: exchanges, modules: getModuleNames(moduleStageNames), @@ -376,6 +383,9 @@ func NewMetrics(registry metrics.Registry, exchanges []openrtb_ext.BidderName, d registerModuleMetrics(registry, module, stages, newMetrics.ModuleMetrics[module]) } + newMetrics.GeoLocationRequestsSuccess = metrics.GetOrRegisterMeter("geolocation_requests.ok", registry) + newMetrics.GeoLocationRequestsFailure = metrics.GetOrRegisterMeter("geolocation_requests.failed", registry) + return newMetrics } @@ -1164,3 +1174,11 @@ func (me *Metrics) getModuleMetric(labels ModuleLabels) (*ModuleMetrics, error) return mm, nil } + +func (me *Metrics) RecordGeoLocationRequest(success bool) { + if success { + me.GeoLocationRequestsSuccess.Mark(1) + } else { + me.GeoLocationRequestsFailure.Mark(1) + } +} diff --git a/metrics/go_metrics_test.go b/metrics/go_metrics_test.go index 7cacd9099db..a89ed87c85e 100644 --- a/metrics/go_metrics_test.go +++ b/metrics/go_metrics_test.go @@ -91,6 +91,9 @@ func TestNewMetrics(t *testing.T) { ensureContainsModuleMetrics(t, registry, fmt.Sprintf("modules.module.%s.stage.%s", module, stage), m.ModuleMetrics[module][stage]) } } + + ensureContains(t, registry, "geolocation_requests.ok", m.GeoLocationRequestsSuccess) + ensureContains(t, registry, "geolocation_requests.failed", m.GeoLocationRequestsFailure) } func TestRecordBidType(t *testing.T) { @@ -1312,3 +1315,35 @@ func TestRecordAdapterRequest(t *testing.T) { }) } } + +func TestRecordGeoLocationRequestMetric(t *testing.T) { + testCases := []struct { + description string + requestSuccess bool + expectedSuccessRequestsCount int64 + expectedFailedRequestsCount int64 + }{ + { + description: "Record GeoLocation failed request, expected success request count is 0 and failed request count is 1", + requestSuccess: false, + expectedSuccessRequestsCount: 0, + expectedFailedRequestsCount: 1, + }, + { + description: "Record GeoLocation successful request, expected success request count is 1 and failed request count is 0", + requestSuccess: true, + expectedSuccessRequestsCount: 1, + expectedFailedRequestsCount: 0, + }, + } + + for _, test := range testCases { + registry := metrics.NewRegistry() + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderName("AnyName")}, config.DisabledMetrics{}, nil, nil) + + m.RecordGeoLocationRequest(test.requestSuccess) + + assert.Equal(t, test.expectedSuccessRequestsCount, m.GeoLocationRequestsSuccess.Count(), test.description) + assert.Equal(t, test.expectedFailedRequestsCount, m.GeoLocationRequestsFailure.Count(), test.description) + } +} diff --git a/metrics/metrics.go b/metrics/metrics.go index 2696c642e21..c215791e1ea 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -472,4 +472,5 @@ type MetricsEngine interface { RecordModuleSuccessRejected(labels ModuleLabels) RecordModuleExecutionError(labels ModuleLabels) RecordModuleTimeout(labels ModuleLabels) + RecordGeoLocationRequest(success bool) } diff --git a/metrics/metrics_mock.go b/metrics/metrics_mock.go index 56e2ab8fb1b..14e0f6ee2d3 100644 --- a/metrics/metrics_mock.go +++ b/metrics/metrics_mock.go @@ -226,3 +226,7 @@ func (me *MetricsEngineMock) RecordModuleExecutionError(labels ModuleLabels) { func (me *MetricsEngineMock) RecordModuleTimeout(labels ModuleLabels) { me.Called(labels) } + +func (me *MetricsEngineMock) RecordGeoLocationRequest(success bool) { + me.Called(success) +} diff --git a/metrics/prometheus/preload.go b/metrics/prometheus/preload.go index 73ea643722b..31f5634872b 100644 --- a/metrics/prometheus/preload.go +++ b/metrics/prometheus/preload.go @@ -172,6 +172,10 @@ func preloadLabelValues(m *Metrics, syncerKeys []string, moduleStageNames map[st successLabel: boolValues, }) + preloadLabelValuesForCounter(m.geoLocationRequests, map[string][]string{ + successLabel: boolValues, + }) + if !m.metricsDisabled.AdapterConnectionMetrics { preloadLabelValuesForCounter(m.adapterCreatedConnections, map[string][]string{ adapterLabel: adapterValues, diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go index 89b589badf8..b59387e3be2 100644 --- a/metrics/prometheus/prometheus.go +++ b/metrics/prometheus/prometheus.go @@ -58,6 +58,7 @@ type Metrics struct { adsCertRequests *prometheus.CounterVec adsCertSignTimer prometheus.Histogram bidderServerResponseTimer prometheus.Histogram + geoLocationRequests *prometheus.CounterVec // Adapter Metrics adapterBids *prometheus.CounterVec @@ -507,6 +508,11 @@ func NewMetrics(cfg config.PrometheusMetrics, disabledMetrics config.DisabledMet createModulesMetrics(cfg, reg, &metrics, moduleStageNames, standardTimeBuckets) + metrics.geoLocationRequests = newCounter(cfg, reg, + "geolocation_requests", + "Count of GeoLocation request, and if they were successfully sent.", + []string{successLabel}) + metrics.Gatherer = reg metricsPrefix := "" @@ -1089,3 +1095,15 @@ func (m *Metrics) RecordModuleTimeout(labels metrics.ModuleLabels) { stageLabel: labels.Stage, }).Inc() } + +func (m *Metrics) RecordGeoLocationRequest(success bool) { + if success { + m.geoLocationRequests.With(prometheus.Labels{ + successLabel: requestSuccessful, + }).Inc() + } else { + m.geoLocationRequests.With(prometheus.Labels{ + successLabel: requestFailed, + }).Inc() + } +} diff --git a/metrics/prometheus/prometheus_test.go b/metrics/prometheus/prometheus_test.go index fcb9ba6452c..dd3b5d60c31 100644 --- a/metrics/prometheus/prometheus_test.go +++ b/metrics/prometheus/prometheus_test.go @@ -2028,3 +2028,32 @@ func TestRecordModuleMetrics(t *testing.T) { } } } + +func TestRecordGeoLocationRequestMetric(t *testing.T) { + testCases := []struct { + description string + requestSuccess bool + expectedSuccessRequestsCount float64 + expectedFailedRequestsCount float64 + }{ + { + description: "Record failed request, expected success request count is 0 and failed request count is 1", + requestSuccess: false, + expectedSuccessRequestsCount: 0, + expectedFailedRequestsCount: 1, + }, + { + description: "Record successful request, expected success request count is 1 and failed request count is 0", + requestSuccess: true, + expectedSuccessRequestsCount: 1, + expectedFailedRequestsCount: 0, + }, + } + + for _, test := range testCases { + m := createMetricsForTesting() + m.RecordGeoLocationRequest(test.requestSuccess) + assertCounterVecValue(t, test.description, "successfully geolocation requests", m.geoLocationRequests, test.expectedSuccessRequestsCount, prometheus.Labels{successLabel: requestSuccessful}) + assertCounterVecValue(t, test.description, "unsuccessfully geolocation requests", m.geoLocationRequests, test.expectedFailedRequestsCount, prometheus.Labels{successLabel: requestFailed}) + } +} diff --git a/privacy/scrubber.go b/privacy/scrubber.go index 7a67737f028..e17e45fb795 100644 --- a/privacy/scrubber.go +++ b/privacy/scrubber.go @@ -110,8 +110,8 @@ func scrubGeoFull(reqWrapper *openrtb_ext.RequestWrapper) { func scrubDeviceIP(reqWrapper *openrtb_ext.RequestWrapper, ipConf IPConf) { if reqWrapper.Device != nil { - reqWrapper.Device.IP = scrubIP(reqWrapper.Device.IP, ipConf.IPV4.AnonKeepBits, iputil.IPv4BitSize) - reqWrapper.Device.IPv6 = scrubIP(reqWrapper.Device.IPv6, ipConf.IPV6.AnonKeepBits, iputil.IPv6BitSize) + reqWrapper.Device.IP = ScrubIP(reqWrapper.Device.IP, ipConf.IPV4.AnonKeepBits, iputil.IPv4BitSize) + reqWrapper.Device.IPv6 = ScrubIP(reqWrapper.Device.IPv6, ipConf.IPV6.AnonKeepBits, iputil.IPv6BitSize) } } @@ -146,7 +146,7 @@ func ScrubGeoAndDeviceIP(reqWrapper *openrtb_ext.RequestWrapper, ipConf IPConf) scrubGEO(reqWrapper) } -func scrubIP(ip string, ones, bits int) string { +func ScrubIP(ip string, ones, bits int) string { if ip == "" { return "" } diff --git a/privacy/scrubber_test.go b/privacy/scrubber_test.go index d06e0c9842e..58821f6f076 100644 --- a/privacy/scrubber_test.go +++ b/privacy/scrubber_test.go @@ -409,7 +409,7 @@ func TestScrubIP(t *testing.T) { for _, test := range testCases { t.Run(test.IP, func(t *testing.T) { // bits: ipv6 - 128, ipv4 - 32 - result := scrubIP(test.IP, test.maskBits, test.bits) + result := ScrubIP(test.IP, test.maskBits, test.bits) assert.Equal(t, test.cleanedIP, result) }) } diff --git a/router/router.go b/router/router.go index 867c856b06c..12bdadc8f24 100644 --- a/router/router.go +++ b/router/router.go @@ -22,6 +22,9 @@ import ( "github.com/prebid/prebid-server/v2/experiment/adscert" "github.com/prebid/prebid-server/v2/floors" "github.com/prebid/prebid-server/v2/gdpr" + "github.com/prebid/prebid-server/v2/geolocation" + "github.com/prebid/prebid-server/v2/geolocation/maxmind" + "github.com/prebid/prebid-server/v2/geolocation/remotefilesyncer" "github.com/prebid/prebid-server/v2/hooks" "github.com/prebid/prebid-server/v2/macros" "github.com/prebid/prebid-server/v2/metrics" @@ -241,10 +244,46 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R requestValidator := ortb.NewRequestValidator(activeBidders, disabledBidders, paramsValidator) priceFloorFetcher := floors.NewPriceFloorFetcher(cfg.PriceFloors, floorFechterHttpClient, r.MetricsEngine) + var geolocationService geolocation.GeoLocation + if cfg.GeoLocation.Enabled { + switch cfg.GeoLocation.Type { + case maxmind.Vendor: + maxmindGeo := &maxmind.GeoLocation{} + maxmindSyncerHttpClient := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + MaxConnsPerHost: cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxConnsPerHost, + MaxIdleConns: cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxIdleConns, + MaxIdleConnsPerHost: cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxIdleConnsPerHost, + IdleConnTimeout: time.Duration(cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.IdleConnTimeout) * time.Second, + }, + } + maxmindSyncer, err := remotefilesyncer.NewRemoteFileSyncer(remotefilesyncer.Options{ + Processor: maxmindGeo, + Client: maxmindSyncerHttpClient, + DownloadURL: cfg.GeoLocation.Maxmind.RemoteFileSyncer.DownloadURL, + SaveFilePath: cfg.GeoLocation.Maxmind.RemoteFileSyncer.SaveFilePath, + TmpFilePath: cfg.GeoLocation.Maxmind.RemoteFileSyncer.TmpFilePath, + RetryCount: cfg.GeoLocation.Maxmind.RemoteFileSyncer.RetryCount, + RetryInterval: time.Duration(cfg.GeoLocation.Maxmind.RemoteFileSyncer.RetryIntervalMillis) * time.Microsecond, + Timeout: time.Duration(cfg.GeoLocation.Maxmind.RemoteFileSyncer.TimeoutMillis) * time.Microsecond, + UpdateInterval: time.Duration(cfg.GeoLocation.Maxmind.RemoteFileSyncer.UpdateIntervalMillis) * time.Microsecond, + }) + if err != nil { + return nil, err + } + _ = maxmindSyncer.Start() + geolocationService = maxmindGeo + default: + return nil, fmt.Errorf("Unknown geolocation type: %s", cfg.GeoLocation.Type) + } + } + geolocationResolver := exchange.NewGeoLocationResolver(geolocationService, r.MetricsEngine) + tmaxAdjustments := exchange.ProcessTMaxAdjustments(cfg.TmaxAdjustments) planBuilder := hooks.NewExecutionPlanBuilder(cfg.Hooks, repo) macroReplacer := macros.NewStringIndexBasedReplacer() - theExchange := exchange.NewExchange(adapters, cacheClient, cfg, requestValidator, syncersByBidder, r.MetricsEngine, cfg.BidderInfos, gdprPermsBuilder, rateConvertor, categoriesFetcher, adsCertSigner, macroReplacer, priceFloorFetcher) + theExchange := exchange.NewExchange(adapters, cacheClient, cfg, requestValidator, syncersByBidder, r.MetricsEngine, cfg.BidderInfos, gdprPermsBuilder, rateConvertor, categoriesFetcher, adsCertSigner, macroReplacer, priceFloorFetcher, geolocationResolver) var uuidGenerator uuidutil.UUIDRandomGenerator openrtbEndpoint, err := openrtb2.NewEndpoint(uuidGenerator, theExchange, requestValidator, fetcher, accounts, cfg, r.MetricsEngine, analyticsRunner, disabledBidders, defReqJSON, activeBidders, storedRespFetcher, planBuilder, tmaxAdjustments) if err != nil { diff --git a/util/task/func_runner.go b/util/task/func_runner.go index fe6de614d2d..7a06246d188 100644 --- a/util/task/func_runner.go +++ b/util/task/func_runner.go @@ -2,14 +2,18 @@ package task import "time" -type funcRunner struct { +type FuncRunner struct { run func() error } -func (r funcRunner) Run() error { +func (r FuncRunner) Run() error { return r.run() } func NewTickerTaskFromFunc(interval time.Duration, runner func() error) *TickerTask { - return NewTickerTask(interval, funcRunner{run: runner}) + return NewTickerTask(interval, FuncRunner{run: runner}) +} + +func NewFuncRunner(f func() error) *FuncRunner { + return &FuncRunner{run: f} } diff --git a/util/task/func_runner_test.go b/util/task/func_runner_test.go index 86481e4ee5f..c8787f82aee 100644 --- a/util/task/func_runner_test.go +++ b/util/task/func_runner_test.go @@ -2,6 +2,7 @@ package task import ( "sync" + "sync/atomic" "testing" "time" @@ -26,3 +27,16 @@ func TestNewTickerTaskFromFunc(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 1, runCount) } + +func TestNewFuncRunner(t *testing.T) { + var count int64 + + runner := NewFuncRunner(func() error { + atomic.AddInt64(&count, 1) + return nil + }) + + err := runner.Run() + assert.NoError(t, err) + assert.Equal(t, int64(1), count) +} diff --git a/util/task/ticker_task.go b/util/task/ticker_task.go index a8d523b75d5..5c1a6b1b454 100644 --- a/util/task/ticker_task.go +++ b/util/task/ticker_task.go @@ -9,23 +9,40 @@ type Runner interface { } type TickerTask struct { - interval time.Duration - runner Runner - done chan struct{} + interval time.Duration + runner Runner + skipInitialRun bool + done chan struct{} } func NewTickerTask(interval time.Duration, runner Runner) *TickerTask { + return NewTickerTaskWithOptions(Options{ + Interval: interval, + Runner: runner, + }) +} + +type Options struct { + Interval time.Duration + Runner Runner + SkipInitialRun bool +} + +func NewTickerTaskWithOptions(opt Options) *TickerTask { return &TickerTask{ - interval: interval, - runner: runner, - done: make(chan struct{}), + interval: opt.Interval, + runner: opt.Runner, + skipInitialRun: opt.SkipInitialRun, + done: make(chan struct{}), } } // Start runs the task immediately and then schedules the task to run periodically // if a positive fetching interval has been specified. func (t *TickerTask) Start() { - t.runner.Run() + if !t.skipInitialRun { + t.runner.Run() + } if t.interval > 0 { go t.runRecurring() @@ -37,6 +54,11 @@ func (t *TickerTask) Stop() { close(t.done) } +// Done exports readonly done channel +func (t *TickerTask) Done() <-chan struct{} { + return t.done +} + // run creates a ticker that ticks at the specified interval. On each tick, // the task is executed func (t *TickerTask) runRecurring() { diff --git a/util/task/ticker_task_test.go b/util/task/ticker_task_test.go index 3ca0280d23e..7c28b45a42c 100644 --- a/util/task/ticker_task_test.go +++ b/util/task/ticker_task_test.go @@ -85,3 +85,43 @@ func TestStartWithPeriodicRun(t *testing.T) { time.Sleep(50 * time.Millisecond) assert.Equal(t, expectedRuns, runner.RunCount(), "runner should not run after Stop is called") } + +func TestSkipInitialRun(t *testing.T) { + // Setup One Periodic Run: + expectedRuns := 0 + runner := NewMockRunner(expectedRuns) + interval := 0 * time.Millisecond + ticker := task.NewTickerTaskWithOptions(task.Options{ + Interval: interval, + Runner: runner, + SkipInitialRun: true, + }) + + // Execute: + ticker.Start() + + // Verify No Additional Runs After Stop: + time.Sleep(50 * time.Millisecond) + assert.Equal(t, expectedRuns, runner.RunCount(), "runner should not run") +} + +func TestChannelDone(t *testing.T) { + runner := NewMockRunner(1) + interval := 10 * time.Millisecond + ticker := task.NewTickerTask(interval, runner) + + // Execute: + ticker.Start() + + go func() { + time.Sleep(10 * time.Millisecond) + ticker.Stop() + }() + + select { + case <-ticker.Done(): + // Expected + case <-time.After(250 * time.Millisecond): + assert.Failf(t, "Ticker Done", "expected stop signal") + } +} diff --git a/util/timeutil/time.go b/util/timeutil/time.go index e8eaae7d61f..8e0fcdb387f 100644 --- a/util/timeutil/time.go +++ b/util/timeutil/time.go @@ -1,6 +1,7 @@ package timeutil import ( + "sync" "time" ) @@ -14,3 +15,49 @@ type RealTime struct{} func (c *RealTime) Now() time.Time { return time.Now() } + +type LocationCache struct { + cache map[string]*LocationCacheResult + mu sync.RWMutex +} + +func NewLocationCache() *LocationCache { + return &LocationCache{ + cache: make(map[string]*LocationCacheResult), + } +} + +type LocationCacheResult struct { + loc *time.Location + err error +} + +// LoadLocation wraps standard package time.LoadLocation, cache the results +func (l *LocationCache) LoadLocation(name string) (*time.Location, error) { + l.mu.RLock() + result, ok := l.cache[name] + l.mu.RUnlock() + + if ok { + return result.loc, result.err + } + + l.mu.Lock() + defer l.mu.Unlock() + + result, ok = l.cache[name] + if ok { + return result.loc, result.err + } + + loc, err := time.LoadLocation(name) + // cache it whether it succeeds or fails. avoid cache penetration caused by invalid timezones. + l.cache[name] = &LocationCacheResult{loc: loc, err: err} + return loc, err +} + +var defaultLocationCache = NewLocationCache() + +func LoadLocation(name string) (*time.Location, error) { + return defaultLocationCache.LoadLocation(name) +} diff --git a/util/timeutil/time_test.go b/util/timeutil/time_test.go new file mode 100644 index 00000000000..229ebe13503 --- /dev/null +++ b/util/timeutil/time_test.go @@ -0,0 +1,64 @@ +package timeutil + +import ( + "testing" + "time" + _ "time/tzdata" + + "github.com/stretchr/testify/assert" +) + +func TestLocationCacheLoadLocation(t *testing.T) { + _, err := LoadLocation("Asia/Shanghai") + assert.Nil(t, err, "should load location Asia/Shanghai") + + c := NewLocationCache() + _, ok := c.cache["America/New_York"] + assert.False(t, ok, "cache should not contain America/New_York") + + newyork, err := c.LoadLocation("America/New_York") + assert.Nil(t, err) + assert.NotNil(t, newyork) + + _, ok = c.cache["America/New_York"] + assert.True(t, ok, "cache should contain America/New_York") + + cacheNewyork, _ := c.LoadLocation("America/New_York") + assert.Equal(t, newyork, cacheNewyork) +} + +func TestLocationCacheLoadLocationUnknown(t *testing.T) { + c := NewLocationCache() + _, ok := c.cache["America/Unknown"] + assert.False(t, ok, "cache should not contain America/Unknown") + + unknown, err := c.LoadLocation("America/Unknown") + assert.NotNil(t, err, "should return error") + assert.Nil(t, unknown, "should return nil location") + + result, ok := c.cache["America/Unknown"] + assert.True(t, ok, "cache should contain America/Unknown") + assert.NotNil(t, result.err, "cache should contain error") +} + +// goos: darwin +// goarch: amd64 +// pkg: github.com/prebid/prebid-server/v2/util/timeutil +// cpu: Intel(R) Core(TM) i5-8257U CPU @ 1.40GHz +// BenchmarkLocationCacheLoadLocation-8 66584589 18.2 ns/op 0 B/op 0 allocs/op +func BenchmarkLocationCacheLoadLocation(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = LoadLocation("America/New_York") + } +} + +// goos: darwin +// goarch: amd64 +// pkg: github.com/prebid/prebid-server/v2/util/timeutil +// cpu: Intel(R) Core(TM) i5-8257U CPU @ 1.40GHz +// BenchmarkTimeLoadLocation-8 51571 23117 ns/op 8635 B/op 13 allocs/op +func BenchmarkTimeLoadLocation(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = time.LoadLocation("America/New_York") + } +}