From 123388aa83360803a109d4d29386ee45a42c9781 Mon Sep 17 00:00:00 2001 From: Sheridan C Rawlins Date: Thu, 21 Nov 2024 20:02:19 -0800 Subject: [PATCH 1/3] jsonpatch is really slow and we shouldn't need to merge if there is no default. --- stored_requests/backends/file_fetcher/fetcher.go | 3 +++ stored_requests/backends/http_fetcher/fetcher.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/stored_requests/backends/file_fetcher/fetcher.go b/stored_requests/backends/file_fetcher/fetcher.go index 1c51a353fb8..ea2930b707f 100644 --- a/stored_requests/backends/file_fetcher/fetcher.go +++ b/stored_requests/backends/file_fetcher/fetcher.go @@ -60,6 +60,9 @@ func (fetcher *eagerFetcher) FetchAccount(ctx context.Context, accountDefaultsJS }} } + if accountDefaultsJSON == nil { + return accountJSON, nil + } completeJSON, err := jsonpatch.MergePatch(accountDefaultsJSON, accountJSON) if err != nil { return nil, []error{err} diff --git a/stored_requests/backends/http_fetcher/fetcher.go b/stored_requests/backends/http_fetcher/fetcher.go index dc4dd03a1fe..6ab21de670c 100644 --- a/stored_requests/backends/http_fetcher/fetcher.go +++ b/stored_requests/backends/http_fetcher/fetcher.go @@ -164,6 +164,9 @@ func (fetcher *HttpFetcher) FetchAccount(ctx context.Context, accountDefaultsJSO DataType: "Account", }} } + if accountDefaultsJSON == nil { + return accountJSON, nil + } completeJSON, err := jsonpatch.MergePatch(accountDefaultsJSON, accountJSON) if err != nil { return nil, []error{err} From 67147d96f629b3c206d8371b2ee5a48a6d560ba6 Mon Sep 17 00:00:00 2001 From: Sheridan C Rawlins Date: Thu, 21 Nov 2024 20:06:30 -0800 Subject: [PATCH 2/3] Add a test case. --- stored_requests/backends/file_fetcher/fetcher_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stored_requests/backends/file_fetcher/fetcher_test.go b/stored_requests/backends/file_fetcher/fetcher_test.go index 80e45bb9445..735045dd9fe 100644 --- a/stored_requests/backends/file_fetcher/fetcher_test.go +++ b/stored_requests/backends/file_fetcher/fetcher_test.go @@ -83,6 +83,10 @@ func TestAccountFetcher(t *testing.T) { assertErrorCount(t, 0, errs) assert.JSONEq(t, `{"disabled":false, "events_enabled":true, "id":"valid" }`, string(account)) + account, errs = fetcher.FetchAccount(context.Background(), nil, "valid") + assertErrorCount(t, 0, errs) + assert.JSONEq(t, `{"disabled":false, "id":"valid" }`, string(account)) + _, errs = fetcher.FetchAccount(context.Background(), json.RawMessage(`{"events_enabled":true}`), "nonexistent") assertErrorCount(t, 1, errs) assert.Error(t, errs[0]) From 867b85f26355dee33bc752b7ee70e37623744431 Mon Sep 17 00:00:00 2001 From: Sheridan C Rawlins Date: Thu, 12 Dec 2024 21:42:40 -0800 Subject: [PATCH 3/3] Update to json-patch.v5. --- adapters/relevantdigital/relevantdigital.go | 2 +- adservertargeting/respdataprocessor.go | 2 +- endpoints/openrtb2/amp_auction.go | 2 +- endpoints/openrtb2/auction.go | 2 +- endpoints/openrtb2/test_utils.go | 2 +- endpoints/openrtb2/video_auction.go | 2 +- exchange/events.go | 2 +- exchange/exchange_test.go | 2 +- firstpartydata/first_party_data.go | 2 +- go.mod | 10 +++++----- go.sum | 4 ++-- hooks/hookexecution/enricher.go | 2 +- stored_requests/backends/file_fetcher/fetcher.go | 2 +- stored_requests/backends/http_fetcher/fetcher.go | 2 +- util/jsonutil/merge.go | 2 +- 15 files changed, 20 insertions(+), 20 deletions(-) diff --git a/adapters/relevantdigital/relevantdigital.go b/adapters/relevantdigital/relevantdigital.go index 35789f3b841..702dca6b271 100644 --- a/adapters/relevantdigital/relevantdigital.go +++ b/adapters/relevantdigital/relevantdigital.go @@ -16,7 +16,7 @@ import ( "github.com/prebid/prebid-server/v3/macros" "github.com/prebid/prebid-server/v3/openrtb_ext" "github.com/prebid/prebid-server/v3/util/jsonutil" - jsonpatch "gopkg.in/evanphx/json-patch.v4" + jsonpatch "gopkg.in/evanphx/json-patch.v5" ) type adapter struct { diff --git a/adservertargeting/respdataprocessor.go b/adservertargeting/respdataprocessor.go index 4b18d64b44b..9977eca15d9 100644 --- a/adservertargeting/respdataprocessor.go +++ b/adservertargeting/respdataprocessor.go @@ -9,7 +9,7 @@ import ( "github.com/prebid/openrtb/v20/openrtb2" "github.com/prebid/prebid-server/v3/openrtb_ext" "github.com/prebid/prebid-server/v3/util/jsonutil" - jsonpatch "gopkg.in/evanphx/json-patch.v4" + jsonpatch "gopkg.in/evanphx/json-patch.v5" ) const MaxKeyLength = 20 diff --git a/endpoints/openrtb2/amp_auction.go b/endpoints/openrtb2/amp_auction.go index b596ec793c6..6f37d49fca5 100644 --- a/endpoints/openrtb2/amp_auction.go +++ b/endpoints/openrtb2/amp_auction.go @@ -19,7 +19,7 @@ import ( "github.com/prebid/prebid-server/v3/hooks/hookexecution" "github.com/prebid/prebid-server/v3/ortb" "github.com/prebid/prebid-server/v3/util/uuidutil" - jsonpatch "gopkg.in/evanphx/json-patch.v4" + jsonpatch "gopkg.in/evanphx/json-patch.v5" accountService "github.com/prebid/prebid-server/v3/account" "github.com/prebid/prebid-server/v3/amp" diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index fa61c484f0d..ff470c320ba 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -29,7 +29,7 @@ import ( "github.com/prebid/prebid-server/v3/privacysandbox" "github.com/prebid/prebid-server/v3/schain" "golang.org/x/net/publicsuffix" - jsonpatch "gopkg.in/evanphx/json-patch.v4" + jsonpatch "gopkg.in/evanphx/json-patch.v5" accountService "github.com/prebid/prebid-server/v3/account" "github.com/prebid/prebid-server/v3/analytics" diff --git a/endpoints/openrtb2/test_utils.go b/endpoints/openrtb2/test_utils.go index e869ded80b8..92e54ed3ed4 100644 --- a/endpoints/openrtb2/test_utils.go +++ b/endpoints/openrtb2/test_utils.go @@ -41,7 +41,7 @@ import ( "github.com/prebid/prebid-server/v3/util/iputil" "github.com/prebid/prebid-server/v3/util/jsonutil" "github.com/prebid/prebid-server/v3/util/uuidutil" - jsonpatch "gopkg.in/evanphx/json-patch.v4" + jsonpatch "gopkg.in/evanphx/json-patch.v5" ) // In this file we define: diff --git a/endpoints/openrtb2/video_auction.go b/endpoints/openrtb2/video_auction.go index cf18840fbd6..f3dbd2246c0 100644 --- a/endpoints/openrtb2/video_auction.go +++ b/endpoints/openrtb2/video_auction.go @@ -21,7 +21,7 @@ import ( "github.com/prebid/prebid-server/v3/hooks/hookexecution" "github.com/prebid/prebid-server/v3/ortb" "github.com/prebid/prebid-server/v3/privacy" - jsonpatch "gopkg.in/evanphx/json-patch.v4" + jsonpatch "gopkg.in/evanphx/json-patch.v5" accountService "github.com/prebid/prebid-server/v3/account" "github.com/prebid/prebid-server/v3/analytics" diff --git a/exchange/events.go b/exchange/events.go index d52db53aac6..20150f90b97 100644 --- a/exchange/events.go +++ b/exchange/events.go @@ -4,7 +4,7 @@ import ( "time" "github.com/prebid/prebid-server/v3/exchange/entities" - jsonpatch "gopkg.in/evanphx/json-patch.v4" + jsonpatch "gopkg.in/evanphx/json-patch.v5" "github.com/prebid/prebid-server/v3/analytics" "github.com/prebid/prebid-server/v3/config" diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index 87b53b101e0..693425ffb32 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -46,7 +46,7 @@ import ( "github.com/prebid/prebid-server/v3/util/ptrutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - jsonpatch "gopkg.in/evanphx/json-patch.v4" + jsonpatch "gopkg.in/evanphx/json-patch.v5" ) func TestNewExchange(t *testing.T) { diff --git a/firstpartydata/first_party_data.go b/firstpartydata/first_party_data.go index 44a3241a53c..a9107efa6fc 100644 --- a/firstpartydata/first_party_data.go +++ b/firstpartydata/first_party_data.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/prebid/openrtb/v20/openrtb2" - jsonpatch "gopkg.in/evanphx/json-patch.v4" + jsonpatch "gopkg.in/evanphx/json-patch.v5" "github.com/prebid/prebid-server/v3/errortypes" "github.com/prebid/prebid-server/v3/openrtb_ext" diff --git a/go.mod b/go.mod index e844fe5bf25..caf5993d34d 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 retract v3.0.0 // Forgot to update major version in import path and module name require ( + github.com/51Degrees/device-detection-go/v4 v4.4.35 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/IABTechLab/adscert v0.34.0 github.com/NYTimes/gziphandler v1.1.1 @@ -32,20 +33,22 @@ require ( github.com/prometheus/client_model v0.2.0 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/rs/cors v1.11.0 + github.com/spf13/cast v1.5.0 github.com/spf13/viper v1.12.0 github.com/stretchr/testify v1.8.1 + github.com/tidwall/gjson v1.17.1 + github.com/tidwall/sjson v1.2.5 github.com/vrischmann/go-metrics-influxdb v0.1.1 github.com/xeipuuv/gojsonschema v1.2.0 github.com/yudai/gojsondiff v1.0.0 golang.org/x/net v0.23.0 golang.org/x/text v0.14.0 google.golang.org/grpc v1.56.3 - gopkg.in/evanphx/json-patch.v4 v4.12.0 + gopkg.in/evanphx/json-patch.v5 v5.9.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/51Degrees/device-detection-go/v4 v4.4.35 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -65,15 +68,12 @@ require ( github.com/prometheus/procfs v0.7.3 // indirect github.com/sergi/go-diff v1.2.0 // indirect github.com/spf13/afero v1.8.2 // indirect - github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.3.0 // indirect - github.com/tidwall/gjson v1.17.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect - github.com/tidwall/sjson v1.2.5 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect diff --git a/go.sum b/go.sum index a6a6226c616..8b022c75aa4 100644 --- a/go.sum +++ b/go.sum @@ -987,8 +987,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v5 v5.9.0 h1:hx1VU2SGj4F8r9b8GUwJLdc8DNO8sy79ZGui0G05GLo= +gopkg.in/evanphx/json-patch.v5 v5.9.0/go.mod h1:/kvTRh1TVm5wuM6OkHxqXtE/1nUZZpihg29RtuIyfvk= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= diff --git a/hooks/hookexecution/enricher.go b/hooks/hookexecution/enricher.go index 7ca9f65f176..17a2eb9cc96 100644 --- a/hooks/hookexecution/enricher.go +++ b/hooks/hookexecution/enricher.go @@ -7,7 +7,7 @@ import ( "github.com/prebid/openrtb/v20/openrtb2" "github.com/prebid/prebid-server/v3/config" "github.com/prebid/prebid-server/v3/util/jsonutil" - jsonpatch "gopkg.in/evanphx/json-patch.v4" + jsonpatch "gopkg.in/evanphx/json-patch.v5" ) const ( diff --git a/stored_requests/backends/file_fetcher/fetcher.go b/stored_requests/backends/file_fetcher/fetcher.go index ea2930b707f..6bb8bc1447b 100644 --- a/stored_requests/backends/file_fetcher/fetcher.go +++ b/stored_requests/backends/file_fetcher/fetcher.go @@ -10,7 +10,7 @@ import ( "github.com/prebid/prebid-server/v3/stored_requests" "github.com/prebid/prebid-server/v3/util/jsonutil" - jsonpatch "gopkg.in/evanphx/json-patch.v4" + jsonpatch "gopkg.in/evanphx/json-patch.v5" ) // NewFileFetcher _immediately_ loads stored request data from local files. diff --git a/stored_requests/backends/http_fetcher/fetcher.go b/stored_requests/backends/http_fetcher/fetcher.go index 6ab21de670c..7f765b99327 100644 --- a/stored_requests/backends/http_fetcher/fetcher.go +++ b/stored_requests/backends/http_fetcher/fetcher.go @@ -11,7 +11,7 @@ import ( "github.com/prebid/prebid-server/v3/stored_requests" "github.com/prebid/prebid-server/v3/util/jsonutil" - jsonpatch "gopkg.in/evanphx/json-patch.v4" + jsonpatch "gopkg.in/evanphx/json-patch.v5" "github.com/golang/glog" "golang.org/x/net/context/ctxhttp" diff --git a/util/jsonutil/merge.go b/util/jsonutil/merge.go index 2b7edc6ee25..6dff8afca64 100644 --- a/util/jsonutil/merge.go +++ b/util/jsonutil/merge.go @@ -8,7 +8,7 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/modern-go/reflect2" - jsonpatch "gopkg.in/evanphx/json-patch.v4" + jsonpatch "gopkg.in/evanphx/json-patch.v5" "github.com/prebid/prebid-server/v3/errortypes" )