From a2cf04ca70671737fde202428364db3f7ccc79e3 Mon Sep 17 00:00:00 2001 From: Petu Eusebiu Date: Fri, 3 Nov 2023 14:48:49 +0200 Subject: [PATCH] feat(sync): use regclient for sync extension replaced containers/image package with regclient/regclient package Signed-off-by: Petu Eusebiu --- .github/workflows/branch-cov.yaml | 4 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/golangci-lint.yaml | 2 +- Makefile | 32 +- demos/dev-getting-started.rec | 2 +- demos/multi-arch-getting-started.rec | 8 +- examples/README.md | 15 +- examples/config-sync.json | 21 +- go.mod | 30 +- go.sum | 52 +- pkg/api/controller.go | 1 - pkg/api/routes.go | 4 +- pkg/common/common.go | 14 +- pkg/extensions/extension_sync.go | 20 +- pkg/extensions/search/search_test.go | 2 +- .../constants/{consts.go => constants.go} | 0 pkg/extensions/sync/content.go | 6 +- ...ntent_internal_test.go => content_test.go} | 19 +- pkg/extensions/sync/destination.go | 170 ++--- pkg/extensions/sync/features/features.go | 53 -- pkg/extensions/sync/httpclient/cache.go | 58 -- pkg/extensions/sync/httpclient/client.go | 507 -------------- .../sync/httpclient/client_internal_test.go | 167 ----- pkg/extensions/sync/oci_layout.go | 42 +- pkg/extensions/sync/on_demand.go | 64 +- pkg/extensions/sync/references/cosign.go | 219 ------ pkg/extensions/sync/references/oci.go | 237 ------- pkg/extensions/sync/references/references.go | 234 ------- .../references/references_internal_test.go | 306 -------- .../sync/references/referrers_tag.go | 172 ----- pkg/extensions/sync/referrers.go | 36 + pkg/extensions/sync/remote.go | 380 +++++++--- pkg/extensions/sync/service.go | 655 +++++++++++------- pkg/extensions/sync/sync.go | 50 +- pkg/extensions/sync/sync_internal_test.go | 380 +--------- pkg/extensions/sync/sync_test.go | 521 ++++++-------- pkg/extensions/sync/utils.go | 277 ++------ pkg/meta/boltdb/boltdb.go | 2 +- pkg/meta/dynamodb/dynamodb.go | 2 +- pkg/meta/parse.go | 2 +- pkg/storage/gc/gc.go | 12 +- pkg/storage/storage.go | 2 +- pkg/test/auth/bearer.go | 27 +- scripts/update_licenses.sh | 2 +- test/blackbox/sync_docker.bats | 2 +- test/scripts/fuzzAll.sh | 2 +- 46 files changed, 1271 insertions(+), 3544 deletions(-) rename pkg/extensions/sync/constants/{consts.go => constants.go} (100%) rename pkg/extensions/sync/{content_internal_test.go => content_test.go} (91%) delete mode 100644 pkg/extensions/sync/features/features.go delete mode 100644 pkg/extensions/sync/httpclient/cache.go delete mode 100644 pkg/extensions/sync/httpclient/client.go delete mode 100644 pkg/extensions/sync/httpclient/client_internal_test.go delete mode 100644 pkg/extensions/sync/references/cosign.go delete mode 100644 pkg/extensions/sync/references/oci.go delete mode 100644 pkg/extensions/sync/references/references.go delete mode 100644 pkg/extensions/sync/references/references_internal_test.go delete mode 100644 pkg/extensions/sync/references/referrers_tag.go create mode 100644 pkg/extensions/sync/referrers.go diff --git a/.github/workflows/branch-cov.yaml b/.github/workflows/branch-cov.yaml index 3228058c2..5e22734bd 100644 --- a/.github/workflows/branch-cov.yaml +++ b/.github/workflows/branch-cov.yaml @@ -34,7 +34,7 @@ jobs: cd $GITHUB_WORKSPACE for i in $(find . -type f \( -name "*.go" -not -name "*_test.go" -not -name "generated.go" \)); do echo $i; - gobco -test '-tags=sync,search,scrub,metrics,containers_image_openpgp' $i; - gobco -test '-tags=containers_image_openpgp' $i; + gobco -test '-tags=sync,search,scrub,metrics' $i; + gobco -test $i; done diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3e75a41f7..122b4902b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,7 +39,7 @@ jobs: # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed env: CGO_ENABLED: 0 - GOFLAGS: "-tags=sync,search,scrub,metrics,userprefs,mgmt,imagetrust,containers_image_openpgp" + GOFLAGS: "-tags=sync,search,scrub,metrics,userprefs,mgmt,imagetrust" steps: - name: Checkout repository diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index 00c6b0ae2..d69fe7054 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -31,7 +31,7 @@ jobs: # Optional: golangci-lint command line arguments. # args: --issues-exit-code=0 - args: --config ./golangcilint.yaml --enable-all --build-tags debug,needprivileges,sync,scrub,search,userprefs,metrics,containers_image_openpgp,lint,mgmt,imagetrust ./cmd/... ./pkg/... + args: --config ./golangcilint.yaml --enable-all --build-tags debug,needprivileges,sync,scrub,search,userprefs,metrics,lint,mgmt,imagetrust ./cmd/... ./pkg/... # Optional: show only new issues if it's a pull request. The default value is `false`. # only-new-issues: true diff --git a/Makefile b/Makefile index 0b1d8f372..27756dc89 100644 --- a/Makefile +++ b/Makefile @@ -109,8 +109,6 @@ swaggercheck: swagger .PHONY: build-metadata build-metadata: $(if $(findstring ui,$(BUILD_LABELS)), ui) - # do not allow empty $(BUILD_TAGS) (at least add containers_image_openpgp that doesn't affect package import & files listing) - $(eval BUILD_TAGS=$(if $(BUILD_LABELS),$(BUILD_LABELS),containers_image_openpgp)) echo "Imports: \n" go list -tags $(BUILD_TAGS) -f '{{ join .Imports "\n" }}' ./... | sort -u echo "\n Files: \n" @@ -164,30 +162,30 @@ gen-protobuf: check-not-freebds $(PROTOC) .PHONY: binary-minimal binary-minimal: EXTENSIONS= binary-minimal: modcheck build-metadata - env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zot-$(OS)-$(ARCH)-minimal $(BUILDMODE_FLAGS) -tags containers_image_openpgp -v -trimpath -ldflags "-X zotregistry.dev/zot/pkg/api/config.ReleaseTag=${RELEASE_TAG} -X zotregistry.dev/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.dev/zot/pkg/api/config.BinaryType=minimal -X zotregistry.dev/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zot + env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zot-$(OS)-$(ARCH)-minimal $(BUILDMODE_FLAGS) -v -trimpath -ldflags "-X zotregistry.dev/zot/pkg/api/config.ReleaseTag=${RELEASE_TAG} -X zotregistry.dev/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.dev/zot/pkg/api/config.BinaryType=minimal -X zotregistry.dev/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zot .PHONY: binary binary: $(if $(findstring ui,$(BUILD_LABELS)), ui) binary: modcheck build-metadata - env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zot-$(OS)-$(ARCH) $(BUILDMODE_FLAGS) -tags $(BUILD_LABELS),containers_image_openpgp -v -trimpath -ldflags "-X zotregistry.dev/zot/pkg/api/config.ReleaseTag=${RELEASE_TAG} -X zotregistry.dev/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.dev/zot/pkg/api/config.BinaryType=$(extended-name) -X zotregistry.dev/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zot + env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zot-$(OS)-$(ARCH) $(BUILDMODE_FLAGS) -tags $(BUILD_LABELS) -v -trimpath -ldflags "-X zotregistry.dev/zot/pkg/api/config.ReleaseTag=${RELEASE_TAG} -X zotregistry.dev/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.dev/zot/pkg/api/config.BinaryType=$(extended-name) -X zotregistry.dev/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zot .PHONY: binary-debug binary-debug: $(if $(findstring ui,$(BUILD_LABELS)), ui) binary-debug: modcheck swaggercheck build-metadata - env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zot-$(OS)-$(ARCH)-debug $(BUILDMODE_FLAGS) -tags $(BUILD_LABELS),debug,containers_image_openpgp -v -gcflags all='-N -l' -ldflags "-X zotregistry.dev/zot/pkg/api/config.ReleaseTag=${RELEASE_TAG} -X zotregistry.dev/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.dev/zot/pkg/api/config.BinaryType=$(extended-name) -X zotregistry.dev/zot/pkg/api/config.GoVersion=${GO_VERSION}" ./cmd/zot + env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zot-$(OS)-$(ARCH)-debug $(BUILDMODE_FLAGS) -tags $(BUILD_LABELS),debug -v -gcflags all='-N -l' -ldflags "-X zotregistry.dev/zot/pkg/api/config.ReleaseTag=${RELEASE_TAG} -X zotregistry.dev/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.dev/zot/pkg/api/config.BinaryType=$(extended-name) -X zotregistry.dev/zot/pkg/api/config.GoVersion=${GO_VERSION}" ./cmd/zot .PHONY: cli cli: modcheck build-metadata - env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zli-$(OS)-$(ARCH) $(BUILDMODE_FLAGS) -tags $(BUILD_LABELS),search,containers_image_openpgp -v -trimpath -ldflags "-X zotregistry.dev/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.dev/zot/pkg/api/config.BinaryType=$(extended-name) -X zotregistry.dev/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zli + env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zli-$(OS)-$(ARCH) $(BUILDMODE_FLAGS) -tags $(BUILD_LABELS),search -v -trimpath -ldflags "-X zotregistry.dev/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.dev/zot/pkg/api/config.BinaryType=$(extended-name) -X zotregistry.dev/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zli .PHONY: bench bench: modcheck build-metadata - env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zb-$(OS)-$(ARCH) $(BUILDMODE_FLAGS) -tags $(BUILD_LABELS),containers_image_openpgp -v -trimpath -ldflags "-X zotregistry.dev/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.dev/zot/pkg/api/config.BinaryType=$(extended-name) -X zotregistry.dev/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zb + env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zb-$(OS)-$(ARCH) $(BUILDMODE_FLAGS) -tags $(BUILD_LABELS) -v -trimpath -ldflags "-X zotregistry.dev/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.dev/zot/pkg/api/config.BinaryType=$(extended-name) -X zotregistry.dev/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zb .PHONY: exporter-minimal exporter-minimal: EXTENSIONS= exporter-minimal: modcheck build-metadata - env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zxp-$(OS)-$(ARCH) $(BUILDMODE_FLAGS) -tags containers_image_openpgp -v -trimpath ./cmd/zxp + env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zxp-$(OS)-$(ARCH) $(BUILDMODE_FLAGS) -v -trimpath ./cmd/zxp .PHONY: test-prereq test-prereq: check-skopeo $(TESTDATA) $(ORAS) @@ -195,22 +193,22 @@ test-prereq: check-skopeo $(TESTDATA) $(ORAS) .PHONY: test-extended test-extended: $(if $(findstring ui,$(BUILD_LABELS)), ui) test-extended: test-prereq - go test -failfast -tags $(BUILD_LABELS),containers_image_openpgp -trimpath -race -timeout 20m -cover -coverpkg ./... -coverprofile=coverage-extended.txt -covermode=atomic ./... + go test -failfast -tags $(BUILD_LABELS) -trimpath -race -timeout 20m -cover -coverpkg ./... -coverprofile=coverage-extended.txt -covermode=atomic ./... rm -rf /tmp/getter*; rm -rf /tmp/trivy* .PHONY: test-minimal test-minimal: test-prereq - go test -failfast -tags containers_image_openpgp -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-minimal.txt -covermode=atomic ./... + go test -failfast -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-minimal.txt -covermode=atomic ./... rm -rf /tmp/getter*; rm -rf /tmp/trivy* .PHONY: test-devmode test-devmode: $(if $(findstring ui,$(BUILD_LABELS)), ui) test-devmode: test-prereq - go test -failfast -tags dev,$(BUILD_LABELS),containers_image_openpgp -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-extended.txt -covermode=atomic ./pkg/test/... ./pkg/api/... ./pkg/storage/... ./pkg/extensions/sync/... -run ^TestInject + go test -failfast -tags dev,$(BUILD_LABELS) -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-extended.txt -covermode=atomic ./pkg/test/... ./pkg/api/... ./pkg/storage/... ./pkg/extensions/sync/... -run ^TestInject rm -rf /tmp/getter*; rm -rf /tmp/trivy* - go test -failfast -tags dev,containers_image_openpgp -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-dev-minimal.txt -covermode=atomic ./pkg/test/... ./pkg/storage/... ./pkg/extensions/sync/... -run ^TestInject + go test -failfast -tags dev -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-dev-minimal.txt -covermode=atomic ./pkg/test/... ./pkg/storage/... ./pkg/extensions/sync/... -run ^TestInject rm -rf /tmp/getter*; rm -rf /tmp/trivy* - go test -failfast -tags stress,$(BUILD_LABELS),containers_image_openpgp -trimpath -race -timeout 15m ./pkg/cli/server/stress_test.go + go test -failfast -tags stress,$(BUILD_LABELS) -trimpath -race -timeout 15m ./pkg/cli/server/stress_test.go .PHONY: test test: $(if $(findstring ui,$(BUILD_LABELS)), ui) @@ -219,7 +217,7 @@ test: test-extended test-minimal test-devmode .PHONY: privileged-test privileged-test: $(if $(findstring ui,$(BUILD_LABELS)), ui) privileged-test: check-skopeo $(TESTDATA) - go test -failfast -tags needprivileges,$(BUILD_LABELS),containers_image_openpgp -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-needprivileges.txt -covermode=atomic ./pkg/storage/local/... ./pkg/cli/client/... -run ^TestElevatedPrivileges + go test -failfast -tags needprivileges,$(BUILD_LABELS) -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-needprivileges.txt -covermode=atomic ./pkg/storage/local/... ./pkg/cli/client/... -run ^TestElevatedPrivileges $(TESTDATA): check-skopeo mkdir -p ${TESTDATA}; \ @@ -320,8 +318,8 @@ check-logs: check: $(if $(findstring ui,$(BUILD_LABELS)), ui) check: ./golangcilint.yaml $(GOLINTER) mkdir -p pkg/extensions/build; touch pkg/extensions/build/.empty - $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags containers_image_openpgp ./... - $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags $(BUILD_LABELS),containers_image_openpgp ./... + $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number ./... + $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags $(BUILD_LABELS) ./... $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags debug ./pkg/debug/swagger/ ./pkg/debug/gqlplayground $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags dev ./pkg/test/inject/ $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags stress ./pkg/cli/server/ @@ -349,7 +347,7 @@ update-licenses: check-linux check-licenses: # note: "printf" works for darwin instead of "echo -n" go install github.com/google/go-licenses@latest - @for tag in "$(BUILD_LABELS),containers_image_openpgp" "containers_image_openpgp"; do \ + @for tag in "$(BUILD_LABELS)"; do \ echo Evaluating tag: $$tag;\ for mod in $$(go list -m -f '{{if not (or .Indirect .Main)}}{{.Path}}{{end}}' all); do \ while [ x$$mod != x ]; do \ diff --git a/demos/dev-getting-started.rec b/demos/dev-getting-started.rec index 3f7bc9705..ee94cf2e4 100644 --- a/demos/dev-getting-started.rec +++ b/demos/dev-getting-started.rec @@ -265,7 +265,7 @@ [8.458937, "i", "\r"] [8.459588, "o", "\r\n"] [8.471366, "o", "go mod tidy\r\n"] -[8.971283, "o", "env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/zot-linux-amd64 -buildmode=pie -tags extended,containers_image_openpgp -v -trimpath -ldflags \"-X zotregistry.dev/zot/pkg/api/config.Commit=v1.4.0-rc3-1-ge583c2f -X zotregistry.dev/zot/pkg/api/config.BinaryType=extended -X zotregistry.dev/zot/pkg/api/config.GoVersion=go1.17.7 -s -w\" ./cmd/zot\r\n"] +[8.971283, "o", "env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/zot-linux-amd64 -buildmode=pie -tags extended -v -trimpath -ldflags \"-X zotregistry.dev/zot/pkg/api/config.Commit=v1.4.0-rc3-1-ge583c2f -X zotregistry.dev/zot/pkg/api/config.BinaryType=extended -X zotregistry.dev/zot/pkg/api/config.GoVersion=go1.17.7 -s -w\" ./cmd/zot\r\n"] [11.652118, "o", "\u001b[32m$ \u001b[39m"] [12.657831, "i", "l"] [12.658095, "o", "l"] diff --git a/demos/multi-arch-getting-started.rec b/demos/multi-arch-getting-started.rec index e50901131..1affc7b63 100644 --- a/demos/multi-arch-getting-started.rec +++ b/demos/multi-arch-getting-started.rec @@ -249,7 +249,7 @@ [7.274013, "i", "\r"] [7.274687, "o", "\r\n"] [7.28773, "o", "go mod tidy\r\n"] -[7.814666, "o", "env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/zot-linux-amd64 -buildmode=pie -tags extended,containers_image_openpgp -v -trimpath -ldflags \"-X zotregistry.dev/zot/pkg/api/config.Commit=v1.4.0-rc3-1-ge583c2f -X zotregistry.dev/zot/pkg/api/config.BinaryType=extended -X zotregistry.dev/zot/pkg/api/config.GoVersion=go1.17.7 -s -w\" ./cmd/zot\r\n"] +[7.814666, "o", "env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/zot-linux-amd64 -buildmode=pie -tags extended -v -trimpath -ldflags \"-X zotregistry.dev/zot/pkg/api/config.Commit=v1.4.0-rc3-1-ge583c2f -X zotregistry.dev/zot/pkg/api/config.BinaryType=extended -X zotregistry.dev/zot/pkg/api/config.GoVersion=go1.17.7 -s -w\" ./cmd/zot\r\n"] [10.405138, "o", "\u001b[32m$ \u001b[39m"] [13.980122, "i", "l"] [13.980725, "o", "l"] @@ -282,7 +282,7 @@ [23.131551, "i", "\r"] [23.132194, "o", "\r\n"] [23.144726, "o", "go mod tidy\r\n"] -[23.578377, "o", "env CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o bin/zot-linux-arm64 -buildmode=pie -tags extended,containers_image_openpgp -v -trimpath -ldflags \"-X zotregistry.dev/zot/pkg/api/config.Commit=v1.4.0-rc3-1-ge583c2f -X zotregistry.dev/zot/pkg/api/config.BinaryType=extended -X zotregistry.dev/zot/pkg/api/config.GoVersion=go1.17.7 -s -w\" ./cmd/zot\r\n"] +[23.578377, "o", "env CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o bin/zot-linux-arm64 -buildmode=pie -tags extended -v -trimpath -ldflags \"-X zotregistry.dev/zot/pkg/api/config.Commit=v1.4.0-rc3-1-ge583c2f -X zotregistry.dev/zot/pkg/api/config.BinaryType=extended -X zotregistry.dev/zot/pkg/api/config.GoVersion=go1.17.7 -s -w\" ./cmd/zot\r\n"] [26.263223, "o", "\u001b[32m$ \u001b[39m"] [28.435297, "i", "\r"] [28.435982, "o", "\r\n\u001b[32m$ \u001b[39m"] @@ -291,14 +291,14 @@ [29.525918, "i", "\r"] [29.526615, "o", "\r\n"] [29.538303, "o", "go mod tidy\r\n"] -[30.014147, "o", "env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o bin/zot-darwin-amd64 -buildmode=pie -tags extended,containers_image_openpgp -v -trimpath -ldflags \"-X zotregistry.dev/zot/pkg/api/config.Commit=v1.4.0-rc3-1-ge583c2f -X zotregistry.dev/zot/pkg/api/config.BinaryType=extended -X zotregistry.dev/zot/pkg/api/config.GoVersion=go1.17.7 -s -w\" ./cmd/zot\r\n"] +[30.014147, "o", "env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o bin/zot-darwin-amd64 -buildmode=pie -tags extended -v -trimpath -ldflags \"-X zotregistry.dev/zot/pkg/api/config.Commit=v1.4.0-rc3-1-ge583c2f -X zotregistry.dev/zot/pkg/api/config.BinaryType=extended -X zotregistry.dev/zot/pkg/api/config.GoVersion=go1.17.7 -s -w\" ./cmd/zot\r\n"] [32.749036, "o", "\u001b[32m$ \u001b[39m"] [34.710083, "i", "make OS=darwin ARCH=arm64 binary"] [34.710825, "o", "make OS=darwin ARCH=arm64 binary"] [35.312014, "i", "\r"] [35.312848, "o", "\r\n"] [35.324987, "o", "go mod tidy\r\n"] -[35.783444, "o", "env CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o bin/zot-darwin-arm64 -buildmode=pie -tags extended,containers_image_openpgp -v -trimpath -ldflags \"-X zotregistry.dev/zot/pkg/api/config.Commit=v1.4.0-rc3-1-ge583c2f -X zotregistry.dev/zot/pkg/api/config.BinaryType=extended -X zotregistry.dev/zot/pkg/api/config.GoVersion=go1.17.7 -s -w\" ./cmd/zot\r\n"] +[35.783444, "o", "env CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o bin/zot-darwin-arm64 -buildmode=pie -tags extended -v -trimpath -ldflags \"-X zotregistry.dev/zot/pkg/api/config.Commit=v1.4.0-rc3-1-ge583c2f -X zotregistry.dev/zot/pkg/api/config.BinaryType=extended -X zotregistry.dev/zot/pkg/api/config.GoVersion=go1.17.7 -s -w\" ./cmd/zot\r\n"] [38.84018, "o", "\u001b[32m$ \u001b[39m"] [40.470596, "i", "l"] [40.471211, "o", "l"] diff --git a/examples/README.md b/examples/README.md index a5f2f7337..e4cbd3ce9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -952,7 +952,7 @@ Configure each registry sync: "onDemand": false, # pull any image which the local registry doesn't have "pollInterval": "6h", # polling interval, if not set then periodically polling will not run "tlsVerify": true, # whether or not to verify tls (default is true) - "certDir": "/home/user/certs", # use certificates at certDir path, if not specified then use the default certs dir + "certDir": "/home/user/certs", # use certificates at certDir path similar to Docker's /etc/docker/certs.d., if not specified then use the default certs dir, "maxRetries": 5, # maxRetries in case of temporary errors (default: no retries) "retryDelay": "10m", # delay between retries, retry options are applied for both on demand and periodically sync and retryDelay is mandatory when using maxRetries. "onlySigned": true, # sync only signed images (either notary or cosign) @@ -1006,5 +1006,16 @@ Configure each registry sync: ] } ``` - Prefixes can be strings that exactly match repositories or they can be [glob](https://en.wikipedia.org/wiki/Glob_(programming)) patterns. + +### Sync's certDir option + +sync uses the same logic for reading cert directory as docker: https://docs.docker.com/engine/security/certificates/#understand-the-configuration +sync can also read the certificates directly under certDir: + - ca.crt - public pem cert of registry. Root CA that signed the registry certificate, in PEM. + - client.cert - public pem cert for client (mTLS) + - client.key - public key cert for client (mTLS) + +### Sync's credentials + +Besides sync-auth.json file, zot also reads ahd uses docker credentials by default: https://docs.docker.com/reference/cli/docker/login/#description diff --git a/examples/config-sync.json b/examples/config-sync.json index 2ce673dfd..67d2ac3c7 100644 --- a/examples/config-sync.json +++ b/examples/config-sync.json @@ -73,10 +73,25 @@ ], "onDemand": true, "tlsVerify": true, - "maxRetries": 6, - "retryDelay": "5m" + "maxRetries": 5, + "retryDelay": "30s" + }, + { + "urls": [ + "https://demo.goharbor.io" + ], + "pollInterval": "12h", + "content": [ + { + "prefix": "zot/**" + } + ], + "onDemand": true, + "tlsVerify": true, + "maxRetries": 5, + "retryDelay": "1m" } ] } } -} +} \ No newline at end of file diff --git a/go.mod b/go.mod index 8c83721be..d710e705a 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/aquasecurity/trivy v0.58.2 github.com/aquasecurity/trivy-db v0.0.0-20241209111357-8c398f13db0e github.com/aws/aws-sdk-go v1.55.6 - github.com/aws/aws-sdk-go-v2 v1.34.0 + github.com/aws/aws-sdk-go-v2 v1.35.0 github.com/aws/aws-sdk-go-v2/config v1.29.1 github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.16.0 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.39.6 @@ -22,7 +22,6 @@ require ( github.com/bmatcuk/doublestar/v4 v4.8.1 github.com/briandowns/spinner v1.23.2 github.com/chartmuseum/auth v0.5.0 - github.com/containers/common v0.61.1 github.com/containers/image/v5 v5.33.1 github.com/dchest/siphash v1.2.3 github.com/didip/tollbooth/v7 v7.0.2 @@ -55,6 +54,7 @@ require ( github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 github.com/redis/go-redis/v9 v9.7.0 + github.com/regclient/regclient v0.8.0 github.com/rs/zerolog v1.33.0 github.com/sigstore/cosign/v2 v2.4.1 github.com/sigstore/sigstore v1.8.12 @@ -130,7 +130,6 @@ require ( github.com/ProtonMail/go-crypto v1.1.3 // indirect github.com/ThalesIgnite/crypto11 v1.2.5 // indirect github.com/VividCortex/ewma v1.2.0 // indirect - github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/agext/levenshtein v1.2.3 // indirect github.com/agnivade/levenshtein v1.2.0 // indirect github.com/alecthomas/chroma v0.10.0 // indirect @@ -161,8 +160,8 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.54 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.30 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.24.16 // indirect github.com/aws/aws-sdk-go-v2/service/ebs v1.25.3 // indirect @@ -170,11 +169,11 @@ require ( github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.25.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.10 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.11 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.68.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.13 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.11 // indirect github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240730143543-a8d7d3c42ca1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect @@ -210,8 +209,6 @@ require ( github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect github.com/containerd/ttrpc v1.2.6 // indirect github.com/containerd/typeurl/v2 v2.2.2 // indirect - github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect - github.com/containers/ocicrypt v1.2.0 // indirect github.com/containers/storage v1.56.1 // indirect github.com/coreos/go-oidc/v3 v3.12.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect @@ -231,6 +228,7 @@ require ( github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/emicklei/proto v1.12.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect @@ -286,6 +284,7 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/licenseclassifier/v2 v2.0.0 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect github.com/google/s2a-go v0.1.8 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/wire v0.6.0 // indirect @@ -319,7 +318,6 @@ require ( github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.17.11 // indirect - github.com/klauspost/pgzip v1.2.6 // indirect github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f // indirect github.com/knqyf263/go-deb-version v0.0.0-20241115132648-6f4aee6ccd23 // indirect github.com/knqyf263/go-rpm-version v0.0.0-20220614171824-631e686d1075 // indirect @@ -347,7 +345,6 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mattn/go-shellwords v1.0.12 // indirect - github.com/mattn/go-sqlite3 v1.14.24 // indirect github.com/microsoft/go-rustaudit v0.0.0-20220808201409-204dfee52032 // indirect github.com/miekg/pkcs11 v1.1.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect @@ -360,7 +357,6 @@ require ( github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/spdystream v0.4.0 // indirect - github.com/moby/sys/capability v0.3.0 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/sys/signal v0.7.1 // indirect @@ -381,6 +377,8 @@ require ( github.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/oleiade/reflections v1.1.0 // indirect + github.com/onsi/ginkgo/v2 v2.21.0 // indirect + github.com/onsi/gomega v1.35.1 // indirect github.com/open-policy-agent/opa v0.70.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opencontainers/runtime-tools v0.9.1-0.20241001195557-6c9570a1678f // indirect @@ -399,7 +397,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/proglottis/gpgme v0.1.3 // indirect github.com/prometheus/common v0.60.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/protocolbuffers/txtpbfmt v0.0.0-20231025115547-084445ff1adf // indirect @@ -435,7 +432,6 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spiffe/go-spiffe/v2 v2.3.0 // indirect - github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/swaggo/files v1.0.1 // indirect @@ -454,7 +450,6 @@ require ( github.com/ulikunitz/xz v0.5.12 // indirect github.com/urfave/cli/v2 v2.27.5 // indirect github.com/vbatts/tar-split v0.11.6 // indirect - github.com/vbauerster/mpb/v8 v8.8.3 // indirect github.com/veraison/go-cose v1.3.0 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect @@ -475,7 +470,6 @@ require ( github.com/zitadel/logging v0.6.1 // indirect github.com/zitadel/schema v1.3.0 // indirect go.mongodb.org/mongo-driver v1.16.0 // indirect - go.mozilla.org/pkcs7 v0.9.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/bridges/prometheus v0.57.0 // indirect diff --git a/go.sum b/go.sum index 3a0f9543c..af1128cf5 100644 --- a/go.sum +++ b/go.sum @@ -313,8 +313,6 @@ github.com/ThalesIgnite/crypto11 v1.2.5 h1:1IiIIEqYmBvUYFeMnHqRft4bwf/O36jryEUpY github.com/ThalesIgnite/crypto11 v1.2.5/go.mod h1:ILDKtnCKiQ7zRoNxcp36Y1ZR8LBPmR2E23+wTQe/MlE= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/agnivade/levenshtein v1.2.0 h1:U9L4IOT0Y3i0TIlUIDJ7rVUziKi/zPbrJGaFrtYH3SY= @@ -425,8 +423,8 @@ github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX github.com/aws/aws-sdk-go v1.47.10/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk= github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/aws/aws-sdk-go-v2 v1.34.0 h1:9iyL+cjifckRGEVpRKZP3eIxVlL06Qk1Tk13vreaVQU= -github.com/aws/aws-sdk-go-v2 v1.34.0/go.mod h1:JgstGg0JjWU1KpVJjD5H0y0yyAIpSdKEq556EI6yOOM= +github.com/aws/aws-sdk-go-v2 v1.35.0 h1:jTPxEJyzjSuuz0wB+302hr8Eu9KUI+Zv8zlujMGJpVI= +github.com/aws/aws-sdk-go-v2 v1.35.0/go.mod h1:JgstGg0JjWU1KpVJjD5H0y0yyAIpSdKEq556EI6yOOM= github.com/aws/aws-sdk-go-v2/config v1.29.1 h1:JZhGawAyZ/EuJeBtbQYnaoftczcb2drR2Iq36Wgz4sQ= github.com/aws/aws-sdk-go-v2/config v1.29.1/go.mod h1:7bR2YD5euaxBhzt2y/oDkt3uNRb6tjFp98GlTFueRwk= github.com/aws/aws-sdk-go-v2/credentials v1.17.54 h1:4UmqeOqJPvdvASZWrKlhzpRahAulBfyTJQUaYy4+hEI= @@ -435,10 +433,10 @@ github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.16.0 h1:bSfq5lT2 github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.16.0/go.mod h1:FBqEl9aG/k3FY7jHAq7CqznoDY4dp6DIm5ktxY4QkDw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 h1:5grmdTdMsovn9kPZPI23Hhvp0ZyNm5cRO+IZFIYiAfw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24/go.mod h1:zqi7TVKTswH3Ozq28PkmBmgzG1tona7mo9G2IJg4Cis= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 h1:Ej0Rf3GMv50Qh4G4852j2djtoDb7AzQ7MuQeFHa3D70= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29/go.mod h1:oeNTC7PwJNoM5AznVr23wxhLnuJv0ZDe5v7w0wqIs9M= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 h1:6e8a71X+9GfghragVevC5bZqvATtc3mAMgxpSNbgzF0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29/go.mod h1:c4jkZiQ+BWpNqq7VtrxjwISrLrt/VvPq3XiopkUIolI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.30 h1:+7AzSGNhHoY53di13lvztf9Dyd/9ofzoYGBllkWp3a0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.30/go.mod h1:Jxd/FrCny99yURiQiMywgXvBhd7tmgdv6KdlUTNzMSo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.30 h1:Ex06eY6I5rO7IX0HalGfa5nGjpBoOsS1Qm3xfjkuszs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.30/go.mod h1:AvyEMA9QcX59kFhVizBpIBpEMThUTXssuJe+emBdcGM= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.39.6 h1:OBoVhuZ7zXKziB4Kyd1lDUzysef2zWY8pC2Doc0zuiQ= @@ -457,20 +455,20 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/C github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.10 h1:dx6ou28o859SdI4UkuH98Awkuwg4RdHawE5s6pYMQiA= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.10/go.mod h1:ilKRWYwq8gS8Wkltnph4MJUTInZefn1C1shAAZchlGg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 h1:TQmKDyETFGiXVhZfQ/I0cCFziqqX58pi4tKJGYGFSz0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9/go.mod h1:HVLPK2iHQBUx7HfZeOQSEu3v2ubZaAY2YPbAm5/WUyY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.11 h1:5JKQ2J3BBW4ovy6A/5Lwx9SpA6IzgH8jB3bquGZ1NUw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.11/go.mod h1:VShCk7rfCzK/b9U1aSkzLwcOoaDlYna16482QqEavis= github.com/aws/aws-sdk-go-v2/service/kms v1.35.7 h1:v0D1LeMkA/X+JHAZWERrr+sUGOt8KrCZKnJA6KszkcE= github.com/aws/aws-sdk-go-v2/service/kms v1.35.7/go.mod h1:K9lwD0Rsx9+NSaJKsdAdlDK4b2G4KKOEve9PzHxPoMI= github.com/aws/aws-sdk-go-v2/service/s3 v1.68.0 h1:bFpcqdwtAEsgpZXvkTxIThFQx/EM0oV6kXmfFIGjxME= github.com/aws/aws-sdk-go-v2/service/s3 v1.68.0/go.mod h1:ralv4XawHjEMaHOWnTFushl0WRqim/gQWesAMF6hTow= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.13 h1:+dFX6kb0ekos09TP4icFIyqq/u3POCQDSrShc9ZkCCI= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.13/go.mod h1:l+Fboycn+g9RMQcYbTfpqF/d3qZn90q5PYmO7Biu+WM= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 h1:kuIyu4fTT38Kj7YCC7ouNbVZSSpqkZ+LzIfhCr6Dg+I= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.11/go.mod h1:Ro744S4fKiCCuZECXgOi760TiYylUM8ZBf6OGiZzJtY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 h1:l+dgv/64iVlQ3WsBbnn+JSbkj01jIi+SM0wYsj3y/hY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10/go.mod h1:Fzsj6lZEb8AkTE5S68OhcbBqeWPsR8RnGuKPr8Todl8= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 h1:BRVDbewN6VZcwr+FBOszDKvYeXY1kJ+GGMCcpghlw0U= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.9/go.mod h1:f6vjfZER1M17Fokn0IzssOTMT2N8ZSq+7jnNF0tArvw= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.13 h1:q4pOAKxypbFoUJzOpgo939bF50qb4DgYshiDfcsdN0M= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.13/go.mod h1:G/0PTg7+vQT42ictQGjJhixzTcVZtHFvrN/OeTXrRfQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.12 h1:4sGSGshSSfO1vrcXruPick3ioSf8nhhD6nuB2ni37P4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.12/go.mod h1:NHpu/pLOelViA4qxkAFH10VLqh+XeLhZfXDaFyMVgSs= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.11 h1:RIXOjp7Dp4siCYJRwBHUcBdVgOWflSJGlq4ZhMI5Ta0= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.11/go.mod h1:ZR17k9bPKPR8u0IkyA6xVsjr56doNQ4ZB1fs7abYBfE= github.com/aws/aws-secretsmanager-caching-go v1.2.0 h1:gUA+CVKvFLj4OUSknhIrnt4dF7Y37+JrChKqfaehJME= github.com/aws/aws-secretsmanager-caching-go v1.2.0/go.mod h1:6t2/zQIsigFMlnpOdGj503Dgaz24tMqIRhass9uoTBo= github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= @@ -586,14 +584,8 @@ github.com/containerd/ttrpc v1.2.6 h1:zG+Kn5EZ6MUYCS1t2Hmt2J4tMVaLSFEJVOraDQwNPC github.com/containerd/ttrpc v1.2.6/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= github.com/containerd/typeurl/v2 v2.2.2 h1:3jN/k2ysKuPCsln5Qv8bzR9cxal8XjkxPogJfSNO31k= github.com/containerd/typeurl/v2 v2.2.2/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk= -github.com/containers/common v0.61.1 h1:jpk385ZFEx3MAX+sjwOoTZElvpgsGi0YJHuRmrhF/j8= -github.com/containers/common v0.61.1/go.mod h1:C+TfkhTV+ADp1Hu+BMIAYPvSFix21swYo9PZuCKoSUM= github.com/containers/image/v5 v5.33.1 h1:nTWKwxAlY0aJrilvvhssqssJVnley6VqxkLiLzTEYIs= github.com/containers/image/v5 v5.33.1/go.mod h1:/FJiLlvVbeBxWNMPVPPIWJxHTAzwBoFvyN0a51zo1CE= -github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= -github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= -github.com/containers/ocicrypt v1.2.0 h1:X14EgRK3xNFvJEfI5O4Qn4T3E25ANudSOZz/sirVuPM= -github.com/containers/ocicrypt v1.2.0/go.mod h1:ZNviigQajtdlxIZGibvblVuIFBKIuUI2M0QM12SD31U= github.com/containers/storage v1.56.1 h1:gDZj/S6Zxus4Xx42X6iNB3ODXuh0qoOdH/BABfrvcKo= github.com/containers/storage v1.56.1/go.mod h1:c6WKowcAlED/DkWGNuL9bvGYqIWCVy7isRMdCSKWNjk= github.com/coreos/go-oidc/v3 v3.12.0 h1:sJk+8G2qq94rDI6ehZ71Bol3oUHy63qNYmkiSjrc/Jo= @@ -1097,8 +1089,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= -github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= -github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f h1:GvCU5GXhHq+7LeOzx/haG7HSIZokl3/0GkoUFzsRJjg= github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f/go.mod h1:q59u9px8b7UTj0nIjEjvmTWekazka6xIt6Uogz5Dm+8= github.com/knqyf263/go-deb-version v0.0.0-20241115132648-6f4aee6ccd23 h1:dWzdsqjh1p2gNtRKqNwuBvKqMNwnLOPLzVZT1n6DK7s= @@ -1207,8 +1197,6 @@ github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkV github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8= github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= -github.com/moby/sys/capability v0.3.0 h1:kEP+y6te0gEXIaeQhIi0s7vKs/w0RPoH1qPa6jROcVg= -github.com/moby/sys/capability v0.3.0/go.mod h1:4g9IK291rVkms3LKCDOoYlnV8xKwoDTpIrNEE35Wq0I= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= @@ -1266,6 +1254,8 @@ github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olareg/olareg v0.1.1 h1:Ui7q93zjcoF+U9U71sgqgZWByDoZOpqHitUXEu2xV+g= +github.com/olareg/olareg v0.1.1/go.mod h1:w8NP4SWrHHtxsFaUiv1lnCnYPm4sN1seCd2h7FK/dc0= github.com/oleiade/reflections v1.1.0 h1:D+I/UsXQB4esMathlt0kkZRJZdUDmhv5zGi/HOwYTWo= github.com/oleiade/reflections v1.1.0/go.mod h1:mCxx0QseeVCHs5Um5HhJeCKVC7AwS8kO67tky4rdisA= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -1336,8 +1326,6 @@ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= -github.com/proglottis/gpgme v0.1.3 h1:Crxx0oz4LKB3QXc5Ea0J19K/3ICfy3ftr5exgUK1AU0= -github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0= github.com/project-zot/mockoidc v0.0.0-20240610203808-d69d9e02020a h1:525aNEKSyDcqJcawiGtA2NPNApJMta8bUe9SoYuhQ+o= github.com/project-zot/mockoidc v0.0.0-20240610203808-d69d9e02020a/go.mod h1:ltIE6ZO/czh/g4xdNQlFGkl7DAfaLLFYmitB4taA5ys= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -1371,6 +1359,8 @@ github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= github.com/redis/rueidis v1.0.19 h1:s65oWtotzlIFN8eMPhyYwxlwLR1lUdhza2KtWprKYSo= github.com/redis/rueidis v1.0.19/go.mod h1:8B+r5wdnjwK3lTFml5VtxjzGOQAC+5UmujoD12pDrEo= +github.com/regclient/regclient v0.8.0 h1:xNAMDlADcyMvFAlGXoqDOxlSUBG4mqWBFgjQqVTP8Og= +github.com/regclient/regclient v0.8.0/go.mod h1:h9+Y6dBvqBkdlrj6EIhbTOv0xUuIFl7CdI1bZvEB42g= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -1486,8 +1476,6 @@ github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/spiffe/go-spiffe/v2 v2.3.0 h1:g2jYNb/PDMB8I7mBGL2Zuq/Ur6hUhoroxGQFyD6tTj8= github.com/spiffe/go-spiffe/v2 v2.3.0/go.mod h1:Oxsaio7DBgSNqhAO9i/9tLClaVlfRok7zvJnTV8ZyIY= -github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 h1:pnnLyeX7o/5aX8qUQ69P/mLojDqwda8hFOCBTmP/6hw= -github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6/go.mod h1:39R/xuhNgVhi+K0/zst4TLrJrVmbm6LVgl4A0+ZFS5M= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -1563,8 +1551,6 @@ github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= github.com/vbatts/tar-split v0.11.6 h1:4SjTW5+PU11n6fZenf2IPoV8/tz3AaYHMWjf23envGs= github.com/vbatts/tar-split v0.11.6/go.mod h1:dqKNtesIOr2j2Qv3W/cHjnvk9I8+G7oAkFDFN6TCBEI= -github.com/vbauerster/mpb/v8 v8.8.3 h1:dTOByGoqwaTJYPubhVz3lO5O6MK553XVgUo33LdnNsQ= -github.com/vbauerster/mpb/v8 v8.8.3/go.mod h1:JfCCrtcMsJwP6ZwMn9e5LMnNyp3TVNpUWWkN+nd4EWk= github.com/vektah/gqlparser/v2 v2.5.21 h1:Zw1rG2dr1pRR4wqwbVq4d6+xk2f4ut/yo+hwr4QjE08= github.com/vektah/gqlparser/v2 v2.5.21/go.mod h1:xMl+ta8a5M1Yo1A1Iwt/k7gSpscwSnHZdw7tfhEGfTM= github.com/veraison/go-cose v1.3.0 h1:2/H5w8kdSpQJyVtIhx8gmwPJ2uSz1PkyWFx0idbd7rk= @@ -1638,8 +1624,6 @@ go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4= go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= -go.mozilla.org/pkcs7 v0.9.0 h1:yM4/HS9dYv7ri2biPtxt8ikvB37a980dg69/pKmS+eI= -go.mozilla.org/pkcs7 v0.9.0/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/pkg/api/controller.go b/pkg/api/controller.go index 056a9bab7..c35d08b56 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -520,5 +520,4 @@ func (c *Controller) StartBackgroundTasks() { type SyncOnDemand interface { SyncImage(ctx context.Context, repo, reference string) error - SyncReference(ctx context.Context, repo string, subjectDigestStr string, referenceType string) error } diff --git a/pkg/api/routes.go b/pkg/api/routes.go index b92c42d56..4e8bc7ed2 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -41,7 +41,6 @@ import ( "zotregistry.dev/zot/pkg/debug/pprof" debug "zotregistry.dev/zot/pkg/debug/swagger" ext "zotregistry.dev/zot/pkg/extensions" - syncConstants "zotregistry.dev/zot/pkg/extensions/sync/constants" "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/meta" mTypes "zotregistry.dev/zot/pkg/meta/types" @@ -569,8 +568,7 @@ func getReferrers(ctx context.Context, routeHandler *RouteHandler, routeHandler.c.Log.Info().Str("repository", name).Str("reference", digest.String()). Msg("referrers not found, trying to get reference by syncing on demand") - if errSync := routeHandler.c.SyncOnDemand.SyncReference(ctx, name, digest.String(), - syncConstants.OCI); errSync != nil { + if errSync := routeHandler.c.SyncOnDemand.SyncImage(ctx, name, digest.String()); errSync != nil { routeHandler.c.Log.Err(errSync).Str("repository", name).Str("reference", digest.String()). Msg("failed to sync OCI reference for image") } diff --git a/pkg/common/common.go b/pkg/common/common.go index cc215d1a9..b97d0b459 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -32,10 +32,20 @@ const ( ArtifactTypeCosign = "application/vnd.dev.cosign.artifact.sig.v1+json" ) -var cosignTagRule = regexp.MustCompile(`sha256\-.+\.sig`) +var cosignSignatureTagRule = regexp.MustCompile(`sha256\-.+\.sig`) + +var cosignSBOMTagRule = regexp.MustCompile(`sha256\-.+\.sbom`) + +func IsCosignSignature(tag string) bool { + return cosignSignatureTagRule.MatchString(tag) +} + +func IsCosignSBOM(tag string) bool { + return cosignSBOMTagRule.MatchString(tag) +} func IsCosignTag(tag string) bool { - return cosignTagRule.MatchString(tag) + return IsCosignSignature(tag) || IsCosignSBOM(tag) } func Contains[T comparable](elems []T, v T) bool { diff --git a/pkg/extensions/extension_sync.go b/pkg/extensions/extension_sync.go index 3e96902d3..5a0d0b4ed 100644 --- a/pkg/extensions/extension_sync.go +++ b/pkg/extensions/extension_sync.go @@ -102,21 +102,6 @@ func getLocalIPs() ([]string, error) { return localIPs, nil } -func getIPFromHostName(host string) ([]string, error) { - addrs, err := net.LookupIP(host) - if err != nil { - return []string{}, err - } - - ips := make([]string, 0, len(addrs)) - - for _, ip := range addrs { - ips = append(ips, ip.String()) - } - - return ips, nil -} - func removeSelfURLs(config *config.Config, registryConfig *syncconf.RegistryConfig, log log.Logger) error { // get IP from config port := config.HTTP.Port @@ -150,7 +135,7 @@ func removeSelfURLs(config *config.Config, registryConfig *syncconf.RegistryConf } // check dns - ips, err := getIPFromHostName(url.Hostname()) + ips, err := net.LookupIP(url.Hostname()) if err != nil { // will not remove, maybe it will get resolved later after multiple retries log.Warn().Str("url", registryURL).Msg("failed to lookup sync registry url's hostname") @@ -163,7 +148,8 @@ func removeSelfURLs(config *config.Config, registryConfig *syncconf.RegistryConf for _, localIP := range localIPs { // if ip resolved from hostname/dns is equal with any local ip for _, ip := range ips { - if net.JoinHostPort(ip, url.Port()) == net.JoinHostPort(localIP, port) { + if (ip.IsLoopback() && (url.Port() == port)) || + (net.JoinHostPort(ip.String(), url.Port()) == net.JoinHostPort(localIP, port)) { registryConfig.URLs = append(registryConfig.URLs[:idx], registryConfig.URLs[idx+1:]...) removed = true diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index 0e3a3f8ce..da5d19302 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -5818,7 +5818,7 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { for _, manifest := range indexContent.Manifests { tag := manifest.Annotations[ispec.AnnotationRefName] - if zcommon.IsCosignTag(tag) { + if zcommon.IsCosignSignature(tag) { signatureTag = tag } } diff --git a/pkg/extensions/sync/constants/consts.go b/pkg/extensions/sync/constants/constants.go similarity index 100% rename from pkg/extensions/sync/constants/consts.go rename to pkg/extensions/sync/constants/constants.go diff --git a/pkg/extensions/sync/content.go b/pkg/extensions/sync/content.go index 124dd5082..1be12124d 100644 --- a/pkg/extensions/sync/content.go +++ b/pkg/extensions/sync/content.go @@ -51,7 +51,7 @@ func (cm ContentManager) MatchesContent(repo string) bool { // FilterTags filters a repo tags based on content config rules (semver, regex). func (cm ContentManager) FilterTags(repo string, tags []string) ([]string, error) { - content := cm.getContentByLocalRepo(repo) + content := cm.GetContentByLocalRepo(repo) var err error // filter based on tags rules @@ -97,7 +97,7 @@ the remote name of a repo given a local repo. - used by on demand sync. */ func (cm ContentManager) GetRepoSource(repo string) string { - content := cm.getContentByLocalRepo(repo) + content := cm.GetContentByLocalRepo(repo) if content == nil { return "" } @@ -133,7 +133,7 @@ func (cm ContentManager) getContentByUpstreamRepo(repo string) *syncconf.Content return nil } -func (cm ContentManager) getContentByLocalRepo(repo string) *syncconf.Content { +func (cm ContentManager) GetContentByLocalRepo(repo string) *syncconf.Content { contentID := -1 repo = strings.Trim(repo, "/") diff --git a/pkg/extensions/sync/content_internal_test.go b/pkg/extensions/sync/content_test.go similarity index 91% rename from pkg/extensions/sync/content_internal_test.go rename to pkg/extensions/sync/content_test.go index eb3961567..f44ced078 100644 --- a/pkg/extensions/sync/content_internal_test.go +++ b/pkg/extensions/sync/content_test.go @@ -1,7 +1,7 @@ //go:build sync // +build sync -package sync +package sync_test //nolint: testpackage import ( "testing" @@ -9,6 +9,7 @@ import ( . "github.com/smartystreets/goconvey/convey" syncconf "zotregistry.dev/zot/pkg/extensions/config/sync" + "zotregistry.dev/zot/pkg/extensions/sync" "zotregistry.dev/zot/pkg/log" ) @@ -67,7 +68,7 @@ func TestContentManager(t *testing.T) { Convey("Test GetRepoDestination()", t, func() { for _, test := range testCases { - cm := NewContentManager([]syncconf.Content{test.content}, log.Logger{}) + cm := sync.NewContentManager([]syncconf.Content{test.content}, log.Logger{}) actualResult := cm.GetRepoDestination(test.expected) So(actualResult, ShouldEqual, test.repo) } @@ -76,7 +77,7 @@ func TestContentManager(t *testing.T) { // this is the inverse function of getRepoDestination() Convey("Test GetRepoSource()", t, func() { for _, test := range testCases { - cm := NewContentManager([]syncconf.Content{test.content}, log.Logger{}) + cm := sync.NewContentManager([]syncconf.Content{test.content}, log.Logger{}) actualResult := cm.GetRepoSource(test.repo) So(actualResult, ShouldEqual, test.expected) } @@ -84,7 +85,7 @@ func TestContentManager(t *testing.T) { Convey("Test MatchesContent() error", t, func() { content := syncconf.Content{Prefix: "[repo%^&"} - cm := NewContentManager([]syncconf.Content{content}, log.Logger{}) + cm := sync.NewContentManager([]syncconf.Content{content}, log.Logger{}) So(cm.MatchesContent("repo"), ShouldEqual, false) }) } @@ -147,8 +148,8 @@ func TestGetContentByLocalRepo(t *testing.T) { Convey("Test getContentByLocalRepo()", t, func() { for _, test := range testCases { - cm := NewContentManager(test.content, log.Logger{}) - actualResult := cm.getContentByLocalRepo(test.repo) + cm := sync.NewContentManager(test.content, log.Logger{}) + actualResult := cm.GetContentByLocalRepo(test.repo) if test.expected == -1 { var tnil *syncconf.Content = nil @@ -162,8 +163,8 @@ func TestGetContentByLocalRepo(t *testing.T) { Convey("Test getContentByLocalRepo() error", t, func() { content := syncconf.Content{Prefix: "[repo%^&"} - cm := NewContentManager([]syncconf.Content{content}, log.Logger{}) - So(cm.getContentByLocalRepo("repo"), ShouldBeNil) + cm := sync.NewContentManager([]syncconf.Content{content}, log.Logger{}) + So(cm.GetContentByLocalRepo("repo"), ShouldBeNil) }) } @@ -266,7 +267,7 @@ func TestFilterTags(t *testing.T) { Convey("Test FilterTags()", t, func() { for _, test := range testCases { - cm := NewContentManager(test.content, log.NewLogger("debug", "")) + cm := sync.NewContentManager(test.content, log.NewLogger("debug", "")) actualResult, err := cm.FilterTags(test.repo, test.tags) So(actualResult, ShouldResemble, test.filteredTags) diff --git a/pkg/extensions/sync/destination.go b/pkg/extensions/sync/destination.go index b93eec625..dcf58a192 100644 --- a/pkg/extensions/sync/destination.go +++ b/pkg/extensions/sync/destination.go @@ -13,9 +13,9 @@ import ( "strings" "time" - "github.com/containers/image/v5/types" - "github.com/opencontainers/go-digest" + godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/regclient/regclient/types/ref" zerr "zotregistry.dev/zot/errors" "zotregistry.dev/zot/pkg/common" @@ -52,7 +52,8 @@ func NewDestinationRegistry( } } -func (registry *DestinationRegistry) CanSkipImage(repo, tag string, imageDigest digest.Digest) (bool, error) { +// Check if image is already synced. +func (registry *DestinationRegistry) CanSkipImage(repo, tag string, digest godigest.Digest) (bool, error) { // check image already synced imageStore := registry.storeController.GetImageStore(repo) @@ -68,10 +69,10 @@ func (registry *DestinationRegistry) CanSkipImage(repo, tag string, imageDigest return false, err } - if localImageManifestDigest != imageDigest { + if localImageManifestDigest != digest { registry.log.Info().Str("repo", repo).Str("reference", tag). Str("localDigest", localImageManifestDigest.String()). - Str("remoteDigest", imageDigest.String()). + Str("remoteDigest", digest.String()). Msg("remote image digest changed, syncing again") return false, nil @@ -80,116 +81,132 @@ func (registry *DestinationRegistry) CanSkipImage(repo, tag string, imageDigest return true, nil } -func (registry *DestinationRegistry) GetContext() *types.SystemContext { - return registry.tempStorage.GetContext() -} - -func (registry *DestinationRegistry) GetImageReference(repo, reference string) (types.ImageReference, error) { +func (registry *DestinationRegistry) GetImageReference(repo, reference string) (ref.Ref, error) { return registry.tempStorage.GetImageReference(repo, reference) } // finalize a syncing image. -func (registry *DestinationRegistry) CommitImage(imageReference types.ImageReference, repo, reference string) error { +func (registry *DestinationRegistry) CommitAll(repo string, imageReference ref.Ref) error { imageStore := registry.storeController.GetImageStore(repo) - tempImageStore := getImageStoreFromImageReference(imageReference, repo, reference, registry.log) + tempImageStore := getImageStoreFromImageReference(repo, imageReference, registry.log) defer os.RemoveAll(tempImageStore.RootDir()) - registry.log.Info().Str("syncTempDir", path.Join(tempImageStore.RootDir(), repo)).Str("reference", reference). + registry.log.Info().Str("syncTempDir", path.Join(tempImageStore.RootDir(), repo)).Str("repository", repo). Msg("pushing synced local image to local registry") var lockLatency time.Time - manifestBlob, manifestDigest, mediaType, err := tempImageStore.GetImageManifest(repo, reference) + index, err := storageCommon.GetIndex(tempImageStore, repo, registry.log) if err != nil { registry.log.Error().Str("errorType", common.TypeOf(err)). - Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("repo", repo).Str("reference", reference). - Msg("couldn't find synced manifest in temporary sync dir") + Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("repo", repo). + Msg("failed to get repo index from temp sync dir") return err } - // is image manifest - switch mediaType { - case ispec.MediaTypeImageManifest: - if err := registry.copyManifest(repo, manifestBlob, reference, tempImageStore); err != nil { - if errors.Is(err, zerr.ErrImageLintAnnotations) { - registry.log.Error().Str("errorType", common.TypeOf(err)). - Err(err).Msg("couldn't upload manifest because of missing annotations") + for _, desc := range index.Manifests { + reference := GetDescriptorReference(desc) - return nil - } + manifestBlob, manifestDigest, mediaType, err := tempImageStore.GetImageManifest(repo, reference) + if err != nil { + registry.log.Error().Str("errorType", common.TypeOf(err)). + Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("repo", repo).Str("reference", reference). + Msg("failed to get manifest from temporary sync dir") return err } - case ispec.MediaTypeImageIndex: - // is image index - var indexManifest ispec.Index - if err := json.Unmarshal(manifestBlob, &indexManifest); err != nil { - registry.log.Error().Str("errorType", common.TypeOf(err)). - Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)). - Msg("invalid JSON") + // is image manifest + switch mediaType { + case ispec.MediaTypeImageManifest: + if err := registry.copyManifest(repo, manifestBlob, reference, tempImageStore); err != nil { + if errors.Is(err, zerr.ErrImageLintAnnotations) { + registry.log.Error().Str("errorType", common.TypeOf(err)). + Err(err).Msg("failed to upload manifest because of missing annotations") - return err - } + return nil + } - for _, manifest := range indexManifest.Manifests { - tempImageStore.RLock(&lockLatency) - manifestBuf, err := tempImageStore.GetBlobContent(repo, manifest.Digest) - tempImageStore.RUnlock(&lockLatency) + return err + } + case ispec.MediaTypeImageIndex: + // is image index + var indexManifest ispec.Index - if err != nil { + if err := json.Unmarshal(manifestBlob, &indexManifest); err != nil { registry.log.Error().Str("errorType", common.TypeOf(err)). - Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("digest", manifest.Digest.String()). - Msg("couldn't find manifest which is part of an image index") + Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)). + Msg("invalid JSON") return err } - if err := registry.copyManifest(repo, manifestBuf, manifest.Digest.String(), - tempImageStore); err != nil { - if errors.Is(err, zerr.ErrImageLintAnnotations) { + for _, manifest := range indexManifest.Manifests { + tempImageStore.RLock(&lockLatency) + manifestBuf, err := tempImageStore.GetBlobContent(repo, manifest.Digest) + tempImageStore.RUnlock(&lockLatency) + + if err != nil { registry.log.Error().Str("errorType", common.TypeOf(err)). - Err(err).Msg("couldn't upload manifest because of missing annotations") + Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("digest", manifest.Digest.String()). + Msg("failed find manifest which is part of an image index") - return nil + return err } - return err - } - } + if err := registry.copyManifest(repo, manifestBuf, manifest.Digest.String(), + tempImageStore); err != nil { + if errors.Is(err, zerr.ErrImageLintAnnotations) { + registry.log.Error().Str("errorType", common.TypeOf(err)). + Err(err).Msg("failed to upload manifest because of missing annotations") - _, _, err = imageStore.PutImageManifest(repo, reference, mediaType, manifestBlob) - if err != nil { - registry.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo).Str("reference", reference). - Err(err).Msg("couldn't upload manifest") + return nil + } - return err - } + return err + } + } - if registry.metaDB != nil { - err = meta.SetImageMetaFromInput(context.Background(), repo, reference, mediaType, - manifestDigest, manifestBlob, imageStore, registry.metaDB, registry.log) + _, _, err = imageStore.PutImageManifest(repo, reference, mediaType, manifestBlob) if err != nil { - return fmt.Errorf("failed to set metadata for image '%s %s': %w", repo, reference, err) + registry.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo).Str("reference", reference). + Err(err).Msg("failed to upload manifest") + + return err } - registry.log.Debug().Str("repo", repo).Str("reference", reference).Str("component", "metadb"). - Msg("successfully set metadata for image") + if registry.metaDB != nil { + err = meta.SetImageMetaFromInput(context.Background(), repo, reference, mediaType, + manifestDigest, manifestBlob, imageStore, registry.metaDB, registry.log) + if err != nil { + return fmt.Errorf("metaDB: failed to set metadata for image '%s %s': %w", repo, reference, err) + } + + registry.log.Debug().Str("repo", repo).Str("reference", reference). + Msg("metaDB: successfully set metadata for image") + } } } - registry.log.Info().Str("image", fmt.Sprintf("%s:%s", repo, reference)).Msg("successfully synced image") - return nil } -func (registry *DestinationRegistry) CleanupImage(imageReference types.ImageReference, repo, reference string) error { - tmpDir := getTempRootDirFromImageReference(imageReference, repo, reference) +func (registry *DestinationRegistry) CleanupImage(imageReference ref.Ref, repo string) error { + var err error + + dir := strings.TrimSuffix(imageReference.Path, repo) + if _, err = os.Stat(dir); err == nil { + if err := os.RemoveAll(strings.TrimSuffix(imageReference.Path, repo)); err != nil { + registry.log.Error().Err(err).Msg("failed to cleanup image from temp storage") + + return err + } + } - return os.RemoveAll(tmpDir) + return nil } func (registry *DestinationRegistry) copyManifest(repo string, manifestContent []byte, reference string, @@ -251,7 +268,7 @@ func (registry *DestinationRegistry) copyManifest(repo string, manifestContent [ } // Copy a blob from one image store to another image store. -func (registry *DestinationRegistry) copyBlob(repo string, blobDigest digest.Digest, blobMediaType string, +func (registry *DestinationRegistry) copyBlob(repo string, blobDigest godigest.Digest, blobMediaType string, tempImageStore storageTypes.ImageStore, ) error { imageStore := registry.storeController.GetImageStore(repo) @@ -282,23 +299,10 @@ func (registry *DestinationRegistry) copyBlob(repo string, blobDigest digest.Dig } // use only with local imageReferences. -func getImageStoreFromImageReference(imageReference types.ImageReference, repo, reference string, log log.Logger, -) storageTypes.ImageStore { - tmpRootDir := getTempRootDirFromImageReference(imageReference, repo, reference) - - return getImageStore(tmpRootDir, log) -} - -func getTempRootDirFromImageReference(imageReference types.ImageReference, repo, reference string) string { - var tmpRootDir string - - if strings.HasSuffix(imageReference.StringWithinTransport(), reference) { - tmpRootDir = strings.ReplaceAll(imageReference.StringWithinTransport(), fmt.Sprintf("%s:%s", repo, reference), "") - } else { - tmpRootDir = strings.ReplaceAll(imageReference.StringWithinTransport(), repo+":", "") - } +func getImageStoreFromImageReference(repo string, imageReference ref.Ref, log log.Logger) storageTypes.ImageStore { + sessionRootDir := strings.TrimSuffix(imageReference.Path, repo) - return tmpRootDir + return getImageStore(sessionRootDir, log) } func getImageStore(rootDir string, log log.Logger) storageTypes.ImageStore { diff --git a/pkg/extensions/sync/features/features.go b/pkg/extensions/sync/features/features.go deleted file mode 100644 index 0dd88a479..000000000 --- a/pkg/extensions/sync/features/features.go +++ /dev/null @@ -1,53 +0,0 @@ -package features - -import ( - "sync" - "time" -) - -const defaultExpireMinutes = 10 - -type featureKey struct { - kind string - repo string -} - -type featureVal struct { - enabled bool - expire time.Time -} - -type Map struct { - store map[featureKey]*featureVal - expireAfter time.Duration - mu *sync.Mutex -} - -func New() *Map { - return &Map{ - store: make(map[featureKey]*featureVal), - expireAfter: defaultExpireMinutes * time.Minute, - mu: new(sync.Mutex), - } -} - -// returns if registry supports this feature and if ok. -func (f *Map) Get(kind, repo string) (bool, bool) { - f.mu.Lock() - defer f.mu.Unlock() - - if feature, ok := f.store[featureKey{kind, repo}]; ok { - if time.Now().Before(feature.expire) { - return feature.enabled, true - } - } - - // feature expired or not found - return false, false -} - -func (f *Map) Set(kind, repo string, enabled bool) { - f.mu.Lock() - f.store[featureKey{kind: kind, repo: repo}] = &featureVal{enabled: enabled, expire: time.Now().Add(f.expireAfter)} - f.mu.Unlock() -} diff --git a/pkg/extensions/sync/httpclient/cache.go b/pkg/extensions/sync/httpclient/cache.go deleted file mode 100644 index e0aa56dee..000000000 --- a/pkg/extensions/sync/httpclient/cache.go +++ /dev/null @@ -1,58 +0,0 @@ -package client - -import ( - "sync" -) - -// Key:Value store for bearer tokens, key is namespace, value is token. -// We are storing only pull scoped tokens, the http client is for pulling only. -type TokenCache struct { - entries sync.Map -} - -func NewTokenCache() *TokenCache { - return &TokenCache{ - entries: sync.Map{}, - } -} - -func (c *TokenCache) Set(namespace string, token *bearerToken) { - if c == nil || token == nil { - return - } - - defer c.prune() - - c.entries.Store(namespace, token) -} - -func (c *TokenCache) Get(namespace string) *bearerToken { - if c == nil { - return nil - } - - val, ok := c.entries.Load(namespace) - if !ok { - return nil - } - - bearerToken, ok := val.(*bearerToken) - if !ok { - return nil - } - - return bearerToken -} - -func (c *TokenCache) prune() { - c.entries.Range(func(key, val any) bool { - bearerToken, ok := val.(*bearerToken) - if ok { - if bearerToken.isExpired() { - c.entries.Delete(key) - } - } - - return true - }) -} diff --git a/pkg/extensions/sync/httpclient/client.go b/pkg/extensions/sync/httpclient/client.go deleted file mode 100644 index be0e30c53..000000000 --- a/pkg/extensions/sync/httpclient/client.go +++ /dev/null @@ -1,507 +0,0 @@ -package client - -import ( - "context" - "encoding/json" - "errors" - "io" - "net/http" - "net/url" - "path/filepath" - "strings" - "sync" - "time" - - zerr "zotregistry.dev/zot/errors" - "zotregistry.dev/zot/pkg/common" - "zotregistry.dev/zot/pkg/log" -) - -const ( - minimumTokenLifetimeSeconds = 60 // in seconds - pingTimeout = 5 * time.Second - // tokenBuffer is used to renew a token before it actually expires - // to account for the time to process requests on the server. - tokenBuffer = 5 * time.Second -) - -type authType int - -const ( - noneAuth authType = iota - basicAuth - tokenAuth -) - -type challengeParams struct { - realm string - service string - scope string - err string -} - -type bearerToken struct { - Token string `json:"token"` //nolint: tagliatelle - AccessToken string `json:"access_token"` //nolint: tagliatelle - ExpiresIn int `json:"expires_in"` //nolint: tagliatelle - IssuedAt time.Time `json:"issued_at"` //nolint: tagliatelle - expirationTime time.Time -} - -func (token *bearerToken) isExpired() bool { - // use tokenBuffer to expire it a bit earlier - return time.Now().After(token.expirationTime.Add(-1 * tokenBuffer)) -} - -type Config struct { - URL string - Username string - Password string - CertDir string - TLSVerify bool -} - -type Client struct { - config *Config - client *http.Client - url *url.URL - authType authType - cache *TokenCache - lock *sync.RWMutex - log log.Logger -} - -func New(config Config, log log.Logger) (*Client, error) { - client := &Client{log: log, lock: new(sync.RWMutex)} - - client.cache = NewTokenCache() - - if err := client.SetConfig(config); err != nil { - return nil, err - } - - return client, nil -} - -func (httpClient *Client) GetConfig() *Config { - httpClient.lock.RLock() - defer httpClient.lock.RUnlock() - - return httpClient.config -} - -func (httpClient *Client) GetHostname() string { - httpClient.lock.RLock() - defer httpClient.lock.RUnlock() - - return httpClient.url.Host -} - -func (httpClient *Client) GetBaseURL() string { - httpClient.lock.RLock() - defer httpClient.lock.RUnlock() - - return httpClient.url.String() -} - -func (httpClient *Client) SetConfig(config Config) error { - httpClient.lock.Lock() - defer httpClient.lock.Unlock() - - clientURL, err := url.Parse(config.URL) - if err != nil { - return err - } - - httpClient.url = clientURL - - // we want TLS enabled if the upstream registry URL is an HTTPS URL - tlsEnabled := clientURL.Scheme == "https" - - clientOpts := common.HTTPClientOptions{ - TLSEnabled: tlsEnabled, - VerifyTLS: config.TLSVerify, - Host: clientURL.Host, - } - - if config.CertDir != "" { - // only configure the default cert file names if the CertDir was specified. - clientOpts.CertOptions = common.HTTPClientCertOptions{ - // filepath is the recommended library to use for joining paths - // taking into account the underlying OS. - // ref: https://stackoverflow.com/a/39182128 - ClientCertFile: filepath.Join(config.CertDir, common.ClientCertFilename), - ClientKeyFile: filepath.Join(config.CertDir, common.ClientKeyFilename), - RootCaCertFile: filepath.Join(config.CertDir, common.CaCertFilename), - } - } - - client, err := common.CreateHTTPClient(&clientOpts) - if err != nil { - return err - } - - httpClient.client = client - httpClient.config = &config - - return nil -} - -func (httpClient *Client) Ping() bool { - httpClient.lock.Lock() - defer httpClient.lock.Unlock() - - pingURL := *httpClient.url - - pingURL = *pingURL.JoinPath("/v2/") - - // for the ping function we want to timeout fast - ctx, cancel := context.WithTimeout(context.Background(), pingTimeout) - defer cancel() - - //nolint: bodyclose - resp, _, err := httpClient.get(ctx, pingURL.String(), "", false) - if err != nil { - return false - } - - httpClient.authType = getAuthType(resp) - - if resp.StatusCode >= http.StatusOK && resp.StatusCode <= http.StatusForbidden { - return true - } - - httpClient.log.Error().Str("url", pingURL.String()).Int("statusCode", resp.StatusCode). - Str("component", "sync").Msg("failed to ping registry") - - return false -} - -func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interface{}, mediaType string, rawQuery string, - route ...string, -) ([]byte, http.Header, int, error) { - httpClient.lock.RLock() - defer httpClient.lock.RUnlock() - - var namespace string - - url := *httpClient.url - for idx, path := range route { - url = *url.JoinPath(path) - - // we know that the second route argument is always the repo name. - // need it for caching tokens, it's not used in requests made to authz server. - if idx == 1 { - namespace = strings.Trim(path, "/") - } - } - - url.RawQuery = rawQuery - - //nolint: bodyclose,contextcheck - resp, body, err := httpClient.makeAndDoRequest(http.MethodGet, mediaType, namespace, url.String()) - if err != nil { - httpClient.log.Error().Err(err).Str("url", url.String()).Str("component", "sync"). - Str("errorType", common.TypeOf(err)). - Msg("failed to make request") - - return nil, nil, -1, err - } - - if resp.StatusCode != http.StatusOK { - return nil, nil, resp.StatusCode, errors.New(string(body)) //nolint:goerr113 - } - - // read blob - if len(body) > 0 { - err = json.Unmarshal(body, &resultPtr) - } - - return body, resp.Header, resp.StatusCode, err -} - -func (httpClient *Client) setupAuth(req *http.Request, namespace string) error { - if httpClient.authType == tokenAuth { - token, err := httpClient.getToken(req.URL.String(), namespace) - if err != nil { - httpClient.log.Error().Err(err).Str("url", req.URL.String()).Str("component", "sync"). - Str("errorType", common.TypeOf(err)). - Msg("failed to get token from authorization realm") - - return err - } - - req.Header.Set("Authorization", "Bearer "+token.Token) - } else if httpClient.authType == basicAuth { - req.SetBasicAuth(httpClient.config.Username, httpClient.config.Password) - } - - return nil -} - -func (httpClient *Client) get(ctx context.Context, url string, mediaType string, - setBasicAuth bool, -) (*http.Response, []byte, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) //nolint - if err != nil { - return nil, nil, err - } - - if mediaType != "" { - req.Header.Set("Accept", mediaType) - } - - if setBasicAuth && httpClient.config.Username != "" && httpClient.config.Password != "" { - req.SetBasicAuth(httpClient.config.Username, httpClient.config.Password) - } - - return httpClient.doRequest(req) -} - -func (httpClient *Client) doRequest(req *http.Request) (*http.Response, []byte, error) { - resp, err := httpClient.client.Do(req) - if err != nil { - httpClient.log.Error().Err(err).Str("url", req.URL.String()).Str("component", "sync"). - Str("errorType", common.TypeOf(err)). - Msg("failed to make request") - - return nil, nil, err - } - - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - httpClient.log.Error().Err(err).Str("url", req.URL.String()). - Str("errorType", common.TypeOf(err)). - Msg("failed to read body") - - return nil, nil, err - } - - return resp, body, nil -} - -func (httpClient *Client) makeAndDoRequest(method, mediaType, namespace, urlStr string, -) (*http.Response, []byte, error) { - req, err := http.NewRequest(method, urlStr, nil) //nolint - if err != nil { - return nil, nil, err - } - - err = httpClient.setupAuth(req, namespace) - if err != nil { - // harbor catalog requests return basicAuth by default, even if bearer is used on the rest of endpoints. - if errors.Is(err, zerr.ErrReceivedUnexpectedAuthHeader) { - httpClient.log.Err(err).Msg("expected bearer auth header, received basic, retrying with basic auth...") - - // try with basic auth - return httpClient.get(context.Background(), urlStr, mediaType, true) - } - - return nil, nil, err - } - - if mediaType != "" { - req.Header.Set("Accept", mediaType) - } - - resp, body, err := httpClient.doRequest(req) - if err != nil { - return nil, nil, err - } - - if httpClient.authType == tokenAuth { - // let's retry one time if we get an insufficient_scope error - if ok, challengeParams := needsRetryWithUpdatedScope(err, resp); ok { - var tokenURL *url.URL - - var token *bearerToken - - tokenURL, err = getTokenURLFromChallengeParams(challengeParams, httpClient.config.Username) - if err != nil { - return nil, nil, err - } - - token, err = httpClient.getTokenFromURL(tokenURL.String(), namespace) - if err != nil { - return nil, nil, err - } - - req.Header.Set("Authorization", "Bearer "+token.Token) - - resp, body, err = httpClient.doRequest(req) - } - } - - return resp, body, err -} - -func (httpClient *Client) getTokenFromURL(urlStr, namespace string) (*bearerToken, error) { - //nolint: bodyclose - resp, body, err := httpClient.get(context.Background(), urlStr, "", true) - if err != nil { - return nil, err - } - - if resp.StatusCode != http.StatusOK { - return nil, zerr.ErrUnauthorizedAccess - } - - token, err := newBearerToken(body) - if err != nil { - return nil, err - } - - // cache it - httpClient.cache.Set(namespace, token) - - return token, nil -} - -// Gets bearer token from Authorization realm. -func (httpClient *Client) getToken(urlStr, namespace string) (*bearerToken, error) { - // first check cache - token := httpClient.cache.Get(namespace) - if token != nil && !token.isExpired() { - return token, nil - } - - //nolint: bodyclose - resp, _, err := httpClient.get(context.Background(), urlStr, "", false) - if err != nil { - return nil, err - } - - challengeParams, err := parseBearerAuthHeader(resp) - if err != nil { - return nil, err - } - - tokenURL, err := getTokenURLFromChallengeParams(challengeParams, httpClient.config.Username) - if err != nil { - return nil, err - } - - return httpClient.getTokenFromURL(tokenURL.String(), namespace) -} - -func getAuthType(resp *http.Response) authType { - authHeader := resp.Header.Get("www-authenticate") - - authHeaderLower := strings.ToLower(authHeader) - - //nolint: gocritic - if strings.Contains(authHeaderLower, "bearer") { - return tokenAuth - } else if strings.Contains(authHeaderLower, "basic") { - return basicAuth - } else { - return noneAuth - } -} - -func newBearerToken(blob []byte) (*bearerToken, error) { - token := new(bearerToken) - if err := json.Unmarshal(blob, &token); err != nil { - return nil, err - } - - if token.Token == "" { - token.Token = token.AccessToken - } - - if token.ExpiresIn < minimumTokenLifetimeSeconds { - token.ExpiresIn = minimumTokenLifetimeSeconds - } - - if token.IssuedAt.IsZero() { - token.IssuedAt = time.Now().UTC() - } - - token.expirationTime = token.IssuedAt.Add(time.Duration(token.ExpiresIn) * time.Second) - - return token, nil -} - -func getTokenURLFromChallengeParams(params challengeParams, account string) (*url.URL, error) { - parsedRealm, err := url.Parse(params.realm) - if err != nil { - return nil, err - } - - query := parsedRealm.Query() - query.Set("service", params.service) - query.Set("scope", params.scope) - - if account != "" { - query.Set("account", account) - } - - parsedRealm.RawQuery = query.Encode() - - return parsedRealm, nil -} - -func parseBearerAuthHeader(resp *http.Response) (challengeParams, error) { - authHeader := resp.Header.Get("www-authenticate") - - authHeaderSlice := strings.Split(authHeader, ",") - - params := challengeParams{} - - for _, elem := range authHeaderSlice { - // expected bearer auth header - if strings.Contains(strings.ToLower(elem), "basic") { - return params, zerr.ErrReceivedUnexpectedAuthHeader - } - - if strings.Contains(strings.ToLower(elem), "bearer") { - elem = strings.Split(elem, " ")[1] - } - - elem := strings.ReplaceAll(elem, "\"", "") - - elemSplit := strings.Split(elem, "=") - if len(elemSplit) != 2 { //nolint:mnd - return params, zerr.ErrParsingAuthHeader - } - - authKey := elemSplit[0] - - authValue := elemSplit[1] - - switch authKey { - case "realm": - params.realm = authValue - case "service": - params.service = authValue - case "scope": - params.scope = authValue - case "error": - params.err = authValue - } - } - - return params, nil -} - -// Checks if the auth headers in the response contain an indication of a failed -// authorization because of an "insufficient_scope" error. -func needsRetryWithUpdatedScope(err error, resp *http.Response) (bool, challengeParams) { - params := challengeParams{} - if err == nil && resp.StatusCode == http.StatusUnauthorized { - params, err = parseBearerAuthHeader(resp) - if err != nil { - return false, params - } - - if params.err == "insufficient_scope" { - if params.scope != "" { - return true, params - } - } - } - - return false, params -} diff --git a/pkg/extensions/sync/httpclient/client_internal_test.go b/pkg/extensions/sync/httpclient/client_internal_test.go deleted file mode 100644 index 48ec04da5..000000000 --- a/pkg/extensions/sync/httpclient/client_internal_test.go +++ /dev/null @@ -1,167 +0,0 @@ -package client - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - . "github.com/smartystreets/goconvey/convey" - - "zotregistry.dev/zot/pkg/log" -) - -func TestTokenCache(t *testing.T) { - Convey("Get/Set tokens", t, func() { - tokenCache := NewTokenCache() - token := &bearerToken{ - Token: "tokenA", - ExpiresIn: 3, - IssuedAt: time.Now(), - } - - token.expirationTime = token.IssuedAt.Add(time.Duration(token.ExpiresIn) * time.Second).Add(tokenBuffer) - - tokenCache.Set("repo", token) - cachedToken := tokenCache.Get("repo") - So(cachedToken.Token, ShouldEqual, token.Token) - - // add token which expires soon - token2 := &bearerToken{ - Token: "tokenB", - ExpiresIn: 1, - IssuedAt: time.Now(), - } - - token2.expirationTime = token2.IssuedAt.Add(time.Duration(token2.ExpiresIn) * time.Second).Add(tokenBuffer) - - tokenCache.Set("repo2", token2) - cachedToken = tokenCache.Get("repo2") - So(cachedToken.Token, ShouldEqual, token2.Token) - - time.Sleep(1 * time.Second) - - // token3 should be expired when adding a new one - token3 := &bearerToken{ - Token: "tokenC", - ExpiresIn: 3, - IssuedAt: time.Now(), - } - - token3.expirationTime = token3.IssuedAt.Add(time.Duration(token3.ExpiresIn) * time.Second).Add(tokenBuffer) - - tokenCache.Set("repo3", token3) - cachedToken = tokenCache.Get("repo3") - So(cachedToken.Token, ShouldEqual, token3.Token) - - // token2 should be expired - token = tokenCache.Get("repo2") - So(token, ShouldBeNil) - - time.Sleep(2 * time.Second) - - // the rest of them should also be expired - tokenCache.Set("repo4", &bearerToken{ - Token: "tokenD", - }) - - // token1 should be expired - token = tokenCache.Get("repo1") - So(token, ShouldBeNil) - }) - - Convey("Error paths", t, func() { - tokenCache := NewTokenCache() - token := tokenCache.Get("repo") - So(token, ShouldBeNil) - - tokenCache = nil - token = tokenCache.Get("repo") - So(token, ShouldBeNil) - - tokenCache = NewTokenCache() - tokenCache.Set("repo", nil) - token = tokenCache.Get("repo") - So(token, ShouldBeNil) - }) -} - -func TestNeedsRetryOnInsuficientScope(t *testing.T) { - resp := http.Response{ - Status: "401 Unauthorized", - StatusCode: http.StatusUnauthorized, - Proto: "HTTP/1.1", - ProtoMajor: 1, - ProtoMinor: 1, - Header: map[string][]string{ - "Content-Length": {"145"}, - "Content-Type": {"application/json"}, - "Date": {"Fri, 26 Aug 2022 08:03:13 GMT"}, - "X-Content-Type-Options": {"nosniff"}, - }, - Request: nil, - } - - Convey("Test client retries on insufficient scope", t, func() { - resp.Header["Www-Authenticate"] = []string{ - `Bearer realm="https://registry.suse.com/auth",service="SUSE Linux Docker Registry"` + - `,scope="registry:catalog:*",error="insufficient_scope"`, - } - - expectedScope := "registry:catalog:*" - expectedRealm := "https://registry.suse.com/auth" - expectedService := "SUSE Linux Docker Registry" - - needsRetry, params := needsRetryWithUpdatedScope(nil, &resp) - - So(needsRetry, ShouldBeTrue) - So(params.scope, ShouldEqual, expectedScope) - So(params.realm, ShouldEqual, expectedRealm) - So(params.service, ShouldEqual, expectedService) - }) - - Convey("Test client fails on insufficient scope", t, func() { - resp.Header["Www-Authenticate"] = []string{ - `Bearer realm="https://registry.suse.com/auth=error"`, - } - - needsRetry, _ := needsRetryWithUpdatedScope(nil, &resp) - So(needsRetry, ShouldBeFalse) - }) -} - -func TestClient(t *testing.T) { - Convey("Test client", t, func() { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - defer server.Close() - - client, err := New(Config{ - URL: server.URL, - TLSVerify: false, - }, log.NewLogger("", "")) - So(err, ShouldBeNil) - - Convey("Test Ping() fails", func() { - ok := client.Ping() - So(ok, ShouldBeFalse) - }) - - Convey("Test makeAndDoRequest() fails", func() { - client.authType = tokenAuth - //nolint: bodyclose - _, _, err := client.makeAndDoRequest(http.MethodGet, "application/json", "catalog", server.URL) - So(err, ShouldNotBeNil) - }) - - Convey("Test setupAuth() fails", func() { - request, err := http.NewRequest(http.MethodGet, server.URL, nil) //nolint: noctx - So(err, ShouldBeNil) - - client.authType = tokenAuth - err = client.setupAuth(request, "catalog") - So(err, ShouldNotBeNil) - }) - }) -} diff --git a/pkg/extensions/sync/oci_layout.go b/pkg/extensions/sync/oci_layout.go index c44a955c5..0e4850b36 100644 --- a/pkg/extensions/sync/oci_layout.go +++ b/pkg/extensions/sync/oci_layout.go @@ -5,71 +5,59 @@ package sync import ( "fmt" - "os" "path" - "github.com/containers/image/v5/oci/layout" - "github.com/containers/image/v5/types" "github.com/gofrs/uuid" + "github.com/regclient/regclient/types/ref" zerr "zotregistry.dev/zot/errors" "zotregistry.dev/zot/pkg/extensions/sync/constants" "zotregistry.dev/zot/pkg/storage" - storageConstants "zotregistry.dev/zot/pkg/storage/constants" "zotregistry.dev/zot/pkg/test/inject" ) type OciLayoutStorageImpl struct { storeController storage.StoreController - context *types.SystemContext } func NewOciLayoutStorage(storeController storage.StoreController) OciLayoutStorage { - context := &types.SystemContext{} - // preserve compression - context.OCIAcceptUncompressedLayers = true - return OciLayoutStorageImpl{ storeController: storeController, - context: context, } } -func (oci OciLayoutStorageImpl) GetContext() *types.SystemContext { - return oci.context -} - -func (oci OciLayoutStorageImpl) GetImageReference(repo string, reference string) (types.ImageReference, error) { +func (oci OciLayoutStorageImpl) GetImageReference(repo string, reference string) (ref.Ref, error) { localImageStore := oci.storeController.GetImageStore(repo) if localImageStore == nil { - return nil, zerr.ErrLocalImgStoreNotFound + return ref.Ref{}, zerr.ErrLocalImgStoreNotFound } + tempSyncPath := path.Join(localImageStore.RootDir(), repo, constants.SyncBlobUploadDir) // create session folder uuid, err := uuid.NewV4() // hard to reach test case, injected error, see pkg/test/dev.go if err := inject.Error(err); err != nil { - return nil, err + return ref.Ref{}, err } sessionRepoPath := path.Join(tempSyncPath, uuid.String()) - localRepo := path.Join(sessionRepoPath, repo) - if err := os.MkdirAll(localRepo, storageConstants.DefaultDirPerms); err != nil { - return nil, err - } + sessionRepo := path.Join(sessionRepoPath, repo) - _, refIsDigest := parseReference(reference) + var imageRefPath string - if !refIsDigest { - localRepo = fmt.Sprintf("%s:%s", localRepo, reference) + digest, ok := parseReference(reference) + if ok { + imageRefPath = fmt.Sprintf("ocidir://%s@%s", sessionRepo, digest.String()) + } else { + imageRefPath = fmt.Sprintf("ocidir://%s:%s", sessionRepo, reference) //nolint: nosprintfhostport } - localImageRef, err := layout.ParseReference(localRepo) + imageReference, err := ref.New(imageRefPath) if err != nil { - return nil, err + return ref.Ref{}, err } - return localImageRef, nil + return imageReference, nil } diff --git a/pkg/extensions/sync/on_demand.go b/pkg/extensions/sync/on_demand.go index 02ef21be4..99ed30029 100644 --- a/pkg/extensions/sync/on_demand.go +++ b/pkg/extensions/sync/on_demand.go @@ -7,9 +7,6 @@ import ( "context" "errors" "sync" - "time" - - "github.com/containers/common/pkg/retry" zerr "zotregistry.dev/zot/errors" "zotregistry.dev/zot/pkg/common" @@ -83,49 +80,16 @@ func (onDemand *BaseOnDemand) SyncImage(ctx context.Context, repo, reference str return err } -func (onDemand *BaseOnDemand) SyncReference(ctx context.Context, repo string, - subjectDigestStr string, referenceType string, -) error { - var err error - - for _, service := range onDemand.services { - err = service.SetNextAvailableURL() - if err != nil { - return err - } - - err = service.SyncReference(ctx, repo, subjectDigestStr, referenceType) - if err != nil { - continue - } else { - return nil - } - } - - return err -} - func (onDemand *BaseOnDemand) syncImage(ctx context.Context, repo, reference string, syncResult chan error) { var err error for serviceID, service := range onDemand.services { - err = service.SetNextAvailableURL() - - isPingErr := errors.Is(err, zerr.ErrSyncPingRegistry) - if err != nil && !isPingErr { - syncResult <- err - - return - } - - // no need to try to sync inline if there is a ping error, we want to retry in background - if !isPingErr { - err = service.SyncImage(ctx, repo, reference) - } - - if err != nil || isPingErr { + err = service.SyncImage(ctx, repo, reference) + if err != nil { if errors.Is(err, zerr.ErrManifestNotFound) || errors.Is(err, zerr.ErrSyncImageFilteredOut) || - errors.Is(err, zerr.ErrSyncImageNotSigned) { + errors.Is(err, zerr.ErrSyncImageNotSigned) || + // some public registries may return 401 for not found. + errors.Is(err, zerr.ErrUnauthorizedAccess) { continue } @@ -141,9 +105,7 @@ func (onDemand *BaseOnDemand) syncImage(ctx context.Context, repo, reference str continue } - retryOptions := service.GetRetryOptions() - - if retryOptions.MaxRetry > 0 { + if service.CanRetryOnError() { // retry in background go func(service Service) { // remove image after syncing @@ -154,18 +116,12 @@ func (onDemand *BaseOnDemand) syncImage(ctx context.Context, repo, reference str }() onDemand.log.Info().Str("repo", repo).Str(reference, "reference").Str("err", err.Error()). - Str("component", "sync").Msg("starting routine to copy image, because of error") - - time.Sleep(retryOptions.Delay) - - // retrying in background, can't use the same context which should be cancelled by now. - if err = retry.RetryIfNecessary(context.Background(), func() error { - err := service.SyncImage(context.Background(), repo, reference) + Msg("sync routine: starting routine to copy image, because of error") - return err - }, retryOptions); err != nil { + err := service.SyncImage(context.Background(), repo, reference) + if err != nil { onDemand.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo).Str("reference", reference). - Err(err).Str("component", "sync").Msg("failed to copy image") + Err(err).Msg("sync routine: error while copying image") } }(service) } diff --git a/pkg/extensions/sync/references/cosign.go b/pkg/extensions/sync/references/cosign.go deleted file mode 100644 index 160b8247c..000000000 --- a/pkg/extensions/sync/references/cosign.go +++ /dev/null @@ -1,219 +0,0 @@ -//go:build sync -// +build sync - -package references - -import ( - "context" - "errors" - "fmt" - "net/http" - "strings" - - godigest "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/sigstore/cosign/v2/pkg/oci/remote" - - zerr "zotregistry.dev/zot/errors" - "zotregistry.dev/zot/pkg/common" - "zotregistry.dev/zot/pkg/extensions/sync/constants" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" - "zotregistry.dev/zot/pkg/log" - "zotregistry.dev/zot/pkg/meta" - mTypes "zotregistry.dev/zot/pkg/meta/types" - "zotregistry.dev/zot/pkg/storage" -) - -type CosignReference struct { - client *client.Client - storeController storage.StoreController - metaDB mTypes.MetaDB - log log.Logger -} - -func NewCosignReference(httpClient *client.Client, storeController storage.StoreController, - metaDB mTypes.MetaDB, log log.Logger, -) CosignReference { - return CosignReference{ - client: httpClient, - storeController: storeController, - metaDB: metaDB, - log: log, - } -} - -func (ref CosignReference) Name() string { - return constants.Cosign -} - -func (ref CosignReference) IsSigned(ctx context.Context, upstreamRepo, subjectDigestStr string) bool { - cosignSignatureTag := getCosignSignatureTagFromSubjectDigest(subjectDigestStr) - _, _, err := ref.getManifest(ctx, upstreamRepo, cosignSignatureTag) - - return err == nil -} - -func (ref CosignReference) canSkipReferences(localRepo, digest string, manifest *ispec.Manifest) ( - bool, error, -) { - if manifest == nil { - return true, nil - } - - imageStore := ref.storeController.GetImageStore(localRepo) - - // check cosign signature already synced - _, localDigest, _, err := imageStore.GetImageManifest(localRepo, digest) - if err != nil { - if errors.Is(err, zerr.ErrManifestNotFound) { - return false, nil - } - - ref.log.Error().Str("errorType", common.TypeOf(err)).Err(err). - Str("repository", localRepo).Str("reference", digest). - Msg("couldn't get local cosign manifest") - - return false, err - } - - if localDigest.String() != digest { - return false, nil - } - - ref.log.Info().Str("repository", localRepo).Str("reference", digest). - Msg("skipping syncing cosign reference, already synced") - - return true, nil -} - -func (ref CosignReference) SyncReferences(ctx context.Context, localRepo, remoteRepo, subjectDigestStr string) ( - []godigest.Digest, error, -) { - cosignTags := getCosignTagsFromSubjectDigest(subjectDigestStr) - - refsDigests := make([]godigest.Digest, 0, len(cosignTags)) - - for _, cosignTag := range cosignTags { - manifest, manifestBuf, err := ref.getManifest(ctx, remoteRepo, cosignTag) - if err != nil { - if errors.Is(err, zerr.ErrSyncReferrerNotFound) { - continue - } - - return refsDigests, err - } - - digest := godigest.FromBytes(manifestBuf) - - skip, err := ref.canSkipReferences(localRepo, digest.String(), manifest) - if err != nil { - ref.log.Error().Err(err).Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("couldn't check if the remote image cosign reference can be skipped") - } - - if skip { - refsDigests = append(refsDigests, digest) - - continue - } - - imageStore := ref.storeController.GetImageStore(localRepo) - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("syncing cosign reference for image") - - for _, blob := range manifest.Layers { - if err := syncBlob(ctx, ref.client, imageStore, localRepo, remoteRepo, blob.Digest, ref.log); err != nil { - return refsDigests, err - } - } - - // sync config blob - if err := syncBlob(ctx, ref.client, imageStore, localRepo, remoteRepo, manifest.Config.Digest, ref.log); err != nil { - return refsDigests, err - } - - // push manifest - referenceDigest, _, err := imageStore.PutImageManifest(localRepo, cosignTag, - ispec.MediaTypeImageManifest, manifestBuf) - if err != nil { - ref.log.Error().Str("errorType", common.TypeOf(err)). - Str("repository", localRepo).Str("subject", subjectDigestStr). - Err(err).Msg("couldn't upload cosign reference manifest for image") - - return refsDigests, err - } - - refsDigests = append(refsDigests, digest) - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("successfully synced cosign reference for image") - - if ref.metaDB != nil { - ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr).Str("component", "metadb"). - Msg("trying to sync cosign reference for image") - - err = meta.SetImageMetaFromInput(ctx, localRepo, cosignTag, ispec.MediaTypeImageManifest, - referenceDigest, manifestBuf, ref.storeController.GetImageStore(localRepo), - ref.metaDB, ref.log) - if err != nil { - return refsDigests, fmt.Errorf("failed to set metadata for cosign reference in '%s@%s': %w", - localRepo, subjectDigestStr, err) - } - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).Str("component", "metadb"). - Msg("successfully added cosign reference for image") - } - } - - return refsDigests, nil -} - -func (ref CosignReference) getManifest(ctx context.Context, repo, cosignTag string) (*ispec.Manifest, []byte, error) { - var cosignManifest ispec.Manifest - - body, _, statusCode, err := ref.client.MakeGetRequest(ctx, &cosignManifest, ispec.MediaTypeImageManifest, "", - "v2", repo, "manifests", cosignTag) - if err != nil { - if statusCode == http.StatusNotFound { - ref.log.Debug().Str("errorType", common.TypeOf(err)). - Str("repository", repo).Str("tag", cosignTag). - Err(err).Msg("couldn't find any cosign manifest for image") - - return nil, nil, zerr.ErrSyncReferrerNotFound - } - - return nil, nil, err - } - - return &cosignManifest, body, nil -} - -func getCosignSignatureTagFromSubjectDigest(digestStr string) string { - return strings.Replace(digestStr, ":", "-", 1) + "." + remote.SignatureTagSuffix -} - -func getCosignSBOMTagFromSubjectDigest(digestStr string) string { - return strings.Replace(digestStr, ":", "-", 1) + "." + remote.SBOMTagSuffix -} - -func getCosignTagsFromSubjectDigest(digestStr string) []string { - var cosignTags []string - - // signature tag - cosignTags = append(cosignTags, getCosignSignatureTagFromSubjectDigest(digestStr)) - // sbom tag - cosignTags = append(cosignTags, getCosignSBOMTagFromSubjectDigest(digestStr)) - - return cosignTags -} - -// this function will check if tag is a cosign tag (signature or sbom). -func IsCosignTag(tag string) bool { - if strings.HasPrefix(tag, "sha256-") && - (strings.HasSuffix(tag, remote.SignatureTagSuffix) || strings.HasSuffix(tag, remote.SBOMTagSuffix)) { - return true - } - - return false -} diff --git a/pkg/extensions/sync/references/oci.go b/pkg/extensions/sync/references/oci.go deleted file mode 100644 index fc391abd7..000000000 --- a/pkg/extensions/sync/references/oci.go +++ /dev/null @@ -1,237 +0,0 @@ -//go:build sync -// +build sync - -package references - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - - godigest "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - - zerr "zotregistry.dev/zot/errors" - "zotregistry.dev/zot/pkg/common" - "zotregistry.dev/zot/pkg/extensions/sync/constants" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" - "zotregistry.dev/zot/pkg/log" - "zotregistry.dev/zot/pkg/meta" - mTypes "zotregistry.dev/zot/pkg/meta/types" - "zotregistry.dev/zot/pkg/storage" - storageTypes "zotregistry.dev/zot/pkg/storage/types" -) - -type OciReferences struct { - client *client.Client - storeController storage.StoreController - metaDB mTypes.MetaDB - log log.Logger -} - -func NewOciReferences(httpClient *client.Client, storeController storage.StoreController, - metaDB mTypes.MetaDB, log log.Logger, -) OciReferences { - return OciReferences{ - client: httpClient, - storeController: storeController, - metaDB: metaDB, - log: log, - } -} - -func (ref OciReferences) Name() string { - return constants.OCI -} - -func (ref OciReferences) IsSigned(ctx context.Context, remoteRepo, subjectDigestStr string) bool { - // use artifactTypeFilter - index, err := ref.getIndex(ctx, remoteRepo, subjectDigestStr) - if err != nil { - return false - } - - if len(getNotationManifestsFromOCIRefs(index)) > 0 || len(getCosignManifestsFromOCIRefs(index)) > 0 { - return true - } - - return false -} - -func (ref OciReferences) canSkipReferences(localRepo, subjectDigestStr string, index ispec.Index) (bool, error) { - imageStore := ref.storeController.GetImageStore(localRepo) - digest := godigest.Digest(subjectDigestStr) - - // check oci references already synced - if len(index.Manifests) > 0 { - localRefs, err := imageStore.GetReferrers(localRepo, digest, nil) - if err != nil { - if errors.Is(err, zerr.ErrManifestNotFound) { - return false, nil - } - - ref.log.Error().Str("errorType", common.TypeOf(err)). - Str("repository", localRepo).Str("subject", subjectDigestStr). - Err(err).Msg("couldn't get local oci references for image") - - return false, err - } - - if !descriptorsEqual(localRefs.Manifests, index.Manifests) { - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("remote oci references for image changed, syncing again") - - return false, nil - } - } - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("skipping oci references for image, already synced") - - return true, nil -} - -func (ref OciReferences) SyncReferences(ctx context.Context, localRepo, remoteRepo, subjectDigestStr string) ( - []godigest.Digest, error, -) { - refsDigests := make([]godigest.Digest, 0, 10) - - index, err := ref.getIndex(ctx, remoteRepo, subjectDigestStr) - if err != nil { - return refsDigests, err - } - - skipOCIRefs, err := ref.canSkipReferences(localRepo, subjectDigestStr, index) - if err != nil { - ref.log.Error().Err(err).Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("couldn't check if the upstream oci references for image can be skipped") - } - - if skipOCIRefs { - /* even if it's skip we need to return the digests, - because maybe in the meantime a reference pointing to this one was pushed */ - for _, man := range index.Manifests { - refsDigests = append(refsDigests, man.Digest) - } - - return refsDigests, nil - } - - imageStore := ref.storeController.GetImageStore(localRepo) - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("syncing oci references for image") - - for _, referrer := range index.Manifests { - referenceBuf, referenceDigest, err := syncManifest(ctx, ref.client, imageStore, localRepo, remoteRepo, - referrer, subjectDigestStr, ref.log) - if err != nil { - return refsDigests, err - } - - refsDigests = append(refsDigests, referenceDigest) - - if ref.metaDB != nil { - ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr).Str("component", "metadb"). - Msg("trying to add oci references for image") - - err = meta.SetImageMetaFromInput(ctx, localRepo, referenceDigest.String(), referrer.MediaType, - referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo), - ref.metaDB, ref.log) - if err != nil { - return refsDigests, fmt.Errorf("failed to set metadata for oci reference in '%s@%s': %w", - localRepo, subjectDigestStr, err) - } - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).Str("component", "metadb"). - Msg("successfully added oci references to MetaDB for image") - } - } - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("successfully synced oci references for image") - - return refsDigests, nil -} - -func (ref OciReferences) getIndex(ctx context.Context, repo, subjectDigestStr string) (ispec.Index, error) { - var index ispec.Index - - _, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex, "", - "v2", repo, "referrers", subjectDigestStr) - if err != nil { - if statusCode == http.StatusNotFound { - ref.log.Debug().Str("repository", repo).Str("subject", subjectDigestStr). - Msg("couldn't find any oci reference for image, skipping") - - return index, zerr.ErrSyncReferrerNotFound - } - - return index, err - } - - return index, nil -} - -func syncManifest(ctx context.Context, client *client.Client, imageStore storageTypes.ImageStore, localRepo, - remoteRepo string, desc ispec.Descriptor, subjectDigestStr string, log log.Logger, -) ([]byte, godigest.Digest, error) { - var manifest ispec.Manifest - - var refDigest godigest.Digest - - OCIRefBuf, _, statusCode, err := client.MakeGetRequest(ctx, &manifest, ispec.MediaTypeImageManifest, "", - "v2", remoteRepo, "manifests", desc.Digest.String()) - if err != nil { - if statusCode == http.StatusNotFound { - return []byte{}, refDigest, zerr.ErrSyncReferrerNotFound - } - - log.Error().Str("errorType", common.TypeOf(err)). - Str("repository", localRepo).Str("subject", subjectDigestStr). - Err(err).Msg("couldn't get oci reference manifest for image") - - return []byte{}, refDigest, err - } - - if desc.MediaType == ispec.MediaTypeImageManifest { - // read manifest - var manifest ispec.Manifest - - err = json.Unmarshal(OCIRefBuf, &manifest) - if err != nil { - log.Error().Str("errorType", common.TypeOf(err)). - Str("repository", localRepo).Str("subject", subjectDigestStr). - Err(err).Msg("couldn't unmarshal oci reference manifest for image") - - return []byte{}, refDigest, err - } - - for _, layer := range manifest.Layers { - if err := syncBlob(ctx, client, imageStore, localRepo, remoteRepo, layer.Digest, log); err != nil { - return []byte{}, refDigest, err - } - } - - // sync config blob - if err := syncBlob(ctx, client, imageStore, localRepo, remoteRepo, manifest.Config.Digest, log); err != nil { - return []byte{}, refDigest, err - } - } else { - return []byte{}, refDigest, nil - } - - refDigest, _, err = imageStore.PutImageManifest(localRepo, desc.Digest.String(), - desc.MediaType, OCIRefBuf) - if err != nil { - log.Error().Str("errorType", common.TypeOf(err)). - Str("repository", localRepo).Str("subject", subjectDigestStr). - Err(err).Msg("couldn't upload oci reference for image") - - return []byte{}, refDigest, err - } - - return OCIRefBuf, refDigest, nil -} diff --git a/pkg/extensions/sync/references/references.go b/pkg/extensions/sync/references/references.go deleted file mode 100644 index 17efc21af..000000000 --- a/pkg/extensions/sync/references/references.go +++ /dev/null @@ -1,234 +0,0 @@ -//go:build sync -// +build sync - -package references - -import ( - "bytes" - "context" - "errors" - "fmt" - "net/http" - - godigest "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/sigstore/cosign/v2/pkg/oci/static" - - zerr "zotregistry.dev/zot/errors" - "zotregistry.dev/zot/pkg/common" - "zotregistry.dev/zot/pkg/extensions/sync/constants" - "zotregistry.dev/zot/pkg/extensions/sync/features" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" - "zotregistry.dev/zot/pkg/log" - mTypes "zotregistry.dev/zot/pkg/meta/types" - "zotregistry.dev/zot/pkg/storage" - storageTypes "zotregistry.dev/zot/pkg/storage/types" -) - -type Reference interface { - // Returns name of reference (OCIReference/CosignReference) - Name() string - // Returns whether or not image is signed - IsSigned(ctx context.Context, upstreamRepo, subjectDigestStr string) bool - // Sync recursively all references for a subject digest (can be image/artifacts/signatures) - SyncReferences(ctx context.Context, localRepo, upstreamRepo, subjectDigestStr string) ([]godigest.Digest, error) -} - -type References struct { - referenceList []Reference - features *features.Map - log log.Logger -} - -func NewReferences(httpClient *client.Client, storeController storage.StoreController, - metaDB mTypes.MetaDB, log log.Logger, -) References { - refs := References{features: features.New(), log: log} - - refs.referenceList = append(refs.referenceList, NewCosignReference(httpClient, storeController, metaDB, log)) - refs.referenceList = append(refs.referenceList, NewTagReferences(httpClient, storeController, metaDB, log)) - refs.referenceList = append(refs.referenceList, NewOciReferences(httpClient, storeController, metaDB, log)) - - return refs -} - -func (refs References) IsSigned(ctx context.Context, upstreamRepo, subjectDigestStr string) bool { - for _, ref := range refs.referenceList { - ok := ref.IsSigned(ctx, upstreamRepo, subjectDigestStr) - if ok { - return true - } - } - - return false -} - -func (refs References) SyncAll(ctx context.Context, localRepo, upstreamRepo, subjectDigestStr string) error { - seen := &[]godigest.Digest{} - - return refs.syncAll(ctx, localRepo, upstreamRepo, subjectDigestStr, seen) -} - -func (refs References) syncAll(ctx context.Context, localRepo, upstreamRepo, - subjectDigestStr string, seen *[]godigest.Digest, -) error { - var err error - - var syncedRefsDigests []godigest.Digest - - // mark subject digest as seen as soon as it comes in - *seen = append(*seen, godigest.Digest(subjectDigestStr)) - - // for each reference type(cosign/oci reference) - for _, ref := range refs.referenceList { - supported, ok := refs.features.Get(ref.Name(), upstreamRepo) - if !supported && ok { - continue - } - - syncedRefsDigests, err = ref.SyncReferences(ctx, localRepo, upstreamRepo, subjectDigestStr) - if err != nil { - // for all referrers we can stop querying same repo (for ten minutes) if the errors are different than 404 - if !errors.Is(err, zerr.ErrSyncReferrerNotFound) { - refs.features.Set(ref.Name(), upstreamRepo, false) - } - - // in the case of oci referrers, it will return 404 only if the repo is not found or refferers API is not supported - // no need to continue to make requests to the same repo - if ref.Name() == constants.OCI && errors.Is(err, zerr.ErrSyncReferrerNotFound) { - refs.features.Set(ref.Name(), upstreamRepo, false) - } - - refs.log.Debug().Err(err). - Str("reference type", ref.Name()). - Str("image", fmt.Sprintf("%s:%s", upstreamRepo, subjectDigestStr)). - Msg("couldn't sync image referrer") - } else { - refs.features.Set(ref.Name(), upstreamRepo, true) - } - - // for each synced references - for _, refDigest := range syncedRefsDigests { - if !common.Contains(*seen, refDigest) { - // sync all references pointing to this one - err = refs.syncAll(ctx, localRepo, upstreamRepo, refDigest.String(), seen) - } - } - } - - return err -} - -func (refs References) SyncReference(ctx context.Context, localRepo, upstreamRepo, - subjectDigestStr, referenceType string, -) error { - var err error - - var syncedRefsDigests []godigest.Digest - - for _, ref := range refs.referenceList { - if ref.Name() == referenceType { - syncedRefsDigests, err = ref.SyncReferences(ctx, localRepo, upstreamRepo, subjectDigestStr) - if err != nil { - refs.log.Debug().Err(err). - Str("reference type", ref.Name()). - Str("image", fmt.Sprintf("%s:%s", upstreamRepo, subjectDigestStr)). - Msg("couldn't sync image referrer") - - return err - } - - for _, refDigest := range syncedRefsDigests { - err = refs.SyncAll(ctx, localRepo, upstreamRepo, refDigest.String()) - } - } - } - - return err -} - -func syncBlob(ctx context.Context, client *client.Client, imageStore storageTypes.ImageStore, - localRepo, remoteRepo string, digest godigest.Digest, log log.Logger, -) error { - var resultPtr interface{} - - body, _, statusCode, err := client.MakeGetRequest(ctx, resultPtr, "", "", "v2", remoteRepo, "blobs", digest.String()) - if err != nil { - if statusCode != http.StatusOK { - log.Info().Str("repo", remoteRepo).Str("digest", digest.String()).Msg("couldn't get remote blob") - - return err - } - } - - _, _, err = imageStore.FullBlobUpload(localRepo, bytes.NewBuffer(body), digest) - if err != nil { - log.Error().Str("errorType", common.TypeOf(err)).Str("digest", digest.String()).Str("repo", localRepo). - Err(err).Msg("couldn't upload blob") - - return err - } - - return nil -} - -func manifestsEqual(manifest1, manifest2 ispec.Manifest) bool { - if manifest1.Config.Digest == manifest2.Config.Digest && - manifest1.Config.MediaType == manifest2.Config.MediaType && - manifest1.Config.Size == manifest2.Config.Size { - if descriptorsEqual(manifest1.Layers, manifest2.Layers) { - return true - } - } - - return false -} - -func descriptorsEqual(desc1, desc2 []ispec.Descriptor) bool { - if len(desc1) != len(desc2) { - return false - } - - for id, desc := range desc1 { - if !descriptorEqual(desc, desc2[id]) { - return false - } - } - - return true -} - -func descriptorEqual(desc1, desc2 ispec.Descriptor) bool { - if desc1.Size == desc2.Size && - desc1.Digest == desc2.Digest && - desc1.MediaType == desc2.MediaType && - desc1.Annotations[static.SignatureAnnotationKey] == desc2.Annotations[static.SignatureAnnotationKey] { - return true - } - - return false -} - -func getNotationManifestsFromOCIRefs(ociRefs ispec.Index) []ispec.Descriptor { - notaryManifests := []ispec.Descriptor{} - - for _, ref := range ociRefs.Manifests { - if ref.ArtifactType == common.ArtifactTypeNotation { - notaryManifests = append(notaryManifests, ref) - } - } - - return notaryManifests -} - -func getCosignManifestsFromOCIRefs(ociRefs ispec.Index) []ispec.Descriptor { - cosignManifests := []ispec.Descriptor{} - - for _, ref := range ociRefs.Manifests { - if ref.ArtifactType == common.ArtifactTypeCosign { - cosignManifests = append(cosignManifests, ref) - } - } - - return cosignManifests -} diff --git a/pkg/extensions/sync/references/references_internal_test.go b/pkg/extensions/sync/references/references_internal_test.go deleted file mode 100644 index a17020ce3..000000000 --- a/pkg/extensions/sync/references/references_internal_test.go +++ /dev/null @@ -1,306 +0,0 @@ -//go:build sync -// +build sync - -package references - -import ( - "context" - "errors" - "testing" - - godigest "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - . "github.com/smartystreets/goconvey/convey" - - zerr "zotregistry.dev/zot/errors" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" - "zotregistry.dev/zot/pkg/log" - "zotregistry.dev/zot/pkg/storage" - "zotregistry.dev/zot/pkg/test/mocks" -) - -var errRef = errors.New("err") - -func TestCosign(t *testing.T) { - Convey("trigger errors", t, func() { - cfg := client.Config{ - URL: "url", - TLSVerify: false, - } - - client, err := client.New(cfg, log.NewLogger("debug", "")) - So(err, ShouldBeNil) - - cosign := NewCosignReference(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{ - GetImageManifestFn: func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return []byte{}, "", "", errRef - }, - }}, nil, log.NewLogger("debug", "")) - - ok, err := cosign.canSkipReferences("repo", "tag", nil) - So(err, ShouldBeNil) - So(ok, ShouldBeTrue) - - // trigger GetImageManifest err - ok, err = cosign.canSkipReferences("repo", "tag", &ispec.Manifest{MediaType: ispec.MediaTypeImageManifest}) - So(err, ShouldNotBeNil) - So(ok, ShouldBeFalse) - - cosign = NewCosignReference(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{ - GetImageManifestFn: func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return []byte{}, "digest", "", nil - }, - }}, nil, log.NewLogger("debug", "")) - - // different digest - ok, err = cosign.canSkipReferences("repo", "tag", &ispec.Manifest{MediaType: ispec.MediaTypeImageManifest}) - So(err, ShouldBeNil) - So(ok, ShouldBeFalse) - }) -} - -func TestOci(t *testing.T) { - Convey("trigger errors", t, func() { - cfg := client.Config{ - URL: "url", - TLSVerify: false, - } - - client, err := client.New(cfg, log.NewLogger("debug", "")) - So(err, ShouldBeNil) - - oci := NewOciReferences(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{ - GetReferrersFn: func(repo string, digest godigest.Digest, artifactTypes []string) (ispec.Index, error) { - return ispec.Index{}, zerr.ErrManifestNotFound - }, - }}, nil, log.NewLogger("debug", "")) - - ok := oci.IsSigned(context.Background(), "repo", "") - So(ok, ShouldBeFalse) - - // trigger GetReferrers err - ok, err = oci.canSkipReferences("repo", "tag", ispec.Index{Manifests: []ispec.Descriptor{{Digest: "digest1"}}}) - So(err, ShouldBeNil) - So(ok, ShouldBeFalse) - }) -} - -func TestReferrersTag(t *testing.T) { - Convey("trigger errors", t, func() { - cfg := client.Config{ - URL: "url", - TLSVerify: false, - } - - client, err := client.New(cfg, log.NewLogger("debug", "")) - So(err, ShouldBeNil) - - referrersTag := NewTagReferences(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{ - GetImageManifestFn: func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return []byte{}, "", "", errRef - }, - }}, nil, log.NewLogger("debug", "")) - - ok := referrersTag.IsSigned(context.Background(), "repo", "") - So(ok, ShouldBeFalse) - - // trigger GetImageManifest err - ok, err = referrersTag.canSkipReferences("repo", "subjectdigest", "digest") - So(err, ShouldNotBeNil) - So(ok, ShouldBeFalse) - - referrersTag = NewTagReferences(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{ - GetImageManifestFn: func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return []byte{}, "", "", zerr.ErrManifestNotFound - }, - }}, nil, log.NewLogger("debug", "")) - - // trigger GetImageManifest err - ok, err = referrersTag.canSkipReferences("repo", "subjectdigest", "digest") - So(err, ShouldBeNil) - So(ok, ShouldBeFalse) - - referrersTag = NewTagReferences(client, storage.StoreController{DefaultStore: mocks.MockedImageStore{ - GetImageManifestFn: func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return []byte{}, "digest", "", nil - }, - }}, nil, log.NewLogger("debug", "")) - - // different digest - ok, err = referrersTag.canSkipReferences("repo", "subjectdigest", "newdigest") - So(err, ShouldBeNil) - So(ok, ShouldBeFalse) - }) -} - -func TestSyncManifest(t *testing.T) { - Convey("sync manifest not found err", t, func() { - cfg := client.Config{ - URL: "url", - TLSVerify: false, - } - - client, err := client.New(cfg, log.NewLogger("debug", "")) - So(err, ShouldBeNil) - - digest := godigest.FromString("test") - - buf, refDigest, err := syncManifest(context.Background(), client, mocks.MockedImageStore{}, - "repo", "repo", ispec.Descriptor{ - Digest: digest, - Size: 10, - MediaType: ispec.MediaTypeImageManifest, - }, digest.String(), log.Logger{}) - - So(buf, ShouldBeEmpty) - So(refDigest, ShouldBeEmpty) - So(err, ShouldNotBeNil) - }) -} - -func TestCompareManifest(t *testing.T) { - testCases := []struct { - manifest1 ispec.Manifest - manifest2 ispec.Manifest - expected bool - }{ - { - manifest1: ispec.Manifest{ - Config: ispec.Descriptor{ - Digest: "digest1", - }, - }, - manifest2: ispec.Manifest{ - Config: ispec.Descriptor{ - Digest: "digest2", - }, - }, - expected: false, - }, - { - manifest1: ispec.Manifest{ - Config: ispec.Descriptor{ - Digest: "digest", - }, - }, - manifest2: ispec.Manifest{ - Config: ispec.Descriptor{ - Digest: "digest", - }, - }, - expected: true, - }, - { - manifest1: ispec.Manifest{ - Layers: []ispec.Descriptor{{ - Digest: "digest", - Size: 1, - }}, - }, - manifest2: ispec.Manifest{ - Layers: []ispec.Descriptor{{ - Digest: "digest", - Size: 1, - }}, - }, - expected: true, - }, - { - manifest1: ispec.Manifest{ - Layers: []ispec.Descriptor{{ - Digest: "digest1", - Size: 1, - }}, - }, - manifest2: ispec.Manifest{ - Layers: []ispec.Descriptor{{ - Digest: "digest2", - Size: 2, - }}, - }, - expected: false, - }, - { - manifest1: ispec.Manifest{ - Layers: []ispec.Descriptor{ - { - Digest: "digest", - Size: 1, - }, - { - Digest: "digest1", - Size: 1, - }, - }, - }, - manifest2: ispec.Manifest{ - Layers: []ispec.Descriptor{{ - Digest: "digest", - Size: 1, - }}, - }, - expected: false, - }, - { - manifest1: ispec.Manifest{ - Layers: []ispec.Descriptor{ - { - Digest: "digest1", - Size: 1, - }, - { - Digest: "digest2", - Size: 2, - }, - }, - }, - manifest2: ispec.Manifest{ - Layers: []ispec.Descriptor{ - { - Digest: "digest1", - Size: 1, - }, - { - Digest: "digest2", - Size: 2, - }, - }, - }, - expected: true, - }, - { - manifest1: ispec.Manifest{ - Layers: []ispec.Descriptor{ - { - Digest: "digest", - Size: 1, - }, - { - Digest: "digest1", - Size: 1, - }, - }, - }, - manifest2: ispec.Manifest{ - Layers: []ispec.Descriptor{ - { - Digest: "digest", - Size: 1, - }, - { - Digest: "digest2", - Size: 2, - }, - }, - }, - expected: false, - }, - } - - Convey("Test manifestsEqual()", t, func() { - for _, test := range testCases { - actualResult := manifestsEqual(test.manifest1, test.manifest2) - So(actualResult, ShouldEqual, test.expected) - } - }) -} diff --git a/pkg/extensions/sync/references/referrers_tag.go b/pkg/extensions/sync/references/referrers_tag.go deleted file mode 100644 index ffed36928..000000000 --- a/pkg/extensions/sync/references/referrers_tag.go +++ /dev/null @@ -1,172 +0,0 @@ -//go:build sync -// +build sync - -package references - -import ( - "context" - "errors" - "fmt" - "net/http" - "strings" - - godigest "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - - zerr "zotregistry.dev/zot/errors" - "zotregistry.dev/zot/pkg/common" - "zotregistry.dev/zot/pkg/extensions/sync/constants" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" - "zotregistry.dev/zot/pkg/log" - "zotregistry.dev/zot/pkg/meta" - mTypes "zotregistry.dev/zot/pkg/meta/types" - "zotregistry.dev/zot/pkg/storage" -) - -type TagReferences struct { - client *client.Client - storeController storage.StoreController - metaDB mTypes.MetaDB - log log.Logger -} - -func NewTagReferences(httpClient *client.Client, storeController storage.StoreController, - metaDB mTypes.MetaDB, log log.Logger, -) TagReferences { - return TagReferences{ - client: httpClient, - storeController: storeController, - metaDB: metaDB, - log: log, - } -} - -func (ref TagReferences) Name() string { - return constants.Tag -} - -func (ref TagReferences) IsSigned(ctx context.Context, remoteRepo, subjectDigestStr string) bool { - return false -} - -func (ref TagReferences) canSkipReferences(localRepo, subjectDigestStr, digest string) (bool, error) { - imageStore := ref.storeController.GetImageStore(localRepo) - - _, localDigest, _, err := imageStore.GetImageManifest(localRepo, getReferrersTagFromSubjectDigest(subjectDigestStr)) - if err != nil { - if errors.Is(err, zerr.ErrManifestNotFound) { - return false, nil - } - - ref.log.Error().Str("errorType", common.TypeOf(err)). - Str("repository", localRepo).Str("subject", subjectDigestStr). - Err(err).Msg("couldn't get local index with referrers tag for image") - - return false, err - } - - if localDigest.String() != digest { - return false, nil - } - - ref.log.Info().Str("repository", localRepo).Str("reference", subjectDigestStr). - Msg("skipping index with referrers tag for image, already synced") - - return true, nil -} - -func (ref TagReferences) SyncReferences(ctx context.Context, localRepo, remoteRepo, subjectDigestStr string) ( - []godigest.Digest, error, -) { - refsDigests := make([]godigest.Digest, 0, 10) - - index, indexContent, err := ref.getIndex(ctx, remoteRepo, subjectDigestStr) - if err != nil { - return refsDigests, err - } - - skipTagRefs, err := ref.canSkipReferences(localRepo, subjectDigestStr, string(godigest.FromBytes(indexContent))) - if err != nil { - ref.log.Error().Err(err).Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("couldn't check if the upstream index with referrers tag for image can be skipped") - } - - if skipTagRefs { - return refsDigests, nil - } - - imageStore := ref.storeController.GetImageStore(localRepo) - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("syncing oci references for image") - - for _, referrer := range index.Manifests { - referenceBuf, referenceDigest, err := syncManifest(ctx, ref.client, imageStore, localRepo, remoteRepo, - referrer, subjectDigestStr, ref.log) - if err != nil { - return refsDigests, err - } - - refsDigests = append(refsDigests, referenceDigest) - - if ref.metaDB != nil { - ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr).Str("component", "metadb"). - Msg("trying to add oci references for image") - - err = meta.SetImageMetaFromInput(ctx, localRepo, referenceDigest.String(), referrer.MediaType, - referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo), - ref.metaDB, ref.log) - if err != nil { - return refsDigests, fmt.Errorf("failed to set metadata for oci reference in '%s@%s': %w", - localRepo, subjectDigestStr, err) - } - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).Str("component", "metadb"). - Msg("successfully added oci references to MetaDB for image") - } - } - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("syncing index with referrers tag for image") - - referrersTag := getReferrersTagFromSubjectDigest(subjectDigestStr) - - _, _, err = imageStore.PutImageManifest(localRepo, referrersTag, index.MediaType, indexContent) - if err != nil { - ref.log.Error().Str("errorType", common.TypeOf(err)). - Str("repository", localRepo).Str("subject", subjectDigestStr). - Err(err).Msg("couldn't upload index with referrers tag for image") - - return refsDigests, err - } - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("successfully synced index with referrers tag for image") - - return refsDigests, nil -} - -func (ref TagReferences) getIndex( - ctx context.Context, repo, subjectDigestStr string, -) (ispec.Index, []byte, error) { - var index ispec.Index - - content, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex, "", - "v2", repo, "manifests", getReferrersTagFromSubjectDigest(subjectDigestStr)) - if err != nil { - if statusCode == http.StatusNotFound { - ref.log.Debug().Str("repository", repo).Str("subject", subjectDigestStr). - Msg("couldn't find any index with referrers tag for image, skipping") - - return index, []byte{}, zerr.ErrSyncReferrerNotFound - } - - return index, []byte{}, err - } - - return index, content, nil -} - -func getReferrersTagFromSubjectDigest(digestStr string) string { - return strings.Replace(digestStr, ":", "-", 1) -} diff --git a/pkg/extensions/sync/referrers.go b/pkg/extensions/sync/referrers.go new file mode 100644 index 000000000..a30d0af20 --- /dev/null +++ b/pkg/extensions/sync/referrers.go @@ -0,0 +1,36 @@ +//go:build sync +// +build sync + +package sync + +import ( + ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/regclient/regclient/types/referrer" + + "zotregistry.dev/zot/pkg/common" +) + +const ( + cosignSignatureTagSuffix = "sig" + SBOMTagSuffix = "sbom" +) + +func hasSignatureReferrers(refs referrer.ReferrerList) bool { + for _, desc := range refs.Descriptors { + tag := desc.Annotations[ispec.AnnotationRefName] + + if common.IsCosignSignature(tag) { + return true + } + + if desc.ArtifactType == common.ArtifactTypeNotation { + return true + } + + if desc.ArtifactType == common.ArtifactTypeCosign { + return true + } + } + + return false +} diff --git a/pkg/extensions/sync/remote.go b/pkg/extensions/sync/remote.go index 174ae94c2..562511918 100644 --- a/pkg/extensions/sync/remote.go +++ b/pkg/extensions/sync/remote.go @@ -5,192 +5,350 @@ package sync import ( "context" + "encoding/json" + "errors" "fmt" - "net/url" - "strings" - - "github.com/containers/image/v5/docker" - dockerReference "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/manifest" - "github.com/containers/image/v5/types" - "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - "zotregistry.dev/zot/pkg/api/constants" + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/regclient/regclient" + "github.com/regclient/regclient/config" + "github.com/regclient/regclient/types/descriptor" + "github.com/regclient/regclient/types/errs" + "github.com/regclient/regclient/types/manifest" + "github.com/regclient/regclient/types/mediatype" + "github.com/regclient/regclient/types/ref" + "github.com/regclient/regclient/types/repo" + + zerr "zotregistry.dev/zot/errors" "zotregistry.dev/zot/pkg/common" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" "zotregistry.dev/zot/pkg/log" ) -type catalog struct { - Repositories []string `json:"repositories"` -} - type RemoteRegistry struct { - client *client.Client - context *types.SystemContext - log log.Logger + client *regclient.RegClient + hosts []config.Host + primaryHost string + log log.Logger } -func NewRemoteRegistry(client *client.Client, logger log.Logger) Remote { +func NewRemoteRegistry(client *regclient.RegClient, hosts []config.Host, logger log.Logger) Remote { registry := &RemoteRegistry{} registry.log = logger + registry.hosts = hosts registry.client = client - clientConfig := client.GetConfig() - registry.context = getUpstreamContext(clientConfig.CertDir, clientConfig.Username, - clientConfig.Password, clientConfig.TLSVerify) + // + registry.primaryHost = hosts[0].Hostname return registry } -func (registry *RemoteRegistry) SetUpstreamAuthConfig(username, password string) { - registry.context.DockerAuthConfig = &types.DockerAuthConfig{ - Username: username, - Password: password, - } +func (registry *RemoteRegistry) GetHostName() string { + return registry.primaryHost } -func (registry *RemoteRegistry) GetContext() *types.SystemContext { - return registry.context +func (registry *RemoteRegistry) GetRepositories(ctx context.Context) ([]string, error) { + var err error + + var repoList *repo.RepoList + + for _, host := range registry.hosts { + repoList, err = registry.client.RepoList(ctx, host.Hostname) + if err != nil { + registry.log.Error().Err(err).Str("remote", host.Name).Msg("failed to list repositories in remote registry") + + continue + } + + return repoList.Repositories, nil + } + + return []string{}, err } -func (registry *RemoteRegistry) GetRepositories(ctx context.Context) ([]string, error) { - var catalog catalog +func (registry *RemoteRegistry) GetImageReference(repo, reference string) (ref.Ref, error) { + digest, ok := parseReference(reference) - _, header, _, err := registry.client.MakeGetRequest(ctx, &catalog, "application/json", "", //nolint: dogsled - constants.RoutePrefix, constants.ExtCatalogPrefix) + var imageRefPath string + if ok { + imageRefPath = fmt.Sprintf("%s/%s@%s", registry.primaryHost, repo, digest.String()) + } else { + // is tag + imageRefPath = fmt.Sprintf("%s/%s:%s", registry.primaryHost, repo, reference) + } + + imageRef, err := ref.New(imageRefPath) if err != nil { - return []string{}, err + return ref.Ref{}, err } - var repos []string + return imageRef, nil +} + +func (registry *RemoteRegistry) GetOCIManifest(ctx context.Context, repo, reference string, +) ([]byte, ispec.Descriptor, bool, error) { + var isConverted bool - repos = append(repos, catalog.Repositories...) + var buf []byte - link := header.Get("Link") - for link != "" { - linkURLPart, _, _ := strings.Cut(link, ";") + var desc ispec.Descriptor - linkURL, err := url.Parse(strings.Trim(linkURLPart, "<>")) - if err != nil { - return catalog.Repositories, err - } + imageReference, err := registry.GetImageReference(repo, reference) + if err != nil { + return nil, ispec.Descriptor{}, false, err + } - _, header, _, err := registry.client.MakeGetRequest(ctx, &catalog, "application/json", - linkURL.RawQuery, constants.RoutePrefix, constants.ExtCatalogPrefix) //nolint: dogsled - if err != nil { - return repos, err + /// check what error it gives when not found + man, err := registry.client.ManifestGet(ctx, imageReference) + if err != nil { + /* public registries may return 401 for image not found + they will try to check private registries as a fallback => 401 */ + if errors.Is(err, errs.ErrHTTPUnauthorized) { + registry.log.Info().Str("errorType", common.TypeOf(err)). + Str("repository", repo).Str("reference", reference). + Err(err).Msg("failed to get manifest: unauthorized") + + return nil, ispec.Descriptor{}, false, zerr.ErrUnauthorizedAccess + } else if errors.Is(err, errs.ErrNotFound) { + registry.log.Info().Str("errorType", common.TypeOf(err)). + Str("repository", repo).Str("reference", reference). + Err(err).Msg("failed to find manifest") + + return nil, ispec.Descriptor{}, false, zerr.ErrManifestNotFound } - repos = append(repos, catalog.Repositories...) + return nil, ispec.Descriptor{}, false, err + } - link = header.Get("Link") + switch man.GetDescriptor().MediaType { + case mediatype.Docker2Manifest: + buf, desc, err = convertDockerManifestToOCI(ctx, man, imageReference, registry.client) + isConverted = true + case mediatype.Docker2ManifestList: + buf, desc, err = convertDockerListToOCI(ctx, man, imageReference, registry.client) + isConverted = true + case mediatype.OCI1Manifest, mediatype.OCI1ManifestList: + buf, err = man.MarshalJSON() + desc = toOCIDescriptor(man.GetDescriptor()) + default: + return nil, desc, false, zerr.ErrMediaTypeNotSupported } - return repos, nil + return buf, desc, isConverted, err } -func (registry *RemoteRegistry) GetDockerRemoteRepo(repo string) string { - dockerNamespace := "library" - dockerRegistry := "docker.io" +func (registry *RemoteRegistry) GetTags(ctx context.Context, repo string) ([]string, error) { + repoRefPath := fmt.Sprintf("%s/%s", registry.primaryHost, repo) - remoteHost := registry.client.GetHostname() - - repoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", remoteHost, repo)) + repoReference, err := ref.New(repoRefPath) if err != nil { - return repo + return []string{}, err } - if !strings.Contains(repo, dockerNamespace) && - strings.Contains(repoRef.String(), dockerNamespace) && - strings.Contains(repoRef.String(), dockerRegistry) { - return fmt.Sprintf("%s/%s", dockerNamespace, repo) + tl, err := registry.client.TagList(ctx, repoReference) + if err != nil { + return []string{}, err } - return repo + return tl.GetTags() } -func (registry *RemoteRegistry) GetImageReference(repo, reference string) (types.ImageReference, error) { - remoteHost := registry.client.GetHostname() +func convertDockerListToOCI(ctx context.Context, man manifest.Manifest, imageReference ref.Ref, + regclient *regclient.RegClient, +) ( + []byte, ispec.Descriptor, error, +) { + var index ispec.Index + + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{} + index.MediaType = ispec.MediaTypeImageIndex + + indexer, ok := man.(manifest.Indexer) + if !ok { + return nil, ispec.Descriptor{}, zerr.ErrMediaTypeNotSupported + } - repoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", remoteHost, repo)) + ociIndex, err := manifest.OCIIndexFromAny(man.GetOrig()) if err != nil { - registry.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo). - Str("reference", reference).Str("remote", remoteHost). - Err(err).Msg("couldn't parse repository reference") + return nil, ispec.Descriptor{}, zerr.ErrMediaTypeNotSupported + } - return nil, err + manifests, err := indexer.GetManifestList() + if err != nil { + return nil, ispec.Descriptor{}, zerr.ErrMediaTypeNotSupported } - var namedRepoRef dockerReference.Named + for _, manDesc := range manifests { + ref := imageReference + ref.Digest = manDesc.Digest.String() - digest, ok := parseReference(reference) - if ok { - namedRepoRef, err = dockerReference.WithDigest(repoRef, digest) + manEntry, err := regclient.ManifestGet(ctx, ref) if err != nil { - return nil, err + return nil, ispec.Descriptor{}, err } - } else { - namedRepoRef, err = dockerReference.WithTag(repoRef, reference) + + regclient.Close(ctx, manEntry.GetRef()) + + _, desc, err := convertDockerManifestToOCI(ctx, manEntry, ref, regclient) if err != nil { - return nil, err + return nil, ispec.Descriptor{}, err + } + + // copy desc platform from docker desc + if manDesc.Platform != nil { + desc.Platform = &ispec.Platform{ + Architecture: manDesc.Platform.Architecture, + OS: manDesc.Platform.OS, + OSVersion: manDesc.Platform.OSVersion, + OSFeatures: manDesc.Platform.OSFeatures, + Variant: manDesc.Platform.Variant, + } } + + index.Manifests = append(index.Manifests, desc) } - imageRef, err := docker.NewReference(namedRepoRef) - if err != nil { - registry.log.Err(err).Str("transport", docker.Transport.Name()).Str("reference", namedRepoRef.String()). - Msg("cannot obtain a valid image reference for given transport and reference") + index.Annotations = ociIndex.Annotations + index.ArtifactType = ociIndex.ArtifactType - return nil, err + if ociIndex.Subject != nil { + subject := toOCIDescriptor(*ociIndex.Subject) + index.Subject = &subject } - return imageRef, nil + indexBuf, err := json.Marshal(index) + if err != nil { + return nil, ispec.Descriptor{}, err + } + + indexDesc := toOCIDescriptor(man.GetDescriptor()) + + indexDesc.MediaType = ispec.MediaTypeImageIndex + indexDesc.Digest = godigest.FromBytes(indexBuf) + indexDesc.Size = int64(len(indexBuf)) + + return indexBuf, indexDesc, nil } -func (registry *RemoteRegistry) GetManifestContent(imageReference types.ImageReference) ( - []byte, string, digest.Digest, error, +func convertDockerManifestToOCI(ctx context.Context, man manifest.Manifest, imageReference ref.Ref, + regclient *regclient.RegClient, +) ( + []byte, ispec.Descriptor, error, ) { - imageSource, err := imageReference.NewImageSource(context.Background(), registry.GetContext()) + imager, ok := man.(manifest.Imager) + if !ok { + return nil, ispec.Descriptor{}, zerr.ErrMediaTypeNotSupported + } + + var ociManifest ispec.Manifest + + manifest, err := man.RawBody() if err != nil { - return []byte{}, "", "", err + return nil, ispec.Descriptor{}, zerr.ErrMediaTypeNotSupported } - defer imageSource.Close() + if err := json.Unmarshal(manifest, &ociManifest); err != nil { + return nil, ispec.Descriptor{}, err + } - manifestBuf, mediaType, err := imageSource.GetManifest(context.Background(), nil) + configDesc, err := imager.GetConfig() if err != nil { - return []byte{}, "", "", err + return nil, ispec.Descriptor{}, err } - // if mediatype is docker then convert to OCI - switch mediaType { - case manifest.DockerV2Schema2MediaType: - manifestBuf, err = convertDockerManifestToOCI(imageSource, manifestBuf) - if err != nil { - return []byte{}, "", "", err - } - case manifest.DockerV2ListMediaType: - manifestBuf, err = convertDockerIndexToOCI(imageSource, manifestBuf) - if err != nil { - return []byte{}, "", "", err - } + // get config blob + config, err := regclient.BlobGetOCIConfig(ctx, imageReference, configDesc) + if err != nil { + return nil, ispec.Descriptor{}, err } - return manifestBuf, ispec.MediaTypeImageManifest, digest.FromBytes(manifestBuf), nil -} + configBuf, err := config.RawBody() + if err != nil { + return nil, ispec.Descriptor{}, err + } + + // var ociConfig ispec.Image -func (registry *RemoteRegistry) GetRepoTags(repo string) ([]string, error) { - remoteHost := registry.client.GetHostname() + // if err := json.Unmarshal(configBuf, &ociConfig); err != nil { + // return nil, ispec.Descriptor{}, err + // } - tags, err := getRepoTags(context.Background(), registry.GetContext(), remoteHost, repo) + // ociConfigContent, err := json.Marshal(ociConfig) + // if err != nil { + // return nil, ispec.Descriptor{}, err + // } + + // convert config and manifest mediatype + ociManifest.Config.Size = int64(len(configBuf)) + ociManifest.Config.Digest = godigest.FromBytes(configBuf) + ociManifest.Config.MediaType = ispec.MediaTypeImageConfig + ociManifest.MediaType = ispec.MediaTypeImageManifest + + layersDesc, err := imager.GetLayers() if err != nil { - registry.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo). - Str("remote", remoteHost).Err(err).Msg("couldn't fetch tags for repo") + return nil, ispec.Descriptor{}, err + } - return []string{}, err + ociManifest.Layers = []ispec.Descriptor{} + + for _, layerDesc := range layersDesc { + ociManifest.Layers = append(ociManifest.Layers, toOCIDescriptor(layerDesc)) + } + + manifestBuf, err := json.Marshal(ociManifest) + if err != nil { + return nil, ispec.Descriptor{}, err } - return tags, nil + manifestDesc := toOCIDescriptor(man.GetDescriptor()) + + manifestDesc.MediaType = ispec.MediaTypeImageManifest + manifestDesc.Digest = godigest.FromBytes(manifestBuf) + manifestDesc.Size = int64(len(manifestBuf)) + + return manifestBuf, manifestDesc, nil +} + +func toOCIDescriptor(desc descriptor.Descriptor) ispec.Descriptor { + ispecPlatform := &ispec.Platform{} + + platform := desc.Platform + if platform != nil { + ispecPlatform.Architecture = platform.Architecture + ispecPlatform.OS = platform.OS + ispecPlatform.OSFeatures = platform.OSFeatures + ispecPlatform.OSVersion = platform.OSVersion + ispecPlatform.Variant = platform.Variant + } else { + ispecPlatform = nil + } + + var mediaType string + + switch desc.MediaType { + case mediatype.Docker2Manifest: + mediaType = ispec.MediaTypeImageManifest + case mediatype.Docker2ManifestList: + mediaType = ispec.MediaTypeImageIndex + case mediatype.Docker2ImageConfig: + mediaType = ispec.MediaTypeImageConfig + case mediatype.Docker2ForeignLayer: + mediaType = ispec.MediaTypeImageLayerNonDistributable //nolint: staticcheck + case mediatype.Docker2LayerGzip: + mediaType = ispec.MediaTypeImageLayerGzip + default: + mediaType = desc.MediaType + } + + return ispec.Descriptor{ + MediaType: mediaType, + Digest: desc.Digest, + Size: desc.Size, + URLs: desc.URLs, + Annotations: desc.Annotations, + Platform: ispecPlatform, + ArtifactType: desc.ArtifactType, + } } diff --git a/pkg/extensions/sync/service.go b/pkg/extensions/sync/service.go index 295f4c56d..3fb63aa1a 100644 --- a/pkg/extensions/sync/service.go +++ b/pkg/extensions/sync/service.go @@ -7,46 +7,52 @@ import ( "context" "errors" "fmt" + "net/url" "strconv" + "strings" + "sync" - "github.com/containers/common/pkg/retry" - "github.com/containers/image/v5/copy" - "github.com/opencontainers/go-digest" + godigest "github.com/opencontainers/go-digest" + "github.com/regclient/regclient" + "github.com/regclient/regclient/config" + "github.com/regclient/regclient/mod" + "github.com/regclient/regclient/scheme/reg" + "github.com/regclient/regclient/types/ref" zerr "zotregistry.dev/zot/errors" - "zotregistry.dev/zot/pkg/api/config" + zconfig "zotregistry.dev/zot/pkg/api/config" "zotregistry.dev/zot/pkg/api/constants" "zotregistry.dev/zot/pkg/cluster" "zotregistry.dev/zot/pkg/common" syncconf "zotregistry.dev/zot/pkg/extensions/config/sync" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" - "zotregistry.dev/zot/pkg/extensions/sync/references" "zotregistry.dev/zot/pkg/log" mTypes "zotregistry.dev/zot/pkg/meta/types" "zotregistry.dev/zot/pkg/storage" ) +// service.refreshRegistryTemporaryCredentials(). type BaseService struct { config syncconf.RegistryConfig credentials syncconf.CredentialsFile credentialHelper CredentialHelper - clusterConfig *config.ClusterConfig remote Remote destination Destination - retryOptions *retry.RetryOptions + clusterConfig *zconfig.ClusterConfig + copyOptions []regclient.ImageOpts contentManager ContentManager storeController storage.StoreController metaDB mTypes.MetaDB repositories []string - references references.References - client *client.Client + rc *regclient.RegClient + hosts []config.Host + clientLock sync.RWMutex log log.Logger } func New( - opts syncconf.RegistryConfig, + config syncconf.RegistryConfig, credentialsFilepath string, - clusterConfig *config.ClusterConfig, + clusterConfig *zconfig.ClusterConfig, tmpDir string, storeController storage.StoreController, metadb mTypes.MetaDB, @@ -54,9 +60,11 @@ func New( ) (*BaseService, error) { service := &BaseService{} - service.config = opts + service.config = config service.log = log service.metaDB = metadb + service.contentManager = NewContentManager(config.Content, log) + service.storeController = storeController var err error @@ -99,7 +107,7 @@ func New( // can be nil if the user did not configure cluster config service.clusterConfig = clusterConfig - service.contentManager = NewContentManager(opts.Content, log) + service.contentManager = NewContentManager(config.Content, log) if len(tmpDir) == 0 { // first it will sync in tmpDir then it will move everything into local ImageStore @@ -116,141 +124,90 @@ func New( ) } - retryOptions := &retry.RetryOptions{} + service.storeController = storeController - if opts.MaxRetries != nil { - retryOptions.MaxRetry = *opts.MaxRetries - if opts.RetryDelay != nil { - retryOptions.Delay = *opts.RetryDelay - } - } + err = service.initClient() + if err != nil { + log.Err(err).Msg("failed to initialize sync client") - service.retryOptions = retryOptions - service.storeController = storeController - // try to set next client. - if err := service.SetNextAvailableClient(); err != nil { - // if it's a ping issue, it will be retried - if !errors.Is(err, zerr.ErrSyncPingRegistry) { - return service, err - } + return nil, err } - service.references = references.NewReferences( - service.client, - service.storeController, - service.metaDB, - service.log, - ) - - service.remote = NewRemoteRegistry( - service.client, - service.log, - ) + // we want referrers using sha-.*" tags + service.copyOptions = append(service.copyOptions, regclient.ImageWithDigestTags()) + // we want oci referrers + service.copyOptions = append(service.copyOptions, regclient.ImageWithReferrers()) return service, nil } -// refreshRegistryTemporaryCredentials refreshes the temporary credentials for the registry if necessary. -// It checks whether a CredentialHelper is configured and if the current credentials have expired. -// If the credentials are expired, it attempts to refresh them and updates the service configuration. -func (service *BaseService) refreshRegistryTemporaryCredentials() error { - // Exit early if no CredentialHelper is configured. - if service.config.CredentialHelper == "" { - return nil - } +func (service *BaseService) initClient() error { + service.clientLock.Lock() + defer service.clientLock.Unlock() - // Strip the transport protocol (e.g., https:// or http://) from the remote address. - remoteAddress := StripRegistryTransport(service.client.GetHostname()) - - // Exit early if the credentials are valid. - if service.credentialHelper.AreCredentialsValid(remoteAddress) { - return nil - } - - // Attempt to refresh the credentials using the CredentialHelper. - credentials, err := service.credentialHelper.RefreshCredentials(remoteAddress) + client, hosts, err := getRegClient(service.config, service.credentials) if err != nil { - service.log.Error(). - Err(err). - Str("url", remoteAddress). - Msg("failed to refresh the credentials") + service.log.Err(err).Msg("failed to parse sync config urls") return err } - service.log.Info(). - Str("url", remoteAddress). - Msg("refreshing the upstream remote registry credentials") + service.rc = client + service.hosts = hosts - // Update the service's credentials map with the new set of credentials. - service.credentials[remoteAddress] = credentials - - // Set the upstream authentication context using the refreshed credentials. - service.remote.SetUpstreamAuthConfig(credentials.Username, credentials.Password) + service.remote = NewRemoteRegistry( + service.rc, + service.hosts, + service.log, + ) - // Return nil to indicate the operation completed successfully. return nil } -func (service *BaseService) SetNextAvailableClient() error { - if service.client != nil && service.client.Ping() { - return service.refreshRegistryTemporaryCredentials() +// refreshRegistryTemporaryCredentials refreshes the temporary credentials for the registry if necessary. +// It checks whether a CredentialHelper is configured and if the current credentials have expired. +// If the credentials are expired, it attempts to refresh them and updates the service configuration. +func (service *BaseService) refreshRegistryTemporaryCredentials() error { + // Exit early if no CredentialHelper is configured. + if service.config.CredentialHelper == "" { + return nil } - found := false - - for _, url := range service.config.URLs { - // skip current client - if service.client != nil && service.client.GetBaseURL() == url { + for _, host := range service.hosts { + // Exit early if the credentials are valid. + if service.credentialHelper.AreCredentialsValid(host.Hostname) { continue } - remoteAddress := StripRegistryTransport(url) - credentials := service.credentials[remoteAddress] - - tlsVerify := true - if service.config.TLSVerify != nil { - tlsVerify = *service.config.TLSVerify - } - - options := client.Config{ - URL: url, - Username: credentials.Username, - Password: credentials.Password, - TLSVerify: tlsVerify, - CertDir: service.config.CertDir, - } - - var err error - - if service.client != nil { - err = service.client.SetConfig(options) - } else { - service.client, err = client.New(options, service.log) - } - + // Attempt to refresh the credentials using the CredentialHelper. + credentials, err := service.credentialHelper.RefreshCredentials(host.Hostname) if err != nil { - service.log.Error().Err(err).Str("url", url).Msg("failed to initialize http client") + service.log.Error(). + Err(err). + Str("url", host.Hostname). + Msg("failed to refresh the credentials") - return err + continue } - if service.client.Ping() { - found = true - - break - } - } + service.log.Info(). + Str("url", host.Hostname). + Msg("refreshing the upstream remote registry credentials") - if service.client == nil || !found { - return zerr.ErrSyncPingRegistry + // Update the service's credentials map with the new set of credentials. + service.credentials[host.Hostname] = credentials } - return nil + // Reinitialize regclient with new credentials + return service.initClient() } -func (service *BaseService) GetRetryOptions() *retry.Options { - return service.retryOptions +func (service *BaseService) CanRetryOnError() bool { + if service.config.MaxRetries != nil && *service.config.MaxRetries > 0 { + return true + } + + return false } func (service *BaseService) getNextRepoFromCatalog(lastRepo string) string { @@ -285,13 +242,13 @@ func (service *BaseService) GetNextRepo(lastRepo string) (string, error) { var err error if len(service.repositories) == 0 { - if err = retry.RetryIfNecessary(context.Background(), func() error { - service.repositories, err = service.remote.GetRepositories(context.Background()) + service.clientLock.RLock() + defer service.clientLock.RUnlock() - return err - }, service.retryOptions); err != nil { - service.log.Error().Str("errorType", common.TypeOf(err)).Str("remote registry", service.client.GetConfig().URL). - Err(err).Msg("failed to get repository list from remote registry") + service.repositories, err = service.remote.GetRepositories(context.Background()) + if err != nil { + service.log.Error().Str("errorType", common.TypeOf(err)).Str("remote registry", service.remote.GetHostName()). + Err(err).Msg("error while getting repositories from remote registry") return "", err } @@ -328,82 +285,45 @@ func (service *BaseService) GetNextRepo(lastRepo string) (string, error) { return lastRepo, nil } -// SyncReference on demand. -func (service *BaseService) SyncReference(ctx context.Context, repo string, - subjectDigestStr string, referenceType string, -) error { - remoteRepo := repo - - remoteURL := service.client.GetConfig().URL - - if len(service.config.Content) > 0 { - remoteRepo = service.contentManager.GetRepoSource(repo) - if remoteRepo == "" { - service.log.Info().Str("remote", remoteURL).Str("repository", repo).Str("subject", subjectDigestStr). - Str("reference type", referenceType).Msg("will not sync reference for image, filtered out by content") - - return zerr.ErrSyncImageFilteredOut - } - } - - remoteRepo = service.remote.GetDockerRemoteRepo(remoteRepo) - - service.log.Info().Str("remote", remoteURL).Str("repository", repo).Str("subject", subjectDigestStr). - Str("reference type", referenceType).Msg("syncing reference for image") - - return service.references.SyncReference(ctx, repo, remoteRepo, subjectDigestStr, referenceType) -} - // SyncImage on demand. func (service *BaseService) SyncImage(ctx context.Context, repo, reference string) error { remoteRepo := repo - remoteURL := service.client.GetConfig().URL + remoteURL := service.remote.GetHostName() if len(service.config.Content) > 0 { remoteRepo = service.contentManager.GetRepoSource(repo) if remoteRepo == "" { - service.log.Info().Str("remote", remoteURL).Str("repository", repo).Str("reference", reference). + service.log.Info().Str("remote", remoteURL).Str("repo", repo).Str("reference", reference). Msg("will not sync image, filtered out by content") return zerr.ErrSyncImageFilteredOut } } - remoteRepo = service.remote.GetDockerRemoteRepo(remoteRepo) - - service.log.Info().Str("remote", remoteURL).Str("repository", repo).Str("reference", reference). - Msg("syncing image") - - manifestDigest, err := service.syncTag(ctx, repo, remoteRepo, reference) - if err != nil { - return err - } + service.log.Info().Str("remote", remoteURL).Str("repo", repo).Str("reference", reference). + Msg("sync: syncing image") - err = service.references.SyncAll(ctx, repo, remoteRepo, manifestDigest.String()) - if err != nil && !errors.Is(err, zerr.ErrSyncReferrerNotFound) { - return err + if err := service.refreshRegistryTemporaryCredentials(); err != nil { + service.log.Error().Err(err).Msg("failed to refresh credentials") } - return nil + return service.syncTagAndReferrers(ctx, repo, remoteRepo, reference) } // sync repo periodically. func (service *BaseService) SyncRepo(ctx context.Context, repo string) error { - service.log.Info().Str("repository", repo).Str("registry", service.client.GetConfig().URL). - Msg("syncing repo") + service.log.Info().Str("repo", repo).Str("registry", service.remote.GetHostName()). + Msg("sync: syncing repo") var err error var tags []string - if err = retry.RetryIfNecessary(ctx, func() error { - tags, err = service.remote.GetRepoTags(repo) - - return err - }, service.retryOptions); err != nil { - service.log.Error().Str("errorType", common.TypeOf(err)).Str("repository", repo). - Err(err).Msg("failed to get tags for repository") + tags, err = service.remote.GetTags(ctx, repo) + if err != nil { + service.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo). + Err(err).Msg("error while getting tags for repo") return err } @@ -414,154 +334,276 @@ func (service *BaseService) SyncRepo(ctx context.Context, repo string) error { return err } - service.log.Info().Str("repository", repo).Msgf("syncing tags %v", tags) + service.log.Info().Str("repo", repo).Msgf("sync: syncing tags %v", tags) // apply content.destination rule - destinationRepo := service.contentManager.GetRepoDestination(repo) + localRepo := service.contentManager.GetRepoDestination(repo) for _, tag := range tags { if common.IsContextDone(ctx) { return ctx.Err() } - if references.IsCosignTag(tag) || common.IsReferrersTag(tag) { + // skip referrers, they are synced in syncTagAndReferrers. + if common.IsCosignTag(tag) || common.IsReferrersTag(tag) { continue } - var manifestDigest digest.Digest - - if err = retry.RetryIfNecessary(ctx, func() error { - manifestDigest, err = service.syncTag(ctx, destinationRepo, repo, tag) - - return err - }, service.retryOptions); err != nil { - if errors.Is(err, zerr.ErrSyncImageNotSigned) || errors.Is(err, zerr.ErrMediaTypeNotSupported) { + err = service.syncTagAndReferrers(ctx, localRepo, repo, tag) + if err != nil { + if errors.Is(err, zerr.ErrSyncImageNotSigned) || + errors.Is(err, zerr.ErrUnauthorizedAccess) || + errors.Is(err, zerr.ErrMediaTypeNotSupported) || + errors.Is(err, zerr.ErrManifestNotFound) { // skip unsigned images or unsupported image mediatype continue } - service.log.Error().Str("errorType", common.TypeOf(err)).Str("repository", repo). - Err(err).Msg("failed to sync tags for repository") + service.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo). + Err(err).Msg("error while syncing tags for repo") return err } - - if manifestDigest != "" { - if err = retry.RetryIfNecessary(ctx, func() error { - err = service.references.SyncAll(ctx, destinationRepo, repo, manifestDigest.String()) - if errors.Is(err, zerr.ErrSyncReferrerNotFound) { - return nil - } - - return err - }, service.retryOptions); err != nil { - service.log.Error().Str("errorType", common.TypeOf(err)).Str("repository", repo). - Err(err).Msg("failed to sync tags for repository") - } - } } - service.log.Info().Str("component", "sync").Str("repository", repo).Msg("finished syncing repository") + service.log.Info().Str("repo", repo).Msg("sync: finished syncing repo") return nil } -func (service *BaseService) syncTag(ctx context.Context, destinationRepo, remoteRepo, tag string, -) (digest.Digest, error) { - copyOptions := getCopyOptions(service.remote.GetContext(), service.destination.GetContext()) +func (service *BaseService) syncReference(ctx context.Context, localRepo string, remoteImageRef, localImageRef ref.Ref, + remoteManifestDigest godigest.Digest, recursive bool, +) (bool, error) { + var reference string + + if remoteImageRef.Tag != "" { + reference = remoteImageRef.Tag + } else { + reference = remoteImageRef.Digest + } + + copyOpts := []regclient.ImageOpts{} + if recursive { + copyOpts = append(copyOpts, service.copyOptions...) + } - policyContext, err := getPolicyContext(service.log) + // check if image digest + its referrers digests are already synced, otherwise sync everything again + skipImage, err := service.destination.CanSkipImage(localRepo, reference, remoteManifestDigest) if err != nil { - return "", err + service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). + Str("repo", localRepo).Str("reference", remoteImageRef.Tag). + Msg("couldn't check if the local image can be skipped") + } + + if !skipImage { + service.log.Info().Str("remote image", remoteImageRef.CommonName()). + Str("local image", fmt.Sprintf("%s:%s", localRepo, remoteImageRef.Tag)).Msg("syncing image") + + err = service.rc.ImageCopy(ctx, remoteImageRef, localImageRef, copyOpts...) + if err != nil { + service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). + Str("remote image", remoteImageRef.CommonName()). + Str("local image", fmt.Sprintf("%s:%s", localRepo, remoteImageRef.Tag)).Msg("failed to sync image") + + return false, err + } + } else { + service.log.Info().Str("image", remoteImageRef.CommonName()). + Msg("skipping image because it's already synced") + + return true, nil } - defer func() { - _ = policyContext.Destroy() - }() + return false, nil +} + +func (service *BaseService) syncTagAndReferrers(ctx context.Context, localRepo, remoteRepo, tag string) error { + service.clientLock.RLock() + defer service.clientLock.RUnlock() remoteImageRef, err := service.remote.GetImageReference(remoteRepo, tag) if err != nil { service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). Str("repository", remoteRepo).Str("reference", tag).Msg("couldn't get a remote image reference") - return "", err + return err } - _, mediaType, manifestDigest, err := service.remote.GetManifestContent(remoteImageRef) + defer service.rc.Close(ctx, remoteImageRef) + + _, remoteManifestDesc, isConverted, err := service.remote.GetOCIManifest(ctx, remoteRepo, tag) if err != nil { service.log.Error().Err(err).Str("repository", remoteRepo).Str("reference", tag). - Msg("couldn't get upstream image manifest details") + Msg("failed to get upstream image manifest details") - return "", err + return err } - if !isSupportedMediaType(mediaType) { - return "", zerr.ErrMediaTypeNotSupported + referrers, err := service.rc.ReferrerList(ctx, remoteImageRef) + if err != nil { + return err } + // if onlySigned flag true in config and the image is not itself a signature if service.config.OnlySigned != nil && *service.config.OnlySigned && - !references.IsCosignTag(tag) && !common.IsReferrersTag(tag) { - signed := service.references.IsSigned(ctx, remoteRepo, manifestDigest.String()) + !common.IsCosignSignature(tag) && !common.IsReferrersTag(tag) { + signed := hasSignatureReferrers(referrers) if !signed { // skip unsigned images - service.log.Info().Str("image", remoteImageRef.DockerReference().String()). + service.log.Info().Str("image", remoteImageRef.CommonName()). Msg("skipping image without mandatory signature") - return "", zerr.ErrSyncImageNotSigned + return zerr.ErrSyncImageNotSigned } } - skipImage, err := service.destination.CanSkipImage(destinationRepo, tag, manifestDigest) + localImageRef, err := service.destination.GetImageReference(localRepo, tag) if err != nil { service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). - Str("repository", destinationRepo).Str("reference", tag). - Msg("couldn't check if the local image can be skipped") + Str("repo", localRepo).Str("reference", localImageRef.Tag).Msg("failed to get a local image reference") + + return err } - if !skipImage { - localImageRef, err := service.destination.GetImageReference(destinationRepo, tag) + defer service.rc.Close(ctx, localImageRef) + + // just in case there is an error before commit() which cleans up. + defer service.destination.CleanupImage(localImageRef, localRepo) //nolint: errcheck + + // first sync image + _, err = service.syncReference(ctx, localRepo, remoteImageRef, localImageRef, remoteManifestDesc.Digest, false) + if err != nil { + return err + } + + tags, err := service.remote.GetTags(ctx, remoteRepo) + if err != nil { + service.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", remoteRepo). + Err(err).Msg("failed to get tags for repo") + + return err + } + + _ = service.syncAllReferrers(ctx, tags, localRepo, remoteRepo, localImageRef, remoteImageRef) + + // convert image to oci if needed + if isConverted { + localImageRef, err = mod.Apply(ctx, service.rc, localImageRef, + mod.WithRefTgt(localImageRef), + mod.WithManifestToOCI(), + mod.WithManifestToOCIReferrers(), + ) if err != nil { - service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). - Str("repository", destinationRepo).Str("reference", tag).Msg("couldn't get a local image reference") + service.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", localRepo). + Err(err).Msg("failed to convert docker image to oci") - return "", err + return err } - service.log.Info().Str("remote image", remoteImageRef.DockerReference().String()). - Str("local image", fmt.Sprintf("%s:%s", destinationRepo, tag)).Msg("syncing image") + defer service.rc.Close(ctx, localImageRef) + } - _, err = copy.Image(ctx, policyContext, localImageRef, remoteImageRef, ©Options) + // commit to storage + err = service.destination.CommitAll(localRepo, localImageRef) + if err != nil { + service.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", localRepo). + Err(err).Msg("failed to commit image") + } + + service.log.Info().Str("repo", localRepo).Str("reference", tag).Msg("successfully synced image") + + return nil +} + +// syncs all referrers recursively. +func (service *BaseService) syncAllReferrers(ctx context.Context, tags []string, localRepo, remoteRepo string, + localImageRef ref.Ref, remoteImageRef ref.Ref, +) error { + seen := []string{} + + var inner func(ctx context.Context, tags []string, localRepo, remoteRepo string, + localImageRef ref.Ref, remoteImageRef ref.Ref, seen []string, + ) error + + inner = func(ctx context.Context, tags []string, localRepo, remoteRepo string, + localImageRef ref.Ref, remoteImageRef ref.Ref, seen []string, + ) error { + // is seen + if common.Contains(seen, remoteImageRef.Digest) { + return nil + } + + remoteDigest := godigest.Digest(remoteImageRef.Digest) + + if remoteImageRef.Tag != "" { + manifest, err := service.rc.ManifestHead(ctx, remoteImageRef, regclient.WithManifestRequireDigest()) + if err != nil { + service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). + Str("repo", remoteRepo).Str("remote reference", remoteImageRef.Tag).Msg("failed to get manifest") + + return err + } + + remoteDigest = manifest.GetDescriptor().Digest + } + + seen = append(seen, remoteDigest.String()) + + referrers, err := service.rc.ReferrerList(ctx, remoteImageRef) if err != nil { - // cleanup in cases of copy.Image errors while copying. - if cErr := service.destination.CleanupImage(localImageRef, destinationRepo, tag); cErr != nil { + return err + } + + for _, desc := range referrers.Descriptors { + remoteImageRef = remoteImageRef.SetDigest(desc.Digest.String()) + + localImageRef = localImageRef.SetDigest(desc.Digest.String()) + + skipped, err := service.syncReference(ctx, localRepo, remoteImageRef, localImageRef, desc.Digest, false) + if err != nil { service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). - Str("local image", fmt.Sprintf("%s:%s", destinationRepo, tag)). - Msg("couldn't cleanup temp local image") + Str("repo", localRepo).Str("local reference", localImageRef.Tag). + Str("remote reference", remoteImageRef.Tag).Msg("failed to sync referrer") } - service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). - Str("remote image", remoteImageRef.DockerReference().String()). - Str("local image", fmt.Sprintf("%s:%s", destinationRepo, tag)).Msg("coulnd't sync image") + if skipped { + service.log.Info().Str("repo", localRepo).Str("local reference", localImageRef.Tag). + Str("remote reference", remoteImageRef.Tag).Msg("skipping syncing referrer because it's already synced") + } - return "", err + _ = inner(ctx, tags, localRepo, remoteRepo, localImageRef, remoteImageRef, seen) } - err = service.destination.CommitImage(localImageRef, destinationRepo, tag) - if err != nil { - service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). - Str("repository", destinationRepo).Str("reference", tag).Msg("couldn't commit image to local image store") + if remoteDigest != "" { + // try cosign + prefix := fmt.Sprintf("%s-%s.", remoteDigest.Algorithm(), remoteDigest.Encoded()) + for _, tag := range tags { + if strings.Contains(tag, prefix) { + remoteImageRef = remoteImageRef.SetTag(tag) - return "", err + localImageRef = localImageRef.SetTag(tag) + + skipped, err := service.syncReference(ctx, localRepo, remoteImageRef, localImageRef, remoteDigest, true) + if err != nil { + service.log.Error().Err(err).Str("errortype", common.TypeOf(err)). + Str("repo", localRepo).Str("local reference", localImageRef.Tag). + Str("remote reference", remoteImageRef.Tag).Msg("failed to sync referrer") + } + + if skipped { + service.log.Info().Str("repo", localRepo).Str("local reference", localImageRef.Tag). + Str("remote reference", remoteImageRef.Tag).Msg("skipping syncing referrer because it's already synced") + } + + _ = inner(ctx, tags, localRepo, remoteRepo, localImageRef, remoteImageRef, seen) + } + } } - } else { - service.log.Info().Str("image", remoteImageRef.DockerReference().String()). - Msg("skipping image because it's already synced") - } - service.log.Info().Str("component", "sync"). - Str("image", remoteImageRef.DockerReference().String()).Msg("finished syncing image") + return err + } - return manifestDigest, nil + return inner(ctx, tags, localRepo, remoteRepo, localImageRef, remoteImageRef, seen) } func (service *BaseService) ResetCatalog() { @@ -570,8 +612,117 @@ func (service *BaseService) ResetCatalog() { service.repositories = []string{} } -func (service *BaseService) SetNextAvailableURL() error { - service.log.Info().Msg("getting available client") +func getTLSConfigOption(url *url.URL, tlsVerify *bool) config.TLSConf { + // by default enabled + tls := config.TLSEnabled + + // tlsVerify is to false + if tlsVerify != nil { + if !*tlsVerify { + tls = config.TLSInsecure + } + } + + // conn is http => disabled + if url.Scheme == "http" { + tls = config.TLSDisabled + } + + return tls +} + +func getRegClient(opts syncconf.RegistryConfig, credentials syncconf.CredentialsFile, +) (*regclient.RegClient, []config.Host, error) { + urls, err := parseRegistryURLs(opts.URLs) + if err != nil { + return nil, nil, err + } + + mirrorsHosts := make([]string, 0) + for _, url := range urls[1:] { + mirrorsHosts = append(mirrorsHosts, url.Host) + } + + mainHost := urls[0].Host + + hostConfig := config.Host{} + + host := config.HostNew() + if host != nil { + hostConfig = *host + } + + hostConfig.Name = mainHost + hostConfig.Hostname = mainHost + hostConfig.Mirrors = mirrorsHosts + + // set TLS configuration + tls := getTLSConfigOption(urls[0], opts.TLSVerify) + hostConfig.TLS = tls + + if opts.CertDir != "" { + clientCert, clientKey, regCert, err := getCertificates(opts.CertDir) + if err != nil { + return nil, nil, err + } + + hostConfig.ClientCert = clientCert + hostConfig.ClientKey = clientKey + hostConfig.RegCert = regCert + } + + if mainHost == regclient.DockerRegistryAuth || + mainHost == regclient.DockerRegistryDNS || + mainHost == regclient.DockerRegistry || + mainHost == "index.docker.io" { + hostConfig.Name = regclient.DockerRegistry + hostConfig.Hostname = regclient.DockerRegistryDNS + hostConfig.CredHost = regclient.DockerRegistryAuth + } + + creds, ok := credentials[mainHost] + if ok { + hostConfig.User = creds.Username + hostConfig.Pass = creds.Password + } + + hostConfigOpts := []config.Host{} + hostConfigOpts = append(hostConfigOpts, hostConfig) + + for _, mirror := range mirrorsHosts { + mirroHostConfig := hostConfig + mirroHostConfig.Name = mirror + mirroHostConfig.Hostname = mirror + + creds, ok := credentials[mirror] + if ok { + mirroHostConfig.User = creds.Username + mirroHostConfig.Pass = creds.Password + } + + hostConfigOpts = append(hostConfigOpts, mirroHostConfig) + } + + regOpts := []reg.Opts{} + + if opts.CertDir != "" { + regOpts = append(regOpts, reg.WithCertDirs([]string{opts.CertDir})) + } + + if opts.MaxRetries != nil { + regOpts = append(regOpts, reg.WithRetryLimit(*opts.MaxRetries)) + } + + if opts.RetryDelay != nil { + regOpts = append(regOpts, reg.WithDelay(*opts.RetryDelay, *opts.RetryDelay)) + } + + client := regclient.New( + regclient.WithDockerCerts(), + regclient.WithDockerCreds(), + regclient.WithRegOpts(regOpts...), + regclient.WithConfigHost(hostConfigOpts...), + ) - return service.SetNextAvailableClient() + return client, hostConfigOpts, nil } diff --git a/pkg/extensions/sync/sync.go b/pkg/extensions/sync/sync.go index 7f5867c12..c57477c62 100644 --- a/pkg/extensions/sync/sync.go +++ b/pkg/extensions/sync/sync.go @@ -9,9 +9,9 @@ import ( "sync" "time" - "github.com/containers/common/pkg/retry" - "github.com/containers/image/v5/types" - "github.com/opencontainers/go-digest" + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/regclient/regclient/types/ref" syncconf "zotregistry.dev/zot/pkg/extensions/config/sync" "zotregistry.dev/zot/pkg/log" @@ -30,23 +30,17 @@ type Service interface { SyncRepo(ctx context.Context, repo string) error // used by periodically sync // Sync an image (repo:tag || repo:digest) into ImageStore. SyncImage(ctx context.Context, repo, reference string) error // used by sync on demand - // Sync a single reference for an image. - SyncReference(ctx context.Context, repo string, subjectDigestStr string, - referenceType string) error // used by sync on demand // Remove all internal catalog entries. ResetCatalog() // used by scheduler to empty out the catalog after a sync periodically roundtrip finishes - // Sync supports multiple urls per registry, before a sync repo/image/ref 'ping' each url. - SetNextAvailableURL() error // used by all sync methods - // Returns retry options from registry config. - GetRetryOptions() *retry.Options // used by sync on demand to retry in background + /* Returns if service has retry option set. + Is used by ondemand to decide if it retries pulling an image in background or not. */ + CanRetryOnError() bool // used by sync on demand to retry in background } // Local and remote registries must implement this interface. type Registry interface { // Get temporary ImageReference, is used by functions in containers/image package - GetImageReference(repo string, tag string) (types.ImageReference, error) - // Get local oci layout context, is used by functions in containers/image package - GetContext() *types.SystemContext + GetImageReference(repo string, tag string) (ref.Ref, error) } // The CredentialHelper interface should be implemented by registries that use temporary tokens. @@ -76,29 +70,25 @@ type OciLayoutStorage interface { // Remote registry. type Remote interface { Registry + // Get host name + GetHostName() string // Get a list of repos (catalog) GetRepositories(ctx context.Context) ([]string, error) // Get a list of tags given a repo - GetRepoTags(repo string) ([]string, error) - // Get manifest content, mediaType, digest given an ImageReference - GetManifestContent(imageReference types.ImageReference) ([]byte, string, digest.Digest, error) - // In the case of public dockerhub images 'library' namespace is added to the repo names of images - // eg: alpine -> library/alpine - GetDockerRemoteRepo(repo string) string - // SetUpstreamAuthConfig sets the upstream credentials used when the credential helper is set. - // This method refreshes the authentication configuration with the provided username and password. - SetUpstreamAuthConfig(username, password string) + GetTags(ctx context.Context, repo string) ([]string, error) + // Get manifest content, mediaType, descriptor given an image(if remote image is docker type then convert it to OCI) + GetOCIManifest(ctx context.Context, repo, reference string) ([]byte, ispec.Descriptor, bool, error) } // Local registry. type Destination interface { Registry - // Check if an image is already synced - CanSkipImage(repo, tag string, imageDigest digest.Digest) (bool, error) - // CommitImage moves a synced repo/ref from temporary oci layout to ImageStore - CommitImage(imageReference types.ImageReference, repo, tag string) error + // Check if descriptors are already synced + CanSkipImage(repo string, tag string, digest godigest.Digest) (bool, error) + // CommitAll moves a synced repo and all its manifests from temporary oci layout to ImageStore + CommitAll(repo string, imageReference ref.Ref) error // Removes image reference, used when copy.Image() errors out - CleanupImage(imageReference types.ImageReference, repo, reference string) error + CleanupImage(imageReference ref.Ref, repo string) error } type TaskGenerator struct { @@ -137,12 +127,6 @@ func (gen *TaskGenerator) Next() (scheduler.Task, error) { return nil, nil //nolint:nilnil } - if err := gen.Service.SetNextAvailableURL(); err != nil { - gen.increaseWaitTime() - - return nil, err - } - repo, err := gen.Service.GetNextRepo(gen.lastRepo) if err != nil { gen.increaseWaitTime() diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index 3609cb4a1..e1204691d 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -7,18 +7,11 @@ import ( "bytes" "context" "encoding/json" - "errors" - "fmt" "os" - "path" "testing" - dockerManifest "github.com/containers/image/v5/manifest" - "github.com/containers/image/v5/oci/layout" - "github.com/containers/image/v5/types" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/rs/zerolog" . "github.com/smartystreets/goconvey/convey" zerr "zotregistry.dev/zot/errors" @@ -26,7 +19,6 @@ import ( syncconf "zotregistry.dev/zot/pkg/extensions/config/sync" "zotregistry.dev/zot/pkg/extensions/lint" "zotregistry.dev/zot/pkg/extensions/monitoring" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" "zotregistry.dev/zot/pkg/log" mTypes "zotregistry.dev/zot/pkg/meta/types" "zotregistry.dev/zot/pkg/storage" @@ -34,139 +26,9 @@ import ( storageConstants "zotregistry.dev/zot/pkg/storage/constants" "zotregistry.dev/zot/pkg/storage/local" . "zotregistry.dev/zot/pkg/test/image-utils" - "zotregistry.dev/zot/pkg/test/inject" "zotregistry.dev/zot/pkg/test/mocks" - ociutils "zotregistry.dev/zot/pkg/test/oci-utils" ) -const ( - testImage = "zot-test" - testImageTag = "0.0.1" - - host = "127.0.0.1:45117" -) - -var ErrTestError = errors.New("testError") - -func TestInjectSyncUtils(t *testing.T) { - Convey("Inject errors in utils functions", t, func() { - repositoryReference := fmt.Sprintf("%s/%s", host, testImage) - ref, err := parseRepositoryReference(repositoryReference) - So(err, ShouldBeNil) - So(ref.Name(), ShouldEqual, repositoryReference) - - injected := inject.InjectFailure(0) - if injected { - _, err = getRepoTags(context.Background(), &types.SystemContext{}, host, testImage) - So(err, ShouldNotBeNil) - } - - injected = inject.InjectFailure(0) - _, err = getPolicyContext(log.NewLogger("debug", "")) - - if injected { - So(err, ShouldNotBeNil) - } else { - So(err, ShouldBeNil) - } - - log := log.Logger{Logger: zerolog.New(os.Stdout)} - metrics := monitoring.NewMetricsServer(false, log) - imageStore := local.NewImageStore(t.TempDir(), false, false, log, metrics, nil, nil, nil) - injected = inject.InjectFailure(0) - - ols := NewOciLayoutStorage(storage.StoreController{DefaultStore: imageStore}) - _, err = ols.GetImageReference(testImage, testImageTag) - - if injected { - So(err, ShouldNotBeNil) - } else { - So(err, ShouldBeNil) - } - }) -} - -func TestNilDefaultStore(t *testing.T) { - Convey("Nil default store", t, func() { - ols := NewOciLayoutStorage(storage.StoreController{}) - _, err := ols.GetImageReference(testImage, testImageTag) - So(err, ShouldEqual, zerr.ErrLocalImgStoreNotFound) - }) -} - -func TestSyncInternal(t *testing.T) { - Convey("Verify parseRepositoryReference func", t, func() { - repositoryReference := fmt.Sprintf("%s/%s", host, testImage) - ref, err := parseRepositoryReference(repositoryReference) - So(err, ShouldBeNil) - So(ref.Name(), ShouldEqual, repositoryReference) - - repositoryReference = fmt.Sprintf("%s/%s:tagged", host, testImage) - _, err = parseRepositoryReference(repositoryReference) - So(err, ShouldEqual, zerr.ErrInvalidRepositoryName) - - repositoryReference = fmt.Sprintf("http://%s/%s", host, testImage) - _, err = parseRepositoryReference(repositoryReference) - So(err, ShouldNotBeNil) - - repositoryReference = fmt.Sprintf("docker://%s/%s", host, testImage) - _, err = parseRepositoryReference(repositoryReference) - So(err, ShouldNotBeNil) - - _, err = getFileCredentials("/path/to/inexistent/file") - So(err, ShouldNotBeNil) - - tempFile, err := os.CreateTemp("", "sync-credentials-") - if err != nil { - panic(err) - } - - content := []byte(`{`) - if err := os.WriteFile(tempFile.Name(), content, 0o600); err != nil { - panic(err) - } - - _, err = getFileCredentials(tempFile.Name()) - So(err, ShouldNotBeNil) - - srcCtx := &types.SystemContext{} - _, err = getRepoTags(context.Background(), srcCtx, host, testImage) - So(err, ShouldNotBeNil) - - _, err = getRepoTags(context.Background(), srcCtx, host, testImage) - So(err, ShouldNotBeNil) - - _, err = getFileCredentials("/invalid/path/to/file") - So(err, ShouldNotBeNil) - - ok := isSupportedMediaType("unknown") - So(ok, ShouldBeFalse) - }) -} - -func TestRemoteRegistry(t *testing.T) { - Convey("test remote registry", t, func() { - logger := log.NewLogger("debug", "") - cfg := client.Config{ - URL: "url", - TLSVerify: false, - } - - client, err := client.New(cfg, logger) - So(err, ShouldBeNil) - - remote := NewRemoteRegistry(client, logger) - imageRef, err := layout.NewReference("dir", "image") - So(err, ShouldBeNil) - _, _, _, err = remote.GetManifestContent(imageRef) - So(err, ShouldNotBeNil) - - tags, err := remote.GetRepoTags("repo") - So(tags, ShouldBeEmpty) - So(err, ShouldNotBeNil) - }) -} - func TestService(t *testing.T) { Convey("trigger fetch tags error", t, func() { conf := syncconf.RegistryConfig{ @@ -181,29 +43,6 @@ func TestService(t *testing.T) { }) } -func TestSyncRepo(t *testing.T) { - Convey("trigger context error", t, func() { - conf := syncconf.RegistryConfig{ - URLs: []string{"http://localhost"}, - } - - service, err := New(conf, "", nil, os.TempDir(), storage.StoreController{}, mocks.MetaDBMock{}, log.Logger{}) - So(err, ShouldBeNil) - - service.remote = mocks.SyncRemote{ - GetRepoTagsFn: func(repo string) ([]string, error) { - return []string{"repo1", "repo2"}, nil - }, - } - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - err = service.SyncRepo(ctx, "repo") - So(err, ShouldEqual, ctx.Err()) - }) -} - func TestDestinationRegistry(t *testing.T) { Convey("make StoreController", t, func() { dir := t.TempDir() @@ -225,7 +64,7 @@ func TestDestinationRegistry(t *testing.T) { So(err, ShouldBeNil) So(imageReference, ShouldNotBeNil) - imgStore := getImageStoreFromImageReference(imageReference, repoName, "1.0", log) + imgStore := getImageStoreFromImageReference(repoName, imageReference, log) // create a blob/layer upload, err := imgStore.NewBlobUpload(repoName) @@ -313,7 +152,7 @@ func TestDestinationRegistry(t *testing.T) { So(ok, ShouldBeFalse) So(err, ShouldBeNil) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldBeNil) }) @@ -321,7 +160,7 @@ func TestDestinationRegistry(t *testing.T) { err = os.Chmod(imgStore.BlobPath(repoName, indexDigest), 0o000) So(err, ShouldBeNil) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldNotBeNil) }) @@ -340,7 +179,7 @@ func TestDestinationRegistry(t *testing.T) { storeController := storage.StoreController{DefaultStore: syncImgStore} registry := NewDestinationRegistry(storeController, storeController, nil, log) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldBeNil) }) @@ -348,7 +187,7 @@ func TestDestinationRegistry(t *testing.T) { err = os.Chmod(imgStore.BlobPath(repoName, digest), 0o000) So(err, ShouldBeNil) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldNotBeNil) }) @@ -356,7 +195,7 @@ func TestDestinationRegistry(t *testing.T) { err = os.Chmod(imgStore.BlobPath(repoName, bdgst1), 0o000) So(err, ShouldBeNil) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldNotBeNil) }) @@ -367,7 +206,7 @@ func TestDestinationRegistry(t *testing.T) { err = os.Chmod(syncImgStore.BlobPath(repoName, indexDigest), 0o000) So(err, ShouldBeNil) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldNotBeNil) }) @@ -383,7 +222,7 @@ func TestDestinationRegistry(t *testing.T) { }, }, log) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldNotBeNil) }) @@ -395,7 +234,7 @@ func TestDestinationRegistry(t *testing.T) { }, }, log) - err = registry.CommitImage(imageReference, repoName, "1.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldNotBeNil) }) @@ -404,7 +243,7 @@ func TestDestinationRegistry(t *testing.T) { So(err, ShouldBeNil) So(imageReference, ShouldNotBeNil) - imgStore := getImageStoreFromImageReference(imageReference, repoName, "2.0", log) + imgStore := getImageStoreFromImageReference(repoName, imageReference, log) // upload image @@ -473,206 +312,9 @@ func TestDestinationRegistry(t *testing.T) { So(ok, ShouldBeFalse) So(err, ShouldBeNil) - err = registry.CommitImage(imageReference, repoName, "2.0") + err = registry.CommitAll(repoName, imageReference) So(err, ShouldBeNil) }) }) }) } - -func TestConvertDockerToOCI(t *testing.T) { - Convey("test converting docker to oci functions", t, func() { - dir := t.TempDir() - - srcStorageCtlr := ociutils.GetDefaultStoreController(dir, log.NewLogger("debug", "")) - - err := WriteImageToFileSystem(CreateDefaultImage(), "zot-test", "0.0.1", srcStorageCtlr) - So(err, ShouldBeNil) - - imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.1") - So(err, ShouldBeNil) - - imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{}) - So(err, ShouldBeNil) - - defer imageSource.Close() - - Convey("trigger Unmarshal manifest error", func() { - _, err = convertDockerManifestToOCI(imageSource, []byte{}) - So(err, ShouldNotBeNil) - }) - - Convey("trigger getImageConfigContent() error", func() { - manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) - So(err, ShouldBeNil) - - var manifest ispec.Manifest - - err = json.Unmarshal(manifestBuf, &manifest) - So(err, ShouldBeNil) - - err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()), 0o000) - So(err, ShouldBeNil) - - _, err = convertDockerManifestToOCI(imageSource, manifestBuf) - So(err, ShouldNotBeNil) - }) - - Convey("trigger Unmarshal config error", func() { - manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) - So(err, ShouldBeNil) - - var manifest ispec.Manifest - - err = json.Unmarshal(manifestBuf, &manifest) - So(err, ShouldBeNil) - - err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()), - []byte{}, storageConstants.DefaultFilePerms) - So(err, ShouldBeNil) - - _, err = convertDockerManifestToOCI(imageSource, manifestBuf) - So(err, ShouldNotBeNil) - }) - - Convey("trigger convertDockerLayersToOCI error", func() { - manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) - So(err, ShouldBeNil) - - var manifest ispec.Manifest - - err = json.Unmarshal(manifestBuf, &manifest) - So(err, ShouldBeNil) - - manifestDigest := godigest.FromBytes(manifestBuf) - - manifest.Layers[0].MediaType = "unknown" - - newManifest, err := json.Marshal(manifest) - So(err, ShouldBeNil) - - err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifestDigest.Encoded()), - newManifest, storageConstants.DefaultFilePerms) - So(err, ShouldBeNil) - - _, err = convertDockerManifestToOCI(imageSource, manifestBuf) - So(err, ShouldNotBeNil) - }) - - Convey("trigger convertDockerIndexToOCI error", func() { - manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) - So(err, ShouldBeNil) - - _, err = convertDockerIndexToOCI(imageSource, manifestBuf) - So(err, ShouldNotBeNil) - - // make zot-test image an index image - - var manifest ispec.Manifest - - err = json.Unmarshal(manifestBuf, &manifest) - So(err, ShouldBeNil) - - dockerNewManifest := ispec.Manifest{ - MediaType: dockerManifest.DockerV2Schema2MediaType, - Config: manifest.Config, - Layers: manifest.Layers, - } - - dockerNewManifestBuf, err := json.Marshal(dockerNewManifest) - So(err, ShouldBeNil) - - dockerManifestDigest := godigest.FromBytes(manifestBuf) - - err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()), - dockerNewManifestBuf, storageConstants.DefaultFilePerms) - So(err, ShouldBeNil) - - var index ispec.Index - - index.Manifests = append(index.Manifests, ispec.Descriptor{ - Digest: dockerManifestDigest, - Size: int64(len(dockerNewManifestBuf)), - MediaType: dockerManifest.DockerV2Schema2MediaType, - }) - - index.MediaType = dockerManifest.DockerV2ListMediaType - - dockerIndexBuf, err := json.Marshal(index) - So(err, ShouldBeNil) - - dockerIndexDigest := godigest.FromBytes(dockerIndexBuf) - - err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerIndexDigest.Encoded()), - dockerIndexBuf, storageConstants.DefaultFilePerms) - So(err, ShouldBeNil) - - // write index.json - - var indexJSON ispec.Index - - indexJSONBuf, err := os.ReadFile(path.Join(dir, "zot-test", "index.json")) - So(err, ShouldBeNil) - - err = json.Unmarshal(indexJSONBuf, &indexJSON) - So(err, ShouldBeNil) - - indexJSON.Manifests = append(indexJSON.Manifests, ispec.Descriptor{ - Digest: dockerIndexDigest, - Size: int64(len(dockerIndexBuf)), - MediaType: ispec.MediaTypeImageIndex, - Annotations: map[string]string{ - ispec.AnnotationRefName: "0.0.2", - }, - }) - - indexJSONBuf, err = json.Marshal(indexJSON) - So(err, ShouldBeNil) - - err = os.WriteFile(path.Join(dir, "zot-test", "index.json"), indexJSONBuf, storageConstants.DefaultFilePerms) - So(err, ShouldBeNil) - - imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.2") - So(err, ShouldBeNil) - - imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{}) - So(err, ShouldBeNil) - - _, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf) - So(err, ShouldNotBeNil) - - err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()), 0o000) - So(err, ShouldBeNil) - - _, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf) - So(err, ShouldNotBeNil) - }) - }) -} - -func TestConvertDockerLayersToOCI(t *testing.T) { - Convey("test converting docker to oci functions", t, func() { - dockerLayers := []ispec.Descriptor{ - { - MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaType, - }, - { - MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaTypeGzip, - }, - { - MediaType: dockerManifest.DockerV2SchemaLayerMediaTypeUncompressed, - }, - { - MediaType: dockerManifest.DockerV2Schema2LayerMediaType, - }, - } - - err := convertDockerLayersToOCI(dockerLayers) - So(err, ShouldBeNil) - - So(dockerLayers[0].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributable) //nolint: staticcheck - So(dockerLayers[1].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributableGzip) //nolint: staticcheck - So(dockerLayers[2].MediaType, ShouldEqual, ispec.MediaTypeImageLayer) - So(dockerLayers[3].MediaType, ShouldEqual, ispec.MediaTypeImageLayerGzip) - }) -} diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 53abcae11..7477b0018 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -21,7 +21,6 @@ import ( "testing" "time" - dockerManifest "github.com/containers/image/v5/manifest" notreg "github.com/notaryproject/notation-go/registry" godigest "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" @@ -56,11 +55,15 @@ import ( ) const ( - ServerCert = "../../../test/data/server.cert" - ServerKey = "../../../test/data/server.key" - CACert = "../../../test/data/ca.crt" - ClientCert = "../../../test/data/client.cert" - ClientKey = "../../../test/data/client.key" + dockerManifestMediaType = "application/vnd.docker.distribution.manifest.v2+json" + dockerIndexManifestMediaType = "application/vnd.docker.distribution.manifest.list.v2+json" + dockerManifestConfigMediaType = "application/vnd.docker.container.image.v1+json" + dockerLayerMediaType = "application/vnd.docker.image.rootfs.diff.tar.gzip" + ServerCert = "../../../test/data/server.cert" + ServerKey = "../../../test/data/server.key" + CACert = "../../../test/data/ca.crt" + ClientCert = "../../../test/data/client.cert" + ClientKey = "../../../test/data/client.key" testImage = "zot-test" testImageTag = "0.0.1" @@ -70,10 +73,13 @@ const ( ) var ( + // no retries unless explicitly configured in each test. + maxRetries = 1 //nolint: gochecknoglobals username = "test" //nolint: gochecknoglobals password = "test" //nolint: gochecknoglobals errSync = errors.New("sync error, src oci repo differs from dest one") errBadStatus = errors.New("bad http status") + ErrTestError = errors.New("testError") ) type TagsList struct { @@ -286,10 +292,11 @@ func TestOnDemand(t *testing.T) { }, }, }, - URLs: []string{srcBaseURL}, - TLSVerify: &tlsVerify, - CertDir: "", - OnDemand: true, + URLs: []string{srcBaseURL}, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, + MaxRetries: &maxRetries, } Convey("Verify sync on demand feature with one registryConfig", func() { @@ -555,6 +562,7 @@ func TestOnDemand(t *testing.T) { hostname, err := os.Hostname() So(err, ShouldBeNil) + So(hostname, ShouldNotBeEmpty) syncRegistryConfig := syncconf.RegistryConfig{ Content: []syncconf.Content{ @@ -568,8 +576,10 @@ func TestOnDemand(t *testing.T) { }, // include self url, should be ignored URLs: []string{ - "http://" + hostname, destBaseURL, - srcBaseURL, "http://localhost:" + destPort, + fmt.Sprintf("http://%s:%s", hostname, destPort), //nolint:nosprintfhostport + destBaseURL, + srcBaseURL, + "http://localhost:" + destPort, }, TLSVerify: &tlsVerify, CertDir: "", @@ -602,7 +612,7 @@ func TestOnDemand(t *testing.T) { sm mTypes.SignatureMetadata, ) error { if sm.SignatureType == zcommon.CosignSignature || sm.SignatureType == zcommon.NotationSignature { - return sync.ErrTestError + return ErrTestError } return nil @@ -612,7 +622,7 @@ func TestOnDemand(t *testing.T) { (strings.HasSuffix(reference, remote.SignatureTagSuffix) || strings.HasSuffix(reference, remote.SBOMTagSuffix)) || strings.HasPrefix(reference, "sha256:") { - return sync.ErrTestError + return ErrTestError } // don't return err for normal image with tag @@ -717,7 +727,7 @@ func TestOnDemand(t *testing.T) { }, // include self url, should be ignored URLs: []string{ - "http://" + hostname, destBaseURL, + fmt.Sprintf("http://%s:%s", hostname, destPort), destBaseURL, //nolint:nosprintfhostport srcBaseURL, "http://localhost:" + destPort, }, TLSVerify: &tlsVerify, @@ -749,7 +759,7 @@ func TestOnDemand(t *testing.T) { dctlr.MetaDB = mocks.MetaDBMock{ SetRepoReferenceFn: func(ctx context.Context, repo, reference string, imageMeta mTypes.ImageMeta) error { if imageMeta.Digest.String() == ociRefImage.ManifestDescriptor.Digest.String() { - return sync.ErrTestError + return ErrTestError } return nil @@ -1363,12 +1373,12 @@ func TestDockerImagesAreSkipped(t *testing.T) { configBlobDigest = manifest.Config.Digest - manifest.MediaType = dockerManifest.DockerV2Schema2MediaType - manifest.Config.MediaType = dockerManifest.DockerV2Schema2ConfigMediaType - index.Manifests[idx].MediaType = dockerManifest.DockerV2Schema2MediaType + manifest.MediaType = dockerManifestMediaType + manifest.Config.MediaType = dockerManifestConfigMediaType + index.Manifests[idx].MediaType = dockerManifestMediaType for idx := range manifest.Layers { - manifest.Layers[idx].MediaType = dockerManifest.DockerV2Schema2LayerMediaType + manifest.Layers[idx].MediaType = dockerLayerMediaType } manifestBuf, err := json.Marshal(manifest) @@ -1503,13 +1513,13 @@ func TestDockerImagesAreSkipped(t *testing.T) { configBlobDigest = manifest.Config.Digest - manifest.MediaType = dockerManifest.DockerV2Schema2MediaType - manifest.Config.MediaType = dockerManifest.DockerV2Schema2ConfigMediaType - newIndex.Manifests[idx].MediaType = dockerManifest.DockerV2Schema2MediaType - indexManifest.Manifests[idx].MediaType = dockerManifest.DockerV2Schema2MediaType + manifest.MediaType = dockerManifestMediaType + manifest.Config.MediaType = dockerManifestConfigMediaType + newIndex.Manifests[idx].MediaType = dockerManifestMediaType + indexManifest.Manifests[idx].MediaType = dockerManifestMediaType for idx := range manifest.Layers { - manifest.Layers[idx].MediaType = dockerManifest.DockerV2Schema2LayerMediaType + manifest.Layers[idx].MediaType = dockerLayerMediaType } manifestBuf, err := json.Marshal(manifest) @@ -1528,7 +1538,7 @@ func TestDockerImagesAreSkipped(t *testing.T) { So(err, ShouldBeNil) } - indexManifest.MediaType = dockerManifest.DockerV2ListMediaType + indexManifest.MediaType = dockerIndexManifestMediaType // write converted multi arch manifest indexManifestContent, err = json.Marshal(indexManifest) So(err, ShouldBeNil) @@ -1538,7 +1548,7 @@ func TestDockerImagesAreSkipped(t *testing.T) { So(err, ShouldBeNil) } - newIndex.Manifests[indexManifestIdx].MediaType = dockerManifest.DockerV2ListMediaType + newIndex.Manifests[indexManifestIdx].MediaType = dockerIndexManifestMediaType newIndex.Manifests[indexManifestIdx].Digest = godigest.FromBytes(indexManifestContent) indexBuf, err := json.Marshal(newIndex) @@ -1711,6 +1721,7 @@ func TestPeriodically(t *testing.T) { PollInterval: updateDuration, TLSVerify: &tlsVerify, CertDir: "", + MaxRetries: &maxRetries, } syncConfig := &syncconf.Config{ @@ -1928,6 +1939,7 @@ func TestPermsDenied(t *testing.T) { TLSVerify: &tlsVerify, CertDir: "", OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -1968,7 +1980,7 @@ func TestPermsDenied(t *testing.T) { dcm.StartAndWait(destPort) found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't get a local image reference", 50*time.Second) + "failed to sync image", 50*time.Second) if err != nil { panic(err) } @@ -2022,6 +2034,7 @@ func TestConfigReloader(t *testing.T) { TLSVerify: &tlsVerify, CertDir: "", OnDemand: true, + MaxRetries: &maxRetries, } syncConfig := &syncconf.Config{ @@ -2299,6 +2312,7 @@ func TestMandatoryAnnotations(t *testing.T) { OnDemand: false, PollInterval: updateDuration, TLSVerify: &tlsVerify, + MaxRetries: &maxRetries, } defaultVal := true @@ -2343,7 +2357,7 @@ func TestMandatoryAnnotations(t *testing.T) { defer dcm.StopServer() found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't upload manifest because of missing annotations", 15*time.Second) + "failed to upload manifest because of missing annotations", 15*time.Second) if err != nil { panic(err) } @@ -2396,6 +2410,7 @@ func TestBadTLS(t *testing.T) { OnDemand: true, PollInterval: updateDuration, TLSVerify: &tlsVerify, + MaxRetries: &maxRetries, } defaultVal := true @@ -2412,7 +2427,7 @@ func TestBadTLS(t *testing.T) { defer dcm.StopServer() found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "x509: certificate signed by unknown authority", 15*time.Second) + "tls: failed to verify certificate: x509: certificate signed by unknown authority", 40*time.Second) if err != nil { panic(err) } @@ -2509,6 +2524,7 @@ func TestTLS(t *testing.T) { PollInterval: updateDuration, TLSVerify: &tlsVerify, CertDir: destClientCertDir, + MaxRetries: &maxRetries, } defaultVal := true @@ -2517,7 +2533,7 @@ func TestTLS(t *testing.T) { Registries: []syncconf.RegistryConfig{syncRegistryConfig}, } - dctlr, _, destDir, _ := makeDownstreamServer(t, true, syncConfig) + dctlr, destBaseURL, destDir, destClient := makeDownstreamServer(t, true, syncConfig) dcm := test.NewControllerManager(dctlr) dcm.StartAndWait(dctlr.Config.HTTP.Port) @@ -2549,6 +2565,10 @@ func TestTLS(t *testing.T) { } waitSyncFinish(dctlr.Config.Log.Output) + + resp, err := destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) }) } @@ -2595,6 +2615,7 @@ func TestBearerAuth(t *testing.T) { PollInterval: updateDuration, TLSVerify: &tlsVerify, CertDir: "", + MaxRetries: &maxRetries, } defaultVal := true @@ -2739,10 +2760,11 @@ func TestBearerAuth(t *testing.T) { Prefix: "**", // sync everything }, }, - URLs: []string{srcBaseURL}, - TLSVerify: &tlsVerify, - OnDemand: true, - CertDir: "", + URLs: []string{srcBaseURL}, + TLSVerify: &tlsVerify, + OnDemand: true, + CertDir: "", + MaxRetries: &maxRetries, } defaultVal := true @@ -2870,6 +2892,7 @@ func TestBasicAuth(t *testing.T) { PollInterval: updateDuration, TLSVerify: &tlsVerify, CertDir: "", + MaxRetries: &maxRetries, } defaultVal := true @@ -2977,6 +3000,7 @@ func TestBasicAuth(t *testing.T) { TLSVerify: &tlsVerify, CertDir: "", OnDemand: true, + MaxRetries: &maxRetries, } destConfig.Extensions = &extconf.ExtensionConfig{} @@ -2996,7 +3020,7 @@ func TestBasicAuth(t *testing.T) { defer dcm.StopServer() found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "authentication required", 15*time.Second) + "unauthorized", 15*time.Second) if err != nil { panic(err) } @@ -3057,6 +3081,7 @@ func TestBasicAuth(t *testing.T) { PollInterval: updateDuration, TLSVerify: &tlsVerify, CertDir: "", + MaxRetries: &maxRetries, } defaultVal := true @@ -3106,19 +3131,22 @@ func TestBasicAuth(t *testing.T) { defaultValue := false syncRegistryConfig := syncconf.RegistryConfig{ - URLs: []string{srcBaseURL}, - TLSVerify: &defaultValue, - OnDemand: true, + URLs: []string{srcBaseURL}, + TLSVerify: &defaultValue, + OnDemand: true, + MaxRetries: &maxRetries, } unreacheableSyncRegistryConfig1 := syncconf.RegistryConfig{ - URLs: []string{"localhost:9999"}, - OnDemand: true, + URLs: []string{"localhost:9999"}, + OnDemand: true, + MaxRetries: &maxRetries, } unreacheableSyncRegistryConfig2 := syncconf.RegistryConfig{ - URLs: []string{"localhost:9999"}, - OnDemand: false, + URLs: []string{"localhost:9999"}, + OnDemand: false, + MaxRetries: &maxRetries, } defaultVal := true @@ -3221,6 +3249,7 @@ func TestBadURL(t *testing.T) { TLSVerify: &tlsVerify, CertDir: "", OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -3270,6 +3299,7 @@ func TestNoImagesByRegex(t *testing.T) { TLSVerify: &tlsVerify, PollInterval: updateDuration, CertDir: "", + MaxRetries: &maxRetries, } defaultVal := true @@ -3334,6 +3364,7 @@ func TestInvalidRegex(t *testing.T) { PollInterval: updateDuration, CertDir: "", OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -3409,6 +3440,7 @@ func TestNotSemver(t *testing.T) { PollInterval: updateDuration, TLSVerify: &tlsVerify, CertDir: "", + MaxRetries: &maxRetries, } defaultVal := true @@ -3507,6 +3539,7 @@ func TestInvalidCerts(t *testing.T) { TLSVerify: &tlsVerify, CertDir: clientCertDir, OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -3570,6 +3603,7 @@ func TestCertsWithWrongPerms(t *testing.T) { TLSVerify: &tlsVerify, CertDir: clientCertDir, OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -3644,6 +3678,7 @@ func TestInvalidUrl(t *testing.T) { TLSVerify: &tlsVerify, CertDir: "", OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -3699,6 +3734,7 @@ func TestInvalidTags(t *testing.T) { TLSVerify: &tlsVerify, CertDir: "", OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -3774,6 +3810,7 @@ func TestSubPaths(t *testing.T) { TLSVerify: &tlsVerify, CertDir: "", OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -3859,10 +3896,11 @@ func TestOnDemandRepoErr(t *testing.T) { Prefix: testImage, }, }, - URLs: []string{"docker://invalid"}, - TLSVerify: &tlsVerify, - CertDir: "", - OnDemand: true, + URLs: []string{"docker://invalid"}, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -3912,10 +3950,11 @@ func TestOnDemandContentFiltering(t *testing.T) { }, }, }, - URLs: []string{srcBaseURL}, - TLSVerify: &tlsVerify, - CertDir: "", - OnDemand: true, + URLs: []string{srcBaseURL}, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -3953,10 +3992,11 @@ func TestOnDemandContentFiltering(t *testing.T) { }, }, }, - URLs: []string{srcBaseURL}, - TLSVerify: &tlsVerify, - CertDir: "", - OnDemand: true, + URLs: []string{srcBaseURL}, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -4005,10 +4045,11 @@ func TestConfigRules(t *testing.T) { }, }, }, - URLs: []string{srcBaseURL}, - TLSVerify: &tlsVerify, - CertDir: "", - OnDemand: false, + URLs: []string{srcBaseURL}, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: false, + MaxRetries: &maxRetries, } defaultVal := true @@ -4040,6 +4081,7 @@ func TestConfigRules(t *testing.T) { TLSVerify: &tlsVerify, CertDir: "", OnDemand: false, + MaxRetries: &maxRetries, } defaultVal := true @@ -4114,10 +4156,11 @@ func TestMultipleURLs(t *testing.T) { }, }, }, - URLs: []string{"badURL", "@!#!$#@%", "http://invalid.invalid/invalid/", srcBaseURL}, + URLs: []string{"http://badURL", srcBaseURL}, PollInterval: updateDuration, TLSVerify: &tlsVerify, CertDir: "", + MaxRetries: &maxRetries, } defaultVal := true @@ -4194,6 +4237,7 @@ func TestNoURLsLeftInConfig(t *testing.T) { PollInterval: updateDuration, TLSVerify: &tlsVerify, CertDir: "", + MaxRetries: &maxRetries, } defaultVal := true @@ -4270,6 +4314,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) { TLSVerify: &tlsVerify, CertDir: "", OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -4307,7 +4352,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) { defer dcm.StopServer() found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "finished syncing all repos", 15*time.Second) + "failed to get upstream image manifest details", 60*time.Second) if err != nil { panic(err) } @@ -4355,7 +4400,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) { defer dcm.StopServer() found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "finished syncing all repos", 15*time.Second) + "failed to sync image", 60*time.Second) if err != nil { panic(err) } @@ -4425,7 +4470,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) { defer dcm.StopServer() found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "finished syncing all repos", 15*time.Second) + "failed to sync image", 30*time.Second) if err != nil { panic(err) } @@ -4454,7 +4499,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) { So(len(index.Manifests), ShouldEqual, 0) }) - Convey("Trigger error on oci refs of both mediatypes", func() { + Convey("Trigger error on oci ref", func() { artifactURLPath := path.Join("/v2", repoName, "referrers", imageManifestDigest.String()) // based on image manifest digest get referrers @@ -4472,191 +4517,82 @@ func TestPeriodicallySignaturesErr(t *testing.T) { err = json.Unmarshal(resp.Body(), &referrers) So(err, ShouldBeNil) - Convey("of type OCI image", func() { //nolint: dupl - // read manifest - var artifactManifest ispec.Manifest - - for _, ref := range referrers.Manifests { - refPath := path.Join(srcDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Encoded()) - body, err := os.ReadFile(refPath) - So(err, ShouldBeNil) - - err = json.Unmarshal(body, &artifactManifest) - So(err, ShouldBeNil) - - // triggers perm denied on artifact blobs - for _, blob := range artifactManifest.Layers { - blobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Encoded()) - err := os.Chmod(blobPath, 0o000) - So(err, ShouldBeNil) - } - } - - // start downstream server - updateDuration, err = time.ParseDuration("1s") - So(err, ShouldBeNil) - - syncConfig.Registries[0].PollInterval = updateDuration - - // start downstream server - dctlr, destBaseURL, _, _ := makeDownstreamServer(t, false, syncConfig) - - dcm := test.NewControllerManager(dctlr) - dcm.StartAndWait(dctlr.Config.HTTP.Port) - defer dcm.StopServer() - - found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't sync image referrer", 15*time.Second) - if err != nil { - panic(err) - } - - if !found { - data, err := os.ReadFile(dctlr.Config.Log.Output) - So(err, ShouldBeNil) - - t.Logf("downstream log: %s", string(data)) - } - - So(found, ShouldBeTrue) + // read manifest + var artifactManifest ispec.Manifest - // should not be synced nor sync on demand - resp, err = resty.R(). - SetHeader("Content-Type", "application/json"). - SetQueryParam("artifactType", "application/vnd.cncf.icecream"). - Get(destBaseURL + artifactURLPath) + for _, ref := range referrers.Manifests { + refPath := path.Join(srcDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Encoded()) + body, err := os.ReadFile(refPath) So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) - - var index ispec.Index - err = json.Unmarshal(resp.Body(), &index) + err = json.Unmarshal(body, &artifactManifest) So(err, ShouldBeNil) - So(len(index.Manifests), ShouldEqual, 0) - }) - Convey("of type OCI artifact", func() { //nolint: dupl - // read manifest - var artifactManifest ispec.Manifest - - for _, ref := range referrers.Manifests { - refPath := path.Join(srcDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Encoded()) - body, err := os.ReadFile(refPath) - So(err, ShouldBeNil) - - err = json.Unmarshal(body, &artifactManifest) - So(err, ShouldBeNil) - - // triggers perm denied on artifact blobs - for _, blob := range artifactManifest.Layers { - blobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Encoded()) - err := os.Chmod(blobPath, 0o000) - So(err, ShouldBeNil) - } - } - - // start downstream server - dctlr, destBaseURL, _, _ := makeDownstreamServer(t, false, syncConfig) - - dcm := test.NewControllerManager(dctlr) - dcm.StartAndWait(dctlr.Config.HTTP.Port) - defer dcm.StopServer() - - found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't sync image referrer", 15*time.Second) - if err != nil { - panic(err) - } - - if !found { - data, err := os.ReadFile(dctlr.Config.Log.Output) + // triggers perm denied on artifact blobs + for _, blob := range artifactManifest.Layers { + blobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Encoded()) + err := os.Chmod(blobPath, 0o000) So(err, ShouldBeNil) - t.Logf("downstream log: %s", string(data)) + break } + } - So(found, ShouldBeTrue) + // start downstream server + updateDuration, err = time.ParseDuration("10m") + So(err, ShouldBeNil) + retries := 1 + syncConfig.Registries[0].PollInterval = updateDuration + syncConfig.Registries[0].MaxRetries = &retries + // syncConfig.Registries[0].OnDemand = false - // should not be synced nor sync on demand - resp, err = resty.R(). - SetHeader("Content-Type", "application/json"). - SetQueryParam("artifactType", "application/vnd.cncf.icecream"). - Get(destBaseURL + artifactURLPath) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) + // start downstream server + dctlr, destBaseURL, _, _ := makeDownstreamServer(t, false, syncConfig) - var index ispec.Index + dcm := test.NewControllerManager(dctlr) + dcm.StartAndWait(dctlr.Config.HTTP.Port) + defer dcm.StopServer() - err = json.Unmarshal(resp.Body(), &index) - So(err, ShouldBeNil) - So(len(index.Manifests), ShouldEqual, 0) - }) + found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, + "failed to sync image", 30*time.Second) + if err != nil { + panic(err) + } - Convey("of type OCI image, error on downstream in canSkipReference()", func() { //nolint: dupl - // start downstream server - updateDuration, err = time.ParseDuration("1s") + if !found { + data, err := os.ReadFile(dctlr.Config.Log.Output) So(err, ShouldBeNil) - syncConfig.Registries[0].PollInterval = updateDuration - dctlr, _, destDir, _ := makeDownstreamServer(t, false, syncConfig) - - dcm := test.NewControllerManager(dctlr) - dcm.StartAndWait(dctlr.Config.HTTP.Port) - defer dcm.StopServer() - - found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "finished syncing all repos", 15*time.Second) - if err != nil { - panic(err) - } - - if !found { - data, err := os.ReadFile(dctlr.Config.Log.Output) - So(err, ShouldBeNil) - - t.Logf("downstream log: %s", string(data)) - } - - So(found, ShouldBeTrue) - - time.Sleep(time.Second) - - blob := referrers.Manifests[0] - blobsDir := path.Join(destDir, repoName, "blobs", string(blob.Digest.Algorithm())) - blobPath := path.Join(blobsDir, blob.Digest.Encoded()) - err = os.MkdirAll(blobsDir, storageConstants.DefaultDirPerms) - So(err, ShouldBeNil) - err = os.WriteFile(blobPath, []byte("blob"), storageConstants.DefaultFilePerms) - So(err, ShouldBeNil) - err = os.Chmod(blobPath, 0o000) - So(err, ShouldBeNil) + t.Logf("downstream log: %s", string(data)) + } - found, err = test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't check if the upstream oci references for image can be skipped", 30*time.Second) - if err != nil { - panic(err) - } + So(found, ShouldBeTrue) - if !found { - data, err := os.ReadFile(dctlr.Config.Log.Output) - So(err, ShouldBeNil) + // should not be synced nor sync on demand + resp, err = resty.R(). + SetHeader("Content-Type", "application/json"). + SetQueryParam("artifactType", "application/vnd.cncf.icecream"). + Get(destBaseURL + artifactURLPath) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) - t.Logf("downstream log: %s", string(data)) - } + var index ispec.Index - So(found, ShouldBeTrue) - }) + err = json.Unmarshal(resp.Body(), &index) + So(err, ShouldBeNil) + So(len(index.Manifests), ShouldEqual, 0) }) }) } func TestSignatures(t *testing.T) { Convey("Verify sync signatures", t, func() { - updateDuration, _ := time.ParseDuration("30m") + updateDuration, _ := time.ParseDuration("1m") sctlr, srcBaseURL, srcDir, _, _ := makeUpstreamServer(t, false, false) scm := test.NewControllerManager(sctlr) + scm.StartAndWait(sctlr.Config.HTTP.Port) defer scm.StopServer() @@ -4753,7 +4689,7 @@ func TestSignatures(t *testing.T) { syncRegistryConfig := syncconf.RegistryConfig{ Content: []syncconf.Content{ { - Prefix: "**", + Prefix: repoName, Tags: &syncconf.Tags{ Regex: ®ex, Semver: &semver, @@ -4766,6 +4702,7 @@ func TestSignatures(t *testing.T) { CertDir: "", OnlySigned: &onlySigned, OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -4781,31 +4718,15 @@ func TestSignatures(t *testing.T) { defer dcm.StopServer() - // wait for sync - var destTagsList TagsList - - for { - resp, err := destClient.R().Get(destBaseURL + "/v2/" + repoName + "/tags/list") - if err != nil { - panic(err) - } - - err = json.Unmarshal(resp.Body(), &destTagsList) - if err != nil { - panic(err) - } - - if len(destTagsList.Tags) > 0 { - break - } - - time.Sleep(500 * time.Millisecond) - } + // sync image with all its refs + resp, err = destClient.R().Get(destBaseURL + "/v2/" + repoName + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) splittedURL = strings.SplitAfter(destBaseURL, ":") destPort := splittedURL[len(splittedURL)-1] - time.Sleep(1 * time.Second) + time.Sleep(5 * time.Second) // notation verify the image image := fmt.Sprintf("localhost:%s/%s@%s", destPort, repoName, digest) @@ -4834,7 +4755,6 @@ func TestSignatures(t *testing.T) { So(resp.StatusCode(), ShouldEqual, http.StatusOK) var index ispec.Index - err = json.Unmarshal(resp.Body(), &index) So(err, ShouldBeNil) @@ -5231,6 +5151,7 @@ func TestSignatures(t *testing.T) { CertDir: "", OnlySigned: &onlySigned, OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -5295,7 +5216,6 @@ func TestSyncedSignaturesMetaDB(t *testing.T) { Convey("Verify that metadb update correctly when syncing a signature", t, func() { repoName := "signed-repo" tag := "random-signed-image" - updateDuration := 30 * time.Minute // Create source registry @@ -5341,11 +5261,10 @@ func TestSyncedSignaturesMetaDB(t *testing.T) { Tags: &syncconf.Tags{Regex: ®ex, Semver: &semver}, }, }, - URLs: []string{srcBaseURL}, - PollInterval: updateDuration, - TLSVerify: &tlsVerify, - CertDir: "", - OnDemand: true, + URLs: []string{srcBaseURL}, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, }, }, } @@ -5363,10 +5282,12 @@ func TestSyncedSignaturesMetaDB(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusOK) + // regclient will put all referrers under ref tag "alg-subjectDigest" repoMeta, err := dctlr.MetaDB.GetRepoMeta(context.Background(), repoName) So(err, ShouldBeNil) So(repoMeta.Tags, ShouldContainKey, tag) - So(len(repoMeta.Tags), ShouldEqual, 1) + // one tag for refs and the tag we pushed earlier + So(len(repoMeta.Tags), ShouldEqual, 2) So(repoMeta.Signatures, ShouldContainKey, signedImage.DigestStr()) imageSignatures := repoMeta.Signatures[signedImage.DigestStr()] @@ -5450,7 +5371,7 @@ func TestOnDemandRetryGoroutine(t *testing.T) { // in the meantime ondemand should retry syncing found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "successfully synced image", 15*time.Second) + "successfully synced image", 60*time.Second) if err != nil { panic(err) } @@ -5500,10 +5421,11 @@ func TestOnDemandWithDigest(t *testing.T) { }, }, }, - URLs: []string{srcBaseURL}, - OnDemand: true, - TLSVerify: &tlsVerify, - CertDir: "", + URLs: []string{srcBaseURL}, + OnDemand: true, + TLSVerify: &tlsVerify, + CertDir: "", + MaxRetries: &maxRetries, } defaultVal := true @@ -5549,10 +5471,11 @@ func TestOnDemandRetryGoroutineErr(t *testing.T) { }, }, }, - URLs: []string{"http://127.0.0.1"}, - OnDemand: true, - TLSVerify: &tlsVerify, - CertDir: "", + URLs: []string{"http://127.0.0.1"}, + OnDemand: true, + TLSVerify: &tlsVerify, + CertDir: "", + MaxRetries: &maxRetries, } maxRetries := 1 @@ -5578,7 +5501,7 @@ func TestOnDemandRetryGoroutineErr(t *testing.T) { So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "failed to copy image", 15*time.Second) + "failed to sync image", 15*time.Second) if err != nil { panic(err) } @@ -5701,7 +5624,7 @@ func TestOnDemandMultipleImage(t *testing.T) { time.Sleep(500 * time.Millisecond) } - waitSync(destDir, testImage) + // waitSync(destDir, testImage) So(len(populatedDirs), ShouldEqual, 1) @@ -5736,10 +5659,11 @@ func TestOnDemandPullsOnce(t *testing.T) { }, }, }, - URLs: []string{srcBaseURL}, - TLSVerify: &tlsVerify, - CertDir: "", - OnDemand: true, + URLs: []string{srcBaseURL}, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -5847,10 +5771,11 @@ func TestSignaturesOnDemand(t *testing.T) { var tlsVerify bool syncRegistryConfig := syncconf.RegistryConfig{ - URLs: []string{srcBaseURL}, - TLSVerify: &tlsVerify, - CertDir: "", - OnDemand: true, + URLs: []string{srcBaseURL}, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -5973,10 +5898,11 @@ func TestSignaturesOnDemand(t *testing.T) { var tlsVerify bool syncRegistryConfig := syncconf.RegistryConfig{ - URLs: []string{srcBaseURL}, - TLSVerify: &tlsVerify, - CertDir: "", - OnDemand: true, + URLs: []string{srcBaseURL}, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, + MaxRetries: &maxRetries, } defaultVal := true @@ -6036,7 +5962,7 @@ func TestSignaturesOnDemand(t *testing.T) { So(resp.StatusCode(), ShouldEqual, http.StatusOK) found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't find any oci reference", 15*time.Second) + "failed to sync referrer", 15*time.Second) if err != nil { panic(err) } @@ -6080,24 +6006,22 @@ func TestOnlySignaturesOnDemand(t *testing.T) { var tlsVerify bool - syncRegistryConfig := syncconf.RegistryConfig{ - URLs: []string{srcBaseURL}, - TLSVerify: &tlsVerify, - CertDir: "", - OnDemand: true, - } + retries := 0 - syncBadRegistryConfig := syncconf.RegistryConfig{ - URLs: []string{"http://invalid.invalid:9999"}, - TLSVerify: &tlsVerify, - CertDir: "", - OnDemand: true, + syncRegistryConfig := syncconf.RegistryConfig{ + URLs: []string{srcBaseURL}, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, + MaxRetries: &retries, } defaultVal := true syncConfig := &syncconf.Config{ - Enable: &defaultVal, - Registries: []syncconf.RegistryConfig{syncBadRegistryConfig, syncRegistryConfig}, + Enable: &defaultVal, + Registries: []syncconf.RegistryConfig{ + syncRegistryConfig, + }, } dctlr, destBaseURL, _, _ := makeDownstreamServer(t, false, syncConfig) @@ -6191,6 +6115,7 @@ func TestSyncOnlyDiff(t *testing.T) { TLSVerify: &tlsVerify, CertDir: "", OnDemand: false, + MaxRetries: &maxRetries, } defaultVal := true @@ -6278,6 +6203,7 @@ func TestSyncWithDiffDigest(t *testing.T) { TLSVerify: &tlsVerify, CertDir: "", OnDemand: false, + MaxRetries: &maxRetries, } defaultVal := true @@ -6455,6 +6381,7 @@ func TestSyncSignaturesDiff(t *testing.T) { TLSVerify: &tlsVerify, CertDir: "", OnDemand: false, + MaxRetries: &maxRetries, } defaultVal := true @@ -6491,7 +6418,7 @@ func TestSyncSignaturesDiff(t *testing.T) { time.Sleep(500 * time.Millisecond) } - time.Sleep(3 * time.Second) + time.Sleep(15 * time.Second) splittedURL = strings.SplitAfter(destBaseURL, ":") destPort := splittedURL[len(splittedURL)-1] @@ -6522,7 +6449,7 @@ func TestSyncSignaturesDiff(t *testing.T) { So(func() { signImage(tdir, srcPort, repoName, digest) }, ShouldNotPanic) // wait for signatures - time.Sleep(12 * time.Second) + time.Sleep(15 * time.Second) // notation verify the image image = fmt.Sprintf("localhost:%s/%s:%s", destPort, repoName, testImageTag) @@ -6614,7 +6541,7 @@ func TestSyncSignaturesDiff(t *testing.T) { So(reflect.DeepEqual(cosignManifest, syncedCosignManifest), ShouldEqual, true) found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "skipping syncing cosign reference", 15*time.Second) + "skipping syncing referrer because it's already synced", 30*time.Second) if err != nil { panic(err) } @@ -6629,7 +6556,7 @@ func TestSyncSignaturesDiff(t *testing.T) { So(found, ShouldBeTrue) found, err = test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "skipping oci references", 15*time.Second) + "skipping syncing referrer because it's already synced", 30*time.Second) if err != nil { panic(err) } @@ -6678,6 +6605,7 @@ func TestOnlySignedFlag(t *testing.T) { TLSVerify: &tlsVerify, CertDir: "", OnlySigned: &onlySigned, + MaxRetries: &maxRetries, } defaultVal := true @@ -6830,6 +6758,7 @@ func TestSyncWithDestination(t *testing.T) { OnDemand: false, PollInterval: updateDuration, TLSVerify: &tlsVerify, + MaxRetries: &maxRetries, } defaultVal := true @@ -6885,10 +6814,11 @@ func TestSyncWithDestination(t *testing.T) { for _, testCase := range testCases { tlsVerify := false syncRegistryConfig := syncconf.RegistryConfig{ - Content: []syncconf.Content{testCase.content}, - URLs: []string{srcBaseURL}, - OnDemand: true, - TLSVerify: &tlsVerify, + Content: []syncconf.Content{testCase.content}, + URLs: []string{srcBaseURL}, + OnDemand: true, + TLSVerify: &tlsVerify, + MaxRetries: &maxRetries, } defaultVal := true @@ -6962,6 +6892,7 @@ func TestSyncImageIndex(t *testing.T) { OnDemand: false, PollInterval: updateDuration, TLSVerify: &tlsVerify, + MaxRetries: &maxRetries, } defaultVal := true diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go index 16aab5e69..95281344b 100644 --- a/pkg/extensions/sync/utils.go +++ b/pkg/extensions/sync/utils.go @@ -4,29 +4,16 @@ package sync import ( - "bytes" - "context" "encoding/json" - "fmt" - "io" + "net/url" "os" + "path" "strings" - "github.com/containers/image/v5/copy" - "github.com/containers/image/v5/docker" - "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/manifest" - "github.com/containers/image/v5/pkg/blobinfocache/none" - "github.com/containers/image/v5/signature" - "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" - zerr "zotregistry.dev/zot/errors" - "zotregistry.dev/zot/pkg/common" syncconf "zotregistry.dev/zot/pkg/extensions/config/sync" - "zotregistry.dev/zot/pkg/log" - "zotregistry.dev/zot/pkg/test/inject" ) // Get sync.FileCredentials from file. @@ -46,71 +33,6 @@ func getFileCredentials(filepath string) (syncconf.CredentialsFile, error) { return creds, nil } -func getUpstreamContext(certDir, username, password string, tlsVerify bool) *types.SystemContext { - upstreamCtx := &types.SystemContext{} - upstreamCtx.DockerCertPath = certDir - upstreamCtx.DockerDaemonCertPath = certDir - - if tlsVerify { - upstreamCtx.DockerDaemonInsecureSkipTLSVerify = false - upstreamCtx.DockerInsecureSkipTLSVerify = types.NewOptionalBool(false) - } else { - upstreamCtx.DockerDaemonInsecureSkipTLSVerify = true - upstreamCtx.DockerInsecureSkipTLSVerify = types.NewOptionalBool(true) - } - - if username != "" && password != "" { - upstreamCtx.DockerAuthConfig = &types.DockerAuthConfig{ - Username: username, - Password: password, - } - } - - return upstreamCtx -} - -// sync needs transport to be stripped to not be wrongly interpreted as an image reference -// at a non-fully qualified registry (hostname as image and port as tag). -func StripRegistryTransport(url string) string { - return strings.Replace(strings.Replace(url, "http://", "", 1), "https://", "", 1) -} - -// getRepoTags lists all tags in a repository. -// It returns a string slice of tags and any error encountered. -func getRepoTags(ctx context.Context, sysCtx *types.SystemContext, host, repo string) ([]string, error) { - repoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", host, repo)) - if err != nil { - return []string{}, err - } - - dockerRef, err := docker.NewReference(reference.TagNameOnly(repoRef)) - // hard to reach test case, injected error, see pkg/test/dev.go - if err = inject.Error(err); err != nil { - return nil, err // Should never happen for a reference with tag and no digest - } - - tags, err := docker.GetRepositoryTags(ctx, sysCtx, dockerRef) - if err != nil { - return nil, err - } - - return tags, nil -} - -// parseRepositoryReference parses input into a reference.Named, and verifies that it names a repository, not an image. -func parseRepositoryReference(input string) (reference.Named, error) { - ref, err := reference.ParseNormalizedNamed(input) - if err != nil { - return nil, err - } - - if !reference.IsNameOnly(ref) { - return nil, zerr.ErrInvalidRepositoryName - } - - return ref, nil -} - // parse a reference, return its digest and if it's valid. func parseReference(reference string) (digest.Digest, bool) { var ok bool @@ -123,176 +45,89 @@ func parseReference(reference string) (digest.Digest, bool) { return d, ok } -func getCopyOptions(upstreamCtx, localCtx *types.SystemContext) copy.Options { - options := copy.Options{ - DestinationCtx: localCtx, - SourceCtx: upstreamCtx, - ReportWriter: io.Discard, - ForceManifestMIMEType: ispec.MediaTypeImageManifest, // force only oci manifest MIME type - ImageListSelection: copy.CopyAllImages, - } - - return options -} +// Given a list of registry string URLs parse them and return *url.URLs slice. +func parseRegistryURLs(rawURLs []string) ([]*url.URL, error) { + urls := make([]*url.URL, 0) -func getPolicyContext(log log.Logger) (*signature.PolicyContext, error) { - policy := &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}} - - policyContext, err := signature.NewPolicyContext(policy) - if err := inject.Error(err); err != nil { - log.Error().Str("errorType", common.TypeOf(err)). - Err(err).Msg("couldn't create policy context") + for _, rawURL := range rawURLs { + u, err := url.Parse(rawURL) + if err != nil { + return nil, err + } - return nil, err + urls = append(urls, u) } - return policyContext, nil -} - -func getSupportedMediaType() []string { - return []string{ - ispec.MediaTypeImageIndex, - ispec.MediaTypeImageManifest, - manifest.DockerV2ListMediaType, - manifest.DockerV2Schema2MediaType, - } + return urls, nil } -func isSupportedMediaType(mediaType string) bool { - mediaTypes := getSupportedMediaType() - for _, m := range mediaTypes { - if m == mediaType { - return true - } +func GetDescriptorReference(desc ispec.Descriptor) string { + v, ok := desc.Annotations[ispec.AnnotationRefName] + if ok { + return v } - return false + return desc.Digest.String() } -// given an imageSource and a docker manifest, convert it to OCI. -func convertDockerManifestToOCI(imageSource types.ImageSource, dockerManifestBuf []byte) ([]byte, error) { - var ociManifest ispec.Manifest - - // unmarshal docker manifest into OCI manifest - err := json.Unmarshal(dockerManifestBuf, &ociManifest) - if err != nil { - return []byte{}, err - } - - configContent, err := getImageConfigContent(imageSource, ociManifest.Config.Digest) - if err != nil { - return []byte{}, err - } +func StripRegistryTransport(url string) string { + return strings.Replace(strings.Replace(url, "http://", "", 1), "https://", "", 1) +} - // marshal config blob into OCI config, will remove keys specific to docker - var ociConfig ispec.Image +func getCertificates(certDir string) (string, string, string, error) { + var clientCert string - err = json.Unmarshal(configContent, &ociConfig) - if err != nil { - return []byte{}, err - } + var clientKey string - ociConfigContent, err := json.Marshal(ociConfig) - if err != nil { - return []byte{}, err - } + var regCert string - // convert layers - err = convertDockerLayersToOCI(ociManifest.Layers) + files, err := os.ReadDir(certDir) if err != nil { - return []byte{}, err - } - - // convert config and manifest mediatype - ociManifest.Config.Size = int64(len(ociConfigContent)) - ociManifest.Config.Digest = digest.FromBytes(ociConfigContent) - ociManifest.Config.MediaType = ispec.MediaTypeImageConfig - ociManifest.MediaType = ispec.MediaTypeImageManifest - - return json.Marshal(ociManifest) -} - -// convert docker layers mediatypes to OCI mediatypes. -func convertDockerLayersToOCI(dockerLayers []ispec.Descriptor) error { - for idx, layer := range dockerLayers { - switch layer.MediaType { - case manifest.DockerV2Schema2ForeignLayerMediaType: - dockerLayers[idx].MediaType = ispec.MediaTypeImageLayerNonDistributable //nolint: staticcheck - case manifest.DockerV2Schema2ForeignLayerMediaTypeGzip: - dockerLayers[idx].MediaType = ispec.MediaTypeImageLayerNonDistributableGzip //nolint: staticcheck - case manifest.DockerV2SchemaLayerMediaTypeUncompressed: - dockerLayers[idx].MediaType = ispec.MediaTypeImageLayer - case manifest.DockerV2Schema2LayerMediaType: - dockerLayers[idx].MediaType = ispec.MediaTypeImageLayerGzip - default: - return zerr.ErrMediaTypeNotSupported + if os.IsNotExist(err) { + return "", "", "", nil } - } - - return nil -} -// given an imageSource and a docker index manifest, convert it to OCI. -func convertDockerIndexToOCI(imageSource types.ImageSource, dockerManifestBuf []byte) ([]byte, error) { - // get docker index - originalIndex, err := manifest.ListFromBlob(dockerManifestBuf, manifest.DockerV2ListMediaType) - if err != nil { - return []byte{}, err + return "", "", "", err } - // get manifests digests - manifestsDigests := originalIndex.Instances() + for _, file := range files { + if file.IsDir() { + continue + } - manifestsUpdates := make([]manifest.ListUpdate, 0, len(manifestsDigests)) + if strings.HasSuffix(file.Name(), ".cert") { + certPath := path.Join(certDir, file.Name()) - // convert each manifests in index from docker to OCI - for _, manifestDigest := range manifestsDigests { - digestCopy := manifestDigest + buf, err := os.ReadFile(certPath) + if err != nil { + return "", "", "", err + } - indexManifestBuf, _, err := imageSource.GetManifest(context.Background(), &digestCopy) - if err != nil { - return []byte{}, err + clientCert = string(buf) } - convertedIndexManifest, err := convertDockerManifestToOCI(imageSource, indexManifestBuf) - if err != nil { - return []byte{}, err - } + if strings.HasSuffix(file.Name(), ".key") { + certPath := path.Join(certDir, file.Name()) - manifestsUpdates = append(manifestsUpdates, manifest.ListUpdate{ - Digest: digest.FromBytes(convertedIndexManifest), - Size: int64(len(convertedIndexManifest)), - MediaType: ispec.MediaTypeImageManifest, - }) - } + buf, err := os.ReadFile(certPath) + if err != nil { + return "", "", "", err + } - // update all manifests in index - if err := originalIndex.UpdateInstances(manifestsUpdates); err != nil { - return []byte{}, err - } + clientKey = string(buf) + } - // convert index to OCI - convertedList, err := originalIndex.ConvertToMIMEType(ispec.MediaTypeImageIndex) - if err != nil { - return []byte{}, err - } + if strings.HasSuffix(file.Name(), ".crt") { + certPath := path.Join(certDir, file.Name()) - return convertedList.Serialize() -} + buf, err := os.ReadFile(certPath) + if err != nil { + return "", "", "", err + } -// given an image source and a config blob digest, get blob config content. -func getImageConfigContent(imageSource types.ImageSource, configDigest digest.Digest, -) ([]byte, error) { - configBlob, _, err := imageSource.GetBlob(context.Background(), types.BlobInfo{ - Digest: configDigest, - }, none.NoCache) - if err != nil { - return nil, err + regCert = string(buf) + } } - configBuf := new(bytes.Buffer) - - _, err = configBuf.ReadFrom(configBlob) - - return configBuf.Bytes(), err + return clientCert, clientKey, regCert, nil } diff --git a/pkg/meta/boltdb/boltdb.go b/pkg/meta/boltdb/boltdb.go index d357c8dd6..04f828b10 100644 --- a/pkg/meta/boltdb/boltdb.go +++ b/pkg/meta/boltdb/boltdb.go @@ -961,7 +961,7 @@ func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godige LayersInfo: mConvert.GetProtoLayersInfo(sigMeta.LayersInfo), } - if zcommon.IsCosignTag(sigMeta.SignatureTag) { + if zcommon.IsCosignSignature(sigMeta.SignatureTag) { // the entry for "sha256-{digest}.sig" signatures should be overwritten if // it exists or added on the first position if it doesn't exist if len(signatureSlice.GetList()) == 0 { diff --git a/pkg/meta/dynamodb/dynamodb.go b/pkg/meta/dynamodb/dynamodb.go index 73aa35fa3..ca3f9d493 100644 --- a/pkg/meta/dynamodb/dynamodb.go +++ b/pkg/meta/dynamodb/dynamodb.go @@ -1265,7 +1265,7 @@ func (dwr *DynamoDB) AddManifestSignature(repo string, signedManifestDigest godi LayersInfo: mConvert.GetProtoLayersInfo(sigMeta.LayersInfo), } - if zcommon.IsCosignTag(sigMeta.SignatureTag) { + if zcommon.IsCosignSignature(sigMeta.SignatureTag) { // the entry for "sha256-{digest}.sig" signatures should be overwritten if // it exists or added on the first position if it doesn't exist if len(signatureSlice.GetList()) == 0 { diff --git a/pkg/meta/parse.go b/pkg/meta/parse.go index 056b0e1e3..a52a67617 100644 --- a/pkg/meta/parse.go +++ b/pkg/meta/parse.go @@ -403,7 +403,7 @@ func isSignature(reference string, manifestContent ispec.Manifest) (bool, string return true, CosignType, manifestContent.Subject.Digest } - if tag := reference; zcommon.IsCosignTag(reference) { + if tag := reference; zcommon.IsCosignSignature(reference) { prefixLen := len("sha256-") digestLen := 64 signedImageManifestDigestEncoded := tag[prefixLen : prefixLen+digestLen] diff --git a/pkg/storage/gc/gc.go b/pkg/storage/gc/gc.go index a28125c6f..e3d0762fb 100644 --- a/pkg/storage/gc/gc.go +++ b/pkg/storage/gc/gc.go @@ -322,7 +322,7 @@ func (gc GarbageCollect) removeReferrer(repo string, index *ispec.Index, manifes // cosign tag, ok := getDescriptorTag(manifestDesc) if ok { - if isCosignTag(tag) { + if zcommon.IsCosignTag(tag) { subjectDigest := getSubjectFromCosignTag(tag) referenced := isManifestReferencedInIndex(index, subjectDigest) @@ -785,16 +785,6 @@ func getDescriptorTag(desc ispec.Descriptor) (string, bool) { return tag, ok } -// this function will check if tag is a cosign tag (signature or sbom). -func isCosignTag(tag string) bool { - if strings.HasPrefix(tag, "sha256-") && - (strings.HasSuffix(tag, cosignSignatureTagSuffix) || strings.HasSuffix(tag, SBOMTagSuffix)) { - return true - } - - return false -} - /* GCTaskGenerator takes all repositories found in the storage.imagestore diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index e76b2c61c..f06941100 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -259,7 +259,7 @@ func CheckIsImageSignature(repoName string, manifestBlob []byte, reference strin return true, CosignType, manifestContent.Subject.Digest, nil } - if tag := reference; zcommon.IsCosignTag(reference) { + if tag := reference; zcommon.IsCosignSignature(reference) { prefixLen := len("sha256-") digestLen := 64 signedImageManifestDigestEncoded := tag[prefixLen : prefixLen+digestLen] diff --git a/pkg/test/auth/bearer.go b/pkg/test/auth/bearer.go index f305d5b02..b2d8a4281 100644 --- a/pkg/test/auth/bearer.go +++ b/pkg/test/auth/bearer.go @@ -36,10 +36,21 @@ func MakeAuthTestServer(serverKey string, unauthorizedNamespace string) *httptes } authTestServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodGet { + response.WriteHeader(http.StatusMethodNotAllowed) + + return + } + var access []auth.AccessEntry - scope := request.URL.Query().Get("scope") - if scope != "" { + scopes := request.URL.Query()["scope"] + + for _, scope := range scopes { + if scope == "" { + continue + } + parts := strings.Split(scope, ":") name := parts[1] actions := strings.Split(parts[2], ",") @@ -48,13 +59,11 @@ func MakeAuthTestServer(serverKey string, unauthorizedNamespace string) *httptes actions = []string{} } - access = []auth.AccessEntry{ - { - Name: name, - Type: "repository", - Actions: actions, - }, - } + access = append(access, auth.AccessEntry{ + Name: name, + Type: "repository", + Actions: actions, + }) } token, err := cmTokenGenerator.GenerateToken(access, time.Minute*1) diff --git a/scripts/update_licenses.sh b/scripts/update_licenses.sh index 352615397..cb2dc0b5a 100755 --- a/scripts/update_licenses.sh +++ b/scripts/update_licenses.sh @@ -2,7 +2,7 @@ set -x -export GOFLAGS="-tags=debug,imagetrust,lint,metrics,mgmt,profile,scrub,search,sync,ui,userprefs,containers_image_openpgp" +export GOFLAGS="-tags=debug,imagetrust,lint,metrics,mgmt,profile,scrub,search,sync,ui,userprefs" echo "Module | License URL | License" > THIRD-PARTY-LICENSES.md echo "---|---|---" >> THIRD-PARTY-LICENSES.md; diff --git a/test/blackbox/sync_docker.bats b/test/blackbox/sync_docker.bats index f2addb435..e51e2cd07 100644 --- a/test/blackbox/sync_docker.bats +++ b/test/blackbox/sync_docker.bats @@ -234,7 +234,7 @@ function teardown_file() { [ $(echo "${lines[-1]}" | jq '.repositories[1]') = '"kube-apiserver"' ] run curl http://127.0.0.1:${zot_port}/v2/kube-apiserver/tags/list [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq '.tags[]') = '"v1.26.0"' ] + [ $(echo "${lines[-1]}" | jq '.tags[1]') = '"v1.26.0"' ] } @test "sync k8s image on demand" { diff --git a/test/scripts/fuzzAll.sh b/test/scripts/fuzzAll.sh index 98af2f4ed..c77095b6f 100644 --- a/test/scripts/fuzzAll.sh +++ b/test/scripts/fuzzAll.sh @@ -13,6 +13,6 @@ do do echo "Fuzzing $func in $file" parentDir=$(dirname $file) - go test $parentDir -run=$func -fuzz=$func$ -fuzztime=${fuzzTime}s -tags sync,metrics,search,scrub,containers_image_openpgp | grep -oP -x '^(?:(?!\blevel\b).)*$' + go test $parentDir -run=$func -fuzz=$func$ -fuzztime=${fuzzTime}s -tags sync,metrics,search,scrub | grep -oP -x '^(?:(?!\blevel\b).)*$' done done