From dfd62f27628f30f07105a33e9190863d1928c116 Mon Sep 17 00:00:00 2001 From: yosifkit Date: Mon, 27 Jan 2025 12:59:41 -0800 Subject: [PATCH] Revert "Implement parallelism in deploy" --- .test/test.sh | 39 +--------- cmd/deploy/input.go | 54 +++++-------- cmd/deploy/input_test.go | 8 +- cmd/deploy/main.go | 142 ++++------------------------------ registry/manifest-children.go | 25 ------ registry/push.go | 13 +++- 6 files changed, 49 insertions(+), 232 deletions(-) delete mode 100644 registry/manifest-children.go diff --git a/.test/test.sh b/.test/test.sh index 2da0a27..f833239 100755 --- a/.test/test.sh +++ b/.test/test.sh @@ -168,43 +168,6 @@ if [ -n "$doDeploy" ]; then data: ("json string" | @json + "\n" | @base64), }, - # test pushing a full, actual image (tianon/true:oci@sha256:9ef42f1d602fb423fad935aac1caa0cfdbce1ad7edce64d080a4eb7b13f7cd9d), all parts - { - # config blob - type: "blob", - refs: [$reg+"/true"], - data: "ewoJImFyY2hpdGVjdHVyZSI6ICJhbWQ2NCIsCgkiY29uZmlnIjogewoJCSJDbWQiOiBbCgkJCSIvdHJ1ZSIKCQldCgl9LAoJImNyZWF0ZWQiOiAiMjAyMy0wMi0wMVQwNjo1MToxMVoiLAoJImhpc3RvcnkiOiBbCgkJewoJCQkiY3JlYXRlZCI6ICIyMDIzLTAyLTAxVDA2OjUxOjExWiIsCgkJCSJjcmVhdGVkX2J5IjogImh0dHBzOi8vZ2l0aHViLmNvbS90aWFub24vZG9ja2VyZmlsZXMvdHJlZS9tYXN0ZXIvdHJ1ZSIKCQl9CgldLAoJIm9zIjogImxpbnV4IiwKCSJyb290ZnMiOiB7CgkJImRpZmZfaWRzIjogWwoJCQkic2hhMjU2OjY1YjVhNDU5M2NjNjFkM2VhNmQzNTVmYjk3YzA0MzBkODIwZWUyMWFhODUzNWY1ZGU0NWU3NWMzMTk1NGI3NDMiCgkJXSwKCQkidHlwZSI6ICJsYXllcnMiCgl9Cn0K", - }, - { - # layer blob - type: "blob", - refs: [$reg+"/true"], - data: "H4sIAAAAAAACAyspKk1loDEwAAJTU1MwDQTotIGhuQmcDRE3MzM0YlAwYKADKC0uSSxSUGAYoaDe1ceNiZERzmdisGMA8SoYHMB8Byx6HBgsGGA6QDQrmiwyXQPl1cDlIUG9wYaflWEUDDgAAIAGdJIABAAA", - }, - { - type: "manifest", - refs: [ "oci", "latest", (range(0; 10)) | $reg+"/true:\(.)", $reg+"/foo/true:\(.)" ], # test pushing a whole bunch of tags in multiple repos - lookup: { - # a few explicit lookup entries for better code coverage (dep calculation during parallelization) - "sha256:1c51fc286aa95d9413226599576bafa38490b1e292375c90de095855b64caea6": ($reg+"/true"), - "": ($reg+"/true"), - }, - data: { - schemaVersion: 2, - mediaType: "application/vnd.oci.image.manifest.v1+json", - config: { - mediaType: "application/vnd.oci.image.config.v1+json", - digest: "sha256:25be82253336f0b8c4347bc4ecbbcdc85d0e0f118ccf8dc2e119c0a47a0a486e", - size: 396, - }, - layers: [ { - mediaType: "application/vnd.oci.image.layer.v1.tar+gzip", - digest: "sha256:1c51fc286aa95d9413226599576bafa38490b1e292375c90de095855b64caea6", - size: 117, - } ], - }, - }, - # test blob mounting between repositories { type: "blob", @@ -250,7 +213,7 @@ if [ -n "$doDeploy" ]; then empty ')" # stored in a variable for easier debugging ("bash -x") - time "$coverage/bin/deploy" <<<"$json" + "$coverage/bin/deploy" <<<"$json" docker rm -vf meta-scripts-test-registry trap - EXIT diff --git a/cmd/deploy/input.go b/cmd/deploy/input.go index 018fcec..8e8e80c 100644 --- a/cmd/deploy/input.go +++ b/cmd/deploy/input.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "maps" "github.com/docker-library/meta-scripts/registry" @@ -47,14 +46,7 @@ type inputNormalized struct { Data []byte `json:"data"` CopyFrom *registry.Reference `json:"copyFrom"` - // if CopyFrom is nil and Type is manifest, this will be set (used by "do") - MediaType string `json:"mediaType,omitempty"` -} - -func (normal inputNormalized) clone() inputNormalized { - // normal.Lookup is the only thing we have concurrency issues with, so it's the only thing we'll explicitly clone 😇 - normal.Lookup = maps.Clone(normal.Lookup) - return normal + Do func(ctx context.Context, dstRef registry.Reference) (ociregistry.Descriptor, error) `json:"-"` } func normalizeInputRefs(deployType deployType, rawRefs []string) ([]registry.Reference, ociregistry.Digest, error) { @@ -230,7 +222,6 @@ func NormalizeInput(raw inputRaw) (inputNormalized, error) { normal.Lookup[d] = ref } - // front-load some validation / data extraction for "normal.do" to work switch normal.Type { case typeManifest: if normal.CopyFrom == nil { @@ -249,42 +240,33 @@ func NormalizeInput(raw inputRaw) (inputNormalized, error) { // and our logic for pushing children needs to know the mediaType (see the GHSAs referenced above) return normal, fmt.Errorf("%s: pushing manifest but missing 'mediaType'", debugId) } - normal.MediaType = mediaTypeHaver.MediaType - } - - case typeBlob: - if normal.CopyFrom != nil && normal.CopyFrom.Digest == "" { - return normal, fmt.Errorf("%s: blobs are always by-digest, and thus need a digest: %s", debugId, normal.CopyFrom) - } - - default: - panic("unknown type: " + string(normal.Type)) - // panic instead of error because this should've already been handled/normalized above (so this is a coding error, not a runtime error) - } - - return normal, nil -} - -// WARNING: many of these codepaths will end up writing to "normal.Lookup", which because it's a map is passed by reference, so this method is *not* safe for concurrent invocation on a single "normal" object! see "normal.clone" (above) -func (normal inputNormalized) do(ctx context.Context, dstRef registry.Reference) (ociregistry.Descriptor, error) { - switch normal.Type { - case typeManifest: - if normal.CopyFrom == nil { - // TODO panic on bad data, like MediaType being empty? - return registry.EnsureManifest(ctx, dstRef, normal.Data, normal.MediaType, normal.Lookup) + normal.Do = func(ctx context.Context, dstRef registry.Reference) (ociregistry.Descriptor, error) { + return registry.EnsureManifest(ctx, dstRef, normal.Data, mediaTypeHaver.MediaType, normal.Lookup) + } } else { - return registry.CopyManifest(ctx, *normal.CopyFrom, dstRef, normal.Lookup) + normal.Do = func(ctx context.Context, dstRef registry.Reference) (ociregistry.Descriptor, error) { + return registry.CopyManifest(ctx, *normal.CopyFrom, dstRef, normal.Lookup) + } } case typeBlob: if normal.CopyFrom == nil { - return registry.EnsureBlob(ctx, dstRef, int64(len(normal.Data)), bytes.NewReader(normal.Data)) + normal.Do = func(ctx context.Context, dstRef registry.Reference) (ociregistry.Descriptor, error) { + return registry.EnsureBlob(ctx, dstRef, int64(len(normal.Data)), bytes.NewReader(normal.Data)) + } } else { - return registry.CopyBlob(ctx, *normal.CopyFrom, dstRef) + if normal.CopyFrom.Digest == "" { + return normal, fmt.Errorf("%s: blobs are always by-digest, and thus need a digest: %s", debugId, normal.CopyFrom) + } + normal.Do = func(ctx context.Context, dstRef registry.Reference) (ociregistry.Descriptor, error) { + return registry.CopyBlob(ctx, *normal.CopyFrom, dstRef) + } } default: panic("unknown type: " + string(normal.Type)) // panic instead of error because this should've already been handled/normalized above (so this is a coding error, not a runtime error) } + + return normal, nil } diff --git a/cmd/deploy/input_test.go b/cmd/deploy/input_test.go index 51b8d75..8304c68 100644 --- a/cmd/deploy/input_test.go +++ b/cmd/deploy/input_test.go @@ -281,7 +281,7 @@ func TestNormalizeInput(t *testing.T) { "refs": [ "localhost:5000/example:test" ], "data": {"mediaType": "application/vnd.oci.image.index.v1+json"} }`, - `{"type":"manifest","refs":["localhost:5000/example:test@sha256:0ae6b7b9d0bc73ee36c1adef005deb431e94cf009c6a947718b31da3d668032d"],"data":"eyJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLm9jaS5pbWFnZS5pbmRleC52MStqc29uIn0=","copyFrom":null,"mediaType":"application/vnd.oci.image.index.v1+json"}`, + `{"type":"manifest","refs":["localhost:5000/example:test@sha256:0ae6b7b9d0bc73ee36c1adef005deb431e94cf009c6a947718b31da3d668032d"],"data":"eyJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLm9jaS5pbWFnZS5pbmRleC52MStqc29uIn0=","copyFrom":null}`, }, { "manifest raw", @@ -290,7 +290,7 @@ func TestNormalizeInput(t *testing.T) { "refs": [ "localhost:5000/example" ], "data": "eyJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLm9jaS5pbWFnZS5pbmRleC52MStqc29uIn0=" }`, - `{"type":"manifest","refs":["localhost:5000/example@sha256:0ae6b7b9d0bc73ee36c1adef005deb431e94cf009c6a947718b31da3d668032d"],"data":"eyJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLm9jaS5pbWFnZS5pbmRleC52MStqc29uIn0=","copyFrom":null,"mediaType":"application/vnd.oci.image.index.v1+json"}`, + `{"type":"manifest","refs":["localhost:5000/example@sha256:0ae6b7b9d0bc73ee36c1adef005deb431e94cf009c6a947718b31da3d668032d"],"data":"eyJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLm9jaS5pbWFnZS5pbmRleC52MStqc29uIn0=","copyFrom":null}`, }, { @@ -301,7 +301,7 @@ func TestNormalizeInput(t *testing.T) { "lookup": { "sha256:9ef42f1d602fb423fad935aac1caa0cfdbce1ad7edce64d080a4eb7b13f7cd9d": "tianon/true" }, "data": {"mediaType": "application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:9ef42f1d602fb423fad935aac1caa0cfdbce1ad7edce64d080a4eb7b13f7cd9d","size":1165}],"schemaVersion":2} }`, - `{"type":"manifest","refs":["localhost:5000/example:test@sha256:0cb474919526d040392883b84e5babb65a149cc605b89b117781ab94e88a5e86"],"lookup":{"sha256:9ef42f1d602fb423fad935aac1caa0cfdbce1ad7edce64d080a4eb7b13f7cd9d":"tianon/true"},"data":"eyJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLm9jaS5pbWFnZS5pbmRleC52MStqc29uIiwibWFuaWZlc3RzIjpbeyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQub2NpLmltYWdlLm1hbmlmZXN0LnYxK2pzb24iLCJkaWdlc3QiOiJzaGEyNTY6OWVmNDJmMWQ2MDJmYjQyM2ZhZDkzNWFhYzFjYWEwY2ZkYmNlMWFkN2VkY2U2NGQwODBhNGViN2IxM2Y3Y2Q5ZCIsInNpemUiOjExNjV9XSwic2NoZW1hVmVyc2lvbiI6Mn0=","copyFrom":null,"mediaType":"application/vnd.oci.image.index.v1+json"}`, + `{"type":"manifest","refs":["localhost:5000/example:test@sha256:0cb474919526d040392883b84e5babb65a149cc605b89b117781ab94e88a5e86"],"lookup":{"sha256:9ef42f1d602fb423fad935aac1caa0cfdbce1ad7edce64d080a4eb7b13f7cd9d":"tianon/true"},"data":"eyJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLm9jaS5pbWFnZS5pbmRleC52MStqc29uIiwibWFuaWZlc3RzIjpbeyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQub2NpLmltYWdlLm1hbmlmZXN0LnYxK2pzb24iLCJkaWdlc3QiOiJzaGEyNTY6OWVmNDJmMWQ2MDJmYjQyM2ZhZDkzNWFhYzFjYWEwY2ZkYmNlMWFkN2VkY2U2NGQwODBhNGViN2IxM2Y3Y2Q5ZCIsInNpemUiOjExNjV9XSwic2NoZW1hVmVyc2lvbiI6Mn0=","copyFrom":null}`, }, { "image", @@ -311,7 +311,7 @@ func TestNormalizeInput(t *testing.T) { "lookup": { "": "tianon/true" }, "data": {"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":1471,"digest":"sha256:690912094c0165c489f874c72cee4ba208c28992c0699fa6e10d8cc59f93fec9"},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","size":129,"digest":"sha256:4c74d744397d4bcbd3079d9c82a87b80d43da376313772978134d1288f20518c"}]} }`, - `{"type":"manifest","refs":["localhost:5000/example@sha256:1c70f9d471b83100c45d5a218d45bbf7e073e11ea5043758a020379a7c78f878"],"lookup":{"":"tianon/true"},"data":"eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjoxNDcxLCJkaWdlc3QiOiJzaGEyNTY6NjkwOTEyMDk0YzAxNjVjNDg5Zjg3NGM3MmNlZTRiYTIwOGMyODk5MmMwNjk5ZmE2ZTEwZDhjYzU5ZjkzZmVjOSJ9LCJsYXllcnMiOlt7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLCJzaXplIjoxMjksImRpZ2VzdCI6InNoYTI1Njo0Yzc0ZDc0NDM5N2Q0YmNiZDMwNzlkOWM4MmE4N2I4MGQ0M2RhMzc2MzEzNzcyOTc4MTM0ZDEyODhmMjA1MThjIn1dfQ==","copyFrom":null,"mediaType":"application/vnd.docker.distribution.manifest.v2+json"}`, + `{"type":"manifest","refs":["localhost:5000/example@sha256:1c70f9d471b83100c45d5a218d45bbf7e073e11ea5043758a020379a7c78f878"],"lookup":{"":"tianon/true"},"data":"eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjoxNDcxLCJkaWdlc3QiOiJzaGEyNTY6NjkwOTEyMDk0YzAxNjVjNDg5Zjg3NGM3MmNlZTRiYTIwOGMyODk5MmMwNjk5ZmE2ZTEwZDhjYzU5ZjkzZmVjOSJ9LCJsYXllcnMiOlt7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLCJzaXplIjoxMjksImRpZ2VzdCI6InNoYTI1Njo0Yzc0ZDc0NDM5N2Q0YmNiZDMwNzlkOWM4MmE4N2I4MGQ0M2RhMzc2MzEzNzcyOTc4MTM0ZDEyODhmMjA1MThjIn1dfQ==","copyFrom":null}`, }, { diff --git a/cmd/deploy/main.go b/cmd/deploy/main.go index 89c812c..91862e6 100644 --- a/cmd/deploy/main.go +++ b/cmd/deploy/main.go @@ -7,11 +7,6 @@ import ( "os" "os/exec" "os/signal" - "sync" - - "github.com/docker-library/meta-scripts/registry" - - ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) func main() { @@ -37,11 +32,6 @@ func main() { panic(err) } - // a set of RWMutex objects for synchronizing the pushing of "child" objects before their parents later in the list of documents - // for every RWMutex, it will be *write*-locked during push, and *read*-locked during reading (which means we won't limit the parallelization of multiple parents after a given child is pushed, but we will stop parents from being pushed before their children) - childMutexes := sync.Map{} - wg := sync.WaitGroup{} - dec := json.NewDecoder(stdout) for dec.More() { var raw inputRaw @@ -58,128 +48,26 @@ func main() { } refsDigest := normal.Refs[0].Digest - var logSuffix string = " (" + string(raw.Type) + ") " - if normal.CopyFrom != nil { - // normal copy (one repo/registry to another) - logSuffix = " 🤝" + logSuffix + normal.CopyFrom.String() - // "localhost:32774/test 🤝 (manifest) tianon/test@sha256:4077658bc7e39f02f81d1682fe49f66b3db2c420813e43f5db0c53046167c12f" + if normal.CopyFrom == nil { + fmt.Printf("Pushing %s %s:\n", raw.Type, refsDigest) } else { - // push (raw/embedded blob or manifest data) - logSuffix = " 🦾" + logSuffix + string(refsDigest) - // "localhost:32774/test 🦾 (blob) sha256:1a51828d59323e0e02522c45652b6a7a44a032b464b06d574f067d2358b0e9f1" + fmt.Printf("Copying %s %s:\n", raw.Type, *normal.CopyFrom) } - startedPrefix := "❔ " - successPrefix := "✅ " - failurePrefix := "❌ " - - // locks are per-digest, but refs might be 20 tags on the same digest, so we need to get one write lock per repo@digest and release it when the first tag completes, and every other tag needs a read lock - seenRefs := map[string]bool{} for _, ref := range normal.Refs { - ref := ref // https://github.com/golang/go/issues/60078 - - necessaryReadLockRefs := []registry.Reference{} - - // before parallelization, collect the pushing "child" mutex we need to lock for writing right away (but only for the first entry) - var mutex *sync.RWMutex - if ref.Digest != "" { - lockRef := ref - lockRef.Tag = "" - lockRefStr := lockRef.String() - if seenRefs[lockRefStr] { - // if we've already seen this specific ref for this input, we need a read lock, not a write lock (since they're per-repo@digest) - necessaryReadLockRefs = append(necessaryReadLockRefs, lockRef) - } else { - seenRefs[lockRefStr] = true - lock, _ := childMutexes.LoadOrStore(lockRefStr, &sync.RWMutex{}) - mutex = lock.(*sync.RWMutex) - // if we have a "child" mutex, lock it immediately so we don't create a race between inputs - mutex.Lock() // (this gets unlocked in the goroutine below) - // this is sane to lock here because interdependent inputs are required to be in-order (children first), so if this hangs it's 100% a bug in the input order - } + fmt.Printf(" - %s", ref.StringWithKnownDigest(refsDigest)) + desc, err := normal.Do(ctx, ref) + if err != nil { + fmt.Fprintf(os.Stderr, " -- ERROR: %v\n", err) + os.Exit(1) + return } - - // make a (deep) copy of "normal" so that we can use it in a goroutine ("normal.do" is not safe for concurrent invocation) - normal := normal.clone() - - wg.Add(1) - go func() { - defer wg.Done() - - if mutex != nil { - defer mutex.Unlock() - } - - // before we start this job (parallelized), if it's a raw data job we need to parse the raw data and see if any of the "children" are objects we're still in the process of pushing (from a previously parallel job) - if len(normal.Data) > 2 { // needs to at least be bigger than "{}" for us to care (anything else either doesn't have data or can't have children) - // explicitly ignoring errors because this might not actually be JSON (or even a manifest at all!); this is best-effort - // TODO optimize this by checking whether normal.Data matches "^\s*{.+}\s*$" first so we have some assurance it might work before we go further? - manifestChildren, _ := registry.ParseManifestChildren(normal.Data) - childDescs := []ocispec.Descriptor{} - childDescs = append(childDescs, manifestChildren.Manifests...) - if manifestChildren.Config != nil { - childDescs = append(childDescs, *manifestChildren.Config) - } - childDescs = append(childDescs, manifestChildren.Layers...) - for _, childDesc := range childDescs { - childRef := ref - childRef.Digest = childDesc.Digest - necessaryReadLockRefs = append(necessaryReadLockRefs, childRef) - - // these read locks are cheap, so let's be aggressive with our "lookup" refs too - if lookupRef, ok := normal.Lookup[childDesc.Digest]; ok { - lookupRef.Digest = childDesc.Digest - necessaryReadLockRefs = append(necessaryReadLockRefs, lookupRef) - } - if fallbackRef, ok := normal.Lookup[""]; ok { - fallbackRef.Digest = childDesc.Digest - necessaryReadLockRefs = append(necessaryReadLockRefs, fallbackRef) - } - } - } - // we don't *know* that all the lookup references are children, but if any of them have an explicit digest, let's treat them as potential children too (which is fair, because they *are* explicit potential references that it's sane to make sure exist) - for digest, lookupRef := range normal.Lookup { - necessaryReadLockRefs = append(necessaryReadLockRefs, lookupRef) - if digest != lookupRef.Digest { - lookupRef.Digest = digest - necessaryReadLockRefs = append(necessaryReadLockRefs, lookupRef) - } - } - // if we're going to do a copy, we need to *also* include the artifact we're copying in our list - if normal.CopyFrom != nil { - necessaryReadLockRefs = append(necessaryReadLockRefs, *normal.CopyFrom) - } - // ok, we've built up a list, let's start grabbing (ro) mutexes - seenChildren := map[string]bool{} - for _, lockRef := range necessaryReadLockRefs { - lockRef.Tag = "" - if lockRef.Digest == "" { - continue - } - lockRefStr := lockRef.String() - if seenChildren[lockRefStr] { - continue - } - seenChildren[lockRefStr] = true - lock, _ := childMutexes.LoadOrStore(lockRefStr, &sync.RWMutex{}) - lock.(*sync.RWMutex).RLock() - defer lock.(*sync.RWMutex).RUnlock() - } - - logText := ref.StringWithKnownDigest(refsDigest) + logSuffix - fmt.Println(startedPrefix + logText) - desc, err := normal.do(ctx, ref) - if err != nil { - fmt.Fprintf(os.Stderr, "%s%s -- ERROR: %v\n", failurePrefix, logText, err) - panic(err) // TODO exit in a more clean way (we can't use "os.Exit" because that causes *more* errors 😭) - } - if ref.Digest == "" && refsDigest == "" { - logText += "@" + string(desc.Digest) - } - fmt.Println(successPrefix + logText) - }() + if ref.Digest == "" && refsDigest == "" { + fmt.Printf("@%s", desc.Digest) + } + fmt.Println() } - } - wg.Wait() + fmt.Println() + } } diff --git a/registry/manifest-children.go b/registry/manifest-children.go deleted file mode 100644 index 15e4c2a..0000000 --- a/registry/manifest-children.go +++ /dev/null @@ -1,25 +0,0 @@ -package registry - -import ( - "encoding/json" - - ocispec "github.com/opencontainers/image-spec/specs-go/v1" -) - -type ManifestChildren struct { - // *technically* this should be two separate structs chosen based on mediaType (https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m), but that makes the code a lot more annoying when we're just collecting a list of potential children we need to copy over for the parent object to push successfully - - // intentional subset of https://github.com/opencontainers/image-spec/blob/v1.1.0/specs-go/v1/index.go#L21 to minimize parsing - Manifests []ocispec.Descriptor `json:"manifests"` - - // intentional subset of https://github.com/opencontainers/image-spec/blob/v1.1.0/specs-go/v1/manifest.go#L20 to minimize parsing - Config *ocispec.Descriptor `json:"config"` // have to turn this into a pointer so we can recognize when it's not set easier / more correctly - Layers []ocispec.Descriptor `json:"layers"` -} - -// opportunistically parse a given manifest for any *potential* child objects; will return JSON parsing errors for non-JSON -func ParseManifestChildren(manifest []byte) (ManifestChildren, error) { - var manifestChildren ManifestChildren - err := json.Unmarshal(manifest, &manifestChildren) - return manifestChildren, err -} diff --git a/registry/push.go b/registry/push.go index e5309f7..9075efb 100644 --- a/registry/push.go +++ b/registry/push.go @@ -74,8 +74,17 @@ func EnsureManifest(ctx context.Context, ref Reference, manifest json.RawMessage errors.Is(err, ociregistry.ErrBlobUnknown) || (errors.As(err, &httpErr) && httpErr.StatusCode() >= 400 && httpErr.StatusCode() <= 499) { // this probably means we need to push some child manifests and/or mount missing blobs (and then retry the manifest push) - manifestChildren, err := ParseManifestChildren(manifest) - if err != nil { + var manifestChildren struct { + // *technically* this should be two separate structs chosen based on mediaType (https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m), but that makes the code a lot more annoying when we're just collecting a list of potential children we need to copy over for the parent object to push successfully + + // intentional subset of https://github.com/opencontainers/image-spec/blob/v1.1.0/specs-go/v1/index.go#L21 to minimize parsing + Manifests []ocispec.Descriptor `json:"manifests"` + + // intentional subset of https://github.com/opencontainers/image-spec/blob/v1.1.0/specs-go/v1/manifest.go#L20 to minimize parsing + Config *ocispec.Descriptor `json:"config"` // have to turn this into a pointer so we can recognize when it's not set easier / more correctly + Layers []ocispec.Descriptor `json:"layers"` + } + if err := json.Unmarshal(manifest, &manifestChildren); err != nil { return desc, fmt.Errorf("%s: failed parsing manifest JSON: %w", ref, err) }