From 4d3f6988380ef196b7be38f64eb026b07b72155d Mon Sep 17 00:00:00 2001 From: David Son Date: Wed, 31 Jul 2024 22:13:50 +0000 Subject: [PATCH] Add SOCI handler wrapper As part of upgrading nerdctl to containerd v2, we must update dependencies to depend on v2 as well. As SOCI maintainers do not wish to upgrade to containerd v2 yet, this is a workaround to avoid depending on SOCI library. We can revert this change if we wish once SOCI updates to containerd v2. Signed-off-by: David Son --- go.mod | 2 +- go.sum | 2 - pkg/imgutil/snapshotter.go | 4 +- pkg/snapshotterutil/socisource.go | 110 ++++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 pkg/snapshotterutil/socisource.go diff --git a/go.mod b/go.mod index 2e7113eb9ad..f5bfb9bd3fc 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/Microsoft/go-winio v0.6.2 github.com/Microsoft/hcsshim v0.12.5 - github.com/awslabs/soci-snapshotter v0.7.0 github.com/compose-spec/compose-go/v2 v2.1.5 github.com/containerd/accelerated-container-image v1.1.4 github.com/containerd/cgroups/v3 v3.0.3 @@ -118,6 +117,7 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect diff --git a/go.sum b/go.sum index a1e57141a6a..04b3fcbe470 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,6 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hcsshim v0.12.5 h1:bpTInLlDy/nDRWFVcefDZZ1+U8tS+rz3MxjKgu9boo0= github.com/Microsoft/hcsshim v0.12.5/go.mod h1:tIUGego4G1EN5Hb6KC90aDYiUI2dqLSTTOCjVNpOgZ8= -github.com/awslabs/soci-snapshotter v0.7.0 h1:p8hKjVNlT3Z8W0ZcsdCjJ5A3qUgD4A0zv6qHrcKjKEg= -github.com/awslabs/soci-snapshotter v0.7.0/go.mod h1:o4LaCms6epXbSpYXcd49a+3QmdjsQAPpKywpp0Ls3TE= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/pkg/imgutil/snapshotter.go b/pkg/imgutil/snapshotter.go index 9fd30e72f8c..0c787767eea 100644 --- a/pkg/imgutil/snapshotter.go +++ b/pkg/imgutil/snapshotter.go @@ -19,13 +19,13 @@ package imgutil import ( "strings" - socisource "github.com/awslabs/soci-snapshotter/fs/source" "github.com/containerd/containerd" "github.com/containerd/containerd/images" ctdsnapshotters "github.com/containerd/containerd/pkg/snapshotters" "github.com/containerd/log" "github.com/containerd/nerdctl/v2/pkg/api/types" "github.com/containerd/nerdctl/v2/pkg/imgutil/pull" + "github.com/containerd/nerdctl/v2/pkg/snapshotterutil" "github.com/containerd/stargz-snapshotter/fs/source" ) @@ -115,5 +115,5 @@ func stargzExtraLabels(f func(images.Handler) images.Handler, rFlags types.Remot } func sociExtraLabels(f func(images.Handler) images.Handler, rFlags types.RemoteSnapshotterFlags) func(images.Handler) images.Handler { - return socisource.AppendDefaultLabelsHandlerWrapper(rFlags.SociIndexDigest, f) + return snapshotterutil.SociAppendDefaultLabelsHandlerWrapper(rFlags.SociIndexDigest, f) } diff --git a/pkg/snapshotterutil/socisource.go b/pkg/snapshotterutil/socisource.go new file mode 100644 index 00000000000..85088388d90 --- /dev/null +++ b/pkg/snapshotterutil/socisource.go @@ -0,0 +1,110 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + Copyright The Soci Snapshotter Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Taken from https://github.com/awslabs/soci-snapshotter/blob/237fc956b8366e49927c84fcfee9a2defbb8f53c/fs/source/source.go +// to avoid taking dependency, as maintainers do not wish to upgrade to containerd v2 yet. + +package snapshotterutil + +import ( + "context" + "fmt" + "strings" + + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/labels" + ctdsnapshotters "github.com/containerd/containerd/pkg/snapshotters" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +const ( + // TargetSizeLabel is a label which contains layer size. + TargetSizeLabel = "containerd.io/snapshot/remote/soci.size" + + // targetImageLayersSizeLabel is a label which contains layer sizes contained in + // the target image. + targetImageLayersSizeLabel = "containerd.io/snapshot/remote/image.layers.size" + + // TargetSociIndexDigestLabel is a label which contains the digest of the soci index. + TargetSociIndexDigestLabel = "containerd.io/snapshot/remote/soci.index.digest" +) + +// SociAppendDefaultLabelsHandlerWrapper makes a handler which appends image's basic +// information to each layer descriptor as annotations during unpack. These +// annotations will be passed to this remote snapshotter as labels and used to +// construct source information. +func SociAppendDefaultLabelsHandlerWrapper(indexDigest string, wrapper func(images.Handler) images.Handler) func(f images.Handler) images.Handler { + return func(f images.Handler) images.Handler { + return images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + children, err := wrapper(f).Handle(ctx, desc) + if err != nil { + return nil, err + } + switch desc.MediaType { + case ocispec.MediaTypeImageManifest, images.MediaTypeDockerSchema2Manifest: + for i := range children { + c := &children[i] + if images.IsLayerType(c.MediaType) { + if c.Annotations == nil { + c.Annotations = make(map[string]string) + } + + c.Annotations[TargetSizeLabel] = fmt.Sprintf("%d", c.Size) + c.Annotations[TargetSociIndexDigestLabel] = indexDigest + + remainingLayerDigestsCount := len(strings.Split(c.Annotations[ctdsnapshotters.TargetImageLayersLabel], ",")) + + var layerSizes string + /* + We must ensure that the counts of layer sizes and layer digests are equal. + We will limit the # of neighboring label sizes to equal the # of neighboring + layer digests for any given layer. + */ + for _, l := range children[i : i+remainingLayerDigestsCount] { + if images.IsLayerType(l.MediaType) { + ls := fmt.Sprintf("%d,", l.Size) + // This avoids the label hits the size limitation. + // Skipping layers is allowed here and only affects performance. + if err := labels.Validate(targetImageLayersSizeLabel, layerSizes+ls); err != nil { + break + } + layerSizes += ls + } + } + c.Annotations[targetImageLayersSizeLabel] = strings.TrimSuffix(layerSizes, ",") + } + } + } + return children, nil + }) + } +}