Merge pull request #132 from vmware-tanzu/tag-metas

Add metadata in imgpkg lock file

pkg/kbld/cmd/image.go

@@ -7,16 +7,16 @@ import (
 	"reflect"
 	"strings"
 
-	ctlimg "github.com/k14s/kbld/pkg/kbld/image"
+	ctlconf "github.com/k14s/kbld/pkg/kbld/config"
 	"sigs.k8s.io/yaml"
 )
 
 type Images []Image
 
 type Image struct {
 	URL      string
-	Metas    []ctlimg.Meta // empty when deserialized
-	metasRaw []interface{} // populated when deserialized
+	Metas    []ctlconf.Meta // empty when deserialized
+	metasRaw []interface{}  // populated when deserialized
 }
 
 func (imgs Images) ForImage(url string) (Image, bool) {

pkg/kbld/cmd/resolve.go

@@ -20,6 +20,7 @@ import (
 	ctlser "github.com/k14s/kbld/pkg/kbld/search"
 	"github.com/k14s/kbld/pkg/kbld/version"
 	"github.com/spf13/cobra"
+	"sigs.k8s.io/yaml"
 )
 
 type ResolveOptions struct {
@@ -234,8 +235,6 @@ func (o *ResolveOptions) emitLockOutput(conf ctlconf.Conf, resolvedImages *Proce
 			})
 		}
 
-		c.Overrides = ctlconf.UniqueImageOverrides(c.Overrides)
-
 		return c.WriteToFile(o.LockOutput)
 	case o.ImgpkgLockOutput != "":
 		iLock := lockconfig.ImagesLock{
@@ -247,11 +246,26 @@ func (o *ResolveOptions) emitLockOutput(conf ctlconf.Conf, resolvedImages *Proce
 		for _, urlImagePair := range resolvedImages.All() {
 			iLock.Images = append(iLock.Images, lockconfig.ImageRef{
 				Image:       urlImagePair.Image.URL,
-				Annotations: map[string]string{ctlconf.ImagesLockKbldID: urlImagePair.UnprocessedImageURL.URL},
+				Annotations: o.imgpkgLockAnnotations(urlImagePair),
 			})
 		}
 		return iLock.WriteToPath(o.ImgpkgLockOutput)
 	default:
 		return nil
 	}
 }
+
+func (o *ResolveOptions) imgpkgLockAnnotations(i ProcessedImageItem) map[string]string {
+	anns := map[string]string{
+		ctlconf.ImagesLockKbldID: i.UnprocessedImageURL.URL,
+	}
+	if len(i.Metas) > 0 {
+		bs, err := yaml.Marshal(i.Metas)
+		if err != nil {
+			return anns
+		}
+		anns[ctlconf.ImagesLockKbldMetas] = string(bs)
+	}
+
+	return anns
+}

pkg/kbld/config/config.go

@@ -68,6 +68,7 @@ type ImageOverride struct {
 	NewImage     string                     `json:"newImage"`
 	Preresolved  bool                       `json:"preresolved,omitempty"`
 	TagSelection *versions.VersionSelection `json:"tagSelection,omitempty"`
+	ImageMetas   []Meta                     `json:"metas,omitempty"`
 }
 
 type ImageDestination struct {
@@ -168,12 +169,17 @@ func NewConfigFromImagesLock(res ctlres.Resource) (Config, error) {
 	overridesConfig := NewConfig()
 
 	for _, image := range imagesLock.Images {
+		imgMeta, err := NewMetasFromString(image.Annotations[ImagesLockKbldMetas])
+		if err != nil {
+			return Config{}, fmt.Errorf("Unmarshaling %s as %s annotation:  %s", res.Description(), ImagesLockKbldMetas, err)
+		}
 		iOverride := ImageOverride{
 			ImageRef: ImageRef{
 				Image: image.Annotations[ImagesLockKbldID],
 			},
 			NewImage:    image.Image,
 			Preresolved: true,
+			ImageMetas:  imgMeta,
 		}
 		overridesConfig.Overrides = append(overridesConfig.Overrides, iOverride)
 	}
@@ -318,12 +324,21 @@ func (d Config) WriteToFile(path string) error {
 	return nil
 }
 
+// Equal reports whether this ImageOverride is equal to another ImageOverride.
+//   (`ImageMeta` is descriptive — not identifying — so not part of equality)
+func (d ImageOverride) Equal(other ImageOverride) bool {
+	return d.ImageRef == other.ImageRef &&
+		d.NewImage == other.NewImage &&
+		d.Preresolved == other.Preresolved &&
+		d.TagSelection == other.TagSelection
+}
+
 func UniqueImageOverrides(overrides []ImageOverride) []ImageOverride {
 	var result []ImageOverride
 	for _, override := range overrides {
 		var found bool
 		for _, addedOverride := range result {
-			if addedOverride == override {
+			if override.Equal(addedOverride) {
 				found = true
 				break
 			}

pkg/kbld/config/image_meta.go

@@ -0,0 +1,120 @@
+// Copyright 2020 VMware, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package config
+
+import (
+	"encoding/json"
+
+	"sigs.k8s.io/yaml"
+)
+
+type Meta interface {
+	meta()
+}
+
+type imageMeta struct {
+	Metas []Meta
+}
+
+const (
+	GitMeta         = "git"
+	LocalMeta       = "local"
+	ResolvedMeta    = "resolved"
+	TaggedMeta      = "tagged"
+	PreresolvedMeta = "preresolved"
+)
+
+type BuiltImageSourceGit struct {
+	Type      string // always set to GitMeta
+	RemoteURL string `json:",omitempty" yaml:",omitempty"`
+	SHA       string
+	Dirty     bool
+	Tags      []string `json:",omitempty" yaml:",omitempty"`
+}
+
+type BuiltImageSourceLocal struct {
+	Type string // always set to LocalMeta
+	Path string
+}
+
+type ResolvedImageSourceURL struct {
+	Type string // always set to ResolvedMeta
+	URL  string
+	Tag  string
+}
+
+type TaggedImageMeta struct {
+	Type string // always set to TaggedMeta
+	Tags []string
+}
+
+type PreresolvedImageSourceURL struct {
+	Type string // always set to PreresolvedMeta
+	URL  string
+	Tag  string `json:",omitempty" yaml:",omitempty"`
+}
+
+func (BuiltImageSourceGit) meta()       {}
+func (BuiltImageSourceLocal) meta()     {}
+func (ResolvedImageSourceURL) meta()    {}
+func (TaggedImageMeta) meta()           {}
+func (PreresolvedImageSourceURL) meta() {}
+
+func NewMetasFromString(metas string) ([]Meta, error) {
+	imgMeta := imageMeta{}
+	err := yaml.Unmarshal([]byte(metas), &imgMeta)
+	if err != nil {
+		return []Meta{}, err
+	}
+	return imgMeta.Metas, nil
+}
+
+var _ json.Unmarshaler = &imageMeta{}
+
+func (m *imageMeta) UnmarshalJSON(data []byte) error {
+	var list []interface{}
+	err := yaml.Unmarshal(data, &list)
+	if err != nil {
+		return err
+	}
+
+	for _, item := range list {
+		var local BuiltImageSourceLocal
+		var git BuiltImageSourceGit
+		var res ResolvedImageSourceURL
+		var preres PreresolvedImageSourceURL
+		var tag TaggedImageMeta
+
+		yamlItem, _ := yaml.Marshal(&item)
+
+		switch {
+		case yaml.Unmarshal(yamlItem, &local) == nil && local.Type == LocalMeta:
+			m.Metas = append(m.Metas, local)
+		case yaml.Unmarshal(yamlItem, &git) == nil && git.Type == GitMeta:
+			m.Metas = append(m.Metas, git)
+		case yaml.Unmarshal(yamlItem, &res) == nil && res.Type == ResolvedMeta:
+			m.Metas = append(m.Metas, res)
+		case yaml.Unmarshal(yamlItem, &preres) == nil && preres.Type == PreresolvedMeta:
+			m.Metas = append(m.Metas, preres)
+		case yaml.Unmarshal(yamlItem, &tag) == nil && tag.Type == TaggedMeta:
+			m.Metas = append(m.Metas, tag)
+		default:
+			// ignore unknown meta.
+			// At this time...
+			// - "Meta" are provided as primarily optional diagnostic information
+			//   rather than operational data (read: less important). Losing
+			//   this information does not change the correctness of kbld's
+			//   primary purpose during deployment: to rewrite image references.
+			//   It would be more than an annoyance to error-out if we were
+			//   unable to parse such data.
+			// - Ideally, yes, we'd at least report a warning. However, if there's
+			//   a systemic condition (e.g. using an older version of kbld to
+			//   deploy than was used to package) there would likely be a flurry
+			//   of warnings. So, the feature would quickly need an enhancement
+			//   to de-dup such warnings. (read: added complexity)
+			// see also https://github.com/vmware-tanzu/carvel-kbld/issues/160
+		}
+	}
+	return nil
+}

pkg/kbld/config/images_lock.go

@@ -4,5 +4,6 @@
 package config
 
 const (
-	ImagesLockKbldID = "kbld.carvel.dev/id"
+	ImagesLockKbldID    = "kbld.carvel.dev/id"
+	ImagesLockKbldMetas = "kbld.carvel.dev/metas"
 )

pkg/kbld/image/built.go

@@ -32,7 +32,7 @@ func NewBuiltImage(url string, buildSource ctlconf.Source, imgDst *ctlconf.Image
 	return BuiltImage{url, buildSource, imgDst, docker, pack, kubectlBuildkit, ko, bazel}
 }
 
-func (i BuiltImage) URL() (string, []Meta, error) {
+func (i BuiltImage) URL() (string, []ctlconf.Meta, error) {
 	metas, err := i.sources()
 	if err != nil {
 		return "", nil, err
@@ -100,7 +100,7 @@ func (i BuiltImage) URL() (string, []Meta, error) {
 	}
 }
 
-func (i BuiltImage) optionalPushWithDocker(dockerTmpRef ctlbdk.DockerTmpRef, metas []Meta) (string, []Meta, error) {
+func (i BuiltImage) optionalPushWithDocker(dockerTmpRef ctlbdk.DockerTmpRef, metas []ctlconf.Meta) (string, []ctlconf.Meta, error) {
 	if i.imgDst != nil {
 		digest, err := i.docker.Push(dockerTmpRef, i.imgDst.NewImage)
 		if err != nil {
@@ -118,41 +118,24 @@ func (i BuiltImage) optionalPushWithDocker(dockerTmpRef ctlbdk.DockerTmpRef, met
 	return dockerTmpRef.AsString(), metas, nil
 }
 
-type BuiltImageSourceGit struct {
-	Type      string // always set to 'git'
-	RemoteURL string `json:",omitempty" yaml:",omitempty"`
-	SHA       string
-	Dirty     bool
-	Tags      []string `json:",omitempty" yaml:",omitempty"`
-}
-
-func (BuiltImageSourceGit) meta() {}
-
-type BuiltImageSourceLocal struct {
-	Type string // always set to 'local'
-	Path string
-}
-
-func (BuiltImageSourceLocal) meta() {}
-
-func (i BuiltImage) sources() ([]Meta, error) {
-	var sources []Meta
+func (i BuiltImage) sources() ([]ctlconf.Meta, error) {
+	var sources []ctlconf.Meta
 
 	absPath, err := filepath.Abs(i.buildSource.Path)
 	if err != nil {
 		return nil, err
 	}
 
-	sources = append(sources, BuiltImageSourceLocal{
-		Type: "local",
+	sources = append(sources, ctlconf.BuiltImageSourceLocal{
+		Type: ctlconf.LocalMeta,
 		Path: absPath,
 	})
 
 	gitRepo := NewGitRepo(absPath)
 
 	if gitRepo.IsValid() {
 		var err error
-		git := BuiltImageSourceGit{Type: "git"}
+		git := ctlconf.BuiltImageSourceGit{Type: ctlconf.GitMeta}
 
 		git.RemoteURL, err = gitRepo.RemoteURL()
 		if err != nil {

pkg/kbld/image/digested.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	regname "github.com/google/go-containerregistry/pkg/name"
+	ctlconf "github.com/k14s/kbld/pkg/kbld/config"
 )
 
 const digestSep = "@"
@@ -40,7 +41,7 @@ func NewDigestedImageFromParts(url, digest string) DigestedImage {
 	return DigestedImage{nameWithDigest, nil}
 }
 
-func (i DigestedImage) URL() (string, []Meta, error) {
+func (i DigestedImage) URL() (string, []ctlconf.Meta, error) {
 	if i.parseErr != nil {
 		return "", nil, i.parseErr
 	}

pkg/kbld/image/factory.go

@@ -15,11 +15,7 @@ import (
 )
 
 type Image interface {
-	URL() (string, []Meta, error)
-}
-
-type Meta interface {
-	meta()
+	URL() (string, []ctlconf.Meta, error)
 }
 
 type Factory struct {
@@ -36,7 +32,7 @@ func (f Factory) New(url string) Image {
 	if overrideConf, found := f.shouldOverride(url); found {
 		url = overrideConf.NewImage
 		if overrideConf.Preresolved {
-			return NewPreresolvedImage(url)
+			return NewPreresolvedImage(url, overrideConf.ImageMetas)
 		} else if overrideConf.TagSelection != nil {
 			return NewTagSelectedImage(url, overrideConf.TagSelection, f.registry)
 		}

pkg/kbld/image/preresolved.go

@@ -3,21 +3,26 @@
 
 package image
 
+import (
+	ctlconf "github.com/k14s/kbld/pkg/kbld/config"
+)
+
 type PreresolvedImage struct {
-	url string
+	url   string
+	metas []ctlconf.Meta
 }
 
-type PreresolvedImageSourceURL struct {
-	Type string // always set to 'preresolved'
-	URL  string
+func NewPreresolvedImage(url string, metas []ctlconf.Meta) PreresolvedImage {
+	return PreresolvedImage{url, copyAndAppendMeta(metas)}
 }
 
-func (PreresolvedImageSourceURL) meta() {}
-
-func NewPreresolvedImage(url string) PreresolvedImage {
-	return PreresolvedImage{url}
+func (i PreresolvedImage) URL() (string, []ctlconf.Meta, error) {
+	imageMetas := copyAndAppendMeta(i.metas, ctlconf.PreresolvedImageSourceURL{Type: ctlconf.PreresolvedMeta, URL: i.url})
+	return i.url, imageMetas, nil
 }
 
-func (i PreresolvedImage) URL() (string, []Meta, error) {
-	return i.url, []Meta{PreresolvedImageSourceURL{Type: "preresolved", URL: i.url}}, nil
+func copyAndAppendMeta(existing []ctlconf.Meta, new ...ctlconf.Meta) []ctlconf.Meta {
+	all := make([]ctlconf.Meta, len(existing), len(existing)+len(new))
+	copy(all, existing)
+	return append(all, new...)
 }