diff --git a/actool/discover.go b/actool/discover.go index f91f526d..4eb1e653 100644 --- a/actool/discover.go +++ b/actool/discover.go @@ -15,12 +15,18 @@ package main import ( + "crypto/tls" "encoding/json" "fmt" + "net" + "net/http" + "net/url" "runtime" "strings" + "time" "github.com/appc/spec/discovery" + "github.com/appc/spec/schema" ) var ( @@ -62,12 +68,37 @@ func runDiscover(args []string) (exit int) { if transportFlags.Insecure { insecure = discovery.InsecureTLS | discovery.InsecureHTTP } + tagsEndpoints, attempts, err := discovery.DiscoverImageTags(*app, nil, insecure) + if err != nil { + stderr("error fetching endpoints for %s: %s", name, err) + return 1 + } + for _, a := range attempts { + fmt.Printf("discover tags walk: prefix: %s error: %v\n", a.Prefix, a.Error) + } + if len(tagsEndpoints) != 0 { + tags, err := fetchImageTags(tagsEndpoints[0].ImageTags, insecure) + if err != nil { + stderr("error fetching tags info: %s", err) + return 1 + } + // Merge tag labels + app, err = app.MergeTag(tags) + if err != nil { + stderr("error resolving tags to labels: %s", err) + return 1 + } + } else { + fmt.Printf("no discover tags found") + } + eps, attempts, err := discovery.DiscoverACIEndpoints(*app, nil, insecure) if err != nil { stderr("error fetching endpoints for %s: %s", name, err) return 1 } for _, a := range attempts { + fmt.Printf("discover endpoints walk: prefix: %s error: %v\n", a.Prefix, a.Error) } publicKeys, attempts, err := discovery.DiscoverPublicKeys(*app, nil, insecure) @@ -104,3 +135,61 @@ func runDiscover(args []string) (exit int) { return } + +func fetchImageTags(urlStr string, insecure discovery.InsecureOption) (*schema.ImageTags, error) { + t := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + Dial: func(n, a string) (net.Conn, error) { + return net.DialTimeout(n, a, 5*time.Second) + }, + } + if insecure&discovery.InsecureTLS != 0 { + t.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + client := &http.Client{ + Transport: t, + } + + fetch := func(scheme string) (res *http.Response, err error) { + u, err := url.Parse(urlStr) + if err != nil { + return nil, err + } + u.Scheme = scheme + urlStr := u.String() + req, err := http.NewRequest("GET", urlStr, nil) + if err != nil { + return nil, err + } + res, err = client.Do(req) + return + } + closeBody := func(res *http.Response) { + if res != nil { + res.Body.Close() + } + } + res, err := fetch("https") + if err != nil || res.StatusCode != http.StatusOK { + if insecure&discovery.InsecureHTTP != 0 { + closeBody(res) + res, err = fetch("http") + } + } + + if res != nil && res.StatusCode != http.StatusOK { + err = fmt.Errorf("expected a 200 OK got %d", res.StatusCode) + } + + if err != nil { + closeBody(res) + return nil, err + } + + var tags *schema.ImageTags + jd := json.NewDecoder(res.Body) + jd.Decode(&tags) + closeBody(res) + + return tags, nil +} diff --git a/discovery/discovery.go b/discovery/discovery.go index 2160f60b..95b80c01 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -37,21 +37,25 @@ type ACIEndpoint struct { ASC string } +type ImageTagsEndpoint struct { + ImageTags string + ASC string +} + // A struct containing both discovered endpoints and keys. Used to avoid // function duplication (one for endpoints and one for keys, so to avoid two // doDiscover, two DiscoverWalkFunc) type discoveryData struct { - ACIEndpoints []ACIEndpoint - PublicKeys []string + ACIEndpoints []ACIEndpoint + PublicKeys []string + ImageTagsEndpoints []ImageTagsEndpoint } type ACIEndpoints []ACIEndpoint type PublicKeys []string -const ( - defaultVersion = "latest" -) +type ImageTagsEndpoints []ImageTagsEndpoint var ( templateExpression = regexp.MustCompile(`{.*?}`) @@ -128,9 +132,6 @@ func createTemplateVars(app App) []string { func doDiscover(pre string, hostHeaders map[string]http.Header, app App, insecure InsecureOption) (*discoveryData, error) { app = *app.Copy() - if app.Labels["version"] == "" { - app.Labels["version"] = defaultVersion - } _, body, err := httpsOrHTTP(pre, hostHeaders, insecure) if err != nil { @@ -165,6 +166,20 @@ func doDiscover(pre string, hostHeaders map[string]http.Header, app App, insecur case "ac-discovery-pubkeys": dd.PublicKeys = append(dd.PublicKeys, m.uri) + case "ac-discovery-tags": + // Only name is used for tags discovery + tplVars := []string{"{name}", app.Name.String()} + // Ignore not handled variables as {ext} isn't already rendered. + uri, _ := renderTemplate(m.uri, tplVars...) + asc, ok := renderTemplate(uri, "{ext}", "aci.asc") + if !ok { + continue + } + tags, ok := renderTemplate(uri, "{ext}", "aci") + if !ok { + continue + } + dd.ImageTagsEndpoints = append(dd.ImageTagsEndpoints, ImageTagsEndpoint{ImageTags: tags, ASC: asc}) } } @@ -175,6 +190,7 @@ func doDiscover(pre string, hostHeaders map[string]http.Header, app App, insecur // optionally will use HTTP if insecure is set. hostHeaders specifies the // header to apply depending on the host (e.g. authentication). Based on the // response of the discoverFn it will continue to recurse up the tree. +// If no discovery data can be found an empty discoveryData will be returned. func DiscoverWalk(app App, hostHeaders map[string]http.Header, insecure InsecureOption, discoverFn DiscoverWalkFunc) (dd *discoveryData, err error) { parts := strings.Split(string(app.Name), "/") for i := range parts { @@ -187,7 +203,7 @@ func DiscoverWalk(app App, hostHeaders map[string]http.Header, insecure Insecure } } - return nil, fmt.Errorf("discovery failed") + return &discoveryData{}, nil } // DiscoverWalkFunc can stop a DiscoverWalk by returning non-nil error. @@ -232,10 +248,13 @@ func DiscoverACIEndpoints(app App, hostHeaders map[string]http.Header, insecure return nil, attempts, err } + if len(dd.ACIEndpoints) == 0 { + return nil, attempts, fmt.Errorf("No ACI endpoints discovered") + } return dd.ACIEndpoints, attempts, nil } -// DiscoverPublicKey will make HTTPS requests to find the ac-public-keys meta +// DiscoverPublicKeys will make HTTPS requests to find the ac-discovery-pubkeys meta // tags and optionally will use HTTP if insecure is set. hostHeaders // specifies the header to apply depending on the host (e.g. authentication). // It will not give up until it has exhausted the path or found an public key. @@ -253,5 +272,29 @@ func DiscoverPublicKeys(app App, hostHeaders map[string]http.Header, insecure In return nil, attempts, err } + if len(dd.PublicKeys) == 0 { + return nil, attempts, fmt.Errorf("No public keys discovered") + } return dd.PublicKeys, attempts, nil } + +// DiscoverImageTags will make HTTPS requests to find the ac-discovery-imagetags meta +// tags and optionally will use HTTP if insecure is set. hostHeaders +// specifies the header to apply depending on the host (e.g. authentication). +// It will not give up until it has exhausted the path or found an imagetag. +func DiscoverImageTags(app App, hostHeaders map[string]http.Header, insecure InsecureOption) (ImageTagsEndpoints, []FailedAttempt, error) { + testFn := func(pre string, dd *discoveryData, err error) error { + if len(dd.ImageTagsEndpoints) != 0 { + return errEnough + } + return nil + } + + attempts := []FailedAttempt{} + dd, err := DiscoverWalk(app, hostHeaders, insecure, walker(&attempts, testFn)) + if err != nil && err != errEnough { + return nil, attempts, err + } + + return dd.ImageTagsEndpoints, attempts, nil +} diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index bb2ca254..7491dadf 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -25,6 +25,7 @@ import ( "strings" "testing" + "github.com/appc/spec/schema" "github.com/appc/spec/schema/types" ) @@ -86,9 +87,11 @@ func fakeHTTPGet(metas []meta, header http.Header) func(req *http.Request) (*htt func TestDiscoverEndpoints(t *testing.T) { tests := []struct { do httpDoer + expectMergeTagSuccess bool expectDiscoveryACIEndpointsSuccess bool expectDiscoveryPublicKeysSuccess bool app App + tags *schema.ImageTags expectedACIEndpoints []ACIEndpoint expectedPublicKeys []string authHeader http.Header @@ -111,6 +114,7 @@ func TestDiscoverEndpoints(t *testing.T) { }, true, true, + true, App{ Name: "example.com/myapp", Labels: map[types.ACIdentifier]string{ @@ -119,6 +123,7 @@ func TestDiscoverEndpoints(t *testing.T) { "arch": "amd64", }, }, + nil, []ACIEndpoint{ ACIEndpoint{ ACI: "https://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci", @@ -144,6 +149,7 @@ func TestDiscoverEndpoints(t *testing.T) { }, true, true, + true, App{ Name: "example.com/myapp/foobar", Labels: map[types.ACIdentifier]string{ @@ -152,6 +158,7 @@ func TestDiscoverEndpoints(t *testing.T) { "arch": "amd64", }, }, + nil, []ACIEndpoint{ ACIEndpoint{ ACI: "https://storage.example.com/example.com/myapp/foobar-1.0.0-linux-amd64.aci", @@ -176,6 +183,7 @@ func TestDiscoverEndpoints(t *testing.T) { nil, ), }, + true, false, false, App{ @@ -189,6 +197,7 @@ func TestDiscoverEndpoints(t *testing.T) { nil, nil, nil, + nil, }, // Test with only 'ac-discovery' at / and only @@ -212,6 +221,7 @@ func TestDiscoverEndpoints(t *testing.T) { }, true, true, + true, App{ Name: "example.com/myapp", Labels: map[types.ACIdentifier]string{ @@ -220,6 +230,7 @@ func TestDiscoverEndpoints(t *testing.T) { "arch": "amd64", }, }, + nil, []ACIEndpoint{ ACIEndpoint{ ACI: "https://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci", @@ -250,6 +261,7 @@ func TestDiscoverEndpoints(t *testing.T) { }, true, true, + true, App{ Name: "example.com/myapp", Labels: map[types.ACIdentifier]string{ @@ -258,6 +270,7 @@ func TestDiscoverEndpoints(t *testing.T) { "arch": "amd64", }, }, + nil, []ACIEndpoint{ ACIEndpoint{ ACI: "https://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci", @@ -281,6 +294,7 @@ func TestDiscoverEndpoints(t *testing.T) { ), }, true, + true, false, App{ Name: "example.com/myapp", @@ -290,6 +304,7 @@ func TestDiscoverEndpoints(t *testing.T) { "arch": "amd64", }, }, + nil, []ACIEndpoint{ ACIEndpoint{ ACI: "https://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci", @@ -312,6 +327,7 @@ func TestDiscoverEndpoints(t *testing.T) { nil, ), }, + true, false, true, App{ @@ -323,6 +339,7 @@ func TestDiscoverEndpoints(t *testing.T) { }, }, nil, + nil, []string{"https://example.com/pubkeys.gpg"}, nil, }, @@ -349,6 +366,7 @@ func TestDiscoverEndpoints(t *testing.T) { }, true, true, + true, App{ Name: "example.com/myapp", Labels: map[types.ACIdentifier]string{ @@ -357,6 +375,7 @@ func TestDiscoverEndpoints(t *testing.T) { "arch": "amd64", }, }, + nil, []ACIEndpoint{ ACIEndpoint{ ACI: "https://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci", @@ -387,12 +406,14 @@ func TestDiscoverEndpoints(t *testing.T) { }, true, true, + true, App{ Name: "example.com/myapp", Labels: map[types.ACIdentifier]string{ "version": "1.0.0", }, }, + nil, []ACIEndpoint{ ACIEndpoint{ ACI: "https://storage.example.com/example.com/myapp-1.0.0.aci", @@ -402,8 +423,7 @@ func TestDiscoverEndpoints(t *testing.T) { []string{"https://example.com/pubkeys.gpg"}, nil, }, - // Test missing labels. version label should default to - // "latest" and the first template should be rendered + // Test with a label called "name". It should be ignored. { &mockHTTPDoer{ doer: fakeHTTPGet( @@ -417,26 +437,31 @@ func TestDiscoverEndpoints(t *testing.T) { }, true, true, + true, App{ - Name: "example.com/myapp", - Labels: map[types.ACIdentifier]string{}, + Name: "example.com/myapp", + Labels: map[types.ACIdentifier]string{ + "name": "labelcalledname", + "version": "1.0.0", + }, }, + nil, []ACIEndpoint{ ACIEndpoint{ - ACI: "https://storage.example.com/example.com/myapp-latest.aci", - ASC: "https://storage.example.com/example.com/myapp-latest.aci.asc", + ACI: "https://storage.example.com/example.com/myapp-1.0.0.aci", + ASC: "https://storage.example.com/example.com/myapp-1.0.0.aci.asc", }, }, []string{"https://example.com/pubkeys.gpg"}, nil, }, - // Test with a label called "name". It should be ignored. + // Test multiple ACIEndpoints. { &mockHTTPDoer{ doer: fakeHTTPGet( []meta{ {"/myapp", - "meta05.html", + "meta06.html", }, }, nil, @@ -444,29 +469,40 @@ func TestDiscoverEndpoints(t *testing.T) { }, true, true, + true, App{ Name: "example.com/myapp", Labels: map[types.ACIdentifier]string{ - "name": "labelcalledname", "version": "1.0.0", + "os": "linux", + "arch": "amd64", }, }, + nil, []ACIEndpoint{ + ACIEndpoint{ + ACI: "https://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci", + ASC: "https://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci.asc", + }, ACIEndpoint{ ACI: "https://storage.example.com/example.com/myapp-1.0.0.aci", ASC: "https://storage.example.com/example.com/myapp-1.0.0.aci.asc", }, + ACIEndpoint{ + ACI: "hdfs://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci", + ASC: "hdfs://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci.asc", + }, }, []string{"https://example.com/pubkeys.gpg"}, nil, }, - // Test multiple ACIEndpoints. + // Test tag alias { &mockHTTPDoer{ doer: fakeHTTPGet( []meta{ {"/myapp", - "meta06.html", + "meta01.html", }, }, nil, @@ -474,26 +510,139 @@ func TestDiscoverEndpoints(t *testing.T) { }, true, true, + true, App{ Name: "example.com/myapp", + Tag: "latest", + Labels: map[types.ACIdentifier]string{ + "os": "linux", + "arch": "amd64", + }, + }, + &schema.ImageTags{ + Aliases: schema.TagAliases{ + "latest": "2.x", + }, + Labels: schema.TagLabels{ + "2.x": map[types.ACIdentifier]string{ + "version": "2.0.0", + }, + }, + }, + []ACIEndpoint{ + ACIEndpoint{ + ACI: "https://storage.example.com/example.com/myapp-2.0.0-linux-amd64.aci", + ASC: "https://storage.example.com/example.com/myapp-2.0.0-linux-amd64.aci.asc", + }, + }, + []string{"https://example.com/pubkeys.gpg"}, + nil, + }, + // Test tag alias should not override required version label + { + &mockHTTPDoer{ + doer: fakeHTTPGet( + []meta{ + {"/myapp", + "meta01.html", + }, + }, + nil, + ), + }, + true, + true, + true, + App{ + Name: "example.com/myapp", + Tag: "latest", Labels: map[types.ACIdentifier]string{ "version": "1.0.0", "os": "linux", "arch": "amd64", }, }, + &schema.ImageTags{ + Aliases: schema.TagAliases{ + "latest": "2.x", + }, + Labels: schema.TagLabels{ + "2.x": map[types.ACIdentifier]string{ + "version": "2.0.0", + }, + }, + }, []ACIEndpoint{ ACIEndpoint{ ACI: "https://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci", ASC: "https://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci.asc", }, + }, + []string{"https://example.com/pubkeys.gpg"}, + nil, + }, + // Test tag without image tags. Should set tag to version label. + { + &mockHTTPDoer{ + doer: fakeHTTPGet( + []meta{ + {"/myapp", + "meta01.html", + }, + }, + nil, + ), + }, + true, + true, + true, + App{ + Name: "example.com/myapp", + Tag: "latest", + Labels: map[types.ACIdentifier]string{ + "os": "linux", + "arch": "amd64", + }, + }, + nil, + []ACIEndpoint{ ACIEndpoint{ - ACI: "https://storage.example.com/example.com/myapp-1.0.0.aci", - ASC: "https://storage.example.com/example.com/myapp-1.0.0.aci.asc", + ACI: "https://storage.example.com/example.com/myapp-latest-linux-amd64.aci", + ASC: "https://storage.example.com/example.com/myapp-latest-linux-amd64.aci.asc", }, + }, + []string{"https://example.com/pubkeys.gpg"}, + nil, + }, + // Test tag without image tags. Should set tag to version label but fail since version is already specified. + { + &mockHTTPDoer{ + doer: fakeHTTPGet( + []meta{ + {"/myapp", + "meta01.html", + }, + }, + nil, + ), + }, + false, + true, + true, + App{ + Name: "example.com/myapp", + Tag: "latest", + Labels: map[types.ACIdentifier]string{ + "version": "1.0.0", + "os": "linux", + "arch": "amd64", + }, + }, + nil, + []ACIEndpoint{ ACIEndpoint{ - ACI: "hdfs://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci", - ASC: "hdfs://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci.asc", + ACI: "https://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci", + ASC: "https://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci.asc", }, }, []string{"https://example.com/pubkeys.gpg"}, @@ -514,6 +663,7 @@ func TestDiscoverEndpoints(t *testing.T) { }, true, true, + true, App{ Name: "example.com/myapp", Labels: map[types.ACIdentifier]string{ @@ -522,6 +672,7 @@ func TestDiscoverEndpoints(t *testing.T) { "arch": "amd64", }, }, + nil, []ACIEndpoint{ ACIEndpoint{ ACI: "https://storage.example.com/example.com/myapp-1.0.0-linux-amd64.aci", @@ -549,7 +700,19 @@ func TestDiscoverEndpoints(t *testing.T) { InsecureTLS | InsecureHTTP, } for _, insecure := range insecureList { - eps, _, err := DiscoverACIEndpoints(tt.app, hostHeaders, insecure) + // Expand App labels with tags info labels + app, err := tt.app.MergeTag(tt.tags) + if err != nil && !tt.expectMergeTagSuccess { + continue + } + if err == nil && !tt.expectMergeTagSuccess { + t.Fatalf("#%d MergeTag should have failed but didn't", i) + } + if err != nil { + t.Fatalf("#%d MergeTag failed: %v", i, err) + } + + eps, _, err := DiscoverACIEndpoints(*app, hostHeaders, insecure) if err != nil && !tt.expectDiscoveryACIEndpointsSuccess { continue } @@ -559,7 +722,8 @@ func TestDiscoverEndpoints(t *testing.T) { if err != nil { t.Fatalf("#%d DiscoverACIEndpoints failed: %v", i, err) } - publicKeys, _, err := DiscoverPublicKeys(tt.app, hostHeaders, insecure) + + publicKeys, _, err := DiscoverPublicKeys(*app, hostHeaders, insecure) if err != nil && !tt.expectDiscoveryPublicKeysSuccess { continue } diff --git a/discovery/parse.go b/discovery/parse.go index 7b9d574a..eb7ab68f 100644 --- a/discovery/parse.go +++ b/discovery/parse.go @@ -19,16 +19,18 @@ import ( "net/url" "strings" + "github.com/appc/spec/schema" "github.com/appc/spec/schema/common" "github.com/appc/spec/schema/types" ) type App struct { Name types.ACIdentifier + Tag string Labels map[types.ACIdentifier]string } -func NewApp(name string, labels map[types.ACIdentifier]string) (*App, error) { +func NewApp(name string, tag string, labels map[types.ACIdentifier]string) (*App, error) { if labels == nil { labels = make(map[types.ACIdentifier]string, 0) } @@ -38,11 +40,13 @@ func NewApp(name string, labels map[types.ACIdentifier]string) (*App, error) { } return &App{ Name: *acn, + Tag: tag, Labels: labels, }, nil } -// NewAppFromString takes a command line app parameter and returns a map of labels. +// NewAppFromString takes a command line app parameter and returns an App +// struct containing the app name, tag and a map of labels. // // Example app parameters: // example.com/reduce-worker:1.0.0 @@ -55,6 +59,7 @@ func NewApp(name string, labels map[types.ACIdentifier]string) (*App, error) { func NewAppFromString(app string) (*App, error) { var ( name string + tag string labels map[types.ACIdentifier]string ) @@ -75,13 +80,17 @@ func NewAppFromString(app string) (*App, error) { name = val[0] continue } + if key == "tag" { + tag = val[0] + continue + } labelName, err := types.NewACIdentifier(key) if err != nil { return nil, err } labels[*labelName] = val[0] } - a, err := NewApp(name, labels) + a, err := NewApp(name, tag, labels) if err != nil { return nil, err } @@ -93,7 +102,7 @@ func prepareAppString(app string) (string, error) { return "", err } - app = "name=" + strings.Replace(app, ":", ",version=", 1) + app = "name=" + strings.Replace(app, ":", ",tag=", 1) return common.MakeQueryString(app) } @@ -112,6 +121,7 @@ func checkColon(app string) error { func (a *App) Copy() *App { ac := &App{ Name: a.Name, + Tag: a.Tag, Labels: make(map[types.ACIdentifier]string, 0), } for k, v := range a.Labels { @@ -123,8 +133,25 @@ func (a *App) Copy() *App { // String returns the URL-like image name func (a *App) String() string { img := a.Name.String() + if a.Tag != "" { + img += ":" + a.Tag + } for n, v := range a.Labels { img += fmt.Sprintf(",%s=%s", n, v) } return img } + +// MergeTag will resolve image tags labels from App.Tag and return a new App +// with the Labels merged and the Tag unsetted. +func (a *App) MergeTag(tags *schema.ImageTags) (*App, error) { + newlabels, err := tags.MergeTag(a.Labels, a.Tag) + if err != nil { + return nil, err + } + newapp := a.Copy() + // Unset newapp.Tag + newapp.Tag = "" + newapp.Labels = newlabels + return newapp, nil +} diff --git a/discovery/parse_test.go b/discovery/parse_test.go index 61bd4a99..b1dcd02c 100644 --- a/discovery/parse_test.go +++ b/discovery/parse_test.go @@ -31,6 +31,16 @@ func TestNewAppFromString(t *testing.T) { { "example.com/reduce-worker:1.0.0", + &App{ + Name: "example.com/reduce-worker", + Tag: "1.0.0", + Labels: map[types.ACIdentifier]string{}, + }, + false, + }, + { + "example.com/reduce-worker,version=1.0.0", + &App{ Name: "example.com/reduce-worker", Labels: map[types.ACIdentifier]string{ @@ -39,6 +49,7 @@ func TestNewAppFromString(t *testing.T) { }, false, }, + { "example.com/reduce-worker,channel=alpha,label=value", @@ -57,8 +68,8 @@ func TestNewAppFromString(t *testing.T) { &App{ Name: "example.com/app", + Tag: "1.2.3", Labels: map[types.ACIdentifier]string{ - "version": "1.2.3", "special": "!*'();@&+$/?#[]", "channel": "beta", }, @@ -102,13 +113,6 @@ func TestNewAppFromString(t *testing.T) { { "example.com/app:3.2.1,channel=beta:1.2.3", - nil, - true, - }, - // two version labels, one implicit, one explicit - { - "example.com/app:3.2.1,version=1.2.3", - nil, true, }, @@ -182,6 +186,15 @@ func TestAppCopy(t *testing.T) { }, "example.com/reduce-worker", }, + { + &App{ + Name: "example.com/reduce-worker", + Tag: "1.0.0", + Labels: map[types.ACIdentifier]string{}, + }, + "example.com/reduce-worker", + }, + { &App{ Name: "example.com/reduce-worker", diff --git a/pkg/acirenderer/acirenderer.go b/pkg/acirenderer/acirenderer.go index 25a097f0..11d4cc8a 100644 --- a/pkg/acirenderer/acirenderer.go +++ b/pkg/acirenderer/acirenderer.go @@ -33,6 +33,7 @@ import ( type ACIRegistry interface { ACIProvider GetImageManifest(key string) (*schema.ImageManifest, error) + GetImageTags(name types.ACIdentifier) (*schema.ImageTags, error) GetACI(name types.ACIdentifier, labels types.Labels) (string, error) } @@ -86,11 +87,11 @@ func GetRenderedACIWithImageID(imageID types.Hash, ap ACIRegistry) (RenderedACI, return GetRenderedACIFromList(imgs, ap) } -// GetRenderedACI, given an image app name and optional labels, starts with the +// GetRenderedACI, given an image app name, an optional tag and optional labels, starts with the // best matching image available in the store, creates the dependencies list // and returns the RenderedACI list. -func GetRenderedACI(name types.ACIdentifier, labels types.Labels, ap ACIRegistry) (RenderedACI, error) { - imgs, err := CreateDepListFromNameLabels(name, labels, ap) +func GetRenderedACI(name types.ACIdentifier, tag string, labels types.Labels, ap ACIRegistry) (RenderedACI, error) { + imgs, err := CreateDepListFromNameTagLabels(name, tag, labels, ap) if err != nil { return nil, err } diff --git a/pkg/acirenderer/acirenderer_test.go b/pkg/acirenderer/acirenderer_test.go index ca834b34..2c848df8 100644 --- a/pkg/acirenderer/acirenderer_test.go +++ b/pkg/acirenderer/acirenderer_test.go @@ -1990,10 +1990,10 @@ func Test3Deps(t *testing.T) { } } -// Given an image app name and optional labels, get the best matching image +// Given an image app name, optional tag and optional labels, get the best matching image // available in the store, build its dependency list and render it inside dir -func RenderACI(name types.ACIdentifier, labels types.Labels, ap ACIRegistry) (map[string]*fileInfo, error) { - renderedACI, err := GetRenderedACI(name, labels, ap) +func RenderACI(name types.ACIdentifier, tag string, labels types.Labels, ap ACIRegistry) (map[string]*fileInfo, error) { + renderedACI, err := GetRenderedACI(name, tag, labels, ap) if err != nil { return nil, err } @@ -2054,7 +2054,7 @@ func renderImage(renderedACI RenderedACI, ap ACIProvider) (map[string]*fileInfo, } func checkRenderACI(app types.ACIdentifier, expectedFiles []*fileInfo, ds *TestStore) error { - files, err := RenderACI(app, nil, ds) + files, err := RenderACI(app, "", nil, ds) if err != nil { return err } diff --git a/pkg/acirenderer/resolve.go b/pkg/acirenderer/resolve.go index 248bcbf2..07373f3f 100644 --- a/pkg/acirenderer/resolve.go +++ b/pkg/acirenderer/resolve.go @@ -32,8 +32,24 @@ func CreateDepListFromImageID(imageID types.Hash, ap ACIRegistry) (Images, error // CreateDepListFromNameLabels returns the flat dependency tree of the image // with the provided app name and optional labels. -func CreateDepListFromNameLabels(name types.ACIdentifier, labels types.Labels, ap ACIRegistry) (Images, error) { - key, err := ap.GetACI(name, labels) +func CreateDepListFromNameTagLabels(name types.ACIdentifier, tag string, labels types.Labels, ap ACIRegistry) (Images, error) { + // Merge image tags labels + imageTags, err := ap.GetImageTags(name) + if err != nil { + return nil, err + } + + newlabelsmap, err := imageTags.MergeTag(labels.ToMap(), tag) + if err != nil { + return nil, err + } + + newlabels, err := types.LabelsFromMap(newlabelsmap) + if err != nil { + return nil, err + } + + key, err := ap.GetACI(name, newlabels) if err != nil { return nil, err } @@ -66,7 +82,23 @@ func createDepList(key string, ap ACIRegistry) (Images, error) { } } else { var err error - depKey, err = ap.GetACI(d.ImageName, d.Labels) + // Merge image tags labels + imageTags, err := ap.GetImageTags(d.ImageName) + if err != nil { + return nil, err + } + + newlabelsmap, err := imageTags.MergeTag(d.Labels.ToMap(), d.Tag) + if err != nil { + return nil, err + } + + newlabels, err := types.LabelsFromMap(newlabelsmap) + if err != nil { + return nil, err + } + + depKey, err = ap.GetACI(d.ImageName, newlabels) if err != nil { return nil, err } diff --git a/pkg/acirenderer/store_test.go b/pkg/acirenderer/store_test.go index 281137bb..3f7a61e5 100644 --- a/pkg/acirenderer/store_test.go +++ b/pkg/acirenderer/store_test.go @@ -75,6 +75,11 @@ func (ts *TestStore) GetImageManifest(key string) (*schema.ImageManifest, error) return aci.ImageManifest, nil } + +func (ts *TestStore) GetImageTags(name types.ACIdentifier) (*schema.ImageTags, error) { + return nil, nil +} + func (ts *TestStore) GetACI(name types.ACIdentifier, labels types.Labels) (string, error) { for _, aci := range ts.acis { if aci.ImageManifest.Name.String() == name.String() { diff --git a/schema/imagetags.go b/schema/imagetags.go new file mode 100644 index 00000000..9e4a74b7 --- /dev/null +++ b/schema/imagetags.go @@ -0,0 +1,119 @@ +// Copyright 2016 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schema + +import ( + "encoding/json" + "fmt" + + "github.com/appc/spec/schema/types" +) + +type ImageTags struct { + Aliases TagAliases `json:"aliases"` + Labels TagLabels `json:"labels"` +} + +type imageTags ImageTags + +type TagAliases map[string]string + +type TagLabels map[string]map[types.ACIdentifier]string + +//TODO(sgotti) add validation, circular references checks etc... +func (t ImageTags) assertValid() error { + return nil +} + +func (t ImageTags) MarshalJSON() ([]byte, error) { + if err := t.assertValid(); err != nil { + return nil, err + } + return json.Marshal(imageTags(t)) +} + +func (t *ImageTags) UnmarshalJSON(data []byte) error { + var jt imageTags + if err := json.Unmarshal(data, &jt); err != nil { + return err + } + nt := ImageTags(jt) + if err := nt.assertValid(); err != nil { + return err + } + *t = nt + return nil +} + +// Resolve will resolve tag Aliases until exausted (checking for circular +// dependencies) and then it will return the Labels referenced from the resolved +// tag if existing or nil if not. +func (t *ImageTags) Resolve(tag string) (map[types.ACIdentifier]string, error) { + curtag := tag + seen := map[string]struct{}{} + seen[curtag] = struct{}{} + for { + end := true + if alias, ok := t.Aliases[tag]; ok { + if _, ok := seen[alias]; ok { + return nil, fmt.Errorf("circular dependency between tag aliases") + } + curtag = alias + seen[curtag] = struct{}{} + end = false + break + } + if end { + break + } + } + if labels, ok := t.Labels[curtag]; ok { + return labels, nil + } + return nil, nil +} + +// MergeTag will resolve image tags labels from tag and return the new merged labels +func (t *ImageTags) MergeTag(labels types.LabelsMap, tag string) (types.LabelsMap, error) { + newlabels := labels.Copy() + // if tag is empty stop here + if tag == "" { + return labels, nil + } + // Not tag data provided. Fallback setting version label value to tag value + if t == nil { + if _, ok := newlabels["version"]; !ok { + newlabels["version"] = tag + return newlabels, nil + } else { + return nil, fmt.Errorf("cannot set tag value to version label since version label is already defined") + } + } + tagLabels, err := t.Resolve(tag) + if err != nil { + return nil, err + } + // No labels resolved from tag. + if tagLabels == nil { + return newlabels, nil + } + // Merge tagLabels with app labels. App specified labels have the precedence. + for n, v := range tagLabels { + if _, ok := newlabels[n]; !ok { + newlabels[n] = v + } + } + return newlabels, nil +} diff --git a/schema/imagetags_test.go b/schema/imagetags_test.go new file mode 100644 index 00000000..61432e5e --- /dev/null +++ b/schema/imagetags_test.go @@ -0,0 +1,42 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schema + +import "testing" + +func TestImageTags(t *testing.T) { + tj := ` + { + "aliases": { + "latest": "3.x", + "3.x": "3.0.x", + "3.0.x": "3.0.1", + "3.0.1": "3.0.1-2" + }, + "labels": { + "3.0.1-2" : { "version": "3.0.1", "build": "2" }, + "3.0.1-3" : { "version": "3.0.1", "build": "3" } + } + } + ` + + var imageTags ImageTags + + err := imageTags.UnmarshalJSON([]byte(tj)) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + +} diff --git a/schema/types/dependencies.go b/schema/types/dependencies.go index fb399e40..c52cc101 100644 --- a/schema/types/dependencies.go +++ b/schema/types/dependencies.go @@ -24,6 +24,7 @@ type Dependencies []Dependency type Dependency struct { ImageName ACIdentifier `json:"imageName"` ImageID *Hash `json:"imageID,omitempty"` + Tag string `json:"tag,omitempty"` Labels Labels `json:"labels,omitempty"` Size uint `json:"size,omitempty"` } diff --git a/schema/types/labels.go b/schema/types/labels.go index ebd2bb1a..8081ebb9 100644 --- a/schema/types/labels.go +++ b/schema/types/labels.go @@ -35,6 +35,8 @@ type Label struct { Value string `json:"value"` } +type LabelsMap map[ACIdentifier]string + // IsValidOsArch checks if a OS-architecture combination is valid given a map // of valid OS-architectures func IsValidOSArch(labels map[ACIdentifier]string, validOSArch map[string][]string) error { @@ -132,3 +134,11 @@ func LabelsFromMap(labelsMap map[ACIdentifier]string) (Labels, error) { } return labels, nil } + +func (lm LabelsMap) Copy() LabelsMap { + nlm := make(LabelsMap, len(lm)) + for k, v := range lm { + nlm[k] = v + } + return nlm +} diff --git a/spec/aci.md b/spec/aci.md index b44849e1..2aa57dfe 100644 --- a/spec/aci.md +++ b/spec/aci.md @@ -242,6 +242,7 @@ JSON Schema for the Image Manifest (app image manifest, ACI manifest), conformin * **dependencies** (list of objects, optional) dependent application images that need to be placed down into the rootfs before the files from this image (if any). The ordering is significant. See [Dependency Matching](#dependency-matching) for how dependencies are retrieved. * **imageName** (string of type [AC Identifier](types.md#ac-identifier-type), required) name of the dependent App Container Image. * **imageID** (string of type [Image ID](types.md#image-id-type), optional) content hash of the dependency. If provided, the retrieved dependency must match the hash. This can be used to produce deterministic, repeatable builds of an App Container Image that has dependencies. + * **tag** (string, optional) a tag that will be resolved to a list of labels (see [Labels Resolution](imagetags.md#labels-resolution)). See [Dependency Matching](#dependency-matching) for how this is used. * **labels** (list of objects, optional) a list of the very same form as the aforementioned label objects in the top level ImageManifest. See [Dependency Matching](#dependency-matching) for how these are used. * **size** (integer, optional) the size of the image referenced dependency, in bytes. This field is optional; if it is present, the ACE SHOULD ensure it matches when retrieving a dependency, to mitigate "endless data" attacks. * **pathWhitelist** (list of strings, optional) whitelist of absolute paths that will exist in the app's rootfs after rendering. This must be a complete and absolute set. An empty list is equivalent to an absent value and means that all files in this image and any dependencies will be available in the rootfs. @@ -253,8 +254,8 @@ JSON Schema for the Image Manifest (app image manifest, ACI manifest), conformin #### Dependency Matching -Dependency matching is based on a combination of the three different fields of the dependency - **imageName**, **imageID**, and **labels**. -First, the image discovery mechanism is used to locate a dependency based on the **imageName** and **labels** (see [App Container Image Discovery](discovery.md)). +Dependency matching is based on a combination of the four different fields of the dependency - **imageName**, **imageID**, **tag**, and **labels**. +First, the image discovery mechanism is used to locate a dependency based on the **imageName**, **tag** and **labels** (see [App Container Image Discovery](discovery.md)). If the image discovery process successfully returns an image and the dependency specification has an image ID, it will be compared against the hash of image returned, and MUST match. diff --git a/spec/discovery.md b/spec/discovery.md index b5033ef3..087cfe32 100644 --- a/spec/discovery.md +++ b/spec/discovery.md @@ -4,21 +4,24 @@ An App Container Image name has a URL-like structure, for example `example.com/r However, there is no scheme on this name, so it cannot be directly resolved to an App Container Image URL. Furthermore, attributes other than the name may be required to unambiguously identify an image (version, OS and architecture). These attributes are expressed in the **labels** field of the [Image Manifest](aci.md#image-manifest-schema). -App Container Image Discovery prescribes a discovery process to retrieve an image based on the name and these attributes. + +App Container Image Discovery prescribes a discovery process to retrieve an image based on a *App Container Image name*, a *tag* and a list of *labels*. Image Discovery is inspired by Go's [remote import paths](https://golang.org/cmd/go/#hdr-Remote_import_paths). -There are three URL types: +There are different URL types: * Image URLs -* Signature URLs * Public key URLs +* Image Tags URLs +* Signature URLs (for Images and Image Tags) ### Discovery Templates -Image Discovery uses one or more templates to render Image and Signature URLs (while the Public keys URLs aren't templates). +Image Discovery uses one or more templates to render Image, ImageTags and Signature URLs (while the Public keys URLs aren't templates). To discriminate between the image and its signature, the templates must contain `{ext}` and its values MUST be either `aci` (for the image) or `aci.asc` (for the signature). +To discriminate between the image tags and its signature, the templates must contain `{ext}` and its values MUST be either `json` (for the image tags) or `json.asc` (for the signature). ### Discovery URL @@ -37,10 +40,12 @@ then inspect the HTML returned for `meta` tags that have the following format: ```html + ``` * `ac-discovery` MUST contain a URL template that can be rendered to retrieve the ACI or associated signature * `ac-discovery-pubkeys` SHOULD contain a URL that provides a set of public keys that can be used to verify the signature of the ACI +* `ac-discovery-imagetags` SHOULD contain a URL that provides [image tags](imagetags.md) data and associated signature. The content of the image tags can be used to resolve a final set of labels to use for resolution of `ac-discovery` templates. Some examples for different schemes and URLs: @@ -48,9 +53,14 @@ Some examples for different schemes and URLs: +