Skip to content

Commit

Permalink
feat(redis): more tests for redis metadb implementation
Browse files Browse the repository at this point in the history 
Signed-off-by: Andrei Aaron <[email protected]>
  • Loading branch information
@andaaron
andaaron committed Jan 22, 2025
1 parent 48f76b8 commit efb88f9
Show file tree
Hide file tree
Showing 4 changed files with 1,480 additions and 9 deletions.
33 changes: 31 additions & 2 deletions pkg/extensions/search/search_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"testing"
"time"

"github.com/alicebob/miniredis/v2"
guuid "github.com/gofrs/uuid"
regTypes "github.com/google/go-containerregistry/pkg/v1/types"
notreg "github.com/notaryproject/notation-go/registry"
Expand Down Expand Up @@ -3905,14 +3906,28 @@ func TestGlobalSearch(t *testing.T) { //nolint: gocyclo
So(len(results.Repos), ShouldEqual, 0)
})

Convey("test nested indexes", t, func() {
Convey("test nested indexes CVE scanning disabled", t, func() {
log := log.NewLogger("debug", "")
rootDir := t.TempDir()
port := GetFreePort()
baseURL := GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
conf.Storage.RootDirectory = rootDir

Convey("test with boltdb", func() {
conf.Storage.CacheDriver = nil
})

Convey("test with redis", func() {
miniRedis := miniredis.RunT(t)

conf.Storage.CacheDriver = map[string]interface{}{
"name": "redis",
"url": "redis://" + miniRedis.Addr(),
}
})

defaultVal := true
conf.Extensions = &extconf.ExtensionConfig{
Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}},
Expand Down Expand Up @@ -4054,14 +4069,28 @@ func TestGlobalSearch(t *testing.T) { //nolint: gocyclo
}
})

Convey("test nested indexes", t, func() {
Convey("test nested indexes CVE scanning enabled", t, func() {
log := log.NewLogger("debug", "")
rootDir := t.TempDir()
port := GetFreePort()
baseURL := GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
conf.Storage.RootDirectory = rootDir

Convey("test with boltdb", func() {
conf.Storage.CacheDriver = nil
})

Convey("test with redis", func() {
miniRedis := miniredis.RunT(t)

conf.Storage.CacheDriver = map[string]interface{}{
"name": "redis",
"url": "redis://" + miniRedis.Addr(),
}
})

defaultVal := true

updateDuration, _ := time.ParseDuration("1h")
Expand Down
198 changes: 197 additions & 1 deletion pkg/meta/meta_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package meta_test

import (
"context"
"encoding/json"
"fmt"
"os"
"path"
Expand All @@ -17,6 +18,7 @@ import (
"github.com/notaryproject/notation-go"
"github.com/notaryproject/notation-go/signer"
godigest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"

Expand Down Expand Up @@ -2018,6 +2020,184 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func
So(digests, ShouldContain, subImage2.DigestStr())
})

Convey("Search Tags with Nested Indexes", func() {
// nested manifest/indexes:
// image111 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop
// image112 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop
// image121 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop
// image122 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop
// image211 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop
// image212 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop
// image31 -> multiArchMiddle3 -> multiArchTop
// image32 -> multiArchMiddle3 -> multiArchTop
repoName := "nested"

image111 := CreateRandomImage()
image112 := CreateRandomImage()
multiArchBottom11 := CreateMultiarchWith().Images([]Image{image111, image112}).Build()

err := metaDB.SetRepoReference(ctx, repoName, image111.DigestStr(), image111.AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(ctx, repoName, image112.DigestStr(), image112.AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(ctx, repoName, multiArchBottom11.DigestStr(), multiArchBottom11.AsImageMeta())
So(err, ShouldBeNil)

image121 := CreateRandomImage()
image122 := CreateRandomImage()
multiArchBottom12 := CreateMultiarchWith().Images([]Image{image121, image122}).Build()

err = metaDB.SetRepoReference(ctx, repoName, image121.DigestStr(), image121.AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(ctx, repoName, image122.DigestStr(), image122.AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(ctx, repoName, multiArchBottom12.DigestStr(), multiArchBottom12.AsImageMeta())
So(err, ShouldBeNil)

indexMultiArchMiddle1 := ispec.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: ispec.MediaTypeImageIndex,
Manifests: []ispec.Descriptor{
{
Digest: multiArchBottom11.IndexDescriptor.Digest,
Size: multiArchBottom11.IndexDescriptor.Size,
MediaType: ispec.MediaTypeImageIndex,
},
{
Digest: multiArchBottom12.IndexDescriptor.Digest,
Size: multiArchBottom12.IndexDescriptor.Size,
MediaType: ispec.MediaTypeImageIndex,
},
},
}

indexMultiArchMiddle1Blob, err := json.Marshal(indexMultiArchMiddle1)
So(err, ShouldBeNil)
indexMultiArchMiddle1Digest := godigest.FromBytes(indexMultiArchMiddle1Blob)

indexMultiArchMiddle1Meta := mTypes.ImageMeta{
MediaType: ispec.MediaTypeImageIndex,
Digest: indexMultiArchMiddle1Digest,
Size: int64(len(indexMultiArchMiddle1Blob)),
Index: &indexMultiArchMiddle1,
}

err = metaDB.SetRepoReference(ctx, repoName, "multiArchMiddle1", indexMultiArchMiddle1Meta)
So(err, ShouldBeNil)

image211 := CreateRandomImage()
image212 := CreateRandomImage()
multiArchBottom21 := CreateMultiarchWith().Images([]Image{image211, image212}).Build()

err = metaDB.SetRepoReference(ctx, repoName, image211.DigestStr(), image211.AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(ctx, repoName, image212.DigestStr(), image212.AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(ctx, repoName, multiArchBottom21.DigestStr(), multiArchBottom21.AsImageMeta())
So(err, ShouldBeNil)

indexMultiArchMiddle2 := ispec.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: ispec.MediaTypeImageIndex,
Manifests: []ispec.Descriptor{
{
Digest: multiArchBottom21.IndexDescriptor.Digest,
Size: multiArchBottom21.IndexDescriptor.Size,
MediaType: ispec.MediaTypeImageIndex,
},
},
}

indexMultiArchMiddle2Blob, err := json.Marshal(indexMultiArchMiddle2)
So(err, ShouldBeNil)

indexMultiArchMiddle2Digest := godigest.FromBytes(indexMultiArchMiddle2Blob)

indexMultiArchMiddle2Meta := mTypes.ImageMeta{
MediaType: ispec.MediaTypeImageIndex,
Digest: indexMultiArchMiddle2Digest,
Size: int64(len(indexMultiArchMiddle2Blob)),
Index: &indexMultiArchMiddle2,
}

err = metaDB.SetRepoReference(ctx, repoName, "multiArchMiddle2", indexMultiArchMiddle2Meta)
So(err, ShouldBeNil)

image31 := CreateRandomImage()
image32 := CreateRandomImage()
multiArchBottom3 := CreateMultiarchWith().Images([]Image{image31, image32}).Build()

err = metaDB.SetRepoReference(ctx, repoName, image31.DigestStr(), image31.AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(ctx, repoName, image32.DigestStr(), image32.AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(ctx, repoName, multiArchBottom3.DigestStr(), multiArchBottom3.AsImageMeta())
So(err, ShouldBeNil)

indexMultiArchTop := ispec.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: ispec.MediaTypeImageIndex,
Manifests: []ispec.Descriptor{
{
Digest: indexMultiArchMiddle1Digest,
Size: int64(len(indexMultiArchMiddle1Blob)),
MediaType: ispec.MediaTypeImageIndex,
},
{
Digest: indexMultiArchMiddle2Digest,
Size: int64(len(indexMultiArchMiddle2Blob)),
MediaType: ispec.MediaTypeImageIndex,
},
{
Digest: multiArchBottom3.IndexDescriptor.Digest,
Size: multiArchBottom3.IndexDescriptor.Size,
MediaType: ispec.MediaTypeImageIndex,
},
},
}

indexMultiArchTopBlob, err := json.Marshal(indexMultiArchTop)
So(err, ShouldBeNil)

indexMultiArchTopBlobDigest := godigest.FromBytes(indexMultiArchTopBlob)

indexMultiArchTopMeta := mTypes.ImageMeta{
MediaType: ispec.MediaTypeImageIndex,
Digest: indexMultiArchTopBlobDigest,
Size: int64(len(indexMultiArchTopBlob)),
Index: &indexMultiArchTop,
}

err = metaDB.SetRepoReference(ctx, repoName, "multiArchTop", indexMultiArchTopMeta)
So(err, ShouldBeNil)

fullImageMetaList, err := metaDB.SearchTags(ctx, repoName+":"+"multiArch")
So(err, ShouldBeNil)

So(len(fullImageMetaList), ShouldEqual, 3)

tags := map[string][]string{}

for _, imageMeta := range fullImageMetaList {
So(imageMeta.MediaType, ShouldEqual, ispec.MediaTypeImageIndex)

digests := []string{}
for _, manifest := range imageMeta.Manifests {
digests = append(digests, manifest.Digest.String())
}

tags[imageMeta.Tag] = digests
}

So(tags, ShouldContainKey, "multiArchMiddle1")
So(tags, ShouldContainKey, "multiArchMiddle2")
So(tags, ShouldContainKey, "multiArchTop")

So(len(tags["multiArchMiddle1"]), ShouldEqual, 4)
So(len(tags["multiArchMiddle2"]), ShouldEqual, 2)
So(len(tags["multiArchTop"]), ShouldEqual, 8)
})

Convey("With referrer", func() {
refImage := CreateRandomImageWith().Subject(image1.DescriptorRef()).Build()
err := metaDB.SetRepoReference(ctx, repo1, "ref-tag", refImage.AsImageMeta())
Expand Down Expand Up @@ -2413,12 +2593,26 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func

Convey("FilterRepos", func() {
repo := "repoFilter"
repoUnmatched := "repoExcluded"
tag1 := "tag1"
tag2 := "tag22"
tag3 := "tag3"
tag4 := "tag44"

userAc := reqCtx.NewUserAccessControl()
userAc.SetUsername("username")
userAc.SetGlobPatterns("read", map[string]bool{
repo: true,
repoUnmatched: false,
})

ctx := userAc.DeriveContext(context.Background())

image := CreateImageWith().DefaultLayers().PlatformConfig("image-platform", "image-os").Build()
err := metaDB.SetRepoReference(ctx, repo, tag1, image.AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(ctx, repoUnmatched, tag3, image.AsImageMeta())
So(err, ShouldBeNil)

multiarch := CreateMultiarchWith().
Images([]Image{
Expand All @@ -2433,9 +2627,11 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func

err = metaDB.SetRepoReference(ctx, repo, tag2, multiarch.AsImageMeta())
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(ctx, repoUnmatched, tag4, multiarch.AsImageMeta())
So(err, ShouldBeNil)

//nolint: contextcheck
repoMetaList, err := metaDB.FilterRepos(context.Background(), mTypes.AcceptAllRepoNames,
repoMetaList, err := metaDB.FilterRepos(ctx, mTypes.AcceptAllRepoNames,
mTypes.AcceptAllRepoMeta)
So(err, ShouldBeNil)
So(len(repoMetaList), ShouldEqual, 1)
Expand Down
14 changes: 8 additions & 6 deletions pkg/meta/redisdb/redis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,8 @@ func (rc *RedisDB) ToggleBookmarkRepo(ctx context.Context, repo string) (mTypes.

// UserDB profile/api key CRUD.
func (rc *RedisDB) GetUserData(ctx context.Context) (mTypes.UserData, error) {
var userData mTypes.UserData
userData := mTypes.UserData{}
userData.APIKeys = make(map[string]mTypes.APIKeyDetails)

userAc, err := reqCtx.UserAcFromContext(ctx)
if err != nil {
Expand All @@ -282,7 +283,7 @@ func (rc *RedisDB) GetUserData(ctx context.Context) (mTypes.UserData, error) {
rc.Log.Error().Err(err).Str("hget", rc.UserDataKey).Str("userid", userid).
Msg("failed to get user data record")

return mTypes.UserData{}, fmt.Errorf("failed to get user data record for identity %s: %w", userid, err)
return userData, fmt.Errorf("failed to get user data record for identity %s: %w", userid, err)
}

if errors.Is(err, redis.Nil) {
Expand All @@ -291,6 +292,11 @@ func (rc *RedisDB) GetUserData(ctx context.Context) (mTypes.UserData, error) {

err = json.Unmarshal(userDataBlob, &userData)

if userData.APIKeys == nil {
// Unmarshal may have reset the value
userData.APIKeys = make(map[string]mTypes.APIKeyDetails)
}

return userData, err
}

Expand Down Expand Up @@ -501,10 +507,6 @@ func (rc *RedisDB) AddUserAPIKey(ctx context.Context, hashedKey string, apiKeyDe
return err
}

if userData.APIKeys == nil {
userData.APIKeys = make(map[string]mTypes.APIKeyDetails)
}

userData.APIKeys[hashedKey] = *apiKeyDetails

userDataBlob, err := json.Marshal(userData)
Expand Down
Loading

0 comments on commit efb88f9

Please sign in to comment.