From 36671617474c6881fd9f03efac06bfb8c2491f7a Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 30 Jan 2024 11:01:36 +0100 Subject: [PATCH] go tests with car archive --- go.mod | 1 + go.sum | 3 + repo/fsrepo/migrations/fetch_test.go | 329 +++++++++++++++------- repo/fsrepo/migrations/migrations_test.go | 13 +- repo/fsrepo/migrations/versions_test.go | 8 +- 5 files changed, 240 insertions(+), 114 deletions(-) diff --git a/go.mod b/go.mod index efab059a36c4..ef4c515534f9 100644 --- a/go.mod +++ b/go.mod @@ -136,6 +136,7 @@ require ( github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-blockservice v0.5.0 // indirect github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect + github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect diff --git a/go.sum b/go.sum index 35eefb2bb6fb..bfb68f05f777 100644 --- a/go.sum +++ b/go.sum @@ -330,11 +330,13 @@ github.com/ipfs/boxo v0.17.1-0.20240126101119-fdfcfcc0708a/go.mod h1:pIZgTWdm3k3 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= +github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-blockservice v0.5.0 h1:B2mwhhhVQl2ntW2EIpaWPwSCxSuqr5fFA93Ms4bYLEY= github.com/ipfs/go-blockservice v0.5.0/go.mod h1:W6brZ5k20AehbmERplmERn8o2Ni3ZZubvAxaIUeaT6w= +github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= @@ -367,6 +369,7 @@ github.com/ipfs/go-ipfs-blockstore v1.3.0 h1:m2EXaWgwTzAfsmt5UdJ7Is6l4gJcaM/A12X github.com/ipfs/go-ipfs-blockstore v1.3.0/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= +github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-cmds v0.10.0 h1:ZB4+RgYaH4UARfJY0uLKl5UXgApqnRjKbuCiJVcErYk= github.com/ipfs/go-ipfs-cmds v0.10.0/go.mod h1:sX5d7jkCft9XLPnkgEfXY0z2UBOB5g6fh/obBS0enJE= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= diff --git a/repo/fsrepo/migrations/fetch_test.go b/repo/fsrepo/migrations/fetch_test.go index 27452d386af6..518624dba062 100644 --- a/repo/fsrepo/migrations/fetch_test.go +++ b/repo/fsrepo/migrations/fetch_test.go @@ -6,7 +6,6 @@ import ( "context" "fmt" "io" - "net/http" "net/http/httptest" "os" "path" @@ -14,92 +13,257 @@ import ( "runtime" "strings" "testing" + + "github.com/ipfs/boxo/blockservice" + "github.com/ipfs/boxo/exchange/offline" + "github.com/ipfs/boxo/gateway" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-unixfsnode/data/builder" + "github.com/ipld/go-car/v2" + carblockstore "github.com/ipld/go-car/v2/blockstore" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/multiformats/go-multicodec" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" +) + +var ( + testIpfsDist string + testServer *httptest.Server ) -func createTestServer() *httptest.Server { - reqHandler := func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - if strings.Contains(r.URL.Path, "not-here") { - http.NotFound(w, r) - } else if strings.HasSuffix(r.URL.Path, "versions") { - fmt.Fprint(w, "v1.0.0\nv1.1.0\nv1.1.2\nv2.0.0-rc1\n2.0.0\nv2.0.1\n") - } else if strings.HasSuffix(r.URL.Path, ".tar.gz") { - createFakeArchive(r.URL.Path, false, w) - } else if strings.HasSuffix(r.URL.Path, "zip") { - createFakeArchive(r.URL.Path, true, w) - } else { - http.NotFound(w, r) +func TestMain(m *testing.M) { + // Setup test data + testDataDir := makeTestData() + defer os.RemoveAll(testDataDir) + + testCar := makeTestCar(testDataDir) + defer os.RemoveAll(testCar) + + // Setup test gateway + fd := setupTestGateway(testCar) + defer fd.Close() + + // Run tests + os.Exit(m.Run()) +} + +func makeTestData() string { + tempDir, err := os.MkdirTemp("", "kubo-migrations-test-*") + if err != nil { + panic(err) + } + + versions := []string{"v1.0.0", "v1.1.0", "v1.1.2", "v2.0.0-rc1", "2.0.0", "v2.0.1"} + packages := []string{"kubo", "go-ipfs", "fs-repo-migrations", "fs-repo-1-to-2", "fs-repo-2-to-3", "fs-repo-9-to-10", "fs-repo-10-to-11"} + + // Generate fake data + for _, name := range packages { + err = os.MkdirAll(filepath.Join(tempDir, name), 0777) + if err != nil { + panic(err) + } + + err = os.WriteFile(filepath.Join(tempDir, name, "versions"), []byte(strings.Join(versions, "\n")+"\n"), 0666) + if err != nil { + panic(err) + } + + for _, version := range versions { + filename, archName := makeArchivePath(name, name, version, "tar.gz") + createFakeArchive(filepath.Join(tempDir, filename), archName, false) + + filename, archName = makeArchivePath(name, name, version, "zip") + createFakeArchive(filepath.Join(tempDir, filename), archName, true) } } - return httptest.NewServer(http.HandlerFunc(reqHandler)) + + return tempDir } -func createFakeArchive(name string, archZip bool, w io.Writer) { +func createFakeArchive(archName, name string, archZip bool) { + err := os.MkdirAll(filepath.Dir(archName), 0777) + if err != nil { + panic(err) + } + fileName := strings.Split(path.Base(name), "_")[0] - root := path.Base(path.Dir(path.Dir(name))) + root := fileName // Simulate fetching go-ipfs, which has "ipfs" as the name in the archive. - if fileName == "go-ipfs" { + if fileName == "go-ipfs" || fileName == "kubo" { fileName = "ipfs" } fileName = ExeName(fileName) - var err error if archZip { - err = writeZip(root, fileName, "FAKE DATA", w) + err = writeZipFile(archName, root, fileName, "FAKE DATA") } else { - err = writeTarGzip(root, fileName, "FAKE DATA", w) + err = writeTarGzipFile(archName, root, fileName, "FAKE DATA") + } + if err != nil { + panic(err) + } +} + +func makeTestCar(testData string) string { + // make a cid with the right length that we eventually will patch with the root. + hasher, err := multihash.GetHasher(multihash.SHA2_256) + if err != nil { + panic(err) + } + digest := hasher.Sum([]byte{}) + hash, err := multihash.Encode(digest, multihash.SHA2_256) + if err != nil { + panic(err) + } + proxyRoot := cid.NewCidV1(uint64(multicodec.DagPb), hash) + + // Make CAR file + fd, err := os.CreateTemp("", "kubo-migrations-test-*.car") + if err != nil { + panic(err) + } + defer fd.Close() + filename := fd.Name() + + rw, err := carblockstore.OpenReadWriteFile(fd, []cid.Cid{proxyRoot}, carblockstore.WriteAsCarV1(true)) + if err != nil { + panic(err) + } + defer rw.Close() + + ctx := context.Background() + + ls := cidlink.DefaultLinkSystem() + ls.TrustedStorage = true + ls.StorageReadOpener = func(_ ipld.LinkContext, l ipld.Link) (io.Reader, error) { + cl, ok := l.(cidlink.Link) + if !ok { + return nil, fmt.Errorf("not a cidlink") + } + blk, err := rw.Get(ctx, cl.Cid) + if err != nil { + return nil, err + } + return bytes.NewBuffer(blk.RawData()), nil + } + ls.StorageWriteOpener = func(_ ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) { + buf := bytes.NewBuffer(nil) + return buf, func(l ipld.Link) error { + cl, ok := l.(cidlink.Link) + if !ok { + return fmt.Errorf("not a cidlink") + } + blk, err := blocks.NewBlockWithCid(buf.Bytes(), cl.Cid) + if err != nil { + return err + } + rw.Put(ctx, blk) + return nil + }, nil + } + + l, _, err := builder.BuildUnixFSRecursive(testData, &ls) + if err != nil { + panic(err) + } + + rcl, ok := l.(cidlink.Link) + if !ok { + panic(fmt.Errorf("could not interpret %s", l)) + } + + if err := rw.Finalize(); err != nil { + panic(err) + } + // re-open/finalize with the final root. + err = car.ReplaceRootsInFile(filename, []cid.Cid{rcl.Cid}) + if err != nil { + panic(err) + } + + return filename +} + +func setupTestGateway(testCar string) io.Closer { + blockService, roots, fd, err := newBlockServiceFromCAR(testCar) + if err != nil { + panic(err) + } + + if len(roots) != 1 { + panic("expected car with 1 root") } + + backend, err := gateway.NewBlocksBackend(blockService) if err != nil { panic(err) } + conf := gateway.Config{ + NoDNSLink: false, + DeserializedResponses: false, + } + + testIpfsDist = "/ipfs/" + roots[0].String() + testServer = httptest.NewServer(gateway.NewHandler(conf, backend)) + + return fd +} + +func newBlockServiceFromCAR(filepath string) (blockservice.BlockService, []cid.Cid, io.Closer, error) { + r, err := os.Open(filepath) + if err != nil { + return nil, nil, nil, err + } + + bs, err := carblockstore.NewReadOnly(r, nil) + if err != nil { + _ = r.Close() + return nil, nil, nil, err + } + + roots, err := bs.Roots() + if err != nil { + return nil, nil, nil, err + } + + blockService := blockservice.New(bs, offline.Exchange(bs)) + return blockService, roots, r, nil } func TestGetDistPath(t *testing.T) { os.Unsetenv(envIpfsDistPath) distPath := GetDistPathEnv("") - if distPath != LatestIpfsDist { - t.Error("did not set default dist path") - } + require.Equal(t, distPath, LatestIpfsDist) testDist := "/unit/test/dist" err := os.Setenv(envIpfsDistPath, testDist) - if err != nil { - panic(err) - } + require.NoError(t, err) defer func() { os.Unsetenv(envIpfsDistPath) }() distPath = GetDistPathEnv("") - if distPath != testDist { - t.Error("did not set dist path from environ") - } + require.Equal(t, distPath, testDist) distPath = GetDistPathEnv("ignored") - if distPath != testDist { - t.Error("did not set dist path from environ") - } + require.Equal(t, distPath, testDist) testDist = "/unit/test/dist2" fetcher := NewHttpFetcher(testDist, "", "", 0) - if fetcher.distPath != testDist { - t.Error("did not set dist path") - } + require.Equal(t, fetcher.distPath, testDist) } func TestHttpFetch(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - ts := createTestServer() - defer ts.Close() - - fetcher := NewHttpFetcher("", ts.URL, "", 0) + fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) - out, err := fetcher.Fetch(ctx, "/versions") - if err != nil { - t.Fatal(err) - } + out, err := fetcher.Fetch(ctx, "/kubo/versions") + require.NoError(t, err) var lines []string scan := bufio.NewScanner(bytes.NewReader(out)) @@ -107,22 +271,14 @@ func TestHttpFetch(t *testing.T) { lines = append(lines, scan.Text()) } err = scan.Err() - if err != nil { - t.Fatal("could not read versions:", err) - } + require.NoError(t, err) - if len(lines) < 6 { - t.Fatal("do not get all expected data") - } - if lines[0] != "v1.0.0" { - t.Fatal("expected v1.0.0 as first line, got", lines[0]) - } + require.Len(t, lines, 6) + require.Equal(t, "v1.0.0", lines[0]) // Check not found _, err = fetcher.Fetch(ctx, "/no_such_file") - if err == nil || !strings.Contains(err.Error(), "404") { - t.Fatal("expected error 404") - } + require.ErrorContains(t, err, "no link named") } func TestFetchBinary(t *testing.T) { @@ -131,51 +287,33 @@ func TestFetchBinary(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - ts := createTestServer() - defer ts.Close() - - fetcher := NewHttpFetcher("", ts.URL, "", 0) + fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) vers, err := DistVersions(ctx, fetcher, distFSRM, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) t.Log("latest version of", distFSRM, "is", vers[len(vers)-1]) bin, err := FetchBinary(ctx, fetcher, distFSRM, vers[0], "", tmpDir) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) fi, err := os.Stat(bin) - if os.IsNotExist(err) { - t.Error("expected file to exist:", bin) - } - + require.NoError(t, err) t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name()) - bin, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir) - if err != nil { - t.Fatal(err) - } + bin, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir) + require.NoError(t, err) fi, err = os.Stat(bin) - if os.IsNotExist(err) { - t.Error("expected file to exist:", bin) - } + require.NoError(t, err) t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name()) // Check error is destination already exists and is not directory - _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", bin) - if !os.IsExist(err) { - t.Fatal("expected 'exists' error, got", err) - } + _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", bin) + require.ErrorIs(t, err, os.ErrExist) - _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir) - if !os.IsExist(err) { - t.Error("expected 'exists' error, got:", err) - } + _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir) + require.ErrorIs(t, err, os.ErrExist) os.Remove(filepath.Join(tmpDir, ExeName("ipfs"))) @@ -192,7 +330,7 @@ func TestFetchBinary(t *testing.T) { if err != nil { panic(err) } - _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir) + _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir) if !os.IsPermission(err) { t.Error("expected 'permission' error, got:", err) } @@ -207,31 +345,24 @@ func TestFetchBinary(t *testing.T) { } // Check error if failure to fetch due to bad dist - _, err = FetchBinary(ctx, fetcher, "not-here", "v0.3.5", "ipfs", tmpDir) - if err == nil || !strings.Contains(err.Error(), "Not Found") { - t.Error("expected 'Not Found' error, got:", err) - } + _, err = FetchBinary(ctx, fetcher, "not-here", "v1.0.0", "ipfs", tmpDir) + require.ErrorContains(t, err, "no link") // Check error if failure to unpack archive - _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "not-such-bin", tmpDir) - if err == nil || err.Error() != "no binary found in archive" { - t.Error("expected 'no binary found in archive' error") - } + _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "not-such-bin", tmpDir) + require.ErrorContains(t, err, "no binary found in archive") } func TestMultiFetcher(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - ts := createTestServer() - defer ts.Close() - badFetcher := NewHttpFetcher("", "bad-url", "", 0) - fetcher := NewHttpFetcher("", ts.URL, "", 0) + fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) mf := NewMultiFetcher(badFetcher, fetcher) - vers, err := mf.Fetch(ctx, "/versions") + vers, err := mf.Fetch(ctx, "/kubo/versions") if err != nil { t.Fatal(err) } diff --git a/repo/fsrepo/migrations/migrations_test.go b/repo/fsrepo/migrations/migrations_test.go index 2fd75b7e9de2..9397af260c9e 100644 --- a/repo/fsrepo/migrations/migrations_test.go +++ b/repo/fsrepo/migrations/migrations_test.go @@ -10,6 +10,7 @@ import ( "testing" config "github.com/ipfs/kubo/config" + "github.com/stretchr/testify/require" ) func TestFindMigrations(t *testing.T) { @@ -110,9 +111,7 @@ func TestFetchMigrations(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - ts := createTestServer() - defer ts.Close() - fetcher := NewHttpFetcher(CurrentIpfsDist, ts.URL, "", 0) + fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) tmpDir := t.TempDir() @@ -121,9 +120,7 @@ func TestFetchMigrations(t *testing.T) { buf.Grow(256) logger := log.New(buf, "", 0) fetched, err := fetchMigrations(ctx, fetcher, needed, tmpDir, logger) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) for _, bin := range fetched { _, err = os.Stat(bin) @@ -162,9 +159,7 @@ func TestRunMigrations(t *testing.T) { t.Fatal(err) } - ts := createTestServer() - defer ts.Close() - fetcher := NewHttpFetcher(CurrentIpfsDist, ts.URL, "", 0) + fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/repo/fsrepo/migrations/versions_test.go b/repo/fsrepo/migrations/versions_test.go index 18de72b779c3..dd62f9bde8d3 100644 --- a/repo/fsrepo/migrations/versions_test.go +++ b/repo/fsrepo/migrations/versions_test.go @@ -13,9 +13,7 @@ func TestDistVersions(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - ts := createTestServer() - defer ts.Close() - fetcher := NewHttpFetcher("", ts.URL, "", 0) + fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) vers, err := DistVersions(ctx, fetcher, testDist, true) if err != nil { @@ -32,9 +30,7 @@ func TestLatestDistVersion(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - ts := createTestServer() - defer ts.Close() - fetcher := NewHttpFetcher("", ts.URL, "", 0) + fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) latest, err := LatestDistVersion(ctx, fetcher, testDist, false) if err != nil {