unixfs/feather: add a basic test

cmd/feather/main.go

@@ -38,15 +38,20 @@ Example:
 %s bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi`, err, os.Args[0], os.Args[0])
 	}
 
-	r, err := feather.DownloadFile(c)
+	f, err := feather.NewClient(feather.WithStaticGateway("http://localhost:8080/"))
 	if err != nil {
-		return fmt.Errorf("error starting file download: %w", err)
+		return fmt.Errorf("creating feather client: %w", err)
+	}
+
+	r, err := f.DownloadFile(c)
+	if err != nil {
+		return fmt.Errorf("starting file download: %w", err)
 	}
 	defer r.Close()
 
 	_, err = io.Copy(os.Stdout, r)
 	if err != nil {
-		return fmt.Errorf("error downloading file: %w", err)
+		return fmt.Errorf("downloading file: %w", err)
 	}
 	return nil
 }

unixfs/feather/entry.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -36,7 +37,6 @@ func init() {
 	cbor.RegisterCborType(carHeader{})
 }
 
-const gateway = "http://localhost:8080/ipfs/"
 const maxHeaderSize = 32 * 1024 * 1024 // 32MiB
 const maxBlockSize = 2 * 1024 * 1024   // 2MiB
 const maxCidSize = 4096
@@ -58,18 +58,66 @@ type downloader struct {
 	readErr  error
 }
 
+type Client struct {
+	httpClient *http.Client
+	hostname   string
+}
+
+type Option func(*Client) error
+
+// WithHTTPClient allows to use a [http.Client] of your choice.
+func WithHTTPClient(client *http.Client) Option {
+	return func(c *Client) error {
+		c.httpClient = client
+		return nil
+	}
+}
+
+// WithStaticGateway sets a static gateway which will be used for all requests.
+func WithStaticGateway(gateway string) Option {
+	if len(gateway) != 0 && gateway[len(gateway)-1] == '/' {
+		gateway = gateway[:len(gateway)-1]
+	}
+	gateway += "/ipfs/"
+
+	return func(c *Client) error {
+		c.hostname = gateway
+		return nil
+	}
+}
+
+var ErrNoAvailableDataSource = errors.New("no data source")
+
+func NewClient(opts ...Option) (*Client, error) {
+	c := &Client{
+		httpClient: http.DefaultClient,
+	}
+
+	for _, opt := range opts {
+		if err := opt(c); err != nil {
+			return nil, err
+		}
+	}
+
+	if c.hostname == "" {
+		return nil, ErrNoAvailableDataSource
+	}
+
+	return c, nil
+}
+
 // DownloadFile takes in a [cid.Cid] and return an [io.ReadCloser] which streams the deserialized file.
 // You MUST always call the Close method when you are done using it else it would leak resources.
-func DownloadFile(c cid.Cid) (io.ReadCloser, error) {
+func (client *Client) DownloadFile(c cid.Cid) (io.ReadCloser, error) {
 	c = normalizeCidv0(c)
 
-	req, err := http.NewRequest("GET", gateway+c.String()+"?dag-scope=entity", bytes.NewReader(nil))
+	req, err := http.NewRequest("GET", client.hostname+c.String()+"?dag-scope=entity", bytes.NewReader(nil))
 	if err != nil {
 		return nil, err
 	}
 	req.Header.Add("Accept", "application/vnd.ipld.car;dups=y;order=dfs;version=1")
 
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := client.httpClient.Do(req)
 	if err != nil {
 		return nil, err
 	}

unixfs/feather/feather_test.go

@@ -0,0 +1,83 @@
+package feather_test
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/ipfs/boxo/blockservice"
+	"github.com/ipfs/boxo/exchange/offline"
+	"github.com/ipfs/boxo/gateway"
+	"github.com/ipfs/boxo/unixfs/feather"
+	"github.com/ipfs/go-cid"
+	carblockstore "github.com/ipld/go-car/v2/blockstore"
+	"github.com/stretchr/testify/assert"
+)
+
+func newGateway(t *testing.T, fixture string) (*httptest.Server, cid.Cid) {
+	t.Helper()
+
+	r, err := os.Open(filepath.Join("./testdata", fixture))
+	assert.NoError(t, err)
+
+	blockStore, err := carblockstore.NewReadOnly(r, nil)
+	assert.NoError(t, err)
+
+	t.Cleanup(func() {
+		blockStore.Close()
+		r.Close()
+	})
+
+	cids, err := blockStore.Roots()
+	assert.NoError(t, err)
+	assert.Len(t, cids, 1)
+
+	blockService := blockservice.New(blockStore, offline.Exchange(blockStore))
+
+	backend, err := gateway.NewBlocksBackend(blockService)
+	assert.NoError(t, err)
+
+	handler := gateway.NewHandler(gateway.Config{}, backend)
+
+	ts := httptest.NewServer(handler)
+	t.Cleanup(func() { ts.Close() })
+	t.Logf("test server url: %s", ts.URL)
+
+	return ts, cids[0]
+}
+
+func newFeather(t *testing.T, fixture string) (*feather.Client, cid.Cid) {
+	t.Helper()
+
+	gw, cid := newGateway(t, fixture)
+	f, err := feather.NewClient(feather.WithHTTPClient(gw.Client()), feather.WithStaticGateway(gw.URL))
+	assert.NoError(t, err)
+	return f, cid
+}
+
+func mustParseHex(s string) []byte {
+	v, err := hex.DecodeString(s)
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+func TestFileWithManyRawLeaves(t *testing.T) {
+	f, root := newFeather(t, "file-with-many-raw-leaves.car")
+	file, err := f.DownloadFile(root)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, file.Close()) }()
+	h := sha256.New()
+	_, err = io.Copy(h, file)
+	assert.NoError(t, err)
+
+	if !bytes.Equal(h.Sum(nil), mustParseHex("1cf7caa4f9894d3316fd730598e1ec7cd7d10c3ca87bc09431821f81aa473298")) {
+		t.Error("decoded content does not match expected")
+	}
+}