Skip to content

Commit

Permalink
feat: add more CAR tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@aschmahmann
aschmahmann committed May 25, 2023
1 parent 8d71bc1 commit 6dcc5b6
Show file tree
Hide file tree
Showing 4 changed files with 180 additions and 23 deletions.
Binary file added fixtures/t0118-multiblock-file-in-directory.car
View file
Binary file not shown.
Binary file added fixtures/t0118-one-layer-hamt.car
View file
Binary file not shown.
110 changes: 87 additions & 23 deletions tests/t0118_gateway_car_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

func TestGatewayCar(t *testing.T) {
fixture := car.MustOpenUnixfsCar("t0118-test-dag.car")
oneLayerHAMTFixture := car.MustOpenUnixfsCar("t0118-one-layer-hamt.car")

tests := SugarTests{
{
Expand Down Expand Up @@ -67,6 +68,30 @@ func TestGatewayCar(t *testing.T) {
InThatOrder(),
),
},
{
Name: "GET CAR with dag-scope=block pathing through a sharded directory",
Hint: `
dag-scope=block should return a CAR file with only the root block and a
block for each optional path component. Pathing through a sharded directory should return
the blocks needed for the traversal, not the entire HAMT and not skipping all intermediate nodes
`,
Request: Request().
Path("ipfs/{{cid}}/1.txt", oneLayerHAMTFixture.MustGetCid()).
Query("format", "car").
Query("dag-scope", "block"),
Response: Expect().
Status(200).
Body(
IsCar().
HasRoot(oneLayerHAMTFixture.MustGetCid()).
HasBlocks(flattenStrings(t,
oneLayerHAMTFixture.MustGetCid(),
oneLayerHAMTFixture.MustGetCIDsInHAMTTraversal(nil, "1.txt"))...,
).
Exactly().
InThatOrder(),
),
},
{
Name: "GET CAR with dag-scope=entity",
Hint: `
Expand All @@ -90,6 +115,30 @@ func TestGatewayCar(t *testing.T) {
InThatOrder(),
),
},
{
Name: "GET CAR with dag-scope=entity for a sharded directory",
Hint: `
dag-scope=entity for a sharded directory should return a CAR file with all of the path blocks as well
as all of the blocks in the HAMT, but not any of blocks below the HAMT.
`,
Request: Request().
Path("ipfs/{{cid}}", oneLayerHAMTFixture.MustGetCid()).
Query("format", "car").
Query("dag-scope", "entity"),
Response: Expect().
Status(200).
Body(
IsCar().
HasRoot(oneLayerHAMTFixture.MustGetCid()).
HasBlocks(
flattenStrings(t,
oneLayerHAMTFixture.MustGetCid(),
oneLayerHAMTFixture.MustGetCidsInHAMT())...,
).
Exactly().
InThatOrder(),
),
},
{
Name: "GET CAR with dag-scope=all",
Hint: `
Expand Down Expand Up @@ -122,7 +171,7 @@ func TestGatewayCar(t *testing.T) {

func TestGatewayCarEntityBytes(t *testing.T) {
multiBlockFileInDirFixture := car.MustOpenUnixfsCar("t0118-multiblock-file-in-directory.car")
fixture := multiBlockFileInDirFixture
oneLayerHAMTFixture := car.MustOpenUnixfsCar("t0118-one-layer-hamt.car")

tests := SugarTests{
{
Expand Down Expand Up @@ -158,16 +207,19 @@ func TestGatewayCarEntityBytes(t *testing.T) {
(i.e. entity-bytes is effectively optional if the entity is not a file)
`,
Request: Request().
Path("ipfs/{{cid}}", fixture.MustGetCid()).
Path("ipfs/{{cid}}", oneLayerHAMTFixture.MustGetCid()).
Query("format", "car").
Query("dag-scope", "entity"),
Query("dag-scope", "entity").
Query("entity-bytes", "0:*"),
Response: Expect().
Status(200).
Body(
IsCar().
HasRoot(fixture.MustGetCid()).
HasRoot(oneLayerHAMTFixture.MustGetCid()).
HasBlocks(
fixture.MustGetCid(),
flattenStrings(t,
oneLayerHAMTFixture.MustGetCid(),
oneLayerHAMTFixture.MustGetCidsInHAMT())...,
).
Exactly().
InThatOrder(),
Expand All @@ -179,58 +231,67 @@ func TestGatewayCarEntityBytes(t *testing.T) {
The response MUST contain only the minimal set of blocks necessary for fulfilling the range request
`,
Request: Request().
Path("ipfs/{{cid}}", fixture.MustGetCid()).
Path("ipfs/{{cid}}", multiBlockFileInDirFixture.MustGetCid()).
Query("format", "car").
Query("dag-scope", "entity"),
Query("dag-scope", "entity").
Query("entity-bytes", "512:*"),
Response: Expect().
Status(200).
Body(
IsCar().
HasRoot(fixture.MustGetCid()).
HasRoot(multiBlockFileInDirFixture.MustGetCid()).
HasBlocks(
fixture.MustGetCid(),
flattenStrings(t,
multiBlockFileInDirFixture.MustGetCid(),
multiBlockFileInDirFixture.MustGetIPLDChildrenCids()[2:])...,
).
Exactly().
InThatOrder(),
),
},
{
Name: "GET CAR with entity-bytes equivalent to a HTTP Range Request for the middle of a large file",
Name: "GET CAR with entity-bytes equivalent to a HTTP Range Request for the middle of a file",
Hint: `
The response MUST contain only the minimal set of blocks necessary for fulfilling the range request
`,
Request: Request().
Path("ipfs/{{cid}}", fixture.MustGetCid()).
Path("ipfs/{{cid}}", multiBlockFileInDirFixture.MustGetCid()).
Query("format", "car").
Query("dag-scope", "entity"),
Query("dag-scope", "entity").
Query("entity-bytes", "512:1024"),
Response: Expect().
Status(200).
Body(
IsCar().
HasRoot(fixture.MustGetCid()).
HasRoot(multiBlockFileInDirFixture.MustGetCid()).
HasBlocks(
fixture.MustGetCid(),
flattenStrings(t,
multiBlockFileInDirFixture.MustGetCid(),
multiBlockFileInDirFixture.MustGetIPLDChildrenCids()[2:4])...,
).
Exactly().
InThatOrder(),
),
},
{
Name: "GET CAR with entity-bytes equivalent to HTTP Suffix Range Request for part of a small file",
Name: "GET CAR with entity-bytes equivalent to HTTP Suffix Range Request for part of a file",
Hint: `
The response MUST contain only the minimal set of blocks necessary for fulfilling the range request
`,
Request: Request().
Path("ipfs/{{cid}}", fixture.MustGetCid()).
Path("ipfs/{{cid}}", multiBlockFileInDirFixture.MustGetCid()).
Query("format", "car").
Query("dag-scope", "entity"),
Query("dag-scope", "entity").
Query("entity-bytes", "-5:*"),
Response: Expect().
Status(200).
Body(
IsCar().
HasRoot(fixture.MustGetCid()).
HasRoot(multiBlockFileInDirFixture.MustGetCid()).
HasBlocks(
fixture.MustGetCid(),
flattenStrings(t,
multiBlockFileInDirFixture.MustGetCid(),
multiBlockFileInDirFixture.MustGetIPLDChildrenCids()[3:])...,
).
Exactly().
InThatOrder(),
Expand All @@ -242,16 +303,19 @@ func TestGatewayCarEntityBytes(t *testing.T) {
The response MUST contain only the minimal set of blocks necessary for fulfilling the range request
`,
Request: Request().
Path("ipfs/{{cid}}", fixture.MustGetCid()).
Path("ipfs/{{cid}}", multiBlockFileInDirFixture.MustGetCid()).
Query("format", "car").
Query("dag-scope", "entity"),
Query("dag-scope", "entity").
Query("entity-bytes", "-999999:-3"),
Response: Expect().
Status(200).
Body(
IsCar().
HasRoot(fixture.MustGetCid()).
HasRoot(multiBlockFileInDirFixture.MustGetCid()).
HasBlocks(
fixture.MustGetCid(),
flattenStrings(t,
multiBlockFileInDirFixture.MustGetCid(),
multiBlockFileInDirFixture.MustGetIPLDChildrenCids()[:5])...,
).
Exactly().
InThatOrder(),
Expand Down
93 changes: 93 additions & 0 deletions tooling/car/unixfs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,13 @@ import (
"bytes"
"context"
"fmt"
"github.com/ipfs/boxo/ipld/unixfs/hamt"
"github.com/ipfs/go-unixfsnode"
"os"
"path"
"sort"
"strings"
"sync"

"github.com/ipfs/boxo/blockservice"
"github.com/ipfs/boxo/ipld/car/v2/blockstore"
Expand Down Expand Up @@ -178,6 +181,59 @@ func (d *UnixfsDag) MustGetChildrenCids(names ...string) []string {
return cids
}

func (d *UnixfsDag) MustGetIPLDChildrenCids(names ...string) []string {
node := d.MustGetNode(names...)
lnks := node.node.Links()
var cids []string
for _, l := range lnks {
cids = append(cids, l.Cid.String())
}
return cids
}

// MustGetCidsInHAMT returns the cids in the HAMT at the given path. Does not include the CID of the HAMT root
func (d *UnixfsDag) MustGetCidsInHAMT(names ...string) []string {
node := d.MustGetNode(names...)
var cids []string
tracker := dservTrackingWrapper{
DAGService: node.dsvc,
}
h, err := hamt.NewHamtFromDag(&tracker, node.node)
if err != nil {
panic(err)
}
_, err = h.EnumLinks(context.Background())
if err != nil {
panic(err)
}
for _, c := range tracker.requestedCids {
cids = append(cids, c.String())
}
return cids
}

// MustGetCIDsInHAMTTraversal returns the cids needed for a given HAMT traversal. Does not include the HAMT root.
func (d *UnixfsDag) MustGetCIDsInHAMTTraversal(path []string, child string) []string {
node := d.MustGetNode(path...)
var cids []string
tracker := dservTrackingWrapper{
DAGService: node.dsvc,
}
unixfsnode.AddUnixFSReificationToLinkSystem()
h, err := hamt.NewHamtFromDag(&tracker, node.node)
if err != nil {
panic(err)
}
_, err = h.Find(context.Background(), child)
if err != nil {
panic(err)
}
for _, c := range tracker.requestedCids {
cids = append(cids, c.String())
}
return cids
}

func (d *UnixfsDag) MustGetRoot() *FixtureNode {
return d.MustGetNode()
}
Expand Down Expand Up @@ -231,3 +287,40 @@ func MustOpenUnixfsCar(file string) *UnixfsDag {
}
return dag
}

type dservTrackingWrapper struct {
format.DAGService
reqMx sync.Mutex
requestedCids []cid.Cid
}

func (d *dservTrackingWrapper) Get(ctx context.Context, c cid.Cid) (format.Node, error) {
nd, err := d.DAGService.Get(ctx, c)
if err != nil {
return nil, err
}
d.reqMx.Lock()
d.requestedCids = append(d.requestedCids, c)
d.reqMx.Unlock()
return nd, nil
}

func (d *dservTrackingWrapper) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption {
innerCh := d.DAGService.GetMany(ctx, cids)
outCh := make(chan *format.NodeOption, 1)
go func() {
defer close(outCh)
for i := range innerCh {
if i.Err == nil {
c := i.Node.Cid()
d.reqMx.Lock()
d.requestedCids = append(d.requestedCids, c)
d.reqMx.Unlock()
} else {
fmt.Println("yolo")
}
outCh <- i
}
}()
return outCh
}

0 comments on commit 6dcc5b6

Please sign in to comment.