Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dagsplitter: bfs #5539

Closed
wants to merge 1 commit into from
Closed

dagsplitter: bfs #5539

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions cmd/lotus-shed/dagspliter/dagsplitter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ type builder struct {
// we only pack one box at a time and don't come back to a box once we're
// done with it we just track a single value here and not in each box.
boxUsedSize uint64

// Walk in breadth-first order instead of the default depth-search.
breadthFirst bool
}

func getSingleNodeSize(node ipld.Node) uint64 {
Expand Down Expand Up @@ -185,6 +188,11 @@ func (b *builder) addExternalLink(node cid.Cid) {
// Pack a DAG delimited by `initialRoot` in boxes. To enforce the maximum
// box size the DAG will be decomposed into smaller sub-DAGs if necessary.
func (b *builder) add(ctx context.Context, initialRoot cid.Cid) error {
walkOptions := make([]mdag.WalkOption, 0)
if b.breadthFirst {
walkOptions = append(walkOptions, mdag.BreadthFirst())
}

// LIFO queue with the roots that need to be scanned and boxed.
// LIFO(-ish, node links pushed in reverse) should result in slightly better
// data layout (less fragmentation in leaves) than FIFO.
Expand Down Expand Up @@ -272,8 +280,7 @@ func (b *builder) add(ctx context.Context, initialRoot cid.Cid) error {
// No need to visit children as not even the parent fits.
return false
},
// FIXME: We're probably not ready for any type of concurrency at this point.
mdag.Concurrency(0),
walkOptions...,
)
if err != nil {
return xerrors.Errorf("error walking dag: %w", err)
Expand Down Expand Up @@ -344,10 +351,6 @@ var Cmd = &cli.Command{
return xerrors.Errorf("parsing chunk size: %w", err)
}

if cctx.Bool("breadth-first") {
return xerrors.Errorf("breadth-first pack not implemented yet")
}

// FIXME: The DAG-to-Box generation and Box-to-CAR generation is now
// coupled in the same command, so for now we don't save the intermediate
// boxes in the block store (IPFS) but keep them in memory and dump
Expand All @@ -364,6 +367,7 @@ var Cmd = &cli.Command{
boxMaxSize: uint64(chunk),
minSubgraphSize: uint64(cctx.Int("minSubgraphSize")),
boxes: make([]*Box, 0),
breadthFirst: cctx.Bool("breadth-first"),
}
bb.newBox() // FIXME: Encapsulate in a constructor.

Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ require (
github.com/ipfs/go-ipld-format v0.2.0
github.com/ipfs/go-log v1.0.4
github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4
github.com/ipfs/go-merkledag v0.3.2
github.com/ipfs/go-merkledag v0.3.3-0.20210205232304-7f10548fe4d7
github.com/ipfs/go-metrics-interface v0.0.1
github.com/ipfs/go-metrics-prometheus v0.0.2
github.com/ipfs/go-path v0.0.7
Expand Down
4 changes: 4 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -656,6 +656,10 @@ github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB
github.com/ipfs/go-merkledag v0.3.1/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
github.com/ipfs/go-merkledag v0.3.2 h1:MRqj40QkrWkvPswXs4EfSslhZ4RVPRbxwX11js0t1xY=
github.com/ipfs/go-merkledag v0.3.2/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
github.com/ipfs/go-merkledag v0.3.3-0.20210205160809-e9b9632687d0 h1:F/xdgTXv5aagyhIjq0Nn7kbatwIAMWUbfMdOi+W6ldc=
github.com/ipfs/go-merkledag v0.3.3-0.20210205160809-e9b9632687d0/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
github.com/ipfs/go-merkledag v0.3.3-0.20210205232304-7f10548fe4d7 h1:B8guDNBRtH4lreZdOV/R5W1lJll1OdglFfFNIHYSc00=
github.com/ipfs/go-merkledag v0.3.3-0.20210205232304-7f10548fe4d7/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s=
Expand Down