wire PR from mdag repo

cmd/lotus-shed/dagspliter/dagsplitter.go

@@ -71,6 +71,9 @@ type builder struct {
 	// we only pack one box at a time and don't come back to a box once we're
 	// done with it we just track a single value here and not in each box.
 	boxUsedSize uint64
+
+	// Walk in breadth-first order instead of the default depth-search.
+	breadthFirst bool
 }
 
 func getSingleNodeSize(node ipld.Node) uint64 {
@@ -185,6 +188,11 @@ func (b *builder) addExternalLink(node cid.Cid) {
 // Pack a DAG delimited by `initialRoot` in boxes. To enforce the maximum
 // box size the DAG will be decomposed into smaller sub-DAGs if necessary.
 func (b *builder) add(ctx context.Context, initialRoot cid.Cid) error {
+	walkOptions := make([]mdag.WalkOption, 0)
+	if b.breadthFirst {
+		walkOptions = append(walkOptions, mdag.BreadthFirst())
+	}
+
 	// LIFO queue with the roots that need to be scanned and boxed.
 	// LIFO(-ish, node links pushed in reverse) should result in slightly better
 	// data layout (less fragmentation in leaves) than FIFO.
@@ -265,8 +273,7 @@ func (b *builder) add(ctx context.Context, initialRoot cid.Cid) error {
 				// No need to visit children as not even the parent fits.
 				return false
 			},
-			// FIXME: We're probably not ready for any type of concurrency at this point.
-			mdag.Concurrency(0),
+			walkOptions...,
 		)
 		if err != nil {
 			return xerrors.Errorf("error walking dag: %w", err)
@@ -337,10 +344,6 @@ var Cmd = &cli.Command{
 			return xerrors.Errorf("parsing chunk size: %w", err)
 		}
 
-		if cctx.Bool("breadth-first") {
-			return xerrors.Errorf("breadth-first pack not implemented yet")
-		}
-
 		// FIXME: The DAG-to-Box generation and Box-to-CAR generation is now
 		//  coupled in the same command, so for now we don't save the intermediate
 		//  boxes in the block store (IPFS) but keep them in memory and dump
@@ -357,6 +360,7 @@ var Cmd = &cli.Command{
 			boxMaxSize:      uint64(chunk),
 			minSubgraphSize: uint64(cctx.Int("minSubgraphSize")),
 			boxes:           make([]*Box, 0),
+			breadthFirst: cctx.Bool("breadth-first"),
 		}
 		bb.newBox() // FIXME: Encapsulate in a constructor.
 

go.mod

@@ -85,7 +85,7 @@ require (
 	github.com/ipfs/go-ipld-format v0.2.0
 	github.com/ipfs/go-log v1.0.4
 	github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4
-	github.com/ipfs/go-merkledag v0.3.2
+	github.com/ipfs/go-merkledag v0.3.3-0.20210205160809-e9b9632687d0
 	github.com/ipfs/go-metrics-interface v0.0.1
 	github.com/ipfs/go-metrics-prometheus v0.0.2
 	github.com/ipfs/go-path v0.0.7

go.sum

@@ -656,6 +656,8 @@ github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB
 github.com/ipfs/go-merkledag v0.3.1/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
 github.com/ipfs/go-merkledag v0.3.2 h1:MRqj40QkrWkvPswXs4EfSslhZ4RVPRbxwX11js0t1xY=
 github.com/ipfs/go-merkledag v0.3.2/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
+github.com/ipfs/go-merkledag v0.3.3-0.20210205160809-e9b9632687d0 h1:F/xdgTXv5aagyhIjq0Nn7kbatwIAMWUbfMdOi+W6ldc=
+github.com/ipfs/go-merkledag v0.3.3-0.20210205160809-e9b9632687d0/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
 github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
 github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s=