builds and looks good

Makefile

@@ -99,7 +99,7 @@ BINS+=lotus-miner
 
 curio: $(BUILD_DEPS)
 	rm -f curio
-	$(GOCC) build $(GOFLAGS) -o curio ./cmd/curio -X github.com/filecoin-project/lotus/curiosrc/build.IsOpencl=$(FFI_USE_OPENCL)
+	$(GOCC) build $(GOFLAGS) -o curio -ldflags "-X github.com/filecoin-project/lotus/curiosrc/build.IsOpencl=$(FFI_USE_OPENCL)" ./cmd/curio 
 .PHONY: curio
 BINS+=curio
 

cmd/curio/deps/deps.go

@@ -18,8 +18,6 @@ import (
 
 	"github.com/BurntSushi/toml"
 	"github.com/gbrlsnchs/jwt/v3"
-	ds "github.com/ipfs/go-datastore"
-	dssync "github.com/ipfs/go-datastore/sync"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/samber/lo"
 	"github.com/urfave/cli/v2"
@@ -28,7 +26,6 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-statestore"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/v1api"
@@ -38,6 +35,7 @@ import (
 	"github.com/filecoin-project/lotus/journal"
 	"github.com/filecoin-project/lotus/journal/alerting"
 	"github.com/filecoin-project/lotus/journal/fsjournal"
+	"github.com/filecoin-project/lotus/lib/ffiselect"
 	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
 	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/node/modules"
@@ -165,20 +163,20 @@ func GetDeps(ctx context.Context, cctx *cli.Context) (*Deps, error) {
 }
 
 type Deps struct {
-	Layers     []string
-	Cfg        *config.CurioConfig // values
-	DB         *harmonydb.DB       // has itest capability
-	Full       api.FullNode
-	Verif      storiface.Verifier
-	LW         *sealer.LocalWorker
-	As         *multictladdr.MultiAddressSelector
-	Maddrs     map[dtypes.MinerAddress]bool
-	ProofTypes map[abi.RegisteredSealProof]bool
-	Stor       *paths.Remote
-	Si         *paths.DBIndex
-	LocalStore *paths.Local
-	LocalPaths *paths.BasicLocalStorage
-	ListenAddr string
+	Layers       []string
+	Cfg          *config.CurioConfig // values
+	DB           *harmonydb.DB       // has itest capability
+	Full         api.FullNode
+	Verif        storiface.Verifier
+	As           *multictladdr.MultiAddressSelector
+	Maddrs       map[dtypes.MinerAddress]bool
+	ProofTypes   map[abi.RegisteredSealProof]bool
+	Stor         *paths.Remote
+	Si           *paths.DBIndex
+	LocalStore   *paths.Local
+	LocalPaths   *paths.BasicLocalStorage
+	ListenAddr   string
+	CurioFfiWrap *ffiselect.CurioFFIWrap
 }
 
 const (
@@ -308,15 +306,9 @@ Get it with: jq .PrivateKey ~/.lotus-miner/keystore/MF2XI2BNNJ3XILLQOJUXMYLUMU`,
 	if deps.Stor == nil {
 		deps.Stor = paths.NewRemote(deps.LocalStore, deps.Si, http.Header(sa), 10, &paths.DefaultPartialFileHandler{})
 	}
-	if deps.LW == nil {
-		wstates := statestore.New(dssync.MutexWrap(ds.NewMapDatastore()))
 
-		// todo localWorker isn't the abstraction layer we want to use here, we probably want to go straight to ffiwrapper
-		//  maybe with a curio specific abstraction. LocalWorker does persistent call tracking which we probably
-		//  don't need (ehh.. maybe we do, the async callback system may actually work decently well with harmonytask)
-		deps.LW = sealer.NewLocalWorker(sealer.WorkerConfig{
-			MaxParallelChallengeReads: deps.Cfg.Proving.ParallelCheckLimit,
-		}, deps.Stor, deps.LocalStore, deps.Si, nil, wstates)
+	if deps.CurioFfiWrap == nil {
+		deps.CurioFfiWrap = &ffiselect.CurioFFIWrap{Layers: deps.Layers}
 	}
 	if deps.Maddrs == nil {
 		deps.Maddrs = map[dtypes.MinerAddress]bool{}

cmd/curio/ffi.go

@@ -1,22 +1,29 @@
 package main
 
 import (
+	"context"
 	"encoding/gob"
 	"fmt"
 	"os"
 	"reflect"
 
 	"github.com/samber/lo"
 	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/cmd/curio/deps"
 	"github.com/filecoin-project/lotus/lib/ffiselect"
+	"github.com/filecoin-project/lotus/storage/paths"
 	"github.com/filecoin-project/lotus/storage/sealer/ffiwrapper"
+	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 var ffiCmd = &cli.Command{
 	Name:   "ffi",
 	Hidden: true,
+	Flags: []cli.Flag{
+		layersFlag,
+	},
 	Action: func(cctx *cli.Context) (err error) {
 		output := os.NewFile(uintptr(3), "out")
 
@@ -33,10 +40,10 @@ var ffiCmd = &cli.Command{
 			return err
 		}
 
-		// TODO duplicate passed-in --layers so we get the same as parent.
+		// wasteful, but *should* work
 		depnd, err := deps.GetDeps(cctx.Context, cctx)
 
-		w, err := ffiwrapper.New(depnd.Stor) // TODO @magik6k: what should I pass in here?
+		w, err := ffiwrapper.New(&sectorProvider{index: depnd.Si, stor: depnd.LocalStore})
 		if err != nil {
 			return err
 		}
@@ -56,3 +63,54 @@ var ffiCmd = &cli.Command{
 		return gob.NewEncoder(output).Encode(ffiselect.ValErr{Val: res, Err: nil})
 	},
 }
+
+type sectorProvider struct {
+	index paths.SectorIndex
+	stor  *paths.Local
+}
+
+func (l *sectorProvider) AcquireSector(ctx context.Context, id storiface.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, sealing storiface.PathType) (storiface.SectorPaths, func(), error) {
+	if allocate != storiface.FTNone {
+		return storiface.SectorPaths{}, nil, xerrors.New("read-only storage")
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+
+	// use TryLock to avoid blocking
+	locked, err := l.index.StorageTryLock(ctx, id.ID, existing, storiface.FTNone)
+	if err != nil {
+		cancel()
+		return storiface.SectorPaths{}, nil, xerrors.Errorf("acquiring sector lock: %w", err)
+	}
+	if !locked {
+		cancel()
+		return storiface.SectorPaths{}, nil, xerrors.Errorf("failed to acquire sector lock")
+	}
+
+	p, _, err := l.stor.AcquireSector(ctx, id, existing, allocate, sealing, storiface.AcquireMove)
+
+	return p, cancel, err
+}
+
+func (l *sectorProvider) AcquireSectorCopy(ctx context.Context, id storiface.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, ptype storiface.PathType) (storiface.SectorPaths, func(), error) {
+	if allocate != storiface.FTNone {
+		return storiface.SectorPaths{}, nil, xerrors.New("read-only storage")
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+
+	// use TryLock to avoid blocking
+	locked, err := l.index.StorageTryLock(ctx, id.ID, existing, storiface.FTNone)
+	if err != nil {
+		cancel()
+		return storiface.SectorPaths{}, nil, xerrors.Errorf("acquiring sector lock: %w", err)
+	}
+	if !locked {
+		cancel()
+		return storiface.SectorPaths{}, nil, xerrors.Errorf("failed to acquire sector lock")
+	}
+
+	p, _, err := l.stor.AcquireSector(ctx, id, existing, allocate, ptype, storiface.AcquireCopy)
+
+	return p, cancel, err
+}

cmd/curio/proving.go

@@ -168,7 +168,7 @@ It will not send any messages to the chain. Since it can compute any deadline, o
 		}
 
 		wdPostTask, wdPoStSubmitTask, derlareRecoverTask, err := curio.WindowPostScheduler(
-			ctx, deps.Cfg.Fees, deps.Cfg.Proving, deps.Full, deps.Verif, deps.LW, nil, nil,
+			ctx, deps.Cfg.Fees, deps.Cfg.Proving, deps.Full, deps.Verif, deps.CurioFfiWrap, nil, nil,
 			deps.As, deps.Maddrs, deps.DB, deps.Stor, deps.Si, deps.Cfg.Subsystems.WindowPostMaxTasks)
 		if err != nil {
 			return err

cmd/curio/run.go

@@ -150,6 +150,11 @@ var runCmd = &cli.Command{
 	},
 }
 
+var layersFlag = &cli.StringSliceFlag{
+	Name:  "layers",
+	Usage: "list of layers to be interpreted (atop defaults). Default: base",
+}
+
 var webCmd = &cli.Command{
 	Name:  "web",
 	Usage: "Start Curio web interface",
@@ -165,10 +170,7 @@ var webCmd = &cli.Command{
 			Name:  "nosync",
 			Usage: "don't check full-node sync status",
 		},
-		&cli.StringSliceFlag{
-			Name:  "layers",
-			Usage: "list of layers to be interpreted (atop defaults). Default: base",
-		},
+		layersFlag,
 	},
 	Action: func(cctx *cli.Context) error {
 

cmd/curio/tasks/tasks.go

@@ -38,7 +38,6 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task
 	db := dependencies.DB
 	full := dependencies.Full
 	verif := dependencies.Verif
-	lw := dependencies.LW
 	as := dependencies.As
 	maddrs := dependencies.Maddrs
 	stor := dependencies.Stor
@@ -61,7 +60,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task
 
 		if cfg.Subsystems.EnableWindowPost {
 			wdPostTask, wdPoStSubmitTask, derlareRecoverTask, err := curio.WindowPostScheduler(
-				ctx, cfg.Fees, cfg.Proving, full, verif, lw, sender, chainSched,
+				ctx, cfg.Fees, cfg.Proving, full, verif, dependencies.CurioFfiWrap, sender, chainSched,
 				as, maddrs, db, stor, si, cfg.Subsystems.WindowPostMaxTasks)
 
 			if err != nil {
@@ -72,14 +71,15 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task
 		}
 
 		if cfg.Subsystems.EnableWinningPost {
-			winPoStTask := winning.NewWinPostTask(cfg.Subsystems.WinningPostMaxTasks, db, lw, verif, full, maddrs)
+			pl := dependencies.LocalStore
+			winPoStTask := winning.NewWinPostTask(cfg.Subsystems.WinningPostMaxTasks, db, dependencies.CurioFfiWrap, pl, verif, full, maddrs)
 			activeTasks = append(activeTasks, winPoStTask)
 			needProofParams = true
 		}
 	}
 
 	slrLazy := lazy.MakeLazy(func() (*ffi.SealCalls, error) {
-		return ffi.NewSealCalls(stor, lstor, si), nil
+		return ffi.NewSealCalls(stor, lstor, si, dependencies.CurioFfiWrap), nil
 	})
 
 	{

curiosrc/builder.go

@@ -9,25 +9,25 @@ import (
 	"github.com/filecoin-project/lotus/curiosrc/message"
 	"github.com/filecoin-project/lotus/curiosrc/multictladdr"
 	"github.com/filecoin-project/lotus/curiosrc/window"
+	"github.com/filecoin-project/lotus/lib/ffiselect"
 	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
 	"github.com/filecoin-project/lotus/node/config"
 	dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/storage/paths"
-	"github.com/filecoin-project/lotus/storage/sealer"
 	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 //var log = logging.Logger("provider")
 
 func WindowPostScheduler(ctx context.Context, fc config.CurioFees, pc config.CurioProvingConfig,
-	api api.FullNode, verif storiface.Verifier, lw *sealer.LocalWorker, sender *message.Sender, chainSched *chainsched.CurioChainSched,
+	api api.FullNode, verif storiface.Verifier, curioFFIWrap *ffiselect.CurioFFIWrap, sender *message.Sender, chainSched *chainsched.CurioChainSched,
 	as *multictladdr.MultiAddressSelector, addresses map[dtypes.MinerAddress]bool, db *harmonydb.DB,
 	stor paths.Store, idx paths.SectorIndex, max int) (*window.WdPostTask, *window.WdPostSubmitTask, *window.WdPostRecoverDeclareTask, error) {
 
 	// todo config
 	ft := window.NewSimpleFaultTracker(stor, idx, pc.ParallelCheckLimit, time.Duration(pc.SingleCheckTimeout), time.Duration(pc.PartitionCheckTimeout))
 
-	computeTask, err := window.NewWdPostTask(db, api, ft, lw, verif, chainSched, addresses, max)
+	computeTask, err := window.NewWdPostTask(db, api, ft, curioFFIWrap, verif, chainSched, addresses, max)
 	if err != nil {
 		return nil, nil, nil, err
 	}

curiosrc/ffi/sdr_funcs.go

@@ -39,21 +39,21 @@ type ExternPrecommit2 func(ctx context.Context, sector storiface.SectorRef, cach
 */
 type SealCalls struct {
 	sectors *storageProvider
-	ffiselect.FFICallWrapper
+	ffiselect.CurioFFIWrap
 
 	/*// externCalls cointain overrides for calling alternative sealing logic
 	externCalls ExternalSealer*/
 }
 
-func NewSealCalls(st *paths.Remote, ls *paths.Local, si paths.SectorIndex) *SealCalls {
+func NewSealCalls(st *paths.Remote, ls *paths.Local, si paths.SectorIndex, cuFFIWrap *ffiselect.CurioFFIWrap) *SealCalls {
 	return &SealCalls{
 		sectors: &storageProvider{
 			storage:             st,
 			localStore:          ls,
 			sindex:              si,
 			storageReservations: xsync.NewIntegerMapOf[harmonytask.TaskID, *StorageReservation](),
-			FFICallWrapper:      ffiselect.FFICallWrapper{},
 		},
+		CurioFFIWrap: *cuFFIWrap,
 	}
 }
 
@@ -260,10 +260,11 @@ func (sb *SealCalls) TreeRC(ctx context.Context, task *harmonytask.TaskID, secto
 		}
 	}
 
-	sl, uns, err := ffi.SealPreCommitPhase2(p1o, fspaths.Cache, fspaths.Sealed)
+	cids, err := sb.CurioFFIWrap.SealPreCommit2(ctx, sector, p1o)
 	if err != nil {
 		return cid.Undef, cid.Undef, xerrors.Errorf("computing seal proof: %w", err)
 	}
+	sl, uns := cids.Sealed, cids.Unsealed
 
 	if uns != unsealed {
 		return cid.Undef, cid.Undef, xerrors.Errorf("unsealed cid changed after sealing")
@@ -311,7 +312,7 @@ func (sb *SealCalls) PoRepSnark(ctx context.Context, sn storiface.SectorRef, sea
 		return nil, xerrors.Errorf("failed to generate vanilla proof: %w", err)
 	}
 
-	proof, err := ffi.SealCommitPhase2(vproof, sn.ID.Number, sn.ID.Miner)
+	proof, err := sb.CurioFFIWrap.SealCommit2(ctx, sn, vproof)
 	if err != nil {
 		return nil, xerrors.Errorf("computing seal proof failed: %w", err)
 	}

curiosrc/window/compute_do.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/ipfs/go-cid"
+	"github.com/samber/lo"
 	"go.uber.org/multierr"
 	"golang.org/x/xerrors"
 
@@ -408,9 +409,14 @@ func (t *WdPostTask) generateWindowPoSt(ctx context.Context, ppt abi.RegisteredP
 				})
 			}
 
-			pr, err := t.prover.GenerateWindowPoStAdv(cctx, ppt, minerID, sectors, int(partIdx), randomness, true)
-			sk := pr.Skipped
-
+			sectorInfo := lo.Map(sectors, func(s storiface.PostSectorChallenge, i int) proof.ExtendedSectorInfo {
+				return proof.ExtendedSectorInfo{
+					SectorNumber: s.SectorNumber,
+					SealProof:    s.SealProof,
+					SealedCID:    s.SealedCID,
+				}
+			})
+			postProofs, sk, err := t.curioFfiWrap.GenerateWindowPoSt(cctx, minerID, ppt, sectorInfo, randomness)
 			if err != nil || len(sk) > 0 {
 				log.Errorf("generateWindowPost part:%d, skipped:%d, sectors: %d, err: %+v", partIdx, len(sk), len(sectors), err)
 				flk.Lock()
@@ -422,7 +428,8 @@ func (t *WdPostTask) generateWindowPoSt(ctx context.Context, ppt abi.RegisteredP
 				flk.Unlock()
 			}
 
-			proofList[partIdx] = ffi.PartitionProof(pr.PoStProofs)
+			// @magik6k help! postProofs is a []proof.PoStProof, but ffi.PartitionProof wants only 1.
+			proofList[partIdx] = ffi.PartitionProof(postProofs[0])
 		}(partIdx)
 	}
 

curiosrc/window/compute_task.go

@@ -23,6 +23,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/curiosrc/chainsched"
+	"github.com/filecoin-project/lotus/lib/ffiselect"
 	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
 	"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
 	"github.com/filecoin-project/lotus/lib/harmony/resources"
@@ -65,7 +66,7 @@ type WdPostTask struct {
 	db  *harmonydb.DB
 
 	faultTracker sealer.FaultTracker
-	prover       ProverPoSt
+	curioFfiWrap *ffiselect.CurioFFIWrap
 	verifier     storiface.Verifier
 
 	windowPoStTF promise.Promise[harmonytask.AddTaskFunc]
@@ -84,7 +85,7 @@ type wdTaskIdentity struct {
 func NewWdPostTask(db *harmonydb.DB,
 	api WDPoStAPI,
 	faultTracker sealer.FaultTracker,
-	prover ProverPoSt,
+	curioFFIWrap *ffiselect.CurioFFIWrap,
 	verifier storiface.Verifier,
 	pcs *chainsched.CurioChainSched,
 	actors map[dtypes.MinerAddress]bool,
@@ -95,7 +96,7 @@ func NewWdPostTask(db *harmonydb.DB,
 		api: api,
 
 		faultTracker: faultTracker,
-		prover:       prover,
+		curioFfiWrap: curioFFIWrap,
 		verifier:     verifier,
 
 		actors: actors,