Merge pull request #1031 from ipfs-force-community/cherry-pick-pr-102…

…2-1024-1027-1028-to-release-v0.9

Cherry pick #1022 #1024 #1027 #1028 to release v0.9

CHANGELOG.zh.md

@@ -1,12 +1,18 @@
 # Changelog
 
+## v0.9.0-rc7
+- damocles-manager
+  - 暂时移除 `move partition` 相关的命令 [#1022](https://github.com/ipfs-force-community/damocles/pull/1022) 
+  - 修复 wait for message, 如果消息失败了则不继续等待消息而是直接返回错误。[#1024](https://github.com/ipfs-force-community/damocles/pull/1024)
+  - 优化 wdpost， 允许 worker 向任意状态的任务发送心跳。[#1028](https://github.com/ipfs-force-community/damocles/pull/1028)
+
 ## v0.9.0-rc6
-  - damocles-manager
-    - 简化扇区存储配置 [#1010](https://github.com/ipfs-force-community/damocles/pull/1010)
+- damocles-manager
+  - 简化扇区存储配置 [#1010](https://github.com/ipfs-force-community/damocles/pull/1010)
 
 ## v0.9.0-rc5
-  - damocles-manager
-    - 升级 venus 依赖: v1.14.0-rc3 -> v1.14.0-rc4
+- damocles-manager
+  - 升级 venus 依赖: v1.14.0-rc3 -> v1.14.0-rc4
 
 ## v0.9.0-rc4
 

damocles-manager/cmd/damocles-manager/internal/global.go

@@ -239,7 +239,8 @@ WAIT_RET:
 		case messager.MessageState.OnChainMsg, messager.MessageState.NonceConflictMsg:
 			mret = ret
 			break WAIT_RET
-
+		case messager.MessageState.FailedMsg:
+			return fmt.Errorf("message failed %s: %s", mid, ret.ErrorMsg)
 		default:
 			mlog.Debugf("msg state: %s", messager.MessageStateToString(ret.State))
 		}

damocles-manager/cmd/damocles-manager/internal/util_sealer_proving.go

@@ -1068,6 +1068,10 @@ var utilSealerProvingRecoverFaultsCmd = &cli.Command{
 			Name:  "from",
 			Usage: "Specify the address of the address to send message, default miner's worker",
 		},
+		&cli.StringFlag{
+			Name:  "exid",
+			Usage: "external identifier of the message, ensure that we could make the message unique, or we could catch up with a previous message",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		api, actx, astop, err := extractAPI(cctx)

damocles-manager/cmd/damocles-manager/internal/util_sealer_sectors.go

@@ -100,7 +100,9 @@ var utilSealerSectorsCmd = &cli.Command{
 		utilSealerSectorsRebuildCmd,
 		utilSealerSectorsExportCmd,
 		utilSealerSectorsUnsealCmd,
-		utilSealerSectorsMovePartitionsCmd,
+
+		// todo: consider add this command back until next update in which FIP0070 maybe be in scope
+		// utilSealerSectorsMovePartitionsCmd,
 	},
 }
 

damocles-manager/go.mod

@@ -20,9 +20,9 @@ require (
 	github.com/filecoin-project/go-jsonrpc v0.3.1
 	github.com/filecoin-project/go-paramfetch v0.0.4
 	github.com/filecoin-project/go-state-types v0.12.5
-	github.com/filecoin-project/lotus v1.24.0-rc1
+	github.com/filecoin-project/lotus v1.24.0-rc6
 	github.com/filecoin-project/specs-storage v0.4.1
-	github.com/filecoin-project/venus v1.14.0-rc4
+	github.com/filecoin-project/venus v1.14.0-rc6
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.3.0
 	github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026

damocles-manager/go.sum

@@ -297,8 +297,8 @@ github.com/filecoin-project/go-storedcounter v0.1.0 h1:Mui6wSUBC+cQGHbDUBcO7rfh5
 github.com/filecoin-project/go-storedcounter v0.1.0/go.mod h1:4ceukaXi4vFURIoxYMfKzaRF5Xv/Pinh2oTnoxpv+z8=
 github.com/filecoin-project/kubo-api-client v0.0.1 h1:IR1b+sm+VYxSRvbgECVv9SbhIgygcXcSoN1Q7xsHDXg=
 github.com/filecoin-project/kubo-api-client v0.0.1/go.mod h1:c36PPMIVOkKfHDwDG5U05gUlPRY9wNuh/BePwo0e+6Y=
-github.com/filecoin-project/lotus v1.24.0-rc1 h1:cuJ0JiJ5MYjGV3rXp0ghqdFkOPLxvgZOSSUwH1K8NSU=
-github.com/filecoin-project/lotus v1.24.0-rc1/go.mod h1:Az8bI5Xpyk04IgonL4TywkqgCkTjtHeiMbPgRbAzXiA=
+github.com/filecoin-project/lotus v1.24.0-rc6 h1:IJd0YjeL1Xq0efQJZdPMaX5/XfF3HC4/IDJZefwpF7s=
+github.com/filecoin-project/lotus v1.24.0-rc6/go.mod h1:hKVsRPYzHjx8+8/hD+Yc3lRmLa6gTqGHLqJR8c9NEhg=
 github.com/filecoin-project/pubsub v1.0.0 h1:ZTmT27U07e54qV1mMiQo4HDr0buo8I1LDHBYLXlsNXM=
 github.com/filecoin-project/pubsub v1.0.0/go.mod h1:GkpB33CcUtUNrLPhJgfdy4FDx4OMNR9k+46DHx/Lqrg=
 github.com/filecoin-project/specs-actors v0.9.13/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao=
@@ -321,8 +321,8 @@ github.com/filecoin-project/specs-actors/v8 v8.0.1 h1:4u0tIRJeT5G7F05lwLRIsDnsrN
 github.com/filecoin-project/specs-actors/v8 v8.0.1/go.mod h1:UYIPg65iPWoFw5NEftREdJwv9b/5yaLKdCgTvNI/2FA=
 github.com/filecoin-project/specs-storage v0.4.1 h1:yvLEaLZj8f+uByhNC4mFOtCUyL2wQku+NGBp6hjTe9M=
 github.com/filecoin-project/specs-storage v0.4.1/go.mod h1:Z2eK6uMwAOSLjek6+sy0jNV2DSsMEENziMUz0GHRFBw=
-github.com/filecoin-project/venus v1.14.0-rc4 h1:DN0lVKUFlNZzPPIltoiex2DtIzP4hfBTfqk/Z1V7Qwc=
-github.com/filecoin-project/venus v1.14.0-rc4/go.mod h1:nCXhYe+mtrtrF+ZAVWDNRprNfXOzRYznXkplAZTofyg=
+github.com/filecoin-project/venus v1.14.0-rc6 h1:NiLlvc11lX1tCvxeLRGbjU+GKZlxC+MT4wiLLDj0GKQ=
+github.com/filecoin-project/venus v1.14.0-rc6/go.mod h1:/apMxQl6VYz1FMeBStrP3BUsBCmDhyGUW91SLjVg5e8=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ=
 github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=

damocles-manager/modules/impl/prover/worker/job_mgr_kv.go

@@ -29,7 +29,7 @@ type kvJobManager struct {
 // TODO(0x5459): Consider putting `txn` into context?
 func (tm *kvJobManager) filter(ctx context.Context, txn kvstore.TxnExt, state core.WdPoStJobState, limit uint32, f func(*core.WdPoStJob) bool) (jobs []*core.WdPoStJob, err error) {
 	var it kvstore.Iter
-	it, err = txn.Scan([]byte(makeWdPoStPrefix(state)))
+	it, err = txn.Scan(kvstore.Prefix(makeWdPoStPrefix(state)))
 	if err != nil {
 		return
 	}
@@ -134,7 +134,7 @@ func (tm *kvJobManager) Create(ctx context.Context, deadlineIdx uint64, partitio
 			CreatedAt:   uint64(now),
 			UpdatedAt:   uint64(now),
 		}
-		return txn.PutJson([]byte(makeWdPoStKey(core.WdPoStJobReadyToRun, jobID)), &job)
+		return txn.PutJson(kvstore.Key(makeWdPoStKey(core.WdPoStJobReadyToRun, jobID)), &job)
 	})
 
 	if err == nil {
@@ -163,7 +163,7 @@ func (tm *kvJobManager) AllocateJobs(ctx context.Context, spec core.AllocateWdPo
 		now := uint64(time.Now().Unix())
 		for _, job := range readyToRun {
 			// Moving ready to run jobs to running jobs
-			if err := txn.Del([]byte(makeWdPoStKey(core.WdPoStJobReadyToRun, job.ID))); err != nil {
+			if err := txn.Del(kvstore.Key(makeWdPoStKey(core.WdPoStJobReadyToRun, job.ID))); err != nil {
 				return err
 			}
 			job.State = string(core.WdPoStJobRunning)
@@ -172,7 +172,7 @@ func (tm *kvJobManager) AllocateJobs(ctx context.Context, spec core.AllocateWdPo
 			job.WorkerName = workerName
 			job.HeartbeatAt = now
 			job.UpdatedAt = now
-			if err := txn.PutJson([]byte(makeWdPoStKey(core.WdPoStJobRunning, job.ID)), job); err != nil {
+			if err := txn.PutJson(kvstore.Key(makeWdPoStKey(core.WdPoStJobRunning, job.ID)), job); err != nil {
 				return err
 			}
 			allocatedJobs = append(allocatedJobs, &core.WdPoStAllocatedJob{
@@ -196,16 +196,28 @@ func (tm *kvJobManager) Heartbeat(ctx context.Context, jobIDs []string, workerNa
 	err := tm.kv.UpdateMustNoConflict(ctx, func(txn kvstore.TxnExt) error {
 		for _, jobID := range jobIDs {
 			var job core.WdPoStJob
-			if err := txn.Peek([]byte(makeWdPoStKey(core.WdPoStJobRunning, jobID)), kvstore.LoadJSON(&job)); err != nil {
+			key, err := txn.PeekAny(
+				kvstore.LoadJSON(&job),
+				kvstore.Key(makeWdPoStKey(core.WdPoStJobRunning, jobID)),
+				kvstore.Key(makeWdPoStKey(core.WdPoStJobReadyToRun, jobID)),
+				kvstore.Key(makeWdPoStKey(core.WdPoStJobFinished, jobID)),
+			)
+			if err != nil {
+				return err
+			}
+
+			if err := txn.Del(key); err != nil {
 				return err
 			}
+
 			if job.StartedAt == 0 {
 				job.StartedAt = now
 			}
 			job.HeartbeatAt = now
 			job.WorkerName = workerName
+			job.State = string(core.WdPoStJobRunning)
 			job.UpdatedAt = now
-			if err := txn.PutJson([]byte(makeWdPoStKey(core.WdPoStJobRunning, jobID)), &job); err != nil {
+			if err := txn.PutJson(kvstore.Key(makeWdPoStKey(core.WdPoStJobRunning, jobID)), &job); err != nil {
 				return err
 			}
 		}
@@ -219,12 +231,17 @@ func (tm *kvJobManager) Heartbeat(ctx context.Context, jobIDs []string, workerNa
 
 func (tm *kvJobManager) Finish(ctx context.Context, jobID string, output *stage.WindowPoStOutput, errorReason string) error {
 	err := tm.kv.UpdateMustNoConflict(ctx, func(txn kvstore.TxnExt) error {
-		runningKey := []byte(makeWdPoStKey(core.WdPoStJobRunning, jobID))
 		var job core.WdPoStJob
-		if err := txn.Peek(runningKey, kvstore.LoadJSON(&job)); err != nil {
+		key, err := txn.PeekAny(
+			kvstore.LoadJSON(&job),
+			kvstore.Key(makeWdPoStKey(core.WdPoStJobRunning, jobID)),
+			kvstore.Key(makeWdPoStKey(core.WdPoStJobReadyToRun, jobID)),
+			kvstore.Key(makeWdPoStKey(core.WdPoStJobFinished, jobID)),
+		)
+		if err != nil {
 			return err
 		}
-		if err := txn.Del(runningKey); err != nil {
+		if err := txn.Del(key); err != nil {
 			return err
 		}
 		now := uint64(time.Now().Unix())
@@ -233,7 +250,7 @@ func (tm *kvJobManager) Finish(ctx context.Context, jobID string, output *stage.
 		job.ErrorReason = errorReason
 		job.FinishedAt = now
 		job.UpdatedAt = now
-		return txn.PutJson([]byte(makeWdPoStKey(core.WdPoStJobFinished, jobID)), &job)
+		return txn.PutJson(kvstore.Key(makeWdPoStKey(core.WdPoStJobFinished, jobID)), &job)
 	})
 
 	if err == nil {
@@ -260,15 +277,15 @@ func (tm *kvJobManager) MakeJobsDie(ctx context.Context, heartbeatTimeout time.D
 		}
 		now := uint64(time.Now().Unix())
 		for _, job := range shouldDead {
-			if err := txn.Del([]byte(makeWdPoStKey(core.WdPoStJobRunning, job.ID))); err != nil {
+			if err := txn.Del(kvstore.Key(makeWdPoStKey(core.WdPoStJobRunning, job.ID))); err != nil {
 				return err
 			}
 			job.State = string(core.WdPoStJobFinished)
 			job.FinishedAt = now
 			job.Output = nil
 			job.ErrorReason = "heartbeat timeout"
 			job.UpdatedAt = now
-			if err := txn.PutJson([]byte(makeWdPoStKey(core.WdPoStJobFinished, job.ID)), job); err != nil {
+			if err := txn.PutJson(kvstore.Key(makeWdPoStKey(core.WdPoStJobFinished, job.ID)), job); err != nil {
 				return err
 			}
 		}
@@ -297,7 +314,7 @@ func (tm *kvJobManager) CleanupExpiredJobs(ctx context.Context, jobLifetime time
 			return err
 		}
 		for _, job := range shouldClean {
-			if err := txn.Del([]byte(makeWdPoStKey(core.WdPoStJobFinished, job.ID))); err != nil {
+			if err := txn.Del(kvstore.Key(makeWdPoStKey(core.WdPoStJobFinished, job.ID))); err != nil {
 				return err
 			}
 		}
@@ -324,7 +341,7 @@ func (tm *kvJobManager) RetryFailedJobs(ctx context.Context, maxTry, limit uint3
 		}
 		now := uint64(time.Now().Unix())
 		for _, job := range shouldRetry {
-			err := txn.Del([]byte(makeWdPoStKey(core.WdPoStJobFinished, job.ID)))
+			err := txn.Del(kvstore.Key(makeWdPoStKey(core.WdPoStJobFinished, job.ID)))
 			if err != nil {
 				return err
 			}
@@ -334,7 +351,7 @@ func (tm *kvJobManager) RetryFailedJobs(ctx context.Context, maxTry, limit uint3
 			job.StartedAt = 0
 			job.FinishedAt = 0
 			job.UpdatedAt = now
-			if err := txn.PutJson([]byte(makeWdPoStKey(core.WdPoStJobReadyToRun, job.ID)), job); err != nil {
+			if err := txn.PutJson(kvstore.Key(makeWdPoStKey(core.WdPoStJobReadyToRun, job.ID)), job); err != nil {
 				return err
 			}
 		}
@@ -379,7 +396,7 @@ func (tm *kvJobManager) Reset(ctx context.Context, jobID string) error {
 		if err := txn.Del(key); err != nil {
 			return err
 		}
-		return txn.PutJson([]byte(makeWdPoStKey(core.WdPoStJobReadyToRun, jobID)), &job)
+		return txn.PutJson(kvstore.Key(makeWdPoStKey(core.WdPoStJobReadyToRun, jobID)), &job)
 	})
 
 	if err == nil {

damocles-manager/ver/ver.go

@@ -2,7 +2,7 @@ package ver
 
 import "fmt"
 
-const Version = "0.9.0-rc6"
+const Version = "0.9.0-rc7"
 
 var Commit string
 

damocles-worker/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "damocles-worker"
-version = "0.9.0-rc6"
+version = "0.9.0-rc7"
 authors = ["dtynn <[email protected]>"]
 edition = "2021"
 exclude = [".github"]