DAOS-6611 control: Enable dmg pool exclude,drain,reint on multiple ranks

src/common/tests_dmg_helpers.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2020-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -1063,7 +1064,9 @@ dmg_pool_target(const char *cmd, const char *dmg_config_file, const uuid_t uuid,
 			D_GOTO(out, rc = -DER_NOMEM);
 	}
 
-	args = cmd_push_arg(args, &argcount, "--rank=%d ", rank);
+	// Exclude, drain and reintegrate take ranks option which can be either a rank-list range or
+	// a single rank identifier.
+	args = cmd_push_arg(args, &argcount, "--ranks=%d ", rank);
 	if (args == NULL)
 		D_GOTO(out, rc = -DER_NOMEM);
 

src/control/cmd/dmg/command_test.go

@@ -160,14 +160,12 @@ func (bci *bridgeConnInvoker) InvokeUnaryRPC(ctx context.Context, uReq control.U
 	case *control.PoolGetACLReq, *control.PoolOverwriteACLReq,
 		*control.PoolUpdateACLReq, *control.PoolDeleteACLReq:
 		resp = control.MockMSResponse("", nil, &mgmtpb.ACLResp{})
-	case *control.PoolExcludeReq:
-		resp = control.MockMSResponse("", nil, &mgmtpb.PoolExcludeResp{})
-	case *control.PoolDrainReq:
-		resp = control.MockMSResponse("", nil, &mgmtpb.PoolDrainResp{})
+	case *control.PoolRanksReq:
+		resp = control.MockMSResponse("", nil, &mgmtpb.PoolRanksResp{
+			SuccessRanks: []uint32{0},
+		})
 	case *control.PoolExtendReq:
 		resp = control.MockMSResponse("", nil, &mgmtpb.PoolExtendResp{})
-	case *control.PoolReintegrateReq:
-		resp = control.MockMSResponse("", nil, &mgmtpb.PoolReintResp{})
 	case *control.SystemCheckEnableReq:
 		resp = control.MockMSResponse("", nil, &mgmtpb.DaosResp{})
 	case *control.SystemCheckDisableReq:

src/control/cmd/dmg/json_test.go

@@ -96,10 +96,8 @@ func TestDmg_JsonOutput(t *testing.T) {
 				testArgs = append(testArgs, test.MockUUID(), "label:foo")
 			case "pool get-prop":
 				testArgs = append(testArgs, test.MockUUID(), "label")
-			case "pool extend":
+			case "pool extend", "pool exclude", "pool drain", "pool reintegrate":
 				testArgs = append(testArgs, test.MockUUID(), "--ranks", "0")
-			case "pool exclude", "pool drain", "pool reintegrate":
-				testArgs = append(testArgs, test.MockUUID(), "--rank", "0")
 			case "pool query-targets":
 				testArgs = append(testArgs, test.MockUUID(), "--rank", "0", "--target-idx", "1,3,5,7")
 			case "container set-owner":

src/control/cmd/dmg/pool.go

@@ -35,10 +35,10 @@ type PoolCmd struct {
 	Destroy      poolDestroyCmd      `command:"destroy" description:"Destroy a DAOS pool"`
 	Evict        poolEvictCmd        `command:"evict" description:"Evict all pool connections to a DAOS pool"`
 	List         poolListCmd         `command:"list" alias:"ls" description:"List DAOS pools"`
-	Extend       poolExtendCmd       `command:"extend" description:"Extend a DAOS pool to include new ranks."`
-	Exclude      poolExcludeCmd      `command:"exclude" description:"Exclude targets from a rank"`
-	Drain        poolDrainCmd        `command:"drain" description:"Drain targets from a rank"`
-	Reintegrate  poolReintegrateCmd  `command:"reintegrate" alias:"reint" description:"Reintegrate targets for a rank"`
+	Extend       poolExtendCmd       `command:"extend" description:"Extend a DAOS pool to include new ranks"`
+	Exclude      poolExcludeCmd      `command:"exclude" description:"Exclude targets from a set of ranks"`
+	Drain        poolDrainCmd        `command:"drain" description:"Drain targets from a set of ranks"`
+	Reintegrate  poolReintegrateCmd  `command:"reintegrate" alias:"reint" description:"Reintegrate targets for a set of rank"`
 	Query        poolQueryCmd        `command:"query" description:"Query a DAOS pool"`
 	QueryTargets poolQueryTargetsCmd `command:"query-targets" description:"Query pool target info"`
 	GetACL       poolGetACLCmd       `command:"get-acl" description:"Get a DAOS pool's Access Control List"`
@@ -531,72 +531,96 @@ func (cmd *poolEvictCmd) Execute(args []string) error {
 	return err
 }
 
+// poolRanksCmd is used as an embedded command type that enables multiple ranks on a pool to be
+// processed.
+type poolRanksCmd struct {
+	poolCmd
+	RankList ui.RankSetFlag `long:"ranks" required:"1" description:"Comma-separated list of rank-range strings to operate on for a single pool"`
+}
+
 // poolExcludeCmd is the struct representing the command to exclude a DAOS target.
 type poolExcludeCmd struct {
-	poolCmd
+	poolRanksCmd
 	Force     bool   `short:"f" long:"force" description:"Force the operation to continue, potentially leading to data loss"`
-	Rank      uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be excluded"`
-	TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be excluded from the rank"`
+	TargetIdx string `long:"target-idx" description:"Comma-separated list of target index(es) to be excluded from each rank"`
 }
 
 // Execute is run when PoolExcludeCmd subcommand is activated
 func (cmd *poolExcludeCmd) Execute(args []string) error {
-	msg := "succeeded"
-
 	var idxList []uint32
 	if err := common.ParseNumberList(cmd.TargetIdx, &idxList); err != nil {
 		return errors.WithMessage(err, "parsing target list")
 	}
 
-	req := &control.PoolExcludeReq{ID: cmd.PoolID().String(), Rank: ranklist.Rank(cmd.Rank), TargetIdx: idxList, Force: cmd.Force}
+	req := &control.PoolRanksReq{
+		ID:        cmd.PoolID().String(),
+		Ranks:     cmd.RankList.Ranks(),
+		TargetIdx: idxList,
+		Force:     cmd.Force,
+	}
+
+	resp, err := control.PoolExclude(cmd.MustLogCtx(), cmd.ctlInvoker, req)
+
+	if cmd.JSONOutputEnabled() {
+		return cmd.OutputJSON(resp, err)
+	}
 
-	err := control.PoolExclude(cmd.MustLogCtx(), cmd.ctlInvoker, req)
+	// Retrieve PoolRanksResults so we can pretty print output.
+	results, err := resp.GetResults(err)
 	if err != nil {
-		msg = errors.WithMessage(err, "failed").Error()
+		cmd.Errorf(errors.WithMessage(err, "Pool exclude failed").Error())
+		return err
 	}
 
-	cmd.Infof("Exclude command %s\n", msg)
+	var out strings.Builder
+	pretty.PrintPoolRankResults(&out, "exclude", results)
+	cmd.Info(out.String())
 
-	return err
+	return resp.Errors()
 }
 
 // poolDrainCmd is the struct representing the command to Drain a DAOS target.
 type poolDrainCmd struct {
-	poolCmd
-	Rank      uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be drained"`
-	TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be drained on the rank"`
+	poolRanksCmd
+	TargetIdx string `long:"target-idx" description:"Comma-separated list of target index(es) to be drained on each rank"`
 }
 
 // Execute is run when PoolDrainCmd subcommand is activated
 func (cmd *poolDrainCmd) Execute(args []string) error {
-	msg := "succeeded"
-
 	var idxList []uint32
 	if err := common.ParseNumberList(cmd.TargetIdx, &idxList); err != nil {
-		err = errors.WithMessage(err, "parsing target list")
-		return err
+		return errors.WithMessage(err, "parsing target list")
 	}
 
-	req := &control.PoolDrainReq{
+	req := &control.PoolRanksReq{
 		ID:        cmd.PoolID().String(),
-		Rank:      ranklist.Rank(cmd.Rank),
+		Ranks:     cmd.RankList.Ranks(),
 		TargetIdx: idxList,
 	}
 
-	err := control.PoolDrain(cmd.MustLogCtx(), cmd.ctlInvoker, req)
+	resp, err := control.PoolDrain(cmd.MustLogCtx(), cmd.ctlInvoker, req)
+
+	if cmd.JSONOutputEnabled() {
+		return cmd.OutputJSON(resp, err)
+	}
+
+	// Retrieve PoolRanksResults so we can pretty print output.
+	results, err := resp.GetResults(err)
 	if err != nil {
-		msg = errors.WithMessage(err, "failed").Error()
+		cmd.Errorf(errors.WithMessage(err, "Pool drain failed").Error())
+		return err
 	}
 
-	cmd.Infof("Drain command %s\n", msg)
+	var out strings.Builder
+	pretty.PrintPoolRankResults(&out, "drain", results)
+	cmd.Info(out.String())
 
-	return err
+	return resp.Errors()
 }
 
 // poolExtendCmd is the struct representing the command to Extend a DAOS pool.
 type poolExtendCmd struct {
-	poolCmd
-	RankList ui.RankSetFlag `long:"ranks" required:"1" description:"Comma-separated list of ranks to add to the pool"`
+	poolRanksCmd
 }
 
 // Execute is run when PoolExtendCmd subcommand is activated
@@ -620,35 +644,41 @@ func (cmd *poolExtendCmd) Execute(args []string) error {
 
 // poolReintegrateCmd is the struct representing the command to Add a DAOS target.
 type poolReintegrateCmd struct {
-	poolCmd
-	Rank      uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be reintegrated"`
-	TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be reintegrated into the rank"`
+	poolRanksCmd
+	TargetIdx string `long:"target-idx" description:"Comma-separated list of target index(es) to be reintegrated into each rank"`
 }
 
 // Execute is run when poolReintegrateCmd subcommand is activated
 func (cmd *poolReintegrateCmd) Execute(args []string) error {
-	msg := "succeeded"
-
 	var idxList []uint32
 	if err := common.ParseNumberList(cmd.TargetIdx, &idxList); err != nil {
-		err = errors.WithMessage(err, "parsing target list")
-		return err
+		return errors.WithMessage(err, "parsing target list")
 	}
 
-	req := &control.PoolReintegrateReq{
+	req := &control.PoolRanksReq{
 		ID:        cmd.PoolID().String(),
-		Rank:      ranklist.Rank(cmd.Rank),
+		Ranks:     cmd.RankList.Ranks(),
 		TargetIdx: idxList,
 	}
 
-	err := control.PoolReintegrate(cmd.MustLogCtx(), cmd.ctlInvoker, req)
+	resp, err := control.PoolReintegrate(cmd.MustLogCtx(), cmd.ctlInvoker, req)
+
+	if cmd.JSONOutputEnabled() {
+		return cmd.OutputJSON(resp, err)
+	}
+
+	// Retrieve PoolRanksResults so we can pretty print output.
+	results, err := resp.GetResults(err)
 	if err != nil {
-		msg = errors.WithMessage(err, "failed").Error()
+		cmd.Errorf(errors.WithMessage(err, "Pool reintegrate failed").Error())
+		return err
 	}
 
-	cmd.Infof("Reintegration command %s\n", msg)
+	var out strings.Builder
+	pretty.PrintPoolRankResults(&out, "reintegrate", results)
+	cmd.Info(out.String())
 
-	return err
+	return resp.Errors()
 }
 
 // poolQueryCmd is the struct representing the command to query a DAOS pool.
@@ -701,7 +731,7 @@ type poolQueryTargetsCmd struct {
 	poolCmd
 
 	Rank    uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be queried"`
-	Targets string `long:"target-idx" description:"Comma-separated list of target idx(s) to be queried"`
+	Targets string `long:"target-idx" description:"Comma-separated list of target index(es) to be queried"`
 }
 
 // Execute is run when PoolQueryTargetsCmd subcommand is activated

src/control/cmd/dmg/pool_test.go

@@ -616,11 +616,11 @@ func TestPoolCommands(t *testing.T) {
 		},
 		{
 			"Exclude a target with single target idx",
-			"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1",
+			"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2 --target-idx 1",
 			strings.Join([]string{
-				printRequest(t, &control.PoolExcludeReq{
+				printRequest(t, &control.PoolRanksReq{
 					ID:        "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
-					Rank:      0,
+					Ranks:     []ranklist.Rank{0, 1, 2},
 					TargetIdx: []uint32{1},
 					Force:     false,
 				}),
@@ -629,11 +629,11 @@ func TestPoolCommands(t *testing.T) {
 		},
 		{
 			"Exclude a target with multiple idx",
-			"pool exclude -f 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1,2,3",
+			"pool exclude -f 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2 --target-idx 1,2,3",
 			strings.Join([]string{
-				printRequest(t, &control.PoolExcludeReq{
+				printRequest(t, &control.PoolRanksReq{
 					ID:        "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
-					Rank:      0,
+					Ranks:     []ranklist.Rank{0, 1, 2},
 					TargetIdx: []uint32{1, 2, 3},
 					Force:     true,
 				}),
@@ -642,11 +642,11 @@ func TestPoolCommands(t *testing.T) {
 		},
 		{
 			"Exclude a target with no idx given",
-			"pool exclude -f 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0",
+			"pool exclude -f 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2",
 			strings.Join([]string{
-				printRequest(t, &control.PoolExcludeReq{
+				printRequest(t, &control.PoolRanksReq{
 					ID:        "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
-					Rank:      0,
+					Ranks:     []ranklist.Rank{0, 1, 2},
 					TargetIdx: []uint32{},
 					Force:     true,
 				}),
@@ -655,35 +655,35 @@ func TestPoolCommands(t *testing.T) {
 		},
 		{
 			"Drain a target with single target idx",
-			"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1",
+			"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2 --target-idx 1",
 			strings.Join([]string{
-				printRequest(t, &control.PoolDrainReq{
+				printRequest(t, &control.PoolRanksReq{
 					ID:        "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
-					Rank:      0,
+					Ranks:     []ranklist.Rank{0, 1, 2},
 					TargetIdx: []uint32{1},
 				}),
 			}, " "),
 			nil,
 		},
 		{
 			"Drain a target with multiple idx",
-			"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1,2,3",
+			"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2 --target-idx 1,2,3",
 			strings.Join([]string{
-				printRequest(t, &control.PoolDrainReq{
+				printRequest(t, &control.PoolRanksReq{
 					ID:        "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
-					Rank:      0,
+					Ranks:     []ranklist.Rank{0, 1, 2},
 					TargetIdx: []uint32{1, 2, 3},
 				}),
 			}, " "),
 			nil,
 		},
 		{
 			"Drain a target with no idx given",
-			"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0",
+			"pool drain 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-2",
 			strings.Join([]string{
-				printRequest(t, &control.PoolDrainReq{
+				printRequest(t, &control.PoolRanksReq{
 					ID:        "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
-					Rank:      0,
+					Ranks:     []ranklist.Rank{0, 1, 2},
 					TargetIdx: []uint32{},
 				}),
 			}, " "),
@@ -720,35 +720,35 @@ func TestPoolCommands(t *testing.T) {
 		},
 		{
 			"Reintegrate a target with single target idx",
-			"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1",
+			"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-1 --target-idx 1",
 			strings.Join([]string{
-				printRequest(t, &control.PoolReintegrateReq{
+				printRequest(t, &control.PoolRanksReq{
 					ID:        "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
-					Rank:      0,
+					Ranks:     []ranklist.Rank{0, 1},
 					TargetIdx: []uint32{1},
 				}),
 			}, " "),
 			nil,
 		},
 		{
 			"Reintegrate a target with multiple idx",
-			"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1,2,3",
+			"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-1 --target-idx 1,2,3",
 			strings.Join([]string{
-				printRequest(t, &control.PoolReintegrateReq{
+				printRequest(t, &control.PoolRanksReq{
 					ID:        "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
-					Rank:      0,
+					Ranks:     []ranklist.Rank{0, 1},
 					TargetIdx: []uint32{1, 2, 3},
 				}),
 			}, " "),
 			nil,
 		},
 		{
 			"Reintegrate a target with no idx given",
-			"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0",
+			"pool reintegrate 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --ranks 0-1",
 			strings.Join([]string{
-				printRequest(t, &control.PoolReintegrateReq{
+				printRequest(t, &control.PoolRanksReq{
 					ID:        "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
-					Rank:      0,
+					Ranks:     []ranklist.Rank{0, 1},
 					TargetIdx: []uint32{},
 				}),
 			}, " "),