Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DAOS-6611 control: Enable dmg pool exclude,drain,reint on multiple ranks #15731

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

DAOS-6611 control: Enable dmg pool exclude,drain,reint on multiple ranks #15731

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b477538
DAOS-6611 control: Enable dmg pool exclude,drain,reint on multiple ranks
tanabarr Feb 1, 2025
715819f
update exclude and reintegrate drpc handlers and fix ftest ranks params
tanabarr Feb 3, 2025
37fd8dc
Merge remote-tracking branch 'origin/master' into tanabarr/control-dr…
tanabarr Feb 3, 2025
79da39d
set system name on mgmt rpcs
tanabarr Feb 3, 2025
7560b12
populate pool id in pool-ranks response
tanabarr Feb 4, 2025
0d117c2
correct bug where rc overwritten in drpc handler
tanabarr Feb 5, 2025
f4e652d
set failed_rank to nil on drpc response init
tanabarr Feb 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/common/tests_dmg_helpers.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/**
* (C) Copyright 2020-2024 Intel Corporation.
* (C) Copyright 2025 Hewlett Packard Enterprise Development LP
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
Expand Down Expand Up @@ -1063,7 +1064,9 @@ dmg_pool_target(const char *cmd, const char *dmg_config_file, const uuid_t uuid,
D_GOTO(out, rc = -DER_NOMEM);
}

args = cmd_push_arg(args, &argcount, "--rank=%d ", rank);
// Exclude, drain and reintegrate take ranks option which can be either a rank-list range or
// a single rank identifier.
args = cmd_push_arg(args, &argcount, "--ranks=%d ", rank);
if (args == NULL)
D_GOTO(out, rc = -DER_NOMEM);

Expand Down
12 changes: 6 additions & 6 deletions src/control/cmd/dmg/command_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/daos-stack/daos/src/control/common/test"
"github.com/daos-stack/daos/src/control/lib/control"
"github.com/daos-stack/daos/src/control/lib/hardware"
"github.com/daos-stack/daos/src/control/lib/ranklist"
"github.com/daos-stack/daos/src/control/logging"
"github.com/daos-stack/daos/src/control/system"
)
Expand Down Expand Up @@ -160,14 +161,13 @@ func (bci *bridgeConnInvoker) InvokeUnaryRPC(ctx context.Context, uReq control.U
case *control.PoolGetACLReq, *control.PoolOverwriteACLReq,
*control.PoolUpdateACLReq, *control.PoolDeleteACLReq:
resp = control.MockMSResponse("", nil, &mgmtpb.ACLResp{})
case *control.PoolExcludeReq:
resp = control.MockMSResponse("", nil, &mgmtpb.PoolExcludeResp{})
case *control.PoolDrainReq:
resp = control.MockMSResponse("", nil, &mgmtpb.PoolDrainResp{})
case *control.PoolRanksReq:
// To meet PoolRanksResp.GetResults() validation requirements.
resp = control.MockMSResponse("", nil, &mgmtpb.PoolRanksResp{
SuccessRanks: []uint32{0}, FailedRank: uint32(ranklist.NilRank),
})
case *control.PoolExtendReq:
resp = control.MockMSResponse("", nil, &mgmtpb.PoolExtendResp{})
case *control.PoolReintegrateReq:
resp = control.MockMSResponse("", nil, &mgmtpb.PoolReintResp{})
case *control.SystemCheckEnableReq:
resp = control.MockMSResponse("", nil, &mgmtpb.DaosResp{})
case *control.SystemCheckDisableReq:
Expand Down
4 changes: 1 addition & 3 deletions src/control/cmd/dmg/json_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,8 @@ func TestDmg_JsonOutput(t *testing.T) {
testArgs = append(testArgs, test.MockUUID(), "label:foo")
case "pool get-prop":
testArgs = append(testArgs, test.MockUUID(), "label")
case "pool extend":
case "pool extend", "pool exclude", "pool drain", "pool reintegrate":
testArgs = append(testArgs, test.MockUUID(), "--ranks", "0")
case "pool exclude", "pool drain", "pool reintegrate":
testArgs = append(testArgs, test.MockUUID(), "--rank", "0")
case "pool query-targets":
testArgs = append(testArgs, test.MockUUID(), "--rank", "0", "--target-idx", "1,3,5,7")
case "container set-owner":
Expand Down
128 changes: 85 additions & 43 deletions src/control/cmd/dmg/pool.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ type PoolCmd struct {
Destroy poolDestroyCmd `command:"destroy" description:"Destroy a DAOS pool"`
Evict poolEvictCmd `command:"evict" description:"Evict all pool connections to a DAOS pool"`
List poolListCmd `command:"list" alias:"ls" description:"List DAOS pools"`
Extend poolExtendCmd `command:"extend" description:"Extend a DAOS pool to include new ranks."`
Exclude poolExcludeCmd `command:"exclude" description:"Exclude targets from a rank"`
Drain poolDrainCmd `command:"drain" description:"Drain targets from a rank"`
Reintegrate poolReintegrateCmd `command:"reintegrate" alias:"reint" description:"Reintegrate targets for a rank"`
Extend poolExtendCmd `command:"extend" description:"Extend a DAOS pool to include new ranks"`
Exclude poolExcludeCmd `command:"exclude" description:"Exclude targets from a set of ranks"`
Drain poolDrainCmd `command:"drain" description:"Drain targets from a set of ranks"`
Reintegrate poolReintegrateCmd `command:"reintegrate" alias:"reint" description:"Reintegrate targets for a set of rank"`
Query poolQueryCmd `command:"query" description:"Query a DAOS pool"`
QueryTargets poolQueryTargetsCmd `command:"query-targets" description:"Query pool target info"`
GetACL poolGetACLCmd `command:"get-acl" description:"Get a DAOS pool's Access Control List"`
Expand Down Expand Up @@ -531,72 +531,104 @@ func (cmd *poolEvictCmd) Execute(args []string) error {
return err
}

// poolRanksCmd is used as an embedded command type that enables multiple ranks on a pool to be
// processed.
type poolRanksCmd struct {
poolCmd
RankList ui.RankSetFlag `long:"ranks" required:"1" description:"Comma-separated list of rank-range strings to operate on for a single pool"`
}

// poolExcludeCmd is the struct representing the command to exclude a DAOS target.
type poolExcludeCmd struct {
poolCmd
poolRanksCmd
Force bool `short:"f" long:"force" description:"Force the operation to continue, potentially leading to data loss"`
Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be excluded"`
TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be excluded from the rank"`
TargetIdx string `long:"target-idx" description:"Comma-separated list of target index(es) to be excluded from each rank"`
}

// Execute is run when PoolExcludeCmd subcommand is activated
func (cmd *poolExcludeCmd) Execute(args []string) error {
msg := "succeeded"

var idxList []uint32
if err := common.ParseNumberList(cmd.TargetIdx, &idxList); err != nil {
return errors.WithMessage(err, "parsing target list")
}

req := &control.PoolExcludeReq{ID: cmd.PoolID().String(), Rank: ranklist.Rank(cmd.Rank), TargetIdx: idxList, Force: cmd.Force}
req := &control.PoolRanksReq{
ID: cmd.PoolID().String(),
Ranks: cmd.RankList.Ranks(),
TargetIdx: idxList,
Force: cmd.Force,
}

resp, err := control.PoolExclude(cmd.MustLogCtx(), cmd.ctlInvoker, req)

err := control.PoolExclude(cmd.MustLogCtx(), cmd.ctlInvoker, req)
if cmd.JSONOutputEnabled() {
return cmd.OutputJSON(resp, err)
}

cmd.Debugf("%T: %+v, %T: %+v", req, req, resp, resp)

// Retrieve PoolRanksResults so we can pretty print output.
results, err := resp.GetResults(err)
if err != nil {
msg = errors.WithMessage(err, "failed").Error()
cmd.Errorf(errors.WithMessage(err, "Pool exclude failed").Error())
return err
}

cmd.Infof("Exclude command %s\n", msg)
cmd.Debugf("%T: %+v", results, results)

return err
var out strings.Builder
pretty.PrintPoolRankResults(&out, results)
cmd.Info(out.String())

return resp.Errors()
}

// poolDrainCmd is the struct representing the command to Drain a DAOS target.
type poolDrainCmd struct {
poolCmd
Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be drained"`
TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be drained on the rank"`
poolRanksCmd
TargetIdx string `long:"target-idx" description:"Comma-separated list of target index(es) to be drained on each rank"`
}

// Execute is run when PoolDrainCmd subcommand is activated
func (cmd *poolDrainCmd) Execute(args []string) error {
msg := "succeeded"

var idxList []uint32
if err := common.ParseNumberList(cmd.TargetIdx, &idxList); err != nil {
err = errors.WithMessage(err, "parsing target list")
return err
return errors.WithMessage(err, "parsing target list")
}

req := &control.PoolDrainReq{
req := &control.PoolRanksReq{
ID: cmd.PoolID().String(),
Rank: ranklist.Rank(cmd.Rank),
Ranks: cmd.RankList.Ranks(),
TargetIdx: idxList,
}

err := control.PoolDrain(cmd.MustLogCtx(), cmd.ctlInvoker, req)
resp, err := control.PoolDrain(cmd.MustLogCtx(), cmd.ctlInvoker, req)

if cmd.JSONOutputEnabled() {
return cmd.OutputJSON(resp, err)
}

cmd.Debugf("%T: %+v, %T: %+v", req, req, resp, resp)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are these debug lines necessary? I think you should already have debug printing from the control library.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wasn't getting any debug for response from interceptors.go but maybe I can just move this to trace. this also prints the native types rather than proto which is what we get from the control library from what I can see


// Retrieve PoolRanksResults so we can pretty print output.
results, err := resp.GetResults(err)
if err != nil {
msg = errors.WithMessage(err, "failed").Error()
cmd.Errorf(errors.WithMessage(err, "Pool drain failed").Error())
return err
}

cmd.Infof("Drain command %s\n", msg)
cmd.Debugf("%T: %+v", results, results)

return err
var out strings.Builder
pretty.PrintPoolRankResults(&out, results)
cmd.Info(out.String())

return resp.Errors()
}

// poolExtendCmd is the struct representing the command to Extend a DAOS pool.
type poolExtendCmd struct {
poolCmd
RankList ui.RankSetFlag `long:"ranks" required:"1" description:"Comma-separated list of ranks to add to the pool"`
poolRanksCmd
}

// Execute is run when PoolExtendCmd subcommand is activated
Expand All @@ -620,35 +652,45 @@ func (cmd *poolExtendCmd) Execute(args []string) error {

// poolReintegrateCmd is the struct representing the command to Add a DAOS target.
type poolReintegrateCmd struct {
poolCmd
Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be reintegrated"`
TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be reintegrated into the rank"`
poolRanksCmd
TargetIdx string `long:"target-idx" description:"Comma-separated list of target index(es) to be reintegrated into each rank"`
}

// Execute is run when poolReintegrateCmd subcommand is activated
func (cmd *poolReintegrateCmd) Execute(args []string) error {
msg := "succeeded"

var idxList []uint32
if err := common.ParseNumberList(cmd.TargetIdx, &idxList); err != nil {
err = errors.WithMessage(err, "parsing target list")
return err
return errors.WithMessage(err, "parsing target list")
}

req := &control.PoolReintegrateReq{
req := &control.PoolRanksReq{
ID: cmd.PoolID().String(),
Rank: ranklist.Rank(cmd.Rank),
Ranks: cmd.RankList.Ranks(),
TargetIdx: idxList,
}

err := control.PoolReintegrate(cmd.MustLogCtx(), cmd.ctlInvoker, req)
resp, err := control.PoolReintegrate(cmd.MustLogCtx(), cmd.ctlInvoker, req)

if cmd.JSONOutputEnabled() {
return cmd.OutputJSON(resp, err)
}

cmd.Debugf("%T: %+v, %T: %+v", req, req, resp, resp)

// Retrieve PoolRanksResults so we can pretty print output.
results, err := resp.GetResults(err)
if err != nil {
msg = errors.WithMessage(err, "failed").Error()
cmd.Errorf(errors.WithMessage(err, "Pool reintegrate failed").Error())
return err
}

cmd.Infof("Reintegration command %s\n", msg)
cmd.Debugf("%T: %+v", results, results)

return err
var out strings.Builder
pretty.PrintPoolRankResults(&out, results)
cmd.Info(out.String())

return resp.Errors()
}

// poolQueryCmd is the struct representing the command to query a DAOS pool.
Expand Down Expand Up @@ -701,7 +743,7 @@ type poolQueryTargetsCmd struct {
poolCmd

Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be queried"`
Targets string `long:"target-idx" description:"Comma-separated list of target idx(s) to be queried"`
Targets string `long:"target-idx" description:"Comma-separated list of target index(es) to be queried"`
}

// Execute is run when PoolQueryTargetsCmd subcommand is activated
Expand Down
Loading
Loading