DAOS-16386 utils: Add DDB Feature and RM_POOL Support (#15062) (#15474)

This PR enhances the DDB functionality for CR purposes with
the following updates:

1. Pool Behavior Control:

Administrators can now control certain vos pool behaviors,
such as skipping vos pool loading or setting a vos pool to immutable  mode.

2. Manual Pool Shard Removal:

A new command ddb rm_pool <vos_pool> has been introduced,
allowing administrators to manually remove pool shards.

3. SPDK Environment Initialization Bug Fix:

Fixed an issue where spdk_env_init() would fail during reinitialization.

These updates aim to improve system flexibility and stability,
providing administrators with more robust management capabilities.

Signed-off-by: Wang Shilong <[email protected]>

src/bio/bio_xstream.c

@@ -20,6 +20,7 @@
 #include <spdk/blob_bdev.h>
 #include <spdk/blob.h>
 #include <spdk/rpc.h>
+#include <spdk/env_dpdk.h>
 #include "bio_internal.h"
 #include <daos_srv/smd.h>
 
@@ -172,7 +173,8 @@ bio_spdk_env_init(void)
 		}
 	}
 
-	rc = spdk_env_init(&opts);
+	/* Don't pass opt for reinitialization, otherwise it will fail */
+	rc = spdk_env_init(spdk_env_dpdk_external_init() ? &opts : NULL);
 	if (rc != 0) {
 		rc = -DER_INVAL; /* spdk_env_init() returns -1 */
 		D_ERROR("Failed to initialize SPDK env, "DF_RC"\n", DP_RC(rc));

src/control/cmd/ddb/command_completers.go

@@ -1,5 +1,5 @@
 //
-// (C) Copyright 2022 Intel Corporation.
+// (C) Copyright 2022-2024 Intel Corporation.
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -16,7 +16,7 @@ const (
 	defMntPrefix = "/mnt"
 )
 
-func listDir(match string) (result []string) {
+func listDirVos(match string) (result []string) {
 	if strings.HasSuffix(match, "vos-") {
 		match = filepath.Dir(match)
 	}
@@ -33,9 +33,9 @@ func listDir(match string) (result []string) {
 	return
 }
 
-func openCompleter(prefix string, args []string) []string {
-	suggestions := []string{"-h", "-w", "--write_mode"}
-	suggestions = append(suggestions, listDir(defMntPrefix)...)
+func filterSuggestions(prefix string, initialSuggestions, additionalSuggestions []string) []string {
+	suggestions := append([]string{}, initialSuggestions...)
+	suggestions = append(suggestions, additionalSuggestions...)
 
 	if len(prefix) > 0 {
 		var newSuggestions []string
@@ -49,5 +49,32 @@ func openCompleter(prefix string, args []string) []string {
 	}
 
 	return suggestions
+}
+
+func openCompleter(prefix string, args []string) []string {
+	return filterSuggestions(prefix, []string{"-h", "-w", "--write_mode"}, listDirVos(defMntPrefix))
+}
+
+func featureCompleter(prefix string, args []string) []string {
+	return filterSuggestions(prefix, []string{"-h", "-e", "--enable", "-d", "--disable", "-s", "--show"},
+		listDirVos(defMntPrefix))
+}
+
+func listDirPool(match string) (result []string) {
+	if strings.HasSuffix(match, "vos-") {
+		match = filepath.Dir(match)
+	}
+	filepath.Walk(match, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			/* ignore error */
+			return nil
+		}
+		result = append(result, path)
+		return nil
+	})
+	return
+}
 
+func rmPoolCompleter(prefix string, args []string) []string {
+	return filterSuggestions(prefix, []string{"-h"}, listDirPool(defMntPrefix))
 }

src/control/cmd/ddb/commands_wrapper.go

@@ -1,5 +1,5 @@
 //
-// (C) Copyright 2022-2023 Intel Corporation.
+// (C) Copyright 2022-2024 Intel Corporation.
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -223,3 +223,36 @@ func ddbDtxActAbort(ctx *DdbContext, path string, dtx_id string) error {
 	/* Run the c code command */
 	return daosError(C.ddb_run_dtx_act_abort(&ctx.ctx, &options))
 }
+
+func ddbFeature(ctx *DdbContext, path, enable, disable string, show bool) error {
+	/* Set up the options */
+	options := C.struct_feature_options{}
+	options.path = C.CString(path)
+	defer freeString(options.path)
+	if enable != "" {
+		err := daosError(C.ddb_feature_string2flags(&ctx.ctx, C.CString(enable),
+			&options.set_compat_flags, &options.set_incompat_flags))
+		if err != nil {
+			return err
+		}
+	}
+	if disable != "" {
+		err := daosError(C.ddb_feature_string2flags(&ctx.ctx, C.CString(disable),
+			&options.clear_compat_flags, &options.clear_incompat_flags))
+		if err != nil {
+			return err
+		}
+	}
+	options.show_features = C.bool(show)
+	/* Run the c code command */
+	return daosError(C.ddb_run_feature(&ctx.ctx, &options))
+}
+
+func ddbRmPool(ctx *DdbContext, path string) error {
+	/* Set up the options */
+	options := C.struct_rm_pool_options{}
+	options.path = C.CString(path)
+	defer freeString(options.path)
+	/* Run the c code command */
+	return daosError(C.ddb_run_rm_pool(&ctx.ctx, &options))
+}

src/control/cmd/ddb/ddb_commands.go

@@ -1,5 +1,5 @@
 //
-// (C) Copyright 2022-2023 Intel Corporation.
+// (C) Copyright 2022-2024 Intel Corporation.
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -297,4 +297,39 @@ the path must include the extent, otherwise, it must not.`,
 		},
 		Completer: nil,
 	})
+	// Command: feature
+	app.AddCommand(&grumble.Command{
+		Name:      "feature",
+		Aliases:   nil,
+		Help:      "Manage vos pool features",
+		LongHelp:  "",
+		HelpGroup: "vos",
+		Flags: func(f *grumble.Flags) {
+			f.String("e", "enable", "", "Enable vos pool features")
+			f.String("d", "disable", "", "Disable vos pool features")
+			f.Bool("s", "show", false, "Show current features")
+		},
+		Args: func(a *grumble.Args) {
+			a.String("path", "Optional, Path to the vos file", grumble.Default(""))
+		},
+		Run: func(c *grumble.Context) error {
+			return ddbFeature(ctx, c.Args.String("path"), c.Flags.String("enable"), c.Flags.String("disable"), c.Flags.Bool("show"))
+		},
+		Completer: featureCompleter,
+	})
+	// Command: rm_pool
+	app.AddCommand(&grumble.Command{
+		Name:      "rm_pool",
+		Aliases:   nil,
+		Help:      "Remove a vos pool.",
+		LongHelp:  "",
+		HelpGroup: "vos",
+		Args: func(a *grumble.Args) {
+			a.String("path", "Optional, Path to the vos file", grumble.Default(""))
+		},
+		Run: func(c *grumble.Context) error {
+			return ddbRmPool(ctx, c.Args.String("path"))
+		},
+		Completer: rmPoolCompleter,
+	})
 }

src/control/cmd/ddb/main.go

@@ -1,5 +1,5 @@
 //
-// (C) Copyright 2022-2023 Intel Corporation.
+// (C) Copyright 2022-2024 Intel Corporation.
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -14,6 +14,7 @@ import (
 	"runtime/debug"
 	"sort"
 	"strings"
+	"unsafe"
 
 	"github.com/desertbit/grumble"
 	"github.com/jessevdk/go-flags"
@@ -24,6 +25,11 @@ import (
 	"github.com/daos-stack/daos/src/control/logging"
 )
 
+/*
+ #include <stdlib.h>
+*/
+import "C"
+
 func exitWithError(log logging.Logger, err error) {
 	cmdName := path.Base(os.Args[0])
 	log.Errorf("%s: %v", cmdName, err)
@@ -51,7 +57,7 @@ func (pathStr vosPathStr) Complete(match string) (comps []flags.Completion) {
 	if match == "" || match == "/" {
 		match = defMntPrefix
 	}
-	for _, comp := range listDir(match) {
+	for _, comp := range listDirVos(match) {
 		comps = append(comps, flags.Completion{Item: comp})
 	}
 	sort.Slice(comps, func(i, j int) bool { return comps[i].Item < comps[j].Item })
@@ -176,22 +182,28 @@ Example Paths:
 	app := createGrumbleApp(ctx)
 
 	if opts.Args.VosPath != "" {
-		log.Debugf("Connect to path: %s\n", opts.Args.VosPath)
-		if err := ddbOpen(ctx, string(opts.Args.VosPath), opts.WriteMode); err != nil {
-			return errors.Wrapf(err, "Error opening path: %s", opts.Args.VosPath)
+		if !strings.HasPrefix(string(opts.Args.RunCmd), "feature") && !strings.HasPrefix(string(opts.Args.RunCmd), "rm_pool") {
+			log.Debugf("Connect to path: %s\n", opts.Args.VosPath)
+			if err := ddbOpen(ctx, string(opts.Args.VosPath), opts.WriteMode); err != nil {
+				return errors.Wrapf(err, "Error opening path: %s", opts.Args.VosPath)
+			}
 		}
 	}
 
 	if opts.Args.RunCmd != "" && opts.CmdFile != "" {
 		return errors.New("Cannot use both command file and a command string")
 	}
 
+	if opts.Args.VosPath != "" {
+		ctx.ctx.dc_pool_path = C.CString(string(opts.Args.VosPath))
+		defer C.free(unsafe.Pointer(ctx.ctx.dc_pool_path))
+	}
 	if opts.Args.RunCmd != "" || opts.CmdFile != "" {
 		// Non-interactive mode
 		if opts.Args.RunCmd != "" {
 			err := runCmdStr(app, string(opts.Args.RunCmd), opts.Args.RunCmdArgs...)
 			if err != nil {
-				log.Errorf("Error running command %s\n", string(opts.Args.RunCmd))
+				log.Errorf("Error running command %q %s\n", string(opts.Args.RunCmd), err)
 			}
 		} else {
 			err := runFileCmds(log, app, opts.CmdFile)

src/dtx/dtx_resync.c

@@ -612,6 +612,12 @@ dtx_resync(daos_handle_t po_hdl, uuid_t po_uuid, uuid_t co_uuid, uint32_t ver, b
 	crt_group_rank(NULL, &myrank);
 
 	pool = cont->sc_pool->spc_pool;
+	if (pool->sp_disable_dtx_resync) {
+		D_DEBUG(DB_MD, "Skip DTX resync (%s) for " DF_UUID "/" DF_UUID " with ver %u\n",
+			block ? "block" : "non-block", DP_UUID(po_uuid), DP_UUID(co_uuid), ver);
+		goto out;
+	}
+
 	ABT_rwlock_rdlock(pool->sp_lock);
 	rc = pool_map_find_target_by_rank_idx(pool->sp_map, myrank,
 					      dss_get_module_info()->dmi_tgt_id, &target);

src/include/daos_srv/pool.h

@@ -82,12 +82,8 @@ struct ds_pool {
 	 */
 	uuid_t			sp_srv_cont_hdl;
 	uuid_t			sp_srv_pool_hdl;
-	uint32_t		sp_stopping:1,
-				sp_cr_checked:1,
-				sp_immutable:1,
-				sp_fetch_hdls:1,
-				sp_need_discard:1,
-				sp_disable_rebuild:1;
+	uint32_t sp_stopping : 1, sp_cr_checked : 1, sp_immutable : 1, sp_fetch_hdls : 1,
+	    sp_need_discard : 1, sp_disable_rebuild : 1, sp_disable_dtx_resync : 1;
 	/* pool_uuid + map version + leader term + rebuild generation define a
 	 * rebuild job.
 	 */

src/include/daos_srv/vos.h

@@ -20,6 +20,56 @@
 #include <daos_srv/dtx_srv.h>
 #include <daos_srv/vos_types.h>
 
+#define VOS_POOL_COMPAT_FLAG_IMMUTABLE       (1ULL << 0)
+#define VOS_POOL_COMPAT_FLAG_SKIP_START      (1ULL << 1)
+#define VOS_POOL_COMPAT_FLAG_SKIP_REBUILD    (1ULL << 2)
+#define VOS_POOL_COMPAT_FLAG_SKIP_DTX_RESYNC (1ULL << 3)
+
+#define VOS_POOL_COMPAT_FLAG_SUPP                                                                  \
+	(VOS_POOL_COMPAT_FLAG_IMMUTABLE | VOS_POOL_COMPAT_FLAG_SKIP_START |                        \
+	 VOS_POOL_COMPAT_FLAG_SKIP_REBUILD | VOS_POOL_COMPAT_FLAG_SKIP_DTX_RESYNC)
+
+#define VOS_POOL_INCOMPAT_FLAG_SUPP 0
+
+#define VOS_MAX_FLAG_NAME_LEN       32
+
+static inline uint64_t
+vos_pool_name2flag(const char *name, bool *compat_feature)
+{
+	*compat_feature = false;
+
+	if (strncmp(name, "immutable", VOS_MAX_FLAG_NAME_LEN) == 0) {
+		*compat_feature = true;
+		return VOS_POOL_COMPAT_FLAG_IMMUTABLE;
+	}
+	if (strncmp(name, "skip_start", VOS_MAX_FLAG_NAME_LEN) == 0) {
+		*compat_feature = true;
+		return VOS_POOL_COMPAT_FLAG_SKIP_START;
+	}
+	if (strncmp(name, "skip_rebuild", VOS_MAX_FLAG_NAME_LEN) == 0) {
+		*compat_feature = true;
+		return VOS_POOL_COMPAT_FLAG_SKIP_REBUILD;
+	}
+	if (strncmp(name, "skip_dtx_resync", VOS_MAX_FLAG_NAME_LEN) == 0) {
+		*compat_feature = true;
+		return VOS_POOL_COMPAT_FLAG_SKIP_DTX_RESYNC;
+	}
+
+	return 0;
+}
+
+bool
+vos_pool_feature_skip_start(daos_handle_t poh);
+
+bool
+vos_pool_feature_immutable(daos_handle_t poh);
+
+bool
+vos_pool_feature_skip_rebuild(daos_handle_t poh);
+
+bool
+vos_pool_feature_skip_dtx_resync(daos_handle_t poh);
+
 /** Initialize the vos reserve/cancel related fields in dtx handle
  *
  * \param dth	[IN]	The dtx handle

src/include/daos_srv/vos_types.h

@@ -78,19 +78,21 @@ D_CASSERT(sizeof(struct dtx_entry) ==
 /** Pool open flags (for vos_pool_create and vos_pool_open) */
 enum vos_pool_open_flags {
 	/** Pool is small (for sys space reservation); implies VOS_POF_EXCL */
-	VOS_POF_SMALL	= (1 << 0),
+	VOS_POF_SMALL = (1 << 0),
 	/** Exclusive (-DER_BUSY if already opened) */
-	VOS_POF_EXCL	= (1 << 1),
+	VOS_POF_EXCL = (1 << 1),
 	/** Ignore the pool uuid passed into vos_pool_open */
 	VOS_POF_SKIP_UUID_CHECK = (1 << 2),
 	/** Caller does VEA flush periodically */
-	VOS_POF_EXTERNAL_FLUSH	= (1 << 3),
+	VOS_POF_EXTERNAL_FLUSH = (1 << 3),
 	/** RDB pool */
-	VOS_POF_RDB	= (1 << 4),
+	VOS_POF_RDB = (1 << 4),
 	/** SYS DB pool */
-	VOS_POF_SYSDB	= (1 << 5),
+	VOS_POF_SYSDB = (1 << 5),
 	/** Open the pool for daos check query, that will bypass EXEL flags. */
 	VOS_POF_FOR_CHECK_QUERY = (1 << 6),
+	/** Open the pool for feature fetch/update, that will skip VEA load */
+	VOS_POF_FOR_FEATURE_FLAG = (1 << 7),
 };
 
 enum vos_oi_attr {

src/pool/srv_target.c

@@ -505,6 +505,27 @@ pool_child_start(struct ds_pool_child *child, bool recreate)
 		goto done;
 	}
 
+	if (!ds_pool_skip_for_check(child->spc_pool) &&
+	    vos_pool_feature_skip_start(child->spc_hdl)) {
+		D_INFO(DF_UUID ": skipped to start\n", DP_UUID(child->spc_uuid));
+		rc = -DER_SHUTDOWN;
+		goto out_close;
+	}
+
+	if (vos_pool_feature_immutable(child->spc_hdl))
+		child->spc_pool->sp_immutable = 1;
+
+	/*
+	 * Rebuild depends on DTX resync, if DTX resync is skipped,
+	 * then rebuild also needs to be skipped.
+	 */
+	if (vos_pool_feature_skip_rebuild(child->spc_hdl) ||
+	    vos_pool_feature_skip_dtx_resync(child->spc_hdl))
+		child->spc_pool->sp_disable_rebuild = 1;
+
+	if (vos_pool_feature_skip_dtx_resync(child->spc_hdl))
+		child->spc_pool->sp_disable_dtx_resync = 1;
+
 	if (!ds_pool_restricted(child->spc_pool, false)) {
 		rc = start_gc_ult(child);
 		if (rc != 0)