DAOS-16695 test: make useradd and groupadd robust

src/tests/ftest/util/exception_utils.py

@@ -1,13 +1,26 @@
 """
   (C) Copyright 2022-2023 Intel Corporation.
+  (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
 
 import os
 
-from env_modules import get_module_list, show_avail
-from general_utils import run_command
+# pylint: disable=import-error,no-name-in-module
+try:
+    from util.env_modules import get_module_list, show_avail
+except (ImportError, ModuleNotFoundError):
+    try:
+        from env_modules import get_module_list, show_avail
+    except (ImportError, ModuleNotFoundError):
+        get_module_list = None
+        show_avail = None
+
+try:
+    from util.general_utils import run_command
+except (ImportError, ModuleNotFoundError):
+    from general_utils import run_command
 
 
 class CommandFailure(Exception):

src/tests/ftest/util/launch_utils.py

@@ -1,5 +1,6 @@
 """
   (C) Copyright 2022-2024 Intel Corporation.
+  (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -17,14 +18,16 @@
 from util.collection_utils import TEST_RESULTS_DIRS, collect_test_result
 from util.data_utils import dict_extract_values, list_flatten, list_unique
 from util.environment_utils import TestEnvironment
+from util.exception_utils import CommandFailure
 from util.host_utils import HostException, HostInfo, get_local_host, get_node_set
 from util.logger_utils import LOG_FILE_FORMAT, get_file_handler
 from util.results_utils import LaunchTestName
 from util.run_utils import RunException, command_as_user, run_local, run_remote
 from util.slurm_utils import create_partition, delete_partition, show_partition
 from util.storage_utils import StorageException, StorageInfo
 from util.systemctl_utils import SystemctlFailure, create_override_config
-from util.user_utils import get_group_id, get_user_groups, groupadd, useradd, userdel
+from util.user_utils import (get_group_id, get_next_uid_gid, get_user_groups, groupadd, useradd,
+                             userdel)
 from util.yaml_utils import YamlUpdater, get_yaml_data
 
 D_TM_SHARED_MEMORY_KEY = 0x10242048
@@ -638,6 +641,16 @@ def _user_setup(self, logger, test, create=False):
         # Keep track of queried groups to avoid redundant work
         group_gid = {}
 
+        # Get the next common UID and GID amongst all clients
+        if create:
+            try:
+                next_uid, next_gid = get_next_uid_gid(logger, clients)
+            except (CommandFailure, ValueError) as error:
+                self.test_result.fail_test(logger, "Prepare", str(error), sys.exc_info())
+                return 128
+        else:
+            next_uid, next_gid = None, None
+
         # Query and optionally create all groups and users
         for _user in users:
             user, *group = _user.split(':')
@@ -646,29 +659,32 @@ def _user_setup(self, logger, test, create=False):
             # Save the group's gid
             if group and group not in group_gid:
                 try:
-                    group_gid[group] = self._query_create_group(logger, clients, group, create)
+                    group_gid[group] = self._query_create_group(
+                        logger, clients, group, create, next_gid)
+                    next_gid += 1
                 except LaunchException as error:
                     self.test_result.fail_test(logger, "Prepare", str(error), sys.exc_info())
                     return 128
 
-            gid = group_gid.get(group, None)
             try:
-                self._query_create_user(logger, clients, user, gid, create)
+                self._query_create_user(logger, clients, user, group_gid[group], create, next_uid)
+                next_uid += 1
             except LaunchException as error:
                 self.test_result.fail_test(logger, "Prepare", str(error), sys.exc_info())
                 return 128
 
         return 0
 
     @staticmethod
-    def _query_create_group(logger, hosts, group, create=False):
+    def _query_create_group(logger, hosts, group, create=False, gid=None):
         """Query and optionally create a group on remote hosts.
 
         Args:
             logger (Logger): logger for the messages produced by this method
             hosts (NodeSet): hosts on which to query and create the group
             group (str): group to query and create
             create (bool, optional): whether to create the group if non-existent
+            gid (int, optional): GID for the new group when creating. Default is None
 
         Raises:
             LaunchException: if there is an error querying or creating the group
@@ -688,7 +704,7 @@ def _query_create_group(logger, hosts, group, create=False):
 
         # Create the group
         logger.info('Creating group %s', group)
-        if not groupadd(logger, hosts, group, True, True).passed:
+        if not groupadd(logger, hosts, group, gid, True, True).passed:
             raise LaunchException(f'Error creating group {group}')
 
         # Get the group id on each node
@@ -701,7 +717,7 @@ def _query_create_group(logger, hosts, group, create=False):
         raise LaunchException(f'Group not setup correctly: {group}')
 
     @staticmethod
-    def _query_create_user(logger, hosts, user, gid=None, create=False):
+    def _query_create_user(logger, hosts, user, gid=None, create=False, uid=None):
         """Query and optionally create a user on remote hosts.
 
         Args:
@@ -710,6 +726,7 @@ def _query_create_user(logger, hosts, user, gid=None, create=False):
             user (str): user to query and create
             gid (str, optional): user's primary gid. Default is None
             create (bool, optional): whether to create the group if non-existent. Default is False
+            uid (int, optional): GID for the new group when creating. Default is None
 
         Raises:
             LaunchException: if there is an error querying or creating the user
@@ -730,7 +747,7 @@ def _query_create_user(logger, hosts, user, gid=None, create=False):
 
         logger.info('Creating user %s in group %s', user, gid)
         test_env = TestEnvironment()
-        if not useradd(logger, hosts, user, gid, test_env.user_dir, True).passed:
+        if not useradd(logger, hosts, user, gid, test_env.user_dir, uid, True).passed:
             raise LaunchException(f'Error creating user {user}')
 
     def _clear_mount_points(self, logger, test, clear_mounts):

src/tests/ftest/util/user_utils.py

@@ -1,5 +1,6 @@
 """
   (C) Copyright 2018-2024 Intel Corporation.
+  (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -12,6 +13,7 @@
 
 from ClusterShell.NodeSet import NodeSet
 # pylint: disable=import-error,no-name-in-module
+from util.exception_utils import CommandFailure
 from util.run_utils import run_remote
 
 
@@ -91,13 +93,55 @@ def getent(log, hosts, database, key, sudo=False):
     return run_remote(log, hosts, command)
 
 
-def groupadd(log, hosts, group, force=False, sudo=False):
+def get_next_uid_gid(log, hosts):
+    """Get the next common UID and GID across some hosts.
+
+    Args:
+        log (logger): logger for the messages produced by this method
+        hosts (NodeSet): hosts on which to run the command
+
+    Returns:
+        (int, int): next UID, next GID common across hosts
+
+    Raises:
+        CommandFailure: if the command fails on one or more hosts
+        ValueError: if the command output is unexpected on one or more hosts
+    """
+    command = '''
+UID_MIN=$(grep -E '^UID_MIN' /etc/login.defs | tr -s ' ' | cut -d ' ' -f 2)
+UID_MAX=$(grep -E '^UID_MAX' /etc/login.defs | tr -s ' ' | cut -d ' ' -f 2)
+GID_MIN=$(grep -E '^GID_MIN' /etc/login.defs | tr -s ' ' | cut -d ' ' -f 2)
+GID_MAX=$(grep -E '^GID_MAX' /etc/login.defs | tr -s ' ' | cut -d ' ' -f 2)
+NEXT_UID=$(cat /etc/passwd | cut -d ":" -f 3 | xargs -n 1 -I % sh -c \
+    "if [[ % -ge $UID_MIN ]] && [[ % -le $UID_MAX ]]; then echo %; fi" \
+        | sort -n | tail -n 1 | awk '{ print $1+1 }')
+NEXT_GID=$(cat /etc/group | cut -d ":" -f 3 | xargs -n 1 -I % sh -c \
+    "if [[ % -ge $GID_MIN ]] && [[ % -le $GID_MAX ]]; then echo %; fi" \
+        | sort -n | tail -n 1 | awk '{ print $1+1 }')
+echo "NEXT_UID=$NEXT_UID"
+echo "NEXT_GID=$NEXT_GID"
+'''
+    result = run_remote(log, hosts, command)
+    if not result.passed:
+        raise CommandFailure(f"Failed to get NEXT_UID and NEXT_GID on {result.failed_hosts}")
+    all_output = "\n".join(result.all_stdout.values())
+    all_uid = re.findall(r'NEXT_UID=([0-9]+)', all_output)
+    all_gid = re.findall(r'NEXT_GID=([0-9]+)', all_output)
+    if len(all_uid) != len(hosts) or len(all_gid) != len(hosts):
+        raise ValueError(f"Failed to get NEXT_UID and NEXT_GID on {hosts}")
+    max_uid = max(map(int, all_uid))
+    max_gid = max(map(int, all_gid))
+    return max_uid, max_gid
+
+
+def groupadd(log, hosts, group, gid=None, force=False, sudo=False):
     """Run groupadd remotely.
 
     Args:
         log (logger): logger for the messages produced by this method
         hosts (NodeSet): hosts on which to run the command
         group (str): the group to create
+        gid (int, optional): GID for the new group. Defaults to None
         force (bool, optional): whether to use the force option. Default is False
         sudo (bool, optional): whether to execute commands with sudo. Default is False
 
@@ -108,12 +152,13 @@ def groupadd(log, hosts, group, force=False, sudo=False):
         'sudo -n' if sudo else None,
         'groupadd',
         '-r',
+        f'-g {gid}' if gid else None,
         '-f' if force else None,
         group]))
     return run_remote(log, hosts, command)
 
 
-def useradd(log, hosts, user, group=None, parent_dir=None, sudo=False):
+def useradd(log, hosts, user, group=None, parent_dir=None, uid=None, sudo=False):
     """Run useradd remotely.
 
     Args:
@@ -122,6 +167,7 @@ def useradd(log, hosts, user, group=None, parent_dir=None, sudo=False):
         user (str): user to create
         group (str, optional): user group. Default is None
         parent_dir (str, optional): parent home directory. Default is None
+        uid (int, optional): UID for the new user. Defaults to None
         sudo (bool): whether to execute commands with sudo. Default is False
 
     Returns:
@@ -133,6 +179,7 @@ def useradd(log, hosts, user, group=None, parent_dir=None, sudo=False):
         '-m',
         f'-g {group}' if group else None,
         f'-d {os.path.join(parent_dir, user)}' if parent_dir else None,
+        f'-u {uid}' if uid else None,
         user]))
     return run_remote(log, hosts, command)