DAOS-623 test: dynamically find hdf5 plugin path

Dynamically find the hdf5 plugin path instead of hardcoding the path for
every test, since it can come from a different path depending on the
system and install location.

Test-tag: MacsioTest
TODO-Test-tag: BasicCheckout BasicCheckoutDm DowngradeTest IoSysAdmin IorSmall MacsioTest SoakSmoke ior LargeFileCount SmallFileCount
Skip-unit-tests: true
Skip-fault-injection-test: true

Required-githooks: true

Signed-off-by: Dalton Bohning <[email protected]>

src/tests/ftest/deployment/basic_checkout.yaml

@@ -159,4 +159,4 @@ mdtest:
 dfuse:
   disable_caching: true
 hdf5_vol:
-  plugin_path: /usr/lib64/mpich/lib
+  plugin_name: libhdf5_vol_daos.so

src/tests/ftest/deployment/io_sys_admin.yaml

@@ -103,7 +103,7 @@ dcp:
   client_processes:
     np: 16
 hdf5_vol:
-  plugin_path: /usr/lib64/mpich/lib
+  plugin_name: libhdf5_vol_daos.so
 
 io_sys_admin:
   steps_to_run:

src/tests/ftest/interoperability/diff_versions.yaml

@@ -37,7 +37,7 @@ ior:
     write_flg: "-w -W -k -G 1 -i 1"
     read_flg: "-C -k -e -r -R -g -G 1 -Q 1 -vv"
 hdf5_vol:
-  plugin_path: "/usr/lib64/mpich/lib"
+  plugin_name: libhdf5_vol_daos.so
 interop:
   # Example of upgrade/downgrade RPMs from local tar file
   # upgrade_rpms: ["/home/dinghwah/RPM/2.1.104/daos-2.1.104-1.el8.x86_64.rpm"]

src/tests/ftest/interoperability/down_grade.yaml

@@ -37,7 +37,7 @@ ior:
     write_flg: "-w -W -k -G 1 -i 1"
     read_flg: "-C -k -e -r -R -g -G 1 -Q 1 -vv"
 hdf5_vol:
-  plugin_path: "/usr/lib64/mpich/lib"
+  plugin_name: libhdf5_vol_daos.so
 interop:
   # Example of upgrade/downgrade RPMs from local tar file
   # upgrade_rpms: ["/home/dinghwah/RPM/2.1.104/daos-2.1.104-1.el8.x86_64.rpm"]

src/tests/ftest/interoperability/updown_grade.yaml

@@ -37,7 +37,7 @@ ior:
     write_flg: "-w -W -k -G 1 -i 1"
     read_flg: "-C -k -e -r -R -g -G 1 -Q 1 -vv"
 hdf5_vol:
-  plugin_path: "/usr/lib64/mpich/lib"
+  plugin_name: libhdf5_vol_daos.so
 interop:
   # Example of upgrade/downgrade RPMs from local tar file
   # upgrade_rpms: ["/home/dinghwah/RPM/2.1.104/daos-2.1.104-1.el8.x86_64.rpm"]

src/tests/ftest/interoperability/updown_grade_8svr.yaml

@@ -37,7 +37,7 @@ ior:
     write_flg: "-w -W -k -G 1 -i 1"
     read_flg: "-C -k -e -r -R -g -G 1 -Q 1 -vv"
 hdf5_vol:
-  plugin_path: "/usr/lib64/mpich/lib"
+  plugin_name: libhdf5_vol_daos.so
 interop:
   # Example of upgrade/downgrade RPMs from local tar file
   # upgrade_rpms: ["/home/dinghwah/RPM/2.1.104/daos-2.1.104-1.el8.x86_64.rpm"]

src/tests/ftest/interoperability/upgrade_downgrade_base.py

@@ -1,5 +1,5 @@
 '''
-  (C) Copyright 2023 Intel Corporation.
+  (C) Copyright 2023-2024 Intel Corporation.
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 '''
@@ -11,7 +11,7 @@
 
 from agent_utils import include_local_host
 from command_utils_base import CommandFailure
-from general_utils import get_random_bytes, pcmd, run_pcmd
+from general_utils import find_library, get_random_bytes, pcmd, run_pcmd
 from ior_test_base import IorTestBase
 from pydaos.raw import DaosApiError
 
@@ -574,7 +574,7 @@ def upgrade_and_downgrade(self, fault_on_pool_upgrade=False):
         # (3.b)ior hdf5
         elif ior_api == "HDF5":
             self.log.info("(3.b)==Run IOR HDF5 write and read.")
-            hdf5_plugin_path = self.params.get("plugin_path", '/run/hdf5_vol/')
+            hdf5_plugin_path = find_library(self.params.get("plugin_name", '/run/hdf5_vol/'))
             self.ior_cmd.flags.update(iorflags_write)
             self.run_ior_with_pool(
                 plugin_path=hdf5_plugin_path, mount_dir=mount_dir,

src/tests/ftest/io/macsio_test.py

@@ -7,7 +7,7 @@
 from apricot import TestWithServers
 from command_utils_base import CommandFailure
 from dfuse_utils import get_dfuse, start_dfuse
-from general_utils import get_log_file, list_to_str
+from general_utils import find_library, get_log_file, list_to_str
 from job_manager_utils import get_job_manager
 from macsio_util import MacsioCommand
 
@@ -44,13 +44,14 @@ def _get_macsio_command(self, pool, cont):
 
         return macsio
 
-    def run_macsio(self, macsio, hosts, processes, plugin=None, slots=None, working_dir=None):
+    def run_macsio(self, job_manager, macsio, hosts, processes, plugin=None, slots=None, working_dir=None):
         """Run the macsio test.
 
         Parameters for the macsio command are obtained from the test yaml file,
         including the path to the macsio executable.
 
         Args:
+            job_manager (JobManager): Orterun, Mpirun, Srun, etc.
             macsio (MacsioCommand): object defining the macsio command
             hosts (NodeSet): hosts on which to run macsio
             processes (int): total number of processes to use to run macsio
@@ -68,7 +69,6 @@ def run_macsio(self, macsio, hosts, processes, plugin=None, slots=None, working_
             # Include DAOS VOL environment settings
             env["HDF5_VOL_CONNECTOR"] = "daos"
             env["HDF5_PLUGIN_PATH"] = str(plugin)
-        job_manager = get_job_manager(self)
         job_manager.job = macsio
         job_manager.assign_hosts(hosts, self.workdir, slots)
         job_manager.assign_processes(processes)
@@ -115,7 +115,8 @@ def test_macsio(self):
         # Run macsio
         self.log_step("Running MACSio")
         macsio = self._get_macsio_command(pool, container)
-        result = self.run_macsio(macsio, self.hostlist_clients, processes)
+        job_manager = get_job_manager(self)
+        result = self.run_macsio(job_manager, macsio, self.hostlist_clients, processes)
         if not macsio.check_results(result, self.hostlist_clients):
             self.fail("MACSio failed")
         self.log.info("Test passed")
@@ -136,7 +137,12 @@ def test_macsio_daos_vol(self):
         :avocado: tags=MacsioTest,test_macsio_daos_vol
         :avocado: tags=DAOS_5610
         """
-        plugin_path = self.params.get("plugin_path", "/run/job_manager/*")
+        # Setup job_manager and find plugin_path
+        job_manager = get_job_manager(self)
+        plugin_name = self.params.get("plugin_name", "/run/job_manager/*")
+        plugin_path = find_library(plugin_name)
+        if not plugin_path:
+            self.fail(f"Failed to find {plugin_name}")
         processes = self.params.get("processes", "/run/macsio/*", len(self.hostlist_clients))
 
         # Create a pool
@@ -160,7 +166,7 @@ def test_macsio_daos_vol(self):
         self.log_step("Running MACSio with DAOS VOL connector")
         macsio = self._get_macsio_command(pool, container)
         result = self.run_macsio(
-            macsio, self.hostlist_clients, processes, plugin_path,
+            job_manager, macsio, self.hostlist_clients, processes, plugin_path,
             working_dir=dfuse.mount_dir.value)
         if not macsio.check_results(result, self.hostlist_clients):
             self.fail("MACSio failed")

src/tests/ftest/io/macsio_test.yaml

@@ -41,15 +41,15 @@ job_manager: !mux
     class_name: Mpirun
     mpi_type: mpich
     macsio_path: /usr/lib64/mpich/bin
-    plugin_path: /usr/lib64/mpich/lib
+    plugin_name: libhdf5_vol_daos.so
     timeout:
       test_macsio: 10
       test_macsio_daos_vol: 20
   openmpi:
     class_name: Orterun
     mpi_type: openmpi
     macsio_path: /usr/lib64/openmpi3/bin
-    plugin_path: /usr/lib64/openmpi3/lib
+    plugin_name: libhdf5_vol_daos.so
     timeout:
       test_macsio: 10
       test_macsio_daos_vol: 20

src/tests/ftest/ior/small.yaml

@@ -65,4 +65,4 @@ ior:
 dfuse:
   disable_caching: true
 hdf5_vol:
-  plugin_path: /usr/lib64/mpich/lib
+  plugin_name: libhdf5_vol_daos.so

src/tests/ftest/soak/faults.yaml

@@ -123,7 +123,7 @@ ior_faults:
     mount_dir: "/tmp/soak_dfuse_ior/"
     disable_caching: true
 hdf5_vol:
-  plugin_path: "/usr/lib64/mpich/lib"
+  plugin_name: libhdf5_vol_daos.so
 events:
   - "mce: [Hardware Error]: Machine check events logged"
   - "Package temperature above threshold"

src/tests/ftest/soak/harassers.yaml

@@ -210,7 +210,7 @@ mdtest_harasser:
     mount_dir: "/tmp/soak_dfuse_mdtest/"
     disable_caching: true
 hdf5_vol:
-  plugin_path: "/usr/lib64/mpich/lib"
+  plugin_name: libhdf5_vol_daos.so
 events:
   - "mce: [Hardware Error]: Machine check events logged"
   - "Package temperature above threshold"

src/tests/ftest/soak/smoke.yaml

@@ -289,7 +289,7 @@ datamover_smoke:
     test_file: "daos:/testFile"
     dfs_destroy: false
 hdf5_vol:
-  plugin_path: "/usr/lib64/mpich/lib"
+  plugin_name: libhdf5_vol_daos.so
 events:
   - "mce: [Hardware Error]: Machine check events logged"
   - "Package temperature above threshold"

src/tests/ftest/soak/soak-extra-suse.yaml

@@ -1,3 +1,3 @@
 hdf5_vol:
-  plugin_path: /usr/lib64/mpi/gcc/mpich/lib
+  plugin_name: libhdf5_vol_daos.so
 mpi_module: "gnu-mpich"

src/tests/ftest/soak/stress.yaml

@@ -329,7 +329,7 @@ datamover_stress:
     test_file: "daos:/testFile"
     dfs_destroy: false
 hdf5_vol:
-  plugin_path: "/usr/lib64/mpich/lib"
+  plugin_name: libhdf5_vol_daos.so
 events:
   - "mce: [Hardware Error]: Machine check events logged"
   - "Package temperature above threshold"

src/tests/ftest/util/file_count_test_base.py

@@ -6,6 +6,7 @@
 import os
 
 from avocado.core.exceptions import TestFail
+from general_utils import find_library
 from ior_test_base import IorTestBase
 from mdtest_test_base import MdtestBase
 from oclass_utils import extract_redundancy_factor
@@ -68,7 +69,10 @@ def run_file_count(self):
         results = []
         dir_oclass = None
         apis = self.params.get("api", "/run/largefilecount/*")
-        hdf5_plugin_path = self.params.get("plugin_path", '/run/hdf5_vol/*')
+        hdf5_plugin_name = self.params.get("plugin_name", '/run/hdf5_vol/*')
+        hdf5_plugin_path = find_library(hdf5_plugin_name)
+        if not hdf5_plugin_path:
+            self.fail(f"Failed to find {hdf5_plugin_name}")
         ior_np = self.params.get("np", '/run/ior/client_processes/*', 1)
         ior_ppn = self.params.get("ppn", '/run/ior/client_processes/*', None)
         mdtest_np = self.params.get("np", '/run/mdtest/client_processes/*', 1)

src/tests/ftest/util/general_utils.py

@@ -1207,3 +1207,31 @@ def check_ssh(log, hosts, cmd_timeout=60, verbose=True):
     """
     result = run_remote(log, hosts, "uname", timeout=cmd_timeout, verbose=verbose)
     return result.passed
+
+
+def find_library(name):
+    """Find a library by a given name.
+
+    In order of preference, searches in
+      LD_LIBRARY_PATH
+      MPI_LIB
+      /usr/lib
+      /usr/lib64
+
+    Args:
+        name (str): library name to find
+
+    Returns:
+        str: directory path containing the library. None if not found
+    """
+    paths = []
+    for env_name in ("LD_LIBRARY_PATH", "MPI_LIB"):
+        env_val = os.environ.get(env_name, None)
+        if env_val is not None:
+            paths.extend(env_val.split(":"))
+    paths.append(os.path.join(os.sep, "usr", "lib"))
+    paths.append(os.path.join(os.sep, "usr", "lib64"))
+    for path in paths:
+        if os.path.exists(os.path.join(path, name)):
+            return path
+    return None

src/tests/ftest/util/ior_test_base.py

@@ -8,7 +8,7 @@
 from apricot import TestWithServers
 from dfuse_utils import get_dfuse, start_dfuse
 from exception_utils import CommandFailure
-from general_utils import get_random_string
+from general_utils import find_library, get_random_string
 from host_utils import get_local_host
 from ior_utils import IorCommand
 from job_manager_utils import get_job_manager
@@ -316,7 +316,11 @@ def run_ior_multiple_variants(self, obj_class, apis, transfer_block_size, flags)
                 flags_w = flags[0]
                 if api == "HDF5-VOL":
                     api = "HDF5"
-                    hdf5_plugin_path = self.params.get("plugin_path", '/run/hdf5_vol/*')
+                    hdf5_plugin_name = self.params.get("plugin_name", '/run/hdf5_vol/*')
+                    hdf5_plugin_path = find_library(hdf5_plugin_name)
+                    if not hdf5_plugin_path:
+                        results.append(["FAIL", f"Failed to find {hdf5_plugin_name}"])
+                        continue
                     flags_w += " -k"
                 elif api == "POSIX+IL":
                     api = "POSIX"

src/tests/ftest/util/soak_utils.py

@@ -27,9 +27,9 @@
 from duns_utils import format_path
 from exception_utils import CommandFailure
 from fio_utils import FioCommand
-from general_utils import (DaosTestError, check_ping, check_ssh, get_journalctl, get_log_file,
-                           get_random_bytes, get_random_string, list_to_str, pcmd, run_command,
-                           run_pcmd, wait_for_result)
+from general_utils import (DaosTestError, check_ping, check_ssh, find_library, get_host_data,
+                           get_journalctl, get_log_file, get_random_bytes, get_random_string,
+                           list_to_str, pcmd, run_command, run_pcmd, wait_for_result)
 from ior_utils import IorCommand
 from job_manager_utils import Mpirun
 from macsio_util import MacsioCommand
@@ -1122,6 +1122,9 @@ def create_ior_cmdline(self, job_spec, pool, ppn, nodesperjob, oclass_list=None,
     if not oclass_list:
         oclass_list = self.params.get("dfs_oclass", ior_params)
     plugin_path = self.params.get("plugin_path", "/run/hdf5_vol/")
+    plugin_name = self.params.get("plugin_name", "/run/hdf5_vol/")
+    if plugin_name and not plugin_path:
+        plugin_path = find_library(plugin_name)
     # update IOR cmdline for each additional IOR obj
     for api in api_list:
         if not self.enable_il and api in ["POSIX-LIBIOIL", "POSIX-LIBPIL4DFS"]:
@@ -1222,6 +1225,9 @@ def create_macsio_cmdline(self, job_spec, pool, ppn, nodesperjob):
     oclass_list = self.params.get("oclass", macsio_params)
     api_list = self.params.get("api", macsio_params)
     plugin_path = self.params.get("plugin_path", "/run/hdf5_vol/")
+    plugin_name = self.params.get("plugin_name", "/run/hdf5_vol/")
+    if plugin_name and not plugin_path:
+        plugin_path = find_library(plugin_name)
     # update macsio cmdline for each additional MACsio obj
     for api in api_list:
         for file_oclass, dir_oclass in oclass_list: