DAOS-16501 build: Add libsanitize

Jenkinsfile

@@ -2,7 +2,7 @@
 /* groovylint-disable-next-line LineLength */
 /* groovylint-disable DuplicateMapLiteral, DuplicateNumberLiteral */
 /* groovylint-disable DuplicateStringLiteral, NestedBlockDepth, VariableName */
 /* Copyright 2019-2024 Intel Corporation
  * Copyright 2025 Hewlett Packard Enterprise Development LP
  * All rights reserved.
  *
@@ -137,6 +137,15 @@
     job_status_update()
 }
 
+String sconsArgs() {
+    if (!params.CI_SCONS_ARGS) {
+        return sconsFaultsArgs()
+    }
+
+    println("Compiling DAOS with libasan")
+    return sconsFaultsArgs() + ' ' + params.CI_SCONS_ARGS
+}
+
 /**
  * Update default commit pragmas based on files modified.
  */
@@ -161,7 +170,7 @@
         TEST_RPMS = cachedCommitPragma(pragma: 'RPM-test', def_val: 'true')
         COVFN_DISABLED = cachedCommitPragma(pragma: 'Skip-fnbullseye', def_val: 'true')
         REPO_FILE_URL = repoFileUrl(env.REPO_FILE_URL)
-        SCONS_FAULTS_ARGS = sconsFaultsArgs()
+        SCONS_FAULTS_ARGS = sconsArgs()
     }
 
     options {
@@ -331,6 +340,9 @@
         string(name: 'CI_BUILD_DESCRIPTION',
                defaultValue: '',
                description: 'A description of the build')
+        string(name: 'CI_SCONS_ARGS',
+               defaultValue: '',
+               description: 'Arguments for scons when building DAOS')
     }
 
     stages {
@@ -612,7 +624,7 @@
                                        stash_files: 'ci/test_files_to_stash.txt',
                                        build_deps: 'no',
                                        stash_opt: true,
-                                       scons_args: sconsFaultsArgs() +
+                                       scons_args: sconsArgs() +
                                                   ' PREFIX=/opt/daos TARGET_TYPE=release'))
                     }
                     post {

ci/provisioning/post_provision_config.sh

@@ -62,6 +62,7 @@ if ! retry_cmd 2400 clush -B -S -l root -w "$NODESTRING" \
            REPO_PATH=\"${REPO_PATH:-}\"
            ARTIFACTS_URL=\"${ARTIFACTS_URL:-}\"
            COVFN_DISABLED=\"${COVFN_DISABLED:-true}\"
+           CI_SCONS_ARGS=\"${CI_SCONS_ARGS:-}\"
            $(cat ci/stacktrace.sh)
            $(cat ci/junit.sh)
            $(cat ci/provisioning/post_provision_config_common_functions.sh)

ci/unit/required_packages.sh

@@ -16,6 +16,7 @@ pkgs="argobots                         \
       fuse3-libs                       \
       gotestsum                        \
       hwloc-devel                      \
+      libasan                          \
       libipmctl-devel                  \
       libisa-l-devel                   \
       libfabric-devel                  \

debian/changelog

@@ -1,3 +1,9 @@
+daos (2.7.101-4) unstable; urgency=medium
+  [ Cedric Koch-Hofer]
+  * Add support of the libasan
+
+ -- Cedric Koch-Hofer <[email protected]>  Mon, 20 Jan 2025 14:12:00 -0700
+
 daos (2.7.101-3) unstable; urgency=medium
   [ Jeff Olivier ]
   * Switch from libfuse3 to libfused

debian/control

@@ -33,7 +33,8 @@ Build-Depends: debhelper (>= 10),
                liblz4-dev,
                libaio-dev,
                libcapstone-dev,
-               libpci-dev
+               libpci-dev,
+               libasan5
 Standards-Version: 4.1.2
 Homepage: https://docs.daos.io/
 Vcs-Git: https://github.com/daos-stack/daos.git

docs/dev/development.md

@@ -122,6 +122,57 @@ scons reduces the clutter from compiler setup.
 Additionally, the tool supports options to filter by directory and file names and specify a lower
 bound value to report.
 
+### Address Sanitizer
+
+To debug memory corruptions, leaks, and other issues, DAOS can be compiled with the
+[AddressSanitizer (ASan)](https://github.com/google/sanitizers/wiki/AddressSanitizer) library.
+
+#### Building with ASan
+
+To build instrumented libraries, executables, RPMs, etc., use the `SANITIZERS=address` flag with the
+`scons` command.  This option is also managed by the CI system through a Jenkins configuration
+variable.
+
+ASan is supported with both Clang and GCC compilers. However, some compiler-specific configurations,
+such as maximum function stack size, may lead to unexpected behavior in the instrumented
+binaries.
+
+#### Customizing ASan Behavior
+ASan behavior can be configured using the `ASAN_OPTIONS` environment variable. For example, you can
+add the following entry to the `env_vars` section of the `daos_server.yml` configuration file:
+```yaml
+engines:
+- ..
+  env_vars:
+  ...
+  - ASAN_OPTIONS=atexit=1:print_stats=1:log_path=/tmp/daos_engine0.asan:disable_coredump=1:handle_segv=2:handle_abort=2:handle_sigfpe=2:handle_sigill=2:handle_sigbus=2:use_sigaltstack=1
+```
+For detailed information, see the
+[Sanitizer Common Flags](https://github.com/google/sanitizers/wiki/SanitizerCommonFlags) and
+[AddressSanitizer Flags](https://github.com/google/sanitizers/wiki/AddressSanitizerFlags)
+documentation pages.
+
+#### Known Issues
+
+ASan support in DAOS is still experimental and has the following known issues:
+1. **Library Path Issues**  
+   ASan's instrumentation of `dlopen()` removes `RPATH` and `RUNPATH` entries passed to the
+   compiler. As a result, libraries like `librdb` may not be found if DAOS is built with SCons and
+   installed in a non-standard location.  
+   **Workaround**: Use the `LD_LIBRARY_PATH` environment variable or the `ldconfig` command.  
+   More details are available in [Bug 27790](https://bugs.llvm.org/show_bug.cgi?id=27790).
+
+2. **Missing Statistics on SIGKILL**  
+   Stopping an application with `kill -s SIGKILL` prevents ASan from printing statistics, such as
+   detected memory leaks. For instance, this occurs when stopping DAOS engines using:  
+   ```bash
+   dmg system stop --force
+   ```
+
+3. **Integration with Test Framework**  
+   ASan is not fully integrated with the DAOS regression testing framework. Some tests, such as
+   those using Valgrind, may fail due to false positives.
+
 ## Go dependencies
 
 Developers contributing Go code may need to change the external dependencies

site_scons/components/__init__.py

@@ -1,4 +1,4 @@
 # Copyright 2016-2024 Intel Corporation
 # Copyright 2025 Hewlett Packard Enterprise Development LP
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -277,7 +277,15 @@
     abt_build = ['./configure',
                  '--prefix=$ARGOBOTS_PREFIX',
                  'CC=gcc',
-                 '--enable-stack-unwind']
+                 '--enable-stack-unwind=yes']
+    try:
+        if reqs.get_env('SANITIZERS') != "":
+            # NOTE the address sanitizer library add some extra info on the stack and thus ULTs
+            # need a bigger stack
+            print("Increase argobots default stack size from 16384 to 32768")
+            abt_build += ['--enable-default-stacksize=32768']
+    except KeyError:
+        pass
 
     if reqs.target_type == 'debug':
         abt_build.append('--enable-debug=most')

site_scons/prereq_tools/base.py

@@ -1,4 +1,5 @@
 # Copyright 2016-2024 Intel Corporation
+# Copyright 2025 Hewlett Packard Enterprise Development LP
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -469,6 +470,7 @@
                               ['gcc', 'covc', 'clang', 'icc'], ignorecase=2))
         opts.Add(EnumVariable('WARNING_LEVEL', "Set default warning level", 'error',
                               ['warning', 'warn', 'error'], ignorecase=2))
+        opts.Add(('SANITIZERS', 'Instrument C code with google sanitizers', None))
 
         opts.Update(self.__env)
 

site_scons/site_tools/compiler_setup.py

@@ -2,6 +2,9 @@
 
 from SCons.Script import Configure, Exit, GetOption
 
+FRAME_SIZE_MAX = 4096
+ASAN_FRAME_SIZE_MAX = {'gcc': 8192,
+                       'clang': 9216}
 DESIRED_FLAGS = ['-fstack-usage',
                  '-Wno-sign-compare',
                  '-Wno-missing-attributes',
@@ -14,7 +17,7 @@
                  '-Wno-unused-command-line-argument',
                  '-Wmismatched-dealloc',
                  '-Wfree-nonheap-object',
-                 '-Wframe-larger-than=4096']
+                 f"-Wframe-larger-than={FRAME_SIZE_MAX}"]
 
 # Compiler flags to prevent optimizing out security checks
 DESIRED_FLAGS.extend(['-fno-strict-overflow', '-fno-delete-null-pointer-checks', '-fwrapv'])
@@ -55,6 +58,28 @@ def _base_setup(env):
 
     env.AppendIfSupported(CCFLAGS=DESIRED_FLAGS)
 
+    if 'SANITIZERS' in env and env['SANITIZERS'] != "":
+        for flag in [f"-Wframe-larger-than={FRAME_SIZE_MAX}", '-fomit-frame-pointer']:
+            if flag in env["CCFLAGS"]:
+                env["CCFLAGS"].remove(flag)
+        cc = 'gcc'
+        if 'COMPILER' in env:
+            cc = env['COMPILER']
+        cc_flags = [f"-Wframe-larger-than={ASAN_FRAME_SIZE_MAX[cc]}",
+                    '-fno-omit-frame-pointer',
+                    '-fno-common']
+
+        asan_flags = []
+        for sanitizer in env['SANITIZERS'].split(','):
+            asan_flags.append(f"-fsanitize={sanitizer}")
+
+        env.AppendIfSupported(CCFLAGS=cc_flags + asan_flags)
+
+        for flag in asan_flags:
+            if flag in env["CCFLAGS"]:
+                env.AppendUnique(LINKFLAGS=flag)
+                print(f"Enabling {flag.split('=')[1]} sanitizer for C code")
+
     if '-Wmismatched-dealloc' in env['CCFLAGS']:
         env.AppendUnique(CPPDEFINES={'HAVE_DEALLOC': '1'})
 
@@ -184,10 +209,30 @@ def _set_fortify_level(env):
     Exit(1)
 
 
+def _check_func(env, func_name):
+    """Check if a function is usable"""
+    denv = env.Clone()
+    # NOTE Remove sanitizers to not scramble the test output
+    if 'SANITIZERS' in denv and denv['SANITIZERS'] != "":
+        for sanitizer in denv['SANITIZERS'].split(','):
+            flag = f"-fsanitize={sanitizer}"
+            if flag not in denv["CCFLAGS"]:
+                continue
+            denv["CCFLAGS"].remove(flag)
+            denv["LINKFLAGS"].remove(flag)
+
+    config = Configure(denv)
+    res = config.CheckFunc(func_name)
+    config.Finish()
+
+    return res
+
+
 def generate(env):
     """Add daos specific method to environment"""
     env.AddMethod(_base_setup, 'compiler_setup')
     env.AddMethod(_append_if_supported, "AppendIfSupported")
+    env.AddMethod(_check_func, "CheckFunc")
 
 
 def exists(_env):

src/SConscript

@@ -83,12 +83,10 @@ def scons():
         base_env_mpi.AppendUnique(CPPPATH=[Dir('include')])
 
     if not env.GetOption('clean') and not env.GetOption('help'):
-        conf = env.Clone().Configure()
         # Detect if we have explicit_bzero
-        if not conf.CheckFunc('explicit_bzero'):
+        if not env.CheckFunc('explicit_bzero'):
             env.Append(CCFLAGS=['-DNEED_EXPLICIT_BZERO'])
             base_env.Append(CCFLAGS=['-DNEED_EXPLICIT_BZERO'])
-        conf.Finish()
 
     for header in HEADERS:
         env.Install(os.path.join('$PREFIX', 'include'), os.path.join('include', header))

src/client/dfs/SConscript

@@ -18,7 +18,7 @@ def configure_lustre(denv):
         _print("No installed Lustre version detected")
     else:
         _print("Installed Lustre version detected")
-        if not conf.CheckFunc('llapi_unlink_foreign'):
+        if not denv.CheckFunc('llapi_unlink_foreign'):
             _print("Lustre version is not compatible")
         else:
             _print("Lustre version is compatible")

src/client/dfs/cont.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2018-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -83,7 +84,7 @@
 			dattr.da_chunk_size = DFS_DEFAULT_CHUNK_SIZE;
 
 		if (attr->da_hints[0] != 0) {
-			strncpy(dattr.da_hints, attr->da_hints, DAOS_CONT_HINT_MAX_LEN);
+			memcpy(dattr.da_hints, attr->da_hints, DAOS_CONT_HINT_MAX_LEN - 1);
 			dattr.da_hints[DAOS_CONT_HINT_MAX_LEN - 1] = '\0';
 		}
 	} else {

src/client/dfs/obj.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2018-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -510,10 +511,11 @@
 		D_GOTO(err, rc = EINVAL);
 	}
 
-	strncpy(new_obj->name, obj->name, DFS_MAX_NAME + 1);
-	new_obj->dfs   = dfs;
-	new_obj->mode  = obj->mode;
-	new_obj->flags = flags;
+	memcpy(new_obj->name, obj->name, DFS_MAX_NAME);
+	new_obj->name[DFS_MAX_NAME] = '\0';
+	new_obj->dfs                = dfs;
+	new_obj->mode               = obj->mode;
+	new_obj->flags              = flags;
 	oid_cp(&new_obj->parent_oid, obj->parent_oid);
 	oid_cp(&new_obj->oid, obj->oid);
 
@@ -616,8 +618,8 @@
 	oid_cp(&obj_glob->parent_oid, obj->parent_oid);
 	uuid_copy(obj_glob->coh_uuid, coh_uuid);
 	uuid_copy(obj_glob->cont_uuid, cont_uuid);
-	strncpy(obj_glob->name, obj->name, DFS_MAX_NAME + 1);
-	obj_glob->name[DFS_MAX_NAME] = 0;
+	memcpy(obj_glob->name, obj->name, DFS_MAX_NAME);
+	obj_glob->name[DFS_MAX_NAME] = '\0';
 	if (S_ISDIR(obj_glob->mode))
 		return 0;
 	rc = dfs_get_chunk_size(obj, &obj_glob->chunk_size);
@@ -674,7 +676,7 @@
 
 	oid_cp(&obj->oid, obj_glob->oid);
 	oid_cp(&obj->parent_oid, obj_glob->parent_oid);
-	strncpy(obj->name, obj_glob->name, DFS_MAX_NAME + 1);
+	memcpy(obj->name, obj_glob->name, DFS_MAX_NAME);
 	obj->name[DFS_MAX_NAME] = '\0';
 	obj->mode               = obj_glob->mode;
 	obj->dfs                = dfs;

src/client/dfuse/ops/lookup.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2016-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -118,11 +119,12 @@
 
 			/* Save the old name so that we can invalidate it in later */
 			wipe_parent = inode->ie_parent;
-			strncpy(wipe_name, inode->ie_name, NAME_MAX);
+			memcpy(wipe_name, inode->ie_name, NAME_MAX);
 			wipe_name[NAME_MAX] = '\0';
 
 			inode->ie_parent = ie->ie_parent;
-			strncpy(inode->ie_name, ie->ie_name, NAME_MAX + 1);
+			memcpy(inode->ie_name, ie->ie_name, NAME_MAX);
+			inode->ie_name[NAME_MAX] = '\0';
 		}
 		atomic_fetch_sub_relaxed(&ie->ie_ref, 1);
 		dfuse_ie_close(dfuse_info, ie);
@@ -295,6 +297,7 @@
 		DFUSE_TRA_DEBUG(ie, "Attr len is %zi", attr_len);
 
 	strncpy(ie->ie_name, name, NAME_MAX);
+	ie->ie_name[NAME_MAX] = '\0';
 
 	dfs_obj2id(ie->ie_obj, &ie->ie_oid);
 

src/control/SConscript

@@ -36,6 +36,8 @@ def get_build_flags(benv):
     """Return string of build flags"""
     if is_release_build(benv):
         return '-buildmode=pie'
+    if 'SANITIZERS' in benv and benv['SANITIZERS'] != "":
+        return '-asan'
     # enable race detector for non-release builds
     return '-race'
 

src/tests/obj_ctl.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2017-2022 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -198,7 +199,7 @@
 	int			 rc;
 
 	if (args) {
-		strncpy(buf, args, CTL_BUF_LEN);
+		strncpy(buf, args, CTL_BUF_LEN - 1);
 		buf[CTL_BUF_LEN - 1] = '\0';
 		str = daos_str_trimwhite(buf);
 	} else {