Skip to content

Commit

Permalink
Add clang detection to SCons (Autodesk#1700)
Browse files Browse the repository at this point in the history 
* Add clang detection to SCons

* Include suggestion about using StringVariable

* Fix StringVariable for completeness

* Remove "warning: -undefined error is deprecated"

* Link deps as usual

* Use -force_load instead of -all_load

* Update the Clang detection

* Fix python imports

* Fix python imports (2)

* Fix python imports (3)

* Link more static libs with -force_load

* Test another way of setting -force_load

* Fix an issue with LLVM's ar and ranlib generating universal libs from FAT object files

* Revert "Test another way of setting -force_load"

This reverts commit 80412f3.
  • Loading branch information
@ansono
ansono authored Oct 19, 2023
1 parent 3d3d3ad commit 8097cb0
Show file tree
Hide file tree
Showing 7 changed files with 322 additions and 79 deletions.
42 changes: 30 additions & 12 deletions SConstruct
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,28 @@ else:
ALLOWED_COMPILERS = ['gcc', 'clang']
arnold_default_api_lib = os.path.join('$ARNOLD_PATH', 'bin')

def compiler_validator(key, val, env):
## Convert the tuple of strings with allowed compilers and versions ...
## ('clang:3.4.0', 'icc:14.0.2', 'gcc')
## ... into a proper dictionary ...
## {'clang' : '3.4.0', 'icc' : '14.0.2', 'gcc' : None}
allowed_versions = {i[0] : i[1] if len(i) == 2 else None for i in (j.split(':') for j in ALLOWED_COMPILERS)}
## Parse the compiler format string <compiler>:<version>
compiler = val.split(':')
## Validate if it's an allowed compiler
if compiler[0] not in allowed_versions.keys():
m = 'Invalid value for option {}: {}. Valid values are: {}'
raise SCons.Errors.UserError(m.format(key, val, allowed_versions.keys()))
## Set valid compiler name and version. If the version was not specified
## we will set the default version which we use in the official releases,
## so the build will fail if it cannot be found.
env['_COMPILER'] = compiler[0]
env['_COMPILER_VERSION'] = len(compiler) == 2 and compiler[1] or allowed_versions[compiler[0]]

# Scons doesn't provide a string variable
def StringVariable(key, help, default):
def StringVariable(key, help='', default=None, validator=None, converter=None):
# We always get string values, so it's always valid and trivial to convert
return (key, help, default, lambda k, v, e: True, lambda s: s)
return (key, help, default, validator, converter)

# Custom variables definitions
vars = Variables('custom.py')
Expand All @@ -47,7 +65,7 @@ vars.AddVariables(
PathVariable('REFERENCE_DIR', 'Directory where the test reference images are stored.', 'testsuite', PathVariable.PathIsDirCreate),
EnumVariable('MODE', 'Set compiler configuration', 'opt', allowed_values=('opt', 'debug', 'profile')),
EnumVariable('WARN_LEVEL', 'Set warning level', 'none', allowed_values=('strict', 'warn-only', 'none')),
EnumVariable('COMPILER', 'Set compiler to use', ALLOWED_COMPILERS[0], allowed_values=ALLOWED_COMPILERS),
StringVariable('COMPILER', 'Set compiler to use', ALLOWED_COMPILERS[0], compiler_validator),
PathVariable('SHCXX', 'C++ compiler used for generating shared-library objects', None),
EnumVariable('CXX_STANDARD', 'C++ standard for gcc/clang.', '11', allowed_values=('11', '14', '17', '20')),
BoolVariable('SHOW_CMDS', 'Display the actual command lines used for building', False),
Expand Down Expand Up @@ -242,9 +260,8 @@ else:

env['PYTHON_LIBRARY'] = File(env['PYTHON_LIB_NAME']) if os.path.isabs(env['PYTHON_LIB_NAME']) else env['PYTHON_LIB_NAME']

if env['COMPILER'] == 'clang':
env['CC'] = 'clang'
env['CXX'] = 'clang++'
if env['_COMPILER'] == 'clang':
env.Tool('clang', version=env['_COMPILER_VERSION'])

# force compiler to match SHCXX
if env['SHCXX'] != '$CXX':
Expand Down Expand Up @@ -272,7 +289,7 @@ elif BUILD_TESTSUITE:
env['USD_VERSION'] = ''
env['USD_HAS_PYTHON_SUPPORT'] = ''

if env['COMPILER'] in ['gcc', 'clang'] and env['SHCXX'] != '$CXX':
if env['_COMPILER'] in ['gcc', 'clang'] and env['SHCXX'] != '$CXX':
env['GCC_VERSION'] = os.path.splitext(os.popen(env['SHCXX'] + ' -dumpversion').read())[0]

print("Building Arnold-USD:")
Expand Down Expand Up @@ -324,16 +341,17 @@ env['ENV']['PREFIX_BIN'] = os.path.abspath(PREFIX_BIN)
env['ENV']['PREFIX_PROCEDURAL'] = os.path.abspath(PREFIX_PROCEDURAL)

# Compiler settings
if env['COMPILER'] in ['gcc', 'clang']:
if env['_COMPILER'] in ['gcc', 'clang']:
env.Append(CCFLAGS = Split('-fno-operator-names -std=c++{}'.format(env['CXX_STANDARD'])))
if IS_DARWIN:
env.Append(LINKFLAGS = '-Wl,-undefined,error')
env_dict = env.Dictionary()
# Minimum compatibility with Mac OSX "env['MACOS_VERSION_MIN']"
env.Append(CCFLAGS = ['-mmacosx-version-min={MACOS_VERSION_MIN}'.format(**env_dict)])
env.Append(LINKFLAGS = ['-mmacosx-version-min={MACOS_VERSION_MIN}'.format(**env_dict)])
env.Append(CCFLAGS = ['-isysroot','{SDK_PATH}/MacOSX{SDK_VERSION}.sdk/'.format(**env_dict)])
env.Append(LINKFLAGS = ['-isysroot','{SDK_PATH}/MacOSX{SDK_VERSION}.sdk/'.format(**env_dict)])
env.Append(CXXFLAGS = ['-stdlib=libc++'])
env.Append(LINKFLAGS = ['-stdlib=libc++'])
if env['MACOS_ARCH'] == 'x86_64':
env.Append(CCFLAGS = ['-arch', 'x86_64'])
env.Append(LINKFLAGS = ['-arch', 'x86_64'])
Expand Down Expand Up @@ -374,7 +392,7 @@ if env['COMPILER'] in ['gcc', 'clang']:
env.Append(LINKFLAGS = Split('-pg'))

# msvc settings
elif env['COMPILER'] == 'msvc':
elif env['_COMPILER'] == 'msvc':
env.Append(CCFLAGS=Split('/EHsc'))
env.Append(LINKFLAGS=Split('/Machine:X64'))
# Ignore all the linking warnings we get on windows, coming from USD
Expand Down Expand Up @@ -438,9 +456,9 @@ env['ROOT_DIR'] = os.getcwd()

# Configure base directory for temp files
if IS_DARWIN:
BUILD_BASE_DIR = os.path.join(BUILD_DIR, '%s_%s' % (system.os, env['MACOS_ARCH']), '%s_%s' % (env['COMPILER'], env['MODE']), 'usd-%s_arnold-%s' % (env['USD_VERSION'], env['ARNOLD_VERSION']))
BUILD_BASE_DIR = os.path.join(BUILD_DIR, '%s_%s' % (system.os, env['MACOS_ARCH']), '%s_%s' % (env['_COMPILER'], env['MODE']), 'usd-%s_arnold-%s' % (env['USD_VERSION'], env['ARNOLD_VERSION']))
else:
BUILD_BASE_DIR = os.path.join(BUILD_DIR, '%s_%s' % (system.os, 'x86_64'), '%s_%s' % (env['COMPILER'], env['MODE']), 'usd-%s_arnold-%s' % (env['USD_VERSION'], env['ARNOLD_VERSION']))
BUILD_BASE_DIR = os.path.join(BUILD_DIR, '%s_%s' % (system.os, 'x86_64'), '%s_%s' % (env['_COMPILER'], env['MODE']), 'usd-%s_arnold-%s' % (env['USD_VERSION'], env['ARNOLD_VERSION']))

env['BUILD_BASE_DIR'] = BUILD_BASE_DIR
# Build target
Expand Down
7 changes: 4 additions & 3 deletions plugins/procedural/SConscript
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,8 @@ elif local_env['USD_BUILD_MODE'] == 'static': # static_monolithic
local_env.Append(LINKFLAGS=['-Wl,--whole-archive,%s,--no-whole-archive' % ','.join(whole_archives)])
local_env.Append(LIBS = ['pthread'])
elif system.IS_DARWIN:
local_env.Append(LINKFLAGS=['-Wl,-all_load,%s,-noall_load' % ','.join(whole_archives)])
for whole_archive in whole_archives:
local_env.Append(LINKFLAGS=['-Wl,-force_load,{}'.format(whole_archive)])
extra_frameworks = local_env['EXTRA_FRAMEWORKS']
if extra_frameworks:
extra_frameworks = extra_frameworks.split(';')
Expand Down Expand Up @@ -155,8 +156,8 @@ if local_env['ENABLE_HYDRA_IN_USD_PROCEDURAL']:
local_env.Append(LINKFLAGS = ['-Wl,--whole-archive,{},--no-whole-archive'.format(os.path.abspath(os.path.join(local_env['BUILD_BASE_DIR'], 'plugins', 'ndr', 'libndrArnold.a')))])
local_env.Append(LINKFLAGS = ['-Wl,--whole-archive,{},--no-whole-archive'.format(os.path.abspath(os.path.join(local_env['BUILD_BASE_DIR'], 'plugins', 'usd_imaging', 'libusdImagingArnold.a')))])
else:
procedural_libs += ['ndrArnold']
procedural_libs += ['usdImagingArnold']
local_env.Append(LINKFLAGS=['-Wl,-force_load,{}'.format(os.path.abspath(os.path.join(local_env['BUILD_BASE_DIR'], 'plugins', 'ndr', 'libndrArnold.a')))])
local_env.Append(LINKFLAGS=['-Wl,-force_load,{}'.format(os.path.abspath(os.path.join(local_env['BUILD_BASE_DIR'], 'plugins', 'usd_imaging', 'libusdImagingArnold.a')))])

procedural_lib_paths += [os.path.abspath(os.path.join(local_env['BUILD_BASE_DIR'], 'plugins', 'ndr'))]
procedural_lib_paths += [os.path.abspath(os.path.join(local_env['BUILD_BASE_DIR'], 'plugins', 'usd_imaging'))]
Expand Down
174 changes: 130 additions & 44 deletions tools/scons-custom/site_tools/clang.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,10 @@
import os
import glob
import re
from sets import Set
import SCons

Version = sa.version.Version

def split_version(v):
"""Splits a version string 'a.b.c' into an int list [a, b, c]"""
try:
Expand All @@ -36,36 +37,67 @@ def closest_version(v, vlist):
"""See if we can match v (string) in vlist (list of strings)
If user requests v=13, it will return the greatest 13.x version.
If user requests v=13.1, it will return the greatest 13.1.x version, and so on"""
versions = sorted((Version(_) for _ in vlist), reverse=True)
if not v:
return vlist[0]
s = split_version(v)
l = len(s)
# Try to match the latest installed version of the requested version
if l > 0:
for vi in vlist:
S = split_version(vi)
if l > len(S):
continue
n = 0
for i in range(l):
if s[i] == S[i]: n = n + 1
else: break
if n == l:
return vi
return None
return versions[0]
closest_v = None
V = Version(v)
V_len = len(V)
for ver in versions:
if V[:V_len] == ver[:V_len]:
closest_v = ver
break
if V > ver:
break
return closest_v

def detect_clang_install_path_windows():
if not sa.system.is_windows:
return []
paths = os.environ.get('PATH').split(';')

def open_key(name):
try:
return SCons.Util.RegOpenKeyEx(SCons.Util.HKEY_LOCAL_MACHINE, name)
except WindowsError:
return None

def get_value(key, name):
try:
v = SCons.Util.RegQueryValueEx(key, name)[0]
except WindowsError:
v = None
return v

keyname = 'SOFTWARE\\Wow6432Node\\LLVM\\LLVM'
key = open_key(keyname)
val = get_value(key,'') if key else None # we want the default key value, so use ''
if val is not None:
paths.extend(glob.glob(os.path.join(os.sep, val, 'bin')))
return paths

def generate(env, version=None):

# Configure tool names
clang_extension = {
'linux' : '',
'darwin' : '',
'windows': '.exe',
}.get(sa.system.os)
clang_name = {
'linux' : r'^(?P<exec>clang)?$',
'darwin': r'^(?P<exec>clang)(?P<suffix>-mp-.\..)?$',
'linux' : r'^(?P<exec>clang)?$',
'darwin' : r'^(?P<exec>clang)(?P<suffix>-mp-[0-9]+(\.[0-9]+)*)?$',
'windows': r'^(?P<exec>clang-cl)\.exe$',
}.get(sa.system.os)
macro_version = ['__clang_major__','__clang_minor__','__clang_patchlevel__']
# Look for clang installations in the PATH and other custom places
toolchain_path = Set(os.environ[sa.system.PATH].split(os.pathsep))
toolchain_path = set(os.environ[sa.system.PATH].split(os.pathsep))
toolchain_path.update({
'linux' : glob.glob(os.path.join(os.sep, 'solidangle', 'toolchain', '*', 'bin')),
'darwin': [os.path.join(os.sep, 'opt', 'local', 'bin')],
'linux' : glob.glob(os.path.join(os.sep, 'solidangle', 'toolchain', '*', 'bin')),
'darwin' : [os.path.join(os.sep, 'opt', 'local', 'bin')] +
glob.glob(os.path.join(os.sep, 'usr', 'local', 'Cellar', 'llvm', '*', 'bin')) +
glob.glob(os.path.join(os.sep, 'usr', 'local', 'Cellar', 'llvm@*', '*', 'bin')),
'windows': detect_clang_install_path_windows(),
}.get(sa.system.os, []))
versions_detected = {}
for p in toolchain_path:
Expand All @@ -83,10 +115,11 @@ def generate(env, version=None):
raise SCons.Errors.UserError("Can't find Clang.")
# Try to match the closest detected version
selected_version = closest_version(version, versions_detected.keys())
path = versions_detected.get(selected_version)
path = versions_detected.get(str(selected_version))
if not path:
versions = sorted(Version(_) for _ in versions_detected.keys())
raise SCons.Errors.UserError("Can't find Clang %s. " % version +
"Installed versions are [%s]." % (', '.join(versions_detected.keys())))
"Installed versions are [%s]." % (', '.join(str(v) for v in versions)))
if len(path) > 1:
# Warn if we found multiple installations of a given version
class ClangWarning(SCons.Warnings.Warning): pass
Expand All @@ -97,38 +130,91 @@ class ClangWarning(SCons.Warnings.Warning): pass
m = re.match(clang_name, clang_exec)
exec_name, suffix = 'clang', ''
if m:
m = m.groupdict()
exec_name = m.get('exec', exec_name)
suffix = m.get('suffix', suffix )
env['CC'] = clang_exec
env['CXX'] = exec_name + '++' + suffix
if sa.system.is_linux:
# In order to use LTO, we need the gold linker, which is able to load plugins
env['LD'] = 'ld.gold'
m = m.groupdict('')
exec_name = m.get('exec' , exec_name)
suffix = m.get('suffix', suffix )
env['CC'] = os.path.join(clang_path, clang_exec)
env['CXX'] = os.path.join(clang_path, exec_name + '++' + suffix)
if sa.system.is_windows:
env['CXX'] = env['CC']
env['LD'] = os.path.join(clang_path, env.get('LINKER_NAME', 'lld-link') + '.exe')
env['LINK'] = os.path.join(clang_path, env.get('LINKER_NAME', 'lld-link') + '.exe')
env['AR'] = os.path.join(clang_path, 'llvm-ar.exe')
env['ARFLAGS'] = 'rcs'
env['ARCOM']= "${TEMPFILE('$AR $ARFLAGS $TARGET $SOURCES','$ARCOMSTR')}"

env.PrependENVPath(sa.system.PATH, clang_path)
# Use LLVM's tools if they are present in the detected LLVM's path
for i in ('AR', 'RANLIB'):
tool = 'llvm-{0}{1}'.format(i.lower(), suffix)
if os.path.exists(os.path.join(clang_path, tool)):
env[i] = tool
# Check the presence of the LLVM Gold plugin, needed for LTO
######
###### TBR: for some reason, adding clang_path to PATH is not letting subsequent executions with this environment to find the executable. Maybe only affects windows?
######
env.PrependENVPath('PATH', clang_path)

env['LLVM_PATH'] = os.path.realpath(os.path.join(clang_path, os.pardir))
# Special linker detection in linux
if sa.system.is_linux:
env['LLVM_GOLD_PLUGIN'] = os.path.join(clang_path, os.pardir, 'lib', 'LLVMgold.so')
# Name of the linker (eg. ld, gold, bfd, lld, etc.). It defaults to "gold"
linker_name = env.get('LINKER_NAME', 'gold').replace('ld.', '', 1)
# Regular expresions for detecting the linker version
regex_version = r'(?:[0-9]+)(?:\.(?:[0-9])+)*'
linker_regex = {
'ld' : re.compile(r'^GNU (ld) (?:(?:\(GNU Binutils\))|(?:version)) ({0})'.format(regex_version)),
'bfd' : re.compile(r'^GNU (ld) (?:(?:\(GNU Binutils\))|(?:version)) ({0})'.format(regex_version)),
'gold': re.compile(r'^GNU (gold) \((?:(?:GNU Binutils)|(?:version)) ({0})[\-\._0-9a-zA-Z]*\) ({0})'.format(regex_version)),
'lld' : re.compile(r'^(LLD) ({0})'.format(regex_version)),
}.get(linker_name)
# Search for the specified linker in the PATHs
linker_detected = set()
for p in env['ENV']['PATH'].split(os.pathsep) + os.environ['PATH'].split(os.pathsep):
# The linker name uses to be prefixed with "ld." (eg. ld.gold). Try both.
for linker_prefix in ['ld.', '']:
linker = os.path.join(p, '{}{}'.format(linker_prefix, linker_name))
if os.path.isfile(linker) and linker_regex:
# Get the version if the found path is a file and we have a proper regex
error, output = sa.system.execute([linker, '-v'])
found = linker_regex.search(output[0]) if not error else None
if found:
linker_detected.add((
found.group(1).lower(), # name
found.group(2), # version
linker # path
))
# Sort detected linkers from newest to oldest
linker_detected = sorted(linker_detected, key=lambda item: Version(item[1]), reverse=True)
if not linker_detected:
raise SCons.Errors.UserError("Can't find linker {}".format(linker_name))
env['LINKER_NAME'] = linker_detected[0][0]
env['LINKER_VERSION'] = Version(linker_detected[0][1])
env['LINKER_PATH'] = linker_detected[0][2]
env['LD'] = env['LINKER_PATH']
# Use LLVM's tools if they are present in the detected LLVM's path
# Get the output after the execution of "kick --version"
# NOTE: Don't do this in macOS for now.
# It seems that when building FAT object files (with "-arch x86_64 -arch arm64"),
# LLVM's ar and ranlib can't generate proper universal static libraries. At
# least with LLVM 15, they simply archive the FAT object files, instead of
# archiving the separated arch slices and "lipo" them.
if not sa.system.is_darwin:
for i in ('AR', 'RANLIB'):
tool = 'llvm-{0}{1}{2}'.format(i.lower(), suffix, clang_extension)
tool = os.path.join(env['LLVM_PATH'], 'bin', tool)
if os.path.exists(tool):
env[i] = tool
# Check the presence of the LLVM Gold plugin, needed for LTO if we use ld.gold
if sa.system.is_linux and linker_name == 'gold':
env['LLVM_GOLD_PLUGIN'] = os.path.join(env['LLVM_PATH'], 'lib', 'LLVMgold.so')
if not os.path.exists(env['LLVM_GOLD_PLUGIN']):
raise SCons.Errors.UserError("Can't find LLVM Gold plugin")
raise SCons.Errors.UserError("Can't find LLVM Gold plugin in {}".format(env['LLVM_GOLD_PLUGIN']))
env['COMPILER_VERSION_DETECTED'] = sa.compiler.detect_version(env, env['CC'], macro_version)
env['COMPILER_VERSION_INSTALLED'] = versions_detected.keys()
env['COMPILER_VERSION_INSTALLED'] = [str(v) for v in sorted(Version(_) for _ in versions_detected.keys())]
[apple] = sa.compiler.get_defines(env, env['CC'], ['__apple_build_version__'])
if apple:
env['COMPILER_PREFIX'] = 'apple'

# If requested, detect and configure the Clang's static analyzer (scan-build)
# More info at http://clang-analyzer.llvm.org/
if env['STATIC_ANALYSIS']:
if env.get('STATIC_ANALYSIS'):
static_analyzer = ['scan-build']
static_analyzer += [] # scan-build options
if os.path.exists(os.path.join(clang_path, static_analyzer[0])):
if os.path.exists(os.path.join(env['LLVM_PATH'], 'bin', static_analyzer[0])):
env['CC'] = ' '.join(static_analyzer + [env['CC'] ])
env['CXX'] = ' '.join(static_analyzer + [env['CXX']])

Expand Down
3 changes: 2 additions & 1 deletion tools/usdgenschema/SConscript
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ elif local_env['USD_BUILD_MODE'] == 'static':
local_env.Append(LINKFLAGS=['-Wl,--whole-archive,%s,--no-whole-archive' % ','.join(whole_archives)])
local_env.Append(LIBS = ['dl', 'pthread'])
elif system.IS_DARWIN:
local_env.Append(LINKFLAGS=['-Wl,-all_load,%s,-noall_load' % ','.join(whole_archives)])
for whole_archive in whole_archives:
local_env.Append(LINKFLAGS=['-Wl,-force_load,{}'.format(whole_archive)])
extra_frameworks = local_env['EXTRA_FRAMEWORKS']
if extra_frameworks:
extra_frameworks = extra_frameworks.split(';')
Expand Down
2 changes: 1 addition & 1 deletion tools/utils/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from . import build_tools, color, compiler, configure, dependencies, elf, git, \
path, regression_test, system, test_stats, testsuite
path, regression_test, system, test_stats, testsuite, version
Loading

0 comments on commit 8097cb0

Please sign in to comment.