From a7573ce8ddd37687514143f230b04551a6abaaf1 Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Wed, 10 May 2023 19:10:21 +0000 Subject: [PATCH 01/13] New config settings code and defaults --- CMakeLists.txt | 264 +++++++++++++++++++----------- build/cmake/MongoSettings.cmake | 278 ++++++++++++++++++++++++++++++++ build/cmake/Sanitizers.cmake | 11 +- 3 files changed, 454 insertions(+), 99 deletions(-) create mode 100644 build/cmake/MongoSettings.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 1bb10023a6..1c8d1d87f4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,58 +1,163 @@ -cmake_minimum_required (VERSION 3.1) - -set (ENABLE_SSL AUTO CACHE STRING - "Enable TLS connections and SCRAM-SHA-1 authentication. Options are - \"DARWIN\" to use Apple's Secure Transport, \"WINDOWS\" to use Windows - Secure Channel, \"OPENSSL\", \"LIBRESSL\", \"AUTO\",\ or \"OFF\". These options are - case-sensitive. The default is \"AUTO\". Note\ that SCRAM-SHA-1 is - required for authenticating to MongoDB 3.0 and later.") - -set_property(CACHE ENABLE_SSL PROPERTY STRINGS "DARWIN;WINDOWS;OPENSSL;LIBRESSL;AUTO;OFF") - -set (ENABLE_SASL AUTO CACHE STRING - "Enable SASL authentication (Kerberos). Options are \"CYRUS\" to use Cyrus - SASL, \"SSPI\" to use Windows Native SSPI, \"AUTO\",\ or \"OFF\". These options are case-sensitive.") - -set_property(CACHE ENABLE_SASL PROPERTY STRINGS "CYRUS|SSPI|AUTO|OFF") - -set (ENABLE_STATIC AUTO CACHE STRING "Build and install static libbson/libmongoc. Set to ON/AUTO/OFF/BUILD_ONLY/DONT_INSTALL, default AUTO.") -option (ENABLE_TESTS "Build MongoDB C Driver tests." ON) -option (ENABLE_EXAMPLES "Build MongoDB C Driver examples." ON) -set (ENABLE_SRV AUTO CACHE STRING "Support mongodb+srv URIs. Set to ON/AUTO/OFF, default AUTO.") -option (ENABLE_MAINTAINER_FLAGS "Use strict compiler checks" OFF) -option (ENABLE_AUTOMATIC_INIT_AND_CLEANUP "Enable automatic init and cleanup (GCC only)" ON) -option (ENABLE_CRYPTO_SYSTEM_PROFILE "Use system crypto profile (OpenSSL only)" OFF) -option (ENABLE_TRACING "Turn on verbose debug output" OFF) -option (ENABLE_COVERAGE "Turn on compile options for lcov" OFF) -set (ENABLE_SHM_COUNTERS AUTO CACHE STRING "Enable memory performance counters that use shared memory on Linux. Set to ON/AUTO/OFF, default AUTO.") -set (ENABLE_MONGOC ON CACHE STRING "Whether to build libmongoc. Set to ON/OFF, default ON.") -set (ENABLE_BSON AUTO CACHE STRING "Whether to build libbson. Set to ON/AUTO/SYSTEM, default AUTO.") -set (ENABLE_SNAPPY AUTO CACHE STRING "Enable snappy support. Set to ON/AUTO/OFF, default AUTO.") -set (ENABLE_ZLIB AUTO CACHE STRING "Enable zlib support") -set (ENABLE_ZSTD AUTO CACHE STRING "Enable zstd support. Set to ON/AUTO/OFF, default AUTO.") -option (ENABLE_MAN_PAGES "Build MongoDB C Driver manual pages." OFF) -option (ENABLE_HTML_DOCS "Build MongoDB C Driver HTML documentation." OFF) -option (ENABLE_EXTRA_ALIGNMENT - "Turn on extra alignment of libbson types. Set to ON/OFF, default ON.\ - Required for the 1.0 ABI but better disabled." - ON +cmake_minimum_required (VERSION 3.15) + +project (mongo-c-driver C) + +list (APPEND CMAKE_MODULE_PATH + "${PROJECT_SOURCE_DIR}/build/cmake" + "${PROJECT_SOURCE_DIR}/build/cmake/make_dist" + ) + +include (MongoSettings) + +# Subcomponents: +mongo_bool_setting(ENABLE_MONGOC "Enable the build of the mongo-c-driver libraries") +mongo_bool_setting(ENABLE_BSON "Enable the build of the C BSON handling libraries") + +# Static libraries: +mongo_setting( + ENABLE_STATIC "Whether we should build (and maybe install) static library archives" + OPTIONS ON OFF BUILD_ONLY + DEFAULT VALUE "ON" ) -option (ENABLE_RDTSCP - "Fast performance counters on Intel using the RDTSCP instruction" - OFF +mongo_bool_setting(ENABLE_PIC "Enable position-independent-code in built static library components") + +# Dev mode checks: +mongo_bool_setting( + ENABLE_MAINTAINER_FLAGS "Enable stricter build-time checks" + DEFAULT VALUE OFF + AUDIT EVAL [[ + if(MSVC) + set(DEFAULT OFF) + else() + set(DEFAULT ON) + endif() + ]]) + +# Toggle instrumentation: +mongo_bool_setting(ENABLE_TRACING "Enable verbose debug output in the build driver" + DEFAULT VALUE OFF AUDIT VALUE ON) +mongo_bool_setting(ENABLE_COVERAGE "Instrument built code for use with lcov" + DEFAULT VALUE OFF AUDIT VALUE ON) +mongo_bool_setting(ENABLE_DEBUG_ASSERTIONS "Build with runtime debug assertions enabled" + DEFAULT VALUE OFF AUDIT VALUE ON) +# for MONGO_SANITIZE: +include(Sanitizers) + +# Toggle optional components: +mongo_bool_setting(ENABLE_TESTS "Whether we should build tests") +mongo_bool_setting(ENABLE_MAN_PAGES "Build the manual pages" DEFAULT VALUE OFF) +mongo_bool_setting(ENABLE_HTML_DOCS "Build the HTML documentation" DEFAULT VALUE OFF) +mongo_bool_setting(ENABLE_UNINSTALL "Generate an 'uninstall' script and an 'uninstall' build target") + +# Optional features that are ENABLED by default: +mongo_bool_setting(ENABLE_SRV "Toggle support for mongodb+srv URIs.") +mongo_bool_setting(ENABLE_SNAPPY "Enable Snappy compression support") +mongo_bool_setting(ENABLE_ZSTD "Enable zstd compression support") +mongo_setting(ENABLE_ZLIB "Enable zlib compression support" + OPTIONS BUNDLED SYSTEM OFF + DEFAULT VALUE BUNDLED) +mongo_setting( + ENABLE_SSL [[Enable TLS connection and SCRAM-SHA-1 authentication. + Note that SCRAM-SHA1 is required for authenticating to + MongoDB 3.0 or later.]] + OPTIONS WINDOWS DARWIN OPENSSL LIBRESSL OFF + DEFAULT EVAL [[ + if(WIN32) + set(DEFAULT "WINDOWS") + elseif(APPLE) + set(DEFAULT "DARWIN") + else() + set(DEFAULT "OPENSSL") + endif() + ]] + VALIDATE CODE [[ + if(ENABLE_SSL STREQUAL "DARWIN" AND NOT APPLE) + message(WARNING "ENABLE_SSL=DARWIN is only supported on Apple platforms") + elseif(ENABLE_SSL STREQUAL "WINDOWS" AND NOT WIN32) + message(WARNING "ENABLE_SSL=WINDOWS is only supported on Windows platforms") + endif() + ]] ) -option (ENABLE_APPLE_FRAMEWORK "Build libraries as frameworks on darwin platforms" OFF) -set (ENABLE_ICU AUTO CACHE STRING "Enable ICU support, necessary to use non-ASCII usernames or passwords, default AUTO.") -option (ENABLE_UNINSTALL "Enable creation of uninstall script and associated uninstall build target." ON) -set (ENABLE_CLIENT_SIDE_ENCRYPTION AUTO CACHE STRING "Enable In-Use Encryption support. Requires libmongocrypt. Set to ON/AUTO/OFF, default AUTO.") -set (ENABLE_MONGODB_AWS_AUTH AUTO CACHE STRING "Enable support for MONGODB-AWS authentication mechanism. Set to ON/AUTO/OFF, default AUTO. (also requires ENABLE_SSL not set to OFF)") -option (ENABLE_PIC - "Enables building of position independent code for static library components." - ON +mongo_bool_setting( + ENABLE_SHM_COUNTERS + "Enable memory performance counters that require shared memeory on Linux" + DEFAULT EVAL [[ + set(DEFAULT OFF) + if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") + set(DEFAULT ON) + endif() + ]] + VALIDATE CODE [[ + if(ENABLE_SHM_COUNTERS AND NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") + message(WARNING "ENABLE_SHM_COUNTERS=ON is only supported on Linux") + endif() + ]] ) -option (ENABLE_DEBUG_ASSERTIONS "Turn on runtime debug assertions" OFF) -project (mongo-c-driver C) +mongo_setting( + ENABLE_SASL "Enable SASL authentication (Kerberos)" + OPTIONS + CYRUS SSPI OFF + DEFAULT EVAL [[ + if(WIN32) + set(DEFAULT SSPI) + else() + set(DEFAULT CYRUS) + endif() + ]] + VALIDATE CODE [[ + if(ENABLE_SASL STREQUAL "SSPI" AND NOT WIN32) + message(WARNING "ENABLE_SASL=SSPI is only supported on Windows platforms") + endif() + ]] +) + +mongo_bool_setting( + ENABLE_EXTRA_ALIGNMENT + [[Enable extra alignment on libbson types. Default is ON for ABI + compatibility, but it is better to disable, if possible. Enabling this + option can often produce false positives with + UndefinedBehaviorSanitizer.]] + DEFAULT VALUE ON + AUDIT VALUE OFF +) + +mongo_bool_setting(ENABLE_ICU "Enable ICU support, needed for non-ASCII usernames and passwords") +mongo_bool_setting(ENABLE_CLIENT_SIDE_ENCRYPTION "Enable In-Use Encryption support. Requires additional support libraries.") +mongo_bool_setting(ENABLE_MONGODB_AWS_AUTH "Enable support for the MONGODB-AWS authentication mechanism") + +# Optional features that are DISABLED by default: +mongo_bool_setting(ENABLE_RDTSCP "Enable fast performance counters using the Intel RDTSCP instruction" + DEFAULT VALUE OFF) +mongo_bool_setting(ENABLE_APPLE_FRAMEWORK "Build libraries as Apple Frameworks on Darwin platforms" + DEFAULT VALUE OFF) +mongo_bool_setting( + ENABLE_CRYPTO_SYSTEM_PROFILE "Use system crypto profile" + DEFAULT VALUE OFF + VALIDATE CODE [[ + if(ENABLE_CRYPTO_SYSTEM_PROFILE AND NOT ENABLE_SSL STREQUAL "OPENSSL") + message(WARNING "ENABLE_CRYPTO_SYSTEM_PROFILE=TRUE is only aplicable when ENABLE_SSL=OPENSSL") + endif() + ]] +) + +# Deprecated options: +mongo_bool_setting( + ENABLE_AUTOMATIC_INIT_AND_CLEANUP + "[Deprecated] Enable automatic initialization of the C driver library" + DEFAULT VALUE ON + AUDIT VALUE OFF + VALIDATE CODE [[ + if(ENABLE_AUTOMATIC_INIT_AND_CLEANUP) + if(NOT CMAKE_C_COMPILER_ID STREQUAL "GNU") + message(WARNING "Automatic init-and-cleanup is only supported on GCC") + endif() + message(DEPRECATION + "Enabling ENABLE_AUTOMATIC_INIT_AND_CLEANUP is deprecated and " + "may be removed in a future release") + endif() + ]] +) # Optionally enable C++ to do some C++-specific tests include (CheckLanguage) @@ -74,12 +179,6 @@ if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) ) endif () -set (CMAKE_MODULE_PATH - ${CMAKE_MODULE_PATH} - ${PROJECT_SOURCE_DIR}/build/cmake - ${PROJECT_SOURCE_DIR}/build/cmake/make_dist -) - include (InstallRequiredSystemLibraries) include (GNUInstallDirs) include (CMakeDependentOption) @@ -93,8 +192,6 @@ include (CCache) if (NOT MSVC) include (LLDLinker) endif () -# Enable sanitizers -include (Sanitizers) set (BUILD_VERSION "0.0.0" CACHE STRING "Library version (for both libbson and libmongoc)") @@ -150,47 +247,20 @@ if ( (ENABLE_BUILD_DEPENDECIES STREQUAL OFF) AND (NOT CMAKE_CURRENT_SOURCE_DIR S set (ENABLE_BUILD_DEPENDECIES ON) endif () -if (ENABLE_EXTRA_ALIGNMENT STREQUAL ON) - set (BSON_EXTRA_ALIGN 1) -else () - set (BSON_EXTRA_ALIGN 0) -endif () - -if (ENABLE_RDTSCP) - set (MONGOC_ENABLE_RDTSCP 1) -else () - set (MONGOC_ENABLE_RDTSCP 0) -endif () +_mongo_pick(BSON_EXTRA_ALIGN 1 0 ENABLE_EXTRA_ALIGNMENT) -if (NOT ENABLE_MONGOC MATCHES "ON|OFF") - message (FATAL_ERROR "ENABLE_MONGOC option must be ON or OFF") -endif () +_mongo_pick(MONGOC_ENABLE_RDTSCP 1 0 ENABLE_RDTSCP) -if (NOT ENABLE_BSON MATCHES "ON|AUTO|SYSTEM") - message (FATAL_ERROR "ENABLE_BSON option must be ON, AUTO, or SYSTEM") -endif () - -if (NOT ENABLE_STATIC MATCHES "^(ON|OFF|AUTO|BUILD_ONLY|DONT_INSTALL)$") - message (FATAL_ERROR - "ENABLE_STATIC option must be ON, OFF, AUTO, BUILD_ONLY, or DONT_INSTALL" - ) -endif () - -set (MONGOC_ENABLE_STATIC_BUILD 0) -set (MONGOC_ENABLE_STATIC_INSTALL 0) - -if (ENABLE_STATIC MATCHES "^(ON|AUTO)$") - message (STATUS "Build and install static libraries") - set (MONGOC_ENABLE_STATIC_BUILD 1) - set (MONGOC_ENABLE_STATIC_INSTALL 1) -elseif (ENABLE_STATIC MATCHES "^(BUILD_ONLY|DONT_INSTALL)$") - message (STATUS "Build only static libraries") - set (MONGOC_ENABLE_STATIC_BUILD 1) -else () - message (STATUS "Don't build static libraries") -endif() +_mongo_pick(MONGOC_ENABLE_STATIC_BUILD 1 0 ENABLE_STATIC) +_mongo_pick(MONGOC_ENABLE_STATIC_INSTALL 1 0 + [[ENABLE_STATIC AND NOT ENABLE_STATIC STREQUAL "BUILD_ONLY"]]) +mongo_bool_setting( + USE_SYSTEM_LIBBSON "Build libmongoc against the system-wide libbson installation" + DEFAULT VALUE OFF + ADVANCED +) -if (ENABLE_BSON STREQUAL SYSTEM) +if (USE_SYSTEM_LIBBSON) # The input variable BSON_ROOT_DIR is respected for backwards compatibility, # but you should use the standard CMAKE_PREFIX_PATH instead. message (STATUS "Searching for libbson CMake packages") diff --git a/build/cmake/MongoSettings.cmake b/build/cmake/MongoSettings.cmake new file mode 100644 index 0000000000..d1bb68bfbc --- /dev/null +++ b/build/cmake/MongoSettings.cmake @@ -0,0 +1,278 @@ +include_guard(DIRECTORY) + +#[[ Bool: Set to TRUE if the environment variable MONGODB_DEVELOPER is a true value ]] +set(MONGODB_DEVELOPER FALSE) +#[[ Bool: Set to TRUE if the environment variable MONGODB_BUILD_AUDIT is a true value ]] +set(MONGODB_BUILD_AUDIT FALSE) + +# Detect developer mode: +set(_is_dev "$ENV{MONGODB_DEVELOPER}") +if(_is_dev) + message(STATUS "MONGODB_DEVELOPER is detected") + set(MONGODB_DEVELOPER TRUE) +endif() + +# Detect audit mode: +set(_is_audit "$ENV{MONGODB_BUILD_AUDIT}") +if(_is_audit) + message(STATUS "MONGODB_BUILD_AUDIT is enabled") + set(MONGODB_BUILD_AUDIT TRUE) +endif() + +#[==[ +Define a new configure-time build setting:: + mongo_setting( + + [TYPE ] + [DEFAULT [[DEVEL|AUDIT] [VALUE | EVAL ]] ...] + [OPTIONS [ ...]] + [VALIDATE [CODE ]] + [ADVANCED] [ALLOW_OTHER_VALUES] + ) + +The `` will be the name of the setting, while `` will be the +documentation string shown to the user. Newlines in the doc string will be +replaced with single spaces. If no other arguments are provided, the default +`TYPE` will be `STRING`, and the DEFAULT value will be an empty string. If the +previous cached value is AUTO, and AUTO is not listed in OPTIONS, then the cache +value will be cleared and reset to the default value. + +Package maintainers Note: Setting a variable `-FORCE` to TRUE will make +this function a no-op. + +TYPE + Sets the type for the generated cache variable. If the type is BOOL and + ALLOW_OTHER_VALUES is not specified, this call will validate that the + setting is a valid boolean value. + +OPTIONS [ ...] + Specify the valid set of values available for this setting. This will set + the STRINGS property on the cache variable and add an information message to + the doc string. Unless ALLOW_OTHER_VALUES is specified, this call will also + validate that the setting's value is one of these options, failing with an + error if it is not. + +DEFAULT [[DEVEL|AUDIT] [VALUE | EVAL ]] ... + Specify the default value of the generated variable. If given VALUE, then + `` will be used as the default, otherwise if given EVAL, `` + will be executed and is expected to define a variable DEFAULT that will + contain the default value. + + - If neither MONGODB_DEVELOPER nor MONGODB_BUILD_AUDIT are true, then the + non-qualified defaults will be used. (If no non-qualified defaults are + provided, then the default value is an empty string.) + - Otherwise, If DEVEL defaults are provided, and MONGODB_DEVELOPER is true, + then the DEVEL defaults will be used. + - Otherwise, if AUDIT defaults are provided, and either MONGODB_DEVELOPER or + MONGODB_BUILD_AUDIT is true, then the AUDIT defaults will be used. + +VALIDATE [CODE ] + If specified, then `` will be evaluated after the setting value is + defined. `` may issue warnings and errors about the value of the + setting. + +ADVANCED + If specified, the cache variable will be marked as an advanced setting + +ALLOW_OTHER_VALUES + If *not* specified, this call will validate that the setting's value is + valid according to TYPE and OPTIONS. + +]==] +function(mongo_setting setting_NAME setting_DOC) + list(APPEND CMAKE_MESSAGE_CONTEXT mongo setting "${setting_NAME}") + # Allow bypassing this code: + set(force "${${setting_NAME}-FORCE}") + if(force) + return() + endif() + + cmake_parse_arguments( + PARSE_ARGV 2 setting + "ALLOW_OTHER_VALUES;ADVANCED" + "TYPE" + "OPTIONS;DEFAULT;VALIDATE") + # Check for unknown arguments: + foreach(arg IN LISTS setting_UNPARSED_ARGUMENTS) + message(SEND_ERROR "Unrecognized argument: “${arg}”") + endforeach() + + # By default, settings are strings: + if(NOT DEFINED setting_TYPE) + set(setting_TYPE STRING) + endif() + + # More arg validation: + if(setting_TYPE STREQUAL "BOOL") + if(DEFINED setting_OPTIONS) + message(FATAL_ERROR [["OPTIONS" cannot be specified with type "BOOL"]]) + endif() + endif() + + # Normalize the doc string for easier writing of doc strings at call sites: + string(REGEX REPLACE "\n[ ]*" " " doc "${setting_DOC}") + # Update the doc string with options: + if(DEFINED setting_OPTIONS) + string(REPLACE ";" ", " opts "${setting_OPTIONS}") + string(APPEND doc " (Options: ${opts})") + endif() + + # Get the default option value: + unset(DEFAULT) + if(DEFINED setting_DEFAULT) + _mongo_compute_default(DEFAULT "${setting_DEFAULT}") + endif() + + if(DEFINED DEFAULT) + # Add that to the doc message: + string(APPEND doc " (Default is “${DEFAULT}”)") + # Check that the default is actually a valid option: + if(DEFINED setting_OPTIONS + AND NOT DEFAULT IN_LIST setting_OPTIONS + AND NOT setting_ALLOW_OTHER_VALUES) + message(AUTHOR_WARNING "${setting_NAME}: Setting's default value is “${DEFAULT}”, which is not one of the provided setting options (${opts})") + endif() + + # Reset "AUTO" values to the default + if(NOT "AUTO" IN_LIST setting_OPTIONS AND "$CACHE{${setting_NAME}}" STREQUAL "AUTO") + message(WARNING "Replacing ${setting_NAME}=“AUTO” with default value ${setting_NAME}=“${DEFAULT}”") + unset("${setting_NAME}" CACHE) + endif() + endif() + + # Actually define it now: + set("${setting_NAME}" "${DEFAULT}" CACHE "${setting_TYPE}" "${doc}") + # Variable properties: + set_property(CACHE "${setting_NAME}" PROPERTY HELPSTRING "${doc}") + set_property(CACHE "${setting_NAME}" PROPERTY TYPE "${setting_TYPE}") + set_property(CACHE "${setting_NAME}" PROPERTY ADVANCED "${setting_ADVANCED}") + if(setting_OPTIONS) + set_property(CACHE "${setting_NAME}" PROPERTY STRINGS "${setting_OPTIONS}") + endif() + + # Report what we set: + message(DEBUG "Build setting ${setting_NAME} := “${${setting_NAME}}”") + + # Validation of options: + if(NOT setting_ALLOW_OTHER_VALUES AND (DEFINED setting_OPTIONS) AND (NOT ("${${setting_NAME}}" IN_LIST setting_OPTIONS))) + message(FATAL_ERROR "The value of “${setting_NAME}” must be one of [${opts}] (Got ${setting_NAME}=“${${setting_NAME}}”)") + endif() + if(setting_TYPE STREQUAL "BOOL" + AND NOT setting_ALLOW_OTHER_VALUES + AND NOT "${${setting_NAME}}" MATCHES "^(TRUE|FALSE|ON|OFF|YES|NO|0|1)$") + message(WARNING "The value of ${setting_NAME}=“${${setting_NAME}}” is not a regular boolean value") + endif() + + # Custom validation: + if(DEFINED setting_VALIDATE) + cmake_parse_arguments(validate "" "CODE" "" ${setting_VALIDATE}) + if(DEFINED validate_CODE) + _mongo_eval_cmake("" "${validate_CODE}") + endif() + if(validate_UNPARSED_ARGUMENTS) + message(FATAL_ERROR "Unrecognized VALIDATE options: ${validate_UNPARSED_ARGUMENTS}") + endif() + endif() +endfunction() + +#[[ Implements DEFAULT setting value logic ]] +function(_mongo_compute_default outvar arglist) + # Clear the value in the caller: + unset("${outvar}" PARENT_SCOPE) + + # Parse arguments: + cmake_parse_arguments(dflt "" "" "DEVEL;AUDIT" ${arglist}) + + # Developer-mode options: + if(DEFINED dflt_DEVEL AND MONGODB_DEVELOPER) + _mongo_compute_default(tmp "${dflt_DEVEL}") + message(DEBUG "Detected MONGODB_DEVELOPER: Default of ${setting_NAME} is “${tmp}”") + set("${outvar}" "${tmp}" PARENT_SCOPE) + return() + # Audit-mode options: + elseif(DEFINED dflt_AUDIT AND (MONGODB_DEVELOPER OR MONGODB_BUILD_AUDIT)) + _mongo_compute_default(tmp "${dflt_AUDIT}") + message(DEBUG "Detected MONGODB_BUILD_AUDIT: Default of ${setting_NAME} is “${tmp}”") + set("${outvar}" "${tmp}" PARENT_SCOPE) + return() + endif() + + # Parse everything else: + set(other_args "${dflt_UNPARSED_ARGUMENTS}") + cmake_parse_arguments(dflt "" "VALUE;EVAL" "" ${other_args}) + + if(DEFINED dflt_VALUE) + # Simple value for the default + if(DEFINED dflt_EVAL) + message(FATAL_ERROR "Only one of VALUE or EVAL may be specified for a DEFAULT") + endif() + set("${outvar}" "${dflt_VALUE}" PARENT_SCOPE) + elseif(DEFINED dflt_EVAL) + # Evaluate some code to determine the default + _mongo_eval_cmake(DEFAULT "${dflt_EVAL}") + set("${outvar}" "${DEFAULT}" PARENT_SCOPE) + if(DEFINED DEFAULT) + message(DEBUG "Computed default ${setting_NAME} value to be “${DEFAULT}”") + else() + message(DEBUG "No default for ${setting_NAME} was computed. Default will be an empty string.") + endif() + elseif(dflt_UNPARSED_ARGUMENTS) + message(FATAL_ERROR + "${setting_NAME}: " + "DEFAULT got unexpected arguments: ${dflt_UNPARSED_ARGUMENTS}") + endif() +endfunction() + +#[==[ +Define a new boolean build setting:: + + mongo_bool_setting( + + [DEFAULT [[DEVEL|AUDIT] [VALUE | EVAL ]] ...] + [VALIDATE [CODE ]] + [ADVANCED] [ALLOW_OTHER_VALUES] + ) + +This is a shorthand for defining a boolean setting. See mongo_setting() for more +option information. The TYPE of the setting will be BOOL, and the implicit +default value for the setting will be ON if neither DEFAULT nor DEFAULT_EVAL are +specified. + +]==] +function(mongo_bool_setting name doc) + set(args ${ARGN}) + # Inject "ON" as a default: + if(NOT "DEFAULT" IN_LIST args AND NOT "DEFAULT_EVAL" IN_LIST args) + list(APPEND args DEFAULT VALUE ON) + endif() + mongo_setting("${name}" "${doc}" TYPE BOOL ${args}) +endfunction() + +# Set the variable named by 'out' to the 'if_true' or 'if_false' value based on 'cond' +function(_mongo_pick out if_true if_false cond) + string(REPLACE "'" "\"" cond "${cond}") + _mongo_eval_cmake("res" "set(res [[${if_false}]])\nif(${cond})\nset(res [[${if_true}]])\nendif()") + set("${out}" "${res}" PARENT_SCOPE) +endfunction() + +# Evaluate CMake code , and lift the given variables into the caller's scope. +function(_mongo_eval_cmake get_variables code) + # Set a name that is unlikely to collide: + set(__eval_liftvars "${get_variables}") + # Clear the values before we evaluate the code: + foreach(__varname IN LISTS __eval_liftvars) + unset("${__varname}" PARENT_SCOPE) + unset("${__varname}") + endforeach() + # We do the "eval" the old fashion way, since we can't yet use cmake_language() + message(TRACE "Evaluating CMake code:\n\n${code}") + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/_eval.tmp.cmake" "${code}") + include("${CMAKE_CURRENT_BINARY_DIR}/_eval.tmp.cmake") + # Lift the variables into the caller's scope + foreach(__varname IN LISTS __eval_liftvars) + if(DEFINED "${__varname}") + message(TRACE "Eval variable result: ${__varname}=${${__varname}}") + set("${__varname}" "${${__varname}}" PARENT_SCOPE) + endif() + endforeach() +endfunction() \ No newline at end of file diff --git a/build/cmake/Sanitizers.cmake b/build/cmake/Sanitizers.cmake index 6826d71851..df3d30746b 100644 --- a/build/cmake/Sanitizers.cmake +++ b/build/cmake/Sanitizers.cmake @@ -1,8 +1,15 @@ include (CheckCSourceCompiles) include (CMakePushCheckState) +include (MongoSettings) -## Directly control the options passed to -fsanitize -set (MONGO_SANITIZE "" CACHE STRING "Semicolon/comma-separated list of sanitizers to apply when building") +mongo_setting ( + MONGO_SANITIZE "Semicolon/comma-separated list of sanitizers to apply when building" + DEFAULT + AUDIT EVAL [[ + if(NOT MSVC) + set(DEFAULT "address,undefined") + endif() + ]]) # Replace commas with semicolons for the genex string(REPLACE ";" "," _sanitize "${MONGO_SANITIZE}") From 3f0fc79d27844c65a9919058feae1f135cf90bb4 Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Wed, 10 May 2023 19:35:24 +0000 Subject: [PATCH 02/13] Better logging when setting change --- build/cmake/MongoSettings.cmake | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/build/cmake/MongoSettings.cmake b/build/cmake/MongoSettings.cmake index d1bb68bfbc..275971ab42 100644 --- a/build/cmake/MongoSettings.cmake +++ b/build/cmake/MongoSettings.cmake @@ -140,6 +140,12 @@ function(mongo_setting setting_NAME setting_DOC) endif() endif() + # Detect the previous value + unset(prev_val) + if(DEFINED "CACHE{${setting_NAME}}") + set(prev_val "$CACHE{${setting_NAME}-PREV}") + endif() + # Actually define it now: set("${setting_NAME}" "${DEFAULT}" CACHE "${setting_TYPE}" "${doc}") # Variable properties: @@ -151,7 +157,14 @@ function(mongo_setting setting_NAME setting_DOC) endif() # Report what we set: - message(DEBUG "Build setting ${setting_NAME} := “${${setting_NAME}}”") + if(NOT DEFINED prev_val) + message(STATUS "Setting: ${setting_NAME} := “${${setting_NAME}}”") + elseif("${${setting_NAME}}" STREQUAL prev_val) + message(DEBUG "Setting: ${setting_NAME} := “${${setting_NAME}}” (Unchanged)") + else() + message(STATUS "Setting: ${setting_NAME} := “${${setting_NAME}}” (Old value was “${prev_val}”)") + endif() + set("${setting_NAME}-PREV" "${${setting_NAME}}" CACHE INTERNAL "Prior value of ${setting_NAME}") # Validation of options: if(NOT setting_ALLOW_OTHER_VALUES AND (DEFINED setting_OPTIONS) AND (NOT ("${${setting_NAME}}" IN_LIST setting_OPTIONS))) From c6381110990565f2223ce267b8d98287e66916d7 Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Wed, 10 May 2023 19:47:29 +0000 Subject: [PATCH 03/13] More option debugging --- .evergreen/scripts/compile-unix.sh | 2 ++ build/cmake/MongoSettings.cmake | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.evergreen/scripts/compile-unix.sh b/.evergreen/scripts/compile-unix.sh index e6ad84f516..0c3f9fc849 100755 --- a/.evergreen/scripts/compile-unix.sh +++ b/.evergreen/scripts/compile-unix.sh @@ -58,6 +58,8 @@ configure_flags_append_if_not_null() { fi } +configure_flags_append --log-level=debug +configure_flags_append --log-context configure_flags_append "-DCMAKE_INSTALL_PREFIX=${install_dir}" configure_flags_append "-DCMAKE_PREFIX_PATH=${cmake_prefix_path}" configure_flags_append "-DCMAKE_SKIP_RPATH=TRUE" # Avoid hardcoding absolute paths to dependency libraries. diff --git a/build/cmake/MongoSettings.cmake b/build/cmake/MongoSettings.cmake index 275971ab42..902c362ca0 100644 --- a/build/cmake/MongoSettings.cmake +++ b/build/cmake/MongoSettings.cmake @@ -142,7 +142,7 @@ function(mongo_setting setting_NAME setting_DOC) # Detect the previous value unset(prev_val) - if(DEFINED "CACHE{${setting_NAME}}") + if(DEFINED "CACHE{${setting_NAME}}-PREV") set(prev_val "$CACHE{${setting_NAME}-PREV}") endif() From 244e35d9d855f09d4985765715b83398694f7f29 Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Wed, 10 May 2023 20:24:55 +0000 Subject: [PATCH 04/13] Better feature detection on SSL/AWS --- CMakeLists.txt | 13 ++++++++++++- build/cmake/MongoSettings.cmake | 14 +++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c8d1d87f4..3dc4932bae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,7 +124,18 @@ mongo_bool_setting( mongo_bool_setting(ENABLE_ICU "Enable ICU support, needed for non-ASCII usernames and passwords") mongo_bool_setting(ENABLE_CLIENT_SIDE_ENCRYPTION "Enable In-Use Encryption support. Requires additional support libraries.") -mongo_bool_setting(ENABLE_MONGODB_AWS_AUTH "Enable support for the MONGODB-AWS authentication mechanism") +if(ENABLE_SSL) + # Only define this as a setting if we have an SSL enabled, otherwise we should implicitly disable it: + mongo_bool_setting(ENABLE_MONGODB_AWS_AUTH "Enable support for the MONGODB-AWS authentication mechanism") +elseif(ENABLE_MONGODB_AWS_AUTH) + # The user requested AWS auth (or it is still enabled in the cache), but we don't have ENABLE_SSL + message(SEND_ERROR "ENABLE_MONGODB_AWS_AUTH requires ENABLE_SSL") +else() + # ENABLE_SSL is disabled, and ENABLE_MONGODB_AWS_AUTH is unset, so we set it off for this configure + # run + set(ENABLE_MONGODB_AWS_AUTH OFF) + message(DEBUG "ENABLE_MONGODB_AWS_AUTH is implicitly disabled because ENABLE_SSL is “${ENABLE_SSL}”") +endif() # Optional features that are DISABLED by default: mongo_bool_setting(ENABLE_RDTSCP "Enable fast performance counters using the Intel RDTSCP instruction" diff --git a/build/cmake/MongoSettings.cmake b/build/cmake/MongoSettings.cmake index 902c362ca0..b99f1fa1d5 100644 --- a/build/cmake/MongoSettings.cmake +++ b/build/cmake/MongoSettings.cmake @@ -142,8 +142,13 @@ function(mongo_setting setting_NAME setting_DOC) # Detect the previous value unset(prev_val) - if(DEFINED "CACHE{${setting_NAME}}-PREV") + if(DEFINED "CACHE{${setting_NAME}-PREV}") set(prev_val "$CACHE{${setting_NAME}-PREV}") + message(DEBUG "Detected previous value was “${prev_val}”") + elseif(DEFINED "CACHE{${setting_NAME}}") + message(DEBUG "Externally defined to be “${${setting_NAME}}”") + else() + message(DEBUG "No previous value detected") endif() # Actually define it now: @@ -158,11 +163,11 @@ function(mongo_setting setting_NAME setting_DOC) # Report what we set: if(NOT DEFINED prev_val) - message(STATUS "Setting: ${setting_NAME} := “${${setting_NAME}}”") + message(VERBOSE "Setting: ${setting_NAME} := “${${setting_NAME}}”") elseif("${${setting_NAME}}" STREQUAL prev_val) message(DEBUG "Setting: ${setting_NAME} := “${${setting_NAME}}” (Unchanged)") else() - message(STATUS "Setting: ${setting_NAME} := “${${setting_NAME}}” (Old value was “${prev_val}”)") + message(VERBOSE "Setting: ${setting_NAME} := “${${setting_NAME}}” (Old value was “${prev_val}”)") endif() set("${setting_NAME}-PREV" "${${setting_NAME}}" CACHE INTERNAL "Prior value of ${setting_NAME}") @@ -190,6 +195,7 @@ endfunction() #[[ Implements DEFAULT setting value logic ]] function(_mongo_compute_default outvar arglist) + list(APPEND CMAKE_MESSAGE_CONTEXT default) # Clear the value in the caller: unset("${outvar}" PARENT_SCOPE) @@ -198,12 +204,14 @@ function(_mongo_compute_default outvar arglist) # Developer-mode options: if(DEFINED dflt_DEVEL AND MONGODB_DEVELOPER) + list(APPEND CMAKE_MESSAGE_CONTEXT "devel") _mongo_compute_default(tmp "${dflt_DEVEL}") message(DEBUG "Detected MONGODB_DEVELOPER: Default of ${setting_NAME} is “${tmp}”") set("${outvar}" "${tmp}" PARENT_SCOPE) return() # Audit-mode options: elseif(DEFINED dflt_AUDIT AND (MONGODB_DEVELOPER OR MONGODB_BUILD_AUDIT)) + list(APPEND CMAKE_MESSAGE_CONTEXT "audit") _mongo_compute_default(tmp "${dflt_AUDIT}") message(DEBUG "Detected MONGODB_BUILD_AUDIT: Default of ${setting_NAME} is “${tmp}”") set("${outvar}" "${tmp}" PARENT_SCOPE) From cf055f8378adf839ffc37a56f626ed92b748a74b Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Wed, 10 May 2023 20:37:01 +0000 Subject: [PATCH 05/13] Use new CMake for debug-compile-aws --- .evergreen/generated_configs/legacy-config.yml | 8 +++++--- .../legacy_config_generator/evergreen_config_lib/tasks.py | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.evergreen/generated_configs/legacy-config.yml b/.evergreen/generated_configs/legacy-config.yml index 9d9335c10e..6674bd2270 100644 --- a/.evergreen/generated_configs/legacy-config.yml +++ b/.evergreen/generated_configs/legacy-config.yml @@ -1841,10 +1841,12 @@ tasks: script: |- set -o errexit # Compile test-awsauth. Disable unnecessary dependencies since test-awsauth is copied to a remote Ubuntu 18.04 ECS cluster for testing, which may not have all dependent libraries. - . .evergreen/scripts/find-cmake.sh + set -euo pipefail + . .evergreen/scripts/find-cmake-latest.sh + cmake=$(find_cmake_latest) export CC='${CC}' - $CMAKE -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF . - $CMAKE --build . --target test-awsauth + "$cmake" -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF . + "$cmake" --build . --target test-awsauth - func: upload-build - name: test-aws-openssl-regular-latest depends_on: diff --git a/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py b/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py index 93f05c82eb..2f44acf979 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py +++ b/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py @@ -825,10 +825,12 @@ def _check_allowed(self): aws_compile_task = NamedTask('debug-compile-aws', commands=[shell_mongoc(''' # Compile test-awsauth. Disable unnecessary dependencies since test-awsauth is copied to a remote Ubuntu 18.04 ECS cluster for testing, which may not have all dependent libraries. - . .evergreen/scripts/find-cmake.sh + set -euo pipefail + . .evergreen/scripts/find-cmake-latest.sh + cmake=$(find_cmake_latest) export CC='${CC}' - $CMAKE -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF . - $CMAKE --build . --target test-awsauth + "$cmake" -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF . + "$cmake" --build . --target test-awsauth '''), func('upload-build')]) all_tasks = chain(all_tasks, [aws_compile_task]) From 772d9e6026fdec1585deaffa2bcaa1560acb14ed Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Thu, 11 May 2023 20:04:08 +0000 Subject: [PATCH 06/13] legacy config generation is now type-safe and checked --- .../generated_configs/legacy-config.yml | 16 +- .../evergreen_config_generator/__init__.py | 73 +- .../evergreen_config_generator/functions.py | 124 +- .../evergreen_config_generator/taskgroups.py | 48 +- .../evergreen_config_generator/tasks.py | 247 ++- .../evergreen_config_generator/variants.py | 18 +- .../evergreen_config_lib/__init__.py | 25 +- .../evergreen_config_lib/functions.py | 2 +- .../evergreen_config_lib/taskgroups.py | 5 +- .../evergreen_config_lib/tasks.py | 1781 +++++++++-------- .../evergreen_config_lib/testazurekms.py | 180 +- .../evergreen_config_lib/testgcpkms.py | 122 +- .../evergreen_config_lib/variants.py | 875 ++++---- .../generate-evergreen-config.py | 36 +- 14 files changed, 1993 insertions(+), 1559 deletions(-) diff --git a/.evergreen/generated_configs/legacy-config.yml b/.evergreen/generated_configs/legacy-config.yml index 6674bd2270..4772382b63 100644 --- a/.evergreen/generated_configs/legacy-config.yml +++ b/.evergreen/generated_configs/legacy-config.yml @@ -1204,7 +1204,7 @@ tasks: - func: run-tests vars: AUTH: auth - CLIENT_SIDE_ENCRYPTION: 'on' + CLIENT_SIDE_ENCRYPTION: 'ON' COVERAGE: 'ON' SSL: openssl - func: upload coverage @@ -9124,11 +9124,11 @@ tasks: tar czf testazurekms.tgz testazurekms/* AZUREKMS_SRC="testazurekms.tgz" \ AZUREKMS_DST="./" \ - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh echo "Copying files ... end" echo "Untarring file ... begin" AZUREKMS_CMD="tar xf testazurekms.tgz" \ - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh echo "Untarring file ... end" - command: shell.exec type: test @@ -9141,7 +9141,7 @@ tasks: export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools AZUREKMS_CMD="LD_LIBRARY_PATH=./testazurekms MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' ./testazurekms/test-azurekms" \ - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - name: testazurekms-fail-task commands: - func: fetch-source @@ -9165,7 +9165,7 @@ tasks: KEY_NAME='${testazurekms_keyname}' \ KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \ EXPECT_ERROR='Error from Azure IMDS server' \ - ./mongoc/src/libmongoc/test-azurekms + ./mongoc/src/libmongoc/test-azurekms - name: testgcpkms-task commands: - func: fetch-source @@ -9265,9 +9265,9 @@ task_groups: script: |- set -o errexit DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools - export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} - export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh + export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} + export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh setup_group_can_fail_task: true setup_group_timeout_secs: 1800 tasks: diff --git a/.evergreen/legacy_config_generator/evergreen_config_generator/__init__.py b/.evergreen/legacy_config_generator/evergreen_config_generator/__init__.py index 77dba46427..c25199a847 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_generator/__init__.py +++ b/.evergreen/legacy_config_generator/evergreen_config_generator/__init__.py @@ -15,26 +15,42 @@ import sys from collections import OrderedDict as OD +from typing import Any, Iterable, Mapping, MutableMapping, MutableSequence, Sequence, Union + +Scalar = Union[str, bool, int, None, float] +"YAML simple schema scalar types" +ValueSequence = Sequence["Value"] +"Sequence of YAML simple values" +MutableValueArray = MutableSequence["Value"] +"A mutable sequence of JSON values" +ValueMapping = Mapping[Scalar, "Value"] +"A YAML mapping type (arbitrary scalars as keys)" +MutableValueMapping = MutableMapping[Scalar, "Value"] +"A mutable YAML mapping type" +Value = Union[ValueSequence, ValueMapping, Scalar] +"Any YAML simple value" +MutableValue = Union[MutableValueMapping, MutableValueArray, Scalar] +"Any YAML simple value, which may be a mutable sequence or map" + +ValueOrderedDict = OD[Scalar, Value] +"An OrderedDict of YAML values" + try: import yaml - import yamlordereddictloader + import yamlordereddictloader # type: ignore except ImportError: - sys.stderr.write( - "try 'pip install -r evergreen_config_generator/requirements.txt'\n") + sys.stderr.write("try 'pip install -r evergreen_config_generator/requirements.txt'\n") raise class ConfigObject(object): - def __init__(self, *args, **kwargs): - super(ConfigObject, self).__init__() - @property - def name(self): - return 'UNSET' + def name(self) -> str: + return "UNSET" - def to_dict(self): - return OD([('name', self.name)]) + def to_dict(self) -> Value: + return OD([("name", self.name)]) # We want legible YAML tasks: @@ -51,36 +67,38 @@ def to_dict(self): # Write values compactly except multiline strings, which use "|" style. Write # tag sets as lists. + class _Dumper(yamlordereddictloader.Dumper): - def __init__(self, *args, **kwargs): - super(_Dumper, self).__init__(*args, **kwargs) + def __init__(self, *args: Value, **kwargs: Value): + super().__init__(*args, **kwargs) # type: ignore self.add_representer(set, type(self).represent_set) # Use "multi_representer" to represent all subclasses of ConfigObject. - self.add_multi_representer(ConfigObject, - type(self).represent_config_object) + self.add_multi_representer(ConfigObject, type(self).represent_config_object) - def represent_scalar(self, tag, value, style=None): - if isinstance(value, (str)) and '\n' in value: - style = '|' - return super(_Dumper, self).represent_scalar(tag, value, style) + def represent_scalar(self, tag: str, value: Value, style: str | None = None) -> yaml.ScalarNode: + if isinstance(value, (str)) and "\n" in value: + style = "|" + return super().represent_scalar(tag, value, style) # type: ignore - def represent_set(self, data): - return super(_Dumper, self).represent_list(sorted(data)) + def represent_set(self, data: Iterable[Value]) -> yaml.MappingNode: + return super().represent_list(sorted(set(data))) # type: ignore - def represent_config_object(self, obj): - return super(_Dumper, self).represent_data(obj.to_dict()) + def represent_config_object(self, obj: ConfigObject) -> yaml.Node: + d = obj.to_dict() + return super().represent_data(d) # type: ignore -def yaml_dump(obj): +def yaml_dump(obj: Any): return yaml.dump(obj, Dumper=_Dumper, default_flow_style=False) -def generate(config, path): +def generate(config: Any, path: str): """Dump config to a file as YAML. config is a dict, preferably an OrderedDict. path is a file path. """ - f = open(path, 'w+') - f.write('''#################################### + f = open(path, "w+") + f.write( + """#################################### # Evergreen configuration # # Generated with evergreen_config_generator from @@ -89,5 +107,6 @@ def generate(config, path): # DO NOT EDIT THIS FILE # #################################### -''') +""" + ) f.write(yaml_dump(config)) diff --git a/.evergreen/legacy_config_generator/evergreen_config_generator/functions.py b/.evergreen/legacy_config_generator/evergreen_config_generator/functions.py index 403c0da248..86025b8a60 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_generator/functions.py +++ b/.evergreen/legacy_config_generator/evergreen_config_generator/functions.py @@ -17,96 +17,106 @@ from evergreen_config_generator import ConfigObject +from . import Value, MutableValueMapping, ValueMapping, ValueOrderedDict -def func(func_name, **kwargs): - od = OD([('func', func_name)]) + +def func(func_name: str, **kwargs: Value) -> MutableValueMapping: + od: MutableValueMapping = OD([("func", func_name)]) if kwargs: - od['vars'] = OD(sorted(kwargs.items())) + od["vars"] = OD(sorted(kwargs.items())) return od -def s3_put(remote_file, project_path=True, **kwargs): +def s3_put(remote_file: str, project_path: bool = True, **kwargs: Value) -> ValueMapping: if project_path: - remote_file = '${project}/' + remote_file - - od = OD([ - ('command', 's3.put'), - ('params', OD([ - ('aws_key', '${aws_key}'), - ('aws_secret', '${aws_secret}'), - ('remote_file', remote_file), - ('bucket', 'mciuploads'), - ('permissions', 'public-read')]))]) - - od['params'].update(kwargs) - return od - - -def strip_lines(s): - return '\n'.join(line for line in s.split('\n') if line.strip()) - - -def shell_exec(script, - test=True, - errexit=True, - xtrace=False, - silent=False, - continue_on_err=False, - working_dir=None, - background=False, - add_expansions_to_env=False, - redirect_standard_error_to_output=False, - ): - dedented = '' + remote_file = "${project}/" + remote_file + + return ValueOrderedDict( + [ + ("command", "s3.put"), + ( + "params", + ValueOrderedDict( + ( + ("aws_key", "${aws_key}"), + ("aws_secret", "${aws_secret}"), + ("remote_file", remote_file), + ("bucket", "mciuploads"), + ("permissions", "public-read"), + *kwargs.items(), + ) + ), + ), + ] + ) + + +def strip_lines(s: str) -> str: + return "\n".join(line for line in s.split("\n") if line.strip()) + + +def shell_exec( + script: str, + test: bool = True, + errexit: bool = True, + xtrace: bool = False, + silent: bool = False, + continue_on_err: bool = False, + working_dir: str | None = None, + background: bool = False, + add_expansions_to_env: bool = False, + redirect_standard_error_to_output: bool = False, +) -> ValueMapping: + dedented = "" if errexit: - dedented += 'set -o errexit\n' + dedented += "set -o errexit\n" if xtrace: - dedented += 'set -o xtrace\n' + dedented += "set -o xtrace\n" dedented += dedent(strip_lines(script)) - command = OD([('command', 'shell.exec')]) + command = ValueOrderedDict([("command", "shell.exec")]) if test: - command['type'] = 'test' + command["type"] = "test" - command['params'] = OD() + command["params"] = OD() if silent: - command['params']['silent'] = True + command["params"]["silent"] = True if working_dir is not None: - command['params']['working_dir'] = working_dir + command["params"]["working_dir"] = working_dir if continue_on_err: - command['params']['continue_on_err'] = True + command["params"]["continue_on_err"] = True if background: - command['params']['background'] = True + command["params"]["background"] = True if add_expansions_to_env: - command['params']['add_expansions_to_env'] = True + command["params"]["add_expansions_to_env"] = True if redirect_standard_error_to_output: - command['params']['redirect_standard_error_to_output'] = True + command["params"]["redirect_standard_error_to_output"] = True - command['params']['shell'] = 'bash' - command['params']['script'] = dedented + command["params"]["shell"] = "bash" + command["params"]["script"] = dedented return command -def targz_pack(target, source_dir, *include): - return OD([ - ('command', 'archive.targz_pack'), - ('params', OD([ - ('target', target), - ('source_dir', source_dir), - ('include', list(include))]))]) +def targz_pack(target: str, source_dir: str, *include: str) -> ValueMapping: + return OD( + [ + ("command", "archive.targz_pack"), + ("params", OD([("target", target), ("source_dir", source_dir), ("include", list(include))])), + ] + ) class Function(ConfigObject): - def __init__(self, *commands): + def __init__(self, *commands: Value): super(Function, self).__init__() self.commands = commands - def to_dict(self): + def to_dict(self) -> Value: return list(self.commands) diff --git a/.evergreen/legacy_config_generator/evergreen_config_generator/taskgroups.py b/.evergreen/legacy_config_generator/evergreen_config_generator/taskgroups.py index 6b1a4e3de3..669555a316 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_generator/taskgroups.py +++ b/.evergreen/legacy_config_generator/evergreen_config_generator/taskgroups.py @@ -12,36 +12,48 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import MutableMapping from evergreen_config_generator import ConfigObject +from . import Value, ValueSequence + class TaskGroup(ConfigObject): - def __init__(self, name): - super(TaskGroup, self).__init__() + def __init__(self, name: str): self._task_group_name = name + self.setup_group: ValueSequence | None = None + self.teardown_group: ValueSequence | None = None + self.setup_task: str | None = None + self.teardown_task: str | None = None + self.max_hosts: str | None = None + self.timeout: str | None = None + self.setup_group_can_fail_task: bool | None = None + self.setup_group_timeout_secs: int | None = None + self.share_processes: str | None = None + self.tasks: ValueSequence | None = None @property - def name(self): + def name(self) -> str: return self._task_group_name - def to_dict(self): - v = super(TaskGroup, self).to_dict() + def to_dict(self) -> Value: + v = super().to_dict() + assert isinstance(v, MutableMapping) # See possible TaskGroup attributes from the Evergreen wiki: # https://github.com/evergreen-ci/evergreen/wiki/Project-Configuration-Files#task-groups attrs = [ - 'setup_group', - 'teardown_group', - 'setup_task', - 'teardown_task', - 'max_hosts', - 'timeout', - 'setup_group_can_fail_task', - 'setup_group_timeout_secs', - 'setup_group_can_fail_task', - 'share_processes', - 'tasks' - ] - + "setup_group", + "teardown_group", + "setup_task", + "teardown_task", + "max_hosts", + "timeout", + "setup_group_can_fail_task", + "setup_group_timeout_secs", + "share_processes", + "tasks", + ] + for i in attrs: if getattr(self, i, None): v[i] = getattr(self, i) diff --git a/.evergreen/legacy_config_generator/evergreen_config_generator/tasks.py b/.evergreen/legacy_config_generator/evergreen_config_generator/tasks.py index a697be4719..68f06048c3 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_generator/tasks.py +++ b/.evergreen/legacy_config_generator/evergreen_config_generator/tasks.py @@ -13,149 +13,218 @@ # limitations under the License. from collections import OrderedDict as OD -from itertools import product - -try: - # Python 3 abstract base classes. - import collections.abc as abc -except ImportError: - import collections as abc +import copy +from itertools import chain, product +import itertools +from typing import ClassVar, Iterable, Literal, Mapping, MutableMapping, Sequence, Union from evergreen_config_generator import ConfigObject from evergreen_config_generator.functions import func +from . import Value, MutableValueMapping, ValueSequence + + +DependencySpec = Union[str, Mapping[str, Value]] + class Task(ConfigObject): - def __init__(self, *args, **kwargs): - super(Task, self).__init__(*args, **kwargs) - self.tags = set() - self.options = OD() - self.depends_on = None - self.commands = kwargs.pop('commands', None) or [] - tags = kwargs.pop('tags', None) - if tags: - self.add_tags(*tags) - depends_on = kwargs.pop('depends_on', None) - if depends_on: - self.add_dependency(depends_on) - - if 'exec_timeout_secs' in kwargs: - self.options['exec_timeout_secs'] = kwargs.pop('exec_timeout_secs') - - name_prefix = 'test' - - def add_tags(self, *args): - self.tags.update(args) - - def has_tags(self, *args): - return bool(self.tags.intersection(args)) - - def add_dependency(self, dependency): - if not isinstance(dependency, abc.Mapping): - dependency = OD([('name', dependency)]) - - if self.depends_on is None: - self.depends_on = dependency - elif isinstance(self.depends_on, abc.Mapping): - self.depends_on = [self.depends_on, dependency] - else: - self.depends_on.append(dependency) + def __init__( + self, + task_name: str | None = None, + commands: Iterable[Value] = (), + tags: Iterable[str] = (), + depends_on: Iterable[DependencySpec] = (), + exec_timeout_secs: int | None = None, + ): + self._name = task_name + self._tags = list(tags) + self.options: MutableValueMapping = OD() + self.commands: ValueSequence = list(commands) + self.exec_timeout_secs = exec_timeout_secs + self._depends_on = list(map(self._normal_dep, depends_on)) + + if exec_timeout_secs is not None: + self.options["exec_timeout_secs"] = exec_timeout_secs - def display(self, axis_name): - value = getattr(self, axis_name) - # E.g., if self.auth is False, return 'noauth'. - if value is False: - return 'no' + axis_name + @property + def dependencies(self) -> Sequence[Mapping[str, Value]]: + main = list(self._depends_on) + main.extend(map(self._normal_dep, self.additional_dependencies())) + return tuple(main) - if value is True: - return axis_name + def _normal_dep(self, spec: DependencySpec) -> Mapping[str, Value]: + if isinstance(spec, str): + return OD([("name", spec)]) + return spec + + @property + def tags(self) -> Sequence[str]: + return tuple(sorted(chain(self.additional_tags(), self._tags))) + + def pre_commands(self) -> Iterable[Value]: + return () + + def main_commands(self) -> Iterable[Value]: + return () - return value + def post_commands(self) -> Iterable[Value]: + return () - def on_off(self, *args, **kwargs): - assert not (args and kwargs) - if args: - axis_name, = args - return 'on' if getattr(self, axis_name) else 'off' + def additional_dependencies(self) -> Iterable[DependencySpec]: + return () + + @property + def name(self) -> str: + assert self._name is not None, f'Task {self} did not set a name, and did not override the "name" property' + return self._name - (axis_name, value), = kwargs.items() - return 'on' if getattr(self, axis_name) == value else 'off' + def additional_tags(self) -> Iterable[str]: + return () + + def add_dependency(self, dependency: DependencySpec): + if isinstance(dependency, str): + dependency = OD([("name", dependency)]) + + self._depends_on.append(dependency) def to_dict(self): - task = super(Task, self).to_dict() + task: MutableValueMapping = super().to_dict() # type: ignore + assert isinstance(task, MutableMapping) if self.tags: - task['tags'] = self.tags + task["tags"] = list(self.tags) task.update(self.options) - if self.depends_on: - task['depends_on'] = self.depends_on - task['commands'] = self.commands + deps: Sequence[MutableValueMapping] = list(self.dependencies) # type: ignore + if deps: + if len(deps) == 1: + task["depends_on"] = OD(deps[0]) + else: + task["depends_on"] = copy.deepcopy(deps) + task["commands"] = list( + itertools.chain( + self.pre_commands(), + self.main_commands(), + self.commands, + self.post_commands(), + ) + ) return task -class NamedTask(Task): - def __init__(self, task_name, commands=None, **kwargs): - super(NamedTask, self).__init__(commands=commands, **kwargs) - self._task_name = task_name - - @property - def name(self): - return self._task_name +NamedTask = Task class FuncTask(NamedTask): - def __init__(self, task_name, *args, **kwargs): - commands = [func(func_name) for func_name in args] - super(FuncTask, self).__init__(task_name, commands=commands, **kwargs) + def __init__( + self, + task_name: str, + functions: Iterable[str], + tags: Iterable[str] = (), + depends_on: Iterable[DependencySpec] = (), + exec_timeout_secs: int | None = None, + ): + commands = [func(func_name) for func_name in functions] + super().__init__(task_name, commands, tags=tags, depends_on=depends_on, exec_timeout_secs=exec_timeout_secs) + super(FuncTask, self).__init__(task_name, commands=commands) class Prohibited(Exception): pass -def require(rule): +def require(rule: bool) -> None: if not rule: raise Prohibited() -def prohibit(rule): +def prohibit(rule: bool) -> None: if rule: raise Prohibited() -def both_or_neither(rule0, rule1): +def both_or_neither(rule0: bool, rule1: bool) -> None: if rule0: require(rule1) else: prohibit(rule1) +class SettingsAccess: + def __init__(self, inst: "MatrixTask") -> None: + self._task = inst + + def __getattr__(self, __setting: str) -> str | bool: + return self._task.setting_value(__setting) + + class MatrixTask(Task): - axes = OD() + axes: ClassVar[Mapping[str, Sequence[str | bool]]] = OD() - def __init__(self, *args, **kwargs): - axis_dict = OD() - for name, values in self.axes.items(): - # First value for each axis is the default value. - axis_dict[name] = kwargs.pop(name, values[0]) + def __init__(self, settings: Mapping[str, str | bool]): + super().__init__() + self._settings = {k: v for k, v in settings.items()} + for axis, options in type(self).axes.items(): + if axis not in self._settings: + self._settings[axis] = options[0] - super(MatrixTask, self).__init__(*args, **kwargs) - self.__dict__.update(axis_dict) + def display(self, axis_name: str) -> str: + value = self.setting_value(axis_name) + if value is False: + # E.g., if self.auth is False, return 'noauth'. + return f"no{axis_name}" + elif value is True: + return axis_name + else: + return value + + def on_off(self, key: str, val: str) -> Literal["on", "off"]: + return "on" if self.setting_value(key) == val else "off" + + @property + def name(self) -> str: + return "-".join(self.name_parts()) + + def name_parts(self) -> Iterable[str]: + raise NotImplementedError + + @property + def settings(self) -> SettingsAccess: + return SettingsAccess(self) + + def setting_value(self, axis: str) -> str | bool: + assert ( + axis in type(self).axes.keys() + ), f'Attempted to inspect setting "{axis}", which is not defined for this task type' + return self._settings[axis] + + def setting_eq(self, axis: str, val: str | bool) -> bool: + current = self.setting_value(axis) + options = type(self).axes[axis] + assert ( + val in options + ), f'Looking for value "{val}" on setting "{axis}", but that is not a supported option (Expects one of {options})' + return current == val + + def is_valid_combination(self) -> bool: + try: + return self.do_is_valid_combination() + except Prohibited: + print(f"Ignoring invalid combination {self.name!r}") + return False + + def do_is_valid_combination(self) -> bool: + return True @classmethod def matrix(cls): for cell in product(*cls.axes.values()): axis_values = dict(zip(cls.axes, cell)) - task = cls(**axis_values) + task = cls(settings=axis_values) if task.allowed: yield task @property def allowed(self): try: - self._check_allowed() - return True + return self.do_is_valid_combination() except Prohibited: return False - - def _check_allowed(self): - pass \ No newline at end of file diff --git a/.evergreen/legacy_config_generator/evergreen_config_generator/variants.py b/.evergreen/legacy_config_generator/evergreen_config_generator/variants.py index b22276504b..ca46139887 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_generator/variants.py +++ b/.evergreen/legacy_config_generator/evergreen_config_generator/variants.py @@ -12,12 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Iterable, Mapping from evergreen_config_generator import ConfigObject +from . import ValueMapping + class Variant(ConfigObject): - def __init__(self, name, display_name, run_on, tasks, expansions=None, - batchtime=None): + def __init__( + self, + name: str, + display_name: str, + run_on: list[str] | str, + tasks: Iterable[str | ValueMapping], + expansions: Mapping[str, str] | None = None, + batchtime: int | None = None, + ): super(Variant, self).__init__() self._variant_name = name self.display_name = display_name @@ -32,7 +42,7 @@ def name(self): def to_dict(self): v = super(Variant, self).to_dict() - for i in 'display_name', 'expansions', 'run_on', 'tasks', 'batchtime': + for i in "display_name", "expansions", "run_on", "tasks", "batchtime": if getattr(self, i): v[i] = getattr(self, i) - return v \ No newline at end of file + return v diff --git a/.evergreen/legacy_config_generator/evergreen_config_lib/__init__.py b/.evergreen/legacy_config_generator/evergreen_config_lib/__init__.py index 254768fe25..e435f4fe54 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_lib/__init__.py +++ b/.evergreen/legacy_config_generator/evergreen_config_lib/__init__.py @@ -15,5 +15,26 @@ from evergreen_config_generator.functions import shell_exec -def shell_mongoc(script, *args, **kwargs): - return shell_exec(script, *args, working_dir='mongoc', **kwargs) +def shell_mongoc( + script: str, + test: bool = True, + errexit: bool = True, + xtrace: bool = False, + silent: bool = False, + continue_on_err: bool = False, + background: bool = False, + add_expansions_to_env: bool = False, + redirect_standard_error_to_output: bool = False, +): + return shell_exec( + script, + working_dir="mongoc", + test=test, + errexit=errexit, + xtrace=xtrace, + silent=silent, + continue_on_err=continue_on_err, + background=background, + add_expansions_to_env=add_expansions_to_env, + redirect_standard_error_to_output=redirect_standard_error_to_output, + ) diff --git a/.evergreen/legacy_config_generator/evergreen_config_lib/functions.py b/.evergreen/legacy_config_generator/evergreen_config_lib/functions.py index 904d11eddd..46832315d4 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_lib/functions.py +++ b/.evergreen/legacy_config_generator/evergreen_config_lib/functions.py @@ -15,7 +15,7 @@ from collections import OrderedDict as OD from evergreen_config_generator.functions import ( - Function, func, s3_put, shell_exec, targz_pack) + Function, s3_put, shell_exec) from evergreen_config_lib import shell_mongoc build_path = '${build_variant}/${revision}/${version_id}/${build_id}' diff --git a/.evergreen/legacy_config_generator/evergreen_config_lib/taskgroups.py b/.evergreen/legacy_config_generator/evergreen_config_lib/taskgroups.py index f54b3033c8..33bee5771d 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_lib/taskgroups.py +++ b/.evergreen/legacy_config_generator/evergreen_config_lib/taskgroups.py @@ -12,4 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -all_task_groups = [] +from typing import Sequence +from evergreen_config_generator.taskgroups import TaskGroup + +all_task_groups: Sequence[TaskGroup] = [] diff --git a/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py b/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py index 2f44acf979..174ba462e7 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py +++ b/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py @@ -14,1023 +14,1186 @@ from collections import OrderedDict as OD from itertools import chain +from typing import ClassVar, Iterable, Literal, Mapping, MutableMapping, MutableSequence, Optional, Sequence -try: - # Python 3 abstract base classes. - import collections.abc as abc -except ImportError: - import collections as abc - -from evergreen_config_generator.functions import (func, s3_put) +from evergreen_config_generator import Value, Scalar +from evergreen_config_generator.functions import func, s3_put from evergreen_config_generator.tasks import ( - both_or_neither, MatrixTask, NamedTask, prohibit, require, Task) + both_or_neither, + MatrixTask, + NamedTask, + prohibit, + require, + Task, + DependencySpec, +) from evergreen_config_lib import shell_mongoc from pkg_resources import parse_version +ToggleStr = Literal["OFF", "ON"] +OptToggleStr = Optional[ToggleStr] +TopologyStr = Literal["server"] + + class CompileTask(NamedTask): - def __init__(self, task_name, tags=None, config='debug', - compression='default', continue_on_err=False, - suffix_commands=None, depends_on=None, - extra_script=None, prefix_commands=None, sanitize=(), - **kwargs): - super(CompileTask, self).__init__(task_name=task_name, - depends_on=depends_on, - tags=tags, - **kwargs) - - self.suffix_commands = suffix_commands or [] - self.prefix_commands = prefix_commands or [] - if extra_script: - self.extra_script = "\n" + extra_script - else: - self.extra_script = "" + cls_compile_sh_env: ClassVar[Mapping[str, str]] = {} + cls_tags: ClassVar[Sequence[str]] = () + cls_sanitize: ClassVar[Sequence[str]] = () + + def __init__( + self, + task_name: str, + tags: Iterable[str] = (), + config: str = "debug", + compression: str | None = "default", + suffix_commands: Iterable[Value] = (), + depends_on: Iterable[DependencySpec] = (), + prefix_commands: Iterable[Value] = (), + sanitize: Iterable[Literal["undefined", "address", "thread"]] = (), + *, + CFLAGS: str | None = None, + LDFLAGS: str | None = None, + EXTRA_CONFIGURE_FLAGS: str | None = None, + SSL: Literal["WINDOWS", "DARWIN", "OPENSSL", "OPENSSL_STATIC", "LIBRESSL", "OFF", None] = None, + ENABLE_SHM_COUNTERS: OptToggleStr = None, + CHECK_LOG: OptToggleStr = None, + TRACING: OptToggleStr = None, + SASL: Literal[None, "OFF", "AUTO", "CYRUS", "SSPI"] = None, + ENABLE_RDTSCP: OptToggleStr = None, + SRV: OptToggleStr = None, + TOPOLOGY: TopologyStr | None = None, + ): + super(CompileTask, self).__init__(task_name=task_name, depends_on=depends_on, tags=tags) + + self.suffix_commands = list(suffix_commands) + self.prefix_commands = list(prefix_commands) # Environment variables for .evergreen/scripts/compile.sh. - self.compile_sh_opt = kwargs - if config == 'debug': - self.compile_sh_opt['DEBUG'] = 'ON' + self.compile_sh_opt: dict[str, str] = {} + if config == "debug": + self.compile_sh_opt["DEBUG"] = "ON" else: - assert config == 'release' - self.compile_sh_opt['RELEASE'] = 'ON' - - if compression != 'default': - self.compile_sh_opt['SNAPPY'] = ( - 'ON' if compression in ('all', 'snappy') else 'OFF') - self.compile_sh_opt['ZLIB'] = ( - 'BUNDLED' if compression in ('all', 'zlib') else 'OFF') - self.compile_sh_opt['ZSTD'] = ( - 'ON' if compression in ('all', 'zstd') else 'OFF') + assert config == "release" + self.compile_sh_opt["RELEASE"] = "ON" + + if CFLAGS: + self.compile_sh_opt["CFLAGS"] = CFLAGS + if LDFLAGS: + self.compile_sh_opt["LDFLAGS"] = LDFLAGS + if EXTRA_CONFIGURE_FLAGS: + self.compile_sh_opt["EXTRA_CONFIGURE_FLAGS"] = EXTRA_CONFIGURE_FLAGS + if SSL: + self.compile_sh_opt["SSL"] = SSL + if ENABLE_SHM_COUNTERS: + self.compile_sh_opt["ENABLE_SHM_COUNTERS"] = ENABLE_SHM_COUNTERS + if CHECK_LOG: + self.compile_sh_opt["CHECK_LOG"] = CHECK_LOG + if TRACING: + self.compile_sh_opt["TRACING"] = TRACING + if SASL: + self.compile_sh_opt["SASL"] = SASL + if ENABLE_RDTSCP: + self.compile_sh_opt["ENABLE_RDTSCP"] = ENABLE_RDTSCP + if SRV: + self.compile_sh_opt["SRV"] = SRV + if TOPOLOGY: + self.compile_sh_opt["TOPOLOGY"] = TOPOLOGY + + if compression != "default": + self.compile_sh_opt["SNAPPY"] = "ON" if compression in ("all", "snappy") else "OFF" + self.compile_sh_opt["ZLIB"] = "BUNDLED" if compression in ("all", "zlib") else "OFF" + self.compile_sh_opt["ZSTD"] = "ON" if compression in ("all", "zstd") else "OFF" if sanitize: - self.compile_sh_opt['SANITIZE'] = ','.join(sanitize) + self.compile_sh_opt["SANITIZE"] = ",".join(sanitize) + + self.compile_sh_opt.update(type(self).cls_compile_sh_env) - self.continue_on_err = continue_on_err + def additional_script_env(self) -> Mapping[str, str]: + return {} def to_dict(self): task = super(CompileTask, self).to_dict() + commands = task["commands"] + assert isinstance(commands, MutableSequence), task - task['commands'].extend(self.prefix_commands) + commands.extend(self.prefix_commands) - script = 'env' + script = "env" for opt, value in sorted(self.compile_sh_opt.items()): script += ' %s="%s"' % (opt, value) - script += ' bash .evergreen/scripts/compile.sh' - script += self.extra_script - task['commands'].append(shell_mongoc( - script, add_expansions_to_env=True)) - task['commands'].append(func('upload-build')) - task['commands'].extend(self.suffix_commands) + script += " bash .evergreen/scripts/compile.sh" + commands.append(shell_mongoc(script, add_expansions_to_env=True)) + commands.append(func("upload-build")) + commands.extend(self.suffix_commands) return task + def additional_tags(self) -> Iterable[str]: + yield from super().additional_tags() + yield from self.cls_tags + class SpecialTask(CompileTask): - def __init__(self, *args, **kwargs): - super(SpecialTask, self).__init__(*args, **kwargs) - self.add_tags('special') + cls_tags: ClassVar[Sequence[str]] = ["special"] class CompileWithClientSideEncryption(CompileTask): - def __init__(self, *args, **kwargs): + cls_compile_sh_env: ClassVar[Mapping[str, str]] = dict( # Compiling with ClientSideEncryption support requires linking against the library libmongocrypt. - super(CompileWithClientSideEncryption, self).__init__(*args, - COMPILE_LIBMONGOCRYPT="ON", - EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON", - **kwargs) - self.add_tags('client-side-encryption', 'special') + COMPILE_LIBMONGOCRYPT="ON", + EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON", + ) + cls_tags: ClassVar[Sequence[str]] = "client-side-encryption", "special" class CompileWithClientSideEncryptionAsan(CompileTask): - def __init__(self, *args, **kwargs): - super(CompileWithClientSideEncryptionAsan, self).__init__(*args, - CFLAGS="-fno-omit-frame-pointer", - COMPILE_LIBMONGOCRYPT="ON", - CHECK_LOG="ON", - sanitize=[ - 'address'], - EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF", - PATH='/usr/lib/llvm-3.8/bin:$PATH', - **kwargs) - self.add_tags('client-side-encryption') + cls_compile_sh_env: ClassVar[Mapping[str, str]] = dict( + CFLAGS="-fno-omit-frame-pointer", + COMPILE_LIBMONGOCRYPT="ON", + CHECK_LOG="ON", + EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF", + PATH="/usr/lib/llvm-3.8/bin:$PATH", + ) + cls_tags: ClassVar[Sequence[str]] = ["client-side-encryption"] + cls_sanitize: ClassVar[Sequence[str]] = ["address"] class LinkTask(NamedTask): - def __init__(self, task_name, suffix_commands, orchestration=True, **kwargs): - if orchestration == 'ssl': + def __init__( + self, task_name: str, suffix_commands: Iterable[Value], orchestration: Literal[True, False, "ssl"] = True + ): + if orchestration == "ssl": # Actual value of SSL does not matter here so long as it is not 'nossl'. - bootstrap_commands = [ - func('fetch-det'), - func('bootstrap-mongo-orchestration', SSL="openssl") - ] + bootstrap_commands = [func("fetch-det"), func("bootstrap-mongo-orchestration", SSL="openssl")] elif orchestration: - bootstrap_commands = [ - func('fetch-det'), - func('bootstrap-mongo-orchestration') - ] + bootstrap_commands = [func("fetch-det"), func("bootstrap-mongo-orchestration")] else: bootstrap_commands = [] - super(LinkTask, self).__init__( + super().__init__( task_name=task_name, - depends_on=OD([('name', 'make-release-archive'), - ('variant', 'releng')]), - commands=bootstrap_commands + suffix_commands, - **kwargs) + depends_on=[OD([("name", "make-release-archive"), ("variant", "releng")])], + commands=bootstrap_commands + list(suffix_commands), + ) all_tasks = [ - CompileTask('hardened-compile', - tags=['hardened'], - compression=None, - CFLAGS='-fno-strict-overflow -D_FORTIFY_SOURCE=2 -fstack-protector-all -fPIE -O', - LDFLAGS='-pie -Wl,-z,relro -Wl,-z,now'), - CompileTask('debug-compile-compression-zlib', - tags=['zlib', 'compression'], - compression='zlib'), - CompileTask('debug-compile-compression-snappy', - tags=['snappy', 'compression'], - compression='snappy'), - CompileTask('debug-compile-compression-zstd', - tags=['zstd', 'compression'], - compression='zstd'), - CompileTask('debug-compile-compression', - tags=['zlib', 'snappy', 'zstd', 'compression'], - compression='all'), - CompileTask('debug-compile-no-align', - tags=['debug-compile'], - compression='zlib', - EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF"), - CompileTask('debug-compile-nosasl-nossl', - tags=['debug-compile', 'nosasl', 'nossl'], - SSL='OFF'), - CompileTask('debug-compile-lto', CFLAGS='-flto'), - CompileTask('debug-compile-lto-thin', CFLAGS='-flto=thin'), - CompileTask('debug-compile-no-counters', - tags=['debug-compile', 'no-counters'], - ENABLE_SHM_COUNTERS='OFF'), - SpecialTask('debug-compile-asan-clang', - tags=['debug-compile', 'asan-clang'], - compression='zlib', - CFLAGS='-fno-omit-frame-pointer', - CHECK_LOG='ON', - sanitize=['address'], - EXTRA_CONFIGURE_FLAGS='-DENABLE_EXTRA_ALIGNMENT=OFF'), - SpecialTask('debug-compile-asan-clang-openssl', - tags=['debug-compile', 'asan-clang'], - compression='zlib', - CFLAGS='-fno-omit-frame-pointer', - CHECK_LOG='ON', - sanitize=['address'], - EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF", - SSL='OPENSSL'), - CompileTask('compile-tracing', - TRACING='ON', CFLAGS='-Werror -Wno-cast-align'), - CompileTask('release-compile', - config='release', - depends_on=OD([('name', 'make-release-archive'), - ('variant', 'releng')])), - CompileTask('debug-compile-nosasl-openssl', - tags=['debug-compile', 'nosasl', 'openssl'], - SSL='OPENSSL'), - CompileTask('debug-compile-nosasl-openssl-static', - tags=['debug-compile', 'nosasl', 'openssl-static'], - SSL='OPENSSL_STATIC'), - CompileTask('debug-compile-nosasl-darwinssl', - tags=['debug-compile', 'nosasl', 'darwinssl'], - SSL='DARWIN'), - CompileTask('debug-compile-nosasl-winssl', - tags=['debug-compile', 'nosasl', 'winssl'], - SSL='WINDOWS'), - CompileTask('debug-compile-sasl-nossl', - tags=['debug-compile', 'sasl', 'nossl'], - SASL='AUTO', - SSL='OFF'), - CompileTask('debug-compile-sasl-openssl', - tags=['debug-compile', 'sasl', 'openssl'], - SASL='AUTO', - SSL='OPENSSL'), - CompileTask('debug-compile-sasl-openssl-static', - tags=['debug-compile', 'sasl', 'openssl-static'], - SASL='AUTO', - SSL='OPENSSL_STATIC'), - CompileTask('debug-compile-sasl-darwinssl', - tags=['debug-compile', 'sasl', 'darwinssl'], - SASL='AUTO', - SSL='DARWIN'), - CompileTask('debug-compile-sasl-winssl', - tags=['debug-compile', 'sasl', 'winssl'], - # Explicitly use CYRUS. - SASL='CYRUS', - SSL='WINDOWS'), - CompileTask('debug-compile-sspi-nossl', - tags=['debug-compile', 'sspi', 'nossl'], - SASL='SSPI', - SSL='OFF'), - CompileTask('debug-compile-sspi-openssl', - tags=['debug-compile', 'sspi', 'openssl'], - SASL='SSPI', - SSL='OPENSSL'), - CompileTask('debug-compile-sspi-openssl-static', - tags=['debug-compile', 'sspi', 'openssl-static'], - SASL='SSPI', - SSL='OPENSSL_STATIC'), - CompileTask('debug-compile-rdtscp', - ENABLE_RDTSCP='ON'), - CompileTask('debug-compile-sspi-winssl', - tags=['debug-compile', 'sspi', 'winssl'], - SASL='SSPI', - SSL='WINDOWS'), - CompileTask('debug-compile-nosrv', - tags=['debug-compile'], - SRV='OFF'), - LinkTask('link-with-cmake', - suffix_commands=[ - func('link sample program', BUILD_SAMPLE_WITH_CMAKE=1)]), - LinkTask('link-with-cmake-ssl', - suffix_commands=[ - func('link sample program', - BUILD_SAMPLE_WITH_CMAKE=1, - ENABLE_SSL="AUTO")]), - LinkTask('link-with-cmake-snappy', - suffix_commands=[ - func('link sample program', - BUILD_SAMPLE_WITH_CMAKE=1, - ENABLE_SNAPPY="ON")]), - LinkTask('link-with-cmake-mac', - suffix_commands=[ - func('link sample program', BUILD_SAMPLE_WITH_CMAKE=1)]), - LinkTask('link-with-cmake-deprecated', - suffix_commands=[ - func('link sample program', - BUILD_SAMPLE_WITH_CMAKE=1, - BUILD_SAMPLE_WITH_CMAKE_DEPRECATED=1)]), - LinkTask('link-with-cmake-ssl-deprecated', - suffix_commands=[ - func('link sample program', - BUILD_SAMPLE_WITH_CMAKE=1, - BUILD_SAMPLE_WITH_CMAKE_DEPRECATED=1, - ENABLE_SSL="AUTO")]), - LinkTask('link-with-cmake-snappy-deprecated', - suffix_commands=[ - func('link sample program', - BUILD_SAMPLE_WITH_CMAKE=1, - BUILD_SAMPLE_WITH_CMAKE_DEPRECATED=1, - ENABLE_SNAPPY="ON")]), - LinkTask('link-with-cmake-mac-deprecated', - suffix_commands=[ - func('link sample program', - BUILD_SAMPLE_WITH_CMAKE=1, - BUILD_SAMPLE_WITH_CMAKE_DEPRECATED=1)]), - LinkTask('link-with-cmake-windows', - suffix_commands=[func('link sample program MSVC')]), - LinkTask('link-with-cmake-windows-ssl', - suffix_commands=[ - func('link sample program MSVC', ENABLE_SSL="AUTO")], - orchestration='ssl'), - LinkTask('link-with-cmake-windows-snappy', - suffix_commands=[ - func('link sample program MSVC', ENABLE_SNAPPY="ON")]), - LinkTask('link-with-cmake-mingw', - suffix_commands=[func('link sample program mingw')]), - LinkTask('link-with-pkg-config', - suffix_commands=[func('link sample program')]), - LinkTask('link-with-pkg-config-mac', - suffix_commands=[func('link sample program')]), - LinkTask('link-with-pkg-config-ssl', - suffix_commands=[func('link sample program', ENABLE_SSL="AUTO")]), - LinkTask('link-with-bson', - suffix_commands=[func('link sample program bson')], - orchestration=False), - LinkTask('link-with-bson-mac', - suffix_commands=[func('link sample program bson')], - orchestration=False), - LinkTask('link-with-bson-windows', - suffix_commands=[func('link sample program MSVC bson')], - orchestration=False), - LinkTask('link-with-bson-mingw', - suffix_commands=[func('link sample program mingw bson')], - orchestration=False), - NamedTask('debian-package-build', - commands=[ - shell_mongoc('export IS_PATCH="${is_patch}"\n' - 'sh .evergreen/scripts/debian_package_build.sh'), - s3_put(local_file='deb.tar.gz', - remote_file='${branch_name}/mongo-c-driver-debian-packages-${CURRENT_VERSION}.tar.gz', - content_type='${content_type|application/x-gzip}'), - s3_put(local_file='deb.tar.gz', - remote_file='${branch_name}/${revision}/${version_id}/${build_id}/${execution}/mongo-c-driver-debian-packages.tar.gz', - content_type='${content_type|application/x-gzip}')]), - NamedTask('rpm-package-build', - commands=[ - shell_mongoc('sh .evergreen/scripts/build_snapshot_rpm.sh'), - s3_put(local_file='rpm.tar.gz', - remote_file='${branch_name}/mongo-c-driver-rpm-packages-${CURRENT_VERSION}.tar.gz', - content_type='${content_type|application/x-gzip}'), - s3_put(local_file='rpm.tar.gz', - remote_file='${branch_name}/${revision}/${version_id}/${build_id}/${execution}/mongo-c-driver-rpm-packages.tar.gz', - content_type='${content_type|application/x-gzip}')]), - NamedTask('install-uninstall-check-mingw', - depends_on=OD([('name', 'make-release-archive'), - ('variant', 'releng')]), - commands=[shell_mongoc(r''' + CompileTask( + "hardened-compile", + tags=["hardened"], + compression=None, + CFLAGS="-fno-strict-overflow -D_FORTIFY_SOURCE=2 -fstack-protector-all -fPIE -O", + LDFLAGS="-pie -Wl,-z,relro -Wl,-z,now", + ), + CompileTask("debug-compile-compression-zlib", tags=["zlib", "compression"], compression="zlib"), + CompileTask("debug-compile-compression-snappy", tags=["snappy", "compression"], compression="snappy"), + CompileTask("debug-compile-compression-zstd", tags=["zstd", "compression"], compression="zstd"), + CompileTask("debug-compile-compression", tags=["zlib", "snappy", "zstd", "compression"], compression="all"), + CompileTask( + "debug-compile-no-align", + tags=["debug-compile"], + compression="zlib", + EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF", + ), + CompileTask("debug-compile-nosasl-nossl", tags=["debug-compile", "nosasl", "nossl"], SSL="OFF"), + CompileTask("debug-compile-lto", CFLAGS="-flto"), + CompileTask("debug-compile-lto-thin", CFLAGS="-flto=thin"), + CompileTask("debug-compile-no-counters", tags=["debug-compile", "no-counters"], ENABLE_SHM_COUNTERS="OFF"), + SpecialTask( + "debug-compile-asan-clang", + tags=["debug-compile", "asan-clang"], + compression="zlib", + CFLAGS="-fno-omit-frame-pointer", + CHECK_LOG="ON", + sanitize=["address"], + EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF", + ), + SpecialTask( + "debug-compile-asan-clang-openssl", + tags=["debug-compile", "asan-clang"], + compression="zlib", + CFLAGS="-fno-omit-frame-pointer", + CHECK_LOG="ON", + sanitize=["address"], + EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF", + SSL="OPENSSL", + ), + CompileTask("compile-tracing", TRACING="ON", CFLAGS="-Werror -Wno-cast-align"), + CompileTask( + "release-compile", config="release", depends_on=[OD([("name", "make-release-archive"), ("variant", "releng")])] + ), + CompileTask("debug-compile-nosasl-openssl", tags=["debug-compile", "nosasl", "openssl"], SSL="OPENSSL"), + CompileTask( + "debug-compile-nosasl-openssl-static", tags=["debug-compile", "nosasl", "openssl-static"], SSL="OPENSSL_STATIC" + ), + CompileTask("debug-compile-nosasl-darwinssl", tags=["debug-compile", "nosasl", "darwinssl"], SSL="DARWIN"), + CompileTask("debug-compile-nosasl-winssl", tags=["debug-compile", "nosasl", "winssl"], SSL="WINDOWS"), + CompileTask("debug-compile-sasl-nossl", tags=["debug-compile", "sasl", "nossl"], SASL="AUTO", SSL="OFF"), + CompileTask("debug-compile-sasl-openssl", tags=["debug-compile", "sasl", "openssl"], SASL="AUTO", SSL="OPENSSL"), + CompileTask( + "debug-compile-sasl-openssl-static", + tags=["debug-compile", "sasl", "openssl-static"], + SASL="AUTO", + SSL="OPENSSL_STATIC", + ), + CompileTask("debug-compile-sasl-darwinssl", tags=["debug-compile", "sasl", "darwinssl"], SASL="AUTO", SSL="DARWIN"), + CompileTask( + "debug-compile-sasl-winssl", + tags=["debug-compile", "sasl", "winssl"], + # Explicitly use CYRUS. + SASL="CYRUS", + SSL="WINDOWS", + ), + CompileTask("debug-compile-sspi-nossl", tags=["debug-compile", "sspi", "nossl"], SASL="SSPI", SSL="OFF"), + CompileTask("debug-compile-sspi-openssl", tags=["debug-compile", "sspi", "openssl"], SASL="SSPI", SSL="OPENSSL"), + CompileTask( + "debug-compile-sspi-openssl-static", + tags=["debug-compile", "sspi", "openssl-static"], + SASL="SSPI", + SSL="OPENSSL_STATIC", + ), + CompileTask("debug-compile-rdtscp", ENABLE_RDTSCP="ON"), + CompileTask("debug-compile-sspi-winssl", tags=["debug-compile", "sspi", "winssl"], SASL="SSPI", SSL="WINDOWS"), + CompileTask("debug-compile-nosrv", tags=["debug-compile"], SRV="OFF"), + LinkTask("link-with-cmake", suffix_commands=[func("link sample program", BUILD_SAMPLE_WITH_CMAKE=1)]), + LinkTask( + "link-with-cmake-ssl", + suffix_commands=[func("link sample program", BUILD_SAMPLE_WITH_CMAKE=1, ENABLE_SSL="AUTO")], + ), + LinkTask( + "link-with-cmake-snappy", + suffix_commands=[func("link sample program", BUILD_SAMPLE_WITH_CMAKE=1, ENABLE_SNAPPY="ON")], + ), + LinkTask("link-with-cmake-mac", suffix_commands=[func("link sample program", BUILD_SAMPLE_WITH_CMAKE=1)]), + LinkTask( + "link-with-cmake-deprecated", + suffix_commands=[func("link sample program", BUILD_SAMPLE_WITH_CMAKE=1, BUILD_SAMPLE_WITH_CMAKE_DEPRECATED=1)], + ), + LinkTask( + "link-with-cmake-ssl-deprecated", + suffix_commands=[ + func( + "link sample program", + BUILD_SAMPLE_WITH_CMAKE=1, + BUILD_SAMPLE_WITH_CMAKE_DEPRECATED=1, + ENABLE_SSL="AUTO", + ) + ], + ), + LinkTask( + "link-with-cmake-snappy-deprecated", + suffix_commands=[ + func( + "link sample program", + BUILD_SAMPLE_WITH_CMAKE=1, + BUILD_SAMPLE_WITH_CMAKE_DEPRECATED=1, + ENABLE_SNAPPY="ON", + ) + ], + ), + LinkTask( + "link-with-cmake-mac-deprecated", + suffix_commands=[func("link sample program", BUILD_SAMPLE_WITH_CMAKE=1, BUILD_SAMPLE_WITH_CMAKE_DEPRECATED=1)], + ), + LinkTask("link-with-cmake-windows", suffix_commands=[func("link sample program MSVC")]), + LinkTask( + "link-with-cmake-windows-ssl", + suffix_commands=[func("link sample program MSVC", ENABLE_SSL="AUTO")], + orchestration="ssl", + ), + LinkTask("link-with-cmake-windows-snappy", suffix_commands=[func("link sample program MSVC", ENABLE_SNAPPY="ON")]), + LinkTask("link-with-cmake-mingw", suffix_commands=[func("link sample program mingw")]), + LinkTask("link-with-pkg-config", suffix_commands=[func("link sample program")]), + LinkTask("link-with-pkg-config-mac", suffix_commands=[func("link sample program")]), + LinkTask("link-with-pkg-config-ssl", suffix_commands=[func("link sample program", ENABLE_SSL="AUTO")]), + LinkTask("link-with-bson", suffix_commands=[func("link sample program bson")], orchestration=False), + LinkTask("link-with-bson-mac", suffix_commands=[func("link sample program bson")], orchestration=False), + LinkTask("link-with-bson-windows", suffix_commands=[func("link sample program MSVC bson")], orchestration=False), + LinkTask("link-with-bson-mingw", suffix_commands=[func("link sample program mingw bson")], orchestration=False), + NamedTask( + "debian-package-build", + commands=[ + shell_mongoc('export IS_PATCH="${is_patch}"\n' "sh .evergreen/scripts/debian_package_build.sh"), + s3_put( + local_file="deb.tar.gz", + remote_file="${branch_name}/mongo-c-driver-debian-packages-${CURRENT_VERSION}.tar.gz", + content_type="${content_type|application/x-gzip}", + ), + s3_put( + local_file="deb.tar.gz", + remote_file="${branch_name}/${revision}/${version_id}/${build_id}/${execution}/mongo-c-driver-debian-packages.tar.gz", + content_type="${content_type|application/x-gzip}", + ), + ], + ), + NamedTask( + "rpm-package-build", + commands=[ + shell_mongoc("sh .evergreen/scripts/build_snapshot_rpm.sh"), + s3_put( + local_file="rpm.tar.gz", + remote_file="${branch_name}/mongo-c-driver-rpm-packages-${CURRENT_VERSION}.tar.gz", + content_type="${content_type|application/x-gzip}", + ), + s3_put( + local_file="rpm.tar.gz", + remote_file="${branch_name}/${revision}/${version_id}/${build_id}/${execution}/mongo-c-driver-rpm-packages.tar.gz", + content_type="${content_type|application/x-gzip}", + ), + ], + ), + NamedTask( + "install-uninstall-check-mingw", + depends_on=[OD([("name", "make-release-archive"), ("variant", "releng")])], + commands=[ + shell_mongoc( + r""" export CC="C:/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin/gcc.exe" BSON_ONLY=1 cmd.exe /c .\\.evergreen\\scripts\\install-uninstall-check-windows.cmd - cmd.exe /c .\\.evergreen\\scripts\\install-uninstall-check-windows.cmd''')]), - NamedTask('install-uninstall-check-msvc', - depends_on=OD([('name', 'make-release-archive'), - ('variant', 'releng')]), - commands=[shell_mongoc(r''' + cmd.exe /c .\\.evergreen\\scripts\\install-uninstall-check-windows.cmd""" + ) + ], + ), + NamedTask( + "install-uninstall-check-msvc", + depends_on=[OD([("name", "make-release-archive"), ("variant", "releng")])], + commands=[ + shell_mongoc( + r""" export CC="Visual Studio 14 2015 Win64" BSON_ONLY=1 cmd.exe /c .\\.evergreen\\scripts\\install-uninstall-check-windows.cmd - cmd.exe /c .\\.evergreen\\scripts\\install-uninstall-check-windows.cmd''')]), - NamedTask('install-uninstall-check', - depends_on=OD([('name', 'make-release-archive'), - ('variant', 'releng')]), - commands=[shell_mongoc(r''' + cmd.exe /c .\\.evergreen\\scripts\\install-uninstall-check-windows.cmd""" + ) + ], + ), + NamedTask( + "install-uninstall-check", + depends_on=[OD([("name", "make-release-archive"), ("variant", "releng")])], + commands=[ + shell_mongoc( + r""" DESTDIR="$(pwd)/dest" sh ./.evergreen/scripts/install-uninstall-check.sh BSON_ONLY=1 sh ./.evergreen/scripts/install-uninstall-check.sh - sh ./.evergreen/scripts/install-uninstall-check.sh''')]), - CompileTask('debug-compile-with-warnings', - CFLAGS='-Werror -Wno-cast-align'), - CompileWithClientSideEncryption('debug-compile-sasl-openssl-cse', tags=[ - 'debug-compile', 'sasl', 'openssl'], SASL="AUTO", SSL="OPENSSL"), - CompileWithClientSideEncryption('debug-compile-sasl-openssl-static-cse', tags=[ - 'debug-compile', 'sasl', 'openssl-static'], SASL="AUTO", SSL="OPENSSL_STATIC"), - CompileWithClientSideEncryption('debug-compile-sasl-darwinssl-cse', tags=[ - 'debug-compile', 'sasl', 'darwinssl'], SASL="AUTO", SSL="DARWIN"), - CompileWithClientSideEncryption('debug-compile-sasl-winssl-cse', tags=[ - 'debug-compile', 'sasl', 'winssl'], SASL="AUTO", SSL="WINDOWS"), - CompileWithClientSideEncryptionAsan('debug-compile-asan-openssl-cse', tags=[ - 'debug-compile', 'asan-clang'], SSL="OPENSSL"), - CompileTask('debug-compile-nosasl-openssl-1.0.1', - prefix_commands=[func("install ssl", SSL="openssl-1.0.1u")], - CFLAGS="-Wno-redundant-decls", SSL="OPENSSL", SASL="OFF"), - NamedTask('build-and-test-with-toolchain', - commands=[ - OD([('command', 's3.get'), - ('params', OD([ - ('aws_key', '${toolchain_aws_key}'), - ('aws_secret', '${toolchain_aws_secret}'), - ('remote_file', - 'mongo-c-toolchain/${distro_id}/mongo-c-toolchain.tar.gz'), - ('bucket', 'mongo-c-toolchain'), - ('local_file', 'mongo-c-toolchain.tar.gz'), - ]))]), - shell_mongoc( - 'bash ./.evergreen/scripts/build-and-test-with-toolchain.sh') - ]) + sh ./.evergreen/scripts/install-uninstall-check.sh""" + ) + ], + ), + CompileTask("debug-compile-with-warnings", CFLAGS="-Werror -Wno-cast-align"), + CompileWithClientSideEncryption( + "debug-compile-sasl-openssl-cse", tags=["debug-compile", "sasl", "openssl"], SASL="AUTO", SSL="OPENSSL" + ), + CompileWithClientSideEncryption( + "debug-compile-sasl-openssl-static-cse", + tags=["debug-compile", "sasl", "openssl-static"], + SASL="AUTO", + SSL="OPENSSL_STATIC", + ), + CompileWithClientSideEncryption( + "debug-compile-sasl-darwinssl-cse", tags=["debug-compile", "sasl", "darwinssl"], SASL="AUTO", SSL="DARWIN" + ), + CompileWithClientSideEncryption( + "debug-compile-sasl-winssl-cse", tags=["debug-compile", "sasl", "winssl"], SASL="AUTO", SSL="WINDOWS" + ), + CompileWithClientSideEncryptionAsan( + "debug-compile-asan-openssl-cse", tags=["debug-compile", "asan-clang"], SSL="OPENSSL", sanitize=["address"] + ), + CompileTask( + "debug-compile-nosasl-openssl-1.0.1", + prefix_commands=[func("install ssl", SSL="openssl-1.0.1u")], + CFLAGS="-Wno-redundant-decls", + SSL="OPENSSL", + SASL="OFF", + ), + NamedTask( + "build-and-test-with-toolchain", + commands=[ + OD( + [ + ("command", "s3.get"), + ( + "params", + OD( + [ + ("aws_key", "${toolchain_aws_key}"), + ("aws_secret", "${toolchain_aws_secret}"), + ("remote_file", "mongo-c-toolchain/${distro_id}/mongo-c-toolchain.tar.gz"), + ("bucket", "mongo-c-toolchain"), + ("local_file", "mongo-c-toolchain.tar.gz"), + ] + ), + ), + ] + ), + shell_mongoc("bash ./.evergreen/scripts/build-and-test-with-toolchain.sh"), + ], + ), ] class CoverageTask(MatrixTask): - axes = OD([('version', ['latest']), - ('topology', ['replica_set']), - ('auth', [True]), - ('sasl', ['sasl']), - ('ssl', ['openssl']), - ('cse', [False, True])]) - - def __init__(self, *args, **kwargs): - super(CoverageTask, self).__init__(*args, **kwargs) - - self.name_prefix = 'test-coverage' - - self.add_tags('test-coverage') - self.add_tags(self.version) - + axes = OD( + [ + ("version", ["latest"]), + ("topology", ["replica_set"]), + ("auth", [True]), + ("sasl", ["sasl"]), + ("ssl", ["openssl"]), + ("cse", [False, True]), + ] + ) + + def additional_tags(self) -> Iterable[str]: + yield from super().additional_tags() + yield "test-coverage" + yield str(self.settings.version) if self.cse: - self.add_tags("client-side-encryption") + yield "client-side-encryption" - @property - def name(self): - def name_part(axis_name): - part = self.display(axis_name) - if part == 'replica_set': - return 'replica-set' - elif part == 'sharded_cluster': - return 'sharded' - return part - - return self.name_prefix + '-' + '-'.join( - name_part(axis_name) for axis_name in self.axes - if getattr(self, axis_name) or axis_name in ('auth', 'sasl', 'ssl')) + def name_parts(self) -> Iterable[str]: + yield "test-coverage" + yield self.display("version") + yield self.display("topology").replace("_", "-") + yield from map(self.display, ("auth", "sasl", "ssl")) + if self.settings.cse: + yield "cse" - def to_dict(self): - task = super(CoverageTask, self).to_dict() - commands = task['commands'] + @property + def cse(self) -> bool: + return bool(self.settings.cse) + def post_commands(self) -> Iterable[Value]: if self.cse: - commands.append(func('compile coverage', - SASL='AUTO', - SSL='OPENSSL', - COMPILE_LIBMONGOCRYPT='ON', - EXTRA_CONFIGURE_FLAGS='EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON"')) + yield func( + "compile coverage", + SASL="AUTO", + SSL="OPENSSL", + COMPILE_LIBMONGOCRYPT="ON", + EXTRA_CONFIGURE_FLAGS='EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON"', + ) else: - commands.append(func('compile coverage', - SASL='AUTO', - SSL='OPENSSL')) - - commands.append(func('fetch-det')) - commands.append(func('bootstrap-mongo-orchestration', - MONGODB_VERSION=self.version, - TOPOLOGY=self.topology, - AUTH='auth' if self.auth else 'noauth', - SSL=self.display('ssl'))) - extra = { - 'COVERAGE': 'ON' - } - - commands.append(func('run-simple-http-server')) + yield func("compile coverage", SASL="AUTO", SSL="OPENSSL") + + yield func("fetch-det") + yield func( + "bootstrap-mongo-orchestration", + MONGODB_VERSION=self.settings.version, + TOPOLOGY=self.settings.topology, + AUTH=self.display("auth"), + SSL=self.display("ssl"), + ) + yield func("run-simple-http-server") + extra = {"COVERAGE": "ON"} if self.cse: - extra["CLIENT_SIDE_ENCRYPTION"] = "on" - commands.append(func('run-mock-kms-servers')) - commands.append(func('run-tests', - AUTH=self.display('auth'), - SSL=self.display('ssl'), - **extra)) - commands.append(func('upload coverage')) - commands.append(func('update codecov.io')) + extra["CLIENT_SIDE_ENCRYPTION"] = "ON" + yield func("run-mock-kms-servers") + yield func("run-tests", AUTH=self.display("auth"), SSL=self.display("ssl"), **extra) + yield func("upload coverage") + yield func("update codecov.io") - return task - - def _check_allowed(self): + def do_is_valid_combination(self) -> bool: # Limit coverage tests to test-coverage-latest-replica-set-auth-sasl-openssl (+ cse). - require(self.topology == 'replica_set') - require(self.auth) - require(self.sasl == 'sasl') - require(self.ssl == 'openssl') - require(self.version == 'latest') - - # Address sanitizer only with auth+SSL or no auth + no SSL. - if self.auth: - require(self.ssl == 'openssl') - else: - prohibit(self.ssl) + require(self.setting_eq("topology", "replica_set")) + require(self.setting_eq("sasl", "sasl")) + require(self.setting_eq("ssl", "openssl")) + require(self.setting_eq("version", "latest")) + require(self.settings.auth is True) - if self.cse: - require(self.version == 'latest' or parse_version( - self.version) >= parse_version("4.2")) - if self.version == 'latest' or parse_version(self.version) >= parse_version("6.0"): - # FLE 2.0 Client-Side Encryption tasks on 6.0 require a non-standalone topology. - require(self.topology in ('server', 'replica_set')) - else: - require(self.topology == 'server') - # limit to SASL=AUTO to reduce redundant tasks. - require(self.sasl) - require(self.sasl != 'sspi') - require(self.ssl) + if not self.cse: + # No further requirements + return True + + # CSE has extra requirements + if self.settings.version != "latest": + # We only work with 4.2 or newer for CSE + require(parse_version(str(self.settings.version)) >= parse_version("4.2")) + return True all_tasks = chain(all_tasks, CoverageTask.matrix()) class DNSTask(MatrixTask): - axes = OD([('auth', [False, True]), - ('loadbalanced', [False, True]), - ('ssl', ['openssl', 'winssl', 'darwinssl']) - ]) + axes = OD( + [ + ("auth", [False, True]), + ("loadbalanced", [False, True]), + ("ssl", ["openssl", "winssl", "darwinssl"]), + ] + ) - name_prefix = 'test-dns' + name_prefix = "test-dns" - def __init__(self, *args, **kwargs): - super(DNSTask, self).__init__(*args, **kwargs) - sasl = 'sspi' if self.ssl == 'winssl' else 'sasl' - self.add_dependency('debug-compile-%s-%s' % - (sasl, self.display('ssl'))) + def additional_dependencies(self) -> Iterable[DependencySpec]: + yield self.build_task_name @property - def name(self): - return self.name_prefix + '-' + '-'.join( - self.display(axis_name) for axis_name in self.axes - if getattr(self, axis_name)) - - def to_dict(self): - task = super(MatrixTask, self).to_dict() - commands = task['commands'] - commands.append( - func('fetch-build', BUILD_NAME=self.depends_on['name'])) - commands.append(func('fetch-det')) - - if self.loadbalanced: - orchestration = func('bootstrap-mongo-orchestration', - TOPOLOGY='sharded_cluster', - AUTH='auth' if self.auth else 'noauth', - SSL='ssl', - LOAD_BALANCER='on') + def build_task_name(self) -> str: + sasl = "sspi" if self.settings.ssl == "winssl" else "sasl" + return f'debug-compile-{sasl}-{self.display("ssl")}' + + def name_parts(self) -> Iterable[str]: + yield "test-dns" + if self.settings.auth: + yield "auth" + if self.settings.loadbalanced: + yield "loadbalanced" + yield self.display("ssl") + + def post_commands(self) -> Iterable[Value]: + yield func("fetch-build", BUILD_NAME=self.build_task_name) + yield func("fetch-det") + if self.settings.loadbalanced: + orchestration = func( + "bootstrap-mongo-orchestration", + TOPOLOGY="sharded_cluster", + AUTH="auth" if self.settings.auth else "noauth", + SSL="ssl", + LOAD_BALANCER="on", + ) else: - orchestration = func('bootstrap-mongo-orchestration', - TOPOLOGY='replica_set', - AUTH='auth' if self.auth else 'noauth', - SSL='ssl') - - if self.auth: - orchestration['vars']['AUTHSOURCE'] = 'thisDB' - - commands.append(orchestration) - - dns = 'on' - if self.loadbalanced: - dns = 'loadbalanced' - commands.append(func("fetch-det")) - commands.append(func( - "start load balancer", MONGODB_URI="mongodb://localhost:27017,localhost:27018")) - elif self.auth: - dns = 'dns-auth' - commands.append(func('run-tests', - SSL='ssl', - AUTH=self.display('auth'), - DNS=dns)) - - return task - - def _check_allowed(self): - prohibit(self.loadbalanced and self.auth) + orchestration = func( + "bootstrap-mongo-orchestration", + TOPOLOGY="replica_set", + AUTH="auth" if self.settings.auth else "noauth", + SSL="ssl", + ) + + if self.settings.auth: + vars = orchestration["vars"] + assert isinstance(vars, MutableMapping) + vars["AUTHSOURCE"] = "thisDB" + + yield orchestration + + dns = "on" + if self.settings.loadbalanced: + dns = "loadbalanced" + yield func("fetch-det") + yield func("start load balancer", MONGODB_URI="mongodb://localhost:27017,localhost:27018") + elif self.settings.auth: + dns = "dns-auth" + yield func("run-tests", SSL="ssl", AUTH=self.display("auth"), DNS=dns) + + def do_is_valid_combination(self) -> bool: + prohibit(bool(self.settings.loadbalanced) and bool(self.settings.auth)) # Load balancer tests only run on some Linux hosts in Evergreen until CDRIVER-4041 is resolved. - prohibit(self.loadbalanced and self.ssl in ["darwinssl", "winssl"]) + prohibit(bool(self.settings.loadbalanced) and self.settings.ssl in ["darwinssl", "winssl"]) + return True all_tasks = chain(all_tasks, DNSTask.matrix()) class CompressionTask(MatrixTask): - axes = OD([('compression', ['zlib', 'snappy', 'zstd', 'compression'])]) - name_prefix = 'test-latest-server' + axes = OD([("compression", ["zlib", "snappy", "zstd", "compression"])]) + name_prefix = "test-latest-server" - def __init__(self, *args, **kwargs): - super(CompressionTask, self).__init__(*args, **kwargs) - self.add_dependency('debug-compile-' + self._compressor_suffix()) - self.add_tags('compression', 'latest') - self.add_tags(*self._compressor_list()) + def additional_dependencies(self) -> Iterable[DependencySpec]: + yield self.build_task_name @property - def name(self): - return self.name_prefix + '-' + self._compressor_suffix() - - def to_dict(self): - task = super(CompressionTask, self).to_dict() - commands = task['commands'] - commands.append( - func('fetch-build', BUILD_NAME=self.depends_on['name'])) - commands.append(func('fetch-det')) - - if self.compression == 'compression': - orchestration_file = 'snappy-zlib-zstd' + def build_task_name(self) -> str: + return f"debug-compile-{self._compressor_suffix()}" + + def additional_tags(self) -> Iterable[str]: + yield from super().additional_tags() + yield "compression" + yield "latest" + yield from self._compressor_list() + + def name_parts(self) -> Iterable[str]: + return [self.name_prefix, self._compressor_suffix()] + + def post_commands(self) -> Iterable[Value]: + yield func("fetch-build", BUILD_NAME=self.build_task_name) + yield func("fetch-det") + if self.settings.compression == "compression": + orc_file = "snappy-zlib-zstd" else: - orchestration_file = self.compression - - commands.append(func('bootstrap-mongo-orchestration', - AUTH='noauth', - SSL='nossl', - ORCHESTRATION_FILE=orchestration_file)) - commands.append(func('run-simple-http-server')) - commands.append(func('run-tests', - AUTH='noauth', - SSL='nossl', - COMPRESSORS=','.join(self._compressor_list()))) - - return task + orc_file = self.settings.compression + yield func("bootstrap-mongo-orchestration", AUTH="noauth", SSL="nossl", ORCHESTRATION_FILE=orc_file) + yield func("run-simple-http-server") + yield func("run-tests", AUTH="noauth", SSL="nossl", COMPRESSORS=",".join(self._compressor_list())) def _compressor_suffix(self): - if self.compression == 'zlib': - return 'compression-zlib' - elif self.compression == 'snappy': - return 'compression-snappy' - elif self.compression == 'zstd': - return 'compression-zstd' + if self.settings.compression == "zlib": + return "compression-zlib" + elif self.settings.compression == "snappy": + return "compression-snappy" + elif self.settings.compression == "zstd": + return "compression-zstd" else: - return 'compression' + return "compression" def _compressor_list(self): - if self.compression == 'zlib': - return ['zlib'] - elif self.compression == 'snappy': - return ['snappy'] - elif self.compression == 'zstd': - return ['zstd'] + if self.settings.compression == "zlib": + return ["zlib"] + elif self.settings.compression == "snappy": + return ["snappy"] + elif self.settings.compression == "zstd": + return ["zstd"] else: - return ['snappy', 'zlib', 'zstd'] + return ["snappy", "zlib", "zstd"] all_tasks = chain(all_tasks, CompressionTask.matrix()) class SpecialIntegrationTask(NamedTask): - def __init__(self, task_name, depends_on='debug-compile-sasl-openssl', - suffix_commands=None, uri=None, - tags=None, version='latest', topology='server'): - commands = [func('fetch-build', BUILD_NAME=depends_on), - func('fetch-det'), - func('bootstrap-mongo-orchestration', - MONGODB_VERSION=version, - TOPOLOGY=topology), - func('run-simple-http-server'), - func('run-tests', URI=uri)] + (suffix_commands or []) - super(SpecialIntegrationTask, self).__init__(task_name, - commands=commands, - depends_on=depends_on, - tags=tags) - - -all_tasks = chain(all_tasks, [ - # Verify that retryWrites=true is ignored with standalone. - SpecialIntegrationTask('retry-true-latest-server', - uri='mongodb://localhost/?retryWrites=true'), - SpecialIntegrationTask('test-latest-server-hardened', - 'hardened-compile', - tags=['hardened', 'latest']), -]) + def __init__( + self, + task_name: str, + main_dep: str = "debug-compile-sasl-openssl", + uri: str | None = None, + tags: Iterable[str] = (), + version: str = "latest", + topology: str = "server", + ): + self._main_dep = main_dep + super().__init__(task_name, depends_on=[self._main_dep], tags=tags) + self._uri = uri + self._version = version + self._topo = topology + + def pre_commands(self) -> Iterable[Value]: + yield func("fetch-build", BUILD_NAME=self._main_dep) + yield func("fetch-det") + yield func("bootstrap-mongo-orchestration", MONGODB_VERSION=self._version, TOPOLOGY=self._topo) + yield func("run-simple-http-server") + yield func("run-tests", URI=self._uri) + + +all_tasks = chain( + all_tasks, + [ + # Verify that retryWrites=true is ignored with standalone. + SpecialIntegrationTask("retry-true-latest-server", uri="mongodb://localhost/?retryWrites=true"), + SpecialIntegrationTask("test-latest-server-hardened", "hardened-compile", tags=["hardened", "latest"]), + ], +) class AuthTask(MatrixTask): - axes = OD([('sasl', ['sasl', 'sspi', False]), - ('ssl', ['openssl', 'openssl-static', 'darwinssl', 'winssl'])]) + axes = OD([("sasl", ["sasl", "sspi", False]), ("ssl", ["openssl", "openssl-static", "darwinssl", "winssl"])]) - name_prefix = 'authentication-tests' + name_prefix = "authentication-tests" - def __init__(self, *args, **kwargs): - super(AuthTask, self).__init__(*args, **kwargs) - self.add_tags('authentication-tests', - self.display('ssl'), - self.display('sasl')) + def additional_tags(self) -> Iterable[str]: + yield from super().additional_tags() + yield "authentication-tests" + yield self.display("ssl") + yield self.display("sasl") - self.add_dependency('debug-compile-%s-%s' % ( - self.display('sasl'), self.display('ssl'))) + def additional_dependencies(self) -> Iterable[DependencySpec]: + yield self.build_task_name - self.commands.extend([ - func('fetch-build', BUILD_NAME=self.depends_on['name']), - func('prepare-kerberos'), - func('run auth tests')]) + def post_commands(self) -> Iterable[Value]: + yield func("fetch-build", BUILD_NAME=self.build_task_name) + yield func("prepare-kerberos") + yield func("run auth tests") @property - def name(self): - rv = self.name_prefix + '-' + self.display('ssl') - if self.sasl: - return rv - else: - return rv + '-nosasl' + def build_task_name(self) -> str: + return f'debug-compile-{self.display("sasl")}-{self.display("ssl")}' - def _check_allowed(self): - both_or_neither(self.ssl == 'winssl', self.sasl == 'sspi') - if not self.sasl: - require(self.ssl == 'openssl') + def name_parts(self) -> Iterable[str]: + yield self.name_prefix + yield self.display("ssl") + if not self.settings.sasl: + yield "nosasl" + + def do_is_valid_combination(self) -> bool: + both_or_neither(self.settings.ssl == "winssl", self.settings.sasl == "sspi") + if not self.settings.sasl: + require(self.settings.ssl == "openssl") + return True all_tasks = chain(all_tasks, AuthTask.matrix()) class PostCompileTask(NamedTask): - def __init__(self, *args, **kwargs): - super(PostCompileTask, self).__init__(*args, **kwargs) - self.commands.insert( - 0, func('fetch-build', BUILD_NAME=self.depends_on['name'])) - - -all_tasks = chain(all_tasks, [ - PostCompileTask( - 'test-asan-memcheck-mock-server', - tags=['test-asan'], - depends_on='debug-compile-asan-clang', - commands=[func('run mock server tests', ASAN='on', SSL='ssl')]), - PostCompileTask( - 'test-mongohouse', - tags=[], - depends_on='debug-compile-sasl-openssl', - commands=[func('fetch-det'), - func('build mongohouse'), - func('run mongohouse'), - func('test mongohouse')]), - NamedTask( - 'authentication-tests-asan-memcheck', - tags=['authentication-tests', 'asan'], - commands=[ - shell_mongoc(""" + def __init__(self, name: str, tags: Iterable[str], get_build: str, commands: Iterable[Value]): + super().__init__(name, commands=commands, tags=tags, depends_on=[get_build]) + self._dep = get_build + + def pre_commands(self) -> Iterable[Value]: + yield func("fetch-build", BUILD_NAME=self._dep) + + +all_tasks = chain( + all_tasks, + [ + PostCompileTask( + "test-asan-memcheck-mock-server", + tags=["test-asan"], + get_build="debug-compile-asan-clang", + commands=[func("run mock server tests", ASAN="on", SSL="ssl")], + ), + PostCompileTask( + "test-mongohouse", + tags=[], + get_build="debug-compile-sasl-openssl", + commands=[func("fetch-det"), func("build mongohouse"), func("run mongohouse"), func("test mongohouse")], + ), + NamedTask( + "authentication-tests-asan-memcheck", + tags=["authentication-tests", "asan"], + commands=[ + shell_mongoc( + """ env SANITIZE=address DEBUG=ON SASL=AUTO SSL=OPENSSL EXTRA_CONFIGURE_FLAGS='-DENABLE_EXTRA_ALIGNMENT=OFF' bash .evergreen/scripts/compile.sh - """, add_expansions_to_env=True), - func('prepare-kerberos'), - func('run auth tests', ASAN='on')]), - PostCompileTask( - 'test-versioned-api', - tags=['versioned-api'], - depends_on='debug-compile-nosasl-openssl', - commands=[func('fetch-det'), - func('bootstrap-mongo-orchestration', TOPOLOGY='server', AUTH='auth', - SSL='ssl', MONGODB_VERSION='5.0', REQUIRE_API_VERSION='true'), - func('run-simple-http-server'), - func('run-tests', MONGODB_API_VERSION=1, AUTH='auth', SSL='ssl')]), - PostCompileTask( - 'test-versioned-api-accept-version-two', - tags=['versioned-api'], - depends_on='debug-compile-nosasl-nossl', - commands=[func('fetch-det'), - func('bootstrap-mongo-orchestration', TOPOLOGY='server', AUTH='noauth', - SSL='nossl', MONGODB_VERSION='5.0', ORCHESTRATION_FILE='versioned-api-testing'), - func('run-simple-http-server'), - func('run-tests', MONGODB_API_VERSION=1, AUTH='noauth', SSL='nossl')]), -]) + """, + add_expansions_to_env=True, + ), + func("prepare-kerberos"), + func("run auth tests", ASAN="on"), + ], + ), + PostCompileTask( + "test-versioned-api", + tags=["versioned-api"], + get_build="debug-compile-nosasl-openssl", + commands=[ + func("fetch-det"), + func( + "bootstrap-mongo-orchestration", + TOPOLOGY="server", + AUTH="auth", + SSL="ssl", + MONGODB_VERSION="5.0", + REQUIRE_API_VERSION="true", + ), + func("run-simple-http-server"), + func("run-tests", MONGODB_API_VERSION=1, AUTH="auth", SSL="ssl"), + ], + ), + PostCompileTask( + "test-versioned-api-accept-version-two", + tags=["versioned-api"], + get_build="debug-compile-nosasl-nossl", + commands=[ + func("fetch-det"), + func( + "bootstrap-mongo-orchestration", + TOPOLOGY="server", + AUTH="noauth", + SSL="nossl", + MONGODB_VERSION="5.0", + ORCHESTRATION_FILE="versioned-api-testing", + ), + func("run-simple-http-server"), + func("run-tests", MONGODB_API_VERSION=1, AUTH="noauth", SSL="nossl"), + ], + ), + ], +) class SSLTask(Task): - def __init__(self, version, patch, cflags=None, fips=False, enable_ssl=False, **kwargs): - full_version = version + patch + ('-fips' if fips else '') - script = 'env' + def __init__( + self, + version: str, + patch: str, + cflags: str = "", + fips: bool = False, + enable_ssl: str | Literal[False] = False, + test_params: Mapping[str, Scalar] | None = None, + ): + full_version = version + patch + ("-fips" if fips else "") + self.enable_ssl = enable_ssl + script = "env" if cflags: - script += f' CFLAGS={cflags}' + script += f" CFLAGS={cflags}" - script += ' DEBUG=ON SASL=OFF' + script += " DEBUG=ON SASL=OFF" - if enable_ssl: + if enable_ssl is not False: script += " SSL=" + enable_ssl - elif 'libressl' in version: + elif "libressl" in version: script += " SSL=LIBRESSL" else: script += " SSL=OPENSSL" script += " bash .evergreen/scripts/compile.sh" - super(SSLTask, self).__init__(commands=[ - func('install ssl', SSL=full_version), - shell_mongoc(script, add_expansions_to_env=True), - func('run auth tests', **kwargs), - func('upload-build')]) + super(SSLTask, self).__init__( + commands=[ + func("install ssl", SSL=full_version), + shell_mongoc(script, add_expansions_to_env=True), + func("run auth tests", **(test_params or {})), + func("upload-build"), + ] + ) self.version = version self.fips = fips - self.enable_ssl = enable_ssl @property def name(self): - s = 'build-and-run-authentication-tests-' + self.version + s = "build-and-run-authentication-tests-" + self.version if self.fips: - return s + '-fips' - if self.enable_ssl: - return s + "-" + self.enable_ssl.lower() + return s + "-fips" + if self.enable_ssl is not False: + return s + "-" + str(self.enable_ssl).lower() return s -all_tasks = chain(all_tasks, [ - SSLTask('openssl-1.0.1', 'u', cflags='-Wno-redundant-decls', ), - SSLTask('openssl-1.0.1', 'u', cflags='-Wno-redundant-decls', fips=True), - SSLTask('openssl-1.0.2', 'l', cflags='-Wno-redundant-decls', ), - SSLTask('openssl-1.1.0', 'l'), - SSLTask('libressl-2.5', '.2', require_tls12=True), - SSLTask('libressl-3.0', '.2', require_tls12=True, enable_ssl="AUTO"), - SSLTask('libressl-3.0', '.2', require_tls12=True), -]) +all_tasks = chain( + all_tasks, + [ + SSLTask( + "openssl-1.0.1", + "u", + cflags="-Wno-redundant-decls", + ), + SSLTask("openssl-1.0.1", "u", cflags="-Wno-redundant-decls", fips=True), + SSLTask( + "openssl-1.0.2", + "l", + cflags="-Wno-redundant-decls", + ), + SSLTask("openssl-1.1.0", "l"), + SSLTask("libressl-2.5", ".2", test_params=dict(require_tls12=True)), + SSLTask("libressl-3.0", ".2", enable_ssl="AUTO", test_params=dict(require_tls12=True)), + SSLTask("libressl-3.0", ".2", test_params=dict(require_tls12=True)), + ], +) class IPTask(MatrixTask): - axes = OD([('client', ['ipv6', 'ipv4', 'localhost']), - ('server', ['ipv6', 'ipv4'])]) + axes = OD( + [ + ("client", ["ipv6", "ipv4", "localhost"]), + ("server", ["ipv6", "ipv4"]), + ] + ) - name_prefix = 'test-latest' + name_prefix = "test-latest" - def __init__(self, *args, **kwargs): - super(IPTask, self).__init__(*args, **kwargs) - self.add_tags('nossl', 'nosasl', 'server', 'ipv4-ipv6', 'latest') - self.add_dependency('debug-compile-nosasl-nossl') - self.commands.extend([ - func('fetch-build', BUILD_NAME=self.depends_on['name']), - func("fetch-det"), - func('bootstrap-mongo-orchestration', - IPV4_ONLY=self.on_off(server='ipv4')), - func('run-simple-http-server'), - func('run-tests', - IPV4_ONLY=self.on_off(server='ipv4'), - URI={'ipv6': 'mongodb://[::1]/', - 'ipv4': 'mongodb://127.0.0.1/', - 'localhost': 'mongodb://localhost/'}[self.client])]) - - def display(self, axis_name): - return axis_name + '-' + getattr(self, axis_name) + def additional_dependencies(self) -> Iterable[DependencySpec]: + yield "debug-compile-nosasl-nossl" - @property - def name(self): - return '-'.join([ - self.name_prefix, self.display('server'), self.display('client'), - 'noauth', 'nosasl', 'nossl']) + def additional_tags(self) -> Iterable[str]: + yield from super().additional_tags() + yield from ("nossl", "nosasl", "server", "ipv4-ipv6", "latest") - def _check_allowed(self): + def post_commands(self) -> Iterable[Value]: + return [ + func("fetch-build", BUILD_NAME="debug-compile-nosasl-nossl"), + func("fetch-det"), + func("bootstrap-mongo-orchestration", IPV4_ONLY=self.on_off("server", "ipv4")), + func("run-simple-http-server"), + func( + "run-tests", + IPV4_ONLY=self.on_off("server", "ipv4"), + URI={ + "ipv6": "mongodb://[::1]/", + "ipv4": "mongodb://127.0.0.1/", + "localhost": "mongodb://localhost/", + }[str(self.settings.client)], + ), + ] + + def name_parts(self) -> Iterable[str]: + return ( + self.name_prefix, + f'server-{self.display("server")}', + f'client-{self.display("client")}', + "noauth", + "nosasl", + "nossl", + ) + + def do_is_valid_combination(self) -> bool: # This would fail by design. - if self.server == 'ipv4': - prohibit(self.client == 'ipv6') + if self.settings.server == "ipv4": + prohibit(self.settings.client == "ipv6") # Default configuration is tested in other variants. - if self.server == 'ipv6': - prohibit(self.client == 'localhost') + if self.settings.server == "ipv6": + prohibit(self.settings.client == "localhost") + return True all_tasks = chain(all_tasks, IPTask.matrix()) -aws_compile_task = NamedTask('debug-compile-aws', commands=[shell_mongoc(''' - # Compile test-awsauth. Disable unnecessary dependencies since test-awsauth is copied to a remote Ubuntu 18.04 ECS cluster for testing, which may not have all dependent libraries. - set -euo pipefail - . .evergreen/scripts/find-cmake-latest.sh - cmake=$(find_cmake_latest) - export CC='${CC}' - "$cmake" -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF . - "$cmake" --build . --target test-awsauth -'''), func('upload-build')]) +aws_compile_task = NamedTask( + "debug-compile-aws", + commands=[ + shell_mongoc( + """ + # Compile test-awsauth. Disable unnecessary dependencies since test-awsauth is copied to a remote Ubuntu 18.04 ECS cluster for testing, which may not have all dependent libraries. + set -euo pipefail + . .evergreen/scripts/find-cmake-latest.sh + cmake=$(find_cmake_latest) + export CC='${CC}' + "$cmake" -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF . + "$cmake" --build . --target test-awsauth + """ + ), + func("upload-build"), + ], +) all_tasks = chain(all_tasks, [aws_compile_task]) class AWSTestTask(MatrixTask): - axes = OD([('testcase', ['regular', 'ec2', 'ecs', 'lambda', 'assume_role', 'assume_role_with_web_identity']), - ('version', ['latest', '5.0', '4.4'])]) - - name_prefix = 'test-aws-openssl' - - def __init__(self, *args, **kwargs): - super(AWSTestTask, self).__init__(*args, **kwargs) - self.add_dependency('debug-compile-aws') - self.commands.extend([ - func('fetch-build', BUILD_NAME=self.depends_on['name']), - func('fetch-det'), - func('bootstrap-mongo-orchestration', - AUTH="auth", - ORCHESTRATION_FILE="auth-aws", - MONGODB_VERSION=self.version, - TOPOLOGY="server"), - func('run aws tests', TESTCASE=self.testcase.upper())]) + axes = OD( + [ + ("testcase", ["regular", "ec2", "ecs", "lambda", "assume_role", "assume_role_with_web_identity"]), + ("version", ["latest", "5.0", "4.4"]), + ] + ) + + name_prefix = "test-aws-openssl" + + def additional_dependencies(self) -> Iterable[DependencySpec]: + yield "debug-compile-aws" + + def post_commands(self) -> Iterable[Value]: + return [ + func("fetch-build", BUILD_NAME="debug-compile-aws"), + func("fetch-det"), + func( + "bootstrap-mongo-orchestration", + AUTH="auth", + ORCHESTRATION_FILE="auth-aws", + MONGODB_VERSION=self.settings.version, + TOPOLOGY="server", + ), + func("run aws tests", TESTCASE=str(self.settings.testcase).upper()), + ] @property def name(self): - return '-'.join([self.name_prefix, self.testcase, self.version]) + return f"{self.name_prefix}-{self.settings.testcase}-{self.settings.version}" all_tasks = chain(all_tasks, AWSTestTask.matrix()) class OCSPTask(MatrixTask): - axes = OD([('test', ['test_1', 'test_2', 'test_3', 'test_4', 'soft_fail_test', 'malicious_server_test_1', - 'malicious_server_test_2', 'cache']), - ('delegate', ['delegate', 'nodelegate']), - ('cert', ['rsa', 'ecdsa']), - ('ssl', ['openssl', 'openssl-1.0.1', 'darwinssl', 'winssl']), - ('version', ['latest', '5.0', '4.4'])]) + axes = OD( + [ + ( + "test", + [ + "test_1", + "test_2", + "test_3", + "test_4", + "soft_fail_test", + "malicious_server_test_1", + "malicious_server_test_2", + "cache", + ], + ), + ("delegate", ["delegate", "nodelegate"]), + ("cert", ["rsa", "ecdsa"]), + ("ssl", ["openssl", "openssl-1.0.1", "darwinssl", "winssl"]), + ("version", ["latest", "5.0", "4.4"]), + ] + ) + + name_prefix = "test-ocsp" + + @property + def build_task_name(self) -> str: + return f'debug-compile-nosasl-{self.display("ssl")}' - name_prefix = 'test-ocsp' + def additional_tags(self) -> Iterable[str]: + yield from super().additional_tags() + yield f'ocsp-{self.display("ssl")}' - def __init__(self, *args, **kwargs): - super(OCSPTask, self).__init__(*args, **kwargs) - self.add_dependency('debug-compile-nosasl-%s' % (self.display('ssl'))) - self.add_tags('ocsp-' + self.display('ssl')) + def additional_dependencies(self) -> Iterable[DependencySpec]: + yield self.build_task_name @property def name(self): - return 'ocsp-' + self.display('ssl') + '-' + self.display('test') + '-' + self.display( - 'cert') + '-' + self.display('delegate') + '-' + self.display('version') + return f"ocsp-{self.settings.ssl}-{self.test}-{self.settings.cert}-{self.settings.delegate}-{self.settings.version}" - def to_dict(self): - task = super(MatrixTask, self).to_dict() - - # OCSP tests should run with a batchtime of 14 days. Avoid running OCSP - # tests in patch builds by default (only in commit builds). - task['patchable'] = False - - commands = task['commands'] - commands.append( - func('fetch-build', BUILD_NAME=self.depends_on['name'])) - commands.append(func('fetch-det')) - - stapling = 'mustStaple' - if self.test in ['test_3', 'test_4', 'soft_fail_test', 'cache']: - stapling = 'disableStapling' - if self.test in ['malicious_server_test_1', 'malicious_server_test_2']: - stapling = 'mustStaple-disableStapling' - - orchestration_file = '%s-basic-tls-ocsp-%s' % (self.cert, stapling) - orchestration = func('bootstrap-mongo-orchestration', - MONGODB_VERSION=self.version, - TOPOLOGY='server', - SSL='ssl', - OCSP='on', - ORCHESTRATION_FILE=orchestration_file) + @property + def test(self) -> str: + return str(self.settings.test) + + def post_commands(self) -> Iterable[Value]: + yield func("fetch-build", BUILD_NAME=self.build_task_name) + yield func("fetch-det") + + stapling = "mustStaple" + if self.test in ["test_3", "test_4", "soft_fail_test", "cache"]: + stapling = "disableStapling" + if self.test in ["malicious_server_test_1", "malicious_server_test_2"]: + stapling = "mustStaple-disableStapling" + + orchestration_file = "%s-basic-tls-ocsp-%s" % (self.settings.cert, stapling) + orchestration = func( + "bootstrap-mongo-orchestration", + MONGODB_VERSION=self.settings.version, + TOPOLOGY="server", + SSL="ssl", + OCSP="on", + ORCHESTRATION_FILE=orchestration_file, + ) # The cache test expects a revoked response from an OCSP responder, exactly like TEST_4. - test_column = 'TEST_4' if self.test == 'cache' else self.test.upper() - use_delegate = 'ON' if self.delegate == 'delegate' else 'OFF' + test_column = "TEST_4" if self.test == "cache" else str(self.test).upper() + use_delegate = "ON" if self.settings.delegate == "delegate" else "OFF" - commands.append(shell_mongoc(f''' - TEST_COLUMN={test_column} CERT_TYPE={self.cert} USE_DELEGATE={use_delegate} bash .evergreen/scripts/run-ocsp-responder.sh - ''')) + yield ( + shell_mongoc( + f""" + TEST_COLUMN={test_column} CERT_TYPE={self.settings.cert} USE_DELEGATE={use_delegate} bash .evergreen/scripts/run-ocsp-responder.sh + """ + ) + ) - commands.append(orchestration) + yield (orchestration) - if self.depends_on['name'] == 'debug-compile-nosasl-openssl-1.0.1': + if self.build_task_name == "debug-compile-nosasl-openssl-1.0.1": # LD_LIBRARY_PATH is needed so the in-tree OpenSSL 1.0.1 is found at runtime - if self.test == 'cache': - commands.append(shell_mongoc(f''' - LD_LIBRARY_PATH=$(pwd)/install-dir/lib CERT_TYPE={self.cert} bash .evergreen/scripts/run-ocsp-cache-test.sh - ''')) + if self.test == "cache": + yield ( + shell_mongoc( + f""" + LD_LIBRARY_PATH=$(pwd)/install-dir/lib CERT_TYPE={self.settings.cert} bash .evergreen/scripts/run-ocsp-cache-test.sh + """ + ) + ) else: - commands.append(shell_mongoc(f''' - LD_LIBRARY_PATH=$(pwd)/install-dir/lib TEST_COLUMN={self.test.upper()} CERT_TYPE={self.cert} bash .evergreen/scripts/run-ocsp-test.sh - ''')) + yield ( + shell_mongoc( + f""" + LD_LIBRARY_PATH=$(pwd)/install-dir/lib TEST_COLUMN={self.test.upper()} CERT_TYPE={self.settings.cert} bash .evergreen/scripts/run-ocsp-test.sh + """ + ) + ) else: - if self.test == 'cache': - commands.append(shell_mongoc(f''' - CERT_TYPE={self.cert} bash .evergreen/scripts/run-ocsp-cache-test.sh - ''')) + if self.test == "cache": + yield ( + shell_mongoc( + f""" + CERT_TYPE={self.settings.cert} bash .evergreen/scripts/run-ocsp-cache-test.sh + """ + ) + ) else: - commands.append(shell_mongoc(f''' - TEST_COLUMN={self.test.upper()} CERT_TYPE={self.cert} bash .evergreen/scripts/run-ocsp-test.sh - ''')) + yield ( + shell_mongoc( + f""" + TEST_COLUMN={self.test.upper()} CERT_TYPE={self.settings.cert} bash .evergreen/scripts/run-ocsp-test.sh + """ + ) + ) + + def to_dict(self): + task = super(MatrixTask, self).to_dict() + + # OCSP tests should run with a batchtime of 14 days. Avoid running OCSP + # tests in patch builds by default (only in commit builds). + task["patchable"] = False return task # Testing in OCSP has a lot of exceptions. - def _check_allowed(self): - if self.ssl == 'darwinssl': + def do_is_valid_combination(self) -> bool: + if self.settings.ssl == "darwinssl": # Secure Transport quietly ignores a must-staple certificate with no stapled response. - prohibit(self.test == 'malicious_server_test_2') + prohibit(self.test == "malicious_server_test_2") # ECDSA certs can't be loaded (in the PEM format they're stored) on Windows/macOS. Skip them. - if self.ssl == 'darwinssl' or self.ssl == 'winssl': - prohibit(self.cert == 'ecdsa') + if self.settings.ssl == "darwinssl" or self.settings.ssl == "winssl": + prohibit(self.settings.cert == "ecdsa") # OCSP stapling is not supported on macOS or Windows. - if self.ssl == 'darwinssl' or self.ssl == 'winssl': - prohibit(self.test in ['test_1', 'test_2', 'cache']) + if self.settings.ssl == "darwinssl" or self.settings.ssl == "winssl": + prohibit(self.test in ["test_1", "test_2", "cache"]) - if self.test == 'soft_fail_test' or self.test == 'malicious_server_test_2' or self.test == 'cache': - prohibit(self.delegate == 'delegate') + if self.test == "soft_fail_test" or self.test == "malicious_server_test_2" or self.test == "cache": + prohibit(self.settings.delegate == "delegate") + return True all_tasks = chain(all_tasks, OCSPTask.matrix()) class LoadBalancedTask(MatrixTask): - axes = OD([ - ('asan', [True]), - # The SSL library the C driver is built with. - ('build_ssl', ['openssl']), - # Whether tests are run with SSL connections. - ('test_ssl', [True, False]), - ('test_auth', [True, False]), - ('version', ['5.0', 'latest']) - ]) - - def _check_allowed(self): + axes = OD( + [ + ("asan", [True]), + # The SSL library the C driver is built with. + ("build_ssl", ["openssl"]), + # Whether tests are run with SSL connections. + ("test_ssl", [True, False]), + ("test_auth", [True, False]), + ("version", ["5.0", "latest"]), + ] + ) + + def do_is_valid_combination(self) -> bool: # Test with both SSL and auth, or neither. - prohibit(self.test_ssl != self.test_auth) + return self.settings.test_ssl == self.settings.test_auth - def __init__(self, *args, **kwargs): - super(LoadBalancedTask, self).__init__(*args, **kwargs) - if self.asan and self.build_ssl == "openssl": - self.add_dependency('debug-compile-asan-clang-openssl') - self.add_tags('test-asan') - else: - raise RuntimeError( - "unimplemented configuration for LoadBalancedTask") + def additional_tags(self) -> Iterable[str]: + yield from super().additional_tags() + if self.settings.asan and self.setting_eq("build_ssl", "openssl"): + yield "test-asan" + yield str(self.settings.version) - self.add_tags(self.version) + def additional_dependencies(self) -> Iterable[DependencySpec]: + if self.settings.asan and self.setting_eq("build_ssl", "openssl"): + yield "debug-compile-asan-clang-openssl" # Return the task name. # Example: test-loadbalanced-asan-auth-openssl-latest @property def name(self): - name = "test-loadbalanced" - if self.asan: - name += "-asan" - if self.test_auth: - name += "-auth" - else: - name += "-noauth" - if self.test_ssl: - name += "-" + self.build_ssl - else: - name += "-nossl" - if self.version: - name += "-" + self.version + name = "test-loadbalanced-" + name += "-".join(self.name_parts()) return name - def to_dict(self): - task = super(MatrixTask, self).to_dict() - commands = task['commands'] - commands.append( - func('fetch-build', BUILD_NAME=self.depends_on['name'])) - commands.append(func("fetch-det")) - - orchestration = func('bootstrap-mongo-orchestration', - TOPOLOGY='sharded_cluster', - AUTH='auth' if self.test_auth else 'noauth', - SSL='ssl' if self.test_ssl else 'nossl', - MONGODB_VERSION=self.version, - LOAD_BALANCER='on') - commands.append(orchestration) - commands.append(func('run-simple-http-server')) - commands.append(func("start load balancer", - MONGODB_URI="mongodb://localhost:27017,localhost:27018")) - commands.append(func('run-tests', - ASAN='on' if self.asan else 'off', - SSL='ssl' if self.test_ssl else 'nossl', - AUTH='auth' if self.test_auth else 'noauth', - LOADBALANCED='loadbalanced')) - - return task + def name_parts(self) -> Iterable[str]: + if self.settings.asan: + yield "asan" + if self.settings.test_auth: + yield "auth" + else: + yield "noauth" + if self.settings.test_ssl: + yield str(self.settings.build_ssl) + else: + yield "nossl" + yield str(self.settings.version) + + def post_commands(self) -> Iterable[Value]: + yield (func("fetch-build", BUILD_NAME=self.dependencies[0]["name"])) + yield (func("fetch-det")) + + orchestration = func( + "bootstrap-mongo-orchestration", + TOPOLOGY="sharded_cluster", + AUTH="auth" if self.settings.test_auth else "noauth", + SSL="ssl" if self.settings.test_ssl else "nossl", + MONGODB_VERSION=self.settings.version, + LOAD_BALANCER="on", + ) + yield (orchestration) + yield (func("run-simple-http-server")) + yield (func("start load balancer", MONGODB_URI="mongodb://localhost:27017,localhost:27018")) + yield ( + func( + "run-tests", + ASAN="on" if self.settings.asan else "off", + SSL="ssl" if self.settings.test_ssl else "nossl", + AUTH="auth" if self.settings.test_auth else "noauth", + LOADBALANCED="loadbalanced", + ) + ) all_tasks = chain(all_tasks, LoadBalancedTask.matrix()) diff --git a/.evergreen/legacy_config_generator/evergreen_config_lib/testazurekms.py b/.evergreen/legacy_config_generator/evergreen_config_lib/testazurekms.py index eebe362e51..acf16b35de 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_lib/testazurekms.py +++ b/.evergreen/legacy_config_generator/evergreen_config_lib/testazurekms.py @@ -16,68 +16,82 @@ from collections import OrderedDict as OD +from typing import MutableSequence + +from evergreen_config_generator.functions import shell_exec, func +from evergreen_config_generator.tasks import NamedTask +from evergreen_config_generator.variants import Variant +from evergreen_config_generator.taskgroups import TaskGroup -from evergreen_config_generator.functions import (shell_exec, func) -from evergreen_config_generator.tasks import (NamedTask) -from evergreen_config_generator.variants import (Variant) -from evergreen_config_generator.taskgroups import (TaskGroup) def _create_tasks(): # passtask is expected to run on a remote Azure VM and succeed at obtaining credentials. - passtask = NamedTask (task_name="testazurekms-task") + passtask = NamedTask(task_name="testazurekms-task") passtask.commands = [ func("fetch-source"), - shell_exec (r''' - echo "Building test-azurekms ... begin" - pushd mongoc - ./.evergreen/scripts/compile-test-azurekms.sh - popd - echo "Building test-azurekms ... end" + shell_exec( + r""" + echo "Building test-azurekms ... begin" + pushd mongoc + ./.evergreen/scripts/compile-test-azurekms.sh + popd + echo "Building test-azurekms ... end" - echo "Copying files ... begin" - export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} - export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} - export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools - mkdir testazurekms - cp ./mongoc/src/libmongoc/test-azurekms ./mongoc/install/lib/libmongocrypt.* testazurekms - tar czf testazurekms.tgz testazurekms/* - AZUREKMS_SRC="testazurekms.tgz" \ - AZUREKMS_DST="./" \ - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh - echo "Copying files ... end" + echo "Copying files ... begin" + export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} + export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} + export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey + DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools + mkdir testazurekms + cp ./mongoc/src/libmongoc/test-azurekms ./mongoc/install/lib/libmongocrypt.* testazurekms + tar czf testazurekms.tgz testazurekms/* + AZUREKMS_SRC="testazurekms.tgz" \ + AZUREKMS_DST="./" \ + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh + echo "Copying files ... end" - echo "Untarring file ... begin" - AZUREKMS_CMD="tar xf testazurekms.tgz" \ - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - echo "Untarring file ... end" - ''', test=False, add_expansions_to_env=True), - shell_exec (r''' - export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} - export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} - export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools - AZUREKMS_CMD="LD_LIBRARY_PATH=./testazurekms MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' ./testazurekms/test-azurekms" \ - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - ''') + echo "Untarring file ... begin" + AZUREKMS_CMD="tar xf testazurekms.tgz" \ + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh + echo "Untarring file ... end" + """, + test=False, + add_expansions_to_env=True, + ), + shell_exec( + r""" + export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} + export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} + export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey + DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools + AZUREKMS_CMD="LD_LIBRARY_PATH=./testazurekms MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' ./testazurekms/test-azurekms" \ + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh + """ + ), ] - failtask = NamedTask (task_name="testazurekms-fail-task") + failtask = NamedTask(task_name="testazurekms-fail-task") failtask.commands = [ func("fetch-source"), - shell_exec (r''' - pushd mongoc - ./.evergreen/scripts/compile-test-azurekms.sh - popd - ''', test=False, add_expansions_to_env=True), - shell_exec (r''' - LD_LIBRARY_PATH=$PWD/install \ - MONGODB_URI='mongodb://localhost:27017' \ - KEY_NAME='${testazurekms_keyname}' \ - KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \ - EXPECT_ERROR='Error from Azure IMDS server' \ - ./mongoc/src/libmongoc/test-azurekms - ''') + shell_exec( + r""" + pushd mongoc + ./.evergreen/scripts/compile-test-azurekms.sh + popd + """, + test=False, + add_expansions_to_env=True, + ), + shell_exec( + r""" + LD_LIBRARY_PATH=$PWD/install \ + MONGODB_URI='mongodb://localhost:27017' \ + KEY_NAME='${testazurekms_keyname}' \ + KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \ + EXPECT_ERROR='Error from Azure IMDS server' \ + ./mongoc/src/libmongoc/test-azurekms + """ + ), ] return [passtask, failtask] @@ -87,10 +101,10 @@ def _create_variant(): name="testazurekms-variant", display_name="Azure KMS", # Azure Virtual Machine created is Debian 10. - run_on="debian10-small", tasks=[ - "testazurekms_task_group", - "testazurekms-fail-task" - ], batchtime=20160) # Use a batchtime of 14 days as suggested by the CSFLE test README + run_on="debian10-small", + tasks=["testazurekms_task_group", "testazurekms-fail-task"], + batchtime=20160, + ) # Use a batchtime of 14 days as suggested by the CSFLE test README def _create_task_group(): @@ -98,8 +112,9 @@ def _create_task_group(): task_group.setup_group_can_fail_task = True task_group.setup_group_timeout_secs = 1800 # 30 minutes task_group.setup_group = [ - func('fetch-det'), - shell_exec(r''' + func("fetch-det"), + shell_exec( + r""" DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools echo '${testazurekms_publickey}' > /tmp/testazurekms_publickey echo '${testazurekms_privatekey}' > /tmp/testazurekms_privatekey @@ -115,32 +130,59 @@ def _create_task_group(): export AZUREKMS_SCOPE=${testazurekms_scope} export AZUREKMS_VMNAME_PREFIX=CDRIVER $DRIVERS_TOOLS/.evergreen/csfle/azurekms/create-and-setup-vm.sh - ''', test=False), + """, + test=False, + ), # Load the AZUREKMS_VMNAME expansion. - OD([('command', 'expansions.update'), - ('params', OD([ - ('file', 'testazurekms-expansions.yml'), - ]))]) + OD( + [ + ("command", "expansions.update"), + ( + "params", + OD( + [ + ("file", "testazurekms-expansions.yml"), + ] + ), + ), + ] + ), ] task_group.teardown_group = [ # Load expansions again. The setup task may have failed before running `expansions.update`. - OD([('command', 'expansions.update'), - ('params', OD([ - ('file', 'testazurekms-expansions.yml'), - ]))]), - shell_exec(r''' - DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools + OD( + [ + ("command", "expansions.update"), + ( + "params", + OD( + [ + ("file", "testazurekms-expansions.yml"), + ] + ), + ), + ] + ), + shell_exec( + r""" + DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh - ''', test=False) + """, + test=False, + ), ] task_group.tasks = ["testazurekms-task"] return task_group -def testazurekms_generate(all_tasks, all_variants, all_task_groups): +def testazurekms_generate( + all_tasks: MutableSequence[NamedTask], + all_variants: MutableSequence[Variant], + all_task_groups: MutableSequence[TaskGroup], +): all_tasks.extend(_create_tasks()) all_variants.append(_create_variant()) all_task_groups.append(_create_task_group()) diff --git a/.evergreen/legacy_config_generator/evergreen_config_lib/testgcpkms.py b/.evergreen/legacy_config_generator/evergreen_config_lib/testgcpkms.py index d3291265cd..bfdb6170b2 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_lib/testgcpkms.py +++ b/.evergreen/legacy_config_generator/evergreen_config_lib/testgcpkms.py @@ -1,32 +1,35 @@ #!/usr/bin/env python # -#Copyright 2022 - present MongoDB, Inc. +# Copyright 2022 - present MongoDB, Inc. # -#Licensed under the Apache License, Version 2.0(the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0(the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -#http: // www.apache.org/licenses/LICENSE-2.0 +# http: // www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from collections import OrderedDict as OD +from typing import MutableSequence -from evergreen_config_generator.functions import (shell_exec, func) -from evergreen_config_generator.tasks import (NamedTask) -from evergreen_config_generator.variants import (Variant) -from evergreen_config_generator.taskgroups import (TaskGroup) +from evergreen_config_generator.functions import shell_exec, func +from evergreen_config_generator.tasks import NamedTask +from evergreen_config_generator.variants import Variant +from evergreen_config_generator.taskgroups import TaskGroup -def _create_tasks(): - passtask = NamedTask (task_name="testgcpkms-task") - passtask.commands = [ - func("fetch-source"), - shell_exec (r''' +def _create_tasks(): + passtask = NamedTask( + task_name="testgcpkms-task", + commands=[ + func("fetch-source"), + shell_exec( + r""" echo "Building test-gcpkms ... begin" pushd mongoc ./.evergreen/scripts/compile-test-gcpkms.sh @@ -46,74 +49,101 @@ def _create_tasks(): echo "Untarring file ... begin" GCPKMS_CMD="tar xf testgcpkms.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Untarring file ... end" - ''', test=False, add_expansions_to_env=True), - shell_exec (r''' + """, + test=False, + add_expansions_to_env=True, + ), + shell_exec( + r""" export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools GCPKMS_CMD="LD_LIBRARY_PATH=./testgcpkms MONGODB_URI='mongodb://localhost:27017' ./testgcpkms/test-gcpkms" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh - ''')] + """ + ), + ], + ) - failtask = NamedTask(task_name="testgcpkms-fail-task") - failtask.commands = [ - shell_exec (r''' + failtask = NamedTask( + task_name="testgcpkms-fail-task", + commands=[ + shell_exec( + r""" pushd mongoc ./.evergreen/scripts/compile-test-gcpkms.sh - popd''', test=False, add_expansions_to_env=True), - shell_exec (r''' + popd""", + test=False, + add_expansions_to_env=True, + ), + shell_exec( + r""" export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - LD_LIBRARY_PATH=$(pwd)/install MONGODB_URI='mongodb://localhost:27017' EXPECT_ERROR='Failed to connect to: metadata.google.internal' ./mongoc/src/libmongoc/test-gcpkms''')] - + LD_LIBRARY_PATH=$(pwd)/install MONGODB_URI='mongodb://localhost:27017' EXPECT_ERROR='Failed to connect to: metadata.google.internal' ./mongoc/src/libmongoc/test-gcpkms""" + ), + ], + ) + return [passtask, failtask] + def _create_variant(): - return Variant( + return Variant( name="testgcpkms-variant", display_name="GCP KMS", # GCP Virtual Machine created is Debian 11. - run_on="debian11-small", tasks=[ - "testgcpkms_task_group", - "testgcpkms-fail-task" - ], batchtime=20160) # Use a batchtime of 14 days as suggested by the CSFLE test README + run_on="debian11-small", + tasks=["testgcpkms_task_group", "testgcpkms-fail-task"], + batchtime=20160, + ) # Use a batchtime of 14 days as suggested by the CSFLE test README + def _create_task_group(): task_group = TaskGroup(name="testgcpkms_task_group") task_group.setup_group_can_fail_task = True task_group.setup_group_timeout_secs = 1800 # 30 minutes task_group.setup_group = [ - func('fetch-det'), + func("fetch-det"), # Create and set up a GCE instance using driver tools script - shell_exec(r''' + shell_exec( + r""" DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools echo '${testgcpkms_key_file}' > /tmp/testgcpkms_key_file.json export GCPKMS_KEYFILE=/tmp/testgcpkms_key_file.json export GCPKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS export GCPKMS_SERVICEACCOUNT="${testgcpkms_service_account}" - $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh''', test=False), + $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh""", + test=False, + ), + # Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_PROJECT, and GCPKMS_ZONE expansions. + OD([("command", "expansions.update"), ("params", OD([("file", "testgcpkms-expansions.yml")]))]), + ] - # Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_PROJECT, and GCPKMS_ZONE expansions. - OD([('command', 'expansions.update'), - ('params', OD([ - ('file', 'testgcpkms-expansions.yml')]))])] - task_group.teardown_group = [ - shell_exec(r''' + shell_exec( + r""" DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh''', test=False) + $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh""", + test=False, + ) ] - task_group.tasks= ["testgcpkms-task"] + task_group.tasks = ["testgcpkms-task"] return task_group -def testgcpkms_generate(all_tasks, all_variants, all_task_groups): + +def testgcpkms_generate( + all_tasks: MutableSequence[NamedTask], + all_variants: MutableSequence[Variant], + all_task_groups: MutableSequence[TaskGroup], +): all_tasks.extend(_create_tasks()) all_variants.append(_create_variant()) all_task_groups.append(_create_task_group()) diff --git a/.evergreen/legacy_config_generator/evergreen_config_lib/variants.py b/.evergreen/legacy_config_generator/evergreen_config_lib/variants.py index 71aa786a0a..2d0848db26 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_lib/variants.py +++ b/.evergreen/legacy_config_generator/evergreen_config_lib/variants.py @@ -18,417 +18,480 @@ mobile_flags = ( - ' -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY' - ' -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ONLY' - ' -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER' - ' -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY' + " -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY" + " -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ONLY" + " -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER" + " -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY" ) -# Returns minutes for batchtime. - -def days(n): +def days(n: int) -> int: + "Calculate the number of minutes in the given number of days" return n * 24 * 60 all_variants = [ - Variant('releng', - '**Release Archive Creator', - 'ubuntu1804-test', - ['make-release-archive', - 'release-compile', - 'debug-compile-no-counters', - 'compile-tracing', - 'link-with-cmake', - 'link-with-cmake-deprecated', - 'abi-compliance-check', - 'link-with-cmake-ssl', - 'link-with-cmake-ssl-deprecated', - 'link-with-cmake-snappy', - 'link-with-cmake-snappy-deprecated', - OD([('name', 'link-with-cmake-mac'), ('distros', ['macos-1014'])]), - OD([('name', 'link-with-cmake-mac-deprecated'), - ('distros', ['macos-1014'])]), - OD([('name', 'link-with-cmake-windows'), - ('distros', ['windows-64-vs2017-test'])]), - OD([('name', 'link-with-cmake-windows-ssl'), - ('distros', ['windows-64-vs2017-test'])]), - OD([('name', 'link-with-cmake-windows-snappy'), - ('distros', ['windows-64-vs2017-test'])]), - OD([('name', 'link-with-cmake-mingw'), - ('distros', ['windows-64-vs2017-test'])]), - OD([('name', 'link-with-pkg-config'), - ('distros', ['ubuntu1804-test'])]), - OD([('name', 'link-with-pkg-config-mac'), - ('distros', ['macos-1014'])]), - 'link-with-pkg-config-ssl', - 'link-with-bson', - OD([('name', 'link-with-bson-windows'), - ('distros', ['windows-64-vs2017-test'])]), - OD([('name', 'link-with-bson-mac'), ('distros', ['macos-1014'])]), - OD([('name', 'link-with-bson-mingw'), - ('distros', ['windows-64-vs2013-compile'])]), - 'check-headers', - 'install-uninstall-check', - OD([('name', 'install-uninstall-check-mingw'), - ('distros', ['windows-64-vs2017-test'])]), - OD([('name', 'install-uninstall-check-msvc'), - ('distros', ['windows-64-vs2017-test'])]), - 'debug-compile-with-warnings', - OD([('name', 'build-and-test-with-toolchain'), - ('distros', ['debian10-small'])])], - {}), - Variant('clang34ubuntu', - 'clang 3.4 (Ubuntu 14.04)', - 'ubuntu1404-build', - ['release-compile', - 'debug-compile-rdtscp'], - {'CC': 'clang'}), - Variant('clang35', - 'clang 3.5 (Debian 8.1)', - 'debian81-test', - ['release-compile', - 'debug-compile-sasl-openssl', - 'debug-compile-nosasl-openssl', - '.authentication-tests .openssl'], - {'CC': 'clang'}), - Variant('clang38', - 'clang 3.8 (Debian 9.2)', - 'debian92-test', - ['release-compile', - 'debug-compile-nosasl-nossl', - '.latest .nossl'], - {'CC': 'clang'}), - Variant('openssl', - 'OpenSSL / LibreSSL', - 'archlinux-build', - ['build-and-run-authentication-tests-openssl-1.0.1', - 'build-and-run-authentication-tests-openssl-1.0.2', - 'build-and-run-authentication-tests-openssl-1.1.0', - 'build-and-run-authentication-tests-openssl-1.0.1-fips', - 'build-and-run-authentication-tests-libressl-2.5', - 'build-and-run-authentication-tests-libressl-3.0-auto', - 'build-and-run-authentication-tests-libressl-3.0'], - {}), - Variant('clang37', - 'clang 3.7 (Archlinux)', - 'archlinux-test', - ['release-compile', - 'debug-compile-sasl-openssl', - 'debug-compile-nosasl-openssl', - '.authentication-tests .openssl'], - {'CC': 'clang'}), - Variant('clang60-i686', - 'clang 6.0 (i686) (Ubuntu 18.04)', - 'ubuntu1804-test', - ['release-compile', - 'debug-compile-nosasl-nossl', - 'debug-compile-no-align', - '.debug-compile !.sspi .nossl .nosasl', - '.latest .nossl .nosasl'], - {'CC': 'clang', 'MARCH': 'i686'}), - Variant('clang38-i686', - 'clang 3.8 (i686) (Ubuntu 16.04)', - 'ubuntu1604-test', - ['release-compile', - 'debug-compile-no-align'], - {'CC': 'clang', 'MARCH': 'i686'}), - Variant('clang38ubuntu', - 'clang 3.8 (Ubuntu 16.04)', - 'ubuntu1604-test', - ['.compression !.zstd', - 'release-compile', - 'debug-compile-sasl-openssl', - 'debug-compile-nosasl-openssl', - 'debug-compile-no-align', - '.authentication-tests .openssl'], - {'CC': 'clang'}), - Variant('gcc48ubuntu', - 'GCC 4.8 (Ubuntu 14.04)', - 'ubuntu1404-build', - ['release-compile'], - {'CC': 'gcc'}), - Variant('gcc82rhel', - 'GCC 8.2 (RHEL 8.0)', - 'rhel80-test', - ['.hardened', - '.compression !.snappy !.zstd', - 'release-compile', - 'debug-compile-nosasl-nossl', - 'debug-compile-nosasl-openssl', - 'debug-compile-sasl-openssl', - '.authentication-tests .openssl', - '.latest .nossl'], - {'CC': 'gcc'}), - Variant('gcc48rhel', - 'GCC 4.8 (RHEL 7.0)', - 'rhel70', - # Skip client-side-encryption tests on RHEL 7.0 due to OCSP errors - # with Azure. See CDRIVER-3620 and CDRIVER-3814. - ['.hardened', - '.compression !.snappy', - 'release-compile', - 'debug-compile-nosasl-nossl', - 'debug-compile-sasl-openssl', - 'debug-compile-nosasl-openssl', - '.authentication-tests .openssl', - '.latest .nossl'], - {'CC': 'gcc'}), - Variant('gcc49', - 'GCC 4.9 (Debian 8.1)', - 'debian81-test', - ['release-compile', - 'debug-compile-sasl-openssl', - 'debug-compile-nosasl-openssl', - '.authentication-tests .openssl'], - {'CC': 'gcc'}), - Variant('gcc63', - 'GCC 6.3 (Debian 9.2)', - 'debian92-test', - ['release-compile', - 'debug-compile-nosasl-nossl', - '.latest .nossl'], - {'CC': 'gcc'}), - Variant('gcc83', - 'GCC 8.3 (Debian 10.0)', - 'debian10-test', - ['release-compile', - 'debug-compile-nosasl-nossl', - '.latest .nossl'], - {'CC': 'gcc'}), - Variant('gcc102', - 'GCC 10.2 (Debian 11.0)', - 'debian11-large', - ['release-compile', - 'debug-compile-nosasl-nossl', - '.latest .nossl'], - {'CC': 'gcc'}), - Variant('gcc94', - 'GCC 9.4 (Ubuntu 20.04)', - 'ubuntu2004-large', - ['release-compile', - 'debug-compile-nosasl-nossl', - '.latest .nossl'], - {'CC': 'gcc'}), - Variant('gcc75-i686', - 'GCC 7.5 (i686) (Ubuntu 18.04)', - 'ubuntu1804-test', - ['release-compile', - 'debug-compile-nosasl-nossl', - 'debug-compile-no-align', - '.latest .nossl .nosasl'], - {'CC': 'gcc', 'MARCH': 'i686'}), - Variant('gcc75', - 'GCC 7.5 (Ubuntu 18.04)', - 'ubuntu1804-test', - ['.compression !.zstd', - 'debug-compile-nosrv', - 'release-compile', - 'debug-compile-nosasl-nossl', - 'debug-compile-no-align', - 'debug-compile-sasl-openssl', - 'debug-compile-nosasl-openssl', - '.authentication-tests .openssl', - '.authentication-tests .asan', - '.test-coverage', - '.latest .nossl', - 'retry-true-latest-server', - 'test-dns-openssl', - 'test-dns-auth-openssl', - 'test-dns-loadbalanced-openssl' - ], - {'CC': 'gcc'}), - Variant('gcc54', - 'GCC 5.4 (Ubuntu 16.04)', - 'ubuntu1604-test', - ['.compression !.zstd', - 'debug-compile-nosrv', - 'release-compile', - 'debug-compile-no-align'], - {'CC': 'gcc'}), - Variant('darwin', - '*Darwin, macOS (Apple LLVM)', - 'macos-1014', - ['.compression !.snappy', - 'release-compile', - 'debug-compile-nosasl-nossl', - 'debug-compile-rdtscp', - 'debug-compile-no-align', - 'debug-compile-nosrv', - 'debug-compile-sasl-darwinssl', - 'debug-compile-nosasl-nossl', - '.authentication-tests .darwinssl', - '.latest .nossl', - 'test-dns-darwinssl', - 'test-dns-auth-darwinssl', - 'debug-compile-lto', - 'debug-compile-lto-thin', - 'debug-compile-aws', - 'test-aws-openssl-regular-4.4', - 'test-aws-openssl-regular-latest' - ], - {'CC': 'clang'}), - Variant('windows-2017-32', - 'Windows (i686) (VS 2017)', - 'windows-64-vs2017-test', - ['debug-compile-nosasl-nossl', - '.latest .nossl .nosasl'], - {'CC': 'Visual Studio 15 2017'}), - Variant('windows-2017', - 'Windows (VS 2017)', - 'windows-64-vs2017-test', - ['debug-compile-nosasl-nossl', - 'debug-compile-nosasl-openssl', - 'debug-compile-sspi-winssl', - '.latest .nossl', - '.nosasl .latest .nossl', - 'test-dns-winssl', - 'test-dns-auth-winssl', - 'debug-compile-aws', - 'test-aws-openssl-regular-4.4', - 'test-aws-openssl-regular-latest', - # Authentication tests with OpenSSL on Windows are only run on the vs2017 variant. - # Older vs variants fail to verify certificates against Atlas tests. - '.authentication-tests .openssl !.sasl', - '.authentication-tests .winssl' - ], - {'CC': 'Visual Studio 15 2017 Win64'}), - Variant('windows-2015', - 'Windows (VS 2015)', - 'windows-64-vs2015-compile', - ['.compression !.snappy !.zstd !.latest', - 'release-compile', - 'debug-compile-sspi-winssl', - 'debug-compile-no-align', - 'debug-compile-nosrv', - '.authentication-tests .winssl'], - {'CC': 'Visual Studio 14 2015 Win64'}), - Variant('windows-2015-32', - 'Windows (i686) (VS 2015)', - 'windows-64-vs2015-compile', - ['.compression !.snappy !.zstd !.latest', - 'release-compile', - 'debug-compile-sspi-winssl', - 'debug-compile-no-align', - '.authentication-tests .winssl'], - {'CC': 'Visual Studio 14 2015'}), - Variant('windows-2013', - 'Windows (VS 2013)', - 'windows-64-vs2013-compile', - ['.compression !.snappy !.zstd !.latest', - 'release-compile', - 'debug-compile-sspi-winssl', - '.authentication-tests .winssl'], - {'CC': 'Visual Studio 12 2013 Win64'}), - Variant('windows-2013-32', - 'Windows (i686) (VS 2013)', - 'windows-64-vs2013-compile', - ['release-compile', - 'debug-compile-rdtscp', - 'debug-compile-sspi-winssl', - '.authentication-tests .winssl'], - {'CC': 'Visual Studio 12 2013'}), - Variant('mingw-windows2016', - 'MinGW-W64 (Windows Server 2016)', - 'windows-64-vs2017-test', - ['debug-compile-nosasl-nossl', - '.latest .nossl .nosasl .server'], - {'CC': 'mingw'}), - Variant('mingw', - 'MinGW-W64', - 'windows-64-vs2013-compile', - ['debug-compile-no-align'], - {'CC': 'mingw'}), - Variant('power8-rhel81', - 'Power8 (ppc64le) (RHEL 8.1)', - 'rhel81-power8-test', - ['release-compile', - 'debug-compile-nosasl-nossl', - 'debug-compile-sasl-openssl', - '.latest .nossl', - 'test-dns-openssl'], - {'CC': 'gcc'}, - batchtime=days(1)), - Variant('arm-ubuntu1804', - '*ARM (aarch64) (Ubuntu 18.04)', - 'ubuntu1804-arm64-large', - ['.compression !.snappy !.zstd', - 'debug-compile-no-align', - 'release-compile', - 'debug-compile-nosasl-nossl', - 'debug-compile-nosasl-openssl', - 'debug-compile-sasl-openssl', - '.authentication-tests .openssl', - '.latest .nossl', - 'test-dns-openssl'], - {'CC': 'gcc'}, - batchtime=days(1)), - Variant('arm-ubuntu1604', - '*ARM (aarch64) (Ubuntu 16.04)', - 'ubuntu1604-arm64-large', - ['.compression !.snappy !.zstd', - 'debug-compile-no-align', - 'release-compile'], - {'CC': 'gcc'}, - batchtime=days(1)), - Variant('zseries-rhel83', - '*zSeries', - 'rhel83-zseries-small', - ['release-compile', - # '.compression', --> TODO: waiting on ticket CDRIVER-3258 - 'debug-compile-no-align', - 'debug-compile-nosasl-nossl', - 'debug-compile-nosasl-openssl', - 'debug-compile-sasl-openssl', - '.authentication-tests .openssl', - '.latest .nossl'], - {'CC': 'gcc'}, - batchtime=days(1)), - Variant('clang60ubuntu', 'clang 6.0 (Ubuntu 18.04)', 'ubuntu1804-test', [ - 'debug-compile-aws', - 'debug-compile-sasl-openssl-static', - '.authentication-tests .asan', - 'test-aws-openssl-regular-latest', - 'test-aws-openssl-ec2-latest', - 'test-aws-openssl-ecs-latest', - 'test-aws-openssl-assume_role-latest', - 'test-aws-openssl-lambda-latest', - 'test-aws-openssl-regular-4.4', - 'test-aws-openssl-ec2-4.4', - 'test-aws-openssl-ecs-4.4', - 'test-aws-openssl-assume_role-4.4', - 'test-aws-openssl-lambda-4.4', - 'test-aws-openssl-assume_role_with_web_identity-latest', - 'test-aws-openssl-assume_role_with_web_identity-5.0', - 'test-aws-openssl-assume_role_with_web_identity-4.4', - ], {'CC': 'clang'}), - Variant('mongohouse', - 'Mongohouse Test', - 'ubuntu1804-test', - ['debug-compile-sasl-openssl', - 'test-mongohouse'], - {}), - Variant('ocsp', 'OCSP tests', 'ubuntu2004-small', [ - OD([('name', 'debug-compile-nosasl-openssl')]), - OD([('name', 'debug-compile-nosasl-openssl-static')]), - OD([('name', 'debug-compile-nosasl-darwinssl'), ('distros', ['macos-1014'])]), - OD([('name', 'debug-compile-nosasl-winssl'), - ('distros', ['windows-64-vs2017-test'])]), - OD([('name', '.ocsp-openssl')]), - OD([('name', '.ocsp-darwinssl'), ('distros', ['macos-1014'])]), - OD([('name', '.ocsp-winssl'), ('distros', ['windows-64-vs2017-test'])]), - OD([('name', 'debug-compile-nosasl-openssl-1.0.1')]), - OD([('name', '.ocsp-openssl-1.0.1')]) - ], {}, batchtime=days(7)), - Variant('packaging', 'Linux Distro Packaging', 'ubuntu1804-test', [ - 'debian-package-build', - OD([('name', 'rpm-package-build'), ('distros', ['rhel82-arm64-small'])]), - ], {}, batchtime=days(1)), - Variant('versioned-api', - 'Versioned API Tests', - 'ubuntu1804-test', - ['debug-compile-nosasl-openssl', - 'debug-compile-nosasl-nossl', - '.versioned-api'], - {}), + Variant( + "releng", + "**Release Archive Creator", + "ubuntu1804-test", + [ + "make-release-archive", + "release-compile", + "debug-compile-no-counters", + "compile-tracing", + "link-with-cmake", + "link-with-cmake-deprecated", + "abi-compliance-check", + "link-with-cmake-ssl", + "link-with-cmake-ssl-deprecated", + "link-with-cmake-snappy", + "link-with-cmake-snappy-deprecated", + OD([("name", "link-with-cmake-mac"), ("distros", ["macos-1014"])]), + OD([("name", "link-with-cmake-mac-deprecated"), ("distros", ["macos-1014"])]), + OD([("name", "link-with-cmake-windows"), ("distros", ["windows-64-vs2017-test"])]), + OD([("name", "link-with-cmake-windows-ssl"), ("distros", ["windows-64-vs2017-test"])]), + OD([("name", "link-with-cmake-windows-snappy"), ("distros", ["windows-64-vs2017-test"])]), + OD([("name", "link-with-cmake-mingw"), ("distros", ["windows-64-vs2017-test"])]), + OD([("name", "link-with-pkg-config"), ("distros", ["ubuntu1804-test"])]), + OD([("name", "link-with-pkg-config-mac"), ("distros", ["macos-1014"])]), + "link-with-pkg-config-ssl", + "link-with-bson", + OD([("name", "link-with-bson-windows"), ("distros", ["windows-64-vs2017-test"])]), + OD([("name", "link-with-bson-mac"), ("distros", ["macos-1014"])]), + OD([("name", "link-with-bson-mingw"), ("distros", ["windows-64-vs2013-compile"])]), + "check-headers", + "install-uninstall-check", + OD([("name", "install-uninstall-check-mingw"), ("distros", ["windows-64-vs2017-test"])]), + OD([("name", "install-uninstall-check-msvc"), ("distros", ["windows-64-vs2017-test"])]), + "debug-compile-with-warnings", + OD([("name", "build-and-test-with-toolchain"), ("distros", ["debian10-small"])]), + ], + {}, + ), + Variant( + "clang34ubuntu", + "clang 3.4 (Ubuntu 14.04)", + "ubuntu1404-build", + ["release-compile", "debug-compile-rdtscp"], + {"CC": "clang"}, + ), + Variant( + "clang35", + "clang 3.5 (Debian 8.1)", + "debian81-test", + [ + "release-compile", + "debug-compile-sasl-openssl", + "debug-compile-nosasl-openssl", + ".authentication-tests .openssl", + ], + {"CC": "clang"}, + ), + Variant( + "clang38", + "clang 3.8 (Debian 9.2)", + "debian92-test", + ["release-compile", "debug-compile-nosasl-nossl", ".latest .nossl"], + {"CC": "clang"}, + ), + Variant( + "openssl", + "OpenSSL / LibreSSL", + "archlinux-build", + [ + "build-and-run-authentication-tests-openssl-1.0.1", + "build-and-run-authentication-tests-openssl-1.0.2", + "build-and-run-authentication-tests-openssl-1.1.0", + "build-and-run-authentication-tests-openssl-1.0.1-fips", + "build-and-run-authentication-tests-libressl-2.5", + "build-and-run-authentication-tests-libressl-3.0-auto", + "build-and-run-authentication-tests-libressl-3.0", + ], + {}, + ), + Variant( + "clang37", + "clang 3.7 (Archlinux)", + "archlinux-test", + [ + "release-compile", + "debug-compile-sasl-openssl", + "debug-compile-nosasl-openssl", + ".authentication-tests .openssl", + ], + {"CC": "clang"}, + ), + Variant( + "clang60-i686", + "clang 6.0 (i686) (Ubuntu 18.04)", + "ubuntu1804-test", + [ + "release-compile", + "debug-compile-nosasl-nossl", + "debug-compile-no-align", + ".debug-compile !.sspi .nossl .nosasl", + ".latest .nossl .nosasl", + ], + {"CC": "clang", "MARCH": "i686"}, + ), + Variant( + "clang38-i686", + "clang 3.8 (i686) (Ubuntu 16.04)", + "ubuntu1604-test", + ["release-compile", "debug-compile-no-align"], + {"CC": "clang", "MARCH": "i686"}, + ), + Variant( + "clang38ubuntu", + "clang 3.8 (Ubuntu 16.04)", + "ubuntu1604-test", + [ + ".compression !.zstd", + "release-compile", + "debug-compile-sasl-openssl", + "debug-compile-nosasl-openssl", + "debug-compile-no-align", + ".authentication-tests .openssl", + ], + {"CC": "clang"}, + ), + Variant("gcc48ubuntu", "GCC 4.8 (Ubuntu 14.04)", "ubuntu1404-build", ["release-compile"], {"CC": "gcc"}), + Variant( + "gcc82rhel", + "GCC 8.2 (RHEL 8.0)", + "rhel80-test", + [ + ".hardened", + ".compression !.snappy !.zstd", + "release-compile", + "debug-compile-nosasl-nossl", + "debug-compile-nosasl-openssl", + "debug-compile-sasl-openssl", + ".authentication-tests .openssl", + ".latest .nossl", + ], + {"CC": "gcc"}, + ), + Variant( + "gcc48rhel", + "GCC 4.8 (RHEL 7.0)", + "rhel70", + # Skip client-side-encryption tests on RHEL 7.0 due to OCSP errors + # with Azure. See CDRIVER-3620 and CDRIVER-3814. + [ + ".hardened", + ".compression !.snappy", + "release-compile", + "debug-compile-nosasl-nossl", + "debug-compile-sasl-openssl", + "debug-compile-nosasl-openssl", + ".authentication-tests .openssl", + ".latest .nossl", + ], + {"CC": "gcc"}, + ), + Variant( + "gcc49", + "GCC 4.9 (Debian 8.1)", + "debian81-test", + [ + "release-compile", + "debug-compile-sasl-openssl", + "debug-compile-nosasl-openssl", + ".authentication-tests .openssl", + ], + {"CC": "gcc"}, + ), + Variant( + "gcc63", + "GCC 6.3 (Debian 9.2)", + "debian92-test", + ["release-compile", "debug-compile-nosasl-nossl", ".latest .nossl"], + {"CC": "gcc"}, + ), + Variant( + "gcc83", + "GCC 8.3 (Debian 10.0)", + "debian10-test", + ["release-compile", "debug-compile-nosasl-nossl", ".latest .nossl"], + {"CC": "gcc"}, + ), + Variant( + "gcc102", + "GCC 10.2 (Debian 11.0)", + "debian11-large", + ["release-compile", "debug-compile-nosasl-nossl", ".latest .nossl"], + {"CC": "gcc"}, + ), + Variant( + "gcc94", + "GCC 9.4 (Ubuntu 20.04)", + "ubuntu2004-large", + ["release-compile", "debug-compile-nosasl-nossl", ".latest .nossl"], + {"CC": "gcc"}, + ), + Variant( + "gcc75-i686", + "GCC 7.5 (i686) (Ubuntu 18.04)", + "ubuntu1804-test", + ["release-compile", "debug-compile-nosasl-nossl", "debug-compile-no-align", ".latest .nossl .nosasl"], + {"CC": "gcc", "MARCH": "i686"}, + ), + Variant( + "gcc75", + "GCC 7.5 (Ubuntu 18.04)", + "ubuntu1804-test", + [ + ".compression !.zstd", + "debug-compile-nosrv", + "release-compile", + "debug-compile-nosasl-nossl", + "debug-compile-no-align", + "debug-compile-sasl-openssl", + "debug-compile-nosasl-openssl", + ".authentication-tests .openssl", + ".authentication-tests .asan", + ".test-coverage", + ".latest .nossl", + "retry-true-latest-server", + "test-dns-openssl", + "test-dns-auth-openssl", + "test-dns-loadbalanced-openssl", + ], + {"CC": "gcc"}, + ), + Variant( + "gcc54", + "GCC 5.4 (Ubuntu 16.04)", + "ubuntu1604-test", + [".compression !.zstd", "debug-compile-nosrv", "release-compile", "debug-compile-no-align"], + {"CC": "gcc"}, + ), + Variant( + "darwin", + "*Darwin, macOS (Apple LLVM)", + "macos-1014", + [ + ".compression !.snappy", + "release-compile", + "debug-compile-nosasl-nossl", + "debug-compile-rdtscp", + "debug-compile-no-align", + "debug-compile-nosrv", + "debug-compile-sasl-darwinssl", + "debug-compile-nosasl-nossl", + ".authentication-tests .darwinssl", + ".latest .nossl", + "test-dns-darwinssl", + "test-dns-auth-darwinssl", + "debug-compile-lto", + "debug-compile-lto-thin", + "debug-compile-aws", + "test-aws-openssl-regular-4.4", + "test-aws-openssl-regular-latest", + ], + {"CC": "clang"}, + ), + Variant( + "windows-2017-32", + "Windows (i686) (VS 2017)", + "windows-64-vs2017-test", + ["debug-compile-nosasl-nossl", ".latest .nossl .nosasl"], + {"CC": "Visual Studio 15 2017"}, + ), + Variant( + "windows-2017", + "Windows (VS 2017)", + "windows-64-vs2017-test", + [ + "debug-compile-nosasl-nossl", + "debug-compile-nosasl-openssl", + "debug-compile-sspi-winssl", + ".latest .nossl", + ".nosasl .latest .nossl", + "test-dns-winssl", + "test-dns-auth-winssl", + "debug-compile-aws", + "test-aws-openssl-regular-4.4", + "test-aws-openssl-regular-latest", + # Authentication tests with OpenSSL on Windows are only run on the vs2017 variant. + # Older vs variants fail to verify certificates against Atlas tests. + ".authentication-tests .openssl !.sasl", + ".authentication-tests .winssl", + ], + {"CC": "Visual Studio 15 2017 Win64"}, + ), + Variant( + "windows-2015", + "Windows (VS 2015)", + "windows-64-vs2015-compile", + [ + ".compression !.snappy !.zstd !.latest", + "release-compile", + "debug-compile-sspi-winssl", + "debug-compile-no-align", + "debug-compile-nosrv", + ".authentication-tests .winssl", + ], + {"CC": "Visual Studio 14 2015 Win64"}, + ), + Variant( + "windows-2015-32", + "Windows (i686) (VS 2015)", + "windows-64-vs2015-compile", + [ + ".compression !.snappy !.zstd !.latest", + "release-compile", + "debug-compile-sspi-winssl", + "debug-compile-no-align", + ".authentication-tests .winssl", + ], + {"CC": "Visual Studio 14 2015"}, + ), + Variant( + "windows-2013", + "Windows (VS 2013)", + "windows-64-vs2013-compile", + [ + ".compression !.snappy !.zstd !.latest", + "release-compile", + "debug-compile-sspi-winssl", + ".authentication-tests .winssl", + ], + {"CC": "Visual Studio 12 2013 Win64"}, + ), + Variant( + "windows-2013-32", + "Windows (i686) (VS 2013)", + "windows-64-vs2013-compile", + ["release-compile", "debug-compile-rdtscp", "debug-compile-sspi-winssl", ".authentication-tests .winssl"], + {"CC": "Visual Studio 12 2013"}, + ), + Variant( + "mingw-windows2016", + "MinGW-W64 (Windows Server 2016)", + "windows-64-vs2017-test", + ["debug-compile-nosasl-nossl", ".latest .nossl .nosasl .server"], + {"CC": "mingw"}, + ), + Variant("mingw", "MinGW-W64", "windows-64-vs2013-compile", ["debug-compile-no-align"], {"CC": "mingw"}), + Variant( + "power8-rhel81", + "Power8 (ppc64le) (RHEL 8.1)", + "rhel81-power8-test", + [ + "release-compile", + "debug-compile-nosasl-nossl", + "debug-compile-sasl-openssl", + ".latest .nossl", + "test-dns-openssl", + ], + {"CC": "gcc"}, + batchtime=days(1), + ), + Variant( + "arm-ubuntu1804", + "*ARM (aarch64) (Ubuntu 18.04)", + "ubuntu1804-arm64-large", + [ + ".compression !.snappy !.zstd", + "debug-compile-no-align", + "release-compile", + "debug-compile-nosasl-nossl", + "debug-compile-nosasl-openssl", + "debug-compile-sasl-openssl", + ".authentication-tests .openssl", + ".latest .nossl", + "test-dns-openssl", + ], + {"CC": "gcc"}, + batchtime=days(1), + ), + Variant( + "arm-ubuntu1604", + "*ARM (aarch64) (Ubuntu 16.04)", + "ubuntu1604-arm64-large", + [".compression !.snappy !.zstd", "debug-compile-no-align", "release-compile"], + {"CC": "gcc"}, + batchtime=days(1), + ), + Variant( + "zseries-rhel83", + "*zSeries", + "rhel83-zseries-small", + [ + "release-compile", + # '.compression', --> TODO: waiting on ticket CDRIVER-3258 + "debug-compile-no-align", + "debug-compile-nosasl-nossl", + "debug-compile-nosasl-openssl", + "debug-compile-sasl-openssl", + ".authentication-tests .openssl", + ".latest .nossl", + ], + {"CC": "gcc"}, + batchtime=days(1), + ), + Variant( + "clang60ubuntu", + "clang 6.0 (Ubuntu 18.04)", + "ubuntu1804-test", + [ + "debug-compile-aws", + "debug-compile-sasl-openssl-static", + ".authentication-tests .asan", + "test-aws-openssl-regular-latest", + "test-aws-openssl-ec2-latest", + "test-aws-openssl-ecs-latest", + "test-aws-openssl-assume_role-latest", + "test-aws-openssl-lambda-latest", + "test-aws-openssl-regular-4.4", + "test-aws-openssl-ec2-4.4", + "test-aws-openssl-ecs-4.4", + "test-aws-openssl-assume_role-4.4", + "test-aws-openssl-lambda-4.4", + "test-aws-openssl-assume_role_with_web_identity-latest", + "test-aws-openssl-assume_role_with_web_identity-5.0", + "test-aws-openssl-assume_role_with_web_identity-4.4", + ], + {"CC": "clang"}, + ), + Variant("mongohouse", "Mongohouse Test", "ubuntu1804-test", ["debug-compile-sasl-openssl", "test-mongohouse"], {}), + Variant( + "ocsp", + "OCSP tests", + "ubuntu2004-small", + [ + OD([("name", "debug-compile-nosasl-openssl")]), + OD([("name", "debug-compile-nosasl-openssl-static")]), + OD([("name", "debug-compile-nosasl-darwinssl"), ("distros", ["macos-1014"])]), + OD([("name", "debug-compile-nosasl-winssl"), ("distros", ["windows-64-vs2017-test"])]), + OD([("name", ".ocsp-openssl")]), + OD([("name", ".ocsp-darwinssl"), ("distros", ["macos-1014"])]), + OD([("name", ".ocsp-winssl"), ("distros", ["windows-64-vs2017-test"])]), + OD([("name", "debug-compile-nosasl-openssl-1.0.1")]), + OD([("name", ".ocsp-openssl-1.0.1")]), + ], + {}, + batchtime=days(7), + ), + Variant( + "packaging", + "Linux Distro Packaging", + "ubuntu1804-test", + [ + "debian-package-build", + OD([("name", "rpm-package-build"), ("distros", ["rhel82-arm64-small"])]), + ], + {}, + batchtime=days(1), + ), + Variant( + "versioned-api", + "Versioned API Tests", + "ubuntu1804-test", + ["debug-compile-nosasl-openssl", "debug-compile-nosasl-nossl", ".versioned-api"], + {}, + ), ] diff --git a/.evergreen/legacy_config_generator/generate-evergreen-config.py b/.evergreen/legacy_config_generator/generate-evergreen-config.py index 7de58be220..c3088d207f 100644 --- a/.evergreen/legacy_config_generator/generate-evergreen-config.py +++ b/.evergreen/legacy_config_generator/generate-evergreen-config.py @@ -24,19 +24,8 @@ from collections import OrderedDict as OD from os.path import dirname, join as joinpath, normpath -import sys -try: - from evergreen_config_generator import generate -except ImportError: - sys.stderr.write("""\ -Could not find evergreen_config_generator package, try: - -python -m pip install -e git+https://github.com/mongodb-labs/\ -drivers-evergreen-tools#subdirectory=evergreen_config_generator\ -&egg=evergreen_config_generator -""") - raise +from evergreen_config_generator import generate from evergreen_config_lib.functions import all_functions from evergreen_config_lib.tasks import all_tasks @@ -45,16 +34,19 @@ from evergreen_config_lib.testgcpkms import testgcpkms_generate from evergreen_config_lib.testazurekms import testazurekms_generate -testazurekms_generate (all_tasks, all_variants, all_task_groups) -testgcpkms_generate(all_tasks, all_variants, all_task_groups) +task_groups = list(all_task_groups) +testazurekms_generate(all_tasks, all_variants, task_groups) +testgcpkms_generate(all_tasks, all_variants, task_groups) -config = OD([ - ('functions', all_functions), - ('tasks', all_tasks), - ('task_groups', all_task_groups), - ('buildvariants', all_variants), -]) +config = OD( + [ + ("functions", all_functions), + ("tasks", all_tasks), + ("task_groups", task_groups), + ("buildvariants", all_variants), + ] +) this_dir = dirname(__file__) -generated_configs_dir = normpath(joinpath(this_dir, '../generated_configs')) -generate(config, joinpath(generated_configs_dir, 'legacy-config.yml')) +generated_configs_dir = normpath(joinpath(this_dir, "../generated_configs")) +generate(config, joinpath(generated_configs_dir, "legacy-config.yml")) From 59af8a5b77dc192ae0b8ba5c7ea31e06d7a948df Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Thu, 11 May 2023 21:40:47 +0000 Subject: [PATCH 07/13] Fix: We haven't been toggling ZSTD in the build --- .evergreen/scripts/compile-unix.sh | 1 + .evergreen/scripts/link-sample-program.sh | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.evergreen/scripts/compile-unix.sh b/.evergreen/scripts/compile-unix.sh index 0c3f9fc849..022e471df5 100755 --- a/.evergreen/scripts/compile-unix.sh +++ b/.evergreen/scripts/compile-unix.sh @@ -78,6 +78,7 @@ configure_flags_append_if_not_null SNAPPY "-DENABLE_SNAPPY=${SNAPPY}" configure_flags_append_if_not_null SRV "-DENABLE_SRV=${SRV}" configure_flags_append_if_not_null TRACING "-DENABLE_TRACING=${TRACING}" configure_flags_append_if_not_null ZLIB "-DENABLE_ZLIB=${ZLIB}" +configure_flags_append_if_not_null ZSTD "-DENABLE_ZSTD=${ZSTD}" if [[ "${DEBUG}" == "ON" ]]; then configure_flags_append "-DCMAKE_BUILD_TYPE=Debug" diff --git a/.evergreen/scripts/link-sample-program.sh b/.evergreen/scripts/link-sample-program.sh index 5a95f6e35c..9606ce9cf8 100644 --- a/.evergreen/scripts/link-sample-program.sh +++ b/.evergreen/scripts/link-sample-program.sh @@ -72,8 +72,6 @@ else STATIC_CMAKE_OPTION="-DENABLE_STATIC=OFF -DENABLE_TESTS=OFF" fi -ZSTD="AUTO" - $CMAKE -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DCMAKE_PREFIX_PATH=$INSTALL_DIR/lib/cmake $SSL_CMAKE_OPTION $SNAPPY_CMAKE_OPTION $STATIC_CMAKE_OPTION -DENABLE_BSON=ON -DENABLE_ZSTD=$ZSTD . $CMAKE --build . $CMAKE --build . --target install From 2a52185c6152b8d477e201494663631ee45f6a57 Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Thu, 11 May 2023 21:41:10 +0000 Subject: [PATCH 08/13] Tweak configs to access a list of compression types --- .../evergreen_config_lib/tasks.py | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py b/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py index 174ba462e7..2529c47d24 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py +++ b/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py @@ -36,6 +36,10 @@ TopologyStr = Literal["server"] +def onoff(b: bool) -> ToggleStr: + return "ON" if b else "OFF" + + class CompileTask(NamedTask): cls_compile_sh_env: ClassVar[Mapping[str, str]] = {} cls_tags: ClassVar[Sequence[str]] = () @@ -46,7 +50,7 @@ def __init__( task_name: str, tags: Iterable[str] = (), config: str = "debug", - compression: str | None = "default", + compression: None | Sequence[Literal["snappy", "zlib", "zstd"]] | Literal["all"] = None, suffix_commands: Iterable[Value] = (), depends_on: Iterable[DependencySpec] = (), prefix_commands: Iterable[Value] = (), @@ -100,10 +104,11 @@ def __init__( if TOPOLOGY: self.compile_sh_opt["TOPOLOGY"] = TOPOLOGY - if compression != "default": - self.compile_sh_opt["SNAPPY"] = "ON" if compression in ("all", "snappy") else "OFF" - self.compile_sh_opt["ZLIB"] = "BUNDLED" if compression in ("all", "zlib") else "OFF" - self.compile_sh_opt["ZSTD"] = "ON" if compression in ("all", "zstd") else "OFF" + if compression is not None: + all_compression = compression == "all" + self.compile_sh_opt["SNAPPY"] = onoff(all_compression or "snappy" in compression) + self.compile_sh_opt["ZLIB"] = "BUNDLED" if (all_compression or "zlib" in compression) else "OFF" + self.compile_sh_opt["ZSTD"] = onoff(all_compression or "zstd" in compression) if sanitize: self.compile_sh_opt["SANITIZE"] = ",".join(sanitize) @@ -183,18 +188,18 @@ def __init__( CompileTask( "hardened-compile", tags=["hardened"], - compression=None, + compression=[], CFLAGS="-fno-strict-overflow -D_FORTIFY_SOURCE=2 -fstack-protector-all -fPIE -O", LDFLAGS="-pie -Wl,-z,relro -Wl,-z,now", ), - CompileTask("debug-compile-compression-zlib", tags=["zlib", "compression"], compression="zlib"), - CompileTask("debug-compile-compression-snappy", tags=["snappy", "compression"], compression="snappy"), - CompileTask("debug-compile-compression-zstd", tags=["zstd", "compression"], compression="zstd"), + CompileTask("debug-compile-compression-zlib", tags=["zlib", "compression"], compression=["zlib"]), + CompileTask("debug-compile-compression-snappy", tags=["snappy", "compression"], compression=["snappy"]), + CompileTask("debug-compile-compression-zstd", tags=["zstd", "compression"], compression=["zstd"]), CompileTask("debug-compile-compression", tags=["zlib", "snappy", "zstd", "compression"], compression="all"), CompileTask( "debug-compile-no-align", tags=["debug-compile"], - compression="zlib", + compression=["zlib"], EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF", ), CompileTask("debug-compile-nosasl-nossl", tags=["debug-compile", "nosasl", "nossl"], SSL="OFF"), @@ -204,7 +209,7 @@ def __init__( SpecialTask( "debug-compile-asan-clang", tags=["debug-compile", "asan-clang"], - compression="zlib", + compression=["zlib"], CFLAGS="-fno-omit-frame-pointer", CHECK_LOG="ON", sanitize=["address"], @@ -213,7 +218,7 @@ def __init__( SpecialTask( "debug-compile-asan-clang-openssl", tags=["debug-compile", "asan-clang"], - compression="zlib", + compression=["zlib"], CFLAGS="-fno-omit-frame-pointer", CHECK_LOG="ON", sanitize=["address"], From fe80c4d08f52519ad18099ce09d2d5bfc37ab7db Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Thu, 11 May 2023 22:23:29 +0000 Subject: [PATCH 09/13] Missing pass-through of more options --- .evergreen/scripts/compile-unix.sh | 8 +++++--- .evergreen/scripts/compile-windows.sh | 9 ++++++--- CMakeLists.txt | 10 +++++++++- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.evergreen/scripts/compile-unix.sh b/.evergreen/scripts/compile-unix.sh index 022e471df5..21afa5667b 100755 --- a/.evergreen/scripts/compile-unix.sh +++ b/.evergreen/scripts/compile-unix.sh @@ -23,12 +23,13 @@ check_var_opt MARCH check_var_opt RELEASE "OFF" check_var_opt SANITIZE check_var_opt SASL "OFF" # CMake default: AUTO. -check_var_opt SNAPPY # CMake default: AUTO. -check_var_opt SRV # CMake default: AUTO. +check_var_opt SNAPPY ON # CMake default: AUTO. +check_var_opt SRV ON # CMake default: AUTO. check_var_opt SSL "OFF" # CMake default: AUTO. check_var_opt TRACING # CMake default: OFF. check_var_opt ZLIB "BUNDLED" # CMake default: AUTO. -check_var_opt ZSTD # CMake default: AUTO. +check_var_opt ZSTD ON # CMake default: AUTO. +check_var_opt ICU ON # CMake default: AUTO. declare script_dir script_dir="$(to_absolute "$(dirname "${BASH_SOURCE[0]}")")" @@ -79,6 +80,7 @@ configure_flags_append_if_not_null SRV "-DENABLE_SRV=${SRV}" configure_flags_append_if_not_null TRACING "-DENABLE_TRACING=${TRACING}" configure_flags_append_if_not_null ZLIB "-DENABLE_ZLIB=${ZLIB}" configure_flags_append_if_not_null ZSTD "-DENABLE_ZSTD=${ZSTD}" +configure_flags_append_if_not_null ICU "-DENABLE_ICU=${ICU}" if [[ "${DEBUG}" == "ON" ]]; then configure_flags_append "-DCMAKE_BUILD_TYPE=Debug" diff --git a/.evergreen/scripts/compile-windows.sh b/.evergreen/scripts/compile-windows.sh index 68c67633a5..fb9cfaa842 100755 --- a/.evergreen/scripts/compile-windows.sh +++ b/.evergreen/scripts/compile-windows.sh @@ -16,10 +16,11 @@ check_var_opt DEBUG "OFF" check_var_opt EXTRA_CONFIGURE_FLAGS check_var_opt RELEASE "OFF" check_var_opt SASL "SSPI" # CMake default: AUTO. -check_var_opt SNAPPY # CMake default: AUTO. -check_var_opt SRV # CMake default: AUTO. +check_var_opt SNAPPY ON # CMake default: AUTO. +check_var_opt SRV ON # CMake default: AUTO. check_var_opt SSL "WINDOWS" # CMake default: OFF. -check_var_opt ZSTD # CMake default: AUTO. +check_var_opt ZSTD ON # CMake default: AUTO. +check_var_opt ICU ON # CMake default: AUTO. declare script_dir script_dir="$(to_absolute "$(dirname "${BASH_SOURCE[0]}")")" @@ -61,6 +62,8 @@ configure_flags_append_if_not_null SASL "-DENABLE_SASL=${SASL}" configure_flags_append_if_not_null SNAPPY "-DENABLE_SNAPPY=${SNAPPY}" configure_flags_append_if_not_null SRV "-DENABLE_SRV=${SRV}" configure_flags_append_if_not_null ZLIB "-DENABLE_ZLIB=${ZLIB}" +configure_flags_append_if_not_null ZSTD "-DENABLE_ZSTD=${ZSTD}" +configure_flags_append_if_not_null ICU "-DENABLE_ICU=${ICU}" if [[ "${DEBUG}" == "ON" ]]; then configure_flags_append "-DCMAKE_BUILD_TYPE=Debug" diff --git a/CMakeLists.txt b/CMakeLists.txt index 3dc4932bae..a9bf357d78 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,7 +126,15 @@ mongo_bool_setting(ENABLE_ICU "Enable ICU support, needed for non-ASCII username mongo_bool_setting(ENABLE_CLIENT_SIDE_ENCRYPTION "Enable In-Use Encryption support. Requires additional support libraries.") if(ENABLE_SSL) # Only define this as a setting if we have an SSL enabled, otherwise we should implicitly disable it: - mongo_bool_setting(ENABLE_MONGODB_AWS_AUTH "Enable support for the MONGODB-AWS authentication mechanism") + mongo_bool_setting( + ENABLE_MONGODB_AWS_AUTH "Enable support for the MONGODB-AWS authentication mechanism" + DEFAULT EVAL [[ + set(DEFAULT ON) + if(MSVC AND MSVC_VERSION LESS 1900) + set(DEFAULT OFF) + message(DEBUG "ENABLE_MONGODB_AWS_AUTH is “OFF” by default on for MSVC older than 19.0 (VS 2015)") + endif() + ]]) elseif(ENABLE_MONGODB_AWS_AUTH) # The user requested AWS auth (or it is still enabled in the cache), but we don't have ENABLE_SSL message(SEND_ERROR "ENABLE_MONGODB_AWS_AUTH requires ENABLE_SSL") From e3c1b859595c991f83a0b177bba3dd1e0b1a9a0d Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Thu, 11 May 2023 22:52:04 +0000 Subject: [PATCH 10/13] Be explicit about compression settings --- .../generated_configs/legacy-config.yml | 57 ++++---- .../evergreen_config_lib/tasks.py | 137 ++++++++++++++---- 2 files changed, 140 insertions(+), 54 deletions(-) diff --git a/.evergreen/generated_configs/legacy-config.yml b/.evergreen/generated_configs/legacy-config.yml index 4772382b63..6d2ae7b74c 100644 --- a/.evergreen/generated_configs/legacy-config.yml +++ b/.evergreen/generated_configs/legacy-config.yml @@ -412,7 +412,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SSL="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" SNAPPY="OFF" SSL="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-lto commands: @@ -424,7 +424,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-flto" DEBUG="ON" bash .evergreen/scripts/compile.sh + env CFLAGS="-flto" DEBUG="ON" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-lto-thin commands: @@ -436,7 +436,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-flto=thin" DEBUG="ON" bash .evergreen/scripts/compile.sh + env CFLAGS="-flto=thin" DEBUG="ON" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-no-counters tags: @@ -451,7 +451,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ENABLE_SHM_COUNTERS="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ENABLE_SHM_COUNTERS="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-asan-clang tags: @@ -495,7 +495,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-Werror -Wno-cast-align" DEBUG="ON" TRACING="ON" bash .evergreen/scripts/compile.sh + env CFLAGS="-Werror -Wno-cast-align" DEBUG="ON" SNAPPY="OFF" TRACING="ON" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: release-compile depends_on: @@ -510,7 +510,7 @@ tasks: shell: bash script: |- set -o errexit - env RELEASE="ON" bash .evergreen/scripts/compile.sh + env RELEASE="ON" SNAPPY="ON" ZLIB="BUNDLED" ZSTD="ON" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-openssl tags: @@ -526,7 +526,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SSL="OPENSSL" bash .evergreen/scripts/compile.sh + env DEBUG="ON" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-openssl-static tags: @@ -542,7 +542,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SSL="OPENSSL_STATIC" bash .evergreen/scripts/compile.sh + env DEBUG="ON" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-darwinssl tags: @@ -558,7 +558,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SSL="DARWIN" bash .evergreen/scripts/compile.sh + env DEBUG="ON" SNAPPY="OFF" SSL="DARWIN" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-winssl tags: @@ -574,7 +574,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SSL="WINDOWS" bash .evergreen/scripts/compile.sh + env DEBUG="ON" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-nossl tags: @@ -590,7 +590,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="AUTO" SSL="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" SASL="AUTO" SNAPPY="OFF" SSL="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-openssl tags: @@ -606,7 +606,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="AUTO" SSL="OPENSSL" bash .evergreen/scripts/compile.sh + env DEBUG="ON" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-openssl-static tags: @@ -622,7 +622,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="AUTO" SSL="OPENSSL_STATIC" bash .evergreen/scripts/compile.sh + env DEBUG="ON" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-darwinssl tags: @@ -638,7 +638,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="AUTO" SSL="DARWIN" bash .evergreen/scripts/compile.sh + env DEBUG="ON" SASL="AUTO" SNAPPY="OFF" SSL="DARWIN" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-winssl tags: @@ -654,7 +654,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="CYRUS" SSL="WINDOWS" bash .evergreen/scripts/compile.sh + env DEBUG="ON" SASL="CYRUS" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sspi-nossl tags: @@ -670,7 +670,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="SSPI" SSL="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" SASL="SSPI" SNAPPY="OFF" SSL="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sspi-openssl tags: @@ -686,7 +686,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="SSPI" SSL="OPENSSL" bash .evergreen/scripts/compile.sh + env DEBUG="ON" SASL="SSPI" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sspi-openssl-static tags: @@ -702,7 +702,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="SSPI" SSL="OPENSSL_STATIC" bash .evergreen/scripts/compile.sh + env DEBUG="ON" SASL="SSPI" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-rdtscp commands: @@ -714,7 +714,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ENABLE_RDTSCP="ON" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ENABLE_RDTSCP="ON" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sspi-winssl tags: @@ -730,7 +730,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="SSPI" SSL="WINDOWS" bash .evergreen/scripts/compile.sh + env DEBUG="ON" SASL="SSPI" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosrv tags: @@ -744,7 +744,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SRV="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" SNAPPY="OFF" SRV="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: link-with-cmake depends_on: @@ -1034,7 +1034,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-Werror -Wno-cast-align" DEBUG="ON" bash .evergreen/scripts/compile.sh + env CFLAGS="-Werror -Wno-cast-align" DEBUG="ON" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-openssl-cse tags: @@ -1052,7 +1052,7 @@ tasks: shell: bash script: |- set -o errexit - env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" SASL="AUTO" SSL="OPENSSL" bash .evergreen/scripts/compile.sh + env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-openssl-static-cse tags: @@ -1070,7 +1070,7 @@ tasks: shell: bash script: |- set -o errexit - env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" SASL="AUTO" SSL="OPENSSL_STATIC" bash .evergreen/scripts/compile.sh + env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-darwinssl-cse tags: @@ -1088,7 +1088,7 @@ tasks: shell: bash script: |- set -o errexit - env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" SASL="AUTO" SSL="DARWIN" bash .evergreen/scripts/compile.sh + env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" SASL="AUTO" SNAPPY="OFF" SSL="DARWIN" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-winssl-cse tags: @@ -1106,7 +1106,7 @@ tasks: shell: bash script: |- set -o errexit - env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" SASL="AUTO" SSL="WINDOWS" bash .evergreen/scripts/compile.sh + env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" SASL="AUTO" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-asan-openssl-cse tags: @@ -1122,7 +1122,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-fno-omit-frame-pointer" CHECK_LOG="ON" COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" PATH="/usr/lib/llvm-3.8/bin:$PATH" SANITIZE="address" SSL="OPENSSL" bash .evergreen/scripts/compile.sh + env CFLAGS="-fno-omit-frame-pointer" CHECK_LOG="ON" COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" PATH="/usr/lib/llvm-3.8/bin:$PATH" SANITIZE="address" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-openssl-1.0.1 commands: @@ -1137,7 +1137,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-Wno-redundant-decls" DEBUG="ON" SASL="OFF" SSL="OPENSSL" bash .evergreen/scripts/compile.sh + env CFLAGS="-Wno-redundant-decls" DEBUG="ON" SASL="OFF" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: build-and-test-with-toolchain commands: @@ -1837,6 +1837,7 @@ tasks: type: test params: working_dir: mongoc + add_expansions_to_env: true shell: bash script: |- set -o errexit diff --git a/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py b/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py index 2529c47d24..93d80e1ca5 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py +++ b/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py @@ -13,6 +13,7 @@ # limitations under the License. from collections import OrderedDict as OD +from copy import copy from itertools import chain from typing import ClassVar, Iterable, Literal, Mapping, MutableMapping, MutableSequence, Optional, Sequence @@ -49,13 +50,13 @@ def __init__( self, task_name: str, tags: Iterable[str] = (), + *, config: str = "debug", - compression: None | Sequence[Literal["snappy", "zlib", "zstd"]] | Literal["all"] = None, + compression: None | Sequence[Literal["snappy", "zlib", "zstd"]] | Literal["all"], suffix_commands: Iterable[Value] = (), depends_on: Iterable[DependencySpec] = (), prefix_commands: Iterable[Value] = (), sanitize: Iterable[Literal["undefined", "address", "thread"]] = (), - *, CFLAGS: str | None = None, LDFLAGS: str | None = None, EXTRA_CONFIGURE_FLAGS: str | None = None, @@ -202,10 +203,12 @@ def __init__( compression=["zlib"], EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF", ), - CompileTask("debug-compile-nosasl-nossl", tags=["debug-compile", "nosasl", "nossl"], SSL="OFF"), - CompileTask("debug-compile-lto", CFLAGS="-flto"), - CompileTask("debug-compile-lto-thin", CFLAGS="-flto=thin"), - CompileTask("debug-compile-no-counters", tags=["debug-compile", "no-counters"], ENABLE_SHM_COUNTERS="OFF"), + CompileTask("debug-compile-nosasl-nossl", tags=["debug-compile", "nosasl", "nossl"], SSL="OFF", compression=[]), + CompileTask("debug-compile-lto", CFLAGS="-flto", compression=[]), + CompileTask("debug-compile-lto-thin", CFLAGS="-flto=thin", compression=[]), + CompileTask( + "debug-compile-no-counters", tags=["debug-compile", "no-counters"], ENABLE_SHM_COUNTERS="OFF", compression=[] + ), SpecialTask( "debug-compile-asan-clang", tags=["debug-compile", "asan-clang"], @@ -225,43 +228,102 @@ def __init__( EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF", SSL="OPENSSL", ), - CompileTask("compile-tracing", TRACING="ON", CFLAGS="-Werror -Wno-cast-align"), + CompileTask("compile-tracing", TRACING="ON", CFLAGS="-Werror -Wno-cast-align", compression=[]), + CompileTask( + "release-compile", + config="release", + depends_on=[OD([("name", "make-release-archive"), ("variant", "releng")])], + compression="all", + ), + CompileTask( + "debug-compile-nosasl-openssl", + tags=["debug-compile", "nosasl", "openssl"], + SSL="OPENSSL", + compression=[], + ), CompileTask( - "release-compile", config="release", depends_on=[OD([("name", "make-release-archive"), ("variant", "releng")])] + "debug-compile-nosasl-openssl-static", + tags=["debug-compile", "nosasl", "openssl-static"], + SSL="OPENSSL_STATIC", + compression=[], ), - CompileTask("debug-compile-nosasl-openssl", tags=["debug-compile", "nosasl", "openssl"], SSL="OPENSSL"), CompileTask( - "debug-compile-nosasl-openssl-static", tags=["debug-compile", "nosasl", "openssl-static"], SSL="OPENSSL_STATIC" + "debug-compile-nosasl-darwinssl", tags=["debug-compile", "nosasl", "darwinssl"], SSL="DARWIN", compression=[] + ), + CompileTask( + "debug-compile-nosasl-winssl", tags=["debug-compile", "nosasl", "winssl"], SSL="WINDOWS", compression=[] + ), + CompileTask( + "debug-compile-sasl-nossl", tags=["debug-compile", "sasl", "nossl"], SASL="AUTO", SSL="OFF", compression=[] + ), + CompileTask( + "debug-compile-sasl-openssl", + tags=["debug-compile", "sasl", "openssl"], + SASL="AUTO", + SSL="OPENSSL", + compression=[], ), - CompileTask("debug-compile-nosasl-darwinssl", tags=["debug-compile", "nosasl", "darwinssl"], SSL="DARWIN"), - CompileTask("debug-compile-nosasl-winssl", tags=["debug-compile", "nosasl", "winssl"], SSL="WINDOWS"), - CompileTask("debug-compile-sasl-nossl", tags=["debug-compile", "sasl", "nossl"], SASL="AUTO", SSL="OFF"), - CompileTask("debug-compile-sasl-openssl", tags=["debug-compile", "sasl", "openssl"], SASL="AUTO", SSL="OPENSSL"), CompileTask( "debug-compile-sasl-openssl-static", tags=["debug-compile", "sasl", "openssl-static"], SASL="AUTO", SSL="OPENSSL_STATIC", + compression=[], + ), + CompileTask( + "debug-compile-sasl-darwinssl", + tags=["debug-compile", "sasl", "darwinssl"], + SASL="AUTO", + SSL="DARWIN", + compression=[], ), - CompileTask("debug-compile-sasl-darwinssl", tags=["debug-compile", "sasl", "darwinssl"], SASL="AUTO", SSL="DARWIN"), CompileTask( "debug-compile-sasl-winssl", tags=["debug-compile", "sasl", "winssl"], # Explicitly use CYRUS. SASL="CYRUS", SSL="WINDOWS", + compression=[], + ), + CompileTask( + "debug-compile-sspi-nossl", + tags=["debug-compile", "sspi", "nossl"], + SASL="SSPI", + SSL="OFF", + compression=[], + ), + CompileTask( + "debug-compile-sspi-openssl", + tags=["debug-compile", "sspi", "openssl"], + SASL="SSPI", + SSL="OPENSSL", + compression=[], ), - CompileTask("debug-compile-sspi-nossl", tags=["debug-compile", "sspi", "nossl"], SASL="SSPI", SSL="OFF"), - CompileTask("debug-compile-sspi-openssl", tags=["debug-compile", "sspi", "openssl"], SASL="SSPI", SSL="OPENSSL"), CompileTask( "debug-compile-sspi-openssl-static", tags=["debug-compile", "sspi", "openssl-static"], SASL="SSPI", SSL="OPENSSL_STATIC", + compression=[], + ), + CompileTask( + "debug-compile-rdtscp", + ENABLE_RDTSCP="ON", + compression=[], + ), + CompileTask( + "debug-compile-sspi-winssl", + tags=["debug-compile", "sspi", "winssl"], + SASL="SSPI", + SSL="WINDOWS", + compression=[], + ), + CompileTask( + "debug-compile-nosrv", + tags=["debug-compile"], + SRV="OFF", + compression=[], ), - CompileTask("debug-compile-rdtscp", ENABLE_RDTSCP="ON"), - CompileTask("debug-compile-sspi-winssl", tags=["debug-compile", "sspi", "winssl"], SASL="SSPI", SSL="WINDOWS"), - CompileTask("debug-compile-nosrv", tags=["debug-compile"], SRV="OFF"), LinkTask("link-with-cmake", suffix_commands=[func("link sample program", BUILD_SAMPLE_WITH_CMAKE=1)]), LinkTask( "link-with-cmake-ssl", @@ -385,24 +447,45 @@ def __init__( ) ], ), - CompileTask("debug-compile-with-warnings", CFLAGS="-Werror -Wno-cast-align"), + CompileTask( + "debug-compile-with-warnings", + CFLAGS="-Werror -Wno-cast-align", + compression=[], + ), CompileWithClientSideEncryption( - "debug-compile-sasl-openssl-cse", tags=["debug-compile", "sasl", "openssl"], SASL="AUTO", SSL="OPENSSL" + "debug-compile-sasl-openssl-cse", + tags=["debug-compile", "sasl", "openssl"], + SASL="AUTO", + SSL="OPENSSL", + compression=[], ), CompileWithClientSideEncryption( "debug-compile-sasl-openssl-static-cse", tags=["debug-compile", "sasl", "openssl-static"], SASL="AUTO", SSL="OPENSSL_STATIC", + compression=[], ), CompileWithClientSideEncryption( - "debug-compile-sasl-darwinssl-cse", tags=["debug-compile", "sasl", "darwinssl"], SASL="AUTO", SSL="DARWIN" + "debug-compile-sasl-darwinssl-cse", + tags=["debug-compile", "sasl", "darwinssl"], + SASL="AUTO", + SSL="DARWIN", + compression=[], ), CompileWithClientSideEncryption( - "debug-compile-sasl-winssl-cse", tags=["debug-compile", "sasl", "winssl"], SASL="AUTO", SSL="WINDOWS" + "debug-compile-sasl-winssl-cse", + tags=["debug-compile", "sasl", "winssl"], + SASL="AUTO", + SSL="WINDOWS", + compression=[], ), CompileWithClientSideEncryptionAsan( - "debug-compile-asan-openssl-cse", tags=["debug-compile", "asan-clang"], SSL="OPENSSL", sanitize=["address"] + "debug-compile-asan-openssl-cse", + tags=["debug-compile", "asan-clang"], + SSL="OPENSSL", + sanitize=["address"], + compression=[], ), CompileTask( "debug-compile-nosasl-openssl-1.0.1", @@ -410,6 +493,7 @@ def __init__( CFLAGS="-Wno-redundant-decls", SSL="OPENSSL", SASL="OFF", + compression=[], ), NamedTask( "build-and-test-with-toolchain", @@ -938,7 +1022,8 @@ def do_is_valid_combination(self) -> bool: export CC='${CC}' "$cmake" -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF . "$cmake" --build . --target test-awsauth - """ + """, + add_expansions_to_env=True, ), func("upload-build"), ], From 1ac0deee43bc5f3d0ba2bc3736daf1e3619fd779 Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Thu, 11 May 2023 23:06:28 +0000 Subject: [PATCH 11/13] Disable ICU (by default) on all builds. --- .../generated_configs/legacy-config.yml | 72 +++++++++---------- .../evergreen_config_lib/tasks.py | 40 ++++++----- 2 files changed, 58 insertions(+), 54 deletions(-) diff --git a/.evergreen/generated_configs/legacy-config.yml b/.evergreen/generated_configs/legacy-config.yml index 6d2ae7b74c..a8b70afe11 100644 --- a/.evergreen/generated_configs/legacy-config.yml +++ b/.evergreen/generated_configs/legacy-config.yml @@ -320,7 +320,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-fno-strict-overflow -D_FORTIFY_SOURCE=2 -fstack-protector-all -fPIE -O" DEBUG="ON" LDFLAGS="-pie -Wl,-z,relro -Wl,-z,now" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-fno-strict-overflow -D_FORTIFY_SOURCE=2 -fstack-protector-all -fPIE -O" DEBUG="ON" ICU="OFF" LDFLAGS="-pie -Wl,-z,relro -Wl,-z,now" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-compression-zlib tags: @@ -335,7 +335,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SNAPPY="OFF" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SNAPPY="OFF" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-compression-snappy tags: @@ -350,7 +350,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SNAPPY="ON" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SNAPPY="ON" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-compression-zstd tags: @@ -365,7 +365,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SNAPPY="OFF" ZLIB="OFF" ZSTD="ON" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="ON" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-compression tags: @@ -382,7 +382,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SNAPPY="ON" ZLIB="BUNDLED" ZSTD="ON" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SNAPPY="ON" ZLIB="BUNDLED" ZSTD="ON" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-no-align tags: @@ -396,7 +396,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" SNAPPY="OFF" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" ICU="OFF" SNAPPY="OFF" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-nossl tags: @@ -412,7 +412,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SNAPPY="OFF" SSL="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SNAPPY="OFF" SSL="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-lto commands: @@ -424,7 +424,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-flto" DEBUG="ON" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-flto" DEBUG="ON" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-lto-thin commands: @@ -436,7 +436,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-flto=thin" DEBUG="ON" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-flto=thin" DEBUG="ON" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-no-counters tags: @@ -451,7 +451,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ENABLE_SHM_COUNTERS="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ENABLE_SHM_COUNTERS="OFF" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-asan-clang tags: @@ -467,7 +467,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-fno-omit-frame-pointer" CHECK_LOG="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" SANITIZE="address" SNAPPY="OFF" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-fno-omit-frame-pointer" CHECK_LOG="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" ICU="OFF" SANITIZE="address" SNAPPY="OFF" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-asan-clang-openssl tags: @@ -483,7 +483,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-fno-omit-frame-pointer" CHECK_LOG="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" SANITIZE="address" SNAPPY="OFF" SSL="OPENSSL" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-fno-omit-frame-pointer" CHECK_LOG="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" ICU="OFF" SANITIZE="address" SNAPPY="OFF" SSL="OPENSSL" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: compile-tracing commands: @@ -495,7 +495,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-Werror -Wno-cast-align" DEBUG="ON" SNAPPY="OFF" TRACING="ON" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-Werror -Wno-cast-align" DEBUG="ON" ICU="OFF" SNAPPY="OFF" TRACING="ON" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: release-compile depends_on: @@ -510,7 +510,7 @@ tasks: shell: bash script: |- set -o errexit - env RELEASE="ON" SNAPPY="ON" ZLIB="BUNDLED" ZSTD="ON" bash .evergreen/scripts/compile.sh + env ICU="OFF" RELEASE="ON" SNAPPY="ON" ZLIB="BUNDLED" ZSTD="ON" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-openssl tags: @@ -526,7 +526,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-openssl-static tags: @@ -542,7 +542,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-darwinssl tags: @@ -558,7 +558,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SNAPPY="OFF" SSL="DARWIN" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SNAPPY="OFF" SSL="DARWIN" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-winssl tags: @@ -574,7 +574,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-nossl tags: @@ -590,7 +590,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="AUTO" SNAPPY="OFF" SSL="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-openssl tags: @@ -606,7 +606,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-openssl-static tags: @@ -622,7 +622,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-darwinssl tags: @@ -638,7 +638,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="AUTO" SNAPPY="OFF" SSL="DARWIN" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="DARWIN" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-winssl tags: @@ -654,7 +654,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="CYRUS" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SASL="CYRUS" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sspi-nossl tags: @@ -670,7 +670,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="SSPI" SNAPPY="OFF" SSL="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SASL="SSPI" SNAPPY="OFF" SSL="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sspi-openssl tags: @@ -686,7 +686,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="SSPI" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SASL="SSPI" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sspi-openssl-static tags: @@ -702,7 +702,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="SSPI" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SASL="SSPI" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-rdtscp commands: @@ -714,7 +714,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ENABLE_RDTSCP="ON" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ENABLE_RDTSCP="ON" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sspi-winssl tags: @@ -730,7 +730,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SASL="SSPI" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SASL="SSPI" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosrv tags: @@ -744,7 +744,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" SNAPPY="OFF" SRV="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env DEBUG="ON" ICU="OFF" SNAPPY="OFF" SRV="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: link-with-cmake depends_on: @@ -1034,7 +1034,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-Werror -Wno-cast-align" DEBUG="ON" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-Werror -Wno-cast-align" DEBUG="ON" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-openssl-cse tags: @@ -1052,7 +1052,7 @@ tasks: shell: bash script: |- set -o errexit - env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-openssl-static-cse tags: @@ -1070,7 +1070,7 @@ tasks: shell: bash script: |- set -o errexit - env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-darwinssl-cse tags: @@ -1088,7 +1088,7 @@ tasks: shell: bash script: |- set -o errexit - env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" SASL="AUTO" SNAPPY="OFF" SSL="DARWIN" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="DARWIN" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-winssl-cse tags: @@ -1106,7 +1106,7 @@ tasks: shell: bash script: |- set -o errexit - env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" SASL="AUTO" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-asan-openssl-cse tags: @@ -1122,7 +1122,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-fno-omit-frame-pointer" CHECK_LOG="ON" COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" PATH="/usr/lib/llvm-3.8/bin:$PATH" SANITIZE="address" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-fno-omit-frame-pointer" CHECK_LOG="ON" COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" ICU="OFF" PATH="/usr/lib/llvm-3.8/bin:$PATH" SANITIZE="address" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-openssl-1.0.1 commands: @@ -1137,7 +1137,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-Wno-redundant-decls" DEBUG="ON" SASL="OFF" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-Wno-redundant-decls" DEBUG="ON" ICU="OFF" SASL="OFF" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: build-and-test-with-toolchain commands: diff --git a/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py b/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py index 93d80e1ca5..e1bda45ade 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py +++ b/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py @@ -68,6 +68,7 @@ def __init__( ENABLE_RDTSCP: OptToggleStr = None, SRV: OptToggleStr = None, TOPOLOGY: TopologyStr | None = None, + icu: bool = False, ): super(CompileTask, self).__init__(task_name=task_name, depends_on=depends_on, tags=tags) @@ -76,45 +77,48 @@ def __init__( # Environment variables for .evergreen/scripts/compile.sh. self.compile_sh_opt: dict[str, str] = {} + opts = self.compile_sh_opt if config == "debug": - self.compile_sh_opt["DEBUG"] = "ON" + opts["DEBUG"] = "ON" else: assert config == "release" - self.compile_sh_opt["RELEASE"] = "ON" + opts["RELEASE"] = "ON" if CFLAGS: - self.compile_sh_opt["CFLAGS"] = CFLAGS + opts["CFLAGS"] = CFLAGS if LDFLAGS: - self.compile_sh_opt["LDFLAGS"] = LDFLAGS + opts["LDFLAGS"] = LDFLAGS if EXTRA_CONFIGURE_FLAGS: - self.compile_sh_opt["EXTRA_CONFIGURE_FLAGS"] = EXTRA_CONFIGURE_FLAGS + opts["EXTRA_CONFIGURE_FLAGS"] = EXTRA_CONFIGURE_FLAGS if SSL: - self.compile_sh_opt["SSL"] = SSL + opts["SSL"] = SSL if ENABLE_SHM_COUNTERS: - self.compile_sh_opt["ENABLE_SHM_COUNTERS"] = ENABLE_SHM_COUNTERS + opts["ENABLE_SHM_COUNTERS"] = ENABLE_SHM_COUNTERS if CHECK_LOG: - self.compile_sh_opt["CHECK_LOG"] = CHECK_LOG + opts["CHECK_LOG"] = CHECK_LOG if TRACING: - self.compile_sh_opt["TRACING"] = TRACING + opts["TRACING"] = TRACING if SASL: - self.compile_sh_opt["SASL"] = SASL + opts["SASL"] = SASL if ENABLE_RDTSCP: - self.compile_sh_opt["ENABLE_RDTSCP"] = ENABLE_RDTSCP + opts["ENABLE_RDTSCP"] = ENABLE_RDTSCP if SRV: - self.compile_sh_opt["SRV"] = SRV + opts["SRV"] = SRV if TOPOLOGY: - self.compile_sh_opt["TOPOLOGY"] = TOPOLOGY + opts["TOPOLOGY"] = TOPOLOGY + + opts["ICU"] = onoff(icu) if compression is not None: all_compression = compression == "all" - self.compile_sh_opt["SNAPPY"] = onoff(all_compression or "snappy" in compression) - self.compile_sh_opt["ZLIB"] = "BUNDLED" if (all_compression or "zlib" in compression) else "OFF" - self.compile_sh_opt["ZSTD"] = onoff(all_compression or "zstd" in compression) + opts["SNAPPY"] = onoff(all_compression or "snappy" in compression) + opts["ZLIB"] = "BUNDLED" if (all_compression or "zlib" in compression) else "OFF" + opts["ZSTD"] = onoff(all_compression or "zstd" in compression) if sanitize: - self.compile_sh_opt["SANITIZE"] = ",".join(sanitize) + opts["SANITIZE"] = ",".join(sanitize) - self.compile_sh_opt.update(type(self).cls_compile_sh_env) + opts.update(type(self).cls_compile_sh_env) def additional_script_env(self) -> Mapping[str, str]: return {} From 20f27a7bc0221fab4c1dd707202583e1ef81f4da Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Fri, 12 May 2023 01:42:29 +0000 Subject: [PATCH 12/13] Disable CSE by default --- .../generated_configs/legacy-config.yml | 74 +++++++++---------- .../evergreen_config_lib/tasks.py | 4 +- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/.evergreen/generated_configs/legacy-config.yml b/.evergreen/generated_configs/legacy-config.yml index a8b70afe11..8a6d8b1b16 100644 --- a/.evergreen/generated_configs/legacy-config.yml +++ b/.evergreen/generated_configs/legacy-config.yml @@ -320,7 +320,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-fno-strict-overflow -D_FORTIFY_SOURCE=2 -fstack-protector-all -fPIE -O" DEBUG="ON" ICU="OFF" LDFLAGS="-pie -Wl,-z,relro -Wl,-z,now" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-fno-strict-overflow -D_FORTIFY_SOURCE=2 -fstack-protector-all -fPIE -O" CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" LDFLAGS="-pie -Wl,-z,relro -Wl,-z,now" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-compression-zlib tags: @@ -335,7 +335,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SNAPPY="OFF" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SNAPPY="OFF" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-compression-snappy tags: @@ -350,7 +350,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SNAPPY="ON" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SNAPPY="ON" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-compression-zstd tags: @@ -365,7 +365,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="ON" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="ON" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-compression tags: @@ -382,7 +382,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SNAPPY="ON" ZLIB="BUNDLED" ZSTD="ON" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SNAPPY="ON" ZLIB="BUNDLED" ZSTD="ON" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-no-align tags: @@ -396,7 +396,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" ICU="OFF" SNAPPY="OFF" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" ICU="OFF" SNAPPY="OFF" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-nossl tags: @@ -412,7 +412,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SNAPPY="OFF" SSL="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SNAPPY="OFF" SSL="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-lto commands: @@ -424,7 +424,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-flto" DEBUG="ON" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-flto" CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-lto-thin commands: @@ -436,7 +436,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-flto=thin" DEBUG="ON" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-flto=thin" CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-no-counters tags: @@ -451,7 +451,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ENABLE_SHM_COUNTERS="OFF" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ENABLE_SHM_COUNTERS="OFF" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-asan-clang tags: @@ -467,7 +467,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-fno-omit-frame-pointer" CHECK_LOG="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" ICU="OFF" SANITIZE="address" SNAPPY="OFF" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-fno-omit-frame-pointer" CHECK_LOG="ON" CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" ICU="OFF" SANITIZE="address" SNAPPY="OFF" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-asan-clang-openssl tags: @@ -483,7 +483,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-fno-omit-frame-pointer" CHECK_LOG="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" ICU="OFF" SANITIZE="address" SNAPPY="OFF" SSL="OPENSSL" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-fno-omit-frame-pointer" CHECK_LOG="ON" CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" ICU="OFF" SANITIZE="address" SNAPPY="OFF" SSL="OPENSSL" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: compile-tracing commands: @@ -495,7 +495,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-Werror -Wno-cast-align" DEBUG="ON" ICU="OFF" SNAPPY="OFF" TRACING="ON" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-Werror -Wno-cast-align" CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SNAPPY="OFF" TRACING="ON" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: release-compile depends_on: @@ -510,7 +510,7 @@ tasks: shell: bash script: |- set -o errexit - env ICU="OFF" RELEASE="ON" SNAPPY="ON" ZLIB="BUNDLED" ZSTD="ON" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" ICU="OFF" RELEASE="ON" SNAPPY="ON" ZLIB="BUNDLED" ZSTD="ON" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-openssl tags: @@ -526,7 +526,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-openssl-static tags: @@ -542,7 +542,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-darwinssl tags: @@ -558,7 +558,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SNAPPY="OFF" SSL="DARWIN" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SNAPPY="OFF" SSL="DARWIN" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-winssl tags: @@ -574,7 +574,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-nossl tags: @@ -590,7 +590,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-openssl tags: @@ -606,7 +606,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-openssl-static tags: @@ -622,7 +622,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-darwinssl tags: @@ -638,7 +638,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="DARWIN" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="DARWIN" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-winssl tags: @@ -654,7 +654,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SASL="CYRUS" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SASL="CYRUS" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sspi-nossl tags: @@ -670,7 +670,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SASL="SSPI" SNAPPY="OFF" SSL="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SASL="SSPI" SNAPPY="OFF" SSL="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sspi-openssl tags: @@ -686,7 +686,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SASL="SSPI" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SASL="SSPI" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sspi-openssl-static tags: @@ -702,7 +702,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SASL="SSPI" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SASL="SSPI" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-rdtscp commands: @@ -714,7 +714,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ENABLE_RDTSCP="ON" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ENABLE_RDTSCP="ON" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sspi-winssl tags: @@ -730,7 +730,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SASL="SSPI" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SASL="SSPI" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosrv tags: @@ -744,7 +744,7 @@ tasks: shell: bash script: |- set -o errexit - env DEBUG="ON" ICU="OFF" SNAPPY="OFF" SRV="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SNAPPY="OFF" SRV="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: link-with-cmake depends_on: @@ -1034,7 +1034,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-Werror -Wno-cast-align" DEBUG="ON" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-Werror -Wno-cast-align" CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SNAPPY="OFF" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-openssl-cse tags: @@ -1052,7 +1052,7 @@ tasks: shell: bash script: |- set -o errexit - env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-openssl-static-cse tags: @@ -1070,7 +1070,7 @@ tasks: shell: bash script: |- set -o errexit - env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="OPENSSL_STATIC" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-darwinssl-cse tags: @@ -1088,7 +1088,7 @@ tasks: shell: bash script: |- set -o errexit - env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="DARWIN" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="DARWIN" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-sasl-winssl-cse tags: @@ -1106,7 +1106,7 @@ tasks: shell: bash script: |- set -o errexit - env COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CLIENT_SIDE_ENCRYPTION="OFF" COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON" ICU="OFF" SASL="AUTO" SNAPPY="OFF" SSL="WINDOWS" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-asan-openssl-cse tags: @@ -1122,7 +1122,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-fno-omit-frame-pointer" CHECK_LOG="ON" COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" ICU="OFF" PATH="/usr/lib/llvm-3.8/bin:$PATH" SANITIZE="address" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-fno-omit-frame-pointer" CHECK_LOG="ON" CLIENT_SIDE_ENCRYPTION="OFF" COMPILE_LIBMONGOCRYPT="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" ICU="OFF" PATH="/usr/lib/llvm-3.8/bin:$PATH" SANITIZE="address" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: debug-compile-nosasl-openssl-1.0.1 commands: @@ -1137,7 +1137,7 @@ tasks: shell: bash script: |- set -o errexit - env CFLAGS="-Wno-redundant-decls" DEBUG="ON" ICU="OFF" SASL="OFF" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh + env CFLAGS="-Wno-redundant-decls" CLIENT_SIDE_ENCRYPTION="OFF" DEBUG="ON" ICU="OFF" SASL="OFF" SNAPPY="OFF" SSL="OPENSSL" ZLIB="OFF" ZSTD="OFF" bash .evergreen/scripts/compile.sh - func: upload-build - name: build-and-test-with-toolchain commands: @@ -1846,7 +1846,7 @@ tasks: . .evergreen/scripts/find-cmake-latest.sh cmake=$(find_cmake_latest) export CC='${CC}' - "$cmake" -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF . + "$cmake" -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ICU=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF . "$cmake" --build . --target test-awsauth - func: upload-build - name: test-aws-openssl-regular-latest diff --git a/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py b/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py index e1bda45ade..9bd9912b4e 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py +++ b/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py @@ -69,6 +69,7 @@ def __init__( SRV: OptToggleStr = None, TOPOLOGY: TopologyStr | None = None, icu: bool = False, + client_side_encryption: bool = False, ): super(CompileTask, self).__init__(task_name=task_name, depends_on=depends_on, tags=tags) @@ -108,6 +109,7 @@ def __init__( opts["TOPOLOGY"] = TOPOLOGY opts["ICU"] = onoff(icu) + opts["CLIENT_SIDE_ENCRYPTION"] = onoff(client_side_encryption) if compression is not None: all_compression = compression == "all" @@ -1024,7 +1026,7 @@ def do_is_valid_combination(self) -> bool: . .evergreen/scripts/find-cmake-latest.sh cmake=$(find_cmake_latest) export CC='${CC}' - "$cmake" -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF . + "$cmake" -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ICU=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF . "$cmake" --build . --target test-awsauth """, add_expansions_to_env=True, From 95f26754a95e713589f6c97845d6b7b4c860a3da Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Fri, 12 May 2023 01:57:15 +0000 Subject: [PATCH 13/13] Pass through CSE enablement --- .../evergreen_config_lib/tasks.py | 2 +- .evergreen/scripts/compile-unix.sh | 9 ++------- .evergreen/scripts/compile-windows.sh | 5 ++--- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py b/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py index 9bd9912b4e..0b9137c7fd 100644 --- a/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py +++ b/.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py @@ -565,7 +565,7 @@ def post_commands(self) -> Iterable[Value]: SASL="AUTO", SSL="OPENSSL", COMPILE_LIBMONGOCRYPT="ON", - EXTRA_CONFIGURE_FLAGS='EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON"', + EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON", ) else: yield func("compile coverage", SASL="AUTO", SSL="OPENSSL") diff --git a/.evergreen/scripts/compile-unix.sh b/.evergreen/scripts/compile-unix.sh index 21afa5667b..b9841844ce 100755 --- a/.evergreen/scripts/compile-unix.sh +++ b/.evergreen/scripts/compile-unix.sh @@ -30,6 +30,7 @@ check_var_opt TRACING # CMake default: OFF. check_var_opt ZLIB "BUNDLED" # CMake default: AUTO. check_var_opt ZSTD ON # CMake default: AUTO. check_var_opt ICU ON # CMake default: AUTO. +check_var_opt CLIENT_SIDE_ENCRYPTION ON declare script_dir script_dir="$(to_absolute "$(dirname "${BASH_SOURCE[0]}")")" @@ -81,6 +82,7 @@ configure_flags_append_if_not_null TRACING "-DENABLE_TRACING=${TRACING}" configure_flags_append_if_not_null ZLIB "-DENABLE_ZLIB=${ZLIB}" configure_flags_append_if_not_null ZSTD "-DENABLE_ZSTD=${ZSTD}" configure_flags_append_if_not_null ICU "-DENABLE_ICU=${ICU}" +configure_flags_append_if_not_null CLIENT_SIDE_ENCRYPTION "-DENABLE_CLIENT_SIDE_ENCRYPTION=${CLIENT_SIDE_ENCRYPTION}" if [[ "${DEBUG}" == "ON" ]]; then configure_flags_append "-DCMAKE_BUILD_TYPE=Debug" @@ -206,13 +208,6 @@ if [[ "${COMPILE_LIBMONGOCRYPT}" == "ON" ]]; then # shellcheck source=.evergreen/scripts/compile-libmongocrypt.sh "${script_dir}/compile-libmongocrypt.sh" "${cmake_binary}" "${mongoc_dir}" "${install_dir}" >/dev/null echo "Installing libmongocrypt... done." - - # Fail if the C driver is unable to find the installed libmongocrypt. - configure_flags_append "-DENABLE_CLIENT_SIDE_ENCRYPTION=ON" -else - # Avoid symbol collisions with libmongocrypt installed via apt/yum. - # Note: may be overwritten by ${EXTRA_CONFIGURE_FLAGS}. - configure_flags_append "-DENABLE_CLIENT_SIDE_ENCRYPTION=OFF" fi "${cmake_binary}" "${configure_flags[@]}" "${extra_configure_flags[@]}" . diff --git a/.evergreen/scripts/compile-windows.sh b/.evergreen/scripts/compile-windows.sh index fb9cfaa842..801e65b95a 100755 --- a/.evergreen/scripts/compile-windows.sh +++ b/.evergreen/scripts/compile-windows.sh @@ -21,6 +21,7 @@ check_var_opt SRV ON # CMake default: AUTO. check_var_opt SSL "WINDOWS" # CMake default: OFF. check_var_opt ZSTD ON # CMake default: AUTO. check_var_opt ICU ON # CMake default: AUTO. +check_var_opt CLIENT_SIDE_ENCRYPTION ON declare script_dir script_dir="$(to_absolute "$(dirname "${BASH_SOURCE[0]}")")" @@ -64,6 +65,7 @@ configure_flags_append_if_not_null SRV "-DENABLE_SRV=${SRV}" configure_flags_append_if_not_null ZLIB "-DENABLE_ZLIB=${ZLIB}" configure_flags_append_if_not_null ZSTD "-DENABLE_ZSTD=${ZSTD}" configure_flags_append_if_not_null ICU "-DENABLE_ICU=${ICU}" +configure_flags_append_if_not_null CLIENT_SIDE_ENCRYPTION "-DENABLE_CLIENT_SIDE_ENCRYPTION=${CLIENT_SIDE_ENCRYPTION}" if [[ "${DEBUG}" == "ON" ]]; then configure_flags_append "-DCMAKE_BUILD_TYPE=Debug" @@ -122,9 +124,6 @@ if [ "${COMPILE_LIBMONGOCRYPT}" = "ON" ]; then # shellcheck source=.evergreen/scripts/compile-libmongocrypt.sh "${script_dir}/compile-libmongocrypt.sh" "${cmake_binary}" "$(to_windows_path "${mongoc_dir}")" "${install_dir}" >/dev/null echo "Installing libmongocrypt... done." - - # Fail if the C driver is unable to find the installed libmongocrypt. - configure_flags_append "-DENABLE_CLIENT_SIDE_ENCRYPTION=ON" fi "${cmake_binary}" -G "$CC" "${configure_flags[@]}" "${extra_configure_flags[@]}"