From ec8ab753a7f055366eac8b9877c8307bc9bbadc4 Mon Sep 17 00:00:00 2001 From: Mikhail Bautin <552936+mbautin@users.noreply.github.com> Date: Thu, 2 Nov 2023 16:40:11 -0700 Subject: [PATCH] Update glog to use stack unwinding based on backtrace function (#242) Also some improvements in handling the `--dev-repo` flag. Issue: https://github.com/yugabyte/yugabyte-db/issues/19085 --- python/build_definitions/glog.py | 2 +- python/yugabyte_db_thirdparty/cmd_line_args.py | 2 +- python/yugabyte_db_thirdparty/compiler_wrapper.py | 14 +++++++++++--- python/yugabyte_db_thirdparty/download_manager.py | 1 + .../yugabyte_db_thirdparty/file_system_layout.py | 7 ++++++- python/yugabyte_db_thirdparty/util.py | 3 ++- thirdparty_src_checksums.txt | 1 + 7 files changed, 23 insertions(+), 7 deletions(-) diff --git a/python/build_definitions/glog.py b/python/build_definitions/glog.py index bc219f93..467d3928 100644 --- a/python/build_definitions/glog.py +++ b/python/build_definitions/glog.py @@ -23,7 +23,7 @@ class GLogDependency(Dependency): def __init__(self) -> None: super(GLogDependency, self).__init__( name='glog', - version='0.4.0-yb-5', + version='0.4.0-yb-8', url_pattern='https://github.com/yugabyte/glog/archive/v{0}.tar.gz', build_group=BuildGroup.POTENTIALLY_INSTRUMENTED) self.patch_version = 1 diff --git a/python/yugabyte_db_thirdparty/cmd_line_args.py b/python/yugabyte_db_thirdparty/cmd_line_args.py index 644b062b..3dde0a2c 100644 --- a/python/yugabyte_db_thirdparty/cmd_line_args.py +++ b/python/yugabyte_db_thirdparty/cmd_line_args.py @@ -258,7 +258,7 @@ def parse_cmd_line_args() -> argparse.Namespace: parser.add_argument( '--dev-repo', help='Specify "development mode" local repository paths for some dependencies. E.g. ' - '--dev-repo- tcmalloc=~/code/tcmalloc. These development repository paths are ' + '--dev-repo tcmalloc=~/code/tcmalloc. These development repository paths are ' 'created automatically if not present.', nargs='+') diff --git a/python/yugabyte_db_thirdparty/compiler_wrapper.py b/python/yugabyte_db_thirdparty/compiler_wrapper.py index 4c94c27c..c81cc30f 100644 --- a/python/yugabyte_db_thirdparty/compiler_wrapper.py +++ b/python/yugabyte_db_thirdparty/compiler_wrapper.py @@ -133,9 +133,14 @@ def run(self) -> None: COMPILER_WRAPPER_ENV_VAR_NAME_LD_FLAGS_TO_REMOVE, '').strip().split()) cmd_args = [arg for arg in cmd_args if arg not in ld_flags_to_remove] - if len(output_files) == 1 and output_files[0].endswith('.o'): + if (len(output_files) == 1 and + output_files[0].endswith('.o') and + # Protobuf build produces a file named libprotobuf.15.dylib-master.o out of multiple + # .o files. + not output_files[0].endswith('.dylib-master.o')): + output_path = output_files[0] - pp_output_path = output_path + 'pp' + pp_output_path = output_path + '.pp' # "pp" for "preprocessed" is_assembly_input = any([arg.endswith('.s') for arg in self.compiler_args]) compile_commands_tmp_dir = compile_commands.get_tmp_dir_env_var() @@ -166,7 +171,10 @@ def run(self) -> None: pp_args.append('-E') subprocess.check_call(pp_args) assert pp_output_path is not None - assert os.path.isfile(pp_output_path) + assert os.path.isfile(pp_output_path), ( + f"Preprocessed output file does not exist: {pp_output_path}. " + f"Preprocessing command arguments: {shlex_join(pp_args)}." + ) # Collect included files from preprocessor output. # https://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html diff --git a/python/yugabyte_db_thirdparty/download_manager.py b/python/yugabyte_db_thirdparty/download_manager.py index 9f5a3944..e017af98 100644 --- a/python/yugabyte_db_thirdparty/download_manager.py +++ b/python/yugabyte_db_thirdparty/download_manager.py @@ -104,6 +104,7 @@ def dest_dir_already_exists(full_out_path: str) -> bool: if os.path.exists(tmp_out_dir): raise IOError("Just-generated unique directory name already exists: %s" % tmp_out_dir) os.makedirs(tmp_out_dir) + assert os.path.isdir(tmp_out_dir), f"Failed to create directory {tmp_out_dir}" archive_extension = None for ext in ARCHIVE_TYPES: diff --git a/python/yugabyte_db_thirdparty/file_system_layout.py b/python/yugabyte_db_thirdparty/file_system_layout.py index 40393b09..e6c1c1ca 100644 --- a/python/yugabyte_db_thirdparty/file_system_layout.py +++ b/python/yugabyte_db_thirdparty/file_system_layout.py @@ -161,7 +161,12 @@ def get_llvm_tool_dir(self) -> str: return os.path.join(self.tp_build_dir, 'llvm-tools') def add_dev_repo_mapping(self, mapping_str: str) -> None: - dep_name, repo_dir = mapping_str.split('=') + components = mapping_str.split('=', 1) + if len(components) != 2: + raise ValueError( + f"Expected a dev repo mapping to be of the form name=directory, got: {mapping_str}") + dep_name, repo_dir = components + repo_dir = os.path.expanduser(repo_dir) if dep_name in self.dev_repo_mappings: raise ValueError( f"Duplicate development repository directory mapping for dependency {dep_name}: " diff --git a/python/yugabyte_db_thirdparty/util.py b/python/yugabyte_db_thirdparty/util.py index 255eb10d..b25b30f6 100644 --- a/python/yugabyte_db_thirdparty/util.py +++ b/python/yugabyte_db_thirdparty/util.py @@ -366,8 +366,9 @@ def capture_all_output( except subprocess.CalledProcessError as ex: cmd_line_str = shlex_join(args) if ex.returncode not in allowed_exit_codes: + expected_exit_codes = set(sorted(allowed_exit_codes | {0})) error_msg = f"Unexpected exit code {ex.returncode} from: {cmd_line_str} " \ - f"(expected one of { set(sorted(allowed_exit_codes | {0})) })" + f"(expected one of {expected_exit_codes})" log(error_msg) log("Output from %s (stdout/stderr combined):", cmd_line_str) log(ex.stdout.decode('utf-8')) diff --git a/thirdparty_src_checksums.txt b/thirdparty_src_checksums.txt index af8f253d..95481329 100644 --- a/thirdparty_src_checksums.txt +++ b/thirdparty_src_checksums.txt @@ -38,6 +38,7 @@ fd167e7c5bd33fa6d15dcd7e7ddcf08cdf1428d4123d60041951cb4210d97990 glog-0.4.0-yb- dfdaf6d5ea276cbb20fabace03821d4a0c08ee143ff3addd34a86f32833fa860 glog-0.4.0-yb-3.tar.gz 4e04b9182904bea9e1f106fc13541aba63d06440d5651b4b712a02398d5cda3a glog-0.4.0-yb-4.tar.gz 8016f57ccffc198a827489c697d3ae4b3dc828359500b4d3f8fcf269d221beac glog-0.4.0-yb-5.tar.gz +ff13d31eca6d5aaf83b13d18730e8d59fdec5d8a6369e533fb34ab13eb0b1968 glog-0.4.0-yb-8.tar.gz 80439c8a92af974e6788347c59d71d59fdf51f808d80d52fabe744eacf4f9cba glog-0.4.0-yb1.tar.gz f28359aeba12f30d73d9e4711ef356dc842886968112162bc73002645139c39c glog-0.4.0.tar.gz 407992e9ef17a08339cd383c33dbaff923969cfa01f8e4ceaeea679400016d85 gmock-1.7.0.zip