libdrgn: orc_info: prefer ORC stack switching entries over DWARF CFI

We have a longstanding issue where stack unwinding will stop at an
interrupt handler frame on x86-64. This is because since Linux 5.12,
interrupt handlers switch to the IRQ stack using inline assembly without
providing CFI directives. Luckily, ORC can unwind this correctly, and we
can recognize the ORC entries that encode this stack switch and prefer
to use them over DWARF CFI.

To do so, we need to identify these entries (there seem to be < 100),
store their address ranges in a lookup table, and check that table to
decide whether to try ORC or DWARF CFI first. We can use the same table
to handle the DRGN_PREFER_ORC_UNWINDER environment variable setting.

Fixes #304.

Signed-off-by: Omar Sandoval <[email protected]>

libdrgn/debug_info.c

@@ -2244,49 +2244,59 @@ drgn_module_find_cfi(struct drgn_program *prog, struct drgn_module *module,
 		 drgn_platforms_equal(&module->debug_file->platform,
 				      &prog->platform));
 
-	if (prog->prefer_orc_unwinder) {
-		if (can_use_debug_file) {
-			*file_ret = module->debug_file;
-			err = drgn_module_find_orc_cfi(module, pc, row_ret,
-						       interrupted_ret,
-						       ret_addr_regno_ret);
-			if (err != &drgn_not_found)
+	bool prefer_orc = false;
+	if (can_use_debug_file) {
+		if (!module->parsed_debug_frame) {
+			err = drgn_module_parse_debug_frame(module);
+			if (err)
 				return err;
-			err = drgn_module_find_dwarf_cfi(module, pc, row_ret,
-							 interrupted_ret,
-							 ret_addr_regno_ret);
-			if (err != &drgn_not_found)
+			module->parsed_debug_frame = true;
+		}
+		if (!module->parsed_orc) {
+			err = drgn_module_parse_orc(module);
+			if (err)
 				return err;
+			module->parsed_orc = true;
 		}
-		if (can_use_loaded_file) {
-			*file_ret = module->loaded_file;
-			return drgn_module_find_eh_cfi(module, pc, row_ret,
+
+		prefer_orc = drgn_module_should_prefer_orc_cfi(module, pc);
+
+		*file_ret = module->debug_file;
+		if (prefer_orc) {
+			err = drgn_module_find_orc_cfi(module, pc, row_ret,
 						       interrupted_ret,
 						       ret_addr_regno_ret);
-		}
-	} else {
-		if (can_use_debug_file) {
-			*file_ret = module->debug_file;
-			err = drgn_module_find_dwarf_cfi(module, pc, row_ret,
-							 interrupted_ret,
-							 ret_addr_regno_ret);
 			if (err != &drgn_not_found)
 				return err;
 		}
-		if (can_use_loaded_file) {
-			*file_ret = module->loaded_file;
-			err = drgn_module_find_eh_cfi(module, pc, row_ret,
-						      interrupted_ret,
-						      ret_addr_regno_ret);
-			if (err != &drgn_not_found)
+		err = drgn_module_find_dwarf_cfi(module, pc, row_ret,
+						 interrupted_ret,
+						 ret_addr_regno_ret);
+		if (err != &drgn_not_found)
+			return err;
+	}
+
+	if (can_use_loaded_file) {
+		if (!module->parsed_eh_frame) {
+			err = drgn_module_parse_eh_frame(module);
+			if (err)
 				return err;
+			module->parsed_eh_frame = true;
 		}
-		if (can_use_debug_file) {
-			*file_ret = module->debug_file;
-			return drgn_module_find_orc_cfi(module, pc, row_ret,
-							interrupted_ret,
-							ret_addr_regno_ret);
-		}
+		*file_ret = module->loaded_file;
+		err = drgn_module_find_eh_cfi(module, pc, row_ret,
+					      interrupted_ret,
+					      ret_addr_regno_ret);
+		if (err != &drgn_not_found)
+			return err;
+	}
+
+	if (can_use_debug_file && !prefer_orc) {
+		err = drgn_module_find_orc_cfi(module, pc, row_ret,
+					       interrupted_ret,
+					       ret_addr_regno_ret);
+		if (err != &drgn_not_found)
+			return err;
 	}
 	return &drgn_not_found;
 }

libdrgn/dwarf_info.c

@@ -7199,21 +7199,13 @@ drgn_find_cfi_row_in_dwarf_fde(struct drgn_dwarf_cfi *cfi,
 }
 
 static struct drgn_error *
-drgn_find_dwarf_cfi(struct drgn_dwarf_cfi *cfi, bool *parsed,
-		    struct drgn_elf_file *file, enum drgn_section_index scn,
-		    uint64_t unbiased_pc, struct drgn_cfi_row **row_ret,
-		    bool *interrupted_ret,
+drgn_find_dwarf_cfi(struct drgn_dwarf_cfi *cfi, struct drgn_elf_file *file,
+		    enum drgn_section_index scn, uint64_t unbiased_pc,
+		    struct drgn_cfi_row **row_ret, bool *interrupted_ret,
 		    drgn_register_number *ret_addr_regno_ret)
 {
 	struct drgn_error *err;
 
-	if (!*parsed) {
-		err = drgn_parse_dwarf_cfi(cfi, file, scn);
-		if (err)
-			return err;
-		*parsed = true;
-	}
-
 	struct drgn_dwarf_fde *fde = drgn_find_dwarf_fde(cfi, unbiased_pc);
 	if (!fde)
 		return &drgn_not_found;
@@ -7226,26 +7218,36 @@ drgn_find_dwarf_cfi(struct drgn_dwarf_cfi *cfi, bool *parsed,
 	return NULL;
 }
 
+struct drgn_error *drgn_module_parse_debug_frame(struct drgn_module *module)
+{
+	return drgn_parse_dwarf_cfi(&module->dwarf.debug_frame,
+				    module->debug_file, DRGN_SCN_DEBUG_FRAME);
+}
+
 struct drgn_error *
 drgn_module_find_dwarf_cfi(struct drgn_module *module, uint64_t pc,
 			   struct drgn_cfi_row **row_ret, bool *interrupted_ret,
 			   drgn_register_number *ret_addr_regno_ret)
 {
 	return drgn_find_dwarf_cfi(&module->dwarf.debug_frame,
-				   &module->parsed_debug_frame,
 				   module->debug_file, DRGN_SCN_DEBUG_FRAME,
 				   pc - module->debug_file_bias, row_ret,
 				   interrupted_ret, ret_addr_regno_ret);
 }
 
+struct drgn_error *drgn_module_parse_eh_frame(struct drgn_module *module)
+{
+	return drgn_parse_dwarf_cfi(&module->dwarf.eh_frame,
+				    module->loaded_file, DRGN_SCN_EH_FRAME);
+}
+
 struct drgn_error *
 drgn_module_find_eh_cfi(struct drgn_module *module, uint64_t pc,
 			struct drgn_cfi_row **row_ret, bool *interrupted_ret,
 			drgn_register_number *ret_addr_regno_ret)
 {
-	return drgn_find_dwarf_cfi(&module->dwarf.eh_frame,
-				   &module->parsed_eh_frame,
-				   module->loaded_file, DRGN_SCN_EH_FRAME,
+	return drgn_find_dwarf_cfi(&module->dwarf.eh_frame, module->loaded_file,
+				   DRGN_SCN_EH_FRAME,
 				   pc - module->loaded_file_bias, row_ret,
 				   interrupted_ret, ret_addr_regno_ret);
 }

libdrgn/dwarf_info.h

@@ -337,11 +337,15 @@ drgn_object_from_dwarf(struct drgn_debug_info *dbinfo,
 		       const struct drgn_register_state *regs,
 		       struct drgn_object *ret);
 
+struct drgn_error *drgn_module_parse_debug_frame(struct drgn_module *module);
+
 struct drgn_error *
 drgn_module_find_dwarf_cfi(struct drgn_module *module, uint64_t pc,
 			   struct drgn_cfi_row **row_ret, bool *interrupted_ret,
 			   drgn_register_number *ret_addr_regno_ret);
 
+struct drgn_error *drgn_module_parse_eh_frame(struct drgn_module *module);
+
 struct drgn_error *
 drgn_module_find_eh_cfi(struct drgn_module *module, uint64_t pc,
 			struct drgn_cfi_row **row_ret, bool *interrupted_ret,

libdrgn/orc_info.c

@@ -18,10 +18,13 @@
 #include "program.h"
 #include "util.h"
 
+DEFINE_VECTOR(uint64_range_vector, struct uint64_range);
+
 void drgn_module_orc_info_deinit(struct drgn_module *module)
 {
 	free(module->orc.entries);
 	free(module->orc.pc_offsets);
+	free(module->orc.preferred);
 }
 
 // Getters for "raw" ORC information, i.e., before it is aligned, byte swapped,
@@ -55,6 +58,24 @@ drgn_raw_orc_entry_is_terminator(struct drgn_module *module, unsigned int i)
 	}
 }
 
+static bool
+drgn_raw_orc_entry_is_preferred(struct drgn_module *module, unsigned int i)
+{
+	uint16_t flags;
+	memcpy(&flags, &module->orc.entries[i].flags, sizeof(flags));
+	if (drgn_elf_file_bswap(module->debug_file))
+		flags = bswap_16(flags);
+	// ORC_REG_SP_INDIRECT is used for the stack switching pattern used in
+	// the Linux kernel's call_on_stack()/call_on_irqstack() macros. See
+	// Linux kernel commits 87ccc826bf1c ("x86/unwind/orc: Change
+	// REG_SP_INDIRECT"), aafeb14e9da2 ("objtool: Support stack-swizzle"),
+	// and a0cfc74d0b00 ("x86/irq: Provide macro for inlining irq stack
+	// switching") (in v5.12). These macros switch the stack pointer in
+	// inline assembly, resulting in inaccurate DWARF CFI. So, we should use
+	// ORC to unwind these instead.
+	return (flags & 0xf) == DRGN_ORC_REG_SP_INDIRECT;
+}
+
 static int compare_orc_entries(const void *a, const void *b, void *arg)
 {
 	struct drgn_module *module = arg;
@@ -104,17 +125,30 @@ static unsigned int keep_orc_entry(struct drgn_module *module,
  * Note that we don't bother checking EH CFI because currently ORC is only used
  * for the Linux kernel on x86-64, which explicitly disables EH data.
  */
-static unsigned int remove_fdes_from_orc(struct drgn_module *module,
-					 unsigned int *indices,
-					 unsigned int num_entries)
+static struct drgn_error *
+remove_fdes_from_orc(struct drgn_module *module, unsigned int *indices,
+		     struct uint64_range_vector *preferred,
+		     unsigned int *num_entriesp)
 {
+	char *env = getenv("DRGN_PREFER_ORC_UNWINDER");
+	if (env && atoi(env)) {
+		struct uint64_range *range =
+			uint64_range_vector_append_entry(preferred);
+		if (!range)
+			return &drgn_enomem;
+		range->start = 0;
+		range->end = UINT64_MAX;
+		return NULL;
+	}
+
 	if (module->dwarf.debug_frame.num_fdes == 0)
-		return num_entries;
+		return NULL;
 
 	struct drgn_dwarf_fde *fde = module->dwarf.debug_frame.fdes;
 	struct drgn_dwarf_fde *last_fde =
 		fde + module->dwarf.debug_frame.num_fdes - 1;
 
+	unsigned int num_entries = *num_entriesp;
 	unsigned int new_num_entries = 0;
 
 	uint64_t start_pc = drgn_raw_orc_pc(module, 0);
@@ -125,6 +159,18 @@ static unsigned int remove_fdes_from_orc(struct drgn_module *module,
 		else
 			end_pc = UINT64_MAX;
 
+		if (drgn_raw_orc_entry_is_preferred(module, i)) {
+			struct uint64_range *range =
+				uint64_range_vector_append_entry(preferred);
+			if (!range)
+				return &drgn_enomem;
+			range->start = start_pc;
+			range->end = end_pc;
+			new_num_entries = keep_orc_entry(module, indices,
+							 new_num_entries, i);
+			continue;
+		}
+
 		if (start_pc < fde->initial_location) {
 			// The current ORC entry starts before the current FDE
 			// (which can only happen if it is the first FDE). Keep
@@ -164,7 +210,8 @@ static unsigned int remove_fdes_from_orc(struct drgn_module *module,
 			fde++;
 		}
 	}
-	return new_num_entries;
+	*num_entriesp = new_num_entries;
+	return NULL;
 }
 
 static int orc_version_from_header(Elf_Data *orc_header)
@@ -313,7 +360,7 @@ static inline void drgn_module_clear_orc(struct drgn_module **modulep)
 	}
 }
 
-static struct drgn_error *drgn_debug_info_parse_orc(struct drgn_module *module)
+struct drgn_error *drgn_module_parse_orc(struct drgn_module *module)
 {
 	struct drgn_error *err;
 
@@ -351,7 +398,12 @@ static struct drgn_error *drgn_debug_info_parse_orc(struct drgn_module *module)
 		}
 	}
 
-	num_entries = remove_fdes_from_orc(module, indices, num_entries);
+	_cleanup_(uint64_range_vector_deinit)
+		struct uint64_range_vector preferred = VECTOR_INIT;
+
+	err = remove_fdes_from_orc(module, indices, &preferred, &num_entries);
+	if (err)
+		return err;
 
 	_cleanup_free_ int32_t *pc_offsets =
 		malloc_array(num_entries, sizeof(pc_offsets[0]));
@@ -414,13 +466,27 @@ static struct drgn_error *drgn_debug_info_parse_orc(struct drgn_module *module)
 		pc_offsets[i] = UINT64_C(4) * index + offset - UINT64_C(4) * i;
 	}
 
+	uint64_range_vector_shrink_to_fit(&preferred);
+	uint64_range_vector_steal(&preferred, &module->orc.preferred,
+				  &module->orc.num_preferred);
 	module->orc.pc_offsets = no_cleanup_ptr(pc_offsets);
 	module->orc.entries = no_cleanup_ptr(entries);
 	module->orc.num_entries = num_entries;
 	clear = NULL;
 	return NULL;
 }
 
+bool drgn_module_should_prefer_orc_cfi(struct drgn_module *module, uint64_t pc)
+{
+	uint64_t unbiased_pc = pc - module->debug_file_bias;
+	#define less_than_uint64_range_start(a, b) (*(a) < (b)->start)
+	size_t i = binary_search_gt(module->orc.preferred,
+				    module->orc.num_preferred, &unbiased_pc,
+				    less_than_uint64_range_start);
+	#undef less_than_uint64_range_start
+	return i > 0 && module->orc.preferred[i - 1].end > unbiased_pc;
+}
+
 static inline uint64_t drgn_orc_pc(struct drgn_module *module, unsigned int i)
 {
 	return module->orc.pc_base + UINT64_C(4) * i + module->orc.pc_offsets[i];
@@ -431,15 +497,6 @@ drgn_module_find_orc_cfi(struct drgn_module *module, uint64_t pc,
 			 struct drgn_cfi_row **row_ret, bool *interrupted_ret,
 			 drgn_register_number *ret_addr_regno_ret)
 {
-	struct drgn_error *err;
-
-	if (!module->parsed_orc) {
-		err = drgn_debug_info_parse_orc(module);
-		if (err)
-			return err;
-		module->parsed_orc = true;
-	}
-
 	uint64_t unbiased_pc = pc - module->debug_file_bias;
 	#define less_than_orc_pc(a, b)	\
 		(*(a) < drgn_orc_pc(module, (b) - module->orc.pc_offsets))

libdrgn/orc_info.h

@@ -27,6 +27,16 @@ struct drgn_module;
 
 /** ORC unwinder data for a @ref drgn_module. */
 struct drgn_module_orc_info {
+	/**
+	 * Ranges where unwinding with ORC should be preferred over DWARF CFI,
+	 * sorted by start address.
+	 *
+	 * ORC may be preferred if configured by the user or for special ORC
+	 * entries; see drgn_raw_orc_entry_is_preferred().
+	 */
+	struct uint64_range *preferred;
+	/** Number of ranges in @ref preferred. */
+	size_t num_preferred;
 	/**
 	 * Base for calculating program counter corresponding to an ORC unwinder
 	 * entry.
@@ -66,6 +76,10 @@ struct drgn_module_orc_info {
 
 void drgn_module_orc_info_deinit(struct drgn_module *module);
 
+struct drgn_error *drgn_module_parse_orc(struct drgn_module *module);
+
+bool drgn_module_should_prefer_orc_cfi(struct drgn_module *module, uint64_t pc);
+
 struct drgn_error *
 drgn_module_find_orc_cfi(struct drgn_module *module, uint64_t pc,
 			 struct drgn_cfi_row **row_ret, bool *interrupted_ret,

libdrgn/program.c

@@ -105,8 +105,6 @@ void drgn_program_init(struct drgn_program *prog,
 	if (platform)
 		drgn_program_set_platform(prog, platform);
 	drgn_thread_set_init(&prog->thread_set);
-	char *env = getenv("DRGN_PREFER_ORC_UNWINDER");
-	prog->prefer_orc_unwinder = env && atoi(env);
 	drgn_program_set_log_level(prog, DRGN_LOG_NONE);
 	drgn_program_set_log_file(prog, stderr);
 	drgn_object_init(&prog->vmemmap, prog);

libdrgn/program.h

@@ -139,7 +139,6 @@ struct drgn_program {
 	 */
 	uint64_t aarch64_insn_pac_mask;
 	bool core_dump_threads_cached;
-	bool prefer_orc_unwinder;
 
 	union {
 		/*

libdrgn/util.h

@@ -222,4 +222,8 @@ static inline uint64_t uint_max(int n)
 void qsort_arg(void *base, size_t nmemb, size_t size,
 	       int (*compar)(const void *, const void *, void*), void *arg);
 
+struct uint64_range {
+	uint64_t start, end;
+};
+
 #endif /* DRGN_UTIL_H */