From a502c1f69fb65722d7215a3ab707d8489aedbf65 Mon Sep 17 00:00:00 2001 From: Luke Craig Date: Fri, 27 Oct 2023 18:26:41 -0400 Subject: [PATCH 01/31] Docker container improvements: slimmer container (#1377) * mean lean docker machine --- .dockerignore | 1 + Dockerfile | 24 +++++++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.dockerignore b/.dockerignore index 94e999f9d8a..aa1f2ae026d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,4 @@ Dockerfile .*sw* .dockerignore .github +.git/FETCH_HEAD \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index d99e3a41e2d..86f243d09a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,4 @@ ARG BASE_IMAGE="ubuntu:20.04" -# Note PANDA supports ubuntu:22.04, but docker versions <= 20.10.7 can't run 22.04 containers - ARG TARGET_LIST="x86_64-softmmu,i386-softmmu,arm-softmmu,aarch64-softmmu,ppc-softmmu,mips-softmmu,mipsel-softmmu,mips64-softmmu" ### BASE IMAGE @@ -96,13 +94,29 @@ RUN python3 -m pip install --ignore-install pycparser && python3 -m pip install RUN ls -alt $(pip show pandare | grep Location: | awk '{print $2}')/pandare/autogen/ RUN bash -c "ls $(pip show pandare | grep Location: | awk '{print $2}')/pandare/autogen/panda_{aarch64_64,arm_32,mips64_64,mips_32,mipsel_32,ppc_32,ppc_64,x86_64_64,i386_32}.py" +# this layer is used to strip shared objects and change python data to be +# symlinks to the installed panda data directory +FROM installer as cleanup +RUN find /usr/local/lib/panda -name "*.so" -exec strip {} \; +RUN PKG=`pip show pandare | grep Location: | awk '{print $2}'`/pandare/data; \ + rm -rf $PKG/pc-bios && ln -s /usr/local/share/panda $PKG/pc-bios; \ + for arch in `find $PKG -name "*-softmmu" -type d -exec basename {} \;` ; do \ + ARCHP=$PKG/$arch; \ + SARCH=`echo $arch | cut -d'-' -f 1`; \ + rm $ARCHP/libpanda-$SARCH.so $ARCHP/llvm-helpers-$SARCH.bc; \ + ln -s /usr/local/share/panda/llvm-helpers-$SARCH.bc $ARCHP/llvm-helpers-$SARCH.bc1; \ + ln -s /usr/local/bin/libpanda-$SARCH.so $ARCHP/libpanda-$SARCH.so; \ + rm -rf $ARCHP/panda/plugins; \ + ln -s /usr/local/lib/panda/$SARCH/ $ARCHP/panda/plugins; \ + done + ### Copy files for panda+pypanda from installer - Stage 5 FROM base as panda # Copy panda + libcapstone.so* + libosi libraries -COPY --from=installer /usr/local /usr/local -COPY --from=installer /usr/lib/libcapstone* /usr/lib/ -COPY --from=installer /lib/libosi.so /lib/libiohal.so /lib/liboffset.so /lib/ +COPY --from=cleanup /usr/local /usr/local +COPY --from=cleanup /usr/lib/libcapstone* /usr/lib/ +COPY --from=cleanup /lib/libosi.so /lib/libiohal.so /lib/liboffset.so /lib/ # Workaround issue #901 - ensure LD_LIBRARY_PATH contains the panda plugins directories #ARG TARGET_LIST="x86_64-softmmu,i386-softmmu,arm-softmmu,ppc-softmmu,mips-softmmu,mipsel-softmmu" From f714e0e4ba993e57314543896b0217edfd515067 Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Wed, 27 Sep 2023 11:08:22 -0400 Subject: [PATCH 02/31] PyPANDA: minor bugfix for interact() with unset expect_prompt --- panda/python/core/pandare/panda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/panda/python/core/pandare/panda.py b/panda/python/core/pandare/panda.py index 3a2fbd6f642..63af77cf5f6 100644 --- a/panda/python/core/pandare/panda.py +++ b/panda/python/core/pandare/panda.py @@ -2634,7 +2634,7 @@ def interact(self, confirm_quit=True): interface after this returns ''' print("PANDA: entering interactive mode. Type pandaquit to exit") - prompt = self.expect_prompt.decode("utf8") if self.expect_prompt else "$ " + prompt = self.expect_prompt.decode("utf8") if self.expect_prompt and isinstance(self.expect_prompt, bytes) else "$ " if not prompt.endswith(" "): prompt += " " while True: cmd = input(prompt) # TODO: Strip all control characters - Ctrl-L breaks things From 60c5c79ae3a5ab0ab69b0e0e904e7a0d5f70607c Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Wed, 27 Sep 2023 11:58:48 -0400 Subject: [PATCH 03/31] Add targetcmp plugin --- panda/plugins/config.panda | 1 + panda/plugins/targetcmp/Makefile | 3 + panda/plugins/targetcmp/README.md | 25 +++++ panda/plugins/targetcmp/targetcmp.cpp | 141 ++++++++++++++++++++++++++ 4 files changed, 170 insertions(+) create mode 100644 panda/plugins/targetcmp/Makefile create mode 100644 panda/plugins/targetcmp/README.md create mode 100644 panda/plugins/targetcmp/targetcmp.cpp diff --git a/panda/plugins/config.panda b/panda/plugins/config.panda index c30be02bb25..4d8c4a90ac4 100644 --- a/panda/plugins/config.panda +++ b/panda/plugins/config.panda @@ -47,6 +47,7 @@ snake_hook stringsearch syscalls2 syscalls_logger +targetcmp textprinter trace track_intexc diff --git a/panda/plugins/targetcmp/Makefile b/panda/plugins/targetcmp/Makefile new file mode 100644 index 00000000000..659a4215597 --- /dev/null +++ b/panda/plugins/targetcmp/Makefile @@ -0,0 +1,3 @@ +CFLAGS+= -std=c++17 +$(PLUGIN_TARGET_DIR)/panda_$(PLUGIN_NAME).so: \ + $(PLUGIN_OBJ_DIR)/$(PLUGIN_NAME).o diff --git a/panda/plugins/targetcmp/README.md b/panda/plugins/targetcmp/README.md new file mode 100644 index 00000000000..c4ea5b036e7 --- /dev/null +++ b/panda/plugins/targetcmp/README.md @@ -0,0 +1,25 @@ +Plugin: TargetCmp +=========== + +Summary +------- + +At every function call, check the first two potential arguments to identify if it's a string pointer to a specified value. If so, examine the other pointer. If that's a string, record it's value. + +The goal here is to dynamically identify values that are compared against a known string. + +Arguments +--------- + +`output_dir`: Optional, directory to store results in. File will be created named targetcmp.txt in this directory. Current directory if unset. +`target_str`: String to search for. + +Dependencies +------------ +`callstack_instr` + +APIs and Callbacks +------------------ + +Example +------- diff --git a/panda/plugins/targetcmp/targetcmp.cpp b/panda/plugins/targetcmp/targetcmp.cpp new file mode 100644 index 00000000000..aaf1100fee9 --- /dev/null +++ b/panda/plugins/targetcmp/targetcmp.cpp @@ -0,0 +1,141 @@ +#include "panda/plugin.h" +#include "panda/plugin_plugin.h" +#include +#include +#include +#include +#include +#include +#include + +#include "callstack_instr/callstack_instr.h" +#include "callwitharg/callwitharg.h" // Unnecessary? + +extern "C" { +#include "callwitharg/callwitharg_ext.h" + bool init_plugin(void *); + void uninit_plugin(void *); +} + +size_t target_str_len; +char *target_str; +std::ofstream outfile; + +// We track the last QUEUE_SIZE addresses we've checked to avoid rereading guest pointers +#define QUEUE_SIZE 100 +std::atomic queue_idx(0); +std::atomic queue[QUEUE_SIZE]; +// Now we'll define a function to add to the queue +void add_to_queue(target_ulong addr) { + size_t idx = queue_idx.fetch_add(1); + queue[idx % QUEUE_SIZE] = addr; +} +// And a function to check if an address is in the queue +bool in_queue(target_ulong addr) { + for (size_t i = 0; i < QUEUE_SIZE; i++) { + if (queue[i] == addr) return true; + } + return false; +} + +// C++ set for storing unique string matches +std::set matches; + +void record_match(char *str) { + if (strlen(str) == 0) return; + + for (int i = 0; i < strlen(str); i++) { + if (!isprint(str[i])) { + return; + } + } + + std::string s(str); + if (matches.find(s) == matches.end()) { + //printf("TargetCMP finds %s with length %u\n", s.c_str(), s.length()); + outfile << s << std::endl; + matches.insert(s); + } +} + +void on_match(CPUState* cpu, target_ulong func_addr, target_ulong *args, char* value, uint matching_idx, uint args_read) { + // We expect 2 args, if matching_idx is 0, arg1 is our target pointer, otherwise arg0 + assert(args_read >= 2); + + //printf("Match in arg %d with arg1=" TARGET_FMT_lx " and arg2=" TARGET_FMT_lx "\n", matching_idx, args[0], args[1]); + + target_ulong target_ptr = args[matching_idx == 0 ? 1 : 0]; // If we matched arg0, we want arg1 and vice versa + + // If it's in the queue, we've already checked it - bail + if (in_queue(target_ptr)) { + return; + } + // Otherwise add it to the queue + add_to_queue(target_ptr); + + size_t short_len = strlen(value); + size_t full_len = 4*short_len; + char* other_arg = (char*)malloc(full_len + 1); + + // Try to read the target string from memory + if (panda_virtual_memory_read(cpu, target_ptr, (uint8_t*)other_arg, full_len) == 0) { + other_arg[target_str_len] = '\0'; // Ensure null termination + } else if (panda_virtual_memory_read(cpu, target_ptr, (uint8_t*)other_arg, short_len) == 0) { + // Recovered short string - move null terminator early + other_arg[short_len] = '\0'; // Ensure null termination + } else { + // Failed to read even the short string - bail + free(other_arg); + return; + } + record_match(other_arg); + free(other_arg); +} + +// logfile default is cwd/targetcmp.txt +std::filesystem::path logfile = std::filesystem::current_path() / "targetcmp.txt"; + +bool init_plugin(void *self) { + if (!init_callwitharg_api()) { + printf("[targetcmp] Fatal error: unable to initialize callwitharg - is it loaded?\n"); + return false; + } + + std::unique_ptr args( + panda_get_args("targetcmp"), panda_free_args); + + const char* logfile_arg = panda_parse_string_opt(args.get(), "output_file", + NULL, "Output file to record compared values into"); + if (logfile_arg) logfile = std::string(logfile_arg); + + target_str = strdup(panda_parse_string_req(args.get(), "target_str", "String to match")); + target_str_len = strlen(target_str); + + if (target_str_len <= 0) { + printf("targetcmp error: invalid target_str argument\n"); + return false; + } + + // On every function call, use our callback to check an argument is the target_str, if so store the other arg +#if defined(TARGET_ARM) || defined(TARGET_MIPS) || defined(TARGET_X86_64) + // Create empty file - Just so we see that something's happening + // Open file for writing, delete anything there. + outfile.open(logfile.string(), std::ios_base::out | std::ios_base::trunc); + + // Call callwitharg's add_target_string function + add_target_string(target_str); + + // Register on_call_match with callwitharg's on_call_match_str PPP callback + PPP_REG_CB("callwitharg", on_call_match_str, on_match); + return true; +#endif + printf("ERROR: Unsupported architecture for targetcmp\n"); + return false; +} + +void uninit_plugin(void *self) { + if (outfile.is_open()) { + outfile.close(); + } + free((void*)target_str); +} From 016a776ecf2eb1cb45add5bffb6ec467837f2b27 Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Wed, 27 Sep 2023 16:52:50 -0400 Subject: [PATCH 04/31] Add CallWithArg plugin --- panda/plugins/callwitharg/Makefile | 3 + panda/plugins/callwitharg/README.md | 45 +++++ panda/plugins/callwitharg/callwitharg.cpp | 191 ++++++++++++++++++ panda/plugins/callwitharg/callwitharg.h | 15 ++ panda/plugins/callwitharg/callwitharg_int.h | 3 + .../plugins/callwitharg/callwitharg_int_fns.h | 16 ++ panda/plugins/config.panda | 1 + 7 files changed, 274 insertions(+) create mode 100644 panda/plugins/callwitharg/Makefile create mode 100644 panda/plugins/callwitharg/README.md create mode 100644 panda/plugins/callwitharg/callwitharg.cpp create mode 100644 panda/plugins/callwitharg/callwitharg.h create mode 100644 panda/plugins/callwitharg/callwitharg_int.h create mode 100644 panda/plugins/callwitharg/callwitharg_int_fns.h diff --git a/panda/plugins/callwitharg/Makefile b/panda/plugins/callwitharg/Makefile new file mode 100644 index 00000000000..659a4215597 --- /dev/null +++ b/panda/plugins/callwitharg/Makefile @@ -0,0 +1,3 @@ +CFLAGS+= -std=c++17 +$(PLUGIN_TARGET_DIR)/panda_$(PLUGIN_NAME).so: \ + $(PLUGIN_OBJ_DIR)/$(PLUGIN_NAME).o diff --git a/panda/plugins/callwitharg/README.md b/panda/plugins/callwitharg/README.md new file mode 100644 index 00000000000..c827ddb2d44 --- /dev/null +++ b/panda/plugins/callwitharg/README.md @@ -0,0 +1,45 @@ +Plugin: CallWithArg +=========== + +Summary +------- + +At every function call, check the first N (potential) arguments to see if any are a specified integer value or a pointer to a specified string value. If so, trigger a custom PPP event. + +The goal here is to dynamically identify functions that take a known value as an argument. **Many false positives are to be expected!** But if you test multiple values, you can use this plugin to easily identify a hookable function in a target program to build custom introspection gadgets. + +Arguments +--------- + +`targets`: an `_` seperated list of hex numbers and/or strings to check for (e.g., `0x1234_hello world_0xABCDEF`) +`verbose`: If set to 1, print on every detected call. +`N`: How many arguments to examine. Default 2. Only supports standard linux calling conventions for now. + + +Dependencies +------------ +`callstack_instr` + +APIs and Callbacks +------------------ +API +* `void add_target_string(char* s)`, `void add_target_num(target_ulong x)`: Add a new target +* `bool remove_target_string(char* s)`, `bool remove_target_num(target_ulong x)`: Remove an existing target. Returns true if the argument was previously a target + +Callbacks: +``` +on_call_match_num(CPUState *cpu, target_ulong* args, uint matching_idx, uint args_read); +``` + +``` +on_call_match_str(CPUState *cpu, target_ulong* args, uint matching_idx, char* value, uint args_read); +``` + +Example +------- +``` +$(python3 -m pandare.qcows x86_64) -panda callwitharg:targets=root_hello_0x41414141,verbose=1 +root@guest# echo hello +root@guest# echo AAAA +root@guest# whoami +``` \ No newline at end of file diff --git a/panda/plugins/callwitharg/callwitharg.cpp b/panda/plugins/callwitharg/callwitharg.cpp new file mode 100644 index 00000000000..aa895df39f0 --- /dev/null +++ b/panda/plugins/callwitharg/callwitharg.cpp @@ -0,0 +1,191 @@ +#include "panda/plugin.h" +#include "panda/plugin_plugin.h" +#include +#include +#include + +#include "callwitharg.h" +#include "callstack_instr/callstack_instr.h" + +extern "C" { +#include "callwitharg_int_fns.h" + bool init_plugin(void *); + void uninit_plugin(void *); + PPP_PROT_REG_CB(on_call_match_str); + PPP_PROT_REG_CB(on_call_match_num); +} + +PPP_CB_BOILERPLATE(on_call_match_str); +PPP_CB_BOILERPLATE(on_call_match_num); + +uint N; +bool verbose; +void on_call_with_args(CPUState *cpu, target_ulong func_pc); + +// Sets of the targets we're looking for +std::unordered_set string_targets; +std::unordered_set int_targets; +void add_target_string(char* target) { + if (verbose) { + printf("Adding string target %s\n", target); + } + std::string target_s = std::string(target); + string_targets.insert(target_s); +} + +bool remove_target_string(char* target) { + if (verbose) { + printf("Removing string target %s\n", target); + } + std::string target_s = std::string(target); + return string_targets.erase(target_s) > 0; +} + +void add_target_num(target_ulong target) { + if (verbose) { + printf("Adding number target " TARGET_FMT_lx "\n", target); + } + int_targets.insert(target); +} + +bool remove_target_num(target_ulong target) { + if (verbose) { + printf("Removing number target " TARGET_FMT_lx "\n", target); + } + return int_targets.erase(target) > 0; +} + +bool init_plugin(void *self) { + std::unique_ptr args( + panda_get_args("callwitharg"), panda_free_args); + + const char * target_str_const = panda_parse_string_opt(args.get(), "targets", "", "Hex values and strings to search for in arguments. Seperated by _s"); + + if (strlen(target_str_const) > 0) { + char *target_str = strdup(target_str_const); // Make a mutable copy + + // First split string on _'s, then iterate over each target and + // add to the appropriate set + char* target = strtok(target_str, "_"); + while (target != NULL) { + // If target is a hex value, add it to the int_targets set + if (target[0] == '0' && target[1] == 'x') { + target_ulong target_num = strtoul(target, NULL, 16); + add_target_num(target_num); + } else { + // Otherwise, add it to the string_targets set + add_target_string(target); + } + target = strtok(NULL, "_"); + } + } + + verbose = panda_parse_bool_opt(args.get(), "verbose", "enable verbose output"); + N = (uint)panda_parse_uint32_opt(args.get(), "N", 2, "Maximum number of arguments to examine on each call"); + +#if defined(TARGET_ARM) || defined(TARGET_MIPS) || defined(TARGET_X86_64) + PPP_REG_CB("callstack_instr", on_call, on_call_with_args); + return true; +#else + printf("ERROR: Unsupported architecture for targetcmp\n"); + return false; +#endif +} + +typedef struct { + target_ulong *args; + size_t count; +} Arguments; + +bool _get_args_for_arch(CPUArchState *env, Arguments *args, int N) { +#ifdef TARGET_ARM + for (int i = 0; i < N; ++i) { + args->args[i] = env->regs[i]; + } +#elif defined(TARGET_MIPS) + for (int i = 0; i < N; ++i) { + args->args[i] = env->active_tc.gpr[4 + i]; + } +#elif defined(TARGET_X86_64) + // Handle SysV ABI here, or make it a parameter to the function + const int regs[] = {7, 6}; // RDI, RSI + for (int i = 0; i < N; ++i) { + args->args[i] = env->regs[regs[i]]; + } +#else + return false; // Error +#endif + + return true; // All good +} + +bool get_n_args(CPUState *cpu, Arguments *args, uint n) { + CPUArchState *UNUSED(env) = (CPUArchState *)cpu->env_ptr; + args->args = (target_ulong*)malloc(n * sizeof(target_ulong)); + args->count = N; + return _get_args_for_arch(env, args, N); +} + + +// Called on every guest function call +void on_call_with_args(CPUState *cpu, target_ulong func_pc) { + + // What kind of target do we have? + bool have_strings = string_targets.size() > 0; + bool have_nums = int_targets.size() > 0; + + size_t max_len = 0; + + if (!have_strings && !have_nums) return; + + if (have_strings) { + for (auto it = string_targets.begin(); it != string_targets.end(); ++it) { + if (it->length() > max_len) { + max_len = it->length(); + } + } + } + + Arguments args; + if (!get_n_args(cpu, &args, N)) { + // Error! + printf("Failed to get %d args\n", N); + return; + } + + // Check the up to N arguments for matches + for (uint i=0; i < N; i++) { + // Is argument a string or a number? + if (have_nums) { + if (int_targets.find(args.args[i]) != int_targets.end()) { + if (verbose) { + printf("Found target " TARGET_FMT_lx " in call at " TARGET_FMT_lx ". In argument %d\n", args.args[i], func_pc, i); + } + PPP_RUN_CB(on_call_match_num, cpu, func_pc, args.args, i, N); + } + } + + if (have_strings) { + // Read string from guest memory + char str[max_len+1]; + if (panda_virtual_memory_read(cpu, args.args[i], (uint8_t*)str, max_len) != 0) { + // Read failed + continue; + } + str[max_len] = '\0'; + + // Check if string is in set of targets + std::string str_s = std::string(str); + if (string_targets.find(str_s) != string_targets.end()) { + if (verbose) { + printf("Found string target %s in call at " TARGET_FMT_lx ". In argument %d\n", str, func_pc, i); + } + PPP_RUN_CB(on_call_match_str, cpu, func_pc, args.args, str, i, N); + } + } + } + free(args.args); +} + +void uninit_plugin(void *self) { +} diff --git a/panda/plugins/callwitharg/callwitharg.h b/panda/plugins/callwitharg/callwitharg.h new file mode 100644 index 00000000000..b629f8772d2 --- /dev/null +++ b/panda/plugins/callwitharg/callwitharg.h @@ -0,0 +1,15 @@ +#ifndef __CALLWITHARG_H +#define __CALLWITHARG_H + +// BEGIN_PYPANDA_NEEDS_THIS -- do not delete this comment bc pypanda +// api autogen needs it. And don't put any compiler directives +// between this and END_PYPANDA_NEEDS_THIS except includes of other +// files in this directory that contain subsections like this one. + +PPP_CB_TYPEDEF(void, on_call_match_num, CPUState *env, target_ulong func_addr, target_ulong *args, uint matching_idx, uint args_read); +PPP_CB_TYPEDEF(void, on_call_match_str, CPUState *env, target_ulong func_addr, target_ulong *args, char* value, uint matching_idx, uint args_read); + +// END_PYPANDA_NEEDS_THIS -- do not delete this comment! + +#endif + diff --git a/panda/plugins/callwitharg/callwitharg_int.h b/panda/plugins/callwitharg/callwitharg_int.h new file mode 100644 index 00000000000..cd70a048c7a --- /dev/null +++ b/panda/plugins/callwitharg/callwitharg_int.h @@ -0,0 +1,3 @@ +typedef void target_ulong; +typedef void bool; +#include "callwitharg_int_fns.h" \ No newline at end of file diff --git a/panda/plugins/callwitharg/callwitharg_int_fns.h b/panda/plugins/callwitharg/callwitharg_int_fns.h new file mode 100644 index 00000000000..e2973885afc --- /dev/null +++ b/panda/plugins/callwitharg/callwitharg_int_fns.h @@ -0,0 +1,16 @@ +#ifndef __CALLWITHARG_INT_FNS_H__ +#define __CALLWITHARG_INT_FNS_H__ + +// BEGIN_PYPANDA_NEEDS_THIS -- do not delete this comment bc pypanda +// api autogen needs it. And don't put any compiler directives +// between this and END_PYPANDA_NEEDS_THIS except includes of other +// files in this directory that contain subsections like this one. + +// Public interface +void add_target_string(char* target); +bool remove_target_string(char* target); +void add_target_num(target_ulong target); +bool remove_target_num(target_ulong target); +// END_PYPANDA_NEEDS_THIS -- do not delete this comment! + +#endif diff --git a/panda/plugins/config.panda b/panda/plugins/config.panda index 4d8c4a90ac4..a1a30d2300f 100644 --- a/panda/plugins/config.panda +++ b/panda/plugins/config.panda @@ -4,6 +4,7 @@ asid_instr_count asidstory callfunc callstack_instr +callwitharg checkpoint collect_code correlatetaps From 87114c4c4ce2c062da9c54de1a5e39c16eb2fb81 Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Fri, 29 Sep 2023 17:06:29 -0400 Subject: [PATCH 05/31] Add findcall plugin --- panda/plugins/config.panda | 1 + panda/plugins/findcall/Makefile | 3 + panda/plugins/findcall/README.md | 23 ++++ panda/plugins/findcall/findcall.cpp | 157 ++++++++++++++++++++++++++++ 4 files changed, 184 insertions(+) create mode 100644 panda/plugins/findcall/Makefile create mode 100644 panda/plugins/findcall/README.md create mode 100644 panda/plugins/findcall/findcall.cpp diff --git a/panda/plugins/config.panda b/panda/plugins/config.panda index a1a30d2300f..e2a3c5227e9 100644 --- a/panda/plugins/config.panda +++ b/panda/plugins/config.panda @@ -14,6 +14,7 @@ coverage dynamic_symbols edge_coverage filereadmon +findcall forcedexec gdb hooks diff --git a/panda/plugins/findcall/Makefile b/panda/plugins/findcall/Makefile new file mode 100644 index 00000000000..659a4215597 --- /dev/null +++ b/panda/plugins/findcall/Makefile @@ -0,0 +1,3 @@ +CFLAGS+= -std=c++17 +$(PLUGIN_TARGET_DIR)/panda_$(PLUGIN_NAME).so: \ + $(PLUGIN_OBJ_DIR)/$(PLUGIN_NAME).o diff --git a/panda/plugins/findcall/README.md b/panda/plugins/findcall/README.md new file mode 100644 index 00000000000..cdef066cba6 --- /dev/null +++ b/panda/plugins/findcall/README.md @@ -0,0 +1,23 @@ +Plugin: FindCall +=========== + +Summary +------- + +At every function call, check if the argument is one or more specified strings. Track both module+offsets and absolute addresses for calls where this occurs. Report the module+offsets and absolute addresses where ALL strings are observed. + +Arguments +--------- + +`output_file`: Optional, path to store results at. Default is `findcall.txt` in the current directory + +Dependencies +------------ +`callwitharg` must be loaded and configured with the target strings you're looking for +`callstack_instr` + +APIs and Callbacks +------------------ + +Example +------- diff --git a/panda/plugins/findcall/findcall.cpp b/panda/plugins/findcall/findcall.cpp new file mode 100644 index 00000000000..69db549a7d8 --- /dev/null +++ b/panda/plugins/findcall/findcall.cpp @@ -0,0 +1,157 @@ +#include "panda/plugin.h" +#include "panda/plugin_plugin.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#include "callstack_instr/callstack_instr.h" +#include "callwitharg/callwitharg.h" +#include "osi/osi_types.h" + +extern "C" { +//#include "callwitharg/callwitharg_int_fns.h" +#include "callwitharg/callwitharg_ext.h" +#include "osi/osi_ext.h" + bool init_plugin(void *); + void uninit_plugin(void *); +} + +//size_t target_str_len; +//char *target_str; +std::ofstream outfile; + +// We want to store a mapping from unique strigs -> set of (module+offset (as a tuple)) +std::map>> matches; + +// Also we store unique strings -> absolute addresses +std::map> matches_abs; + +void on_match(CPUState* cpu, target_ulong func_addr, target_ulong *args, char* value, uint matching_idx, uint args_read) { + assert(args_read >= 1); + + // A match happend! Use OSI to get our current module+offst + OsiProc *current = get_current_process(cpu); + + OsiModule *m = get_mapping_by_addr(cpu, current, func_addr); + if (m) { + // If matches doesn't have a key for this string, add it + if (matches.find(value) == matches.end()) { + matches[value] = std::set>(); + } + + if (matches_abs.find(value) == matches_abs.end()) { + matches_abs[value] = std::set(); + } + + // Now check the matches[value] set for this module+offset tuple - if it's not there, add it + std::tuple t(m->name, func_addr - m->base); + if (matches[value].find(t) == matches[value].end()) { + matches[value].insert(t); + printf("Match of %s at func_addr " TARGET_FMT_lx " which is in module %s + " TARGET_FMT_lx "\n", value, func_addr, m->name, func_addr - m->base); + } + + if (matches_abs[value].find(func_addr) == matches_abs[value].end()) { + matches_abs[value].insert(func_addr); + printf("Match of %s at func_addr " TARGET_FMT_lx "\n", value, func_addr); + } + + // cleanup + free_osimodule(m); + }else { + printf("Match of %s at unknown\n", value); + } +} + +// logfile default is cwd/findcall.txt +std::filesystem::path logfile = std::filesystem::current_path() / "findcall.txt"; + +bool init_plugin(void *self) { + init_callwitharg_api(); + init_osi_api(); + + std::unique_ptr args( + panda_get_args("findcall"), panda_free_args); + + const char* logfile_arg = panda_parse_string_opt(args.get(), "output_file", + NULL, "Output file to record compared values into"); + if (logfile_arg) logfile = std::string(logfile_arg); + + /* + target_str = strdup(panda_parse_string_req(args.get(), "target_str", "String to match")); + target_str_len = strlen(target_str); + + if (target_str_len <= 0) { + printf("targetcmp error: invalid target_str argument\n"); + return false; + } + */ + +#if defined(TARGET_ARM) || defined(TARGET_MIPS) || defined(TARGET_X86_64) + outfile.open(logfile.string(), std::ios_base::out | std::ios_base::trunc); // Empty file to start + + // Tell callwitharg to call on_match when it finds a match of our string + //add_target_string(target_str); + + // Register on_call_match with callwitharg's on_call_match_str PPP callback + PPP_REG_CB("callwitharg", on_call_match_str, on_match); + return true; +#endif + printf("ERROR: Unsupported architecture for targetcmp\n"); + return false; +} + +void uninit_plugin(void *self) { + // Find all unique module+offset values in matches, then check if all keys ever have each value - if so print + std::map, std::set> modoff_to_keys; + for (auto const& [key, val] : matches) { + for (auto const& [mod, off] : val) { + modoff_to_keys[std::make_tuple(mod, off)].insert(key); + } + } + // Now we have a map from module+offset to set of keys that have that module+offset - print all that have all keys + for (auto const& [modoff, keys] : modoff_to_keys) { + if (keys.size() == matches.size()) { + + // Write module, offset to outfile + outfile << std::get<0>(modoff) << "," << std::get<1>(modoff) << std::endl; + + // Print the common module+offset then the two keys + printf("%s + " TARGET_FMT_lx "\n", std::get<0>(modoff).c_str(), std::get<1>(modoff)); + for (auto const& key : keys) { + printf(" %s\n", key.c_str()); + } + } + } + + // Now write down all absolute addresses that are in every key + std::map> abs_to_keys; + for (auto const& [key, val] : matches_abs) { + for (auto const& abs : val) { + abs_to_keys[abs].insert(key); + } + } + // Now we have a map from absolute address to set of keys that have that absolute address - print all that have all keys + for (auto const& [abs, keys] : abs_to_keys) { + if (keys.size() == matches_abs.size()) { + + // Write absolute address to outfile: just raw address + outfile << abs << std::endl; + + // Print the common absolute address then the two keys + printf(TARGET_FMT_lx "\n", abs); + for (auto const& key : keys) { + printf(" %s\n", key.c_str()); + } + } + } + + if (outfile.is_open()) { + outfile.close(); + } + +} From 3ee9f67ab68af5b5b0dab53d36edde83dfca3074 Mon Sep 17 00:00:00 2001 From: Mark Mankins Date: Mon, 9 Oct 2023 07:53:40 -0400 Subject: [PATCH 06/31] Extract logic to dlopen libpanda. Moved the logic to dlopen libpanda into seperate utility functions. This makes it easier to understand what _panda_load_plugin is doing. --- panda/src/callbacks.c | 87 +++++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 36 deletions(-) diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index e8f5d6790e5..2299d9d46e7 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -165,8 +165,51 @@ bool panda_load_plugin(const char *filename, const char *plugin_name) { return _panda_load_plugin(filename, plugin_name, false); } +static void *try_open_libpanda(const char *panda_lib) { + void *libpanda = NULL; + if(panda_lib != NULL) { + libpanda = dlopen(panda_lib, RTLD_LAZY | RTLD_NOLOAD | RTLD_GLOBAL); + if (NULL == libpanda) { + fprintf(stderr, "Failed to load libpanda: %s from %s\n", dlerror(), panda_lib); + } + } + return libpanda; +} + +// When running as a library, load libpanda +static bool load_libpanda(void) { + + const char *panda_lib = g_getenv("PANDA_LIB"); + void *libpanda; + + if(panda_lib != NULL) { + libpanda = try_open_libpanda(panda_lib); + } else { +#ifndef LIBRARY_DIR + assert(0 && "Library dir unset but library mode is enabled - Unsupported architecture?"); + printf("Library dir not set"); +#endif + const char *lib_dir = g_getenv("PANDA_DIR"); + + if (lib_dir != NULL) { + panda_lib = g_strdup_printf("%s%s", lib_dir, LIBRARY_DIR); + } else { + fprintf(stderr, "WARNING: using hacky dlopen code that will be removed soon\n"); + panda_lib = g_strdup_printf("../../../build/%s", LIBRARY_DIR); // XXX This is bad, need a less hardcoded path + } + + libpanda = try_open_libpanda(panda_lib); + g_free((char *)panda_lib); + } + + return libpanda != NULL; +} + + bool _panda_load_plugin(const char *filename, const char *plugin_name, bool library_mode) { + static bool libpanda_loaded = false; + #ifndef CONFIG_LLVM // Taint2 seems to be our most commonly used LLVM plugin and it causes some confusion // when users build PANDA without LLVM and then claim taint2 is "missing" @@ -193,41 +236,13 @@ bool _panda_load_plugin(const char *filename, const char *plugin_name, bool libr panda_plugins_loaded[nb_panda_plugins_loaded] = strdup(filename); nb_panda_plugins_loaded ++; - // Ensure pypanda has been dlopened so its symbols can be used in the plugin we're - // now loading. XXX: This should probably happen earlier and only once - if (library_mode) { - // When running as a library, load libpanda -#ifndef LIBRARY_DIR - assert(0 && "Library dir unset but library mode is enabled - Unsupported architecture?"); - printf("Library dir not set"); -#endif - const char *panda_lib = g_getenv("PANDA_LIB"); - if(panda_lib != NULL) { - void *libpanda = dlopen(panda_lib, RTLD_LAZY | RTLD_NOLOAD | RTLD_GLOBAL); - - if (!libpanda) { - fprintf(stderr, "Failed to load libpanda: %s from %s\n", dlerror(), panda_lib); - return false; - } - } else { - const char *lib_dir = g_getenv("PANDA_DIR"); - char *library_path; - if (lib_dir != NULL) { - library_path = g_strdup_printf("%s%s", lib_dir, LIBRARY_DIR); - }else{ - fprintf(stderr, "WARNING: using hacky dlopen code that will be removed soon\n"); - library_path = g_strdup_printf("../../../build/%s", LIBRARY_DIR); // XXX This is bad, need a less hardcoded path - } - - void *libpanda = dlopen(library_path, RTLD_LAZY | RTLD_NOLOAD | RTLD_GLOBAL); - - if (!libpanda) { - fprintf(stderr, "Failed to load libpanda: %s from %s\n", dlerror(), library_path); - g_free(library_path); - return false; - } - g_free(library_path); + // Ensure libpanda has been dlopened so its symbols can be used in the plugin we're + // now loading. XXX: This should probably happen earlier. + if (library_mode && (!libpanda_loaded)) { + if(!load_libpanda()) { + return false; } + libpanda_loaded = true; } void *plugin = dlopen(filename, RTLD_NOW); @@ -244,8 +259,8 @@ bool _panda_load_plugin(const char *filename, const char *plugin_name, bool libr // Populate basic plugin info *before* calling init_fn. // This allows plugins accessing handles of other plugins before - // initialization completes. E.g. osi does a panda_require("win7x86intro"), - // and then win7x86intro does a PPP_REG_CB("osi", ...) while initializing. + // initialization completes. E.g. osi does a panda_require("wintrospection"), + // and then wintrospection does a PPP_REG_CB("osi", ...) while initializing. panda_plugins[nb_panda_plugins].plugin = plugin; if (plugin_name) { strncpy(panda_plugins[nb_panda_plugins].name, plugin_name, 256); From 76c87b092722e10185ca29ab8235930ff6ca7490 Mon Sep 17 00:00:00 2001 From: Mark Mankins Date: Mon, 9 Oct 2023 09:20:31 -0400 Subject: [PATCH 07/31] Remove panda_load_external_plugin. --- panda/include/panda/plugin.h | 4 --- panda/src/callbacks.c | 48 ------------------------------------ 2 files changed, 52 deletions(-) diff --git a/panda/include/panda/plugin.h b/panda/include/panda/plugin.h index 671533f71a0..171d6b372b5 100644 --- a/panda/include/panda/plugin.h +++ b/panda/include/panda/plugin.h @@ -194,10 +194,6 @@ bool _panda_load_plugin(const char *filename, const char *plugin_name, bool libr */ bool panda_add_arg(const char *plugin_name, const char *plugin_arg); - -// I think this is not used anywhere? -bool panda_load_external_plugin(const char *filename, const char *plugin_name, void *plugin_uuid, void *init_fn_ptr); - /** * panda_get_plugin_by_name() - Returns pointer to the plugin of this name. * @name: The name of the desired plugin. diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index 2299d9d46e7..95239659d1b 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -113,54 +113,6 @@ static char* attempt_normalize_path(char* path){ return new_path; } -bool panda_load_external_plugin(const char *filename, const char *plugin_name, void *plugin_uuid, void *init_fn_ptr) { - // don't load the same plugin twice - char* rfilename = attempt_normalize_path(strdup(filename)); - uint32_t i; - for (i=0; i dlopen(filename, RTLD_NOW); - bool (*init_fn)(void *) = init_fn_ptr; //normally dlsym init_fun - - // Populate basic plugin info *before* calling init_fn. - // This allows plugins accessing handles of other plugins before - // initialization completes. E.g. osi does a panda_require("win7x86intro"), - // and then win7x86intro does a PPP_REG_CB("osi", ...) while initializing. - panda_plugins[nb_panda_plugins].plugin = plugin; - if (plugin_name) { - strncpy(panda_plugins[nb_panda_plugins].name, plugin_name, 256); - } else { - char *pn = g_path_get_basename(rfilename); - *g_strrstr(pn, HOST_DSOSUF) = '\0'; - strncpy(panda_plugins[nb_panda_plugins].name, pn, 256); - g_free(pn); - } - nb_panda_plugins++; - - // Call init_fn and check status. - LOG_INFO(PANDA_MSG_FMT "initializing %s\n", PANDA_CORE_NAME, panda_plugins[nb_panda_plugins-1].name); - panda_help_wanted = false; - panda_args_set_help_wanted(plugin_name); - if (panda_help_wanted) { - printf("Options for plugin %s:\n", plugin_name); - fprintf(stderr, "PLUGIN ARGUMENT REQUIRED DESCRIPTION\n"); - fprintf(stderr, "====== ======== ======== ===========\n"); - } - if(!init_fn(plugin) || panda_plugin_load_failed) { - return false; - } - return true; -} - - bool panda_load_plugin(const char *filename, const char *plugin_name) { return _panda_load_plugin(filename, plugin_name, false); } From c75d0c9af5203fb9b274054d19f55b59c04ee0fb Mon Sep 17 00:00:00 2001 From: Mark Mankins Date: Mon, 9 Oct 2023 09:44:29 -0400 Subject: [PATCH 08/31] Remove panda_plugins_loaded. Use the existing panda_plugins array to determine if a plugin has already been loaded. --- panda/src/callbacks.c | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index 95239659d1b..53bb6beb374 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -61,9 +61,6 @@ bool panda_plugins_to_unload[MAX_PANDA_PLUGINS]; bool panda_plugin_to_unload = false; -int nb_panda_plugins_loaded = 0; -char *panda_plugins_loaded[MAX_PANDA_PLUGINS]; - bool panda_please_flush_tb = false; bool panda_please_break_exec = false; bool panda_update_pc = false; @@ -177,16 +174,12 @@ bool _panda_load_plugin(const char *filename, const char *plugin_name, bool libr // don't load the same plugin twice uint32_t i; - for (i=0; i Date: Mon, 9 Oct 2023 10:03:40 -0400 Subject: [PATCH 09/31] Remove panda_plugins_to_unload array. Added a new boolean field to panda_plugins structure to track when a request to unload a plugin has been made. --- panda/include/panda/plugin.h | 3 +-- panda/src/callbacks.c | 5 ++--- panda/src/cb-support.c | 11 ++++++++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/panda/include/panda/plugin.h b/panda/include/panda/plugin.h index 171d6b372b5..30cf85a402d 100644 --- a/panda/include/panda/plugin.h +++ b/panda/include/panda/plugin.h @@ -69,6 +69,7 @@ void panda_disable_plugin(void *plugin); typedef struct panda_plugin { char name[256]; // Currently basename(filename) void *plugin; // Handle to the plugin (for use with dlsym()) + bool unload; // When true, unload plugin when safe } panda_plugin; @@ -228,8 +229,6 @@ void panda_unload_plugins(void); extern bool panda_update_pc; extern bool panda_use_memcb; extern panda_cb_list *panda_cbs[PANDA_CB_LAST]; -extern bool panda_plugins_to_unload[MAX_PANDA_PLUGINS]; -extern bool panda_plugin_to_unload; extern bool panda_tb_chaining; // this stuff is used by the new qemu cmd-line arg '-os os_name' diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index 53bb6beb374..64bd67c48a0 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -57,8 +57,6 @@ int panda_argc; int nb_panda_plugins = 0; panda_plugin panda_plugins[MAX_PANDA_PLUGINS]; -bool panda_plugins_to_unload[MAX_PANDA_PLUGINS]; - bool panda_plugin_to_unload = false; bool panda_please_flush_tb = false; @@ -207,6 +205,7 @@ bool _panda_load_plugin(const char *filename, const char *plugin_name, bool libr // initialization completes. E.g. osi does a panda_require("wintrospection"), // and then wintrospection does a PPP_REG_CB("osi", ...) while initializing. panda_plugins[nb_panda_plugins].plugin = plugin; + panda_plugins[nb_panda_plugins].unload = false; if (plugin_name) { strncpy(panda_plugins[nb_panda_plugins].name, plugin_name, 256); } else { @@ -397,7 +396,7 @@ void panda_unload_plugin_idx(int plugin_idx) return; } panda_plugin_to_unload = true; - panda_plugins_to_unload[plugin_idx] = true; + panda_plugins[plugin_idx].unload = true; } void panda_unload_plugins(void) diff --git a/panda/src/cb-support.c b/panda/src/cb-support.c index ecf4c1d718f..beb9517ca73 100644 --- a/panda/src/cb-support.c +++ b/panda/src/cb-support.c @@ -196,13 +196,18 @@ MAKE_CALLBACK_NO_ARGS(void, PRE_SHUTDOWN, pre_shutdown); // Non-standard callbacks below here +extern bool panda_plugin_to_unload; +extern int nb_panda_plugins; +extern panda_plugin panda_plugins[MAX_PANDA_PLUGINS]; + void PCB(before_find_fast)(void) { if (panda_plugin_to_unload) { panda_plugin_to_unload = false; - for (int i = 0; i < MAX_PANDA_PLUGINS; i++) { - if (panda_plugins_to_unload[i]) { + for (int i = 0; i < nb_panda_plugins;) { + if (panda_plugins[i].unload) { panda_do_unload_plugin(i); - panda_plugins_to_unload[i] = false; + } else { + i++; } } } From 9b01c097c09fe0e3802c76a0e480ccb7a151cb83 Mon Sep 17 00:00:00 2001 From: Mark Mankins Date: Mon, 9 Oct 2023 10:20:22 -0400 Subject: [PATCH 10/31] Create _panda_require. Combine similar functions panda_require_from_library and panda_require into a new utility. --- panda/src/callbacks.c | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index 64bd67c48a0..78071fbe3e5 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -302,7 +302,7 @@ char *panda_plugin_path(const char *plugin_name) { return resolve_file_from_plugin_directory("panda_%s" HOST_DSOSUF, plugin_name); } -void panda_require_from_library(const char *plugin_name, char **plugin_args, uint32_t num_args) { +static void _panda_require(const char *plugin_name, char **plugin_args, uint32_t num_args, bool library_mode) { // If we're printing help, panda_require will be a no-op. if (panda_help_wanted) return; @@ -313,36 +313,25 @@ void panda_require_from_library(const char *plugin_name, char **plugin_args, uin // translate plugin name into a path to .so char *plugin_path = panda_plugin_path(plugin_name); // May be NULL, would raise assert in in _panda_load_plugin + if (NULL == plugin_path) { + LOG_ERROR(PANDA_MSG_FMT "FAILED to find required plugin %s\n", PANDA_CORE_NAME, plugin_name); + abort(); + } // load plugin same as in vl.c - if (!_panda_load_plugin(plugin_path, plugin_name, true)) { // Load in library mode + if (!_panda_load_plugin(plugin_path, plugin_name, library_mode)) { LOG_ERROR(PANDA_MSG_FMT "FAILED to load required plugin %s from %s\n", PANDA_CORE_NAME, plugin_name, plugin_path); abort(); } g_free(plugin_path); } -void panda_require(const char *plugin_name) { - // If we're printing help, panda_require will be a no-op. - if (panda_help_wanted) return; - - LOG_INFO(PANDA_MSG_FMT "loading required plugin %s\n", PANDA_CORE_NAME, plugin_name); - - // translate plugin name into a path to .so - char *plugin_path = panda_plugin_path(plugin_name); - if (NULL == plugin_path) { - LOG_ERROR(PANDA_MSG_FMT "FAILED to find required plugin %s\n", - PANDA_CORE_NAME, plugin_name); - abort(); - } +void panda_require_from_library(const char *plugin_name, char **plugin_args, uint32_t num_args) { + _panda_require(plugin_name, plugin_args, num_args, true); +} - // load plugin same as in vl.c - if (!panda_load_plugin(plugin_path, plugin_name)) { - LOG_ERROR(PANDA_MSG_FMT "FAILED to load required plugin %s from %s\n", - PANDA_CORE_NAME, plugin_name, plugin_path); - abort(); - } - g_free(plugin_path); +void panda_require(const char *plugin_name) { + _panda_require(plugin_name, NULL, 0, false); } // Internal: remove a plugin from the global array panda_plugins From c656b7bab11c148f151b03f94d15f172c4b0187b Mon Sep 17 00:00:00 2001 From: Mark Mankins Date: Mon, 9 Oct 2023 10:37:54 -0400 Subject: [PATCH 11/31] Make _panda_load_plugin static. Not a public API. --- panda/include/panda/plugin.h | 4 ---- panda/src/callbacks.c | 11 +++++------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/panda/include/panda/plugin.h b/panda/include/panda/plugin.h index 30cf85a402d..4ca36ba9793 100644 --- a/panda/include/panda/plugin.h +++ b/panda/include/panda/plugin.h @@ -182,10 +182,6 @@ void panda_unregister_callbacks(void *plugin); */ bool panda_load_plugin(const char *filename, const char *plugin_name); - -bool _panda_load_plugin(const char *filename, const char *plugin_name, bool library_mode); - - /** * panda_add_arg() - Add an argument to those for a plugin. * @plugin_name: The name of the plugin. diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index 78071fbe3e5..6b04fc667f6 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -108,10 +108,6 @@ static char* attempt_normalize_path(char* path){ return new_path; } -bool panda_load_plugin(const char *filename, const char *plugin_name) { - return _panda_load_plugin(filename, plugin_name, false); -} - static void *try_open_libpanda(const char *panda_lib) { void *libpanda = NULL; if(panda_lib != NULL) { @@ -152,8 +148,7 @@ static bool load_libpanda(void) { return libpanda != NULL; } - -bool _panda_load_plugin(const char *filename, const char *plugin_name, bool library_mode) { +static bool _panda_load_plugin(const char *filename, const char *plugin_name, bool library_mode) { static bool libpanda_loaded = false; @@ -232,6 +227,10 @@ bool _panda_load_plugin(const char *filename, const char *plugin_name, bool libr return true; } +bool panda_load_plugin(const char *filename, const char *plugin_name) { + return _panda_load_plugin(filename, plugin_name, false); +} + extern const char *qemu_file; // Resolve a file in the plugin directory to a path. If the file doesn't From 495a89ee56c220d4a7377596ac760a5c8d3113c2 Mon Sep 17 00:00:00 2001 From: Mark Mankins Date: Mon, 9 Oct 2023 11:20:52 -0400 Subject: [PATCH 12/31] Export symbols to subsequently opened plugins... if magic symbol PANDA_EXPORT_SYMBOLS_plugin_name is present in the plugin being loaded. --- panda/include/panda/plugin.h | 7 ++++--- panda/src/callbacks.c | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/panda/include/panda/plugin.h b/panda/include/panda/plugin.h index 4ca36ba9793..4bf4b54d2ea 100644 --- a/panda/include/panda/plugin.h +++ b/panda/include/panda/plugin.h @@ -67,9 +67,10 @@ void panda_disable_plugin(void *plugin); // Structure to store metadata about a plugin typedef struct panda_plugin { - char name[256]; // Currently basename(filename) - void *plugin; // Handle to the plugin (for use with dlsym()) - bool unload; // When true, unload plugin when safe + char name[256]; // Currently basename(filename) + void *plugin; // Handle to the plugin (for use with dlsym()) + bool unload; // When true, unload plugin when safe + bool exported_symbols; // True if plugin dlopened with RTLD_GLOBAL } panda_plugin; diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index 6b04fc667f6..317ed1dca45 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -188,6 +188,7 @@ static bool _panda_load_plugin(const char *filename, const char *plugin_name, bo fprintf(stderr, "Failed to load %s: %s\n", filename, dlerror()); return false; } + bool (*init_fn)(void *) = dlsym(plugin, "init_plugin"); if(!init_fn) { fprintf(stderr, "Couldn't get symbol %s: %s\n", "init_plugin", dlerror()); @@ -201,6 +202,8 @@ static bool _panda_load_plugin(const char *filename, const char *plugin_name, bo // and then wintrospection does a PPP_REG_CB("osi", ...) while initializing. panda_plugins[nb_panda_plugins].plugin = plugin; panda_plugins[nb_panda_plugins].unload = false; + panda_plugins[nb_panda_plugins].exported_symbols = false; + if (plugin_name) { strncpy(panda_plugins[nb_panda_plugins].name, plugin_name, 256); } else { @@ -209,6 +212,17 @@ static bool _panda_load_plugin(const char *filename, const char *plugin_name, bo strncpy(panda_plugins[nb_panda_plugins].name, pn, 256); g_free(pn); } + + char *export_symbol = g_strdup_printf("PANDA_EXPORT_SYMBOLS_%s", plugin_name); + + if(NULL != dlsym(plugin, export_symbol)) { + LOG_DEBUG(PANDA_MSG_FMT "Exporting symbols for plugin %s\n", PANDA_CORE_NAME, plugin_name); + assert(plugin == dlopen(filename, RTLD_NOW | RTLD_GLOBAL)); + panda_plugins[nb_panda_plugins].exported_symbols = true; + } + + g_free(export_symbol); + nb_panda_plugins++; // Call init_fn and check status. @@ -222,6 +236,10 @@ static bool _panda_load_plugin(const char *filename, const char *plugin_name, bo } if(!init_fn(plugin) || panda_plugin_load_failed) { dlclose(plugin); + if(panda_plugins[nb_panda_plugins].exported_symbols) { + // This plugin was dlopened twice. dlclose it twice to fully unload it. + dlclose(plugin); + } return false; } return true; @@ -346,6 +364,7 @@ static void panda_delete_plugin(int i) void panda_do_unload_plugin(int plugin_idx) { void *plugin = panda_plugins[plugin_idx].plugin; + bool exported_symbols = panda_plugins[plugin_idx].exported_symbols; void (*uninit_fn)(void *) = dlsym(plugin, "uninit_plugin"); if (!uninit_fn) { LOG_ERROR("Couldn't get symbol %s: %s\n", "uninit_plugin", @@ -356,6 +375,10 @@ void panda_do_unload_plugin(int plugin_idx) panda_unregister_callbacks(plugin); panda_delete_plugin(plugin_idx); dlclose(plugin); + if(exported_symbols) { + // This plugin was dlopened twice. dlclose it twice to fully unload it. + dlclose(plugin); + } } void panda_unload_plugin(void *plugin) From fa79a3abf7134b50a3e00bd5bcc39927970bf459 Mon Sep 17 00:00:00 2001 From: Mark Mankins Date: Tue, 10 Oct 2023 18:39:39 -0400 Subject: [PATCH 13/31] Require plugin_name be not null and not empty. Added an empty check for filename too. --- panda/src/callbacks.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index 317ed1dca45..019b2376edd 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -152,6 +152,11 @@ static bool _panda_load_plugin(const char *filename, const char *plugin_name, bo static bool libpanda_loaded = false; + if ((plugin_name == NULL) || (*plugin_name == '\0')) { + LOG_ERROR(PANDA_MSG_FMT "Fatal error: plugin_name is required\n", PANDA_CORE_NAME); + abort(); + } + #ifndef CONFIG_LLVM // Taint2 seems to be our most commonly used LLVM plugin and it causes some confusion // when users build PANDA without LLVM and then claim taint2 is "missing" @@ -160,10 +165,10 @@ static bool _panda_load_plugin(const char *filename, const char *plugin_name, bo } #endif - if (filename == NULL) { + if ((filename == NULL) || (*filename == '\0')) { LOG_ERROR(PANDA_MSG_FMT "Fatal error: could not find path for plugin %s\n", PANDA_CORE_NAME, plugin_name); + abort(); } - assert(filename != NULL); // don't load the same plugin twice uint32_t i; @@ -204,14 +209,7 @@ static bool _panda_load_plugin(const char *filename, const char *plugin_name, bo panda_plugins[nb_panda_plugins].unload = false; panda_plugins[nb_panda_plugins].exported_symbols = false; - if (plugin_name) { - strncpy(panda_plugins[nb_panda_plugins].name, plugin_name, 256); - } else { - char *pn = g_path_get_basename((char *) filename); - *g_strrstr(pn, HOST_DSOSUF) = '\0'; - strncpy(panda_plugins[nb_panda_plugins].name, pn, 256); - g_free(pn); - } + strncpy(panda_plugins[nb_panda_plugins].name, plugin_name, 256); char *export_symbol = g_strdup_printf("PANDA_EXPORT_SYMBOLS_%s", plugin_name); @@ -329,7 +327,7 @@ static void _panda_require(const char *plugin_name, char **plugin_args, uint32_t LOG_INFO(PANDA_MSG_FMT "loading required plugin %s\n", PANDA_CORE_NAME, plugin_name); // translate plugin name into a path to .so - char *plugin_path = panda_plugin_path(plugin_name); // May be NULL, would raise assert in in _panda_load_plugin + char *plugin_path = panda_plugin_path(plugin_name); // May be NULL, would abort in in _panda_load_plugin if (NULL == plugin_path) { LOG_ERROR(PANDA_MSG_FMT "FAILED to find required plugin %s\n", PANDA_CORE_NAME, plugin_name); abort(); From d057dcb71e31f2cfcfff36a43395a88f479c32a5 Mon Sep 17 00:00:00 2001 From: Mark Mankins Date: Tue, 10 Oct 2023 18:51:06 -0400 Subject: [PATCH 14/31] Remove length limit for plugin name. This was probably never going to cause a problem, but the code assumed a plugin was never greater than 256 bytes, but allowed for that case to occur, which would cause things to break if it ever happened. --- panda/include/panda/plugin.h | 2 +- panda/src/callbacks.c | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/panda/include/panda/plugin.h b/panda/include/panda/plugin.h index 4bf4b54d2ea..77f2993b9d6 100644 --- a/panda/include/panda/plugin.h +++ b/panda/include/panda/plugin.h @@ -67,7 +67,7 @@ void panda_disable_plugin(void *plugin); // Structure to store metadata about a plugin typedef struct panda_plugin { - char name[256]; // Currently basename(filename) + char *name; // Plugin name: basename(filename) void *plugin; // Handle to the plugin (for use with dlsym()) bool unload; // When true, unload plugin when safe bool exported_symbols; // True if plugin dlopened with RTLD_GLOBAL diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index 019b2376edd..5789e74bc1e 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -173,7 +173,7 @@ static bool _panda_load_plugin(const char *filename, const char *plugin_name, bo // don't load the same plugin twice uint32_t i; for (i=0; i Date: Wed, 11 Oct 2023 06:56:57 -0400 Subject: [PATCH 15/31] Make utility to dlclose plugin. --- panda/src/callbacks.c | 47 +++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index 5789e74bc1e..8af2c67c589 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -148,6 +148,27 @@ static bool load_libpanda(void) { return libpanda != NULL; } +// Internal: remove a plugin from the global array panda_plugins +static void panda_delete_plugin(int i) +{ + if (i != nb_panda_plugins - 1) { // not the last element + memmove(&panda_plugins[i], &panda_plugins[i + 1], + (nb_panda_plugins - i - 1) * sizeof(panda_plugin)); + } + nb_panda_plugins--; +} + +static void dlclose_plugin(int plugin_idx) { + void *plugin = panda_plugins[plugin_idx].plugin; + bool exported_symbols = panda_plugins[plugin_idx].exported_symbols; + panda_delete_plugin(plugin_idx); + dlclose(plugin); + if(exported_symbols) { + // This plugin was dlopened twice. dlclose it twice to fully unload it. + dlclose(plugin); + } +} + static bool _panda_load_plugin(const char *filename, const char *plugin_name, bool library_mode) { static bool libpanda_loaded = false; @@ -231,14 +252,12 @@ static bool _panda_load_plugin(const char *filename, const char *plugin_name, bo fprintf(stderr, "PLUGIN ARGUMENT REQUIRED DESCRIPTION\n"); fprintf(stderr, "====== ======== ======== ===========\n"); } + if(!init_fn(plugin) || panda_plugin_load_failed) { - dlclose(plugin); - if(panda_plugins[nb_panda_plugins].exported_symbols) { - // This plugin was dlopened twice. dlclose it twice to fully unload it. - dlclose(plugin); - } + dlclose_plugin(nb_panda_plugins - 1); return false; } + return true; } @@ -348,20 +367,9 @@ void panda_require(const char *plugin_name) { _panda_require(plugin_name, NULL, 0, false); } -// Internal: remove a plugin from the global array panda_plugins -static void panda_delete_plugin(int i) -{ - if (i != nb_panda_plugins - 1) { // not the last element - memmove(&panda_plugins[i], &panda_plugins[i + 1], - (nb_panda_plugins - i - 1) * sizeof(panda_plugin)); - } - nb_panda_plugins--; -} - void panda_do_unload_plugin(int plugin_idx) { void *plugin = panda_plugins[plugin_idx].plugin; - bool exported_symbols = panda_plugins[plugin_idx].exported_symbols; void (*uninit_fn)(void *) = dlsym(plugin, "uninit_plugin"); if (!uninit_fn) { LOG_ERROR("Couldn't get symbol %s: %s\n", "uninit_plugin", @@ -370,12 +378,7 @@ void panda_do_unload_plugin(int plugin_idx) uninit_fn(plugin); } panda_unregister_callbacks(plugin); - panda_delete_plugin(plugin_idx); - dlclose(plugin); - if(exported_symbols) { - // This plugin was dlopened twice. dlclose it twice to fully unload it. - dlclose(plugin); - } + dlclose_plugin(plugin_idx); } void panda_unload_plugin(void *plugin) From e952f325db0eae4e2355e1bebb3a84204e8cd3fa Mon Sep 17 00:00:00 2001 From: Mark Mankins Date: Wed, 11 Oct 2023 07:09:02 -0400 Subject: [PATCH 16/31] Free plugin name. --- panda/src/callbacks.c | 1 + 1 file changed, 1 insertion(+) diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index 8af2c67c589..dcfd6335996 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -151,6 +151,7 @@ static bool load_libpanda(void) { // Internal: remove a plugin from the global array panda_plugins static void panda_delete_plugin(int i) { + g_free(panda_plugins[i].name); if (i != nb_panda_plugins - 1) { // not the last element memmove(&panda_plugins[i], &panda_plugins[i + 1], (nb_panda_plugins - i - 1) * sizeof(panda_plugin)); From 49b8134768479f71c5b1d35c0e12141ae78198d5 Mon Sep 17 00:00:00 2001 From: Mark Mankins Date: Wed, 11 Oct 2023 07:56:46 -0400 Subject: [PATCH 17/31] Use logging macros for most output For help output, always print to stdout --- panda/src/callbacks.c | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index dcfd6335996..f957c9abd37 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -113,7 +113,7 @@ static void *try_open_libpanda(const char *panda_lib) { if(panda_lib != NULL) { libpanda = dlopen(panda_lib, RTLD_LAZY | RTLD_NOLOAD | RTLD_GLOBAL); if (NULL == libpanda) { - fprintf(stderr, "Failed to load libpanda: %s from %s\n", dlerror(), panda_lib); + LOG_ERROR(PANDA_MSG_FMT "Failed to load libpanda: %s from %s\n", PANDA_CORE_NAME, dlerror(), panda_lib); } } return libpanda; @@ -129,15 +129,15 @@ static bool load_libpanda(void) { libpanda = try_open_libpanda(panda_lib); } else { #ifndef LIBRARY_DIR - assert(0 && "Library dir unset but library mode is enabled - Unsupported architecture?"); - printf("Library dir not set"); + assert(0 && "LIBRARY_DIR undefined, but library mode is enabled - Unsupported architecture?"); + LOG_ERROR(PANDA_MSG_FMT "LIBRARY_DIR not defined\n", PANDA_CORE_NAME); #endif const char *lib_dir = g_getenv("PANDA_DIR"); if (lib_dir != NULL) { panda_lib = g_strdup_printf("%s%s", lib_dir, LIBRARY_DIR); } else { - fprintf(stderr, "WARNING: using hacky dlopen code that will be removed soon\n"); + LOG_WARNING(PANDA_MSG_FMT "WARNING: using hacky dlopen code that will be removed soon\n", PANDA_CORE_NAME); panda_lib = g_strdup_printf("../../../build/%s", LIBRARY_DIR); // XXX This is bad, need a less hardcoded path } @@ -212,13 +212,14 @@ static bool _panda_load_plugin(const char *filename, const char *plugin_name, bo void *plugin = dlopen(filename, RTLD_NOW); if(!plugin) { - fprintf(stderr, "Failed to load %s: %s\n", filename, dlerror()); + LOG_ERROR(PANDA_MSG_FMT "Failed to load %s: %s\n", PANDA_CORE_NAME, filename, dlerror()); return false; } - bool (*init_fn)(void *) = dlsym(plugin, "init_plugin"); + const char *init_plugin = "init_plugin"; + bool (*init_fn)(void *) = dlsym(plugin, init_plugin); if(!init_fn) { - fprintf(stderr, "Couldn't get symbol %s: %s\n", "init_plugin", dlerror()); + LOG_ERROR(PANDA_MSG_FMT "Couldn't get symbol %s: %s\n", PANDA_CORE_NAME, init_plugin, dlerror()); dlclose(plugin); return false; } @@ -250,8 +251,8 @@ static bool _panda_load_plugin(const char *filename, const char *plugin_name, bo panda_args_set_help_wanted(plugin_name); if (panda_help_wanted) { printf("Options for plugin %s:\n", plugin_name); - fprintf(stderr, "PLUGIN ARGUMENT REQUIRED DESCRIPTION\n"); - fprintf(stderr, "====== ======== ======== ===========\n"); + printf("PLUGIN ARGUMENT REQUIRED DESCRIPTION\n"); + printf("====== ======== ======== ===========\n"); } if(!init_fn(plugin) || panda_plugin_load_failed) { @@ -1102,7 +1103,7 @@ static bool panda_parse_bool_internal(panda_arg_list *args, const char *argname, } help: if (panda_help_wanted) { - fprintf(stderr, "%-20s%-24sOptional %s (default=true)\n", args->plugin_name, argname, help); + printf("%-20s%-24sOptional %s (default=true)\n", args->plugin_name, argname, help); } // not found @@ -1138,13 +1139,13 @@ static target_ulong panda_parse_ulong_internal(panda_arg_list *args, const char error_handling: if (required) { LOG_ERROR("ERROR: plugin required ulong argument \"%s\" but you did not provide it\n", argname); - fprintf(stderr, "Help for \"%s\": %s\n", argname, help); + LOG_ERROR(PANDA_MSG_FMT "Help for \"%s\": %s\n", PANDA_CORE_NAME, argname, help); panda_plugin_load_failed = true; } help: if (panda_help_wanted) { - if (required) fprintf(stderr, "%-20s%-24sRequired %s\n", args->plugin_name, argname, help); - else fprintf(stderr, "%-20s%-24sOptional %s (default=" TARGET_FMT_ld ")\n", args->plugin_name, argname, help, defval); + if (required) printf("%-20s%-24sRequired %s\n", args->plugin_name, argname, help); + else printf("%-20s%-24sOptional %s (default=" TARGET_FMT_ld ")\n", args->plugin_name, argname, help, defval); } return defval; @@ -1180,8 +1181,8 @@ static uint32_t panda_parse_uint32_internal(panda_arg_list *args, const char *ar } help: if (panda_help_wanted) { - if (required) fprintf(stderr, "%-20s%-24sRequired %s\n", args->plugin_name, argname, help); - else fprintf(stderr, "%-20s%-24sOptional %s (default=%d)\n", args->plugin_name, argname, help, defval); + if (required) printf("%-20s%-24sRequired %s\n", args->plugin_name, argname, help); + else printf("%-20s%-24sOptional %s (default=%d)\n", args->plugin_name, argname, help, defval); } return defval; @@ -1217,8 +1218,8 @@ static uint64_t panda_parse_uint64_internal(panda_arg_list *args, const char *ar } help: if (panda_help_wanted) { - if (required) fprintf(stderr, "%-20s%-24sRequired %s)\n", args->plugin_name, argname, help); - else fprintf(stderr, "%-20s%-24sOptional %s (default=%" PRId64 ")\n", args->plugin_name, argname, help, defval); + if (required) printf("%-20s%-24sRequired %s)\n", args->plugin_name, argname, help); + else printf("%-20s%-24sOptional %s (default=%" PRId64 ")\n", args->plugin_name, argname, help, defval); } return defval; @@ -1254,8 +1255,8 @@ static double panda_parse_double_internal(panda_arg_list *args, const char *argn } help: if (panda_help_wanted) { - if (required) fprintf(stderr, "%-20s%-24sRequired %s\n", args->plugin_name, argname, help); - else fprintf(stderr, "%-20s%-24sOptional %s (default=%f)\n", args->plugin_name, argname, help, defval); + if (required) printf("%-20s%-24sRequired %s\n", args->plugin_name, argname, help); + else printf("%-20s%-24sOptional %s (default=%f)\n", args->plugin_name, argname, help, defval); } return defval; @@ -1329,8 +1330,8 @@ static const char *panda_parse_string_internal(panda_arg_list *args, const char } help: if (panda_help_wanted) { - if (required) fprintf(stderr, "%-20s%-24sRequired %s\n", args->plugin_name, argname, help); - else fprintf(stderr, "%-20s%-24sOptional %s (default=\"%s\")\n", args->plugin_name, argname, help, defval); + if (required) printf("%-20s%-24sRequired %s\n", args->plugin_name, argname, help); + else printf("%-20s%-24sOptional %s (default=\"%s\")\n", args->plugin_name, argname, help, defval); } return defval; From b583b3ece9859645d2abf4db36c4837709c972bc Mon Sep 17 00:00:00 2001 From: Mark Mankins Date: Wed, 11 Oct 2023 08:00:54 -0400 Subject: [PATCH 18/31] Extract code to export symbols into a new function. --- panda/src/callbacks.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index f957c9abd37..902eaa074f6 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -170,6 +170,22 @@ static void dlclose_plugin(int plugin_idx) { } } +// Determine if the plugin being loaded wants to export symbols to +// subsequently loaded plugins. If it does, dlopen it a second time +// with RTLD_GLOBAL. +static void do_check_export_symbols(panda_plugin *panda_plugin, const char *filename) { + + char *export_symbol = g_strdup_printf("PANDA_EXPORT_SYMBOLS_%s", panda_plugin->name); + + if(NULL != dlsym(panda_plugin->plugin, export_symbol)) { + LOG_DEBUG(PANDA_MSG_FMT "Exporting symbols for plugin %s\n", PANDA_CORE_NAME, panda_plugin->name); + assert(panda_plugin->plugin == dlopen(filename, RTLD_NOW | RTLD_GLOBAL)); + panda_plugin->exported_symbols = true; + } + + g_free(export_symbol); +} + static bool _panda_load_plugin(const char *filename, const char *plugin_name, bool library_mode) { static bool libpanda_loaded = false; @@ -233,15 +249,7 @@ static bool _panda_load_plugin(const char *filename, const char *plugin_name, bo panda_plugins[nb_panda_plugins].exported_symbols = false; panda_plugins[nb_panda_plugins].name = g_strdup(plugin_name); - char *export_symbol = g_strdup_printf("PANDA_EXPORT_SYMBOLS_%s", plugin_name); - - if(NULL != dlsym(plugin, export_symbol)) { - LOG_DEBUG(PANDA_MSG_FMT "Exporting symbols for plugin %s\n", PANDA_CORE_NAME, plugin_name); - assert(plugin == dlopen(filename, RTLD_NOW | RTLD_GLOBAL)); - panda_plugins[nb_panda_plugins].exported_symbols = true; - } - - g_free(export_symbol); + do_check_export_symbols(&panda_plugins[nb_panda_plugins], filename); nb_panda_plugins++; From 4e3819ea5957f45928ce898290e3786aedaa9b77 Mon Sep 17 00:00:00 2001 From: Mark Mankins Date: Wed, 11 Oct 2023 08:49:07 -0400 Subject: [PATCH 19/31] Remove commented out code. --- panda/src/panda_api.c | 50 ------------------------------------------- 1 file changed, 50 deletions(-) diff --git a/panda/src/panda_api.c b/panda/src/panda_api.c index a43be94e591..36bce437597 100644 --- a/panda/src/panda_api.c +++ b/panda/src/panda_api.c @@ -200,56 +200,6 @@ void panda_setup_signal_handling(void (*f) (int, void*, void *)) panda_external_signal_handler = (void(*)(int,siginfo_t*,void*)) f; } -// we have this temporarily in callbacks.c -> to be moved here -/* -bool panda_load_external_plugin(const char *filename, const char *plugin_name, void *plugin_uuid, void *init_fn_ptr) { - // don't load the same plugin twice - uint32_t i; - for (i=1; i dlopen(filename, RTLD_NOW); - bool (*init_fn)(void *) = init_fn_ptr; //normally dlsym init_fun - - // Populate basic plugin info *before* calling init_fn. - // This allows plugins accessing handles of other plugins before - // initialization completes. E.g. osi does a panda_require("win7x86intro"), - // and then win7x86intro does a PPP_REG_CB("osi", ...) while initializing. - panda_plugins[nb_panda_plugins].plugin = plugin; - if (plugin_name) { - strncpy(panda_plugins[nb_panda_plugins].name, plugin_name, 256); - } else { - char *pn = g_path_get_basename((char *) filename); - *g_strrstr(pn, HOST_DSOSUF) = '\0'; - strncpy(panda_plugins[nb_panda_plugins].name, pn, 256); - g_free(pn); - } - nb_panda_plugins++; - - // Call init_fn and check status. - fprintf(stderr, PANDA_MSG_FMT "initializing %s\n", PANDA_CORE_NAME, panda_plugins[nb_panda_plugins-1].name); - panda_help_wanted = false; - panda_args_set_help_wanted(plugin_name); - if (panda_help_wanted) { - printf("Options for plugin %s:\n", plugin_name); - fprintf(stderr, "PLUGIN ARGUMENT REQUIRED DESCRIPTION\n"); - fprintf(stderr, "====== ======== ======== ===========\n"); - } - if(!init_fn(plugin) || panda_plugin_load_failed) { - return false; - } - return true; -}*/ - - - // Taken from Avatar2's Configurable Machine - see hw/avatar/configurable_machine.c void map_memory(char* name, uint64_t size, uint64_t address) { //const char * name; /// XXX const? From 961403148e57dd2b01dad204561b7c6960fd4d9a Mon Sep 17 00:00:00 2001 From: Mark Mankins Date: Wed, 11 Oct 2023 12:42:11 -0400 Subject: [PATCH 20/31] Clear dlerror if magic export symbol is not found. Previously, running panda-system-i386 -panda osi_linux:help=y would output that osi failed to load because PANDA_EXPORT_SYMBOLS_osi_linux was not defined. The error message output had nothing to do with why osi_linux failed to load. In this case, osi_linux doesn't panda_require('osi') (which may be a bug) - so when it calls init_osi_api things go sideways. The error message was being obtained in init_osi_api from calling dlerror() - which was reporting the missing symbol, which isn't an error at all as the symbol is optional and not expected to be found when loading most plugins. --- panda/src/callbacks.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index 902eaa074f6..787e71fe3f7 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -181,6 +181,10 @@ static void do_check_export_symbols(panda_plugin *panda_plugin, const char *file LOG_DEBUG(PANDA_MSG_FMT "Exporting symbols for plugin %s\n", PANDA_CORE_NAME, panda_plugin->name); assert(panda_plugin->plugin == dlopen(filename, RTLD_NOW | RTLD_GLOBAL)); panda_plugin->exported_symbols = true; + } else { + // Error condition is not unexpected, clear dlerror(), + // otherwise someone might call it later and be confused + dlerror(); } g_free(export_symbol); From f7660454fbce3e093dd0be01666358b04d1054af Mon Sep 17 00:00:00 2001 From: "Laura L. Mann" Date: Thu, 9 Nov 2023 06:59:21 -0500 Subject: [PATCH 21/31] A538: add fetchers for FS and DS Also log more information before crashing due to missing register fetcher so the user has a clue as to what went wrong. --- .../coverage/EdgeInstrumentationDelegate.cpp | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/panda/plugins/coverage/EdgeInstrumentationDelegate.cpp b/panda/plugins/coverage/EdgeInstrumentationDelegate.cpp index 0400878f93c..014e6710cb2 100644 --- a/panda/plugins/coverage/EdgeInstrumentationDelegate.cpp +++ b/panda/plugins/coverage/EdgeInstrumentationDelegate.cpp @@ -98,6 +98,10 @@ const static std::unordered_map { X86_REG_CS, MK_SEG_FETCHER(segs[R_CS]) }, + { X86_REG_DS, MK_SEG_FETCHER(segs[R_DS]) }, + + { X86_REG_FS, MK_SEG_FETCHER(segs[R_FS]) }, + { X86_REG_SS, MK_SEG_FETCHER(segs[R_SS]) }, { X86_REG_GS, MK_SEG_FETCHER(segs[R_GS]) }, @@ -128,7 +132,7 @@ const static std::unordered_map // control flow instructions have at most two targets, we can get away with // this struct and avoid the worst case O(N) complexity of a hash table and // this should be more cache friendly because setting the jump targets for -// the current thread basiclaly a memcpy or assignment. +// the current thread basically a memcpy or assignment. struct JumpTargets { bool has_dst1; @@ -146,7 +150,7 @@ struct EdgeState // Flag for enabling or disabling coverage. bool cov_enabled; - // Maps Thread ID -> Previos Blocks + // Maps Thread ID -> Previous Blocks std::unordered_map prev_blocks; // Previous Block for Current Thread Block* prev_block; @@ -414,6 +418,21 @@ static void instrument_jcc(EdgeState *edge_state, CPUState *cpu, TCGOp *op, insert_call(&op, &jcc_callback, edge_state, tb->pc, tb->size, nit, jt); } +static const RegisterFetcher *get_register_fetcher(cs_insn *insn, unsigned int reg) +{ + // print some useful debugging information before give up if can't get + // the requested register fetcher + const RegisterFetcher *rf = nullptr; + try { + rf = &CS_TO_QEMU_REG_FETCH.at(reg); + } catch (std::out_of_range& err) { + printf("Error getting fetcher for register %d at address 0x%lx\n", reg, + insn->address); + throw; + } + return rf; +} + static void instrument_jmp(EdgeState *edge_state, CPUState *cpu, TCGOp *op, TranslationBlock *tb, cs_insn *insn) { @@ -422,14 +441,13 @@ static void instrument_jmp(EdgeState *edge_state, CPUState *cpu, TCGOp *op, target_ulong jt = static_cast(jmp_op.imm); insert_call(&op, static_jmp_callback, edge_state, tb->pc, tb->size, jt); } else if (X86_OP_MEM == jmp_op.type) { - static const RegisterFetcher *INVALID_REGISTER = - &CS_TO_QEMU_REG_FETCH.at(X86_REG_INVALID); - const RegisterFetcher *srf = &CS_TO_QEMU_REG_FETCH.at( + get_register_fetcher(insn, X86_REG_INVALID); + const RegisterFetcher *srf = get_register_fetcher(insn, jmp_op.mem.segment); - const RegisterFetcher *brf = &CS_TO_QEMU_REG_FETCH.at( + const RegisterFetcher *brf = get_register_fetcher(insn, jmp_op.mem.base); - const RegisterFetcher *irf = &CS_TO_QEMU_REG_FETCH.at( + const RegisterFetcher *irf = get_register_fetcher(insn, jmp_op.mem.index); #ifdef EDGE_INST_DEBUG @@ -491,7 +509,7 @@ static void instrument_jmp(EdgeState *edge_state, CPUState *cpu, TCGOp *op, } } else if (X86_OP_REG == jmp_op.type) { - const RegisterFetcher *reg_fetcher = &CS_TO_QEMU_REG_FETCH.at( + const RegisterFetcher *reg_fetcher = get_register_fetcher(insn, jmp_op.reg); insert_call(&op, jmp_reg_callback, edge_state, cpu, tb->pc, tb->size, reg_fetcher); From 392bb77a412c1bd51efb137fc23f155bde588ba4 Mon Sep 17 00:00:00 2001 From: "Laura L. Mann" Date: Thu, 9 Nov 2023 07:57:08 -0500 Subject: [PATCH 22/31] A718: respond to plugin_cmd help --- panda/plugins/coverage/coverage.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/panda/plugins/coverage/coverage.cpp b/panda/plugins/coverage/coverage.cpp index d19ceb02910..f6237d8db8d 100644 --- a/panda/plugins/coverage/coverage.cpp +++ b/panda/plugins/coverage/coverage.cpp @@ -42,8 +42,7 @@ using namespace coverage; const char *DEFAULT_FILE = "coverage.csv"; // commands that can be accessed through the QEMU monitor -const char *MONITOR_HELP = "help"; -constexpr size_t MONITOR_HELP_LEN = 4; +const std::string MONITOR_HELP = "help"; const std::string MONITOR_ENABLE = "coverage_enable"; const std::string MONITOR_DISABLE = "coverage_disable"; @@ -124,6 +123,11 @@ int monitor_callback(Monitor *mon, const char *cmd_cstr) std::cerr << "Error enabling instrumentation: " << err.code().message() << "\n"; } } + } else if (0 == cmd.find(MONITOR_HELP)) { + log_message("coverage_enable=filename: start logging coverage " + "information to the named file"); + log_message("coverage_disable: stop logging coverage information " + "and close the current file"); } return 0; } From 93be8bb2eeb40b6fc8749c8c53200e8e356c0991 Mon Sep 17 00:00:00 2001 From: "Laura L. Mann" Date: Tue, 14 Nov 2023 13:22:07 -0500 Subject: [PATCH 23/31] A538: remove pointer arithmetic abuse Also note when cannot read jump destination from panda memory. --- .../coverage/EdgeInstrumentationDelegate.cpp | 41 +++++++++++++++---- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/panda/plugins/coverage/EdgeInstrumentationDelegate.cpp b/panda/plugins/coverage/EdgeInstrumentationDelegate.cpp index 014e6710cb2..0769ec07b23 100644 --- a/panda/plugins/coverage/EdgeInstrumentationDelegate.cpp +++ b/panda/plugins/coverage/EdgeInstrumentationDelegate.cpp @@ -39,7 +39,8 @@ static target_ulong fetch_register_value(size_t cpu_offset, target_ulong mask, target_ulong shr) { CPUArchState *env = static_cast(first_cpu->env_ptr); - return (*(reinterpret_cast((env + cpu_offset))) & mask) + // don't want to do pointer arithmetic, as cpu_offset in wrong units + return (*(reinterpret_cast((((size_t)env) + cpu_offset))) & mask) >> shr; } #define MK_REG_FETCHER(cpu_state_variable, mask, shift_right) \ @@ -49,7 +50,7 @@ static target_ulong fetch_register_value(size_t cpu_offset, target_ulong mask, static target_ulong fetch_segment_value(size_t cpu_offset) { CPUArchState *env = static_cast(first_cpu->env_ptr); - return reinterpret_cast((env + cpu_offset))->base; + return reinterpret_cast((((size_t)env) + cpu_offset))->base; } #define MK_SEG_FETCHER(cpu_state_variable) \ std::bind(fetch_segment_value, offsetof(CPUArchState, cpu_state_variable)) @@ -251,8 +252,12 @@ static void jmp_mem_direct_callback(EdgeState *edge_state, return; } target_ulong jump_target = 0x0; - panda_virtual_memory_read(cpu, direct_address, + int retcode = panda_virtual_memory_read(cpu, direct_address, reinterpret_cast(&jump_target), sizeof(jump_target)); + if (retcode != 0) { + printf("error reading panda memory for block 0x" TARGET_FMT_lx "\n", + prev_block_addr); + } update_jump_targets(edge_state, prev_block_addr, prev_block_size, { .has_dst1 = true, .dst1 = jump_target, @@ -273,8 +278,12 @@ static void jmp_mem_indexed_callback(EdgeState *edge_state, } target_ulong address = (*index_register_fetcher)() * scale + disp; target_ulong jump_target = 0x0; - panda_virtual_memory_read(cpu, address, + int retcode = panda_virtual_memory_read(cpu, address, reinterpret_cast(&jump_target), sizeof(jump_target)); + if (retcode != 0) { + printf("error reading panda memory for block 0x" TARGET_FMT_lx "\n", + prev_block_addr); + } update_jump_targets(edge_state, prev_block_addr, prev_block_size, { .has_dst1 = true, .dst1 = jump_target, @@ -294,8 +303,12 @@ static void jmp_mem_indirect(EdgeState* edge_state, CPUState *cpu, } target_ulong address = (*base_register_fetch)() + disp; target_ulong jump_target = 0x0; - panda_virtual_memory_read(cpu, address, + int retcode = panda_virtual_memory_read(cpu, address, reinterpret_cast(&jump_target), sizeof(jump_target)); + if (retcode != 0) { + printf("error reading panda memory for block 0x" TARGET_FMT_lx "\n", + prev_block_addr); + } update_jump_targets(edge_state, prev_block_addr, prev_block_size, { .has_dst1 = true, .dst1 = jump_target, @@ -317,8 +330,12 @@ static void jmp_mem_indirect_no_index(EdgeState* edge_state, CPUState *cpu, target_ulong address = (*segment_register_fetch)() + (*base_register_fetch)() + disp; target_ulong jump_target = 0x0; - panda_virtual_memory_read(cpu, address, + int retcode = panda_virtual_memory_read(cpu, address, reinterpret_cast(&jump_target), sizeof(jump_target)); + if (retcode != 0) { + printf("error reading panda memory for block 0x" TARGET_FMT_lx "\n", + prev_block_addr); + } update_jump_targets(edge_state, prev_block_addr, prev_block_size, { .has_dst1 = true, .dst1 = jump_target, @@ -340,8 +357,12 @@ static void jmp_mem_indirect_disp_si_callback(EdgeState *edge_state, } target_ulong address = (*brf)() + disp + (*irf)() * scale; target_ulong jump_target = 0x0; - panda_virtual_memory_read(cpu, address, + int retcode = panda_virtual_memory_read(cpu, address, reinterpret_cast(&jump_target), sizeof(jump_target)); + if (retcode != 0) { + printf("error reading panda memory for block 0x" TARGET_FMT_lx "\n", + prev_block_addr); + } update_jump_targets(edge_state, prev_block_addr, prev_block_size, { .has_dst1 = true, .dst1 = jump_target, @@ -379,8 +400,12 @@ static void ret_callback(EdgeState *edge_state, // Read the return target address off the stack. CPUArchState *env_ptr = static_cast(cpu->env_ptr); target_ulong return_addr = 0x0; - panda_virtual_memory_read(cpu, env_ptr->regs[R_ESP], + int retcode = panda_virtual_memory_read(cpu, env_ptr->regs[R_ESP], reinterpret_cast(&return_addr), sizeof(return_addr)); + if (retcode != 0) { + printf("error reading panda memory for block 0x" TARGET_FMT_lx "\n", + prev_block_addr); + } update_jump_targets(edge_state, prev_block_addr, prev_block_size, { .has_dst1 = true, .dst1 = return_addr, From 7fb79ad2fa55fd3176994bd8b60f4db95cbb6f1f Mon Sep 17 00:00:00 2001 From: "Laura L. Mann" Date: Fri, 17 Nov 2023 09:12:47 -0500 Subject: [PATCH 24/31] A538: handle RIP relative addressing Fix calculation of mem_indirect target address when base register is RIP. --- .../coverage/EdgeInstrumentationDelegate.cpp | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/panda/plugins/coverage/EdgeInstrumentationDelegate.cpp b/panda/plugins/coverage/EdgeInstrumentationDelegate.cpp index 0769ec07b23..ee0c0634b50 100644 --- a/panda/plugins/coverage/EdgeInstrumentationDelegate.cpp +++ b/panda/plugins/coverage/EdgeInstrumentationDelegate.cpp @@ -317,6 +317,35 @@ static void jmp_mem_indirect(EdgeState* edge_state, CPUState *cpu, }); } +static void jmp_mem_indirect_rip(EdgeState* edge_state, CPUState *cpu, + target_ulong prev_block_addr, + target_ulong prev_block_size, + int64_t disp) +{ + if (!edge_state->cov_enabled) { + return; + } + // this instrumentation is called just before the prev_block is executed + // so the RIP register has prev_block_addr in it, not the address of the + // next block in memory after this one; fortunately, we have enough info + // to calculate the address of the next block without using a register + // fetcher + target_ulong address = prev_block_addr + prev_block_size + disp; + target_ulong jump_target = 0x0; + int retcode = panda_virtual_memory_read(cpu, address, + reinterpret_cast(&jump_target), sizeof(jump_target)); + if (retcode != 0) { + printf("error reading panda memory for block 0x" TARGET_FMT_lx "\n", + prev_block_addr); + } + update_jump_targets(edge_state, prev_block_addr, prev_block_size, { + .has_dst1 = true, + .dst1 = jump_target, + .has_dst2 = false, + .dst2 = 0x0 + }); +} + static void jmp_mem_indirect_no_index(EdgeState* edge_state, CPUState *cpu, target_ulong prev_block_addr, target_ulong prev_block_size, @@ -494,9 +523,15 @@ static void instrument_jmp(EdgeState *edge_state, CPUState *cpu, TCGOp *op, } else if (INVALID_REGISTER == srf && INVALID_REGISTER == irf) { - // indirect addressing - insert_call(&op, jmp_mem_indirect, edge_state, cpu, tb->pc, - tb->size, brf, jmp_op.mem.disp); + if (X86_REG_RIP == jmp_op.mem.base) { + // RIP relative addressing + insert_call(&op, jmp_mem_indirect_rip, edge_state, cpu, tb->pc, + tb->size, jmp_op.mem.disp); + } else { + // indirect addressing + insert_call(&op, jmp_mem_indirect, edge_state, cpu, tb->pc, + tb->size, brf, jmp_op.mem.disp); + } } else if (INVALID_REGISTER == srf && INVALID_REGISTER == brf) { From 1704a0aed6b329523022e7758fc6674cb2f6c3a3 Mon Sep 17 00:00:00 2001 From: "Laura L. Mann" Date: Fri, 17 Nov 2023 09:13:28 -0500 Subject: [PATCH 25/31] A538: add missing full default value Also fix some grammar and formatting issues. --- panda/plugins/coverage/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/panda/plugins/coverage/README.md b/panda/plugins/coverage/README.md index 89c02883dfb..ed0ee7d3589 100644 --- a/panda/plugins/coverage/README.md +++ b/panda/plugins/coverage/README.md @@ -16,7 +16,7 @@ execution is also stored, regardless of mode. (A value of 1 means the block was executed in kernel mode.) For the `osi-block` and `asid-block` modes, the mode used to create the CSV file is also written at the top of the file, to cue parsers in as to how to interpret the file. `edge` mode writes two blocks per -records where the first block represents the "from" node in a control flow +record where the first block represents the "from" node in a control flow graph (CFG) and the second represents the "to" node. Note that the `edge` mode requires OSI and only works with X86 guests. @@ -69,10 +69,10 @@ adjusted manually for DOS operating systems. Arguments --------- -* `filename` - The name of the file to output (default: `coverage.csv`). +* `filename` - The name of the file to output (default: `coverage.csv`). * `mode` - Output mode, one of `asid-block`, `osi-block`, or `edge` (default: `asid-block`). Note that `edge` requires OSI. -* `full` - When `true`, logs each record every time it is generated (default: +* `full` - When `true`, logs each record every time it is generated (default: `false`) * `summary` - When `true`, only report total distinct blocks executed per process. Requires OSI and a mode of `osi-block` (default: `false`) * `start_disabled` - When `true`, does not start data collection when the @@ -111,12 +111,14 @@ None Example ------- Use the coverage plugin with a recording: + ``` panda-system-i386 -m 2G -replay test \ -os windows-32-xpsp3 \ -panda coverage:filename=test_coverage.csv,mode=osi-block ``` Use the coverage plugin on a live system: + ``` panda-system-i386 -monitor stdio -m 2G -net nic -net user -os linux-32-.+ -panda osi -panda osi_linux:kconf_file=myconf.conf,kconf_group=mygroup -hda myimage.img (qemu) load_plugin coverage,filename=test01.csv,mode=osi-block From 8d661ba8ed7afa08739ae6e06d9e9e5e0ea0da34 Mon Sep 17 00:00:00 2001 From: zacogen Date: Wed, 8 Nov 2023 13:23:22 -0500 Subject: [PATCH 26/31] Update RR2 sha functions for newer openssl --- panda/src/rr/panda_rr2.c | 42 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/panda/src/rr/panda_rr2.c b/panda/src/rr/panda_rr2.c index 08310cdf2b9..0355fec867f 100644 --- a/panda/src/rr/panda_rr2.c +++ b/panda/src/rr/panda_rr2.c @@ -6,7 +6,7 @@ #include #include -#include +#include #include "config-host.h" #include "panda/rr/panda_rr2.h" @@ -15,7 +15,7 @@ bool write_file_to_archive(struct rr_file_state* rstate, const char* fname, const uint8_t* contents, size_t len); void write_magic_file(struct rr_file_state* rstate); -void write_hash_to_log(struct rr_file_state* rstate, const char* fname, SHA_CTX* ctx); +void write_hash_to_log(struct rr_file_state* rstate, const char* fname, EVP_MD_CTX* mdctx); bool is_valid_rrv2_file(const char* state); void add_file_hash_for_content(struct rr_file_state* rstate, const char* fname, const void* content, size_t len); @@ -287,12 +287,13 @@ bool rrfile_add_recording_file(struct rr_file_state* rstate, const char* type, archive_entry_free(entry); } - // Initialize a SHA_CTX for openssl for this file - SHA_CTX* ctx = (SHA_CTX*)malloc(sizeof(SHA_CTX)); - if (!ctx || !SHA1_Init(ctx)) { + // Initialize a EVP_MD_CTX for openssl for this file + EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); + const EVP_MD *md = EVP_sha1(); + if (!mdctx || !md || !EVP_DigestInit_ex(mdctx, md, NULL)) { fprintf(stderr, "Failed to find hash for file contents of %s\n", type); - if (ctx) { - free(ctx); + if (mdctx) { + EVP_MD_CTX_destroy(mdctx); } return false; } @@ -302,7 +303,7 @@ bool rrfile_add_recording_file(struct rr_file_state* rstate, const char* type, uint8_t buffer[1024 * 1024]; len = fread(buffer, 1, sizeof(buffer), fp); while (len > 0) { - SHA1_Update(ctx, buffer, len); + EVP_DigestUpdate(mdctx, buffer, len); int status = archive_write_data(a, buffer, len); if (status <= 0) { fprintf(stderr, "Failed to archive_write_data\n"); @@ -315,8 +316,8 @@ bool rrfile_add_recording_file(struct rr_file_state* rstate, const char* type, } unlink(fpath); // Write the hash for this file out to the log - write_hash_to_log(rstate, type, ctx); - free(ctx); + write_hash_to_log(rstate, type, mdctx); + EVP_MD_CTX_destroy(mdctx); return true; } @@ -418,7 +419,7 @@ bool write_file_to_archive(struct rr_file_state* rstate, const char* fname, return true; } -void write_hash_to_log(struct rr_file_state* rstate, const char* fname, SHA_CTX* ctx) +void write_hash_to_log(struct rr_file_state* rstate, const char* fname, EVP_MD_CTX* mdctx) { // If the log isn't open for writing, exit if (!rstate->hash_fp) { @@ -426,11 +427,11 @@ void write_hash_to_log(struct rr_file_state* rstate, const char* fname, SHA_CTX* } size_t hexsize = 41; - unsigned char* hash_md = (unsigned char*)malloc(SHA_DIGEST_LENGTH); + unsigned char* hash_md = (unsigned char*)malloc(EVP_MAX_MD_SIZE); char* hex = (char*)calloc(1, hexsize); // Finalize the hash and snprintf the hexified string for it - if (!SHA1_Final(hash_md, ctx)) { + if (!EVP_DigestFinal_ex(mdctx, hash_md, 0)) { fprintf(stderr, "Failed to find hash for file contents of %s\n", fname); goto cleanup; } @@ -453,23 +454,24 @@ void write_hash_to_log(struct rr_file_state* rstate, const char* fname, SHA_CTX* void add_file_hash_for_content(struct rr_file_state* rstate, const char* fname, const void* content, size_t len) { - // Initialize a SHA_CTX for openssl - SHA_CTX* ctx = (SHA_CTX*)malloc(sizeof(SHA_CTX)); - if (!ctx || !SHA1_Init(ctx)) { + // Initialize a EVP_MD_CTX for openssl + EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); + const EVP_MD *md = EVP_sha1(); + if (!mdctx || !md || !EVP_DigestInit_ex(mdctx, md, NULL)) { fprintf(stderr, "Failed to find hash for file contents of %s\n", fname); goto cleanup; } // Calculate the SHA1 hash for these file contents - if (!SHA1_Update(ctx, content, len)) { + if (!EVP_DigestUpdate(mdctx, content, len)) { fprintf(stderr, "Failed to find hash for file contents of %s\n", fname); goto cleanup; } // Write it to the log - write_hash_to_log(rstate, fname, ctx); + write_hash_to_log(rstate, fname, mdctx); cleanup: - if (ctx) { - free(ctx); + if (mdctx) { + EVP_MD_CTX_destroy(mdctx); } } From 4f40d77229962eb8f876a7dfc04fe00fee3b6b05 Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Fri, 1 Dec 2023 17:28:38 -0500 Subject: [PATCH 27/31] Add QMP callback --- panda/include/panda/callbacks/cb-defs.h | 35 ++++++++++++++ panda/include/panda/callbacks/cb-support.h | 1 + .../include/panda/callbacks/cb-trampolines.h | 1 + panda/src/callbacks.c | 1 + panda/src/cb-support.c | 2 + qapi/qmp-dispatch.c | 47 ++++++++++++++++--- 6 files changed, 80 insertions(+), 7 deletions(-) diff --git a/panda/include/panda/callbacks/cb-defs.h b/panda/include/panda/callbacks/cb-defs.h index 6170f6d699f..39f94216cc0 100644 --- a/panda/include/panda/callbacks/cb-defs.h +++ b/panda/include/panda/callbacks/cb-defs.h @@ -57,6 +57,7 @@ typedef enum panda_cb_type { PANDA_CB_HD_WRITE, // Each HDD write PANDA_CB_GUEST_HYPERCALL, // Hypercall from the guest (e.g. CPUID) PANDA_CB_MONITOR, // Monitor callback + PANDA_CB_QMP, // QMP callback PANDA_CB_CPU_RESTORE_STATE, // In cpu_restore_state() (fault/exception) PANDA_CB_BEFORE_LOADVM, // at start of replay, before loadvm PANDA_CB_ASID_CHANGED, // When CPU asid (address space identifier) changes @@ -592,6 +593,23 @@ typedef union panda_cb { */ int (*monitor)(Monitor *mon, const char *cmd); + /* Callback ID: PANDA_CB_QMP + + qmp: + Called when someone sends an unhandled QMP command + + Arguments: + char *command: the command string as json + char *args: the arguments string as json + char **result: pointer to a json result or NULL + + Helper call location: TBA + + Return value: + bool: true IFF the command was handled by the plugin + */ + bool (*qmp)(char *command, char* args, char **result); + /* Callback ID: PANDA_CB_CPU_RESTORE_STATE @@ -1544,6 +1562,23 @@ typedef union panda_cb_with_context { */ int (*monitor)(void* context, Monitor *mon, const char *cmd); + /* Callback ID: PANDA_CB_QMP + + qmp: + Called when someone sends an unhandled QMP command + + Arguments: + char *command: the command string as json + char *args: the arguments string as json + char **result: pointer to a json result or NULL + + Helper call location: TBA + + Return value: + bool: true IFF the command was handled by the plugin + */ + bool (*qmp)(void* context, char *command, char* args, char **result); + /* Callback ID: PANDA_CB_CPU_RESTORE_STATE diff --git a/panda/include/panda/callbacks/cb-support.h b/panda/include/panda/callbacks/cb-support.h index af894b7f4d5..b7009d5b971 100644 --- a/panda/include/panda/callbacks/cb-support.h +++ b/panda/include/panda/callbacks/cb-support.h @@ -75,6 +75,7 @@ bool panda_callbacks_after_find_fast(CPUState *cpu, TranslationBlock *tb, bool b int panda_callbacks_insn_exec(CPUState *env, target_ptr_t pc); int panda_callbacks_after_insn_exec(CPUState *env, target_ptr_t pc); int panda_callbacks_monitor(Monitor *mon, const char *cmd); +bool panda_callbacks_qmp(char *command, char *args, char **result); int panda_callbacks_before_loadvm(void); void panda_callbacks_replay_hd_transfer(CPUState *env, uint32_t type, target_ptr_t src_addr, target_ptr_t dest_addr, size_t num_bytes); void panda_callbacks_after_machine_init(CPUState *env); diff --git a/panda/include/panda/callbacks/cb-trampolines.h b/panda/include/panda/callbacks/cb-trampolines.h index f13ca5be3a3..2fbae221422 100644 --- a/panda/include/panda/callbacks/cb-trampolines.h +++ b/panda/include/panda/callbacks/cb-trampolines.h @@ -19,6 +19,7 @@ void panda_cb_trampoline_phys_mem_after_write(void* context, CPUState *env, targ int panda_cb_trampoline_insn_exec(void* context, CPUState *env, target_ptr_t pc); int panda_cb_trampoline_after_insn_exec(void* context, CPUState *env, target_ptr_t pc); int panda_cb_trampoline_monitor(void* context, Monitor *mon, const char *cmd); +bool panda_cb_trampoline_qmp(void* context, char *command, char *args, char **result); //int panda_cb_trampoline_before_loadvm(void* context); void panda_cb_trampoline_replay_hd_transfer(void* context, CPUState *env, uint32_t type, target_ptr_t src_addr, target_ptr_t dest_addr, size_t num_bytes); void panda_cb_trampoline_after_machine_init(void* context, CPUState *env); diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index 787e71fe3f7..73ec90c1bb0 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -474,6 +474,7 @@ panda_cb_with_context panda_get_cb_trampoline(panda_cb_type type) { CASE_CB_TRAMPOLINE(HD_WRITE,hd_write) CASE_CB_TRAMPOLINE(GUEST_HYPERCALL,guest_hypercall) CASE_CB_TRAMPOLINE(MONITOR,monitor) + CASE_CB_TRAMPOLINE(QMP,qmp) CASE_CB_TRAMPOLINE(CPU_RESTORE_STATE,cpu_restore_state) //CASE_CB_TRAMPOLINE(BEFORE_LOADVM,before_loadvm) diff --git a/panda/src/cb-support.c b/panda/src/cb-support.c index beb9517ca73..0f72386ed18 100644 --- a/panda/src/cb-support.c +++ b/panda/src/cb-support.c @@ -101,7 +101,9 @@ void panda_cb_trampoline_start_block_exec(void* context, CPUState *cpu, Translat // these aren't used MAKE_CALLBACK(void, HD_READ, hd_read, CPUState*, env); MAKE_CALLBACK(void, HD_WRITE, hd_write, CPUState*, env); + MAKE_CALLBACK(int, MONITOR, monitor, Monitor*, mon, const char*, cmd); +MAKE_CALLBACK(bool, QMP, qmp, char*, cmd, char*, args, char **, result); // Helper - get a physical address static inline hwaddr get_paddr(CPUState *cpu, target_ptr_t addr, void *ram_ptr) { diff --git a/qapi/qmp-dispatch.c b/qapi/qmp-dispatch.c index dc502129d80..5bc2b2d36d4 100644 --- a/qapi/qmp-dispatch.c +++ b/qapi/qmp-dispatch.c @@ -19,6 +19,8 @@ #include "qapi/qmp/qjson.h" #include "qapi-types.h" #include "qapi/qmp/qerror.h" +//#include "panda/callbacks/cb-support.h" +extern bool panda_callbacks_qmp(char *command, char* args, char **result); static QDict *qmp_dispatch_check_obj(const QObject *request, Error **errp) { @@ -83,24 +85,55 @@ static QObject *do_qmp_dispatch(QmpCommandList *cmds, QObject *request, command = qdict_get_str(dict, "execute"); cmd = qmp_find_command(cmds, command); + + if (!qdict_haskey(dict, "arguments")) { + args = qdict_new(); + } else { + args = qdict_get_qdict(dict, "arguments"); + QINCREF(args); + } + if (cmd == NULL) { + // Call any PANDA consumers of the unhandled command + // Provide them with arguments in json format. + // If any plugin returns true, we assume it handled the command + // and we expect a json output in result. + char *result = NULL; + const QString* cmd_args_q = qobject_to_json(QOBJECT(args)); + const char *cmd_args = qstring_get_str(cmd_args_q); + + if (panda_callbacks_qmp((char*)command, (char*)cmd_args, &result)) { + if (result != NULL) { + // We have a return value from the callback. Let's convert it to a qobject + ret = qobject_from_json(result, &local_err); + if (local_err) { + printf("PANDA ERROR decoding result json in callback"); + error_propagate(errp, local_err); + return NULL; + } + } + + if (!ret) { + printf("PANDA WARNING: a qmp callback consumer returned TRUE without providing \ + a return value! Creating empty dictionary"); + + ret = QOBJECT(qdict_new()); + } + QDECREF(args); + return ret; + } + error_set(errp, ERROR_CLASS_COMMAND_NOT_FOUND, "The command %s has not been found", command); return NULL; } + if (!cmd->enabled) { error_setg(errp, "The command %s has been disabled for this instance", command); return NULL; } - if (!qdict_haskey(dict, "arguments")) { - args = qdict_new(); - } else { - args = qdict_get_qdict(dict, "arguments"); - QINCREF(args); - } - cmd->fn(args, &ret, &local_err); if (local_err) { error_propagate(errp, local_err); From 6447808e6bdcab0ab35ee7fd165906b92c1e9497 Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Tue, 5 Dec 2023 21:42:58 -0500 Subject: [PATCH 28/31] Add example for consuming QMP callbacks --- panda/python/examples/cb_qmp.py | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 panda/python/examples/cb_qmp.py diff --git a/panda/python/examples/cb_qmp.py b/panda/python/examples/cb_qmp.py new file mode 100644 index 00000000000..00ea50b7f57 --- /dev/null +++ b/panda/python/examples/cb_qmp.py @@ -0,0 +1,45 @@ +from pandare import Panda +from time import sleep +import json +import tempfile + +path = tempfile.mktemp(".sock") +panda = Panda(generic="x86_64", extra_args=["-qmp", f"unix:{path},server,nowait"]) + +print() +print("QMP example running! To send a QMP command to this VM, run:") +print("""echo '{ "execute": "mycmd", "arguments": { "arg_key" : "arg_val" } }' | socat - UNIX-CONNECT:""" + path + """ | cat""") +print() + +@panda.cb_qmp +def on_qmp_command(cmd, arg_json, result_json): + args = {} + cmd = panda.ffi.string(cmd).decode() if cmd != panda.ffi.NULL else None + if cmd != 'mycmd': + return False + if arg_json != panda.ffi.NULL: + args = json.loads(panda.ffi.string(arg_json).decode()) + + print(f"PyPANDA handling QMP command {cmd} with args {args}") + + data = { + "hello": "world", + "key": "value", + "0": 1, + } + + # Dump our result to json and copy it into the result_json buffer + encoded_data = json.dumps(data).encode() + result_buffer = panda.ffi.new("char[]", len(encoded_data) + 1) # Null term + panda.ffi.memmove(result_buffer, encoded_data, len(encoded_data)) + result_json[0] = result_buffer + return True + +@panda.queue_blocking +def driver(): + panda.revert_sync("root") + panda.run_serial_cmd("whoami") + sleep(300) + panda.end_analysis() + +panda.run() From b07c387ca9c8b40005a23ec1ce5f228fd513af89 Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Wed, 6 Dec 2023 09:42:48 -0500 Subject: [PATCH 29/31] Add panda_callbacks_qmp stub in qtest so qmp tests can run without panda --- qapi/qmp-dispatch.c | 3 +++ stubs/qtest.c | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/qapi/qmp-dispatch.c b/qapi/qmp-dispatch.c index 5bc2b2d36d4..8311ff99bce 100644 --- a/qapi/qmp-dispatch.c +++ b/qapi/qmp-dispatch.c @@ -19,8 +19,11 @@ #include "qapi/qmp/qjson.h" #include "qapi-types.h" #include "qapi/qmp/qerror.h" + //#include "panda/callbacks/cb-support.h" extern bool panda_callbacks_qmp(char *command, char* args, char **result); +// And we have a stub in stubs/qtest.c that always returns false if we build +// without panda for tests static QDict *qmp_dispatch_check_obj(const QObject *request, Error **errp) { diff --git a/stubs/qtest.c b/stubs/qtest.c index 891eb954fbe..f68f66e9395 100644 --- a/stubs/qtest.c +++ b/stubs/qtest.c @@ -18,3 +18,9 @@ bool qtest_driver(void) { return false; } + +/* Needed for qmp-dispatch tests to pass when we don't have panda */ +bool panda_callbacks_qmp(char *command, char* args, char **result); +bool panda_callbacks_qmp(char *command, char* args, char **result) { + return false; +} From 50cc9aae3e6c6d4302b43686749d18adb83dab4f Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Thu, 7 Dec 2023 14:55:37 -0500 Subject: [PATCH 30/31] Ensure variables are initialized in pri_taint and loaded to silence compiler warnings --- panda/plugins/loaded/loaded.cpp | 2 +- panda/plugins/pri_taint/pri_taint.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/panda/plugins/loaded/loaded.cpp b/panda/plugins/loaded/loaded.cpp index 844ec55be0a..f3303c5fb73 100644 --- a/panda/plugins/loaded/loaded.cpp +++ b/panda/plugins/loaded/loaded.cpp @@ -69,7 +69,7 @@ uint32_t guest_strncpy(CPUState *cpu, char *buf, size_t maxlen, target_ulong gue buf[0] = 0; unsigned i; for (i=0; i Date: Fri, 15 Dec 2023 13:39:24 -0500 Subject: [PATCH 31/31] Arc resource sharing (#1387) Migrate to ARC for CI testing --- .github/workflows/local_tests.yml | 6 +- .github/workflows/parallel_tests.yml | 165 ++++++++++++++------------- .github/workflows/publish_docker.yml | 29 ++--- .github/workflows/stale.yml | 2 +- .gitmodules | 22 ++-- Dockerfile | 4 +- panda/python/core/requirements.txt | 3 + panda/python/tests/run_all_tests.sh | 22 ++++ panda/scripts/install_ubuntu.sh | 3 +- 9 files changed, 148 insertions(+), 108 deletions(-) create mode 100644 panda/python/core/requirements.txt create mode 100644 panda/python/tests/run_all_tests.sh diff --git a/.github/workflows/local_tests.yml b/.github/workflows/local_tests.yml index 6f01b98de03..1da5e317390 100644 --- a/.github/workflows/local_tests.yml +++ b/.github/workflows/local_tests.yml @@ -7,12 +7,12 @@ name: Local jobs: local_build_container: - runs-on: ubuntu:22.04 + runs-on: panda-arc steps: - uses: actions/checkout@v2 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory - name: Build docker container from project root - run: echo $GITHUB_WORKSPACE; cd $GITHUB_WORKSPACE && DOCKER_BUILDKIT=1 docker build --progress=plain --target developer -t panda_local_${{ github.sha }} . + run: echo $GITHUB_WORKSPACE; cd $GITHUB_WORKSPACE && DOCKER_BUILDKIT=1 docker build --progress=plain --target developer -t panda_local:${{ github.sha }} . - name: Minimal test of built container # Just test to see if one of our binaries is built - run: docker run --rm "panda_local_${{ github.sha }}" /bin/bash -c 'exit $(/panda/build/arm-softmmu/panda-system-arm -help | grep -q "usage. panda-system-arm")' + run: docker run --rm "panda_local:${{ github.sha }}" /bin/bash -c 'exit $(/panda/build/arm-softmmu/panda-system-arm -help | grep -q "usage. panda-system-arm")' diff --git a/.github/workflows/parallel_tests.yml b/.github/workflows/parallel_tests.yml index b4dc7c88644..4e0a618ab42 100644 --- a/.github/workflows/parallel_tests.yml +++ b/.github/workflows/parallel_tests.yml @@ -1,8 +1,7 @@ name: Parallel Tests # For PRs to dev or pushes that modify the root Dockerfile, build from scratch # then run CI tests using that container in parallel -# For forked repos that can't use our self-hosted test suite, just build and run make check - +# For forked repos that can't use our panda-arc test suite, just build and run make check on: pull_request: branches: @@ -17,14 +16,26 @@ on: jobs: test_installer: # test install_ubuntu.sh - runs-on: ubuntu-20.04 # Note 22.04 would work, but it requires docker > 20.10.7 which is not on our CI box (yet) + runs-on: panda-arc # Note 22.04 would work, but it requires docker > 20.10.7 which is not on our CI box (yet) + container: + image: ubuntu:20.04 steps: - - uses: actions/checkout@v2 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory + - name: Update + run: apt-get -qq update -y + - name: Install ssl + run: apt-get -qq install -y libssl-dev + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install Python dev headers + run: apt-get -qq install -y libpython3.9-dev + - uses: actions/checkout@v4 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory - name: Lint PyPANDA with flake8 run: | - python -m pip install --upgrade pip - python -m pip install flake8 - python -m flake8 $GITHUB_WORKSPACE/panda/python/core/pandare/ --count --select=E9,F63,F7,F82 --show-source --statistics + pip install --upgrade pip + pip install flake8 + flake8 $GITHUB_WORKSPACE/panda/python/core/pandare/ --count --select=E9,F63,F7,F82 --show-source --statistics # python -m flake8 $GITHUB_WORKSPACE/panda/python/core/pandare/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Run install_ubuntu.sh run: cd $GITHUB_WORKSPACE && ./panda/scripts/install_ubuntu.sh @@ -32,105 +43,103 @@ jobs: build_container: if: github.repository == 'panda-re/panda' - runs-on: self-hosted + runs-on: panda-arc steps: - - - uses: actions/checkout@v2 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory - - - name: Build docker container from project root - run: cd $GITHUB_WORKSPACE && DOCKER_BUILDKIT=1 docker build --progress=plain --target developer -t panda_local_${{ github.sha }} . - - - name: Minimal test of built container # Just test to see if one of our binaries is built - run: docker run --rm "panda_local_${{ github.sha }}" /bin/bash -c 'exit $(/panda/build/arm-softmmu/panda-system-arm -help | grep -q "usage. panda-system-arm")' - - taint_tests: + - name: Install git + run: sudo apt-get -qq update -y && sudo apt-get -qq install git -y + - uses: actions/checkout@v4 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory + with: + fetch-depth: 0 + + #- name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: ${{ github.workspace }} + tags: ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }} + target: developer + - name: Minimal test of built container # Just test to see if one of our binaries is built + run: docker run --rm "ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}" /bin/bash -c 'exit $(/panda/build/arm-softmmu/panda-system-arm -help | grep -q "usage. panda-system-arm")' + - name: 'Login to Github Container Registry' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push to ghcr.io + run: docker push ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }} + + tests: if: github.repository == 'panda-re/panda' - runs-on: self-hosted + runs-on: panda-arc needs: [build_container] strategy: matrix: - target: [i386, x86_64] - + include: + - test_type: "taint" + target: "i386" + - test_type: "taint" + target: "x86_64" + - test_type: "pypanda" + test_script: "all" + - test_type: "make_check" + test_script: "all" + steps: # Given a container with PANDA installed at /panda, run the taint tests - - name: Run taint tests inside current container + - name: Update + run: sudo apt-get -qq update -y + - name: Install ssl + run: sudo apt-get -qq install -y wget + - name: Run Taint Tests + if: matrix.test_type == 'taint' run: >- + wget -q -O wheezy_panda2.qcow2 https://panda-re.mit.edu/qcows/linux/debian/7.3/x86/debian_7.3_x86.qcow; + wget -q https://panda-re.mit.edu/qcows/linux/ubuntu/1804/x86_64/bionic-server-cloudimg-amd64-noaslr-nokaslr.qcow2; docker run --name panda_test_${{ matrix.target }}_${GITHUB_RUN_ID} - --mount type=bind,source=/home/panda/regdir/qcows/wheezy_panda2.qcow2,target=/home/panda/regdir/qcows/wheezy_panda2.qcow2 - --mount type=bind,source=/home/panda/regdir/qcows/bionic-server-cloudimg-amd64-noaslr-nokaslr.qcow2,target=/home/panda/regdir/qcows/bionic-server-cloudimg-amd64-noaslr-nokaslr.qcow2 - --rm -t "panda_local_${{ github.sha }}" bash -c + --mount type=bind,source=$(pwd)/wheezy_panda2.qcow2,target=/home/panda/regdir/qcows/wheezy_panda2.qcow2 + --mount type=bind,source=$(pwd)/bionic-server-cloudimg-amd64-noaslr-nokaslr.qcow2,target=/home/panda/regdir/qcows/bionic-server-cloudimg-amd64-noaslr-nokaslr.qcow2 + --rm -t "ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}" bash -c "cd /tmp; git clone https://github.com/panda-re/panda_test; cd ./panda_test/tests/taint2; + echo 'Running Record:'; python3 taint2_multi_arch_record_or_replay.py --arch ${{ matrix.target }} --mode record; + echo 'Running Replay:'; python3 taint2_multi_arch_record_or_replay.py --arch ${{ matrix.target }} --mode replay; sed -i '/^\s*$/d' taint2_log; if cat taint2_log; then echo 'Taint unit test log found!'; else echo 'Taint unit test log NOT found!' && exit 1; fi; echo -e '\nFailures:'; if grep 'fail' taint2_log; then echo 'TEST FAILED!' && exit 1; else echo -e 'None.\nTEST PASSED!' && exit 0; fi" - sym_trace_tests: - if: github.repository == 'panda-re/panda' - runs-on: self-hosted - needs: [build_container] - - strategy: - matrix: - target: [x86_64] - - steps: - # Given a container with PANDA installed at /panda, run the taint tests - - name: Run symbolic tracing tests inside current container + - name: Run PyPanda Tests + if: matrix.test_type == 'pypanda' run: >- + wget -q https://panda-re.mit.edu/qcows/linux/ubuntu/1604/x86/ubuntu_1604_x86.qcow; + docker run --name panda_test_${{ matrix.test_script }}_${GITHUB_RUN_ID} + --mount type=bind,source=$(pwd)/ubuntu_1604_x86.qcow,target=/root/.panda/ubuntu_1604_x86.qcow + -e PANDA_TEST=yes --cap-add SYS_NICE + --rm -t "ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}" bash -c + "cd /panda/panda/python/tests/ && make && pip3 install -r requirements.txt && chmod +x ./run_all_tests.sh && ./run_all_tests.sh"; + docker run --name panda_sym_test_${{ matrix.target }}_${GITHUB_RUN_ID} - --rm -t "panda_local_${{ github.sha }}" bash -c + --rm -t "ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}" bash -c "pip3 install capstone keystone-engine z3-solver; python3 /panda/panda/python/examples/unicorn/taint_sym_x86_64.py; if [ $? -eq 0 ]; then echo -e 'TEST PASSED!' && exit 0; else echo 'TEST FAILED!' && exit 1; fi" - make_check: - if: github.repository == 'panda-re/panda' - runs-on: self-hosted - needs: [build_container] - - strategy: - matrix: - # See output from `make check-help`: we're just splitting `make check` into all the things it does - # so we can run them in parallel: arch-specific qtests, plus a few others - target: [check-qtest-x86_64, check-qtest-i386, check-qtest-arm, check-qtest-mips, check-qtest-mipsel, check-qtest-ppc, check-block, check-unit, check-qapi-schema] - - steps: - - name: Run Individual QEMU tests - run: >- - docker run --name panda_test_${{ matrix.target }}_${GITHUB_RUN_ID} - -e PANDA_TEST=yes --cap-add SYS_NICE - --rm -t "panda_local_${{ github.sha }}" bash -c - "cd /panda/build && make ${{ matrix.target }}" - - pypanda_tests: - if: github.repository == 'panda-re/panda' - runs-on: self-hosted - needs: [build_container] - - strategy: - matrix: - # See output from `make check-help`: we're just splitting `make check` into all the things it does - # so we can run them in parallel: arch-specific qtests, plus a few others - test_script: [dyn_hooks, copy_test, file_fake, file_hook, generic_tests, monitor_cmds, multi_proc_cbs, sleep_in_cb, syscalls, record_no_snap, sig_suppress] - - steps: - - name: Run individual pypanda tests - # TODO: pip requirements install here should be moved to Docker image build to save test time + - name: Run make Tests + if: matrix.test_type == 'make_check' run: >- docker run --name panda_test_${{ matrix.test_script }}_${GITHUB_RUN_ID} - --mount type=bind,source=/home/panda/regdir/qcows/ubuntu_1604_x86.qcow,target=/root/.panda/ubuntu_1604_x86.qcow -e PANDA_TEST=yes --cap-add SYS_NICE - --rm -t "panda_local_${{ github.sha }}" bash -c - "cd /panda/panda/python/tests/ && make && pip3 install -r requirements.txt && python3 ${{ matrix.test_script }}.py" + --rm -t "ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}" bash -c + "cd /panda/build && make check" cleanup: # Cleanup after prior jobs finish - even if they fail - needs: [taint_tests, sym_trace_tests, make_check, pypanda_tests] - runs-on: self-hosted + needs: [tests] + runs-on: panda-arc if: always() steps: @@ -143,9 +152,9 @@ jobs: docker image prune --all -f --filter "until=72h" docker builder prune -af --filter "until=72h" - build_and_check_fork: # Forked repos can't use self-hosted test suite - just checkout and run make check + build_and_check_fork: # Forked repos can't use panda-arc test suite - just checkout and run make check if: github.repository != 'panda-re/panda' - runs-on: ubuntu-latest + runs-on: panda-arc steps: - uses: actions/checkout@v1 # Clones code into to /home/runner/work/panda diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml index 5fdafc34bc7..1148f73755d 100644 --- a/.github/workflows/publish_docker.yml +++ b/.github/workflows/publish_docker.yml @@ -9,24 +9,27 @@ on: jobs: build_dev: if: github.repository == 'panda-re/panda' && github.ref == 'refs/heads/dev' - runs-on: self-hosted + runs-on: panda-arc steps: - name: Checkout PANDA at current commit uses: actions/checkout@v2 - name: Docker login run: docker login -u pandare -p ${{secrets.pandare_dockerhub}} - - - name: Build Bionic container - # Push both dev and regular container - run: DOCKER_BUILDKIT=1 docker build --progress=plain --target=panda -t pandare/panda:${GITHUB_SHA} $GITHUB_WORKSPACE; - docker tag pandare/panda:${GITHUB_SHA} pandare/panda:latest; - docker push pandare/panda:${GITHUB_SHA}; - docker push pandare/panda; - DOCKER_BUILDKIT=1 docker build --progress=plain --target=developer -t pandare/pandadev:${GITHUB_SHA} $GITHUB_WORKSPACE; - docker tag pandare/pandadev:${GITHUB_SHA} pandare/pandadev:latest; - docker push pandare/pandadev:${GITHUB_SHA}; - docker push pandare/pandadev; + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build panda:latest + uses: docker/build-push-action@v5 + with: + tags: pandare/panda:${GITHUB_SHA} + target: panda + - name: Build pandadev:latest + uses: docker/build-push-action@v5 + with: + tags: pandare/pandadev:${GITHUB_SHA} + target: developer - name: Checkout docs and reset run: rm -rf "${GITHUB_WORKSPACE}/auto_pydoc"; @@ -55,7 +58,7 @@ jobs: build_stable: if: github.repository == 'panda-re/panda' && github.ref == 'refs/heads/stable' - runs-on: self-hosted + runs-on: panda-arc steps: - name: Checkout PANDA at current commit uses: actions/checkout@v1 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 4863a746a46..d2425d5a088 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -7,7 +7,7 @@ on: jobs: stale: - runs-on: ubuntu-latest + runs-on: panda-arc permissions: issues: write pull-requests: write diff --git a/.gitmodules b/.gitmodules index beb2b4fb3b4..aa8c2e6edb3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,36 +1,36 @@ [submodule "roms/vgabios"] path = roms/vgabios - url = https://git.qemu.org/git/vgabios.git + url = https://gitlab.com/qemu-project/vgabios.git [submodule "roms/seabios"] path = roms/seabios - url = https://git.qemu.org/git/seabios.git + url = https://gitlab.com/qemu-project/seabios.git [submodule "roms/SLOF"] path = roms/SLOF - url = https://git.qemu.org/git/SLOF.git + url = https://gitlab.com/qemu-project/SLOF.git [submodule "roms/ipxe"] path = roms/ipxe - url = https://git.qemu.org/git/ipxe.git + url = https://gitlab.com/qemu-project/ipxe.git [submodule "roms/openbios"] path = roms/openbios - url = https://git.qemu.org/git/openbios.git + url = https://gitlab.com/qemu-project/openbios.git [submodule "roms/openhackware"] path = roms/openhackware - url = https://git.qemu.org/git/openhackware.git + url = https://gitlab.com/qemu-project/openhackware.git [submodule "roms/qemu-palcode"] path = roms/qemu-palcode url = https://github.com/rth7680/qemu-palcode.git [submodule "roms/sgabios"] path = roms/sgabios - url = https://git.qemu.org/git/sgabios.git + url = https://gitlab.com/qemu-project/sgabios.git [submodule "pixman"] path = pixman - url = https://anongit.freedesktop.org/git/pixman + url = https://github.com/coolkingcole/pixman.git [submodule "dtc"] path = dtc - url = https://git.qemu.org/git/dtc.git + url = https://github.com/qemu/dtc.git [submodule "roms/u-boot"] path = roms/u-boot - url = https://git.qemu.org/git/u-boot.git + url = https://gitlab.com/qemu-project/u-boot.git [submodule "roms/skiboot"] path = roms/skiboot - url = https://git.qemu.org/git/skiboot.git + url = https://gitlab.com/qemu-project/skiboot.git diff --git a/Dockerfile b/Dockerfile index 86f243d09a2..b933350fa05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,7 +62,9 @@ RUN git -C /panda submodule update --init dtc && \ --prefix=/usr/local \ --disable-numa \ --enable-llvm && \ - (make -C /panda/build -j "$(nproc)" || make) # If multi-core make fails, remake once to give a good error at the end + rm -rf /panda/.git + +RUN make -C /panda/build -j "$(nproc)" #### Develop setup: panda built + pypanda installed (in develop mode) - Stage 3 FROM builder as developer diff --git a/panda/python/core/requirements.txt b/panda/python/core/requirements.txt new file mode 100644 index 00000000000..b88df3026f7 --- /dev/null +++ b/panda/python/core/requirements.txt @@ -0,0 +1,3 @@ +cffi>=1.14.3 +protobuf==3.0.0 +colorama diff --git a/panda/python/tests/run_all_tests.sh b/panda/python/tests/run_all_tests.sh new file mode 100644 index 00000000000..f05a5a0d754 --- /dev/null +++ b/panda/python/tests/run_all_tests.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Define the list of test scripts to run +declare -a tests=("dyn_hooks" "copy_test" "file_fake" "file_hook" "generic_tests" "monitor_cmds" "multi_proc_cbs" "sleep_in_cb" "syscalls" "record_no_snap" "sig_suppress") + +# Base directory for test scripts +TEST_DIR="/panda/panda/python/tests" + +# Iterate over the test scripts array +for test_script in "${tests[@]}"; do + # Construct the full path to the script + full_script_path="$TEST_DIR/${test_script}.py" + + echo "Running $full_script_path..." + python3 "$full_script_path" + + # Check the exit status of the script + if [ $? -ne 0 ]; then + echo "Test $full_script_path failed" + exit 1 + fi +done diff --git a/panda/scripts/install_ubuntu.sh b/panda/scripts/install_ubuntu.sh index 8cad2a52f63..d54c40a4898 100755 --- a/panda/scripts/install_ubuntu.sh +++ b/panda/scripts/install_ubuntu.sh @@ -110,7 +110,7 @@ if [[ !$(ldconfig -p | grep -q libcapstone.so.4) ]]; then echo "Installing libcapstone v4" pushd /tmp && \ curl -o /tmp/cap.tgz -L https://github.com/aquynh/capstone/archive/4.0.2.tar.gz && \ - tar xvf cap.tgz && cd capstone-4.0.2/ && ./make.sh && $SUDO make install && cd /tmp && \ + tar xvf cap.tgz && cd capstone-4.0.2/ && MAKE_JOBS=$(nproc) ./make.sh && $SUDO make install && cd /tmp && \ rm -rf /tmp/capstone-4.0.2 $SUDO ldconfig popd @@ -153,6 +153,7 @@ pushd build progress "PANDA is built and ready to use in panda/build/[arch]-softmmu/panda-system-[arch]." cd ../panda/python/core +$SUDO python3 -m pip install -r requirements.txt $SUDO python3 setup.py install python3 -c "import pandare; panda = pandare.Panda(generic='i386')" # Make sure it worked progress "Pypanda successfully installed"