From 98b9da2b1461abf3b4b6794c5feb98c9cadd697c Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Fri, 15 Dec 2023 11:03:28 -0500 Subject: [PATCH 1/7] Rename dependency filenames to not use colons (fixes #1380) --- Dockerfile | 17 ++++++++++------- ...ntu:18.04_base.txt => ubuntu_18.04_base.txt} | 0 ...u:18.04_build.txt => ubuntu_18.04_build.txt} | 0 ...ntu:20.04_base.txt => ubuntu_20.04_base.txt} | 0 ...u:20.04_build.txt => ubuntu_20.04_build.txt} | 0 ...ntu:22.04_base.txt => ubuntu_22.04_base.txt} | 0 ...u:22.04_build.txt => ubuntu_22.04_build.txt} | 0 panda/python/core/README.md | 2 +- panda/scripts/install_ubuntu.sh | 2 +- 9 files changed, 12 insertions(+), 9 deletions(-) rename panda/dependencies/{ubuntu:18.04_base.txt => ubuntu_18.04_base.txt} (100%) rename panda/dependencies/{ubuntu:18.04_build.txt => ubuntu_18.04_build.txt} (100%) rename panda/dependencies/{ubuntu:20.04_base.txt => ubuntu_20.04_base.txt} (100%) rename panda/dependencies/{ubuntu:20.04_build.txt => ubuntu_20.04_build.txt} (100%) rename panda/dependencies/{ubuntu:22.04_base.txt => ubuntu_22.04_base.txt} (100%) rename panda/dependencies/{ubuntu:22.04_build.txt => ubuntu_22.04_build.txt} (100%) diff --git a/Dockerfile b/Dockerfile index b933350fa05..d666ddda12c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,14 +5,17 @@ ARG TARGET_LIST="x86_64-softmmu,i386-softmmu,arm-softmmu,aarch64-softmmu,ppc-sof FROM $BASE_IMAGE as base ARG BASE_IMAGE -# Copy dependencies lists into container. Note this -# will rarely change so caching should still work well -COPY ./panda/dependencies/${BASE_IMAGE}*.txt /tmp/ +# Copy dependencies lists into container. We copy them all and then do a mv because +# we need to transform base_image into a windows compatible filename which we can't +# do in a COPY command. +COPY ./panda/dependencies/* /tmp +RUN mv /tmp/$(echo "$BASE_IMAGE" | sed 's/:/_/g')_build.txt /tmp/build_dep.txt && \ + mv /tmp/$(echo "$BASE_IMAGE" | sed 's/:/_/g')_base.txt /tmp/base_dep.txt # Base image just needs runtime dependencies -RUN [ -e /tmp/${BASE_IMAGE}_base.txt ] && \ +RUN [ -e /tmp/base_dep.txt ] && \ apt-get -qq update && \ - DEBIAN_FRONTEND=noninteractive apt-get -qq install -y --no-install-recommends curl $(cat /tmp/${BASE_IMAGE}_base.txt | grep -o '^[^#]*') && \ + DEBIAN_FRONTEND=noninteractive apt-get -qq install -y --no-install-recommends curl $(cat /tmp/base_dep.txt | grep -o '^[^#]*') && \ apt-get clean ### BUILD IMAGE - STAGE 2 @@ -20,9 +23,9 @@ FROM base AS builder ARG BASE_IMAGE ARG TARGET_LIST -RUN [ -e /tmp/${BASE_IMAGE}_build.txt ] && \ +RUN [ -e /tmp/build_dep.txt ] && \ apt-get -qq update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $(cat /tmp/${BASE_IMAGE}_build.txt | grep -o '^[^#]*') && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $(cat /tmp/build_dep.txt | grep -o '^[^#]*') && \ apt-get clean && \ python3 -m pip install --upgrade --no-cache-dir pip && \ python3 -m pip install --upgrade --no-cache-dir "cffi>1.14.3" && \ diff --git a/panda/dependencies/ubuntu:18.04_base.txt b/panda/dependencies/ubuntu_18.04_base.txt similarity index 100% rename from panda/dependencies/ubuntu:18.04_base.txt rename to panda/dependencies/ubuntu_18.04_base.txt diff --git a/panda/dependencies/ubuntu:18.04_build.txt b/panda/dependencies/ubuntu_18.04_build.txt similarity index 100% rename from panda/dependencies/ubuntu:18.04_build.txt rename to panda/dependencies/ubuntu_18.04_build.txt diff --git a/panda/dependencies/ubuntu:20.04_base.txt b/panda/dependencies/ubuntu_20.04_base.txt similarity index 100% rename from panda/dependencies/ubuntu:20.04_base.txt rename to panda/dependencies/ubuntu_20.04_base.txt diff --git a/panda/dependencies/ubuntu:20.04_build.txt b/panda/dependencies/ubuntu_20.04_build.txt similarity index 100% rename from panda/dependencies/ubuntu:20.04_build.txt rename to panda/dependencies/ubuntu_20.04_build.txt diff --git a/panda/dependencies/ubuntu:22.04_base.txt b/panda/dependencies/ubuntu_22.04_base.txt similarity index 100% rename from panda/dependencies/ubuntu:22.04_base.txt rename to panda/dependencies/ubuntu_22.04_base.txt diff --git a/panda/dependencies/ubuntu:22.04_build.txt b/panda/dependencies/ubuntu_22.04_build.txt similarity index 100% rename from panda/dependencies/ubuntu:22.04_build.txt rename to panda/dependencies/ubuntu_22.04_build.txt diff --git a/panda/python/core/README.md b/panda/python/core/README.md index 246c7ba169b..7b44ee00084 100644 --- a/panda/python/core/README.md +++ b/panda/python/core/README.md @@ -12,7 +12,7 @@ Installation -- Note the PANDARE package provided on pypi only a binary distribution and will certainly only work on `x86_64` hosts as pre-built PANDA libraries from Ubuntu 20.04 are included. -Prior to using pandare **you must install some dependencies**, these are fully described in the code's dependencies folder, i.e., [the list for ubuntu:20.04 is here](https://github.com/panda-re/panda/blob/dev/panda/dependencies/ubuntu:20.04_base.txt). On our development systems, we typically find we just need to install: +Prior to using pandare **you must install some dependencies**, these are fully described in the code's dependencies folder, i.e., [the list for ubuntu:20.04 is here](https://github.com/panda-re/panda/blob/dev/panda/dependencies/ubuntu_20.04_base.txt). On our development systems, we typically find we just need to install: ``` apt install libvdeplug-dev libpng16-16 libsdl2-2.0-0 ``` diff --git a/panda/scripts/install_ubuntu.sh b/panda/scripts/install_ubuntu.sh index d54c40a4898..164051ed8fc 100755 --- a/panda/scripts/install_ubuntu.sh +++ b/panda/scripts/install_ubuntu.sh @@ -79,7 +79,7 @@ fi # Dependencies are for a major version, but the filenames include minor versions # So take our major version, find the first match in dependencies directory and run with it. # This will give us "./panda/dependencies/ubuntu:20.04" where ubuntu:20.04_build.txt or 20.04_base.txt exists -dep_base=$(find ./panda/dependencies/ubuntu:${version}.* -print -quit | sed -e "s/_build\.txt\|_base\.txt//") +dep_base=$(find ./panda/dependencies/ubuntu_${version}.* -print -quit | sed -e "s/_build\.txt\|_base\.txt//") if [ -e ${dep_base}_build.txt ] || [ -e ${dep_base}_base.txt ]; then echo "Found dependency file(s) at ${dep_base}*.txt" From 0eadd64134322228af83423421a42008d6b72391 Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Thu, 14 Dec 2023 19:10:04 -0500 Subject: [PATCH 2/7] Build debian package with docker --- .dockerignore | 1 + Dockerfile | 6 ++++- panda/debian/.gitignore | 1 + panda/debian/control | 12 ++++++++++ panda/debian/setup.sh | 52 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 panda/debian/.gitignore create mode 100644 panda/debian/control create mode 100755 panda/debian/setup.sh diff --git a/.dockerignore b/.dockerignore index aa1f2ae026d..0c91294c87b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,6 +2,7 @@ build venv local Dockerfile +panda/debian .*sw* .dockerignore .github diff --git a/Dockerfile b/Dockerfile index d666ddda12c..1770c79d8fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -118,6 +118,10 @@ RUN PKG=`pip show pandare | grep Location: | awk '{print $2}'`/pandare/data; \ ### Copy files for panda+pypanda from installer - Stage 5 FROM base as panda +# Include dependency lists for packager +COPY --from=base /tmp/base_dep.txt /tmp +COPY --from=base /tmp/build_dep.txt /tmp + # Copy panda + libcapstone.so* + libosi libraries COPY --from=cleanup /usr/local /usr/local COPY --from=cleanup /usr/lib/libcapstone* /usr/lib/ @@ -134,4 +138,4 @@ ENV PANDA_PATH /usr/local/lib/python3.8/dist-packages/pandare/data RUN ldconfig && \ update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && \ if (ldd /usr/local/lib/python*/dist-packages/pandare/data/*-softmmu/libpanda-*.so | grep 'not found'); then exit 1; fi && \ - if (ldd /usr/local/lib/python*/dist-packages/pandare/data/*-softmmu/panda/plugins/*.so | grep 'not found'); then exit 1; fi + if (ldd /usr/local/lib/python*/dist-packages/pandare/data/*-softmmu/panda/plugins/*.so | grep 'not found'); then exit 1; fi \ No newline at end of file diff --git a/panda/debian/.gitignore b/panda/debian/.gitignore new file mode 100644 index 00000000000..7bbf39a4c32 --- /dev/null +++ b/panda/debian/.gitignore @@ -0,0 +1 @@ +panda.deb diff --git a/panda/debian/control b/panda/debian/control new file mode 100644 index 00000000000..c5457a4457d --- /dev/null +++ b/panda/debian/control @@ -0,0 +1,12 @@ +Package: pandare +Version: 3.1.0 +Architecture: all +BUILD_DEPENDS_LIST +DEPENDS_LIST +Maintainer: Andrew Fasano +Description: dynamic analysis platform + Platform for Architecture Neutral Dynamic Analysis (PANDA) is a processor + emulator designed to support analyses of guest code. PANDA supports record- + and-replay based analyses as well as analyses on live systems. PANDA is forked + from the QEMU emulator. + Panda currently supports i386, x86_64, ARM, MIPS, and PPC. diff --git a/panda/debian/setup.sh b/panda/debian/setup.sh new file mode 100755 index 00000000000..810720bec3d --- /dev/null +++ b/panda/debian/setup.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -eu + +# Function to get the current Ubuntu version +get_ubuntu_version() { + lsb_release -i -s 2>/dev/null +} + +if [[ $# -eq 0 ]]; then + # No argument given, try building a package for current Ubuntu version + + # Check if we're running Ubuntu, exit otherwise + OS=$(get_ubuntu_version) +else + OS=$1 +fi + +if [[ $(get_ubuntu_version) != "Ubuntu" ]]; then + echo "ERROR: OS of $OS is not Ubuntu and unsupported" + exit 1 +fi + +if [[ $# -eq 1 ]]; then + echo "USAGE:" + echo " To build a package for current Ubuntu version:" + echo " $0" + echo " To build a package for a specific OS/version (only Ubuntu supported for now):" + echo " $0 " + exit 1 +fi + +if [[ $# -eq 2 ]]; then + version=$2 + +else + version=$(lsb_release -r | awk '{print $2}') +fi + +# Check if the given version is supported +if [[ ! -f "../dependencies/ubuntu_${version}_base.txt" ]]; then + echo "ERROR: Ubuntu ${version} is not supported, no dependencies file found" + exit 1 +fi + +# First build main panda container for the target ubuntu version +DOCKER_BUILDKIT=1 docker build --target panda -t panda --build-arg BASE_IMAGE="ubuntu:${version}" ../.. + +# Now build the packager container from that +docker build -t packager . + +# Copy deb file out of container to host +docker run --rm -v $(pwd):/out packager bash -c "cp /pandare.deb /out" From a8f910f1a7c9705957faa5a9d00da0a655fa5b82 Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Sat, 16 Dec 2023 23:43:40 -0500 Subject: [PATCH 3/7] callbacks: better resolve path to libpanda and plugin dirs to support debian package --- panda/src/callbacks.c | 73 ++++++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 18 deletions(-) diff --git a/panda/src/callbacks.c b/panda/src/callbacks.c index 73ec90c1bb0..81078650304 100644 --- a/panda/src/callbacks.c +++ b/panda/src/callbacks.c @@ -35,9 +35,13 @@ PANDAENDCOMMENT */ #include "panda/rr/rr_api.h" #include "panda/callbacks/cb-trampolines.h" -#define LIBRARY_DIR "/" TARGET_NAME "-softmmu/libpanda-" TARGET_NAME ".so" +#define SOFTMMU_DIR "/" TARGET_NAME "-softmmu" +#define LIBRARY_NAME "/libpanda-" TARGET_NAME ".so" #define PLUGIN_DIR "/" TARGET_NAME "-softmmu/panda/plugins/" +#define INSTALL_PLUGIN_DIR "/usr/local/lib/panda/" +#define INSTALL_BIN_DIR "/usr/local/bin/" // libpanda-arch.so and panda-system-arch in here + const gchar *panda_bool_true_strings[] = {"y", "yes", "true", "1", NULL}; const gchar *panda_bool_false_strings[] = {"n", "no", "false", "0", NULL}; @@ -121,31 +125,54 @@ static void *try_open_libpanda(const char *panda_lib) { // When running as a library, load libpanda static bool load_libpanda(void) { - - const char *panda_lib = g_getenv("PANDA_LIB"); + const char *panda_lib = g_getenv("PANDA_LIB"); // Direct path to libpanda + const char *lib_dir = g_getenv("PANDA_DIR"); // Path to directory containing libpanda void *libpanda; + // We'll search through paths for libpanda. As soon as we find a valid path, we'll call try_open_libpanda on it + // First: if we have PANDA_LIB set, try that. + // Next, if we have LIB_DIR set, try to load from there. + // Next, try loading from standard /usr/local/bin/libpanda-arch.so + // Finally try the hacky dlopen code that will be removed soon + + // Try PANDA_LIB + if (panda_lib != NULL) { + if (g_file_test(panda_lib, G_FILE_TEST_EXISTS)) { + libpanda = try_open_libpanda(panda_lib); + return libpanda != NULL; + } + } - if(panda_lib != NULL) { - libpanda = try_open_libpanda(panda_lib); - } else { -#ifndef LIBRARY_DIR - assert(0 && "LIBRARY_DIR undefined, but library mode is enabled - Unsupported architecture?"); - LOG_ERROR(PANDA_MSG_FMT "LIBRARY_DIR not defined\n", PANDA_CORE_NAME); -#endif - const char *lib_dir = g_getenv("PANDA_DIR"); - - if (lib_dir != NULL) { - panda_lib = g_strdup_printf("%s%s", lib_dir, LIBRARY_DIR); - } else { - LOG_WARNING(PANDA_MSG_FMT "WARNING: using hacky dlopen code that will be removed soon\n", PANDA_CORE_NAME); - panda_lib = g_strdup_printf("../../../build/%s", LIBRARY_DIR); // XXX This is bad, need a less hardcoded path + // Try relative to PANDA_DIR + if (lib_dir != NULL) { + panda_lib = g_strdup_printf("%s%s%s", lib_dir, SOFTMMU_DIR, LIBRARY_NAME); + if (g_file_test(panda_lib, G_FILE_TEST_EXISTS)) { + libpanda = try_open_libpanda(panda_lib); + g_free((char *)panda_lib); + return libpanda != NULL; } + g_free((char *)panda_lib); + } + // Try standard install location + panda_lib = g_strdup_printf("%s%s", INSTALL_BIN_DIR, LIBRARY_NAME); + if (g_file_test(panda_lib, G_FILE_TEST_EXISTS)) { libpanda = try_open_libpanda(panda_lib); g_free((char *)panda_lib); + return libpanda != NULL; } + g_free((char *)panda_lib); - return libpanda != NULL; + // XXX terrible hack: relative path to build directory from the binary(?) + panda_lib = g_strdup_printf("../../../build/%s%s", SOFTMMU_DIR, LIBRARY_NAME); + if (g_file_test(panda_lib, G_FILE_TEST_EXISTS)) { + LOG_WARNING(PANDA_MSG_FMT "WARNING: using hacky dlopen code that will be removed soon\n", PANDA_CORE_NAME); + libpanda = try_open_libpanda(panda_lib); + g_free((char *)panda_lib); + return libpanda != NULL; + } + g_free((char *)panda_lib); + + return false; } // Internal: remove a plugin from the global array panda_plugins @@ -287,6 +314,7 @@ extern const char *qemu_file; // // - Relative to the PANDA_DIR environment variable. // - Relative to the QEMU binary +// - Relative to the standard install location (/usr/local/lib/panda/[arch]/) // - Relative to the install prefix directory. char* resolve_file_from_plugin_directory(const char* file_name_fmt, const char* name){ char *plugin_path, *name_formatted; @@ -320,6 +348,15 @@ char* resolve_file_from_plugin_directory(const char* file_name_fmt, const char* } g_free(plugin_path); + + // Third, check relative to the standard install location. + plugin_path = attempt_normalize_path( + g_strdup_printf("%s/%s/%s", INSTALL_PLUGIN_DIR, + TARGET_NAME, name_formatted)); + if (TRUE == g_file_test(plugin_path, G_FILE_TEST_EXISTS)) { + return plugin_path; + } + // Finally, try relative to the installation path. plugin_path = attempt_normalize_path( g_strdup_printf("%s/%s/%s", CONFIG_PANDA_PLUGINDIR, From 8c413c26086a1a9a373d2a007bde23512b82ad87 Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Sat, 16 Dec 2023 16:23:27 -0500 Subject: [PATCH 4/7] PyPANDA: update setup scripts to use debian package instead of self-installing --- panda/python/core/pandare/panda.py | 11 ++++- panda/python/core/pandare/utils.py | 10 +++- panda/python/core/setup.py | 73 ++++++++---------------------- 3 files changed, 37 insertions(+), 57 deletions(-) diff --git a/panda/python/core/pandare/panda.py b/panda/python/core/pandare/panda.py index 63af77cf5f6..3809f3409e9 100644 --- a/panda/python/core/pandare/panda.py +++ b/panda/python/core/pandare/panda.py @@ -157,7 +157,16 @@ def __init__(self, arch="i386", mem="128M", if libpanda_path: environ["PANDA_LIB"] = self.libpanda_path = libpanda_path else: - self.libpanda_path = pjoin(self.get_build_dir(), "{0}-softmmu/libpanda-{0}.so".format(self.arch_name)) + build_dir = self.get_build_dir() + lib_paths = ["libpanda-{0}.so".format(self.arch_name), "{0}-softmmu/libpanda-{0}.so".format(self.arch_name)] + # Select the first path that exists - we'll have libpanda-{arch}.so for a system install versus arch-softmmu/libpanda-arch.so for a build + for p in lib_paths: + if isfile(pjoin(build_dir, p)): + self.libpanda_path = pjoin(build_dir, p) + break + else: + raise RuntimeError("Couldn't find libpanda-{0}.so in {1} (in either root or {0}-libpanda directory)".format(self.arch_name, build_dir)) + self.panda = self.libpanda_path # Necessary for realpath to work inside core-panda, may cause issues? self.ffi = self._do_types_import() diff --git a/panda/python/core/pandare/utils.py b/panda/python/core/pandare/utils.py index dda2172415b..c3978b621fb 100644 --- a/panda/python/core/pandare/utils.py +++ b/panda/python/core/pandare/utils.py @@ -179,12 +179,20 @@ def _find_build_dir(arch_name, find_executable=False): ''' Internal function to return the build directory for the specified architecture ''' + + system_build = "/usr/local/bin/" python_package = pjoin(*[dirname(__file__), "data"]) local_build = realpath(pjoin(dirname(__file__), "../../../../build")) + + arch_dir = f"{arch_name}-softmmu" file_name = f"panda-system-{arch_name}" if find_executable else \ f"libpanda-{arch_name}.so" - pot_paths = [pjoin(python_package, arch_dir), pjoin(local_build, arch_dir)] + + # system path could have panda-system-X or libpanda-X.so. Others would have an arch_name - softmmu directory + pot_paths = [system_build, + pjoin(python_package, arch_dir), + pjoin(local_build, arch_dir)] if find_executable and 'PATH' in environ: # If we're looking for the panda executable, also search the user's path diff --git a/panda/python/core/setup.py b/panda/python/core/setup.py index 8e0945891ca..10607ac46a2 100644 --- a/panda/python/core/setup.py +++ b/panda/python/core/setup.py @@ -1,19 +1,18 @@ #!/usr/bin/env python -# Install with python setup.py (develop|install) from setuptools import setup, find_packages from setuptools.command.install import install as install_orig from setuptools.command.develop import develop as develop_orig from setuptools.dist import Distribution from subprocess import check_output +import shutil import os -import shutil -################################################ -# 1) Copy panda object files: libpanda-XYZ.so, # -# pc-bios/*, all .so files for plugins, # -# pypanda's include directory, llvm-helpers # -################################################ +arches = ['arm', 'aarch64', 'i386', 'x86_64', 'ppc', 'mips', 'mipsel', 'mips64'] + +# Check for PANDA binaries in /usr/local/bin/ or in our build directory +#panda_binaries = ['/usr/local/bin/panda-system-{arch}' for arch in arches] +# Do we actually care at this point? root_dir = os.path.join(*[os.path.dirname(__file__), "..", "..", ".."]) # panda-git/ root dir @@ -28,16 +27,9 @@ def copy_objs(): build_root = os.path.join(root_dir, "build") if os.path.isdir(lib_dir): - assert('panda' in lib_dir), "Refusing to rm -rf directory without 'panda' in it" shutil.rmtree(lib_dir) os.mkdir(lib_dir) - # Copy bios directory - biosdir = os.path.join(root_dir, "pc-bios") - if not os.path.isdir(biosdir): - raise RuntimeError(f"Could not find PC-bios directory at {biosdir}") - shutil.copytree(biosdir, lib_dir+"/pc-bios") - # Copy pypanda's include directory (different than core panda's) into a datadir pypanda_inc = os.path.join(*[root_dir, "panda", "python", "core", "pandare", "include"]) if not os.path.isdir(pypanda_inc): @@ -47,21 +39,18 @@ def copy_objs(): shutil.rmtree(pypanda_inc_dest) shutil.copytree(pypanda_inc, pypanda_inc_dest) + # For each arch, copy llvm-helpers + # XXX Should these be in standard panda deb? + # What actually uses these? Taint? Disabling for now + ''' # Check if we have llvm-support with open(os.path.join(*[build_root, 'config-host.mak']), 'r') as cfg: llvm_enabled = True if 'CONFIG_LLVM=y' in cfg.read() else False - # For each arch, copy library, plugins, plog_pb2.py and llvm-helpers - arches = ['arm', 'aarch64', 'i386', 'x86_64', 'ppc', 'mips', 'mipsel', 'mips64'] - if pypi_build: - # Nobody really wants mips32 anymore, shrink our distribution size by dropping - arches = ['arm', 'aarch64', 'i386', 'x86_64', 'ppc', 'mips64'] - for arch in arches: libname = "libpanda-"+arch+".so" softmmu = arch+"-softmmu" path = os.path.join(*[build_root, softmmu, libname]) - plugindir = os.path.join(*[build_root, softmmu, "panda", "plugins"]) llvm1 = os.path.join(*[build_root, softmmu, "llvm-helpers.bc1"]) llvm2 = os.path.join(*[build_root, softmmu, f"llvm-helpers-{arch}.bc"]) @@ -71,25 +60,10 @@ def copy_objs(): continue os.mkdir(os.path.join(lib_dir, softmmu)) - - new_plugindir = os.path.join(lib_dir, softmmu, "panda/plugins") - os.mkdir(os.path.dirname(new_plugindir)) # When we copy the whole tree, it will make the plugins directory - - shutil.copy( path, os.path.join(lib_dir, softmmu)) if llvm_enabled: shutil.copy( llvm1, os.path.join(lib_dir, softmmu)) shutil.copy( llvm2, os.path.join(lib_dir, softmmu)) - - shutil.copytree(plugindir, new_plugindir, ignore=shutil.ignore_patterns('*.o', '*.d')) - - # Strip libpandas and plugins to save space (Need <100mb for pypi) - if pypi_build: - check_output(f"find {lib_dir} -type f -executable -exec strip {{}} \;", shell=True) - - -######################### -# 3) Build the package # -######################### + ''' from setuptools.command.install import install as install_orig from setuptools.command.develop import develop as develop_orig @@ -100,11 +74,6 @@ class custom_develop(develop_orig): 2) Running regular setup tools logic ''' def run(self): - # Delete pandare/data in the case of `setup.py develop` - # Don't copy objects, use them in the current path - if os.path.isdir(lib_dir): - assert('panda' in lib_dir), "Refusing to rm -rf directory without 'panda' in it" - shutil.rmtree(lib_dir) from create_panda_datatypes import main as create_datatypes create_datatypes(install=False) super().run() @@ -114,18 +83,16 @@ class custom_install(install_orig): We're going to install to the system. Two possible states to handle 1) Running from within the panda repo with panda built - need to create_datatypes 2) Running from a python sdist where all the files are already prepared - - Install to the system by: - 1) Creating datatype files for an install - 2) Copying objects into local module - 3) Running regular setup tools logic ''' def run(self): try: from create_panda_datatypes import main as create_datatypes + # If we can do the import, we're in the panda repo create_datatypes(install=True) copy_objs() + except ImportError: + # Import failed, we're either in a python sdist or something has gone very wrong assert(os.path.isfile("pandare/include/panda_datatypes.h")), \ "panda_datatypes.h missing and can't be generated" assert(os.path.isfile("pandare/autogen/panda_datatypes.py")), \ @@ -140,7 +107,7 @@ def run(self): long_description = fh.read() setup(name='pandare', - version='0.1.1.6', + version='0.1.2.0', description='Python Interface to PANDA', long_description=long_description, long_description_content_type="text/markdown", @@ -148,14 +115,10 @@ def run(self): author_email='fasano@mit.edu', url='https://github.com/panda-re/panda/', packages=find_packages(), - package_data = { 'pandare': ['data/**/*', # Copy everything (fails?) - 'data/*-softmmu/libpanda-*.so', # Libpandas - 'data/*-softmmu/llvm-helpers*.bc*', # Llvm-helpers - 'data/*-softmmu/panda/plugins/*', # All plugins - 'data/*-softmmu/panda/plugins/**/*',# All plugin files + package_data = { 'pandare': [ \ + 'data/*-softmmu/llvm-helpers*.bc*', # LLVM Helpers + 'data/pypanda/include/*.h', # Includes files 'data/pypanda/include/*.h', # Includes files - 'data/pc-bios/*', # BIOSes - 'data/pc-bios/**/*', # Keymaps 'qcows.json' # Generic Images ]}, install_requires=[ 'cffi>=1.14.3', 'colorama'], From 7dce13ce448717dff94b7940a697fb506c5a6503 Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Sat, 16 Dec 2023 16:51:03 -0500 Subject: [PATCH 5/7] PyPANDA: require protobuf >= 4.25.1 because we don't work with (much) older versions --- panda/python/core/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/panda/python/core/setup.py b/panda/python/core/setup.py index 10607ac46a2..c0fa18b7055 100644 --- a/panda/python/core/setup.py +++ b/panda/python/core/setup.py @@ -121,7 +121,7 @@ def run(self): 'data/pypanda/include/*.h', # Includes files 'qcows.json' # Generic Images ]}, - install_requires=[ 'cffi>=1.14.3', 'colorama'], + install_requires=[ 'cffi>=1.14.3', 'colorama', 'protobuf>=4.25.1'], python_requires='>=3.6', cmdclass={'install': custom_install, 'develop': custom_develop}, ) From 6d95bb1e380227721b905c5614857d03691ae5d3 Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Sat, 16 Dec 2023 23:43:57 -0500 Subject: [PATCH 6/7] PyPANDA: better resolve plugin paths to support debian package --- panda/python/core/pandare/panda.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/panda/python/core/pandare/panda.py b/panda/python/core/pandare/panda.py index 3809f3409e9..fc73fe677ba 100644 --- a/panda/python/core/pandare/panda.py +++ b/panda/python/core/pandare/panda.py @@ -18,7 +18,7 @@ # seems to work with python's but not vice-versa. This allows for # stdio interactions later (e.g., pdb, input()) without segfaults -from os.path import realpath, exists, abspath, isfile, dirname, join as pjoin +from os.path import realpath, exists, abspath, isfile, dirname, isdir, join as pjoin from os import dup, getenv, environ, path from random import randint from inspect import signature @@ -261,7 +261,16 @@ def __init__(self, arch="i386", mem="128M", def get_plugin_path(self): if self.plugin_path is None: - self.plugin_path = pjoin(*[self.get_build_dir(), self.arch_name+"-softmmu", "panda", "plugins"]) + build_dir = self.get_build_dir() + rel_dir = pjoin(*[build_dir, self.arch_name+"-softmmu", "panda", "plugins"]) + + if build_dir == "/usr/local/bin/": + # Installed - use /usr/local/lib/panda/plugins + self.plugin_path = f"/usr/local/lib/panda/{self.arch_name}" + elif isdir(rel_dir): + self.plugin_path = rel_dir + else: + raise ValueError(f"Could not find plugin path. Build dir={build_dir}") return self.plugin_path def get_build_dir(self): From 21f1c8ebc8042dd7ba761a48777241c5f951f5db Mon Sep 17 00:00:00 2001 From: Andrew Fasano Date: Tue, 19 Dec 2023 11:17:18 -0500 Subject: [PATCH 7/7] PyPANDA bugfix: fix typo for shim when plog reader is unavilable --- panda/python/core/pandare/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/panda/python/core/pandare/__init__.py b/panda/python/core/pandare/__init__.py index 3dfbd4a171b..bef65cbc94b 100644 --- a/panda/python/core/pandare/__init__.py +++ b/panda/python/core/pandare/__init__.py @@ -21,7 +21,7 @@ from .plog_reader import PLogReader except ImportError: # plog reader isn't widely used - let's make this optional - PlogReader = None + PLogReader = None __all__ = ['Panda', 'PLogReader', 'Callbacks', 'PyPlugin']