Support building debian packages for PANDA and pip packages for PyPANDA

.dockerignore

@@ -2,6 +2,7 @@ build
 venv
 local
 Dockerfile
+panda/debian
 .*sw*
 .dockerignore
 .github

Dockerfile

@@ -5,24 +5,27 @@ ARG TARGET_LIST="x86_64-softmmu,i386-softmmu,arm-softmmu,aarch64-softmmu,ppc-sof
 FROM $BASE_IMAGE as base
 ARG BASE_IMAGE
 
-# Copy dependencies lists into container. Note this
-#  will rarely change so caching should still work well
-COPY ./panda/dependencies/${BASE_IMAGE}*.txt /tmp/ 
+# Copy dependencies lists into container. We copy them all and then do a mv because
+# we need to transform base_image into a windows compatible filename which we can't
+# do in a COPY command.
+COPY ./panda/dependencies/* /tmp
+RUN mv /tmp/$(echo "$BASE_IMAGE" | sed 's/:/_/g')_build.txt /tmp/build_dep.txt && \
+    mv /tmp/$(echo "$BASE_IMAGE" | sed 's/:/_/g')_base.txt /tmp/base_dep.txt
 
 # Base image just needs runtime dependencies
-RUN [ -e /tmp/${BASE_IMAGE}_base.txt ] && \
+RUN [ -e /tmp/base_dep.txt ] && \
     apt-get -qq update && \
-    DEBIAN_FRONTEND=noninteractive apt-get -qq install -y --no-install-recommends curl $(cat /tmp/${BASE_IMAGE}_base.txt | grep -o '^[^#]*') && \
+    DEBIAN_FRONTEND=noninteractive apt-get -qq install -y --no-install-recommends curl $(cat /tmp/base_dep.txt | grep -o '^[^#]*') && \
     apt-get clean
 
 ### BUILD IMAGE - STAGE 2
 FROM base AS builder
 ARG BASE_IMAGE
 ARG TARGET_LIST
 
-RUN [ -e /tmp/${BASE_IMAGE}_build.txt ] && \
+RUN [ -e /tmp/build_dep.txt ] && \
     apt-get -qq update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $(cat /tmp/${BASE_IMAGE}_build.txt | grep -o '^[^#]*') && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $(cat /tmp/build_dep.txt | grep -o '^[^#]*') && \
     apt-get clean && \
     python3 -m pip install --upgrade --no-cache-dir pip && \
     python3 -m pip install --upgrade --no-cache-dir "cffi>1.14.3" && \
@@ -115,6 +118,10 @@ RUN PKG=`pip show pandare | grep Location: | awk '{print $2}'`/pandare/data; \
 ### Copy files for panda+pypanda from installer  - Stage 5
 FROM base as panda
 
+# Include dependency lists for packager
+COPY --from=base /tmp/base_dep.txt /tmp
+COPY --from=base /tmp/build_dep.txt /tmp
+
 # Copy panda + libcapstone.so* + libosi libraries
 COPY --from=cleanup /usr/local /usr/local
 COPY --from=cleanup /usr/lib/libcapstone* /usr/lib/
@@ -131,4 +138,4 @@ ENV PANDA_PATH /usr/local/lib/python3.8/dist-packages/pandare/data
 RUN ldconfig && \
     update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && \
     if (ldd /usr/local/lib/python*/dist-packages/pandare/data/*-softmmu/libpanda-*.so | grep 'not found'); then exit 1; fi && \
-    if (ldd /usr/local/lib/python*/dist-packages/pandare/data/*-softmmu/panda/plugins/*.so | grep 'not found'); then exit 1; fi
+    if (ldd /usr/local/lib/python*/dist-packages/pandare/data/*-softmmu/panda/plugins/*.so | grep 'not found'); then exit 1; fi

panda/debian/.gitignore

@@ -0,0 +1 @@
+panda.deb

panda/debian/control

@@ -0,0 +1,12 @@
+Package: pandare
+Version: 3.1.0
+Architecture: all
+BUILD_DEPENDS_LIST
+DEPENDS_LIST
+Maintainer: Andrew Fasano <[email protected]>
+Description: dynamic analysis platform
+ Platform for Architecture Neutral Dynamic Analysis (PANDA) is a processor
+ emulator designed to support analyses of guest code. PANDA supports record-
+ and-replay based analyses as well as analyses on live systems. PANDA is forked
+ from the QEMU emulator.
+ Panda currently supports i386, x86_64, ARM, MIPS, and PPC.

panda/debian/setup.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -eu
+
+# Function to get the current Ubuntu version
+get_ubuntu_version() {
+    lsb_release -i -s 2>/dev/null
+}
+
+if [[ $# -eq 0 ]]; then
+	# No argument given, try building a package for current Ubuntu version
+
+	# Check if we're running Ubuntu, exit otherwise
+	OS=$(get_ubuntu_version)
+else
+	OS=$1
+fi
+
+if [[ $(get_ubuntu_version) != "Ubuntu" ]]; then
+	echo "ERROR: OS of $OS is not Ubuntu and unsupported"
+	exit 1
+fi
+
+if [[ $# -eq 1 ]]; then
+	echo "USAGE:"
+	echo "	To build a package for current Ubuntu version:"
+	echo "	  $0"
+	echo "	To build a package for a specific OS/version (only Ubuntu supported for now):"
+	echo "	  $0 <OS> <version>"
+	exit 1
+fi
+
+if [[ $# -eq 2 ]]; then
+	version=$2
+
+else
+	version=$(lsb_release -r | awk '{print $2}')
+fi
+
+# Check if the given version is supported
+if [[ ! -f "../dependencies/ubuntu_${version}_base.txt" ]]; then
+	echo "ERROR: Ubuntu ${version} is not supported, no dependencies file found"
+	exit 1
+fi
+
+# First build main panda container for the target ubuntu version
+DOCKER_BUILDKIT=1 docker build --target panda -t panda --build-arg BASE_IMAGE="ubuntu:${version}" ../.. 
+
+# Now build the packager container from that
+docker build -t packager .
+
+# Copy deb file out of container to host
+docker run --rm -v $(pwd):/out packager bash -c "cp /pandare.deb /out"

panda/python/core/README.md

@@ -12,7 +12,7 @@ Installation
 --
 Note the PANDARE package provided on pypi only a binary distribution and will certainly only work on `x86_64` hosts as pre-built PANDA libraries from Ubuntu 20.04 are included.
 
-Prior to using pandare **you must install some dependencies**, these are fully described in the code's dependencies folder, i.e., [the list for ubuntu:20.04 is here](https://github.com/panda-re/panda/blob/dev/panda/dependencies/ubuntu:20.04_base.txt). On our development systems, we typically find we just need to install:
+Prior to using pandare **you must install some dependencies**, these are fully described in the code's dependencies folder, i.e., [the list for ubuntu:20.04 is here](https://github.com/panda-re/panda/blob/dev/panda/dependencies/ubuntu_20.04_base.txt). On our development systems, we typically find we just need to install:
 ```
 apt install libvdeplug-dev libpng16-16 libsdl2-2.0-0
 ```

panda/python/core/pandare/__init__.py

@@ -21,7 +21,7 @@
     from .plog_reader import PLogReader
 except ImportError:
     # plog reader isn't widely used - let's make this optional
-    PlogReader = None
+    PLogReader = None
 
 __all__ = ['Panda', 'PLogReader', 'Callbacks', 'PyPlugin']
 

panda/python/core/pandare/panda.py

@@ -18,7 +18,7 @@
 # seems to work with python's but not vice-versa. This allows for
 # stdio interactions later (e.g., pdb, input())  without segfaults
 
-from os.path import realpath, exists, abspath, isfile, dirname, join as pjoin
+from os.path import realpath, exists, abspath, isfile, dirname, isdir, join as pjoin
 from os import dup, getenv, environ, path
 from random import randint
 from inspect import signature
@@ -157,7 +157,16 @@ def __init__(self, arch="i386", mem="128M",
         if libpanda_path:
             environ["PANDA_LIB"] = self.libpanda_path = libpanda_path
         else:
-            self.libpanda_path = pjoin(self.get_build_dir(), "{0}-softmmu/libpanda-{0}.so".format(self.arch_name))
+            build_dir = self.get_build_dir()
+            lib_paths = ["libpanda-{0}.so".format(self.arch_name), "{0}-softmmu/libpanda-{0}.so".format(self.arch_name)]
+            # Select the first path that exists - we'll have libpanda-{arch}.so for a system install versus arch-softmmu/libpanda-arch.so for a build
+            for p in lib_paths:
+                if isfile(pjoin(build_dir, p)):
+                    self.libpanda_path = pjoin(build_dir, p)
+                    break
+            else:
+                raise RuntimeError("Couldn't find libpanda-{0}.so in {1} (in either root or {0}-libpanda directory)".format(self.arch_name, build_dir))
+
         self.panda = self.libpanda_path # Necessary for realpath to work inside core-panda, may cause issues?
 
         self.ffi = self._do_types_import()
@@ -252,7 +261,16 @@ def __init__(self, arch="i386", mem="128M",
 
     def get_plugin_path(self):
         if self.plugin_path is None:
-            self.plugin_path = pjoin(*[self.get_build_dir(), self.arch_name+"-softmmu", "panda", "plugins"])
+            build_dir = self.get_build_dir()
+            rel_dir = pjoin(*[build_dir, self.arch_name+"-softmmu", "panda", "plugins"])
+
+            if build_dir == "/usr/local/bin/":
+                # Installed - use /usr/local/lib/panda/plugins
+                self.plugin_path = f"/usr/local/lib/panda/{self.arch_name}"
+            elif isdir(rel_dir):
+                self.plugin_path = rel_dir
+            else:
+                raise ValueError(f"Could not find plugin path. Build dir={build_dir}")
         return self.plugin_path
 
     def get_build_dir(self):

panda/python/core/pandare/utils.py

@@ -179,12 +179,20 @@ def _find_build_dir(arch_name, find_executable=False):
     '''
     Internal function to return the build directory for the specified architecture
     '''
+
+    system_build = "/usr/local/bin/"
     python_package = pjoin(*[dirname(__file__), "data"])
     local_build = realpath(pjoin(dirname(__file__), "../../../../build"))
+
+
     arch_dir = f"{arch_name}-softmmu"
     file_name = f"panda-system-{arch_name}" if find_executable else \
                 f"libpanda-{arch_name}.so"
-    pot_paths = [pjoin(python_package, arch_dir), pjoin(local_build, arch_dir)]
+
+    # system path could have panda-system-X or libpanda-X.so. Others would have an arch_name - softmmu directory
+    pot_paths = [system_build,
+                 pjoin(python_package, arch_dir),
+                 pjoin(local_build, arch_dir)]
 
     if find_executable and 'PATH' in environ:
         # If we're looking for the panda executable, also search the user's path

panda/python/core/setup.py

@@ -1,19 +1,18 @@
 #!/usr/bin/env python
-# Install with python setup.py (develop|install)
 
 from setuptools import setup, find_packages
 from setuptools.command.install import install as install_orig
 from setuptools.command.develop import develop as develop_orig
 from setuptools.dist import Distribution
 from subprocess import check_output
+import shutil
 
 import os
-import shutil
-################################################
-# 1) Copy panda object files: libpanda-XYZ.so, #
-#    pc-bios/*, all .so files for plugins,     #
-#    pypanda's include directory, llvm-helpers #
-################################################
+arches = ['arm', 'aarch64', 'i386', 'x86_64', 'ppc', 'mips', 'mipsel', 'mips64']
+
+# Check for PANDA binaries in /usr/local/bin/ or in our build directory
+#panda_binaries = ['/usr/local/bin/panda-system-{arch}' for arch in arches]
+# Do we actually care at this point?
 
 root_dir = os.path.join(*[os.path.dirname(__file__), "..", "..", ".."]) # panda-git/ root dir
 
@@ -28,16 +27,9 @@ def copy_objs():
     build_root = os.path.join(root_dir, "build")
 
     if os.path.isdir(lib_dir):
-        assert('panda' in lib_dir), "Refusing to rm -rf directory without 'panda' in it"
         shutil.rmtree(lib_dir)
     os.mkdir(lib_dir)
 
-    # Copy bios directory
-    biosdir = os.path.join(root_dir, "pc-bios")
-    if not os.path.isdir(biosdir):
-        raise RuntimeError(f"Could not find PC-bios directory at {biosdir}")
-    shutil.copytree(biosdir, lib_dir+"/pc-bios")
-
     # Copy pypanda's include directory (different than core panda's) into a datadir
     pypanda_inc = os.path.join(*[root_dir, "panda", "python", "core", "pandare", "include"])
     if not os.path.isdir(pypanda_inc):
@@ -47,21 +39,18 @@ def copy_objs():
         shutil.rmtree(pypanda_inc_dest)
     shutil.copytree(pypanda_inc, pypanda_inc_dest)
 
+    # For each arch, copy llvm-helpers
+    # XXX Should these be in standard panda deb?
+    # What actually uses these? Taint? Disabling for now
+    '''
     # Check if we have llvm-support
     with open(os.path.join(*[build_root, 'config-host.mak']), 'r') as cfg:
         llvm_enabled = True if 'CONFIG_LLVM=y' in cfg.read() else False
 
-    # For each arch, copy library, plugins, plog_pb2.py and llvm-helpers
-    arches = ['arm', 'aarch64', 'i386', 'x86_64', 'ppc', 'mips', 'mipsel', 'mips64']
-    if pypi_build:
-        # Nobody really wants mips32 anymore, shrink our distribution size by dropping
-        arches = ['arm', 'aarch64', 'i386', 'x86_64', 'ppc', 'mips64']
-
     for arch in arches:
         libname = "libpanda-"+arch+".so"
         softmmu = arch+"-softmmu"
         path      = os.path.join(*[build_root, softmmu, libname])
-        plugindir = os.path.join(*[build_root, softmmu, "panda", "plugins"])
         llvm1     = os.path.join(*[build_root, softmmu, "llvm-helpers.bc1"])
         llvm2     = os.path.join(*[build_root, softmmu, f"llvm-helpers-{arch}.bc"])
 
@@ -71,25 +60,10 @@ def copy_objs():
             continue
 
         os.mkdir(os.path.join(lib_dir, softmmu))
-
-        new_plugindir = os.path.join(lib_dir, softmmu, "panda/plugins")
-        os.mkdir(os.path.dirname(new_plugindir)) # When we copy the whole tree, it will make the plugins directory
-
-        shutil.copy(        path,       os.path.join(lib_dir, softmmu))
         if llvm_enabled:
             shutil.copy(    llvm1,      os.path.join(lib_dir, softmmu))
             shutil.copy(    llvm2,      os.path.join(lib_dir, softmmu))
-
-        shutil.copytree(plugindir,  new_plugindir, ignore=shutil.ignore_patterns('*.o', '*.d'))
-
-    # Strip libpandas and plugins to save space (Need <100mb for pypi)
-    if pypi_build:
-        check_output(f"find {lib_dir} -type f -executable -exec strip {{}} \;", shell=True)
-
-
-#########################
-# 3)  Build the package #
-#########################
+    '''
 
 from setuptools.command.install import install as install_orig
 from setuptools.command.develop import develop as develop_orig
@@ -100,11 +74,6 @@ class custom_develop(develop_orig):
         2) Running regular setup tools logic
     '''
     def run(self):
-        # Delete pandare/data in the case of `setup.py develop`
-        # Don't copy objects, use them in the current path
-        if os.path.isdir(lib_dir):
-            assert('panda' in lib_dir), "Refusing to rm -rf directory without 'panda' in it"
-            shutil.rmtree(lib_dir)
         from create_panda_datatypes import main as create_datatypes
         create_datatypes(install=False)
         super().run()
@@ -114,18 +83,16 @@ class custom_install(install_orig):
     We're going to install to the system. Two possible states to handle
     1) Running from within the panda repo with panda built - need to create_datatypes
     2) Running from a python sdist where all the files are already prepared
-
-    Install to the system by:
-        1) Creating datatype files for an install
-        2) Copying objects into local module
-        3) Running regular setup tools logic
     '''
     def run(self):
         try:
             from create_panda_datatypes import main as create_datatypes
+            # If we can do the import, we're in the panda repo
             create_datatypes(install=True)
             copy_objs()
+
         except ImportError:
+            # Import failed, we're either in a python sdist or something has gone very wrong
             assert(os.path.isfile("pandare/include/panda_datatypes.h")), \
                             "panda_datatypes.h missing and can't be generated"
             assert(os.path.isfile("pandare/autogen/panda_datatypes.py")), \
@@ -140,25 +107,21 @@ def run(self):
     long_description = fh.read()
 
 setup(name='pandare',
-      version='0.1.1.6',
+      version='0.1.2.0',
       description='Python Interface to PANDA',
       long_description=long_description,
       long_description_content_type="text/markdown",
       author='Andrew Fasano, Luke Craig, and Tim Leek',
       author_email='[email protected]',
       url='https://github.com/panda-re/panda/',
       packages=find_packages(),
-      package_data = { 'pandare': ['data/**/*', # Copy everything (fails?)
-          'data/*-softmmu/libpanda-*.so',     # Libpandas
-          'data/*-softmmu/llvm-helpers*.bc*', # Llvm-helpers
-          'data/*-softmmu/panda/plugins/*',   # All plugins
-          'data/*-softmmu/panda/plugins/**/*',# All plugin files
+      package_data = { 'pandare': [ \
+          'data/*-softmmu/llvm-helpers*.bc*', # LLVM Helpers
+          'data/pypanda/include/*.h',         # Includes files
           'data/pypanda/include/*.h',         # Includes files
-          'data/pc-bios/*',                   # BIOSes
-          'data/pc-bios/**/*',                # Keymaps
           'qcows.json'                        # Generic Images
           ]},
-      install_requires=[ 'cffi>=1.14.3', 'colorama'],
+      install_requires=[ 'cffi>=1.14.3', 'colorama', 'protobuf>=4.25.1'],
       python_requires='>=3.6',
       cmdclass={'install': custom_install, 'develop': custom_develop},
      )

panda/scripts/install_ubuntu.sh

@@ -79,7 +79,7 @@ fi
 # Dependencies are for a major version, but the filenames include minor versions
 # So take our major version, find the first match in dependencies directory and run with it.
 # This will give us "./panda/dependencies/ubuntu:20.04" where ubuntu:20.04_build.txt or 20.04_base.txt exists
-dep_base=$(find ./panda/dependencies/ubuntu:${version}.* -print -quit | sed  -e "s/_build\.txt\|_base\.txt//")
+dep_base=$(find ./panda/dependencies/ubuntu_${version}.* -print -quit | sed  -e "s/_build\.txt\|_base\.txt//")
 
 if [ -e ${dep_base}_build.txt ] || [ -e ${dep_base}_base.txt ]; then
   echo "Found dependency file(s) at ${dep_base}*.txt"