Scripts to build from source (platform agnostic) and bootstrap a macO…

…S launchctl service daemon (#215)

* build script

* echo_log

* plist generation

* write the launcher script

* Working

* Installer scripts

* settings option

* rename

* shorter vars

* tweak varnames

* Cleanup

* varnames

* commetn

* Scripts

* remove pdb

* style

---------

Co-authored-by: ashariyar <[email protected]>

.gitignore

@@ -18,6 +18,7 @@ dist/
 downloads/
 eggs/
 .eggs/
+launchctl/
 lib/
 lib64/
 parts/
@@ -121,3 +122,6 @@ dmypy.json
 
 # PyCharm user settings
 .idea/
+
+# MacOS detritus
+.DS_Store

build_scripts/build_opencanary.sh

@@ -0,0 +1,141 @@
+#!/bin/bash -e
+# Build opencanary package from git repo and create virtualenv where it can be run.
+# Tested on macOS but should work elsewhere as long as the compile flags are configured.
+#
+# Environment variable options:
+#    OPENCANARY_BUILD_FULL_CLEAN=True - do a new git checkout
+#    OPENCANARY_BUILD_FRESH_VENV=True - recreate the venv
+
+# Configurable variables
+OPENCANARY_GIT_REPO="https://github.com/thinkst/opencanary.git"
+HOMEBREW_OPENSSL_FORMULA="[email protected]"
+VENV_DIR=env
+VENV_CREATION_CMD='python3 -m venv'
+
+# System info
+SYSTEM_INFO=`uname -a`
+BUILD_SCRIPT_DIR=$(dirname -- "$(readlink -f -- "$0";)";)
+OPENCANARY_DIR="$(readlink -f -- "$BUILD_SCRIPT_DIR/../";)"
+BUILD_LOG="$BUILD_SCRIPT_DIR/build.log"
+VENV_PATH="$OPENCANARY_DIR/$VENV_DIR"
+
+
+# Echo a string to both STDOUT and the build log
+echo_log() {
+    echo -e "$1"
+    echo >> $BUILD_LOG
+    echo -e "$1" >> "$BUILD_LOG"
+    echo >> $BUILD_LOG
+}
+
+# Bash fxn to move files/dirs out of the way
+mv_to_old() {
+    local FILE_TO_MOVE="$1"
+    local OLD_COPY="$1.old"
+    [[ -a $OLD_COPY ]] && mv_to_old $OLD_COPY   # Recursively move old copies out the way
+    echo_log "Moving '$FILE_TO_MOVE' out of the way to '$OLD_COPY'"
+    mv -v "$FILE_TO_MOVE" "$OLD_COPY" >> "$BUILD_LOG"
+}
+
+# Create python virtual env
+create_venv() {
+    echo_log "Creating new virtualenv in '$VENV_DIR'..."
+    $VENV_CREATION_CMD "$VENV_PATH" >> "$BUILD_LOG"
+}
+
+
+echo -e "Build log will be written to '$BUILD_LOG'..."
+pushd "$OPENCANARY_DIR" >> "$BUILD_LOG"
+
+
+if [[ $SYSTEM_INFO =~ 'Darwin' ]]; then
+    echo_log 'macOS detected...'
+
+    if ! command -v brew &>/dev/null; then
+        echo_log 'ERROR: homebrew not found. Try visiting https://brew.sh/'
+        exit 1
+    fi
+
+    set +e
+    OPENSSL_PATH=$(brew --prefix "$HOMEBREW_OPENSSL_FORMULA" 2>/dev/null)
+
+    if [ $? -ne 0 ] ; then
+        echo_log "ERROR: $HOMEBREW_OPENSSL_FORMULA not found. Try 'brew install $HOMEBREW_OPENSSL_FORMULA'."
+        exit 1
+    fi
+
+    set -e
+
+    if [[ $SYSTEM_INFO =~ 'X86' ]]; then
+        echo_log 'x86 detected...'
+        export ARCHFLAGS="-arch x86_64"
+    elif [[ $SYSTEM_INFO =~ 'ARM64' ]]; then
+        echo_log 'm1 detected...'
+        export ARCHFLAGS="-arch arm64"
+    else
+        echo_log "ERROR: Architecture not identifiable from system info, exiting."
+        echo_log "'uname -a' output was: $SYSTEM_INFO"
+        exit 1
+    fi
+
+    export LDFLAGS="-L$OPENSSL_PATH/lib"
+    export CPPFLAGS="-I$OPENSSL_PATH/include"
+
+    echo_log "Found $HOMEBREW_OPENSSL_FORMULA at '$OPENSSL_PATH'"
+    echo_log "    LDFLAGS set to '$LDFLAGS'"
+    echo_log "    CPPFLAGS set to '$CPPFLAGS'"
+    echo_log "    ARCHFLAGS set to '$ARCHFLAGS'"
+else
+    echo_log "Unknown system. You may need to set LDFLAGS, CPPFLAGS, and ARCHFLAGS to compile 'cryptography'."
+fi
+
+
+# Backup current checkout and pull a fresh one from github if requested
+if [ "${OPENCANARY_BUILD_FULL_CLEAN+set}" = set ]; then
+    echo_log "OPENCANARY_BUILD_FULL_CLEAN requested; backing up repo and rebuilding from scratch..."
+    pushd .. >> "$BUILD_LOG"
+    [[ -a opencanary ]] && mv_to_old opencanary
+    git clone "$OPENCANARY_GIT_REPO"
+    popd >> "$BUILD_LOG"
+else
+    echo_log "Using current repo at '$OPENCANARY_DIR'"
+    echo "    (Set OPENCANARY_BUILD_FULL_CLEAN=true to start from a fresh git checkout)"
+fi
+
+
+# Backup current virtual env and make a new one if requested
+if [[ -d $VENV_PATH ]]; then
+    if [[ "${OPENCANARY_BUILD_FRESH_VENV+set}" = set ]]; then
+        mv_to_old "$VENV_PATH"
+        create_venv
+    else
+        echo_log "Using current virtualenv in '$VENV_PATH'"
+        echo "    (Set OPENCANARY_BUILD_FRESH_VENV=true to rebuild a new virtualenv)"
+    fi
+else
+    create_venv
+fi
+
+
+echo_log "Activating virtual env in subshell..."
+. "$VENV_PATH/bin/activate"
+
+echo_log "Installing cryptography package..."
+pip3 install cryptography >> "$BUILD_LOG"
+
+echo_log "Building..."
+python3 setup.py sdist >> "$BUILD_LOG" 2>&1
+BUILT_PKG=$(ls dist/opencanary*.tar.gz)
+
+echo_log "Installing built package '$BUILT_PKG'..."
+pip install dist/opencanary-0.7.1.tar.gz >> "$BUILD_LOG"
+
+echo_log "Install complete.\n"
+
+if [[ "${VIRTUAL_ENV+set}" = set ]]; then
+    echo_log "IMPORTANT: virtualenv is NOT active!"
+fi
+
+echo_log "To activate the virtualenv now and in the future:"
+echo_log "\n    . '$OPENCANARY_DIR/env/bin/activate'\n\n"
+popd >> "$BUILD_LOG"

build_scripts/generate_macOS_launchctl_service_files.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+# Python script to generate a .plist file that launchctl can use to manage opencanary as a service
+# as well as bootstrap and bootout scripts to get the service up and running.
+# NOTE: Requires homebrew.
+
+import json
+import pathlib
+import plistlib
+import re
+import stat
+import sys
+from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
+from functools import partial
+from os import chmod, pardir, path
+from os.path import dirname, join, realpath
+from shutil import copyfile
+from subprocess import CalledProcessError, check_output
+
+from pkg_resources import resource_filename
+
+OPENCANARY = 'opencanary'
+LAUNCH_DAEMONS_DIR = '/Library/LaunchDaemons'
+DEFAULT_SERVICE_NAME = 'com.thinkst.opencanary'
+CONFIG_FILE_BASENAME = 'opencanary.conf'
+DEFAULT_CONFIG_FILE = resource_filename('opencanary', 'data/settings.json')
+
+# opencanary dirs
+OPENCANARY_DIR = realpath(join(dirname(__file__), pardir))
+OPENCANARY_BIN_DIR = join(OPENCANARY_DIR, 'bin')
+VENV_DIR = join(OPENCANARY_DIR, 'env')
+VENV_BIN_DIR = join(VENV_DIR, 'bin')
+DEFAULT_LOG_DIR = join(OPENCANARY_DIR, 'log')
+
+# daemon config
+DAEMON_CONFIG_DIR = '/etc/opencanaryd'
+DAEMON_CONFIG_PATH = join(DAEMON_CONFIG_DIR, CONFIG_FILE_BASENAME)
+DAEMON_PATH = join(VENV_BIN_DIR, 'opencanaryd')
+DAEMON_RUNTIME_OPTIONS = "--dev"
+
+# This script writes to the launchctl/ folder
+LAUNCHCTL_DIR = join(OPENCANARY_DIR, 'launchctl')
+
+# Homebrew (TODO: is this necessary?)
+try:
+    homebrew_bin_dir = join(check_output(['brew', '--prefix']).decode().rstrip(), 'bin')
+except CalledProcessError as e:
+    print(f"Couldn't get homebrew install location: {e}")
+    sys.exit()
+
+# Load opencanary.conf default config
+with open(DEFAULT_CONFIG_FILE, 'r') as file:
+    config = json.load(file)
+    canaries = [k.split(".")[0] for k in config.keys() if re.match("[a-z]+\\.enabled", k)]
+
+
+# Parse arguments.
+parser = ArgumentParser(
+    description='Generate .plist, opencanary.conf, and scripts to bootstrap opencanary as a launchctl daemon.',
+    formatter_class=ArgumentDefaultsHelpFormatter
+)
+
+parser.add_argument('--service-name',
+                    help='string you would like launchctl to use as the name of the opencanary service',
+                    metavar='NAME',
+                    default=DEFAULT_SERVICE_NAME)
+
+parser.add_argument('--log-output-dir',
+                    help='opencanary will write its logs to files in DIR when the service is running',
+                    metavar='DIR',
+                    default=DEFAULT_LOG_DIR)
+
+parser.add_argument('--canary', action='append',
+                    help=f'enable canary service in the generated opencanary.conf file ' + \
+                         '(can be supplied more than once)',
+                    choices=canaries,
+                    dest='canaries')
+
+
+args = parser.parse_args()
+args.canaries = args.canaries or []
+plist_basename = args.service_name + ".plist"
+
+# Setup dirs
+for dir in [LAUNCHCTL_DIR, args.log_output_dir]:
+    print(f"Creating '{dir}'...")
+    pathlib.Path(dir).mkdir(parents=True, exist_ok=True)
+
+# File builders
+build_launchctl_dir_path = partial(join, LAUNCHCTL_DIR)
+build_logfile_name = lambda log_name: join(args.log_output_dir, f"opencanary.{log_name}.log")
+
+
+# daemon launcher script
+launcher_script = build_launchctl_dir_path(f"launch_{args.service_name}.sh")
+
+with open(launcher_script, 'w') as file:
+    file.write(f'. "{VENV_BIN_DIR}/activate"\n')
+    file.write(f'"{DAEMON_PATH}" {DAEMON_RUNTIME_OPTIONS}\n')
+
+
+# Write launchctl service .plist
+plist_output_file = build_launchctl_dir_path(plist_basename)
+
+plist_contents = {
+    'Label': args.service_name,
+    'RunAtLoad': True,
+    'KeepAlive': True,
+    'WorkingDirectory': VENV_BIN_DIR,
+    'StandardOutPath':  build_logfile_name('err'),
+    'StandardErrorPath': build_logfile_name('out'),
+    'EnvironmentVariables': {
+        'PATH': f"{VENV_BIN_DIR}:{homebrew_bin_dir}:/usr/bin:/bin",
+        'VIRTUAL_ENV': VENV_DIR
+    },
+    'ProgramArguments': [launcher_script]
+}
+
+with open(plist_output_file, 'wb+') as _plist_file:
+    plistlib.dump(plist_contents, _plist_file)
+
+
+# opencanary config
+for canary in canaries:
+    config[f"{canary}.enabled"] = canary in args.canaries
+
+log_handlers = config["logger"]["kwargs"]["handlers"]
+log_handlers["file"]["filename"] = build_logfile_name('run')
+
+# TODO: This config doesn't work even though direct calls to syslog do
+# log_handlers["syslog-unix"] = {
+#     "class": "logging.handlers.SysLogHandler",
+#     "formatter":"syslog_rfc",
+#     "address": [
+#         "localhost",
+#         514
+#     ],
+#     "socktype": "ext://socket.SOCK_DGRAM"
+# }
+
+config_output_file = build_launchctl_dir_path(CONFIG_FILE_BASENAME)
+
+with open(config_output_file, 'w') as file:
+    file.write(json.dumps(config, indent=4))
+
+
+# service bootstrap script
+install_service_script = build_launchctl_dir_path(f"install_service_{args.service_name}.sh")
+daemon_plist_path = join(LAUNCH_DAEMONS_DIR, plist_basename)
+
+with open(install_service_script, 'w') as file:
+    script_contents = [
+        f"set -e\n\n"
+        f"chown root '{launcher_script}'",
+        f"mkdir -p '{DAEMON_CONFIG_DIR}'",
+        f"cp '{config_output_file}' {DAEMON_CONFIG_PATH}",
+        f"cp '{plist_output_file}' {LAUNCH_DAEMONS_DIR}",
+        f"launchctl bootstrap system '{daemon_plist_path}'",
+        ""
+    ]
+
+    file.write("\n".join(script_contents))
+
+
+# uninstall/bootout script
+uninstall_service_script = build_launchctl_dir_path(f"uninstall_service_{args.service_name}.sh")
+
+with open(uninstall_service_script, 'w') as file:
+    file.write(f"launchctl bootout system/{args.service_name}\n")
+
+
+# Set permissions
+chmod(launcher_script, stat.S_IRWXU)  # stat.S_IEXEC | stat.S_IREAD
+chmod(uninstall_service_script, stat.S_IRWXU)
+chmod(install_service_script, stat.S_IRWXU)
+
+
+# Print results
+print("Generated files...\n")
+print(f"    Service definition: ./{path.relpath(plist_output_file)}")
+print(f"       Launcher script: ./{path.relpath(launcher_script)}")
+print(f"      Bootstrap script: ./{path.relpath(install_service_script)}")
+print(f"        Bootout script: ./{path.relpath(uninstall_service_script)}\n")
+print(f"                Config: ./{path.relpath(config_output_file)}")
+print(f"      Enabled canaries: {', '.join(args.canaries)}\n")
+print(f"To install as a system service run:\n    'sudo {install_service_script}'\n")