From 82844a17b4100fc8bdc0e2d10b1989bbbf2dd115 Mon Sep 17 00:00:00 2001 From: Manu <3916435+m3nu@users.noreply.github.com> Date: Tue, 3 Mar 2020 13:19:36 +0800 Subject: [PATCH] Add macOS notarization, use Github Workflows for testing (#407) * Improve macOS packaging, add notarization. * Properly use QApplication while testing, remove workarounds. * Use Github Workflows instead of Travis. * Remove outdated test workaround. --- .github/workflows/main.yml | 69 +++++++++++ .travis.yml | 82 ------------- Makefile | 43 +++---- Vagrantfile | 180 ----------------------------- appdmg.json => package/appdmg.json | 2 +- package/borg.spec | 53 +++++++++ package/entitlements.plist | 13 +++ package/macos-package-app.sh | 62 ++++++++++ package/vorta.spec | 81 +++++++++++++ requirements.d/dev.txt | 2 +- setup.cfg | 23 +++- src/vorta/borg/borg_thread.py | 19 +-- src/vorta/config.py | 3 +- src/vorta/models.py | 13 +-- src/vorta/utils.py | 4 +- src/vorta/views/main_window.py | 4 +- tests/conftest.py | 40 ++++--- tests/test_archives.py | 36 +++--- tests/test_borg.py | 4 +- tests/test_notifications.py | 2 +- tests/test_repo.py | 47 ++++---- tests/test_schedule.py | 4 +- tests/test_scheduler.py | 4 +- tests/test_source.py | 8 +- tests/test_utils.py | 2 +- vorta.spec | 73 ------------ 26 files changed, 415 insertions(+), 458 deletions(-) create mode 100644 .github/workflows/main.yml delete mode 100644 .travis.yml delete mode 100644 Vagrantfile rename appdmg.json => package/appdmg.json (81%) create mode 100644 package/borg.spec create mode 100644 package/entitlements.plist create mode 100644 package/macos-package-app.sh create mode 100644 package/vorta.spec delete mode 100644 vorta.spec diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..0640e2fed --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,69 @@ +name: Test + +on: [push, pull_request] + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + + matrix: + python-version: [3.6, 3.7, 3.8] + os: [ubuntu-latest, macos-latest] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install system dependencies (Linux) + if: runner.os == 'Linux' + run: | + sudo apt install -y \ + xvfb herbstluftwm libssl-dev openssl libacl1-dev libacl1 build-essential \ + libxkbcommon-x11-0 dbus-x11 + - name: Install system dependencies (macOS) + if: runner.os == 'macOS' + run: | + brew upgrade openssl readline xz # pyenv pyenv-virtualenv + - name: Install Vorta + run: | + pip install . + pip install borgbackup + pip install -r requirements.d/dev.txt + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v1 + - name: Test with pytest (Linux) + if: runner.os == 'Linux' + run: | + export DISPLAY=:99.0 + /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile \ + --background --exec /usr/bin/Xvfb -- :99 -screen 0 1920x1200x24 -ac +extension GLX +render -noreset + sleep 3 + export $(dbus-launch) + (herbstluftwm) & + sleep 3 + pytest + - name: Test with pytest (macOS) + if: runner.os == 'macOS' + run: | + pytest + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install Vorta + run: | + pip install . + pip install -r requirements.d/dev.txt + - name: Run Flake8 + run: flake8 + - name: Run PyLint (info only) + run: pylint --rcfile=setup.cfg src --exit-zero diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9ab773d8c..000000000 --- a/.travis.yml +++ /dev/null @@ -1,82 +0,0 @@ -language: generic -sudo: required -dist: xenial - -addons: - apt: - packages: - - xvfb - - herbstluftwm - - libssl-dev - - openssl - - libacl1-dev - - libacl1 - - build-essential - - libxkbcommon-x11-0 - homebrew: - update: false - packages: - - openssl - - readline - - xz - - pyenv - - pyenv-virtualenv - casks: - - xquartz - -cache: - directories: - - $HOME/.cache/pip - - $HOME/.pyenv/versions - - $HOME/Library/Caches/Homebrew - -env: - global: - - SETUP_XVFB=true - - PYTHON36=3.6.9 - - PYTHON37=3.7.5 - - PYTHON38=3.8.2 - -matrix: - include: - - os: linux - dist: xenial - env: - - RUN_PYINSTALLER=true - - os: osx - env: - - RUN_PYINSTALLER=true - -install: -- | - if [ $TRAVIS_OS_NAME = "linux" ]; then - export DISPLAY=:99.0 - /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -screen 0 1920x1200x24 -ac +extension GLX +render -noreset - sleep 3 - cd $(pyenv root) && git pull origin master && cd $TRAVIS_BUILD_DIR - elif [ $TRAVIS_OS_NAME = "osx" ]; then - brew upgrade pyenv - fi - pyenv install -s $PYTHON37 - pyenv install -s $PYTHON36 - eval "$(pyenv init -)" - pyenv shell $PYTHON36 $PYTHON37 - -- pip install -U setuptools pip -- pip install . -- pip install borgbackup -- pip install -r requirements.d/dev.txt - -before_script: -- if [ $TRAVIS_OS_NAME = "linux" ]; then (herbstluftwm)& fi -- sleep 3 - -script: -- tox - -branches: - only: - - master - -notifications: - email: false diff --git a/Makefile b/Makefile index 411d6bb23..68a420807 100644 --- a/Makefile +++ b/Makefile @@ -1,33 +1,35 @@ export VORTA_SRC := src/vorta -export QT_SELECT=5 +export CERTIFICATE_NAME := "Developer ID Application: Manuel Riel (CNMSCAXT48)" .PHONY : help .DEFAULT_GOAL := help DATE = "$(shell date +%F)" +clean: + rm -rf dist/* + icon-resources: ## Compile SVG icons to importable resource files. pyrcc5 -o src/vorta/views/dark/collection_rc.py src/vorta/assets/icons/dark/collection.qrc pyrcc5 -o src/vorta/views/light/collection_rc.py src/vorta/assets/icons/light/collection.qrc -Vorta.app: translations-to-qm - pyinstaller --clean --noconfirm vorta.spec +dist/Vorta.app: translations-to-qm clean + pyinstaller --clean --noconfirm package/vorta.spec cp -R bin/darwin/Sparkle.framework dist/Vorta.app/Contents/Frameworks/ - cd dist; codesign --deep --sign 'Developer ID Application: Manuel Riel (CNMSCAXT48)' Vorta.app + cp -R ../borg/dist/borg-dir dist/Vorta.app/Contents/Resources/ + rm -rf build + rm -rf dist/vorta -Vorta.dmg-Vagrant: - vagrant up darwin64 - rm -rf dist/* - vagrant scp darwin64:/vagrant/dist/Vorta.app dist/ - vagrant halt darwin64 - cp -R bin/darwin/Sparkle.framework dist/Vorta.app/Contents/Frameworks/ - cd dist; codesign --deep --sign 'Developer ID Application: Manuel Riel (CNMSCAXT48)' Vorta.app - sleep 2; appdmg appdmg.json dist/vorta-0.6.23.dmg +borg: + cd ../borg && pyinstaller --clean --noconfirm ../vorta/package/borg.spec . + find ../borg/dist/borg-dir -type f \( -name \*.so -or -name \*.dylib -or -name borg.exe \) \ + -exec codesign --verbose --force --sign $(CERTIFICATE_NAME) \ + --entitlements package/entitlements.plist --timestamp --deep --options runtime {} \; -Vorta.dmg: Vorta.app - rm -rf dist/vorta-0.6.23.dmg - sleep 2; appdmg appdmg.json dist/vorta-0.6.23.dmg +dist/Vorta.dmg: dist/Vorta.app + sh package/macos-package-app.sh -github-release: Vorta.dmg +github-release: dist/Vorta.dmg + cp dist/Vorta.dmg dist/dist/vorta-0.6.23.dmg hub release create --attach=dist/vorta-0.6.23.dmg v0.6.23 git checkout gh-pages git commit -m 'rebuild pages' --allow-empty @@ -45,15 +47,6 @@ bump-version: ## Add new version tag and push to upstream repo. git commit -a -m 'Bump version' git push upstream -travis-debug: ## Prepare connecting to Travis instance via SSH. - curl -s -X POST \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - -H "Travis-API-Version: 3" \ - -H "Authorization: token ${TRAVIS_TOKEN}" \ - -d '{ "quiet": true }' \ - https://api.travis-ci.org/job/${TRAVIS_JOB_ID}/debug - translations-from-source: ## Extract strings from source code / UI files, merge into .ts. pylupdate5 -verbose -translate-function trans_late \ $$VORTA_SRC/*.py $$VORTA_SRC/views/*.py $$VORTA_SRC/borg/*.py \ diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index 647d6b9af..000000000 --- a/Vagrantfile +++ /dev/null @@ -1,180 +0,0 @@ - -# Inspired by https://github.com/borgbackup/borg/blob/master/Vagrantfile - -$cpus = Integer(ENV.fetch('VMCPUS', '4')) # create VMs with that many cpus -$xdistn = Integer(ENV.fetch('XDISTN', '4')) # dispatch tests to that many pytest workers -$wmem = $xdistn * 256 # give the VM additional memory for workers [MB] - -def fs_init(user) - return <<-EOF - # clean up (wrong/outdated) stuff we likely got via rsync: - rm -rf /vagrant/vorta/.tox 2> /dev/null - find /vagrant/vorta/src -name '__pycache__' -exec rm -rf {} \\; 2> /dev/null - chown -R #{user} /vagrant/vorta - touch ~#{user}/.bash_profile ; chown #{user} ~#{user}/.bash_profile - echo 'export LANG=en_US.UTF-8' >> ~#{user}/.bash_profile - echo 'export LC_CTYPE=en_US.UTF-8' >> ~#{user}/.bash_profile - echo 'export XDISTN=#{$xdistn}' >> ~#{user}/.bash_profile - EOF -end - -def packages_debianoid(user) - return <<-EOF - apt update - # install all the (security and other) updates - apt dist-upgrade -y - # for building borgbackup and dependencies: - apt install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config - usermod -a -G fuse #{user} - chgrp fuse /dev/fuse - chmod 666 /dev/fuse - apt install -y fakeroot build-essential git curl - apt install -y python3-dev python3-setuptools python-virtualenv python3-virtualenv - # for building python: - apt install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev libffi-dev - # minimal window manager and system tray icon support - apt install xvfb herbstluftwm gnome-keyring - EOF -end - -def install_pyenv(boxname) - return <<-EOF - curl -s -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash - echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> ~/.bash_profile - echo 'eval "$(pyenv init -)"' >> ~/.bash_profile - echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile - echo 'export PYTHON_CONFIGURE_OPTS="--enable-shared"' >> ~/.bash_profile - EOF -end - -def install_pythons(boxname) - return <<-EOF - . ~/.bash_profile - pyenv install 3.6.8 - pyenv rehash - EOF -end - -def build_pyenv_venv(boxname) - return <<-EOF - . ~/.bash_profile - cd /vagrant/vorta - pyenv global 3.6.8 - pyenv virtualenv 3.6.8 vorta-env - ln -s ~/.pyenv/versions/vorta-env . - EOF -end - -def install_pyinstaller() - return <<-EOF - . ~/.bash_profile - cd /vagrant/vorta - . vorta-env/bin/activate - pip install pyinstaller - EOF -end - -def build_binary_with_pyinstaller(boxname) - return <<-EOF - . ~/.bash_profile - cd /vagrant/vorta - . vorta-env/bin/activate - pip uninstall pyqt5 - # Use older PyQt5 to avoid DBus issue. - pip install pyqt5==5.11.3 - pyinstaller --clean --noconfirm vorta.spec - EOF -end - -def run_tests(boxname) - return <<-EOF - . ~/.bash_profile - cd /vagrant/vorta - . vorta-env/bin/activate - tox - fi - EOF -end - -def darwin_prepare() - return <<-EOF - /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - brew install python - echo 'export PATH="/usr/local/opt/qt/bin/:$PATH"' >> ~/.bash_profile - cd /vagrant - pip3 install -e . - pip3 install -r requirements.d/dev.txt - brew bundle --file=requirements.d/Brewfile - EOF -end - -def darwin_build() - return <<-EOF - cd /vagrant - make Vorta.app - EOF -end - -Vagrant.configure(2) do |config| - - config.vm.define "jessie64" do |b| - b.vm.box = "debian/jessie64" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") - b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant") - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("jessie64") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("jessie64") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("jessie64") - b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() - b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("jessie64") -# b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("jessie64") - end - - config.vm.define "darwin64" do |b| - b.vm.box = "monsenso/macos-10.13" - b.vm.provider :virtualbox do |v| - v.memory = 1536 + $wmem - v.customize ['modifyvm', :id, '--ostype', 'MacOS_64'] - v.customize ['modifyvm', :id, '--paravirtprovider', 'default'] - v.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"] - v.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"] - # Adjust CPU settings according to - # https://github.com/geerlingguy/macos-virtualbox-vm -# v.customize ['modifyvm', :id, '--cpuidset', -# '00000001', '000306a9', '00020800', '80000201', '178bfbff'] - # Disable USB variant requiring Virtualbox proprietary extension pack - v.customize ["modifyvm", :id, '--usbehci', 'off', '--usbxhci', 'off'] - end - - b.vm.synced_folder ".", "/vagrant", type: "rsync", user: "vagrant", group: "staff" - b.vm.provision "darwin_prepare", :type => :shell, :privileged => false, :inline => darwin_prepare() - b.vm.provision "darwin_build", :type => :shell, :privileged => false, run: "always", :inline => darwin_build() - end - - config.vm.define "win64" do |b| - b.vm.box = "gusztavvargadr/windows-10" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - end - - # config.vm.define "freebsd64" do |b| - # b.vm.box = "freebsd12-amd64" - # b.vm.provider :virtualbox do |v| - # v.memory = 1024 + $wmem - # end - # b.ssh.shell = "sh" - # b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") - # b.vm.provision "packages freebsd", :type => :shell, :inline => packages_freebsd - # b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("freebsd64") - # b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) - # b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() - # b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd64") - # b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd64") - # end - - # TODO: create more VMs with python 3.6 and openssl 1.1. - # See branch 1.1-maint for a better equipped Vagrantfile (but still on py34 and openssl 1.0). -end diff --git a/appdmg.json b/package/appdmg.json similarity index 81% rename from appdmg.json rename to package/appdmg.json index 8915a3695..9cd15f43d 100644 --- a/appdmg.json +++ b/package/appdmg.json @@ -2,7 +2,7 @@ "title": "Vorta Backups", "contents": [ { "x": 448, "y": 144, "type": "link", "path": "/Applications" }, - { "x": 162, "y": 144, "type": "file", "path": "dist/Vorta.app" } + { "x": 162, "y": 144, "type": "file", "path": "../dist/Vorta.app" } ], "format": "ULFO", "code-sign": { diff --git a/package/borg.spec b/package/borg.spec new file mode 100644 index 000000000..1b1453b2f --- /dev/null +++ b/package/borg.spec @@ -0,0 +1,53 @@ +# -*- mode: python -*- +# this pyinstaller spec file is used to build borg binaries on posix platforms +# adapted from Borg project to package noatrized folder-style app + +import os, sys + +## Pass borg source dir as last argument +basepath = os.path.abspath(os.path.join(sys.argv[-1])) + +block_cipher = None + +a = Analysis([os.path.join(basepath, 'src', 'borg', '__main__.py'), ], + pathex=[basepath, ], + binaries=[], + datas=[ + (os.path.join(basepath, 'src', 'borg', 'paperkey.html'), 'borg'), + ], + hiddenimports=[ + 'borg.platform.posix', + 'borg.platform.darwin', + ], + hookspath=[], + runtime_hooks=[], + excludes=[ + '_ssl', 'ssl', + ], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher) + +if sys.platform == 'darwin': + # do not bundle the osxfuse libraries, so we do not get a version + # mismatch to the installed kernel driver of osxfuse. + a.binaries = [b for b in a.binaries if 'libosxfuse' not in b[0]] + +pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) + +exe = EXE(pyz, + a.scripts, + exclude_binaries=True, + name='borg.exe', + debug=False, + strip=False, + upx=False, + console=True) + +coll = COLLECT(exe, + a.binaries, + a.zipfiles, + a.datas, + strip=False, + upx=False, + name='borg-dir') diff --git a/package/entitlements.plist b/package/entitlements.plist new file mode 100644 index 000000000..33740fa5b --- /dev/null +++ b/package/entitlements.plist @@ -0,0 +1,13 @@ + + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.cs.disable-library-validation + + + \ No newline at end of file diff --git a/package/macos-package-app.sh b/package/macos-package-app.sh new file mode 100644 index 000000000..5e2a4255a --- /dev/null +++ b/package/macos-package-app.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Inspired by https://github.com/metabrainz/picard/blob/master/scripts/package/macos-notarize-app.sh + +set -e + +CERTIFICATE_NAME="Developer ID Application: Manuel Riel (CNMSCAXT48)" +APP_BUNDLE_ID="com.borgbase.client.macos" +APP_BUNDLE="Vorta" +APPLE_ID_USER="manu@snapdragon.cc" +APPLE_ID_PASSWORD="@keychain:Notarization" + +cd dist + +# codesign --deep is only 1 level deep. It misses Sparkle embedded app AutoUpdate +codesign --verbose --force --sign "$CERTIFICATE_NAME" --timestamp --deep --options runtime \ + $APP_BUNDLE.app/Contents/Frameworks/Sparkle.framework/Resources/Autoupdate.app + +codesign --verify --force --verbose --deep \ + --options runtime --timestamp \ + --entitlements ../package/entitlements.plist \ + --sign "$CERTIFICATE_NAME" $APP_BUNDLE.app + +# ditto -c -k --rsrc --keepParent "$APP_BUNDLE.app" "${APP_BUNDLE}.zip" +rm -rf $APP_BUNDLE.dmg +appdmg ../package/appdmg.json $APP_BUNDLE.dmg + +RESULT=$(xcrun altool --notarize-app --type osx \ + --primary-bundle-id $APP_BUNDLE_ID \ + --username $APPLE_ID_USER --password $APPLE_ID_PASSWORD \ + --file "$APP_BUNDLE.dmg" --output-format xml) + +REQUEST_UUID=$(echo "$RESULT" | xpath \ + "//key[normalize-space(text()) = 'RequestUUID']/following-sibling::string[1]/text()" 2> /dev/null) + +# Poll for notarization status +echo "Submitted notarization request $REQUEST_UUID, waiting for response..." +sleep 60 +while true +do + RESULT=$(xcrun altool --notarization-info "$REQUEST_UUID" \ + --username "$APPLE_ID_USER" \ + --password "$APPLE_ID_PASSWORD" \ + --output-format xml) + STATUS=$(echo "$RESULT" | xpath "//key[normalize-space(text()) = 'Status']/following-sibling::string[1]/text()" 2> /dev/null) + + if [ "$STATUS" = "success" ]; then + echo "Notarization of $APP_BUNDLE succeeded!" + break + elif [ "$STATUS" = "in progress" ]; then + echo "Notarization in progress..." + sleep 20 + else + echo "Notarization of $APP_BUNDLE failed:" + echo "$RESULT" + exit 1 + fi +done + +# Staple the notary ticket +xcrun stapler staple $APP_BUNDLE.dmg +xcrun stapler staple $APP_BUNDLE.app +xcrun stapler validate $APP_BUNDLE.dmg \ No newline at end of file diff --git a/package/vorta.spec b/package/vorta.spec new file mode 100644 index 000000000..c37bebf8d --- /dev/null +++ b/package/vorta.spec @@ -0,0 +1,81 @@ +# -*- mode: python -*- + +import os +import sys +from pathlib import Path + +from vorta.config import ( + APP_NAME, + APP_ID_DARWIN +) +from vorta._version import __version__ as APP_VERSION + +BLOCK_CIPHER = None +APP_APPCAST_URL = 'https://borgbase.github.io/vorta/appcast.xml' + + +# it is assumed that the cwd is the git repo dir: +SRC_DIR = os.path.join(os.getcwd(), 'src', 'vorta') + +a = Analysis([os.path.join(SRC_DIR, '__main__.py')], + pathex=[SRC_DIR], + binaries=[], + datas=[ + (os.path.join(SRC_DIR, 'assets/UI/*'), 'assets/UI'), + (os.path.join(SRC_DIR, 'assets/icons/*'), 'assets/icons'), + (os.path.join(SRC_DIR, 'i18n/qm/*'), 'vorta/i18n/qm'), + ], + hiddenimports=[ + 'vorta.views.dark.collection_rc', + 'vorta.views.light.collection_rc', + 'pkg_resources.py2_warn', + ], + hookspath=[], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=BLOCK_CIPHER, + noarchive=False) + +pyz = PYZ(a.pure, a.zipped_data, cipher=BLOCK_CIPHER) + +exe = EXE(pyz, + a.scripts, + exclude_binaries=True, + name=f"vorta-{sys.platform}", + bootloader_ignore_signals=True, + console=False, + debug=False, + strip=False, + upx=True) + +coll = COLLECT(exe, + a.binaries, + a.zipfiles, + a.datas, + debug=False, + strip=False, + upx=False, + name='vorta') + +app = BUNDLE(coll, + name='Vorta.app', + icon=os.path.join(SRC_DIR, 'assets/icons/app-icon.icns'), + bundle_identifier=None, + info_plist={ + 'CFBundleName': APP_NAME, + 'CFBundleDisplayName': APP_NAME, + 'CFBundleIdentifier': APP_ID_DARWIN, + 'NSHighResolutionCapable': 'True', + 'LSUIElement': '1', + 'LSMinimumSystemVersion': '10.14', + 'CFBundleShortVersionString': APP_VERSION, + 'CFBundleVersion': APP_VERSION, + 'SUFeedURL': APP_APPCAST_URL, + 'LSEnvironment': { + 'LC_CTYPE': 'en_US.UTF-8', + 'PATH': '/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin' + } + }) + diff --git a/requirements.d/dev.txt b/requirements.d/dev.txt index 928d2c08c..4dd335a01 100644 --- a/requirements.d/dev.txt +++ b/requirements.d/dev.txt @@ -3,8 +3,8 @@ pytest pytest-qt pytest-mock pytest-faulthandler -pytest-xdist pyinstaller tox bump2version flake8 +pylint diff --git a/setup.cfg b/setup.cfg index 4973da518..e86586c88 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,15 +48,13 @@ tests_require = pytest pytest-qt pytest-mock - pytest-xdist - pytest-faulthandler [options.entry_points] gui_scripts = vorta = vorta.__main__:main [tool:pytest] -addopts = --forked -vs +addopts = -vs testpaths = tests qt_default_raising = true filterwarnings = @@ -74,7 +72,7 @@ exclude = ./src/vorta/views/light/collection_rc.py [tox:tox] -envlist = py36,py37,flake8 +envlist = py36,py37,py38,flake8 skip_missing_interpreters = true [testenv] @@ -82,8 +80,6 @@ deps = pytest pytest-qt pytest-mock - pytest-xdist - pytest-faulthandler commands=pytest passenv = DISPLAY @@ -91,3 +87,18 @@ passenv = DISPLAY deps = flake8 commands=flake8 src tests + +[pycodestyle] +max_line_length = 120 + +[pylint.master] +extension-pkg-whitelist=PyQt5 +load-plugins= +ignore= + collection_rc.py + +[pylint.messages control] +disable= W0511,C0301,R0903,R0201,W0212,C0114,C0115,C0116,C0103,E0611,E1120,C0415,R0914,R0912,R0915 + +[pylint.format] +max-line-length=120 diff --git a/src/vorta/borg/borg_thread.py b/src/vorta/borg/borg_thread.py index 177990409..42559169a 100644 --- a/src/vorta/borg/borg_thread.py +++ b/src/vorta/borg/borg_thread.py @@ -144,18 +144,19 @@ def prepare(cls, profile): def prepare_bin(cls): """Find packaged borg binary. Prefer globally installed.""" - # Look in current PATH. borg_in_path = shutil.which('borg') + if borg_in_path: return borg_in_path - else: - # Look in pyinstaller package - cwd = getattr(sys, '_MEIPASS', os.getcwd()) - meipass_borg = os.path.join(cwd, 'bin', 'borg') - if os.path.isfile(meipass_borg): - return meipass_borg - else: - return None + elif sys.platform == 'darwin': + # macOS: Look in pyinstaller bundle + from Foundation import NSBundle + mainBundle = NSBundle.mainBundle() + + bundled_borg = os.path.join(mainBundle.bundlePath(), 'Contents', 'Resources', 'borg-dir', 'borg.exe') + if os.path.isfile(bundled_borg): + return bundled_borg + return None def run(self): self.started_event() diff --git a/src/vorta/config.py b/src/vorta/config.py index f4e1371ae..a6da37055 100644 --- a/src/vorta/config.py +++ b/src/vorta/config.py @@ -1,8 +1,9 @@ -import appdirs import os +import appdirs APP_NAME = 'Vorta' APP_AUTHOR = 'BorgBase' +APP_ID_DARWIN = 'com.borgbase.client.macos' dirs = appdirs.AppDirs(APP_NAME, APP_AUTHOR) SETTINGS_DIR = dirs.user_data_dir LOG_DIR = dirs.user_log_dir diff --git a/src/vorta/models.py b/src/vorta/models.py index 2a4f6bfdd..7fcabbc22 100644 --- a/src/vorta/models.py +++ b/src/vorta/models.py @@ -241,10 +241,11 @@ def get_misc_settings(): return settings -def init_db(con): - os.umask(0o0077) - db.initialize(con) - db.connect() +def init_db(con=None): + if con is not None: + os.umask(0o0077) + db.initialize(con) + db.connect() db.create_tables([RepoModel, RepoPassword, BackupProfileModel, SourceFileModel, SettingsModel, ArchiveModel, WifiSettingModel, EventLogModel, SchemaVersion]) @@ -345,9 +346,7 @@ def init_db(con): 'extra_borg_arguments', pw.CharField(default=''))) if current_schema.version < 13: - """ - Migrate ArchiveModel data to new table to remove unique constraint from snapshot_id column. - """ + # Migrate ArchiveModel data to new table to remove unique constraint from snapshot_id column. tables = db.get_tables() if ArchiveModel.select().count() == 0 and 'snapshotmodel' in tables: cursor = db.execute_sql('select * from snapshotmodel;') diff --git a/src/vorta/utils.py b/src/vorta/utils.py index 6b8a50a10..fe065b517 100644 --- a/src/vorta/utils.py +++ b/src/vorta/utils.py @@ -79,7 +79,7 @@ def get_private_keys(): 'fingerprint': parsed_key.get_fingerprint().hex() } available_private_keys.append(key_details) - except (SSHException, UnicodeDecodeError, IsADirectoryError): + except (SSHException, UnicodeDecodeError, IsADirectoryError, IndexError): continue except OSError as e: if e.errno == errno.ENXIO: @@ -254,7 +254,7 @@ def get_mount_points(repo_url): mount_point = proc.cmdline()[idx + 1] mount_points[archive_name] = mount_point break - except (psutil.ZombieProcess, psutil.AccessDenied): + except (psutil.ZombieProcess, psutil.AccessDenied, psutil.NoSuchProcess): # Getting process details may fail (e.g. zombie process on macOS) # or because the process is owned by another user. # Also see https://github.com/giampaolo/psutil/issues/783 diff --git a/src/vorta/views/main_window.py b/src/vorta/views/main_window.py index 95b64f112..b71b1c961 100644 --- a/src/vorta/views/main_window.py +++ b/src/vorta/views/main_window.py @@ -31,8 +31,6 @@ def __init__(self, parent=None): self.current_profile = BackupProfileModel.select().order_by('id').first() self.setWindowFlags(QtCore.Qt.WindowCloseButtonHint | QtCore.Qt.WindowMinimizeButtonHint) - self.tests_running = False - # Load tab models self.repoTab = RepoTab(self.repoTabSlot) self.sourceTab = SourceTab(self.sourceTabSlot) @@ -149,7 +147,7 @@ def backup_cancelled_event(self): self.set_status(self.tr('Task cancelled')) def closeEvent(self, event): - if not is_system_tray_available() and not self.tests_running: + if not is_system_tray_available(): run_in_background = QMessageBox.question(self, trans_late("MainWindow QMessagebox", "Quit"), diff --git a/tests/conftest.py b/tests/conftest.py index 44a706c46..fcc95da63 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,22 +2,25 @@ import peewee import sys from datetime import datetime as dt +from unittest.mock import MagicMock import vorta -from vorta.application import VortaApp -from vorta.models import RepoModel, SourceFileModel, ArchiveModel, BackupProfileModel +from vorta.models import (RepoModel, RepoPassword, BackupProfileModel, SourceFileModel, + SettingsModel, ArchiveModel, WifiSettingModel, EventLogModel, SchemaVersion) + + +models = [RepoModel, RepoPassword, BackupProfileModel, SourceFileModel, + SettingsModel, ArchiveModel, WifiSettingModel, EventLogModel, SchemaVersion] def pytest_configure(config): sys._called_from_test = True -@pytest.fixture -def app(tmpdir, qtbot, mocker): - tmp_db = tmpdir.join('settings.sqlite') - mock_db = peewee.SqliteDatabase(str(tmp_db)) - vorta.models.init_db(mock_db) - mocker.patch.object(vorta.application.VortaApp, 'set_borg_details_action', return_value=None) +@pytest.fixture(scope='function', autouse=True) +def init_db(qapp): + vorta.models.db.drop_tables(models) + vorta.models.init_db() new_repo = RepoModel(url='i0fi93@i593.repo.borgbase.com:repo') new_repo.save() @@ -32,11 +35,22 @@ def app(tmpdir, qtbot, mocker): source_dir = SourceFileModel(dir='/tmp/another', repo=new_repo) source_dir.save() - app = VortaApp([]) - app.open_main_window_action() - qtbot.addWidget(app.main_window) - app.main_window.tests_running = True - return app + qapp.open_main_window_action() + + +@pytest.fixture(scope='session') +def qapp(tmpdir_factory): + tmp_db = tmpdir_factory.mktemp('Vorta').join('settings.sqlite') + mock_db = peewee.SqliteDatabase(str(tmp_db)) + vorta.models.init_db(mock_db) + + from vorta.application import VortaApp + VortaApp.set_borg_details_action = MagicMock() # Can't use pytest-mock in session scope + VortaApp.scheduler = MagicMock() + + qapp = VortaApp([]) # Only init QApplication once to avoid segfaults while testing. + + yield qapp @pytest.fixture diff --git a/tests/test_archives.py b/tests/test_archives.py index fbbc73d9e..a9738053d 100644 --- a/tests/test_archives.py +++ b/tests/test_archives.py @@ -15,9 +15,9 @@ def selectedFiles(self): return ['/tmp'] -def test_prune_intervals(app, qtbot): +def test_prune_intervals(qapp, qtbot): prune_intervals = ['hour', 'day', 'week', 'month', 'year'] - main = app.main_window + main = qapp.main_window tab = main.archiveTab profile = BackupProfileModel.get(id=1) @@ -28,25 +28,28 @@ def test_prune_intervals(app, qtbot): assert getattr(profile, f'prune_{i}') == 9 -def test_repo_list(app, qtbot, mocker, borg_json_output): - main = app.main_window +def test_repo_list(qapp, qtbot, mocker, borg_json_output): + main = qapp.main_window tab = main.archiveTab - main.tabWidget.setCurrentIndex(3) - tab.list_action() - assert not tab.checkButton.isEnabled() stdout, stderr = borg_json_output('list') popen_result = mocker.MagicMock(stdout=stdout, stderr=stderr, returncode=0) mocker.patch.object(vorta.borg.borg_thread, 'Popen', return_value=popen_result) + main.tabWidget.setCurrentIndex(3) + tab.list_action() + qtbot.waitUntil(lambda: not tab.checkButton.isEnabled(), timeout=3000) + + assert not tab.checkButton.isEnabled() + qtbot.waitUntil(lambda: main.createProgressText.text() == 'Refreshing archives done.', timeout=3000) assert ArchiveModel.select().count() == 6 assert main.createProgressText.text() == 'Refreshing archives done.' assert tab.checkButton.isEnabled() -def test_repo_prune(app, qtbot, mocker, borg_json_output): - main = app.main_window +def test_repo_prune(qapp, qtbot, mocker, borg_json_output): + main = qapp.main_window tab = main.archiveTab main.tabWidget.setCurrentIndex(3) tab.populate_from_profile() @@ -59,8 +62,8 @@ def test_repo_prune(app, qtbot, mocker, borg_json_output): qtbot.waitUntil(lambda: main.createProgressText.text().startswith('Refreshing archives done.'), timeout=5000) -def test_check(app, mocker, borg_json_output, qtbot): - main = app.main_window +def test_check(qapp, mocker, borg_json_output, qtbot): + main = qapp.main_window tab = main.archiveTab main.tabWidget.setCurrentIndex(3) tab.populate_from_profile() @@ -74,7 +77,7 @@ def test_check(app, mocker, borg_json_output, qtbot): qtbot.waitUntil(lambda: main.createProgressText.text().startswith(success_text), timeout=3000) -def test_archive_mount(app, qtbot, mocker, borg_json_output, monkeypatch, choose_file_dialog): +def test_archive_mount(qapp, qtbot, mocker, borg_json_output, monkeypatch, choose_file_dialog): def psutil_disk_partitions(**kwargs): DiskPartitions = namedtuple('DiskPartitions', ['device', 'mountpoint']) return [DiskPartitions('borgfs', '/tmp')] @@ -83,7 +86,7 @@ def psutil_disk_partitions(**kwargs): psutil, "disk_partitions", psutil_disk_partitions ) - main = app.main_window + main = qapp.main_window tab = main.archiveTab main.tabWidget.setCurrentIndex(3) tab.populate_from_profile() @@ -106,17 +109,14 @@ def psutil_disk_partitions(**kwargs): qtbot.waitUntil(lambda: tab.mountErrors.text().startswith('Un-mounted successfully.'), timeout=5000) -def test_archive_extract(app, qtbot, mocker, borg_json_output, monkeypatch): - main = app.main_window +def test_archive_extract(qapp, qtbot, mocker, borg_json_output, monkeypatch): + main = qapp.main_window tab = main.archiveTab main.tabWidget.setCurrentIndex(3) tab.populate_from_profile() qtbot.waitUntil(lambda: tab.archiveTable.rowCount() == 1) - qtbot.mouseClick(tab.extractButton, QtCore.Qt.LeftButton) - qtbot.waitUntil(lambda: tab.mountErrors.text().startswith('Select an archive')) - monkeypatch.setattr( vorta.views.extract_dialog.ExtractDialog, "exec_", lambda *args: True ) diff --git a/tests/test_borg.py b/tests/test_borg.py index b6dfc38ee..91a4c7062 100644 --- a/tests/test_borg.py +++ b/tests/test_borg.py @@ -3,13 +3,13 @@ from vorta.borg.prune import BorgPruneThread -def test_borg_prune(app, qtbot, mocker, borg_json_output): +def test_borg_prune(qapp, qtbot, mocker, borg_json_output): stdout, stderr = borg_json_output('prune') popen_result = mocker.MagicMock(stdout=stdout, stderr=stderr, returncode=0) mocker.patch.object(vorta.borg.borg_thread, 'Popen', return_value=popen_result) params = BorgPruneThread.prepare(vorta.models.BackupProfileModel.select().first()) - thread = BorgPruneThread(params['cmd'], params, app) + thread = BorgPruneThread(params['cmd'], params, qapp) with qtbot.waitSignal(thread.result, timeout=10000) as blocker: blocker.connect(thread.updated) diff --git a/tests/test_notifications.py b/tests/test_notifications.py index ad9123e70..a90f90ed5 100644 --- a/tests/test_notifications.py +++ b/tests/test_notifications.py @@ -8,7 +8,7 @@ @pytest.mark.skipif(sys.platform != 'linux', reason="DBus notifications only on Linux") -def test_linux_background_notifications(app, mocker): +def test_linux_background_notifications(qapp, mocker): """We can't see notifications, but we watch for exceptions and errors.""" notifier = vorta.notifications.VortaNotifications.pick() diff --git a/tests/test_repo.py b/tests/test_repo.py index dee2fc559..92a5f7eb0 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -10,9 +10,9 @@ from vorta.models import EventLogModel, RepoModel, ArchiveModel -def test_repo_add_failures(app, qtbot, mocker, borg_json_output): +def test_repo_add_failures(qapp, qtbot, mocker, borg_json_output): # Add new repo window - main = app.main_window + main = qapp.main_window add_repo_window = AddRepoWindow(main) qtbot.addWidget(add_repo_window) @@ -25,12 +25,27 @@ def test_repo_add_failures(app, qtbot, mocker, borg_json_output): assert add_repo_window.errorText.text() == 'Please use a longer passphrase.' -def test_repo_add_success(app, qtbot, mocker, borg_json_output): +def test_repo_unlink(qapp, qtbot, monkeypatch): + monkeypatch.setattr(QMessageBox, "exec_", lambda *args: QMessageBox.Yes) + main = qapp.main_window + tab = main.repoTab + + main.tabWidget.setCurrentIndex(0) + qtbot.mouseClick(tab.repoRemoveToolbutton, QtCore.Qt.LeftButton) + qtbot.waitUntil(lambda: tab.repoSelector.count() == 4, timeout=5000) + assert RepoModel.select().count() == 0 + + qtbot.mouseClick(main.createStartBtn, QtCore.Qt.LeftButton) + assert main.createProgressText.text() == 'Add a backup repository first.' + + +def test_repo_add_success(qapp, qtbot, mocker, borg_json_output): LONG_PASSWORD = 'long-password-long' + # Add new repo window - main = app.main_window + main = qapp.main_window + main.repoTab.repo_added.disconnect() add_repo_window = AddRepoWindow(main) - qtbot.addWidget(add_repo_window) test_repo_url = f'vorta-test-repo.{uuid.uuid4()}.com:repo' # Random repo URL to avoid macOS keychain qtbot.keyClicks(add_repo_window.repoURL, test_repo_url) @@ -47,28 +62,13 @@ def test_repo_add_success(app, qtbot, mocker, borg_json_output): main.repoTab.process_new_repo(blocker.args[0]) - qtbot.waitUntil(lambda: EventLogModel.select().count() == 2) - assert EventLogModel.select().count() == 2 + assert EventLogModel.select().count() == 1 assert RepoModel.get(id=2).url == test_repo_url from vorta.utils import keyring assert keyring.get_password("vorta-repo", RepoModel.get(id=2).url) == LONG_PASSWORD -def test_repo_unlink(app, qtbot, monkeypatch): - monkeypatch.setattr(QMessageBox, "exec_", lambda *args: QMessageBox.Yes) - main = app.main_window - tab = main.repoTab - main.tabWidget.setCurrentIndex(0) - qtbot.mouseClick(tab.repoRemoveToolbutton, QtCore.Qt.LeftButton) - - qtbot.waitUntil(lambda: tab.repoSelector.count() == 4, timeout=5000) - assert RepoModel.select().count() == 0 - - qtbot.mouseClick(main.createStartBtn, QtCore.Qt.LeftButton) - assert main.createProgressText.text() == 'Add a backup repository first.' - - def test_ssh_dialog(qtbot, tmpdir): ssh_dialog = SSHAddWindow() ssh_dir = tmpdir @@ -79,6 +79,7 @@ def test_ssh_dialog(qtbot, tmpdir): qtbot.mouseClick(ssh_dialog.generateButton, QtCore.Qt.LeftButton) qtbot.waitUntil(lambda: key_tmpfile.check(file=1)) + qtbot.waitUntil(lambda: pub_tmpfile.check(file=1)) key_tmpfile_content = key_tmpfile.read() pub_tmpfile_content = pub_tmpfile.read() @@ -90,8 +91,8 @@ def test_ssh_dialog(qtbot, tmpdir): qtbot.waitUntil(lambda: ssh_dialog.errors.text().startswith('Key file already')) -def test_create(app, borg_json_output, mocker, qtbot): - main = app.main_window +def test_create(qapp, borg_json_output, mocker, qtbot): + main = qapp.main_window stdout, stderr = borg_json_output('create') popen_result = mocker.MagicMock(stdout=stdout, stderr=stderr, returncode=0) mocker.patch.object(vorta.borg.borg_thread, 'Popen', return_value=popen_result) diff --git a/tests/test_schedule.py b/tests/test_schedule.py index 3e6928047..698ed7d07 100644 --- a/tests/test_schedule.py +++ b/tests/test_schedule.py @@ -2,8 +2,8 @@ from PyQt5 import QtCore -def test_schedule_tab(app, qtbot): - main = app.main_window +def test_schedule_tab(qapp, qtbot): + main = qapp.main_window tab = main.scheduleTab qtbot.mouseClick(tab.scheduleApplyButton, QtCore.Qt.LeftButton) assert tab.nextBackupDateTimeLabel.text() == 'None scheduled' diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index a7601d1f9..4c77beef7 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -2,11 +2,11 @@ import vorta.models -def test_scheduler_create_backup(app, qtbot, mocker, borg_json_output): +def test_scheduler_create_backup(qapp, qtbot, mocker, borg_json_output): stdout, stderr = borg_json_output('create') popen_result = mocker.MagicMock(stdout=stdout, stderr=stderr, returncode=0) mocker.patch.object(vorta.borg.borg_thread, 'Popen', return_value=popen_result) - app.scheduler.create_backup(1) + qapp.scheduler.create_backup(1) qtbot.waitUntil(lambda: vorta.models.EventLogModel.select().count() == 2, timeout=5000) diff --git a/tests/test_source.py b/tests/test_source.py index 8089d0770..b0a31b687 100644 --- a/tests/test_source.py +++ b/tests/test_source.py @@ -1,19 +1,15 @@ -import logging from PyQt5 import QtCore import vorta.models import vorta.views -def test_add_folder(app, qtbot, tmpdir, monkeypatch, choose_file_dialog): +def test_add_folder(qapp, qtbot, tmpdir, monkeypatch, choose_file_dialog): monkeypatch.setattr( vorta.views.source_tab, "choose_file_dialog", choose_file_dialog ) - main = app.main_window + main = qapp.main_window main.tabWidget.setCurrentIndex(1) tab = main.sourceTab qtbot.mouseClick(tab.sourceAddFolder, QtCore.Qt.LeftButton) qtbot.waitUntil(lambda: tab.sourceFilesWidget.count() == 2) - - for src in vorta.models.SourceFileModel.select(): - logging.error(src.dir, src.profile) diff --git a/tests/test_utils.py b/tests/test_utils.py index 9f568574a..aabd0f7bc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,7 +2,7 @@ from vorta.utils import keyring -def test_keyring(app): +def test_keyring(qapp): UNICODE_PW = 'kjalsdfüadsfäadsfß' REPO = f'vorta-test-repo.{uuid.uuid4()}.com:repo' # Random repo URL diff --git a/vorta.spec b/vorta.spec deleted file mode 100644 index 4d1cbbff9..000000000 --- a/vorta.spec +++ /dev/null @@ -1,73 +0,0 @@ -# -*- mode: python -*- - -import os -import sys - -CREATE_VORTA_DIR = False # create dist/vorta-dir/ output? -BLOCK_CIPHER = None - -# it is assumed that the cwd is the git repo dir: -REPO_DIR = os.path.abspath('.') -SRC_DIR = os.path.join(REPO_DIR, 'src') - -a = Analysis(['src/vorta/__main__.py'], - pathex=[SRC_DIR], - binaries=[ - (f"bin/{sys.platform}/borg", 'bin'), # (, ) - ], - datas=[ - ('src/vorta/assets/UI/*', 'assets/UI'), - ('src/vorta/assets/icons/*', 'assets/icons'), - ('src/vorta/i18n/qm/*', 'vorta/i18n/qm'), - ], - hiddenimports=[ - 'vorta.views.dark.collection_rc', - 'vorta.views.light.collection_rc', - ], - hookspath=[], - runtime_hooks=[], - excludes=[], - win_no_prefer_redirects=False, - win_private_assemblies=False, - cipher=BLOCK_CIPHER, - noarchive=False) - -pyz = PYZ(a.pure, a.zipped_data, cipher=BLOCK_CIPHER) - -exe = EXE(pyz, - a.scripts, - a.binaries, - a.zipfiles, - a.datas, - [], - name=f"vorta-{sys.platform}", - debug=False, - bootloader_ignore_signals=True, - strip=False, - upx=True, - runtime_tmpdir=None, - console=True) - -app = BUNDLE(exe, - name='Vorta.app', - icon='src/vorta/assets/icons/app-icon.icns', - bundle_identifier='com.borgbase.client.macos', - info_plist={ - 'NSHighResolutionCapable': 'True', - 'LSUIElement': '1', - 'CFBundleShortVersionString': '0.6.23', - 'CFBundleVersion': '0.6.23', - 'NSAppleEventsUsageDescription': 'Please allow', - 'SUFeedURL': 'https://borgbase.github.io/vorta/appcast.xml', - 'LSEnvironment': { - 'LC_CTYPE': 'en_US.UTF-8' - } - }) - -if CREATE_VORTA_DIR: - coll = COLLECT(exe, - a.binaries, - a.zipfiles, - a.datas, - strip=False, - name='vorta-dir')