From f980d59e883cc68e2de3ad9d885f4ec4d6a7dbc6 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 3 Jun 2017 09:39:10 +0200 Subject: [PATCH] flake8 + isort --- docs/conf.py | 17 +-- src/pyload/core/__init__.py | 2 - src/pyload/core/api/__init__.py | 4 - src/pyload/core/api/account.py | 2 +- src/pyload/core/api/addon.py | 2 +- src/pyload/core/api/config.py | 52 ++++----- src/pyload/core/api/core.py | 12 +- src/pyload/core/api/download.py | 7 +- src/pyload/core/api/exchange.py | 2 +- src/pyload/core/api/file.py | 15 +-- src/pyload/core/api/init.py | 14 +-- src/pyload/core/api/predownload.py | 3 +- src/pyload/core/api/stat.py | 2 - src/pyload/core/api/user.py | 2 +- src/pyload/core/cli.py | 27 +++-- src/pyload/core/config/__init__.py | 4 - src/pyload/core/config/default.py | 162 +++++++++++++++++---------- src/pyload/core/database/__init__.py | 4 - src/pyload/core/database/account.py | 6 +- src/pyload/core/database/backend.py | 29 +++-- src/pyload/core/database/config.py | 10 +- src/pyload/core/database/file.py | 126 +++++++++++++-------- src/pyload/core/database/storage.py | 20 ++-- src/pyload/core/database/user.py | 32 +++--- src/pyload/core/datatype/__init__.py | 4 - src/pyload/core/datatype/check.py | 1 - src/pyload/core/datatype/file.py | 74 +++++++----- src/pyload/core/datatype/init.py | 24 ++-- src/pyload/core/datatype/package.py | 34 +++--- src/pyload/core/datatype/task.py | 19 ++-- src/pyload/core/datatype/user.py | 12 +- src/pyload/core/iface.py | 29 +++-- src/pyload/core/init.py | 52 +++++---- src/pyload/core/manager/__init__.py | 4 - src/pyload/core/manager/account.py | 14 ++- src/pyload/core/manager/addon.py | 11 +- src/pyload/core/manager/base.py | 8 +- src/pyload/core/manager/config.py | 6 +- src/pyload/core/manager/exchange.py | 32 ++++-- src/pyload/core/manager/file.py | 64 +++++++---- src/pyload/core/manager/info.py | 3 +- src/pyload/core/manager/plugin.py | 21 ++-- src/pyload/core/manager/remote.py | 3 +- src/pyload/core/manager/transfer.py | 23 ++-- src/pyload/core/network/account.py | 50 +++++---- src/pyload/core/network/addon.py | 7 +- src/pyload/core/network/base.py | 64 ++++++----- src/pyload/core/network/crypter.py | 65 ++++++----- src/pyload/core/network/extractor.py | 2 +- src/pyload/core/network/factory.py | 7 +- src/pyload/core/network/hoster.py | 65 ++++++----- src/pyload/core/network/loader.py | 17 +-- src/pyload/core/network/ocr.py | 9 +- src/pyload/core/thread/__init__.py | 4 - src/pyload/core/thread/addon.py | 2 +- src/pyload/core/thread/decrypter.py | 28 +++-- src/pyload/core/thread/download.py | 11 +- src/pyload/core/thread/info.py | 25 +++-- src/pyload/core/thread/plugin.py | 17 +-- tests/api/apiproxy.py | 7 +- tests/api/apitester.py | 1 + tests/api/test_api.py | 1 + tests/api/test_jsonbackend.py | 8 +- tests/api/test_noargs.py | 4 +- tests/api/test_wsbackend.py | 1 + tests/crypter_tester.py | 24 ++-- tests/helper/parser.py | 1 + tests/helper/plugintester.py | 25 +++-- tests/helper/stubs.py | 1 + tests/hoster_tester.py | 6 +- tests/manager/test_account.py | 1 + tests/manager/test_config.py | 1 + tests/manager/test_download.py | 8 +- tests/manager/test_file.py | 13 ++- tests/manager/test_interaction.py | 1 + tests/misc/test_configparser.py | 1 + tests/misc/test_curldownload.py | 4 +- tests/misc/test_curlrequest.py | 1 + tests/misc/test_filedatabase.py | 20 ++-- tests/misc/test_networkrequest.py | 1 + tests/misc/test_statdatabase.py | 1 + tests/misc/test_syntax.py | 3 +- 82 files changed, 872 insertions(+), 629 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index e695e031a2..fb0da34bf5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,17 +11,16 @@ # All configuration values have a default; values that are commented out # serve to show the default. -from __future__ import unicode_literals -from __future__ import print_function -from __future__ import division -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals -from future import standard_library -standard_library.install_aliases() import os import sys from os.path import abspath, dirname, exists, join +from future import standard_library + +standard_library.install_aliases() + dir_name = join(dirname(abspath(""))) sys.path.append(dir_name) sys.path.append(join(dir_name, "pyload", "lib")) @@ -39,8 +38,10 @@ # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', - 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autosummary', + 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', + 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode'] autosummary_generate = True autodoc_default_flags = ['members'] diff --git a/src/pyload/core/__init__.py b/src/pyload/core/__init__.py index 893dd48342..8f792f32c3 100644 --- a/src/pyload/core/__init__.py +++ b/src/pyload/core/__init__.py @@ -14,10 +14,8 @@ from future import standard_library standard_library.install_aliases() import builtins -import codecs import locale import os -import sys import tempfile builtins.USERDIR = os.path.expanduser('~') diff --git a/src/pyload/core/api/__init__.py b/src/pyload/core/api/__init__.py index c6135e9871..2961b3f76f 100644 --- a/src/pyload/core/api/__init__.py +++ b/src/pyload/core/api/__init__.py @@ -1,10 +1,6 @@ # -*- coding: utf-8 -*- # @author: vuolter -from __future__ import unicode_literals -from __future__ import absolute_import -from future import standard_library -standard_library.install_aliases() from .init import Api, requireperm, statestring from .base import AbstractApi, BaseApi from .account import AccountApi diff --git a/src/pyload/core/api/account.py b/src/pyload/core/api/account.py index 4448a51926..dfc06a8895 100644 --- a/src/pyload/core/api/account.py +++ b/src/pyload/core/api/account.py @@ -6,7 +6,7 @@ from ..datatype.init import Permission from .base import BaseApi -from .init import Api, requireperm +from .init import requireperm standard_library.install_aliases() diff --git a/src/pyload/core/api/addon.py b/src/pyload/core/api/addon.py index f060cdf5a1..f0fd3049fd 100644 --- a/src/pyload/core/api/addon.py +++ b/src/pyload/core/api/addon.py @@ -6,7 +6,7 @@ from ..datatype.init import Permission from .base import BaseApi -from .init import Api, requireperm +from .init import requireperm standard_library.install_aliases() diff --git a/src/pyload/core/api/config.py b/src/pyload/core/api/config.py index fda1d48bb7..567aebf7f9 100644 --- a/src/pyload/core/api/config.py +++ b/src/pyload/core/api/config.py @@ -3,31 +3,27 @@ from __future__ import absolute_import, unicode_literals from future import standard_library -from pyload.utils.convert import to_str -from ..datatype.init import ConfigHolder, ConfigInfo, ConfigItem, Permission from .base import BaseApi -from .init import Api, requireperm standard_library.install_aliases() -# @TODO: Recheck... +# TODO: Recheck... # helper function to create a ConfigHolder # def to_config_holder(section, config, values): - # holder = ConfigHolder(section, config.label, - # config.description, config.explanation) - # holder.items = [ConfigItem(option, x.label, x.description, x.input, - # to_str(values.get(option, x.input.default), values.get(option, x.input.default))) for option, x in - # config.config.items()] - # return holder +# holder = ConfigHolder(section, config.label, +# config.description, config.explanation) +# holder.items = [ConfigItem(option, x.label, x.description, x.input, +# to_str(values.get(option, x.input.default), values.get(option, x.input.default))) for option, x in +# config.config.items()] +# return holder class ConfigApi(BaseApi): """ Everything related to configuration. """ - def get_config_value(self, section, option): """ Retrieve config value. @@ -57,7 +53,7 @@ def set_config_value(self, section, option, value): # """ # data = {} # for section, config, values in self.pyload.config.iter_core_sections(): - # data[section] = to_config_holder(section, config, values) + # data[section] = to_config_holder(section, config, values) # return data # def get_core_config(self): @@ -67,7 +63,7 @@ def set_config_value(self, section, option, value): # :rtype: list of PluginInfo # """ # return [ConfigInfo(section, config.label, config.description, False, False) - # for section, config, values in self.pyload.config.iter_core_sections()] + # for section, config, values in self.pyload.config.iter_core_sections()] # @requireperm(Permission.Plugins) # def get_plugin_config(self): @@ -82,17 +78,17 @@ def set_config_value(self, section, option, value): # data = [] # active = [x.get_name() for x in self.pyload.adm.active_plugins()] # for name, config, values in self.pyload.config.iter_sections(): - # # skip unmodified and inactive addons - # if not values and name not in active: - # continue - - # item = ConfigInfo(name, config.label, config.description, - # self.pyload.pgm.get_category(name), - # self.pyload.pgm.is_user_plugin(name), - # # TODO: won't work probably - # values.get("activated", - # None if "activated" not in config.config else config.config['activated'].input.default)) - # data.append(item) + # # skip unmodified and inactive addons + # if not values and name not in active: + # continue + + # item = ConfigInfo(name, config.label, config.description, + # self.pyload.pgm.get_category(name), + # self.pyload.pgm.is_user_plugin(name), + # # TODO: won't work probably + # values.get("activated", + # None if "activated" not in config.config else config.config['activated'].input.default)) + # data.append(item) # return data @@ -105,9 +101,9 @@ def set_config_value(self, section, option, value): # """ # # TODO: filter user_context / addons when not allowed # plugins = [ConfigInfo(name, config.label, config.description, - # self.pyload.pgm.get_category(name), - # self.pyload.pgm.is_user_plugin(name)) - # for name, config, values in self.pyload.config.iter_sections()] + # self.pyload.pgm.get_category(name), + # self.pyload.pgm.is_user_plugin(name)) + # for name, config, values in self.pyload.config.iter_sections()] # return plugins @@ -132,7 +128,7 @@ def set_config_value(self, section, option, value): # :param config: :class:`ConfigHolder` # """ # for item in config.items: - # self.pyload.config.set(config.name, item.name, item.value, store=False) + # self.pyload.config.set(config.name, item.name, item.value, store=False) # # save the changes # self.pyload.config.store() diff --git a/src/pyload/core/api/core.py b/src/pyload/core/api/core.py index 3034b2f570..90e4977485 100644 --- a/src/pyload/core/api/core.py +++ b/src/pyload/core/api/core.py @@ -5,13 +5,13 @@ import os from future import standard_library + from pyload.utils.fs import availspace, lopen -from semver import format_version from ..datatype.init import Permission, StatusInfo from ..datatype.task import Interaction from .base import BaseApi -from .init import Api, requireperm +from .init import requireperm standard_library.install_aliases() @@ -48,9 +48,9 @@ def is_ws_secure(self): # Gets and address for the websocket based on configuration. # """ # if self.is_ws_secure(): - # ws = "wss" + # ws = "wss" # else: - # ws = "ws" + # ws = "ws" # return "{0}://{{0}}:{1:d}".format(ws, self.pyload.config.get('rpc', 'port')) @@ -69,7 +69,7 @@ def get_status_info(self): total[1], queue[1], self.is_interaction_waiting( Interaction.All), - not self.pyload.tsm.pause, #: and self.is_time_download(), + not self.pyload.tsm.pause, # and self.is_time_download(), self.pyload.tsm.pause, # and self.is_time_reconnect(), self.pyload.config.get( @@ -77,7 +77,7 @@ def get_status_info(self): self.get_quota()) for file in self.pyload.tsm.active_downloads(): - server_status.speed += file.get_speed() #: bytes/s + server_status.speed += file.get_speed() # bytes/s return server_status diff --git a/src/pyload/core/api/download.py b/src/pyload/core/api/download.py index c31b1e094a..1ccc3e24bc 100644 --- a/src/pyload/core/api/download.py +++ b/src/pyload/core/api/download.py @@ -6,12 +6,13 @@ from builtins import str from future import standard_library + from pyload.utils.fs import lopen from ..datatype.init import Permission from ..datatype.user import Role from .base import BaseApi -from .init import Api, requireperm +from .init import requireperm standard_library.install_aliases() @@ -105,8 +106,8 @@ def add_links(self, pid, links): if hoster: self.pyload.iom.create_info_thread(hoster, pid) - self.pyload.log.info( - (self._("Added {0:d} links to package") + " #{0:d}".format(pid)).format(len(hoster + crypter))) + self.pyload.log.info((self._( + "Added {0:d} links to package") + " #{0:d}".format(pid)).format(len(hoster + crypter))) self.pyload.files.save() @requireperm(Permission.Add) diff --git a/src/pyload/core/api/exchange.py b/src/pyload/core/api/exchange.py index 8779ccf13a..329657f523 100644 --- a/src/pyload/core/api/exchange.py +++ b/src/pyload/core/api/exchange.py @@ -7,7 +7,7 @@ from ..datatype.init import Permission from ..datatype.task import Interaction from .base import BaseApi -from .init import Api, requireperm +from .init import requireperm standard_library.install_aliases() diff --git a/src/pyload/core/api/file.py b/src/pyload/core/api/file.py index a54a459eb2..6a10fa137b 100644 --- a/src/pyload/core/api/file.py +++ b/src/pyload/core/api/file.py @@ -3,13 +3,14 @@ from __future__ import absolute_import, unicode_literals from future import standard_library + from pyload.utils.purge import uniqify from ..datatype.file import FileDoesNotExist from ..datatype.init import DownloadState, Permission from ..datatype.package import PackageDoesNotExist, PackageStatus from .base import BaseApi -from .init import Api, requireperm +from .init import requireperm standard_library.install_aliases() @@ -21,12 +22,12 @@ class FileApi(BaseApi): """ # def check_result(self, info): - # """ - # Internal method to verify result and owner. - # """ - # TODO: shared? - # return info and (not self.primary_uid or info.owner == - # self.primary_uid) + # """ + # Internal method to verify result and owner. + # """ + # TODO: shared? + # return info and (not self.primary_uid or info.owner == + # self.primary_uid) @requireperm(Permission.All) def get_all_files(self): diff --git a/src/pyload/core/api/init.py b/src/pyload/core/api/init.py index e11252d1f9..abb3f25967 100644 --- a/src/pyload/core/api/init.py +++ b/src/pyload/core/api/init.py @@ -4,9 +4,9 @@ from __future__ import absolute_import, unicode_literals from builtins import object, str +from types import MethodType from future import standard_library -from types import MethodType from ..datatype.init import DownloadState, DownloadStatus, Permission from ..datatype.user import User @@ -34,7 +34,7 @@ def __new__(cls, func, *args, **kwargs): statemap = { DownloadState.All: frozenset(getattr(DownloadStatus, x) for x in dir(DownloadStatus) if not x.startswith("_")), DownloadState.Finished: frozenset((DownloadStatus.Finished, DownloadStatus.Skipped)), - DownloadState.Unfinished: None, #: set below + DownloadState.Unfinished: None, # set below DownloadState.Failed: frozenset((DownloadStatus.Failed, DownloadStatus.TempOffline, DownloadStatus.Aborted, DownloadStatus.NotPossible, DownloadStatus.FileMismatch)), DownloadState.Unmanaged: None, @@ -60,8 +60,8 @@ class Api(AbstractApi): These can be configured via web interface. Admin user have all permissions, and are the only ones who can access the methods with no specific permission """ - EXTERNAL = AbstractApi #: let the json api know which methods are external - EXTEND = False #: only extendable when set too true + EXTERNAL = AbstractApi # let the json api know which methods are external + EXTEND = False # only extendable when set too true def __init__(self, core): self.pyload = core @@ -144,7 +144,8 @@ def check_auth(self, username, password, remoteip=None): :param remoteip: :return: dict with info, empty when login is incorrect """ - self.pyload.log.info(self._("User '{0}' tries to log in").format(username)) + self.pyload.log.info( + self._("User '{0}' tries to log in").format(username)) return self.pyload.db.check_auth(username, password) @@ -169,9 +170,8 @@ class UserApi(Api): """ Proxy object for api that provides all methods in user context. """ - def __init__(self, core, user): - #: No need to init super class + # No need to init super class self.pyload = core self._user = user diff --git a/src/pyload/core/api/predownload.py b/src/pyload/core/api/predownload.py index 6f165b3be8..c884cfdae8 100644 --- a/src/pyload/core/api/predownload.py +++ b/src/pyload/core/api/predownload.py @@ -8,6 +8,7 @@ from itertools import chain from future import standard_library + from pyload.utils import parse from pyload.utils.fs import lopen from pyload.utils.purge import uniqify @@ -15,7 +16,7 @@ from ..datatype.check import OnlineCheck from ..datatype.init import DownloadStatus, LinkStatus, Permission from .base import BaseApi -from .init import Api, requireperm +from .init import requireperm standard_library.install_aliases() diff --git a/src/pyload/core/api/stat.py b/src/pyload/core/api/stat.py index 781fc15a01..c35c68c803 100644 --- a/src/pyload/core/api/stat.py +++ b/src/pyload/core/api/stat.py @@ -5,7 +5,6 @@ from future import standard_library from .base import BaseApi -from .init import Api standard_library.install_aliases() @@ -18,7 +17,6 @@ class StatisticsApi(BaseApi): """ Retrieve download statistics and quota. """ - def record_download(self, file): """ Add download record to the statistics. diff --git a/src/pyload/core/api/user.py b/src/pyload/core/api/user.py index c464220356..5f8a58b598 100644 --- a/src/pyload/core/api/user.py +++ b/src/pyload/core/api/user.py @@ -6,7 +6,7 @@ from ..datatype.init import Permission from .base import BaseApi -from .init import Api, requireperm +from .init import requireperm standard_library.install_aliases() diff --git a/src/pyload/core/cli.py b/src/pyload/core/cli.py index 6e6dd8650e..bcb5cf1883 100644 --- a/src/pyload/core/cli.py +++ b/src/pyload/core/cli.py @@ -9,10 +9,9 @@ import sys from builtins import map +from future import standard_library from pkg_resources import resource_filename -from future import standard_library -from pyload.utils import format from pyload.utils.system import set_console_icon, set_console_title from . import iface @@ -23,6 +22,7 @@ try: import colorclass except ImportError: + colorclass = None autoblue = autogreen = autored = autowhite = autoyellow = lambda msg: msg else: for tag, reset, _, _ in (_f for _f in colorclass.list_tags() if _f): @@ -50,7 +50,7 @@ def _gen_logo(): \/ {0} """.format(text)) -logo = _gen_logo() +LOGO = _gen_logo() def parse_args(argv=None): @@ -58,7 +58,8 @@ def parse_args(argv=None): desc = autored("""Free and Open Source download manager written in Pure Python and designed to be extremely lightweight, fully customizable and remotely manageable""") - epilog = autogreen("""*** Visit https://pyload.net for further details ***""") + epilog = autogreen( + """*** Visit https://pyload.net for further details ***""") ap = argparse.ArgumentParser( prog=prog, description=desc, epilog=epilog, add_help=False) @@ -90,19 +91,26 @@ def parse_args(argv=None): for prsr in (pg, sp_start, sp_quit, sp_restart, sp_status, sp_version): prsr.add_argument( - '-h', '--help', action='help', help=autored("Show this help message and exit")) + '-h', '--help', action='help', + help=autored("Show this help message and exit")) for prsr in (pg, sp_start, sp_stop, sp_restart, sp_status): profile_help = ''.join( - autored("Profile name to use ("), autoyellow("`default`"), autored(" if missing)")) + autored("Profile name to use ("), + autoyellow("`default`"), + autored(" if missing)")) configdir_help = autored("Change path of config directory") prsr.add_argument('-p', '--profile', help=profile_help) prsr.add_argument('-c', '--configdir', help=configdir_help) for prsr in (pg, sp_start, sp_restart): tmpdir_help = autored("Change path of temp files directory") - debug_help = ''.join(autored("Enable debug mode ("), autoyellow("`-dd`"), autored(" for extended debug)")) - restore_help = ''.join(autored("Restore default login credentials "), autoyellow("`admin|pyload`"), autored(")")) + debug_help = ''.join(autored("Enable debug mode ("), autoyellow( + "`-dd`"), autored(" for extended debug)")) + restore_help = ''.join( + autored("Restore default login credentials "), + autoyellow("`admin|pyload`"), + autored(")")) daemon_help = autored("Run as daemon") prsr.add_argument('-t', '--tmpdir', help=tmpdir_help) prsr.add_argument('-d', '--debug', action='count', help=debug_help) @@ -115,7 +123,7 @@ def parse_args(argv=None): if not set(map(operator.itemgetter(0), sc)).intersection(argv): argv.append('start') - print(logo + os.linesep) + print(LOGO + os.linesep) return ap.parse_args(argv) @@ -129,7 +137,6 @@ def _setup_console(): def main(argv=sys.argv[1:]): - exc = None args = parse_args(argv) _setup_console() diff --git a/src/pyload/core/config/__init__.py b/src/pyload/core/config/__init__.py index 069549d3d8..440794f5b4 100644 --- a/src/pyload/core/config/__init__.py +++ b/src/pyload/core/config/__init__.py @@ -1,8 +1,4 @@ # -*- coding: utf-8 -*- # @author: vuolter -from __future__ import unicode_literals -from __future__ import absolute_import -from future import standard_library -standard_library.install_aliases() from .default import config_defaults, session_defaults diff --git a/src/pyload/core/config/default.py b/src/pyload/core/config/default.py index 5df0e1175c..23bdba0361 100644 --- a/src/pyload/core/config/default.py +++ b/src/pyload/core/config/default.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals from builtins import range from copy import deepcopy from future import standard_library + from pyload.config.types import InputType standard_library.install_aliases() @@ -46,88 +46,134 @@ def _gen_session_defaults(): # TODO: write descriptions def _gen_config_defaults(): general_config = ( - ('language', ('option', None, 'Language', None, (None, 'english'), InputType.Str)), - ('storage_folder', ('option', None, 'Storage folder', None, None, InputType.Folder)), - ('min_storage_size', ('option', 1024, 'Min storage space (in MiB)', None, None, InputType.Size)), - ('folder_pack', ('option', True, 'Create folder for each package', None, None, InputType.Bool)), - ('local_access', ('option', True, 'No authentication on local access', None, None, InputType.Bool)), - ('niceness', ('option', 0, 'Process priority', None, range(-19, 20), InputType.Int)), - ('ioniceness', ('option', 0, 'Process I/O priority', None, range(0, 3), InputType.Int)) + ('language', ('option', None, 'Language', + None, (None, 'english'), InputType.Str)), + ('storage_folder', ('option', None, 'Storage folder', + None, None, InputType.Folder)), + ('min_storage_size', ('option', 1024, 'Min storage space (in MiB)', + None, None, InputType.Size)), + ('folder_pack', ('option', True, 'Create folder for each package', + None, None, InputType.Bool)), + ('local_access', ('option', True, 'No authentication on local access', + None, None, InputType.Bool)), + ('niceness', ('option', 0, 'Process priority', + None, range(-19, 20), InputType.Int)), + ('ioniceness', ('option', 0, 'Process I/O priority', + None, range(0, 3), InputType.Int)) ) log_config = ( - ('activated', ('option', True, 'Activated', None, None, InputType.Bool)), - ('syslog', ('option', None, 'Sent log to syslog', None, (None, 'remote', 'local'), InputType.Str)), - ('syslog_folder', ('option', None, 'Syslog local folder', None, None, InputType.Folder)), - ('syslog_host', ('option', 'localhost:514', 'Syslog remote IP address', None, None, InputType.Address)), - ('logfile', ('option', False, 'Save log to file', None, None, InputType.Bool)), - ('logfile_size', ('option', 100, 'Max file size (in KiB)', None, None, InputType.Size)), - ('logfile_folder', ('option', None, 'File folder', None, None, InputType.Foider)), - ('logfile_name', ('option', None, 'File name', None, None, InputType.File)), - ('max_logfiles', ('option', 5, 'Max log files', None, None, InputType.Int)), - ('rotate', ('option', True, 'Log rotate', None, None, InputType.Bool)), - ('debug', ('option', False, 'Debug mode', None, None, InputType.Bool)), - ('verbose', ('option', False, 'Verbose mode', None, None, InputType.Bool)), - ('color_console', ('option', True, 'Color console', None, None, InputType.Bool)) + ('activated', ('option', True, 'Activated', + None, None, InputType.Bool)), + ('syslog', ('option', None, 'Sent log to syslog', + None, (None, 'remote', 'local'), InputType.Str)), + ('syslog_folder', ('option', None, 'Syslog local folder', + None, None, InputType.Folder)), + ('syslog_host', ('option', 'localhost:514', 'Syslog remote IP address', + None, None, InputType.Address)), + ('logfile', ('option', False, 'Save log to file', + None, None, InputType.Bool)), + ('logfile_size', ('option', 100, 'Max file size (in KiB)', + None, None, InputType.Size)), + ('logfile_folder', ('option', None, 'File folder', + None, None, InputType.Foider)), + ('logfile_name', ('option', None, 'File name', + None, None, InputType.File)), + ('max_logfiles', ('option', 5, 'Max log files', + None, None, InputType.Int)), + ('rotate', ('option', True, 'Log rotate', + None, None, InputType.Bool)), + ('debug', ('option', False, 'Debug mode', + None, None, InputType.Bool)), + ('verbose', ('option', False, 'Verbose mode', + None, None, InputType.Bool)), + ('color_console', ('option', True, 'Color console', + None, None, InputType.Bool)) ) perm_config = ( ('user', ('option', 'user', 'Username', None, None, InputType.Str)), ('group', ('option', 'users', 'Groupname', None, None, InputType.Str)), - ('foldermode', ('option', 0o755, 'Folder mode', None, None, InputType.Octal)), - ('filemode', ('option', 0o644, 'File mode', None, None, InputType.Octal)), - ('change_user', ('option', False, 'Change user of pyLoad process', None, None, InputType.Bool)), - ('change_group', ('option', False, 'Change group of pyLoad process', None, None, InputType.Bool)), - ('change_fileowner', ('option', False, 'Change user and group of saved files', None, None, InputType.Bool)), - ('change_filemode', ('option', False, 'Change file mode of saved files', None, None, InputType.Bool)) + ('foldermode', ('option', 0o755, 'Folder mode', + None, None, InputType.Octal)), + ('filemode', ('option', 0o644, 'File mode', + None, None, InputType.Octal)), + ('change_user', ('option', False, 'Change user of pyLoad process', + None, None, InputType.Bool)), + ('change_group', ('option', False, 'Change group of pyLoad process', + None, None, InputType.Bool)), + ('change_fileowner', ('option', False, 'Change user and group of saved files', + None, None, InputType.Bool)), + ('change_filemode', ('option', False, 'Change file mode of saved files', + None, None, InputType.Bool)) ) conn_config = ( - ('max_transfers', ('option', 5, 'Max parallel transfers', None, None, InputType.Int)), - ('max_speed', ('option', -1, 'Max transfer speed (in KiB/s)', None, None, InputType.Size)), - ('max_chunks', ('option', -1, 'Max connections for single transfer', None, None, InputType.Int)), - ('wait', ('option', 2, 'Active transfers while waiting', None, None, InputType.Int)), # TODO: Recheck... - ('skip', ('option', False, 'Skip existing files', None, None, InputType.Bool)), - ('preallocate', ('option', True, 'Pre-allocate files on disk', None, None, InputType.Bool)), - ('interface', ('option', None, 'Interface address to bind', None, None, InputType.Address)), + ('max_transfers', ('option', 5, 'Max parallel transfers', + None, None, InputType.Int)), + ('max_speed', ('option', -1, 'Max transfer speed (in KiB/s)', + None, None, InputType.Size)), + ('max_chunks', ('option', -1, 'Max connections for single transfer', + None, None, InputType.Int)), + # TODO: Recheck... + ('wait', ('option', 2, 'Active transfers while waiting', + None, None, InputType.Int)), + ('skip', ('option', False, 'Skip existing files', + None, None, InputType.Bool)), + ('preallocate', ('option', True, 'Pre-allocate files on disk', + None, None, InputType.Bool)), + ('interface', ('option', None, 'Interface address to bind', + None, None, InputType.Address)), ('ipv6', ('option', False, 'Allow IPv6', None, None, InputType.Bool)), ) ssl_config = ( - ('activated', ('option', False, 'Activated', None, None, InputType.Bool)), - ('cert', ('option', 'ssl.crt', 'Cert file', None, None, InputType.File)), + ('activated', ('option', False, 'Activated', + None, None, InputType.Bool)), + ('cert', ('option', 'ssl.crt', 'Cert file', + None, None, InputType.File)), ('key', ('option', 'ssl.key', 'Key file', None, None, InputType.File)) ) reconn_config = ( - ('activated', ('option', False, 'Activated', None, None, InputType.Bool)), + ('activated', ('option', False, 'Activated', + None, None, InputType.Bool)), ('script', ('option', None, 'Script file', None, None, InputType.File)), - ('wait', ('option', False, 'Don\'t reconnect while waiting', None, None, InputType.Bool)) + ('wait', ('option', False, 'Don\'t reconnect while waiting', + None, None, InputType.Bool)) ) proxy_config = ( - ('activated', ('option', False, 'Activated', None, None, InputType.Bool)), - ('type', ('option', 'http', 'Protocol', None, ('http', 'socks4', 'socks5'), InputType.Str)), - ('host', ('option', 'localhost:7070', 'IP address', None, None, InputType.Address)), + ('activated', ('option', False, 'Activated', + None, None, InputType.Bool)), + ('type', ('option', 'http', 'Protocol', None, + ('http', 'socks4', 'socks5'), InputType.Str)), + ('host', ('option', 'localhost:7070', + 'IP address', None, None, InputType.Address)), ('username', ('option', None, 'Username', None, None, InputType.Str)), - ('password', ('option', None, 'Password', None, None, InputType.Password)) + ('password', ('option', None, 'Password', + None, None, InputType.Password)) ) up_config = ( - ('activated', ('option', False, 'Activated', None, None, InputType.Bool)), - ('nodebug', ('option', False, 'Don\'t update in debug mode', None, None, InputType.Bool)), - ('periodical', ('option', True, 'Check for updates on schedule', None, None, InputType.Bool)), - ('interval', ('option', 1, 'Check interval (in days)', None, None, InputType.Int)) + ('activated', ('option', False, 'Activated', + None, None, InputType.Bool)), + ('nodebug', ('option', False, 'Don\'t update in debug mode', + None, None, InputType.Bool)), + ('periodical', ('option', True, 'Check for updates on schedule', + None, None, InputType.Bool)), + ('interval', ('option', 1, 'Check interval (in days)', + None, None, InputType.Int)) ) # self.config.add_section('webui', 'Web User Interface', desc, expl, # [ - # ('activated', 'bool', 'Activated', True), - # ('server', ('auto', 'threaded', 'fallback', 'fastcgi'), 'Webserver', 'auto'), - # ('host', 'str', 'IP address', 'localhost'), - # ('port', 'port', 'Port', 8010), - # ('force_server', 'str', 'Forced webserver', None), - # ('external', 'bool', 'Served external', False), - # ('prefix', 'str', 'Path prefix', None), - # ('debug', 'bool', 'Debug mode', False) + # ('activated', 'bool', 'Activated', True), + # ('server', ('auto', 'threaded', 'fallback', 'fastcgi'), 'Webserver', 'auto'), + # ('host', 'str', 'IP address', 'localhost'), + # ('port', 'port', 'Port', 8010), + # ('force_server', 'str', 'Forced webserver', None), + # ('external', 'bool', 'Served external', False), + # ('prefix', 'str', 'Path prefix', None), + # ('debug', 'bool', 'Debug mode', False) # ]) # self.config.add_section('rpc', 'REST API Interface', desc, expl, # [ - # ('activated', 'bool', 'Activated', False), - # ('host', 'str', 'IP address', '0.0.0.0'), - # ('port', 'port', 'Port', 7227) + # ('activated', 'bool', 'Activated', False), + # ('host', 'str', 'IP address', '0.0.0.0'), + # ('port', 'port', 'Port', 7227) # ]) defaults = ( diff --git a/src/pyload/core/database/__init__.py b/src/pyload/core/database/__init__.py index 035e6808b6..4c814706ba 100644 --- a/src/pyload/core/database/__init__.py +++ b/src/pyload/core/database/__init__.py @@ -1,10 +1,6 @@ # -*- coding: utf-8 -*- # @author: vuolter -from __future__ import unicode_literals -from __future__ import absolute_import -from future import standard_library -standard_library.install_aliases() from .backend import DatabaseBackend, DatabaseMethods, async, inner, queue from .account import AccountMethods from .config import ConfigMethods diff --git a/src/pyload/core/database/account.py b/src/pyload/core/database/account.py index 63f9a2a78e..426e4f209a 100644 --- a/src/pyload/core/database/account.py +++ b/src/pyload/core/database/account.py @@ -12,7 +12,6 @@ class AccountMethods(DatabaseMethods): - @queue def load_accounts(self): self.c.execute( @@ -23,8 +22,9 @@ def load_accounts(self): @queue def create_account(self, plugin, loginname, password, owner): - self.c.execute('INSERT INTO accounts(plugin, loginname, password, owner) VALUES(?,?,?,?)', - (plugin, loginname, password, owner)) + self.c.execute( + 'INSERT INTO accounts(plugin, loginname, password, owner) VALUES(?,?,?,?)', + (plugin, loginname, password, owner)) return self.c.lastrowid diff --git a/src/pyload/core/database/backend.py b/src/pyload/core/database/backend.py index ca6255a118..2e51916257 100644 --- a/src/pyload/core/database/backend.py +++ b/src/pyload/core/database/backend.py @@ -5,13 +5,14 @@ import os import shutil -from builtins import int, object, range, str +from builtins import int, object, str +from queue import Queue from traceback import print_exc from future import standard_library + from pyload.utils.fs import lopen, remove from pyload.utils.layer.safethreading import Event, Thread -from queue import Queue standard_library.install_aliases() @@ -128,7 +129,7 @@ def __init__(self, core): self.setDaemon(True) self.pyload = core self._ = core._ - self.manager = None #: set later + self.manager = None # set later self.error = None self.__running = Event() @@ -228,12 +229,12 @@ def _convert_db(self, v): except Exception: return False - #--convert scripts start + # -- convert scripts start -- def _convert_v6(self): return False - #--convert scripts end + # -- convert scripts end -- def _create_tables(self): """ @@ -247,11 +248,11 @@ def _create_tables(self): '"site" TEXT DEFAULT "" NOT NULL, ' '"comment" TEXT DEFAULT "" NOT NULL, ' '"password" TEXT DEFAULT "" NOT NULL, ' - '"added" INTEGER DEFAULT 0 NOT NULL,' #: set by trigger + '"added" INTEGER DEFAULT 0 NOT NULL,' # set by trigger '"status" INTEGER DEFAULT 0 NOT NULL,' '"tags" TEXT DEFAULT "" NOT NULL,' '"shared" INTEGER DEFAULT 0 NOT NULL,' - '"packageorder" INTEGER DEFAULT -1 NOT NULL,' #: incremented by trigger + '"packageorder" INTEGER DEFAULT -1 NOT NULL,' # incremented by trigger '"root" INTEGER DEFAULT -1 NOT NULL, ' '"owner" INTEGER NOT NULL, ' 'FOREIGN KEY(owner) REFERENCES users(uid), ' @@ -265,16 +266,14 @@ def _create_tables(self): 'UPDATE packages SET added = strftime("%s", "now"), ' 'packageorder = (SELECT max(p.packageorder) + 1 FROM packages p WHERE p.root=new.root) ' 'WHERE rowid = new.rowid;' - 'END' - ) + 'END') self.c.execute( 'CREATE TRIGGER IF NOT EXISTS "delete_package" AFTER DELETE ON "packages"' 'BEGIN ' 'DELETE FROM files WHERE package = old.pid;' 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > old.packageorder AND root=old.pid;' - 'END' - ) + 'END') self.c.execute( 'CREATE INDEX IF NOT EXISTS "package_index" ON packages(root, owner)') self.c.execute( @@ -313,8 +312,7 @@ def _create_tables(self): 'UPDATE files SET added = strftime("%s", "now"), ' 'fileorder = (SELECT max(f.fileorder) + 1 FROM files f WHERE f.package=new.package) ' 'WHERE rowid = new.rowid;' - 'END' - ) + 'END') self.c.execute( 'CREATE TABLE IF NOT EXISTS "collector" (' @@ -348,7 +346,7 @@ def _create_tables(self): '"dlquota" TEXT DEFAULT "" NOT NULL, ' '"hddquota" INTEGER DEFAULT -1 NOT NULL, ' '"template" TEXT DEFAULT "default" NOT NULL, ' - '"user" INTEGER DEFAULT -1 NOT NULL, ' #: set by trigger to self + '"user" INTEGER DEFAULT -1 NOT NULL, ' # set by trigger to self 'FOREIGN KEY(user) REFERENCES users(uid)' ')' ) @@ -360,8 +358,7 @@ def _create_tables(self): 'BEGIN ' 'UPDATE users SET user = new.uid, folder=new.name ' 'WHERE rowid = new.rowid;' - 'END' - ) + 'END') self.c.execute( 'CREATE TABLE IF NOT EXISTS "settings" (' diff --git a/src/pyload/core/database/config.py b/src/pyload/core/database/config.py index e4936ad60d..a2b49b0c3a 100644 --- a/src/pyload/core/database/config.py +++ b/src/pyload/core/database/config.py @@ -11,20 +11,21 @@ class ConfigMethods(DatabaseMethods): - @async def save_config(self, plugin, config, user=None): if user is None: user = -1 self.c.execute( - 'INSERT INTO settings(plugin, config, user) VALUES(?,?,?)', (plugin, config, user)) + 'INSERT INTO settings(plugin, config, user) VALUES(?,?,?)', + (plugin, config, user)) @queue def load_config(self, plugin, user=None): if user is None: user = -1 self.c.execute( - 'SELECT config FROM settings WHERE plugin=? AND user=?', (plugin, user)) + 'SELECT config FROM settings WHERE plugin=? AND user=?', + (plugin, user)) r = self.c.fetchone() return r[0] if r else "" @@ -35,7 +36,8 @@ def delete_config(self, plugin, user=None): self.c.execute('DELETE FROM settings WHERE plugin=?', (plugin,)) else: self.c.execute( - 'DELETE FROM settings WHERE plugin=? AND user=?', (plugin, user)) + 'DELETE FROM settings WHERE plugin=? AND user=?', + (plugin, user)) @queue def load_all_configs(self): diff --git a/src/pyload/core/database/file.py b/src/pyload/core/database/file.py index 17f67c3fea..4a05fa7345 100644 --- a/src/pyload/core/database/file.py +++ b/src/pyload/core/database/file.py @@ -6,6 +6,7 @@ from builtins import int from future import standard_library + from pyload.utils.layer.legacy.collections_ import OrderedDict from ..api.init import statestring @@ -22,7 +23,6 @@ class FileMethods(DatabaseMethods): - @queue def filecount(self): """ @@ -74,7 +74,8 @@ def processcount(self, fid=-1, user=None): """ # status in online, queued, starting, waiting, downloading self.c.execute( - "SELECT COUNT(*), SUM(size) FROM files WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (fid,)) + "SELECT COUNT(*), SUM(size) FROM files WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", + (fid,)) return self.c.fetchone()[0] @queue @@ -92,8 +93,9 @@ def processstats(self, user=None): @queue def add_link(self, url, name, plugin, package, owner): # mark file status initially as missing, dlstatus - queued - self.c.execute('INSERT INTO files(url, name, plugin, status, dlstatus, package, owner) VALUES(?,?,?,1,3,?,?)', - (url, name, plugin, package, owner)) + self.c.execute( + 'INSERT INTO files(url, name, plugin, status, dlstatus, package, owner) VALUES(?,?,?,1,3,?,?)', + (url, name, plugin, package, owner)) return self.c.lastrowid @async @@ -109,15 +111,17 @@ def add_links(self, links, package, owner): @queue def add_file(self, name, size, media, package, owner): # file status - ok, dl status NA - self.c.execute('INSERT INTO files(name, size, media, package, owner) VALUES(?,?,?,?,?)', - (name, size, media, package, owner)) + self.c.execute( + 'INSERT INTO files(name, size, media, package, owner) VALUES(?,?,?,?,?)', + (name, size, media, package, owner)) return self.c.lastrowid @queue def add_package(self, name, folder, root, password, site, comment, status, owner): self.c.execute( - 'INSERT INTO packages(name, folder, root, password, site, comment, status, owner) VALUES(?,?,?,?,?,?,?,?)', (name, folder, root, password, site, comment, status, owner)) + 'INSERT INTO packages(name, folder, root, password, site, comment, status, owner) VALUES(?,?,?,?,?,?,?,?)', + (name, folder, root, password, site, comment, status, owner)) return self.c.lastrowid @async @@ -136,13 +140,15 @@ def delete_file(self, fid, order, package, owner=None): """ if owner is None: self.c.execute('DELETE FROM files WHERE fid=?', (fid,)) - self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=?', - (order, package)) + self.c.execute( + 'UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=?', + (order, package)) else: self.c.execute( 'DELETE FROM files WHERE fid=? AND owner=?', (fid, owner)) - self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=? AND owner=?', - (order, package, owner)) + self.c.execute( + 'UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=? AND owner=?', + (order, package, owner)) @queue def get_all_files(self, package=None, search=None, state=None, owner=None): @@ -183,10 +189,16 @@ def get_all_files(self, package=None, search=None, state=None, owner=None): data = OrderedDict() for r in self.c.fetchall(): - finfo = FileInfo(r[0], r[1], r[13], r[2], r[3], r[4], r[5], r[6], r[7]) - if r[11] > 0: #: dl status != NA + finfo = FileInfo(r[0], r[1], r[13], r[2], + r[3], r[4], r[5], r[6], r[7]) + if r[11] > 0: # dl status != NA finfo.download = DownloadInfo( - r[8], r[9], r[10], r[11], self.manager.status_msg[r[11]], r[12]) + r[8], + r[9], + r[10], + r[11], + self.manager.status_msg[r[11]], + r[12]) data[r[0]] = finfo return data @@ -230,8 +242,9 @@ def get_all_packages(self, root=None, owner=None, tags=None): self.c.execute(qry.format( ' WHERE root=? OR pid=?'), (root, root)) else: - self.c.execute(qry.format( - ' WHERE (root=? OR pid=?) AND owner=?'), (root, root, owner)) + self.c.execute( + qry.format(' WHERE (root=? OR pid=?) AND owner=?'), + (root, root, owner)) data = OrderedDict() for r in self.c.fetchall(): @@ -244,11 +257,12 @@ def get_all_packages(self, root=None, owner=None, tags=None): @inner def get_package_stats(self, pid=None, root=None, owner=None): - qry = ("SELECT p.pid, SUM(f.size) AS sizetotal, COUNT(f.fid) AS linkstotal, sizedone, linksdone " - "FROM packages p JOIN files f ON p.pid = f.package AND f.dlstatus > 0 {0} LEFT OUTER JOIN " - "(SELECT p.pid AS pid, SUM(f.size) AS sizedone, COUNT(f.fid) AS linksdone " - "FROM packages p JOIN files f ON p.pid = f.package {0} AND f.dlstatus in (5,6) GROUP BY p.pid) s ON s.pid = p.pid " - "GROUP BY p.pid") + qry = ( + "SELECT p.pid, SUM(f.size) AS sizetotal, COUNT(f.fid) AS linkstotal, sizedone, linksdone " + "FROM packages p JOIN files f ON p.pid = f.package AND f.dlstatus > 0 {0} LEFT OUTER JOIN " + "(SELECT p.pid AS pid, SUM(f.size) AS sizedone, COUNT(f.fid) AS linksdone " + "FROM packages p JOIN files f ON p.pid = f.package {0} AND f.dlstatus in (5,6) GROUP BY p.pid) s ON s.pid = p.pid " + "GROUP BY p.pid") # status in (finished, skipped, processing) @@ -282,16 +296,22 @@ def get_file_info(self, fid, force=False): """ Get data for specific file, when force is true download info will be appended. """ - self.c.execute('SELECT fid, name, owner, size, status, media, added, fileorder, ' - 'url, plugin, hash, dlstatus, error, package FROM files ' - 'WHERE fid=?', (fid,)) + self.c.execute( + 'SELECT fid, name, owner, size, status, media, added, fileorder, ' + 'url, plugin, hash, dlstatus, error, package FROM files ' + 'WHERE fid=?', (fid,)) r = self.c.fetchone() if not r: return None finfo = FileInfo(r[0], r[1], r[13], r[2], r[3], r[4], r[5], r[6], r[7]) if r[11] > 0 or force: finfo.download = DownloadInfo( - r[8], r[9], r[10], r[11], self.manager.status_msg[r[11]], r[12]) + r[8], + r[9], + r[10], + r[11], + self.manager.status_msg[r[11]], + r[12]) return finfo @queue @@ -337,10 +357,11 @@ def update_link_info(self, data): @async def update_file(self, f): - self.c.execute('UPDATE files SET name=?, size=?, status=?,' - 'media=?, url=?, hash=?, dlstatus=?, error=? WHERE fid=?', - (f.name, f.size, f.filestatus, f.media, f.url, - f.hash, f.status, f.error, f.fid)) + self.c.execute( + 'UPDATE files SET name=?, size=?, status=?,' + 'media=?, url=?, hash=?, dlstatus=?, error=? WHERE fid=?', + (f.name, f.size, f.filestatus, f.media, f.url, f.hash, + f.status, f.error, f.fid)) @async def set_download_status(self, fid, status): @@ -357,12 +378,14 @@ def update_package(self, p): # beforehand @async def order_package(self, pid, root, oldorder, order): - if oldorder > order: #: package moved upwards + if oldorder > order: # package moved upwards self.c.execute( - 'UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND root=? AND packageorder >= 0', (order, oldorder, root)) - elif oldorder < order: #: moved downwards + 'UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND root=? AND packageorder >= 0', + (order, oldorder, root)) + elif oldorder < order: # moved downwards self.c.execute( - 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND root=? AND packageorder >= 0', (order, oldorder, root)) + 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND root=? AND packageorder >= 0', + (order, oldorder, root)) self.c.execute( 'UPDATE packages SET packageorder=? WHERE pid=?', (order, pid)) @@ -372,13 +395,15 @@ def order_files(self, pid, fids, oldorder, order): diff = len(fids) data = [] - if oldorder > order: #: moved upwards + if oldorder > order: # moved upwards self.c.execute( - 'UPDATE files SET fileorder=fileorder+? WHERE fileorder >= ? AND fileorder < ? AND package=?', (diff, order, oldorder, pid)) + 'UPDATE files SET fileorder=fileorder+? WHERE fileorder >= ? AND fileorder < ? AND package=?', + (diff, order, oldorder, pid)) data = [(order + i, fid) for i, fid in enumerate(fids)] elif oldorder < order: self.c.execute( - 'UPDATE files SET fileorder=fileorder-? WHERE fileorder <= ? AND fileorder >= ? AND package=?', (diff, order, oldorder + diff, pid)) + 'UPDATE files SET fileorder=fileorder-? WHERE fileorder <= ? AND fileorder >= ? AND package=?', + (diff, order, oldorder + diff, pid)) data = [(order - diff + i + 1, fid) for i, fid in enumerate(fids)] self.c.executemany('UPDATE files SET fileorder=? WHERE fid=?', data) @@ -390,8 +415,9 @@ def move_files(self, pid, fids, package): r = self.c.fetchone() order = (r[0] if r[0] else 0) + 1 - self.c.execute('UPDATE files SET fileorder=fileorder-? WHERE fileorder > ? AND package=?', - (len(fids), order, pid)) + self.c.execute( + 'UPDATE files SET fileorder=fileorder-? WHERE fileorder > ? AND package=?', + (len(fids), order, pid)) data = [(package, order + i, fid) for i, fid in enumerate(fids)] self.c.executemany( @@ -404,11 +430,13 @@ def move_package(self, root, order, pid, dpid): r = self.c.fetchone() max = (r[0] if r[0] else 0) + 1 - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND root=?', - (order, root)) + self.c.execute( + 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND root=?', + (order, root)) self.c.execute( - 'UPDATE packages SET root=?, packageorder=? WHERE pid=?', (dpid, max, pid)) + 'UPDATE packages SET root=?, packageorder=? WHERE pid=?', + (dpid, max, pid)) @async def restart_file(self, fid): @@ -429,9 +457,11 @@ def get_jobs(self, occ): cmd = "({0})".format(", ".join("'{0}'".format(x) for x in occ)) # dlstatus in online, queued, occupied | package status = ok - cmd = ("SELECT f.owner, f.fid FROM files as f INNER JOIN packages as p ON f.package=p.pid " - "WHERE f.owner=? AND f.plugin NOT IN {} AND f.dlstatus IN (2,3,16) AND p.status=0 " - "ORDER BY p.packageorder ASC, f.fileorder ASC LIMIT 1").format(cmd) + cmd = ( + "SELECT f.owner, f.fid FROM files as f INNER JOIN packages as p ON f.package=p.pid " + "WHERE f.owner=? AND f.plugin NOT IN {} AND f.dlstatus IN (2,3,16) AND p.status=0 " + "ORDER BY p.packageorder ASC, f.fileorder ASC LIMIT 1").format( + cmd) self.c.execute("SELECT uid FROM users") uids = self.c.fetchall() @@ -452,7 +482,8 @@ def get_unfinished(self, pid): """ # status in finished, skipped, processing self.c.execute( - "SELECT fid FROM files WHERE package=? AND dlstatus NOT IN (5, 6, 14) LIMIT 3", (pid,)) + "SELECT fid FROM files WHERE package=? AND dlstatus NOT IN (5, 6, 14) LIMIT 3", + (pid,)) return [r[0] for r in self.c.fetchall()] @queue @@ -469,7 +500,8 @@ def find_duplicates(self, id, folder, filename): """ # TODO: also check root of package self.c.execute( - "SELECT f.plugin FROM files f INNER JOIN packages as p ON f.package=p.pid AND p.folder=? WHERE f.fid!=? AND f.dlstatus=5 AND f.name=?", (folder, id, filename)) + "SELECT f.plugin FROM files f INNER JOIN packages as p ON f.package=p.pid AND p.folder=? WHERE f.fid!=? AND f.dlstatus=5 AND f.name=?", + (folder, id, filename)) return self.c.fetchone() @queue @@ -478,7 +510,7 @@ def purge_links(self): self.c.execute("DELETE FROM files WHERE status == 1") @queue - def purge_all(self): #: only used for debugging + def purge_all(self): # only used for debugging self.c.execute("DELETE FROM packages") self.c.execute("DELETE FROM files") self.c.execute("DELETE FROM collector") diff --git a/src/pyload/core/database/storage.py b/src/pyload/core/database/storage.py index ab744a1e69..a2e86eeb11 100644 --- a/src/pyload/core/database/storage.py +++ b/src/pyload/core/database/storage.py @@ -14,29 +14,33 @@ class StorageMethods(object): - @queue def set_storage(self, db, identifier, key, value): db.c.execute( - "SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) + "SELECT id FROM storage WHERE identifier=? AND key=?", + (identifier, key)) if db.c.fetchone() is not None: db.c.execute( - "UPDATE storage SET value=? WHERE identifier=? AND key=?", (value, identifier, key)) + "UPDATE storage SET value=? WHERE identifier=? AND key=?", + (value, identifier, key)) else: db.c.execute( - "INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value)) + "INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", + (identifier, key, value)) @queue def get_storage(self, db, identifier, key=None): if key is not None: db.c.execute( - "SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key)) + "SELECT value FROM storage WHERE identifier=? AND key=?", + (identifier, key)) row = db.c.fetchone() if row is not None: return row[0] else: db.c.execute( - "SELECT key, value FROM storage WHERE identifier=?", (identifier,)) + "SELECT key, value FROM storage WHERE identifier=?", + (identifier,)) d = {} for row in db.c: d[row[0]] = row[1] @@ -45,6 +49,8 @@ def get_storage(self, db, identifier, key=None): @queue def del_storage(self, db, identifier, key): db.c.execute( - "DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key)) + "DELETE FROM storage WHERE identifier=? AND key=?", + (identifier, key)) + DatabaseBackend.register_sub(StorageMethods) diff --git a/src/pyload/core/database/user.py b/src/pyload/core/database/user.py index 01fa3a123f..5eeb36c6c0 100644 --- a/src/pyload/core/database/user.py +++ b/src/pyload/core/database/user.py @@ -25,7 +25,6 @@ def random_salt(): class UserMethods(DatabaseMethods): - @queue def add_user(self, user, password, role, permission): salt = random_salt() @@ -34,25 +33,29 @@ def add_user(self, user, password, role, permission): self.c.execute('SELECT name FROM users WHERE name=?', (user,)) if self.c.fetchone() is not None: - self.c.execute('UPDATE users SET password=?, role=?, permission=? WHERE name=?', - (password, role, permission, user)) + self.c.execute( + 'UPDATE users SET password=?, role=?, permission=? WHERE name=?', + (password, role, permission, user)) else: - self.c.execute('INSERT INTO users (name, role, permission, password) VALUES (?, ?, ?, ?)', - (user, role, permission, password)) + self.c.execute( + 'INSERT INTO users (name, role, permission, password) VALUES (?, ?, ?, ?)', + (user, role, permission, password)) @queue def add_debug_user(self, uid): # just add a user with uid to db try: - self.c.execute('INSERT INTO users (uid, name, password) VALUES (?, ?, ?)', - (uid, "debugUser", random_salt())) + self.c.execute( + 'INSERT INTO users (uid, name, password) VALUES (?, ?, ?)', + (uid, "debugUser", random_salt())) except Exception: pass @queue def get_user_data(self, name=None, uid=None, role=None): - qry = ('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' - 'hddquota, user, template FROM "users" WHERE ') + qry = ( + 'SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' + 'hddquota, user, template FROM "users" WHERE ') if name is not None: self.c.execute(qry + "name=?", (name,)) @@ -76,8 +79,9 @@ def get_user_data(self, name=None, uid=None, role=None): @queue def get_all_user_data(self): - self.c.execute('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' - 'hddquota, user, template FROM "users"') + self.c.execute( + 'SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' + 'hddquota, user, template FROM "users"') user = {} for r in self.c.fetchall(): user[r[0]] = UserData(*r) @@ -86,8 +90,10 @@ def get_all_user_data(self): @queue def check_auth(self, user, password): - self.c.execute('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' - 'hddquota, user, template, password FROM "users" WHERE name=?', (user,)) + self.c.execute( + 'SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' + 'hddquota, user, template, password FROM "users" WHERE name=?', + (user,)) r = self.c.fetchone() if not r: return None diff --git a/src/pyload/core/datatype/__init__.py b/src/pyload/core/datatype/__init__.py index 2a100e75ed..bc80d24278 100644 --- a/src/pyload/core/datatype/__init__.py +++ b/src/pyload/core/datatype/__init__.py @@ -1,10 +1,6 @@ # -*- coding: utf-8 -*- # @author: vuolter -from __future__ import unicode_literals -from __future__ import absolute_import -from future import standard_library -standard_library.install_aliases() from .init import * from .check import OnlineCheck from .file import File, FileDoesNotExist, FileInfo, FileStatus diff --git a/src/pyload/core/datatype/check.py b/src/pyload/core/datatype/check.py index bd32227afe..ae429d9b37 100644 --- a/src/pyload/core/datatype/check.py +++ b/src/pyload/core/datatype/check.py @@ -3,7 +3,6 @@ from __future__ import absolute_import, unicode_literals import time -from builtins import object from future import standard_library diff --git a/src/pyload/core/datatype/file.py b/src/pyload/core/datatype/file.py index 40f777b984..c724364b28 100644 --- a/src/pyload/core/datatype/file.py +++ b/src/pyload/core/datatype/file.py @@ -5,9 +5,10 @@ import re import time -from builtins import int, object +from builtins import int from future import standard_library + from pyload.utils import purge from pyload.utils.decorator import trycatch from pyload.utils.struct.lock import RWLock, lock @@ -47,13 +48,20 @@ 'unknown': 20 } filetypes = { - MediaType.Audio: re.compile(r'\.(m3u|m4a|mp3|wav|wma|aac?|flac|midi|m4b)$', flags=re.I), - MediaType.Image: re.compile(r'\.(jpe?g|bmp|png|gif|ico|tiff?|svg|psd)$', flags=re.I), - MediaType.Video: re.compile(r'\.(3gp|flv|m4v|avi|mp4|mov|swf|vob|wmv|divx|mpe?g|rm|mkv)$', flags=re.I), - MediaType.Document: re.compile(r'\.(epub|mobi|acsm|azw[0-9]|pdf|txt|md|abw|docx?|tex|odt|rtf||log)$', flags=re.I), - MediaType.Archive: re.compile(r'\.(rar|r[0-9]+|7z|7z.[0-9]+|zip|gz|bzip2?|tar|lzma)$', flags=re.I), - MediaType.Executable: re.compile(r'\.(jar|exe|dmg|sh|apk)$', flags=re.I), -} + MediaType.Audio: re.compile( + r'\.(m3u|m4a|mp3|wav|wma|aac?|flac|midi|m4b)$', flags=re.I), + MediaType.Image: re.compile( + r'\.(jpe?g|bmp|png|gif|ico|tiff?|svg|psd)$', flags=re.I), + MediaType.Video: re.compile( + r'\.(3gp|flv|m4v|avi|mp4|mov|swf|vob|wmv|divx|mpe?g|rm|mkv)$', + flags=re.I), + MediaType.Document: re.compile( + r'\.(epub|mobi|acsm|azw[0-9]|pdf|txt|md|abw|docx?|tex|odt|rtf||log)$', + flags=re.I), + MediaType.Archive: re.compile( + r'\.(rar|r[0-9]+|7z|7z.[0-9]+|zip|gz|bzip2?|tar|lzma)$', flags=re.I), + MediaType.Executable: re.compile( + r'\.(jar|exe|dmg|sh|apk)$', flags=re.I), } def guess_type(name): @@ -80,8 +88,9 @@ class FileInfo(BaseObject): __slots__ = ['fid', 'name', 'package', 'owner', 'size', 'status', 'media', 'added', 'fileorder', 'download'] - def __init__(self, fid=None, name=None, package=None, owner=None, size=None, - status=None, media=None, added=None, fileorder=None, download=None): + def __init__(self, fid=None, name=None, package=None, owner=None, + size=None, status=None, media=None, added=None, + fileorder=None, download=None): self.fid = fid self.name = name self.package = package @@ -98,15 +107,16 @@ class File(BaseObject): """ Represents a file object at runtime. """ - __slots__ = ['_name', '_size', 'abort', 'added', 'error', 'fid', 'fileorder', - 'filestatus', 'hash', 'lock', 'manager', 'media', 'owner', - 'packageid', 'plugin', 'pluginclass', 'pluginname', + __slots__ = ['_name', '_size', 'abort', 'added', 'error', 'fid', + 'fileorder', 'filestatus', 'hash', 'lock', 'manager', 'media', + 'owner', 'packageid', 'plugin', 'pluginclass', 'pluginname', 'reconnected', 'status', 'statusname', 'url', 'wait_until'] @staticmethod def from_info_data(m, info): - file = File(m, info.fid, info.name, info.size, info.status, info.media, info.added, info.fileorder, - "", "", "", DownloadStatus.NA, "", info.package, info.owner) + file = File(m, info.fid, info.name, info.size, info.status, info.media, + info.added, info.fileorder, "", "", "", DownloadStatus.NA, + "", info.package, info.owner) if info.download: file.url = info.download.url file.pluginname = info.download.plugin @@ -115,8 +125,9 @@ def from_info_data(m, info): file.error = info.download.error return file - def __init__(self, manager, fid, name, size, filestatus, media, added, fileorder, - url, pluginname, hash, status, error, package, owner): + def __init__( + self, manager, fid, name, size, filestatus, media, added, + fileorder, url, pluginname, hash, status, error, package, owner): self.manager = manager self.pyload = manager.pyload @@ -140,7 +151,7 @@ def __init__(self, manager, fid, name, size, filestatus, media, added, fileorder self.plugin = None - self.wait_until = 0 #: time.time() + time to wait + self.wait_until = 0 # time.time() + time to wait # status attributes self.abort = False @@ -184,7 +195,8 @@ def set_name(self, name): name = property(get_name, set_name) def __repr__(self): - return "".format(self.id, self.name, self.pluginname) + return "".format( + self.id, self.name, self.pluginname) @lock def init_plugin(self): @@ -245,10 +257,13 @@ def release(self): self.manager.release_file(self.fid) def to_info_data(self): - return FileInfo(self.fid, self.get_name(), self.packageid, self.owner, self.size, self.filestatus, - self.media, self.added, self.fileorder, DownloadInfo( - self.url, self.pluginname, self.hash, self.status, self.get_status_name(), self.error) - ) + return FileInfo(self.fid, self.get_name(), + self.packageid, self.owner, self.size, self.filestatus, + self.media, self.added, self.fileorder, + DownloadInfo( + self.url, self.pluginname, self.hash, self.status, + self.get_status_name(), + self.error)) def get_path(self): raise NotImplementedError @@ -327,10 +342,11 @@ def get_flags(self): def get_progress_info(self): return ProgressInfo( - self.pluginname, self.name, self.get_status_name(), self.get_eta(), - self.get_bytes_arrived(), self.size, self.owner, ProgressType.Download, + self.pluginname, self.name, self.get_status_name(), + self.get_eta(), + self.get_bytes_arrived(), + self.size, self.owner, ProgressType.Download, DownloadProgress( - self.fid, self.packageid, self.get_speed(), self.get_flags(), - self.status - ) - ) + self.fid, self.packageid, self.get_speed(), + self.get_flags(), + self.status)) diff --git a/src/pyload/core/datatype/init.py b/src/pyload/core/datatype/init.py index 44d6f9735b..dd650cc8f1 100644 --- a/src/pyload/core/datatype/init.py +++ b/src/pyload/core/datatype/init.py @@ -150,11 +150,15 @@ class ProgressType(IntEnum): class AccountInfo(BaseObject): - __slots__ = ['aid', 'plugin', 'loginname', 'owner', 'valid', 'validuntil', - 'trafficleft', 'maxtraffic', 'premium', 'activated', 'shared', 'config'] - - def __init__(self, aid=None, plugin=None, loginname=None, owner=None, valid=None, validuntil=None, - trafficleft=None, maxtraffic=None, premium=None, activated=None, shared=None, config=None): + __slots__ = [ + 'aid', 'plugin', 'loginname', 'owner', 'valid', 'validuntil', + 'trafficleft', 'maxtraffic', 'premium', 'activated', 'shared', + 'config'] + + def __init__( + self, aid=None, plugin=None, loginname=None, owner=None, + valid=None, validuntil=None, trafficleft=None, maxtraffic=None, + premium=None, activated=None, shared=None, config=None): self.aid = aid self.plugin = plugin self.loginname = loginname @@ -304,11 +308,13 @@ def __init__(self, plugin=None, name=None, statusmsg=None, eta=None, class StatusInfo(BaseObject): - __slots__ = ['speed', 'linkstotal', 'linksqueue', 'sizetotal', - 'sizequeue', 'notifications', 'paused', 'download', 'reconnect', 'quota'] + __slots__ = [ + 'speed', 'linkstotal', 'linksqueue', 'sizetotal', 'sizequeue', + 'notifications', 'paused', 'download', 'reconnect', 'quota'] - def __init__(self, speed=None, linkstotal=None, linksqueue=None, sizetotal=None, sizequeue=None, - notifications=None, paused=None, download=None, reconnect=None, quota=None): + def __init__(self, speed=None, linkstotal=None, linksqueue=None, + sizetotal=None, sizequeue=None, notifications=None, + paused=None, download=None, reconnect=None, quota=None): self.speed = speed self.linkstotal = linkstotal self.linksqueue = linksqueue diff --git a/src/pyload/core/datatype/package.py b/src/pyload/core/datatype/package.py index 937027a988..bf0eef16d1 100644 --- a/src/pyload/core/datatype/package.py +++ b/src/pyload/core/datatype/package.py @@ -5,7 +5,6 @@ import os import time -from builtins import object from future import standard_library @@ -16,7 +15,6 @@ except ImportError: from aenum import IntEnum - standard_library.install_aliases() @@ -35,11 +33,16 @@ def __init__(self, pid=None): class PackageInfo(BaseObject): - __slots__ = ['pid', 'name', 'folder', 'root', 'owner', 'site', 'comment', 'password', - 'added', 'tags', 'status', 'shared', 'packageorder', 'stats', 'fids', 'pids'] - - def __init__(self, pid=None, name=None, folder=None, root=None, owner=None, site=None, comment=None, password=None, - added=None, tags=None, status=None, shared=None, packageorder=None, stats=None, fids=None, pids=None): + __slots__ = [ + 'pid', 'name', 'folder', 'root', 'owner', 'site', 'comment', + 'password', 'added', 'tags', 'status', 'shared', 'packageorder', + 'stats', 'fids', 'pids'] + + def __init__( + self, pid=None, name=None, folder=None, root=None, owner=None, + site=None, comment=None, password=None, added=None, tags=None, + status=None, shared=None, packageorder=None, stats=None, fids=None, + pids=None): self.pid = pid self.name = name self.folder = folder @@ -80,11 +83,14 @@ class Package(BaseObject): @staticmethod def from_info_data(m, info): - return Package(m, info.pid, info.name, info.folder, info.root, info.owner, - info.site, info.comment, info.password, info.added, info.tags, info.status, info.shared, info.packageorder) - - def __init__(self, manager, pid, name, folder, root, owner, site, comment, password, added, tags, status, - shared, packageorder): + return Package( + m, info.pid, info.name, info.folder, info.root, info.owner, info. + site, info.comment, info.password, info.added, info.tags, info. + status, info.shared, info.packageorder) + + def __init__( + self, manager, pid, name, folder, root, owner, site, comment, + password, added, tags, status, shared, packageorder): self.manager = manager self.pyload = manager.pyload @@ -103,7 +109,7 @@ def __init__(self, manager, pid, name, folder, root, owner, site, comment, passw self.packageorder = packageorder self.timestamp = time.time() - #: Finish event already fired + # Finish event already fired self.set_finished = False def is_stale(self): @@ -169,7 +175,7 @@ class RootPackage(Package): def __init__(self, m, owner): Package.__init__(self, m, -1, "root", "", owner, -2, - "", "", "", 0, [], PackageStatus.Ok, False, 0) + "", "", "", 0, [], PackageStatus.Ok, False, 0) def get_path(self, name=""): return os.path.join(self.pyload.config.get( diff --git a/src/pyload/core/datatype/task.py b/src/pyload/core/datatype/task.py index 70f2a95e4d..193bb9254b 100644 --- a/src/pyload/core/datatype/task.py +++ b/src/pyload/core/datatype/task.py @@ -15,7 +15,6 @@ except ImportError: from aenum import IntEnum - standard_library.install_aliases() @@ -35,23 +34,23 @@ class InteractionTask(BaseObject): 'owner', 'plugin', 'result', 'seen', 'shared', 'storage', 'title', 'type', 'wait_until'] - #: Plugins can put needed data here + # Plugins can put needed data here storage = None - #: Timestamp when task expires + # Timestamp when task expires wait_until = 0 - #: The received result + # The received result result = None - #: List of registered handles + # List of registered handles handler = None - #: Error Message + # Error Message error = None - #: Timeout locked + # Timeout locked locked = False - #: A task that was retrieved counts as seen + # A task that was retrieved counts as seen seen = False - #: A task that is relevant to every user + # A task that is relevant to every user shared = False - #: primary uid of the owner + # primary uid of the owner owner = None def __init__(self, iid=None, type_=None, input=None, title=None, diff --git a/src/pyload/core/datatype/user.py b/src/pyload/core/datatype/user.py index 108892a94e..fbd5c27413 100644 --- a/src/pyload/core/datatype/user.py +++ b/src/pyload/core/datatype/user.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, unicode_literals from future import standard_library + from pyload.utils.check import bitset from .init import BaseObject, ExceptionObject, Permission @@ -33,8 +34,9 @@ class UserData(BaseObject): 'traffic', 'dllimit', 'dlquota', 'hddquota', 'user', 'templatename'] - def __init__(self, uid=None, name=None, email=None, role=None, permission=None, folder=None, - traffic=None, dllimit=None, dlquota=None, hddquota=None, user=None, templatename=None): + def __init__(self, uid=None, name=None, email=None, role=None, + permission=None, folder=None, traffic=None, dllimit=None, + dlquota=None, hddquota=None, user=None, templatename=None): self.uid = uid self.name = name self.email = email @@ -57,8 +59,10 @@ class User(UserData): @staticmethod def from_user_data(api, user): - return User(api, user.uid, user.name, user.email, user.role, user.permission, user.folder, - user.traffic, user.dllimit, user.dlquota, user.hddquota, user.user, user.templatename) + return User( + api, user.uid, user.name, user.email, user.role, user.permission, + user.folder, user.traffic, user.dllimit, user.dlquota, user. + hddquota, user.user, user.templatename) def __init__(self, api, *args, **kwargs): UserData.__init__(self, *args, **kwargs) diff --git a/src/pyload/core/iface.py b/src/pyload/core/iface.py index e6d2cbcc02..b257279166 100644 --- a/src/pyload/core/iface.py +++ b/src/pyload/core/iface.py @@ -1,25 +1,28 @@ # -*- coding: utf-8 -*- # @author: vuolter -from __future__ import unicode_literals -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals -from future import standard_library -standard_library.install_aliases() import os - -import autoupgrade -import daemonize - +from builtins import DATADIR from tempfile import mkstemp -from builtins import DATADIR +from future import standard_library from pkg_resources import get_default_cache + +import autoupgrade +import daemonize from pyload.utils.fs import cleanpy, makedirs -from .__about__ import __namespace__, __version__, __package_name__ +from .__about__ import __namespace__, __package_name__, __version__ from .init import Core, _pmap +standard_library.install_aliases() + + + + + def _mkdprofile(profile=None, rootdir=None): DEFAULT_PROFILE = 'default' @@ -62,7 +65,9 @@ def quit(profile=None, configdir=None): inst.shutdown() -def start(profile=None, configdir=None, tempdir=None, debug=None, restore=None, daemon=False): +def start( + profile=None, configdir=None, tempdir=None, debug=None, restore=None, + daemon=False): profiledir = _mkdprofile(profile, configdir) inst = Core(profiledir, tempdir, debug, restore) @@ -74,7 +79,7 @@ def start(profile=None, configdir=None, tempdir=None, debug=None, restore=None, d = daemonize.Daemonize("pyLoad", pidfile, inst.join, logger=inst.log) d.start() - return inst #: returns process instance + return inst # returns process instance def restart(*args, **kwargs): diff --git a/src/pyload/core/init.py b/src/pyload/core/init.py index 1aeeaa5b2f..fe769b14ab 100644 --- a/src/pyload/core/init.py +++ b/src/pyload/core/init.py @@ -3,10 +3,6 @@ from __future__ import absolute_import, unicode_literals -import builtins -import errno -import fcntl -import imp import locale import logging import logging.handlers @@ -16,26 +12,25 @@ import sys import tempfile import time -from builtins import TMPDIR, USERDIR, int, map, str +from builtins import TMPDIR, USERDIR, int, str from contextlib import closing from multiprocessing import Event, Process import portalocker import psutil +from future import standard_library from pkg_resources import resource_filename -from future import standard_library from pyload.config import ConfigParser -from pyload.utils import convert, format +from pyload.utils import format from pyload.utils.check import ismodule -from pyload.utils.fs import availspace, fullpath, lopen, makedirs, remove -from pyload.utils.misc import get_translation, write_pid -from pyload.utils.struct.info import Info +from pyload.utils.fs import availspace, fullpath, makedirs +from pyload.utils.misc import get_translation from pyload.utils.system import (ionice, renice, set_process_group, set_process_name, set_process_user) -from semver import format_version -from .__about__ import __namespace__, __version__, __version_info__, __package__ +from .__about__ import (__namespace__, __package__, __version__, + __version_info__) from .config import config_defaults, session_defaults standard_library.install_aliases() @@ -43,7 +38,7 @@ try: import colorlog except ImportError: - pass + colorlog = None _pmap = {} @@ -109,7 +104,7 @@ def _init_consolelogger(self): self.log.addHandler(consolehdlr) def _init_syslogger(self): - #: try to mimic to normal syslog messages + # try to mimic to normal syslog messages fmt = "%(asctime)s %(name)s: %(message)s" datefmt = "%b %e %H:%M:%S" syslogform = logging.Formatter(fmt, datefmt) @@ -156,7 +151,8 @@ def _init_filelogger(self): encoding=locale.getpreferredencoding(do_setlocale=False)) else: filehdlr = logging.FileHandler( - logfile, encoding=locale.getpreferredencoding(do_setlocale=False)) + logfile, encoding=locale.getpreferredencoding( + do_setlocale=False)) filehdlr.setFormatter(fileform) self.log.addHandler(filehdlr) @@ -223,7 +219,8 @@ def _setup_language(self): if lang == default: raise self.log.warning( - self._("Unable to load `{0}` language, use default `{1}`").format(lang, default), + self._("Unable to load `{0}` language, use default `{1}`").format( + lang, default), str(e)) self.set_language(default) @@ -305,7 +302,7 @@ def _init_managers(self): self.addonmanager = self.adm = AddonManager(self) # self.remotemanager = self.rem = RemoteManager(self) # self.servermanager = self.svm = ServerManager(self) - self.db.manager = self.files #: ugly? + self.db.manager = self.files # ugly? def _init_requests(self): from .network.factory import RequestFactory @@ -328,7 +325,7 @@ def _init_config(self): self.session = session def _init_cache(self): - #: Re-use cache + # Re-use cache tempdir = self.__tempdir if tempdir is None: tempdir = self.session.get('previous', 'cache', 'path') @@ -339,11 +336,11 @@ def _init_cache(self): self.session.set('current', 'cache', 'path', tempdir) self.cachedir = tempdir # if tempdir not in sys.path: - # sys.path.append(tempdir) + # sys.path.append(tempdir) def _register_signals(self): - shutfn = lambda s, f: self.shutdown() - quitfn = lambda s, f: self.terminate() + def shutfn(s, f): return self.shutdown() + def quitfn(s, f): return self.terminate() try: if os.name == 'nt': # signal.signal(signal.CTRL_C_EVENT, shutfn) @@ -369,7 +366,7 @@ def __init__(self, profiledir=None, tempdir=None, debug=None, restore=None): self._init_profile(profiledir) # if refresh: - # cleanpy(PACKDIR) + # cleanpy(PACKDIR) Process.__init__(self) @@ -397,7 +394,7 @@ def _init_profile(self, profiledir): def _setup_process(self): try: set_process_name('pyLoad') - except NameError: + except AttributeError: pass niceness = self.config.get('general', 'niceness') renice(niceness=niceness) @@ -411,7 +408,8 @@ def _setup_storage(self): self.log.debug("Storage: {0}".format(storage_folder)) makedirs(storage_folder, exist_ok=True) avail_space = format.size(availspace(storage_folder)) - self.log.info(self._("Available storage space: {0}").format(avail_space)) + self.log.info( + self._("Available storage space: {0}").format(avail_space)) def _workloop(self): self.__running.set() @@ -465,7 +463,7 @@ def run(self): self.log.info(self._("pyLoad is up and running")) self.evm.fire('pyload:started') - # #: some memory stats + # # some memory stats # from guppy import hpy # hp=hpy() # print(hp.heap()) @@ -538,8 +536,8 @@ def shutdown(self): self.config.close() self._remove_loggers() # if cleanup: - # self.log.info(self._("Deleting temp files ...")) - # remove(self.tempdir, ignore_errors=True) + # self.log.info(self._("Deleting temp files ...")) + # remove(self.tempdir, ignore_errors=True) finally: self.terminate() diff --git a/src/pyload/core/manager/__init__.py b/src/pyload/core/manager/__init__.py index 48074cbe70..4454a7f0d5 100644 --- a/src/pyload/core/manager/__init__.py +++ b/src/pyload/core/manager/__init__.py @@ -1,10 +1,6 @@ # -*- coding: utf-8 -*- # @author: vuolter -from __future__ import unicode_literals -from __future__ import absolute_import -from future import standard_library -standard_library.install_aliases() from .plugin import PluginManager from .account import AccountManager from .addon import AddonManager diff --git a/src/pyload/core/manager/account.py b/src/pyload/core/manager/account.py index 83ce0bac54..0a9bc72249 100644 --- a/src/pyload/core/manager/account.py +++ b/src/pyload/core/manager/account.py @@ -7,6 +7,7 @@ import random from future import standard_library + from pyload.utils.struct.lock import lock from ..datatype.init import AccountInfo @@ -46,7 +47,8 @@ def _create_account(self, info, password, options): if plugin not in self.accounts: self.accounts[plugin] = [] - self.pyload.log.debug("Create account {0}:{1}".format(plugin, loginname)) + self.pyload.log.debug( + "Create account {0}:{1}".format(plugin, loginname)) # New account instance account = klass.from_info_data(self, info, password, options) @@ -82,9 +84,10 @@ def save_accounts(self): data = [] for plugin, accounts in self.accounts.items(): data.extend( - [(acc.loginname, 1 if acc.activated else 0, 1 if acc.shared else 0, acc.password, - json.dumps(acc.options), acc.aid) for acc in - accounts]) + [(acc.loginname, 1 if acc.activated else 0, 1 + if acc.shared else 0, acc.password, json.dumps( + acc.options), + acc.aid) for acc in accounts]) self.pyload.db.save_accounts(data) def get_account(self, aid, plugin, user=None): @@ -163,7 +166,8 @@ def get_all_accounts(self, uid): accounts = [] for plugin, accs in self.accounts.items(): accounts.extend( - acc for acc in accs if acc.shared or not uid or acc.owner == uid) + acc for acc in accs + if acc.shared or not uid or acc.owner == uid) return accounts diff --git a/src/pyload/core/manager/addon.py b/src/pyload/core/manager/addon.py index ca78e96a1e..1635240825 100644 --- a/src/pyload/core/manager/addon.py +++ b/src/pyload/core/manager/addon.py @@ -7,13 +7,14 @@ from builtins import str from collections import defaultdict from gettext import gettext +from types import MethodType from _thread import start_new_thread from future import standard_library + from pyload.utils.layer.legacy.collections_ import namedtuple from pyload.utils.layer.safethreading import RLock from pyload.utils.struct.lock import lock -from types import MethodType from ..datatype.init import (AddonInfo, AddonService, ServiceDoesNotExist, ServiceException) @@ -33,7 +34,7 @@ class AddonManager(BaseManager): def __init__(self, core): BaseManager.__init__(self, core) - builtins.ADDONMANAGER = self #: needed to let addons register themselves + builtins.ADDONMANAGER = self # needed to let addons register themselves # TODO: multiuser addons @@ -69,7 +70,8 @@ def call(self, plugin, f, *args): func = getattr(plugin, f) return func(*args) except Exception as e: - plugin.log_error(self._("Error when executing {0}".format(f)), str(e)) + plugin.log_error( + self._("Error when executing {0}".format(f)), str(e)) # self.pyload.print_exc() def invoke(self, plugin, func_name, args): @@ -113,7 +115,8 @@ def create_index(self): active.append(pluginclass.__name__) else: self.pyload.log.debug( - "Loaded internal plugin: {0}".format(pluginclass.__name__)) + "Loaded internal plugin: {0}".format( + pluginclass.__name__)) else: deactive.append(pluginname) diff --git a/src/pyload/core/manager/base.py b/src/pyload/core/manager/base.py index b6cc6e17f3..76c4401d98 100644 --- a/src/pyload/core/manager/base.py +++ b/src/pyload/core/manager/base.py @@ -3,18 +3,20 @@ from __future__ import absolute_import, unicode_literals -from future import standard_library -standard_library.install_aliases() from builtins import object +from future import standard_library + from pyload.utils.layer.safethreading import Lock +standard_library.install_aliases() + + class BaseManager(object): """ Base manager """ - def __init__(self, core): """ Constructor. diff --git a/src/pyload/core/manager/config.py b/src/pyload/core/manager/config.py index 735f869c99..1954b1281a 100644 --- a/src/pyload/core/manager/config.py +++ b/src/pyload/core/manager/config.py @@ -5,6 +5,7 @@ import json from future import standard_library + from pyload.config import ConfigParser # from pyload.config.convert import from_string from pyload.utils.layer.legacy.collections_ import OrderedDict @@ -18,7 +19,6 @@ def convertkeyerror(func): """ Converts KeyError into InvalidConfigSection. """ - def conv(*args, **kwargs): try: return func(*args, **kwargs) @@ -71,7 +71,7 @@ def get(self, section, option, user=None): assert self.config[section].config[option] return self.load_values(user, section)[option] except KeyError: - pass #: Returns default value later + pass # Returns default value later return self.config[section].config[option].input.default @@ -80,7 +80,7 @@ def load_values(self, user, section): conf = self.pyload.db.load_config(section, user) try: self.values[user, section] = json.loads(conf) if conf else {} - except ValueError: #: Something did go wrong when parsing + except ValueError: # Something did go wrong when parsing self.values[user, section] = {} # self.pyload.print_exc() diff --git a/src/pyload/core/manager/exchange.py b/src/pyload/core/manager/exchange.py index d4b2ce027e..a586de9aa5 100644 --- a/src/pyload/core/manager/exchange.py +++ b/src/pyload/core/manager/exchange.py @@ -7,6 +7,7 @@ from base64 import standard_b64encode from future import standard_library + from pyload.utils.check import bitset from pyload.utils.layer.legacy.collections_ import OrderedDict from pyload.utils.struct.lock import lock @@ -30,12 +31,13 @@ class ExchangeManager(BaseManager): def __init__(self, core): BaseManager.__init__(self, core) - self.tasks = OrderedDict() #: task store, for all outgoing tasks + self.tasks = OrderedDict() # task store, for all outgoing tasks self.last_clients = {} - self.ids = 0 #: uniue interaction ids + self.ids = 0 # uniue interaction ids def is_client_connected(self, user): - return self.last_clients.get(user, 0) + self.CLIENT_THRESHOLD > time.time() + return self.last_clients.get( + user, 0) + self.CLIENT_THRESHOLD > time.time() @lock def work(self): @@ -44,8 +46,10 @@ def work(self): del self.tasks[n] # keep notifications count limited - n = [k for k, v in self.tasks.items() if v.type == Interaction.Notification][ - ::-1] + n = [ + k for k, v in self.tasks.items() + if v.type == Interaction.Notification][ + :: -1] for v in n[:self.MAX_NOTIFICATIONS]: del self.tasks[v] @@ -61,8 +65,10 @@ def create_notification(self, title, content, :param plugin: plugin name :return: :class:`InteractionTask` """ - task = InteractionTask(self.ids, Interaction.Notification, Input(InputType.Str, None, content), title, desc, plugin, - owner=owner) + task = InteractionTask( + self.ids, Interaction.Notification, + Input(InputType.Str, None, content), + title, desc, plugin, owner=owner) self.ids += 1 self.queue_task(task) return task @@ -101,8 +107,8 @@ def create_captcha_task(self, img, format, filename, input = Input(type_, data=[standard_b64encode(img), format, filename]) # TODO: title desc plugin - task = InteractionTask(self.ids, Interaction.Captcha, input, - self._("Captcha request"), self._("Please solve the captcha"), plugin, owner=owner) + task = InteractionTask(self.ids, Interaction.Captcha, input, self._( + "Captcha request"), self._("Please solve the captcha"), plugin, owner=owner) self.ids += 1 self.queue_task(task) @@ -127,13 +133,15 @@ def get_tasks(self, user, mode=Interaction.All): tasks = [tsk for tsk in self.tasks.values() if mode == Interaction.All or bitset(tsk.type, mode)] # filter correct user / or shared - tasks = [tsk for tsk in tasks if user is None or user == tsk.owner or tsk.shared] + tasks = [tsk for tsk in tasks if user is None or user == + tsk.owner or tsk.shared] return tasks def is_task_waiting(self, user, mode=Interaction.All): - tasks = [tsk for tsk in self.get_tasks( - user, mode) if not tsk.type == Interaction.Notification or not tsk.seen] + tasks = [ + tsk for tsk in self.get_tasks(user, mode) + if not tsk.type == Interaction.Notification or not tsk.seen] return len(tasks) > 0 def queue_task(self, task): diff --git a/src/pyload/core/manager/file.py b/src/pyload/core/manager/file.py index 77212d8669..c82b8ccfdc 100644 --- a/src/pyload/core/manager/file.py +++ b/src/pyload/core/manager/file.py @@ -8,6 +8,7 @@ from functools import reduce from future import standard_library + from pyload.utils.struct.lock import RWLock, lock from ..datatype.file import File @@ -45,17 +46,31 @@ def __init__(self, core): BaseManager.__init__(self, core) # translations - self.status_msg = [self._("none"), self._("offline"), self._("online"), self._("queued"), self._("paused"), - self._("finished"), self._("skipped"), self._( - "failed"), self._("starting"), self._("waiting"), - self._("downloading"), self._("temp. offline"), self._( - "aborted"), self._("not possible"), self._("missing"), - self._("file mismatch"), self._("occupied"), self._( - "decrypting"), self._("processing"), self._("custom"), - self._("unknown")] - - self.files = {} #: holds instances for files - self.packages = {} #: same for packages + self.status_msg = [ + self._("none"), + self._("offline"), + self._("online"), + self._("queued"), + self._("paused"), + self._("finished"), + self._("skipped"), + self._("failed"), + self._("starting"), + self._("waiting"), + self._("downloading"), + self._("temp. offline"), + self._("aborted"), + self._("not possible"), + self._("missing"), + self._("file mismatch"), + self._("occupied"), + self._("decrypting"), + self._("processing"), + self._("custom"), + self._("unknown")] + + self.files = {} # holds instances for files + self.packages = {} # same for packages self.job_cache = {} @@ -63,8 +78,8 @@ def __init__(self, core): self.lock = RWLock() #self.lock._Verbose__verbose = True - self.downloadstats = {} #: cached dl stats - self.queuestats = {} #: cached queue stats + self.downloadstats = {} # cached dl stats + self.queuestats = {} # cached queue stats self.db = self.pyload.db @@ -110,11 +125,13 @@ def add_package(self, name, folder, root, password, """ Adds a package to database. """ - pid = self.db.add_package(name, folder, root, password, site, comment, - PackageStatus.Paused if paused else PackageStatus.Ok, owner) + pid = self.db.add_package( + name, folder, root, password, site, comment, PackageStatus.Paused + if paused else PackageStatus.Ok, owner) pinfo = self.db.get_package_info(pid) - self.pyload.evm.fire("package:inserted", pid, pinfo.root, pinfo.packageorder) + self.pyload.evm.fire("package:inserted", pid, + pinfo.root, pinfo.packageorder) return pid @lock @@ -273,7 +290,7 @@ def get_tree(self, pid, full, state, owner=None): packs[self.ROOT_PACKAGE] = view.root elif pid in packs: view.root = packs[pid] - else: #: package does not exists + else: # package does not exists return view self._sanitize_tree(packs, files) @@ -462,7 +479,8 @@ def check_package_finished(self, file): if not ids or (file.fid in ids and len(ids) == 1): if not file.package().set_finished: self.pyload.log.info( - self._("Package finished: {0}").format(file.package().name)) + self._("Package finished: {0}").format( + file.package().name)) self.pyload.adm.package_finished(file.package()) file.package().set_finished = True @@ -523,13 +541,15 @@ def order_package(self, pid, position): self.pyload.evm.fire("package:reordered", pid, position, pinfo.root) def _get_first_fileinfo(self, files): - return reduce(lambda x, y: x if x.fileorder <= y.fileorder else y, files) # NOTE: Equality between fileorders should never happen... + # NOTE: Equality between fileorders should never happen... + return reduce( + lambda x, y: x if x.fileorder <= y.fileorder else y, files) def _order_files(self, fids, finfo, position): diff = len(fids) incr = 0 - files = (file for file in self.files.values() - if not (file.fileorder < 0 or file.packageid != finfo.package)) + files = (file for file in self.files.values() if not ( + file.fileorder < 0 or file.packageid != finfo.package)) if finfo.fileorder > position: for file in files: if not (position <= file.fileorder < finfo.fileorder): @@ -557,7 +577,7 @@ def order_files(self, fids, pid, position): finfo = self._get_first_fileinfo(files) - order = finfo.fileorder #: minimum fileorder + order = finfo.fileorder # : minimum fileorder self.db.order_files(pid, fids, order, position) self._order_files(fids, finfo, position) diff --git a/src/pyload/core/manager/info.py b/src/pyload/core/manager/info.py index 10e6a60241..b6903eb17d 100644 --- a/src/pyload/core/manager/info.py +++ b/src/pyload/core/manager/info.py @@ -6,6 +6,7 @@ import time from future import standard_library + from pyload.utils.convert import to_list from pyload.utils.layer.safethreading import RLock from pyload.utils.struct.lock import lock @@ -27,7 +28,7 @@ def __init__(self, core): """ BaseManager.__init__(self, core) - self.thread = [] #: thread list + self.thread = [] # thread list self.lock = RLock() diff --git a/src/pyload/core/manager/plugin.py b/src/pyload/core/manager/plugin.py index a0dd91753a..5b8cb6f89e 100644 --- a/src/pyload/core/manager/plugin.py +++ b/src/pyload/core/manager/plugin.py @@ -7,14 +7,14 @@ import sys from builtins import object, str +from future import standard_library from pkg_resources import resource_filename -from future import standard_library from pyload.utils.fs import fullpath -from .base import BaseManager from ..__about__ import __package__ from ..network.loader import LoaderFactory, PluginLoader +from .base import BaseManager standard_library.install_aliases() @@ -23,7 +23,6 @@ class PluginMatcher(object): """ Abstract class that allows modify which plugins to match and to load. """ - def match_url(self, url): """ Returns (type, name) of a plugin if a match is found. @@ -60,7 +59,7 @@ def __init__(self, core): sys.path.append(os.getcwd()) # TODO: Recheck... self.loader = LoaderFactory( PluginLoader(fullpath(self.LOCALROOT), - self.LOCALROOT, self.pyload.config), + self.LOCALROOT, self.pyload.config), PluginLoader(resource_filename(__package__, 'network')), self.ROOT, self.pyload.config) @@ -75,7 +74,8 @@ def add_matcher(self, matcher, index=0): """ if not isinstance(matcher, PluginMatcher): raise TypeError( - "Expected type of PluginMatcher, got '{0}' instead".format(type(matcher))) + "Expected type of PluginMatcher, got '{0}' instead".format( + type(matcher))) if matcher in self.matcher: self.matcher.remove(matcher) @@ -93,7 +93,7 @@ def parse_urls(self, urls): """ Parse plugins for given list of urls, separate to crypter and hoster. """ - res = {'hoster': [], 'crypter': []} #: tupels of (url, plugin) + res = {'hoster': [], 'crypter': []} # tupels of (url, plugin) for url in urls: if not isinstance(url, str): @@ -108,10 +108,10 @@ def parse_urls(self, urls): if self.loader.get_plugin(ptype, name).re.match(url): res[ptype].append((url, name)) found = (ptype, name) - break #: need to exit this loop first + break # need to exit this loop first - if found: #: found match - if self.history[0] != found: #: update history + if found: # found match + if self.history[0] != found: # update history self.history.remove(found) self.history.insert(0, found) continue @@ -205,7 +205,8 @@ def load_module(self, type_, name): return module except Exception as e: self.pyload.log.error( - self._("Error importing {0}: {1}").format(name, str(e))) + self._("Error importing {0}: {1}").format( + name, str(e))) # self.pyload.print_exc() def load_class(self, type_, name): diff --git a/src/pyload/core/manager/remote.py b/src/pyload/core/manager/remote.py index 4e6588a4cb..580c9c162f 100644 --- a/src/pyload/core/manager/remote.py +++ b/src/pyload/core/manager/remote.py @@ -39,7 +39,8 @@ def start(self): self._("Starting {0}: {1}:{2}").format(b, host, port)) except Exception as e: self.pyload.log.error( - self._("Failed loading backend {0} | {1}").format(b, str(e))) + self._("Failed loading backend {0} | {1}").format( + b, str(e))) if self.pyload.debug: print_exc() else: diff --git a/src/pyload/core/manager/transfer.py b/src/pyload/core/manager/transfer.py index e1f97b5cfc..8954cdd2f6 100644 --- a/src/pyload/core/manager/transfer.py +++ b/src/pyload/core/manager/transfer.py @@ -9,6 +9,7 @@ from collections import defaultdict from future import standard_library + from pyload.utils.fs import availspace from pyload.utils.layer.legacy import subprocess_ as subprocess from pyload.utils.layer.safethreading import Event @@ -29,17 +30,17 @@ class TransferManager(BaseManager): def __init__(self, core): BaseManager.__init__(self, core) - #: won't start download when true + # won't start download when true self.pause = True - #: each thread is in exactly one category + # each thread is in exactly one category self.free = [] - #: a thread that in working must have a file as active attribute + # a thread that in working must have a file as active attribute self.downloading = [] - #: holds the decrypter threads + # holds the decrypter threads self.decrypting = [] - #: indicates when reconnect has occurred + # indicates when reconnect has occurred self.reconnecting = Event() self.lock = RWLock() @@ -226,7 +227,8 @@ def start_job(self, info, limit): # this plugin does not exits if plugin is None: self.pyload.log.error( - self._("Plugin '{0}' does not exists").format(info.download.plugin)) + self._("Plugin '{0}' does not exists").format( + info.download.plugin)) self.pyload.files.set_download_status( info.fid, DownloadStatus.Failed) return False @@ -287,7 +289,8 @@ def try_reconnect(self): 'script'), shell=True) except Exception: - self.pyload.log.warning(self._("Failed executing reconnect script!")) + self.pyload.log.warning( + self._("Failed executing reconnect script!")) self.pyload.config.set('reconnect', 'activated', False) self.reconnecting.clear() # self.pyload.print_exc() @@ -309,8 +312,10 @@ def want_reconnect(self): """ Number of downloads that are waiting for reconnect. """ - active = [x.active.has_plugin( - ) and x.active.plugin.want_reconnect and x.active.plugin.waiting for x in self.downloading] + active = [ + x.active.has_plugin() and + x.active.plugin.want_reconnect and x.active.plugin.waiting + for x in self.downloading] return active.count(True) @lock(shared=True) diff --git a/src/pyload/core/network/account.py b/src/pyload/core/network/account.py index f2454304fe..e799b2c434 100644 --- a/src/pyload/core/network/account.py +++ b/src/pyload/core/network/account.py @@ -6,6 +6,7 @@ from builtins import dict, str from future import standard_library + from pyload.requests.cookie import CookieJar from pyload.utils import format, parse from pyload.utils.convert import to_str @@ -45,15 +46,16 @@ class Account(Base): maxtraffic = -1 premium = True - #: after that time [in minutes] pyload will relogin the account + # after that time [in minutes] pyload will relogin the account login_timeout = 600 - #: account data will be reloaded after this time + # account data will be reloaded after this time info_threshold = 600 @classmethod def from_info_data(cls, m, info, password, options): - return cls(m, info.aid, info.loginname, info.owner, - True if info.activated else False, True if info.shared else False, password, options) + return cls(m, info.aid, info.loginname, info.owner, True + if info.activated else False, True + if info.shared else False, password, options) __type__ = "account" @@ -73,7 +75,7 @@ def __init__(self, manager, aid, loginname, owner, self.lock = RLock() self.timestamp = 0 - self.login_ts = 0 #: timestamp for login + self.login_ts = 0 # timestamp for login self.cj = CookieJar() self.error = None @@ -86,14 +88,16 @@ def __init__(self, manager, aid, loginname, owner, self.init() def to_info_data(self): - info = AccountInfo(self.aid, self.__name__, self.loginname, self.owner, self.valid, self.validuntil, self.trafficleft, - self.maxtraffic, self.premium, self.activated, self.shared, self.options) + info = AccountInfo( + self.aid, self.__name__, self.loginname, self.owner, self.valid, + self.validuntil, self.trafficleft, self.maxtraffic, self.premium, + self.activated, self.shared, self.options) info.config = [ ConfigItem( name, item.label, item.description, item.input, to_str(self.get_config(name), self.get_config(name))) - for name, item in self.config_data.items()] + for name, item in self.config_data.items()] return info def init(self): @@ -159,12 +163,14 @@ def _login(self, req): self.valid = True except WrongPassword: self.log_warning( - self._("Could not login with account {0} | {1}").format(self.loginname, self._("Wrong Password"))) + self._("Could not login with account {0} | {1}").format( + self.loginname, self._("Wrong Password"))) self.valid = False except Exception as e: self.log_warning( - self._("Could not login with account {0} | {1}").format(self.loginname, str(e))) + self._("Could not login with account {0} | {1}").format( + self.loginname, str(e))) self.valid = False # self.pyload.print_exc() @@ -182,7 +188,7 @@ def set_login(self, loginname, password): """ if password != self.password or loginname != self.loginname: self.login_ts = 0 - self.valid = True #: set valid, so the login will be retried + self.valid = True # set valid, so the login will be retried self.loginname = loginname self.password = password @@ -241,13 +247,14 @@ def get_account_info(self, force=False): infos = {'error': str(e)} self.log_error(self._("Error: {0}").format(str(e))) - self.restore_defaults() #: reset to initial state - if isinstance(infos, dict): #: copy result from dict to class + self.restore_defaults() # reset to initial state + if isinstance(infos, dict): # copy result from dict to class for k, v in infos.items(): if hasattr(self, k): setattr(self, k, v) else: - self.log_debug("Unknown attribute {0}={1}".format(k, v)) + self.log_debug( + "Unknown attribute {0}={1}".format(k, v)) self.log_debug("Account Info: {0}".format(infos)) self.timestamp = time.time() @@ -306,12 +313,12 @@ def is_usable(self): if 0 <= self.validuntil < time.time(): return False - if self.trafficleft is 0: #: test explicitly for 0 + if self.trafficleft is 0: # test explicitly for 0 return False return True - def parse_traffic(self, string): #: returns kbyte + def parse_traffic(self, string): # returns kbyte return parse.bytesize(string) >> 10 def format_trafficleft(self): @@ -326,8 +333,8 @@ def empty(self, user=None): if user: self.log_debug("Deprecated argument user for .empty()", user) - self.log_warning( - self._("Account {0} has not enough traffic, checking again in 30min").format(self.login)) + self.log_warning(self._( + "Account {0} has not enough traffic, checking again in 30min").format(self.login)) self.trafficleft = 0 self.schedule_refresh(30 * 60) @@ -346,8 +353,9 @@ def schedule_refresh(self, time=0, force=True): """ Add a task for refreshing the account info to the scheduler. """ - self.log_debug("Scheduled Account refresh for {0} in {1} seconds".format( - self.loginname, time)) + self.log_debug( + "Scheduled Account refresh for {0} in {1} seconds".format( + self.loginname, time)) self.pyload.scheduler.enter(time, 1, self.get_account_info, [force]) @lock @@ -356,7 +364,7 @@ def check_login(self, req): Checks if the user is still logged in. """ if self.login_ts + self.login_timeout * 60 < time.time(): - if self.login_ts: #: separate from fresh login to have better debug logs + if self.login_ts: # separate from fresh login to have better debug logs self.log_debug( "Reached login timeout for {0}".format(self.loginname)) else: diff --git a/src/pyload/core/network/addon.py b/src/pyload/core/network/addon.py index e714a29f05..8d181b6051 100644 --- a/src/pyload/core/network/addon.py +++ b/src/pyload/core/network/addon.py @@ -5,6 +5,7 @@ from builtins import ADDONMANAGER, object, str from future import standard_library + from pyload.utils.check import hasmethod from pyload.utils.convert import to_list @@ -109,7 +110,7 @@ class Addon(Base): """ - #: periodic call interval in seconds + # periodic call interval in seconds interval = 0 __type__ = "addon" @@ -117,10 +118,10 @@ class Addon(Base): def __init__(self, core, manager, user=None): Base.__init__(self, core, user) - #: Callback of periodical job task, used by AddonManager + # Callback of periodical job task, used by AddonManager self.cb = None - #: `AddonManager` + # `AddonManager` self.manager = manager self.init() diff --git a/src/pyload/core/network/base.py b/src/pyload/core/network/base.py index 8aa06a8d50..eabe1acb77 100644 --- a/src/pyload/core/network/base.py +++ b/src/pyload/core/network/base.py @@ -3,13 +3,13 @@ from __future__ import absolute_import, unicode_literals -import builtins import os import random import time from builtins import object from future import standard_library + from pyload.utils.convert import to_str from pyload.utils.fs import lopen, makedirs, remove @@ -46,40 +46,40 @@ class Base(object): The Base plugin class with all shared methods and every possible attribute for plugin definition. """ - #: Version as string or number + # Version as string or number __version__ = "0.1" # Type of the plugin, will be inherited and should not be set! __type__ = "" - #: Regexp pattern which will be matched for download/crypter plugins + # Regexp pattern which will be matched for download/crypter plugins __pattern__ = r'' - #: Internal addon plugin which is always loaded + # Internal addon plugin which is always loaded __internal__ = False - #: When True this addon can be enabled by every user + # When True this addon can be enabled by every user __user_context__ = False - #: Config definition: list of (name, type, label, default) or - #: (name, label, desc, Input(...)) + # Config definition: list of (name, type, label, default) or + # (name, label, desc, Input(...)) __config__ = tuple() - #: Short description, one liner + # Short description, one liner __description__ = "" - #: More detailed text + # More detailed text __explanation__ = """""" - #: List of needed modules + # List of needed modules __dependencies__ = tuple() - #: Used to assign a category for addon plugins + # Used to assign a category for addon plugins __category__ = "" - #: Tags to categorize the plugin, see documentation for further info + # Tags to categorize the plugin, see documentation for further info __tags__ = tuple() - #: Base64 encoded .png icon, should be 32x32, please do not use sizes above ~2KB, for bigger icons use url. + # Base64 encoded .png icon, should be 32x32, please do not use sizes above ~2KB, for bigger icons use url. __icon__ = "" - #: Alternative, link to png icon + # Alternative, link to png icon __icon_url__ = "" - #: Domain name of the service + # Domain name of the service __domain__ = "" - #: Url with general information/support/discussion + # Url with general information/support/discussion __url__ = "" - #: Url to term of content, user is accepting these when using the plugin + # Url to term of content, user is accepting these when using the plugin __toc_url__ = "" - #: Url to service (to buy premium) for accounts + # Url to service (to buy premium) for accounts __ref_url__ = "" __author__ = tuple() @@ -88,23 +88,23 @@ class Base(object): def __init__(self, core, owner=None): self.__name__ = self.__class__.__name__ - #: Core instance + # Core instance self.pyload = core self._ = core._ if owner is not None: - #: :class:`Api`, user api when user is set + # :class:`Api`, user api when user is set self.api = self.pyload.api.with_user_context(owner) if not self.api: raise Exception("Plugin running with invalid user") - #: :class:`User`, user related to this plugin + # :class:`User`, user related to this plugin self.owner = self.api.user else: self.api = self.pyload.api self.owner = None - #: last interaction task + # last interaction task self.task = None def __getitem__(self, item): @@ -241,14 +241,15 @@ def load(self, url, get={}, post={}, ref=True, from inspect import currentframe frame = currentframe() - dumpdir = os.path.join(self.pyload.cachedir, 'plugins', self.__name__) + dumpdir = os.path.join(self.pyload.cachedir, + 'plugins', self.__name__) makedirs(dumpdir, exist_ok=True) filepath = os.path.join(dumpdir, "dump_{0}_line{1}.html".format( frame.f_back.f_code.co_name, frame.f_back.f_lineno)) with lopen(filepath, mode='wb') as fp: fp.write(res) - del frame #: delete the frame or it wont be cleaned + del frame # delete the frame or it wont be cleaned if just_header: # parse header @@ -289,8 +290,11 @@ def correct_captcha(self): self.log_debug("Deprecated method .correct_captcha, use .correct_task") self.correct_task() - def decrypt_captcha(self, url, get={}, post={}, cookies=True, forceuser=False, imgtype='jpg', - result_type='textual'): + def decrypt_captcha( + self, url, get={}, + post={}, + cookies=True, forceuser=False, imgtype='jpg', + result_type='textual'): """ Loads a captcha and decrypts it with ocr, plugin, user input @@ -340,14 +344,16 @@ def decrypt_captcha(self, url, get={}, post={}, cookies=True, forceuser=False, i # TODO: task handling self.pyload.exm.remove_task(task) - if task.error and has_plugin: #: ignore default error message since the user could use OCR + if task.error and has_plugin: # ignore default error message since the user could use OCR self.fail( - self._("Pil and tesseract not installed and no Client connected for captcha decrypting")) + self._( + "Pil and tesseract not installed and no Client connected for captcha decrypting")) elif task.error: self.fail(task.error) elif not task.result: self.fail( - self._("No captcha result obtained in appropriate time")) + self._( + "No captcha result obtained in appropriate time")) result = task.result self.pyload.log.debug( diff --git a/src/pyload/core/network/crypter.py b/src/pyload/core/network/crypter.py index 836881f2cc..24d266581d 100644 --- a/src/pyload/core/network/crypter.py +++ b/src/pyload/core/network/crypter.py @@ -7,9 +7,9 @@ from builtins import int, str from future import standard_library + from pyload.requests.curl.download import CurlDownload from pyload.requests.curl.request import CurlRequest -from pyload.utils import format from pyload.utils.convert import chunks as _chunks from pyload.utils.fs import lopen, makedirs, remove @@ -46,10 +46,10 @@ class Hoster(Base): Base plugin for hoster plugin. Overwrite get_info for online status retrieval, process for downloading. """ - #: Class used to make requests with `self.load` + # Class used to make requests with `self.load` REQUEST_CLASS = CurlRequest - #: Class used to make download + # Class used to make download DOWNLOAD_CLASS = CurlDownload @staticmethod @@ -69,26 +69,26 @@ def __init__(self, file): Base.__init__(self, file.manager.pyload) self.want_reconnect = False - #: enables simultaneous processing of multiple downloads + # enables simultaneous processing of multiple downloads self.limit_dl = 0 - #: chunk limit + # chunk limit self.chunk_limit = 1 - #: enables resume (will be ignored if server dont accept chunks) + # enables resume (will be ignored if server dont accept chunks) self.resume_download = False - #: plugin is waiting + # plugin is waiting self.waiting = False - self.ocr = None #: captcha reader instance - #: account handler instance, see :py:class:`Account` + self.ocr = None # captcha reader instance + # account handler instance, see :py:class:`Account` self.account = self.pyload.acm.select_account( self.__name__, self.owner) - #: premium status + # premium status self.premium = False if self.account: - #: Request instance bound to account + # Request instance bound to account self.req = self.account.get_account_request() # Default: -1, True, True self.chunk_limit, self.limit_dl, self.resume_download = self.account.get_download_settings() @@ -96,20 +96,20 @@ def __init__(self, file): else: self.req = self.pyload.req.get_request(class_=self.REQUEST_CLASS) - #: Will hold the download class + # Will hold the download class self.dl = None - #: associated file instance, see `File` + # associated file instance, see `File` self.filename = file - self.thread = None #: holds thread in future + self.thread = None # holds thread in future - #: location where the last call to download was saved + # location where the last call to download was saved self.last_download = "" - #: re match of the last call to `check_download` + # re match of the last call to `check_download` self.last_check = None - self.retries = 0 #: amount of retries already made - self.html = None #: some plugins store html code here + self.retries = 0 # amount of retries already made + self.html = None # some plugins store html code here self.init() @@ -126,7 +126,7 @@ def get_multi_dl(self): def set_multi_dl(self, val): self.limit_dl = 0 if val else 1 - #: virtual attribute using self.limit_dl on behind + # virtual attribute using self.limit_dl on behind multi_dl = property(get_multi_dl, set_multi_dl) def get_chunk_count(self): @@ -140,7 +140,7 @@ def get_download_limit(self): limit = self.account.options.get("limitDL", 0) if limit == "": limit = 0 - if self.limit_dl > 0: #: a limit is already set, we use the minimum + if self.limit_dl > 0: # a limit is already set, we use the minimum return min(int(limit), self.limit_dl) else: return int(limit) @@ -312,7 +312,8 @@ def download(self, url, get={}, post={}, ref=True, os.chown(location, uid, gid) except Exception as e: self.pyload.log.warning( - self._("Setting User and Group failed: {0}").format(str(e))) + self._("Setting User and Group failed: {0}").format( + str(e))) name = self.file.name @@ -325,14 +326,18 @@ def download(self, url, get={}, post={}, ref=True, self.req, self.DOWNLOAD_CLASS) try: # TODO: hardcoded arguments - newname = self.dl.download(url, filepath, get=get, post=post, referer=ref, chunks=self.get_chunk_count(), - resume=self.resume_download, cookies=cookies, disposition=disposition) + newname = self.dl.download( + url, filepath, get=get, post=post, referer=ref, + chunks=self.get_chunk_count(), + resume=self.resume_download, cookies=cookies, + disposition=disposition) finally: self.dl.close() self.file.size = self.dl.size - if disposition and newname and newname != name: #: triple check, just to be sure - self.pyload.log.info(self._("{0} saved as {1}").format(name, newname)) + if disposition and newname and newname != name: # triple check, just to be sure + self.pyload.log.info( + self._("{0} saved as {1}").format(name, newname)) self.file.name = newname filepath = os.path.join(location, newname) @@ -354,7 +359,8 @@ def download(self, url, get={}, post={}, ref=True, os.chown(fs_filename, uid, gid) except Exception as e: self.pyload.log.warning( - self._("Setting User and Group failed: {0}").format(str(e))) + self._("Setting User and Group failed: {0}").format( + str(e))) self.last_download = fs_filename return self.last_download @@ -421,10 +427,10 @@ def check_for_same_files(self, starting=False): for file in self.pyload.files.cached_files(): if file != self.file and file.name == self.file.name and file.package( ).folder == pack.folder: - if file.status in (0, 12): #: finished or downloading + if file.status in (0, 12): # finished or downloading raise Skip(file.pluginname) elif file.status in ( - 5, 7) and starting: #: a download is waiting/starting and was apparently started before + 5, 7) and starting: # a download is waiting/starting and was apparently started before raise Skip(file.pluginname) download_folder = self.pyload.config.get('general', 'storage_folder') @@ -443,7 +449,8 @@ def check_for_same_files(self, starting=False): raise Skip(file[0]) self.pyload.log.debug( - "File {0} not skipped, because it does not exists".format(self.file.name)) + "File {0} not skipped, because it does not exists".format( + self.file.name)) def clean(self): """ diff --git a/src/pyload/core/network/extractor.py b/src/pyload/core/network/extractor.py index 4557f3e143..65cfb5b73a 100644 --- a/src/pyload/core/network/extractor.py +++ b/src/pyload/core/network/extractor.py @@ -61,7 +61,7 @@ def __init__(self, m, filename, out, fullpath, self.overwrite = overwrite self.excludefiles = excludefiles self.renice = renice - self.files = [] #: Store extracted files here + self.files = [] # Store extracted files here def init(self): """ diff --git a/src/pyload/core/network/factory.py b/src/pyload/core/network/factory.py index ab0f7693f1..5b883f7d90 100644 --- a/src/pyload/core/network/factory.py +++ b/src/pyload/core/network/factory.py @@ -6,6 +6,7 @@ from builtins import object from future import standard_library + from pyload.requests.bucket import Bucket from pyload.requests.curl.download import CurlDownload from pyload.requests.curl.request import CurlRequest @@ -67,12 +68,14 @@ def get_proxies(self): _type = "socks5" username = None - if self.pyload.config.get('proxy', 'username') and self.pyload.config.get( + if self.pyload.config.get( + 'proxy', 'username') and self.pyload.config.get( 'proxy', 'username').lower() != "none": username = self.pyload.config.get('proxy', 'username') pw = None - if self.pyload.config.get('proxy', 'password') and self.pyload.config.get( + if self.pyload.config.get( + 'proxy', 'password') and self.pyload.config.get( 'proxy', 'password').lower() != "none": pw = self.pyload.config.get('proxy', 'password') diff --git a/src/pyload/core/network/hoster.py b/src/pyload/core/network/hoster.py index 836881f2cc..24d266581d 100644 --- a/src/pyload/core/network/hoster.py +++ b/src/pyload/core/network/hoster.py @@ -7,9 +7,9 @@ from builtins import int, str from future import standard_library + from pyload.requests.curl.download import CurlDownload from pyload.requests.curl.request import CurlRequest -from pyload.utils import format from pyload.utils.convert import chunks as _chunks from pyload.utils.fs import lopen, makedirs, remove @@ -46,10 +46,10 @@ class Hoster(Base): Base plugin for hoster plugin. Overwrite get_info for online status retrieval, process for downloading. """ - #: Class used to make requests with `self.load` + # Class used to make requests with `self.load` REQUEST_CLASS = CurlRequest - #: Class used to make download + # Class used to make download DOWNLOAD_CLASS = CurlDownload @staticmethod @@ -69,26 +69,26 @@ def __init__(self, file): Base.__init__(self, file.manager.pyload) self.want_reconnect = False - #: enables simultaneous processing of multiple downloads + # enables simultaneous processing of multiple downloads self.limit_dl = 0 - #: chunk limit + # chunk limit self.chunk_limit = 1 - #: enables resume (will be ignored if server dont accept chunks) + # enables resume (will be ignored if server dont accept chunks) self.resume_download = False - #: plugin is waiting + # plugin is waiting self.waiting = False - self.ocr = None #: captcha reader instance - #: account handler instance, see :py:class:`Account` + self.ocr = None # captcha reader instance + # account handler instance, see :py:class:`Account` self.account = self.pyload.acm.select_account( self.__name__, self.owner) - #: premium status + # premium status self.premium = False if self.account: - #: Request instance bound to account + # Request instance bound to account self.req = self.account.get_account_request() # Default: -1, True, True self.chunk_limit, self.limit_dl, self.resume_download = self.account.get_download_settings() @@ -96,20 +96,20 @@ def __init__(self, file): else: self.req = self.pyload.req.get_request(class_=self.REQUEST_CLASS) - #: Will hold the download class + # Will hold the download class self.dl = None - #: associated file instance, see `File` + # associated file instance, see `File` self.filename = file - self.thread = None #: holds thread in future + self.thread = None # holds thread in future - #: location where the last call to download was saved + # location where the last call to download was saved self.last_download = "" - #: re match of the last call to `check_download` + # re match of the last call to `check_download` self.last_check = None - self.retries = 0 #: amount of retries already made - self.html = None #: some plugins store html code here + self.retries = 0 # amount of retries already made + self.html = None # some plugins store html code here self.init() @@ -126,7 +126,7 @@ def get_multi_dl(self): def set_multi_dl(self, val): self.limit_dl = 0 if val else 1 - #: virtual attribute using self.limit_dl on behind + # virtual attribute using self.limit_dl on behind multi_dl = property(get_multi_dl, set_multi_dl) def get_chunk_count(self): @@ -140,7 +140,7 @@ def get_download_limit(self): limit = self.account.options.get("limitDL", 0) if limit == "": limit = 0 - if self.limit_dl > 0: #: a limit is already set, we use the minimum + if self.limit_dl > 0: # a limit is already set, we use the minimum return min(int(limit), self.limit_dl) else: return int(limit) @@ -312,7 +312,8 @@ def download(self, url, get={}, post={}, ref=True, os.chown(location, uid, gid) except Exception as e: self.pyload.log.warning( - self._("Setting User and Group failed: {0}").format(str(e))) + self._("Setting User and Group failed: {0}").format( + str(e))) name = self.file.name @@ -325,14 +326,18 @@ def download(self, url, get={}, post={}, ref=True, self.req, self.DOWNLOAD_CLASS) try: # TODO: hardcoded arguments - newname = self.dl.download(url, filepath, get=get, post=post, referer=ref, chunks=self.get_chunk_count(), - resume=self.resume_download, cookies=cookies, disposition=disposition) + newname = self.dl.download( + url, filepath, get=get, post=post, referer=ref, + chunks=self.get_chunk_count(), + resume=self.resume_download, cookies=cookies, + disposition=disposition) finally: self.dl.close() self.file.size = self.dl.size - if disposition and newname and newname != name: #: triple check, just to be sure - self.pyload.log.info(self._("{0} saved as {1}").format(name, newname)) + if disposition and newname and newname != name: # triple check, just to be sure + self.pyload.log.info( + self._("{0} saved as {1}").format(name, newname)) self.file.name = newname filepath = os.path.join(location, newname) @@ -354,7 +359,8 @@ def download(self, url, get={}, post={}, ref=True, os.chown(fs_filename, uid, gid) except Exception as e: self.pyload.log.warning( - self._("Setting User and Group failed: {0}").format(str(e))) + self._("Setting User and Group failed: {0}").format( + str(e))) self.last_download = fs_filename return self.last_download @@ -421,10 +427,10 @@ def check_for_same_files(self, starting=False): for file in self.pyload.files.cached_files(): if file != self.file and file.name == self.file.name and file.package( ).folder == pack.folder: - if file.status in (0, 12): #: finished or downloading + if file.status in (0, 12): # finished or downloading raise Skip(file.pluginname) elif file.status in ( - 5, 7) and starting: #: a download is waiting/starting and was apparently started before + 5, 7) and starting: # a download is waiting/starting and was apparently started before raise Skip(file.pluginname) download_folder = self.pyload.config.get('general', 'storage_folder') @@ -443,7 +449,8 @@ def check_for_same_files(self, starting=False): raise Skip(file[0]) self.pyload.log.debug( - "File {0} not skipped, because it does not exists".format(self.file.name)) + "File {0} not skipped, because it does not exists".format( + self.file.name)) def clean(self): """ diff --git a/src/pyload/core/network/loader.py b/src/pyload/core/network/loader.py index 3fec2bbb60..1913a7262c 100644 --- a/src/pyload/core/network/loader.py +++ b/src/pyload/core/network/loader.py @@ -3,14 +3,15 @@ from __future__ import absolute_import, unicode_literals +import ast import logging import os import re import time from builtins import object, range, str -import ast from future import standard_library + from pyload.utils.fs import fullpath, lopen, makefile from pyload.utils.layer.legacy.collections_ import defaultdict, namedtuple @@ -26,7 +27,6 @@ class BaseAttributes(defaultdict): """ Dictionary that loads defaults values from Base object. """ - def __missing__(self, key): attr = "__{0}__".format(key) if not hasattr(Base, attr): @@ -39,7 +39,6 @@ class LoaderFactory(object): """ Container for multiple plugin loaders. """ - def __init__(self, *loader): self.loader = list(loader) @@ -230,7 +229,8 @@ def parse_plugin(self, filename, folder, name): version = float(attrs['version']) except ValueError: self.log_debug( - folder, name, "Invalid version {0}".format(attrs['version'])) + folder, name, "Invalid version {0}".format( + attrs['version'])) version = 9 # TODO: remove when plugins are fixed, causing update loops else: self.log_debug(folder, name, "No version attribute") @@ -240,7 +240,8 @@ def parse_plugin(self, filename, folder, name): re_plugin = re.compile(attrs['pattern'], flags=re.I) except Exception: self.log_debug( - folder, name, "Invalid regexp pattern '{0}'".format(attrs['pattern'])) + folder, name, "Invalid regexp pattern '{0}'".format( + attrs['pattern'])) re_plugin = self.RE_NO_MATCH else: re_plugin = self.RE_NO_MATCH @@ -250,8 +251,8 @@ def parse_plugin(self, filename, folder, name): # create plugin tuple # user_context=True is the default for non addon plugins - plugin = PluginTuple( - version, re_plugin, deps, category, bool(folder != "addon" or attrs['user_context']), filename) + plugin = PluginTuple(version, re_plugin, deps, category, bool( + folder != "addon" or attrs['user_context']), filename) # These have none or their own config if folder in ("internal", "account", "network"): @@ -273,7 +274,7 @@ def parse_plugin(self, filename, folder, name): for item in config: if item[0] == "activated": break - else: #: activated flag missing + else: # activated flag missing config.insert(0, ("activated", "bool", "Activated", False)) try: diff --git a/src/pyload/core/network/ocr.py b/src/pyload/core/network/ocr.py index c0ff961960..8efe19bf73 100644 --- a/src/pyload/core/network/ocr.py +++ b/src/pyload/core/network/ocr.py @@ -24,15 +24,15 @@ import os from builtins import object, range +from future import standard_library from pkg_resources import resource_filename import Image -from future import standard_library - -from ..__about__ import __package__ from pyload.utils.fs import lopen, remove from pyload.utils.layer.legacy.subprocess_ import PIPE, Popen +from ..__about__ import __package__ + standard_library.install_aliases() @@ -91,7 +91,8 @@ def run_tesser(self, subset=False, digits=True, self.image.save(tmp.name, 'TIFF') if os.name == 'nt': - tessparams = [resource_filename(__package__, 'tesseract/tesseract.exe')] + tessparams = [resource_filename( + __package__, 'tesseract/tesseract.exe')] else: tessparams = ['tesseract'] diff --git a/src/pyload/core/thread/__init__.py b/src/pyload/core/thread/__init__.py index c77371194f..df450970d4 100644 --- a/src/pyload/core/thread/__init__.py +++ b/src/pyload/core/thread/__init__.py @@ -1,10 +1,6 @@ # -*- coding: utf-8 -*- # @author: vuolter -from __future__ import unicode_literals -from __future__ import absolute_import -from future import standard_library -standard_library.install_aliases() from .plugin import PluginThread from .addon import AddonThread from .decrypter import DecrypterThread diff --git a/src/pyload/core/thread/addon.py b/src/pyload/core/thread/addon.py index dcd50fc305..18882cf017 100644 --- a/src/pyload/core/thread/addon.py +++ b/src/pyload/core/thread/addon.py @@ -31,7 +31,7 @@ def __init__(self, manager, func, args, kwargs): self.kwargs = kwargs self.active = [] - self.__pi = None #: ProgressInfo + self.__pi = None # ProgressInfo def start(self): self.manager.add_thread(self) diff --git a/src/pyload/core/thread/decrypter.py b/src/pyload/core/thread/decrypter.py index 92360de89e..908f68844e 100644 --- a/src/pyload/core/thread/decrypter.py +++ b/src/pyload/core/thread/decrypter.py @@ -6,12 +6,13 @@ from builtins import str from future import standard_library + from pyload.utils.misc import accumulate from pyload.utils.purge import uniqify from ..datatype.init import (DownloadStatus, LinkStatus, ProgressInfo, ProgressType) -from ..network.base import Abort, Fail, Retry +from ..network.base import Abort, Retry from ..network.crypter import Package from .plugin import PluginThread @@ -51,16 +52,17 @@ def run(self): if links: self.pyload.log.info( - self._("Decrypted {0:d} links into package {1}").format(len(links), pack.name)) + self._("Decrypted {0:d} links into package {1}").format( + len(links), + pack.name)) api.add_links(self.pid, [l.url for l in links]) for pack_ in packages: api.add_package(pack_.name, pack_.get_urls(), pack.password) self.pyload.files.set_download_status( - self.fid, - DownloadStatus.Finished if not self.error else DownloadStatus.Failed - ) + self.fid, DownloadStatus.Finished + if not self.error else DownloadStatus.Failed) self.manager.done(self) def _decrypt(self, name, urls, password): @@ -78,8 +80,10 @@ def _decrypt(self, name, urls, password): if not klass: self.error = True # if err: - result.extend(LinkStatus( - url, url, -1, DownloadStatus.NotPossible, name) for url in urls) + result.extend( + LinkStatus( + url, url, -1, DownloadStatus.NotPossible, name) + for url in urls) self.pyload.log.debug( "Plugin '{0}' for decrypting was not loaded".format(name)) self.__pi.done += len(urls) @@ -108,8 +112,8 @@ def _decrypt(self, name, urls, password): # no debug for intentional errors # if self.pyload.debug and not isinstance(e, Fail): - # self.pyload.print_exc() - # self.debug_report(plugin.__name__, plugin=plugin) + # self.pyload.print_exc() + # self.debug_report(plugin.__name__, plugin=plugin) finally: if plugin: plugin.clean() @@ -120,12 +124,14 @@ def _decrypt(self, name, urls, password): def decrypt(self, plugin_map, password=None): result = [] self.__pi = ProgressInfo( - "BasePlugin", "", self._("decrypting"), 0, 0, len(self.data), self.owner, + "BasePlugin", "", self._("decrypting"), 0, 0, len( + self.data), self.owner, ProgressType.Decrypting ) # TODO: QUEUE_DECRYPT result = self._pack_result( - self._decrypt(name, urls, password) for name, urls in plugin_map.items()) + self._decrypt(name, urls, password) for name, + urls in plugin_map.items()) # clear the progress self.__pi = None return result diff --git a/src/pyload/core/thread/download.py b/src/pyload/core/thread/download.py index da0de43cbd..08f95dcb44 100644 --- a/src/pyload/core/thread/download.py +++ b/src/pyload/core/thread/download.py @@ -6,13 +6,14 @@ import sys import time from builtins import str +from queue import Queue from traceback import print_exc from future import standard_library + from pycurl import error from pyload.requests.request import ResponseException from pyload.utils.layer.safethreading import Event -from queue import Queue from ..network.base import Abort, Fail, Retry from ..network.hoster import Reconnect, Skip @@ -34,7 +35,7 @@ def __init__(self, manager): PluginThread.__init__(self, manager) self.__running = Event() - self.queue = Queue() #: job queue + self.queue = Queue() # job queue self.active = None @property @@ -116,7 +117,8 @@ def _handle_skip(self, file, errmsg): file.set_status("skipped") self.pyload.log.info( - self._("Download skipped: {0} due to {1}").format(file.name, errmsg)) + self._("Download skipped: {0} due to {1}").format( + file.name, errmsg)) self.clean(file) @@ -131,7 +133,8 @@ def _handle_error(self, file, errmsg, errcode=None): if errcode in (7, 18, 28, 52, 56): self.pyload.log.warning( - self._("Couldn't connect to host or connection reset, waiting 1 minute and retry")) + self._( + "Couldn't connect to host or connection reset, waiting 1 minute and retry")) wait = time.time() + 60 file.wait_until = wait diff --git a/src/pyload/core/thread/info.py b/src/pyload/core/thread/info.py index 80b0d890b2..cf9f48223f 100644 --- a/src/pyload/core/thread/info.py +++ b/src/pyload/core/thread/info.py @@ -6,6 +6,7 @@ from builtins import int, str from future import standard_library + from pyload.utils import parse from pyload.utils.check import hasmethod from pyload.utils.misc import accumulate @@ -27,11 +28,11 @@ def __init__(self, manager, owner, data, pid=-1, oc=None): # [...(url, plugin)...] self.data = data self.pid = pid - self.oc = oc #: online check + self.oc = oc # online check # urls that already have a package name self.names = {} - self.__pi = None #: ProgressInfo + self.__pi = None # ProgressInfo def start(self): self.manager.add_thread(self) @@ -135,8 +136,8 @@ def fetch_for_plugin(self, plugin, urls, cb): # final number of links to be checked done = self.__pi.done + len(urls) try: - cached = [] #: results loaded from cache - process = [] #: urls to process + cached = [] # results loaded from cache + process = [] # urls to process for url in urls: if url in self.manager.info_cache: cached.append(self.manager.info_cache[url]) @@ -145,7 +146,9 @@ def fetch_for_plugin(self, plugin, urls, cb): if cached: self.manager.log.debug( - "Fetched {0:d} links from cache for {1}".format(len(cached), pluginname)) + "Fetched {0:d} links from cache for {1}".format( + len(cached), + pluginname)) self.__pi.done += len(cached) cb(cached) @@ -163,8 +166,13 @@ def fetch_for_plugin(self, plugin, urls, cb): if isinstance(res, LinkStatus): links.append(res) elif isinstance(res, tuple) and len(res) == 4: - links.append(LinkStatus( - res[3], res[0], int(res[1]), res[2], pluginname)) + links.append( + LinkStatus( + res[3], + res[0], + int(res[1]), + res[2], + pluginname)) elif isinstance(res, tuple) and len(res) == 5: links.append(LinkStatus(res[3], res[0], int( res[1]), res[2], pluginname, res[4])) @@ -183,7 +191,8 @@ def fetch_for_plugin(self, plugin, urls, cb): "Finished Info Fetching for {0}".format(pluginname)) except Exception as e: self.manager.log.warning( - self._("Info Fetching for {0} failed | {1}").format(pluginname, str(e))) + self._("Info Fetching for {0} failed | {1}").format( + pluginname, str(e))) # self.pyload.print_exc() finally: self.__pi.done = done diff --git a/src/pyload/core/thread/plugin.py b/src/pyload/core/thread/plugin.py index 318820df43..29f8d25f5a 100644 --- a/src/pyload/core/thread/plugin.py +++ b/src/pyload/core/thread/plugin.py @@ -6,13 +6,12 @@ import sys import time import zipfile -from builtins import str from future import standard_library -from pyload.utils import debug, format -from pyload.utils.fs import lopen, makedirs + +from pyload.utils import debug +from pyload.utils.fs import makedirs from pyload.utils.layer.safethreading import Thread -from semver import format_version standard_library.install_aliases() @@ -29,10 +28,10 @@ def __init__(self, manager, owner=None): """ Thread.__init__(self) self.setDaemon(True) - self.manager = manager #: Thread manager + self.manager = manager # Thread manager self.pyload = manager.pyload self._ = manager.pyload._ - #: Owner of the thread, every type should set it or overwrite user + # Owner of the thread, every type should set it or overwrite user self.owner = owner @property @@ -95,10 +94,12 @@ def _zip(self, filepath, reports, dumpdir): zip.writestr(arcname, data) def debug_report(self, file): - dumpdir = os.path.join(self.pyload.cachedir, 'plugins', file.pluginname) + dumpdir = os.path.join(self.pyload.cachedir, + 'plugins', file.pluginname) makedirs(dumpdir, exist_ok=True) - reportdir = os.path.join('crashes', 'plugins', file.pluginname) # NOTE: Relpath to configdir + # NOTE: Relpath to configdir + reportdir = os.path.join('crashes', 'plugins', file.pluginname) makedirs(reportdir, exist_ok=True) filename = "debug-report_{0}_{1}.zip".format( diff --git a/tests/api/apiproxy.py b/tests/api/apiproxy.py index ef7c373e23..827e207405 100644 --- a/tests/api/apiproxy.py +++ b/tests/api/apiproxy.py @@ -6,6 +6,7 @@ from builtins import object, zip from future import standard_library + from pyload.core.datatype.debug import classes, methods from tests.helper.config import credentials @@ -16,7 +17,6 @@ class ApiProxy(object): """ Proxy that does type checking on the api. """ - def __init__(self, api, user=credentials[0], pw=credentials[1]): self.api = api self.user = user @@ -27,7 +27,7 @@ def __init__(self, api, user=credentials[0], pw=credentials[1]): def assert_type(self, result, type_): if not type_: - return #: void + return # void try: # Complex attribute if isinstance(type_, tuple): @@ -54,7 +54,7 @@ def assert_type(self, result, type_): for attr, atype in zip(result.__slots__, classes[ result.__name__]): self.assert_type(getattr(result, attr), atype) - else: #: simple object + else: # simple object assert isinstance(result, type_) except AssertionError: print("Assertion for {0} as {1} failed".format(result, type_)) @@ -72,6 +72,7 @@ def call(*args, **kwargs): return call + if __name__ == '__main__': from pyload.rpc.jsonclient import JSONClient diff --git a/tests/api/apitester.py b/tests/api/apitester.py index c57e57bc4c..76a204746c 100644 --- a/tests/api/apitester.py +++ b/tests/api/apitester.py @@ -5,6 +5,7 @@ from builtins import object from future import standard_library + from pyload.rpc.jsonclient import JSONClient from pyload.rpc.wsclient import WSClient from tests.api.apiproxy import ApiProxy diff --git a/tests/api/test_api.py b/tests/api/test_api.py index c6bb19df2b..be6e1e7431 100644 --- a/tests/api/test_api.py +++ b/tests/api/test_api.py @@ -6,6 +6,7 @@ from builtins import range from future import standard_library + from pyload.core import Core from tests.api.apitester import ApiTester from unittest2 import TestCase diff --git a/tests/api/test_jsonbackend.py b/tests/api/test_jsonbackend.py index 5576b58176..2df570c4d2 100644 --- a/tests/api/test_jsonbackend.py +++ b/tests/api/test_jsonbackend.py @@ -6,9 +6,9 @@ from builtins import object import requests +from future import standard_library from requests.auth import HTTPBasicAuth -from future import standard_library from nose.tools import assertEqual, raises from pyload.core.datatype import Forbidden from pyload.rpc.jsonclient import JSONClient @@ -42,8 +42,10 @@ def test_jsonbody(self): payload = {'section': 'webui', 'option': 'port'} headers = {'content-type': 'application/json'} - ret = requests.get(webaddress + "/getConfigValue", headers=headers, - auth=HTTPBasicAuth(*credentials), data=json.dumps(payload)) + ret = requests.get( + webaddress + "/getConfigValue", headers=headers, + auth=HTTPBasicAuth(*credentials), + data=json.dumps(payload)) assertEqual(ret.status_code, 200) assert ret.text diff --git a/tests/api/test_noargs.py b/tests/api/test_noargs.py index 585a29d92f..42a178e936 100644 --- a/tests/api/test_noargs.py +++ b/tests/api/test_noargs.py @@ -5,6 +5,7 @@ import inspect from future import standard_library + from pyload.core.api import AbstractApi from tests.api.apitester import ApiTester @@ -19,6 +20,7 @@ class TestNoArgs(ApiTester): def setUp(self): self.enable_json() + # Setup test_methods dynamically, only these which require no arguments for name in dir(AbstractApi): if name.startswith("_") or name in IGNORE: @@ -26,7 +28,7 @@ def setUp(self): spec = inspect.getargspec(getattr(AbstractApi, name)) if len(spec.args) == 1 and (not spec.varargs or len(spec.varargs) == 0): - def meta_test(name): #: retain local scope + def meta_test(name): # retain local scope def test(self): getattr(self.api, name)() test.__name__ = "test_{0}".format(name) diff --git a/tests/api/test_wsbackend.py b/tests/api/test_wsbackend.py index 3ff8296e99..6ac536f7f8 100644 --- a/tests/api/test_wsbackend.py +++ b/tests/api/test_wsbackend.py @@ -5,6 +5,7 @@ from builtins import object from future import standard_library + from nose.tools import raises from pyload.core.datatype import Forbidden from pyload.rpc.wsclient import WSClient diff --git a/tests/crypter_tester.py b/tests/crypter_tester.py index 19fbd768ea..72ece2a47e 100644 --- a/tests/crypter_tester.py +++ b/tests/crypter_tester.py @@ -7,6 +7,7 @@ from logging import DEBUG, log from future import standard_library + from nose.tools import nottest from pyload.core.plugin.base import Fail from pyload.utils.convert import accumulate, to_int @@ -46,21 +47,20 @@ def test_plugin(self, name, url, flag): c = Core() -crypterlinks = os.path.join(os.path.dirname(__file__), "crypterlinks.txt") -with lopen(crypterlinks) as fp: - links = [line.strip() for line in bufread(fp, buffering=1):] urls = [] flags = {} -for l in links: - if not l or l.startswith("#"): - continue - if l.startswith("http"): - if "||" in l: - l, flag = l.split("||") - flags[l] = flag - - urls.append(l) +crypterlinks = os.path.join(os.path.dirname(__file__), "crypterlinks.txt") +with lopen(crypterlinks) as fp: + links = (line.strip() for line in bufread(fp, buffering=1)) + for l in links: + if not l or l.startswith("#"): + continue + if l.startswith("http"): + if "||" in l: + l, flag = l.split("||") + flags[l] = flag + urls.append(l) h, crypter = c.pgm.parse_urls(urls) plugins = accumulate(crypter) diff --git a/tests/helper/parser.py b/tests/helper/parser.py index d8eef5b37f..4e5f431144 100644 --- a/tests/helper/parser.py +++ b/tests/helper/parser.py @@ -3,6 +3,7 @@ from __future__ import absolute_import, unicode_literals from future import standard_library + from pyload.utils.fs import bufread, lopen standard_library.install_aliases() diff --git a/tests/helper/plugintester.py b/tests/helper/plugintester.py index f7dbac55b7..75cd20b247 100644 --- a/tests/helper/plugintester.py +++ b/tests/helper/plugintester.py @@ -12,11 +12,12 @@ from logging import DEBUG, log from future import standard_library + from pycurl import FORM_FILE, LOW_SPEED_TIME from pyload.core.plugin.base import Abort, Fail from pyload.core.plugin.hoster import Hoster from pyload.requests import get_request -from pyload.utils.fs import bufread, lopen, makedirs, remove +from pyload.utils.fs import lopen, makedirs, remove from tests.helper.stubs import Core, Thread, noop from unittest2 import TestCase @@ -45,10 +46,13 @@ def _wait(self): self.waiting = False self.file.set_status("starting") + Hoster.wait = _wait -def decrypt_captcha(self, url, get={}, post={}, cookies=False, forceuser=False, imgtype='jpg', +def decrypt_captcha(self, url, get={}, + post={}, + cookies=False, forceuser=False, imgtype='jpg', result_type='textual'): img = self.load(url, get=get, post=post, cookies=cookies) @@ -67,12 +71,14 @@ def decrypt_captcha(self, url, get={}, post={}, cookies=False, forceuser=False, # raise timeout threshold req.c.setopt(LOW_SPEED_TIME, 300) - json = req.load("http://captchatrader.com/api/submit", - post={'api_key': "9f65e7f381c3af2b076ea680ae96b0b7", - 'username': fp.readline().strip(), - 'password': fp.readline().strip(), - 'value': (FORM_FILE, fp.name), - 'type': "file"}, multipart=True) + json = req.load( + "http://captchatrader.com/api/submit", + post={'api_key': "9f65e7f381c3af2b076ea680ae96b0b7", + 'username': fp.readline().strip(), + 'password': fp.readline().strip(), + 'value': (FORM_FILE, fp.name), + 'type': "file"}, + multipart=True) response = loads(json) log(DEBUG, str(response)) @@ -82,18 +88,21 @@ def decrypt_captcha(self, url, get={}, post={}, cookies=False, forceuser=False, return result + Hoster.decrypt_captcha = decrypt_captcha def invalid_captcha(self): log(DEBUG, "Captcha invalid") + Hoster.invalid_captcha = invalid_captcha def correct_captcha(self): log(DEBUG, "Captcha correct") + Hoster.correct_captcha = correct_captcha Hoster.check_for_same_files = noop diff --git a/tests/helper/stubs.py b/tests/helper/stubs.py index 78015fb584..390a742eaa 100644 --- a/tests/helper/stubs.py +++ b/tests/helper/stubs.py @@ -11,6 +11,7 @@ from traceback import format_exc from future import standard_library + from pyload.config import ConfigParser from pyload.core import Core from pyload.core.datatype import Role, User diff --git a/tests/hoster_tester.py b/tests/hoster_tester.py index 8f8a6b4e84..d7b045531d 100644 --- a/tests/hoster_tester.py +++ b/tests/hoster_tester.py @@ -10,6 +10,7 @@ from logging import DEBUG, log from future import standard_library + from nose.tools import nottest from pyload.core.datatype import File from pyload.core.datatype.file import statusmap @@ -51,7 +52,7 @@ def test_plugin(self, name, url, status): # url and plugin should be only important thing file = File(self.pyload, -1, url, url, 0, 0, - 0, 0, url, name, "", 0, 0, 0, 0) + 0, 0, url, name, "", 0, 0, 0, 0) file.init_plugin() self.thread.file = file @@ -88,7 +89,7 @@ def test_plugin(self, name, url, status): log(DEBUG, "Hash is {0}".format(hash.hexdigest())) size = os.stat(fp.name).st_size - if size < 10 << 20: #: 10MB + if size < 10 << 20: # 10MB # Copy for debug report log(DEBUG, "Downloaded file copied to report") shutil.move(fp.name, os.path.join(plugin, fp.name)) @@ -103,6 +104,7 @@ def test_plugin(self, name, url, status): else: raise + # setup methods c = Core() diff --git a/tests/manager/test_account.py b/tests/manager/test_account.py index 039e842045..b58568d967 100644 --- a/tests/manager/test_account.py +++ b/tests/manager/test_account.py @@ -3,6 +3,7 @@ from __future__ import absolute_import, unicode_literals from future import standard_library + from pyload.core.manager import AccountManager from tests.helper.stubs import Core, admin_user, normal_user from unittest2 import TestCase diff --git a/tests/manager/test_config.py b/tests/manager/test_config.py index 05acd9d33e..edcbccbb8a 100644 --- a/tests/manager/test_config.py +++ b/tests/manager/test_config.py @@ -5,6 +5,7 @@ from collections import defaultdict from future import standard_library + from nose.tools import raises from pyload.core.database import DatabaseBackend, InvalidConfigSection from pyload.core.manager import ConfigManager diff --git a/tests/manager/test_download.py b/tests/manager/test_download.py index e72a88e5d6..4e9d9020be 100644 --- a/tests/manager/test_download.py +++ b/tests/manager/test_download.py @@ -6,6 +6,7 @@ from builtins import range from future import standard_library + from pyload.core.database import DatabaseBackend from tests.helper.benchmark import BenchmarkTest from tests.helper.stubs import Core, admin_user, normal_user, other_user @@ -52,8 +53,11 @@ def test_add_links(self): for i in range(self.PACKAGES): pid = self.files.add_package( "name {0:d}", "folder", -1, "", "", "", False, user.uid) - self.files.add_links((("url{0:d}".format(i), "plugin{0:d}".format( - i % self.PLUGINS)) for i in range(self.LINKS)), pid, user.uid) + self.files.add_links( + (("url{0:d}".format(i), + "plugin{0:d}".format(i % self.PLUGINS)) + for i in range(self.LINKS)), + pid, user.uid) def test_simple(self): jobs = self.db.get_jobs([]) diff --git a/tests/manager/test_file.py b/tests/manager/test_file.py index 890fed6f03..481427defb 100644 --- a/tests/manager/test_file.py +++ b/tests/manager/test_file.py @@ -7,6 +7,7 @@ from builtins import range from future import standard_library + from pyload.core.database import DatabaseBackend from pyload.core.datatype import DownloadState from tests.helper.benchmark import BenchmarkTest @@ -61,8 +62,10 @@ def test_add_packages(self): def test_add_files(self): for pid in self.pids: - self.manager.add_links((("plugin {0:d}".format(i), "url {0}".format( - i)) for i in range(self.count)), pid, normal_user.uid) + self.manager.add_links( + (("plugin {0:d}".format(i), + "url {0}".format(i)) for i in range(self.count)), + pid, normal_user.uid) count = self.manager.get_queue_stats()[0] files = self.count * len(self.pids) @@ -138,8 +141,10 @@ def test_order_package(self): self.manager.add_links((("url", "plugin") for i in range(100)), parent, normal_user.uid) - pids = [self.manager.add_package( - "c", "", parent, "", "", "", False, normal_user.uid) for i in range(5)] + pids = [ + self.manager.add_package( + "c", "", parent, "", "", "", False, normal_user.uid) + for i in range(5)] v = self.manager.get_tree(parent, False, None) self.assert_ordered(pids, 0, 5, v.root.pids, v.packages, True) diff --git a/tests/manager/test_interaction.py b/tests/manager/test_interaction.py index 76c359808c..8d233d8c7c 100644 --- a/tests/manager/test_interaction.py +++ b/tests/manager/test_interaction.py @@ -5,6 +5,7 @@ from builtins import range from future import standard_library + from pyload.core.datatype import InputType, Interaction from pyload.core.manager import ExchangeManager from tests.helper.stubs import Core diff --git a/tests/misc/test_configparser.py b/tests/misc/test_configparser.py index c5d4b91dfa..b73d15d5af 100644 --- a/tests/misc/test_configparser.py +++ b/tests/misc/test_configparser.py @@ -5,6 +5,7 @@ from builtins import object from future import standard_library + from nose.tools import raises from tests.helper.stubs import Core diff --git a/tests/misc/test_curldownload.py b/tests/misc/test_curldownload.py index d5fcbfec07..6b07c9f902 100644 --- a/tests/misc/test_curldownload.py +++ b/tests/misc/test_curldownload.py @@ -5,6 +5,7 @@ import os from future import standard_library + from pyload.requests.bucket import Bucket from pyload.requests.curl.download import CurlDownload from pyload.requests.curl.request import CurlRequest @@ -31,7 +32,8 @@ def test_download(self): assert self.dl.context is not None self.dl.download( - "https://pyload.net/lib/tpl/pyload/images/pyload-logo-edited3.5-new-font-small.png", "/tmp/random.bin") + "https://pyload.net/lib/tpl/pyload/images/pyload-logo-edited3.5-new-font-small.png", + "/tmp/random.bin") print(self.dl.size, self.dl.arrived) assert self.dl.size == self.dl.arrived > 0 diff --git a/tests/misc/test_curlrequest.py b/tests/misc/test_curlrequest.py index 7dce432f17..14f765d159 100644 --- a/tests/misc/test_curlrequest.py +++ b/tests/misc/test_curlrequest.py @@ -5,6 +5,7 @@ from builtins import dict from future import standard_library + from pyload.requests.curl.request import CurlRequest # needed to register globals from tests.helper import stubs diff --git a/tests/misc/test_filedatabase.py b/tests/misc/test_filedatabase.py index ddbdec9c5d..18470d339c 100644 --- a/tests/misc/test_filedatabase.py +++ b/tests/misc/test_filedatabase.py @@ -7,6 +7,7 @@ from builtins import range from future import standard_library + from pyload.core.database import DatabaseBackend from pyload.core.datatype import DownloadState, FileInfo, PackageInfo from tests.helper.benchmark import BenchmarkTest @@ -21,8 +22,9 @@ class TestDatabase(BenchmarkTest): bench = ["insert", "insert_links", "insert_many", "get_packages", - "get_files", "get_files_queued", "get_package_childs", "get_package_files", - "get_package_data", "get_file_data", "find_files", "collector", "purge"] + "get_files", "get_files_queued", "get_package_childs", + "get_package_files", "get_package_data", "get_file_data", + "find_files", "collector", "purge"] pids = None fids = None owner = 123 @@ -56,8 +58,9 @@ def setUp(self): def test_insert(self, n=200): for i in range(n): - pid = self.db.add_package("name", "folder", random.choice(self.pids), "password", "site", "comment", self.pstatus, - self.owner) + pid = self.db.add_package( + "name", "folder", random.choice(self.pids), + "password", "site", "comment", self.pstatus, self.owner) self.pids.append(pid) def test_insert_links(self): @@ -200,13 +203,13 @@ def assert_file(self, finfo): try: assert finfo is not None assert isinstance(finfo, FileInfo) - self.assert_in( - finfo, ("fid", "status", "size", "media", "fileorder", "added", "package", "owner")) + self.assert_in(finfo, ("fid", "status", "size", "media", + "fileorder", "added", "package", "owner")) assert finfo.status in range(5) assert finfo.owner == self.owner assert finfo.media in range(1024) assert finfo.package in self.pids - assert finfo.added > 10 ** 6 #: date is usually big integer + assert finfo.added > 10 ** 6 # date is usually big integer except Exception: print(finfo) raise @@ -216,7 +219,7 @@ def assert_pack(self, pinfo): assert pinfo is not None assert isinstance(pinfo, PackageInfo) self.assert_in(pinfo, ("pid", "root", "added", - "status", "packageorder", "owner")) + "status", "packageorder", "owner")) assert pinfo.pid in self.pids assert pinfo.owner == self.owner assert pinfo.status in range(5) @@ -232,5 +235,6 @@ def assert_in(self, obj, list): for attr in list: assert isinstance(getattr(obj, attr), int) + if __name__ == '__main__': TestDatabase.benchmark() diff --git a/tests/misc/test_networkrequest.py b/tests/misc/test_networkrequest.py index 3168187759..1acafee250 100644 --- a/tests/misc/test_networkrequest.py +++ b/tests/misc/test_networkrequest.py @@ -6,6 +6,7 @@ from contextlib import closing from future import standard_library + from pyload.core.network.factory import RequestFactory from pyload.requests.curl.request import CurlRequest from tests.helper.stubs import Core diff --git a/tests/misc/test_statdatabase.py b/tests/misc/test_statdatabase.py index 079996b4d1..6c19808328 100644 --- a/tests/misc/test_statdatabase.py +++ b/tests/misc/test_statdatabase.py @@ -5,6 +5,7 @@ from builtins import object from future import standard_library + from tests.helper.stubs import Core standard_library.install_aliases() diff --git a/tests/misc/test_syntax.py b/tests/misc/test_syntax.py index 352c72bd0e..08b95c3847 100644 --- a/tests/misc/test_syntax.py +++ b/tests/misc/test_syntax.py @@ -5,6 +5,7 @@ import os from future import standard_library + # needed to register globals from tests.helper import stubs from unittest2 import TestCase @@ -26,7 +27,7 @@ class TestSyntax(TestCase): if not fname.endswith(".py") or fname.startswith("__"): continue path = os.path.join(PACKDIR, fname) - pack = path.replace(PACKDIR, "")[1:-3] #: replace / and .py + pack = path.replace(PACKDIR, "")[1:-3] # replace / and .py imp = pack.replace("/", ".") packages = imp.split(".") #__import__(imp)