diff --git a/GUIEventBinder.py b/GUIEventBinder.py index 5938f6c..7ba190c 100644 --- a/GUIEventBinder.py +++ b/GUIEventBinder.py @@ -32,12 +32,12 @@ def bindEvent(): class File: @staticmethod def bineEvent(): - items = ('parse', 'settings', 'exit') + items = ('logs', 'settings', 'exit') FrameDownloader.MenuBar.batchBind(FrameDownloader.MenuBar.File, gui.frame_downloader.menu_bar.file, items) @staticmethod - def parse(event): - pass + def logs(event): + gui.dialog_dllog.ShowModal() @staticmethod def settings(event): diff --git a/README.md b/README.md index 4157f06..578d8b3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# 爱奇艺、哔哩哔哩、腾讯视频解析器 (IQIYI-BILIBILI-parser) +# 爱奇艺、哔哩哔哩、腾讯视频解析器 (IQIYI-BILIBILI-TENCENT-parser) 爱奇艺、哔哩哔哩、(腾讯视频)解析下载器。【支持导入会员Cookie下载会员视频】 @@ -7,9 +7,12 @@ ## 更新说明 +* **2019/05/04** + * 修复腾讯视频导入Cookie无效问题。 + * 添加下载器进行组下载,将M3U8碎片化以单文件下载。 + * **2019/05/03** - * 加入支持下载腾讯视频。(由于FFMPEG不支持SAMPLE-AES-CTR解密,所以该程序无法下载电影视频,只支持解析出M3U8)。 - *(由于FFMPEG不支持SAMPLE-AES-CTR解密,所以该程序无法下载电影类加密的视频,只支持解析出M3U8,注意需要会员的是需要导入会员Cookie)。 + * 加入支持下载腾讯视频。 *(由于FFMPEG不支持SAMPLE-AES-CTR解密,所以该程序无法下载电影类加密的视频,只支持解析出M3U8,注意需要会员的视频是需要导入会员Cookie)。 * 修复bilibili的flv格式合并错误问题。 * **2019/04/28** diff --git a/core/common.py b/core/common.py index 1c88f18..ed164e0 100644 --- a/core/common.py +++ b/core/common.py @@ -232,6 +232,34 @@ def getVideoLegalTitle(self): +class BasicUserCookie: + def __init__(self): + self.extra_info = {} + + def extract(self, cookie_str): + + rex_query = re.compile('((\w+)=(.+?))[;$\s"]') + ele_query = rex_query.findall(cookie_str + ' ') + for i in ele_query: + self.checkQuery(i[0]) + + def checkQuery(self, query): + if not query: + return + key, value = [i.strip().strip('"').strip("'") for i in splitvalue(query)] + self.extra_info[key] = value + + def dumps(self): + _list = ['%s=%s' % (i[0], i[1]) for i in self.extra_info.items()] + return '; '.join(_list) + + def extract_headers(self, headers_list): + if headers_list: + for i in headers_list: + self.checkQuery(i.split(';')[0]) + + + class BasicUrlGroup: def __init__(self, _init_items=None): self._members = [] @@ -307,3 +335,11 @@ def __getattr__(self, item): return object.__getattribute__(self, item) +def matchParse(url, quality, features): + global WEBSITE + res = WEBSITE.parse(url, [quality]) + for i in res: + if i.matchFeature(features): + return i + + return None \ No newline at end of file diff --git a/core/tencent.py b/core/tencent.py index 38dd9ff..d8408f5 100644 --- a/core/tencent.py +++ b/core/tencent.py @@ -1,13 +1,13 @@ import os -from core.common import BasicParser, BasicRespond, BasicVideoInfo, BasicAudioInfo, \ - CONTENT_TYPE_MAP, make_query, dict_get_key, format_byte, BasicUrlGroup +from core.common import BasicParser, BasicRespond, BasicVideoInfo, BasicAudioInfo, BasicUserCookie, \ + CONTENT_TYPE_MAP, make_query, dict_get_key, format_byte, BasicUrlGroup, raw_decompress from bs4 import BeautifulSoup import re import json import PyJSCaller import time -from urllib.parse import splittype, splithost, urlencode, unquote, splitvalue, splitquery +from urllib.parse import splittype, splithost, urlencode, unquote, splitvalue, splitquery, quote import CommonVar as cv @@ -185,16 +185,31 @@ def saveCookie(self, new_cookie_str): def parse(self, url, quality): text = self.request(url) - # bs4 = BeautifulSoup(text, features='html.parser') + bs4 = BeautifulSoup(text, features='html.parser') video_info_rex = re.compile('var VIDEO_INFO\s*=\s*({.+?})\s*') - video_info = json.loads(video_info_rex.search(text).group(1)) + s1, s2 = splittype(url) + host, path = splithost(s2) + seg_path = path.split('/') + if seg_path[-2] == 'cover': + url = bs4.find('link', rel='canonical').get('href') + # text = self.request(url) + # video_info_rex = re.compile('var VIDEO_INFO\s*=\s*({.+?})\s*') + + # video_info = json.loads(video_info_rex.search(text).group(1)) + s1, s2 = splittype(url) + host, path = splithost(s2) + seg_path = path.split('/') + + vid = seg_path[-1].split('.html')[0] + + # scripts = bs4.findAll('script') title = video_info['title'] - - self.auth_refresh() + if self.cookie_str: + self.auth_refresh() res_videos = [] for i in quality: @@ -203,11 +218,7 @@ def parse(self, url, quality): getckey, createGUID, setdocument = sess.require('getckey', 'createGUID', 'setdocument') setdocument(url) - s1, s2 = splittype(url) - host, path = splithost(s2) - seg_path = path.split('/') - vid = seg_path[-1].split('.html')[0] tm = int(time.time()) @@ -240,7 +251,7 @@ def parse(self, url, quality): 'cKey': ckey.getValue(), 'guid': guid.getValue(), 'defn': i, - 'logintoken': str(new_token).replace("'", '"') + "logintoken": new_token } req_vinfoparam = vinfoparam.copy() @@ -250,11 +261,12 @@ def parse(self, url, quality): req_data = str(req_globalparams).replace("'", '"').encode('utf-8') text = self.request(method='POST', url='https://vd.l.qq.com/proxyhttp', data=req_data) - + # urlencode() res_json = json.loads(text) res_json['vinfo'] = json.loads(res_json['vinfo']) - res_videos.append(TencentRespond(self, res_json, res_json, BasicVideoInfo(url, title, i))) + if res_json['vinfo'].get('vl'): + res_videos.append(TencentRespond(self, res_json, res_json, BasicVideoInfo(url, title, i))) @@ -269,16 +281,26 @@ def auth_refresh(self): new_url = make_query(AUTH_REFRESH_URL, add_params) res = self.requestRaw(url=new_url) + raw = res.read() + text = raw_decompress(raw, res.info()) + + res.close() + res_josn = json.loads(text[text.index('{'):]) + + if res_josn['errcode'] != 0: + raise Exception('导入的cookie不正确,返回:', res_josn['msg']) self.user.extract_headers(res.info().get_all('set-cookie')) new_cookie_str = self.user.dumps() self.saveCookie(new_cookie_str) self.loadCookie(new_cookie_str) + return True -class TencentUser: +class TencentUser(BasicUserCookie): def __init__(self): + BasicUserCookie.__init__(self) self.main_login = '' self.openid = '' self.appid = '' @@ -286,16 +308,11 @@ def __init__(self): self.vuserid = '' self.vusession = '' - self.extra_info = {} - - def extract(self, cookie_str): - - eles = cookie_str.split(';') - for i in eles: - self.checkQuery(i) - def checkQuery(self, query): - key, value = [i.strip() for i in splitvalue(query)] + BasicUserCookie.checkQuery(self, query) + if not query: + return + key, value = [i.strip().strip('"').strip("'") for i in splitvalue(query)] if key == 'main_login': self.main_login = value elif 'openid' in key: @@ -308,37 +325,27 @@ def checkQuery(self, query): self.vuserid = value elif 'vusession' in key: self.vusession = value - else: - self.extra_info[key] = value - def dumps(self): - if self.main_login == 'qq': - add_info = { - 'main_login': self.main_login, - 'vqq_openid': self.openid, - 'vqq_appid': self.appid, - 'vqq_access_token': self.access_token, - 'vqq_vuserid': self.vuserid, - 'vqq_vusession': self.vusession, - } - elif self.main_login == 'wx': - add_info = { - 'main_login': self.main_login, - 'openid': self.openid, - 'appid': self.appid, - 'access_token': self.access_token, - 'vuserid': self.vuserid, - 'vusession': self.vusession, - } - else: - raise AttributeError('unknown "main_login"') - - add_info.update(self.extra_info) - _list = ['%s=%s' % (i[0], i[1]) for i in add_info.items()] + _list = [] + for key, value in self.extra_info.items(): + if key == 'main_login': + value = self.main_login + elif 'openid' in key: + value = self.openid + elif 'appid' in key: + value = self.appid + elif 'access_token' in key: + value = self.access_token + elif 'vuserid' in key: + value = self.vuserid + elif 'vusession' in key: + value = self.vusession + _list.append('%s=%s' % (key, value)) return '; '.join(_list) def extract_headers(self, headers_list): + if headers_list: for i in headers_list: self.checkQuery(i.split(';')[0]) @@ -403,6 +410,8 @@ def getConcatMethod(self): return cv.MER_CONCAT_DEMUXER + + def init(): global TENCENT TENCENT = Tencent() @@ -428,6 +437,15 @@ def parse(url, qualitys): return TENCENT.parse(url, [QUALITY[i] for i in qualitys]) +def matchParse(url, quality, features): + global TENCENT + res = TENCENT.parse(url, [quality]) + for i in res: + if i.matchFeature(features): + return i + + return None + if __name__ == '__main__': init() diff --git a/flow.py b/flow.py index fbbf1d5..2b1b877 100644 --- a/flow.py +++ b/flow.py @@ -41,7 +41,8 @@ import nbdler from zipfile import ZipFile from core.common import BasicUrlGroup - +import traceback +import io TOOL_REQ_URL = { 'ffmpeg': 'https://ffmpeg.zeranoe.com/builds/win64/static/ffmpeg-3.2-win64-static.zip', @@ -199,9 +200,17 @@ def __(sel_res): except (socket.timeout, URLError, SSLError): wx.CallAfter(UndoneJob.timeout) else: - cv.SEL_RES = sel_res - wx.CallAfter(__, sel_res) + if not sel_res: + wx.CallAfter(UndoneJob.empty) + else: + cv.SEL_RES = sel_res + wx.CallAfter(__, sel_res) + @staticmethod + def empty(): + dlg = wx.MessageDialog(gui.frame_parse, u'数据返回为空。', u'错误', wx.OK | wx.ICON_ERROR) + dlg.ShowModal() + dlg.Destroy() @staticmethod @@ -259,12 +268,22 @@ def empty(): dlg.ShowModal() dlg.Destroy() + @staticmethod + def exception(msg): + wx.MessageDialog(gui.frame_parse, msg, u'解析异常', wx.OK | wx.ICON_ERROR).ShowModal() + + @staticmethod def _parse(url, qualitys): try: res = parser.parse(url, qualitys) except (socket.timeout, URLError, SSLError): wx.CallAfter(FrameParser.ButtonParse.timeout) + except Exception as e: + msg = traceback.format_exc() + # print(traceback.format_exc()) + wx.CallAfter(FrameParser.ButtonParse.exception, msg) + else: if not res: wx.CallAfter(FrameParser.ButtonParse.empty) @@ -295,15 +314,15 @@ def appendItem(res): gui.frame_parse.SetTitle(res[0].getVideoLegalTitle()) - class ButtonPath: - """Frame Parser Button-[Path] Handler""" - @staticmethod - def handle(): - dlg = wx.DirDialog(gui.frame_parse, style=wx.FD_DEFAULT_STYLE) - if dlg.ShowModal() == wx.ID_OK: - gui.frame_parse.textctrl_path.SetValue(dlg.GetPath()) - cv.FILEPATH = dlg.GetPath() - dlg.Destroy() + # class ButtonPath: + # """Frame Parser Button-[Path] Handler""" + # @staticmethod + # def handle(): + # dlg = wx.DirDialog(gui.frame_parse, style=wx.FD_DEFAULT_STYLE) + # if dlg.ShowModal() == wx.ID_OK: + # gui.frame_parse.textctrl_path.SetValue(dlg.GetPath()) + # cv.FILEPATH = dlg.GetPath() + # dlg.Destroy() class MenuCopyLink: @@ -466,6 +485,8 @@ class FrameDownload: """Frame Download Handler""" @staticmethod def handle(): + # io + # gui.dialog_dllog.start_logger() gui.frame_parse.Hide() FrameDownload.Download.handle() diff --git a/gui/__init__.py b/gui/__init__.py index e34409a..78d90c6 100644 --- a/gui/__init__.py +++ b/gui/__init__.py @@ -8,6 +8,7 @@ from gui.dialog_gettool import DialogGetTool from gui.dialog_settings import DialogSettings from gui.dialog_copylink import DialogCopyLink +from gui.dialog_dllog import DialogDLLog app = None frame_downloader = None @@ -15,14 +16,16 @@ timer = None frame_merger = None dialog_copylink = None +dialog_dllog = None def init(): - global app, frame_downloader, frame_parse, frame_merger, dialog_copylink + global app, frame_downloader, frame_parse, frame_merger, dialog_copylink, dialog_dllog app = wx.App() frame_downloader = FrameMain(None) frame_parse = FrameParser(None) frame_merger = FrameMerger(None) dialog_copylink = DialogCopyLink(frame_parse) + dialog_dllog = DialogDLLog(frame_downloader) def MainLoop(): global app diff --git a/gui/about.py b/gui/about.py index 65d837b..5057e86 100644 --- a/gui/about.py +++ b/gui/about.py @@ -39,7 +39,7 @@ def __init__(self, parent): wx.Font(13, wx.FONTFAMILY_DEFAULT, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_BOLD, False, wx.EmptyString)) - text_sites = wx.StaticText(global_panel, wx.ID_ANY, u"爱奇艺、哔哩哔哩", wx.DefaultPosition, + text_sites = wx.StaticText(global_panel, wx.ID_ANY, u"爱奇艺、哔哩哔哩、腾讯视频", wx.DefaultPosition, wx.DefaultSize, 0) text_sites.Wrap(-1) diff --git a/gui/dialog_dllog.py b/gui/dialog_dllog.py new file mode 100644 index 0000000..186368f --- /dev/null +++ b/gui/dialog_dllog.py @@ -0,0 +1,65 @@ +import wx +# import wx.xrc +from gui.listctrl import ListCtrl_DLLog +import threading, time +# import io + +EVT_DLLOG_APPEND = wx.NewId() + + +class DialogDLLog(wx.Dialog): + + def __init__(self, parent): + wx.Dialog.__init__(self, parent, id=wx.ID_ANY, title=u"下载器日志", pos=wx.DefaultPosition, size=wx.Size(500, 500), + style=wx.DEFAULT_DIALOG_STYLE) + + self.SetSizeHints(wx.DefaultSize, wx.DefaultSize) + + sizer_global = wx.BoxSizer(wx.VERTICAL) + + self.listctrl_log = ListCtrl_DLLog(self, wx.ID_ANY, wx.DefaultPosition, wx.DefaultSize, wx.LC_REPORT) + sizer_global.Add(self.listctrl_log, 1, wx.ALL | wx.EXPAND, 5) + + self.SetSizer(sizer_global) + self.Layout() + + self.Centre(wx.BOTH) + self.Bind(wx.EVT_CLOSE, self.onClose) + + + def __del__(self): + pass + + def onClose(self, event): + self.Hide() + + def start_logger(self, stdout): + threading.Thread(target=self._stdout_thread, name='log_stdout_read_thread', daemon=True, args=(stdout,)).start() + + def _stdout_thread(self, stdout): + while True: + time.sleep(0.1) + line_text = stdout.readline() + while line_text: + self.listctrl_log.Append((line_text,)) + + def append_log(self, event): + self.listctrl_log.Append(event.data) + + + + + +class DialogDLLogAppendEvent(wx.PyEvent): + def __init__(self, data): + wx.PyEvent.__init__(self) + self.SetEventType(EVT_DLLOG_APPEND) + self.data = data + + +# app = wx.App() +# # # frame_main = FrameParser(None) +# # # +# DialogCopyLink(None).Show() +# # # frame_main.Show() +# app.MainLoop() diff --git a/gui/frame_downloader.py b/gui/frame_downloader.py index c2b6e18..37326a7 100644 --- a/gui/frame_downloader.py +++ b/gui/frame_downloader.py @@ -198,20 +198,20 @@ class Menu_File(wx.Menu): def __init__(self, *args): wx.Menu.__init__(self, *args) - self.parse = None + self.logs = None self.settings = None self.exit = None self.initMenuItems() def initMenuItems(self): - self.parse = wx.MenuItem(self, wx.ID_ANY, 'Parse', wx.EmptyString, wx.ITEM_NORMAL) - self.parse.Enable(False) + self.logs = wx.MenuItem(self, wx.ID_ANY, 'Logs', wx.EmptyString, wx.ITEM_NORMAL) + # self.parse.Enable(False) self.settings = wx.MenuItem(self, wx.ID_ANY, 'Settings', wx.EmptyString, wx.ITEM_NORMAL) # self.settings.Enable(False) self.exit = wx.MenuItem(self, wx.ID_ANY, 'Exit', wx.EmptyString, wx.ITEM_NORMAL) - self.Append(self.parse) + self.Append(self.logs) self.AppendSeparator() self.Append(self.settings) self.AppendSeparator() diff --git a/gui/listctrl.py b/gui/listctrl.py index 8372f09..e5992c8 100644 --- a/gui/listctrl.py +++ b/gui/listctrl.py @@ -135,3 +135,37 @@ def initItems(self): self.copygroup = wx.MenuItem(self, wx.ID_ANY, u'复制所在组所有链接', wx.EmptyString, wx.ITEM_NORMAL) self.Append(self.copygroup) + + +class ListCtrl_DLLog(wx.ListCtrl): + def __init__(self, *args): + wx.ListCtrl.__init__(self, *args) + self.initColumn() + + # self.menu = Menu_CopyLink() + # self.Bind(wx.EVT_RIGHT_DOWN, self.OnContextMenu) + + def initColumn(self): + self.AppendColumn('信息', format=wx.LIST_FORMAT_RIGHT, width=400) + + + def Append(self, entry, fgcolor=None): + wx.ListCtrl.Append(self, entry) + item_count = self.GetItemCount() + if not item_count % 2: + self.SetItemBackgroundColour(item_count - 1, ODD_BGCOLOR) + + if fgcolor: + self.SetItemTextColour(item_count-1, wx.Colour(fgcolor)) + self.ScrollLines(1) + + def DeleteItem(self, item): + wx.ListCtrl.DeleteItem(self, item) + item_count = self.GetItemCount() + odd = True if item % 2 else False + + for i in range(item_count - item): + + self.SetItemBackgroundColour(i + item, ODD_BGCOLOR if odd else EVEN_BGCOLOR) + + odd = not odd \ No newline at end of file diff --git a/handler/downloader.py b/handler/downloader.py index c62c9b2..876df85 100644 --- a/handler/downloader.py +++ b/handler/downloader.py @@ -45,6 +45,8 @@ def __init__(self): self._thr = None + self.wait_for_run = False + def prepare(self, res): self.filepath = cv.FILEPATH self.max_task = cv.MAX_TASK @@ -72,18 +74,18 @@ def add_handler(self, dlm, urls, path, name, dlm_name=None): if os.path.exists(filepath + '.nbdler') and os.path.exists(filepath): group_done_flag = False - dl = nbdler.open(filepath) + dl = nbdler.open(filepath, wait_for_run=self.wait_for_run) kwargs = { 'urls': [*list(urls)] if isinstance(urls, list) or isinstance( urls, tuple) else [urls], 'range_formats': [self._range_format], 'headers': [self._headers], } - dl.batchAdd(**kwargs) + dl.batchAdd(wait_for_run=self.wait_for_run, **kwargs) # dlm.addHandler(dl, name=index) dlm.addHandler(dl) - elif os.path.exists(filepath + '.nbdler') or not os.path.exists(filepath): + elif not os.path.exists(filepath): group_done_flag = False # for mul_url in self.video_urls: @@ -98,7 +100,7 @@ def add_handler(self, dlm, urls, path, name, dlm_name=None): 'buffer_size': cv.BUFFER_SIZE * 1024 * 1024, 'block_size': cv.BLOCK_SIZE * 1024 } - dl = nbdler.open(**kwargs) + dl = nbdler.open(wait_for_run=self.wait_for_run, **kwargs) # dlm.addHandler(dl, name=index) if dlm_name is not None: dlm.addHandler(dl, name=dlm_name) @@ -113,6 +115,8 @@ def __run__(self): self.dlm.config(max_task=self.max_task) self.generate_name() + if len(self.video_filenames) + len(self.audio_filenames) > 100: + self.wait_for_run = True path = os.path.join(self.filepath, self._title) ######################## video diff --git a/handler/logs.py b/handler/logs.py new file mode 100644 index 0000000..65ba6d8 --- /dev/null +++ b/handler/logs.py @@ -0,0 +1,34 @@ + +import sys, time +from threading import Thread, Lock +import gui, wx + +class StdoutPipe: + def __init__(self): + self._buff = [] + self.__console__ = sys.stdout + self._writhread = Thread(target=self._writethread, daemon=True).start() + + def write(self, data): + self._buff.append(data) + self.__console__.write(data) + + + def _writethread(self): + while True: + if self._buff: + line_text = self._buff.pop(0) + if line_text.strip(): + wx.CallAfter(gui.dialog_dllog.listctrl_log.Append, (str(line_text.strip()),)) + time.sleep(0.1) + + + def print_stdconsole(self): + self.__console__.write() + + def flush(self): + # self._buff = '' + pass + + # def + diff --git a/handler/parser.py b/handler/parser.py index 11c8d58..c4c4b27 100644 --- a/handler/parser.py +++ b/handler/parser.py @@ -35,18 +35,18 @@ def get_parser_from_url(url): def parse(url, *args): global LASTREQ, LASTRES LASTREQ = (url, *args) - _parser = get_parser_from_url(url) - _parser.init() - LASTRES = _parser.parse(url, *args) + sel_parser = get_parser_from_url(url) + sel_parser.init() + LASTRES = sel_parser.parse(url, *args) return LASTRES def matchParse(url, quality, features): global LASTREQ, LASTRES LASTREQ = (url, quality, features) - _parser = get_parser_from_url(url) - _parser.init() - return _parser.matchParse(*LASTREQ) + sel_parser = get_parser_from_url(url) + sel_parser.init() + return sel_parser.matchParse(*LASTREQ) diff --git a/main.py b/main.py index cac754f..decf6af 100644 --- a/main.py +++ b/main.py @@ -15,6 +15,7 @@ import threading import flow import handler +from handler.logs import StdoutPipe socket.setdefaulttimeout(3) @@ -39,10 +40,11 @@ def wait_thread(): ferr.write('%s\n' % time.asctime(time.localtime(time.time()))) ferr.write('------------------------\n') sys.stderr = ferr + gui.init() GUIEventBinder.init() - + sys.stdout = StdoutPipe() main() gui.MainLoop() diff --git a/nbdler/DLHandler.py b/nbdler/DLHandler.py index 5f04667..b84b347 100644 --- a/nbdler/DLHandler.py +++ b/nbdler/DLHandler.py @@ -42,6 +42,8 @@ def __init__(self): self.shutdown_flag = False + self._wait_for_run = False + self._batchnode_bak = None def uninstall(self): self.globalprog = self.__globalprog__ @@ -77,7 +79,6 @@ def batchAdd(self, **kwargs): def addNode(self, *args, **kwargs): self.url.addNode(*args, **kwargs) - def delete(self, url=None, urlid=None): if urlid: self.url.delete(urlid) @@ -114,6 +115,7 @@ def __config_params__(self): 'max_conn': 'url.max_conn', 'buffer_size': 'file.buffer_size', 'max_speed': 'url.max_speed', + 'wait_for_run': '_wait_for_run' } # def setRangeFormat(self, range_format='Range: bytes=%d-%d'): @@ -223,8 +225,11 @@ def fix(self, segs): # return sample_type def __run__(self): + if self.file.size == -1 and self._batchnode_bak and self._wait_for_run: + self.batchAdd(**self._batchnode_bak) if self.__new_project__: - self.file.makeFile() + if not self.file.makeFile(): + return if self.file.size == -1: return self.globalprog.allotter.makeBaseConn() @@ -233,7 +238,7 @@ def __run__(self): self.globalprog.run() def run(self): - self.thrpool.Thread(target=self.__run__).start() + self.thrpool.Thread(target=self.__run__, name='Nbdler-Launch').start() def pause(self): self.globalprog.pause() @@ -246,10 +251,10 @@ def unpack(self, packet): self.__new_project__ = False def shutdown(self): - if not self.isEnd(): - self.shutdown_flag = True - self.globalprog.shutdown() - self.shutdown_flag = False + # if not self.isEnd(): + self.shutdown_flag = True + self.globalprog.shutdown() + self.shutdown_flag = False def __packet_params__(self): diff --git a/nbdler/DLInfos.py b/nbdler/DLInfos.py index f669429..c6f2534 100644 --- a/nbdler/DLInfos.py +++ b/nbdler/DLInfos.py @@ -384,6 +384,8 @@ def makeFile(self, withdir=True): thrs = self.parent.thrpool.getThreadsFromName('Nbdler-AddNode') if len(thrs) == 1 and self.size == -1: thrs[0].join() + if self.parent.shutdown_flag: + return False else: while len(self.parent.thrpool.getThreadsFromName('Nbdler-AddNode')): if self.size != -1: @@ -391,10 +393,11 @@ def makeFile(self, withdir=True): time.sleep(0.01) else: if self.size == -1: - return + return False if self.size == -1: - raise Exception('UrlTimeout.') + return False + # raise Exception('UrlTimeout.') if withdir: try: @@ -409,6 +412,7 @@ def makeFile(self, withdir=True): with open(os.path.join(self.path, self.name), 'wb') as f: f.seek(self.size - 1) f.write(b'\x00') + return True def checkName(self): diff --git a/nbdler/DLManager.py b/nbdler/DLManager.py index ad95910..14710cb 100644 --- a/nbdler/DLManager.py +++ b/nbdler/DLManager.py @@ -17,7 +17,9 @@ def __init__(self): self._insp_thr = None self.shutdown_flag = False self.all_pause_flag = True - self.__queue_lock__ = threading.Lock() + self.__queue_lock__ = threading.RLock() + + self._done_buff = [] def __insp__(self): @@ -32,16 +34,16 @@ def __insp__(self): if not self.queue.undone and self.isEnd(): self.checkRunQueue() break - time.sleep(0.1) + time.sleep(0.01) def checkRunQueue(self): with self.__queue_lock__: - tmp = self.queue.run[:] - for i in tmp: + for i in list(self.queue.run): if self.tasks[i].isEnd(): self.tasks[i].close() self.queue.run.remove(i) self.queue.done.append(i) + self._done_buff.append(i) def getHandler(self, name=None, id=None): if name is None and id is None: @@ -78,15 +80,16 @@ def getNameFromId(self, id): return i def addHandler(self, Handler, name=None): - id = self.newId() + with self.__queue_lock__: + id = self.newId() - name = id if not name else name + name = id if not name else name - self.tasks[id] = Handler - self.name_id[name] = id + self.tasks[id] = Handler + self.name_id[name] = id - self.id_map[id] = True - self.queue.undone.append(id) + self.id_map[id] = True + self.queue.undone.append(id) return id @@ -99,9 +102,10 @@ def newId(self): return len(self.id_map) - 1 def remove(self, id): - del self.tasks[id] - self.id_map[id] = False - del self.name_id[self.getIdFromName(id)] + with self.__queue_lock__: + del self.tasks[id] + self.id_map[id] = False + del self.name_id[self.getIdFromName(id)] def run(self, id=None): with self.__queue_lock__: @@ -124,35 +128,32 @@ def run(self, id=None): if i in self.queue.pause: self.queue.pause.remove(i) self.queue.undone.remove(i) - # else: - # - # if i in self.queue.undone: - # self.queue.done.append(i) - # self.queue.undone.remove(i) def pause(self, id=None): - + self._done_buff = [] if id is not None: self.tasks[id].pause() - if id in self.queue.run: - self.queue.run.remove(id) - if id not in self.queue.pause: - self.queue.pause.append(id) + with self.__queue_lock__: + if id in self.queue.run: + self.queue.run.remove(id) + if id not in self.queue.pause: + self.queue.pause.append(id) else: self.all_pause_flag = True self._insp_thr.join() - tmp = self.getRunQueue()[:] - for i in tmp: - threading.Thread(target=self.tasks[i].pause).start() - if i in self.queue.run: - self.queue.run.remove(i) - if i not in self.queue.pause: - self.queue.pause.append(i) + with self.__queue_lock__: + for i in list(self.getRunQueue()): + threading.Thread(target=self.tasks[i].pause).start() + if i in self.queue.run: + self.queue.run.remove(i) + if i not in self.queue.pause: + self.queue.pause.append(i) self.checkRunQueue() def shutdown(self): + self._done_buff = [] self.shutdown_flag = True if self._insp_thr: self._insp_thr.join() @@ -177,35 +178,45 @@ def join(self): i.join() def getAvgSpeed(self, id=None): - if id is not None: - return self.tasks[id].getAvgSpeed() + with self.__queue_lock__: + if id is not None: + return self.tasks[id].getAvgSpeed() - speed = 0 - for i in self.queue.run: - if not self.tasks[i].isEnd(): - speed += self.tasks[i].getAvgSpeed() + speed = 0 + for i in self.queue.run: + if not self.tasks[i].isEnd(): + speed += self.tasks[i].getAvgSpeed() return speed def getInsSpeed(self, id=None): - if id is not None: - return self.tasks[id].getInsSpeed() - - speed = 0 - for i in self.queue.run: - speed += self.tasks[i].getInsSpeed() + with self.__queue_lock__: + if id is not None: + return self.tasks[id].getInsSpeed() + + speed = 0 + for i in self.queue.run: + speed += self.tasks[i].getInsSpeed() + + for i in list(self._done_buff): + tmp = self.tasks[i].getInsSpeed() + speed += tmp + if tmp < 100: + self._done_buff.remove(i) + # self._done_buff = [] return speed def getIncByte(self, id=None): - if id is not None: - return self.tasks[id].getIncByte() + with self.__queue_lock__: + if id is not None: + return self.tasks[id].getIncByte() - inc = 0 - for i in self.queue.done: - inc += self.tasks[i].getFileSize() - for i in self.queue.run: - dl = self.tasks[i] - inc += dl.getFileSize() - dl.getLeft() + inc = 0 + for i in self.queue.done: + inc += self.tasks[i].getFileSize() + for i in self.queue.run: + dl = self.tasks[i] + inc += dl.getFileSize() - dl.getLeft() return inc @@ -223,27 +234,29 @@ def getTotalSize(self): return size def getLeft(self, id=None): - if id is not None: - return self.tasks[id].getLeft() + with self.__queue_lock__: + if id is not None: + return self.tasks[id].getLeft() - left = 0 - for i in self.queue.run: - if not self.tasks[i].isEnd(): + left = 0 + for i in self.queue.run: + if not self.tasks[i].isEnd(): + left += self.tasks[i].getLeft() + for i in self.queue.undone: left += self.tasks[i].getLeft() - for i in self.queue.undone: - left += self.tasks[i].getLeft() return left def isEnd(self, id=None): - if id is not None: - return self.tasks[id].isEnd() + with self.__queue_lock__: + if id is not None: + return self.tasks[id].isEnd() - for i in self.queue.run: - if not self.tasks[i].isEnd(): - break - else: - return True if not self.queue.undone else False + for i in self.queue.run: + if not self.tasks[i].isEnd(): + break + else: + return True if not self.queue.undone else False return False def config(self, **kwargs): diff --git a/nbdler/DLProgress.py b/nbdler/DLProgress.py index c8bb78b..fb34d31 100644 --- a/nbdler/DLProgress.py +++ b/nbdler/DLProgress.py @@ -340,7 +340,8 @@ def join(self): def pause(self): - + if self.isEnd(): + return self.pause_req = True for i in self.handler.thrpool.getThreadsFromName('Nbdler-AddNode'): i.join() @@ -436,6 +437,8 @@ def getInsSpeed(self): curleft = self.getLeft() curclock = time.time() + + if self.piece.last_left is None: self.piece.last_left = curleft if self.piece.last_clock is None: @@ -447,12 +450,28 @@ def getInsSpeed(self): incbyte = self.piece.last_left - curleft incclock = curclock - self.piece.last_clock + + + if incclock >= 1: self.piece.last_left = curleft + incbyte / 2.0 self.piece.last_clock = curclock - incclock / 2.0 + if self.isEnd() and curleft == 0: + if incbyte > 0 and incclock > 0: + # self.piece.last_left = curleft + # self.piece.last_clock = curclock + return incbyte * 1.0 / incclock + else: + return 0 + + + if incclock > 0 and incbyte > 0: + ins_speed = incbyte * 1.0 / incclock + else: + ins_speed = 0 - return incbyte * 1.0 / incclock if incclock > 0 else 0 + return ins_speed def close(self): self.releaseBuffer() diff --git a/nbdler/__init__.py b/nbdler/__init__.py index 4108d27..60bc445 100644 --- a/nbdler/__init__.py +++ b/nbdler/__init__.py @@ -34,7 +34,10 @@ def open(fp=None, **kwargs): dlh.unpack(packet) else: dlh.config(**kwargs) - dlh.batchAdd(**kwargs) + if kwargs.get('wait_for_run', False): + dlh._batchnode_bak = kwargs + else: + dlh.batchAdd(**kwargs) return dlh