头像缓存持久化

.gitignore

@@ -105,3 +105,4 @@ venv.bak/
 
 
 .idea/
+data/database.db

views/chat.py → api/chat.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 import asyncio
-import datetime
 import enum
 import json
 import logging
@@ -12,6 +11,7 @@
 import tornado.websocket
 
 import blivedm.blivedm as blivedm
+import models.avatar
 
 logger = logging.getLogger(__name__)
 
@@ -26,74 +26,14 @@ class Command(enum.IntEnum):
     DEL_SUPER_CHAT = 6
 
 
-DEFAULT_AVATAR_URL = '//static.hdslb.com/images/member/noface.gif'
-
 _http_session = aiohttp.ClientSession()
-_avatar_url_cache: Dict[int, str] = {}
-_last_fetch_avatar_time = datetime.datetime.now()
-_last_avatar_failed_time = None
-_uids_to_fetch_avatar = asyncio.Queue(15)
 
+room_manager: Optional['RoomManager'] = None
 
-async def get_avatar_url(user_id):
-    if user_id in _avatar_url_cache:
-        return _avatar_url_cache[user_id]
 
-    global _last_avatar_failed_time, _last_fetch_avatar_time
-    cur_time = datetime.datetime.now()
-    # 防止获取头像频率太高被ban
-    if (cur_time - _last_fetch_avatar_time).total_seconds() < 0.2:
-        # 由_fetch_avatar_loop过一段时间再获取并缓存
-        try:
-            _uids_to_fetch_avatar.put_nowait(user_id)
-        except asyncio.QueueFull:
-            pass
-        return DEFAULT_AVATAR_URL
-
-    if _last_avatar_failed_time is not None:
-        if (cur_time - _last_avatar_failed_time).total_seconds() < 3 * 60 + 3:
-            # 3分钟以内被ban，解封大约要15分钟
-            return DEFAULT_AVATAR_URL
-        else:
-            _last_avatar_failed_time = None
-
-    _last_fetch_avatar_time = cur_time
-    try:
-        async with _http_session.get('https://api.bilibili.com/x/space/acc/info',
-                                     params={'mid': user_id}) as r:
-            if r.status != 200:  # 可能会被B站ban
-                logger.warning('Failed to fetch avatar: status=%d %s uid=%d', r.status, r.reason, user_id)
-                _last_avatar_failed_time = cur_time
-                return DEFAULT_AVATAR_URL
-            data = await r.json()
-    except aiohttp.ClientConnectionError:
-        return DEFAULT_AVATAR_URL
-    url = data['data']['face'].replace('http:', '').replace('https:', '')
-    if not url.endswith('noface.gif'):
-        url += '@48w_48h'
-    _avatar_url_cache[user_id] = url
-
-    if len(_avatar_url_cache) > 50000:
-        for _, key in zip(range(100), _avatar_url_cache):
-            del _avatar_url_cache[key]
-
-    return url
-
-
-async def _fetch_avatar_loop():
-    while True:
-        try:
-            user_id = await _uids_to_fetch_avatar.get()
-            if user_id in _avatar_url_cache:
-                continue
-            # 延时长一些使实时弹幕有机会获取头像
-            await asyncio.sleep(0.4 - (datetime.datetime.now() - _last_fetch_avatar_time).total_seconds())
-            asyncio.ensure_future(get_avatar_url(user_id))
-        except:
-            pass
-
-
-asyncio.ensure_future(_fetch_avatar_loop())
+def init():
+    global room_manager
+    room_manager = RoomManager()
 
 
 class Room(blivedm.BLiveClient):
@@ -119,7 +59,7 @@ def __parse_gift(self, command):
         data = command['data']
         return self._on_receive_gift(blivedm.GiftMessage(
             data['giftName'], data['num'], data['uname'], data['face'], None,
-            None, data['timestamp'], None, None,
+            data['uid'], data['timestamp'], None, None,
             None, None, None, data['coin_type'], data['total_coin']
         ))
 
@@ -135,7 +75,7 @@ def __parse_super_chat(self, command):
         return self._on_super_chat(blivedm.SuperChatMessage(
             data['price'], data['message'], None, data['start_time'],
             None, None, data['id'], None,
-            None, None, data['user_info']['uname'],
+            None, data['uid'], data['user_info']['uname'],
             data['user_info']['face'], None,
             None, None,
             None, None, None,
@@ -182,7 +122,7 @@ async def __on_receive_danmaku(self, danmaku: blivedm.DanmakuMessage):
         else:
             author_type = 0
         self.send_message(Command.ADD_TEXT, {
-            'avatarUrl': await get_avatar_url(danmaku.uid),
+            'avatarUrl': await models.avatar.get_avatar_url(danmaku.uid),
             'timestamp': danmaku.timestamp,
             'authorName': danmaku.uname,
             'authorType': author_type,
@@ -196,10 +136,12 @@ async def __on_receive_danmaku(self, danmaku: blivedm.DanmakuMessage):
         })
 
     async def _on_receive_gift(self, gift: blivedm.GiftMessage):
+        avatar_url = gift.face.replace('http:', '').replace('https:', '')
+        models.avatar.update_avatar_cache(gift.uid, avatar_url)
         if gift.coin_type != 'gold':  # 丢人
             return
         self.send_message(Command.ADD_GIFT, {
-            'avatarUrl': gift.face.replace('http:', '').replace('https:', ''),
+            'avatarUrl': avatar_url,
             'timestamp': gift.timestamp,
             'authorName': gift.uname,
             'giftName': gift.gift_name,
@@ -212,14 +154,16 @@ async def _on_buy_guard(self, message: blivedm.GuardBuyMessage):
 
     async def __on_buy_guard(self, message: blivedm.GuardBuyMessage):
         self.send_message(Command.ADD_MEMBER, {
-            'avatarUrl':  await get_avatar_url(message.uid),
+            'avatarUrl':  await models.avatar.get_avatar_url(message.uid),
             'timestamp': message.start_time,
             'authorName': message.username
         })
 
     async def _on_super_chat(self, message: blivedm.SuperChatMessage):
+        avatar_url = message.face.replace('http:', '').replace('https:', '')
+        models.avatar.update_avatar_cache(message.uid, avatar_url)
         self.send_message(Command.ADD_SUPER_CHAT, {
-            'avatarUrl': message.face.replace('http:', '').replace('https:', ''),
+            'avatarUrl': avatar_url,
             'timestamp': message.start_time,
             'authorName': message.uname,
             'price': message.price,
@@ -282,9 +226,6 @@ def _del_room(self, room_id):
         del self._rooms[room_id]
 
 
-room_manager = RoomManager()
-
-
 # noinspection PyAbstractClass
 class ChatHandler(tornado.websocket.WebSocketHandler):
     def __init__(self, *args, **kwargs):

config.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+
+import configparser
+import logging
+import os
+from typing import *
+
+logger = logging.getLogger(__name__)
+
+CONFIG_PATH = os.path.join('data', 'config.ini')
+
+_config: Optional['AppConfig'] = None
+
+
+def init():
+    reload()
+
+
+def reload():
+    config = AppConfig()
+    if config.load(CONFIG_PATH):
+        global _config
+        _config = config
+
+
+def get_config():
+    return _config
+
+
+class AppConfig:
+    def __init__(self):
+        self.database_url = 'sqlite:///data/database.db'
+
+    def load(self, path):
+        config = configparser.ConfigParser()
+        config.read(path)
+        try:
+            app_section = config['app']
+            self.database_url = app_section['database_url']
+        except (KeyError, ValueError):
+            logger.exception('Failed to load config:')
+            return False
+        return True

data/config.ini

@@ -0,0 +1,8 @@
+[app]
+# See https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls
+database_url = sqlite:///data/database.db
+
+
+# DON'T modify this section
+[DEFAULT]
+database_url = sqlite:///data/database.db

main.py

@@ -1,60 +1,79 @@
 # -*- coding: utf-8 -*-
 
 import argparse
-import asyncio
 import logging
 import os
 import webbrowser
 
 import tornado.ioloop
 import tornado.web
 
+import api.chat
+import api.main
+import config
+import models.avatar
+import models.database
 import update
-import views.chat
-import views.main
 
 logger = logging.getLogger(__name__)
 
 WEB_ROOT = os.path.join(os.path.dirname(__file__), 'frontend', 'dist')
 
+routes = [
+    (r'/chat', api.chat.ChatHandler),
+
+    (r'/((css|fonts|img|js|static)/.*)', tornado.web.StaticFileHandler, {'path': WEB_ROOT}),
+    (r'/(favicon\.ico)', tornado.web.StaticFileHandler, {'path': WEB_ROOT}),
+    (r'/.*', api.main.MainHandler, {'path': WEB_ROOT})
+]
+
 
 def main():
+    args = parse_args()
+
+    init_logging(args.debug)
+    config.init()
+    models.database.init(args.debug)
+    models.avatar.init()
+    api.chat.init()
+    update.check_update()
+
+    run_server(args.host, args.port, args.debug)
+
+
+def parse_args():
     parser = argparse.ArgumentParser(description='用于OBS的仿YouTube风格的bilibili直播聊天层')
     parser.add_argument('--host', help='服务器host，默认为127.0.0.1', default='127.0.0.1')
     parser.add_argument('--port', help='服务器端口，默认为12450', type=int, default=12450)
     parser.add_argument('--debug', help='调试模式', action='store_true')
-    args = parser.parse_args()
+    return parser.parse_args()
+
 
+def init_logging(debug):
     logging.basicConfig(
         format='{asctime} {levelname} [{name}]: {message}',
         datefmt='%Y-%m-%d %H:%M:%S',
         style='{',
-        level=logging.INFO if not args.debug else logging.DEBUG
+        level=logging.INFO if not debug else logging.DEBUG
     )
 
-    asyncio.ensure_future(update.check_update())
 
+def run_server(host, port, debug):
     app = tornado.web.Application(
-        [
-            (r'/chat', views.chat.ChatHandler),
-
-            (r'/((css|fonts|img|js|static)/.*)', tornado.web.StaticFileHandler, {'path': WEB_ROOT}),
-            (r'/(favicon\.ico)', tornado.web.StaticFileHandler, {'path': WEB_ROOT}),
-            (r'/.*', views.main.MainHandler, {'path': WEB_ROOT})
-        ],
-        websocket_ping_interval=30,
-        debug=args.debug,
+        routes,
+        websocket_ping_interval=10,
+        debug=debug,
         autoreload=False
     )
     try:
-        app.listen(args.port, args.host)
+        app.listen(port, host)
     except OSError:
-        logger.warning('Address is used %s:%d', args.host, args.port)
+        logger.warning('Address is used %s:%d', host, port)
         return
     finally:
-        url = 'http://localhost' if args.port == 80 else f'http://localhost:{args.port}'
+        url = 'http://localhost' if port == 80 else f'http://localhost:{port}'
         webbrowser.open(url)
-    logger.info('Server started: %s:%d', args.host, args.port)
+    logger.info('Server started: %s:%d', host, port)
     tornado.ioloop.IOLoop.current().start()