From 8d1d969ba8adf7eb7fcf61f874197da9f8cc2a48 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Thu, 2 Jun 2022 15:34:29 +0800 Subject: [PATCH 01/23] feat(server): add multiprocessing worker class add support for multiprocessing worker. --- sea/__init__.py | 2 - sea/app.py | 15 +++- sea/cmds.py | 2 +- sea/server/__init__.py | 1 + sea/server/multiprocessing.py | 117 +++++++++++++++++++++++++ sea/{server.py => server/threading.py} | 10 --- tests/test_server.py | 2 +- 7 files changed, 133 insertions(+), 16 deletions(-) create mode 100644 sea/server/__init__.py create mode 100644 sea/server/multiprocessing.py rename sea/{server.py => server/threading.py} (82%) diff --git a/sea/__init__.py b/sea/__init__.py index 4cad558..e57cdc6 100644 --- a/sea/__init__.py +++ b/sea/__init__.py @@ -28,8 +28,6 @@ def create_app(root_path=None): app_class = import_string("app:App") _app = app_class(root_path, env=env) _app.config.from_object(config) - # only filter default configurations - _app.config.load_config_from_env() _app.load_middlewares() _app.load_extensions_in_module(import_string("app.extensions")) diff --git a/sea/app.py b/sea/app.py index ed43e08..9fccdcc 100644 --- a/sea/app.py +++ b/sea/app.py @@ -1,5 +1,6 @@ import inspect import logging +import os import os.path import sys @@ -20,11 +21,9 @@ class BaseApp: :param env: the env """ config_class = Config - debug = ConfigAttribute('DEBUG') testing = ConfigAttribute('TESTING') tz = ConfigAttribute('TIMEZONE') default_config = ImmutableDict({ - 'DEBUG': False, 'TESTING': False, 'TIMEZONE': 'UTC', 'GRPC_WORKERS': 3, @@ -47,13 +46,25 @@ def __init__(self, root_path, env): self.root_path = root_path self.name = os.path.basename(root_path) self.env = env + self.debug = (os.environ.get("SEA_DEBUG") or env == "development") self.config = self.config_class(root_path, self.default_config) self._servicers = {} self._extensions = {} self._middlewares = [] + def _setup_root_logger(self): + fmt = self.config['GRPC_LOG_FORMAT'] + lvl = self.config['GRPC_LOG_LEVEL'] + h = self.config['GRPC_LOG_HANDLER'] + h.setFormatter(logging.Formatter(fmt)) + root = logging.getLogger() + root.setLevel(lvl) + root.addHandler(h) + @cached_property def logger(self): + self._setup_root_logger() + logger = logging.getLogger('sea.app') if self.debug and logger.level == logging.NOTSET: logger.setLevel(logging.DEBUG) diff --git a/sea/cmds.py b/sea/cmds.py index 0d76a9f..a7389d2 100644 --- a/sea/cmds.py +++ b/sea/cmds.py @@ -2,7 +2,7 @@ from sea.cli import jobm, JobException from sea import current_app -from sea.server import Server +from sea.server.multiprocessing import Server @jobm.job('server', aliases=['s'], help='Run Server') diff --git a/sea/server/__init__.py b/sea/server/__init__.py new file mode 100644 index 0000000..b19bbc6 --- /dev/null +++ b/sea/server/__init__.py @@ -0,0 +1 @@ +from .threading import Server as Server \ No newline at end of file diff --git a/sea/server/multiprocessing.py b/sea/server/multiprocessing.py new file mode 100644 index 0000000..4a39bd4 --- /dev/null +++ b/sea/server/multiprocessing.py @@ -0,0 +1,117 @@ +import contextlib +import multiprocessing +import signal +import socket +import time +import logging +from concurrent import futures +import grpc + +from sea import signals + +""" +GRPC_WORKERS +GRPC_THREADS +GRPC_HOST +GRPC_PORT +GRPC_GRACE + +GRPC_LOG_FORMAT +GRPC_LOG_LEVEL +GRPC_LOG_HANDLER +""" + + +class Server: + """sea multiprocessing server implements + + :param app: application instance + """ + + def __init__(self, app): + self.app = app + self.worker_num = self.app.config['GRPC_WORKERS'] + self.thread_num = self.app.config.get('GRPC_THREADS', 1) + self.host = self.app.config['GRPC_HOST'] + self.port = self.app.config['GRPC_PORT'] + self.workers = [] + self._stopped = False + + self.server = None # slave process server object + + def _run_server(self, bind_address): + server = grpc.server( + futures.ThreadPoolExecutor(max_workers=self.thread_num), + options=[ + ("grpc.so_reuseport", 1), + # ("grpc.use_local_subchannel_pool", 1), + ], + ) + for _, (add_func, servicer) in self.app.servicers.items(): + add_func(servicer(), server) + server.add_insecure_port(bind_address) + server.start() + self.server = server # set server in slave process + + signals.server_started.send(self) + server.wait_for_termination() + + def run(self): + # # run prometheus client + # if self.app.config['PROMETHEUS_SCRAPE']: + # from prometheus_client import start_http_server + # start_http_server(self.app.config['PROMETHEUS_PORT']) + + self.register_signal() + + with _reserve_address_port(self.host, self.port) as port: + bind_address = "{}:{}".format(self.host, self.port) + for _ in range(self.worker_num): + worker = multiprocessing.Process(target=self._run_server, args=(bind_address,)) + worker.start() + self.workers.append(worker) + for worker in self.workers: + worker.join() + + return True + + + def register_signal(self): + signal.signal(signal.SIGINT, self._stop_handler) + signal.signal(signal.SIGHUP, self._stop_handler) + signal.signal(signal.SIGTERM, self._stop_handler) + signal.signal(signal.SIGQUIT, self._stop_handler) + + def _stop_handler(self, signum, frame): + grace = max(self.app.config['GRPC_GRACE'], 5) if self.app.config['GRPC_GRACE'] else 5 + if not self.server: + self.app.logger.warning("master process received signal {}, sleep {} to wait slave done".format(signum, grace)) + + # master process sleep to wait slaves end their lives + time.sleep(grace) + + # kill the slave process which don't wanna die + for worker in self.workers: + if worker.is_alive(): + self.app.logger.warning("master found process {} still alive after {} timeout".format(worker.pid, grace)) + worker.kill() + self.app.logger.warning("master exit") + else: + # slave process sleep less 3s to make grace more reliable + self.app.logger.warning("slave process received signal {}, try to stop process".format(signum)) + self.server.stop(grace - 3) + time.sleep(grace - 3) + signals.server_stopped.send(self) + +@contextlib.contextmanager +def _reserve_address_port(host, port): + """Find and reserve a port for all subprocesses to use.""" + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 0: + raise RuntimeError("Failed to set SO_REUSEPORT.") + sock.bind(("", port)) + try: + yield sock.getsockname()[1] + finally: + sock.close() \ No newline at end of file diff --git a/sea/server.py b/sea/server/threading.py similarity index 82% rename from sea/server.py rename to sea/server/threading.py index 704c3c7..7b48918 100644 --- a/sea/server.py +++ b/sea/server/threading.py @@ -15,7 +15,6 @@ class Server: def __init__(self, app): self.app = app - self.setup_logger() self.workers = self.app.config['GRPC_WORKERS'] self.host = self.app.config['GRPC_HOST'] self.port = self.app.config['GRPC_PORT'] @@ -42,15 +41,6 @@ def run(self): signals.server_stopped.send(self) return True - def setup_logger(self): - fmt = self.app.config['GRPC_LOG_FORMAT'] - lvl = self.app.config['GRPC_LOG_LEVEL'] - h = self.app.config['GRPC_LOG_HANDLER'] - h.setFormatter(logging.Formatter(fmt)) - logger = logging.getLogger() - logger.setLevel(lvl) - logger.addHandler(h) - def register_signal(self): signal.signal(signal.SIGINT, self._stop_handler) signal.signal(signal.SIGHUP, self._stop_handler) diff --git a/tests/test_server.py b/tests/test_server.py index 2bb4e36..13ceb7a 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -4,7 +4,7 @@ import threading from unittest import mock -from sea.server import Server +from sea.server.threading import Server from sea.signals import server_started, server_stopped From 33fbd4f8877ba09aadb833f1985de68a62a9aea6 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Wed, 8 Jun 2022 23:35:43 +0800 Subject: [PATCH 02/23] feat(metrics): add prometheus support for multiprocessing --- sea/app.py | 1 + sea/server/multiprocessing.py | 46 +++++++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/sea/app.py b/sea/app.py index 9fccdcc..1dbc278 100644 --- a/sea/app.py +++ b/sea/app.py @@ -27,6 +27,7 @@ class BaseApp: 'TESTING': False, 'TIMEZONE': 'UTC', 'GRPC_WORKERS': 3, + 'GRPC_THREADS': 1, # Only appliable in multiprocessing server 'GRPC_HOST': '[::]', 'GRPC_PORT': 6000, 'GRPC_LOG_LEVEL': 'WARNING', diff --git a/sea/server/multiprocessing.py b/sea/server/multiprocessing.py index 4a39bd4..f0d0821 100644 --- a/sea/server/multiprocessing.py +++ b/sea/server/multiprocessing.py @@ -1,12 +1,14 @@ import contextlib +import glob import multiprocessing import signal import socket import time -import logging from concurrent import futures import grpc +import os + from sea import signals """ @@ -19,6 +21,9 @@ GRPC_LOG_FORMAT GRPC_LOG_LEVEL GRPC_LOG_HANDLER + +PROMETHEUS_SCRAPE +PROMETHEUS_PORT """ @@ -31,7 +36,7 @@ class Server: def __init__(self, app): self.app = app self.worker_num = self.app.config['GRPC_WORKERS'] - self.thread_num = self.app.config.get('GRPC_THREADS', 1) + self.thread_num = self.app.config.get('GRPC_THREADS') self.host = self.app.config['GRPC_HOST'] self.port = self.app.config['GRPC_PORT'] self.workers = [] @@ -47,20 +52,42 @@ def _run_server(self, bind_address): # ("grpc.use_local_subchannel_pool", 1), ], ) + self.server = server # set server in slave process + for _, (add_func, servicer) in self.app.servicers.items(): add_func(servicer(), server) server.add_insecure_port(bind_address) server.start() - self.server = server # set server in slave process signals.server_started.send(self) + + # hang up here, to make slave run always server.wait_for_termination() + + def _run_prometheus_http_server(self): + """Run prometheus_client built-in http server. + + Duing to prometheus_client multiprocessing details, + PROMETHEUS_MULTIPROC_DIR must set in environment variables.""" + if not self.app.config['PROMETHEUS_SCRAPE']: + return + + from prometheus_client import start_http_server, REGISTRY + from prometheus_client.multiprocess import MultiProcessCollector + + MultiProcessCollector(REGISTRY) + start_http_server(self.app.config['PROMETHEUS_PORT']) + + def _clean_prometheus(self): + if not self.app.config['PROMETHEUS_SCRAPE']: + return + dir = os.getenv("PROMETHEUS_MULTIPROC_DIR") + self.app.logger.info(f"clean prometheus dir {dir}") + for f in glob.glob(os.path.join(dir, "*")): + os.remove(f) def run(self): - # # run prometheus client - # if self.app.config['PROMETHEUS_SCRAPE']: - # from prometheus_client import start_http_server - # start_http_server(self.app.config['PROMETHEUS_PORT']) + self._run_prometheus_http_server() self.register_signal() @@ -73,6 +100,8 @@ def run(self): for worker in self.workers: worker.join() + self._clean_prometheus() + return True @@ -86,6 +115,7 @@ def _stop_handler(self, signum, frame): grace = max(self.app.config['GRPC_GRACE'], 5) if self.app.config['GRPC_GRACE'] else 5 if not self.server: self.app.logger.warning("master process received signal {}, sleep {} to wait slave done".format(signum, grace)) + signals.server_stopped.send(self) # master process sleep to wait slaves end their lives time.sleep(grace) @@ -98,10 +128,10 @@ def _stop_handler(self, signum, frame): self.app.logger.warning("master exit") else: # slave process sleep less 3s to make grace more reliable + signals.server_stopped.send(self) self.app.logger.warning("slave process received signal {}, try to stop process".format(signum)) self.server.stop(grace - 3) time.sleep(grace - 3) - signals.server_stopped.send(self) @contextlib.contextmanager def _reserve_address_port(host, port): From 830e3369ce94e90f41576760b9ed93b94e2135d0 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Wed, 8 Jun 2022 23:41:42 +0800 Subject: [PATCH 03/23] chore(commit): add pre-commit hooks --- .isort.cfg | 2 ++ .pre-commit-config.yaml | 34 +++++++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..504b2f3 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,2 @@ +[settings] +known_third_party = app,blinker,cachext,celery,configs,google,grpc,helloworld_pb2,helloworld_pb2_grpc,mock,peewee,peeweext,pendulum,pkg_resources,pytest,sentry_sdk,setuptools,worldhello_pb2,worldhello_pb2_grpc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a9ed456..db5d2ee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,31 @@ repos: - - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook - rev: v7.0.1 + - repo: "https://github.com/pre-commit/pre-commit-hooks" + rev: v2.2.3 hooks: - - id: commitlint - stages: [commit-msg] - additional_dependencies: ["@commitlint/config-conventional"] + - id: trailing-whitespace + - id: end-of-file-fixer + - id: no-commit-to-branch + args: [ --branch, master, --branch, develop ] + - repo: https://github.com/asottile/seed-isort-config + rev: v1.9.1 + hooks: + - id: seed-isort-config + - repo: "https://github.com/pre-commit/mirrors-isort" + rev: v5.10.1 + hooks: + - id: isort + - repo: "https://github.com/ambv/black" + rev: 22.3.0 + hooks: + - id: black + language_version: python3 + exclude: "(configs|protos|tests)" + - repo: "https://github.com/pre-commit/pre-commit-hooks" + rev: v2.2.3 + hooks: + - id: flake8 + exclude: "(configs|protos|tests)" + - repo: https://github.com/jorisroovers/gitlint + rev: v0.17.0 + hooks: + - id: gitlint From 8db8dcbed47685354dae26345b1e4927fed6bc25 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Thu, 9 Jun 2022 10:33:31 +0800 Subject: [PATCH 04/23] style: add flake8 config --- .flake8 | 12 ++++++ sea/server/multiprocessing.py | 78 +++++++++++++++++++++-------------- 2 files changed, 60 insertions(+), 30 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..f0fb19f --- /dev/null +++ b/.flake8 @@ -0,0 +1,12 @@ +[flake8] +# https://github.com/ambv/black/blob/master/.flake8 +# E203 whitespace before ':' +# E266 Too many leading '#' for block comment +# E501 Line too long +# W503 Line break occurred before a binary operator +ignore = E203, E266, E501, W503, B950 +max-line-length = 88 +max-complexity = 18 +exclude = + .git, + tests, diff --git a/sea/server/multiprocessing.py b/sea/server/multiprocessing.py index f0d0821..754aa60 100644 --- a/sea/server/multiprocessing.py +++ b/sea/server/multiprocessing.py @@ -1,13 +1,13 @@ import contextlib import glob import multiprocessing +import os import signal import socket import time from concurrent import futures -import grpc -import os +import grpc from sea import signals @@ -35,24 +35,26 @@ class Server: def __init__(self, app): self.app = app - self.worker_num = self.app.config['GRPC_WORKERS'] - self.thread_num = self.app.config.get('GRPC_THREADS') - self.host = self.app.config['GRPC_HOST'] - self.port = self.app.config['GRPC_PORT'] + self.worker_num = self.app.config["GRPC_WORKERS"] + self.thread_num = self.app.config.get("GRPC_THREADS") + self.host = self.app.config["GRPC_HOST"] + self.port = self.app.config["GRPC_PORT"] self.workers = [] self._stopped = False - self.server = None # slave process server object - + self.server = None # slave process server object + def _run_server(self, bind_address): server = grpc.server( futures.ThreadPoolExecutor(max_workers=self.thread_num), options=[ - ("grpc.so_reuseport", 1), - # ("grpc.use_local_subchannel_pool", 1), + ( + "grpc.so_reuseport", + 1, + ), # multiprocessing worker must reuse port to pass between processes. ], - ) - self.server = server # set server in slave process + ) + self.server = server # set server in slave process for _, (add_func, servicer) in self.app.servicers.items(): add_func(servicer(), server) @@ -63,23 +65,23 @@ def _run_server(self, bind_address): # hang up here, to make slave run always server.wait_for_termination() - + def _run_prometheus_http_server(self): """Run prometheus_client built-in http server. - Duing to prometheus_client multiprocessing details, + Duing to prometheus_client multiprocessing details, PROMETHEUS_MULTIPROC_DIR must set in environment variables.""" - if not self.app.config['PROMETHEUS_SCRAPE']: + if not self.app.config["PROMETHEUS_SCRAPE"]: return - from prometheus_client import start_http_server, REGISTRY + from prometheus_client import REGISTRY, start_http_server from prometheus_client.multiprocess import MultiProcessCollector MultiProcessCollector(REGISTRY) - start_http_server(self.app.config['PROMETHEUS_PORT']) - + start_http_server(self.app.config["PROMETHEUS_PORT"]) + def _clean_prometheus(self): - if not self.app.config['PROMETHEUS_SCRAPE']: + if not self.app.config["PROMETHEUS_SCRAPE"]: return dir = os.getenv("PROMETHEUS_MULTIPROC_DIR") self.app.logger.info(f"clean prometheus dir {dir}") @@ -88,22 +90,23 @@ def _clean_prometheus(self): def run(self): self._run_prometheus_http_server() - + self.register_signal() - with _reserve_address_port(self.host, self.port) as port: + with _reserve_address_port(self.host, self.port): bind_address = "{}:{}".format(self.host, self.port) for _ in range(self.worker_num): - worker = multiprocessing.Process(target=self._run_server, args=(bind_address,)) + worker = multiprocessing.Process( + target=self._run_server, args=(bind_address,) + ) worker.start() self.workers.append(worker) for worker in self.workers: worker.join() - + self._clean_prometheus() - - return True + return True def register_signal(self): signal.signal(signal.SIGINT, self._stop_handler) @@ -112,9 +115,17 @@ def register_signal(self): signal.signal(signal.SIGQUIT, self._stop_handler) def _stop_handler(self, signum, frame): - grace = max(self.app.config['GRPC_GRACE'], 5) if self.app.config['GRPC_GRACE'] else 5 + grace = ( + max(self.app.config["GRPC_GRACE"], 5) + if self.app.config["GRPC_GRACE"] + else 5 + ) if not self.server: - self.app.logger.warning("master process received signal {}, sleep {} to wait slave done".format(signum, grace)) + self.app.logger.warning( + "master process received signal {}, sleep {} to wait slave done".format( + signum, grace + ) + ) signals.server_stopped.send(self) # master process sleep to wait slaves end their lives @@ -123,16 +134,23 @@ def _stop_handler(self, signum, frame): # kill the slave process which don't wanna die for worker in self.workers: if worker.is_alive(): - self.app.logger.warning("master found process {} still alive after {} timeout".format(worker.pid, grace)) + self.app.logger.warning( + "master found process {} still alive after {} timeout".format( + worker.pid, grace + ) + ) worker.kill() self.app.logger.warning("master exit") else: # slave process sleep less 3s to make grace more reliable signals.server_stopped.send(self) - self.app.logger.warning("slave process received signal {}, try to stop process".format(signum)) + self.app.logger.warning( + "slave process received signal {}, try to stop process".format(signum) + ) self.server.stop(grace - 3) time.sleep(grace - 3) + @contextlib.contextmanager def _reserve_address_port(host, port): """Find and reserve a port for all subprocesses to use.""" @@ -144,4 +162,4 @@ def _reserve_address_port(host, port): try: yield sock.getsockname()[1] finally: - sock.close() \ No newline at end of file + sock.close() From c1bc78640bd031ce432960dbd1e3f0038b711ec5 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Thu, 9 Jun 2022 10:47:12 +0800 Subject: [PATCH 05/23] chore(commit): change commitlint to gitlint --- .gitlint | 6 ++++++ commitlint.config.js | 3 --- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 .gitlint delete mode 100644 commitlint.config.js diff --git a/.gitlint b/.gitlint new file mode 100644 index 0000000..d2115ac --- /dev/null +++ b/.gitlint @@ -0,0 +1,6 @@ +[general] +# Ignore certain rules (comma-separated list), you can reference them by +# their id or by their full name +ignore=body-is-missing + +contrib=contrib-title-conventional-commits diff --git a/commitlint.config.js b/commitlint.config.js deleted file mode 100644 index a989bfc..0000000 --- a/commitlint.config.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = { - extends: ['@commitlint/config-conventional'] -}; From 6f893a1178df8874e4655773d86613a387a6ef53 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Thu, 9 Jun 2022 10:48:44 +0800 Subject: [PATCH 06/23] chore(commit): fix pre-commit config --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index db5d2ee..025a7f2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,12 +19,12 @@ repos: hooks: - id: black language_version: python3 - exclude: "(configs|protos|tests)" + exclude: "(tests)" - repo: "https://github.com/pre-commit/pre-commit-hooks" rev: v2.2.3 hooks: - id: flake8 - exclude: "(configs|protos|tests)" + exclude: "(tests)" - repo: https://github.com/jorisroovers/gitlint rev: v0.17.0 hooks: From 2a9821e983abe6415d03505a8e6d41379daccddf Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Fri, 17 Jun 2022 17:29:57 +0800 Subject: [PATCH 07/23] feat: ignore stop signal when stopping server --- sea/server/multiprocessing.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/sea/server/multiprocessing.py b/sea/server/multiprocessing.py index 754aa60..46116bb 100644 --- a/sea/server/multiprocessing.py +++ b/sea/server/multiprocessing.py @@ -120,7 +120,16 @@ def _stop_handler(self, signum, frame): if self.app.config["GRPC_GRACE"] else 5 ) + + if self._stopped: + self.app.logger.debug( + "stop signal has received, ignore duplicated function signal" + ) + return + self._stopped = True + if not self.server: + # master self.app.logger.warning( "master process received signal {}, sleep {} to wait slave done".format( signum, grace @@ -142,11 +151,12 @@ def _stop_handler(self, signum, frame): worker.kill() self.app.logger.warning("master exit") else: - # slave process sleep less 3s to make grace more reliable + # slave signals.server_stopped.send(self) self.app.logger.warning( "slave process received signal {}, try to stop process".format(signum) ) + # slave process sleep less 3s to make grace more reliable self.server.stop(grace - 3) time.sleep(grace - 3) From 6e2c0679d3742e78c318258cbfed43561f16b217 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Thu, 23 Jun 2022 14:47:24 +0800 Subject: [PATCH 08/23] test: add multiprocessing unit test --- .isort.cfg | 2 +- sea/server/multiprocessing.py | 4 +-- .../test_extensions/test_celery.py | 3 +- tests/test_server.py | 36 +++++++++++++++++-- 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 504b2f3..c85a536 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,2 +1,2 @@ [settings] -known_third_party = app,blinker,cachext,celery,configs,google,grpc,helloworld_pb2,helloworld_pb2_grpc,mock,peewee,peeweext,pendulum,pkg_resources,pytest,sentry_sdk,setuptools,worldhello_pb2,worldhello_pb2_grpc +known_third_party = app,blinker,cachext,celery,configs,google,grpc,helloworld_pb2,helloworld_pb2_grpc,peewee,peeweext,pendulum,pkg_resources,pytest,sentry_sdk,setuptools,worldhello_pb2,worldhello_pb2_grpc diff --git a/sea/server/multiprocessing.py b/sea/server/multiprocessing.py index 46116bb..ad2084c 100644 --- a/sea/server/multiprocessing.py +++ b/sea/server/multiprocessing.py @@ -91,7 +91,7 @@ def _clean_prometheus(self): def run(self): self._run_prometheus_http_server() - self.register_signal() + self._register_signals() with _reserve_address_port(self.host, self.port): bind_address = "{}:{}".format(self.host, self.port) @@ -108,7 +108,7 @@ def run(self): return True - def register_signal(self): + def _register_signals(self): signal.signal(signal.SIGINT, self._stop_handler) signal.signal(signal.SIGHUP, self._stop_handler) signal.signal(signal.SIGTERM, self._stop_handler) diff --git a/tests/test_contrib/test_extensions/test_celery.py b/tests/test_contrib/test_extensions/test_celery.py index 3c5ae3a..609b92d 100644 --- a/tests/test_contrib/test_extensions/test_celery.py +++ b/tests/test_contrib/test_extensions/test_celery.py @@ -1,6 +1,7 @@ import os import sys -import mock +from unittest import mock + from sea import import_string from sea.contrib.extensions.celery import AsyncTask, Bus from sea.contrib.extensions.celery.cmd import async_task, bus diff --git a/tests/test_server.py b/tests/test_server.py index 13ceb7a..70007e9 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -4,11 +4,12 @@ import threading from unittest import mock -from sea.server.threading import Server from sea.signals import server_started, server_stopped -def test_server(app, logstream): +def test_thread_server(app, logstream): + from sea.server.threading import Server + s = Server(app) assert not s._stopped @@ -34,3 +35,34 @@ def _mocked(*args, **kwargs): content = logstream.getvalue() assert "started!" in content and "stopped!" in content + + +def test_multiprocessing_server(app, logstream): + from sea.server.multiprocessing import Server + + s = Server(app) + assert not s._stopped + + def log_started(s): + app.logger.warning("started!") + + def log_stopped(s): + app.logger.warning("stopped!") + + def _mocked(*args, **kwargs): + curframe = inspect.currentframe() + caller_name = inspect.getouterframes(curframe, 2)[1][3] + if caller_name == "run": + os.kill(os.getpid(), signal.SIGINT) + + server_started.connect(log_started) + server_stopped.connect(log_stopped) + + with mock.patch("time.sleep", new=_mocked): + assert s.run() + process_num = os.open("ps ax | grep sea | grep -v grep | wc -l") + print(process_num) + assert s._stopped + + content = logstream.getvalue() + assert "started!" in content and "stopped!" in content From 0185a3d62f24c71bac1dd733d9d79208c45ff8b9 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Thu, 23 Jun 2022 16:40:58 +0800 Subject: [PATCH 09/23] fix(app): fix SEA_DEBUG default value --- sea/app.py | 74 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/sea/app.py b/sea/app.py index 1dbc278..7bba207 100644 --- a/sea/app.py +++ b/sea/app.py @@ -6,7 +6,7 @@ from sea import exceptions, utils from sea.config import Config, ConfigAttribute -from sea.datatypes import ImmutableDict, ConstantsObject +from sea.datatypes import ConstantsObject, ImmutableDict if sys.version_info.minor >= 8: from functools import cached_property @@ -20,26 +20,27 @@ class BaseApp: :param root_path: the root path :param env: the env """ + config_class = Config - testing = ConfigAttribute('TESTING') - tz = ConfigAttribute('TIMEZONE') - default_config = ImmutableDict({ - 'TESTING': False, - 'TIMEZONE': 'UTC', - 'GRPC_WORKERS': 3, - 'GRPC_THREADS': 1, # Only appliable in multiprocessing server - 'GRPC_HOST': '[::]', - 'GRPC_PORT': 6000, - 'GRPC_LOG_LEVEL': 'WARNING', - 'GRPC_LOG_HANDLER': logging.StreamHandler(), - 'GRPC_LOG_FORMAT': '[%(asctime)s %(levelname)s in %(module)s] %(message)s', # NOQA - 'GRPC_GRACE': 5, - 'PROMETHEUS_SCRAPE': False, - 'PROMETHEUS_PORT': 9091, - 'MIDDLEWARES': [ - 'sea.middleware.RpcErrorMiddleware' - ] - }) + testing = ConfigAttribute("TESTING") + tz = ConfigAttribute("TIMEZONE") + default_config = ImmutableDict( + { + "TESTING": False, + "TIMEZONE": "UTC", + "GRPC_WORKERS": 3, + "GRPC_THREADS": 1, # Only appliable in multiprocessing server + "GRPC_HOST": "[::]", + "GRPC_PORT": 6000, + "GRPC_LOG_LEVEL": "WARNING", + "GRPC_LOG_HANDLER": logging.StreamHandler(), + "GRPC_LOG_FORMAT": "[%(asctime)s %(levelname)s in %(module)s] %(message)s", # NOQA + "GRPC_GRACE": 5, + "PROMETHEUS_SCRAPE": False, + "PROMETHEUS_PORT": 9091, + "MIDDLEWARES": ["sea.middleware.RpcErrorMiddleware"], + } + ) def __init__(self, root_path, env): if not os.path.isabs(root_path): @@ -47,16 +48,19 @@ def __init__(self, root_path, env): self.root_path = root_path self.name = os.path.basename(root_path) self.env = env - self.debug = (os.environ.get("SEA_DEBUG") or env == "development") + self.debug = ( + os.environ.get("SEA_DEBUG") not in (None, "0", "false", "no") + or env == "development" + ) self.config = self.config_class(root_path, self.default_config) self._servicers = {} self._extensions = {} self._middlewares = [] def _setup_root_logger(self): - fmt = self.config['GRPC_LOG_FORMAT'] - lvl = self.config['GRPC_LOG_LEVEL'] - h = self.config['GRPC_LOG_HANDLER'] + fmt = self.config["GRPC_LOG_FORMAT"] + lvl = self.config["GRPC_LOG_LEVEL"] + h = self.config["GRPC_LOG_HANDLER"] h.setFormatter(logging.Formatter(fmt)) root = logging.getLogger() root.setLevel(lvl) @@ -66,12 +70,12 @@ def _setup_root_logger(self): def logger(self): self._setup_root_logger() - logger = logging.getLogger('sea.app') + logger = logging.getLogger("sea.app") if self.debug and logger.level == logging.NOTSET: logger.setLevel(logging.DEBUG) if not utils.logger_has_level_handler(logger): h = logging.StreamHandler() - h.setFormatter(logging.Formatter('%(message)s')) + h.setFormatter(logging.Formatter("%(message)s")) logger.addHandler(h) return logger @@ -100,16 +104,15 @@ def _register_servicer(self, servicer): """ name = servicer.__name__ if name in self._servicers: - raise exceptions.ConfigException( - 'servicer duplicated: {}'.format(name)) + raise exceptions.ConfigException("servicer duplicated: {}".format(name)) add_func = self._get_servicer_add_func(servicer) self._servicers[name] = (add_func, servicer) def _get_servicer_add_func(self, servicer): for b in servicer.__bases__: - if b.__name__.endswith('Servicer'): + if b.__name__.endswith("Servicer"): m = inspect.getmodule(b) - return getattr(m, 'add_{}_to_server'.format(b.__name__)) + return getattr(m, "add_{}_to_server".format(b.__name__)) def _register_extension(self, name, ext): """register extension @@ -119,13 +122,11 @@ def _register_extension(self, name, ext): """ ext.init_app(self) if name in self._extensions: - raise exceptions.ConfigException( - 'extension duplicated: {}'.format(name)) + raise exceptions.ConfigException("extension duplicated: {}".format(name)) self._extensions[name] = ext def load_middlewares(self): - mids = ['sea.middleware.GuardMiddleware'] + \ - self.config.get('MIDDLEWARES') + mids = ["sea.middleware.GuardMiddleware"] + self.config.get("MIDDLEWARES") for mn in mids: m = utils.import_string(mn) self._middlewares.insert(0, m) @@ -133,14 +134,15 @@ def load_middlewares(self): def load_extensions_in_module(self, module): def is_ext(ins): - return not inspect.isclass(ins) and hasattr(ins, 'init_app') + return not inspect.isclass(ins) and hasattr(ins, "init_app") + for n, ext in inspect.getmembers(module, is_ext): self._register_extension(n, ext) return self.extensions def load_servicers_in_module(self, module): for _, _servicer in inspect.getmembers(module, inspect.isclass): - if _servicer.__name__.endswith('Servicer'): + if _servicer.__name__.endswith("Servicer"): self._register_servicer(_servicer) return self.servicers From ba854bc641fcad88b67728f2dd5f15a349ba41e4 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Thu, 23 Jun 2022 16:41:29 +0800 Subject: [PATCH 10/23] test: fix multiprocessing server unit test --- tests/test_app.py | 9 ++++++--- tests/test_server.py | 45 ++++++++++++++++++++++++++------------------ 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/tests/test_app.py b/tests/test_app.py index 6c47ded..70f963c 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,9 +1,10 @@ -import pytest -import sys import logging import os.path +import sys from unittest import mock +import pytest + from sea import app, exceptions @@ -19,6 +20,8 @@ def test_baseapp(caplog): from configs import testing + os.environ["SEA_DEBUG"] = "true" + _app = app.BaseApp(root_path, env='testing') _app.config.from_object(testing) assert _app.config["PORT"] == 4000 os.environ["PORT"] = "4001" @@ -31,7 +34,7 @@ def test_baseapp(caplog): assert len(_app.middlewares) == 3 with mock.patch('sea._app', new=_app): - from app import servicers, extensions + from app import extensions, servicers with pytest.raises(exceptions.ConfigException): _app._register_servicer(servicers.GreeterServicer) diff --git a/tests/test_server.py b/tests/test_server.py index 70007e9..f0d7761 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -2,6 +2,7 @@ import os import signal import threading +import time from unittest import mock from sea.signals import server_started, server_stopped @@ -38,31 +39,39 @@ def _mocked(*args, **kwargs): def test_multiprocessing_server(app, logstream): - from sea.server.multiprocessing import Server + # In multiprocessing mode, prometheus dir must be set + try: + os.mkdir("/tmp/prometheus_metrics") + os.environ.setdefault("PROMETHEUS_MULTIPROC_DIR", "/tmp/prometheus_metrics") - s = Server(app) - assert not s._stopped + app.config["PROMETHEUS_PORT"] = 9092 - def log_started(s): - app.logger.warning("started!") + from sea.server.multiprocessing import Server - def log_stopped(s): - app.logger.warning("stopped!") + s = Server(app) + assert not s._stopped - def _mocked(*args, **kwargs): - curframe = inspect.currentframe() - caller_name = inspect.getouterframes(curframe, 2)[1][3] - if caller_name == "run": + def log_started(s): + app.logger.warning("started!") + + def log_stopped(s): + app.logger.warning("stopped!") + + server_started.connect(log_started) + server_stopped.connect(log_stopped) + + def kill_later(sec): + time.sleep(sec) os.kill(os.getpid(), signal.SIGINT) - server_started.connect(log_started) - server_stopped.connect(log_stopped) + # 3 seconds to wait before killing server + threading.Thread(target=kill_later, args=[3]).start() - with mock.patch("time.sleep", new=_mocked): + # with mock.patch("time.sleep", new=_mocked): assert s.run() - process_num = os.open("ps ax | grep sea | grep -v grep | wc -l") - print(process_num) assert s._stopped - content = logstream.getvalue() - assert "started!" in content and "stopped!" in content + content = logstream.getvalue() + assert "stopped!" in content + finally: + os.rmdir("/tmp/prometheus_metrics") From 6411e8901af77cb5bce1230ec4b971c7f2cb149b Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Thu, 23 Jun 2022 17:51:02 +0800 Subject: [PATCH 11/23] test: add unittest proto --- tests/wd/protos/helloworld.proto | 38 ++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/wd/protos/helloworld.proto diff --git a/tests/wd/protos/helloworld.proto b/tests/wd/protos/helloworld.proto new file mode 100644 index 0000000..be878ce --- /dev/null +++ b/tests/wd/protos/helloworld.proto @@ -0,0 +1,38 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "io.grpc.examples.helloworld"; +option java_outer_classname = "HelloWorldProto"; +option objc_class_prefix = "HLW"; + +package helloworld; + +// The greeting service definition. +service Greeter { + // Sends a greeting + rpc SayHello (HelloRequest) returns (HelloReply) {} +} + +// The request message containing the user's name. +message HelloRequest { + string name = 1; +} + +// The response message containing the greetings +message HelloReply { + string message = 1; +} From 59a6b108983fbcc967cf0f9fbd56ae8255ae8de7 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Thu, 23 Jun 2022 17:51:51 +0800 Subject: [PATCH 12/23] style: black code --- sea/cmds.py | 100 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 43 deletions(-) diff --git a/sea/cmds.py b/sea/cmds.py index a7389d2..c0f56fb 100644 --- a/sea/cmds.py +++ b/sea/cmds.py @@ -1,107 +1,121 @@ import os -from sea.cli import jobm, JobException from sea import current_app +from sea.cli import JobException, jobm from sea.server.multiprocessing import Server -@jobm.job('server', aliases=['s'], help='Run Server') +@jobm.job("server", aliases=["s"], help="Run Server") def server(): s = Server(current_app) s.run() return 0 -@jobm.job('console', aliases=['c'], help='Run Console') +@jobm.job("console", aliases=["c"], help="Run Console") def console(): banner = """ [Sea Console]: the following vars are included: `app` (the current app) """ - ctx = {'app': current_app} + ctx = {"app": current_app} try: from IPython import embed + h, kwargs = embed, dict(banner1=banner, user_ns=ctx, colors="neutral") except ImportError: import code + h, kwargs = code.interact, dict(banner=banner, local=ctx) h(**kwargs) return 0 -@jobm.job('generate', aliases=['g'], inapp=False, help='Generate RPC') -@jobm.option('-I', '--proto_path', required=True, action='append', - help="the dir in which we'll search the proto files") -@jobm.option('protos', nargs='+', - help='the proto files which will be compiled.' - 'the paths are related to the path defined in "-I"') +@jobm.job("generate", aliases=["g"], inapp=False, help="Generate RPC") +@jobm.option( + "-I", + "--proto_path", + required=True, + action="append", + help="the dir in which we'll search the proto files", +) +@jobm.option( + "protos", + nargs="+", + help="the proto files which will be compiled." + 'the paths are related to the path defined in "-I"', +) def generate(proto_path, protos): from grpc_tools import protoc - well_known_path = os.path.join(os.path.dirname(protoc.__file__), '_proto') - proto_out = os.path.join(os.getcwd(), 'protos') + well_known_path = os.path.join(os.path.dirname(protoc.__file__), "_proto") + + proto_out = os.path.join(os.getcwd(), "protos") proto_path.append(well_known_path) proto_path_args = [] for protop in proto_path: - proto_path_args += ['--proto_path', protop] + proto_path_args += ["--proto_path", protop] cmd = [ - 'grpc_tools.protoc', + "grpc_tools.protoc", *proto_path_args, - '--python_out', proto_out, - '--grpc_python_out', proto_out, - *protos + "--python_out", + proto_out, + "--grpc_python_out", + proto_out, + *protos, ] return protoc.main(cmd) -@jobm.job('test', env='testing', inapp=False, proxy=True, help='run test') +@jobm.job("test", env="testing", inapp=False, proxy=True, help="run test") def runtest(argv): import pytest + from sea import create_app class AppPlugin: - def pytest_load_initial_conftests(early_config, parser, args): create_app() return pytest.main(argv, plugins=[AppPlugin]) -@jobm.job('new', aliases=['n'], inapp=False, help='Create Sea Project') -@jobm.option('project', help='project name') -@jobm.option('--skip-git', action='store_true', - help='skip add git files and run git init') -@jobm.option('--skip-peewee', action='store_true', help='skip peewee') -@jobm.option('--skip-cache', action='store_true', help='skip cache') -@jobm.option('--skip-async-task', action='store_true', help='skip async_task') -@jobm.option('--skip-bus', action='store_true', help='skip bus') -@jobm.option('--skip-sentry', action='store_true', help='skip sentry') +@jobm.job("new", aliases=["n"], inapp=False, help="Create Sea Project") +@jobm.option("project", help="project name") +@jobm.option( + "--skip-git", action="store_true", help="skip add git files and run git init" +) +@jobm.option("--skip-peewee", action="store_true", help="skip peewee") +@jobm.option("--skip-cache", action="store_true", help="skip cache") +@jobm.option("--skip-async-task", action="store_true", help="skip async_task") +@jobm.option("--skip-bus", action="store_true", help="skip bus") +@jobm.option("--skip-sentry", action="store_true", help="skip sentry") def new(project, **extra): PACKAGE_DIR = os.path.dirname(__file__) - TMPLPATH = os.path.join(PACKAGE_DIR, 'template') + TMPLPATH = os.path.join(PACKAGE_DIR, "template") IGNORED_FILES = { - 'git': ['gitignore'], - 'peewee': ['configs/default/peewee.py.tmpl', - 'app/models.py.tmpl'], - 'cache': ['configs/default/cache.py.tmpl'], - 'sentry': [], - 'async_task': ['configs/default/async_task.py.tmpl', - 'app/tasks.py.tmpl'], - 'bus': ['configs/default/bus.py.tmpl', 'app/buses.py.tmpl'], + "git": ["gitignore"], + "peewee": ["configs/default/peewee.py.tmpl", "app/models.py.tmpl"], + "cache": ["configs/default/cache.py.tmpl"], + "sentry": [], + "async_task": ["configs/default/async_task.py.tmpl", "app/tasks.py.tmpl"], + "bus": ["configs/default/bus.py.tmpl", "app/buses.py.tmpl"], } def _build_skip_files(extra): skipped = set() for ignore_key in IGNORED_FILES.keys(): - if extra[('skip_' + ignore_key)]: + if extra[("skip_" + ignore_key)]: for f in IGNORED_FILES[ignore_key]: skipped.add(os.path.join(TMPLPATH, f)) return skipped def _gen_project(path, skip=set(), ctx={}): import shutil + from jinja2 import Environment, FileSystemLoader + env = Environment(loader=FileSystemLoader(TMPLPATH)) for dirpath, dirnames, filenames in os.walk(TMPLPATH): for fn in filenames: @@ -112,19 +126,19 @@ def _gen_project(path, skip=set(), ctx={}): # create the parentdir if not exists os.makedirs(os.path.dirname(dst), exist_ok=True) r, ext = os.path.splitext(dst) - if ext == '.tmpl': - with open(r, 'w') as f: + if ext == ".tmpl": + with open(r, "w") as f: tmpl = env.get_template(relfn) f.write(tmpl.render(**ctx)) else: shutil.copyfile(src, dst) - print('created: {}'.format(dst)) + print("created: {}".format(dst)) path = os.path.join(os.getcwd(), project) if os.path.exists(path): - raise JobException('{} already exists'.format(path)) + raise JobException("{} already exists".format(path)) ctx = extra.copy() - ctx['project'] = os.path.basename(path) + ctx["project"] = os.path.basename(path) _gen_project(path, skip=_build_skip_files(extra), ctx=ctx) return 0 From 8a748eafdeef969e194f3217f11ee67963f5667b Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Thu, 23 Jun 2022 17:52:33 +0800 Subject: [PATCH 13/23] feat(server): change server mode to default mode in cmd --- sea/cmds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sea/cmds.py b/sea/cmds.py index c0f56fb..a90020c 100644 --- a/sea/cmds.py +++ b/sea/cmds.py @@ -2,7 +2,7 @@ from sea import current_app from sea.cli import JobException, jobm -from sea.server.multiprocessing import Server +from sea.server import Server @jobm.job("server", aliases=["s"], help="Run Server") From 28888e2243a7cde57bb24bc89d1549f4cbe780fb Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Thu, 23 Jun 2022 22:01:20 +0800 Subject: [PATCH 14/23] chore: add more comments --- sea/server/multiprocessing.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/sea/server/multiprocessing.py b/sea/server/multiprocessing.py index ad2084c..1cc49e5 100644 --- a/sea/server/multiprocessing.py +++ b/sea/server/multiprocessing.py @@ -11,38 +11,27 @@ from sea import signals -""" -GRPC_WORKERS -GRPC_THREADS -GRPC_HOST -GRPC_PORT -GRPC_GRACE - -GRPC_LOG_FORMAT -GRPC_LOG_LEVEL -GRPC_LOG_HANDLER - -PROMETHEUS_SCRAPE -PROMETHEUS_PORT -""" - class Server: - """sea multiprocessing server implements + """sea multiprocessing server implementation :param app: application instance """ def __init__(self, app): + # application instance self.app = app + # worker process number self.worker_num = self.app.config["GRPC_WORKERS"] + # worker thread number self.thread_num = self.app.config.get("GRPC_THREADS") self.host = self.app.config["GRPC_HOST"] self.port = self.app.config["GRPC_PORT"] + # slave worker refs, master node contains all slave workers refs self.workers = [] self._stopped = False - - self.server = None # slave process server object + # slave worker server instance ref + self.server = None def _run_server(self, bind_address): server = grpc.server( From 431295e8302e71d0e1860b5a799a56e1ac446814 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Thu, 23 Jun 2022 23:11:13 +0800 Subject: [PATCH 15/23] fix(server): respect GRPC_HOST config in multiprocessing --- sea/app.py | 2 +- sea/server/multiprocessing.py | 24 +++++++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/sea/app.py b/sea/app.py index 7bba207..0dba5b0 100644 --- a/sea/app.py +++ b/sea/app.py @@ -30,7 +30,7 @@ class BaseApp: "TIMEZONE": "UTC", "GRPC_WORKERS": 3, "GRPC_THREADS": 1, # Only appliable in multiprocessing server - "GRPC_HOST": "[::]", + "GRPC_HOST": "0.0.0.0", "GRPC_PORT": 6000, "GRPC_LOG_LEVEL": "WARNING", "GRPC_LOG_HANDLER": logging.StreamHandler(), diff --git a/sea/server/multiprocessing.py b/sea/server/multiprocessing.py index 1cc49e5..4f92e40 100644 --- a/sea/server/multiprocessing.py +++ b/sea/server/multiprocessing.py @@ -82,8 +82,7 @@ def run(self): self._register_signals() - with _reserve_address_port(self.host, self.port): - bind_address = "{}:{}".format(self.host, self.port) + with _reserve_address_port(self.host, self.port) as bind_address: for _ in range(self.worker_num): worker = multiprocessing.Process( target=self._run_server, args=(bind_address,) @@ -153,12 +152,27 @@ def _stop_handler(self, signum, frame): @contextlib.contextmanager def _reserve_address_port(host, port): """Find and reserve a port for all subprocesses to use.""" - sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + + from ipaddress import IPv6Address, ip_address + + ipv6 = False + if host and type(ip_address(host)) is IPv6Address: + ipv6 = True + + sock = socket.socket( + socket.AF_INET6 if ipv6 else socket.AF_INET, socket.SOCK_STREAM + ) + + # ENABLE SO_REUSEPORt sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 0: raise RuntimeError("Failed to set SO_REUSEPORT.") - sock.bind(("", port)) + + sock.bind((host, port)) try: - yield sock.getsockname()[1] + if ipv6: + yield "[{0}]:{1}".format(*sock.getsockname()) + else: + yield "{0}:{1}".format(*sock.getsockname()) finally: sock.close() From 2e92ffbb6f26257f48152d54540fabe4be309879 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Fri, 24 Jun 2022 00:35:21 +0800 Subject: [PATCH 16/23] chore: change GRPC_WORKERS default to 4 --- sea/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sea/app.py b/sea/app.py index 0dba5b0..2c9b67c 100644 --- a/sea/app.py +++ b/sea/app.py @@ -28,7 +28,7 @@ class BaseApp: { "TESTING": False, "TIMEZONE": "UTC", - "GRPC_WORKERS": 3, + "GRPC_WORKERS": 4, "GRPC_THREADS": 1, # Only appliable in multiprocessing server "GRPC_HOST": "0.0.0.0", "GRPC_PORT": 6000, From 636db771134f3bf318b2e9621339e9fec5dd7ee6 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Fri, 24 Jun 2022 00:35:44 +0800 Subject: [PATCH 17/23] docs(benchmark): add server benchmark --- docs/_sidebar.md | 2 ++ docs/assets/images/2022-06-23-latency.jpg | Bin 0 -> 81697 bytes docs/assets/images/2022-06-23-qps.jpg | Bin 0 -> 77233 bytes docs/benchmark.md | 18 ++++++++++++++++++ 4 files changed, 20 insertions(+) create mode 100644 docs/assets/images/2022-06-23-latency.jpg create mode 100644 docs/assets/images/2022-06-23-qps.jpg create mode 100644 docs/benchmark.md diff --git a/docs/_sidebar.md b/docs/_sidebar.md index 957574b..e504845 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -18,3 +18,5 @@ - 内置扩展 - [celery](contrib/celery) - [Sentry](contrib/sentry) +- 性能测试 + - [benchmark](benchmark) diff --git a/docs/assets/images/2022-06-23-latency.jpg b/docs/assets/images/2022-06-23-latency.jpg new file mode 100644 index 0000000000000000000000000000000000000000..799fd8de907cd4c1e19a98b89217dcba2c27697a GIT binary patch literal 81697 zcmeEv2Ut_t+VufM6bquDSfligC@7*3dX*9&p(;u*0sI zy3$2D(mM%+|9~@hocZULx$3<$_lxu783oQEXJ_xXuC?CH51Su>19DQbQUD1F2@nMS z2W)l&;=qADd-w09Jg|S?{)5y94$-k5qdR<^#nZ`3XZv8)POmF8K-^LjR}yD1MJp*wnviJ66! zjh&DG!bO2gg5naAQqnTAa;j=@gt~^NmcD_Zk+F%Xne83ByY}}S93MV%d;G-xsfV9` zKwwbt^N`T!nAo`Zgv6xe?3~=Zd~88sQRSegfRBbQfF%4X4BsCTkuGf@PZd-jAY5JC}Vh(jws zNkvY9S$~)0>kD3(Rp}|{Xl(yV(r-cHZg?w zWe*TT2x16949Ne#l!xdjiH`CYBS}nef2ju$6MtwWu?q2Z@#jAlLL`}W=^RGXGb)_X z8*tX+ZNfXLOKS?#ZvyBYn?QQZ+OcFi#vy+LMQWJE>Dig}d(aI#=o>_y!Rjw1ulR)O z`h?k-b21q$L7$|HoMd7I+*x%FP;homP1VSyFvoF*-VL8uS~e=syVkbf<&?|Aw!wwC z!sk7Ga>388A2;bXlRUwRxQaS-;L_1C_MZ2`x}6EyEGB0~H<}md7b}-u8);JZ#g&HG z+0%Z$e(O4!%LU9;K**F4Z}h^#C}?W*piEYSS0qWXS2_h9|K1M22^Qrb*6WO+v=Rro zTN%j02eCIBW%=eObPXO@7<&h)`RUWs!uQl3op~1rSg2|SIxmnc@lBYsWBZhyqg@U) zpz)vAnn$EJme$AS2MM1DY7W+$fHjI8xbkKbC@qUPWq|6}eT(LXPV)>)2SZkl%tw+D zFCs1kaYqo32GK?kjU~}*5Zxy+Bp}8{#2}FvEfdp*@033YhYZ(jyk1XQ1Q1H@WrG#a z&xr)7ycCuy-MNpLr8X3Umg;IVt?=PfC&AQ}0!&>K)+jqtG5a@xnunXfd)0L}bC~Mj zO{D=*Ng9Nq8x2f^ocJQH!Eg6~!Kx?W${|7#ddOTW`@ZBRaBBA^a7}uiKMv|mtvYaX zpnr81g_}SR{7q7*{LXe? zK|}hH?!tAB3&?9dllkbQV=}etiDY^nu8r~Lh8M{W+7HqS0bptKn_jg3pLkYhm2eYK zU$AmIcVzt@%Z44~O-BHMPg2S4$rIFI#U}7Mi691+^k0?DecS|=+BShRT5W2pA5>Ri zA&lz&_0|l%Lb0%L2^#KQWI?xD&y9jg`KBWsrj~uyjGian)B(o}>ONXORPNDMNX}cVKUAG&dB$C)SpJ#4 zF-0U1TSmmo5ix*7FeDKbN`%J}al%BvG!Z#YM8gwl0e=mCgs4n;tr}o|TAAd_sa;?o z;0ShNgRoprdOl6<57uk#T<`5isjF$Bopg{0!0cV3cKM~-jT`MFIfiZ1md6eDWI`0n zA~dkg&9{v$M)4`-X**mQbvR`xG9&l8^EHkl8z#=T?W|q@thg&Jt(yKr>Dmy1pJ$sa z5mS%Gd#?TJSKs*wJ$@!Z65nq#EREN})VZ(}bWq!zRv5)MmM2&4`MeOa zTWu3Cf3Sv#C~+$(rHmgTEZCT}1IpNz$S35RfMfsEo%Q5dU@wUan@J?82kZBH5x4JO zb=!U8`;KK~WpW4X`NakiuAJ`R9_#Cc3Ia{Z!N z=HmBGz1#%u+-4;$o;YG}k=;v)UUq;+2=6=7GgIiFq5Kl_HqVdG`);(Gt_WKj3C-cz zZZC-kNIa%G?gv>oj<2Agxh{yZ={^WA^*E*w$IWc%dOP z+JhA!zW&T>Aax@$=rl=ND4-=5Wt}2^SVH7YUk_zm=_UXZ&~bm4um_|iMDFA8mg?K< z+TD#$X*3qv7#^g-ip6ZaD0`Z60e40A;pL-LkslEr^wm)?vBb%ptYqx_PClOo4eo4) z?%UibtOt3O%#r@Wr0*OW{O@?K3PO$L9uI(^ISQ!_e3c0A|}XW z;983SY}%aN0%00vm}+R-G{|X6-!jd89NFMzJ7LpC!d0C~QEwTBUI|-gFY4;+KhVkx z2P0ba8$?=3z~S|1!?KqS{CX~I+T%g9A2Sz@w--6XZ4>0Q{_{sS6+?}xaYem{t$Uk) zg%8V3n2Tn=&w1rCNcNxt$sS7cv|uv#HH|1bp}G=N>FU9nTK_}6!jJSg|Eu?AORe+J zbXOMiGjprX*y}(|#^AU}LlOn&=I6L~?xK~IaJ*i@eAE%iNkqoG=A zag3ETrLXULmJChuCXltEAa51pXRlKR?bV%2Qat`{LfExtpU`@=KVy7q7Ad-sDjwlNs~;%=w;Q60QtBAJ$E zjGMjOfOLsbk2%?Ncf8V;O%=#39JN?ENBb_R9uZg)es~j@XtE^C4K?l5>bl;4*7O6o zeX$$ly4Y5(j};sv7gJDf*P>kx2K@=*L!!a`fhoj)#GuFLu(KK8h_q*+0Vd?$7PNmt z(;qOfy6&2+2S%|=)R+nsZjmI)I!U|Y+3^M|kS=*F@_jHiRfU|rY!ao4m|Aq#uXrAE zyfrdqBH4phjrAlzLlSHivkB0?8rlTxP}RB7j|Gn0lA&=@3-}Y9%&#ouugbX~k@#<+ zgQ?P>gxadcSCa49_rDGuB$d3~pl%@cSTpEbEI#qtf7jc>r+IE*Hvx@7`xR+s7fv-> zC7L}&>XbsCnXjpQy4z`SR0p>=@4&d~)$Zw7ju$bP^hKx&I?mngX5KcpOxZ6>ZY*Sh zA*>aSvK^ZXM#ED(QD6uw1gK(LEk0IsZ<|={xIgL9CRP^yQdN=A>8vE<8_AdbJ>G%V z^{Ttt8>AAc3!hm;b%ND?F${0Xk%UoAyw%bvMG52gZXiUmy&lO-H*3he9l*w1>+9#$ zPy-`?oSG3cbL0u+=O?vF%sGZ-K?C{PB1eQh z{Y8$=_lh1I9)_(Bb=XyBD%D$JW|_eeYX-+8vQNI}!3sTL&W*~!y45I-`Ef zCg{ymH7|RknXs*Gm5eV6y#n7^0d+S>{>duErDGKxDDz(x*ryu_ZDxEJJb>V7i{P8l!UY+0}) zP-dO1{pwi)STi4AYu{N|TE?WOl0U0+U-sicbxw`&$9_!Pt#sS?%Y=uhh9;Au+}~= z1wF*_LSd-Ev497%~}IqmG2T{Q-vR}lRn00Ckf?Vs4%A6Aq9o!<-2x}nKA3u0x=g-6vr zuB0+FfUR5?KH2ixWOc#V14CyPjyF3x+lgB&N88vi2*2Di<>}p~XC)u9%xL!~42#fs zzg}A7pgVJU>Z3DrYWLV!f%%gjRMAVyR)7_pXgJqC{66`G)uob=KkP2C+8_hh--EZhL(nqfFyIL(>UhurbUbfhvzm#;Q z$;cPmtKjPRU;*+rciHs*RkTTerRR}j>$`nP`JS}RvSYfI6vW-(fmACyo7Xz>N`pn_eBu9Q+@4~Qt{xAb_b7JxOB z3Rf2ek;9R{kIjwtyBC9>XB7?^Ns&#G68>}^c`ub+k4<$oR>!nXn||rc87W!7sl&Ky zI^h#j`6gh1(jxb+N;p>v!j}sPMWgGi7grgS^sSzLXx%~m%tk25jN_gJ^{xt*lM`A~ zo|bI2n$L@MML>V}=(uLnJ zNI?CL4~d4I<%Ii~mo!mb)?eptdV&T3hTVpJ`9 zIGd@Kt5MLvujb&iZ~KvU9}bc{ZG1JG`Ke|?dF}eRezWEDab#Pu?SwtQ>15j6W$G!O z*_2>Q{%(_O`h6Tux1oZOT|qI9a@}WOgmOY&f>lgsg{x|VmHO!7xO%&o3k^VVQ<{$< zyTInHb`B}m{!d2zb{AS47YbA4`ekz-Qq+Tx)3psj_yoJ5%GEeURuO=Xj5O$X*&lKq z^3V10EqihM%?t%9t}DKhEqR)le9+7c+8!l5>hWaiNHAogf^5SV-BHNo6B`lZ0A+wn zX_xG~U?>`NBc?4hgK2$tvnOO(5^+t~bid&4Zv*6F0I^Detw~>gnj0HT8;@ ziv#;re0}J>s4|$thWmPupUR!Wsp@iD_`Gh?@HD9=;$lbFwY+gAr#d(Wo?=cthYrVL zTs?A5#!@eO9hz`R8cRzWU~OI1lSqE6oh5h&q}Ys~|1ME5F#`VYM8I-@7JJ0oIIEo^ zR$$A2-CQVr#_D-2B^-GK7Yei%l(=?3xsP2@_t(sx0HfaF?lU?#o2`C7Rp`J9WIAKD zBLF5%Ips}?zA184+4P`{eXop^ql=Bwpq8?>xFzpF@|9`SvC!`rTdp&;>Vrl`+IQv8fA*$8-o-@{wABi ztsw$DBO6;aX*^{$n-Y5QSbOXx%4)=-Xw9hRyVUEw7wR?gH-Vyy$I7gtwO&6wZK}PI zjtMl0W;D;|hr>FhSVtb+Ro|~!Z?BJ0IJG>E?Ul!F3>I{FmwKA+-@_JIqBFOK@TTQX zecmn*l?(#UXjo6WycYD}I1l>AuW{#!5hf%iiHyar!r7$ABcUnG@zCLLl;^RLB+nS7 zPrk9HirNfP&*vHSxhtIGbMDu7ql@UMZa!FR^;huq0grbAa~7@_OpGbRoUOz*0@PpA zSA{;}Sg{%X0KePD{Ol8JIc{%g&dEo++vOq4-WqT2Kj{6*(33zDT~8(r3S^^e60gD7JE^5c`>Su^74{))zq%J zaIY|{+u!Q!e=krV1_1x*0O0){Z}7ChdO`uaGiO8@T_|idg9R(O5#)WSm-lEjM3IRH zS`H==-jmJJklY>JqHLCJGdAOj@Gya+)ELFB>`#3vx8O{LHHb|E>B3_lZz%7y%lDvcUu#cOIc6|T z?p}97M=!txpFPgLd(1QFO^&2dX}l7(^lIMu4Z%Tt9dyQ52MtjuR#jB0J(hf~PwN;D zr}f4P92c6#9CL7^{4MGXjQP3JIoVY%O`p0Qgnc#bg>j9ZN!hULj#*s+_V(<@KR;EZ zzlz+c05K>EA3=a&y-CM&U@rhH$Hh!2qF#rgT&XOq+%#fNh0)71WvV}ip=HjZF>`i;HpLrq?q*-Nr zw-|eA*!41kD2=eL^ z-XyV*nxq!Xu+`*5iY-rjwpO#%XsohIg+Ij_H zlf1OhrYU}%Pf>R^C~DO|$U2C3 zPg$hFv8Lw+z8WkWH-T1+3XpTip^gseABNX#Mgtm_`J(sT?1tYxEeH({ut_P6~ zwFFv_V$m+s!r$dVD}4hLS~zi5l62}RDcW+0>09%8TJqP!C0N`zTq)iL#lGd!^M@7@ zTs7_o41Os&zfDP8?90L^`J!-3(XlIzM~hMnld#rE3k%Ez!!2vJhA^ygr;sTP~v>_kbBOA=RK>;Wz9ZmOQG7SX#=)n-Yf-W7hiZjfamL%K~?)_>#c zRZNZx&0vnA+~_G>rVW&98(+7{f<21M=4wHhn;nv|rnvz+;)|0j!H~~%4+zaQ<2o59 zkDAJHOO?LPc>1A?seyN6d?%8EUrRwj8kft`GbiDwN44aml=&(3Z}B%|%I>lxauR9> zIrpTCha1}NN`Ys9q+PnR7JDg31bQO0B`)fQK9^bT8=)Ls$}ANjVH$j=?X+2)l&=@P z8;;VBwq4O%5Vhl$AzPg`SSaLB5&K7E>mU6+BANR;=EPqUyW?d!t-pu?lSceTQDdPTRccu}*k}4s?BYQxcKobx-hR ztm_j_D5{~Kx?7eiEq2$Nn`oI{=COB=FXw;Ytt@^rcS=Dk+M_Xlg-@2t)q^$mY_x~> zun)9!qD)mbkM=HKMivvToPrb+{y{iFEb`*}gW(d-!Bl(bg`n3R#hEwFu&{YPoL!>9 z&zXJPvV788!&_9#=&c-EAA$@i4W~p~xhJg)Xun_g7)pRV-FU^=Rxumf}|GeZCYWmuV7oqBXV!z zVs}{30=G^&s-!lJ!-z5 z#NC(cjso-Z8z`-I760}zqlwoRNoAaKBE@mO%BWmxP3*CymktMFadZP-Cqr7BochQWa9&se@WMD)8;%D>>$tG~$ z!_oB70w>0`Bao(pG|WD+2NQNEjVC8c>oV)1)r%0sNGbr715{>O9O*Oz`{+{*hWaMK z;63YyO`ZAHg!G$WV+Q8Bw^R?MzR4%tzmc5O<9xGiB(BY~5ov71c*jR!hPk>+M)~ad zC4TOt9l?j^V&w}Ty{R(TPey7Q>UC%%zGJ?Nm!n*R&&h=|_41(}y&TRMTWjWk_f4-E zvbhRExh6HDSgcgLqmo$iEzWAaJF##+MCB6pjfQ~b?ur?CCb#NnE2aYUn{$TI@+k+{D(n@J>EUT|Z)E&kdKB8l}oW7cDoOfeQuN{1X{1emap6b@iJAstmWPm13$g( zUxQ{&K0?j6*x}excq_~yB9CzXp_s48UMXhxz*5HH%^K*WVuPb4lxGx2C7wEn~;6?UN{;_H45!1^O?Xdly0 z54P3u09gflnYNrv_{rMT6z?1U`JI=xV6YNFN9?^$4b-8RiwLAHoIRhZs71d~+g&L6V}==7NLQv1oc) zQzfty5VZ|eze*w`vW6bjx`0y%r=#bZnpg*wE0emy?d@qNl;NJVsOF%W(AWo+bWJ#*zYj zvCA@8(!0dhNjIVi;B3sP(%vS_CSgYwc{-(_G@b{|4kYx}V{XE`l|#kdW5{M}Q+Pj! zM5twfn3s09pNip5oNy%{GvNn~ikCS;Fvc&zF((>TRLee4xlkcq3S14#2W$1QWS^aW z_JKp|RiEVzqxRlys#(!L3U=3XbJcZj(s4*?)aI{%VAn*^Z$>SDYsd88xSq&|`R5#j z{~ziAx3rp?nm@X75rn{s98TWjS+K1HztLQqn#40b-ECMz3z=M6pU_60d_Gg?thA+$ z?Q--&3SVP($GO5W>mWYyxObNwf>mhWm1^bPbYHF7!&NigWQ?!z2fNCD zaKx#ylD(BB6v>{n_zoJ#7xv-!R>PdX#+_cro!(eDtvtEZIVBO5vNsV`bOHS~S1qo% zskYsxUiGe1B3HFhpy~&Z_0?GLen7{a!IUcHHai_3g?`bK<}w$+rorp*VFmk{(w0^8)stv0!4#%REl8}H&#{@N=R zbf!*aTnaH#R#EXXWQ-rx4qmT-%uNe>HmV*DRt>NF0GWb}tJWk)l-&=8K1cGQryrDghs_3LL?*7At8CB7NdBx8mrBTqTjwzZ5_1rt;vUY7rRtZ|#A7k0I z@r|`9YHBLi#E zYuApmpX(hnspYdqu7Hjnp&-ZqiJuc}_4x?#y)g8j)yg{=PQ56ilwvC%c0}DEx^(%n ziVrTJwAJ3a$_xapgDf?h@dX)s1?Da6%ecH)d6GFoQTLgXQ7Nrn>fHk@$~v1su32Sr zVx`MgGaPJ|MJatmoC^rYf_ziNBe!quYW>-+|E4Y*==uH+h%>;HtZ9oz)b~vFbQs`@ z3qkekpJt9DWL21Y8Qf0Ac34zr#tdT>YVeJ%^NIScmX=`+g*K6(9E$%Y&;ho^>3(X9 z^N~Ey55_heGh#1Is~KY7$PcsHxm%aFM0=fmiUtVym%%ZCh(Q$)&h8MfMY2&~UOr@M zsU4k}Fh_PMzY7wDnx&-*OfiGpFZvb26&axr<}KFIPcIv6zs1eTS$&xI5(=W&ToSZb zo%t&Y0l=YrL7KpXuEYd&?F?4Rq9(c;Uj#(J!%kT72q8tO6ZuS86FyFKkfS z`7d0y7^|I9Q&Su5smm`*O7)(x!54KWOvqF~oQ%~HdI5FE#{Aqh#ZSX8iw!t1qW&t~ zV}mm~!b&UZ>*k8AF>ZIOA4r1(G@tW@Fj}mrP``*g;lJ7uez@u9pqT&bJ?(*7k zwP}8BfpdGKx|=V>`#bH<#d5bG_I531pUiJ^Q4;IH)WzVLlTZx}AU183Y(7oMU-wWJ zCQoTVoqwADvi$wW0q0S6m-xekODFGki+e3{)vmiKSVf9GKmVo$B3hot$`y&Fd(5PC z`_lS&U-2pJA%Fd*b!;lXm`ymw7Zd?e+n0fXqC24+Iu)?d2of79Y2V9 zn~-Qw^q8mQ^t;HBcoY|=SRlM(c%Y&tdMB%TTbjiNuhEOs1+asXRbx)@zs?Vzgt0*Cc9hZk^MZ;QklFv0i6j!31a{lM}&tR}Vf=+I`_A z{X=Y{OaslqD&TTxjCHE!t7nX$gvF%|M^%{B%}Y#$&Rc#F*gulV^q$+uA}G_1CE>XD zS<|k-@!s3R>Kpk4*SuxK79%ZN_MC3eVX!vpi#I2&qd2qy>n1AaQjSZxvS^SNX zwpBjZrlhf!2u!9QPU7nIBGFIkKk;@l8C7bHqb#(FEPsCfvK*3vG;cipi`PyHG`4(W z7J!rzx7tf~pekvRmP^fF*-dy_2M;$YEqMNkrl!f@J^58Ty5j3I^{VlF1~`pYMiRf< zLK828o20jiYH;$1hS78R(7sD4)IT|G4yeUb5W}izj~Y^E5?EIxJmN{Zd_s~Aa$Vtk z?aKoEkju8`Zjou>eI!yQsuVNfpR{g@9m*A*Qghy+jJ_(`-~&!M26ON)F1$ou@{Z~& zm4w-8s_&tLsI;^CJCCEdlD+NhCh&mdP9;0Ene!P69|$eM-7eUuzW zmqIi?v8o{zO~aPteIB$q?aL*}ZAd=X04y=5)Fx`xblgzFU+HihKXULyBRje(D}{Qj ziKpKlq(A{4$QziTuJiuAWtGwL$Lp)9ZiCZtuc1h^GPOi4ZE0>QWvHq)k;_{M_x11K z1}Ea$BF6rS!)HtXd~EtIWlW&}Eyg-cOZ=Iaq1CeVR{uPx(nj7^4kU%eMYulb2}h89 zz8K7v-N98h?OpAACk&sz&yKm7&}~3~Cw$FFZw$XgIRY}$H_U|<%%%tOwO-|7L<^Ib zN}re?-WwKYWeb+J9$8Btobslni1vbQ6|~(FKubX0%D<=b*a$epFaO=RdK$=1oi*fQ zTlt8HkYsB6iC_|ORhoSz!WR~ukyj%LDmHzHQ9I?qtAe-_iZKBZsxggI?-VMO)_165 zIFT2ob_}ii1d1Wvi%ZGIc+M1nm{>4GkP`5gcX~m7PU!4USvn{U@#`!d9U?t-#Qua- zEY-5~QUY@X=aW0xB)GrvQ|jlbZoRKeh5JfiT{{BH* zaMq@0zaQlNk;ctm|7X7%|MGuhava-kz+jerH;+XOzLlrx#3sNFWJwy;rk0mkp9E9I zVzZTlC&Uk;%d;Z)C2v!&lF_yv)t$*YcHA*NQwj0j94G3}p(7iCYyPTvXo8ldu*Z40 zK}Rq_vr?(}gj)0X;rf;OvbMoBc;eIz&uK9i++ZkBi?`dT!Ns0NWn$WzmyhGoOX*;R z$zl_TOe@j?YSEIqYo@%yb-aj}RxbTbAjk=wa-zMCmE`F$sk83MLXX)Ne+QcXtNKta zlMhobM=}*cAoa7iO$Vp@iX8Is&nh+2wkjcvPk&LVxh6K}(Jh^e$}EINa60l2*IN|@ zTj6W0(}V&`S@eYB2(&g!)mYQRVuu6EI!`#;i7q*P&hIoo+`n?fdi}Fhzhv$c@_@R& z^n2^cIQr-*Ny?Z^x|mSvPMu4}T)jR>KBw;Zemm~|6`w(^!#Ji`4enzIF!lzOD;^e| zuFr%`o27<&v4L6=-ElY7kI>y$$+{hd46K(uCRMC2315VHsE8b8B$?6lMq2Y$v7sYt zBQiDqkdL zV6`4ebFz{SmR~UN%=3dYeiV3VviU7V3R{Mt1Wb-s6S&Va%glq}jB(VFOpVS=k z4$rV|mg>lH%J4ao2A;SWrZ$l}x)DR5AwejZEP7Bc>BBY>RKT7wA{DF!+TDhp*QTzZ z(VZBS#8LXW@a_z8xYW=>LRc_pn;XkQFZt!5ig>jGl*u&~KWpG5c7t^J(9Gn-v&l4$ zB#Qqep7^;~#LkxBLEezfPGzh4{udt1mV)EKp_pR+w{w$^{?C-M6-M zQzwL=Crsx|>2n8hyvtV^b|OI zzl;|o)3nB22UyF&Mm?-?*W|+>o8ItsN2a!^(g};$rP`kM7^@h~SFXx$`qFK2@DT1O z^o-2z#S`tC{KCzkaD@$`ruDI3aF6+3{$WvICUefNGWeO?D~~SK^4N zig@|Vwh~H-Z)j_cvn|pDHWV~{s&`BjgF%FuN6U--Ui^9eFJkLb`(dPRbjrXYNKw3I z^na>s@b6?k|5Mq9>M02?!(KI~aJVV|Wl*hjIDT*}hL6O?qt;(bcg=XW@T4^iSxOk( zkh_7(3?bYh$%Dz-?a*~rqiDU-q4dFS&z9?l_M^twO%`FG5CbzyrzD`joN=qn-f3ee zGy;J%k`nNsg{X!Wp9(RDtmkkdHAYfnt*f`=09G=vYsM`|0h_$glJ0NI*3goO^Q+Du zi`@HR(d5WXaoHGq+uaMz8z0msq)-tW(JlwJVKaZE9r!D0*KTiTT2R{c8AyZuB1Q>H zyXr-UjvTiRiBrS9FsJz4uKoEo9+rhPp-|mJ3|$vzqhD8-ks642Z&7dKt?c9%+-U=q zR(4X1m}~XR=R73W+uxVbsAVkke#8(qZ7tjOh_(g)%Fm{Eqxyv4-WJf%xUeKbc(=k1 zQ1fnw@mG{9*gn(yi6sG4m63k9 zU7LWX^`cfHXS&g)z&XNPEm&h{$^aV!Wu%$QA3`xZIE!hFq@&P-@ty3|JC>z>y-y_^gs1>5|QBhp-x0x2-iQdqp;Prt}Bj?_0UMnp$noon(|E@`jimUFs20tvNM>NKBK>#X>0d|=Um1<4j~OmC%T6?Ph8yb zaV5^L5-C2F{f-WMUX0#*52@5S8|xY^b@|VO=6jb*L{pEuc-u{r_vF#SBG3PO$5~jZv zbiNMq+8Z$a^2wsXk9`wp>X7cr<3KN&fw7=E*#cjTLP_|DDKBBc$b~OVIZrW%s{9Z< zw&S#JfzRkX&4)#e8eFYrSub4*Y`i0D(VcfImIK}*_3Q;am~`9PjI`Hq#h@Niskt3o z`KB7{A{osDP@!o!$g=9%$t%lgqT!@f5gIrT2t}ZSOgLE*g>} z+uO?I%bt^$1SG$H91s;s{mZ+bX==+=-l4m_FJt~gH;`wXeND(#Q(AdWnZlfA2+@26 zSr+AM)9&0|%hhT8^~uRGm?H;6WOW+vrSjx=pLRe0#BqnBCkPZN%#*W<+Mjwv)#~dX z&n$hP(u1tBqrhi5Zuj?~KNcYw?RuzCf{SD`VTyP3;wmDyZ^e0E_^S$(Wpg_wZ$ zF>V5GX3}w;(k;D8_?}HbFf)mUVOY(}31G8Ty+#%kGhsUlYiGv0fS;owv^r$S?EEH> z77>8T9fdqgcT11Fo}Ka07Iq;D7}G3u%?`Q{AQ-Fis!qG7=ApD$XF(y|qKLz~Oxk59 z1>t`31Lsi=519QFt}gmOUwcav!fZD_CJ;zD{2am!ruKla7wJ9qDNQfm-{#Q-*7u1r z$3Ap#!WMoCITg~U2_;W~b%{cHx1?56XFOA~VY*Iyd8}(8Q}(tOdoN3G3BJ&4Q(TUF zPF3!&wV2Df*I)*rD7M^qpZzu6Kn^;i0oAoLb@u?wMdXx=TAR8Kn^5Z7QU6#6Ue129;LB>ij1P5HST_IND+oE|?P399uc z*QRu{^!9B6qUcePV>F2KpU?4DDJSB>Tdy=K4o*K&M(*tIaJkUwx>W}Qg+ZgtA({c0 zWd{O0yhx!T;k+D0TM)mI&@oWCO#_TT6v+X`zg5T5?c@u6&UjI+sAZ+GPJ+eu%#5~$ z^@(y|FUbQDx))Lj)+?QD5JdU9o9ro%nh+zhPAPLe47*|+r~`N?Wc}4*7VBKKqNq>CNzit5-8awCyk#!w4Q|Luxu)c zTb7y(J^)54CaoJIq$#o~ozh^ufQ3lz=(z8E3Oe8Nk;+gjJCrusW{P1$n=~*f?Zfq$ z&b|Is2dI{;W%_j}({bz5)-=1Q3c0j6Oygf|0-OVzz=M_>*ZVFt*T; z=ZiWSiucrzEvfTebwK0lU4#N{5&BIQ>NC0MB>|c;lM?^w6sl<0P`~U-h|9_?gw^?g z(p|Odo*FV~U7XBe`Ul=U!_0D{B4n8I9^KU9U)Dj%CGvG@3=e@3sAoLOoJ`FauksvS z(v;GSVe*}rw!pkS*3F(sa*3``)ZSap3AF6v%Y(^5xKLk<$m_m5pU%q%ALSZ)HY0oI zR;(SH#sGtETZ_Z{7G(PkpI^(g{e`%@9&Bx^foHy@ow(J&dtIGm{k{P#xZQ8{mkxPr7$V)u*hD_g&+&5! z50pkU1)1HlEBYQmLZ^D2*X(iOlsPBkUh3M=P`p3NM-sddg|a5ZorQk5+Jf{{2J;ZD zA+n88f@}V`2^q5x4XR!i-7Kkzmd{iA$c@x!?*hAbG}c~H9%Z5)$iMqJ1OT~e=dN|X zUK#a9dV={+jzI(yH>y)rErw-lxAli%4UU5y2apGV3>_Nk*KmWd>ek+Sx!>RQbWOrxh=c6%*mb``q~dw z4+^>k@`o3lkS7$)NoPVf90Tuca6?NN-Z+;HXv)Q(g3#2-<>tGOUl9chXX<;f_$sQo zi((PN;4I>A{ehH%jv<{kMw!r&%`U3u7$SBjnLll;|mmP2kB+f;cW=L4J-ydSH6K zhh#EEmD@J~w6rQUO$CrT%m(K6`pbuNjG!T+`V>GQjT_{(T{Z1)56E29FQ!Y1d$pn` zB|M?M)`saiAjL}~L3M*%=*}3$wZ$TK+WHpXuj8`Qc156w(jZbUVs-Y0OED?;bN}}v42Y+joQ7jyePo@Be zUZf4~su@6>*(Ud;60OqTJq`bYoBtQC{WDoCX~t%7BTMQgy+-)M^AvqR%1Aal+hUYZ zeVx&1-nYDs!a~gL!n4Mp!Zlb!0Mz}pHh~w)1Wq%aJhO|CiDnmQuMSm1$IH85jU=5> z_cps0P(^4g!Gy$u1cJ~PLWg+?*$Ar@DGn76An8#{^9ZcKc&8==hfMp=+0-m12IoAf;d>m% zcAwW+oM|{$*ZE#e{s0(1<{3mVf=_CvV8D&BEOT}^-gyYp(3WClfSG?i1db*KXE0_4 zou}A0iV|yQpYOWB`L*d2v8yjsJmW^ubeNyhi2@fUP+EKu(vbkMyy><#rU}TyEC*7p z*;CM$rm0WK(g%m6>D)P(h}#jLAL7Kae!mB_S?^`fYeDpi{AbEG7CRHTpQAirtCYee z?xsoOb*doou6LD&&PezZYY69dj zvoY>IrXS)50zg89@#xo2)8yEYVpN0vC|>;CeA%P*ahrgn?i;m~_)5WGtLk)^mNfa6 zMbWZKBJJ$Xka>VxP~kMe*wG-#g+0c+S)~VL4}4MP%NG05YP!Vx|9z(4pELE_#s@n! zkv79)OLNgPJyJpsjZ$ROHCcL%Q6Lw=EP!!YqVTn4BRj`Ir2`r3Iarq7FxqQ*^|b;% ze7ezoW+eeZ8Dhv($CD?whF?6;DZVzLY8P13x(9aLbof%x)lJ}~Y^ovRkubaG0-KX8 zc-9q|mQqp9pUxsL8nm187c-7cZUS2aeCRcdQ+S0dQ)}Y-W7Tg^+QFP(C0 zG^m<)B_eE$m%z`iCEk3KX9+CXvX3El1y5H$oK099R~hicyhil7$0pX8(D`=BuOF@} zn+2e%s-BmzbVlWiTw1wmR}Kcxv9K&~!LgNo=hd%TeF~t8Y?LdyB|dW*(~{l{Ke2Du zhOb~NQm!6+1M}~oDrWGR z$ow??Ud&J0k&^re9`)xq_IHlXCuf@nnXbr};Zq3wS~JpqPPL2>ySI)W3{5Wh*Qj%5 z$UOdKgZ_jyPq5qY%y zx)F+U+5|YBA38Y6go61nx+WwBb~+pBeEAEHnQnaKPB`kZj@&Mh@8%9a1Lxy zef?Eu%V+zAOYVyy$5#!YlCL$X;+^=v3I%!#Ooq8b;K7EangI zKkj2x^;m}@ ziLkQ20K)UH1m^kE!eohy_}4;(5jWDx7;z1VYp_LW`3vp?aP;z@N7MXOiLXJsv8Kvg z3u_ZOmh;jj#&GH>k>ddf&g_~ig2=S+`CZ&6v+-}Il6Nz(ym+bBQFkj3on6D|y(2S( zv^c1%e9+bDM$^u?UnvIs9yH(wevMWYrk#(q%9cMNZQ9cii+L3qX=(VRa<=xk?|CT? zx_ME(ohU7!D>`fVA^zM*?LerJrs07%*)rO9MJq<5-b^XKBI`F@=+CRFzr?M_afXD{ z52#UWBPo$o&kXPRp#8SrQKlW;=8^Sph@5|?r>4c&n!G<^Bh;1}#_l zo4?oas9MH}hnP`ykfbZwnjVS8M$ld#Ve2#Uxo>n_d2akCzR34vvG4B*yo% z-!HfJ@}bz73w~lzp4_&3vO$bC>zfm)KMWqdZD22!X!oF zN!tf4Sd+*d;Naphwk_B3v&zDM`2XiRm*_g~oq9hd8uGI6BI~t;$eU9zChztqAL8AF zo0=+b29g0X0#}TiJUXLYco|!#tCrSZ30571RQigGHAe05c> zA_+^Ls$hUrSH~R$n=O6xsO$Njmbl1c4%h-=lhdRAOQ|wb`h?0>Qqsrf?)BMzyFaff zQz*!5q*>7VBpz(PpBvOAHnF>cfew1zEVQc01cBXkZM0y1(7sRVfN!}Fg~)l4lU%e< zEokyX1IW2za=c&dKJ>2lB}w>$2_ZALHuH_pl_u@dsfvzi&D*0RlN7@~tYpc_zWqn9 zx(b|pS&U}Nbi7lKyp(KibvP=yFYV*P4)VhSwF1ggwrJb_uEg?a0Xc>o_KCWvr?m_| z8)Ibgv8fzRyLgVkoLt@_K8b|r&)*qxYPEP;h7{bBZ|vT+<^~2~8f#=2o&6?c_n(n4 ze%v41cKE$Ke?pgrgLQZ&C+bPox-`F=U|Lc~3|0@u)kNO=|Ju9mpr*cU9h$VDNK-nJ z2pEdeiy|PsBhs5l?qumnSIWzv)2CBUi(|$&dH?Yi3kYUeL#n-pTsAR@&t;iSw_g+#=`Ty zz(l^=6rW+-S^LP^caCQU zJCo<~pq$|hJD(U+y4igd*;w8@SRa@w4lB0+!O^_h*=(j5In8%uiHxlHIf zEK!UP7s;sAShg94XV}r6%~7A0k-aOxj74pERV7ube))24WII^fYjEj~f)zGJ`k{H7 z4>DM+P&ED~JF8Nwlh=7(4!~d1W@Xi~iN>fthd*6(Iy$o0R#1};-CuoQMQsOUo1hVI z$qFFSk9=oy*C@pcfn{f75 zVFUbx6xLe9qmQwTJ5=i0;DN-`CyVR?@a@f`$hH=NfMst zLauBO{siC)BAgoVuPRt2%IlhSjd;gq;$GG{`I%)xZnF|?|kP(CQ`Iq z)$vb&vp&(a8Hs)N`q@`czH!GEu1mJyu_s?AzaGWQY4wdKqNIGcpENErrOC*zVX(*- z=mWTKwmC?ne0y zPA-8XcwzkYqjs6X?C!T3$@ACY!0fSCOU}v0E}5kC%!nP|QPaDg&g8uXtRTAP>(cy* zOi70GH7t$ueQIDGZao&59Fb~yFhWa(+(Ob)pN?8j$BIPKKcza9C%U5 z3cscw=?3^-R26|sadh_T5BAc@q~n`i4!I@Y+I?DnFLwNHudz#I(lAYj^DMd+J9>#g zTu?B_cqC2P1bcgpGiWjb3>Al^Wck0xy_ln@aW@wc?#WrZKjAI1$QRjkMMi?81sH!L z#b1|e0wR8@JYHM{HE_^m3>Dcl^)gzBiW;fZq;Z|fJ7)8^;t66RPwZ=GJ07|;ChR9I zMeNT18P7hudGO|yxNg&O#y)qIs5Nh7#pC>Z%D>?an!l~ekzXp?(P1|BMaOa@n3Ydx z{x}cqB=W+!en8;F>1)aA2>W>+zMdQ zPcv+L0N-0KBMV8#oK%E4 zZR~!1~MGr!=srK(#c^d4lms<(m!B(v7(wx!{;@`%M@h@zroSUqwzJRJHO8O2C=|9W1aEUXgEqdRP)7-yy>WS?_+NWitA^+mc`Ja-p|3_!MB#@#I$0CigB>D=T zB(;OqCqoI5S{PM;@=cVX&c{Z_F)gx`q5X5C=n)bZ;jn63+WIF;cBKU{h^6az-3Ti$ ze7j?IVEGl-Gyb^OCoEs>Z@^Mzk7#R*2lEFg7cH0!C}dH+=5gX3(@BAsOl{jk7%uDM zw7IgJne1d1X`&wr0CG9}6^bp1aytog0?(ruNlipfb2ZnEygd}-<^(*(U3keU7?}O{ zAd~0lQdNMX>#^5xpi|Tn?HDRvx`r44o*yGC32ozx8;~jO_|?&e_(2AXylO&}>sgGJ zVNSm}#riteFIi8s#LB!4y3^F9tpa^6N*-Y>&qTr$xT6B5%3!DoTH4jOa4<3XiPGf= z)e*4Q&-nQ{ub3#z1+gO3)r)1`gl~g@#ywkHFDc(@`uA$8D1NK&VK?abM#YuQl?IIi zl)SZ0b%#`vd9+emC0~|r4kv&{JPMM)W*Lnf6%z1o5t;I`1XGuM8Zm+tv}MV%R74k9AqVVf2iqRn`9SWjm)6#jM213pW4bytIBE_!T}aijn+|bi zxjcr>e{eYL&iHL&(O@~xA_i&(@fCn+s4V!|1USzvxyK~8W6rsQ05Xx;2T3wb6{j*t zKy_a#zc46Dny)_0rwErCUWg;RI(g-|=`LwYf}g>f38^J5stz7JLC(?}{zG(q1Jd1G zN3SlO{n_Xx&Cx>ME}cq%usK9F{k zod@8nCFBX#m;udoorL2lZKnp!#CFU~JI;GzB+IjG%DT%w+IrwpU>Zb&DE~D??n{t* z*i%(~Ha_#-EbYYy~hFS~)sXJ<<_-sSg3 z2Fom3Yfvdb4Y4dGyRV}pT9UNN?gx4DVcEE#mJ&Jq6pZ~+mOFW_9Y`GK8~Ul=iig0LWB zX3BlQ?tO$OwyYDOu_=NmKcn$e4O@8<84_n~$qx(J`DNIDBaY9Py-WQkpID?^X|av& zQ?I69O@$nvjIS8nxgtT5Bl1RQ2cZqxMznq9`hbpKoGD-MUk9P3 zPYipz|7hA5u(ai2Y&g06mqs|Nf{L4G71NSkIbo@b89UEb!wJG@c1sM%F2^l4XH(WeG&OLOOxt&UP?@gSniL;a7G=?y!Wi7U* zUB=P({$`w7?7`WH(%P*Fw)kRHiEIfiAITI`veF*e2Q2`*J~}ios-Vwf<$*MJb2eVK zYo(wWJG`;QGOG~wGgvYGUeu7^>6+uHu+5BK?Jan7mxH%(z9O%WREp!f?21yo2D~^=qtmu@Ry1jv71$yHf96+ z*MXHrI4lS)Wv$o43OCey7uB|~sE=7yV6)6o8yQzj(SF&n0ZHcx$-XQ=GH;7!$#d8> z?K;QrAD>`tv7WXfCc+)6Q{TR01PR^u413EgRDsZ@uMRDKE<%>8loG2Ski~WG&oje{ z23GEQc2TXu&S!2FwdY4(uiVKNF z_ej$#Jwn)oKDsgHQJQI&;A5U}4Wj7%4s;0y4rO;`aeTVV&(kH%odLHe3>hW2aSXC| zVO+AH9J!Z8&s^VNo6&{6Oit#PNb|QezZqG+p0@0DshVRoTu3{cVist19$g{=&fa7( zL+L@-16OJ;Xz#0|b75itc%d;&b^?Z?!%PCFkP>8s#QL=i+ZGja2I^A?E83;-Q~iAx zmJzkhb$M~b8= zMr58~JSVQIesz2bEuDu?=qQ57o#Fs(uX`IB8dGPq!wS9^X5b8{6uwZc&AQedLh4x_ zPR|zadIQwK_I=Oow;lXDoBaVKBX9te(gI*HfK|Y46BQ^vAkI=v;+aTtKKPc12by0+ zR0AP0bGJ?vQ`D5pYkP>I0}*JK{L5ZD%;V=$+=4dkRL7hB7Gq>e<#wNS+2%68ZqYgN0YVLP$Mej_T24&b>g|h>9@vpy=G$w4YP~|pNANO zIDi2PIo>$cc}$B25htvIsThT}!YVadkpSnq;xd(9osh zF={W*%`kP)3_WbRjNxkQ(oXK=c^cX{eUz3UQtuq*mgT#ud5$89@n;}-^C@MSZ`m5l zv_damN9-IRX*;xa?dDb%JnQIDZoiJ~JrX`=a+n~QNCvG1ALnSf(_2VKJ%T2HZ8c@~ zadC~3wCw5V7gX)rP0Wh2uQ>-x1P$RdfaT%OgRb)E{u5O`{1wXN;ymyk(T~!1k$evi zCOw?#d4$EHKf)9;R)dkJz#&C>H5am6hK}(%aXnr5v!@9ziBz5PZG>D@w~c#+V01B9 zqY4z*@Z$FVd%5f&5Td8?f!G}vVmMF@&puZ9a-fZ4ljW631<>Hg;Q@PPiwzpziUR8e z{d4VN&~NWJAm%~uxb}w^E~nZ$%dEfz-ArFUaX18u*8_k!@K4B#Jo)U2SfPpGnYlYR znDvs<1IXHQv%y`rP_W_I?Osngk!ZXGU}4$8n7vvA_E18n0&&+pe&VjJ2B zgP*5)o5R~Z9OC=#Jiz-wE>Z6;aD+q(jeold@Iwd)2L*8Smoe2(%{P*Yrwrk$WEHf) znE0x_gA?dLtN>-NVoixr)UOk##ybFl+9gq(>tT{#)P$gjF=8 zxmKAoHwQGM>3lo!!63{4zFVr0h-;9@v)pR_nSwCkU z9%Y=SUWuCgY?e{2<70W9^5;tF|J&B#|7pwef5pr8d#hJYkyfh?=`zE3rvm-C&c}kz z8EEy7@X}|y3LWo@Wo4z3=(CLsJ;n<@MlSN8h@~#S$a&DI%b$Sf(b)6gexk|^VyFDT zZ{slv;b#%o{Wt#?{x%lE`F`6bXy$($r*LwxI#lCOvaB-imxizcK70PKi|9iNzdxAx zD6SFtem6n>-v|HwcKPfR%8Qrz>nB89SmxB_!e=g`>G%2SVxIYXGx>#OE-Z6_+H4mo z#6@y-k#-_>F5ut-4ldx}0uCNkDuuEoYlL{Tps1z@Q5kCpNwy@WQkLv% zwoLXVj4Z>LnP>iIbi21(_ulXF@BY5OW$t}Vc;-B3`JDIX{W;6$a}K?Q{t;TPaX|e5 z#KOV?g@gYgdOM^FEnmE3*^))emn~hka>ephYeo6ju3o)XT4=L?sKQPqMfsh(}+fxwGa3nVq3w!a-;NKj#auRIX5};?6`P4cA?O|%vZd6b>qT2EnF^fEn2;1 zEg%2pEh3^~;xe+kJSlWPHTL)Y9tIX=@u>J6E@J?jGkoy)Iwz z4+y+^Ehr@PPFQ%v-N>l8_=LoVNy#aXo@Qm|>H&yR_@j1Jn6h@(~gS^dG_6o&3wfrv{P@K*TSW4(Q09tuFanrO`C7oKi9BJ|5nR> zZP;(^YJ--rvVhKGT>+t?5A|-(q#UBuO;ujKqb69!kY-Zx?ZAi zzj-W5hpzZGAYO;*kPr8Gx}7K~bWE^X=`Zt9N}vR0VJ$_F6%=jqkQ{9a{TlmivCVN< zTjj2io|f|8Wgh=c`kdtWHtlEL%*xD*ohg|`Ow4?v9ZYSS=PM@9{27xoCBu{q(`qw^ z5Wiu7IfP&iA(#X5AErD^L&-FhzmX(!di&dY0CVDxe$8Bk_;&HzouV#P zTb*t#nH)q6+@t$$`C}iJlBLk`xRj1z_vZR+tz!=k46G`DZ6J^EX z7-tpkxFAu8`C-a{fq}FK<_LO^k|!$UHZoLu%pDbW}B%Xq1m~cId+{3M`I42JeC>rl80qu^)l(`)?_*a)1gCj$TU0%3Z9`uQ#I4^Uw)+6 z#4N=B@+F#?h$#Y7rU;lKfS4j+iU61kCKoV8@V^xWfe&uuqy#=kh5x|};wOwGpD<@7 z%vs4FVStG(CNBO67eC|K|7mQ6T*vCFELM|qnZs7!_(ba>*Rm!*^t?u!rVdV8B6Elv zG$$5)2A-@#6Jyl})aa0sJ~GEMvPL*AezFUMv!;NU$wk`PukL%+4bZyO?moU|lU!8C zO(kJLGM_jUc@#yP63tp1XDTk0tqNX`r$a^lD4RgU8^d^2^7UYNQ1Iw-I#gprhlsJ5 zwv8ayIyywy7rg!X=D0Y@e)MD;nlh+Mx(W|u!!xyUmD`wQFlF^W09SWIga#~AEo=L6 z2T!W;=&<~6`Iso+sqheC`u$wG{qI)4^=Gf9j0Xn7J+J7{rxau#SWx_S`LNMowncwd zM^}wE@@L9}V{?Dr*22cyl8dkGCSNO`Nrt;N;E?mBbIsRhjKy|}zkR8AA-@Vks=$*6 z=+I660nZx?i;OvAkaXQomAW7*{!$MkbsY`wN{19Jso2>jVooPqgQhywkqT{ zh@nwrV7p=S$u}?|%lO47Iy8o-ea75_2f+(3LH%%43hd-#1Xt@KjbM3q0E_ICfo+Fz zlWzd~#v8x*pbBW1co7qs?1iRHh$8h#bVwQ3fijGkyaQ*SGbeWwW(MnMv8iy)8c?qT z8IFxgD1^*&wh`S<^+&bo&>@`m826(J@;kI+6DS}U- zUIF}@h-w3UG5-R7K$Mnc0?dFOOu!*IDEnr3;682!v>@FW-+dAx3>}c=Cea}cban)F z4z$6L4xzhsDMmT8ax{@~)`D_FI_$1Z0cRm(Sb}OePVzV~Dh0%Zp6a8$nl>jQn^D>J z@P|-@h8>TuZl(!?7FbfnNmZ(#roaqEBzxh3;X>-*$zNZPGn)!UW>H-SXpZ&TlwL!k zl+H9v8PV-CpTpBgAZ-Md%%}|vD)S+o4mD_(={|5oqI|==%kju0XuX@K93AR}DP!1C zKDZ`ghO^0d64|`o9-eBLia(o2hYq8qjb8*4iFn!!rj>{<0$nu$(gGNKNqpx*p<}Z) zu!ep1zE5F-CBOnB83Clq0wD9j6Q85A#lK{2N$o?4m`#Tw2>0jskvYOl0%&7A;|)i+ z2qr#PuJ8o*m>ZNt@;4_>6c1+t?6l@&5Wz_iFlM%nbj+SI0Tb#FS?`eu+$4}I;WRe$ z1E`BO^#bU3W8}hxf9;XNps8b+J;0DS9pfvh2 zGA9Gu7=d2iIDG_`$BZbHQc0l*cyW4=!U4R)m2lt>{zB5a4DukF7>{W=3lGGBVh>

QP>khaE3{#4*v%1rB;HAM| zUW!VhlF&0#b+iaDWT|naUbUMc`IJ(pv@7Y5KdQ9>Jls#MLYcpTYhvco(G2xFWoJIw z0uzccqys>qluBt-S#OGhBY6-P8Bf_|4t}PC^x(yeH_GXnZYmM)H$l^a3lV1JT2G<kRlbkf+k>2#Pe&n>fV4$D0=@AAb2<4ys=iwm3HPEEt(Xcd63?T|2v_C_ttCZQ#>r z>Xi9A>#ctQDLZ7(9>8-KlX7CdVtVZU6lsT8UXlA+4T9X;m_(u1a< zZ~L0lyy}n&I`oh*3&`v^b_74?09YDqISu)^t^)q!K0%;=@qK zO5hs&Th69r4{`Xl0z5DX6HqiS^I42GmBvk(0ZmIqw}t>H1Eig;7wNZ|LnJ`|NKk>y z;k$rb77l@eW9g7D9h#f^ngt#yG(l>fjc!6tW1n~g_-9e-6IMQI$_1rR(q;aK_^ z&7hMTj^r7sXF&0uOMFu^)c8#tc@8xLx;ew>LZONW40tS02Ss4;vZ2Cjb0;4?f%lz& zAFjB8asl7qVVW1ZO8`ANg(d=1csz{`y?F`4Q}Mnrd*)42W5f|nU}03BQ{EB2$sI!# zNlx0*_Yr?tJzDM{TmfKxAxfFxg6@j~)U^dWYC{GV2R+>P7osO!r}b!;)zK2A&NocQ zQ%rFlps*J~f7+&uQF-)0R2}UhVY18@2rDNAmivg(^9jQLnH2D&s$fhINSoHA0scD# zeJ?i-QS{T^M@9uyt821WG-_7_6}jj9nMHz19PR5}z$h34)d zdJ%u|vNz>0df=X~8@|mMytylhmZV93JB^_xa{nn3=@99@c?Vn{LD4{sm6N$YC(#nM z$+bTEl!x$NloU@(WVFFvv-&fCe=PlhhD(q zuzBEQ#!KZ?TEot7H3N|Aw}5P<_~3zler~|Dr{f2b25o6xsOq9AsZ`hv(Ac*rzlCm& zZ@VL4bWE?UZ!6;$HT2_B^81e{WAk@bfGWN)08rm=mBf?|*6;tW4WMyOfU|Y7vV&b9y$4E?J&EOP0OYPJ`J>7 z{1BQ&y19he!e~IvQ&h$b#CebniJi1jF-pTc5%y_0r=^(?2ufaSp0~>^QkE9*N#Y` zn}I&5enha`L!OXjuUXfNCN;u0*2(4&MQBq?-3)X`&8@(f)_d?~$LnFjhC0#%Td#yb zq?LR8aZr%bBg8+p0E!!K26wy<(jie7P4ab?ZoYz-xX2{#<#_K?00zKPG(9uaHslww zwM9o8DV$ri`kg*6QTiA<$ zA9;eF0Gz+xaXeWo;vsyVn__!NK?^9e(p5T?dkJf+itk;W-jXY8wu%l(Ehh6jHKt*y zFGL0mmfcQ*Ybvm_{9sVDeQ3476uV0EXqVM0AIpoYmhb{({ zDjdey7n2vAtatD>hkI}|ujB8zse@b1$cO8kS%ze+5jL^c2Wv3;v+ot%?(Q+?rhuy7 zb$eEDT!c1m>wQ#?O5JNmD06#xxwbz%U;kn9&^Z)sg3zT-;^X}A97~4;QS}iy#c)mI z9z*27IX7c5l6TmU3!1}mJpR=1RKDYDRn4|8FwX0y6wlzwsvigSdDyWgzDYvb_!Vgf z;R;(c@wsn(AeD;kyi7naGXjRZ6+RjIs0lDt(Qs;%-AjkIf+;7e%h1cwHsi(7dc=4} zK#3O&Y;A`vQ-OHSl+NniFxM4Op;{(YtjUh(+y-xP%Qlvy8xoLaoP){TL8UXDO+BA?a2%zC{>IEXK>wM z{so3uXF{pAWCKODqBlkT#^SK8PyYwrYqZCuh$)7wOM5hcm|^A&6s?R~B07w-@P;$@ z+>AsUAD#`#7}8o3fr-bK8Gr#sxzsb?w=)Oc@`i{Mp1p#5sh?+#Bl_-(cpotV1{aq7 zH#8)qkS(ZDolH(-8g%zJpKqo|95A`0mRRZL8?f;kbGGKRs!V7k^4|0x6r#D(HN z5-~adz0|zw&3rhL71SrZ#d@=IZLVi((zQFiaYNZ+`s(J9Yk(8g$c{y#aehDTQx2ff zLx;3Lv^KuU-t_4?7z|1q@SifKP3v)Q)U>ZCg1!+1+;U8Nf;Zhn84S{Y4egFgDtV22 zeJMyZtwOAWKsH8M2XJlMvbZp8`!1}@;qAkj>tvrD?6s)(JZ=P?AeSt&$$z%un2eDV zm!q)GktsvgSWEI}!+To$Z?+a?^a*!D61h9v=tCta)M6>%qx(jN;w&r|P`$MA6^4+(EANl0_*?#-Oz zU7H9Dp@QCu8Iz8T2IrWi;$q7V?%u7OBUla(83rxkgf<6P2LO0!~=43D!xETb2 zRcR~y{QQOIOax=JALj>Fqg$osXBcYhHUoo9m>rDavPsSbN}z=QTz7z(9aDBppfI)K zUtpAkWYYKSafn;va4M~itiS8`--ww&z|Nze;lo72P}E8DclW+>)z^fY;Yxw-QlV?8 z6Tmv^`9!10ePAFW`aNeQj&!xiGR6J7#)4I=38`n`#Hd?z$a?P};eiOrO)!JpNckbl6jibg zEsX+(3|S|UNyHMYqg4h{h-B<|>Q|Qjt^6dA*xO`A0zR!qo1uxJvxEST-!RJked*%9 ziK=AGXi`Ti?Jj<@5i!8ES|VOJN`Lyu7gKQK7jvOEcN0a1w}Tmn{x@q_p%lq^YvKVG zI*fm4m?}3FAB@UN=}4sAB}_K@g3PhNfUKEFZU4S)_^fdnezjny&pXt&Ye9C%FP7$O zk4cRT3*eWNCPl&Or~7Sqe&sZfnvI&NOZ`D!uyx%erMSH^mb536fv3-tm{v#3XB=fL zD4GlLd*44QR#m z`4#`nAJblkqs`BITetEkf>{TVv9R@Jsr`fbSGoU+ui@p0txpe$2qYCXMTZ!o@(Gzl zyx^YxAEY|H$cNA=<+!aZmLw4mdVrHQWkX>vr$gbvpg#Qg@3k63^4UftayxDcoL{dt z2dlF8NEd8dyPum$Q z!H>V|e|@9|jMSyNi`#4RwZDNhhGb3y>wTs0+_dX=y@qStPKPRH@sr$+`uX}4Nw`KI z^jzH+Ei!ccA+-}oO%86?&Nrq=qX*)B4e+hEf$AJc-SfRFAc=v94$w0rq%xR7oYSL2 zr&Vb))VD{z?~9~q1=Kq!d*wcOqy;V%=|{hgpzPu{rVjgjKg5F#VyVeWX_Q;A^1_$)9md=4upzjCA%lcOTOZ%k7%9oj_k9~J4qGPrHB z1`M!ed;;1LPvl|3kG1qdmlgF2*87zB1Q8HVY-K&|CiuC?{rloiYLbmR{A(B->U$2DsK$Zvlg!+x!=-W8)VV+)@s0&-rBG5=S!J@d;-!1oo^t2J7C#(|q! zpvp*R+mKd6Ex?cs3cl}1GS1oM@f%Lfr|5|>eB{Fin_dD8a8o_;C28N|Zy>_i1(T^W zm@9&qfJr8+Ohards>_)UElB+#TUtUvaEMY6=iV%(KGgU|gr}j4a=iX9umb4<-*@~k z=MZk6^66m%DcWRy_21$8AL?|WN8 zvE|IX=1;?=N~z<GlG<$QXq)xXc)zEhhrg8F2fjUPqH7q#M~ z9M@&OAOC@}dAB7tr3Iw4c8>fw)5(%yPIdlpUbi7+!Om@%hjn_jKR9#z1gD5oV%Y`i z1D?G%sHG*{PoED6DDXQsI4s5Astdoqt1G|dmd{?1P3d(iyrXI9qaI|jOLWLHGI;e| zXqD>f&{In)B%MB{yu0+~5l(04MdRd{MIH*L5*r_?TWl^X=5;9ujD6Lgj^S71FXi`F zTMM}@c%TM>1KXG%rVQpW@Vit0nscncV%BFmlmx#wAI?ccUk$=7{N|R6rx(6mKT^}a z;g9xb=g~C@XCjKJQGbL1Cc6GeD$Hj;M_ZWaVj}71AnA|f_><6O$VU{ByJ2s==-^;% zf-TDKvDY$|fuKOj(VV9|d0YDWw|j1WzWY;6Ow3XZ3FWPO>rG;du33bLz8Hx=Yoyid zq+p<#(vjxWanv-;J!?D5l}IbjSRs@K)D)MMP?R6iZ^!`GhK(qLsG_a+M+DJEaBBU*i~IU#qK z>p`pVX0G+qr?0Ps`CZz~(zfvCUETjdXTu_R5mefUuP2VJ>t<1v{!HXyPM9+?>4#ZCbBRWrZht3V7!ZW0|=z<;Fw^Qx}*V!emRPU0~W%=E&h+ z9V0CdW<%{z#Tup9{DD{8Px5a2Y2J4KmginV40~LtP@|rsIgz{CPe4tsQk$4Dpx$4~ zh`nZ+GVpaXpO3o6;MUB$bL+6(F);r%Zd$p4lM!Q0;bGaz%iRXTGX`Fm1K-0@cfw>R zaBtONYcxQe$pssLU=5eBFLX2)JV#T8{Umh_kq`F3-=JYKf#Q=vozbrLldLl=A~2pc zhf?-6Al_2FHVc|~jVTAf@7!heu&pUva}w<8;l^VYyBVTK_n_zfyl~_36ulb63wRnP zQAHQ@@oG}^z~#j#N^c!4)s{91-iU$<)O3;NY-AE9$l%(Vi$Fy$)UwZs2a*aiQbEnY zLlOb(s}f{bV%4mHL#LUSI&ddWKw5K=5j^ny;D_qsvhcYMbSPh{7hMucISqD688HFC zjZb1H#g3aiATop@HKHfMa9smxh%>fBGkvR}2qD#vKAx+?h67ahYM-Fc&Ef+rt_9hADG&}Rr1V%<+ zexSpV&Km)C;HaM+tFfcW_<>Nw!cSH-t!6Zt5u=zWMXE$iR%?S0Bgj_QgwPK5$vYw@ zP(2vZ+q@YoYT#@w#|>T_Ez@C0q)(HUVUG}KlPYTlAEAJ3GJtIAfy;jz?1?9as;o(V zWZsthCGKuZ#Jp&6l-^qQcj7qG1KhkDqV}vF@bUd!@0gFS(pFi&hIUk*W zgyG|=`RfS)ESq_1cA-Hi(*voyID+{+fL-X&d0)4D!E7j7?+!7C8PM z_|_7mRMQwx1(A!a`+}+!s_-+LLGLzb<#5~d(nMLcf+`dmkHx{2dIAo6R0uOokX@aL zh?zio4Gnn3G(UWBhLnez0dO<*+a;Q2nU1zGM&=R}8VlpJD{DELfD?0;{16X(8^i*Q zOkG10LlGI4JT(j0)?Dy5_p7BS$42gh*e<@9rOzc+je126tph-Pq+It1*L4eSmqvIP zi9MrIM&?6(ku6Dxm+zhVXI?45je4dwzN|_6^Fj1z3`4`(<1&@YQ=*@uX&?~nQyocQ z|MVSX$NY12mLt{#)7_69NzH0D05ZVN>TrD6xZ#{^Fb(VERe|caJrFGAsXrmZ^5ht& zk$H90>8s~%gR>$`RMmVZrD&5HGOC?m$NmC{FdJk&)UofEZfW}3Em;9q_jJrV-8tOp@g!GSa#3pvT(bp4&YjCsgDgOh0N^d)HsXt) zYxGGnKzhGKQJcBxaEP?YnuCCF3aSh9;L#kK@yq<A8UM>YJWBedPCWj z0&EKkK^PdIY)Sg{k5BX83MPUFW(cB762LI@ZFc@8qy_{BMB~ZJfwz+nNPwY{zC5oi zL{Buq#4HTSaA4jp0&Qhf3fC%ktPeyzfj?0#GfI8)alP|jv@`(XGoaZZZ|@;qNuL<4 zlp;yuCz}{7DT$kGg~i)IEAg4;!1U23HECA~lN~TYHd765}l2MPl?+k%@s1`}~ep}npOAG#$NRJewbQxM1XmEw~;#as*rriA%JtQ{lt6)WS9DXrRRSmS&*V5tFDcQyCo& z50Y%&Od=J8a?l~mcGye1685(N#nGYG)tz9S3iwW~#xqQ*p>GfrOAECJqL^utA|1|{ zF+=838+F*FMpCuM zXhH4Zcm%Q@?IzrFSAcdb zNR0E_NZE>~jt3*NJU$^Hmw6u~ko@up80Wl3GrU|Gg^2VSiGnJeoUZ~|m$VqvA?I`u zluE7a}R{9O?KT_5MS6Pt{}=PMOiIH2HWhn5|FpR zXcNNhK=+$}MUOh|DtaIvP%e6II$aI2f-T^NQbZm^%OoIg(37B#Zo<=_EP}vIeQz8Y z7zOYXAxQwNp8_x0gXX_Q-i885_W_j8qutB^s69#MLn0#--8;HIsULowI~cP>d*ohb z{TFiArA3@a_dNpAx28%k)Dx*HLQOQnAo72N0*c%UPi&l5_7R1vzxZa>MZecZ5w?IYZzZ3gVjPi3#@z+XYid=-1fn;mfnA`Jm<0c?LQD5Zwn{G=TwZ zPsWn12s0zijC@Hn6}UnOkY5$5O|BR*;ii0?e@;vCLidG$^c>hxvrIZ98H$k6J@aLd z3g4d(onWB7lGd$V_9c0a9B`;^j-c3S1Gl=!^r&6fy((+zP`ma#wBjia_yIp4@ld zfJe|^Xpo@LItqcOc#xKKo~0~+f%2;s0z$y2TZcop8OI3927%Ic$LmD;?N4l47K9HI zh1_^pf@zsfUTUab8>Fc~X%k2dq5$try0)F@c=W_lH zM5lRoJJGsTF94xkXin#0370ui@D8X-BEO_kZ|J=G?6&zzj~X)GjwWa{k^}gVhvW7a z<`bHi;ECJ)lgYfjkn5!m6<@qoU2ezPy7*V`PF!*wx~{|C;Oy<;>J;v{ z#JN4Uvdlj(cLzrIVwFk5-*eyTJ4xy?gZ-^Y$a}J*p|^<083nWXeI13t}^tg zK5fwXg_o7D#T{$OK5gA_VK9e}Roe(%;#SqAMN5q#4;dk;6h-q7w8JRtUD%nX<^VB&zO3rs#>aw5|% z{ExAXS5{nFcpQ_TV)MfdUf;O!{ku0gJxKs-ZmW_sw(pEG6=jhI|1jMpnJ%-xVc;jY zjlO2e^EY`iW$~LVnArMF7EE0HTv;%&^>gLP#OH7F{3+O~e7{Y~VI+JO&gDm~Y)pZ41|n3hP;rfgBXaHGN%m)OjLnR?mS=q zi06`)mta^SaPTOV7|JsaA~lqSs`r=FZ9Eggff3;5wa4Uzv}@hivhLL82Z6D|_kDA` zf}njnwtMMcct(f70f)*#kaYxVX3a!=1(p&_)*p=F@N}j_z^z7+nSdz=CLWl2!Q=-f zPcqHH{}r>iuy*~GqcpLpKZxY`Zx@mAQ{zp3Vx$POUVng|ey z_0rIFH=@zf`!h4Es+`xf){Ry!OuT3r8lttED!wHE8LnTd?~+O0r9^|vFF#d_ zvV7i4OFHK@d3Z;f9$7cdg}2>r$N6DTqEWVI@3RI=+k0+7CVYiX2Wy6O*k3$({(QUX zy{k#Ew8pbz>DQOLwg=@TwiQ0O;#9UbG~7kODnxrSX7PSME+uvh=_j(6J@`7Cv7RpCYVBqO#u-{MRjXt@T!=$O~}lkovG*oscbbGv9^wx z0aFG{958i($p=hMWZH%QIks_C$w8KU<&wm>APC&-_`8pUh%tygrtLwV(;>rjyWbyV zr2((tRbD<$q@G99&Y{bUa;#=O5JeR79M>yh8iZR14od|`R%JJPQF~P+I*Z&M~*(*Gj+s-!>+Wg=MDfgD0obkkmH&4Uv zzK#uWPi9R@GR=%FBPNyz1n=0nrKi#F4R^htjA()gbbl3A16vIal-C0XVS*z&*E|4M zpMygsi)<$GLw9g9Rgk8d+5p@j1}vC``4;L3jdK9Rxz$c zI}Sn~GT?;t1#3?o+%Us)@oUvUjcm{9v7F!<&1@JjU7d<%PmPCT|^h$%gLd(SxLvNKAW*I zcrDvwh|PPYo%g}|2D9ikcCID2BnvoM%!Hlx=)tEn9oZvuZu5IvlL)| z>1N9#uSlENIh~Rr8PgC_ZGWU$ex=ocQ^Uhr3zc*1w#qyZx0jEEy{cVa^a$V94cvP1JAMk_uD3aWm$RL$}9^BY#T``wJrYKVSh88VhJiS9X=8?w*j-HA}b zdRW7+mC&yA&Lo>dtmD~^mHa!rrq(xok~o!>74%`*e!DfwMN=$F{{h+XWTe#(A2jta z&r-{e-Zfa7sazaYZp&U6o$7jGiF=~Keq;Zwb&)>#yz!4rj~!EouI?M(DEax_8&0Jx zKNohT^s%!e1Gz5_3SkaEI%(ldzBsDFL;O*W*}3jIb`svQjrPLWopD(%x)_LC(^#>z8xl#gpEpOV)yn*~0rt)JCzO}G{Aa#wh! z6jh&Lu-d4?+M4(I`m2rGe0+wzelRXSb8YO8@)CbQi#nREz3cc|kDaP{$|p-v5s611 zdE7cCGhSo6X$-qD-euo2v{tvD^cBMCr%k=NM!C?ovb7F26Wh%MvS{K>eRud>Oseep zOE(?5T3L8Z{S{~PLxnSW=62rAYM$LI*n@>P@X41CyV$Jum|1!z;szANx$KZt_^q1{ zFgz^05AN^>byl6@FBhsDOzXI#`;xW(u}zjz$!!PWD_M75i4^Td!dSqq9o~1cY?kQf zJ6Q?KSWOSgr#62pw!sBn|B~(d2e$L~TWO}>QeHjojhglq#cyw{Hc=J;`*56e=y4Kx zQFB~U$!px}OF^P(75@=~^)uzb%6&8!B@H*N1|WfPvcV+D-+sKyP!^&Tt`Ni>S+?tC@ZhhkQ3Dq!EStBEszDt6$SP{@JX?wAJyXb%Bbm6f^&6Lth}Zn%C~^Y_K@ zu5ZAw9hOX6u_jfpX*0y5tM+q02I|g>ANPeh<4?O{@9i#azH^xG?(4OlL9szI&`b>1 z4;vxL^)|hpE7q>Q-W3>IVyE$6z6HrTV$m#?3GT&J?0s;iZr7LmVg!L8yG>#z~8NJ;x=nxlNe%JYq`lkIBBmMmJF z$MaLGlH{X5hksl+7O}*C=jFV!uDE*K*1*W5SN8oq?hc;+t}fZFL`&K;P{CF&_|?MJSxJkqfbOR`;7HM~&#&8-8z`OicOtL6KB{$Lu&=uZOq%& zmFXIpi}l>LGHNeiLunjz;NMf4@2Q&gCRydE%g!2ea5q|1oBwdzrU1+XN8idLk6CYe z3gw->lP8ze-jR6!efwIGyOWZ9A|n2*x0G`nDN?v;!o&;-R{-wOou=8DrOFHTopnfB zUb*IE$Gxx!`B&>bVixHP?<_y*vcHaNqf?wGPh6PNxc78VbEIQ(Iq%Sc=l01L!f$1% zX0iJ$?d{% zw$YBQaGw#oc5>T^iIUm`ujy*Bh7UE1OG1toCO-3Dr16Bf#JO5)nYeZRHvXzKfAZ>f zjbwl&cea7svxWgrpJSFfcs}7Vw7M;q!OF)f#KU4_Yt@?gA?vO1!;`BNG2-u9IZ?M{ zvyKc|Yzlu@&UMbe>dnmq@&oI=Ltycif7-c(BY)%Cu~yC&A^mOHtn>;eG_J6UL?C?NNOP%-e|Q0UWa1=GT;Vx|k+$ zxTaaK8T~HE-L*q=FmG39ys|GL z0_9=z_MG13D-L2us}IY$rCIr8=_+7Ip+`5gf@^%QST12b(K|iRmOc7dI8?9D|DM>I zh$og04;;qOBIwY{(+$+Id|E%jsOC^ZV&?th<<9nQ8z&98c;i3ZcN2=1(9_`XJgXLZ zu=S+Z@G=Ex?V?k8ZCgijJa3fHLeyVfKkKW-jUd#BbU13rjl2Xd6qUUn+{py-r0=ab}7#{3vGm_hu!NBg-1x2JDGZi8+p2W zt?=XUa#mRqVm1}MSk)=|+$-yCLfxAoCr)2&ebwVNNmYt+dW-#T@zs(1Mb`7~?x|1j zY`qr|bR=R5NTSL)LE z9hQC3&y-x0h7+PvD`s^#cYO=#N9aAfWmYz!ZtW<%=X&rmsjdU*+U=go-{5~CI9L734WV6x z+rj79UF-eC3#ag{av_Nwa>tMFQm$R=vV+e#=*d^)?B&_Ga3kyb+eFnNZW&`=pO-t2 zK1w-Rdj)oQTzrzw_t(JW3u%rEgr&}xQUpQ>(gbwvy{PgxPTg0v5AzP)FQCs}) z?|PXe&w3w@wv$NRJu+&!RVJu>&%w|H&lTaq)Z&Kw`feZJ^OxtZ;po#_5TJ8UU--Hx z^%~WN_3Xdtx3#3oXl$a>z|1CZG@wjBm{n*$RZe~g#+cs=@^l(kg9JYiHxA32YsOxeQO)8M*xm{|8XAc*m` zV6}Or^{|}&t9Ye%exd&QmlReWuzN9@TweRN&^#%f8g7a|Mm_ta?My?wjcxd%LFZ_v}bauD;R_fCb9;geuPr;D#>DtEp zr~A)HJ{|Q{xBr4Wit0ttphTeZS+VS(WQ~;qALVuHG6mj!~SB zS(0?qIOl4q^mW+g_Kldy7atK`-?8EDPP4ZrEH-C~16p>NMZ;QQJ8k#3d1^Z6ynXe| zxb?n3Yst(j4u_!WSD$W2 zg>VtKNsUER9~m9(<0G?3ZeK=l%qZMso42Z6b?MuFiVn}6iSbI#4XbOkUq5a)rA6CE z5DIESeWkBEaiBGz>YEM+x?c{j=-9Miaf09xiEMX~$68{ak9F9&j^x}MsY_57+V!$h zbIHY;wQu)O#N95YZ!8qEX2`Qh&9Q%EN0mWF3#*DF z-xN9BTOT-Gnsi1#yrlbp`}0C~Nq?UGicj*1jgu-oQ#0$v8SQPZm{9;=J1XWAJS4-1CM1oSYQs_kFshzR{2CJ zwR0@jwe0-NZoX*=?U+BqM^?Za3;o!WyJm32F$%#Y_thIE>An(>Vp9T~8Ad};fx z`>S4kYA;kiCak`+*TNyLFRWObWI`xoW@|6gr;g29FLo$s1DyE&env)bWM!f~On6Nfx< zlh$N=FU=xwMfjxfl6cLrXq-yDQnSCdTEHZkn~X7jM0lKhZbQuybUhgk!tXT>tG8(PQgW z#V$?9-^f!rO(}L~y|i28x~qx+uU@09rWreJB*<#_n-kPh%8k}X9FGZi%;drobo9^Sb7VAobW#@a?Zqvl=N`tCdy4CL=kXvY^ z;^O<+GE{|swcNgmx^27~fi;{hAjw;o&qs>s(87jew+J?B9_n)IAFk~Y40CU8b(Hna zyPqZLtIx)EvA|J32x^PVzuOZ%)@&cSmGzJb9jaei-LRUgg!6>7k%`*W;M13mTW+#- zK2*sR6OtE9sO6Y8d{(f%qjW*ZarG4+JYSNXOM9Z;|)F zN1Y_el5|vS<8zy|(Pqtjp4Cdc22P7nH9=68^*MoF!80c#*it2y@Cqf%${IC{E`5-1 zceLucxaHr3jMPgHa9Sp;=+OAhQ= zET~eGG@Yt_3TpiL#DQ>HIy+G_w7*EDccv!_YEJ;5$24fgFAd#(V}r?p0cHJFm$6#% zsh71gv%PKO!{Tl|V<}#Fy2kQT6x8^qU87&_xV14=CQ}ur+~Hn+k7HMf%N|#ibyGPt zci)bt-|~xWR_aQBY}HnCbI-ZA7~zt2T0!nb8E&zvlfaOa$><;19+C|#19 zcY7pY&%zJucT-dxTKGO-bwBBvz0BQezxi#xej%o9w68{JA`u|ZH+Y_Am!Ny}_=n0DFOTLz|zR=;I5kGwoxBNPAq*9w}Y_s%8Dh9XCHXPru#N zP79$!+BT!gRYm*KN{-cRX!9Lk@0-n18|tT>sCvppYU`$FhT_6U{k)A1KS)@UyR!4x zDpE*Iq}+0gR~uM{{Me21a_xf!&~|DrD;JnH^|nX*jw45blDsXuWV7OZZ)C1x&4CfL zpxs;7tQ|`i3~e-tKfmQDW@Gt{JTVBRtudJV!zH4>u62H0^ZYOP`7(eSG+i7 zV7(sr2*R@WP8`#8L49yz1IUB`Y+I<{z!>K-ZKwj~_<(xsb4*QyV$>Rsy z1Btu(kG+vV>E*vl*_KVQe`p-Ov+w|`1n=&tvQeYtm_iFE}()Q*{1J z+*0M{4^~`)7hVN!dmGHXPung>4bltL$hx%O?Z!3jXMC-fXe_&Vf)h+60#>Z&=vd`_ zhV%VdXJe38gn~nQ z^|I!!d$x*2_9d#Li!|hqVpT6!9gXu7yRofIW&A4V9pa0}S}fCoIR8zy&CO}nmsU1B zdXRWOq*Z`Rqz$ZW@@7*k?QJqMKWyBb;AkWwGMy^(Z=6I8*6o*nFiUJod!jyLrI*x} z@;0$p;aY~!Nw;@w*F&Bx!SH=-4&z~2TKK{}lkCi~_&&ATYeSg%^+7dXdzExecHpYg!4C#_PuoB z#XjTYc%XdG-g(3Ay0Dk+&*el9PJLwbnZD1Cx~`j2B6e?Q7D?+BJrjy@^YTe^QeSw5 zCzLq0IoLr}g$=i*X6$pak)5ih6swnPesp5h7mwCX~jONzUAQsV*!P8Oa` zKUr|4SRs7h-j@~%KH_WLDvoS6u{2?oI#k)~yT1u2fqd7^?lnQDxYu3Zt~9(mExpU8 zb@|vejaW%_=zjFWG*DDxW^P3Fq)-hmuab79ja7|~Zt@L+61Rj|OIAq~Ps&$7KvSHS zEnS+}nyriJ<6VCJfMLrH?4iq11!|DBdeVj?4j->?#YgtPa}9ap!crq5f(o;#`^j4= z|K`>VZ)hr5%0oaTLP|pp72R?HIu7Rn}_}4ep-Ip<6?)pZaOo1^$!c>f} z{KHfXCVMh12GdH;4@Q_|z$61E88FF!Ne1hv%xMC1n(*IYn$WD3L5EOt9<*~vsy-e1 zbz+}PIKJ28|E)gpiRb>DJs;eR!mq6poHT7k=PBKcGppbK_|LEdcvjZhKW80XMQ_Va uJn Date: Fri, 24 Jun 2022 10:08:51 +0800 Subject: [PATCH 18/23] style: fix flake8 --- .flake8 | 2 +- sea/server/__init__.py | 2 +- sea/server/threading.py | 22 ++++++++++------------ 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.flake8 b/.flake8 index f0fb19f..156633a 100644 --- a/.flake8 +++ b/.flake8 @@ -4,7 +4,7 @@ # E266 Too many leading '#' for block comment # E501 Line too long # W503 Line break occurred before a binary operator -ignore = E203, E266, E501, W503, B950 +ignore = E203, E266, E501, W503, W504, B950 max-line-length = 88 max-complexity = 18 exclude = diff --git a/sea/server/__init__.py b/sea/server/__init__.py index b19bbc6..1012e3a 100644 --- a/sea/server/__init__.py +++ b/sea/server/__init__.py @@ -1 +1 @@ -from .threading import Server as Server \ No newline at end of file +from .threading import Server as Server # noqa diff --git a/sea/server/threading.py b/sea/server/threading.py index 7b48918..820e070 100644 --- a/sea/server/threading.py +++ b/sea/server/threading.py @@ -1,7 +1,7 @@ import signal import time -import logging from concurrent import futures + import grpc from sea import signals @@ -15,21 +15,19 @@ class Server: def __init__(self, app): self.app = app - self.workers = self.app.config['GRPC_WORKERS'] - self.host = self.app.config['GRPC_HOST'] - self.port = self.app.config['GRPC_PORT'] - self.server = grpc.server( - futures.ThreadPoolExecutor( - max_workers=self.workers)) - self.server.add_insecure_port( - '{}:{}'.format(self.host, self.port)) + self.workers = self.app.config["GRPC_WORKERS"] + self.host = self.app.config["GRPC_HOST"] + self.port = self.app.config["GRPC_PORT"] + self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=self.workers)) + self.server.add_insecure_port("{}:{}".format(self.host, self.port)) self._stopped = False def run(self): # run prometheus client - if self.app.config['PROMETHEUS_SCRAPE']: + if self.app.config["PROMETHEUS_SCRAPE"]: from prometheus_client import start_http_server - start_http_server(self.app.config['PROMETHEUS_PORT']) + + start_http_server(self.app.config["PROMETHEUS_PORT"]) # run grpc server for name, (add_func, servicer) in self.app.servicers.items(): add_func(servicer(), self.server) @@ -48,7 +46,7 @@ def register_signal(self): signal.signal(signal.SIGQUIT, self._stop_handler) def _stop_handler(self, signum, frame): - grace = self.app.config['GRPC_GRACE'] + grace = self.app.config["GRPC_GRACE"] self.server.stop(grace) time.sleep(grace or 1) self._stopped = True From e80039063b28cd76e539684fecdfacf726a586d0 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Fri, 24 Jun 2022 10:40:47 +0800 Subject: [PATCH 19/23] fix(server): using os.kill to kill process before 3.7 --- sea/server/multiprocessing.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/sea/server/multiprocessing.py b/sea/server/multiprocessing.py index 4f92e40..e4118e8 100644 --- a/sea/server/multiprocessing.py +++ b/sea/server/multiprocessing.py @@ -6,6 +6,7 @@ import socket import time from concurrent import futures +from typing import List import grpc @@ -22,16 +23,16 @@ def __init__(self, app): # application instance self.app = app # worker process number - self.worker_num = self.app.config["GRPC_WORKERS"] + self.worker_num: int = self.app.config["GRPC_WORKERS"] # worker thread number - self.thread_num = self.app.config.get("GRPC_THREADS") - self.host = self.app.config["GRPC_HOST"] - self.port = self.app.config["GRPC_PORT"] + self.thread_num: int = self.app.config.get("GRPC_THREADS") + self.host: str = self.app.config["GRPC_HOST"] + self.port: int = self.app.config["GRPC_PORT"] # slave worker refs, master node contains all slave workers refs - self.workers = [] - self._stopped = False + self.workers: List[multiprocessing.Process] = [] + self._stopped: bool = False # slave worker server instance ref - self.server = None + self.server: grpc.Server = None def _run_server(self, bind_address): server = grpc.server( @@ -136,7 +137,11 @@ def _stop_handler(self, signum, frame): worker.pid, grace ) ) - worker.kill() + # compatitable with 3.6 and before + if callable(getattr(worker, "kill", None)): + worker.kill() + else: + os.kill(worker.pid, signal.SIGKILL) self.app.logger.warning("master exit") else: # slave From 0db9b99cf36b32fbc18d35a4e6464b64cd68b181 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Fri, 24 Jun 2022 13:39:09 +0800 Subject: [PATCH 20/23] chore: fix comment --- sea/server/multiprocessing.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/sea/server/multiprocessing.py b/sea/server/multiprocessing.py index e4118e8..7d125f9 100644 --- a/sea/server/multiprocessing.py +++ b/sea/server/multiprocessing.py @@ -104,11 +104,7 @@ def _register_signals(self): signal.signal(signal.SIGQUIT, self._stop_handler) def _stop_handler(self, signum, frame): - grace = ( - max(self.app.config["GRPC_GRACE"], 5) - if self.app.config["GRPC_GRACE"] - else 5 - ) + grace = max(self.app.config.get("GRPC_GRACE", 0), 5) if self._stopped: self.app.logger.debug( @@ -168,7 +164,7 @@ def _reserve_address_port(host, port): socket.AF_INET6 if ipv6 else socket.AF_INET, socket.SOCK_STREAM ) - # ENABLE SO_REUSEPORt + # ENABLE SO_REUSEPORT sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 0: raise RuntimeError("Failed to set SO_REUSEPORT.") From 5a82e82cdb1d3322a7533589c95c3a78431af1e0 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Fri, 24 Jun 2022 14:05:27 +0800 Subject: [PATCH 21/23] feat(server): add GRPC_WORKER_MODE config to change worker mode --- sea/app.py | 1 + sea/cmds.py | 20 +++++++++++++++++--- tests/test_cmds.py | 12 +++++++++--- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/sea/app.py b/sea/app.py index 2c9b67c..c7231e2 100644 --- a/sea/app.py +++ b/sea/app.py @@ -30,6 +30,7 @@ class BaseApp: "TIMEZONE": "UTC", "GRPC_WORKERS": 4, "GRPC_THREADS": 1, # Only appliable in multiprocessing server + "GRPC_WORKER_MODE": "threading", # Worker mode. threading|multiprocessing "GRPC_HOST": "0.0.0.0", "GRPC_PORT": 6000, "GRPC_LOG_LEVEL": "WARNING", diff --git a/sea/cmds.py b/sea/cmds.py index a90020c..1654dc5 100644 --- a/sea/cmds.py +++ b/sea/cmds.py @@ -2,12 +2,26 @@ from sea import current_app from sea.cli import JobException, jobm -from sea.server import Server @jobm.job("server", aliases=["s"], help="Run Server") -def server(): - s = Server(current_app) +@jobm.option( + "-M", + "--worker_mode", + required=False, + action="store", + help="Worker mode. threading|multiprocessing", +) +def server(worker_mode): + worker_mode = worker_mode or current_app.config["GRPC_WORKER_MODE"] + if worker_mode == "threading": + from sea.server.threading import Server + + s = Server(current_app) + else: + from sea.server.multiprocessing import Server + + s = Server(current_app) s.run() return 0 diff --git a/tests/test_cmds.py b/tests/test_cmds.py index 7fd4ad5..4183e08 100644 --- a/tests/test_cmds.py +++ b/tests/test_cmds.py @@ -1,15 +1,21 @@ -import sys -import pytest import os import shutil +import sys from unittest import mock +import pytest + from sea import cli def test_cmd_server(app): sys.argv = "sea s".split() - with mock.patch("sea.cmds.Server", autospec=True) as mocked: + with mock.patch("sea.server.threading.Server", autospec=True) as mocked: + assert cli.main() == 0 + mocked.return_value.run.assert_called_with() + + sys.argv = "sea s -M multiprocessing".split() + with mock.patch("sea.server.multiprocessing.Server", autospec=True) as mocked: assert cli.main() == 0 mocked.return_value.run.assert_called_with() From d118ef19523d481312044928f6d63f382b1b189c Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Fri, 24 Jun 2022 14:25:26 +0800 Subject: [PATCH 22/23] docs: added multiprocessing changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index af5d54d..a370c03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,13 @@ # CHANGELOG ## [Unreleased] + +### Added +- Added multiprocessing worker class ### Changed - Use cached_property from standard lib when possible + ## [2.3.3] - 2022-04-22 ### Added - Added `post_ready` signal that will be sended after sea app is ready From 3ac543c2ac235778f2801fa8702f36f27c98d801 Mon Sep 17 00:00:00 2001 From: Weidong Feng Date: Fri, 24 Jun 2022 14:44:27 +0800 Subject: [PATCH 23/23] ci: fix gitlint config --- .travis.yml | 2 +- test-requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8cda423..c9723f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ install: - pip list - pip install -r test-requirements.txt -U script: -- echo $(git log -1 --pretty=%B) > latest_commit_msg; pre-commit run --hook-stage commit-msg commitlint --commit-msg-filename latest_commit_msg; rm -rf latest_commit_msg +- gitlint --commits "8d1d969b..HEAD" - flake8 sea --exclude=*_pb2.py - pytest tests/test_contrib/test_extensions/test_celery.py::test_celery_no_app --cov-fail-under=10 - pytest tests diff --git a/test-requirements.txt b/test-requirements.txt index 39227d3..a2329e3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -12,3 +12,4 @@ coverage>=4.0,<6.4 prometheus_client versioneer pre-commit +gitlint