Create a python daemon (microsoft#8420)

news/3 Code Health/8451.md

@@ -0,0 +1 @@
+Create python daemon for execution of python code.

package.json

@@ -2738,6 +2738,7 @@
         "vscode-debugadapter": "^1.28.0",
         "vscode-debugprotocol": "^1.28.0",
         "vscode-extension-telemetry": "0.1.0",
+        "vscode-jsonrpc": "^4.0.0",
         "vscode-languageclient": "^5.2.1",
         "vscode-languageserver": "^5.2.1",
         "vscode-languageserver-protocol": "^3.14.1",
@@ -2769,6 +2770,7 @@
         "@types/chai-arrays": "^1.0.2",
         "@types/chai-as-promised": "^7.1.0",
         "@types/copy-webpack-plugin": "^4.4.2",
+        "@types/dedent": "^0.7.0",
         "@types/del": "^3.0.0",
         "@types/diff-match-patch": "^1.0.32",
         "@types/download": "^6.2.2",
@@ -2831,6 +2833,7 @@
         "css-loader": "^1.0.1",
         "cucumber-html-reporter": "^4.0.5",
         "decache": "^4.4.0",
+        "dedent": "^0.7.0",
         "del": "^3.0.0",
         "download": "^7.0.0",
         "electron-download": "^4.1.1",

pvsc.code-workspace

@@ -5,6 +5,9 @@
         },
         {
             "path": "uitests"
+        },
+        {
+            "path": "pythonFiles"
         }
     ],
     "settings": {

pythonFiles/.env

@@ -0,0 +1 @@
+PYTHONPATH=./lib/python

pythonFiles/.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "files.exclude": {
+        "**/__pycache__/**": true,
+        "**/**/*.pyc": true
+    },
+    "python.formatting.provider": "black"
+}

pythonFiles/datascience/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.

pythonFiles/datascience/daemon/README.md

@@ -0,0 +1,25 @@
+# Sample usage in node.js
+
+```javascript
+const cp  = require('child_process');
+const rpc = require('vscode-jsonrpc');
+const env = {
+    PYTHONUNBUFFERED: '1',
+    PYTHONPATH: '<extension dir>/pythonFiles:<extension dir>/pythonFiles/lib/python'
+}
+const childProcess = cp.spawn('<fully qualifieid python path>', ['-m', 'datascience.daemon', '-v', '--log-file=log.log'], {env});
+const connection = rpc.createMessageConnection(new rpc.StreamMessageReader(childProcess.stdout),new rpc.StreamMessageWriter(childProcess.stdin));
+
+connection.onClose(() => console.error('Closed'));
+connection.onError(ex => console.error(ex));
+connection.onDispose(() => console.error('disposed'));
+connection.onNotification((e, data) => console.log(`Notification from daemon, such as stdout/stderr, ${JSON.stringify(data)}`);
+connection.onUnhandledNotification(e => console.error(e));
+
+// Start
+connection.listen();
+
+const pingRequest = new rpc.RequestType('ping');
+connection.sendRequest(pingRequest, {data: 'one₹😄'})
+	.then(response => console.log(`Pong received ${JSON.stringify(response)}`),	ex => console.error(ex));
+```

pythonFiles/datascience/daemon/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.

pythonFiles/datascience/daemon/__main__.py

@@ -0,0 +1,101 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import argparse
+import importlib
+import json
+import logging
+import logging.config
+import sys
+
+
+log = logging.getLogger(__name__)
+
+LOG_FORMAT = "%(asctime)s UTC - %(levelname)s - %(name)s - %(message)s"
+
+
+def add_arguments(parser):
+    parser.description = "Daemon"
+
+    parser.add_argument(
+        "--daemon-module",
+        default="datascience.daemon.daemon_python",
+        help="Daemon Module",
+    )
+
+    log_group = parser.add_mutually_exclusive_group()
+    log_group.add_argument(
+        "--log-config", help="Path to a JSON file containing Python logging config."
+    )
+    log_group.add_argument(
+        "--log-file",
+        help="Redirect logs to the given file instead of writing to stderr."
+        "Has no effect if used with --log-config.",
+    )
+
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="Increase verbosity of log output, overrides log config file",
+    )
+
+
+def _configure_logger(verbose=0, log_config=None, log_file=None):
+    root_logger = logging.root
+
+    if log_config:
+        with open(log_config, "r") as f:
+            logging.config.dictConfig(json.load(f))
+    else:
+        formatter = logging.Formatter(LOG_FORMAT)
+        if log_file:
+            log_handler = logging.handlers.RotatingFileHandler(
+                log_file,
+                mode="a",
+                maxBytes=50 * 1024 * 1024,
+                backupCount=10,
+                encoding=None,
+                delay=0,
+            )
+        else:
+            log_handler = logging.StreamHandler()
+        log_handler.setFormatter(formatter)
+        root_logger.addHandler(log_handler)
+
+    if verbose == 0:
+        level = logging.WARNING
+    elif verbose == 1:
+        level = logging.INFO
+    elif verbose >= 2:
+        level = logging.DEBUG
+
+    root_logger.setLevel(level)
+
+
+def main():
+    """ Starts the daemon.
+    The daemon_module allows authors of modules to provide a custom daemon implementation.
+    E.g. we have a base implementation for standard python functionality,
+    and a custom daemon implementation for DS work (related to jupyter).
+    """
+    parser = argparse.ArgumentParser()
+    add_arguments(parser)
+    args = parser.parse_args()
+    _configure_logger(args.verbose, args.log_config, args.log_file)
+
+    log.info("Starting daemon from %s.PythonDaemon", args.daemon_module)
+    try:
+        daemon_module = importlib.import_module(args.daemon_module)
+        daemon_cls = daemon_module.PythonDaemon
+        daemon_cls.start_daemon()
+    except Exception:
+        import traceback
+
+        log.error(traceback.format_exc())
+        raise Exception("Failed to start daemon")
+
+
+if __name__ == "__main__":
+    main()

pythonFiles/datascience/daemon/daemon_output.py

@@ -0,0 +1,131 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+# Code borrowed from `pydevd` (https://github.com/microsoft/ptvsd/blob/608803cb99b450aedecc45167a7339b9b7b93b75/src/ptvsd/_vendored/pydevd/pydevd.py)
+
+import os
+import sys
+import logging
+from threading import Lock
+
+log = logging.getLogger(__name__)
+
+
+class IORedirector:
+    """
+    This class works to wrap a stream (stdout/stderr) with an additional redirect.
+    """
+
+    def __init__(self, name, original, new_redirect, wrap_buffer=False):
+        """
+        :param stream original:
+            The stream to be wrapped (usually stdout/stderr, but could be None).
+
+        :param stream new_redirect:
+
+        :param bool wrap_buffer:
+            Whether to create a buffer attribute (needed to mimick python 3 s
+            tdout/stderr which has a buffer to write binary data).
+        """
+        self._name = name
+        self._lock = Lock()
+        self._writing = False
+        self._redirect_to = (new_redirect,)
+        if wrap_buffer and hasattr(original, "buffer"):
+            self.buffer = IORedirector(
+                name, original.buffer, new_redirect.buffer, False
+            )
+
+    def write(self, s):
+        # Note that writing to the original stream may fail for some reasons
+        # (such as trying to write something that's not a string or having it closed).
+        with self._lock:
+            if self._writing:
+                return
+            self._writing = True
+            try:
+                for r in self._redirect_to:
+                    if hasattr(r, "write"):
+                        r.write(s)
+            finally:
+                self._writing = False
+
+    def isatty(self):
+        for r in self._redirect_to:
+            if hasattr(r, "isatty"):
+                return r.isatty()
+        return False
+
+    def flush(self):
+        for r in self._redirect_to:
+            if hasattr(r, "flush"):
+                r.flush()
+
+    def __getattr__(self, name):
+        log.info("getting attr for %s: %s", self._name, name)
+        for r in self._redirect_to:
+            if hasattr(r, name):
+                return getattr(r, name)
+        raise AttributeError(name)
+
+
+class CustomWriter(object):
+    def __init__(self, name, wrap_stream, wrap_buffer, on_write=None):
+        """
+        :param wrap_stream:
+            Either sys.stdout or sys.stderr.
+
+        :param bool wrap_buffer:
+            If True the buffer attribute (which wraps writing bytes) should be
+            wrapped.
+
+        :param callable(str) on_write:
+            Call back with the string that has been written.
+        """
+        self._name = name
+        encoding = getattr(wrap_stream, "encoding", None)
+        if not encoding:
+            encoding = os.environ.get("PYTHONIOENCODING", "utf-8")
+        self.encoding = encoding
+        if wrap_buffer:
+            self.buffer = CustomWriter(
+                name, wrap_stream, wrap_buffer=False, on_write=on_write
+            )
+        self._on_write = on_write
+
+    def flush(self):
+        pass  # no-op here
+
+    def write(self, s):
+        if s:
+            # Need s in str
+            if isinstance(s, bytes):
+                s = s.decode(self.encoding, errors="replace")
+            log.info("write to %s: %s", self._name, s)
+            if self._on_write is not None:
+                self._on_write(s)
+
+
+_stdin = sys.stdin.buffer
+_stdout = sys.stdout.buffer
+
+
+def get_io_buffers():
+    return _stdin, _stdout
+
+
+def redirect_output(stdout_handler, stderr_handler):
+    log.info("Redirect stdout/stderr")
+
+    sys._vsc_out_buffer_ = CustomWriter("stdout", sys.stdout, True, stdout_handler)
+    sys.stdout_original = sys.stdout
+    _stdout_redirector = sys.stdout = IORedirector(
+        "stdout", sys.stdout, sys._vsc_out_buffer_, True
+    )
+
+    sys._vsc_err_buffer_ = CustomWriter("stderr", sys.stderr, True, stderr_handler)
+    sys.stderr_original = sys.stderr
+    _stderr_redirector = sys.stderr = IORedirector(
+        "stderr", sys.stderr, sys._vsc_err_buffer_, True
+    )
+