Use python debug adapter when in experiment (microsoft#8119)

* Use DebugAdapterServer for attaches
* Add mock to test DebugAdapterServer
* Custom debugger packaging and path fixes
* Minor cleanups
* Fix DebugAdapterDescriptorFactory tests
* Fix ptvsd install path
* Linter fixes
* Fix install ptvsd test
* Address comments
* Remove python packages that are no longer needed
* Cleanup distribution.toml

pythonFiles/install_ptvsd.py

@@ -1,63 +1,57 @@
-from io import BytesIO
-from os import path
-from zipfile import ZipFile
+import io
 import json
-import urllib.request
-import sys
+import os
+import urllib.request as url_lib
+import zipfile
 
-ROOT = path.dirname(path.dirname(path.abspath(__file__)))
-REQUIREMENTS = path.join(ROOT, "requirements.txt")
-PYTHONFILES = path.join(ROOT, "pythonFiles", "lib", "python")
-PYPI_PTVSD_URL = "https://pypi.org/pypi/ptvsd/json"
 
+EXTENSION_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+DEBUGGER_DEST = os.path.join(EXTENSION_ROOT, "pythonFiles", "lib", "python")
+DEBUGGER_PACKAGE = "ptvsd"
+DEBUGGER_VERSION = "5.0.0a5"
+DEBUGGER_PYTHON_VERSIONS = ("cp37",)
 
-def install_ptvsd():
-    sys.path.insert(0, PYTHONFILES)
-    from packaging.requirements import Requirement
 
-    with open(REQUIREMENTS, "r", encoding="utf-8") as reqsfile:
-        for line in reqsfile:
-            pkgreq = Requirement(line)
-            if pkgreq.name == "ptvsd":
-                specs = pkgreq.specifier
-                version = next(iter(specs)).version
-                break
+def _contains(s, parts=()):
+    return any(p for p in parts if p in s)
 
-    try:
-        version
-    except NameError:
-        raise Exception("ptvsd requirement not found.")
 
+def _get_debugger_wheel_urls():
+    json_uri = "https://pypi.org/pypi/{0}/json".format(DEBUGGER_PACKAGE)
     # Response format: https://warehouse.readthedocs.io/api-reference/json/#project
-    with urllib.request.urlopen(PYPI_PTVSD_URL) as response:
+    # Release metadata format: https://github.com/pypa/interoperability-peps/blob/master/pep-0426-core-metadata.rst
+    with url_lib.urlopen(json_uri) as response:
         json_response = json.loads(response.read())
-    releases = json_response["releases"]
+        return list(
+            r["url"]
+            for r in json_response["releases"][DEBUGGER_VERSION]
+            if _contains(r["url"], DEBUGGER_PYTHON_VERSIONS)
+        )
 
-    # Release metadata format: https://github.com/pypa/interoperability-peps/blob/master/pep-0426-core-metadata.rst
-    for wheel_info in releases[version]:
-        # Download only if it's a 3.7 wheel.
-        if not wheel_info["python_version"].endswith(("37", "3.7")):
-            continue
-
-        # Trim the file extension and remove the ptvsd version from the folder name.
-        filename = wheel_info["filename"].rpartition(".")[0]
-        filename = filename.replace(f"{version}-", "")
-        ptvsd_path = path.join(PYTHONFILES, filename)
-
-        with urllib.request.urlopen(wheel_info["url"]) as wheel_response:
-            wheel_file = BytesIO(wheel_response.read())
+
+def _download_and_extract(root, url):
+    root = os.getcwd() if root is None or root == "." else root
+    prefix = os.path.join("ptvsd-{0}.data".format(DEBUGGER_VERSION), "purelib")
+    with url_lib.urlopen(url) as response:
         # Extract only the contents of the purelib subfolder (parent folder of ptvsd),
         # since ptvsd files rely on the presence of a 'ptvsd' folder.
-        prefix = path.join(f"ptvsd-{version}.data", "purelib")
-
-        with ZipFile(wheel_file, "r") as wheel:
+        with zipfile.ZipFile(io.BytesIO(response.read()), "r") as wheel:
             for zip_info in wheel.infolist():
-                # Normalize path for Windows, the wheel folder structure uses forward slashes.
-                normalized = path.normpath(zip_info.filename)
+                # Ignore dist info since we are merging multiple wheels
+                if ".dist-info" in zip_info.filename:
+                    continue
+                # Normalize path for Windows, the wheel folder structure
+                # uses forward slashes.
+                normalized = os.path.normpath(zip_info.filename)
                 # Flatten the folder structure.
                 zip_info.filename = normalized.split(prefix)[-1]
-                wheel.extract(zip_info, ptvsd_path)
+                wheel.extract(zip_info, root)
+
+
+def main(root):
+    for url in _get_debugger_wheel_urls():
+        _download_and_extract(root, url)
 
 
 if __name__ == "__main__":
-    install_ptvsd()
+    main(DEBUGGER_DEST)

pythonFiles/tests/debug_adapter/test_install_ptvsd.py

@@ -0,0 +1,35 @@
+import os
+import pytest
+import subprocess
+import sys
+
+
+def _check_binaries(dir_path):
+    expected_endswith = (
+        "win_amd64.pyd",
+        "win32.pyd",
+        "darwin.so",
+        "i386-linux-gnu.so",
+        "x86_64-linux-gnu.so",
+    )
+
+    binaries = list(p for p in os.listdir(dir_path) if p.endswith(expected_endswith))
+
+    assert len(binaries) == len(expected_endswith)
+
+@pytest.mark.skipif(
+    sys.version_info[:2] != (3, 7),
+    reason="PTVSD wheels shipped for Python 3.7 only",
+)
+def test_install_ptvsd(tmpdir):
+    import install_ptvsd
+    install_ptvsd.main(str(tmpdir))
+    dir_path = os.path.join(
+        str(tmpdir), "ptvsd", "_vendored", "pydevd", "_pydevd_bundle"
+    )
+    _check_binaries(dir_path)
+
+    dir_path = os.path.join(
+        str(tmpdir), "ptvsd", "_vendored", "pydevd", "_pydevd_frame_eval"
+    )
+    _check_binaries(dir_path)

requirements.txt

@@ -1,7 +1,4 @@
 jedi==0.15.1
 parso==0.5.1
 isort==4.3.21
-ptvsd==5.0.0a5
 pyparsing==2.4.0
-six==1.12.0
-packaging==19.2

src/client/api.ts

@@ -49,13 +49,13 @@ export function buildApi(ready: Promise<any>, experimentsManager: IExperimentsMa
                 const useNewDAPtvsd = experimentsManager.inExperiment(DebugAdapterExperiment.experiment) && (await debugFactory.useNewPtvsd(pythonSettings.pythonPath));
 
                 if (!useNewDAPtvsd) {
-                return new RemoteDebuggerExternalLauncherScriptProvider().getLauncherArgs({ host, port, waitUntilDebuggerAttaches });
-            }
+                    return new RemoteDebuggerExternalLauncherScriptProvider().getLauncherArgs({ host, port, waitUntilDebuggerAttaches });
+                }
 
                 // Same logic as in RemoteDebuggerExternalLauncherScriptProvider, but eventually launcherProvider.ts will be deleted.
                 const args = debugFactory.getRemotePtvsdArgs({ host, port, waitUntilDebuggerAttaches });
-                return [await debugFactory.getPtvsdPath(pythonSettings.pythonPath), ...args];
-        }
+                return [debugFactory.getPtvsdPath(), ...args];
+            }
         }
     };
 }