Ship 2 versions of PTVSD in the extension (microsoft#7115)

* Install old PTVSD in CI & update ptvsd_launcher
* Update ptvsd in requirements.txt
* Parse requirements to install new ptvsd
* Pass args properly
* Add comment
* Fix typo in ptvsd launcher
* Add comment
* Move install_ptvsd to pythonFiles folder
* Move everything to a main function
* Update comment

CONTRIBUTING.md

@@ -36,7 +36,7 @@ cd vscode-python
 npm ci
 python3 -m venv .venv
 # Activate the virtual environment as appropriate for your shell.
-python3 -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt
+npx gulp installPythonLibs
 # Optionally Update `launch.json` to set a value for the environment variable `CI_PYTHON_PATH` pointing to the fully qualified path of the above interpreter.
 ```
 You may see warnings that ```The engine "vscode" appears to be invalid.```, you can ignore these.

build/ci/templates/build_compile_steps.yml

@@ -63,6 +63,7 @@ steps:
     - bash: |
           python -m pip install -U pip
           python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt
+          python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python/old_ptvsd --no-cache-dir --implementation py --no-deps --upgrade 'ptvsd==4.3.2'
       failOnStderr: true
       displayName: "pip install requirements"
       condition: and(succeeded(), eq(variables['build'], 'true'))
@@ -71,7 +72,8 @@ steps:
       displayName: "Install PTVSD wheels"
       inputs:
           scriptSource: "filePath"
-          scriptPath: "./build/ci/install_ptvsd.py"
+          scriptPath: "./pythonFiles/install_ptvsd.py"
+          arguments: "--ci"
           failOnStderr: true
       condition: and(succeeded(), eq(variables['build'], 'true'))
 

build/ci/templates/test_phases.yml

@@ -179,6 +179,7 @@ steps:
       python -m pip install -U pip
       python -m pip install --upgrade -r build/test-requirements.txt
       python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt
+      python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python/old_ptvsd --no-cache-dir --implementation py --no-deps --upgrade 'ptvsd==4.3.2'
     displayName: 'pip install system test requirements'
     condition: and(succeeded(), eq(variables['NeedsPythonTestReqs'], 'true'))
 

gulpfile.js

@@ -231,7 +231,7 @@ gulp.task('verifyBundle', async () => {
 gulp.task('prePublishBundle', gulp.series('webpack', 'renameSourceMaps'));
 gulp.task('prePublishNonBundle', gulp.series('checkNativeDependencies', 'check-datascience-dependencies', 'compile', 'compile-webviews'));
 
-gulp.task('installPythonLibs', async () => {
+gulp.task('installPythonRequirements', async () => {
     const requirements = fs
         .readFileSync(path.join(__dirname, 'requirements.txt'), 'utf8')
         .split('\n')
@@ -254,6 +254,41 @@ gulp.task('installPythonLibs', async () => {
     );
 });
 
+// Install new PTVSD wheels for python 3.7
+// See https://github.com/microsoft/vscode-python/issues/7136
+gulp.task('installPtvsdWheels', async () => {
+    const args = ['./pythonFiles/install_ptvsd.py']
+    const success = await spawnAsync(process.env.CI_PYTHON_PATH || 'python3', args)
+        .then(() => true)
+        .catch(ex => {
+            console.error("Failed to install new PTVSD wheels using 'python3'", ex);
+            return false;
+        });
+    if (!success) {
+        console.info("Failed to install new PTVSD wheels using 'python3', attempting to install using 'python'");
+        await spawnAsync('python', args.concat(requirement)).catch(ex => console.error("Failed to install PTVSD 5.0 wheels using 'python'", ex));
+    }
+});
+
+// Install the last stable version of old PTVSD (which includes a middle layer adapter and requires ptvsd_launcher.py)
+// until all users have migrated to the new debug adapter + new PTVSD (specified in requirements.txt)
+// See https://github.com/microsoft/vscode-python/issues/7136
+gulp.task('installOldPtvsd', async () => {
+    const args = ['-m', 'pip', '--disable-pip-version-check', 'install', '-t', './pythonFiles/lib/python/old_ptvsd', '--no-cache-dir', '--implementation', 'py', '--no-deps', '--upgrade', 'ptvsd==4.3.2']
+    const success = await spawnAsync(process.env.CI_PYTHON_PATH || 'python3', args)
+        .then(() => true)
+        .catch(ex => {
+            console.error("Failed to install PTVSD using 'python3'", ex);
+            return false;
+        });
+    if (!success) {
+        console.info("Failed to install PTVSD using 'python3', attempting to install using 'python'");
+        await spawnAsync('python', args.concat(requirement)).catch(ex => console.error("Failed to install PTVSD using 'python'", ex));
+    }
+});
+
+gulp.task('installPythonLibs', gulp.series('installPythonRequirements', 'installOldPtvsd', 'installPtvsdWheels'));
+
 function uploadExtension(uploadBlobName) {
     const azure = require('gulp-azure-storage');
     const rename = require('gulp-rename');

build/ci/install_ptvsd.py → pythonFiles/install_ptvsd.py

@@ -3,15 +3,27 @@
 from zipfile import ZipFile
 import json
 import urllib.request
+import sys
 
-ROOT_DIRNAME = path.dirname(path.dirname(path.dirname(path.abspath(__file__))))
+ROOT_DIRNAME = path.dirname(path.dirname(path.abspath(__file__)))
+REQUIREMENTS_PATH = path.join(ROOT_DIRNAME, "requirements.txt")
 PYTHONFILES_PATH = path.join(ROOT_DIRNAME, "pythonFiles", "lib", "python")
 PYPI_PTVSD_URL = "https://pypi.org/pypi/ptvsd/json"
 
-if __name__ == "__main__":
-    # Remove this when the version of PTVSD in requirements.txt gets updated.
-    # (and add code leveraging the packaging module to parse requirements.txt) in #7002
-    ptvsd_version = "5.0.0a3"
+
+def install_ptvsd():
+    # If we are in CI use the packaging module installed in PYTHONFILES_PATH.
+    if len(sys.argv) == 2 and sys.argv[1] == "--ci":
+        sys.path.insert(0, PYTHONFILES_PATH)
+    from packaging.requirements import Requirement
+
+    with open(REQUIREMENTS_PATH, "r", encoding="utf-8") as requirements:
+        for line in requirements:
+            package_requirement = Requirement(line)
+            if package_requirement.name != "ptvsd":
+                continue
+            requirement_specifier = package_requirement.specifier
+            ptvsd_version = next(requirement_specifier.__iter__()).version
 
     # Response format: https://warehouse.readthedocs.io/api-reference/json/#project
     with urllib.request.urlopen(PYPI_PTVSD_URL) as response:
@@ -23,7 +35,7 @@
         # Download only if it's a 3.7 wheel.
         if not wheel_info["python_version"].endswith(("37", "3.7")):
             continue
-        filename = wheel_info["filename"].rpartition(".")[0]  # Trim the file extension
+        filename = wheel_info["filename"].rpartition(".")[0]  # Trim the file extension.
         ptvsd_path = path.join(PYTHONFILES_PATH, filename)
 
         with urllib.request.urlopen(wheel_info["url"]) as wheel_response:
@@ -38,3 +50,7 @@
                 # Flatten the folder structure.
                 zip_info.filename = zip_info.filename.split(prefix)[-1]
                 wheel.extract(zip_info, ptvsd_path)
+
+
+if __name__ == "__main__":
+    install_ptvsd()

pythonFiles/ptvsd_launcher.py

@@ -12,11 +12,11 @@
 
 # Load the debugger package
 try:
-    ptvs_lib_path = os.path.join(os.path.dirname(__file__), 'lib', 'python')
+    ptvsd_lib_path = os.path.join(os.path.dirname(__file__), 'lib', 'python', 'old_ptvsd')
     if useCustomPtvsd:
-        sys.path.append(ptvs_lib_path)
+        sys.path.append(ptvsd_lib_path)
     else:
-        sys.path.insert(0, ptvs_lib_path)
+        sys.path.insert(0, ptvsd_lib_path)
     try:
         import ptvsd
         from ptvsd.__main__ import main
@@ -37,7 +37,7 @@
         input()
     sys.exit(1)
 finally:
-    if ptvs_lib_path:
-        sys.path.remove(ptvs_lib_path)
+    if ptvsd_lib_path:
+        sys.path.remove(ptvsd_lib_path)
 
 main(ptvsdArgs)

requirements.txt

@@ -1,4 +1,7 @@
 jedi==0.15.1
 parso==0.5.1
 isort==4.3.21
-ptvsd==4.3.2
+ptvsd==5.0.0a3
+pyparsing==2.4.0
+six==1.12.0
+packaging==19.1