From 91834053084ee8c6aaad1844590bf4df5070d086 Mon Sep 17 00:00:00 2001 From: Kim-Adeline Miguel <51720070+kimadeline@users.noreply.github.com> Date: Thu, 29 Aug 2019 15:39:11 -0700 Subject: [PATCH] Ship 2 versions of PTVSD in the extension (#7115) * Install old PTVSD in CI & update ptvsd_launcher * Update ptvsd in requirements.txt * Parse requirements to install new ptvsd * Pass args properly * Add comment * Fix typo in ptvsd launcher * Add comment * Move install_ptvsd to pythonFiles folder * Move everything to a main function * Update comment --- CONTRIBUTING.md | 2 +- build/ci/templates/build_compile_steps.yml | 4 ++- build/ci/templates/test_phases.yml | 1 + gulpfile.js | 37 +++++++++++++++++++++- {build/ci => pythonFiles}/install_ptvsd.py | 28 ++++++++++++---- pythonFiles/ptvsd_launcher.py | 10 +++--- requirements.txt | 5 ++- 7 files changed, 72 insertions(+), 15 deletions(-) rename {build/ci => pythonFiles}/install_ptvsd.py (65%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5cfc0ee77616..2b62301a0fc7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,7 +36,7 @@ cd vscode-python npm ci python3 -m venv .venv # Activate the virtual environment as appropriate for your shell. -python3 -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt +npx gulp installPythonLibs # Optionally Update `launch.json` to set a value for the environment variable `CI_PYTHON_PATH` pointing to the fully qualified path of the above interpreter. ``` You may see warnings that ```The engine "vscode" appears to be invalid.```, you can ignore these. diff --git a/build/ci/templates/build_compile_steps.yml b/build/ci/templates/build_compile_steps.yml index ee5952d10cc6..337851878142 100644 --- a/build/ci/templates/build_compile_steps.yml +++ b/build/ci/templates/build_compile_steps.yml @@ -63,6 +63,7 @@ steps: - bash: | python -m pip install -U pip python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt + python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python/old_ptvsd --no-cache-dir --implementation py --no-deps --upgrade 'ptvsd==4.3.2' failOnStderr: true displayName: "pip install requirements" condition: and(succeeded(), eq(variables['build'], 'true')) @@ -71,7 +72,8 @@ steps: displayName: "Install PTVSD wheels" inputs: scriptSource: "filePath" - scriptPath: "./build/ci/install_ptvsd.py" + scriptPath: "./pythonFiles/install_ptvsd.py" + arguments: "--ci" failOnStderr: true condition: and(succeeded(), eq(variables['build'], 'true')) diff --git a/build/ci/templates/test_phases.yml b/build/ci/templates/test_phases.yml index 75f772f5b462..257d1ccd400d 100644 --- a/build/ci/templates/test_phases.yml +++ b/build/ci/templates/test_phases.yml @@ -179,6 +179,7 @@ steps: python -m pip install -U pip python -m pip install --upgrade -r build/test-requirements.txt python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt + python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python/old_ptvsd --no-cache-dir --implementation py --no-deps --upgrade 'ptvsd==4.3.2' displayName: 'pip install system test requirements' condition: and(succeeded(), eq(variables['NeedsPythonTestReqs'], 'true')) diff --git a/gulpfile.js b/gulpfile.js index 5b5e88ff792d..b57071c8b46f 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -231,7 +231,7 @@ gulp.task('verifyBundle', async () => { gulp.task('prePublishBundle', gulp.series('webpack', 'renameSourceMaps')); gulp.task('prePublishNonBundle', gulp.series('checkNativeDependencies', 'check-datascience-dependencies', 'compile', 'compile-webviews')); -gulp.task('installPythonLibs', async () => { +gulp.task('installPythonRequirements', async () => { const requirements = fs .readFileSync(path.join(__dirname, 'requirements.txt'), 'utf8') .split('\n') @@ -254,6 +254,41 @@ gulp.task('installPythonLibs', async () => { ); }); +// Install new PTVSD wheels for python 3.7 +// See https://github.com/microsoft/vscode-python/issues/7136 +gulp.task('installPtvsdWheels', async () => { + const args = ['./pythonFiles/install_ptvsd.py'] + const success = await spawnAsync(process.env.CI_PYTHON_PATH || 'python3', args) + .then(() => true) + .catch(ex => { + console.error("Failed to install new PTVSD wheels using 'python3'", ex); + return false; + }); + if (!success) { + console.info("Failed to install new PTVSD wheels using 'python3', attempting to install using 'python'"); + await spawnAsync('python', args.concat(requirement)).catch(ex => console.error("Failed to install PTVSD 5.0 wheels using 'python'", ex)); + } +}); + +// Install the last stable version of old PTVSD (which includes a middle layer adapter and requires ptvsd_launcher.py) +// until all users have migrated to the new debug adapter + new PTVSD (specified in requirements.txt) +// See https://github.com/microsoft/vscode-python/issues/7136 +gulp.task('installOldPtvsd', async () => { + const args = ['-m', 'pip', '--disable-pip-version-check', 'install', '-t', './pythonFiles/lib/python/old_ptvsd', '--no-cache-dir', '--implementation', 'py', '--no-deps', '--upgrade', 'ptvsd==4.3.2'] + const success = await spawnAsync(process.env.CI_PYTHON_PATH || 'python3', args) + .then(() => true) + .catch(ex => { + console.error("Failed to install PTVSD using 'python3'", ex); + return false; + }); + if (!success) { + console.info("Failed to install PTVSD using 'python3', attempting to install using 'python'"); + await spawnAsync('python', args.concat(requirement)).catch(ex => console.error("Failed to install PTVSD using 'python'", ex)); + } +}); + +gulp.task('installPythonLibs', gulp.series('installPythonRequirements', 'installOldPtvsd', 'installPtvsdWheels')); + function uploadExtension(uploadBlobName) { const azure = require('gulp-azure-storage'); const rename = require('gulp-rename'); diff --git a/build/ci/install_ptvsd.py b/pythonFiles/install_ptvsd.py similarity index 65% rename from build/ci/install_ptvsd.py rename to pythonFiles/install_ptvsd.py index b3b777e6e69f..7065293f6018 100644 --- a/build/ci/install_ptvsd.py +++ b/pythonFiles/install_ptvsd.py @@ -3,15 +3,27 @@ from zipfile import ZipFile import json import urllib.request +import sys -ROOT_DIRNAME = path.dirname(path.dirname(path.dirname(path.abspath(__file__)))) +ROOT_DIRNAME = path.dirname(path.dirname(path.abspath(__file__))) +REQUIREMENTS_PATH = path.join(ROOT_DIRNAME, "requirements.txt") PYTHONFILES_PATH = path.join(ROOT_DIRNAME, "pythonFiles", "lib", "python") PYPI_PTVSD_URL = "https://pypi.org/pypi/ptvsd/json" -if __name__ == "__main__": - # Remove this when the version of PTVSD in requirements.txt gets updated. - # (and add code leveraging the packaging module to parse requirements.txt) in #7002 - ptvsd_version = "5.0.0a3" + +def install_ptvsd(): + # If we are in CI use the packaging module installed in PYTHONFILES_PATH. + if len(sys.argv) == 2 and sys.argv[1] == "--ci": + sys.path.insert(0, PYTHONFILES_PATH) + from packaging.requirements import Requirement + + with open(REQUIREMENTS_PATH, "r", encoding="utf-8") as requirements: + for line in requirements: + package_requirement = Requirement(line) + if package_requirement.name != "ptvsd": + continue + requirement_specifier = package_requirement.specifier + ptvsd_version = next(requirement_specifier.__iter__()).version # Response format: https://warehouse.readthedocs.io/api-reference/json/#project with urllib.request.urlopen(PYPI_PTVSD_URL) as response: @@ -23,7 +35,7 @@ # Download only if it's a 3.7 wheel. if not wheel_info["python_version"].endswith(("37", "3.7")): continue - filename = wheel_info["filename"].rpartition(".")[0] # Trim the file extension + filename = wheel_info["filename"].rpartition(".")[0] # Trim the file extension. ptvsd_path = path.join(PYTHONFILES_PATH, filename) with urllib.request.urlopen(wheel_info["url"]) as wheel_response: @@ -38,3 +50,7 @@ # Flatten the folder structure. zip_info.filename = zip_info.filename.split(prefix)[-1] wheel.extract(zip_info, ptvsd_path) + + +if __name__ == "__main__": + install_ptvsd() diff --git a/pythonFiles/ptvsd_launcher.py b/pythonFiles/ptvsd_launcher.py index 14661c43fc18..98bdbe614ee2 100644 --- a/pythonFiles/ptvsd_launcher.py +++ b/pythonFiles/ptvsd_launcher.py @@ -12,11 +12,11 @@ # Load the debugger package try: - ptvs_lib_path = os.path.join(os.path.dirname(__file__), 'lib', 'python') + ptvsd_lib_path = os.path.join(os.path.dirname(__file__), 'lib', 'python', 'old_ptvsd') if useCustomPtvsd: - sys.path.append(ptvs_lib_path) + sys.path.append(ptvsd_lib_path) else: - sys.path.insert(0, ptvs_lib_path) + sys.path.insert(0, ptvsd_lib_path) try: import ptvsd from ptvsd.__main__ import main @@ -37,7 +37,7 @@ input() sys.exit(1) finally: - if ptvs_lib_path: - sys.path.remove(ptvs_lib_path) + if ptvsd_lib_path: + sys.path.remove(ptvsd_lib_path) main(ptvsdArgs) diff --git a/requirements.txt b/requirements.txt index 5fe6d7d3d310..aefcfc2c437f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,7 @@ jedi==0.15.1 parso==0.5.1 isort==4.3.21 -ptvsd==4.3.2 +ptvsd==5.0.0a3 +pyparsing==2.4.0 +six==1.12.0 +packaging==19.1