Pin python packages used in extension and remove them repo (microsoft…

…#3513)

* Move python dependencies into libs
* Delete parso, isort and jedi
* Deleted experimental folder from pythonFiles
* deleted old ptvsd folder
* rename requirements file
* Updated travis
* Install deps using gulp
* Use CI python
* Fix path to ptvsd
* install distro
* upgrade in place

.gitignore

@@ -16,6 +16,7 @@ coverage/
 **/.venv*/
 precommit.hook
 pythonFiles/experimental/ptvsd/**
+pythonFiles/lib/**
 debug_coverage*/**
 languageServer/**
 languageServer.*/**

.travis.yml

@@ -59,8 +59,8 @@ install:
   - npx gulp prePublishNonBundle
   - npx gulp hygiene-modified
   - python -m pip install --upgrade pip
-  - python -m pip install --upgrade -r requirements.txt
-  - python -m pip install -t ./pythonFiles/experimental/ptvsd --no-cache-dir --implementation py --no-deps ptvsd
+  - python -m pip install --upgrade -r ./build/test-requirements.txt
+  - npx gulp installPythonLibs
 
 script:
   - if [ $UNIT_TEST == "true" ]; then
@@ -73,7 +73,7 @@ script:
     fi
   - npm run debugger-coverage
   - if [ $FUNCTIONAL_TEST == "true" ]; then
-        python -m pip install --upgrade -r functionalTestRequirements.txt;
+        python -m pip install --upgrade -r ./build/functional-test-requirements.txt;
         npm run test:functional;
     fi
   - if [ $SINGLE_WORKSPACE_TEST == "true" ]; then

.vscodeignore

@@ -15,7 +15,6 @@ CODING_STANDARDS.md
 CONTRIBUTING.md
 CONTRIBUTING - LANGUAGE SERVER.md
 coverconfig.json
-functionalTestRequirements.txt
 gulpfile.js
 package.datascience-ui.dependencies.json
 package-lock.json
@@ -56,6 +55,9 @@ out/test/**
 out/testMultiRootWkspc/**
 precommit.hook
 pythonFiles/**/*.pyc
+pythonFiles/lib/**/*.dist-info/**
+pythonFiles/lib/**/*.egg-info/**
+pythonFiles/lib/python/bin/**
 requirements.txt
 scripts/**
 src/**

CONTRIBUTING.md

@@ -37,8 +37,7 @@ cd vscode-python
 npm install
 python3 -m venv .venv
 # Activate the virtual environment as appropriate for your shell.
-python -m pip install -r requirements.txt
-python -m pip install -t pythonFiles/experimental/ptvsd ptvsd
+python3 -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt
 # Specifying the virtual environment simply varies between shells.
 export CI_PYTHON_PATH=`pwd`/.venv/bin/python
 ```

build/ci/templates/compile-and-validate.yml

@@ -156,19 +156,19 @@ jobs:
 
 
     - task: CmdLine@1
-      displayName: 'pip install requirements'
+      displayName: 'pip install test requirements'
       inputs:
         filename: python
 
-        arguments: '-m pip install --upgrade -r ./requirements.txt'
+        arguments: '-m pip install --upgrade -r ./build/test-requirements.txt'
 
 
     - task: CmdLine@1
-      displayName: 'pip install ptvsd'
+      displayName: 'pip install python packages'
       inputs:
         filename: python
 
-        arguments: '-m pip install -t ./pythonFiles/experimental/ptvsd --no-cache-dir --implementation py --no-deps ptvsd'
+        arguments: '-m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt'
 
 
     - task: ArchiveFiles@2

build/ci/templates/test-phase.yml

@@ -149,15 +149,15 @@ steps:
     inputs:
       filename: python
 
-      arguments: '-m pip install --upgrade -r ./requirements.txt'
+      arguments: '-m pip install --upgrade -r ./build/test-requirements.txt'
 
 
   - task: CmdLine@1
     displayName: 'pip install ptvsd'
     inputs:
       filename: python
 
-      arguments: '-m pip install -t ./pythonFiles/experimental/ptvsd --no-cache-dir --implementation py --no-deps ptvsd'
+      arguments: '-m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt'
 
 
   - task: Npm@1

build/test-requirements.txt

@@ -0,0 +1,16 @@
+# Install flake8 first, as both flake8 and autopep8 require pycodestyle,
+# but flake8 has a tighter pinning.
+flake8
+autopep8
+bandit
+black ; python_version>='3.6'
+yapf
+pylint
+pep8
+prospector
+pydocstyle
+nose
+pytest==3.6.3
+rope
+flask
+django

gulpfile.js

@@ -22,7 +22,7 @@ const path = require('path');
 const jeditor = require("gulp-json-editor");
 const del = require('del');
 const sourcemaps = require('gulp-sourcemaps');
-const fs = require('fs');
+const fs = require('fs-extra');
 const fsExtra = require('fs-extra');
 const remapIstanbul = require('remap-istanbul');
 const istanbul = require('istanbul');
@@ -107,8 +107,6 @@ gulp.task('clean:out', () => del(['out']));
 
 gulp.task('clean', gulp.parallel('output:clean', 'cover:clean', 'clean:vsix'));
 
-gulp.task('clean:ptvsd', () => del(['coverage', 'pythonFiles/experimental/ptvsd/*']));
-
 gulp.task('checkNativeDependencies', (done) => {
     if (hasNativeDependencies()) {
         throw new Error('Native dependencies deteced');
@@ -202,11 +200,32 @@ gulp.task('webpack', async () => {
 gulp.task('prePublishBundle', gulp.series('checkNativeDependencies', 'check-datascience-dependencies', 'compile', 'clean:cleanExceptTests', 'webpack'));
 gulp.task('prePublishNonBundle', gulp.series('checkNativeDependencies', 'check-datascience-dependencies', 'compile', 'compile-webviews'));
 
+const installPythonLibArgs = ['-m', 'pip', '--disable-pip-version-check', 'install',
+    '-t', './pythonFiles/lib/python', '--no-cache-dir', '--implementation', 'py', '--no-deps',
+    '--upgrade', '-r', 'requirements.txt'];
+gulp.task('installPythonLibs', async () => {
+    const requirements = fs.readFileSync(path.join(__dirname, 'requirements.txt'), 'utf8').split('\n').map(item => item.trim()).filter(item => item.length > 0);
+    const args = ['-m', 'pip', '--disable-pip-version-check', 'install', '-t', './pythonFiles/lib/python', '--no-cache-dir', '--implementation', 'py', '--no-deps', '--upgrade'];
+    await Promise.all(requirements.map(async requirement => {
+        const success = await spawnAsync(process.env.CI_PYTHON_PATH || 'python3', args.concat(requirement))
+            .then(() => true)
+            .catch(ex => {
+                console.error('Failed to install Python Libs using \'python3\'', ex);
+                return false
+            });
+        if (!success) {
+            console.info('Failed to install Python Libs using \'python3\', attempting to install using \'python\'');
+            await spawnAsync('python', args.concat(requirement))
+                .catch(ex => console.error('Failed to install Python Libs using \'python\'', ex));
+        }
+    }));
+});
+
 function spawnAsync(command, args) {
     return new Promise((resolve, reject) => {
         const proc = spawn(command, args, { cwd: __dirname });
         proc.stdout.on('data', data => {
-            // Log output on CI.
+            // Log output on CI (else travis times out when there's not output).
             if (isCI) {
                 console.log(data.toString());
             }

package.json

@@ -1708,7 +1708,6 @@
         "lint-staged": "node gulpfile.js",
         "lint": "tslint src/**/*.ts -t verbose",
         "clean": "gulp clean",
-        "clean:ptvsd": "gulp clean:ptvsd",
         "cover:enable": "gulp cover:enable",
         "debugger-coverage": "gulp debugger-coverage",
         "cover:inlinesource": "gulp inlinesource"

pvsc-dev-ext.py

@@ -63,10 +63,26 @@ def build_typescript(checkout):
     run_command(["node", os.fspath(tsc_path), "-p", os.fspath(checkout)], cwd=checkout)
 
 
-def install_ptvsd(checkout):
-    """Install ptvsd from PyPI."""
-    ptvsd_path = checkout / "pythonFiles" / "experimental" / "ptvsd"
-    cmd = [sys.executable, "-m", "pip", "-q", "--disable-pip-version-check", "install", "--target", os.fspath(ptvsd_path), "--upgrade", "ptvsd"]
+def install_libs(checkout):
+    """Install libs from PyPI."""
+    libs_path = checkout / "pythonFiles" / "libs" / "python"
+    cmd = [
+        sys.executable,
+        "-m",
+        "pip",
+        "-q",
+        "--disable-pip-version-check",
+        "install",
+        "--target",
+        os.fspath(libs_path),
+        "--no-cache-dir",
+        "--implementation",
+        "py",
+        "--no-deps",
+        "--upgrade",
+        "-r",
+        "requirements.txt",
+    ]
     run_command(cmd)
 
 
@@ -83,8 +99,8 @@ def build(checkout):
     install_npm_dependencies(checkout)
     print("Building TypeScript files ...")
     build_typescript(checkout)
-    print("Installing ptvsd ...")
-    install_ptvsd(checkout)
+    print("Installing libs ...")
+    install_libs(checkout)
 
 
 def setup(install_type):