Merge pull request winpython#40 from stonebig/master

pre-install pip via get-pip or ensurepip

make.py

@@ -304,7 +304,7 @@ def get_package_fname(self, pattern):
             raise RuntimeError(
                 'Could not found required package matching %s' % pattern)
 
-    def install_package(self, pattern):
+    def install_package(self, pattern, install_options=None):
         """Install package matching pattern"""
         fname = self.get_package_fname(pattern)
         if fname not in [p.fname for p in self.installed_packages]:
@@ -313,8 +313,11 @@ def install_package(self, pattern):
                 self.distribution._print(pack, "Installing")
                 self.distribution._print_done()
             else:
-                self.distribution.install(pack,
-                                          install_options=self.install_options)
+                if install_options:
+                    self.distribution.install(pack, install_options)
+                else:
+                    self.distribution.install(pack, 
+                                         install_options=self.install_options)
             self.installed_packages.append(pack)
 
     def create_batch_script(self, name, contents):
@@ -362,8 +365,8 @@ def create_launcher(self, name, icon, command=None,
 
         # handle well Flavor with R included
         data += [('R_HOME', '$EXEDIR%s' % r'\tools\R'),
-                 ('JULIA_HOME','$EXEDIR\%s' % r'tools\Julia\bin'),
-                 ('JULIA', '$EXEDIR\%s' % r'tools\Julia\bin\julia.exe')]
+                 ('JULIA_HOME','$EXEDIR%s' % r'\tools\Julia\bin'),
+                 ('JULIA', '$EXEDIR%s' % r'\tools\Julia\bin\julia.exe')]
 
         if settingspath is not None:
             data += [('SETTINGSDIR', osp.dirname(settingspath)),
@@ -424,7 +427,8 @@ def _extract_python(self):
         os.mkdir(self.python_dir)
         utils.extract_msi(self.python_fname, targetdir=self.python_dir)
         os.remove(osp.join(self.python_dir, osp.basename(self.python_fname)))
-        os.mkdir(osp.join(self.python_dir, 'Scripts'))
+        if not os.path.exists(osp.join(self.python_dir, 'Scripts')):
+            os.mkdir(osp.join(self.python_dir, 'Scripts'))
         self._print_done()
 
     def _add_msvc_files(self):
@@ -469,18 +473,20 @@ def _install_required_packages(self):
         print("Installing required packages")
         self.install_package('pywin32-([0-9\.]*[a-z]*).%s-py%s.exe'
                              % (self.py_arch, self.python_version))
-        self.install_package('setuptools-([0-9\.]*[a-z]*[0-9]?).%s(-py%s)?.exe'
-                             % (self.py_arch, self.python_version))
         # Install First these two packages to support wheel format
-        self.install_package('pip-([0-9\.]*[a-z]*[0-9]?).%s(-py%s)?.exe'
-                             % (self.py_arch, self.python_version))
+        if self.python_version == '3.3':
+            self.install_package('get-pip-([0-9\.]*[a-z]*[0-9]?).%s(-py%s)?.exe'
+                             % (self.py_arch, self.python_version))           
+        #else:
+        self.install_package('%s-([0-9\.]*[a-z]*[0-9]?)(.*)(\.exe|\.whl)' %
+                              'setuptools' , install_options=['--upgrade','--no-deps'])
+        self.install_package('%s-([0-9\.]*[a-z]*[0-9]?)(.*)(\.exe|\.whl)' %
+                              'pip', install_options=['--upgrade','--no-deps'])
+
         self.install_package('wheel-([0-9\.]*[a-z]*[0-9]?).tar.gz')
         # six is needed early
         self.install_package('six-([0-9\.]*[a-z]*[0-9]?)-py2.py3-none-any.whl')
 
-        self.install_package(
-            'spyder(lib)?-([0-9\.]*[a-z]*[0-9]?).%s(-py%s)?.exe'
-            % (self.py_arch, self.python_version))
         # PyQt module is now like :PyQt4-4.10.4-gpl-Py3.4-Qt4.8.6-x32.exe
         self.install_package(
             'PyQt4-([0-9\.\-]*)-gpl-Py%s-Qt([0-9\.\-]*)%s.exe'
@@ -489,6 +495,10 @@ def _install_required_packages(self):
             'PyQwt-([0-9\.]*)-py%s-%s-([a-z0-9\.\-]*).exe'
             % (self.python_version, self.pyqt_arch))
 
+        self.install_package(
+            'spyder(lib)?-([0-9\.]*[a-z]*[0-9]?).%s(-py%s)?.exe'
+            % (self.py_arch, self.python_version))
+
         # Install 'main packages' first (was before Wheel idea, keep for now)
         for happy_few in['numpy-MKL', 'scipy', 'matplotlib', 'pandas']:
             # can be a wheel now
@@ -1126,14 +1136,14 @@ def make_all(build_number, release_level, pyver,
 
     #make_all(4, '', pyver='3.4', rootdir=r'D:\Winpython',
     #         verbose=False, archis=(32, ))
-    #make_all(4, '', pyver='3.4', rootdir=r'D:\Winpython',
-    #          verbose=False, archis=(64, ), flavor='')
+    make_all(4, '', pyver='3.4', rootdir=r'D:\Winpython',
+              verbose=False, archis=(64, ), flavor='')
     #make_all(5, '', pyver='3.3', rootdir=r'D:\Winpython',
     #          verbose=False, archis=(32, ))
     #make_all(5, '', pyver='3.3', rootdir=r'D:\Winpython',
     #          verbose=False, archis=(64, ))
-    make_all(2, '', pyver='2.7', rootdir=r'D:\Winpython',
-            verbose=False, archis=(32, ))
+    #make_all(2, '', pyver='2.7', rootdir=r'D:\Winpython',
+    #        verbose=False, archis=(32, ))
     #make_all(2, '', pyver='2.7', rootdir=r'D:\Winpython',
     #         verbose=False, archis=(64, ))
     #make_all(2, '', pyver='2.7', rootdir=r'D:\Winpython',

portable/launcher.nsi

@@ -71,11 +71,12 @@ end_Rsettings:
 
 ; Addition of JULIA and JULIA_HOME Environment Variable if %JULIA% program exists
 StrCmp "${JULIA}" "" end_Julia_settings
-IfFileExists "${JULIA}\bin\*.*" 0 end_Julia_settings
+IfFileExists "${JULIA}" 0 end_Julia_settings
 
 System::Call 'Kernel32::SetEnvironmentVariableA(t, t) i("JULIA", "${JULIA}").r0'
 
 StrCmp "${JULIA_HOME}" "" end_Julia_settings
+IfFileExists "${JULIA_HOME}\*.*" 0 end_Julia_settings
 System::Call 'Kernel32::SetEnvironmentVariableA(t, t) i("JULIA_HOME", "${JULIA_HOME}").r0'
 
 end_Julia_settings:

winpython/utils.py

@@ -278,6 +278,33 @@ def get_python_long_version(path):
         ver = None
     return ver
 
+# =============================================================================
+# Patch chebang line (courtesy of Christoph Gohlke)
+# =============================================================================
+def patch_shebang_line(fname, pad=b' '):
+    """Remove absolute path to python.exe in shebang lines."""
+    if sys.version_info[0] == 2:
+        shebang_line = re.compile(r"(#!.+pythonw?\.exe)")  # Python2.7
+    else:
+        shebang_line = re.compile(b"(#!.+pythonw?\.exe)")  # Python3+
+
+    with open(fname, 'rb') as fh:
+        content = fh.read()
+
+    content = shebang_line.split(content, maxsplit=1)
+    if len(content) != 3:
+        return
+    exe = os.path.basename(content[1][2:])
+    content[1] = b'#!' + exe + (pad * (len(content[1]) - len(exe) - 2))
+    content = b''.join(content)
+
+    try:
+        with open(fname, 'wb') as fh:
+            fh.write(content)
+            print("patched", fname)
+    except Exception:
+        print("failed to patch", fname)
+
 
 # =============================================================================
 # Extract functions
@@ -302,6 +329,17 @@ def extract_msi(fname, targetdir=None, verbose=False):
         args += ['/qn']
     args += ['TARGETDIR=%s' % targetdir]
     subprocess.call([extract]+args, cwd=osp.dirname(fname))
+    print ('fname=%s' % fname)
+    print ('TARGETDIR=%s' % targetdir)
+    # ensure pip if it's not 3.3
+    if '-3.3' not in targetdir:
+        subprocess.call([r'%s\%s' %(targetdir, 'python.exe'), '-m' , 'ensurepip'],
+                     cwd=osp.dirname(r'%s\%s' %(targetdir, 'pythons.exe')))
+        # We patch ensurepip live (shame) !!!!
+        # rational: https://github.com/pypa/pip/issues/2328
+        import glob
+        for fname in glob.glob(r'%s\Scripts\*.exe' % targetdir):
+            patch_shebang_line(fname)
     return targetdir
 
 
@@ -354,7 +392,7 @@ def extract_archive(fname, targetdir=None, verbose=False):
 #         . joblib-0.8.3_r1-py2.py3-none-any.whl,
 #         . joblib-0.8.3-r1.tar.gz
 
-SOURCE_PATTERN = r'([a-zA-Z0-9\-\_\.]*)-([0-9\.\_]*[a-z]*[0-9]?)(\.zip|\.tar\.gz|\-(py2|py2\.py3|py3)\-none\-any\.whl)'
+SOURCE_PATTERN = r'([a-zA-Z0-9\-\_\.]*)-([0-9\.\_]*[a-z]*[0-9]?)(\.zip|\.tar\.gz|\-(py[2-7]*|py[2-7]*\.py[2-7]*)\-none\-any\.whl)'
 
 # WHEELBIN_PATTERN defines what an acceptable binary wheel package is
 # "cp([0-9]*)" to replace per cp(34) for python3.4
@@ -446,7 +484,8 @@ def build_wheel(this_whl, python_exe=None, copy_to=None,
     assert osp.isfile(python_exe)
     myroot = os.path.dirname(python_exe)
 
-    cmd = [python_exe, myroot + r'\Scripts\pip-script.py', 'install']
+    #cmd = [python_exe, myroot + r'\Scripts\pip-script.py', 'install']
+    cmd = [python_exe, '-m', 'pip', 'install']
     if install_options:
         cmd += install_options  # typically ['--no-deps']
         print('wheel install_options', install_options)
@@ -469,7 +508,35 @@ def build_wheel(this_whl, python_exe=None, copy_to=None,
             print("Installed %s" % src_fname)
         return src_fname
 
+def do_script(this_script, python_exe=None, copy_to=None,
+                architecture=None, verbose=False, install_options=None):
+    """Execute a script (get-pip typically)"""
+    if python_exe is None:
+        python_exe = sys.executable
+    assert osp.isfile(python_exe)
+    myroot = os.path.dirname(python_exe)
+
+    #cmd = [python_exe, myroot + r'\Scripts\pip-script.py', 'install']
+    cmd = [python_exe]
+    if install_options:
+        cmd += install_options  # typically ['--no-deps']
+        print('script install_options', install_options)
+    cmd += [this_script]
+    #  print('build_wheel', myroot, cmd)
+    print("Executing " , cmd)
 
+    if verbose:
+        subprocess.call(cmd, cwd=myroot)
+    else:
+        p = subprocess.Popen(cmd, cwd=myroot, stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+        p.communicate()
+        p.stdout.close()
+        p.stderr.close()
+    if verbose:
+            print("Executed " % cmd)
+    return 'ok'
+
 def wheel_to_wininst(fname, python_exe=None,
                      architecture=None, verbose=False, install_options=None):
     """Just install a wheel !"""

winpython/wppm.py

@@ -372,19 +372,28 @@ def install(self, package, install_options=None):
 
         # We patch pip live (around line 100) !!!!
         # rational: https://github.com/pypa/pip/issues/2328
-        if package.name == "pip":
+        if package.name == "get-pip":
+            # self.exec_script
+            my_script_is=osp.join(self.target, 'Scripts', 'get-pip.py')
+            self.install_script(my_script_is, install_options=None)
+        if package.name == "pip" or package.name == "get-pip":
+            import glob
+            for ffname in glob.glob(r'%s\Scripts\*.exe' % self.target):
+                utils.patch_shebang_line(ffname)
             do_replace = self.target + (r"\Lib\site-packages\pip\_vendor" +
                          r"\distlib\scripts.py"  )
             print("do_replace" , do_replace)
             fh = open(do_replace,"r")
             the_thing =  fh.read()             
             fh.close()
-            the_thing = the_thing.replace(
+            the_thing_after = the_thing.replace(
               " executable = get_executable()",
               " executable = os.path.join(os.path.basename(get_executable()))") 
-            fh = open(do_replace,"w")
-            fh.write(the_thing)
-            fh.close()
+            if not the_thing_after == the_thing:
+                print("do_replace_ok" , do_replace)
+                fh = open(do_replace,"w")
+                fh.write(the_thing_after)
+                fh.close()
 
     def handle_specific_packages(self, package):
         """Packages requiring additional configuration"""
@@ -474,7 +483,7 @@ def install_bdist_wininst(self, package):
         targetdir = utils.extract_archive(package.fname)
         self._print_done()
 
-        self._print(package, "Installing")
+        self._print(package, "Installing %s from " % targetdir)
         self.copy_files(package, targetdir, 'PURELIB',
                         osp.join('Lib', 'site-packages'))
         self.copy_files(package, targetdir, 'PLATLIB',
@@ -501,6 +510,17 @@ def install_bdist_wheel(self, package, install_options=None):
         package = Package(fname)
         self._print_done()
 
+    def install_script(self, script, install_options=None):
+        try:
+            fname = utils.do_script(script,
+                        python_exe=osp.join(self.target, 'python.exe'),
+                        architecture=self.architecture, verbose=self.verbose,
+                        install_options=install_options)
+        except RuntimeError:
+            if not self.verbose:
+                print("Failed!")
+                raise
+
     def install_bdist_msi(self, package):
         """Install a distutils package built with the bdist_msi option
         (binary distribution, .msi file)"""