Merge pull request winpython#36 from stonebig/master

Accept binary wheels, patch standard pip package

make.py

@@ -128,6 +128,8 @@ class WinPythonDistribution(object):
     THG_PATH = r'\tools\TortoiseHg\thgw.exe'
     WINMERGE_PATH = r'\tools\WinMerge\WinMergeU.exe'
     MINGW32_PATH = r'\tools\mingw32\bin'
+    R_PATH = r'\tools\R\bin'
+    JULIA_PATH = r'\tools\Julia\bin'
 
     def __init__(self, build_number, release_level, target, instdirs,
                  srcdirs=None, toolsdirs=None, verbose=False, simulation=False,
@@ -185,6 +187,12 @@ def get_tool_path(relpath, checkfunc):
         if gccpath is not None:
             gccver = utils.get_gcc_version(gccpath)
             installed_tools += [('MinGW32', gccver)]
+
+        rpath = get_tool_path(self.R_PATH, osp.isdir)
+        if rpath is not None:
+            rver = utils.get_r_version(rpath)
+            installed_tools += [('R', rver)]
+
         tools = []
         for name, ver in installed_tools:
             metadata = wppm.get_package_metadata('tools.ini', name)
@@ -249,12 +257,19 @@ def prepath(self):
         path = [r"Lib\site-packages\PyQt4",
                 "",  # Python root directory (python.exe)
                 "DLLs", "Scripts", r"..\tools", r"..\tools\mingw32\bin"
-                # , r"..\tools\Julia\bin"
-                # , r"..\tools\R\bin"
                 ]
         if self.distribution.architecture == 32 \
            and osp.isdir(self.winpydir + self.MINGW32_PATH):
             path += [r".." + self.MINGW32_PATH]
+
+        if self.distribution.architecture == 32:
+            path += [r".." + self.R_PATH + r"\i386"]
+
+        if self.distribution.architecture == 64:
+            path += [r".." + self.R_PATH + r"\x64"]
+
+        path += [r".." + self.JULIA_PATH]
+
         return path
 
     @property
@@ -337,16 +352,19 @@ def create_launcher(self, name, icon, command=None,
 
         data = [('WINPYDIR', '$EXEDIR\%s' % self.python_name),
                 ('WINPYVER', self.winpyver),
-                # ('JULIA_HOME','$EXEDIR\%s' % r'\tools\Julia\bin'),
-                # ('JULIA', '$EXEDIR\%s' % r'\tools\Julia\bin\julia.exe'),
-                # ('R_HOME', '$EXEDIR\%s' % r'\tools\R'),
                 ('COMMAND', command),
                 ('PARAMETERS', args),
                 ('WORKDIR', workdir),
                 ('PREPATH', prepath),
                 ('POSTPATH', postpath),
                 ('Icon', icon_fname),
                 ('OutFile', name)]
+
+        # handle well Flavor with R included
+        data += [('R_HOME', '$EXEDIR%s' % r'\tools\R'),
+                 ('JULIA_HOME','$EXEDIR\%s' % r'tools\Julia\bin'),
+                 ('JULIA', '$EXEDIR\%s' % r'tools\Julia\bin\julia.exe')]
+
         if settingspath is not None:
             data += [('SETTINGSDIR', osp.dirname(settingspath)),
                      ('SETTINGSNAME', osp.basename(settingspath))]
@@ -473,9 +491,9 @@ def _install_required_packages(self):
 
         # Install 'main packages' first (was before Wheel idea, keep for now)
         for happy_few in['numpy-MKL', 'scipy', 'matplotlib', 'pandas']:
+            # can be a wheel now
             self.install_package(
-                '%s-([0-9\.]*[a-z]*[0-9]?).%s(-py%s)?.exe'
-                % (happy_few, self.py_arch, self.python_version))
+                '%s-([0-9\.]*[a-z]*[0-9]?)(.*)(\.exe|\.whl)' % happy_few) 
 
     def _install_all_other_packages(self):
         """Try to install all other packages in instdirs"""
@@ -606,8 +624,25 @@ def _create_batch_scripts(self):
         path = conv(self.prepath) + ";%PATH%;" + conv(self.postpath)
         self.create_batch_script('env.bat', """@echo off
 set WINPYDIR=%~dp0..\\""" + self.python_name + r"""
-set WINPYVER=""" + self.winpyver + """
+set WINPYVER=""" + self.winpyver + r"""
 set HOME=%WINPYDIR%\..\settings
+set WINPYARCH="WIN32"
+if  "%WINPYDIR:~-5%"=="amd64" set WINPYARCH="WIN-AMD64"
+
+rem handle R if included
+if not exist "%WINPYDIR%\..\tools\R\bin" goto r_bad
+set R_HOME=%WINPYDIR%\..\tools\R
+if %WINPYARCH%=="WIN32"     set R_HOMEbin=%R_HOME%\bin\i386
+if not %WINPYARCH%=="WIN32" set R_HOMEbin=%R_HOME%\bin\x64
+:r_bad
+
+rem handle Julia if included
+if not exist "%WINPYDIR%\..\tools\Julia\bin" goto julia_bad
+set JULIA_HOME=%WINPYDIR%\..\tools\Julia\bin\
+set JULIA_EXE=julia.exe
+set JULIA=%JULIA_HOME%%JULIA_EXE%
+:julia_bad
+
 set PATH=""" + path)
 
         self.create_batch_script('start_ijulia.bat', r"""@echo off
@@ -709,42 +744,49 @@ def _create_batch_scripts(self):
         self.create_batch_script('start_with_r.bat', r"""@echo off
 call %~dp0env.bat
 
+rem **get Base of winpython in pure path form
+pushd
+cd /d  %WINPYDIR%
+cd..
+set WINPYDIR..=%CD%
+popd
+
 rem ******************
 rem R part (supposing you install it in \tools\R of winpython)
 rem ******************
 set tmp_Rdirectory=R
-if not exist "%WINPYDIR%\..\tools\%tmp_Rdirectory%\bin" goto r_bad
+if not exist "%WINPYDIR..%\tools\%tmp_Rdirectory%\bin" goto r_bad
 
 rem  R_HOME for rpy2, R_HOMEBIN for PATH
-set R_HOME=%WINPYDIR%\..\tools\%tmp_Rdirectory%\
-set R_HOMEbin=%WINPYDIR%\..\tools\%tmp_Rdirectory%\bin
+set R_HOME=%WINPYDIR..%\tools\%tmp_Rdirectory%
+if %WINPYARCH%=="WIN32"     set R_HOMEbin=%R_HOME%\bin\i386
+if not %WINPYARCH%=="WIN32" set R_HOMEbin=%R_HOME%\bin\x64
 
 set SYS_PATH=%PATH%
 set PATH=%SYS_PATH%;%R_HOMEbin%
 
 echo "r!"
-echo "We are going to  update %WINPYDIR%\settings\winpython.ini with"
+echo "We are going to  update %WINPYDIR..%\settings\winpython.ini with"
 echo "R_HOME = %R_HOME%"
 echo "(relaunch this batch, if you move your winpython)"
 pause
 
-%~dp0Add_or_removeLine.vbs ..\settings\winpython.ini  "R_HOME = " -remove
-
 rem Handle case when winpython.ini is not already created
-if exist "..\settings\winpython.ini" goto ini_exists 
+if exist "%WINPYDIR..%\settings\winpython.ini" goto ini_exists 
 
-echo [debug]>"..\settings\winpython.ini"
-echo state = disabled>>"..\settings\winpython.ini"
-echo [environment]>>"..\settings\winpython.ini"
+echo [debug]>"%WINPYDIR..%\settings\winpython.ini"
+echo state = disabled>>"%WINPYDIR..%\settings\winpython.ini"
+echo [environment]>>"%WINPYDIR..%\settings\winpython.ini"
 
 :ini_exists 
-%~dp0Add_or_removeLine.vbs ..\settings\winpython.ini  "[environment]" "R_HOME = %R_HOME%"
+%~dp0Add_or_removeLine.vbs %WINPYDIR..%\settings\winpython.ini  "R_HOME = " -remove
+%~dp0Add_or_removeLine.vbs %WINPYDIR..%\settings\winpython.ini  "[environment]" "R_HOME = %R_HOME%"
 goto r_end
 
 :r_bad 
 
-echo directory "%WINPYDIR%\..\tools\%tmp_Rdirectory%\bin" not found
-echo please install R at "%WINPYDIR%\..\tools\%tmp_Rdirectory%" 
+echo directory "%WINPYDIR..%\tools\%tmp_Rdirectory%\bin" not found
+echo please install R at "%WINPYDIR..%\tools\%tmp_Rdirectory%" 
 pause
 
 :r_end
@@ -1082,19 +1124,25 @@ def make_all(build_number, release_level, pyver,
     # DO create only what version at a time
     # You may have to manually delete previous build\winpython-.. directory
 
-    #make_all(3, '', pyver='3.4', rootdir=r'D:\Winpython',
+    #make_all(4, '', pyver='3.4', rootdir=r'D:\Winpython',
     #         verbose=False, archis=(32, ))
-    make_all(3, '', pyver='3.4', rootdir=r'D:\Winpython',
-              verbose=False, archis=(64, ), flavor='')
-    #make_all(4, '', pyver='3.3', rootdir=r'D:\Winpython',
+    #make_all(4, '', pyver='3.4', rootdir=r'D:\Winpython',
+    #          verbose=False, archis=(64, ), flavor='')
+    #make_all(5, '', pyver='3.3', rootdir=r'D:\Winpython',
     #          verbose=False, archis=(32, ))
-    #make_all(4, '', pyver='3.3', rootdir=r'D:\Winpython',
+    #make_all(5, '', pyver='3.3', rootdir=r'D:\Winpython',
     #          verbose=False, archis=(64, ))
-    #make_all(4, '', pyver='2.7', rootdir=r'D:\Winpython',
-    #         verbose=False, archis=(32, ))
-    #make_all(4, '', pyver='2.7', rootdir=r'D:\Winpython',
+    #make_all(2, '', pyver='2.7', rootdir=r'D:\Winpython',
+    #        verbose=False, archis=(32, ))
+    #make_all(2, '', pyver='2.7', rootdir=r'D:\Winpython',
     #         verbose=False, archis=(64, ))
-    #make_all(3, '', pyver='3.4', rootdir=r'D:\Winpython',
+    #make_all(2, '', pyver='2.7', rootdir=r'D:\Winpython',
+    #         verbose=False, archis=(64, ), flavor='FlavorRfull')
+    #make_all(4, '', pyver='3.4', rootdir=r'D:\Winpython',
     #          verbose=False, archis=(64, ), flavor='FlavorIgraph')
-    #make_all(3, '', pyver='3.4', rootdir=r'D:\Winpython',
+    #make_all(4, '', pyver='3.4', rootdir=r'D:\Winpython',
     #          verbose=False, archis=(32, ), flavor='FlavorKivy')
+    make_all(4, '', pyver='3.4', rootdir=r'D:\Winpython',
+              verbose=False, archis=(32, ), flavor='FlavorRfull')
+    #make_all(4, '', pyver='3.4', rootdir=r'D:\Winpython',
+    #          verbose=False, archis=(64, ), flavor='FlavorRfull')

portable/launcher.nsi

@@ -14,6 +14,12 @@ Licensed under the terms of the MIT License
 !addincludedir ""
 !define WINPYDIR ""
 !define WINPYVER ""
+;  Addition for R_HOME
+!define R_HOME ""
+;  Addition for JULIA_HOME and JULIA
+!define JULIA_HOME ""
+!define JULIA ""
+
 !define COMMAND ""
 !define PARAMETERS ""
 !define WORKDIR ""
@@ -55,6 +61,24 @@ end_workdir:
 System::Call 'Kernel32::SetEnvironmentVariableA(t, t) i("WINPYDIR", "${WINPYDIR}").r0'
 System::Call 'Kernel32::SetEnvironmentVariableA(t, t) i("WINPYVER", "${WINPYVER}").r0'
 
+; Addition of R_HOME Environment Variable if %R_Home%\bin exists
+StrCmp "${R_HOME}" "" end_Rsettings
+IfFileExists "${R_HOME}\bin\*.*" 0 end_Rsettings
+
+System::Call 'Kernel32::SetEnvironmentVariableA(t, t) i("R_HOME", "${R_HOME}").r0'
+
+end_Rsettings:
+
+; Addition of JULIA and JULIA_HOME Environment Variable if %JULIA% program exists
+StrCmp "${JULIA}" "" end_Julia_settings
+IfFileExists "${JULIA}\bin\*.*" 0 end_Julia_settings
+
+System::Call 'Kernel32::SetEnvironmentVariableA(t, t) i("JULIA", "${JULIA}").r0'
+
+StrCmp "${JULIA_HOME}" "" end_Julia_settings
+System::Call 'Kernel32::SetEnvironmentVariableA(t, t) i("JULIA_HOME", "${JULIA_HOME}").r0'
+
+end_Julia_settings:
 
 ;================================================================
 ; Settings directory

winpython/data/packages.ini

@@ -330,6 +330,9 @@ category=util
 [python-igraph]
 description=High performance graph data structures and algorithms
 
+[python-twitter]
+description=A Python wrapper around the Twitter API
+
 [pytz]
 description=World Timezone Definitions for Python
 url=http://pytz.sourceforge.net/
@@ -364,6 +367,9 @@ category=docgen
 [requests]
 description=Requests is an Apache2 Licensed HTTP library, written in Python, for human beings.
 
+[requests_oauthlib]
+description=OAuthlib authentication support for Requests.
+
 [rope]
 description=a python refactoring library...
 
@@ -474,6 +480,9 @@ category=dataproc
 [tqdm]
 description=A Simple Python Progress Meter
 
+[tweepy]
+description=Twitter library for python
+
 [vispy]
 description=Interactive visualization in Python
 

winpython/data/tools.ini

@@ -10,14 +10,14 @@ url=http://julialang.org/
 description=C/C++ and Fortran compilers (Mingw64 static toolchain Numpy version, with OpenBLAS) 
 url=https://github.com/numpy/numpy/wiki/Mingw-static-toolchain
 
-[scite]
-description=SCIntilla based Text Editor - Multilanguage, powerful and light-weight text editor
-url=http://www.scintilla.org/SciTE.html
-
 [r]
 description=The R Project for Statistical Computing
 url=http://www.r-project.org
 
+[scite]
+description=SCIntilla based Text Editor - Multilanguage, powerful and light-weight text editor
+url=http://www.scintilla.org/SciTE.html
+
 [tortoisehg]
 description=Set of graphical tools and a shell extension for the Mercurial distributed revision control system
 url=http://tortoisehg.bitbucket.org

winpython/utils.py

@@ -237,6 +237,10 @@ def get_gcc_version(path):
     """Return version of the GCC compiler installed in *path*"""
     return exec_shell_cmd('gcc --version', path).splitlines()[0].split()[-1]
 
+def get_r_version(path):
+    """Return version of the R installed in *path*"""
+    return exec_shell_cmd('dir ..\README.R*', path).splitlines()[-3].split("-")[-1]
+
 
 def get_thg_version(path):
     """Return version of TortoiseHg installed in *path*"""
@@ -352,6 +356,10 @@ def extract_archive(fname, targetdir=None, verbose=False):
 
 SOURCE_PATTERN = r'([a-zA-Z0-9\-\_\.]*)-([0-9\.\_]*[a-z]*[0-9]?)(\.zip|\.tar\.gz|\-[a-z\.0-9]*\-none\-any\.whl)'
 
+# WHEELBIN_PATTERN defines what an acceptable binary wheel package is
+# "cp([0-9]*)" to replace per cp(34) for python3.4
+# "win32|win\_amd64" to replace per "win\_amd64" for 64bit
+WHEELBIN_PATTERN = r'([a-zA-Z0-9\-\_\.]*)-([0-9\.\_]*[a-z]*[0-9]?)-cp([0-9]*)\-none\-(win32|win\_amd64)\.whl'
 
 def get_source_package_infos(fname):
     """Return a tuple (name, version) of the Python source package"""
@@ -373,7 +381,7 @@ def build_wininst(root, python_exe=None, copy_to=None,
         archstr = 'win32' if architecture == 32 else 'win-amd64'
         cmd += ['--plat-name=%s' % archstr]
     cmd += ['bdist_wininst']
-    #  print('build_wininst', root, cmd)
+    # root = a tmp dir in windows\tmp, 
     if verbose:
         subprocess.call(cmd, cwd=root)
     else:
@@ -406,6 +414,8 @@ def build_wininst(root, python_exe=None, copy_to=None,
         shutil.move(src_fname, dst_fname)
         if verbose:
             print(("Move: %s --> %s" % (src_fname, (dst_fname))))
+            # remove tempo dir 'root' no more needed
+            shutil.rmtree(root, onerror=onerror)
         return dst_fname
 
 

winpython/wppm.py

@@ -128,6 +128,17 @@ def extract_infos(self):
                 self.name, self.version, self.pyversion, arch, _pyqt = match.groups()
                 self.architecture = int(arch)
                 return
+        # New : Binary wheel case
+        elif bname.endswith(('32.whl', '64.whl')):
+            match = re.match(utils.WHEELBIN_PATTERN, bname)
+            # typical macht is ('scipy', '0.14.1rc1', '34', 'win32')
+            if match is not None:
+                self.name, self.version,  self.pywheel , arch  = match.groups()
+                # self.pywheel version is '34' not 3.4
+                self.pyversion = self.pywheel[:1] + '.' + self.pywheel[1:]
+                # wheel arch is 'win32' or 'win_amd64'
+                self.architecture = 32 if arch == 'win32' else 64
+                return
         elif bname.endswith(('.zip', '.tar.gz', '.whl')):
             # distutils sdist
             infos = utils.get_source_package_infos(bname)
@@ -358,6 +369,22 @@ def install(self, package, install_options=None):
         package.save_log(self.logdir)
         if tmp_fname is not None:
             os.remove(tmp_fname)
+
+        # We patch pip live (around line 100) !!!!
+        # rational: https://github.com/pypa/pip/issues/2328
+        if package.name == "pip":
+            do_replace = self.target + (r"\Lib\site-packages\pip\_vendor" +
+                         r"\distlib\scripts.py"  )
+            print("do_replace" , do_replace)
+            fh = open(do_replace,"r")
+            the_thing =  fh.read()             
+            fh.close()
+            the_thing = the_thing.replace(
+              " executable = get_executable()",
+              " executable = os.path.join(os.path.basename(get_executable()))") 
+            fh = open(do_replace,"w")
+            fh.write(the_thing)
+            fh.close()
 
     def handle_specific_packages(self, package):
         """Packages requiring additional configuration"""