[Windows] Fix pipstrap (#6775)

* Fix pipstrap on windows

* Pipstrap pin setuptools version, so explicit install it is not needed anymore.

* Rebuild letsencrypt-auto source

* Use sys.executable in pipstrap to allow straightforward execution in a venv by choosing the python interpreter from this venv.

* Update letsencrypt-auto

* Simulate test-everything

* Revert "Simulate test-everything"

This reverts commit b62c4d719a.

* Clean pipstrap code

letsencrypt-auto-source/letsencrypt-auto

@@ -1221,7 +1221,6 @@ from distutils.version import StrictVersion
 from hashlib import sha256
 from os import environ
 from os.path import join
-from pipes import quote
 from shutil import rmtree
 try:
     from subprocess import check_output
@@ -1241,7 +1240,7 @@ except ImportError:
                 cmd = popenargs[0]
             raise CalledProcessError(retcode, cmd)
         return output
-from sys import exit, version_info
+import sys
 from tempfile import mkdtemp
 try:
     from urllib2 import build_opener, HTTPHandler, HTTPSHandler
@@ -1263,7 +1262,7 @@ maybe_argparse = (
     [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/'
       'argparse-1.4.0.tar.gz',
       '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')]
-    if version_info < (2, 7, 0) else [])
+    if sys.version_info < (2, 7, 0) else [])
 
 
 PACKAGES = maybe_argparse + [
@@ -1344,7 +1343,8 @@ def get_index_base():
 
 
 def main():
-    pip_version = StrictVersion(check_output(['pip', '--version'])
+    python = sys.executable or 'python'
+    pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version'])
                                 .decode('utf-8').split()[1])
     has_pip_cache = pip_version >= StrictVersion('6.0')
     index_base = get_index_base()
@@ -1354,12 +1354,12 @@ def main():
                                      temp,
                                      digest)
                      for path, digest in PACKAGES]
-        check_output('pip install --no-index --no-deps -U ' +
+        # On Windows, pip self-upgrade is not possible, it must be done through python interpreter.
-                     # Disable cache since we're not using it and it otherwise
+        command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U']
-                     # sometimes throws permission warnings:
+        # Disable cache since it is not used and it otherwise sometimes throws permission warnings:
-                     ('--no-cache-dir ' if has_pip_cache else '') +
+        command.extend(['--no-cache-dir'] if has_pip_cache else [])
-                     ' '.join(quote(d) for d in downloads),
+        command.extend(downloads)
-                     shell=True)
+        check_output(command)
     except HashError as exc:
         print(exc)
     except Exception:
@@ -1372,7 +1372,7 @@ def main():
 
 
 if __name__ == '__main__':
-    exit(main())
+    sys.exit(main())
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

letsencrypt-auto-source/pieces/pipstrap.py

@@ -23,7 +23,6 @@ from distutils.version import StrictVersion
 from hashlib import sha256
 from os import environ
 from os.path import join
-from pipes import quote
 from shutil import rmtree
 try:
     from subprocess import check_output
@@ -43,7 +42,7 @@ except ImportError:
                 cmd = popenargs[0]
             raise CalledProcessError(retcode, cmd)
         return output
-from sys import exit, version_info
+import sys
 from tempfile import mkdtemp
 try:
     from urllib2 import build_opener, HTTPHandler, HTTPSHandler
@@ -65,7 +64,7 @@ maybe_argparse = (
     [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/'
       'argparse-1.4.0.tar.gz',
       '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')]
-    if version_info < (2, 7, 0) else [])
+    if sys.version_info < (2, 7, 0) else [])
 
 
 PACKAGES = maybe_argparse + [
@@ -146,7 +145,8 @@ def get_index_base():
 
 
 def main():
-    pip_version = StrictVersion(check_output(['pip', '--version'])
+    python = sys.executable or 'python'
+    pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version'])
                                 .decode('utf-8').split()[1])
     has_pip_cache = pip_version >= StrictVersion('6.0')
     index_base = get_index_base()
@@ -156,12 +156,12 @@ def main():
                                      temp,
                                      digest)
                      for path, digest in PACKAGES]
-        check_output('pip install --no-index --no-deps -U ' +
+        # On Windows, pip self-upgrade is not possible, it must be done through python interpreter.
-                     # Disable cache since we're not using it and it otherwise
+        command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U']
-                     # sometimes throws permission warnings:
+        # Disable cache since it is not used and it otherwise sometimes throws permission warnings:
-                     ('--no-cache-dir ' if has_pip_cache else '') +
+        command.extend(['--no-cache-dir'] if has_pip_cache else [])
-                     ' '.join(quote(d) for d in downloads),
+        command.extend(downloads)
-                     shell=True)
+        check_output(command)
     except HashError as exc:
         print(exc)
     except Exception:
@@ -174,4 +174,4 @@ def main():
 
 
 if __name__ == '__main__':
-    exit(main())
+    sys.exit(main())

tools/_venv_common.py

@@ -107,13 +107,13 @@ def subprocess_with_print(cmd, env=os.environ, shell=False):
     subprocess.check_call(cmd, env=env, shell=shell)
 
 
-def get_venv_bin_path(venv_path):
+def get_venv_python_path(venv_path):
     python_linux = os.path.join(venv_path, 'bin/python')
     if os.path.isfile(python_linux):
-        return os.path.abspath(os.path.dirname(python_linux))
+        return os.path.abspath(python_linux)
     python_windows = os.path.join(venv_path, 'Scripts\\python.exe')
     if os.path.isfile(python_windows):
-        return os.path.abspath(os.path.dirname(python_windows))
+        return os.path.abspath(python_windows)
 
     raise ValueError((
         'Error, could not find python executable in venv path {0}: is it a valid venv ?'
@@ -149,17 +149,12 @@ def main(venv_name, venv_args, args):
     command.extend(shlex.split(venv_args))
     subprocess_with_print(command)
 
-    # We execute the following commands in the context of the virtual environment, to install
+    # Using the python executable from venv, we ensure to execute following commands in this venv.
-    # the packages in it. To do so, we append the venv binary to the PATH that will be used for
+    py_venv = get_venv_python_path(venv_name)
-    # these commands. With this trick, correct python executable will be selected.
+    subprocess_with_print([py_venv, os.path.abspath('letsencrypt-auto-source/pieces/pipstrap.py')])
-    new_environ = os.environ.copy()
+    command = [py_venv, os.path.abspath('tools/pip_install.py')]
-    new_environ['PATH'] = os.pathsep.join([get_venv_bin_path(venv_name), new_environ['PATH']])
+    command.extend(args)
-    subprocess_with_print('python {0}'.format('./letsencrypt-auto-source/pieces/pipstrap.py'),
+    subprocess_with_print(command)
-                          env=new_environ, shell=True)
-    subprocess_with_print('python -m pip install --upgrade "setuptools>=30.3"',
-                          env=new_environ, shell=True)
-    subprocess_with_print('python {0} {1}'.format('./tools/pip_install.py', ' '.join(args)),
-                          env=new_environ, shell=True)
 
     if os.path.isdir(os.path.join(venv_name, 'bin')):
         # Linux/OSX specific
@@ -179,6 +174,4 @@ def main(venv_name, venv_args, args):
 
 
 if __name__ == '__main__':
-    main('venv',
+    main('venv', '', sys.argv[1:])
-         '',
-         sys.argv[1:])