From 1dd7db12e05c7a52f1b1092d76a14caf0bf4c4cc Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Tue, 20 Nov 2018 00:38:37 +0100 Subject: [PATCH] Workaround for old pip versions that are not exposed as importable modules. (#6500) Fallback to pipstrap 1.5.0. Manipulate PATH variable on higher level to activate the virtual environment before calling pipstrap. --- letsencrypt-auto-source/letsencrypt-auto | 13 +++----- letsencrypt-auto-source/pieces/pipstrap.py | 14 +++------ tools/_venv_common.py | 36 ++++++++++++---------- 3 files changed, 29 insertions(+), 34 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9973607be..1417811f3 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1244,13 +1244,11 @@ UNLIKELY_EOF cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" #!/usr/bin/env python """A small script that can act as a trust root for installing pip >=8 - Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, with which you can install the rest of your dependencies safely. All it assumes is Python 2.6 or better and *some* version of pip already installed. If anything goes wrong, it will exit with a non-zero status code. - """ # This is here so embedded copies are MIT-compliant: # Copyright (c) 2016 Erik Rose @@ -1289,7 +1287,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info, executable +from sys import exit, version_info from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1301,7 +1299,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 2, 0, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -1377,10 +1375,8 @@ def hashed_download(url, temp, digest): def get_index_base(): """Return the URL to the dir containing the "packages" folder. - Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the end if it's there; that is likely to give us the right dir. - """ env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') if env_var: @@ -1394,7 +1390,7 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version']) + pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) min_pip_version = StrictVersion(PIP_VERSION) if pip_version >= min_pip_version: @@ -1407,7 +1403,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) + + check_output('pip install --no-index --no-deps -U ' + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: ('--no-cache-dir ' if has_pip_cache else '') + @@ -1426,6 +1422,7 @@ def main(): if __name__ == '__main__': exit(main()) + UNLIKELY_EOF # ------------------------------------------------------------------------- # Set PATH so pipstrap upgrades the right (v)env: diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index f21d36657..727040c3c 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -1,12 +1,10 @@ #!/usr/bin/env python """A small script that can act as a trust root for installing pip >=8 - Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, with which you can install the rest of your dependencies safely. All it assumes is Python 2.6 or better and *some* version of pip already installed. If anything goes wrong, it will exit with a non-zero status code. - """ # This is here so embedded copies are MIT-compliant: # Copyright (c) 2016 Erik Rose @@ -45,7 +43,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info, executable +from sys import exit, version_info from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -57,7 +55,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 2, 0, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -133,10 +131,8 @@ def hashed_download(url, temp, digest): def get_index_base(): """Return the URL to the dir containing the "packages" folder. - Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the end if it's there; that is likely to give us the right dir. - """ env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') if env_var: @@ -150,7 +146,7 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version']) + pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) min_pip_version = StrictVersion(PIP_VERSION) if pip_version >= min_pip_version: @@ -163,7 +159,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) + + check_output('pip install --no-index --no-deps -U ' + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: ('--no-cache-dir ' if has_pip_cache else '') + @@ -181,4 +177,4 @@ def main(): if __name__ == '__main__': - exit(main()) \ No newline at end of file + exit(main()) diff --git a/tools/_venv_common.py b/tools/_venv_common.py index bddda6302..2a48b0d2e 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -19,6 +19,7 @@ import time import subprocess import sys import re +import shlex VERSION_PATTERN = re.compile(r'^(\d+)\.(\d+).*$') @@ -101,18 +102,18 @@ def _check_version(version_str, major_version): return False -def subprocess_with_print(command): - print(command) - subprocess.check_call(command, shell=True) +def subprocess_with_print(cmd, env=os.environ, shell=False): + print('+ {0}'.format(subprocess.list2cmdline(cmd)) if isinstance(cmd, list) else cmd) + subprocess.check_call(cmd, env=env, shell=shell) -def get_venv_python(venv_path): +def get_venv_bin_path(venv_path): python_linux = os.path.join(venv_path, 'bin/python') if os.path.isfile(python_linux): - return python_linux + return os.path.abspath(os.path.dirname(python_linux)) python_windows = os.path.join(venv_path, 'Scripts\\python.exe') if os.path.isfile(python_windows): - return python_windows + return os.path.abspath(os.path.dirname(python_windows)) raise ValueError(( 'Error, could not find python executable in venv path {0}: is it a valid venv ?' @@ -144,18 +145,19 @@ def main(venv_name, venv_args, args): if os.path.isdir(venv_name): os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) - subprocess_with_print('"{0}" -m virtualenv --no-site-packages --setuptools {1} {2}' - .format(sys.executable, venv_name, venv_args)) + command = [sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', venv_name] + command.extend(shlex.split(venv_args)) + subprocess_with_print(command) - python_executable = get_venv_python(venv_name) - - subprocess_with_print('"{0}" {1}'.format( - python_executable, - os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py'))) - subprocess_with_print('"{0}" {1} {2}'.format( - python_executable, - os.path.normpath('./tools/pip_install.py'), - ' '.join(args))) + # We execute the two following commands in the context of the virtual environment, to install + # the packages in it. To do so, we append the venv binary to the PATH that will be used for + # these commands. With this trick, correct python executable will be selected. + new_environ = os.environ.copy() + new_environ['PATH'] = os.pathsep.join([get_venv_bin_path(venv_name), new_environ['PATH']]) + subprocess_with_print('python {0}'.format('./letsencrypt-auto-source/pieces/pipstrap.py'), + env=new_environ, shell=True) + subprocess_with_print('python {0} {1}'.format('./tools/pip_install.py', ' '.join(args)), + env=new_environ, shell=True) if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific