Use current, but pinned versions of pip, setuptools and wheel
This patch provides a convenience script to check for the current version of any package on PyPI, then output it in various ways. This script is used in the SHA updating script in order to provide a current set of critical packages to ensure that each SHA bump includes an update to a current version of pip, setuptools and wheel but also to ensure that they are pinned to a specific version with this particular set of packages. This ensures that we keep current with these packages as they change, but also ensures that the versions tested for each tag are the versions used forever. The patch also ensures that any package installed by pip is upgraded to the expected versions. Depends-On: I0a78135737c40a505d77df6ed67da0ef6695bfcb Change-Id: I61795b3afb4804060d494a08975c10adcf52f468
This commit is contained in:
parent
669116f923
commit
ab2834ef8e
@ -71,6 +71,12 @@ pip_get_pip_options: >
|
||||
pip_links:
|
||||
- { name: "openstack_release", link: "{{ openstack_repo_url }}/os-releases/{{ openstack_release }}/" }
|
||||
|
||||
# These are pinned to ensure exactly the same behaviour forever!
|
||||
# These pins are updated through the sources-branch-updater script
|
||||
pip_packages:
|
||||
- pip==8.0.3
|
||||
- setuptools==20.1.1
|
||||
- wheel==0.29.0
|
||||
|
||||
## Memcached options
|
||||
memcached_listen: "{{ ansible_ssh_host }}"
|
||||
|
@ -0,0 +1,8 @@
|
||||
---
|
||||
fixes:
|
||||
- The python packages `pip`, `setuptools` and `wheel` are now all pinned on
|
||||
a per-tag basis. The pins are updated along with every OpenStack Service
|
||||
update. This is done to ensure a consistent build experience with the
|
||||
latest available packages at the time the tag is released. A deployer may
|
||||
override the pins by adding a list of required pins using the
|
||||
`pip_packages` variable in `user_variables.yml`.
|
@ -1,7 +1,13 @@
|
||||
Jinja2>=2.6 # ansible
|
||||
netaddr>=0.7.12 # playbooks/inventory/dynamic_inventory.py
|
||||
paramiko>=1.13.0 # ansible
|
||||
pip>7,<8 # ansible
|
||||
PrettyTable>=0.7,<0.8 # scripts/inventory-manage.py
|
||||
pycrypto>=2.6 # ansible
|
||||
PyYAML>=3.1.0 # ansible
|
||||
###
|
||||
### These are pinned to ensure exactly the same behaviour forever! ###
|
||||
### These pins are updated through the sources-branch-updater script ###
|
||||
###
|
||||
pip==8.0.3
|
||||
setuptools==20.1.1
|
||||
wheel==0.29.0
|
||||
|
100
scripts/get-pypi-pkg-version.py
Executable file
100
scripts/get-pypi-pkg-version.py
Executable file
@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright 2016, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# (c) 2016, Jesse Pretorius <jesse.pretorius@rackspace.co.uk>
|
||||
#
|
||||
|
||||
|
||||
"""Returns the versions of a list of pypi packages you specify."""
|
||||
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import xmlrpclib
|
||||
|
||||
|
||||
def get_package_version(pypiConn, pkg_name):
|
||||
"""Get the current package version from PyPI."""
|
||||
pkg_result = pypiConn.package_releases(pkg_name)
|
||||
if pkg_result:
|
||||
pkg_version = pkg_result[0]
|
||||
else:
|
||||
pkg_version = 'Not available.'
|
||||
|
||||
return pkg_version
|
||||
|
||||
|
||||
def main():
|
||||
"""Run the main application."""
|
||||
|
||||
# Setup argument parsing
|
||||
parser = argparse.ArgumentParser(
|
||||
description='PyPI Current Package Version Checker',
|
||||
epilog='Licensed "Apache 2.0"')
|
||||
|
||||
parser.add_argument(
|
||||
'-f',
|
||||
'--format',
|
||||
choices=['requirements', 'bare'],
|
||||
default='requirements',
|
||||
help='<Optional> Output format',
|
||||
required=False
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-l',
|
||||
'--layout',
|
||||
choices=['vertical', 'horizontal'],
|
||||
default='vertical',
|
||||
help='<Optional> Output layout',
|
||||
required=False
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-p',
|
||||
'--packages',
|
||||
nargs='+',
|
||||
help='<Required> Space-delimited list of packages',
|
||||
required=True
|
||||
)
|
||||
|
||||
# Parse arguments
|
||||
args = parser.parse_args()
|
||||
|
||||
# Setup pypi object
|
||||
pypi = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
|
||||
|
||||
# Setup the newline if the results layout should be vertical
|
||||
# Also add a space delimiter appropriately
|
||||
if args.layout == 'vertical':
|
||||
delimiter = ''
|
||||
endline = '\n'
|
||||
else:
|
||||
delimiter = ' '
|
||||
endline = ''
|
||||
|
||||
# Print the results to stdout
|
||||
for pkg_name in args.packages:
|
||||
pkg_version = get_package_version(pypi, pkg_name)
|
||||
if args.format == 'requirements':
|
||||
print(pkg_name + '==' + pkg_version + delimiter, end=endline)
|
||||
else:
|
||||
print(pkg_version + delimiter, end=endline)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
@ -21,7 +21,7 @@ MAX_RETRIES=${MAX_RETRIES:-5}
|
||||
REPORT_DATA=${REPORT_DATA:-""}
|
||||
ANSIBLE_PARAMETERS=${ANSIBLE_PARAMETERS:-""}
|
||||
STARTTIME="${STARTTIME:-$(date +%s)}"
|
||||
PIP_MAJOR_VERSION=${PIP_MAJOR_VERSION:-"7"}
|
||||
PIP_INSTALL_OPTIONS=${PIP_INSTALL_OPTIONS:-'pip==8.0.3 setuptools==20.1.1 wheel==0.29.0 '}
|
||||
|
||||
# The default SSHD configuration has MaxSessions = 10. If a deployer changes
|
||||
# their SSHD config, then the FORKS may be set to a higher number. We set the
|
||||
@ -216,10 +216,8 @@ function get_pip {
|
||||
# check if pip is already installed
|
||||
if [ "$(which pip)" ]; then
|
||||
|
||||
# if the version installed is the wrong version, fix it
|
||||
if [ "$(pip --version | awk '{print $2}' | cut -d. -f1)" != ${PIP_MAJOR_VERSION} ]; then
|
||||
pip install -I "pip>=${PIP_MAJOR_VERSION},<$((PIP_MAJOR_VERSION+1))"
|
||||
fi
|
||||
# make sure that the right pip base packages are installed
|
||||
pip install --upgrade ${PIP_INSTALL_OPTIONS}
|
||||
|
||||
# when pip is not installed, install it
|
||||
else
|
||||
@ -228,7 +226,7 @@ function get_pip {
|
||||
if [ -n "${GET_PIP_URL:-}" ]; then
|
||||
curl --silent ${GET_PIP_URL} > /opt/get-pip.py
|
||||
if head -n 1 /opt/get-pip.py | grep python; then
|
||||
python /opt/get-pip.py "pip>=${PIP_MAJOR_VERSION},<$((PIP_MAJOR_VERSION+1))"
|
||||
python /opt/get-pip.py "${PIP_INSTALL_OPTIONS}"
|
||||
return
|
||||
fi
|
||||
fi
|
||||
@ -236,14 +234,14 @@ function get_pip {
|
||||
# Try getting pip from bootstrap.pypa.io as a primary source
|
||||
curl --silent https://bootstrap.pypa.io/get-pip.py > /opt/get-pip.py
|
||||
if head -n 1 /opt/get-pip.py | grep python; then
|
||||
python /opt/get-pip.py "pip>=${PIP_MAJOR_VERSION},<$((PIP_MAJOR_VERSION+1))"
|
||||
python /opt/get-pip.py "${PIP_INSTALL_OPTIONS}"
|
||||
return
|
||||
fi
|
||||
|
||||
# Try the get-pip.py from the github repository as a primary source
|
||||
curl --silent https://raw.githubusercontent.com/pypa/get-pip/master/get-pip.py > /opt/get-pip.py
|
||||
if head -n 1 /opt/get-pip.py | grep python; then
|
||||
python /opt/get-pip.py "pip>=${PIP_MAJOR_VERSION},<$((PIP_MAJOR_VERSION+1))"
|
||||
python /opt/get-pip.py "${PIP_INSTALL_OPTIONS}"
|
||||
return
|
||||
fi
|
||||
|
||||
|
@ -134,3 +134,14 @@ for repo in $(grep 'git_repo\:' ${SERVICE_FILE}); do
|
||||
echo -e "Processed $repo_name @ $branch_entry\n"
|
||||
|
||||
done
|
||||
|
||||
# Finally, update the PIP_INSTALL_OPTIONS with the current versions of pip, wheel and setuptools
|
||||
PIP_CURRENT_OPTIONS=$(./scripts/get-pypi-pkg-version.py -p pip setuptools wheel -l horizontal)
|
||||
sed -i.bak "s|^PIP_INSTALL_OPTIONS=.*|PIP_INSTALL_OPTIONS=\$\{PIP_INSTALL_OPTIONS:-'${PIP_CURRENT_OPTIONS}'\}|" scripts/scripts-library.sh
|
||||
|
||||
for pin in ${PIP_CURRENT_OPTIONS}; do
|
||||
sed -i.bak "s|^$(echo ${pin} | cut -f1 -d=).*|${pin}|" *requirements.txt
|
||||
sed -i.bak "s|^ - $(echo ${pin} | cut -f1 -d=).*| - ${pin}|" playbooks/inventory/group_vars/hosts.yml
|
||||
done
|
||||
|
||||
echo "Updated pip install options/pins"
|
||||
|
Loading…
Reference in New Issue
Block a user