From 8df6eb01184672ffdc752437ea4d98d13f16026e Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Fri, 8 Mar 2019 13:19:03 +0100 Subject: [PATCH] replace 'distro_python3' variable with 'distro_python_version' RHEL7 derived distros use Python 2.7 RHEL8 uses Python 3.6 Ubuntu 18.04 uses Python 3.6, but we haven't enabled it yet. Debian:buster uses Python 3.7, but we haven't enabled it yet. We have several places in code where "/lib/pythonX.Y/site-packages" is used so we need to know value of X.Y rather than py2/py3 switch. How to check for Python 2: {% if distro_python_version == '2.7' %} Breaks: Iad6b70b433a0dd1b0f8ae6790fd280594517661a Change-Id: Ie24f109af19ae0438c21e0d43cb3cc947812b4ce --- docker/base/Dockerfile.j2 | 3 +- docker/horizon/Dockerfile.j2 | 6 +-- docker/horizon/extend_start.sh | 4 +- docker/kolla-toolbox/Dockerfile.j2 | 2 +- docker/macros.j2 | 7 +++- docker/openstack-base/Dockerfile.j2 | 10 ++++- kolla/common/config.py | 4 -- kolla/image/build.py | 15 +++---- kolla/tests/test_build.py | 40 ++++++++++++++----- ...hon_version-variable-3688288558f4e586.yaml | 6 +++ 10 files changed, 66 insertions(+), 31 deletions(-) create mode 100644 releasenotes/notes/Add-distro_python_version-variable-3688288558f4e586.yaml diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index bd75f6a90c..98e8ff1326 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -17,7 +17,8 @@ LABEL kolla_version="{{ kolla_version }}" ENV KOLLA_BASE_DISTRO={{ base_distro }} \ KOLLA_INSTALL_TYPE={{ install_type }} \ - KOLLA_INSTALL_METATYPE={{ install_metatype }} + KOLLA_INSTALL_METATYPE={{ install_metatype }} \ + KOLLA_DISTRO_PYTHON_VERSION={{ distro_python_version }} #### Customize PS1 to be used with bash shell COPY kolla_bashrc /tmp/ diff --git a/docker/horizon/Dockerfile.j2 b/docker/horizon/Dockerfile.j2 index 0fbeb39dbd..f2f3412024 100644 --- a/docker/horizon/Dockerfile.j2 +++ b/docker/horizon/Dockerfile.j2 @@ -74,7 +74,7 @@ RUN echo > /etc/apache2/ports.conf \ && cp /usr/share/openstack-dashboard/openstack_dashboard/conf/*.json /etc/openstack-dashboard \ && cp /usr/share/openstack-dashboard/manage.py /usr/bin/manage.py \ && rm /etc/apache2/conf-enabled/openstack-dashboard.conf \ - && for locale in /usr/lib/python2.7/site-packages/*/locale; do \ + && for locale in /usr/lib/python{{distro_python_version}}/site-packages/*/locale; do \ (cd ${locale%/*} && /usr/bin/python /usr/bin/manage.py compilemessages) \ done {% endblock %} @@ -136,10 +136,10 @@ RUN ln -s horizon-source/* horizon \ && if [ "$(ls /plugins)" ]; then \ {{ macros.install_pip(horizon_plugins_pip_packages) }}; \ fi \ - && for locale in /var/lib/kolla/venv/lib/python2.7/site-packages/*/locale; do \ + && for locale in /var/lib/kolla/venv/lib/python{{distro_python_version}}/site-packages/*/locale; do \ (cd ${locale%/*} && /var/lib/kolla/venv/bin/python /var/lib/kolla/venv/bin/manage.py compilemessages) \ done \ - && chown -R horizon: /etc/openstack-dashboard /var/lib/kolla/venv/lib/python2.7/site-packages/static + && chown -R horizon: /etc/openstack-dashboard /var/lib/kolla/venv/lib/python{{distro_python_version}}/site-packages/static {% endif %} diff --git a/docker/horizon/extend_start.sh b/docker/horizon/extend_start.sh index 8a32271669..1dc6aeb60c 100644 --- a/docker/horizon/extend_start.sh +++ b/docker/horizon/extend_start.sh @@ -5,9 +5,9 @@ set -o errexit FORCE_GENERATE="${FORCE_GENERATE}" if [[ ${KOLLA_INSTALL_TYPE} == "binary" ]]; then - SITE_PACKAGES="/usr/lib/python2.7/site-packages" + SITE_PACKAGES="/usr/lib/python${KOLLA_DISTRO_PYTHON_VERSION}/site-packages" elif [[ ${KOLLA_INSTALL_TYPE} == "source" ]]; then - SITE_PACKAGES="/var/lib/kolla/venv/lib/python2.7/site-packages" + SITE_PACKAGES="/var/lib/kolla/venv/lib/python${KOLLA_DISTRO_PYTHON_VERSION}/site-packages" fi if [[ ${KOLLA_INSTALL_TYPE} == "source" ]] && [[ ! -f ${SITE_PACKAGES}/openstack_dashboard/local/local_settings.py ]]; then diff --git a/docker/kolla-toolbox/Dockerfile.j2 b/docker/kolla-toolbox/Dockerfile.j2 index ca616074b7..4af30dede3 100644 --- a/docker/kolla-toolbox/Dockerfile.j2 +++ b/docker/kolla-toolbox/Dockerfile.j2 @@ -4,7 +4,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {% block kolla_toolbox_header %}{% endblock %} {% set virtualenv_path='/opt/ansible' %} -{% set os_client_config= virtualenv_path + '/lib/python2.7/site-packages/os_client_config/defaults.json' %} +{% set os_client_config= virtualenv_path + '/lib/python' + distro_python_version + '/site-packages/os_client_config/defaults.json' %} {% import "macros.j2" as macros with context %} diff --git a/docker/macros.j2 b/docker/macros.j2 index aee933e7b2..419fa0f4fa 100644 --- a/docker/macros.j2 +++ b/docker/macros.j2 @@ -22,8 +22,13 @@ {%- endmacro %} {% macro get_pip() %} +{% if distro_python_version == '2.7' %} + {% set python='python' %} +{% else %} + {% set python='python3' %} +{% endif %} RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ - && python get-pip.py \ + && {{python}} get-pip.py \ && rm get-pip.py {% endmacro %} diff --git a/docker/openstack-base/Dockerfile.j2 b/docker/openstack-base/Dockerfile.j2 index c3dab166ad..4f66271977 100644 --- a/docker/openstack-base/Dockerfile.j2 +++ b/docker/openstack-base/Dockerfile.j2 @@ -211,11 +211,9 @@ ENV DEBIAN_FRONTEND noninteractive 'debtcollector', 'decorator', 'elasticsearch', - 'enum34', 'eventlet', 'fasteners', 'funcsigs', - 'functools32', 'futures', 'futurist', 'gnocchiclient', @@ -297,6 +295,14 @@ ENV DEBIAN_FRONTEND noninteractive ] %} +{% if distro_python_version == '2.7' %} + {% set openstack_base_pip_packages = openstack_base_pip_packages + [ + 'enum34', + 'functools32' + ] + %} +{% endif %} + ADD openstack-base-archive /openstack-base-source RUN ln -s openstack-base-source/* /requirements \ && mkdir -p /var/lib/kolla \ diff --git a/kolla/common/config.py b/kolla/common/config.py index 5d1082f658..05ccb13e0c 100755 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -266,10 +266,6 @@ _BASE_OPTS = [ cfg.StrOpt('maintainer', default='Kolla Project (https://launchpad.net/kolla)', help='Content of the maintainer label'), - cfg.BoolOpt('distro_python3', default=None, - help=('Enable this to force python3 packaging names. By ' - 'default this will try and determine the value of this ' - 'based on the base_distro and base_distro_tag.')), cfg.StrOpt('base_package_type', default=None, help=('Set the package type of the distro. If not set then ' 'the packaging type is set to "rpm" if a RHEL based ' diff --git a/kolla/image/build.py b/kolla/image/build.py index 0306d9b2bc..624db3406e 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -693,13 +693,13 @@ class KollaWorker(object): deb_base = ['ubuntu', 'debian'] deb_type = ['source', 'binary'] - if self.conf.distro_python3 is not None: - self.distro_python3 = self.conf.distro_python3 - elif self.base in rh_base and self.base_tag in ['8']: - # RHEL 8+ is python3 - self.distro_python3 = True + if self.base in rh_base and self.base_tag.startswith('7'): + self.conf.distro_python_version = "2.7" + elif self.base in rh_base and self.base_tag.startswith('8'): + self.conf.distro_python_version = "3.6" else: - self.distro_python3 = False + # Assume worst + self.conf.distro_python_version = "2.7" # Determine base packaging type for use in Dockerfiles. if self.conf.base_package_type: @@ -738,6 +738,7 @@ class KollaWorker(object): self.image_statuses_unmatched = dict() self.image_statuses_skipped = dict() self.maintainer = conf.maintainer + self.distro_python_version = conf.distro_python_version docker_kwargs = docker.utils.kwargs_from_env() try: @@ -915,7 +916,7 @@ class KollaWorker(object): 'kolla_version': kolla_version, 'image_name': image_name, 'users': self.get_users(), - 'distro_python3': self.distro_python3, + 'distro_python_version': self.distro_python_version, 'rpm_setup': self.rpm_setup, 'build_date': build_date} env = jinja2.Environment( # nosec: not used to render HTML diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py index 6ed50fbf1a..32c5a9928b 100644 --- a/kolla/tests/test_build.py +++ b/kolla/tests/test_build.py @@ -381,25 +381,45 @@ class KollaWorkerTest(base.TestCase): kolla = build.KollaWorker(self.conf) self.assertEqual(2, len(kolla.rpm_setup)) - def test_build_distro_python3(self): - """check distro_python3 conf value is taken""" - self.conf.set_override('distro_python3', True) + def test_build_distro_python_version_debian(self): + """check distro_python_version for Debian""" + self.conf.set_override('base', 'debian') kolla = build.KollaWorker(self.conf) - self.assertTrue(kolla.distro_python3) + self.assertEqual('2.7', kolla.distro_python_version) - def test_build_distro_python3_rhel8(self): - """check distro_python3 true for rhel8""" + def test_build_distro_python_version_rhel80(self): + """check distro_python_version for RHEL8.0""" + self.conf.set_override('base', 'rhel') + self.conf.set_override('base_tag', '8.0') + kolla = build.KollaWorker(self.conf) + self.assertEqual('3.6', kolla.distro_python_version) + + def test_build_distro_python_version_rhel8(self): + """check distro_python_version for RHEL8""" self.conf.set_override('base', 'rhel') self.conf.set_override('base_tag', '8') kolla = build.KollaWorker(self.conf) - self.assertTrue(kolla.distro_python3) + self.assertEqual('3.6', kolla.distro_python_version) - def test_build_distro_python3_non_rhel8(self): - """check distro_python3 false for non-rhel8""" + def test_build_distro_python_version_ubuntu(self): + """check distro_python_version for Ubuntu""" + self.conf.set_override('base', 'ubuntu') + kolla = build.KollaWorker(self.conf) + self.assertEqual('2.7', kolla.distro_python_version) + + def test_build_distro_python_version_centos7(self): + """check distro_python_version for CentOS 7.6.1810""" + self.conf.set_override('base', 'centos') + self.conf.set_override('base_tag', '7.6.1810') + kolla = build.KollaWorker(self.conf) + self.assertEqual('2.7', kolla.distro_python_version) + + def test_build_distro_python_version_rhel7(self): + """check distro_python_version for RHEL7""" self.conf.set_override('base', 'rhel') self.conf.set_override('base_tag', '7') kolla = build.KollaWorker(self.conf) - self.assertFalse(kolla.distro_python3) + self.assertEqual('2.7', kolla.distro_python_version) def test_base_package_type(self): """check base_package_type conf value is taken""" diff --git a/releasenotes/notes/Add-distro_python_version-variable-3688288558f4e586.yaml b/releasenotes/notes/Add-distro_python_version-variable-3688288558f4e586.yaml new file mode 100644 index 0000000000..a218da9f75 --- /dev/null +++ b/releasenotes/notes/Add-distro_python_version-variable-3688288558f4e586.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + New variable 'distro_python_version' to store version of Python used in + distribution used as a base. Can be used to handle py2/py3 checks and to + find 'site-packages' directory.