Debian/Ubuntu: move to Python 3 for source images

Commit 43b74ccc15 enabled use of Python 3
based packages but not switched to use Python 3.

Some of images still contain Python 2. There are two reasons:

- Ceph (ceph-common depends on Py2)
- python3-ldappool on Ubuntu 18.04

In Ceph situation Py3 packages were added. For second one we can not do
anything - Py2 dependency got dropped in Ubuntu 18.10 version.

Removed neutron-server-plugin-networking-infoblox due to being not
maintained. Once https://review.opendev.org/#/c/657578/ get merged
someone may revert that part.

Implements: blueprint debian-ubuntu-python3

Depends-on: Ie2a1077f7def0743f1403341985e2109aa490026

Change-Id: Ibfe0c2b8be98db56c61f74fb0247488ab3749ef4

docker/aodh/aodh-base/Dockerfile.j2

@@ -78,8 +78,8 @@ RUN mkdir -p /var/www/cgi-bin/aodh \
 
        {% set aodh_base_packages = [
             'apache2',
-            'libapache2-mod-wsgi',
+            'libapache2-mod-wsgi-py3',
-            'python-ldappool'
+            'python3-ldappool'
         ] %}
 
 {{ macros.install_packages(aodh_base_packages | customizable("packages")) }}

docker/base/Dockerfile.j2

@@ -317,7 +317,7 @@ COPY apt_preferences.{{ base_distro }} /etc/apt/preferences
    'netbase',
    'open-iscsi',
    'procps',
-   'python',
+   'python3',
    'socat',
    'sudo',
    'tgt']
@@ -390,17 +390,19 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 {% endblock %}
 
-{% if base_distro == 'ubuntu' %}
+# FIXME(mgoddard): Some images, including Horizon, depend on Ubuntu having a
-RUN sed -i \
+# site-packages directory.
-        -e "s|\('purelib': '\$base/\)local/\(lib/python\$py_version_short/dist-packages',\)|\1\2|" \
+#{% if base_distro == 'ubuntu' %}
-        -e "s|\('platlib': '\$platbase/\)local/\(lib/python\$py_version_short/dist-packages',\)|\1\2|" \
+#RUN sed -i \
-        -e "s|\('headers': '\$base/\)local/\(include/python\$py_version_short/\$dist_name',\)|\1\2|" \
+#        -e "s|\('purelib': '\$base/\)local/\(lib/python\$py_version_short/dist-packages',\)|\1\2|" \
-        -e "s|\('scripts': '\$base/\)local/\(bin',\)|\1\2|" \
+#        -e "s|\('platlib': '\$platbase/\)local/\(lib/python\$py_version_short/dist-packages',\)|\1\2|" \
-        -e "s|\('data'   : '\$base\)/local\(',\)|\1\2|" \
+#        -e "s|\('headers': '\$base/\)local/\(include/python\$py_version_short/\$dist_name',\)|\1\2|" \
-        /usr/lib/python2.7/distutils/command/install.py \
+#        -e "s|\('scripts': '\$base/\)local/\(bin',\)|\1\2|" \
-    && rm -rf /usr/lib/python2.7/site-packages \
+#        -e "s|\('data'   : '\$base\)/local\(',\)|\1\2|" \
-    && ln -s dist-packages /usr/lib/python2.7/site-packages
+#        /usr/lib/python{{ distro_python_version }}/distutils/command/install.py \
-{% endif %}
+#    && rm -rf /usr/lib/python{{ distro_python_version }}/site-packages \
+#    && ln -s dist-packages /usr/lib/{{ distro_python_version }}/site-packages
+#{% endif %}
 
 {# endif base_package_type deb #}
 {% endif %}
@@ -410,6 +412,10 @@ RUN sed -ri '/-session(\s+)optional(\s+)pam_systemd.so/d' /etc/pam.d/system-auth
 {% endif %}
 
 COPY set_configs.py /usr/local/bin/kolla_set_configs
+{% if distro_python_version.startswith('3') %}
+RUN sed -i -e "s+#\!/usr/bin/env python+#\!/usr/bin/env python3+g" /usr/local/bin/kolla_set_configs
+{% endif %}
+
 COPY start.sh /usr/local/bin/kolla_start
 COPY sudoers /etc/sudoers
 COPY curlrc /root/.curlrc

docker/ceilometer/ceilometer-compute/Dockerfile.j2

@@ -25,7 +25,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
          ] %}
     {% elif base_package_type == 'deb' %}
         {% set ceilometer_compute_packages = [
-            'python-libvirt'
+            'python3-libvirt'
          ] %}
     {% endif %}
 

docker/cinder/cinder-api/Dockerfile.j2

@@ -61,7 +61,7 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \
     {% elif base_package_type == 'deb' %}
         {% set cinder_api_packages = [
                 'apache2',
-                'libapache2-mod-wsgi'
+                'libapache2-mod-wsgi-py3'
         ] %}
 {{ macros.install_packages(cinder_api_packages | customizable("packages")) }}
 RUN echo > /etc/apache2/ports.conf

docker/cinder/cinder-base/Dockerfile.j2

@@ -59,6 +59,9 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
                 'ceph-common',
                 'lvm2',
                 'cryptsetup',
+                'python3-cephfs',
+                'python3-rados',
+                'python3-rbd',
                 'qemu-utils'
         ] %}
     {% endif %}

docker/cinder/cinder-volume/Dockerfile.j2

@@ -33,7 +33,7 @@ RUN sed -i '1 i include /var/lib/cinder/volumes/*' /etc/tgt/tgtd.conf
     {% set cinder_volume_packages = [
             'nfs-common',
             'sysfsutils',
-            'python-rtslib-fb',
+            'python3-rtslib-fb',
             'targetcli-fb',
             'thin-provisioning-tools',
             'tgt'

docker/cloudkitty/cloudkitty-api/Dockerfile.j2

@@ -48,7 +48,7 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \
     {% elif base_package_type == 'deb' %}
         {% set cloudkitty_api_packages = [
                 'apache2',
-                'libapache2-mod-wsgi'
+                'libapache2-mod-wsgi-py3'
         ] %}
 {{ macros.install_packages(cloudkitty_api_packages | customizable("packages")) }}
 RUN echo > /etc/apache2/ports.conf

docker/freezer/freezer-api/Dockerfile.j2

@@ -30,7 +30,7 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \
     {% elif base_package_type == 'deb' %}
         {% set freezer_api_packages = [
             'apache2',
-            'libapache2-mod-wsgi'
+            'libapache2-mod-wsgi-py3'
         ] %}
 
 {{ macros.install_packages(freezer_api_packages | customizable("packages")) }}

docker/glance/glance-base/Dockerfile.j2

@@ -38,10 +38,24 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 {{ macros.install_packages(glance_base_packages | customizable("packages")) }}
 
 {% elif install_type == 'source' %}
-    {% set glance_base_packages = [
+    {% if base_package_type == 'rpm' %}
-        'python-rados',
+        {% if distro_python_version.startswith('3') %}
-        'python-rbd'
+        {% set glance_base_packages = [
-    ] %}
+            'python3-rados',
+            'python3-rbd'
+        ] %}
+        {% else %}
+        {% set glance_base_packages = [
+            'python-rados',
+            'python-rbd'
+        ] %}
+        {% endif %}
+    {% elif base_package_type == 'deb' %}
+        {% set glance_base_packages = [
+            'python3-rados',
+            'python3-rbd'
+        ] %}
+    {% endif %}
 
 {{ macros.install_packages(glance_base_packages | customizable("packages")) }}
 

docker/gnocchi/gnocchi-base/Dockerfile.j2

@@ -96,9 +96,9 @@ RUN mkdir -p /var/www/cgi-bin/gnocchi \
 
        {% set gnocchi_base_packages = [
             'apache2',
-            'libapache2-mod-wsgi',
+            'libapache2-mod-wsgi-py3',
             'librados-dev',
-            'python-ldappool'
+            'python3-ldappool'
         ] %}
 
         # on x86-64 pip grabs precompiled numpy/scipy - on aarch64/ppc64le

docker/heat/heat-base/Dockerfile.j2

@@ -56,7 +56,7 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \
 
     {% set heat_base_packages = [
         'apache2',
-        'libapache2-mod-wsgi'
+        'libapache2-mod-wsgi-py3'
     ] %}
 
 {{ macros.install_packages(heat_base_packages | customizable("packages")) }}

docker/helm-repository/Dockerfile.j2

@@ -22,7 +22,7 @@ ENV helm_version=2.3.0 \
     {% set helm_repo_packages = [
         'git',
         'jq',
-        'python-pip'
+        'python3-pip'
      ] %}
 
 {% endif %}

docker/horizon/Dockerfile.j2

@@ -83,7 +83,7 @@ RUN echo > /etc/apache2/ports.conf \
     && cp /usr/share/openstack-dashboard/openstack_dashboard/conf/*.json /etc/openstack-dashboard \
     && cp /usr/share/openstack-dashboard/manage.py /usr/bin/manage.py \
     && rm /etc/apache2/conf-enabled/openstack-dashboard.conf \
-    && for locale in /usr/lib/python{{distro_python_version}}/site-packages/*/locale; do \
+    && for locale in /usr/lib/python3/dist-packages/*/locale; do \
            (cd ${locale%/*} && /usr/bin/python3 /usr/bin/manage.py compilemessages) \
        done
 {% endblock %}
@@ -115,7 +115,7 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \
         {% set horizon_packages = [
                 'apache2',
                 'gettext',
-                'libapache2-mod-wsgi',
+                'libapache2-mod-wsgi-py3',
                 'locales',
                 'tzdata'
         ] %}

docker/horizon/extend_start.sh

@@ -11,7 +11,11 @@ if [[ ${KOLLA_INSTALL_TYPE} == "binary" ]] && [[ "${KOLLA_BASE_DISTRO}" =~ ubunt
 fi
 
 if [[ ${KOLLA_INSTALL_TYPE} == "binary" ]]; then
-    SITE_PACKAGES="/usr/lib/python${KOLLA_DISTRO_PYTHON_VERSION}/site-packages"
+    if [[ ${KOLLA_BASE_DISTRO} == "debian" ]] || [[ ${KOLLA_BASE_DISTRO} == "ubuntu" ]]; then
+        SITE_PACKAGES="/usr/lib/python3/dist-packages"
+    else
+        SITE_PACKAGES="/usr/lib/python${KOLLA_DISTRO_PYTHON_VERSION}/site-packages"
+    fi
 elif [[ ${KOLLA_INSTALL_TYPE} == "source" ]]; then
     SITE_PACKAGES="/var/lib/kolla/venv/lib/python${KOLLA_DISTRO_PYTHON_VERSION}/site-packages"
 fi

docker/ironic/ironic-api/Dockerfile.j2

@@ -39,7 +39,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     {% elif base_package_type == 'deb' %}
         {% set ironic_api_packages = [
             'apache2',
-            'libapache2-mod-wsgi'
+            'libapache2-mod-wsgi-py3'
         ] %}
     {% endif %}
 

docker/keystone/keystone-base/Dockerfile.j2

@@ -85,8 +85,8 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \
             'apache2',
             'libapache2-mod-auth-mellon',
             'libapache2-mod-auth-openidc',
-            'libapache2-mod-wsgi',
+            'libapache2-mod-wsgi-py3',
-            'python-ldappool'
+            'python3-ldappool'
         ] %}
 {{ macros.install_packages(keystone_base_packages | customizable("packages")) }}
 RUN echo > /etc/apache2/ports.conf

docker/kolla-toolbox/Dockerfile.j2

@@ -41,7 +41,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         'libssl-dev',
         'libxslt1-dev',
         'mariadb-client',
-        'python-dev'
+        'python3-dev'
      ] %}
 {% endif %}
 {{ macros.install_packages(kolla_toolbox_packages | customizable("packages")) }}

docker/kube/kube-controller-manager/Dockerfile.j2

@@ -13,7 +13,10 @@ ENV KUBERNETES_COMPONENT=kube-controller-manager
     ] %}
 {% elif base_package_type == 'deb' %}
    {% set ceph_support_packages = [
-        'ceph-common'
+        'ceph-common',
+        'python3-cephfs',
+        'python3-rados',
+        'python3-rbd'
     ] %}
 {% endif %}
 {{ macros.install_packages(ceph_support_packages | customizable("packages")) }}

docker/kubetoolbox/Dockerfile.j2

@@ -11,6 +11,14 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
    'jq'
 ] %}
 
+{% if base_package_type == 'deb' %}
+     {% set kubetoolbox_packages = kubetoolbox_packages + [
+          'python3-cephfs',
+          'python3-rados',
+          'python3-rbd'
+       ] %}
+{% endif %}
+
 {{ macros.install_packages(kubetoolbox_packages | customizable("packages")) }}
 
 {% block kubetoolbox_install_kube_binary %}

docker/manila/manila-api/Dockerfile.j2

@@ -54,7 +54,7 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \
     {% elif base_package_type == 'deb' %}
         {% set manila_api_packages = [
                 'apache2',
-                'libapache2-mod-wsgi'
+                'libapache2-mod-wsgi-py3'
         ] %}
 {{ macros.install_packages(manila_api_packages | customizable("packages")) }}
 RUN echo > /etc/apache2/ports.conf

docker/manila/manila-share/Dockerfile.j2

@@ -33,6 +33,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     {% elif base_package_type == 'deb' %}
         {% set manila_share_packages = [
             'ceph-common',
+            'python3-rados',
+            'python3-rbd',
             'sqlite3'
         ] %}
 

docker/mistral/mistral-api/Dockerfile.j2

@@ -49,7 +49,7 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \
     {% elif base_package_type == 'deb' %}
         {% set mistral_api_packages = [
             'apache2',
-            'libapache2-mod-wsgi'
+            'libapache2-mod-wsgi-py3'
         ] %}
 
 {{ macros.install_packages(mistral_api_packages | customizable("packages")) }}

docker/neutron/neutron-base/Dockerfile.j2

@@ -115,7 +115,7 @@ RUN mkdir -p /usr/share/neutron \
             'keepalived',
             'net-tools',
             'openvswitch-switch',
-            'python-openvswitch',
+            'python3-openvswitch',
             'radvd',
             'uuid-runtime'
         ] %}

docker/nova/nova-api/Dockerfile.j2

@@ -58,7 +58,7 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \
 
         {% set nova_api_packages = [
             'apache2',
-            'libapache2-mod-wsgi'
+            'libapache2-mod-wsgi-py3'
         ] %}
 
 {{ macros.install_packages(nova_api_packages | customizable("packages")) }}

docker/nova/nova-base/Dockerfile.j2

@@ -85,7 +85,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'iptables',
             'openssh-client',
             'openvswitch-switch',
-            'python-libvirt'
+            'python3-libvirt'
         ] %}
         {% if base_arch == 'x86_64' %}
             {% set nova_base_packages = nova_base_packages + [

docker/nova/nova-compute/Dockerfile.j2

@@ -100,9 +100,12 @@ RUN rm -f /etc/nova/nova-compute.conf
             'nfs-common',
             'open-iscsi',
             'parted',
-            'python-guestfs',
+            'python3-cephfs',
-            'python-libvirt',
+            'python3-guestfs',
-            'python-rtslib-fb',
+            'python3-libvirt',
+            'python3-rados',
+            'python3-rbd',
+            'python3-rtslib-fb',
             'qemu-utils',
             'targetcli-fb'
         ] %}

docker/octavia/octavia-api/Dockerfile.j2

@@ -46,7 +46,7 @@ RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \
     {% elif base_package_type == 'deb' %}
         {% set octavia_api_packages = [
             'apache2',
-            'libapache2-mod-wsgi'
+            'libapache2-mod-wsgi-py3'
         ] %}
     {% endif %}
 

docker/openstack-base/Dockerfile.j2

@@ -256,7 +256,7 @@ ENV DEBIAN_FRONTEND noninteractive
             'libyaml-dev',
             'libz-dev',
             'pkg-config',
-            'python-dev',
+            'python3-dev',
             'zip'
         ] %}
 
@@ -295,7 +295,6 @@ ENV DEBIAN_FRONTEND noninteractive
         'eventlet',
         'fasteners',
         'funcsigs',
-        'futures',
         'futurist',
         'gnocchiclient',
         'greenlet',
@@ -379,7 +378,8 @@ ENV DEBIAN_FRONTEND noninteractive
 {% if distro_python_version == '2.7' %}
     {% set openstack_base_pip_packages = openstack_base_pip_packages + [
              'enum34',
-             'functools32'
+             'functools32',
+             'futures'
          ]
     %}
 {% endif %}

docker/panko/panko-base/Dockerfile.j2

@@ -64,8 +64,8 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \
 
        {% set panko_base_packages = [
             'apache2',
-            'libapache2-mod-wsgi',
+            'libapache2-mod-wsgi-py3',
-            'python-ldappool'
+            'python3-ldappool'
         ] %}
 
 {{ macros.install_packages(panko_base_packages | customizable("packages")) }}

docker/placement/placement-api/Dockerfile.j2

@@ -56,7 +56,7 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \
 
         {% set placement_api_packages = [
             'apache2',
-            'libapache2-mod-wsgi'
+            'libapache2-mod-wsgi-py3'
         ] %}
 
 {{ macros.install_packages(placement_api_packages | customizable("packages")) }}

docker/vitrage/vitrage-base/Dockerfile.j2

@@ -56,8 +56,8 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \
 
         {% set vitrage_base_packages = [
             'apache2',
-            'libapache2-mod-wsgi',
+            'libapache2-mod-wsgi-py3',
-            'python-ldappool'
+            'python3-ldappool'
         ] %}
 
 {{ macros.install_packages(vitrage_base_packages | customizable("packages")) }}

docker/zaqar/zaqar-base/Dockerfile.j2

@@ -76,8 +76,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     {% elif base_package_type == 'deb' %}
         {% set zaqar_base_packages = [
             'apache2',
-            'libapache2-mod-wsgi',
+            'libapache2-mod-wsgi-py3',
-            'python-ldappool'
+            'python3-ldappool'
         ] %}
 
 {{ macros.install_packages(zaqar_base_packages | customizable("packages")) }} \

docker/zun/zun-base/Dockerfile.j2

@@ -42,9 +42,9 @@ RUN mkdir -p /var/www/cgi-bin/zun \
 
         {% set zun_base_packages = [
             'apache2',
-            'libapache2-mod-wsgi',
+            'libapache2-mod-wsgi-py3',
             'numactl',
-            'python-ldappool'
+            'python3-ldappool'
         ] %}
 
 {{ macros.install_packages(zun_base_packages | customizable("packages")) }}

kolla/common/config.py

@@ -628,10 +628,6 @@ SOURCES = {
         'type': 'url',
         'location': ('$tarballs_base/networking-sfc/'
                      'networking-sfc-master.tar.gz')},
-    'neutron-server-plugin-networking-infoblox': {
-        'type': 'url',
-        'location': ('$tarballs_base/networking-infoblox/'
-                     'networking-infoblox-master.tar.gz')},
     'neutron-server-plugin-neutron-dynamic-routing': {
         'type': 'url',
         'location': ('$tarballs_base/neutron-dynamic-routing/'

kolla/image/build.py

@@ -711,6 +711,10 @@ class KollaWorker(object):
             self.conf.distro_python_version = "2.7"
         elif self.base in rh_base and self.base_tag.startswith('8'):
             self.conf.distro_python_version = "3.6"
+        elif self.base in ['debian']:
+            self.conf.distro_python_version = "3.7"
+        elif self.base in ['ubuntu']:
+            self.conf.distro_python_version = "3.6"
         else:
             # Assume worst
             self.conf.distro_python_version = "2.7"

kolla/tests/test_build.py

@@ -385,7 +385,7 @@ class KollaWorkerTest(base.TestCase):
         """check distro_python_version for Debian"""
         self.conf.set_override('base', 'debian')
         kolla = build.KollaWorker(self.conf)
-        self.assertEqual('2.7', kolla.distro_python_version)
+        self.assertEqual('3.7', kolla.distro_python_version)
 
     def test_build_distro_python_version_rhel80(self):
         """check distro_python_version for RHEL8.0"""
@@ -405,7 +405,7 @@ class KollaWorkerTest(base.TestCase):
         """check distro_python_version for Ubuntu"""
         self.conf.set_override('base', 'ubuntu')
         kolla = build.KollaWorker(self.conf)
-        self.assertEqual('2.7', kolla.distro_python_version)
+        self.assertEqual('3.6', kolla.distro_python_version)
 
     def test_build_distro_python_version_centos7(self):
         """check distro_python_version for CentOS 7.6.1810"""