From d1702568960bee8524204794c68085529c4b8b08 Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Wed, 13 Feb 2019 09:58:17 -0500 Subject: [PATCH] Introduce clean_package_cache Default to true (to maintain backward compatibility), this parameter will allow to disable the Package cache cleaning when building the Kolla images. In TripleO, we are investigating how we can accelerate the builds by not cleaning the Yum cache and this patch will help us to do so. Also included same for deb package_install method. Co-Authored-By: Jon Schlueter Change-Id: I0adc85302dfe0411d362c899f3e0d6a0019680d4 --- docker/base/Dockerfile.j2 | 12 ++++++------ docker/macros.j2 | 6 ++++-- kolla/common/config.py | 4 +++- kolla/image/build.py | 5 ++++- kolla/template/methods.py | 7 ++++--- .../notes/clean_package_cache-fa08d1808a2f2b49.yaml | 6 ++++++ 6 files changed, 27 insertions(+), 13 deletions(-) create mode 100644 releasenotes/notes/clean_package_cache-fa08d1808a2f2b49.yaml diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index 46c4a8d79b..d126d26091 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -156,9 +156,9 @@ RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 rpm --import {{ key }} \ {% endfor -%} {%- if base_centos_yum_repo_keys|customizable('centos_yum_repo_keys')|length == 0 %}RUN {% else %} && {% endif -%} - yum -y update --security --sec-severity=Important --sec-severity=Critical \ + yum -y update --security --sec-severity=Important --sec-severity=Critical{% if clean_package_cache %} \ && yum clean all \ - && rm -rf /var/cache/yum + && rm -rf /var/cache/yum{% endif %} {% endif %} {# Endif for base_distro centos #} @@ -180,9 +180,9 @@ RUN yum-config-manager --enable rhel-7-server-optional-rpms \ && yum-config-manager --enable rhel-7-server-rhceph-2-osd-rpms \ && yum-config-manager --enable rhel-7-server-rhceph-2-mon-rpms \ && yum-config-manager --enable rhel-7-server-rhceph-2-tools-rpms \ - && yum -y update --security --sec-severity=Important --sec-severity=Critical \ + && yum -y update --security --sec-severity=Important --sec-severity=Critical{% if clean_package_cache %} \ && yum clean all \ - && rm -rf /var/cache/yum + && rm -rf /var/cache/yum{% endif %} {% endblock %} {% endif %} @@ -210,9 +210,9 @@ RUN yum -y install \ && yum-config-manager --enable ol7_optional_latest ol7_addons \ && yum -y install \ yum-plugin-priorities \ - && yum -y update --security --sec-severity=Important --sec-severity=Critical \ + && yum -y update --security --sec-severity=Important --sec-severity=Critical{% if clean_package_cache %} \ && yum clean all \ - && rm -rf /var/cache/yum + && rm -rf /var/cache/yum{% endif %} {% endblock %} {% endif %} diff --git a/docker/macros.j2 b/docker/macros.j2 index 3e1353e71c..dcc9bb0bb6 100644 --- a/docker/macros.j2 +++ b/docker/macros.j2 @@ -1,14 +1,16 @@ {% macro install_packages(packages) -%} {% if packages is defined and packages|length > 0 -%} {% if base_package_type == 'rpm' -%} - RUN {{ distro_package_manager }} -y install {{ packages | join(' ') }} && {{ distro_package_manager }} clean all && rm -rf /var/cache/{{ distro_package_manager }} + RUN {{ distro_package_manager }} -y install {{ packages | join(' ') }} + {%- if clean_package_cache %} \ + && {{ distro_package_manager }} clean all && rm -rf /var/cache/{{ distro_package_manager }}{% endif %} {%- elif base_package_type == 'deb' -%} {#- debian_package_install is a utility method to build up an appropriate set of commands to install packages in a debian-based environment that may include URL links to a .deb package -#} - RUN {{ debian_package_install(packages) }} + RUN {{ debian_package_install(packages, clean_package_cache) }} {%- endif %} {%- endif %} {%- endmacro %} diff --git a/kolla/common/config.py b/kolla/common/config.py index e364ae2904..8a7943c052 100755 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -283,7 +283,9 @@ _BASE_OPTS = [ cfg.BoolOpt('squash-cleanup', default=True, help='Remove source image from Docker after squashing'), cfg.StrOpt('squash-tmp-dir', - help='Temporary directory to be used during squashing') + help='Temporary directory to be used during squashing'), + cfg.BoolOpt('clean_package_cache', default=True, + help='Clean all package cache.') ] diff --git a/kolla/image/build.py b/kolla/image/build.py index 7c85f8cdce..72ee0d7acb 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -713,6 +713,8 @@ class KollaWorker(object): package_manager = 'apt' self.distro_package_manager = package_manager + self.clean_package_cache = self.conf.clean_package_cache + # Determine base packaging type for use in Dockerfiles. if self.conf.base_package_type: self.base_package_type = self.conf.base_package_type @@ -931,7 +933,8 @@ class KollaWorker(object): 'distro_python_version': self.distro_python_version, 'distro_package_manager': self.distro_package_manager, 'rpm_setup': self.rpm_setup, - 'build_date': build_date} + 'build_date': build_date, + 'clean_package_cache': self.clean_package_cache} env = jinja2.Environment( # nosec: not used to render HTML loader=jinja2.FileSystemLoader(self.working_dir)) env.filters.update(self._get_filters()) diff --git a/kolla/template/methods.py b/kolla/template/methods.py index e11fa61cc6..017b219c7a 100644 --- a/kolla/template/methods.py +++ b/kolla/template/methods.py @@ -13,7 +13,7 @@ # limitations under the License. -def debian_package_install(packages): +def debian_package_install(packages, clean_package_cache=True): """Jinja utility method for building debian-based package install command. apt-get is not capable of installing .deb files from a URL and the @@ -48,8 +48,9 @@ def debian_package_install(packages): cmds.append('apt-get -y install --no-install-recommends {}'.format( ' '.join(reg_packages) )) - cmds.append('apt-get clean') - cmds.append('rm -rf /var/lib/apt/lists/*') + if clean_package_cache: + cmds.append('apt-get clean') + cmds.append('rm -rf /var/lib/apt/lists/*') # handle URL packages for url in url_packages: diff --git a/releasenotes/notes/clean_package_cache-fa08d1808a2f2b49.yaml b/releasenotes/notes/clean_package_cache-fa08d1808a2f2b49.yaml new file mode 100644 index 0000000000..84a0800c53 --- /dev/null +++ b/releasenotes/notes/clean_package_cache-fa08d1808a2f2b49.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + clean_package_cache is a parameter (set to True by default for backward + compatibility) that controls the package cache cleaning when building the + images.