From 908875eeabae5a0c90da7b1a7cae41c36d3df51a Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Thu, 19 Sep 2019 11:46:29 +0200 Subject: [PATCH] Add support for yum caching when buildah updates When yum_cache is set, that directory will be automatically picked as either the source or destination for the containers being updated as the following: * when that host directory is missing (or empty), the container under update will start populating it, while it gets updated. That path going to become the lower overlay FS layer for future use among other containers under concurrent yum update executions. * when the yum_cache directory exists and is not empty, it will be bind-mounted as an upper overlay FS layer for other containers under update. So those can benefit from some of the already prefetched contents in its yum cache without data races or conflicts when concurrently accessing the cached data. Overlaying ensures data safety as each container can only see the lower layer of the overlay, while storing its local changes on top of it as an ephemeral. The yum_cache directory existance & non-emptiness facts act as a single mutex, which only grants a dedicated writing access to the lower layer to a single "populating" container at a time. This behavior may be forcefully reset via the force_purge_yum_cache flag. The container update playbook invoked with it, instantly creates a new populator and creates a fresh yum cache. Note that the 100% saturation of the cache is only expected, when the populating container finishes its execution. The feature can be used only for buildah in yum update scenarios using yum or dnf. Change-Id: I30c6dd12454a0b1781803ab16ef79b5914178114 Related-bug: #1844446 Signed-off-by: Bogdan Dobrelya --- README.rst | 13 +++- defaults/main.yml | 1 + tasks/yum_update_buildah.yml | 65 ++++++++++++++++++- .../yum_update.sh.j2 | 6 +- 4 files changed, 80 insertions(+), 5 deletions(-) rename files/yum_update.sh => templates/yum_update.sh.j2 (94%) diff --git a/README.rst b/README.rst index a7715e7..182a00f 100644 --- a/README.rst +++ b/README.rst @@ -54,7 +54,15 @@ Role Variables * - `container_build_tool` - `docker` - See modify image variables - + * - `yum_cache` + - `None` + - Optional path to the host directory for yum cache during the update. + Requires an overlay-enabled FS that also supports SE context relabling. + Works only with container_build_tool=buildah. + * - `force_purge_yum_cache` + - `False` + - Optional argument that tells buildah to forcefully re-populate the yum + cache with new contents. .. list-table:: Variables used for yum install :widths: auto @@ -186,7 +194,8 @@ of an `import_role` parameter. compare_host_packages: true yum_repos_dir_path: /etc/yum.repos.d modified_append_tag: updated - container_build_tool: docker # or buildah + container_build_tool: buildah # or docker + yum_cache: /tmp/containers-updater/yum_cache Note, if you have a locally installed gating repo, you can add ``update_repo: gating-repo``. This may be the case for the consequent in-place diff --git a/defaults/main.yml b/defaults/main.yml index 9937e08..d46f9b9 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -4,3 +4,4 @@ container_build_tool: 'docker' python_dir: [] refspecs: [] yum_packages: [] +force_purge_yum_cache: false diff --git a/tasks/yum_update_buildah.yml b/tasks/yum_update_buildah.yml index af5950d..ab06ec2 100644 --- a/tasks/yum_update_buildah.yml +++ b/tasks/yum_update_buildah.yml @@ -22,9 +22,21 @@ state: file register: yum_update +- name: Identify the primary pkg mgr (dnf or yum) + shell: command -v dnf || command -v yum + register: pkg_mgr_output + +- name: Set fact for the used pkg mgr binary + set_fact: + pkg_mgr: "{{ pkg_mgr_output.stdout }}" + +- name: Set fact for the used pkg mgr cache path + set_fact: + cache_path: /var/cache/{{ pkg_mgr.split('/')[-1] }} + - name: Prepare yum_update.sh script - copy: - src: files/yum_update.sh + template: + src: yum_update.sh.j2 dest: "{{ yum_update.path }}" mode: 0755 @@ -34,12 +46,58 @@ chdir: "{{ yum_repos_dir_path }}" register: file_repos +- name: Define bind-mount modes for yum cache to be populated or used + when: yum_cache is defined and yum_cache + block: + - name: Check for the pkg mgr cache existence + stat: + path: "{{ yum_cache }}" + get_checksum: false + register: yum_cache_stat + + - name: Check for the pkg mgr cach contents + shell: ls -A {{ yum_cache }} + register: yum_cache_contents + when: yum_cache_stat.stat.exists|default() + + - name: Purge the pgh mgr cache on host + file: + path: "{{ yum_cache }}" + state: absent + when: + - force_purge_yum_cache|bool + - yum_cache_contents is defined + - yum_cache_contents.stdout + + - name: Ensure the pgh mgr cache path exists + file: + path: "{{ yum_cache }}" + state: directory + mode: 0755 + setype: svirt_sandbox_file_t + when: not yum_cache_stat.stat.exists|default() + + - name: Use the pre-populated non-empty cache as an overlay fs + set_fact: + cache_volume: "{{ yum_cache }}:{{ cache_path }}:O" + when: + - yum_cache_stat.stat.exists|default() + - yum_cache_contents.stdout + + - name: Define the cache populating mode otherwise + set_fact: + cache_volume: "{{ yum_cache }}:{{ cache_path }}:rw,z" + when: cache_volume is not defined + - block: - name: Run yum_update.sh command: > buildah run --volume {{ yum_update.path }}:/tmp/yum_update.sh --volume {{ yum_repos_dir_path }}:/etc/yum.repos.d + {% if cache_volume is defined and cache_volume %} + --volume {{ cache_volume }} + {% endif %} {% for repo in file_repos.stdout_lines %} {% if repo|exists %} --volume {{ repo }}:{{ repo }} @@ -57,6 +115,9 @@ buildah --debug run --volume {{ yum_update.path }}:/tmp/yum_update.sh --volume {{ yum_repos_dir_path }}:/etc/yum.repos.d + {% if cache_volume is defined and cache_volume %} + --volume {{ cache_volume }} + {% endif %} {% for repo in file_repos.stdout_lines %} {% if repo|exists %} --volume {{ repo }}:{{ repo }} diff --git a/files/yum_update.sh b/templates/yum_update.sh.j2 similarity index 94% rename from files/yum_update.sh rename to templates/yum_update.sh.j2 index 3277728..c0e5c06 100755 --- a/files/yum_update.sh +++ b/templates/yum_update.sh.j2 @@ -2,7 +2,7 @@ set -eou pipefail -PKG="$(command -v dnf || command -v yum)" +PKG={{ pkg_mgr }} PKG_MGR="$(echo ${PKG:(-3)})" if [ $PKG_MGR == "dnf" ]; then @@ -44,4 +44,8 @@ if $(! echo $installed | grep -qw $plugin) && $($PKG list available $plugin >/de fi $PKG -y update $packages_for_update +{% if yum_cache is defined and yum_cache %} +sync +{% else %} rm -rf /var/cache/$PKG_MGR +{% endif %}