Add support for refreshing images by cron

This change adds charm config parameters: `auto-retrofit` and
`frequency` that control if a cron job is added to periodically
refresh retrofitted images based on corresponding glance images.

Closes-Bug: #1937013
Change-Id: I5e12f1a5ea4fe94aebc307484ac68a00a1f3f5fa

src/config.yaml

@@ -56,3 +56,18 @@ options:
       Note that the charm will allways add the ``octavia-diskimage-retrofit``
       tag to created images, this is for convenience and may be used by the
       charm for housekeeping in the future.
+  auto-retrofit:
+    type: boolean
+    default: False
+    description: |
+      Enable cron-based retrofitting.
+      .
+      If True, cron job to automatically retrofit images according to the
+      ``frequency`` param, will be set on the leader unit.
+  frequency:
+    type: string
+    default: "daily"
+    description: |
+      Frequency of the auto retrofit cron job execution
+      .
+      It is one of ['hourly', 'daily', 'weekly']

src/files/auto-retrofit.tmpl

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Copyright 2022 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set_context () {
+  if [ -z "$HOME" ]; then
+      export HOME=/root
+  fi
+
+  if [ -f /etc/profile.d/juju-proxy.sh ]; then
+      source /etc/profile.d/juju-proxy.sh
+  elif [ -f /etc/juju-proxy.conf ]; then
+      source /etc/juju-proxy.conf
+  elif [ -f /home/ubuntu/.juju-proxy ]; then
+      source /home/ubuntu/.juju-proxy
+  fi
+
+  readonly script_file="./files/retrofit-image.py"
+  readonly SCRIPT_COMMAND="/usr/bin/juju-run -u {{ unit_name }} -- $script_file"
+}
+
+retrofit () {
+    logger -p syslog.info "Starting image retrofitting process."
+    if $SCRIPT_COMMAND; then
+        logger -p syslog.info "Image retrofitting process completed successfully."
+    else
+        logger -p syslog.info "Image retrofitting process failed. Return code: $?"
+    fi
+}
+
+main () {
+    set -euo pipefail
+    set_context
+    retrofit
+}
+
+main

src/files/retrofit-image.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+# Copyright 2022 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import traceback
+
+# Load basic layer module from $CHARM_DIR/lib
+sys.path.append('lib')
+from charms.layer import basic
+
+# setup module loading from charm venv
+basic.bootstrap_charm_deps()
+
+import charms.reactive as reactive
+import charmhelpers.core as ch_core
+import charms_openstack.bus
+import charms_openstack.charm as charm
+
+from charm.openstack.octavia_diskimage_retrofit import DestinationImageExists
+
+# load reactive interfaces
+reactive.bus.discover()
+# load Endpoint based interface data
+ch_core.hookenv._run_atstart()
+
+# load charm class
+charms_openstack.bus.discover()
+
+
+def retrofit_image():
+    """Trigger image retrofitting process."""
+    keystone_endpoint = reactive.endpoint_from_flag(
+        'identity-credentials.available')
+    with charm.provide_charm_instance() as instance:
+        try:
+            ch_core.hookenv.log('Starting image retrofitting...',
+                                level=ch_core.hookenv.INFO)
+            instance.retrofit(keystone_endpoint)
+            ch_core.hookenv.log('Image retrofitting completed.',
+                                level=ch_core.hookenv.INFO)
+        except DestinationImageExists as e:
+            ch_core.hookenv.log('Skipping image retrofitting: {}'
+                                .format(str(e)),
+                                level=ch_core.hookenv.INFO)
+
+
+def main(args):
+    try:
+        retrofit_image()
+    except Exception as e:
+        ch_core.hookenv.log('Image retrofitting failed: "{}" "{}"'
+                            .format(str(e), traceback.format_exc()),
+                            level=ch_core.hookenv.ERROR)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

src/lib/charm/openstack/octavia_diskimage_retrofit.py

@@ -15,16 +15,23 @@
 import os
 import subprocess
 import tempfile
+import time
 
 import charms_openstack.adapters
 import charms_openstack.charm
 import charms_openstack.charm.core
 
 import charmhelpers.core as ch_core
+import charmhelpers.core.unitdata as unitdata
 
 import charm.openstack.glance_retrofitter as glance_retrofitter
 
 TMPDIR = '/var/snap/octavia-diskimage-retrofit/common/tmp'
+SCRIPT_WRAPPER_NAME = "auto-retrofit.sh"
+SCRIPT_WRAPPER_TEMPLATE_NAME = "auto-retrofit.tmpl"
+CRON_JOB_LINKNAME = 'auto-retrofit'
+ERR_FILE_EXISTS, ERR_FILE_NOT_EXISTS = 17, 2
+KEY_LAST_RUN_IMAGE_ID, KEY_LAST_RUN_TIME = 'last-image-id', 'last-timestamp'
 
 
 class SourceImageNotFound(Exception):
@@ -42,6 +49,7 @@ class OctaviaDiskimageRetrofitCharm(charms_openstack.charm.OpenStackCharm):
     packages = ['distro-info']
     adapters_class = charms_openstack.adapters.OpenStackRelationAdapters
     required_relations = ['juju-info', 'identity-credentials']
+    db = unitdata.kv()
 
     @property
     def application_version(self):
@@ -83,6 +91,79 @@ class OctaviaDiskimageRetrofitCharm(charms_openstack.charm.OpenStackCharm):
             return 'internalURL'
         return 'publicURL'
 
+    def handle_auto_retrofit(self):
+        """Setup or clear cron job for auto-retrofitting
+
+        Depending on the current values of ``auto-retrofit`` and ``frequency``
+        config options, the cron job with the specified frequency will be
+        added or removed.
+
+        :raises:OSError
+        """
+        target_script = os.path.join("files", SCRIPT_WRAPPER_NAME)
+
+        # By default remove links for current and previous frequency values
+        # on all units.
+        # The second one is current
+        previous_linkname = self.config.previous('frequency')
+        currnet_linkname = self.config['frequency']
+        for freq in [previous_linkname, currnet_linkname]:
+            if freq:
+                linkname = '/etc/cron.{}/{}'.format(
+                    freq,
+                    CRON_JOB_LINKNAME)
+                self.remove_cron_job(linkname)
+
+        # Setup cron job only on the leader
+        if self.config['auto-retrofit'] and ch_core.hookenv.is_leader():
+            try:
+                self.render_shell_wrapper()
+                ch_core.hookenv.log('Creating symlink: "{}" -> "{}"'
+                                    .format(target_script, linkname))
+                os.symlink(os.path.abspath(target_script), linkname)
+            except OSError as ex:
+                if ex.errno == ERR_FILE_EXISTS:
+                    ch_core.hookenv.log('symlink "{}" already exists'
+                                        .format(linkname),
+                                        level=ch_core.hookenv.INFO)
+                else:
+                    raise ex
+
+    def remove_cron_job(self, linkname):
+        """Remove existing cron job for auto retrofitting
+
+        :param linkname: cron job symbolic link to be removed
+        :type linkname: str
+        :raises:OSError
+        """
+        try:
+            ch_core.hookenv.log('Removing symlink: "{}"'
+                                .format(linkname))
+            os.unlink(linkname)
+        except OSError as ex:
+            if ex.errno == ERR_FILE_NOT_EXISTS:
+                ch_core.hookenv.log('symlink "{}" does not exist'
+                                    .format(linkname),
+                                    level=ch_core.hookenv.INFO)
+            else:
+                raise ex
+
+    def render_shell_wrapper(self):
+        """Render shell script that will be run by cron
+        """
+        target_script = os.path.join("files", SCRIPT_WRAPPER_NAME)
+        if not os.path.exists(target_script):
+            unit_name = ch_core.hookenv.local_unit()
+            ch_core.templating.render(
+                source=SCRIPT_WRAPPER_TEMPLATE_NAME,
+                target=target_script,
+                perms=0o755,
+                context={
+                    'unit_name': unit_name,
+                },
+                templates_dir='files'
+            )
+
     def retrofit(self, keystone_endpoint, force=False, image_id=''):
         """Use ``octavia-diskimage-retrofit`` tool to retrofit an image.
 
@@ -234,6 +315,19 @@ class OctaviaDiskimageRetrofitCharm(charms_openstack.charm.OpenStackCharm):
             source_product_name=source_image.product_name or 'custom',
             source_version_name=source_image.version_name or 'custom',
             tags=tags)
+        ts = time.strftime("%x %X")
+        self.db.set(KEY_LAST_RUN_IMAGE_ID, dest_image.id)
+        self.db.set(KEY_LAST_RUN_TIME, ts)
+        self.db.flush()
         ch_core.hookenv.log('Successfully created image "{}" with id "{}"'
                             .format(dest_image.name, dest_image.id),
                             level=ch_core.hookenv.INFO)
+
+    def custom_assess_status_last_check(self):
+        image_id = self.db.get(KEY_LAST_RUN_IMAGE_ID)
+        last_run_time = self.db.get(KEY_LAST_RUN_TIME)
+        if image_id and last_run_time:
+            return 'active', 'Unit is ready (Image {} retrofitting ' \
+                   'completed at {})'.format(image_id, last_run_time)
+        else:
+            return None, None

src/reactive/octavia_diskimage_retrofit_handlers.py

@@ -41,3 +41,10 @@ def request_credentials():
 def credentials_available():
     with charm.provide_charm_instance() as instance:
         instance.assess_status()
+
+
+@reactive.when_any('config.changed.auto-retrofit',
+                   'config.changed.frequency')
+def retrofit_by_cron():
+    with charm.provide_charm_instance() as instance:
+        instance.handle_auto_retrofit()

unit_tests/test_lib_charm_openstack_octavia_diskimage_retrofit.py

@@ -15,6 +15,7 @@
 from unittest import mock
 import os
 import subprocess
+import time
 
 import charms_openstack.test_utils as test_utils
 
@@ -74,7 +75,100 @@ class TestOctaviaDiskimageRetrofitCharm(test_utils.PatchHelper):
             ['distro-info', '-r', '--series', 'focal'],
             universal_newlines=True)
 
+    def test_handle_auto_retrofit(self):
+        current_link = ('/etc/cron.weekly/' + octavia_diskimage_retrofit
+                        .CRON_JOB_LINKNAME)
+        previous_link = ('/etc/cron.hourly/' + octavia_diskimage_retrofit
+                         .CRON_JOB_LINKNAME)
+        installed_script = os.path.join("files", octavia_diskimage_retrofit
+                                        .SCRIPT_WRAPPER_NAME)
+        self.patch_object(
+            octavia_diskimage_retrofit.os, 'symlink')
+        self.patch_object(
+            octavia_diskimage_retrofit.os.path, 'abspath')
+        self.abspath.return_value = installed_script
+        self.patch_target('config')
+        self.config.__getitem__ = lambda _, key: {
+            'auto-retrofit': True,
+            'frequency': 'weekly',
+        }.get(key)
+        self.config.previous.return_value = 'hourly'
+        self.patch_target('remove_cron_job')
+        self.patch_object(octavia_diskimage_retrofit.ch_core.hookenv,
+                          'is_leader')
+        self.patch_target('render_shell_wrapper')
+        self.is_leader.return_value = True
+        self.target.handle_auto_retrofit()
+        self.is_leader.assert_called_once()
+        self.target.remove_cron_job.assert_has_calls([
+            mock.call(previous_link),
+            mock.call(current_link)],
+            any_order=False)
+        self.target.render_shell_wrapper.assert_called_once()
+        self.symlink.assert_called_once_with(installed_script, current_link)
+        os_error = OSError()
+        os_error.errno = octavia_diskimage_retrofit.ERR_FILE_EXISTS
+        self.symlink.reset_mock()
+        self.symlink.side_effect = os_error
+        self.patch_object(octavia_diskimage_retrofit.ch_core.hookenv,
+                          'log')
+        self.target.handle_auto_retrofit()
+        self.assertEquals(self.log.call_args[0][0],
+                          'symlink "' + current_link + '" already exists')
+        self.symlink.reset_mock()
+        self.is_leader.return_value = False
+        self.target.handle_auto_retrofit()
+        self.symlink.assert_not_called()
+
+    def test_remove_cron_job(self):
+        fake_link = 'fake_link'
+        self.patch_object(
+            octavia_diskimage_retrofit.os, 'unlink')
+        self.target.remove_cron_job(fake_link)
+        self.unlink.assert_called_once_with(fake_link)
+        os_error = OSError()
+        os_error.errno = octavia_diskimage_retrofit.ERR_FILE_NOT_EXISTS
+        self.unlink.reset_mock()
+        self.unlink.side_effect = os_error
+        self.patch_object(octavia_diskimage_retrofit.ch_core.hookenv, 'log')
+        self.target.remove_cron_job(fake_link)
+        self.assertEquals(self.log.call_args[0][0],
+                          'symlink "' + fake_link + '" does not exist')
+
+    def test_render_shell_wrapper(self):
+        unit_name = 'octavia-diskimage-retrofit/0'
+        self.patch_object(
+            octavia_diskimage_retrofit.os.path, 'exists')
+        self.exists.return_value = False
+        self.patch_object(octavia_diskimage_retrofit.ch_core.hookenv,
+                          'local_unit')
+        self.patch_object(octavia_diskimage_retrofit.ch_core,
+                          'templating')
+        self.local_unit.return_value = unit_name
+        target = os.path.join("files",
+                              octavia_diskimage_retrofit.SCRIPT_WRAPPER_NAME)
+        render_params = {
+            'source': octavia_diskimage_retrofit.SCRIPT_WRAPPER_TEMPLATE_NAME,
+            'target': target,
+            'perms': 0o755,
+            'context': {
+                'unit_name': unit_name
+            },
+            'templates_dir': 'files'
+        }
+        self.target.render_shell_wrapper()
+        self.local_unit.assert_called_once()
+        self.templating.render.assert_called_once_with(**render_params)
+        self.exists.reset_mock()
+        self.exists.return_value = True
+        self.local_unit.reset_mock()
+        self.templating.render.reset_mock()
+        self.target.render_shell_wrapper()
+        self.local_unit.assert_not_called()
+        self.templating.render.assert_not_called()
+
     def test_retrofit(self):
+        timestamp = '03/07/22 15:54:22'
         self.patch_object(octavia_diskimage_retrofit, 'glance_retrofitter')
         glance = mock.MagicMock()
         self.glance_retrofitter.get_glance_client.return_value = glance
@@ -155,6 +249,9 @@ class TestOctaviaDiskimageRetrofitCharm(test_utils.PatchHelper):
             self.glance_retrofitter.find_destination_image.return_value = \
                 []
             self.hookenv.env_proxy_settings.return_value = proxy_envvars
+            self.patch_target('db')
+            self.patch_object(time, 'strftime')
+            self.strftime.return_value = timestamp
             self.target.retrofit('aKeystone')
             self.get_ubuntu_release.assert_called_once_with(series='bionic')
 
@@ -193,3 +290,11 @@ class TestOctaviaDiskimageRetrofitCharm(test_utils.PatchHelper):
                 source_product_name='aProductName',
                 source_version_name='aVersionName',
                 tags=['octavia-diskimage-retrofit', 'octavia-amphora'])
+            self.strftime.assert_called_once()
+            self.db.set.assert_has_calls([
+                mock.call(octavia_diskimage_retrofit.KEY_LAST_RUN_IMAGE_ID,
+                          'aId'),
+                mock.call(octavia_diskimage_retrofit.KEY_LAST_RUN_TIME,
+                          timestamp)
+            ])
+            self.db.flush.assert_called_once()

unit_tests/test_reactive_octavia_diskimage_retrofit_handlers.py

@@ -37,6 +37,11 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks):
                 'build': (
                     'octavia-diskimage-retrofit.build',),
             },
+            'when_any': {
+                'retrofit_by_cron': (
+                    'config.changed.auto-retrofit',
+                    'config.changed.frequency',),
+            },
             'when_not': {
                 'request_credentials': (
                     'identity-credentials.available',),