Local file based image uploader

This change modifies the "openstack overcloud image upload" to add a
--local argument which will copy the files to /var/lib/ironic/images
instead of uploading them to glance. /var/lib/ironic is chosen as it
is bind-mounted into the ironic containers on the undercloud.

The on-disk layout is prefixed by the architecture (and optionally the
platform) for example:
/var/lib/ironic/images/x86_64/overcloud-full.qcow2
/var/lib/ironic/images/x86_64/overcloud-full.initrd
/var/lib/ironic/images/x86_64/overcloud-full.vmlinuz
/var/lib/ironic/images/power9-ppc64le/overcloud-full.qcow2

When glance is switched off this will become the default way of
preparing image files for deployment.

Blueprint: nova-less-deploy

Change-Id: I4e1b8a7effca21b5aa054a78ddeaae7338ca9d2c

tripleoclient/constants.py

@@ -47,6 +47,7 @@ PUPPET_BASE = "/etc/puppet/"
 STACK_TIMEOUT = 240
 
 IRONIC_HTTP_BOOT_BIND_MOUNT = '/var/lib/ironic/httpboot'
+IRONIC_LOCAL_IMAGE_PATH = '/var/lib/ironic/images'
 
 # The default ffwd upgrade ansible playbooks generated from heat stack output
 FFWD_UPGRADE_PLAYBOOK = "fast_forward_upgrade_playbook.yaml"

tripleoclient/tests/v1/overcloud_image/test_overcloud_image.py

@@ -21,6 +21,7 @@ import tripleo_common.arch
 from tripleoclient.tests import base
 from tripleoclient.tests.fakes import FakeHandle
 from tripleoclient.tests.v1.test_plugin import TestPluginV1
+from tripleoclient import utils as plugin_utils
 from tripleoclient.v1 import overcloud_image
 
 
@@ -150,6 +151,245 @@ class TestBaseClientAdapter(base.TestCommand):
         mock_copy_file.assert_called_once_with(
             self.adapter, 'discimg', 'discimgprod')
 
+    @mock.patch('subprocess.check_call', autospec=True)
+    def test_copy_file(self, mock_subprocess_call):
+        self.adapter._copy_file('/foo.qcow2', 'bar.qcow2')
+        mock_subprocess_call.assert_called_once_with(
+            'sudo cp -f "/foo.qcow2" "bar.qcow2"', shell=True)
+
+    @mock.patch('subprocess.check_call', autospec=True)
+    def test_move_file(self, mock_subprocess_call):
+        self.adapter._move_file('/foo.qcow2', 'bar.qcow2')
+        mock_subprocess_call.assert_called_once_with(
+            'sudo mv "/foo.qcow2" "bar.qcow2"', shell=True)
+
+    @mock.patch('subprocess.check_call', autospec=True)
+    def test_make_dirs(self, mock_subprocess_call):
+        self.adapter._make_dirs('/foo/bar/baz')
+        mock_subprocess_call.assert_called_once_with(
+            'sudo mkdir -m 0775 -p "/foo/bar/baz"', shell=True)
+
+
+class TestFileImageClientAdapter(TestPluginV1):
+
+    def setUp(self):
+        super(TestFileImageClientAdapter, self).setUp()
+        self.updated = []
+        self.adapter = overcloud_image.FileImageClientAdapter(
+            image_path='/home/foo',
+            local_path='/my/images',
+            updated=self.updated
+        )
+        self.image = mock.Mock()
+        self.image.id = 'file:///my/images/x86_64/overcloud-full.qcow2'
+        self.image.name = 'overcloud-full'
+        self.image.checksum = 'asdf'
+        self.image.created_at = '2019-11-14T01:33:39'
+        self.image.size = 982802432
+
+    @mock.patch('os.path.exists')
+    def test_get_image_property(self, mock_exists):
+        mock_exists.side_effect = [
+            True, True, False, False
+        ]
+        image = mock.Mock()
+        image.id = 'file:///my/images/x86_64/overcloud-full.qcow2'
+        # file exists
+        self.assertEqual(
+            'file:///my/images/x86_64/overcloud-full.vmlinuz',
+            self.adapter.get_image_property(image, 'kernel_id')
+        )
+        self.assertEqual(
+            'file:///my/images/x86_64/overcloud-full.initrd',
+            self.adapter.get_image_property(image, 'ramdisk_id')
+        )
+        # file doesn't exist
+        self.assertIsNone(
+            self.adapter.get_image_property(image, 'kernel_id')
+        )
+        self.assertIsNone(
+            self.adapter.get_image_property(image, 'ramdisk_id')
+        )
+        self.assertRaises(ValueError, self.adapter.get_image_property,
+                          image, 'foo')
+
+    def test_paths(self):
+        self.assertEqual(
+            ('/my/images/x86_64',
+             'overcloud-full.vmlinuz'),
+            self.adapter._paths(
+                'overcloud-full',
+                plugin_utils.overcloud_kernel,
+                'x86_64',
+                None
+            )
+        )
+        self.assertEqual(
+            ('/my/images',
+             'overcloud-full.qcow2'),
+            self.adapter._paths(
+                'overcloud-full',
+                plugin_utils.overcloud_image,
+                None,
+                None
+            )
+        )
+        self.assertEqual(
+            ('/my/images/power9-ppc64le',
+             'overcloud-full.initrd'),
+            self.adapter._paths(
+                'overcloud-full',
+                plugin_utils.overcloud_ramdisk,
+                'ppc64le',
+                'power9'
+            )
+        )
+
+    @mock.patch('os.path.exists')
+    @mock.patch('os.stat')
+    @mock.patch('tripleoclient.utils.file_checksum')
+    def test_get_image(self, mock_checksum, mock_stat, mock_exists):
+        mock_exists.return_value = True
+        mock_stat.return_value.st_size = 982802432
+        mock_stat.return_value.st_mtime = 1573695219
+        mock_checksum.return_value = 'asdf'
+
+        image = self.adapter._get_image(
+            '/my/images/x86_64/overcloud-full.qcow2')
+        self.assertEqual(
+            'file:///my/images/x86_64/overcloud-full.qcow2',
+            image.id
+        )
+        self.assertEqual('overcloud-full', image.name)
+        self.assertEqual('asdf', image.checksum)
+        self.assertEqual('2019-11-14T01:33:39', image.created_at)
+        self.assertEqual(982802432, image.size)
+
+    @mock.patch('tripleoclient.utils.file_checksum')
+    @mock.patch('tripleoclient.v1.overcloud_image.'
+                'FileImageClientAdapter._get_image', autospec=True)
+    @mock.patch('tripleoclient.v1.overcloud_image.'
+                'BaseClientAdapter._move_file', autospec=True)
+    def test_image_try_update(self, mock_move, mock_get_image, mock_checksum):
+
+        # existing image with identical checksum
+        mock_checksum.return_value = 'asdf'
+        mock_get_image.return_value = self.image
+        self.assertEqual(
+            self.image,
+            self.adapter._image_try_update(
+                '/home/foo/overcloud-full.qcow2',
+                '/my/images/x86_64/overcloud-full.qcow2'
+            )
+        )
+        self.assertEqual([], self.updated)
+
+        # no image to update
+        mock_get_image.return_value = None
+        self.assertIsNone(
+            self.adapter._image_try_update(
+                '/home/foo/overcloud-full.qcow2',
+                '/my/images/x86_64/overcloud-full.qcow2'
+            )
+        )
+        self.assertEqual([], self.updated)
+
+        # existing image with different checksum, but update_existing=False
+        mock_checksum.return_value = 'fdsa'
+        mock_get_image.return_value = self.image
+        self.assertEqual(
+            self.image,
+            self.adapter._image_try_update(
+                '/home/foo/overcloud-full.qcow2',
+                '/my/images/x86_64/overcloud-full.qcow2'
+            )
+        )
+        self.assertEqual([], self.updated)
+
+        # existing image with different checksum, update_existing=True
+        self.adapter.update_existing = True
+        self.assertIsNone(
+            self.adapter._image_try_update(
+                '/home/foo/overcloud-full.qcow2',
+                '/my/images/x86_64/overcloud-full.qcow2'
+            )
+        )
+        self.assertEqual(
+            ['/my/images/x86_64/overcloud-full.qcow2'],
+            self.updated
+        )
+        mock_move.assert_called_once_with(
+            self.adapter,
+            '/my/images/x86_64/overcloud-full.qcow2',
+            '/my/images/x86_64/overcloud-full_20191114T013339.qcow2'
+        )
+
+    @mock.patch('subprocess.check_call', autospec=True)
+    @mock.patch('tripleoclient.v1.overcloud_image.'
+                'FileImageClientAdapter._get_image', autospec=True)
+    @mock.patch('os.path.isdir')
+    def test_upload_image(self, mock_isdir, mock_get_image,
+                          mock_subprocess_call):
+        mock_isdir.return_value = False
+
+        mock_get_image.return_value = self.image
+
+        result = self.adapter._upload_image(
+            '/home/foo/overcloud-full.qcow2',
+            '/my/images/x86_64/overcloud-full.qcow2'
+        )
+        self.assertEqual(self.image, result)
+        mock_subprocess_call.assert_has_calls([
+            mock.call('sudo mkdir -m 0775 -p "/my/images/x86_64"', shell=True),
+            mock.call('sudo cp -f "/home/foo/overcloud-full.qcow2" '
+                      '"/my/images/x86_64/overcloud-full.qcow2"', shell=True)
+        ])
+
+    @mock.patch('tripleoclient.v1.overcloud_image.'
+                'FileImageClientAdapter._upload_image', autospec=True)
+    @mock.patch('tripleoclient.v1.overcloud_image.'
+                'FileImageClientAdapter._image_try_update', autospec=True)
+    def test_update_or_upload(self, mock_image_try_update, mock_upload_image):
+
+        # image exists
+        mock_image_try_update.return_value = self.image
+        self.assertEqual(
+            self.image,
+            self.adapter.update_or_upload(
+                image_name='overcloud-full',
+                properties={},
+                names_func=plugin_utils.overcloud_image,
+                arch='x86_64'
+            )
+        )
+        mock_upload_image.assert_not_called()
+
+        # image needs uploading
+        mock_image_try_update.return_value = None
+        mock_upload_image.return_value = self.image
+        self.assertEqual(
+            self.image,
+            self.adapter.update_or_upload(
+                image_name='overcloud-full',
+                properties={},
+                names_func=plugin_utils.overcloud_image,
+                arch='x86_64'
+            )
+        )
+        mock_upload_image.assert_called_once_with(
+            self.adapter,
+            '/home/foo/overcloud-full.qcow2',
+            '/my/images/x86_64/overcloud-full.qcow2'
+        )
+        mock_image_try_update.assert_has_calls([
+            mock.call(self.adapter,
+                      '/home/foo/overcloud-full.qcow2',
+                      '/my/images/x86_64/overcloud-full.qcow2'),
+            mock.call(self.adapter,
+                      '/home/foo/overcloud-full.qcow2',
+                      '/my/images/x86_64/overcloud-full.qcow2')
+        ])
+
 
 class TestGlanceClientAdapter(TestPluginV1):
 

tripleoclient/v1/overcloud_image.py

@@ -16,6 +16,8 @@
 from __future__ import print_function
 
 import abc
+import collections
+from datetime import datetime
 import logging
 import os
 import re
@@ -118,8 +120,19 @@ class BaseClientAdapter(object):
         pass
 
     def _copy_file(self, src, dest):
-        subprocess.check_call('sudo cp -f "{0}" "{1}"'.format(src, dest),
+        cmd = 'sudo cp -f "{0}" "{1}"'.format(src, dest)
-                              shell=True)
+        self.log.debug(cmd)
+        subprocess.check_call(cmd, shell=True)
+
+    def _move_file(self, src, dest):
+        cmd = 'sudo mv "{0}" "{1}"'.format(src, dest)
+        self.log.debug(cmd)
+        subprocess.check_call(cmd, shell=True)
+
+    def _make_dirs(self, path):
+        cmd = 'sudo mkdir -m 0775 -p "{0}"'.format(path)
+        self.log.debug(cmd)
+        subprocess.check_call(cmd, shell=True)
 
     def _files_changed(self, filepath1, filepath2):
         return (plugin_utils.file_checksum(filepath1) !=
@@ -154,6 +167,115 @@ class BaseClientAdapter(object):
         return file_descriptor
 
 
+class FileImageClientAdapter(BaseClientAdapter):
+
+    def __init__(self, local_path, **kwargs):
+        super(FileImageClientAdapter, self).__init__(**kwargs)
+        self.local_path = local_path
+
+    def get_image_property(self, image, prop):
+        if prop == 'kernel_id':
+            path = os.path.splitext(image.id)[0] + '.vmlinuz'
+            if os.path.exists(path):
+                return path
+            return None
+        elif prop == 'ramdisk_id':
+            path = os.path.splitext(image.id)[0] + '.initrd'
+            if os.path.exists(path):
+                return path
+            return None
+        raise ValueError('Unsupported property %s' % prop)
+
+    def _print_image_info(self, image):
+        table = PrettyTable(['Path', 'Name', 'Size'])
+        table.add_row([image.id, image.name, image.size])
+        print(table, file=sys.stdout)
+
+    def _paths(self, image_name, names_func, arch, platform):
+        (arch_path, extension) = names_func(
+            image_name, arch=arch, platform=platform, use_subdir=True)
+        image_file = image_name + extension
+
+        dest_dir = os.path.split(
+            os.path.join(self.local_path, arch_path))[0]
+        return (dest_dir, image_file)
+
+    def _get_image(self, path):
+        if not os.path.exists(path):
+            return
+        stat = os.stat(path)
+        created_at = datetime.fromtimestamp(
+            stat.st_mtime).isoformat()
+
+        Image = collections.namedtuple(
+            'Image',
+            'id, name, checksum, created_at, size'
+        )
+        (dir_path, filename) = os.path.split(path)
+        (name, extension) = os.path.splitext(filename)
+        checksum = plugin_utils.file_checksum(path)
+
+        return Image(
+            id='file://%s' % path,
+            name=name,
+            checksum=checksum,
+            created_at=created_at,
+            size=stat.st_size
+        )
+
+    def _image_changed(self, image, filename):
+        return image.checksum != plugin_utils.file_checksum(filename)
+
+    def _image_try_update(self, src_path, dest_path):
+        image = self._get_image(dest_path)
+        if image:
+            if self._image_changed(image, src_path):
+                if self.update_existing:
+                    dest_base, dest_ext = os.path.splitext(dest_path)
+                    dest_datestamp = re.sub(
+                        r'[\-:\.]|(0+$)', '', image.created_at)
+                    dest_mv = dest_base + '_' + dest_datestamp + dest_ext
+                    self._move_file(dest_path, dest_mv)
+
+                    if self.updated is not None:
+                        self.updated.append(dest_path)
+                    return None
+                else:
+                    print('Image "%s" already exists and can be updated'
+                          ' with --update-existing.' % dest_path)
+                    return image
+            else:
+                print('Image "%s" is up-to-date, skipping.' % dest_path)
+                return image
+        else:
+            return None
+
+    def _upload_image(self, src_path, dest_path):
+        dest_dir = os.path.split(dest_path)[0]
+        if not os.path.isdir(dest_dir):
+            self._make_dirs(dest_dir)
+
+        self._copy_file(src_path, dest_path)
+        image = self._get_image(dest_path)
+        print('Image "%s" was copied.' % image.id, file=sys.stdout)
+        self._print_image_info(image)
+        return image
+
+    def update_or_upload(self, image_name, properties, names_func,
+                         arch, platform=None,
+                         disk_format='qcow2', container_format='bare'):
+        (dest_dir, image_file) = self._paths(
+            image_name, names_func, arch, platform)
+
+        src_path = os.path.join(self.image_path, image_file)
+        dest_path = os.path.join(dest_dir, image_file)
+        existing_image = self._image_try_update(src_path, dest_path)
+        if existing_image:
+            return existing_image
+
+        return self._upload_image(src_path, dest_path)
+
+
 class GlanceClientAdapter(BaseClientAdapter):
 
     def __init__(self, client, **kwargs):
@@ -263,13 +385,15 @@ class UploadOvercloudImage(command.Command):
     log = logging.getLogger(__name__ + ".UploadOvercloudImage")
 
     def _get_client_adapter(self, parsed_args):
-        return GlanceClientAdapter(
+        kwargs = {
-            self.app.client_manager.image,
+            'progress': parsed_args.progress,
-            progress=parsed_args.progress,
+            'image_path': parsed_args.image_path,
-            image_path=parsed_args.image_path,
+            'update_existing': parsed_args.update_existing,
-            update_existing=parsed_args.update_existing,
+            'updated': self.updated
-            updated=self.updated
+        }
-        )
+        if parsed_args.local:
+            return FileImageClientAdapter(parsed_args.local_path, **kwargs)
+        return GlanceClientAdapter(self.app.client_manager.image, **kwargs)
 
     def _get_environment_var(self, envvar, default, deprecated=[]):
         for env_key in deprecated:
@@ -353,6 +477,21 @@ class UploadOvercloudImage(command.Command):
             action="store_true",
             default=False,
             help=_('Show progress bar for upload files action'))
+        parser.add_argument(
+            "--local",
+            dest="local",
+            action="store_true",
+            default=False,
+            help=_('Copy files locally, even if there is an image service '
+                   'endpoint'))
+        parser.add_argument(
+            "--local-path",
+            default=self._get_environment_var(
+                'LOCAL_IMAGE_PATH',
+                constants.IRONIC_LOCAL_IMAGE_PATH),
+            help=_("Root directory for image file copy destination when there "
+                   "is no image endpoint, or when --local is specified")
+        )
 
         return parser