Local file based image uploader
This change modifies the "openstack overcloud image upload" to add a --local argument which will copy the files to /var/lib/ironic/images instead of uploading them to glance. /var/lib/ironic is chosen as it is bind-mounted into the ironic containers on the undercloud. The on-disk layout is prefixed by the architecture (and optionally the platform) for example: /var/lib/ironic/images/x86_64/overcloud-full.qcow2 /var/lib/ironic/images/x86_64/overcloud-full.initrd /var/lib/ironic/images/x86_64/overcloud-full.vmlinuz /var/lib/ironic/images/power9-ppc64le/overcloud-full.qcow2 When glance is switched off this will become the default way of preparing image files for deployment. Blueprint: nova-less-deploy Change-Id: I4e1b8a7effca21b5aa054a78ddeaae7338ca9d2c
This commit is contained in:
Steve Baker
parent
4c4e6aacd2
commit
5304c34324
@ -47,6 +47,7 @@ PUPPET_BASE = "/etc/puppet/"
|
||||
STACK_TIMEOUT = 240
|
||||
|
||||
IRONIC_HTTP_BOOT_BIND_MOUNT = '/var/lib/ironic/httpboot'
|
||||
IRONIC_LOCAL_IMAGE_PATH = '/var/lib/ironic/images'
|
||||
|
||||
# The default ffwd upgrade ansible playbooks generated from heat stack output
|
||||
FFWD_UPGRADE_PLAYBOOK = "fast_forward_upgrade_playbook.yaml"
|
||||
|
||||
@ -21,6 +21,7 @@ import tripleo_common.arch
|
||||
from tripleoclient.tests import base
|
||||
from tripleoclient.tests.fakes import FakeHandle
|
||||
from tripleoclient.tests.v1.test_plugin import TestPluginV1
|
||||
from tripleoclient import utils as plugin_utils
|
||||
from tripleoclient.v1 import overcloud_image
|
||||
|
||||
|
||||
@ -150,6 +151,245 @@ class TestBaseClientAdapter(base.TestCommand):
|
||||
mock_copy_file.assert_called_once_with(
|
||||
self.adapter, 'discimg', 'discimgprod')
|
||||
|
||||
@mock.patch('subprocess.check_call', autospec=True)
|
||||
def test_copy_file(self, mock_subprocess_call):
|
||||
self.adapter._copy_file('/foo.qcow2', 'bar.qcow2')
|
||||
mock_subprocess_call.assert_called_once_with(
|
||||
'sudo cp -f "/foo.qcow2" "bar.qcow2"', shell=True)
|
||||
|
||||
@mock.patch('subprocess.check_call', autospec=True)
|
||||
def test_move_file(self, mock_subprocess_call):
|
||||
self.adapter._move_file('/foo.qcow2', 'bar.qcow2')
|
||||
mock_subprocess_call.assert_called_once_with(
|
||||
'sudo mv "/foo.qcow2" "bar.qcow2"', shell=True)
|
||||
|
||||
@mock.patch('subprocess.check_call', autospec=True)
|
||||
def test_make_dirs(self, mock_subprocess_call):
|
||||
self.adapter._make_dirs('/foo/bar/baz')
|
||||
mock_subprocess_call.assert_called_once_with(
|
||||
'sudo mkdir -m 0775 -p "/foo/bar/baz"', shell=True)
|
||||
|
||||
|
||||
class TestFileImageClientAdapter(TestPluginV1):
|
||||
|
||||
def setUp(self):
|
||||
super(TestFileImageClientAdapter, self).setUp()
|
||||
self.updated = []
|
||||
self.adapter = overcloud_image.FileImageClientAdapter(
|
||||
image_path='/home/foo',
|
||||
local_path='/my/images',
|
||||
updated=self.updated
|
||||
)
|
||||
self.image = mock.Mock()
|
||||
self.image.id = 'file:///my/images/x86_64/overcloud-full.qcow2'
|
||||
self.image.name = 'overcloud-full'
|
||||
self.image.checksum = 'asdf'
|
||||
self.image.created_at = '2019-11-14T01:33:39'
|
||||
self.image.size = 982802432
|
||||
|
||||
@mock.patch('os.path.exists')
|
||||
def test_get_image_property(self, mock_exists):
|
||||
mock_exists.side_effect = [
|
||||
True, True, False, False
|
||||
]
|
||||
image = mock.Mock()
|
||||
image.id = 'file:///my/images/x86_64/overcloud-full.qcow2'
|
||||
# file exists
|
||||
self.assertEqual(
|
||||
'file:///my/images/x86_64/overcloud-full.vmlinuz',
|
||||
self.adapter.get_image_property(image, 'kernel_id')
|
||||
)
|
||||
self.assertEqual(
|
||||
'file:///my/images/x86_64/overcloud-full.initrd',
|
||||
self.adapter.get_image_property(image, 'ramdisk_id')
|
||||
)
|
||||
# file doesn't exist
|
||||
self.assertIsNone(
|
||||
self.adapter.get_image_property(image, 'kernel_id')
|
||||
)
|
||||
self.assertIsNone(
|
||||
self.adapter.get_image_property(image, 'ramdisk_id')
|
||||
)
|
||||
self.assertRaises(ValueError, self.adapter.get_image_property,
|
||||
image, 'foo')
|
||||
|
||||
def test_paths(self):
|
||||
self.assertEqual(
|
||||
('/my/images/x86_64',
|
||||
'overcloud-full.vmlinuz'),
|
||||
self.adapter._paths(
|
||||
'overcloud-full',
|
||||
plugin_utils.overcloud_kernel,
|
||||
'x86_64',
|
||||
None
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
('/my/images',
|
||||
'overcloud-full.qcow2'),
|
||||
self.adapter._paths(
|
||||
'overcloud-full',
|
||||
plugin_utils.overcloud_image,
|
||||
None,
|
||||
None
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
('/my/images/power9-ppc64le',
|
||||
'overcloud-full.initrd'),
|
||||
self.adapter._paths(
|
||||
'overcloud-full',
|
||||
plugin_utils.overcloud_ramdisk,
|
||||
'ppc64le',
|
||||
'power9'
|
||||
)
|
||||
)
|
||||
|
||||
@mock.patch('os.path.exists')
|
||||
@mock.patch('os.stat')
|
||||
@mock.patch('tripleoclient.utils.file_checksum')
|
||||
def test_get_image(self, mock_checksum, mock_stat, mock_exists):
|
||||
mock_exists.return_value = True
|
||||
mock_stat.return_value.st_size = 982802432
|
||||
mock_stat.return_value.st_mtime = 1573695219
|
||||
mock_checksum.return_value = 'asdf'
|
||||
|
||||
image = self.adapter._get_image(
|
||||
'/my/images/x86_64/overcloud-full.qcow2')
|
||||
self.assertEqual(
|
||||
'file:///my/images/x86_64/overcloud-full.qcow2',
|
||||
image.id
|
||||
)
|
||||
self.assertEqual('overcloud-full', image.name)
|
||||
self.assertEqual('asdf', image.checksum)
|
||||
self.assertEqual('2019-11-14T01:33:39', image.created_at)
|
||||
self.assertEqual(982802432, image.size)
|
||||
|
||||
@mock.patch('tripleoclient.utils.file_checksum')
|
||||
@mock.patch('tripleoclient.v1.overcloud_image.'
|
||||
'FileImageClientAdapter._get_image', autospec=True)
|
||||
@mock.patch('tripleoclient.v1.overcloud_image.'
|
||||
'BaseClientAdapter._move_file', autospec=True)
|
||||
def test_image_try_update(self, mock_move, mock_get_image, mock_checksum):
|
||||
|
||||
# existing image with identical checksum
|
||||
mock_checksum.return_value = 'asdf'
|
||||
mock_get_image.return_value = self.image
|
||||
self.assertEqual(
|
||||
self.image,
|
||||
self.adapter._image_try_update(
|
||||
'/home/foo/overcloud-full.qcow2',
|
||||
'/my/images/x86_64/overcloud-full.qcow2'
|
||||
)
|
||||
)
|
||||
self.assertEqual([], self.updated)
|
||||
|
||||
# no image to update
|
||||
mock_get_image.return_value = None
|
||||
self.assertIsNone(
|
||||
self.adapter._image_try_update(
|
||||
'/home/foo/overcloud-full.qcow2',
|
||||
'/my/images/x86_64/overcloud-full.qcow2'
|
||||
)
|
||||
)
|
||||
self.assertEqual([], self.updated)
|
||||
|
||||
# existing image with different checksum, but update_existing=False
|
||||
mock_checksum.return_value = 'fdsa'
|
||||
mock_get_image.return_value = self.image
|
||||
self.assertEqual(
|
||||
self.image,
|
||||
self.adapter._image_try_update(
|
||||
'/home/foo/overcloud-full.qcow2',
|
||||
'/my/images/x86_64/overcloud-full.qcow2'
|
||||
)
|
||||
)
|
||||
self.assertEqual([], self.updated)
|
||||
|
||||
# existing image with different checksum, update_existing=True
|
||||
self.adapter.update_existing = True
|
||||
self.assertIsNone(
|
||||
self.adapter._image_try_update(
|
||||
'/home/foo/overcloud-full.qcow2',
|
||||
'/my/images/x86_64/overcloud-full.qcow2'
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
['/my/images/x86_64/overcloud-full.qcow2'],
|
||||
self.updated
|
||||
)
|
||||
mock_move.assert_called_once_with(
|
||||
self.adapter,
|
||||
'/my/images/x86_64/overcloud-full.qcow2',
|
||||
'/my/images/x86_64/overcloud-full_20191114T013339.qcow2'
|
||||
)
|
||||
|
||||
@mock.patch('subprocess.check_call', autospec=True)
|
||||
@mock.patch('tripleoclient.v1.overcloud_image.'
|
||||
'FileImageClientAdapter._get_image', autospec=True)
|
||||
@mock.patch('os.path.isdir')
|
||||
def test_upload_image(self, mock_isdir, mock_get_image,
|
||||
mock_subprocess_call):
|
||||
mock_isdir.return_value = False
|
||||
|
||||
mock_get_image.return_value = self.image
|
||||
|
||||
result = self.adapter._upload_image(
|
||||
'/home/foo/overcloud-full.qcow2',
|
||||
'/my/images/x86_64/overcloud-full.qcow2'
|
||||
)
|
||||
self.assertEqual(self.image, result)
|
||||
mock_subprocess_call.assert_has_calls([
|
||||
mock.call('sudo mkdir -m 0775 -p "/my/images/x86_64"', shell=True),
|
||||
mock.call('sudo cp -f "/home/foo/overcloud-full.qcow2" '
|
||||
'"/my/images/x86_64/overcloud-full.qcow2"', shell=True)
|
||||
])
|
||||
|
||||
@mock.patch('tripleoclient.v1.overcloud_image.'
|
||||
'FileImageClientAdapter._upload_image', autospec=True)
|
||||
@mock.patch('tripleoclient.v1.overcloud_image.'
|
||||
'FileImageClientAdapter._image_try_update', autospec=True)
|
||||
def test_update_or_upload(self, mock_image_try_update, mock_upload_image):
|
||||
|
||||
# image exists
|
||||
mock_image_try_update.return_value = self.image
|
||||
self.assertEqual(
|
||||
self.image,
|
||||
self.adapter.update_or_upload(
|
||||
image_name='overcloud-full',
|
||||
properties={},
|
||||
names_func=plugin_utils.overcloud_image,
|
||||
arch='x86_64'
|
||||
)
|
||||
)
|
||||
mock_upload_image.assert_not_called()
|
||||
|
||||
# image needs uploading
|
||||
mock_image_try_update.return_value = None
|
||||
mock_upload_image.return_value = self.image
|
||||
self.assertEqual(
|
||||
self.image,
|
||||
self.adapter.update_or_upload(
|
||||
image_name='overcloud-full',
|
||||
properties={},
|
||||
names_func=plugin_utils.overcloud_image,
|
||||
arch='x86_64'
|
||||
)
|
||||
)
|
||||
mock_upload_image.assert_called_once_with(
|
||||
self.adapter,
|
||||
'/home/foo/overcloud-full.qcow2',
|
||||
'/my/images/x86_64/overcloud-full.qcow2'
|
||||
)
|
||||
mock_image_try_update.assert_has_calls([
|
||||
mock.call(self.adapter,
|
||||
'/home/foo/overcloud-full.qcow2',
|
||||
'/my/images/x86_64/overcloud-full.qcow2'),
|
||||
mock.call(self.adapter,
|
||||
'/home/foo/overcloud-full.qcow2',
|
||||
'/my/images/x86_64/overcloud-full.qcow2')
|
||||
])
|
||||
|
||||
|
||||
class TestGlanceClientAdapter(TestPluginV1):
|
||||
|
||||
|
||||
@ -16,6 +16,8 @@
|
||||
from __future__ import print_function
|
||||
|
||||
import abc
|
||||
import collections
|
||||
from datetime import datetime
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
@ -118,8 +120,19 @@ class BaseClientAdapter(object):
|
||||
pass
|
||||
|
||||
def _copy_file(self, src, dest):
|
||||
subprocess.check_call('sudo cp -f "{0}" "{1}"'.format(src, dest),
|
||||
shell=True)
|
||||
cmd = 'sudo cp -f "{0}" "{1}"'.format(src, dest)
|
||||
self.log.debug(cmd)
|
||||
subprocess.check_call(cmd, shell=True)
|
||||
|
||||
def _move_file(self, src, dest):
|
||||
cmd = 'sudo mv "{0}" "{1}"'.format(src, dest)
|
||||
self.log.debug(cmd)
|
||||
subprocess.check_call(cmd, shell=True)
|
||||
|
||||
def _make_dirs(self, path):
|
||||
cmd = 'sudo mkdir -m 0775 -p "{0}"'.format(path)
|
||||
self.log.debug(cmd)
|
||||
subprocess.check_call(cmd, shell=True)
|
||||
|
||||
def _files_changed(self, filepath1, filepath2):
|
||||
return (plugin_utils.file_checksum(filepath1) !=
|
||||
@ -154,6 +167,115 @@ class BaseClientAdapter(object):
|
||||
return file_descriptor
|
||||
|
||||
|
||||
class FileImageClientAdapter(BaseClientAdapter):
|
||||
|
||||
def __init__(self, local_path, **kwargs):
|
||||
super(FileImageClientAdapter, self).__init__(**kwargs)
|
||||
self.local_path = local_path
|
||||
|
||||
def get_image_property(self, image, prop):
|
||||
if prop == 'kernel_id':
|
||||
path = os.path.splitext(image.id)[0] + '.vmlinuz'
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
return None
|
||||
elif prop == 'ramdisk_id':
|
||||
path = os.path.splitext(image.id)[0] + '.initrd'
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
return None
|
||||
raise ValueError('Unsupported property %s' % prop)
|
||||
|
||||
def _print_image_info(self, image):
|
||||
table = PrettyTable(['Path', 'Name', 'Size'])
|
||||
table.add_row([image.id, image.name, image.size])
|
||||
print(table, file=sys.stdout)
|
||||
|
||||
def _paths(self, image_name, names_func, arch, platform):
|
||||
(arch_path, extension) = names_func(
|
||||
image_name, arch=arch, platform=platform, use_subdir=True)
|
||||
image_file = image_name + extension
|
||||
|
||||
dest_dir = os.path.split(
|
||||
os.path.join(self.local_path, arch_path))[0]
|
||||
return (dest_dir, image_file)
|
||||
|
||||
def _get_image(self, path):
|
||||
if not os.path.exists(path):
|
||||
return
|
||||
stat = os.stat(path)
|
||||
created_at = datetime.fromtimestamp(
|
||||
stat.st_mtime).isoformat()
|
||||
|
||||
Image = collections.namedtuple(
|
||||
'Image',
|
||||
'id, name, checksum, created_at, size'
|
||||
)
|
||||
(dir_path, filename) = os.path.split(path)
|
||||
(name, extension) = os.path.splitext(filename)
|
||||
checksum = plugin_utils.file_checksum(path)
|
||||
|
||||
return Image(
|
||||
id='file://%s' % path,
|
||||
name=name,
|
||||
checksum=checksum,
|
||||
created_at=created_at,
|
||||
size=stat.st_size
|
||||
)
|
||||
|
||||
def _image_changed(self, image, filename):
|
||||
return image.checksum != plugin_utils.file_checksum(filename)
|
||||
|
||||
def _image_try_update(self, src_path, dest_path):
|
||||
image = self._get_image(dest_path)
|
||||
if image:
|
||||
if self._image_changed(image, src_path):
|
||||
if self.update_existing:
|
||||
dest_base, dest_ext = os.path.splitext(dest_path)
|
||||
dest_datestamp = re.sub(
|
||||
r'[\-:\.]|(0+$)', '', image.created_at)
|
||||
dest_mv = dest_base + '_' + dest_datestamp + dest_ext
|
||||
self._move_file(dest_path, dest_mv)
|
||||
|
||||
if self.updated is not None:
|
||||
self.updated.append(dest_path)
|
||||
return None
|
||||
else:
|
||||
print('Image "%s" already exists and can be updated'
|
||||
' with --update-existing.' % dest_path)
|
||||
return image
|
||||
else:
|
||||
print('Image "%s" is up-to-date, skipping.' % dest_path)
|
||||
return image
|
||||
else:
|
||||
return None
|
||||
|
||||
def _upload_image(self, src_path, dest_path):
|
||||
dest_dir = os.path.split(dest_path)[0]
|
||||
if not os.path.isdir(dest_dir):
|
||||
self._make_dirs(dest_dir)
|
||||
|
||||
self._copy_file(src_path, dest_path)
|
||||
image = self._get_image(dest_path)
|
||||
print('Image "%s" was copied.' % image.id, file=sys.stdout)
|
||||
self._print_image_info(image)
|
||||
return image
|
||||
|
||||
def update_or_upload(self, image_name, properties, names_func,
|
||||
arch, platform=None,
|
||||
disk_format='qcow2', container_format='bare'):
|
||||
(dest_dir, image_file) = self._paths(
|
||||
image_name, names_func, arch, platform)
|
||||
|
||||
src_path = os.path.join(self.image_path, image_file)
|
||||
dest_path = os.path.join(dest_dir, image_file)
|
||||
existing_image = self._image_try_update(src_path, dest_path)
|
||||
if existing_image:
|
||||
return existing_image
|
||||
|
||||
return self._upload_image(src_path, dest_path)
|
||||
|
||||
|
||||
class GlanceClientAdapter(BaseClientAdapter):
|
||||
|
||||
def __init__(self, client, **kwargs):
|
||||
@ -263,13 +385,15 @@ class UploadOvercloudImage(command.Command):
|
||||
log = logging.getLogger(__name__ + ".UploadOvercloudImage")
|
||||
|
||||
def _get_client_adapter(self, parsed_args):
|
||||
return GlanceClientAdapter(
|
||||
self.app.client_manager.image,
|
||||
progress=parsed_args.progress,
|
||||
image_path=parsed_args.image_path,
|
||||
update_existing=parsed_args.update_existing,
|
||||
updated=self.updated
|
||||
)
|
||||
kwargs = {
|
||||
'progress': parsed_args.progress,
|
||||
'image_path': parsed_args.image_path,
|
||||
'update_existing': parsed_args.update_existing,
|
||||
'updated': self.updated
|
||||
}
|
||||
if parsed_args.local:
|
||||
return FileImageClientAdapter(parsed_args.local_path, **kwargs)
|
||||
return GlanceClientAdapter(self.app.client_manager.image, **kwargs)
|
||||
|
||||
def _get_environment_var(self, envvar, default, deprecated=[]):
|
||||
for env_key in deprecated:
|
||||
@ -353,6 +477,21 @@ class UploadOvercloudImage(command.Command):
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=_('Show progress bar for upload files action'))
|
||||
parser.add_argument(
|
||||
"--local",
|
||||
dest="local",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=_('Copy files locally, even if there is an image service '
|
||||
'endpoint'))
|
||||
parser.add_argument(
|
||||
"--local-path",
|
||||
default=self._get_environment_var(
|
||||
'LOCAL_IMAGE_PATH',
|
||||
constants.IRONIC_LOCAL_IMAGE_PATH),
|
||||
help=_("Root directory for image file copy destination when there "
|
||||
"is no image endpoint, or when --local is specified")
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user