# Copyright 2015 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # import abc import collections from datetime import datetime import logging import os import re import subprocess import sys from glanceclient.common.progressbar import VerboseFileWrapper from keystoneauth1.exceptions import catalog as exc_catalog from osc_lib import exceptions from osc_lib.i18n import _ from prettytable import PrettyTable import tripleo_common.arch from tripleo_common.image import build from tripleoclient import command from tripleoclient import constants from tripleoclient import utils as plugin_utils class BuildOvercloudImage(command.Command): """Build images for the overcloud""" auth_required = False log = logging.getLogger(__name__ + ".BuildOvercloudImage") IMAGE_YAML_PATH = "/usr/share/openstack-tripleo-common/image-yaml" DEFAULT_YAML = ['overcloud-images-python3.yaml', 'overcloud-images-centos8.yaml'] REQUIRED_PACKAGES = [ 'openstack-tripleo-common', 'openstack-ironic-python-agent-builder', 'openstack-tripleo-image-elements', 'openstack-tripleo-puppet-elements', 'xfsprogs' ] def get_parser(self, prog_name): parser = super(BuildOvercloudImage, self).get_parser(prog_name) parser.add_argument( "--config-file", dest="config_files", metavar='', default=[], action="append", help=_("YAML config file specifying the image build. May be " "specified multiple times. Order is preserved, and later " "files will override some options in previous files. " "Other options will append."), ) parser.add_argument( "--image-name", dest="image_names", metavar='', default=None, help=_("Name of image to build. May be specified multiple " "times. If unspecified, will build all images in " "given YAML files."), ) parser.add_argument( "--no-skip", dest="skip", action="store_false", default=True, help=_("Skip build if cached image exists."), ) parser.add_argument( "--output-directory", dest="output_directory", default=os.environ.get('TRIPLEO_ROOT', '.'), help=_("Output directory for images. Defaults to $TRIPLEO_ROOT," "or current directory if unset."), ) parser.add_argument( "--temp-dir", dest="temp_dir", default=os.environ.get('TMPDIR', os.getcwd()), help=_("Temporary directory to use when building the images. " "Defaults to $TMPDIR or current directory if unset."), ) return parser def _ensure_packages_installed(self): cmd = ['sudo', 'dnf', 'install', '-y'] + self.REQUIRED_PACKAGES output = plugin_utils.run_command(cmd, name="Install required packages") self.log.info(output) def take_action(self, parsed_args): self.log.debug("take_action(%s)" % parsed_args) self._ensure_packages_installed() if not parsed_args.config_files: parsed_args.config_files = [os.path.join(self.IMAGE_YAML_PATH, f) for f in self.DEFAULT_YAML] os.environ.update({'TMPDIR': parsed_args.temp_dir}) manager = build.ImageBuildManager( parsed_args.config_files, output_directory=parsed_args.output_directory, skip=parsed_args.skip, images=parsed_args.image_names) manager.build() class BaseClientAdapter(object): log = logging.getLogger(__name__ + ".BaseClientAdapter") def __init__(self, image_path, progress=False, update_existing=False, updated=None): self.progress = progress self.image_path = image_path self.update_existing = update_existing self.updated = updated @abc.abstractmethod def get_image_property(self, image, prop): pass @abc.abstractmethod def update_or_upload(self, image_name, properties, names_func, arch, platform=None, disk_format='raw', container_format='bare'): pass def _copy_file(self, src, dest): cmd = 'sudo cp -f "{0}" "{1}"'.format(src, dest) self.log.debug(cmd) subprocess.check_call(cmd, shell=True) def _move_file(self, src, dest): cmd = 'sudo mv "{0}" "{1}"'.format(src, dest) self.log.debug(cmd) subprocess.check_call(cmd, shell=True) def _convert_image(self, src, dest): cmd = 'sudo qemu-img convert -O raw "{0}" "{1}"'.format(src, dest) self.log.debug(cmd) subprocess.check_call(cmd, shell=True) def _make_dirs(self, path): cmd = 'sudo mkdir -m 0775 -p "{0}"'.format(path) self.log.debug(cmd) subprocess.check_call(cmd, shell=True) def _files_changed(self, filepath1, filepath2): return (plugin_utils.file_checksum(filepath1) != plugin_utils.file_checksum(filepath2)) def file_create_or_update(self, src_file, dest_file): if os.path.isfile(dest_file): if self._files_changed(src_file, dest_file): if self.update_existing: self._copy_file(src_file, dest_file) else: print('Image file "%s" already exists and can be updated' ' with --update-existing.' % dest_file) else: print('Image file "%s" is up-to-date, skipping.' % dest_file) else: self._copy_file(src_file, dest_file) def check_file_exists(self, file_path): if not os.path.isfile(file_path): raise exceptions.CommandError( 'Required file "%s" does not exist.' % file_path ) def read_image_file_pointer(self, filepath): self.check_file_exists(filepath) file_descriptor = open(filepath, 'rb') if self.progress: file_descriptor = VerboseFileWrapper(file_descriptor) return file_descriptor class FileImageClientAdapter(BaseClientAdapter): def __init__(self, local_path, **kwargs): super(FileImageClientAdapter, self).__init__(**kwargs) self.local_path = local_path def get_image_property(self, image, prop): if prop == 'kernel_id': path = os.path.splitext(image.id)[0] + '.vmlinuz' if os.path.exists(path.replace("file://", "")): return path return None elif prop == 'ramdisk_id': path = os.path.splitext(image.id)[0] + '.initrd' if os.path.exists(path.replace("file://", "")): return path return None raise ValueError('Unsupported property %s' % prop) def _print_image_info(self, image): table = PrettyTable(['Path', 'Name', 'Size']) table.add_row([image.id, image.name, image.size]) print(table, file=sys.stdout) def _paths(self, image_name, names_func, arch, platform): (arch_path, extension) = names_func( image_name, arch=arch, platform=platform, use_subdir=True) image_file = image_name + extension dest_dir = os.path.split( os.path.join(self.local_path, arch_path))[0] return (dest_dir, image_file) def _get_image(self, path): if not os.path.exists(path): return stat = os.stat(path) created_at = datetime.fromtimestamp( stat.st_mtime).isoformat() Image = collections.namedtuple( 'Image', 'id, name, checksum, created_at, size' ) (dir_path, filename) = os.path.split(path) (name, extension) = os.path.splitext(filename) checksum = plugin_utils.file_checksum(path) return Image( id='file://%s' % path, name=name, checksum=checksum, created_at=created_at, size=stat.st_size ) def _image_changed(self, image, filename): return image.checksum != plugin_utils.file_checksum(filename) def _image_try_update(self, src_path, dest_path): image = self._get_image(dest_path) if image: if self._image_changed(image, src_path): if self.update_existing: dest_base, dest_ext = os.path.splitext(dest_path) dest_datestamp = re.sub( r'[\-:\.]|(0+$)', '', image.created_at) dest_mv = dest_base + '_' + dest_datestamp + dest_ext self._move_file(dest_path, dest_mv) if self.updated is not None: self.updated.append(dest_path) return None else: print('Image "%s" already exists and can be updated' ' with --update-existing.' % dest_path) return image else: print('Image "%s" is up-to-date, skipping.' % dest_path) return image else: return None def _upload_image(self, src_path, dest_path): dest_dir = os.path.split(dest_path)[0] if not os.path.isdir(dest_dir): self._make_dirs(dest_dir) self._copy_file(src_path, dest_path) image = self._get_image(dest_path) print('Image "%s" was copied.' % image.id, file=sys.stdout) self._print_image_info(image) return image def update_or_upload(self, image_name, properties, names_func, arch, platform=None, disk_format='raw', container_format='bare'): (dest_dir, image_file) = self._paths( image_name, names_func, arch, platform) src_path = os.path.join(self.image_path, image_file) dest_path = os.path.join(dest_dir, image_file) existing_image = self._image_try_update(src_path, dest_path) if existing_image: return existing_image return self._upload_image(src_path, dest_path) class GlanceClientAdapter(BaseClientAdapter): def __init__(self, client, **kwargs): super(GlanceClientAdapter, self).__init__(**kwargs) self.client = client def _print_image_info(self, image): table = PrettyTable(['ID', 'Name', 'Disk Format', 'Size', 'Status']) table.add_row([image.id, image.name, image.disk_format, image.size, image.status]) print(table, file=sys.stdout) def _get_image(self, name): """Retrieves 'openstack.image.v2.image.Image' object. Uses 'openstack.image.v2._proxy.Proxy.find_image' method. :param name: name or ID of an image :type name: `string` :returns: Requested image object if one exists, otherwise `None` :rtype: `openstack.image.v2.image.Image` or `NoneType` """ # This would return None by default for an non-existent resorurce # And DuplicateResource exception if there more than one. return self.client.find_image(name) def _image_changed(self, image, filename): """Compare the precomputed hash value with the one derived here. :param image: Image resource :type image: `openstack.image.v2.image.Image` :param filename: path to the image file :type filname: `string` :returns: True if image hashes don't match, false otherwise :rtype: `bool` """ if not hasattr(image, 'hash_value'): raise RuntimeError( ("The supplied image does not have a hash value set.")) return image.hash_value != plugin_utils.file_checksum( filename, image.hash_algo) def _image_try_update(self, image_name, image_file): image = self._get_image(image_name) if image: if self._image_changed(image, image_file): if self.update_existing: self.client.update_image( image.id, name='%s_%s' % (image.name, re.sub(r'[\-:\.]|(0+$)', '', image.created_at)) ) if self.updated is not None: self.updated.append(image.id) return None else: print('Image "%s" already exists and can be updated' ' with --update-existing.' % image_name) return image else: print('Image "%s" is up-to-date, skipping.' % image_name) return image else: return None def _upload_image(self, name, data, properties=None, visibility='public', disk_format='raw', container_format='bare'): image = self.client.create_image( name=name, visibility=visibility, disk_format=disk_format, container_format=container_format, data=data, validate_checksum=False ) if properties: self.client.update_image(image.id, **properties) # Refresh image info image = self.client.get_image(image.id) print('Image "%s" was uploaded.' % image.name, file=sys.stdout) self._print_image_info(image) return image def get_image_property(self, image, prop): return getattr(image, prop) def update_or_upload(self, image_name, properties, names_func, arch, platform=None, disk_format='raw', container_format='bare'): if arch == 'x86_64' and platform is None: arch = None (glance_name, extension) = names_func( image_name, arch=arch, platform=platform) file_path = os.path.join(self.image_path, image_name + extension) updated_image = self._image_try_update(glance_name, file_path) if updated_image: return updated_image with self.read_image_file_pointer(file_path) as data: return self._upload_image( name=glance_name, disk_format=disk_format, container_format=container_format, properties=properties, data=data) class UploadOvercloudImage(command.Command): """Make existing image files available for overcloud deployment.""" log = logging.getLogger(__name__ + ".UploadOvercloudImage") def _get_client_adapter(self, parsed_args): kwargs = { 'progress': parsed_args.progress, 'image_path': parsed_args.image_path, 'update_existing': parsed_args.update_existing, 'updated': self.updated } if not parsed_args.local: try: return GlanceClientAdapter(self.app.client_manager.image, **kwargs) except exc_catalog.EndpointNotFound: # Missing endpoint implies local-only upload pass return FileImageClientAdapter(parsed_args.local_path, **kwargs) def _get_environment_var(self, envvar, default, deprecated=[]): for env_key in deprecated: if env_key in os.environ: self.log.warning(('Found deprecated environment var \'%s\', ' 'please use \'%s\' instead' % (env_key, envvar))) return os.environ.get(env_key) return os.environ.get(envvar, default) def _get_image_filename(self, parsed_args): if parsed_args.os_image_name: return parsed_args.os_image_name if os.path.exists(os.path.join(parsed_args.image_path, constants.DEFAULT_WHOLE_DISK_IMAGE)): return constants.DEFAULT_WHOLE_DISK_IMAGE return constants.DEFAULT_PARTITION_IMAGE def get_parser(self, prog_name): parser = super(UploadOvercloudImage, self).get_parser(prog_name) parser.add_argument( "--image-path", default=self._get_environment_var('IMAGE_PATH', './'), help=_("Path to directory containing image files"), ) parser.add_argument( "--os-image-name", default=self._get_environment_var('OS_IMAGE_NAME', None), help=_("OpenStack disk image filename"), ) parser.add_argument( "--ironic-python-agent-name", dest='ipa_name', default=self._get_environment_var('IRONIC_PYTHON_AGENT_NAME', 'ironic-python-agent', deprecated=['AGENT_NAME']), help=_("OpenStack ironic-python-agent (agent) image filename"), ) parser.add_argument( "--http-boot", default=self._get_environment_var( 'HTTP_BOOT', constants.IRONIC_HTTP_BOOT_BIND_MOUNT), help=_("Root directory for the ironic-python-agent image. If " "uploading images for multiple architectures/platforms, " "vary this argument such that a distinct folder is " "created for each architecture/platform.") ) parser.add_argument( "--update-existing", dest="update_existing", action="store_true", help=_("Update images if already exist"), ) parser.add_argument( "--whole-disk", dest="whole_disk", action="store_true", default=False, help=_("When set, the overcloud-full image to be uploaded " "will be considered as a whole disk one"), ) parser.add_argument( "--architecture", help=_("Architecture type for these images, " "\'x86_64\', \'i386\' and \'ppc64le\' " "are common options. This option should match at least " "one \'arch\' value in instackenv.json"), ) parser.add_argument( "--platform", help=_("Platform type for these images. Platform is a " "sub-category of architecture. For example you may have " "generic images for x86_64 but offer images specific to " "SandyBridge (SNB)."), ) parser.add_argument( "--image-type", dest="image_type", choices=["os", "ironic-python-agent"], help=_("If specified, allows to restrict the image type to upload " "(os for the overcloud image or ironic-python-agent for " "the ironic-python-agent one)"), ) parser.add_argument( "--progress", dest="progress", action="store_true", default=False, help=_('Show progress bar for upload files action')) parser.add_argument( "--local", dest="local", action="store_true", default=True, help=_('DEPRECATED: Copy files locally, even if there is an image ' 'service endpoint. The default has been changed to copy ' 'files locally.')) parser.add_argument( "--no-local", dest="local", action="store_false", default=True, help=_('Upload files to image service.')) parser.add_argument( "--local-path", default=self._get_environment_var( 'LOCAL_IMAGE_PATH', constants.IRONIC_LOCAL_IMAGE_PATH), help=_("Root directory for image file copy destination when there " "is no image endpoint, or when --local is specified") ) return parser def take_action(self, parsed_args): self.log.debug("take_action(%s)" % parsed_args) self.updated = [] self.adapter = self._get_client_adapter(parsed_args) if parsed_args.platform and not parsed_args.architecture: raise exceptions.CommandError('You supplied a platform (%s) ' 'without specifying the ' 'architecture') self.log.debug("checking if image files exist") image_files = [] image_filename = self._get_image_filename(parsed_args) image_name = os.path.splitext(image_filename)[0] if image_filename == constants.DEFAULT_WHOLE_DISK_IMAGE: parsed_args.whole_disk = True if parsed_args.image_type is None or \ parsed_args.image_type == 'ironic-python-agent': image_files.append('%s.initramfs' % parsed_args.ipa_name) image_files.append('%s.kernel' % parsed_args.ipa_name) if parsed_args.image_type is None or parsed_args.image_type == 'os': image_files.append(image_filename) if parsed_args.whole_disk: overcloud_image_type = 'whole disk' else: overcloud_image_type = 'partition' for image in image_files: extension = image.split('.')[-1] image_path = os.path.join(parsed_args.image_path, image) self.adapter.check_file_exists(image_path) # Convert qcow2 image to raw, see bug/1893912 if extension == 'qcow2': self.adapter._convert_image( image_path, os.path.join(parsed_args.image_path, image_name + '.raw')) self.log.debug("uploading %s overcloud images " % overcloud_image_type) properties = {} arch = parsed_args.architecture if arch: properties['hw_architecture'] = arch else: properties['hw_architecture'] = tripleo_common.arch.kernel_arch() platform = parsed_args.platform if platform: properties['tripleo_platform'] = platform if parsed_args.image_type is None or parsed_args.image_type == 'os': # vmlinuz and initrd only need to be uploaded for a partition image if not parsed_args.whole_disk: kernel = self.adapter.update_or_upload( image_name=image_name, properties=properties, names_func=plugin_utils.overcloud_kernel, arch=arch, platform=platform, disk_format='aki' ) ramdisk = self.adapter.update_or_upload( image_name=image_name, properties=properties, names_func=plugin_utils.overcloud_ramdisk, arch=arch, platform=platform, disk_format='ari' ) overcloud_image = self.adapter.update_or_upload( image_name=image_name, properties=dict( {'kernel_id': kernel.id, 'ramdisk_id': ramdisk.id}, **properties), names_func=plugin_utils.overcloud_image, arch=arch, platform=platform ) img_kernel_id = self.adapter.get_image_property( overcloud_image, 'kernel_id') img_ramdisk_id = self.adapter.get_image_property( overcloud_image, 'ramdisk_id') # check overcloud image links if img_kernel_id is None or img_ramdisk_id is None: self.log.error('Link of overcloud image %s to its initrd' ' or kernel images is MISSING.' 'You can keep it or fix it manually.' % overcloud_image.name) elif (img_kernel_id != kernel.id or img_ramdisk_id != ramdisk.id): self.log.error('Link of overcloud image %s to its initrd' ' or kernel images leads to OLD image.' 'You can keep it or fix it manually.' % overcloud_image.name) else: overcloud_image = self.adapter.update_or_upload( image_name=image_name, properties=properties, names_func=plugin_utils.overcloud_image, arch=arch, platform=platform ) self.log.debug("uploading bm images") if parsed_args.image_type is None or \ parsed_args.image_type == 'ironic-python-agent': self.log.debug("copy agent images to HTTP BOOT dir") self.adapter.file_create_or_update( os.path.join(parsed_args.image_path, '%s.kernel' % parsed_args.ipa_name), os.path.join(parsed_args.http_boot, 'agent.kernel') ) self.adapter.file_create_or_update( os.path.join(parsed_args.image_path, '%s.initramfs' % parsed_args.ipa_name), os.path.join(parsed_args.http_boot, 'agent.ramdisk') ) if self.updated: print('%s images have been updated, make sure to ' 'rerun\n\topenstack overcloud node configure\nto reflect ' 'the changes on the nodes' % len(self.updated))