diff --git a/freezer/common/config.py b/freezer/common/config.py index 0ef418e3..f5b26af6 100644 --- a/freezer/common/config.py +++ b/freezer/common/config.py @@ -73,6 +73,9 @@ DEFAULT_PARAMS = { 'cinder_vol_name': '', 'nova_inst_id': '', '__version__': FREEZER_VERSION, 'nova_inst_name': '', + 'glance_image_id': '', + 'glance_image_name': '', + 'glance_image_name_filter': '', 'remove_older_than': None, 'restore_from_date': None, 'upload_limit': -1, 'always_level': False, 'version': None, 'dry_run': False, 'lvm_snapsize': DEFAULT_LVM_SNAPSIZE, @@ -122,11 +125,13 @@ _COMMON = [ "(filesystem),mongo (MongoDB), mysql (MySQL), " "sqlserver(SQL Server), " "cinder(OpenStack Volume backup by freezer), " + "glance(OpenStack Image backup by freezer), " "cindernative(OpenStack native cinder-volume backup), " "nova(OpenStack Instance). Default set to fs"), cfg.StrOpt('engine', short='e', - choices=['tar', 'rsync', 'rsyncv2', 'nova', 'osbrick'], + choices=['tar', 'rsync', 'rsyncv2', + 'nova', 'osbrick', 'glance'], dest='engine_name', default=DEFAULT_PARAMS['engine_name'], help="Engine to be used for backup/restore. " @@ -401,6 +406,21 @@ _COMMON = [ dest='cindernative_backup_id', help="Id of the cindernative backup to be restored" ), + cfg.StrOpt('glance-image-id', + dest='glance_image_id', + default=DEFAULT_PARAMS['glance_image_id'], + help="Id of glance image for backup" + ), + cfg.StrOpt('glance-image-name', + dest='glance_image_name', + default=DEFAULT_PARAMS['glance_image_name'], + help="Name of glance image for backup" + ), + cfg.StrOpt('glance-image-name-filter', + dest='glance_image_name_filter', + default=DEFAULT_PARAMS['glance_image_name_filter'], + help="Name filter of glance image for backup" + ), cfg.StrOpt('nova-inst-id', dest='nova_inst_id', default=DEFAULT_PARAMS['nova_inst_id'], @@ -714,6 +734,11 @@ def get_backup_args(): backup_args.nova_inst_id or backup_args.nova_inst_name): backup_media = 'nova' + elif backup_args.engine_name == 'glance': + if (backup_args.project_id or backup_args.glance_image_id or + backup_args.glance_image_name or + backup_args.glance_image_name_filter): + backup_media = 'glance' elif backup_args.cinderbrick_vol_id: backup_media = 'cinderbrick' diff --git a/freezer/engine/glance/__init__.py b/freezer/engine/glance/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freezer/engine/glance/glance.py b/freezer/engine/glance/glance.py new file mode 100644 index 00000000..396d5f05 --- /dev/null +++ b/freezer/engine/glance/glance.py @@ -0,0 +1,285 @@ +# (c) Copyright 2019 ZTE Corporation.. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from concurrent import futures +import os + +from oslo_config import cfg +from oslo_log import log +from oslo_serialization import jsonutils as json + +from freezer.common import client_manager +from freezer.engine import engine +from freezer.engine.tar import tar +from freezer.utils import utils + +import tempfile + +LOG = log.getLogger(__name__) +CONF = cfg.CONF + + +class GlanceEngine(engine.BackupEngine): + + def __init__(self, storage, **kwargs): + super(GlanceEngine, self).__init__(storage=storage) + self.client = client_manager.get_client_manager(CONF) + self.glance = self.client.create_glance() + self.encrypt_pass_file = kwargs.get('encrypt_key') + self.exclude = kwargs.get('exclude') + self.server_info = None + self.openssl_path = None + self.compression_algo = 'gzip' + self.is_windows = None + self.dry_run = kwargs.get('dry_run', False) + self.max_segment_size = kwargs.get('max_segment_size') + self.storage = storage + self.dereference_symlink = kwargs.get('symlinks') + + @property + def name(self): + return "glance" + + def stream_image(self, pipe): + """Reading bytes from a pipe and converting it to a stream-like""" + try: + while True: + chunk = pipe.recv_bytes() + yield chunk + except EOFError: + pass + + def get_glance_tenant(self, project_id): + # Load info about tenant images + if self.storage._type == 'swift': + swift_connection = self.client.create_swift() + headers, data = swift_connection.get_object( + self.storage.storage_path, + "project_glance_" + project_id) + elif self.storage._type == 's3': + bucket_name, object_name = self.get_storage_info(project_id) + data = self.storage.get_object( + bucket_name=bucket_name, + key=object_name + )['Body'].read() + elif self.storage._type in ['local', 'ssh', 'ftp', 'ftps']: + backup_basepath = os.path.join(self.storage.storage_path, + 'project_glance_' + project_id) + with self.storage.open(backup_basepath, 'rb') as backup_file: + data = backup_file.readline() + elif self.storage._type in ['ftp', 'ftps']: + backup_basepath = os.path.join(self.storage.storage_path, + 'project_glance_' + project_id) + file = tempfile.NamedTemporaryFile('wb', delete=True) + self.storage.get_file(backup_basepath, file.name) + with open(file.name) as f: + data = f.readline() + LOG.info("get_glance_tenant download {0}".format(data)) + + return json.loads(data) + + def restore_glance_tenant(self, project_id, hostname_backup_name, + overwrite, recent_to_date): + + image_ids = self.get_glance_tenant(project_id) + for image_id in image_ids: + LOG.info("Restore glance image ID: {0} from container {1}". + format(image_id, self.storage.storage_path)) + backup_name = os.path.join(hostname_backup_name, + image_id) + self.restore( + hostname_backup_name=backup_name, + restore_resource=image_id, + overwrite=overwrite, + recent_to_date=recent_to_date) + + def restore_level(self, restore_resource, read_pipe, backup, except_queue): + try: + metadata = backup.metadata() + if (not self.encrypt_pass_file and + metadata.get("encryption", False)): + raise Exception("Cannot restore encrypted backup without key") + engine_metadata = backup.engine_metadata() + image_info = metadata.get('image', {}) + container_format = image_info.get('container_format', 'bare') + disk_format = image_info.get('disk_format', 'raw') + + length = int(engine_metadata.get('length')) + + stream = self.stream_image(read_pipe) + data = utils.ReSizeStream(stream, length, 1) + image = self.client.create_image( + "Restore: {0}".format( + image_info.get('name', image_info.get('id', None)) + ), + container_format, + disk_format, + data=data + ) + + if self.encrypt_pass_file: + try: + tmpdir = tempfile.mkdtemp() + except Exception: + LOG.error("Unable to create a tmp directory") + raise + + tar_engine = tar.TarEngine(self.compression_algo, + self.dereference_symlink, + self.exclude, self.storage, + self.max_segment_size, + self.encrypt_pass_file, + self.dry_run) + + tar_engine.restore_level(tmpdir, read_pipe, backup, + except_queue) + + utils.wait_for( + GlanceEngine.image_active, + 1, + CONF.timeout, + message="Waiting for image to finish uploading {0} and become" + " active".format(image.id), + kwargs={"glance_client": self.glance, "image_id": image.id} + ) + return image + except Exception as e: + LOG.exception(e) + except_queue.put(e) + raise + + def backup_glance_tenant(self, project_id, hostname_backup_name, + no_incremental, max_level, always_level, + restart_always_level): + # import pdb;pdb.set_trace() + image_ids = [image.id for image in + self.glance.images.list(detailed=False)] + data = json.dumps(image_ids) + LOG.info("Saving information about image {0}".format(data)) + + if self.storage._type == 'swift': + swift_connection = self.client.create_swift() + swift_connection.put_object(self.storage.storage_path, + "project_glance_{0}". + format(project_id), + data) + elif self.storage._type == 's3': + bucket_name, object_name = self.get_storage_info(project_id) + self.storage.put_object( + bucket_name=bucket_name, + key=object_name, + data=data + ) + elif self.storage._type in ['local', 'ssh']: + backup_basepath = os.path.join(self.storage.storage_path, + "project_glance_" + project_id) + with self.storage.open(backup_basepath, 'wb') as backup_file: + backup_file.write(data) + elif self.storage._type in ['ftp', 'ftps']: + backup_basepath = os.path.join(self.storage.storage_path, + 'project_glance_' + project_id) + file = tempfile.NamedTemporaryFile('wb', delete=True) + with open(file.name, 'wb') as f: + f.write(data) + LOG.info("backup_glance_tenant data={0}".format(data)) + self.storage.put_file(file.name, backup_basepath) + + executor = futures.ThreadPoolExecutor( + max_workers=len(image_ids)) + futures_list = [] + for image_id in image_ids: + LOG.info("Backup glance image ID: {0} to container {1}". + format(image_id, self.storage.storage_path)) + backup_name = os.path.join(hostname_backup_name, + image_id) + + futures_list.append(executor.submit( + self.backup, + backup_resource=image_id, + hostname_backup_name=backup_name, + no_incremental=no_incremental, + max_level=max_level, + always_level=always_level, + restart_always_level=restart_always_level)) + + futures.wait(futures_list, CONF.timeout) + + def get_storage_info(self, project_id): + if self.storage.get_object_prefix() != '': + object_name = "{0}/project_{1}".format( + self.storage.get_object_prefix(), + project_id + ) + else: + object_name = "project_{0}".format(project_id) + return self.storage.get_bucket_name(), object_name + + def backup_data(self, backup_resource, manifest_path): + # import pdb;pdb.set_trace() + image = self.glance.images.get(backup_resource) + if not image: + raise Exception( + "Image {0} can't be found.".format(backup_resource) + ) + LOG.info('Image backup') + stream = self.client.download_image(image) + + LOG.info("Uploading image to storage path") + + headers = {"image_name": image.name, + "image_id": image.get('id'), + "disk_format": image.get('disk_format'), + "container_format": image.get('container_format'), + "visibility": image.get('visibility'), + 'length': str(len(stream)), + "protected": image.protected} + self.set_tenant_meta(manifest_path, headers) + for chunk in stream: + yield chunk + + if self.encrypt_pass_file: + tar_engine = tar.TarEngine(self.compression_algo, + self.dereference_symlink, + self.exclude, self.storage, + self.max_segment_size, + self.encrypt_pass_file, self.dry_run) + + for data_chunk in tar_engine.backup_data('.', manifest_path): + yield data_chunk + + @staticmethod + def image_active(glance_client, image_id): + """Check if the image is in the active state or not""" + image = glance_client.images.get(image_id) + return image.status == 'active' + + def metadata(self, backup_resource): + """Construct metadata""" + # import pdb;pdb.set_trace() + image_info = self.glance.images.get(backup_resource) + return { + "engine_name": self.name, + "image": image_info, + "encryption": bool(self.encrypt_pass_file) + } + + def set_tenant_meta(self, path, metadata): + """push data to the manifest file""" + with open(path, 'wb') as fb: + fb.writelines(json.dumps(metadata)) + + def get_tenant_meta(self, path): + with open(path, 'rb') as fb: + json.loads(fb.read()) diff --git a/freezer/job.py b/freezer/job.py index ae48105f..04331431 100644 --- a/freezer/job.py +++ b/freezer/job.py @@ -53,6 +53,7 @@ class Job(object): self.client = client_manager.get_client_manager(CONF) self.nova = self.client.get_nova() self.cinder = self.client.get_cinder() + self.glance = self.client.get_glance() self._general_validation() self._validate() if self.conf.nova_inst_name: @@ -66,6 +67,18 @@ class Job(object): if volume.name == self.conf.cinder_inst_name] + if self.conf.glance_image_name: + self.glance_image_ids = [image.id for image in + self.glance.images.list() + if image.name == + self.conf.glance_image_name] + + if self.conf.glance_image_name_filter: + self.glance_image_ids = [image.id for image in + self.glance.images.list() + if self.conf.glance_image_name_filter + not in image.name] + @abc.abstractmethod def _validate(self): """ @@ -143,6 +156,17 @@ class BackupJob(Job): and not self.conf.nova_inst_name: raise ValueError("nova-inst-id or project-id or nova-inst-name" " argument must be provided") + if self.conf.mode == 'glance': + if not self.conf.no_incremental: + raise ValueError("Incremental glance backup is not supported") + + if not self.conf.glance_image_id and not self.conf.project_id \ + and not self.conf.glance_image_name \ + and not self.conf.glance_image_name_filter: + raise ValueError("glance-image-id or project-id or" + " glance-image-name or " + " glance-image-name_filter " + "argument must be provided") if self.conf.mode == 'cinder': if not self.conf.cinder_vol_id and not self.conf.cinder_vol_name: @@ -317,6 +341,50 @@ class BackupJob(Job): futures.wait(futures_list, CONF.timeout) + elif backup_media == 'glance': + if self.conf.project_id: + return self.engine.backup_glance_tenant( + project_id=self.conf.project_id, + hostname_backup_name=self.conf.hostname_backup_name, + no_incremental=self.conf.no_incremental, + max_level=self.conf.max_level, + always_level=self.conf.always_level, + restart_always_level=self.conf.restart_always_level) + + elif self.conf.glance_image_id: + LOG.info('Executing glance backup. Image ID: {0}'.format( + self.conf.glance_image_id)) + + hostname_backup_name = os.path.join( + self.conf.hostname_backup_name, + self.conf.glance_image_id) + return self.engine.backup( + backup_resource=self.conf.glance_image_id, + hostname_backup_name=hostname_backup_name, + no_incremental=self.conf.no_incremental, + max_level=self.conf.max_level, + always_level=self.conf.always_level, + restart_always_level=self.conf.restart_always_level) + + else: + executor = futures.ThreadPoolExecutor( + max_workers=len(self.glance_image_ids)) + futures_list = [] + for image_id in self.glance_image_ids: + hostname_backup_name = os.path.join( + self.conf.hostname_backup_name, image_id) + futures_list.append(executor.submit( + self.engine.backup( + backup_resource=image_id, + hostname_backup_name=hostname_backup_name, + no_incremental=self.conf.no_incremental, + max_level=self.conf.max_level, + always_level=self.conf.always_level, + restart_always_level=self.conf.restart_always_level + ))) + + futures.wait(futures_list, CONF.timeout) + elif backup_media == 'cindernative': LOG.info('Executing cinder native backup. Volume ID: {0}, ' 'incremental: {1}'.format(self.conf.cindernative_vol_id, @@ -362,10 +430,13 @@ class RestoreJob(Job): if not any([self.conf.restore_abs_path, self.conf.nova_inst_id, self.conf.nova_inst_name, + self.conf.glance_image_id, self.conf.cinder_vol_id, self.conf.cinder_vol_name, self.conf.cindernative_vol_id, self.conf.cinderbrick_vol_id, + self.conf.glance_image_name, + self.conf.glance_image_name_filter, self.conf.project_id]): raise ValueError("--restore-abs-path is required") if not self.conf.container: @@ -450,6 +521,42 @@ class RestoreJob(Job): recent_to_date=restore_timestamp, backup_media=conf.mode) + elif conf.backup_media == 'glance': + if self.conf.project_id: + return self.engine.restore_glance_tenant( + project_id=self.conf.project_id, + hostname_backup_name=self.conf.hostname_backup_name, + overwrite=conf.overwrite, + recent_to_date=restore_timestamp) + + elif conf.glance_image_id: + LOG.info("Restoring glance backup. Image ID: {0}, " + "timestamp: {1} ".format(conf.glance_image_id, + restore_timestamp)) + hostname_backup_name = os.path.join( + self.conf.hostname_backup_name, + self.conf.glance_image_id) + self.engine.restore( + hostname_backup_name=hostname_backup_name, + restore_resource=conf.glance_image_id, + overwrite=conf.overwrite, + recent_to_date=restore_timestamp, + backup_media=conf.mode) + + else: + for image_id in self.glance_image_ids: + LOG.info("Restoring glance backup. Image ID: {0}, " + "timestamp: {1}".format(image_id, + restore_timestamp)) + hostname_backup_name = os.path.join( + self.conf.hostname_backup_name, image_id) + self.engine.restore( + hostname_backup_name=hostname_backup_name, + restore_resource=image_id, + overwrite=conf.overwrite, + recent_to_date=restore_timestamp, + backup_media=conf.mode) + elif conf.backup_media == 'cinder': if conf.cinder_vol_id: LOG.info("Restoring cinder backup from glance. " diff --git a/freezer/main.py b/freezer/main.py index 7d557551..2e2df932 100644 --- a/freezer/main.py +++ b/freezer/main.py @@ -46,7 +46,6 @@ LOG = log.getLogger(__name__) def freezer_main(backup_args): """Freezer main loop for job execution. """ - if not backup_args.quiet: LOG.info("Begin freezer agent process with args: {0}".format(sys.argv)) LOG.info('log file at {0}'.format(CONF.get('log_file'))) @@ -61,7 +60,7 @@ def freezer_main(backup_args): if (backup_args.storage == 'swift' or backup_args.backup_media in ['nova', 'cinder', 'cindernative', - 'cinderbrick']): + 'cinderbrick', 'glance']): backup_args.client_manager = client_manager.get_client_manager( backup_args.__dict__) diff --git a/freezer/mode/glance.py b/freezer/mode/glance.py new file mode 100644 index 00000000..b389a8f6 --- /dev/null +++ b/freezer/mode/glance.py @@ -0,0 +1,37 @@ +# (c) Copyright 2019 ZTE Corporation.. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from freezer.mode import mode + + +class GlanceMode(mode.Mode): + """ + Execute a glance backup/restore + """ + def __init__(self, conf): + self.conf = conf + + @property + def name(self): + return "glance" + + @property + def version(self): + return "1.0" + + def release(self): + pass + + def prepare(self): + pass diff --git a/freezer/tests/commons.py b/freezer/tests/commons.py index cffe90ce..8c4ada73 100644 --- a/freezer/tests/commons.py +++ b/freezer/tests/commons.py @@ -375,7 +375,9 @@ class BackupOpt1(object): self.nova_inst_id = '' self.nova_inst_name = '' self.lvm_snapperm = 'ro' - + self.glance_image_name = '' + self.glance_image_name_filter = '' + self.glance_image_id = '' self.compression = 'gzip' self.storage = mock.MagicMock() self.engine = mock.MagicMock()