openstack/freezer
Code Issues Proposed changes

[WIP] Add the backup and restore of glance image

Browse Source 
Currently, Freezer doesn't support image backup and restore,
It is necessary for us to increase support for image backup and
restore.

Implements: bp Add-support-for-glance-image-backup
Co-authored-by: gengchc2 <geng.changcai2@zte.com.cn>
Change-Id: I2a3ce511aa3c5357799a97babbe3eb0a3818fdfd
Story: #2005855
Task: #33648
This commit is contained in:
gecong1973 2019-06-10 02:14:27 -07:00
parent 17e04f02bd
commit 76a42bd162
7 changed files with 459 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
freezer/common/config.py
View File

@ -73,6 +73,9 @@ DEFAULT_PARAMS = {
'cinder_vol_name': '',
'nova_inst_id': '', '__version__': FREEZER_VERSION,
'nova_inst_name': '',
'glance_image_id': '',
'glance_image_name': '',
'glance_image_name_filter': '',
'remove_older_than': None, 'restore_from_date': None,
'upload_limit': -1, 'always_level': False, 'version': None,
'dry_run': False, 'lvm_snapsize': DEFAULT_LVM_SNAPSIZE,
@ -122,11 +125,13 @@ _COMMON = [
"(filesystem),mongo (MongoDB), mysql (MySQL), "
"sqlserver(SQL Server), "
"cinder(OpenStack Volume backup by freezer), "
"glance(OpenStack Image backup by freezer), "
"cindernative(OpenStack native cinder-volume backup), "
"nova(OpenStack Instance). Default set to fs"),
cfg.StrOpt('engine',
short='e',
choices=['tar', 'rsync', 'rsyncv2', 'nova', 'osbrick'],
choices=['tar', 'rsync', 'rsyncv2',
'nova', 'osbrick', 'glance'],
dest='engine_name',
default=DEFAULT_PARAMS['engine_name'],
help="Engine to be used for backup/restore. "
@ -401,6 +406,21 @@ _COMMON = [
dest='cindernative_backup_id',
help="Id of the cindernative backup to be restored"
),
cfg.StrOpt('glance-image-id',
dest='glance_image_id',
default=DEFAULT_PARAMS['glance_image_id'],
help="Id of glance image for backup"
),
cfg.StrOpt('glance-image-name',
dest='glance_image_name',
default=DEFAULT_PARAMS['glance_image_name'],
help="Name of glance image for backup"
),
cfg.StrOpt('glance-image-name-filter',
dest='glance_image_name_filter',
default=DEFAULT_PARAMS['glance_image_name_filter'],
help="Name filter of glance image for backup"
),
cfg.StrOpt('nova-inst-id',
dest='nova_inst_id',
default=DEFAULT_PARAMS['nova_inst_id'],
@ -714,6 +734,11 @@ def get_backup_args():
backup_args.nova_inst_id or
backup_args.nova_inst_name):
backup_media = 'nova'
elif backup_args.engine_name == 'glance':
if (backup_args.project_id or backup_args.glance_image_id or
backup_args.glance_image_name or
backup_args.glance_image_name_filter):
backup_media = 'glance'
elif backup_args.cinderbrick_vol_id:
backup_media = 'cinderbrick'

0
freezer/engine/glance/__init__.py Normal file
View File

285
freezer/engine/glance/glance.py Normal file
View File

@ -0,0 +1,285 @@
# (c) Copyright 2019 ZTE Corporation..
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from concurrent import futures
import os
from oslo_config import cfg
from oslo_log import log
from oslo_serialization import jsonutils as json
from freezer.common import client_manager
from freezer.engine import engine
from freezer.engine.tar import tar
from freezer.utils import utils
import tempfile
LOG = log.getLogger(__name__)
CONF = cfg.CONF
class GlanceEngine(engine.BackupEngine):
def __init__(self, storage, **kwargs):
super(GlanceEngine, self).__init__(storage=storage)
self.client = client_manager.get_client_manager(CONF)
self.glance = self.client.create_glance()
self.encrypt_pass_file = kwargs.get('encrypt_key')
self.exclude = kwargs.get('exclude')
self.server_info = None
self.openssl_path = None
self.compression_algo = 'gzip'
self.is_windows = None
self.dry_run = kwargs.get('dry_run', False)
self.max_segment_size = kwargs.get('max_segment_size')
self.storage = storage
self.dereference_symlink = kwargs.get('symlinks')
@property
def name(self):
return "glance"
def stream_image(self, pipe):
"""Reading bytes from a pipe and converting it to a stream-like"""
try:
while True:
chunk = pipe.recv_bytes()
yield chunk
except EOFError:
pass
def get_glance_tenant(self, project_id):
# Load info about tenant images
if self.storage._type == 'swift':
swift_connection = self.client.create_swift()
headers, data = swift_connection.get_object(
self.storage.storage_path,
"project_glance_" + project_id)
elif self.storage._type == 's3':
bucket_name, object_name = self.get_storage_info(project_id)
data = self.storage.get_object(
bucket_name=bucket_name,
key=object_name
)['Body'].read()
elif self.storage._type in ['local', 'ssh', 'ftp', 'ftps']:
backup_basepath = os.path.join(self.storage.storage_path,
'project_glance_' + project_id)
with self.storage.open(backup_basepath, 'rb') as backup_file:
data = backup_file.readline()
elif self.storage._type in ['ftp', 'ftps']:
backup_basepath = os.path.join(self.storage.storage_path,
'project_glance_' + project_id)
file = tempfile.NamedTemporaryFile('wb', delete=True)
self.storage.get_file(backup_basepath, file.name)
with open(file.name) as f:
data = f.readline()
LOG.info("get_glance_tenant download {0}".format(data))
return json.loads(data)
def restore_glance_tenant(self, project_id, hostname_backup_name,
overwrite, recent_to_date):
image_ids = self.get_glance_tenant(project_id)
for image_id in image_ids:
LOG.info("Restore glance image ID: {0} from container {1}".
format(image_id, self.storage.storage_path))
backup_name = os.path.join(hostname_backup_name,
image_id)
self.restore(
hostname_backup_name=backup_name,
restore_resource=image_id,
overwrite=overwrite,
recent_to_date=recent_to_date)
def restore_level(self, restore_resource, read_pipe, backup, except_queue):
try:
metadata = backup.metadata()
if (not self.encrypt_pass_file and
metadata.get("encryption", False)):
raise Exception("Cannot restore encrypted backup without key")
engine_metadata = backup.engine_metadata()
image_info = metadata.get('image', {})
container_format = image_info.get('container_format', 'bare')
disk_format = image_info.get('disk_format', 'raw')
length = int(engine_metadata.get('length'))
stream = self.stream_image(read_pipe)
data = utils.ReSizeStream(stream, length, 1)
image = self.client.create_image(
"Restore: {0}".format(
image_info.get('name', image_info.get('id', None))
),
container_format,
disk_format,
data=data
)
if self.encrypt_pass_file:
try:
tmpdir = tempfile.mkdtemp()
except Exception:
LOG.error("Unable to create a tmp directory")
raise
tar_engine = tar.TarEngine(self.compression_algo,
self.dereference_symlink,
self.exclude, self.storage,
self.max_segment_size,
self.encrypt_pass_file,
self.dry_run)
tar_engine.restore_level(tmpdir, read_pipe, backup,
except_queue)
utils.wait_for(
GlanceEngine.image_active,
1,
CONF.timeout,
message="Waiting for image to finish uploading {0} and become"
" active".format(image.id),
kwargs={"glance_client": self.glance, "image_id": image.id}
)
return image
except Exception as e:
LOG.exception(e)
except_queue.put(e)
raise
def backup_glance_tenant(self, project_id, hostname_backup_name,
no_incremental, max_level, always_level,
restart_always_level):
# import pdb;pdb.set_trace()
image_ids = [image.id for image in
self.glance.images.list(detailed=False)]
data = json.dumps(image_ids)
LOG.info("Saving information about image {0}".format(data))
if self.storage._type == 'swift':
swift_connection = self.client.create_swift()
swift_connection.put_object(self.storage.storage_path,
"project_glance_{0}".
format(project_id),
data)
elif self.storage._type == 's3':
bucket_name, object_name = self.get_storage_info(project_id)
self.storage.put_object(
bucket_name=bucket_name,
key=object_name,
data=data
)
elif self.storage._type in ['local', 'ssh']:
backup_basepath = os.path.join(self.storage.storage_path,
"project_glance_" + project_id)
with self.storage.open(backup_basepath, 'wb') as backup_file:
backup_file.write(data)
elif self.storage._type in ['ftp', 'ftps']:
backup_basepath = os.path.join(self.storage.storage_path,
'project_glance_' + project_id)
file = tempfile.NamedTemporaryFile('wb', delete=True)
with open(file.name, 'wb') as f:
f.write(data)
LOG.info("backup_glance_tenant data={0}".format(data))
self.storage.put_file(file.name, backup_basepath)
executor = futures.ThreadPoolExecutor(
max_workers=len(image_ids))
futures_list = []
for image_id in image_ids:
LOG.info("Backup glance image ID: {0} to container {1}".
format(image_id, self.storage.storage_path))
backup_name = os.path.join(hostname_backup_name,
image_id)
futures_list.append(executor.submit(
self.backup,
backup_resource=image_id,
hostname_backup_name=backup_name,
no_incremental=no_incremental,
max_level=max_level,
always_level=always_level,
restart_always_level=restart_always_level))
futures.wait(futures_list, CONF.timeout)
def get_storage_info(self, project_id):
if self.storage.get_object_prefix() != '':
object_name = "{0}/project_{1}".format(
self.storage.get_object_prefix(),
project_id
)
else:
object_name = "project_{0}".format(project_id)
return self.storage.get_bucket_name(), object_name
def backup_data(self, backup_resource, manifest_path):
# import pdb;pdb.set_trace()
image = self.glance.images.get(backup_resource)
if not image:
raise Exception(
"Image {0} can't be found.".format(backup_resource)
)
LOG.info('Image backup')
stream = self.client.download_image(image)
LOG.info("Uploading image to storage path")
headers = {"image_name": image.name,
"image_id": image.get('id'),
"disk_format": image.get('disk_format'),
"container_format": image.get('container_format'),
"visibility": image.get('visibility'),
'length': str(len(stream)),
"protected": image.protected}
self.set_tenant_meta(manifest_path, headers)
for chunk in stream:
yield chunk
if self.encrypt_pass_file:
tar_engine = tar.TarEngine(self.compression_algo,
self.dereference_symlink,
self.exclude, self.storage,
self.max_segment_size,
self.encrypt_pass_file, self.dry_run)
for data_chunk in tar_engine.backup_data('.', manifest_path):
yield data_chunk
@staticmethod
def image_active(glance_client, image_id):
"""Check if the image is in the active state or not"""
image = glance_client.images.get(image_id)
return image.status == 'active'
def metadata(self, backup_resource):
"""Construct metadata"""
# import pdb;pdb.set_trace()
image_info = self.glance.images.get(backup_resource)
return {
"engine_name": self.name,
"image": image_info,
"encryption": bool(self.encrypt_pass_file)
}
def set_tenant_meta(self, path, metadata):
"""push data to the manifest file"""
with open(path, 'wb') as fb:
fb.writelines(json.dumps(metadata))
def get_tenant_meta(self, path):
with open(path, 'rb') as fb:
json.loads(fb.read())

107
freezer/job.py
View File

@ -53,6 +53,7 @@ class Job(object):
self.client = client_manager.get_client_manager(CONF)
self.nova = self.client.get_nova()
self.cinder = self.client.get_cinder()
self.glance = self.client.get_glance()
self._general_validation()
self._validate()
if self.conf.nova_inst_name:
@ -66,6 +67,18 @@ class Job(object):
if volume.name ==
self.conf.cinder_inst_name]
if self.conf.glance_image_name:
self.glance_image_ids = [image.id for image in
self.glance.images.list()
if image.name ==
self.conf.glance_image_name]
if self.conf.glance_image_name_filter:
self.glance_image_ids = [image.id for image in
self.glance.images.list()
if self.conf.glance_image_name_filter
not in image.name]
@abc.abstractmethod
def _validate(self):
"""
@ -143,6 +156,17 @@ class BackupJob(Job):
and not self.conf.nova_inst_name:
raise ValueError("nova-inst-id or project-id or nova-inst-name"
" argument must be provided")
if self.conf.mode == 'glance':
if not self.conf.no_incremental:
raise ValueError("Incremental glance backup is not supported")
if not self.conf.glance_image_id and not self.conf.project_id \
and not self.conf.glance_image_name \
and not self.conf.glance_image_name_filter:
raise ValueError("glance-image-id or project-id or"
" glance-image-name or "
" glance-image-name_filter "
"argument must be provided")
if self.conf.mode == 'cinder':
if not self.conf.cinder_vol_id and not self.conf.cinder_vol_name:
@ -317,6 +341,50 @@ class BackupJob(Job):
futures.wait(futures_list, CONF.timeout)
elif backup_media == 'glance':
if self.conf.project_id:
return self.engine.backup_glance_tenant(
project_id=self.conf.project_id,
hostname_backup_name=self.conf.hostname_backup_name,
no_incremental=self.conf.no_incremental,
max_level=self.conf.max_level,
always_level=self.conf.always_level,
restart_always_level=self.conf.restart_always_level)
elif self.conf.glance_image_id:
LOG.info('Executing glance backup. Image ID: {0}'.format(
self.conf.glance_image_id))
hostname_backup_name = os.path.join(
self.conf.hostname_backup_name,
self.conf.glance_image_id)
return self.engine.backup(
backup_resource=self.conf.glance_image_id,
hostname_backup_name=hostname_backup_name,
no_incremental=self.conf.no_incremental,
max_level=self.conf.max_level,
always_level=self.conf.always_level,
restart_always_level=self.conf.restart_always_level)
else:
executor = futures.ThreadPoolExecutor(
max_workers=len(self.glance_image_ids))
futures_list = []
for image_id in self.glance_image_ids:
hostname_backup_name = os.path.join(
self.conf.hostname_backup_name, image_id)
futures_list.append(executor.submit(
self.engine.backup(
backup_resource=image_id,
hostname_backup_name=hostname_backup_name,
no_incremental=self.conf.no_incremental,
max_level=self.conf.max_level,
always_level=self.conf.always_level,
restart_always_level=self.conf.restart_always_level
)))
futures.wait(futures_list, CONF.timeout)
elif backup_media == 'cindernative':
LOG.info('Executing cinder native backup. Volume ID: {0}, '
'incremental: {1}'.format(self.conf.cindernative_vol_id,
@ -362,10 +430,13 @@ class RestoreJob(Job):
if not any([self.conf.restore_abs_path,
self.conf.nova_inst_id,
self.conf.nova_inst_name,
self.conf.glance_image_id,
self.conf.cinder_vol_id,
self.conf.cinder_vol_name,
self.conf.cindernative_vol_id,
self.conf.cinderbrick_vol_id,
self.conf.glance_image_name,
self.conf.glance_image_name_filter,
self.conf.project_id]):
raise ValueError("--restore-abs-path is required")
if not self.conf.container:
@ -450,6 +521,42 @@ class RestoreJob(Job):
recent_to_date=restore_timestamp,
backup_media=conf.mode)
elif conf.backup_media == 'glance':
if self.conf.project_id:
return self.engine.restore_glance_tenant(
project_id=self.conf.project_id,
hostname_backup_name=self.conf.hostname_backup_name,
overwrite=conf.overwrite,
recent_to_date=restore_timestamp)
elif conf.glance_image_id:
LOG.info("Restoring glance backup. Image ID: {0}, "
"timestamp: {1} ".format(conf.glance_image_id,
restore_timestamp))
hostname_backup_name = os.path.join(
self.conf.hostname_backup_name,
self.conf.glance_image_id)
self.engine.restore(
hostname_backup_name=hostname_backup_name,
restore_resource=conf.glance_image_id,
overwrite=conf.overwrite,
recent_to_date=restore_timestamp,
backup_media=conf.mode)
else:
for image_id in self.glance_image_ids:
LOG.info("Restoring glance backup. Image ID: {0}, "
"timestamp: {1}".format(image_id,
restore_timestamp))
hostname_backup_name = os.path.join(
self.conf.hostname_backup_name, image_id)
self.engine.restore(
hostname_backup_name=hostname_backup_name,
restore_resource=image_id,
overwrite=conf.overwrite,
recent_to_date=restore_timestamp,
backup_media=conf.mode)
elif conf.backup_media == 'cinder':
if conf.cinder_vol_id:
LOG.info("Restoring cinder backup from glance. "

3
freezer/main.py
View File

@ -46,7 +46,6 @@ LOG = log.getLogger(__name__)
def freezer_main(backup_args):
"""Freezer main loop for job execution.
"""
if not backup_args.quiet:
LOG.info("Begin freezer agent process with args: {0}".format(sys.argv))
LOG.info('log file at {0}'.format(CONF.get('log_file')))
@ -61,7 +60,7 @@ def freezer_main(backup_args):
if (backup_args.storage ==
'swift' or
backup_args.backup_media in ['nova', 'cinder', 'cindernative',
'cinderbrick']):
'cinderbrick', 'glance']):
backup_args.client_manager = client_manager.get_client_manager(
backup_args.__dict__)

37
freezer/mode/glance.py Normal file
View File

@ -0,0 +1,37 @@
# (c) Copyright 2019 ZTE Corporation..
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from freezer.mode import mode
class GlanceMode(mode.Mode):
"""
Execute a glance backup/restore
"""
def __init__(self, conf):
self.conf = conf
@property
def name(self):
return "glance"
@property
def version(self):
return "1.0"
def release(self):
pass
def prepare(self):
pass

4
freezer/tests/commons.py
View File

@ -375,7 +375,9 @@ class BackupOpt1(object):
self.nova_inst_id = ''
self.nova_inst_name = ''
self.lvm_snapperm = 'ro'
self.glance_image_name = ''
self.glance_image_name_filter = ''
self.glance_image_id = ''
self.compression = 'gzip'
self.storage = mock.MagicMock()
self.engine = mock.MagicMock()