openstack
/
freezer
Code Issues Proposed changes

Merge "[WIP] Add the backup and restore of glance image"

This commit is contained in:
Zuul 2019-06-14 07:18:51 +00:00 committed by Gerrit Code Review
parent c8981898e3 76a42bd162
commit 88cd6c6f01
7 changed files with 459 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
freezer/common/config.py
View File

@ -73,6 +73,9 @@ DEFAULT_PARAMS = {
'cinder_vol_name': '',
'nova_inst_id': '', '__version__': FREEZER_VERSION,
'nova_inst_name': '',
'glance_image_id': '',
'glance_image_name': '',
'glance_image_name_filter': '',
'remove_older_than': None, 'restore_from_date': None,
'upload_limit': -1, 'always_level': False, 'version': None,
'dry_run': False, 'lvm_snapsize': DEFAULT_LVM_SNAPSIZE,
@ -122,11 +125,13 @@ _COMMON = [
"(filesystem),mongo (MongoDB), mysql (MySQL), "
"sqlserver(SQL Server), "
"cinder(OpenStack Volume backup by freezer), "
"glance(OpenStack Image backup by freezer), "
"cindernative(OpenStack native cinder-volume backup), "
"nova(OpenStack Instance). Default set to fs"),
cfg.StrOpt('engine',
short='e',
choices=['tar', 'rsync', 'rsyncv2', 'nova', 'osbrick'],
choices=['tar', 'rsync', 'rsyncv2',
'nova', 'osbrick', 'glance'],
dest='engine_name',
default=DEFAULT_PARAMS['engine_name'],
help="Engine to be used for backup/restore. "
@ -401,6 +406,21 @@ _COMMON = [
dest='cindernative_backup_id',
help="Id of the cindernative backup to be restored"
),
cfg.StrOpt('glance-image-id',
dest='glance_image_id',
default=DEFAULT_PARAMS['glance_image_id'],
help="Id of glance image for backup"
),
cfg.StrOpt('glance-image-name',
dest='glance_image_name',
default=DEFAULT_PARAMS['glance_image_name'],
help="Name of glance image for backup"
),
cfg.StrOpt('glance-image-name-filter',
dest='glance_image_name_filter',
default=DEFAULT_PARAMS['glance_image_name_filter'],
help="Name filter of glance image for backup"
),
cfg.StrOpt('nova-inst-id',
dest='nova_inst_id',
default=DEFAULT_PARAMS['nova_inst_id'],
@ -714,6 +734,11 @@ def get_backup_args():
backup_args.nova_inst_id or
backup_args.nova_inst_name):
backup_media = 'nova'
elif backup_args.engine_name == 'glance':
if (backup_args.project_id or backup_args.glance_image_id or
backup_args.glance_image_name or
backup_args.glance_image_name_filter):
backup_media = 'glance'
elif backup_args.cinderbrick_vol_id:
backup_media = 'cinderbrick'

0
freezer/engine/glance/__init__.py Normal file
View File

285
freezer/engine/glance/glance.py Normal file
View File

@ -0,0 +1,285 @@
# (c) Copyright 2019 ZTE Corporation..
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from concurrent import futures
import os
from oslo_config import cfg
from oslo_log import log
from oslo_serialization import jsonutils as json
from freezer.common import client_manager
from freezer.engine import engine
from freezer.engine.tar import tar
from freezer.utils import utils
import tempfile
LOG = log.getLogger(__name__)
CONF = cfg.CONF
class GlanceEngine(engine.BackupEngine):
def __init__(self, storage, **kwargs):
super(GlanceEngine, self).__init__(storage=storage)
self.client = client_manager.get_client_manager(CONF)
self.glance = self.client.create_glance()
self.encrypt_pass_file = kwargs.get('encrypt_key')
self.exclude = kwargs.get('exclude')
self.server_info = None
self.openssl_path = None
self.compression_algo = 'gzip'
self.is_windows = None
self.dry_run = kwargs.get('dry_run', False)
self.max_segment_size = kwargs.get('max_segment_size')
self.storage = storage
self.dereference_symlink = kwargs.get('symlinks')
@property
def name(self):
return "glance"
def stream_image(self, pipe):
"""Reading bytes from a pipe and converting it to a stream-like"""
try:
while True:
chunk = pipe.recv_bytes()
yield chunk
except EOFError:
pass
def get_glance_tenant(self, project_id):
# Load info about tenant images
if self.storage._type == 'swift':
swift_connection = self.client.create_swift()
headers, data = swift_connection.get_object(
self.storage.storage_path,
"project_glance_" + project_id)
elif self.storage._type == 's3':
bucket_name, object_name = self.get_storage_info(project_id)
data = self.storage.get_object(
bucket_name=bucket_name,
key=object_name
)['Body'].read()
elif self.storage._type in ['local', 'ssh', 'ftp', 'ftps']:
backup_basepath = os.path.join(self.storage.storage_path,
'project_glance_' + project_id)
with self.storage.open(backup_basepath, 'rb') as backup_file:
data = backup_file.readline()
elif self.storage._type in ['ftp', 'ftps']:
backup_basepath = os.path.join(self.storage.storage_path,
'project_glance_' + project_id)
file = tempfile.NamedTemporaryFile('wb', delete=True)
self.storage.get_file(backup_basepath, file.name)
with open(file.name) as f:
data = f.readline()
LOG.info("get_glance_tenant download {0}".format(data))
return json.loads(data)
def restore_glance_tenant(self, project_id, hostname_backup_name,
overwrite, recent_to_date):
image_ids = self.get_glance_tenant(project_id)
for image_id in image_ids:
LOG.info("Restore glance image ID: {0} from container {1}".
format(image_id, self.storage.storage_path))
backup_name = os.path.join(hostname_backup_name,
image_id)
self.restore(
hostname_backup_name=backup_name,
restore_resource=image_id,
overwrite=overwrite,
recent_to_date=recent_to_date)
def restore_level(self, restore_resource, read_pipe, backup, except_queue):
try:
metadata = backup.metadata()
if (not self.encrypt_pass_file and
metadata.get("encryption", False)):
raise Exception("Cannot restore encrypted backup without key")
engine_metadata = backup.engine_metadata()
image_info = metadata.get('image', {})
container_format = image_info.get('container_format', 'bare')
disk_format = image_info.get('disk_format', 'raw')
length = int(engine_metadata.get('length'))
stream = self.stream_image(read_pipe)
data = utils.ReSizeStream(stream, length, 1)
image = self.client.create_image(
"Restore: {0}".format(
image_info.get('name', image_info.get('id', None))
),
container_format,
disk_format,
data=data
)
if self.encrypt_pass_file:
try:
tmpdir = tempfile.mkdtemp()
except Exception:
LOG.error("Unable to create a tmp directory")
raise
tar_engine = tar.TarEngine(self.compression_algo,
self.dereference_symlink,
self.exclude, self.storage,
self.max_segment_size,
self.encrypt_pass_file,
self.dry_run)
tar_engine.restore_level(tmpdir, read_pipe, backup,
except_queue)
utils.wait_for(
GlanceEngine.image_active,
1,
CONF.timeout,
message="Waiting for image to finish uploading {0} and become"
" active".format(image.id),
kwargs={"glance_client": self.glance, "image_id": image.id}
)
return image
except Exception as e:
LOG.exception(e)
except_queue.put(e)
raise
def backup_glance_tenant(self, project_id, hostname_backup_name,
no_incremental, max_level, always_level,
restart_always_level):
# import pdb;pdb.set_trace()
image_ids = [image.id for image in
self.glance.images.list(detailed=False)]
data = json.dumps(image_ids)
LOG.info("Saving information about image {0}".format(data))
if self.storage._type == 'swift':
swift_connection = self.client.create_swift()
swift_connection.put_object(self.storage.storage_path,
"project_glance_{0}".
format(project_id),
data)
elif self.storage._type == 's3':
bucket_name, object_name = self.get_storage_info(project_id)
self.storage.put_object(
bucket_name=bucket_name,
key=object_name,
data=data
)
elif self.storage._type in ['local', 'ssh']:
backup_basepath = os.path.join(self.storage.storage_path,
"project_glance_" + project_id)
with self.storage.open(backup_basepath, 'wb') as backup_file:
backup_file.write(data)
elif self.storage._type in ['ftp', 'ftps']:
backup_basepath = os.path.join(self.storage.storage_path,
'project_glance_' + project_id)
file = tempfile.NamedTemporaryFile('wb', delete=True)
with open(file.name, 'wb') as f:
f.write(data)
LOG.info("backup_glance_tenant data={0}".format(data))
self.storage.put_file(file.name, backup_basepath)
executor = futures.ThreadPoolExecutor(
max_workers=len(image_ids))
futures_list = []
for image_id in image_ids:
LOG.info("Backup glance image ID: {0} to container {1}".
format(image_id, self.storage.storage_path))
backup_name = os.path.join(hostname_backup_name,
image_id)
futures_list.append(executor.submit(
self.backup,
backup_resource=image_id,
hostname_backup_name=backup_name,
no_incremental=no_incremental,
max_level=max_level,
always_level=always_level,
restart_always_level=restart_always_level))
futures.wait(futures_list, CONF.timeout)
def get_storage_info(self, project_id):
if self.storage.get_object_prefix() != '':
object_name = "{0}/project_{1}".format(
self.storage.get_object_prefix(),
project_id
)
else:
object_name = "project_{0}".format(project_id)
return self.storage.get_bucket_name(), object_name
def backup_data(self, backup_resource, manifest_path):
# import pdb;pdb.set_trace()
image = self.glance.images.get(backup_resource)
if not image:
raise Exception(
"Image {0} can't be found.".format(backup_resource)
)
LOG.info('Image backup')
stream = self.client.download_image(image)
LOG.info("Uploading image to storage path")
headers = {"image_name": image.name,
"image_id": image.get('id'),
"disk_format": image.get('disk_format'),
"container_format": image.get('container_format'),
"visibility": image.get('visibility'),
'length': str(len(stream)),
"protected": image.protected}
self.set_tenant_meta(manifest_path, headers)
for chunk in stream:
yield chunk
if self.encrypt_pass_file:
tar_engine = tar.TarEngine(self.compression_algo,
self.dereference_symlink,
self.exclude, self.storage,
self.max_segment_size,
self.encrypt_pass_file, self.dry_run)
for data_chunk in tar_engine.backup_data('.', manifest_path):
yield data_chunk
@staticmethod
def image_active(glance_client, image_id):
"""Check if the image is in the active state or not"""
image = glance_client.images.get(image_id)
return image.status == 'active'
def metadata(self, backup_resource):
"""Construct metadata"""
# import pdb;pdb.set_trace()
image_info = self.glance.images.get(backup_resource)
return {
"engine_name": self.name,
"image": image_info,
"encryption": bool(self.encrypt_pass_file)
}
def set_tenant_meta(self, path, metadata):
"""push data to the manifest file"""
with open(path, 'wb') as fb:
fb.writelines(json.dumps(metadata))
def get_tenant_meta(self, path):
with open(path, 'rb') as fb:
json.loads(fb.read())

107
freezer/job.py
View File

@ -53,6 +53,7 @@ class Job(object):
self.client = client_manager.get_client_manager(CONF)
self.nova = self.client.get_nova()
self.cinder = self.client.get_cinder()
self.glance = self.client.get_glance()
self._general_validation()
self._validate()
if self.conf.nova_inst_name:
@ -66,6 +67,18 @@ class Job(object):
if volume.name ==
self.conf.cinder_inst_name]
if self.conf.glance_image_name:
self.glance_image_ids = [image.id for image in
self.glance.images.list()
if image.name ==
self.conf.glance_image_name]
if self.conf.glance_image_name_filter:
self.glance_image_ids = [image.id for image in
self.glance.images.list()
if self.conf.glance_image_name_filter
not in image.name]
@abc.abstractmethod
def _validate(self):
"""
@ -143,6 +156,17 @@ class BackupJob(Job):
and not self.conf.nova_inst_name:
raise ValueError("nova-inst-id or project-id or nova-inst-name"
" argument must be provided")
if self.conf.mode == 'glance':
if not self.conf.no_incremental:
raise ValueError("Incremental glance backup is not supported")
if not self.conf.glance_image_id and not self.conf.project_id \
and not self.conf.glance_image_name \
and not self.conf.glance_image_name_filter:
raise ValueError("glance-image-id or project-id or"
" glance-image-name or "
" glance-image-name_filter "
"argument must be provided")
if self.conf.mode == 'cinder':
if not self.conf.cinder_vol_id and not self.conf.cinder_vol_name:
@ -317,6 +341,50 @@ class BackupJob(Job):
futures.wait(futures_list, CONF.timeout)
elif backup_media == 'glance':
if self.conf.project_id:
return self.engine.backup_glance_tenant(
project_id=self.conf.project_id,
hostname_backup_name=self.conf.hostname_backup_name,
no_incremental=self.conf.no_incremental,
max_level=self.conf.max_level,
always_level=self.conf.always_level,
restart_always_level=self.conf.restart_always_level)
elif self.conf.glance_image_id:
LOG.info('Executing glance backup. Image ID: {0}'.format(
self.conf.glance_image_id))
hostname_backup_name = os.path.join(
self.conf.hostname_backup_name,
self.conf.glance_image_id)
return self.engine.backup(
backup_resource=self.conf.glance_image_id,
hostname_backup_name=hostname_backup_name,
no_incremental=self.conf.no_incremental,
max_level=self.conf.max_level,
always_level=self.conf.always_level,
restart_always_level=self.conf.restart_always_level)
else:
executor = futures.ThreadPoolExecutor(
max_workers=len(self.glance_image_ids))
futures_list = []
for image_id in self.glance_image_ids:
hostname_backup_name = os.path.join(
self.conf.hostname_backup_name, image_id)
futures_list.append(executor.submit(
self.engine.backup(
backup_resource=image_id,
hostname_backup_name=hostname_backup_name,
no_incremental=self.conf.no_incremental,
max_level=self.conf.max_level,
always_level=self.conf.always_level,
restart_always_level=self.conf.restart_always_level
)))
futures.wait(futures_list, CONF.timeout)
elif backup_media == 'cindernative':
LOG.info('Executing cinder native backup. Volume ID: {0}, '
'incremental: {1}'.format(self.conf.cindernative_vol_id,
@ -362,10 +430,13 @@ class RestoreJob(Job):
if not any([self.conf.restore_abs_path,
self.conf.nova_inst_id,
self.conf.nova_inst_name,
self.conf.glance_image_id,
self.conf.cinder_vol_id,
self.conf.cinder_vol_name,
self.conf.cindernative_vol_id,
self.conf.cinderbrick_vol_id,
self.conf.glance_image_name,
self.conf.glance_image_name_filter,
self.conf.project_id]):
raise ValueError("--restore-abs-path is required")
if not self.conf.container:
@ -450,6 +521,42 @@ class RestoreJob(Job):
recent_to_date=restore_timestamp,
backup_media=conf.mode)
elif conf.backup_media == 'glance':
if self.conf.project_id:
return self.engine.restore_glance_tenant(
project_id=self.conf.project_id,
hostname_backup_name=self.conf.hostname_backup_name,
overwrite=conf.overwrite,
recent_to_date=restore_timestamp)
elif conf.glance_image_id:
LOG.info("Restoring glance backup. Image ID: {0}, "
"timestamp: {1} ".format(conf.glance_image_id,
restore_timestamp))
hostname_backup_name = os.path.join(
self.conf.hostname_backup_name,
self.conf.glance_image_id)
self.engine.restore(
hostname_backup_name=hostname_backup_name,
restore_resource=conf.glance_image_id,
overwrite=conf.overwrite,
recent_to_date=restore_timestamp,
backup_media=conf.mode)
else:
for image_id in self.glance_image_ids:
LOG.info("Restoring glance backup. Image ID: {0}, "
"timestamp: {1}".format(image_id,
restore_timestamp))
hostname_backup_name = os.path.join(
self.conf.hostname_backup_name, image_id)
self.engine.restore(
hostname_backup_name=hostname_backup_name,
restore_resource=image_id,
overwrite=conf.overwrite,
recent_to_date=restore_timestamp,
backup_media=conf.mode)
elif conf.backup_media == 'cinder':
if conf.cinder_vol_id:
LOG.info("Restoring cinder backup from glance. "

3
freezer/main.py
View File

@ -46,7 +46,6 @@ LOG = log.getLogger(__name__)
def freezer_main(backup_args):
"""Freezer main loop for job execution.
"""
if not backup_args.quiet:
LOG.info("Begin freezer agent process with args: {0}".format(sys.argv))
LOG.info('log file at {0}'.format(CONF.get('log_file')))
@ -61,7 +60,7 @@ def freezer_main(backup_args):
if (backup_args.storage ==
'swift' or
backup_args.backup_media in ['nova', 'cinder', 'cindernative',
'cinderbrick']):
'cinderbrick', 'glance']):
backup_args.client_manager = client_manager.get_client_manager(
backup_args.__dict__)

37
freezer/mode/glance.py Normal file
View File

@ -0,0 +1,37 @@
# (c) Copyright 2019 ZTE Corporation..
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from freezer.mode import mode
class GlanceMode(mode.Mode):
"""
Execute a glance backup/restore
"""
def __init__(self, conf):
self.conf = conf
@property
def name(self):
return "glance"
@property
def version(self):
return "1.0"
def release(self):
pass
def prepare(self):
pass

4
freezer/tests/commons.py
View File

@ -375,7 +375,9 @@ class BackupOpt1(object):
self.nova_inst_id = ''
self.nova_inst_name = ''
self.lvm_snapperm = 'ro'
self.glance_image_name = ''
self.glance_image_name_filter = ''
self.glance_image_id = ''
self.compression = 'gzip'
self.storage = mock.MagicMock()
self.engine = mock.MagicMock()