openstack
/
freezer
Code Issues Proposed changes

Backup nova/cinder to local 

Freezer-agent backup volume_by_glance upload data to swift by default.
This patch backup it and metadata to local,Now.

usage:
backup nova / cinder by glance to local:
freezer-agent --cinder-vol-id 0bd0d291-e75c-4fd9-b13b-058c335a8497
--storage local --container /home/nfs --mode cinder

restore:

freezer-agent --action restore
--cinder-vol-id 0bd0d291-e75c-4fd9-b13b-058c335a8497
--storage local --container /home/nfs
--restore-from-date 2016-05-10T00:01:00

Implements blueprint: backup-nova-cinder-to-non-swift

Change-Id: I9bc8496fea7727edbf35130b7dbf9bb435d7038b
This commit is contained in:
yangyapeng 2016-09-28 08:49:18 +00:00 committed by Dmitry Stepanenko
parent 29a51a492e
commit eee345b4c6
7 changed files with 148 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
freezer/job.py
View File

@ -289,7 +289,8 @@ class RestoreJob(Job):
"Backup Consistency Check failed: could not checksum file"
" {0} ({1})".format(e.filename, e.strerror))
return {}
res = restore.RestoreOs(conf.client_manager, conf.container)
res = restore.RestoreOs(conf.client_manager, conf.container,
conf.storage)
if conf.backup_media == 'nova':
LOG.info("Restoring nova backup. Instance ID: {0}, timestamp: {1}"
.format(conf.nova_inst_id, restore_timestamp))

12
freezer/openstack/backup.py
View File

@ -70,7 +70,8 @@ class BackupOs(object):
package = "{0}/{1}".format(instance_id, utils.DateTime.now().timestamp)
LOG.info("Uploading image to swift")
headers = {"x-object-meta-name": instance.name,
"x-object-meta-flavor-id": str(instance.flavor.get('id'))}
"x-object-meta-flavor-id": str(instance.flavor.get('id')),
'x-object-meta-length': str(len(stream))}
self.storage.add_stream(stream, package, headers)
LOG.info("Deleting temporary image {0}".format(image))
glance.images.delete(image.id)
@ -99,7 +100,14 @@ class BackupOs(object):
stream = client_manager.download_image(image)
package = "{0}/{1}".format(volume_id, utils.DateTime.now().timestamp)
LOG.debug("Uploading image to swift")
headers = {}
headers = {'x-object-meta-length': str(len(stream)),
'volume_name': volume.name,
'volume_type': volume.volume_type,
'availability_zone': volume.availability_zone
}
attachments = volume._info['attachments']
if attachments:
headers['server'] = attachments[0]['server_id']
self.storage.add_stream(stream, package, headers=headers)
LOG.debug("Deleting temporary snapshot")
client_manager.clean_snapshot(snapshot)

22
freezer/openstack/osclients.py
View File

@ -286,6 +286,28 @@ class OSClientManager(object):
LOG.debug("Stream with size {0}".format(image.size))
return utils.ReSizeStream(stream, image.size, 1000000)
def create_image(self, name, container_format, disk_format, data=None):
LOG.info("Creating glance image")
glance = self.get_glance()
image = glance.images.create(name=name,
container_format=container_format,
disk_format=disk_format)
if image is None:
msg = "Failed to create glance image {}".format(name)
LOG.error(msg)
raise BaseException(msg)
if data is None:
return image
glance.images.upload(image.id, data)
while image.status not in ('active', 'killed'):
LOG.info("Waiting for glance image upload")
time.sleep(5)
image = glance.images.get(image.id)
if image.status == 'killed':
raise BaseException('Failed to upload data into image')
LOG.info("Created glance image {}".format(image.id))
return image
class OpenstackOpts(object):
"""

93
freezer/openstack/restore.py
View File

@ -16,6 +16,8 @@ limitations under the License.
Freezer restore modes related functions
"""
import json
import os
import time
from oslo_log import log
@ -26,9 +28,10 @@ LOG = log.getLogger(__name__)
class RestoreOs(object):
def __init__(self, client_manager, container):
def __init__(self, client_manager, container, storage):
self.client_manager = client_manager
self.container = container
self.storage = storage
def _get_backups(self, path, restore_from_timestamp):
"""
@ -38,11 +41,21 @@ class RestoreOs(object):
:type restore_from_timestamp: int
:return:
"""
swift = self.client_manager.get_swift()
path = "{0}_segments/{1}/".format(self.container, path)
info, backups = swift.get_container(self.container, prefix=path)
backups = sorted(
map(lambda x: int(x["name"].rsplit("/", 1)[-1]), backups))
if self.storage == "swift":
swift = self.client_manager.get_swift()
path = "{0}_segments/{1}/".format(self.container, path)
info, backups = swift.get_container(self.container, prefix=path)
backups = sorted(
map(lambda x: int(x["name"].rsplit("/", 1)[-1]), backups))
elif self.storage == "local":
path = "{0}/{1}".format(self.container, path)
backups = os.listdir(os.path.abspath(path))
else:
# TODO(dstepanenko): handle ssh storage type here
msg = ("{} storage type is not supported at the moment."
" Try (local or swift)".format(self.storage))
print(msg)
raise BaseException(msg)
backups = list(filter(lambda x: x >= restore_from_timestamp, backups))
if not backups:
msg = "Cannot find backups for path: %s" % path
@ -58,19 +71,43 @@ class RestoreOs(object):
:return:
"""
swift = self.client_manager.get_swift()
glance = self.client_manager.get_glance()
backup = self._get_backups(path, restore_from_timestamp)
path = "{0}_segments/{1}/{2}".format(self.container, path, backup)
stream = swift.get_object(self.container, "%s/%s" % (path, backup),
resp_chunk_size=10000000)
length = int(stream[0]["x-object-meta-length"])
LOG.info("Creation glance image")
image = glance.images.create(
container_format="bare", disk_format="raw")
glance.images.upload(image.id,
utils.ReSizeStream(stream[1], length, 1))
return stream[0], image
if self.storage == 'swift':
path = "{0}_segments/{1}/{2}".format(self.container, path, backup)
stream = swift.get_object(self.container,
"{}/{}".format(path, backup),
resp_chunk_size=10000000)
length = int(stream[0]["x-object-meta-length"])
data = utils.ReSizeStream(stream[1], length, 1)
info = stream[0]
image = self.client_manager.create_image(
name="restore_{}".format(path),
container_format="bare",
disk_format="raw",
data=data)
return info, image
elif self.storage == 'local':
image_file = "{0}/{1}/{2}/{3}".format(self.container, path,
backup, path)
metadata_file = "{0}/{1}/{2}/metadata".format(self.container,
path, backup)
try:
data = open(image_file, 'rb')
except Exception:
msg = "Failed to open image file {}".format(image_file)
LOG.error(msg)
raise BaseException(msg)
info = json.load(file(metadata_file))
image = self.client_manager.create_image(
name="restore_{}".format(path),
container_format="bare",
disk_format="raw",
data=data)
return info, image
else:
# TODO(yangyapeng) ssh storage need to implement
# storage is ssh storage
return {}
def restore_cinder(self, volume_id=None,
backup_id=None,
@ -127,9 +164,23 @@ class RestoreOs(object):
if length % gb > 0:
size += 1
LOG.info("Creating volume from image")
self.client_manager.get_cinder().volumes.create(size,
imageRef=image.id)
LOG.info("Deleting temporary image")
cinder_client = self.client_manager.get_cinder()
volume = cinder_client.volumes.create(size,
imageRef=image.id,
name=info['volume_name'])
while volume.status != "available":
try:
LOG.info("Volume copy status: " + volume.status)
volume = cinder_client.volumes.get(volume.id)
if volume.status == "error":
raise Exception("Volume copy status: error")
time.sleep(5)
except Exception as e:
LOG.exception(e)
if volume.status != "error":
LOG.warn("Exception getting volume status")
LOG.info("Deleting temporary image {}".format(image.id))
self.client_manager.get_glance().images.delete(image.id)
def restore_nova(self, instance_id, restore_from_timestamp,

25
freezer/storage/fslike.py
View File

@ -16,6 +16,7 @@ import abc
import six
from freezer.storage import physical
import json
@six.add_metaclass(abc.ABCMeta)
@ -73,3 +74,27 @@ class FsLikeStorage(physical.PhysicalStorage):
:return:
"""
pass
def add_stream(self, stream, package_name, headers=None):
"""
:param stream: data
:param package_name: path
:param headers: backup metadata infomation
:return:
"""
split = package_name.rsplit('/', 1)
# create backup_basedir
backup_basedir = "{0}/{1}".format(self.storage_path,
package_name)
self.create_dirs(backup_basedir)
# define backup_data_name
backup_basepath = "{0}/{1}".format(backup_basedir,
split[0])
backup_metadata = "%s/metadata" % backup_basedir
# write backup to backup_basepath
with self.open(backup_basepath, 'wb') as backup_file:
for el in stream:
backup_file.write(el)
# write data matadata to backup_metadata
with self.open(backup_metadata, 'wb') as backup_meta:
backup_meta.write(json.dumps(headers))

3
freezer/storage/swift.py
View File

@ -170,13 +170,12 @@ class SwiftStorage(physical.PhysicalStorage):
full_path = "{0}/{1}".format(backup_basepath, package_name)
headers['x-object-manifest'] = full_path
headers['x-object-meta-length'] = str(len(stream))
objname = package_name.rsplit('/', 1)[1]
# This call sets the metadata on a file which will be used to download
# the whole backup later. Do not remove it ! (szaher)
self.swift().put_object(container=full_path, obj=objname, contents=u'',
headers=headers)
content_length=len(u''), headers=headers)
def backup_blocks(self, backup):
"""

20
freezer/tests/unit/openstack/test_restore.py
View File

@ -26,20 +26,32 @@ class TestRestore(commons.FreezerBaseTestCase):
def test_restore_cinder_by_glance(self):
backup_opt = commons.BackupOpt1()
restore.RestoreOs(backup_opt.client_manager, backup_opt.container)
restore.RestoreOs(backup_opt.client_manager, backup_opt.container,
backup_opt.storage)
def test_restore_cinder_by_glance_from_local(self):
backup_opt = commons.BackupOpt1()
restore.RestoreOs(backup_opt.client_manager, backup_opt.container,
'local')
def test_restore_cinder_with_backup_id(self):
backup_opt = commons.BackupOpt1()
ros = restore.RestoreOs(backup_opt.client_manager,
backup_opt.container)
backup_opt.container, backup_opt.storage)
ros.restore_cinder(35, 34, 33)
def test_restore_cinder_without_backup_id(self):
backup_opt = commons.BackupOpt1()
ros = restore.RestoreOs(backup_opt.client_manager,
backup_opt.container)
backup_opt.container, backup_opt.storage)
ros.restore_cinder(35, 34)
def test_restore_nova(self):
backup_opt = commons.BackupOpt1()
restore.RestoreOs(backup_opt.client_manager, backup_opt.container)
restore.RestoreOs(backup_opt.client_manager, backup_opt.container,
backup_opt.storage)
def test_restore_nova_from_local(self):
backup_opt = commons.BackupOpt1()
restore.RestoreOs(backup_opt.client_manager, backup_opt.container,
'local')