Merge "Backup/restore cinder/nova to ssh"

This commit is contained in:
Jenkins 2016-12-23 10:16:05 +00:00 committed by Gerrit Code Review
commit 01b8f89220
8 changed files with 67 additions and 12 deletions

View File

@ -290,7 +290,7 @@ class RestoreJob(Job):
" {0} ({1})".format(e.filename, e.strerror))
return {}
res = restore.RestoreOs(conf.client_manager, conf.container,
conf.storage)
self.storage)
if conf.backup_media == 'nova':
LOG.info("Restoring nova backup. Instance ID: {0}, timestamp: {1}"
.format(conf.nova_inst_id, restore_timestamp))

View File

@ -41,19 +41,21 @@ class RestoreOs(object):
:type restore_from_timestamp: int
:return:
"""
if self.storage == "swift":
if self.storage.type == "swift":
swift = self.client_manager.get_swift()
path = "{0}_segments/{1}/".format(self.container, path)
info, backups = swift.get_container(self.container, prefix=path)
backups = sorted(
map(lambda x: int(x["name"].rsplit("/", 1)[-1]), backups))
elif self.storage == "local":
elif self.storage.type == "local":
path = "{0}/{1}".format(self.container, path)
backups = os.listdir(os.path.abspath(path))
elif self.storage.type == "ssh":
path = "{0}/{1}".format(self.container, path)
backups = self.storage.listdir(path)
else:
# TODO(dstepanenko): handle ssh storage type here
msg = ("{} storage type is not supported at the moment."
" Try (local or swift)".format(self.storage))
" Try local, swift or ssh".format(self.storage.type))
print(msg)
raise BaseException(msg)
backups = list(filter(lambda x: x >= restore_from_timestamp, backups))
@ -72,7 +74,7 @@ class RestoreOs(object):
"""
swift = self.client_manager.get_swift()
backup = self._get_backups(path, restore_from_timestamp)
if self.storage == 'swift':
if self.storage.type == 'swift':
path = "{0}_segments/{1}/{2}".format(self.container, path, backup)
stream = swift.get_object(self.container,
"{}/{}".format(path, backup),
@ -86,7 +88,7 @@ class RestoreOs(object):
disk_format="raw",
data=data)
return info, image
elif self.storage == 'local':
elif self.storage.type == 'local':
image_file = "{0}/{1}/{2}/{3}".format(self.container, path,
backup, path)
metadata_file = "{0}/{1}/{2}/metadata".format(self.container,
@ -104,9 +106,25 @@ class RestoreOs(object):
disk_format="raw",
data=data)
return info, image
elif self.storage.type == 'ssh':
image_file = "{0}/{1}/{2}/{3}".format(self.container, path,
backup, path)
metadata_file = "{0}/{1}/{2}/metadata".format(self.container,
path, backup)
try:
data = self.storage.open(image_file, 'rb')
except Exception:
msg = "Failed to open remote image file {}".format(image_file)
LOG.error(msg)
raise BaseException(msg)
info = json.loads(self.storage.read_metadata_file(metadata_file))
image = self.client_manager.create_image(
name="restore_{}".format(path),
container_format="bare",
disk_format="raw",
data=data)
return info, image
else:
# TODO(yangyapeng) ssh storage need to implement
# storage is ssh storage
return {}
def restore_cinder(self, volume_id=None,

View File

@ -32,6 +32,7 @@ class Storage(object):
Any freezer storage implementation should be inherited from this abstract
class.
"""
_type = None
def __init__(self, skip_prepare=False):
if not skip_prepare:
@ -76,6 +77,10 @@ class Storage(object):
"""
pass
@property
def type(self):
return self._type
def get_latest_level_zero_increments(self, engine, hostname_backup_name,
recent_to_date=None):
"""

View File

@ -21,6 +21,8 @@ import json
@six.add_metaclass(abc.ABCMeta)
class FsLikeStorage(physical.PhysicalStorage):
_type = 'fslike'
def __init__(self, storage_path,
max_segment_size, skip_prepare=False):
super(FsLikeStorage, self).__init__(

View File

@ -24,6 +24,8 @@ from freezer.utils import utils
class LocalStorage(fslike.FsLikeStorage):
_type = 'local'
def get_file(self, from_path, to_path):
shutil.copyfile(from_path, to_path)

View File

@ -24,6 +24,7 @@ LOG = log.getLogger(__name__)
class MultipleStorage(base.Storage):
_type = 'multiple'
def info(self):
for s in self.storages:

View File

@ -25,11 +25,14 @@ from freezer.storage import fslike
from freezer.utils import utils
CHUNK_SIZE = 32768
class SshStorage(fslike.FsLikeStorage):
"""
:type ftp: paramiko.SFTPClient
"""
_type = 'ssh'
def __init__(self, storage_path, ssh_key_path,
remote_username, remote_ip, port, max_segment_size):
@ -51,8 +54,8 @@ class SshStorage(fslike.FsLikeStorage):
def init(self):
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(self.remote_ip, username=self.remote_username,
key_filename=self.ssh_key_path, port=self.port)
@ -100,7 +103,11 @@ class SshStorage(fslike.FsLikeStorage):
def listdir(self, directory):
try:
return self.ftp.listdir(directory)
# paramiko SFTPClient.listdir_attr returns
# directories in arbitarary order, so we should
# sort results of this command
res = self.ftp.listdir(directory)
return sorted(res)
except IOError as e:
if e.errno == errno.ENOENT:
return list()
@ -108,7 +115,25 @@ class SshStorage(fslike.FsLikeStorage):
raise
def open(self, filename, mode):
return self.ftp.open(filename, mode=mode)
return self.ftp.open(filename, mode=mode,
bufsize=self.max_segment_size)
def read_metadata_file(self, path):
file_stats = self.ftp.stat(path)
file_size = file_stats.st_size
data = ""
received_size = 0
with self.open(path, 'r') as reader:
reader.prefetch(file_size)
chunk = reader.read(CHUNK_SIZE)
while chunk:
received_size += len(chunk)
data += chunk
chunk = reader.read(CHUNK_SIZE)
if file_size != received_size:
raise IOError('Size mismatch: expected {} received {}'.format(
file_size, received_size))
return data
def backup_blocks(self, backup):
self.init() # should recreate ssh for new process

View File

@ -33,6 +33,8 @@ class SwiftStorage(physical.PhysicalStorage):
:type client_manager: freezer.osclients.ClientManager
"""
_type = 'swift'
def rmtree(self, path):
split = path.rsplit('/', 1)
for file in self.swift().get_container(split[0],