Data consistent way of making cinder backups
fs_backup_mode now checks volume_id parameter in conf. If we have volume_id - we execute cinder backup instead of tar or lvm backup. As far as mongo_mode, mysql_mode, sql_server_mode are using fs_backup_mode it will be execute cinder backup in case of provided volume_id parameter, otherwise it will execute lvm or tar backup. Implements blueprint: cinder-backup-mode Change-Id: I7e70d3a0888cafee87232ab28c8fea48788e5de4
This commit is contained in:
eldar nugaev
Fausto Marzi
parent
8184cf27ee
commit
79e5c3cb18
19
README.rst
19
README.rst
|
|
@ -196,9 +196,22 @@ Execute a MySQL backup using lvm snapshot::
|
|||
--mysql-conf /root/.freezer/freezer-mysql.conf--container
|
||||
freezer_mysql-backup-prod --mode mysql --backup-name mysql-ops002
|
||||
|
||||
Cinder backups
|
||||
|
||||
To make a cinder backup you should provide volume-id parameter in arguments.
|
||||
Freezer doesn't do any additional checks and assumes that making backup
|
||||
of that image will be sufficient to restore your data in future.
|
||||
|
||||
Execute a cinder backup::
|
||||
$ freezerc --volume-id 3ad7a62f-217a-48cd-a861-43ec0a04a78b
|
||||
|
||||
Execute a mysql backup with cinder::
|
||||
|
||||
$ freezerc --mysql-conf /root/.freezer/freezer-mysql.conf
|
||||
--container freezer_mysql-backup-prod --mode mysql
|
||||
--backup-name mysql-ops002
|
||||
--volume-id 3ad7a62f-217a-48cd-a861-43ec0a04a78b
|
||||
|
||||
All the freezerc activities are logged into /var/log/freezer.log.
|
||||
|
||||
Restore
|
||||
|
|
@ -257,6 +270,12 @@ Remove backups older then 1 day::
|
|||
|
||||
$ freezerc --action admin --container freezer_dev-test --remove-older-then 1 --backup-name dev-test-01
|
||||
|
||||
|
||||
Cinder restore currently creates a volume with content of saved one, but
|
||||
doesn't implement deattach of existing volume and attach the new one to the
|
||||
vm. You should implement this steps manually. To create new volume from
|
||||
existing content run next command:
|
||||
|
||||
Execute a cinder restore::
|
||||
$ freezerc --action restore --volume-id 3ad7a62f-217a-48cd-a861-43ec0a04a78b
|
||||
|
||||
|
|
|
|||
|
|
@ -147,7 +147,7 @@ def backup_mode_mongo(backup_opt_dict, time_stamp, manifest_meta_dict):
|
|||
return True
|
||||
|
||||
|
||||
def backup_mode_cinder(backup_dict, time_stamp, create_clients=True):
|
||||
def backup_cinder(backup_dict, time_stamp, create_clients=True):
|
||||
"""
|
||||
Implements cinder backup:
|
||||
1) Gets a stream of the image from glance
|
||||
|
|
@ -165,15 +165,22 @@ def backup_mode_cinder(backup_dict, time_stamp, create_clients=True):
|
|||
|
||||
volume_id = backup_dict.volume_id
|
||||
volume = backup_dict.cinder.volumes.get(volume_id)
|
||||
logging.info("[*] Creation temporary snapshot")
|
||||
snapshot = provide_snapshot(backup_dict, volume,
|
||||
"backup_snapshot_for_volume_%s" % volume_id)
|
||||
logging.info("[*] Creation temporary volume")
|
||||
copied_volume = do_copy_volume(backup_dict, snapshot)
|
||||
logging.info("[*] Creation temporary glance image")
|
||||
image = make_glance_image(backup_dict, "name", copied_volume)
|
||||
stream = download_image(backup_dict, image)
|
||||
package = "{0}/{1}".format(backup_dict, volume_id, time_stamp)
|
||||
logging.info("[*] Uploading image to swift")
|
||||
swift.add_stream(backup_dict, stream, package)
|
||||
logging.info("[*] Deleting temporary snapshot")
|
||||
clean_snapshot(backup_dict, snapshot)
|
||||
logging.info("[*] Deleting temporary volume")
|
||||
backup_dict.cinder.volumes.delete(copied_volume)
|
||||
logging.info("[*] Deleting temporary image")
|
||||
backup_dict.glance.images.delete(image)
|
||||
|
||||
|
||||
|
|
@ -184,6 +191,12 @@ def backup_mode_fs(backup_opt_dict, time_stamp, manifest_meta_dict):
|
|||
|
||||
logging.info('[*] File System backup is being executed...')
|
||||
|
||||
if backup_opt_dict.volume_id:
|
||||
logging.info('[*] Detected volume_id parameter')
|
||||
logging.info('[*] Executing cinder snapshot')
|
||||
backup_cinder(backup_opt_dict, time_stamp, manifest_meta_dict)
|
||||
return
|
||||
|
||||
try:
|
||||
|
||||
if is_windows():
|
||||
|
|
|
|||
|
|
@ -105,10 +105,7 @@ class BackupJob(Job):
|
|||
self.conf, manifest_meta_dict)
|
||||
|
||||
self.conf.manifest_meta_dict = manifest_meta_dict
|
||||
if self.conf.volume_id:
|
||||
backup.backup_mode_cinder(
|
||||
self.conf, self.start_time.timestamp)
|
||||
elif self.conf.mode == 'fs':
|
||||
if self.conf.mode == 'fs':
|
||||
backup.backup_mode_fs(
|
||||
self.conf, self.start_time.timestamp, manifest_meta_dict)
|
||||
elif self.conf.mode == 'mongo':
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ import os
|
|||
import logging
|
||||
import re
|
||||
import datetime
|
||||
import time
|
||||
|
||||
from freezer.tar import tar_restore
|
||||
from freezer.swift import object_to_stream
|
||||
|
|
@ -34,7 +33,7 @@ from freezer.glance import glance
|
|||
from freezer.cinder import cinder
|
||||
from freezer.glance import ReSizeStream
|
||||
from freezer.utils import (
|
||||
validate_all_args, get_match_backup, sort_backup_list)
|
||||
validate_all_args, get_match_backup, sort_backup_list, date_to_timestamp)
|
||||
|
||||
|
||||
def restore_fs(backup_opt_dict):
|
||||
|
|
@ -100,10 +99,7 @@ def restore_fs_sort_obj(backup_opt_dict):
|
|||
'''
|
||||
|
||||
# Convert backup_opt_dict.restore_from_date to timestamp
|
||||
fmt = '%Y-%m-%dT%H:%M:%S'
|
||||
opt_backup_date = datetime.datetime.strptime(
|
||||
backup_opt_dict.restore_from_date, fmt)
|
||||
opt_backup_timestamp = int(time.mktime(opt_backup_date.timetuple()))
|
||||
opt_backup_timestamp = date_to_timestamp(backup_opt_dict.restore_from_date)
|
||||
|
||||
# Sort remote backup list using timestamp in reverse order,
|
||||
# that is from the newest to the oldest executed backup
|
||||
|
|
@ -175,6 +171,8 @@ def restore_cinder(backup_opt_dict, create_clients=True):
|
|||
recreates cinder and glance clients,
|
||||
False - uses existing from backup_opt_dict
|
||||
"""
|
||||
timestamp = date_to_timestamp(backup_opt_dict.restore_from_date)
|
||||
|
||||
if create_clients:
|
||||
backup_opt_dict = cinder(backup_opt_dict)
|
||||
backup_opt_dict = glance(backup_opt_dict)
|
||||
|
|
@ -182,18 +180,22 @@ def restore_cinder(backup_opt_dict, create_clients=True):
|
|||
container = backup_opt_dict.container
|
||||
connector = backup_opt_dict.sw_connector
|
||||
info, backups = connector.get_container(container, path=volume_id)
|
||||
backups = sorted(map(lambda x: x["name"].rsplit("/", 1)[-1], backups))
|
||||
backups = sorted(map(lambda x: int(x["name"].rsplit("/", 1)[-1]), backups))
|
||||
backups = filter(lambda x: x >= timestamp, backups)
|
||||
|
||||
if not backups:
|
||||
msg = "Cannot find backups for volume: %s" % volume_id
|
||||
logging.error(msg)
|
||||
raise BaseException(msg)
|
||||
backup = backups[-1]
|
||||
|
||||
stream = connector.get_object(
|
||||
backup_opt_dict.container, "%s/%s" % (volume_id, backup),
|
||||
resp_chunk_size=10000000)
|
||||
length = int(stream[0]["x-object-meta-length"])
|
||||
stream = stream[1]
|
||||
images = backup_opt_dict.glance.images
|
||||
logging.info("[*] Creation glance image")
|
||||
image = images.create(data=ReSizeStream(stream, length, 1),
|
||||
container_format="bare",
|
||||
disk_format="raw")
|
||||
|
|
@ -201,6 +203,7 @@ def restore_cinder(backup_opt_dict, create_clients=True):
|
|||
size = length / gb
|
||||
if length % gb > 0:
|
||||
size += 1
|
||||
|
||||
logging.info("[*] Creation volume from image")
|
||||
backup_opt_dict.cinder.volumes.create(size, imageRef=image.id)
|
||||
logging.info("[*] Deleting temporary image")
|
||||
images.delete(image)
|
||||
|
|
|
|||
|
|
@ -633,3 +633,9 @@ def create_subprocess(cmd):
|
|||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
return process.communicate()
|
||||
|
||||
|
||||
def date_to_timestamp(date):
|
||||
fmt = '%Y-%m-%dT%H:%M:%S'
|
||||
opt_backup_date = datetime.datetime.strptime(date, fmt)
|
||||
return int(time.mktime(opt_backup_date.timetuple()))
|
||||
|
|
|
|||
2
setup.py
2
setup.py
|
|
@ -81,6 +81,8 @@ setup(
|
|||
install_requires=[
|
||||
'python-swiftclient>=1.6.0',
|
||||
'python-keystoneclient>=0.7.0',
|
||||
'python-cinderclient',
|
||||
'python-glanceclient',
|
||||
'pymysql',
|
||||
'pymongo',
|
||||
'docutils>=0.8.1'],
|
||||
|
|
|
|||
|
|
@ -614,8 +614,8 @@ class FakeSwiftClient:
|
|||
])
|
||||
elif container == "test-container" and 'path' in kwargs:
|
||||
return ({'container_metadata': True}, [
|
||||
{'bytes': 251, 'last_modified': '2015-03-09T10:37:01.701170', 'hash': '9a8cbdb30c226d11bf7849f3d48831b9', 'name': 'hostname_backup_name_1234567890_0/1234567890/67108864/00000000', 'content_type': 'application/octet-stream'},
|
||||
{'bytes': 632, 'last_modified': '2015-03-09T11:54:27.860730', 'hash': 'd657a4035d0dcc18deaf9bfd2a3d0ebf', 'name': 'hostname_backup_name_1234567891_1/1234567891/67108864/00000000', 'content_type': 'application/octet-stream'}
|
||||
{'bytes': 251, 'last_modified': '2015-03-09T10:37:01.701170', 'hash': '9a8cbdb30c226d11bf7849f3d48831b9', 'name': 'hostname_backup_name_1234567890_0/11417649003', 'content_type': 'application/octet-stream'},
|
||||
{'bytes': 632, 'last_modified': '2015-03-09T11:54:27.860730', 'hash': 'd657a4035d0dcc18deaf9bfd2a3d0ebf', 'name': 'hostname_backup_name_1234567891_1/1417649003', 'content_type': 'application/octet-stream'}
|
||||
])
|
||||
else:
|
||||
return [{}, []]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
from freezer.backup import backup_mode_mysql, backup_mode_fs, backup_mode_mongo
|
||||
from freezer.backup import backup_mode_cinder
|
||||
from freezer.backup import backup_cinder
|
||||
import freezer
|
||||
from freezer import cinder
|
||||
from freezer import glance
|
||||
|
|
@ -193,7 +193,7 @@ class TestBackUP:
|
|||
assert backup_mode_mongo(
|
||||
backup_opt, 123456789, test_meta) is True
|
||||
|
||||
def test_backup_mode_cinder(self, monkeypatch):
|
||||
def test_backup_cinder(self, monkeypatch):
|
||||
backup_opt = BackupOpt1()
|
||||
backup_opt.volume_id = 34
|
||||
|
||||
|
|
@ -202,4 +202,4 @@ class TestBackUP:
|
|||
fakeswiftclient = FakeSwiftClient()
|
||||
monkeypatch.setattr(swiftclient, 'client', fakeswiftclient.client)
|
||||
|
||||
backup_mode_cinder(backup_opt, 123456789, False)
|
||||
backup_cinder(backup_opt, 1417649003, False)
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ class TestRestore:
|
|||
backup_opt.backup_name = 'abcdtest'
|
||||
pytest.raises(Exception, restore_fs_sort_obj, backup_opt)
|
||||
|
||||
def test_backup_mode_cinder(self, monkeypatch):
|
||||
def test_restore_cinder(self, monkeypatch):
|
||||
backup_opt = BackupOpt1()
|
||||
backup_opt.volume_id = 34
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from freezer.utils import (
|
|||
eval_restart_backup, set_backup_level,
|
||||
get_vol_fs_type, check_backup_and_tar_meta_existence,
|
||||
add_host_name_ts_level, get_mount_from_path, human2bytes, DateTime,
|
||||
OpenstackOptions)
|
||||
date_to_timestamp)
|
||||
|
||||
from freezer import utils
|
||||
import pytest
|
||||
|
|
@ -299,6 +299,10 @@ class TestUtils:
|
|||
env_dict = dict(OS_USERNAME='testusername', OS_TENANT_NAME='testtenantename', OS_AUTH_URL='testauthurl')
|
||||
pytest.raises(Exception, OpenstackOptions.create_from_dict, env_dict)
|
||||
|
||||
def test_date_to_timestamp(self):
|
||||
assert 1417649003 == date_to_timestamp("2014-12-03T23:23:23")
|
||||
|
||||
|
||||
class TestDateTime:
|
||||
def setup(self):
|
||||
d = datetime.datetime(2015, 3, 7, 17, 47, 44, 716799)
|
||||
|
|
|
|||
Loading…
Reference in New Issue