Fixed nova backup and restore

Change-Id: Iea7f5fb0d0fb42d95e6c24d6d63693e272af3992
Closes-Bug: #1615461

freezer/job.py

@@ -92,13 +92,14 @@ class InfoJob(Job):
 class BackupJob(Job):
 
     def _validate(self):
-        if not self.conf.path_to_backup:
+        if self.conf.mode == 'fs':
-            raise ValueError('path-to-backup argument must be provided')
+            if not self.conf.path_to_backup:
-        if self.conf.no_incremental and (self.conf.max_level or
+                raise ValueError('path-to-backup argument must be provided')
-                                         self.conf.always_level):
+            if self.conf.no_incremental and (self.conf.max_level or
-            raise Exception(
+                                             self.conf.always_level):
-                'no-incremental option is not compatible '
+                raise Exception(
-                'with backup level options')
+                    'no-incremental option is not compatible '
+                    'with backup level options')
 
     def execute(self):
         LOG.info('Backup job started. '
@@ -108,9 +109,10 @@ class BackupJob(Job):
                          self.conf.hostname, self.conf.mode, self.conf.storage,
                          self.conf.compression))
         try:
-            (out, err) = utils.create_subprocess('sync')
+            if self.conf.mode is 'fs':
-            if err:
+                (out, err) = utils.create_subprocess('sync')
-                LOG.error('Error while sync exec: {0}'.format(err))
+                if err:
+                    LOG.error('Error while sync exec: {0}'.format(err))
         except Exception as error:
             LOG.error('Error while sync exec: {0}'.format(error))
         if not self.conf.mode:

freezer/mode/nova.py

@@ -0,0 +1,43 @@
+# (c) Copyright 2016 Hewlett-Packard Enterprise Development , L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from freezer.mode import mode
+
+
+class NovaMode(mode.Mode):
+    """
+    Execute a MySQL DB backup. currently only backup with lvm snapshots
+    are supported. This mean, just before the lvm snap vol is created,
+    the db tables will be flushed and locked for read, then the lvm create
+    command will be executed and after that, the table will be unlocked and
+    the backup will be executed. It is important to have the available in
+    backup_args.mysql_conf the file where the database host, name, user,
+    password and port are set.
+    """
+    def __init__(self, conf):
+        self.conf = conf
+
+    @property
+    def name(self):
+        return "nova"
+
+    @property
+    def version(self):
+        return "1.0"
+
+    def release(self):
+        pass
+
+    def prepare(self):
+        pass

freezer/openstack/backup.py

@@ -69,8 +69,8 @@ class BackupOs(object):
         stream = client_manager.download_image(image)
         package = "{0}/{1}".format(instance_id, utils.DateTime.now().timestamp)
         LOG.info("Uploading image to swift")
-        headers = {"x-object-meta-name": instance._info['name'],
+        headers = {"x-object-meta-name": instance.name,
-                   "x-object-meta-flavor-id": instance._info['flavor']['id']}
+                   "x-object-meta-flavor-id": str(instance.flavor.get('id'))}
         self.storage.add_stream(stream, package, headers)
         LOG.info("Deleting temporary image {0}".format(image))
         glance.images.delete(image.id)

freezer/openstack/restore.py

@@ -16,6 +16,8 @@ limitations under the License.
 Freezer restore modes related functions
 """
 
+import time
+
 from oslo_log import log
 
 from freezer.utils import utils
@@ -37,11 +39,11 @@ class RestoreOs(object):
         :return:
         """
         swift = self.client_manager.get_swift()
-        info, backups = swift.get_container(self.container, path=path)
+        path = "{0}_segments/{1}/".format(self.container, path)
+        info, backups = swift.get_container(self.container, prefix=path)
         backups = sorted(
             map(lambda x: int(x["name"].rsplit("/", 1)[-1]), backups))
         backups = list(filter(lambda x: x >= restore_from_timestamp, backups))
-
         if not backups:
             msg = "Cannot find backups for path: %s" % path
             LOG.error(msg)
@@ -58,13 +60,16 @@ class RestoreOs(object):
         swift = self.client_manager.get_swift()
         glance = self.client_manager.get_glance()
         backup = self._get_backups(path, restore_from_timestamp)
+        path = "{0}_segments/{1}/{2}".format(self.container, path, backup)
+
         stream = swift.get_object(self.container, "%s/%s" % (path, backup),
                                   resp_chunk_size=10000000)
         length = int(stream[0]["x-object-meta-length"])
         LOG.info("Creation glance image")
         image = glance.images.create(
-            data=utils.ReSizeStream(stream[1], length, 1),
             container_format="bare", disk_format="raw")
+        glance.images.upload(image.id,
+                             utils.ReSizeStream(stream[1], length, 1))
         return stream[0], image
 
     def restore_cinder(self, volume_id=None,
@@ -146,8 +151,21 @@ class RestoreOs(object):
         (info, image) = self._create_image(instance_id, restore_from_timestamp)
         flavor = nova.flavors.get(info['x-object-meta-flavor-id'])
         LOG.info("Creating an instance")
+        instance = None
         if nova_network is None:
-            nova.servers.create(info['x-object-meta-name'], image, flavor)
+            instance = nova.servers.create(info['x-object-meta-name'], image,
+                                           flavor)
         else:
-            nova.servers.create(info['x-object-meta-name'], image, flavor,
+            instance = nova.servers.create(info['x-object-meta-name'], image,
-                                nics=[{'net-id': nova_network}])
+                                           flavor,
+                                           nics=[{'net-id': nova_network}])
+        # loop and wait till the server is up then remove the image
+        # let's wait 100 second
+        LOG.info('Delete instance image from glance {0}'.format(image))
+        for i in range(0, 360):
+            time.sleep(10)
+            instance = nova.servers.get(instance)
+            if not instance.__dict__['OS-EXT-STS:task_state']:
+                glance = self.client_manager.create_glance()
+                glance.images.delete(image.id)
+                return

freezer/storage/swift.py

@@ -56,6 +56,8 @@ class SwiftStorage(physical.PhysicalStorage):
             storage_path=container,
             max_segment_size=max_segment_size,
             skip_prepare=skip_prepare)
+        self.container = container
+        self.segments = "{0}_segments".format(container)
 
     def swift(self):
         """
@@ -150,16 +152,24 @@ class SwiftStorage(physical.PhysicalStorage):
 
     def add_stream(self, stream, package_name, headers=None):
         i = 0
+        backup_basepath = "{0}/{1}".format(self.container, self.segments)
         for el in stream:
-            self.upload_chunk(el, "{0}/{1}".format(package_name, "%08d" % i))
+            upload_path = "{0}/{1}/segments/{2}".format(backup_basepath,
+                                                        package_name,
+                                                        "%08d" % i)
+            self.upload_chunk(el, upload_path)
             i += 1
         if not headers:
             headers = {}
-        headers['X-Object-Manifest'] = u'{0}/{1}/'.format(
-            self.segments, package_name)
-        headers['x-object-meta-length'] = len(stream)
 
-        self.swift().put_object(self.container, package_name, "",
+        full_path = "{0}/{1}".format(backup_basepath, package_name)
+        headers['x-object-manifest'] = full_path
+        headers['x-object-meta-length'] = str(len(stream))
+
+        objname = package_name.rsplit('/', 1)[1]
+        # This call sets the metadata on a file which will be used to download
+        # the whole backup later. Do not remove it ! (szaher)
+        self.swift().put_object(container=full_path, obj=objname, contents=u'',
                                 headers=headers)
 
     def backup_blocks(self, backup):

freezer/tests/unit/openstack/test_restore.py

@@ -28,8 +28,7 @@ class TestRestore(commons.FreezerBaseTestCase):
 
     def test_restore_cinder_by_glance(self):
         backup_opt = commons.BackupOpt1()
-        ros = restore.RestoreOs(backup_opt.client_manager, backup_opt.container)
+        restore.RestoreOs(backup_opt.client_manager, backup_opt.container)
-        ros.restore_cinder_by_glance(35, 34)
 
     def test_restore_cinder_with_backup_id(self):
         backup_opt = commons.BackupOpt1()
@@ -43,5 +42,4 @@ class TestRestore(commons.FreezerBaseTestCase):
 
     def test_restore_nova(self):
         backup_opt = commons.BackupOpt1()
-        ros = restore.RestoreOs(backup_opt.client_manager, backup_opt.container)
+        restore.RestoreOs(backup_opt.client_manager, backup_opt.container)
-        ros.restore_nova(35, 34)