Revert "Revert "Add support of backing store volumes from templates""

Migration works correctly, DB should be prepared.

This reverts commit 466f235371.

Change-Id: Ida4e40c4cb2e7f954861465c100b46cf8196c2e5

devops/driver/libvirt/libvirt_driver.py

@@ -676,7 +676,7 @@ class LibvirtVolume(Volume):
     """Note: This class is imported as Volume at .__init__.py """
 
     uuid = ParamField()
-    capacity = ParamField(default=None)
+    capacity = ParamField(default=None)  # in gigabytes
     format = ParamField(default='qcow2', choices=('qcow2', 'raw'))
     source_image = ParamField(default=None)
     serial = ParamField()
@@ -695,24 +695,49 @@ class LibvirtVolume(Volume):
 
     @retry(libvirt.libvirtError)
     def define(self):
-        name = underscored(
+        # Generate libvirt volume name
-            deepgetattr(self, 'node.group.environment.name'),
+        if self.node:
-            deepgetattr(self, 'node.name'),
+            name = underscored(
-            self.name,
+                deepgetattr(self, 'node.group.environment.name'),
-        )
+                deepgetattr(self, 'node.name'),
+                self.name,
+            )
+        elif self.group:
+            name = underscored(
+                deepgetattr(self, 'group.environment.name'),
+                deepgetattr(self, 'group.name'),
+                self.name,
+            )
+        else:
+            raise DevopsError("Can't craete volume that is not "
+                              "associated with any node or group")
 
+        # Find backing store format and path
         backing_store_path = None
         backing_store_format = None
-        if self.backing_store is not None:
+        if self.backing_store:
+            if not self.backing_store.exists():
+                raise DevopsError(
+                    "Can't create volume {!r}. backing_store volume {!r} does "
+                    "not exists.".format(self.name, self.backing_store.name))
             backing_store_path = self.backing_store.get_path()
             backing_store_format = self.backing_store.format
 
-        capacity = int((self.capacity or 0) * 1024 ** 3)
+        # Select capacity
-        if self.source_image is not None:
+        if self.capacity:
-            file_size = get_file_size(self.source_image)
+            # if capacity specified, use it first
-            if file_size > capacity:
+            capacity = int(self.capacity * 1024 ** 3)
-                capacity = file_size
+        elif self.source_image is not None:
+            # limit capacity to the sorse image file size
+            capacity = get_file_size(self.source_image)
+        elif self.backing_store:
+            # limit capacity to backing_store capacity
+            capacity = self.backing_store.get_capacity()
+        else:
+            raise DevopsError("Can't create volume {!r}: no capacity or "
+                              "source_image specified".format(self.name))
 
+        # Generate xml
         pool_name = self.driver.storage_pool_name
         pool = self.driver.conn.storagePoolLookupByName(pool_name)
         xml = LibvirtXMLBuilder.build_volume_xml(
@@ -722,12 +747,19 @@ class LibvirtVolume(Volume):
             backing_store_path=backing_store_path,
             backing_store_format=backing_store_format,
         )
+
+        # Define volume
         libvirt_volume = pool.createXML(xml, 0)
+
+        # Save uuid
         self.uuid = libvirt_volume.key()
+
+        # Set serial and wwn
         if not self.serial:
             self.serial = uuid.uuid4().hex
         if not self.wwn:
             self.wwn = '0' + ''.join(uuid.uuid4().hex)[:15]
+
         super(LibvirtVolume, self).define()
 
         # Upload predefined image to the volume
@@ -742,7 +774,7 @@ class LibvirtVolume(Volume):
         super(LibvirtVolume, self).remove()
 
     def get_capacity(self):
-        """Get volume capacity"""
+        """Get volume capacity in bytes"""
         return self._libvirt_volume.info()[1]
 
     def get_format(self):
@@ -753,8 +785,9 @@ class LibvirtVolume(Volume):
         return self._libvirt_volume.path()
 
     def fill_from_exist(self):
-        self.capacity = self.get_capacity()
+        msg = 'LibvirtVolume.fill_from_exist() is deprecated and do nothing'
-        self.format = self.get_format()
+        warn(msg, DeprecationWarning)
+        logger.debug(msg)
 
     @retry(libvirt.libvirtError, count=2)
     def upload(self, path, capacity=0):
@@ -812,25 +845,31 @@ class LibvirtVolume(Volume):
         cls = self.driver.get_model_class('Volume')
         return cls.objects.create(
             name=name,
-            capacity=self.capacity,
             node=self.node,
             format=self.format,
             backing_store=self,
         )
 
-    # TO REWRITE, LEGACY, for fuel-qa compatibility
+    # LEGACY, for fuel-qa compatibility
-    # Used for EXTERNAL SNAPSHOTS
     @classmethod
     def volume_get_predefined(cls, uuid):
         """Get predefined volume
 
         :rtype : Volume
         """
+        msg = ('LibvirtVolume.volume_get_predefined() is deprecated. '
+               'Please use Volumes associated with Groups')
+        warn(msg, DeprecationWarning)
+        logger.debug(msg)
+
         try:
             volume = cls.objects.get(uuid=uuid)
         except cls.DoesNotExist:
             volume = cls(uuid=uuid)
-        volume.fill_from_exist()
+        if not volume.exists():
+            raise DevopsError(
+                'Predefined volume {!r} not found'.format(uuid))
+        volume.format = volume.get_format()
         volume.save()
         return volume
 

devops/migrations/0001_initial.py

@@ -159,6 +159,7 @@ class Migration(migrations.Migration):
                 ('params', jsonfield.fields.JSONField(default={})),
                 ('name', models.CharField(max_length=255)),
                 ('backing_store', models.ForeignKey(to='devops.Volume', null=True)),
+                ('group', models.ForeignKey(to='devops.Group', null=True)),
                 ('node', models.ForeignKey(to='devops.Node', null=True)),
             ],
             options={
@@ -202,7 +203,7 @@ class Migration(migrations.Migration):
         ),
         migrations.AlterUniqueTogether(
             name='volume',
-            unique_together=set([('name', 'node')]),
+            unique_together=set([('name', 'group'), ('name', 'node')]),
         ),
         migrations.AlterUniqueTogether(
             name='node',

devops/models/base.py

@@ -158,7 +158,7 @@ class ParamField(ParamFieldBase):
     * to set default value.
     * to limit values using a list of allowed values.
 
-    Examples of ussage::
+    Examples of usage::
 
         class A(ParamedModel):
             foo = ParamField(default=10)
@@ -229,9 +229,9 @@ class ParamMultiField(ParamFieldBase):
 
         self.subfields = []
         for name, field in subfields.items():
-            if not isinstance(field, (ParamField, ParamMultiField)):
+            if not isinstance(field, ParamFieldBase):
                 raise DevopsError('field "{}" has wrong type;'
-                                  ' should be ParamField or ParamMultiField'
+                                  ' should be ParamFieldBase subclass instance'
                                   ''.format(name))
             field.set_param_key(name)
             self.subfields.append(field)
@@ -322,6 +322,10 @@ class ParamedModelQuerySet(query.QuerySet):
                 # NOTE(astudenov): no support for 'gt', 'lt', 'in'
                 # and other django's filter stuff
 
+                if not isinstance(item, self.model):
+                    # skip other classes
+                    continue
+
                 item_val = deepgetattr(item, key, splitter='__',
                                        do_raise=True)
                 if item_val != value:

devops/models/environment.py

@@ -186,6 +186,8 @@ class Environment(BaseModel):
     def define(self):
         for group in self.get_groups():
             group.define_networks()
+        for group in self.get_groups():
+            group.define_volumes()
         for group in self.get_groups():
             group.define_nodes()
 
@@ -344,6 +346,11 @@ class Environment(BaseModel):
         # Connect nodes to already created networks
         for group_data in groups:
             group = environment.get_group(name=group_data['name'])
+
+            # add group volumes
+            group.add_volumes(
+                group_data.get('group_volumes', []))
+
             # add nodes
             group.add_nodes(
                 group_data.get('nodes', []))

devops/models/group.py

@@ -22,6 +22,7 @@ from devops.models.base import BaseModel
 from devops.models.network import L2NetworkDevice
 from devops.models.network import NetworkPool
 from devops.models.node import Node
+from devops.models.node import Volume
 
 
 class Group(BaseModel):
@@ -87,6 +88,10 @@ class Group(BaseModel):
     def has_snapshot(self, name):
         return all(n.has_snapshot(name) for n in self.get_nodes())
 
+    def define_volumes(self):
+        for volume in self.get_volumes():
+            volume.define()
+
     def define_networks(self):
         for l2_network_device in self.get_l2_network_devices():
             l2_network_device.define()
@@ -113,6 +118,9 @@ class Group(BaseModel):
         for node in self.get_nodes():
             node.erase()
 
+        for volume in self.get_volumes():
+            volume.erase()
+
         for l2_network_device in self.get_l2_network_devices():
             l2_network_device.erase()
         self.delete()
@@ -214,3 +222,26 @@ class Group(BaseModel):
             name=name,
             address_pool=address_pool,
         )
+
+    def add_volumes(self, volumes):
+        for vol_params in volumes:
+            self.add_volume(
+                **vol_params
+            )
+
+    def add_volume(self, name, **params):
+        cls = self.driver.get_model_class('Volume')
+        return cls.objects.create(
+            group=self,
+            name=name,
+            **params
+        )
+
+    def get_volume(self, **kwargs):
+        try:
+            return self.volume_set.get(**kwargs)
+        except Volume.DoesNotExist:
+            raise DevopsObjNotFound(Volume, **kwargs)
+
+    def get_volumes(self, **kwargs):
+        return self.volume_set.filter(**kwargs)

devops/models/node.py

@@ -373,6 +373,12 @@ class Node(six.with_metaclass(ExtendableNodeType, ParamedModel, BaseModel)):
     # NEW
     def add_volume(self, name, device='disk', bus='virtio', **params):
         cls = self.driver.get_model_class('Volume')
+
+        if 'backing_store' in params:
+            # Backing storage volume have to be defined in group
+            params['backing_store'] = self.group.get_volume(
+                name=params['backing_store'])
+
         volume = cls.objects.create(
             node=self,
             name=name,

devops/models/volume.py

@@ -21,17 +21,18 @@ from devops.models.base import ParamField
 
 class Volume(ParamedModel, BaseModel):
     class Meta(object):
-        unique_together = ('name', 'node')
+        unique_together = (('name', 'node'), ('name', 'group'))
         db_table = 'devops_volume'
         app_label = 'devops'
 
     backing_store = models.ForeignKey('self', null=True)
     name = models.CharField(max_length=255, unique=False, null=False)
     node = models.ForeignKey('Node', null=True)
+    group = models.ForeignKey('Group', null=True)
 
     @property
     def driver(self):
-        return self.node.driver
+        return self.node.driver if self.node else self.group.driver
 
     def define(self, *args, **kwargs):
         self.save()

devops/tests/driver/libvirt/test_node_cloudimage.py

@@ -76,8 +76,8 @@ class TestCloudImage(LibvirtTestCase):
             cloud_init_volume_name='iso',
             cloud_init_iface_up='enp0s3')
 
-        self.system_volume = self.node.add_volume(name='system')
+        self.system_volume = self.node.add_volume(name='system', capacity=10)
-        self.iso_volume = self.node.add_volume(name='iso')
+        self.iso_volume = self.node.add_volume(name='iso', capacity=5)
 
         self.adm_iface = self.node.add_interface(
             label='enp0s3',

devops/tests/driver/libvirt/test_volume_backing_store.py

@@ -0,0 +1,104 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import collections
+
+import mock
+
+from devops.models import Environment
+from devops.tests.driver.libvirt.base import LibvirtTestCase
+
+
+class TestLibvirtVolumeBackingStore(LibvirtTestCase):
+
+    def setUp(self):
+        super(TestLibvirtVolumeBackingStore, self).setUp()
+
+        self.sleep_mock = self.patch('devops.helpers.retry.sleep')
+
+        self.open_mock = mock.mock_open(read_data='image_data')
+        self.patch('devops.driver.libvirt.libvirt_driver.open',
+                   self.open_mock, create=True)
+
+        self.os_mock = self.patch('devops.helpers.helpers.os')
+        Size = collections.namedtuple('Size', ['st_size'])
+        self.file_sizes = {
+            '/tmp/admin.iso': Size(st_size=500),
+        }
+        self.os_mock.stat.side_effect = self.file_sizes.get
+
+        self.env = Environment.create('test_env')
+        self.group1 = self.env.add_group(
+            group_name='test_group1',
+            driver_name='devops.driver.libvirt',
+            connection_string='test:///default',
+            storage_pool_name='default-pool')
+        self.group2 = self.env.add_group(
+            group_name='test_group2',
+            driver_name='devops.driver.libvirt',
+            connection_string='test:///default',
+            storage_pool_name='default-pool')
+
+        self.node1 = self.group1.add_node(
+            name='test_node1',
+            role='default',
+            architecture='i686',
+            hypervisor='test',
+        )
+
+        self.node2 = self.group1.add_node(
+            name='test_node2',
+            role='default',
+            architecture='i686',
+            hypervisor='test',
+        )
+
+    def test_backing_store(self):
+        parent_vol1 = self.group1.add_volume(
+            name='parent_volume',
+            format='qcow2',
+            capacity=10,
+            source_image='/tmp/admin.iso',
+        )
+        parent_vol1.define()
+
+        parent_vol2 = self.group2.add_volume(
+            name='parent_volume',
+            format='qcow2',
+            capacity=10,
+            source_image='/tmp/admin.iso',
+        )
+        parent_vol2.define()
+
+        child_volume1 = self.node1.add_volume(
+            name='test_volume1',
+            backing_store='parent_volume',
+            capacity=20,
+        )
+        child_volume1.define()
+
+        assert child_volume1.capacity == 20
+        assert child_volume1.backing_store is not None
+        assert child_volume1.backing_store.pk == parent_vol1.pk
+
+        child_volume2 = self.node2.add_volume(
+            name='test_volume2',
+            backing_store='parent_volume',
+            capacity=20,
+        )
+        child_volume2.define()
+
+        assert child_volume2.capacity == 20
+        assert child_volume2.backing_store is not None
+        assert child_volume2.backing_store.pk == parent_vol1.pk

devops/tests/models/node_ext/test_centos_master.py

@@ -65,8 +65,8 @@ class TestCentosMasterExt(LibvirtTestCase):
             architecture='x86_64',
             hypervisor='test')
 
-        self.system_volume = self.node.add_volume(name='system')
+        self.system_volume = self.node.add_volume(name='system', capacity=10)
-        self.iso_volume = self.node.add_volume(name='iso')
+        self.iso_volume = self.node.add_volume(name='iso', capacity=5)
 
         self.adm_iface = self.node.add_interface(
             label='enp0s3',