From b24847454266f197201b981511d8bb66a7f59c46 Mon Sep 17 00:00:00 2001 From: Dennis Dmitriev Date: Thu, 21 Jul 2016 12:05:15 +0000 Subject: [PATCH] Revert "Revert "Add support of backing store volumes from templates"" Migration works correctly, DB should be prepared. This reverts commit 466f235371a6d031767ca8f90fc4b4bf7fda4339. Change-Id: Ida4e40c4cb2e7f954861465c100b46cf8196c2e5 --- devops/driver/libvirt/libvirt_driver.py | 77 +++++++++---- devops/migrations/0001_initial.py | 3 +- devops/models/base.py | 10 +- devops/models/environment.py | 7 ++ devops/models/group.py | 31 ++++++ devops/models/node.py | 6 + devops/models/volume.py | 5 +- .../driver/libvirt/test_node_cloudimage.py | 4 +- .../libvirt/test_volume_backing_store.py | 104 ++++++++++++++++++ .../models/node_ext/test_centos_master.py | 4 +- 10 files changed, 222 insertions(+), 29 deletions(-) create mode 100644 devops/tests/driver/libvirt/test_volume_backing_store.py diff --git a/devops/driver/libvirt/libvirt_driver.py b/devops/driver/libvirt/libvirt_driver.py index aa12c74b..62e5b5e4 100644 --- a/devops/driver/libvirt/libvirt_driver.py +++ b/devops/driver/libvirt/libvirt_driver.py @@ -676,7 +676,7 @@ class LibvirtVolume(Volume): """Note: This class is imported as Volume at .__init__.py """ uuid = ParamField() - capacity = ParamField(default=None) + capacity = ParamField(default=None) # in gigabytes format = ParamField(default='qcow2', choices=('qcow2', 'raw')) source_image = ParamField(default=None) serial = ParamField() @@ -695,24 +695,49 @@ class LibvirtVolume(Volume): @retry(libvirt.libvirtError) def define(self): - name = underscored( - deepgetattr(self, 'node.group.environment.name'), - deepgetattr(self, 'node.name'), - self.name, - ) + # Generate libvirt volume name + if self.node: + name = underscored( + deepgetattr(self, 'node.group.environment.name'), + deepgetattr(self, 'node.name'), + self.name, + ) + elif self.group: + name = underscored( + deepgetattr(self, 'group.environment.name'), + deepgetattr(self, 'group.name'), + self.name, + ) + else: + raise DevopsError("Can't craete volume that is not " + "associated with any node or group") + # Find backing store format and path backing_store_path = None backing_store_format = None - if self.backing_store is not None: + if self.backing_store: + if not self.backing_store.exists(): + raise DevopsError( + "Can't create volume {!r}. backing_store volume {!r} does " + "not exists.".format(self.name, self.backing_store.name)) backing_store_path = self.backing_store.get_path() backing_store_format = self.backing_store.format - capacity = int((self.capacity or 0) * 1024 ** 3) - if self.source_image is not None: - file_size = get_file_size(self.source_image) - if file_size > capacity: - capacity = file_size + # Select capacity + if self.capacity: + # if capacity specified, use it first + capacity = int(self.capacity * 1024 ** 3) + elif self.source_image is not None: + # limit capacity to the sorse image file size + capacity = get_file_size(self.source_image) + elif self.backing_store: + # limit capacity to backing_store capacity + capacity = self.backing_store.get_capacity() + else: + raise DevopsError("Can't create volume {!r}: no capacity or " + "source_image specified".format(self.name)) + # Generate xml pool_name = self.driver.storage_pool_name pool = self.driver.conn.storagePoolLookupByName(pool_name) xml = LibvirtXMLBuilder.build_volume_xml( @@ -722,12 +747,19 @@ class LibvirtVolume(Volume): backing_store_path=backing_store_path, backing_store_format=backing_store_format, ) + + # Define volume libvirt_volume = pool.createXML(xml, 0) + + # Save uuid self.uuid = libvirt_volume.key() + + # Set serial and wwn if not self.serial: self.serial = uuid.uuid4().hex if not self.wwn: self.wwn = '0' + ''.join(uuid.uuid4().hex)[:15] + super(LibvirtVolume, self).define() # Upload predefined image to the volume @@ -742,7 +774,7 @@ class LibvirtVolume(Volume): super(LibvirtVolume, self).remove() def get_capacity(self): - """Get volume capacity""" + """Get volume capacity in bytes""" return self._libvirt_volume.info()[1] def get_format(self): @@ -753,8 +785,9 @@ class LibvirtVolume(Volume): return self._libvirt_volume.path() def fill_from_exist(self): - self.capacity = self.get_capacity() - self.format = self.get_format() + msg = 'LibvirtVolume.fill_from_exist() is deprecated and do nothing' + warn(msg, DeprecationWarning) + logger.debug(msg) @retry(libvirt.libvirtError, count=2) def upload(self, path, capacity=0): @@ -812,25 +845,31 @@ class LibvirtVolume(Volume): cls = self.driver.get_model_class('Volume') return cls.objects.create( name=name, - capacity=self.capacity, node=self.node, format=self.format, backing_store=self, ) - # TO REWRITE, LEGACY, for fuel-qa compatibility - # Used for EXTERNAL SNAPSHOTS + # LEGACY, for fuel-qa compatibility @classmethod def volume_get_predefined(cls, uuid): """Get predefined volume :rtype : Volume """ + msg = ('LibvirtVolume.volume_get_predefined() is deprecated. ' + 'Please use Volumes associated with Groups') + warn(msg, DeprecationWarning) + logger.debug(msg) + try: volume = cls.objects.get(uuid=uuid) except cls.DoesNotExist: volume = cls(uuid=uuid) - volume.fill_from_exist() + if not volume.exists(): + raise DevopsError( + 'Predefined volume {!r} not found'.format(uuid)) + volume.format = volume.get_format() volume.save() return volume diff --git a/devops/migrations/0001_initial.py b/devops/migrations/0001_initial.py index be1c6b3e..62c7a35f 100644 --- a/devops/migrations/0001_initial.py +++ b/devops/migrations/0001_initial.py @@ -159,6 +159,7 @@ class Migration(migrations.Migration): ('params', jsonfield.fields.JSONField(default={})), ('name', models.CharField(max_length=255)), ('backing_store', models.ForeignKey(to='devops.Volume', null=True)), + ('group', models.ForeignKey(to='devops.Group', null=True)), ('node', models.ForeignKey(to='devops.Node', null=True)), ], options={ @@ -202,7 +203,7 @@ class Migration(migrations.Migration): ), migrations.AlterUniqueTogether( name='volume', - unique_together=set([('name', 'node')]), + unique_together=set([('name', 'group'), ('name', 'node')]), ), migrations.AlterUniqueTogether( name='node', diff --git a/devops/models/base.py b/devops/models/base.py index 2b998552..62f6072a 100644 --- a/devops/models/base.py +++ b/devops/models/base.py @@ -158,7 +158,7 @@ class ParamField(ParamFieldBase): * to set default value. * to limit values using a list of allowed values. - Examples of ussage:: + Examples of usage:: class A(ParamedModel): foo = ParamField(default=10) @@ -229,9 +229,9 @@ class ParamMultiField(ParamFieldBase): self.subfields = [] for name, field in subfields.items(): - if not isinstance(field, (ParamField, ParamMultiField)): + if not isinstance(field, ParamFieldBase): raise DevopsError('field "{}" has wrong type;' - ' should be ParamField or ParamMultiField' + ' should be ParamFieldBase subclass instance' ''.format(name)) field.set_param_key(name) self.subfields.append(field) @@ -322,6 +322,10 @@ class ParamedModelQuerySet(query.QuerySet): # NOTE(astudenov): no support for 'gt', 'lt', 'in' # and other django's filter stuff + if not isinstance(item, self.model): + # skip other classes + continue + item_val = deepgetattr(item, key, splitter='__', do_raise=True) if item_val != value: diff --git a/devops/models/environment.py b/devops/models/environment.py index cb79b555..85e0b50b 100644 --- a/devops/models/environment.py +++ b/devops/models/environment.py @@ -186,6 +186,8 @@ class Environment(BaseModel): def define(self): for group in self.get_groups(): group.define_networks() + for group in self.get_groups(): + group.define_volumes() for group in self.get_groups(): group.define_nodes() @@ -344,6 +346,11 @@ class Environment(BaseModel): # Connect nodes to already created networks for group_data in groups: group = environment.get_group(name=group_data['name']) + + # add group volumes + group.add_volumes( + group_data.get('group_volumes', [])) + # add nodes group.add_nodes( group_data.get('nodes', [])) diff --git a/devops/models/group.py b/devops/models/group.py index 069d70c6..a208fd3e 100644 --- a/devops/models/group.py +++ b/devops/models/group.py @@ -22,6 +22,7 @@ from devops.models.base import BaseModel from devops.models.network import L2NetworkDevice from devops.models.network import NetworkPool from devops.models.node import Node +from devops.models.node import Volume class Group(BaseModel): @@ -87,6 +88,10 @@ class Group(BaseModel): def has_snapshot(self, name): return all(n.has_snapshot(name) for n in self.get_nodes()) + def define_volumes(self): + for volume in self.get_volumes(): + volume.define() + def define_networks(self): for l2_network_device in self.get_l2_network_devices(): l2_network_device.define() @@ -113,6 +118,9 @@ class Group(BaseModel): for node in self.get_nodes(): node.erase() + for volume in self.get_volumes(): + volume.erase() + for l2_network_device in self.get_l2_network_devices(): l2_network_device.erase() self.delete() @@ -214,3 +222,26 @@ class Group(BaseModel): name=name, address_pool=address_pool, ) + + def add_volumes(self, volumes): + for vol_params in volumes: + self.add_volume( + **vol_params + ) + + def add_volume(self, name, **params): + cls = self.driver.get_model_class('Volume') + return cls.objects.create( + group=self, + name=name, + **params + ) + + def get_volume(self, **kwargs): + try: + return self.volume_set.get(**kwargs) + except Volume.DoesNotExist: + raise DevopsObjNotFound(Volume, **kwargs) + + def get_volumes(self, **kwargs): + return self.volume_set.filter(**kwargs) diff --git a/devops/models/node.py b/devops/models/node.py index cae23358..ede85f24 100644 --- a/devops/models/node.py +++ b/devops/models/node.py @@ -373,6 +373,12 @@ class Node(six.with_metaclass(ExtendableNodeType, ParamedModel, BaseModel)): # NEW def add_volume(self, name, device='disk', bus='virtio', **params): cls = self.driver.get_model_class('Volume') + + if 'backing_store' in params: + # Backing storage volume have to be defined in group + params['backing_store'] = self.group.get_volume( + name=params['backing_store']) + volume = cls.objects.create( node=self, name=name, diff --git a/devops/models/volume.py b/devops/models/volume.py index cf4073fb..7aae70f6 100644 --- a/devops/models/volume.py +++ b/devops/models/volume.py @@ -21,17 +21,18 @@ from devops.models.base import ParamField class Volume(ParamedModel, BaseModel): class Meta(object): - unique_together = ('name', 'node') + unique_together = (('name', 'node'), ('name', 'group')) db_table = 'devops_volume' app_label = 'devops' backing_store = models.ForeignKey('self', null=True) name = models.CharField(max_length=255, unique=False, null=False) node = models.ForeignKey('Node', null=True) + group = models.ForeignKey('Group', null=True) @property def driver(self): - return self.node.driver + return self.node.driver if self.node else self.group.driver def define(self, *args, **kwargs): self.save() diff --git a/devops/tests/driver/libvirt/test_node_cloudimage.py b/devops/tests/driver/libvirt/test_node_cloudimage.py index 9c9d39fe..9095a2c3 100644 --- a/devops/tests/driver/libvirt/test_node_cloudimage.py +++ b/devops/tests/driver/libvirt/test_node_cloudimage.py @@ -76,8 +76,8 @@ class TestCloudImage(LibvirtTestCase): cloud_init_volume_name='iso', cloud_init_iface_up='enp0s3') - self.system_volume = self.node.add_volume(name='system') - self.iso_volume = self.node.add_volume(name='iso') + self.system_volume = self.node.add_volume(name='system', capacity=10) + self.iso_volume = self.node.add_volume(name='iso', capacity=5) self.adm_iface = self.node.add_interface( label='enp0s3', diff --git a/devops/tests/driver/libvirt/test_volume_backing_store.py b/devops/tests/driver/libvirt/test_volume_backing_store.py new file mode 100644 index 00000000..bbca2d30 --- /dev/null +++ b/devops/tests/driver/libvirt/test_volume_backing_store.py @@ -0,0 +1,104 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections + +import mock + +from devops.models import Environment +from devops.tests.driver.libvirt.base import LibvirtTestCase + + +class TestLibvirtVolumeBackingStore(LibvirtTestCase): + + def setUp(self): + super(TestLibvirtVolumeBackingStore, self).setUp() + + self.sleep_mock = self.patch('devops.helpers.retry.sleep') + + self.open_mock = mock.mock_open(read_data='image_data') + self.patch('devops.driver.libvirt.libvirt_driver.open', + self.open_mock, create=True) + + self.os_mock = self.patch('devops.helpers.helpers.os') + Size = collections.namedtuple('Size', ['st_size']) + self.file_sizes = { + '/tmp/admin.iso': Size(st_size=500), + } + self.os_mock.stat.side_effect = self.file_sizes.get + + self.env = Environment.create('test_env') + self.group1 = self.env.add_group( + group_name='test_group1', + driver_name='devops.driver.libvirt', + connection_string='test:///default', + storage_pool_name='default-pool') + self.group2 = self.env.add_group( + group_name='test_group2', + driver_name='devops.driver.libvirt', + connection_string='test:///default', + storage_pool_name='default-pool') + + self.node1 = self.group1.add_node( + name='test_node1', + role='default', + architecture='i686', + hypervisor='test', + ) + + self.node2 = self.group1.add_node( + name='test_node2', + role='default', + architecture='i686', + hypervisor='test', + ) + + def test_backing_store(self): + parent_vol1 = self.group1.add_volume( + name='parent_volume', + format='qcow2', + capacity=10, + source_image='/tmp/admin.iso', + ) + parent_vol1.define() + + parent_vol2 = self.group2.add_volume( + name='parent_volume', + format='qcow2', + capacity=10, + source_image='/tmp/admin.iso', + ) + parent_vol2.define() + + child_volume1 = self.node1.add_volume( + name='test_volume1', + backing_store='parent_volume', + capacity=20, + ) + child_volume1.define() + + assert child_volume1.capacity == 20 + assert child_volume1.backing_store is not None + assert child_volume1.backing_store.pk == parent_vol1.pk + + child_volume2 = self.node2.add_volume( + name='test_volume2', + backing_store='parent_volume', + capacity=20, + ) + child_volume2.define() + + assert child_volume2.capacity == 20 + assert child_volume2.backing_store is not None + assert child_volume2.backing_store.pk == parent_vol1.pk diff --git a/devops/tests/models/node_ext/test_centos_master.py b/devops/tests/models/node_ext/test_centos_master.py index 1b38f6a6..7b14ae41 100644 --- a/devops/tests/models/node_ext/test_centos_master.py +++ b/devops/tests/models/node_ext/test_centos_master.py @@ -65,8 +65,8 @@ class TestCentosMasterExt(LibvirtTestCase): architecture='x86_64', hypervisor='test') - self.system_volume = self.node.add_volume(name='system') - self.iso_volume = self.node.add_volume(name='iso') + self.system_volume = self.node.add_volume(name='system', capacity=10) + self.iso_volume = self.node.add_volume(name='iso', capacity=5) self.adm_iface = self.node.add_interface( label='enp0s3',