openstack/fuel-devops
Code Issues Proposed changes

Revert "Revert "Add support of backing store volumes from templates""

Browse Source 
Migration works correctly, DB should be prepared.

This reverts commit 466f235371.

Change-Id: Ida4e40c4cb2e7f954861465c100b46cf8196c2e5
This commit is contained in:
Dennis Dmitriev 2016-07-21 12:05:15 +00:00
parent 466f235371
commit b248474542
10 changed files with 222 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
devops/driver/libvirt/libvirt_driver.py
View File

@ -676,7 +676,7 @@ class LibvirtVolume(Volume):
"""Note: This class is imported as Volume at .__init__.py """
uuid = ParamField()
capacity = ParamField(default=None)
capacity = ParamField(default=None) # in gigabytes
format = ParamField(default='qcow2', choices=('qcow2', 'raw'))
source_image = ParamField(default=None)
serial = ParamField()
@ -695,24 +695,49 @@ class LibvirtVolume(Volume):
@retry(libvirt.libvirtError)
def define(self):
name = underscored(
deepgetattr(self, 'node.group.environment.name'),
deepgetattr(self, 'node.name'),
self.name,
)
# Generate libvirt volume name
if self.node:
name = underscored(
deepgetattr(self, 'node.group.environment.name'),
deepgetattr(self, 'node.name'),
self.name,
)
elif self.group:
name = underscored(
deepgetattr(self, 'group.environment.name'),
deepgetattr(self, 'group.name'),
self.name,
)
else:
raise DevopsError("Can't craete volume that is not "
"associated with any node or group")
# Find backing store format and path
backing_store_path = None
backing_store_format = None
if self.backing_store is not None:
if self.backing_store:
if not self.backing_store.exists():
raise DevopsError(
"Can't create volume {!r}. backing_store volume {!r} does "
"not exists.".format(self.name, self.backing_store.name))
backing_store_path = self.backing_store.get_path()
backing_store_format = self.backing_store.format
capacity = int((self.capacity or 0) * 1024 ** 3)
if self.source_image is not None:
file_size = get_file_size(self.source_image)
if file_size > capacity:
capacity = file_size
# Select capacity
if self.capacity:
# if capacity specified, use it first
capacity = int(self.capacity * 1024 ** 3)
elif self.source_image is not None:
# limit capacity to the sorse image file size
capacity = get_file_size(self.source_image)
elif self.backing_store:
# limit capacity to backing_store capacity
capacity = self.backing_store.get_capacity()
else:
raise DevopsError("Can't create volume {!r}: no capacity or "
"source_image specified".format(self.name))
# Generate xml
pool_name = self.driver.storage_pool_name
pool = self.driver.conn.storagePoolLookupByName(pool_name)
xml = LibvirtXMLBuilder.build_volume_xml(
@ -722,12 +747,19 @@ class LibvirtVolume(Volume):
backing_store_path=backing_store_path,
backing_store_format=backing_store_format,
)
# Define volume
libvirt_volume = pool.createXML(xml, 0)
# Save uuid
self.uuid = libvirt_volume.key()
# Set serial and wwn
if not self.serial:
self.serial = uuid.uuid4().hex
if not self.wwn:
self.wwn = '0' + ''.join(uuid.uuid4().hex)[:15]
super(LibvirtVolume, self).define()
# Upload predefined image to the volume
@ -742,7 +774,7 @@ class LibvirtVolume(Volume):
super(LibvirtVolume, self).remove()
def get_capacity(self):
"""Get volume capacity"""
"""Get volume capacity in bytes"""
return self._libvirt_volume.info()[1]
def get_format(self):
@ -753,8 +785,9 @@ class LibvirtVolume(Volume):
return self._libvirt_volume.path()
def fill_from_exist(self):
self.capacity = self.get_capacity()
self.format = self.get_format()
msg = 'LibvirtVolume.fill_from_exist() is deprecated and do nothing'
warn(msg, DeprecationWarning)
logger.debug(msg)
@retry(libvirt.libvirtError, count=2)
def upload(self, path, capacity=0):
@ -812,25 +845,31 @@ class LibvirtVolume(Volume):
cls = self.driver.get_model_class('Volume')
return cls.objects.create(
name=name,
capacity=self.capacity,
node=self.node,
format=self.format,
backing_store=self,
)
# TO REWRITE, LEGACY, for fuel-qa compatibility
# Used for EXTERNAL SNAPSHOTS
# LEGACY, for fuel-qa compatibility
@classmethod
def volume_get_predefined(cls, uuid):
"""Get predefined volume
:rtype : Volume
"""
msg = ('LibvirtVolume.volume_get_predefined() is deprecated. '
'Please use Volumes associated with Groups')
warn(msg, DeprecationWarning)
logger.debug(msg)
try:
volume = cls.objects.get(uuid=uuid)
except cls.DoesNotExist:
volume = cls(uuid=uuid)
volume.fill_from_exist()
if not volume.exists():
raise DevopsError(
'Predefined volume {!r} not found'.format(uuid))
volume.format = volume.get_format()
volume.save()
return volume

3
devops/migrations/0001_initial.py
View File

@ -159,6 +159,7 @@ class Migration(migrations.Migration):
('params', jsonfield.fields.JSONField(default={})),
('name', models.CharField(max_length=255)),
('backing_store', models.ForeignKey(to='devops.Volume', null=True)),
('group', models.ForeignKey(to='devops.Group', null=True)),
('node', models.ForeignKey(to='devops.Node', null=True)),
],
options={
@ -202,7 +203,7 @@ class Migration(migrations.Migration):
),
migrations.AlterUniqueTogether(
name='volume',
unique_together=set([('name', 'node')]),
unique_together=set([('name', 'group'), ('name', 'node')]),
),
migrations.AlterUniqueTogether(
name='node',

10
devops/models/base.py
View File

@ -158,7 +158,7 @@ class ParamField(ParamFieldBase):
* to set default value.
* to limit values using a list of allowed values.
Examples of ussage::
Examples of usage::
class A(ParamedModel):
foo = ParamField(default=10)
@ -229,9 +229,9 @@ class ParamMultiField(ParamFieldBase):
self.subfields = []
for name, field in subfields.items():
if not isinstance(field, (ParamField, ParamMultiField)):
if not isinstance(field, ParamFieldBase):
raise DevopsError('field "{}" has wrong type;'
' should be ParamField or ParamMultiField'
' should be ParamFieldBase subclass instance'
''.format(name))
field.set_param_key(name)
self.subfields.append(field)
@ -322,6 +322,10 @@ class ParamedModelQuerySet(query.QuerySet):
# NOTE(astudenov): no support for 'gt', 'lt', 'in'
# and other django's filter stuff
if not isinstance(item, self.model):
# skip other classes
continue
item_val = deepgetattr(item, key, splitter='__',
do_raise=True)
if item_val != value:

7
devops/models/environment.py
View File

@ -186,6 +186,8 @@ class Environment(BaseModel):
def define(self):
for group in self.get_groups():
group.define_networks()
for group in self.get_groups():
group.define_volumes()
for group in self.get_groups():
group.define_nodes()
@ -344,6 +346,11 @@ class Environment(BaseModel):
# Connect nodes to already created networks
for group_data in groups:
group = environment.get_group(name=group_data['name'])
# add group volumes
group.add_volumes(
group_data.get('group_volumes', []))
# add nodes
group.add_nodes(
group_data.get('nodes', []))

31
devops/models/group.py
View File

@ -22,6 +22,7 @@ from devops.models.base import BaseModel
from devops.models.network import L2NetworkDevice
from devops.models.network import NetworkPool
from devops.models.node import Node
from devops.models.node import Volume
class Group(BaseModel):
@ -87,6 +88,10 @@ class Group(BaseModel):
def has_snapshot(self, name):
return all(n.has_snapshot(name) for n in self.get_nodes())
def define_volumes(self):
for volume in self.get_volumes():
volume.define()
def define_networks(self):
for l2_network_device in self.get_l2_network_devices():
l2_network_device.define()
@ -113,6 +118,9 @@ class Group(BaseModel):
for node in self.get_nodes():
node.erase()
for volume in self.get_volumes():
volume.erase()
for l2_network_device in self.get_l2_network_devices():
l2_network_device.erase()
self.delete()
@ -214,3 +222,26 @@ class Group(BaseModel):
name=name,
address_pool=address_pool,
)
def add_volumes(self, volumes):
for vol_params in volumes:
self.add_volume(
**vol_params
)
def add_volume(self, name, **params):
cls = self.driver.get_model_class('Volume')
return cls.objects.create(
group=self,
name=name,
**params
)
def get_volume(self, **kwargs):
try:
return self.volume_set.get(**kwargs)
except Volume.DoesNotExist:
raise DevopsObjNotFound(Volume, **kwargs)
def get_volumes(self, **kwargs):
return self.volume_set.filter(**kwargs)

6
devops/models/node.py
View File

@ -373,6 +373,12 @@ class Node(six.with_metaclass(ExtendableNodeType, ParamedModel, BaseModel)):
# NEW
def add_volume(self, name, device='disk', bus='virtio', **params):
cls = self.driver.get_model_class('Volume')
if 'backing_store' in params:
# Backing storage volume have to be defined in group
params['backing_store'] = self.group.get_volume(
name=params['backing_store'])
volume = cls.objects.create(
node=self,
name=name,

5
devops/models/volume.py
View File

@ -21,17 +21,18 @@ from devops.models.base import ParamField
class Volume(ParamedModel, BaseModel):
class Meta(object):
unique_together = ('name', 'node')
unique_together = (('name', 'node'), ('name', 'group'))
db_table = 'devops_volume'
app_label = 'devops'
backing_store = models.ForeignKey('self', null=True)
name = models.CharField(max_length=255, unique=False, null=False)
node = models.ForeignKey('Node', null=True)
group = models.ForeignKey('Group', null=True)
@property
def driver(self):
return self.node.driver
return self.node.driver if self.node else self.group.driver
def define(self, *args, **kwargs):
self.save()

4
devops/tests/driver/libvirt/test_node_cloudimage.py
View File

@ -76,8 +76,8 @@ class TestCloudImage(LibvirtTestCase):
cloud_init_volume_name='iso',
cloud_init_iface_up='enp0s3')
self.system_volume = self.node.add_volume(name='system')
self.iso_volume = self.node.add_volume(name='iso')
self.system_volume = self.node.add_volume(name='system', capacity=10)
self.iso_volume = self.node.add_volume(name='iso', capacity=5)
self.adm_iface = self.node.add_interface(
label='enp0s3',

104
devops/tests/driver/libvirt/test_volume_backing_store.py Normal file
View File

@ -0,0 +1,104 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import mock
from devops.models import Environment
from devops.tests.driver.libvirt.base import LibvirtTestCase
class TestLibvirtVolumeBackingStore(LibvirtTestCase):
def setUp(self):
super(TestLibvirtVolumeBackingStore, self).setUp()
self.sleep_mock = self.patch('devops.helpers.retry.sleep')
self.open_mock = mock.mock_open(read_data='image_data')
self.patch('devops.driver.libvirt.libvirt_driver.open',
self.open_mock, create=True)
self.os_mock = self.patch('devops.helpers.helpers.os')
Size = collections.namedtuple('Size', ['st_size'])
self.file_sizes = {
'/tmp/admin.iso': Size(st_size=500),
}
self.os_mock.stat.side_effect = self.file_sizes.get
self.env = Environment.create('test_env')
self.group1 = self.env.add_group(
group_name='test_group1',
driver_name='devops.driver.libvirt',
connection_string='test:///default',
storage_pool_name='default-pool')
self.group2 = self.env.add_group(
group_name='test_group2',
driver_name='devops.driver.libvirt',
connection_string='test:///default',
storage_pool_name='default-pool')
self.node1 = self.group1.add_node(
name='test_node1',
role='default',
architecture='i686',
hypervisor='test',
)
self.node2 = self.group1.add_node(
name='test_node2',
role='default',
architecture='i686',
hypervisor='test',
)
def test_backing_store(self):
parent_vol1 = self.group1.add_volume(
name='parent_volume',
format='qcow2',
capacity=10,
source_image='/tmp/admin.iso',
)
parent_vol1.define()
parent_vol2 = self.group2.add_volume(
name='parent_volume',
format='qcow2',
capacity=10,
source_image='/tmp/admin.iso',
)
parent_vol2.define()
child_volume1 = self.node1.add_volume(
name='test_volume1',
backing_store='parent_volume',
capacity=20,
)
child_volume1.define()
assert child_volume1.capacity == 20
assert child_volume1.backing_store is not None
assert child_volume1.backing_store.pk == parent_vol1.pk
child_volume2 = self.node2.add_volume(
name='test_volume2',
backing_store='parent_volume',
capacity=20,
)
child_volume2.define()
assert child_volume2.capacity == 20
assert child_volume2.backing_store is not None
assert child_volume2.backing_store.pk == parent_vol1.pk

4
devops/tests/models/node_ext/test_centos_master.py
View File

@ -65,8 +65,8 @@ class TestCentosMasterExt(LibvirtTestCase):
architecture='x86_64',
hypervisor='test')
self.system_volume = self.node.add_volume(name='system')
self.iso_volume = self.node.add_volume(name='iso')
self.system_volume = self.node.add_volume(name='system', capacity=10)
self.iso_volume = self.node.add_volume(name='iso', capacity=5)
self.adm_iface = self.node.add_interface(
label='enp0s3',