Parser for new object model

Parser that processes deployement config and produces "storage claim".

Change-Id: I8a250f911144776f2cf7ae9b561095cbcd067637
This commit is contained in:
Dmitry Bogun 2016-12-26 16:56:06 +02:00 committed by Andrii Ostapenko
parent 36f518cc3b
commit 1786de8c21
3 changed files with 441 additions and 14 deletions

View File

@ -13,8 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import defaultdict
import collections
import fnmatch
import itertools
import json
import math
import os
@ -24,6 +26,7 @@ from oslo_log import log as logging
from bareon.drivers.data.generic import GenericDataDriver
from bareon import errors
from bareon import objects
from bareon.utils import block_device
from bareon.utils import hardware as hu
from bareon.utils import partition as pu
from bareon.utils import utils
@ -46,8 +49,13 @@ class Ironic(GenericDataDriver):
def __init__(self, data):
super(Ironic, self).__init__(data)
self._original_data = data
convert_size(self.data['partitions'])
@property
def storage_claim(self):
return StorageParser(self._original_data, self.image_scheme).claim
def _get_image_meta(self):
pass
@ -640,7 +648,7 @@ def _set_vg_sizes(vgs, disks):
for disk in disks:
pvs += [vol for vol in disk['volumes'] if vol['type'] == 'pv']
vg_sizes = defaultdict(int)
vg_sizes = collections.defaultdict(int)
for pv in pvs:
vg_sizes[pv['vg']] += pv['size'] - pv.get(
'lvm_meta_size', DEFAULT_LVM_META_SIZE)
@ -699,10 +707,11 @@ def _resolve_sizes(spaces, retain_space_size=True):
claimed_space, unsized_volume = _process_space_claims(space)
taken_space += claimed_space
delta = space_size - taken_space
delta_MiB = utils.B2MiB(abs(delta))
if delta < 0:
raise ValueError('Sum of requested filesystem sizes exceeds space '
'available on {type} "{id}" by {delta} '
'MiB'.format(delta=abs(delta), type=space['type'],
'MiB'.format(delta=delta_MiB, type=space['type'],
id=_get_disk_id(space)))
elif unsized_volume:
ref = (unsized_volume['mount'] if unsized_volume.get(
@ -710,7 +719,7 @@ def _resolve_sizes(spaces, retain_space_size=True):
if delta:
LOG.info('Claiming remaining {delta} MiB for {ref} '
'volume/partition on {type} {id}.'
''.format(delta=abs(delta),
''.format(delta=delta_MiB,
type=space['type'],
id=_get_disk_id(space),
ref=ref))
@ -724,7 +733,7 @@ def _resolve_sizes(spaces, retain_space_size=True):
ref=ref))
else:
LOG.info('{delta} MiB of unclaimed space remains on {type} "{id}" '
'after completing allocations.'.format(delta=abs(delta),
'after completing allocations.'.format(delta=delta_MiB,
type=space['type'],
id=_get_disk_id(
space)))
@ -749,3 +758,264 @@ def convert_string_sizes(data, target=None):
else:
data[k] = convert_string_sizes(v, target=target)
return data
class StorageParser(object):
def __init__(self, data, image_schema):
self.storage = objects.block_device.StorageSubsystem()
self.disk_finder = block_device.DeviceFinder()
operation_systems = self._collect_operation_systems(image_schema)
self._existing_os_binding = set(operation_systems)
self._default_os_binding = operation_systems[:1]
self.mdfs_by_mount = {}
self.mddev_by_mount = collections.defaultdict(list)
self.lvm_pv_reference = collections.defaultdict(list)
LOG.debug('--- Preparing partition scheme ---')
LOG.debug('Looping over all disks in provision data')
try:
self.claim = self._parse(data)
except KeyError:
raise errors.DataSchemaCorruptError()
self._assemble_lvm_vg()
self._assemble_mdraid()
self._validate()
def _collect_operation_systems(self, image_schema):
return [image.os_id for image in image_schema.images]
def _parse(self, data):
for raw in data['partitions']:
kind = raw['type']
if kind == 'disk':
item = self._parse_disk(raw)
elif kind == 'vg':
item = self._parse_lvm_vg(raw)
else:
raise errors.DataSchemaCorruptError(exc_info=False)
self.storage.add(item)
def _assemble_lvm_vg(self):
vg_by_id = {
i.idnr: i for i in self.storage.items_by_kind(
objects.block_device.LVMvg)}
defined_vg = set(vg_by_id)
referred_vg = set(self.lvm_pv_reference)
empty_vg = defined_vg - referred_vg
orphan_pv = referred_vg - defined_vg
if empty_vg:
raise errors.WrongInputDataError(
'Following LVMvg have no any PV: "{}"'.format(
'", "'.join(sorted(empty_vg))))
if orphan_pv:
raise errors.WrongInputDataError(
'Following LVMpv refer to missing VG: "{}"'.format(
'", "'.join(sorted(orphan_pv))))
for idnr in defined_vg:
vg = vg_by_id[idnr]
for pv in self.lvm_pv_reference[idnr]:
vg.add(pv)
def _assemble_mdraid(self):
name_template = '/dev/md{:d}'
idx = itertools.count()
for mount in sorted(self.mddev_by_mount):
components = self.mddev_by_mount[mount]
fields = self.mdfs_by_mount[mount]
try:
name = fields.pop('name')
if not name.startswith('/dev/'):
name = '/dev/{}'.format(name)
except KeyError:
name = name_template.format(next(idx))
md = objects.block_device.MDRaid(name, **fields)
for item in components:
md.add(item)
self.storage.add(md)
def _validate(self):
for item in self.storage.items:
if isinstance(item, objects.block_device.Disk):
self._validate_disk(item)
elif isinstance(item, objects.block_device.LVMvg):
self._validate_lvm_vg(item)
def _validate_disk(self, disk):
remaining = []
for item in disk.items:
if item.size.kind != item.size.KIND_BIGGEST:
continue
remaining.append(item)
if len(remaining) < 2:
return
raise errors.WrongInputDataError(
'Multiple requests on "remaining" space.\n'
'disk:\n{}\npartitions:\n{}'.format(disk.idnr, '\n'.join(
repr(x) for x in remaining)))
def _validate_lvm_vg(self, vg):
remaining = []
for item in vg.items_by_kind(objects.block_device.LVMlv):
if item.size.kind != item.size.KIND_BIGGEST:
continue
remaining.append(item)
if len(remaining) < 2:
return
raise errors.WrongInputDataError(
'Multiple requests on "remaining" space.\n'
'lvm-vg: {}\nlogical volumes:\n{}'.format(vg.idnr, '\n'.join(
repr(x) for x in remaining)))
def _parse_disk(self, data):
size = self._size(data['size'])
idnr = self._disk_id(data['id'])
disk = objects.block_device.Disk(
idnr, size, **self._get_fields(data, 'name'))
for raw in data['volumes']:
kind = raw['type']
if kind == 'pv':
item = self._parse_lvm_pv(raw)
elif kind == 'raid':
item = self._parse_mdraid_dev(raw)
elif kind == 'partition':
item = self._parse_disk_partition(raw)
elif kind == 'boot':
item = self._parse_disk_partition(raw)
item.is_boot = True
# FIXME(dbogun): unsupported but allowed by data-schema type
elif kind == 'lvm_meta_pool':
item = None
else:
raise errors.DataSchemaCorruptError(exc_info=False)
if item is not None:
disk.add(item)
return disk
def _parse_lvm_vg(self, data):
vg = objects.block_device.LVMvg(
data['id'], **self._get_fields(
data, 'label', 'min_size', 'keep_data', '_allocate_size'))
for raw in data['volumes']:
item = self._parse_lvm_lv(raw)
vg.add(item)
return vg
def _parse_lvm_pv(self, data):
size = self._size(data['size'])
fields = self._get_fields(data, 'keep_data', 'lvm_meta_size')
if 'lvm_meta_size' in fields:
fields['lvm_meta_size'] = self._size(fields['lvm_meta_size'])
pv = objects.block_device.LVMpv(data['vg'], size, **fields)
self.lvm_pv_reference[pv.vg_idnr].append(pv)
return pv
def _parse_mdraid_dev(self, data):
fields = self._get_filesystem_fields(data, 'name')
size = fields.pop('size')
mddev = objects.block_device.MDDev(size)
mount = fields['mount']
self.mdfs_by_mount.setdefault(mount, fields)
self.mddev_by_mount[mount].append(mddev)
return mddev
def _parse_disk_partition(self, data):
fields = self._get_filesystem_fields(data, 'disk_label')
self._rename_fields(fields, {'disk_label': 'label'})
if fields.get('file_system') == 'swap':
fields['guid_code'] = 0x8200
size = fields.pop('size')
return objects.block_device.Partition(size, **fields)
def _parse_lvm_lv(self, data):
fields = self._get_filesystem_fields(data)
size = fields.pop('size')
return objects.block_device.LVMlv(data['name'], size, **fields)
def _get_filesystem_fields(self, data, *extra):
fields = self._get_fields(
data,
'mount', 'keep_data', 'file_system', 'size', 'images',
'fstab_options', 'fstab_enabled', *extra)
self._rename_fields(fields, {
'fstab_enabled': 'fstab_member',
'fstab_options': 'mount_options',
'images': 'os_binding'})
fields.setdefault('os_binding', self._default_os_binding)
fields['size'] = self._size(fields['size'])
if 'mount' in fields:
if fields['mount'].lower() == 'none':
fields.pop('mount')
if 'file_system' in fields:
fields['file_system'] = fields['file_system'].lower()
fields['os_binding'] = set(fields['os_binding'])
missing = fields['os_binding'] - self._existing_os_binding
if missing:
# FIXME(dbogun): it must be treated as error
LOG.warn(
'Try to claim not existing operating systems: '
'"{}"\n\n{}'.format(
'", "'.join(sorted(missing)),
json.dumps(data, indent=2)))
fields['os_binding'] -= missing
return fields
@staticmethod
def _get_fields(data, *fields):
result = {}
for f in fields:
try:
result[f] = data[f]
except KeyError:
pass
return result
@staticmethod
def _rename_fields(data, mapping):
for src in mapping:
try:
value = data.pop(src)
except KeyError:
continue
data[mapping[src]] = value
@staticmethod
def _size(size):
if size == 'remaining':
result = block_device.SpaceClaim.new_biggest()
else:
result = block_device.SizeUnit.new_by_string(
size, default_unit='MiB')
result = block_device.SpaceClaim.new_by_sizeunit(result)
return result
def _disk_id(self, idnr):
idnr = objects.block_device.DevIdnr(idnr['type'], idnr['value'])
idnr(self.disk_finder)
return idnr

View File

@ -13,6 +13,7 @@
# limitations under the License.
import sys
import traceback
class BaseError(Exception):
@ -22,14 +23,24 @@ class BaseError(Exception):
class InternalError(BaseError):
exc_info = None
def __init__(self, message=None, exc_info=True):
if message is None:
message = 'Internall error'
super(InternalError, self).__init__(message)
if exc_info:
self.exc_info = sys.exc_info()
message += '\nOriginal exception {}'.format(
''.join(traceback.format_exception(*self.exc_info)))
super(InternalError, self).__init__(message)
class DataSchemaCorruptError(InternalError):
def __init__(self, message=None, **kwargs):
if message is None:
message = (
'Integrity error in data processed by data validator. This '
'mean an error in data validation scheme or in parsing code.')
super(DataSchemaCorruptError, self).__init__(message, **kwargs)
class ApplicationDataCorruptError(BaseError):

View File

@ -13,11 +13,15 @@
# limitations under the License.
import copy
import difflib
import mock
import unittest2
from bareon.drivers.data import ironic
from bareon import errors
from bareon import objects
from bareon.utils import block_device
SAMPLE_CHUNK_PARTITIONS = [
{
@ -49,7 +53,7 @@ SAMPLE_CHUNK_PARTITIONS = [
"vg": "os"
},
{
"size": "45597",
"size": "45573",
"type": "pv",
"lvm_meta_size": "64",
"vg": "image"
@ -87,7 +91,7 @@ SAMPLE_CHUNK_PARTITIONS = [
"vg": "os"
},
{
"size": "64971",
"size": "64947",
"type": "pv",
"lvm_meta_size": "64",
"vg": "image"
@ -121,7 +125,7 @@ SAMPLE_CHUNK_PARTITIONS = [
"vg": "os"
},
{
"size": "64971",
"size": "64947",
"type": "pv",
"lvm_meta_size": "64",
"vg": "image"
@ -164,7 +168,7 @@ SAMPLE_CHUNK_PARTITIONS = [
"volumes": [
{
"mount": "/var/lib/glance",
"size": "175347",
"size": "175275",
"type": "lv",
"name": "glance",
"file_system": "xfs"
@ -175,7 +179,7 @@ SAMPLE_CHUNK_PARTITIONS = [
}
]
SAMPLE_PAYLOAD = {
PAYLOAD_SAMPLE0 = {
'partitions': SAMPLE_CHUNK_PARTITIONS,
'images': []
}
@ -184,7 +188,7 @@ SAMPLE_PAYLOAD = {
class TestIronicDataValidator(unittest2.TestCase):
def setUp(self):
super(TestIronicDataValidator, self).setUp()
self.payload = copy.deepcopy(SAMPLE_PAYLOAD)
self.payload = copy.deepcopy(PAYLOAD_SAMPLE0)
def test_no_error(self):
ironic.Ironic.validate_data(self.payload)
@ -280,3 +284,145 @@ class TestIronicDataValidator(unittest2.TestCase):
@staticmethod
def _get_disks(payload):
return payload['partitions']
class TestIronicDataModel(unittest2.TestCase):
def setUp(self):
super(TestIronicDataModel, self).setUp()
self.block_device_list = mock.Mock()
self.device_info = mock.Mock()
self.device_finder = mock.Mock()
for path, m in (
('bareon.utils.hardware.'
'get_block_data_from_udev', self.block_device_list),
('bareon.utils.hardware.'
'get_device_info', self.device_info)):
patch = mock.patch(path, m)
patch.start()
self.addCleanup(patch.stop)
def test_sample0(self):
self.block_device_list.side_effect = [
['/dev/sda', '/dev/sdb', '/dev/sdc'],
[]]
self.device_info.side_effect = [
{'uspec': {'DEVNAME': '/dev/sda', 'DEVLINKS': []}},
{'uspec': {'DEVNAME': '/dev/sdb', 'DEVLINKS': []}},
{'uspec': {
'DEVNAME': '/dev/sdc',
'DEVLINKS': ['/dev/disk/by-path/pci-0000:00:0d.0-'
'scsi-0:0:0:0']}}]
device_finder = block_device.DeviceFinder()
patch = mock.patch(
'bareon.utils.block_device.DeviceFinder', self.device_finder)
patch.start()
self.addCleanup(patch.stop)
self.device_finder.return_value = device_finder
expect_storage_claim = objects.block_device.StorageSubsystem()
idnr = objects.block_device.DevIdnr('name', 'sda')
idnr(device_finder)
sda = objects.block_device.Disk(
idnr, self._size(65535, 'MiB'), name='sda')
sda.add(objects.block_device.Partition(
self._size(24, 'MiB'), guid_code=0xEF02, is_service=True))
sda.add(objects.block_device.Partition(
self._size(300, 'MiB'), is_boot=True))
sda.add(objects.block_device.MDDev(
self._size(200, 'MiB')))
sda.add(objects.block_device.LVMpv(
'os', self._size(19438, 'MiB'),
lvm_meta_size=self._size(64, 'MiB')))
sda.add(objects.block_device.LVMpv(
'image', self._size(45573, 'MiB'),
lvm_meta_size=self._size(64, 'MiB')))
expect_storage_claim.add(sda)
idnr = objects.block_device.DevIdnr('name', 'sdb')
idnr(device_finder)
sdb = objects.block_device.Disk(
idnr, self._size(65535, 'MiB'), name='sdb')
sdb.add(objects.block_device.Partition(
self._size(24, 'MiB'), guid_code=0xEF02, is_service=True))
sdb.add(objects.block_device.Partition(
self._size(300, 'MiB'), is_boot=True))
sdb.add(objects.block_device.MDDev(
self._size(200, 'MiB')))
sdb.add(objects.block_device.LVMpv(
'os', self._size(0, 'MiB'), lvm_meta_size=self._size(0, 'MiB')))
sdb.add(objects.block_device.LVMpv(
'image', self._size(64947, 'MiB'),
lvm_meta_size=self._size(64, 'MiB')))
expect_storage_claim.add(sdb)
idnr = objects.block_device.DevIdnr(
'path', 'disk/by-path/pci-0000:00:0d.0-scsi-0:0:0:0')
idnr(device_finder)
sdc = objects.block_device.Disk(
idnr, self._size(65535, 'MiB'), name='sdc')
sdc.add(objects.block_device.Partition(
self._size(24, 'MiB'), guid_code=0xEF02, is_service=True))
sdc.add(objects.block_device.Partition(
self._size(300, 'MiB'), is_boot=True))
sdc.add(objects.block_device.MDDev(
self._size(200, 'MiB')))
sdc.add(objects.block_device.LVMpv(
'os', self._size(0, 'MiB'), lvm_meta_size=self._size(0, 'MiB')))
sdc.add(objects.block_device.LVMpv(
'image', self._size(64947, 'MiB'),
lvm_meta_size=self._size(64, 'MiB')))
expect_storage_claim.add(sdc)
lvm_os = objects.block_device.LVMvg(
'os', _allocate_size='min', label='Base System', min_size=19374)
lvm_os.add(objects.block_device.LVMlv(
'root', self._size(15360, 'MiB'), mount='/', file_system='ext4'))
lvm_os.add(objects.block_device.LVMlv(
'swap', self._size(4014, 'MiB'), mount='swap', file_system='swap'))
for component in expect_storage_claim.items_by_kind(
objects.block_device.LVMpv, recursion=True):
if component.vg_idnr != 'os':
continue
lvm_os.add(component)
expect_storage_claim.add(lvm_os)
lvm_image = objects.block_device.LVMvg(
'image',
_allocate_size='all',
label='Image Storage',
min_size=5120)
lvm_image.add(objects.block_device.LVMlv(
'glance', self._size(175275, 'MiB'),
mount='/var/lib/glance', file_system='xfs'))
for component in expect_storage_claim.items_by_kind(
objects.block_device.LVMpv, recursion=True):
if component.vg_idnr != 'image':
continue
lvm_image.add(component)
expect_storage_claim.add(lvm_image)
md_boot = objects.block_device.MDRaid(
'/dev/Boot', file_system='ext2', mount='/boot')
for component in expect_storage_claim.items_by_kind(
objects.block_device.MDDev, recursion=True):
md_boot.add(component)
expect_storage_claim.add(md_boot)
data_driver = ironic.Ironic(PAYLOAD_SAMPLE0)
if expect_storage_claim != data_driver.storage_claim:
diff = difflib.unified_diff(
repr(data_driver.storage_claim).splitlines(True),
repr(expect_storage_claim).splitlines(True),
'actual', 'expect')
raise AssertionError(
'Parsed storage claim is not match expected value:\n'
'{}'.format(''.join(diff)))
@staticmethod
def _size(value, unit):
value = block_device.SizeUnit(value, unit)
return block_device.SpaceClaim.new_by_sizeunit(value)