cinder plugin

Change-Id: Ib01a5dd1f99faa9756b098ae1dacb4f71ea4764d
This commit is contained in:
Alexey Weyl
2016-03-24 12:23:38 +02:00
parent 6c679d8c5f
commit 08b21b1996
15 changed files with 554 additions and 5 deletions

View File

@@ -6,6 +6,7 @@ pbr>=1.6
Babel>=1.3
lxml>=2.3
python-ceilometerclient>=2.2.1 # Apache-2.0
python-cinderclient>=1.3.1 # Apache-2.0
python-dateutil>=2.4.2
python-novaclient>=2.26.0
networkx>=1.10

View File

@@ -9,6 +9,7 @@ discover
lxml>=2.3
networkx>=1.10
python-ceilometerclient>=2.2.1 # Apache-2.0
python-cinderclient>=1.3.1 # Apache-2.0
python-novaclient>=2.26.0
python-subunit>=0.0.18
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2

View File

@@ -16,6 +16,7 @@ from oslo_config import cfg
from oslo_log import log
from ceilometerclient import client as cm_client
from cinderclient import client as cin_client
from novaclient import client as n_client
@@ -25,6 +26,7 @@ LOG = log.getLogger(__name__)
OPTS = [
cfg.StrOpt('aodh_version', default='2', help='Aodh version'),
cfg.FloatOpt('nova_version', default='2.0', help='Nova version'),
cfg.StrOpt('cinder_version', default='1', help='Cinder version'),
]
@@ -58,3 +60,19 @@ def nova_client(conf):
return client
except Exception as e:
LOG.exception('Create Nova client - Got Exception: %s', e)
def cinder_client(conf):
"""Get an instance of cinder client"""
auth_config = conf.service_credentials
try:
client = cin_client.Client(
version=conf.cinder_version,
session=keystone_client.get_session(conf),
region_name=auth_config.region_name,
interface=auth_config.interface,
)
LOG.info('Cinder client created')
return client
except Exception as e:
LOG.exception('Create Cinder client - Got Exception: %s', e)

View File

@@ -44,6 +44,7 @@ class EdgeLabels(object):
ON = 'on'
CONTAINS = 'contains'
CAUSES = 'causes'
ATTACHED = 'attached'
class SyncMode(object):

View File

@@ -107,11 +107,11 @@ class EntityGraphApis(object):
ga = create_algorithm(self.entity_graph)
if graph_type == 'tree':
final_query = query if query else TOPOLOGY_QUERY
return ga.graph_query_vertices(
query_dict=final_query,
root_id=root)
else:
final_query = {}
return ga.graph_query_vertices(
query_dict=final_query,
root_id=root)
return self.entity_graph
@staticmethod
def _get_first(lst):

View File

@@ -15,6 +15,7 @@
from oslo_config import cfg
from vitrage.synchronizer.plugins.aodh import AODH_PLUGIN
from vitrage.synchronizer.plugins.cinder.volume import CINDER_VOLUME_PLUGIN
from vitrage.synchronizer.plugins.nagios import NAGIOS_PLUGIN
from vitrage.synchronizer.plugins.nova.host import NOVA_HOST_PLUGIN
from vitrage.synchronizer.plugins.nova.instance import NOVA_INSTANCE_PLUGIN
@@ -32,6 +33,7 @@ OPTS = [
NOVA_ZONE_PLUGIN,
NAGIOS_PLUGIN,
STATIC_PHYSICAL_PLUGIN,
AODH_PLUGIN],
AODH_PLUGIN,
CINDER_VOLUME_PLUGIN],
help='Names of supported plugins'),
]

View File

@@ -0,0 +1,15 @@
# Copyright 2016 - Alcatel-Lucent
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
__author__ = 'stack'

View File

@@ -0,0 +1,30 @@
# Copyright 2016 - Alcatel-Lucent
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
CINDER_VOLUME_PLUGIN = 'cinder.volume'
OPTS = [
cfg.StrOpt('transformer',
default='vitrage.synchronizer.plugins.cinder.volume.'
'transformer.CinderVolumeTransformer',
help='Nova host transformer class path',
required=True),
cfg.StrOpt('synchronizer',
default='vitrage.synchronizer.plugins.cinder.volume.'
'synchronizer.CinderVolumeSynchronizer',
help='Nova host synchronizer class path',
required=True),
]

View File

@@ -0,0 +1,38 @@
# Copyright 2016 - Alcatel-Lucent
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
from vitrage import clients
from vitrage.synchronizer.plugins.cinder.volume import CINDER_VOLUME_PLUGIN
from vitrage.synchronizer.plugins.synchronizer_base import SynchronizerBase
LOG = logging.getLogger(__name__)
class CinderVolumeSynchronizer(SynchronizerBase):
def __init__(self, conf):
self.client = clients.cinder_client(conf)
self.conf = conf
@staticmethod
def filter_instances(volumes):
return [volume.__dict__ for volume in volumes]
def get_all(self, sync_mode):
return self.make_pickleable(
self.filter_instances(self.client.volumes.list()),
CINDER_VOLUME_PLUGIN,
sync_mode)

View File

@@ -0,0 +1,187 @@
# Copyright 2016 - Alcatel-Lucent
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
from vitrage.common.constants import EdgeLabels
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import EventAction
from vitrage.common.constants import SynchronizerProperties as SyncProps
from vitrage.common.constants import SyncMode
from vitrage.common.constants import VertexProperties as VProps
from vitrage.common.exception import VitrageTransformerError
import vitrage.graph.utils as graph_utils
from vitrage.synchronizer.plugins.base.resource.transformer import \
BaseResourceTransformer
from vitrage.synchronizer.plugins.cinder.volume import CINDER_VOLUME_PLUGIN
from vitrage.synchronizer.plugins.nova.instance import NOVA_INSTANCE_PLUGIN
from vitrage.synchronizer.plugins import transformer_base
from vitrage.synchronizer.plugins.transformer_base import extract_field_value
LOG = logging.getLogger(__name__)
class CinderVolumeTransformer(BaseResourceTransformer):
# Fields returned from Nova Instance snapshot
VOLUME_ID = {
SyncMode.SNAPSHOT: ('id',),
SyncMode.INIT_SNAPSHOT: ('id',),
SyncMode.UPDATE: ('payload', 'instance_id')
}
VOLUME_STATE = {
SyncMode.SNAPSHOT: ('status',),
SyncMode.INIT_SNAPSHOT: ('status',),
SyncMode.UPDATE: ('payload', 'state')
}
VOLUME_UPDATE_TIMESTAMP = {
SyncMode.SNAPSHOT: ('created_at',),
SyncMode.INIT_SNAPSHOT: ('created_at',),
SyncMode.UPDATE: ('metadata', 'timestamp')
}
VOLUME_NAME = {
SyncMode.SNAPSHOT: ('display_name',),
SyncMode.INIT_SNAPSHOT: ('display_name',),
SyncMode.UPDATE: ('payload', 'hostname')
}
# Event types which need to refer them differently
EVENT_TYPES = {
'compute.instance.delete.end': EventAction.DELETE_ENTITY,
'compute.instance.create.start': EventAction.CREATE_ENTITY
}
def __init__(self, transformers):
self.transformers = transformers
def _create_entity_vertex(self, entity_event):
sync_mode = entity_event[SyncProps.SYNC_MODE]
metadata = {
VProps.NAME: extract_field_value(entity_event,
self.VOLUME_NAME[sync_mode])
}
entity_key = self._create_entity_key(entity_event)
entity_id = extract_field_value(
entity_event,
self.VOLUME_ID[sync_mode])
state = extract_field_value(
entity_event,
self.VOLUME_STATE[sync_mode])
update_timestamp = extract_field_value(
entity_event,
self.VOLUME_UPDATE_TIMESTAMP[sync_mode])
sample_timestamp = entity_event[SyncProps.SAMPLE_DATE]
update_timestamp = self._format_update_timestamp(update_timestamp,
sample_timestamp)
return graph_utils.create_vertex(
entity_key,
entity_id=entity_id,
entity_category=EntityCategory.RESOURCE,
entity_type=CINDER_VOLUME_PLUGIN,
entity_state=state,
sample_timestamp=sample_timestamp,
update_timestamp=update_timestamp,
metadata=metadata)
def _create_neighbors(self, entity_event):
return self._create_instance_neighbors(entity_event)
def _extract_action_type(self, entity_event):
sync_mode = entity_event[SyncProps.SYNC_MODE]
if SyncMode.INIT_SNAPSHOT == sync_mode:
return EventAction.CREATE_ENTITY
if SyncMode.SNAPSHOT == sync_mode:
return EventAction.UPDATE_ENTITY
if SyncMode.UPDATE == sync_mode:
return self.EVENT_TYPES.get(
entity_event[self.UPDATE_EVENT_TYPE],
EventAction.UPDATE_ENTITY)
raise VitrageTransformerError(
'Invalid sync mode: (%s)' % sync_mode)
def _create_entity_key(self, entity_event):
volume_id = extract_field_value(
entity_event,
self.VOLUME_ID[entity_event[SyncProps.SYNC_MODE]])
key_fields = self._key_values(CINDER_VOLUME_PLUGIN, volume_id)
return transformer_base.build_key(key_fields)
def create_placeholder_vertex(self, **kwargs):
if VProps.ID not in kwargs:
LOG.error('Cannot create placeholder vertex. Missing property ID')
raise ValueError('Missing property ID')
key_fields = self._key_values(CINDER_VOLUME_PLUGIN, kwargs[VProps.ID])
return graph_utils.create_vertex(
transformer_base.build_key(key_fields),
entity_id=kwargs[VProps.ID],
entity_category=EntityCategory.RESOURCE,
entity_type=CINDER_VOLUME_PLUGIN,
sample_timestamp=kwargs[VProps.SAMPLE_TIMESTAMP],
is_placeholder=True)
def _create_instance_neighbors(self, entity_event):
transformer = self.transformers[NOVA_INSTANCE_PLUGIN]
if transformer:
return [self._create_instance_neighbor(entity_event,
attachment,
transformer)
for attachment in entity_event['attachments']]
else:
LOG.warning('Cannot find instance transformer')
def _create_instance_neighbor(self,
entity_event,
attachment,
instance_transformer):
volume_vitrage_id = self._create_entity_key(entity_event)
instance_id = attachment['server_id']
sample_timestamp = entity_event[SyncProps.SAMPLE_DATE]
properties = {
VProps.ID: instance_id,
VProps.SAMPLE_TIMESTAMP: sample_timestamp
}
instance_vertex = \
instance_transformer.create_placeholder_vertex(
**properties)
relationship_edge = graph_utils.create_edge(
source_id=volume_vitrage_id,
target_id=instance_vertex.vertex_id,
relationship_type=EdgeLabels.ATTACHED)
return transformer_base.Neighbor(instance_vertex, relationship_edge)

View File

@@ -191,6 +191,39 @@ def simple_zone_generators(zone_num, host_num, snapshot_events=0,
return tg.get_trace_generators(test_entity_spec_list)
def simple_volume_generators(volume_num, instance_num, snapshot_events=0,
snap_vals=None):
"""A function for returning vm event generators.
Returns generators for a given number of volumes and
instances. Instances will be distributed across hosts in round-robin style.
:param volume_num: number of volumes
:param instance_num: number of instances
:param snapshot_events: number of snapshot events per host
:param snap_vals: preset vals for ALL snapshot events
:return: generators for volume_num volumes as specified
"""
mapping = [('volume-{0}'.format(index % volume_num),
'vm-{0}'.format(index))
for index in range(instance_num)
]
test_entity_spec_list = []
if snapshot_events:
test_entity_spec_list.append(
{tg.DYNAMIC_INFO_FKEY: tg.SYNC_VOLUME_SNAPSHOT_D,
tg.STATIC_INFO_FKEY: None,
tg.EXTERNAL_INFO_KEY: snap_vals,
tg.MAPPING_KEY: mapping,
tg.NAME_KEY: 'Volume snapshot generator',
tg.NUM_EVENTS: snapshot_events
}
)
return tg.get_trace_generators(test_entity_spec_list)
def simple_switch_generators(switch_num, host_num, snapshot_events=0,
snap_vals=None):
"""A function for returning switch event generators.

View File

@@ -48,6 +48,7 @@ SYNC_INST_SNAPSHOT_S = 'sync_inst_snapshot_static.json'
SYNC_INST_UPDATE_D = 'sync_inst_update_dynamic.json'
SYNC_HOST_SNAPSHOT_D = 'sync_host_snapshot_dynamic.json'
SYNC_ZONE_SNAPSHOT_D = 'sync_zone_snapshot_dynamic.json'
SYNC_VOLUME_SNAPSHOT_D = 'sync_volume_snapshot_dynamic.json'
SYNC_SWITCH_SNAPSHOT_D = 'sync_switch_snapshot_dynamic.json'
SYNC_NAGIOS_SNAPSHOT_D = 'sync_nagios_snapshot_dynamic.json'
SYNC_NAGIOS_SNAPSHOT_S = 'sync_nagios_snapshot_static.json'
@@ -98,6 +99,7 @@ class EventTraceGenerator(object):
SYNC_INST_UPDATE_D: _get_sync_vm_update_values,
SYNC_HOST_SNAPSHOT_D: _get_sync_host_snapshot_values,
SYNC_ZONE_SNAPSHOT_D: _get_sync_zone_snapshot_values,
SYNC_VOLUME_SNAPSHOT_D: _get_sync_volume_snapshot_values,
SYNC_SWITCH_SNAPSHOT_D: _get_sync_switch_snapshot_values,
SYNC_NAGIOS_SNAPSHOT_D: _get_sync_nagios_alarm_values,
@@ -257,6 +259,31 @@ def _get_sync_zone_snapshot_values(spec):
return static_values
def _get_sync_volume_snapshot_values(spec):
"""Generates the static synchronizer values for each volume.
:param spec: specification of event generation.
:type spec: dict
:return: list of static synchronizer values for each volume.
:rtype: list
"""
volume_instance_mapping = spec[MAPPING_KEY]
static_info_re = None
if spec[STATIC_INFO_FKEY] is not None:
static_info_re = utils.load_specs(spec[STATIC_INFO_FKEY])
static_values = []
for volume_name, instance_name in volume_instance_mapping:
mapping = {'id': volume_name,
'display_name': volume_name,
'attachments': [{'server_id': instance_name}]}
static_values.append(combine_data(
static_info_re, mapping, spec.get(EXTERNAL_INFO_KEY, None)
))
return static_values
def _get_trans_vm_snapshot_values(spec):
"""Generates the static transformer values for each vm.

View File

@@ -0,0 +1,10 @@
{
"attachments": [{"server_id": "54321"}],
"display_name": "volume-0",
"created_at": "2015-12-01T12:46:41Z",
"status": "In-use",
"id": "12345",
"sync_type": "cinder\\.volume",
"sync_mode": "snapshot",
"sample_date": "2015-12-01T12:46:41Z"
}

View File

@@ -0,0 +1,15 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
__author__ = 'stack'

View File

@@ -0,0 +1,171 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from oslo_log import log as logging
from vitrage.common.constants import EdgeLabels
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import SynchronizerProperties as SyncProps
from vitrage.common.constants import VertexProperties as VProps
from vitrage.synchronizer.plugins.cinder.volume import CINDER_VOLUME_PLUGIN
from vitrage.synchronizer.plugins.cinder.volume.transformer \
import CinderVolumeTransformer
from vitrage.synchronizer.plugins.nova.instance import NOVA_INSTANCE_PLUGIN
from vitrage.synchronizer.plugins.nova.instance.transformer \
import InstanceTransformer
from vitrage.synchronizer.plugins.transformer_base import TransformerBase
from vitrage.tests import base
from vitrage.tests.mocks import mock_syncronizer as mock_sync
LOG = logging.getLogger(__name__)
class TestCinderVolumeTransformer(base.BaseTest):
# noinspection PyAttributeOutsideInit
@classmethod
def setUpClass(cls):
cls.transformers = {}
cls.transformers[CINDER_VOLUME_PLUGIN] = \
CinderVolumeTransformer(cls.transformers)
cls.transformers[NOVA_INSTANCE_PLUGIN] = \
InstanceTransformer(cls.transformers)
def test_create_placeholder_vertex(self):
LOG.debug('Cinder Volume transformer test: Create placeholder '
'vertex')
# Tests setup
volume_id = 'Instance123'
timestamp = datetime.datetime.utcnow()
properties = {
VProps.ID: volume_id,
VProps.SAMPLE_TIMESTAMP: timestamp
}
transformer = CinderVolumeTransformer(self.transformers)
# Test action
placeholder = transformer.create_placeholder_vertex(**properties)
# Test assertions
observed_id_values = placeholder.vertex_id.split(
TransformerBase.KEY_SEPARATOR)
expected_id_values = transformer._key_values(CINDER_VOLUME_PLUGIN,
volume_id)
self.assertEqual(tuple(observed_id_values), expected_id_values)
observed_time = placeholder.get(VProps.SAMPLE_TIMESTAMP)
self.assertEqual(observed_time, timestamp)
observed_type = placeholder.get(VProps.TYPE)
self.assertEqual(observed_type, CINDER_VOLUME_PLUGIN)
observed_entity_id = placeholder.get(VProps.ID)
self.assertEqual(observed_entity_id, volume_id)
observed_category = placeholder.get(VProps.CATEGORY)
self.assertEqual(observed_category, EntityCategory.RESOURCE)
is_placeholder = placeholder.get(VProps.IS_PLACEHOLDER)
self.assertEqual(is_placeholder, True)
def test_key_values(self):
LOG.debug('Cinder Volume transformer test: get key values')
# Test setup
volume_type = CINDER_VOLUME_PLUGIN
volume_id = '12345'
transformer = CinderVolumeTransformer(self.transformers)
# Test action
observed_key_fields = transformer._key_values(volume_type,
volume_id)
# Test assertions
self.assertEqual(EntityCategory.RESOURCE, observed_key_fields[0])
self.assertEqual(CINDER_VOLUME_PLUGIN, observed_key_fields[1])
self.assertEqual(volume_id, observed_key_fields[2])
def test_snapshot_transform(self):
LOG.debug('Cinder Volume transformer test: transform entity event '
'snapshot')
# Test setup
spec_list = mock_sync.simple_volume_generators(3, 7, 7)
static_events = mock_sync.generate_random_events_list(spec_list)
for event in static_events:
# Test action
wrapper = self.transformers[CINDER_VOLUME_PLUGIN].transform(event)
# Test assertions
vertex = wrapper.vertex
self._validate_volume_vertex_props(vertex, event)
neighbors = wrapper.neighbors
self._validate_neighbors(neighbors, vertex.vertex_id, event)
def _validate_volume_vertex_props(self, vertex, event):
sync_mode = event[SyncProps.SYNC_MODE]
self.assertEqual(EntityCategory.RESOURCE, vertex[VProps.CATEGORY])
self.assertEqual(event[SyncProps.SYNC_TYPE], vertex[VProps.TYPE])
self.assertEqual(
event[CinderVolumeTransformer.VOLUME_ID[sync_mode][0]],
vertex[VProps.ID])
self.assertEqual(event[SyncProps.SAMPLE_DATE],
vertex[VProps.SAMPLE_TIMESTAMP])
self.assertEqual(
event[CinderVolumeTransformer.VOLUME_NAME[sync_mode][0]],
vertex[VProps.NAME])
self.assertEqual(
event[CinderVolumeTransformer.VOLUME_STATE[sync_mode][0]],
vertex[VProps.STATE])
self.assertFalse(vertex[VProps.IS_PLACEHOLDER])
self.assertFalse(vertex[VProps.IS_DELETED])
def _validate_neighbors(self, neighbors, volume_vertex_id, event):
instance_counter = 0
for neighbor in neighbors:
self._validate_instance_neighbor(
neighbor,
event['attachments'][0]['server_id'],
volume_vertex_id)
instance_counter += 1
self.assertEqual(1,
instance_counter,
'Zone can belongs to only one Node')
def _validate_instance_neighbor(self,
instance_neighbor,
instance_id,
volume_vertex_id):
# validate neighbor vertex
self.assertEqual(EntityCategory.RESOURCE,
instance_neighbor.vertex[VProps.CATEGORY])
self.assertEqual(NOVA_INSTANCE_PLUGIN,
instance_neighbor.vertex[VProps.TYPE])
self.assertEqual(instance_id, instance_neighbor.vertex[VProps.ID])
self.assertTrue(instance_neighbor.vertex[VProps.IS_PLACEHOLDER])
self.assertFalse(instance_neighbor.vertex[VProps.IS_DELETED])
# Validate neighbor edge
edge = instance_neighbor.edge
self.assertEqual(edge.target_id, instance_neighbor.vertex.vertex_id)
self.assertEqual(edge.source_id, volume_vertex_id)
self.assertEqual(edge.label, EdgeLabels.ATTACHED)