diff --git a/doc/source/add-new-datasource.rst b/doc/source/add-new-datasource.rst new file mode 100644 index 000000000..d1fe4b187 --- /dev/null +++ b/doc/source/add-new-datasource.rst @@ -0,0 +1,220 @@ +================== +Add New Datasource +================== + +Add Datasource Package - HOW TO +------------------------------- + +In order to add a new datasource to Vitrage do the following steps: + + 1. Have your datasource enclosed in a package with the datasources' name and + put it under 'vitrage.datasources', For example: + vitrage.datasource.cinder.volume. + 2. Under your datasource package, have both your datasources' driver class + your datasources' transformer class. See below for details on those + classes. + 3. Under your datasources' package __init__.py you must import cfg + from oslo_config and declare a list named OPTS. Under OPTS, you can define + your needed options using the oslo_config.cfg module. + There are three options you must have: + + a. Driver and transformer with the path to your driver and transformer + classes respectively. + b. update_method property that describes the type of update mechanism for + this datasource. The options are: push, pull or none. + c. In addition to those three, you may add any other configuration options + you may need for your datasource. + + 4. In case you want your datasource to get registered under other names i.e. + for other sub-entities, add a list option named 'entities' under which + list all of your sub-entities names (more details below). + 5. In case you want your datasource to be automatically configured when + devstack is installed, you need to add it to the 'types' property in the + datasources section in the configuration. To do so, do the following: + + a. add the datasource name to the types property in the "devstack.settings" + file. + b. if the datasource is not is not one of the main and basic projects of + devstack, add the following data in the "devstack.plugin.sh" file": + + # remove vitrage datasource if + datasource not installed + + if ! is_service_enabled ; then + + disable_vitrage_datasource + + fi + 6. You are done! + + +Driver Class +__________________ + +Responsible for importing information regarding entities in the cloud. +Entities in this context refer both to resources (physical, virtual, +applicative) and alarms (Aodh, Nagios, Zabbix, Monasca, etc.) +The datasource has two modes of action: + + 1. get_all (snapshot): Query all entities and send events to the vitrage + events queue. + When done for the first time, send an "end" event to inform it has finished + the get_all for the datasource (because it is done asynchronously). + 2. notify: Send an event to the vitrage events queue upon any change. + This can be done in two ways: + + a. Built in polling mechanism called get_changes. + b. Built in pushing mechanism using the oslo bus. + +A driver should inherit from 'vitrage.datasources.driver_base.DriverBase' class +and must implement the following methods: + ++----------------------+------------------------------------+--------------------------------+--------------------------------+ +| Name | Input | Output | Comments | ++======================+====================================+================================+================================+ +| get_all | sync mode | void | for snapshot mechanism | ++----------------------+------------------------------------+--------------------------------+--------------------------------+ +| get_changes | sync mode | void | for update pulling mechanism | ++----------------------+------------------------------------+--------------------------------+--------------------------------+ +| get_event_types | conf | void | for update pushing mechanism | ++----------------------+------------------------------------+--------------------------------+--------------------------------+ +| enrich_event | event, event_type | void | for update pushing mechanism | ++----------------------+------------------------------------+--------------------------------+--------------------------------+ + + +Transformer Class +_________________ + +The Transformer class understands the specific entity details and outputs a +tuple with the following details: + + 1. The vertex with its new details to be added / updated / deleted. + 2. List of tuples where each tuple consists of: + + a. Neighbor vertex with it's partial data so vitrage will know to where + to connect the edge. + b. Edge that connects the vertex to its' neighbor. + +Note that for every driver there should be a matching Transformer. +A transformer should inherit from +'vitrage.datasoures.transformer_base.TransformerBase' class and +must implement the following methods: + ++----------------------------------+------------------------------------+----------------------------------------+ +| Name | Input | Output | ++==================================+====================================+========================================+ +| _create_snapshot_entity_vertex | entity event | vertex | ++----------------------------------+------------------------------------+----------------------------------------+ +| _create_update_entity_vertex | entity event | vertex | ++----------------------------------+------------------------------------+----------------------------------------+ +| _create_snapshot_neighbors | entity event | neighbor tuple | ++----------------------------------+------------------------------------+----------------------------------------+ +| _create_update_neighbors | entity event | neighbor tuple | ++----------------------------------+------------------------------------+----------------------------------------+ +| _create_entity_key | entity event | the unique key of this entity | ++----------------------------------+------------------------------------+----------------------------------------+ +| get_type | | datasources' type | ++----------------------------------+------------------------------------+----------------------------------------+ + + +Configuration +_____________ + +Holds the following fields: + ++----------------------------+------------------------------------+-------------------------------------------------------------+ +| Name | Type | Description | ++============================+====================================+=============================================================+ +| transformer | string - Required! | Transformer class path under vitrage | ++----------------------------+------------------------------------+-------------------------------------------------------------+ +| driver | string - Required! | Driver class path under vitrage | ++----------------------------+------------------------------------+-------------------------------------------------------------+ +| update_method | string - Required! | need to be one of: pull, push or none values | ++----------------------------+------------------------------------+-------------------------------------------------------------+ +| changes_interval | integer - Optional | Interval between checking for changes in polling mechanism | ++----------------------------+------------------------------------+-------------------------------------------------------------+ +| entities | string list - Optional | Sub-entities of the datasource | ++----------------------------+------------------------------------+-------------------------------------------------------------+ + +**Example** + +Datasource __init__.py OPTS: + +.. code:: python + + from oslo_config import cfg + + OPTS = [ + cfg.StrOpt('transformer', + default='vitrage.datasources.cinder.volume.transformer.' + 'CinderVolumeTransformer', + help='Cinder volume transformer class path', + required=True), + cfg.StrOpt('driver', + default='vitrage.datasources.cinder.volume.driver.' + 'CinderVolumeDriver', + help='Cinder volume driver class path', + required=True), + cfg.StrOpt('update_method', + default=UpdateMethod.PUSH, + help='None: updates only via Vitrage periodic snapshots.' + 'Pull: updates every [changes_interval] seconds.' + 'Push: updates by getting notifications from the' + ' datasource itself.', + required=True), + ] + + +Instantiation flow +------------------ + +Now, when loading Vitrage, vitrage.datasources.launcher.Launcher +will get instantiated and will register all of the datasources +into Vitrage. Note, that if you want your datasource to also run as a +service i.e. get changes every you need to set under your +datasources' OPTS an Integer option named 'changes_interval'. +Additionally, vitrage.entity_graph.transformer_manager.TransformerManager +will get instantiated and will register all of the datasources transformers +into Vitrage. +These two steps are using your previously configured driver and +transformer path options under your datasources' package __init__.OPTS. + + +Datasource Configuration Options +-------------------------------- + +Any option your datasource defined can be accessed using oslo_config.cfg +or by configuring vitrage.conf. + +**Example** + +.. code:: python + + cfg.. + + +**Example** + +/etc/vitrage/vitrage.conf + + ... + + [datasources] + + snapshots_interval = 300 + + # Names of supported plugins (list value) + + types = nova.host,nova.instance,nova.zone,static_physical,aodh,cinder.volume,neutron.network,neutron.port,heat.stack + + + [zabbix] + + url = http://135.248.18.30 + + password = zabbix + + user = admin + + config_file = /etc/vitrage/zabbix_conf.yaml + diff --git a/vitrage/datasources/__init__.py b/vitrage/datasources/__init__.py index cc7ff580d..fdfaef95b 100644 --- a/vitrage/datasources/__init__.py +++ b/vitrage/datasources/__init__.py @@ -14,14 +14,12 @@ from oslo_config import cfg -from vitrage.datasources.aodh import AODH_DATASOURCE from vitrage.datasources.cinder.volume import CINDER_VOLUME_DATASOURCE -from vitrage.datasources.nagios import NAGIOS_DATASOURCE from vitrage.datasources.neutron.network import NEUTRON_NETWORK_DATASOURCE +from vitrage.datasources.neutron.port import NEUTRON_PORT_DATASOURCE from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE from vitrage.datasources.nova.zone import NOVA_ZONE_DATASOURCE -from vitrage.datasources.static_physical import STATIC_PHYSICAL_DATASOURCE OPENSTACK_CLUSTER = 'openstack.cluster' @@ -31,10 +29,8 @@ OPTS = [ default=[NOVA_HOST_DATASOURCE, NOVA_INSTANCE_DATASOURCE, NOVA_ZONE_DATASOURCE, - NAGIOS_DATASOURCE, - STATIC_PHYSICAL_DATASOURCE, - AODH_DATASOURCE, CINDER_VOLUME_DATASOURCE, + NEUTRON_PORT_DATASOURCE, NEUTRON_NETWORK_DATASOURCE], help='Names of supported data sources'), cfg.ListOpt('path', diff --git a/vitrage/datasources/cinder/volume/__init__.py b/vitrage/datasources/cinder/volume/__init__.py index aafbc43e6..f928985b6 100644 --- a/vitrage/datasources/cinder/volume/__init__.py +++ b/vitrage/datasources/cinder/volume/__init__.py @@ -21,12 +21,12 @@ OPTS = [ cfg.StrOpt('transformer', default='vitrage.datasources.cinder.volume.transformer.' 'CinderVolumeTransformer', - help='Nova host transformer class path', + help='Cinder volume transformer class path', required=True), cfg.StrOpt('driver', default='vitrage.datasources.cinder.volume.driver.' 'CinderVolumeDriver', - help='Nova host driver class path', + help='Cinder volume driver class path', required=True), cfg.StrOpt('update_method', default=UpdateMethod.PUSH, diff --git a/vitrage/tests/functional/datasources/cinder/test_cinder_volume.py b/vitrage/tests/functional/datasources/cinder/test_cinder_volume.py index a7ec0817b..6ecdbe355 100644 --- a/vitrage/tests/functional/datasources/cinder/test_cinder_volume.py +++ b/vitrage/tests/functional/datasources/cinder/test_cinder_volume.py @@ -16,8 +16,8 @@ from oslo_config import cfg from vitrage.common.constants import EntityCategory from vitrage.common.constants import VertexProperties as VProps -from vitrage.datasources import CINDER_VOLUME_DATASOURCE -from vitrage.datasources import NAGIOS_DATASOURCE +from vitrage.datasources.cinder.volume import CINDER_VOLUME_DATASOURCE +from vitrage.datasources.nagios import NAGIOS_DATASOURCE from vitrage.datasources import NOVA_HOST_DATASOURCE from vitrage.datasources import NOVA_INSTANCE_DATASOURCE from vitrage.datasources import NOVA_ZONE_DATASOURCE diff --git a/vitrage/tests/functional/datasources/nagios/test_nagios.py b/vitrage/tests/functional/datasources/nagios/test_nagios.py index 162e00de7..0dd3c9d8c 100644 --- a/vitrage/tests/functional/datasources/nagios/test_nagios.py +++ b/vitrage/tests/functional/datasources/nagios/test_nagios.py @@ -16,7 +16,7 @@ from oslo_config import cfg from vitrage.common.constants import EntityCategory from vitrage.common.constants import VertexProperties as VProps -from vitrage.datasources import NAGIOS_DATASOURCE +from vitrage.datasources.nagios import NAGIOS_DATASOURCE from vitrage.datasources import NOVA_HOST_DATASOURCE from vitrage.datasources import NOVA_INSTANCE_DATASOURCE from vitrage.datasources import NOVA_ZONE_DATASOURCE diff --git a/vitrage/tests/functional/datasources/static_physical/test_static_physical.py b/vitrage/tests/functional/datasources/static_physical/test_static_physical.py index c2d819fbd..d07a91bd7 100644 --- a/vitrage/tests/functional/datasources/static_physical/test_static_physical.py +++ b/vitrage/tests/functional/datasources/static_physical/test_static_physical.py @@ -17,7 +17,7 @@ from oslo_config import cfg from vitrage.common.constants import DatasourceProperties as DSProps from vitrage.common.constants import EntityCategory from vitrage.common.constants import VertexProperties as VProps -from vitrage.datasources import NAGIOS_DATASOURCE +from vitrage.datasources.nagios import NAGIOS_DATASOURCE from vitrage.datasources import NOVA_HOST_DATASOURCE from vitrage.datasources import NOVA_INSTANCE_DATASOURCE from vitrage.datasources import NOVA_ZONE_DATASOURCE diff --git a/vitrage_tempest_tests/tests/api/base.py b/vitrage_tempest_tests/tests/api/base.py index 0faa353e5..7c0f790df 100644 --- a/vitrage_tempest_tests/tests/api/base.py +++ b/vitrage_tempest_tests/tests/api/base.py @@ -20,8 +20,8 @@ from oslotest import base from vitrage import clients from vitrage.common.constants import EntityCategory from vitrage.common.constants import VertexProperties as VProps -from vitrage.datasources import AODH_DATASOURCE -from vitrage.datasources import CINDER_VOLUME_DATASOURCE +from vitrage.datasources.aodh import AODH_DATASOURCE +from vitrage.datasources.cinder.volume import CINDER_VOLUME_DATASOURCE from vitrage.datasources.heat.stack import HEAT_STACK_DATASOURCE from vitrage.datasources.neutron.network import NEUTRON_NETWORK_DATASOURCE from vitrage.datasources.neutron.port import NEUTRON_PORT_DATASOURCE diff --git a/vitrage_tempest_tests/tests/api/rca/base.py b/vitrage_tempest_tests/tests/api/rca/base.py index 546bcc5df..49413b61c 100644 --- a/vitrage_tempest_tests/tests/api/rca/base.py +++ b/vitrage_tempest_tests/tests/api/rca/base.py @@ -17,7 +17,7 @@ import json from oslo_log import log as logging from vitrage.common.constants import VertexProperties as VProps -from vitrage.datasources import AODH_DATASOURCE +from vitrage.datasources.aodh import AODH_DATASOURCE from vitrage.datasources import NOVA_HOST_DATASOURCE from vitrage.datasources import NOVA_INSTANCE_DATASOURCE from vitrage.entity_graph.mappings.operational_alarm_severity \