Add tool for datasource scaffold

Implements: blueprint datasource-scaffold
Change-Id: If853de3cd00e9d7ca4a93fdc409c6f0059dbc817

doc/source/contributor/add-new-datasource.rst

@@ -216,3 +216,31 @@ or by configuring ``vitrage.conf``.
     password = omd
     url = http://<ip>:<port>/<site>/nagios/cgi-bin/status.cgi
     config_file = /etc/vitrage/nagios_conf.yaml
+
+
+Using the scaffold tool
+-----------------------
+
+A datasource scaffold tool is provided to get you started to create a new
+datasource. See ``tools\datasoruce-scaffold`` for details.
+
+This tool uses `cookiecutter`_ to generate the scaffold of new datasource.
+
+.. _cookiecutter: https://github.com/audreyr/cookiecutter
+
+**Install**
+
+.. code-block:: shell
+
+    pip install -r requirements.txt
+
+**Usage**
+
+.. code-block:: shell
+
+    $ cookiecutter .
+    name [sample]:
+
+Enter the name of new datasource. It will create a new folder in current
+directory including the scaffold of the new data source. Move the directory to
+``vitrage/datasources`` as a start point for a complete implemenation.

tools/datasource-scaffold/README

@@ -0,0 +1 @@
+See https://docs.openstack.org/vitrage/latest/contributor/add-new-datasource.html#using-the-scaffold-tool

tools/datasource-scaffold/cookiecutter.json

@@ -0,0 +1,3 @@
+{
+    "name": "sample"
+}

tools/datasource-scaffold/requirements.txt

@@ -0,0 +1 @@
+cookiecutter # BSD 3-Clause License

tools/datasource-scaffold/sample/__init__.py

@@ -0,0 +1,48 @@
+# Copyright 2018 - Vitrage team
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR  CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+from vitrage.common.constants import DatasourceOpts as DSOpts
+from vitrage.common.constants import UpdateMethod
+
+SAMPLE_DATASOURCE = 'sample'
+
+OPTS = [
+    cfg.StrOpt(DSOpts.TRANSFORMER,
+               default='vitrage.datasources.sample.transformer.'
+                       'SampleTransformer',
+               help='Sample transformer class path',
+               required=True),
+    cfg.StrOpt(DSOpts.DRIVER,
+               default='vitrage.datasources.sample.driver.'
+                       'SampleDriver',
+               help='Sample driver class path',
+               required=True),
+    cfg.StrOpt(DSOpts.UPDATE_METHOD,
+               default=UpdateMethod.PULL,
+               help='None: updates only via Vitrage periodic snapshots.'
+                    'Pull: updates periodically.'
+                    'Push: updates by getting notifications from the'
+                    ' datasource itself.',
+               required=True),
+    cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
+               default=30,
+               min=10,
+               help='interval in seconds between checking changes in the'
+                    'sample configuration files')]
+
+
+class SampleFields(object):
+    TYPE = 'type'
+    ID = 'id'

tools/datasource-scaffold/sample/driver.py

@@ -0,0 +1,59 @@
+# Copyright 2018 - Vitrage team
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_log import log
+
+from vitrage.datasources.driver_base import DriverBase
+from vitrage.datasources.sample import SAMPLE_DATASOURCE
+
+LOG = log.getLogger(__name__)
+
+
+class SampleDriver(DriverBase):
+
+    def __init__(self, conf):
+        super(SampleDriver, self).__init__()
+        self.cfg = conf
+
+    @staticmethod
+    def get_event_types():
+        return []
+
+    def enrich_event(self, event, event_type):
+        pass
+
+    def get_all(self, datasource_action):
+        """Query all entities and send events to the vitrage events queue.
+
+        When done for the first time, send an "end" event to inform it has
+        finished the get_all for the datasource (because it is done
+        asynchronously).
+        """
+
+        return self.make_pickleable(self._get_all_entities(),
+                                    SAMPLE_DATASOURCE,
+                                    datasource_action)
+
+    def get_changes(self, datasource_action):
+        """Send an event to the vitrage events queue upon any change."""
+
+        return self.make_pickleable(self._get_changes_entities(),
+                                    SAMPLE_DATASOURCE,
+                                    datasource_action)
+
+    def _get_all_entities(self):
+        return []
+
+    def _get_changes_entities(self):
+        return []

tools/datasource-scaffold/sample/transformer.py

@@ -0,0 +1,69 @@
+# Copyright 2018 - Vitrage team
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_log import log as logging
+
+from vitrage.common.constants import EntityCategory
+from vitrage.common.constants import VertexProperties as VProps
+from vitrage.datasources.resource_transformer_base import \
+    ResourceTransformerBase
+from vitrage.datasources.sample import SAMPLE_DATASOURCE
+from vitrage.datasources.sample import SampleFields
+from vitrage.datasources import transformer_base
+import vitrage.graph.utils as graph_utils
+
+LOG = logging.getLogger(__name__)
+
+
+class SampleTransformer(ResourceTransformerBase):
+
+    def __init__(self, transformers, conf):
+        super(SampleTransformer, self).__init__(transformers, conf)
+
+    def _create_snapshot_entity_vertex(self, entity_event):
+        return self._create_vertex(entity_event)
+
+    def _create_update_entity_vertex(self, entity_event):
+        return self._create_vertex(entity_event)
+
+    def _create_snapshot_neighbors(self, entity_event):
+        return self._create_sample_neighbors(entity_event)
+
+    def _create_update_neighbors(self, entity_event):
+        return self._create_sample_neighbors(entity_event)
+
+    def _create_entity_key(self, entity_event):
+        """the unique key of this entity"""
+        entity_id = entity_event[VProps.ID]
+        entity_type = entity_event[SampleFields.TYPE]
+        key_fields = self._key_values(entity_type, entity_id)
+        return transformer_base.build_key(key_fields)
+
+    @staticmethod
+    def get_vitrage_type():
+        return SAMPLE_DATASOURCE
+
+    def _create_vertex(self, entity_event):
+        return graph_utils.create_vertex(
+            self._create_entity_key(entity_event),
+            vitrage_category=EntityCategory.RESOURCE,
+            vitrage_type=None,  # FIXME
+            vitrage_sample_timestamp=None,  # FIXME
+            entity_id=None,  # FIXME
+            update_timestamp=None,  # FIXME
+            entity_state=None,  # FIXME
+            metadata=None)  # FIXME
+
+    def _create_sample_neighbors(self, entity_event):
+        return []

tools/datasource-scaffold/{{cookiecutter.name}}/__init__.py

@@ -0,0 +1,48 @@
+# Copyright 2018 - Vitrage team
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR  CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+from vitrage.common.constants import DatasourceOpts as DSOpts
+from vitrage.common.constants import UpdateMethod
+
+{{cookiecutter.name|upper}}_DATASOURCE = '{{cookiecutter.name}}'
+
+OPTS = [
+    cfg.StrOpt(DSOpts.TRANSFORMER,
+               default='vitrage.datasources.{{cookiecutter.name}}.transformer.'
+                       '{{cookiecutter.name|capitalize}}Transformer',
+               help='{{cookiecutter.name|capitalize}} transformer class path',
+               required=True),
+    cfg.StrOpt(DSOpts.DRIVER,
+               default='vitrage.datasources.{{cookiecutter.name}}.driver.'
+                       '{{cookiecutter.name|capitalize}}Driver',
+               help='{{cookiecutter.name|capitalize}} driver class path',
+               required=True),
+    cfg.StrOpt(DSOpts.UPDATE_METHOD,
+               default=UpdateMethod.PULL,
+               help='None: updates only via Vitrage periodic snapshots.'
+                    'Pull: updates periodically.'
+                    'Push: updates by getting notifications from the'
+                    ' datasource itself.',
+               required=True),
+    cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
+               default=30,
+               min=10,
+               help='interval in seconds between checking changes in the'
+                    '{{cookiecutter.name}} configuration files')]
+
+
+class {{cookiecutter.name|capitalize}}Fields(object):
+    TYPE = 'type'
+    ID = 'id'

tools/datasource-scaffold/{{cookiecutter.name}}/driver.py

@@ -0,0 +1,59 @@
+# Copyright 2018 - Vitrage team
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_log import log
+
+from vitrage.datasources.driver_base import DriverBase
+from vitrage.datasources.{{cookiecutter.name}} import {{cookiecutter.name|upper}}_DATASOURCE
+
+LOG = log.getLogger(__name__)
+
+
+class {{cookiecutter.name|capitalize}}Driver(DriverBase):
+
+    def __init__(self, conf):
+        super({{cookiecutter.name|capitalize}}Driver, self).__init__()
+        self.cfg = conf
+
+    @staticmethod
+    def get_event_types():
+        return []
+
+    def enrich_event(self, event, event_type):
+        pass
+
+    def get_all(self, datasource_action):
+        """Query all entities and send events to the vitrage events queue.
+
+        When done for the first time, send an "end" event to inform it has
+        finished the get_all for the datasource (because it is done
+        asynchronously).
+        """
+
+        return self.make_pickleable(self._get_all_entities(),
+                                    {{cookiecutter.name|upper}}_DATASOURCE,
+                                    datasource_action)
+
+    def get_changes(self, datasource_action):
+        """Send an event to the vitrage events queue upon any change."""
+
+        return self.make_pickleable(self._get_changes_entities(),
+                                    {{cookiecutter.name|upper}}_DATASOURCE,
+                                    datasource_action)
+
+    def _get_all_entities(self):
+        return []
+
+    def _get_changes_entities(self):
+        return []

tools/datasource-scaffold/{{cookiecutter.name}}/transformer.py

@@ -0,0 +1,72 @@
+# Copyright 2018 - Vitrage team
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_log import log as logging
+
+from vitrage.common.constants import EntityCategory
+from vitrage.common.constants import VertexProperties as VProps
+from vitrage.datasources.resource_transformer_base import \
+    ResourceTransformerBase
+from vitrage.datasources.{{cookiecutter.name}} import {{cookiecutter.name|upper}}_DATASOURCE
+from vitrage.datasources.{{cookiecutter.name}} import {{cookiecutter.name|capitalize}}Fields
+from vitrage.datasources import transformer_base
+import vitrage.graph.utils as graph_utils
+
+LOG = logging.getLogger(__name__)
+
+
+class {{cookiecutter.name|capitalize}}Transformer(ResourceTransformerBase):
+
+    def __init__(self, transformers, conf):
+        super({{cookiecutter.name|capitalize}}Transformer, self).__init__(transformers, conf)
+
+    def _create_snapshot_entity_vertex(self, entity_event):
+        return self._create_vertex(entity_event)
+
+    def _create_update_entity_vertex(self, entity_event):
+        return self._create_vertex(entity_event)
+
+    def _create_snapshot_neighbors(self, entity_event):
+        return self._create_{{cookiecutter.name}}_neighbors(entity_event)
+
+    def _create_update_neighbors(self, entity_event):
+        return self._create_{{cookiecutter.name}}_neighbors(entity_event)
+
+    def _create_entity_key(self, entity_event):
+        """the unique key of this entity"""
+
+        # FIXME: Below is just an example. It could be different in a real
+        #        datasource
+        entity_id = entity_event[VProps.ID]
+        entity_type = entity_event[{{cookiecutter.name|capitalize}}Fields.TYPE]
+        key_fields = self._key_values(entity_type, entity_id)
+        return transformer_base.build_key(key_fields)
+
+    @staticmethod
+    def get_vitrage_type():
+        return {{cookiecutter.name|upper}}_DATASOURCE
+
+    def _create_vertex(self, entity_event):
+        return graph_utils.create_vertex(
+            self._create_entity_key(entity_event),
+            vitrage_category=EntityCategory.RESOURCE,
+            vitrage_type=None,  # FIXME
+            vitrage_{{cookiecutter.name}}_timestamp=None,  # FIXME
+            entity_id=None,  # FIXME
+            update_timestamp=None,  # FIXME
+            entity_state=None,  # FIXME
+            metadata=None)  # FIXME
+
+    def _create_{{cookiecutter.name}}_neighbors(self, entity_event):
+        return []

tox.ini

@@ -56,7 +56,7 @@ ignore = E123,E125
 enable-extensions=H106,H203
 builtins = _
 filename = *.py,app.wsgi
-exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools/datasource-scaffold
 
 [hacking]
 local-check-factory = vitrage.hacking.checks.factory

vitrage/datasources/sample

@@ -0,0 +1 @@
+tools/datasource-scaffold/sample