From 89a0d974e51b4f5ef8c7f3c4b9ee736835a83dea Mon Sep 17 00:00:00 2001 From: Yujun Zhang Date: Fri, 4 May 2018 15:37:36 +0800 Subject: [PATCH] Add tool for datasource scaffold Implements: blueprint datasource-scaffold Change-Id: If853de3cd00e9d7ca4a93fdc409c6f0059dbc817 --- doc/source/contributor/add-new-datasource.rst | 28 ++++++++ tools/datasource-scaffold/README | 1 + tools/datasource-scaffold/cookiecutter.json | 3 + tools/datasource-scaffold/requirements.txt | 1 + tools/datasource-scaffold/sample/__init__.py | 48 +++++++++++++ tools/datasource-scaffold/sample/driver.py | 59 +++++++++++++++ .../datasource-scaffold/sample/transformer.py | 69 ++++++++++++++++++ .../{{cookiecutter.name}}/__init__.py | 48 +++++++++++++ .../{{cookiecutter.name}}/driver.py | 59 +++++++++++++++ .../{{cookiecutter.name}}/transformer.py | 72 +++++++++++++++++++ tox.ini | 2 +- vitrage/datasources/sample | 1 + 12 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 tools/datasource-scaffold/README create mode 100644 tools/datasource-scaffold/cookiecutter.json create mode 100644 tools/datasource-scaffold/requirements.txt create mode 100644 tools/datasource-scaffold/sample/__init__.py create mode 100644 tools/datasource-scaffold/sample/driver.py create mode 100644 tools/datasource-scaffold/sample/transformer.py create mode 100644 tools/datasource-scaffold/{{cookiecutter.name}}/__init__.py create mode 100644 tools/datasource-scaffold/{{cookiecutter.name}}/driver.py create mode 100644 tools/datasource-scaffold/{{cookiecutter.name}}/transformer.py create mode 120000 vitrage/datasources/sample diff --git a/doc/source/contributor/add-new-datasource.rst b/doc/source/contributor/add-new-datasource.rst index 508a31a41..9492d1730 100644 --- a/doc/source/contributor/add-new-datasource.rst +++ b/doc/source/contributor/add-new-datasource.rst @@ -216,3 +216,31 @@ or by configuring ``vitrage.conf``. password = omd url = http://://nagios/cgi-bin/status.cgi config_file = /etc/vitrage/nagios_conf.yaml + + +Using the scaffold tool +----------------------- + +A datasource scaffold tool is provided to get you started to create a new +datasource. See ``tools\datasoruce-scaffold`` for details. + +This tool uses `cookiecutter`_ to generate the scaffold of new datasource. + +.. _cookiecutter: https://github.com/audreyr/cookiecutter + +**Install** + +.. code-block:: shell + + pip install -r requirements.txt + +**Usage** + +.. code-block:: shell + + $ cookiecutter . + name [sample]: + +Enter the name of new datasource. It will create a new folder in current +directory including the scaffold of the new data source. Move the directory to +``vitrage/datasources`` as a start point for a complete implemenation. diff --git a/tools/datasource-scaffold/README b/tools/datasource-scaffold/README new file mode 100644 index 000000000..b58c34d7a --- /dev/null +++ b/tools/datasource-scaffold/README @@ -0,0 +1 @@ +See https://docs.openstack.org/vitrage/latest/contributor/add-new-datasource.html#using-the-scaffold-tool diff --git a/tools/datasource-scaffold/cookiecutter.json b/tools/datasource-scaffold/cookiecutter.json new file mode 100644 index 000000000..7402915af --- /dev/null +++ b/tools/datasource-scaffold/cookiecutter.json @@ -0,0 +1,3 @@ +{ + "name": "sample" +} diff --git a/tools/datasource-scaffold/requirements.txt b/tools/datasource-scaffold/requirements.txt new file mode 100644 index 000000000..01935af01 --- /dev/null +++ b/tools/datasource-scaffold/requirements.txt @@ -0,0 +1 @@ +cookiecutter # BSD 3-Clause License diff --git a/tools/datasource-scaffold/sample/__init__.py b/tools/datasource-scaffold/sample/__init__.py new file mode 100644 index 000000000..cd943583c --- /dev/null +++ b/tools/datasource-scaffold/sample/__init__.py @@ -0,0 +1,48 @@ +# Copyright 2018 - Vitrage team +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from vitrage.common.constants import DatasourceOpts as DSOpts +from vitrage.common.constants import UpdateMethod + +SAMPLE_DATASOURCE = 'sample' + +OPTS = [ + cfg.StrOpt(DSOpts.TRANSFORMER, + default='vitrage.datasources.sample.transformer.' + 'SampleTransformer', + help='Sample transformer class path', + required=True), + cfg.StrOpt(DSOpts.DRIVER, + default='vitrage.datasources.sample.driver.' + 'SampleDriver', + help='Sample driver class path', + required=True), + cfg.StrOpt(DSOpts.UPDATE_METHOD, + default=UpdateMethod.PULL, + help='None: updates only via Vitrage periodic snapshots.' + 'Pull: updates periodically.' + 'Push: updates by getting notifications from the' + ' datasource itself.', + required=True), + cfg.IntOpt(DSOpts.CHANGES_INTERVAL, + default=30, + min=10, + help='interval in seconds between checking changes in the' + 'sample configuration files')] + + +class SampleFields(object): + TYPE = 'type' + ID = 'id' diff --git a/tools/datasource-scaffold/sample/driver.py b/tools/datasource-scaffold/sample/driver.py new file mode 100644 index 000000000..91f6f24a4 --- /dev/null +++ b/tools/datasource-scaffold/sample/driver.py @@ -0,0 +1,59 @@ +# Copyright 2018 - Vitrage team +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log + +from vitrage.datasources.driver_base import DriverBase +from vitrage.datasources.sample import SAMPLE_DATASOURCE + +LOG = log.getLogger(__name__) + + +class SampleDriver(DriverBase): + + def __init__(self, conf): + super(SampleDriver, self).__init__() + self.cfg = conf + + @staticmethod + def get_event_types(): + return [] + + def enrich_event(self, event, event_type): + pass + + def get_all(self, datasource_action): + """Query all entities and send events to the vitrage events queue. + + When done for the first time, send an "end" event to inform it has + finished the get_all for the datasource (because it is done + asynchronously). + """ + + return self.make_pickleable(self._get_all_entities(), + SAMPLE_DATASOURCE, + datasource_action) + + def get_changes(self, datasource_action): + """Send an event to the vitrage events queue upon any change.""" + + return self.make_pickleable(self._get_changes_entities(), + SAMPLE_DATASOURCE, + datasource_action) + + def _get_all_entities(self): + return [] + + def _get_changes_entities(self): + return [] diff --git a/tools/datasource-scaffold/sample/transformer.py b/tools/datasource-scaffold/sample/transformer.py new file mode 100644 index 000000000..5a4eb76ec --- /dev/null +++ b/tools/datasource-scaffold/sample/transformer.py @@ -0,0 +1,69 @@ +# Copyright 2018 - Vitrage team +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging + +from vitrage.common.constants import EntityCategory +from vitrage.common.constants import VertexProperties as VProps +from vitrage.datasources.resource_transformer_base import \ + ResourceTransformerBase +from vitrage.datasources.sample import SAMPLE_DATASOURCE +from vitrage.datasources.sample import SampleFields +from vitrage.datasources import transformer_base +import vitrage.graph.utils as graph_utils + +LOG = logging.getLogger(__name__) + + +class SampleTransformer(ResourceTransformerBase): + + def __init__(self, transformers, conf): + super(SampleTransformer, self).__init__(transformers, conf) + + def _create_snapshot_entity_vertex(self, entity_event): + return self._create_vertex(entity_event) + + def _create_update_entity_vertex(self, entity_event): + return self._create_vertex(entity_event) + + def _create_snapshot_neighbors(self, entity_event): + return self._create_sample_neighbors(entity_event) + + def _create_update_neighbors(self, entity_event): + return self._create_sample_neighbors(entity_event) + + def _create_entity_key(self, entity_event): + """the unique key of this entity""" + entity_id = entity_event[VProps.ID] + entity_type = entity_event[SampleFields.TYPE] + key_fields = self._key_values(entity_type, entity_id) + return transformer_base.build_key(key_fields) + + @staticmethod + def get_vitrage_type(): + return SAMPLE_DATASOURCE + + def _create_vertex(self, entity_event): + return graph_utils.create_vertex( + self._create_entity_key(entity_event), + vitrage_category=EntityCategory.RESOURCE, + vitrage_type=None, # FIXME + vitrage_sample_timestamp=None, # FIXME + entity_id=None, # FIXME + update_timestamp=None, # FIXME + entity_state=None, # FIXME + metadata=None) # FIXME + + def _create_sample_neighbors(self, entity_event): + return [] diff --git a/tools/datasource-scaffold/{{cookiecutter.name}}/__init__.py b/tools/datasource-scaffold/{{cookiecutter.name}}/__init__.py new file mode 100644 index 000000000..2a48470b2 --- /dev/null +++ b/tools/datasource-scaffold/{{cookiecutter.name}}/__init__.py @@ -0,0 +1,48 @@ +# Copyright 2018 - Vitrage team +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from vitrage.common.constants import DatasourceOpts as DSOpts +from vitrage.common.constants import UpdateMethod + +{{cookiecutter.name|upper}}_DATASOURCE = '{{cookiecutter.name}}' + +OPTS = [ + cfg.StrOpt(DSOpts.TRANSFORMER, + default='vitrage.datasources.{{cookiecutter.name}}.transformer.' + '{{cookiecutter.name|capitalize}}Transformer', + help='{{cookiecutter.name|capitalize}} transformer class path', + required=True), + cfg.StrOpt(DSOpts.DRIVER, + default='vitrage.datasources.{{cookiecutter.name}}.driver.' + '{{cookiecutter.name|capitalize}}Driver', + help='{{cookiecutter.name|capitalize}} driver class path', + required=True), + cfg.StrOpt(DSOpts.UPDATE_METHOD, + default=UpdateMethod.PULL, + help='None: updates only via Vitrage periodic snapshots.' + 'Pull: updates periodically.' + 'Push: updates by getting notifications from the' + ' datasource itself.', + required=True), + cfg.IntOpt(DSOpts.CHANGES_INTERVAL, + default=30, + min=10, + help='interval in seconds between checking changes in the' + '{{cookiecutter.name}} configuration files')] + + +class {{cookiecutter.name|capitalize}}Fields(object): + TYPE = 'type' + ID = 'id' diff --git a/tools/datasource-scaffold/{{cookiecutter.name}}/driver.py b/tools/datasource-scaffold/{{cookiecutter.name}}/driver.py new file mode 100644 index 000000000..699a5e8f6 --- /dev/null +++ b/tools/datasource-scaffold/{{cookiecutter.name}}/driver.py @@ -0,0 +1,59 @@ +# Copyright 2018 - Vitrage team +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log + +from vitrage.datasources.driver_base import DriverBase +from vitrage.datasources.{{cookiecutter.name}} import {{cookiecutter.name|upper}}_DATASOURCE + +LOG = log.getLogger(__name__) + + +class {{cookiecutter.name|capitalize}}Driver(DriverBase): + + def __init__(self, conf): + super({{cookiecutter.name|capitalize}}Driver, self).__init__() + self.cfg = conf + + @staticmethod + def get_event_types(): + return [] + + def enrich_event(self, event, event_type): + pass + + def get_all(self, datasource_action): + """Query all entities and send events to the vitrage events queue. + + When done for the first time, send an "end" event to inform it has + finished the get_all for the datasource (because it is done + asynchronously). + """ + + return self.make_pickleable(self._get_all_entities(), + {{cookiecutter.name|upper}}_DATASOURCE, + datasource_action) + + def get_changes(self, datasource_action): + """Send an event to the vitrage events queue upon any change.""" + + return self.make_pickleable(self._get_changes_entities(), + {{cookiecutter.name|upper}}_DATASOURCE, + datasource_action) + + def _get_all_entities(self): + return [] + + def _get_changes_entities(self): + return [] diff --git a/tools/datasource-scaffold/{{cookiecutter.name}}/transformer.py b/tools/datasource-scaffold/{{cookiecutter.name}}/transformer.py new file mode 100644 index 000000000..0e9cc7751 --- /dev/null +++ b/tools/datasource-scaffold/{{cookiecutter.name}}/transformer.py @@ -0,0 +1,72 @@ +# Copyright 2018 - Vitrage team +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging + +from vitrage.common.constants import EntityCategory +from vitrage.common.constants import VertexProperties as VProps +from vitrage.datasources.resource_transformer_base import \ + ResourceTransformerBase +from vitrage.datasources.{{cookiecutter.name}} import {{cookiecutter.name|upper}}_DATASOURCE +from vitrage.datasources.{{cookiecutter.name}} import {{cookiecutter.name|capitalize}}Fields +from vitrage.datasources import transformer_base +import vitrage.graph.utils as graph_utils + +LOG = logging.getLogger(__name__) + + +class {{cookiecutter.name|capitalize}}Transformer(ResourceTransformerBase): + + def __init__(self, transformers, conf): + super({{cookiecutter.name|capitalize}}Transformer, self).__init__(transformers, conf) + + def _create_snapshot_entity_vertex(self, entity_event): + return self._create_vertex(entity_event) + + def _create_update_entity_vertex(self, entity_event): + return self._create_vertex(entity_event) + + def _create_snapshot_neighbors(self, entity_event): + return self._create_{{cookiecutter.name}}_neighbors(entity_event) + + def _create_update_neighbors(self, entity_event): + return self._create_{{cookiecutter.name}}_neighbors(entity_event) + + def _create_entity_key(self, entity_event): + """the unique key of this entity""" + + # FIXME: Below is just an example. It could be different in a real + # datasource + entity_id = entity_event[VProps.ID] + entity_type = entity_event[{{cookiecutter.name|capitalize}}Fields.TYPE] + key_fields = self._key_values(entity_type, entity_id) + return transformer_base.build_key(key_fields) + + @staticmethod + def get_vitrage_type(): + return {{cookiecutter.name|upper}}_DATASOURCE + + def _create_vertex(self, entity_event): + return graph_utils.create_vertex( + self._create_entity_key(entity_event), + vitrage_category=EntityCategory.RESOURCE, + vitrage_type=None, # FIXME + vitrage_{{cookiecutter.name}}_timestamp=None, # FIXME + entity_id=None, # FIXME + update_timestamp=None, # FIXME + entity_state=None, # FIXME + metadata=None) # FIXME + + def _create_{{cookiecutter.name}}_neighbors(self, entity_event): + return [] diff --git a/tox.ini b/tox.ini index fe9a61c49..fbb704eab 100644 --- a/tox.ini +++ b/tox.ini @@ -56,7 +56,7 @@ ignore = E123,E125 enable-extensions=H106,H203 builtins = _ filename = *.py,app.wsgi -exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools/datasource-scaffold [hacking] local-check-factory = vitrage.hacking.checks.factory diff --git a/vitrage/datasources/sample b/vitrage/datasources/sample new file mode 120000 index 000000000..63bcf0ba7 --- /dev/null +++ b/vitrage/datasources/sample @@ -0,0 +1 @@ +tools/datasource-scaffold/sample \ No newline at end of file