From 45254745c330b85cafb5a2c88c25ef034a1a269e Mon Sep 17 00:00:00 2001 From: Tomi Juvonen Date: Tue, 29 Jan 2019 08:50:00 +0200 Subject: [PATCH] Support for action plug-in Add ability to give action plug-ins in session creation. Add handling for action-plug-ins in defautl workflow Add database support Dummy action plug-in added to see how it works. Will be removed before this merges. Documentation will be updated as review done Story: 2003846 Task: #29160 Change-Id: I56c77df4937c16f419b6d963f5a5fa0642fc0d43 Signed-off-by: Tomi Juvonen --- fenix/db/api.py | 16 +- .../versions/001_initial.py | 24 ++- fenix/db/sqlalchemy/api.py | 145 +++++++++++++----- fenix/db/sqlalchemy/models.py | 27 +++- fenix/workflow/actions/__init__.py | 0 fenix/workflow/actions/dummy.py | 31 ++++ fenix/workflow/workflow.py | 35 ++++- fenix/workflow/workflows/default.py | 44 +++++- 8 files changed, 263 insertions(+), 59 deletions(-) create mode 100644 fenix/workflow/actions/__init__.py create mode 100644 fenix/workflow/actions/dummy.py diff --git a/fenix/db/api.py b/fenix/db/api.py index cc27005..b181229 100644 --- a/fenix/db/api.py +++ b/fenix/db/api.py @@ -120,9 +120,21 @@ def remove_session(session_id): return IMPL.remove_session(session_id) -def create_action(values): +def create_action_plugin(values): """Create a action from the values.""" - return IMPL.create_action(values) + return IMPL.create_action_plugin(values) + + +def create_action_plugins(session_id, action_dict_list): + return IMPL.create_action_plugins(action_dict_list) + + +def create_action_plugin_instance(values): + return IMPL.create_action_plugin_instance(values) + + +def remove_action_plugin_instance(ap_instance): + return IMPL.remove_action_plugin_instance(ap_instance) def create_host(values): diff --git a/fenix/db/migration/alembic_migrations/versions/001_initial.py b/fenix/db/migration/alembic_migrations/versions/001_initial.py index d41dfd9..6f49f58 100644 --- a/fenix/db/migration/alembic_migrations/versions/001_initial.py +++ b/fenix/db/migration/alembic_migrations/versions/001_initial.py @@ -13,9 +13,6 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -revision = '001' -down_revision = None - import uuid from alembic import op @@ -23,6 +20,9 @@ import six import sqlalchemy as sa from sqlalchemy.dialects.mysql import MEDIUMTEXT +revision = '001' +down_revision = None + def _generate_unicode_uuid(): return six.text_type(str(uuid.uuid4())) @@ -57,6 +57,8 @@ def upgrade(): sa.Column('maintained', sa.Boolean, default=False), sa.Column('disabled', sa.Boolean, default=False), sa.Column('details', sa.String(length=255), nullable=True), + sa.Column('plugin', sa.String(length=255), nullable=True), + sa.Column('plugin_state', sa.String(length=32), nullable=True), sa.UniqueConstraint('session_id', 'hostname', name='_session_host_uc'), sa.PrimaryKeyConstraint('id')) @@ -106,12 +108,26 @@ def upgrade(): sa.Column('session_id', sa.String(36), sa.ForeignKey('sessions.session_id')), sa.Column('plugin', sa.String(length=255), nullable=False), - sa.Column('state', sa.String(length=32), nullable=True), sa.Column('type', sa.String(length=32), nullable=True), sa.Column('meta', MediumText(), nullable=False), sa.UniqueConstraint('session_id', 'plugin', name='_session_plugin_uc'), sa.PrimaryKeyConstraint('id')) + op.create_table( + 'action_plugin_instances', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.String(36), primary_key=True, + default=_generate_unicode_uuid), + sa.Column('session_id', sa.String(36), + sa.ForeignKey('sessions.session_id')), + sa.Column('plugin', sa.String(length=255), nullable=False), + sa.Column('hostname', sa.String(length=255), nullable=False), + sa.Column('state', MediumText(), nullable=True), + sa.UniqueConstraint('session_id', 'plugin', 'hostname', + name='_session_plugin_instance_uc'), + sa.PrimaryKeyConstraint('id')) + def downgrade(): op.drop_table('sessions') diff --git a/fenix/db/sqlalchemy/api.py b/fenix/db/sqlalchemy/api.py index 67c2115..4bbc84c 100644 --- a/fenix/db/sqlalchemy/api.py +++ b/fenix/db/sqlalchemy/api.py @@ -54,7 +54,8 @@ def setup_db(): engine = db_session.EngineFacade(cfg.CONF.database.connection, sqlite_fk=True).get_engine() models.MaintenanceSession.metadata.create_all(engine) - models.MaintenanceAction.metadata.create_all(engine) + models.MaintenanceActionPlugin.metadata.create_all(engine) + models.MaintenanceActionPluginInstance.metadata.create_all(engine) models.MaintenanceHost.metadata.create_all(engine) models.MaintenanceProject.metadata.create_all(engine) models.MaintenanceInstance.metadata.create_all(engine) @@ -148,72 +149,138 @@ def remove_session(session_id): session = get_session() with session.begin(): + action_plugin_instances = _action_plugin_instances_get_all(session, + session_id) + if action_plugin_instances: + for action in action_plugin_instances: + session.delete(action) + + action_plugins = _action_plugins_get_all(session, session_id) + if action_plugins: + for action in action_plugins: + session.delete(action) + hosts = _hosts_get(session, session_id) - - if not hosts: - # raise not found error - raise db_exc.FenixDBNotFound(session, session_id=session_id, - model='hosts') - - for host in hosts: - session.delete(host) + if hosts: + for host in hosts: + session.delete(host) projects = _projects_get(session, session_id) - - if not projects: - # raise not found error - raise db_exc.FenixDBNotFound(session, session_id=session_id, - model='projects') - - for project in projects: - session.delete(project) + if projects: + for project in projects: + session.delete(project) instances = _instances_get(session, session_id) - - if not instances: - # raise not found error - raise db_exc.FenixDBNotFound(session, session_id=session_id, - model='instances') - - for instance in instances: - session.delete(instance) + if instances: + for instance in instances: + session.delete(instance) msession = _maintenance_session_get(session, session_id) - if not msession: # raise not found error raise db_exc.FenixDBNotFound(session, session_id=session_id, model='sessions') session.delete(msession) - # TBD Other tables content when implemented -# Action -def _action_get(session, session_id, plugin): - query = model_query(models.MaintenanceActions, session) +# Action Plugin +def _action_plugin_get(session, session_id, plugin): + query = model_query(models.MaintenanceActionPlugin, session) return query.filter_by(session_id=session_id, plugin=plugin).first() -def action_get(session_id, plugin): - return _action_get(get_session(), session_id, plugin) +def action_plugin_get(session_id, plugin): + return _action_plugin_get(get_session(), session_id, plugin) -def create_action(values): +def _action_plugins_get_all(session, session_id): + query = model_query(models.MaintenanceActionPlugin, session) + return query.filter_by(session_id=session_id).all() + + +def action_plugins_get_all(session_id): + return _action_plugins_get_all(get_session(), session_id) + + +def create_action_plugin(values): values = values.copy() - maction = models.MaintenanceActions() - maction.update(values) + ap = models.MaintenanceActionPlugin() + ap.update(values) session = get_session() with session.begin(): try: - maction.save(session=session) + ap.save(session=session) except common_db_exc.DBDuplicateEntry as e: # raise exception about duplicated columns (e.columns) raise db_exc.FenixDBDuplicateEntry( - model=maction.__class__.__name__, columns=e.columns) + model=ap.__class__.__name__, columns=e.columns) - return action_get(maction.session_id, maction.plugin) + return action_plugin_get(ap.session_id, ap.plugin) + + +def create_action_plugins(values_list): + for values in values_list: + vals = values.copy() + session = get_session() + with session.begin(): + ap = models.MaintenanceActionPlugin() + ap.update(vals) + try: + ap.save(session=session) + except common_db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise db_exc.FenixDBDuplicateEntry( + model=ap.__class__.__name__, columns=e.columns) + + return action_plugins_get_all(ap.session_id) + + +# Action Plugin Instance +def _action_plugin_instance_get(session, session_id, plugin, hostname): + query = model_query(models.MaintenanceActionPluginInstance, session) + return query.filter_by(session_id=session_id, plugin=plugin, + hostname=hostname).first() + + +def action_plugin_instance_get(session_id, plugin, hostname): + return _action_plugin_instance_get(get_session(), session_id, plugin, + hostname) + + +def _action_plugin_instances_get_all(session, session_id): + query = model_query(models.MaintenanceActionPluginInstance, session) + return query.filter_by(session_id=session_id).all() + + +def action_plugin_instances_get_all(session_id): + return _action_plugin_instances_get_all(get_session(), session_id) + + +def create_action_plugin_instance(values): + values = values.copy() + ap_instance = models.MaintenanceActionPluginInstance() + ap_instance.update(values) + + session = get_session() + with session.begin(): + try: + ap_instance.save(session=session) + except common_db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise db_exc.FenixDBDuplicateEntry( + model=ap_instance.__class__.__name__, columns=e.columns) + + return action_plugin_instance_get(ap_instance.session_id, + ap_instance.plugin, + ap_instance.hostname) + + +def remove_action_plugin_instance(ap_instance): + session = get_session() + with session.begin(): + session.delete(ap_instance) # Host @@ -386,6 +453,6 @@ def remove_instance(session_id, instance_id): if not minstance: # raise not found error raise db_exc.FenixDBNotFound(session, session_id=session_id, - model='sessions') + model='instances') session.delete(minstance) diff --git a/fenix/db/sqlalchemy/models.py b/fenix/db/sqlalchemy/models.py index d7e39dd..b0b9ec3 100644 --- a/fenix/db/sqlalchemy/models.py +++ b/fenix/db/sqlalchemy/models.py @@ -53,21 +53,36 @@ class MaintenanceSession(mb.FenixBase): return super(MaintenanceSession, self).to_dict() -class MaintenanceAction(mb.FenixBase): - """Maintenance action""" +class MaintenanceActionPlugin(mb.FenixBase): + """Maintenance action plugin""" - __tablename__ = 'actions' + __tablename__ = 'action_plugins' id = _id_column() session_id = sa.Column(sa.String(36), sa.ForeignKey('sessions.session_id'), nullable=False) plugin = sa.Column(sa.String(length=255), nullable=False) - state = sa.Column(sa.String(length=32), nullable=True) type = sa.Column(sa.String(length=32), nullable=True) meta = sa.Column(MediumText(), nullable=False) def to_dict(self): - return super(MaintenanceAction, self).to_dict() + return super(MaintenanceActionPlugin, self).to_dict() + + +class MaintenanceActionPluginInstance(mb.FenixBase): + """Maintenance action instance""" + + __tablename__ = 'action_plugin_instances' + + id = _id_column() + session_id = sa.Column(sa.String(36), sa.ForeignKey('sessions.session_id'), + nullable=False) + hostname = sa.Column(sa.String(255), nullable=False) + plugin = sa.Column(sa.String(255), nullable=False) + state = sa.Column(sa.String(length=32), nullable=True) + + def to_dict(self): + return super(MaintenanceActionPluginInstance, self).to_dict() class MaintenanceHost(mb.FenixBase): @@ -83,6 +98,8 @@ class MaintenanceHost(mb.FenixBase): maintained = sa.Column(sa.Boolean, default=False) disabled = sa.Column(sa.Boolean, default=False) details = sa.Column(sa.String(length=255), nullable=True) + plugin = sa.Column(sa.String(length=255), nullable=True) + plugin_state = sa.Column(sa.String(length=32), nullable=True) def to_dict(self): return super(MaintenanceHost, self).to_dict() diff --git a/fenix/workflow/actions/__init__.py b/fenix/workflow/actions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fenix/workflow/actions/dummy.py b/fenix/workflow/actions/dummy.py new file mode 100644 index 0000000..70e6afb --- /dev/null +++ b/fenix/workflow/actions/dummy.py @@ -0,0 +1,31 @@ +# Copyright (c) 2019 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_log import log as logging + +LOG = logging.getLogger(__name__) + + +class ActionPlugin(object): + + def __init__(self, wf, ap_dbi): + self.hostname = ap_dbi.hostname + self.wf = wf + self.ap_dbi = ap_dbi + LOG.info("%s: Dummy action plugin initialized" % self.wf.session_id) + + def run(self): + LOG.info("%s: Dummy action plugin run %s" % (self.wf.session_id, + self.hostname)) + self.ap_dbi.state = "DONE" diff --git a/fenix/workflow/workflow.py b/fenix/workflow/workflow.py index 92a9103..d38a511 100644 --- a/fenix/workflow/workflow.py +++ b/fenix/workflow/workflow.py @@ -40,15 +40,16 @@ class BaseWorkflow(Thread): self.thg = threadgroup.ThreadGroup() self.timer = {} self.session = self._init_session(data) - LOG.info('%s: session %s' % (self.session_id, self.session)) self.hosts = [] if "hosts" in data and data['hosts']: # Hosts given as input, not to be discovered in workflow self.hosts = self.init_hosts(self.convert(data['hosts'])) else: LOG.info('%s: No hosts as input' % self.session_id) - # TBD API to support action plugins - # self.actions = + if "actions" in data: + self.actions = self._init_action_plugins(data["actions"]) + else: + self.actions = [] self.projects = [] self.instances = [] self.proj_instance_actions = {} @@ -104,9 +105,35 @@ class BaseWorkflow(Thread): 'maintenance_at': str(data['maintenance_at']), 'meta': str(self.convert(data['metadata'])), 'workflow': self.convert((data['workflow']))} - LOG.info('%s: _init_session: %s' % (self.session_id, session)) + LOG.info('Initializing maintenance session: %s' % session) return db_api.create_session(session) + def _init_action_plugins(self, ap_list): + actions = [] + for action in ap_list: + adict = { + 'session_id': self.session_id, + 'plugin': str(action['plugin']), + 'type': str(action['type'])} + if 'metadata' in action: + adict['meta'] = str(self.convert(action['metadata'])) + actions.append(adict) + return db_api.create_action_plugins(self.session_id, actions) + + def _create_action_plugin_instance(self, plugin, hostname, state=None): + ap_instance = { + 'session_id': self.session_id, + 'plugin': plugin, + 'hostname': hostname, + 'state': state} + return db_api.create_action_plugin_instance(ap_instance) + + def get_action_plugins_by_type(self, ap_type): + aps = [ap for ap in self.actions if ap.type == ap_type] + if aps: + aps = sorted(aps, key=lambda k: k['plugin']) + return aps + def get_compute_hosts(self): return [host.hostname for host in self.hosts if host.type == 'compute'] diff --git a/fenix/workflow/workflows/default.py b/fenix/workflow/workflows/default.py index 2352b72..5bbfe79 100644 --- a/fenix/workflow/workflows/default.py +++ b/fenix/workflow/workflows/default.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. import datetime +from importlib import import_module from novaclient import API_MAX_VERSION as nova_max_version import novaclient.client as novaclient @@ -554,11 +555,44 @@ class Workflow(BaseWorkflow): (server_id, instance.state)) return False - def host_maintenance(self, host): - LOG.info('maintaining host %s' % host) - # TBD Here we should call maintenance plugin given in maintenance - # session creation - time.sleep(5) + def host_maintenance_by_plugin_type(self, hostname, plugin_type): + aps = self.get_action_plugins_by_type(plugin_type) + if aps: + LOG.info("%s: Calling action plug-ins with type %s" % + (self.session_id, plugin_type)) + for ap in aps: + ap_name = "fenix.workflow.actions.%s" % ap.plugin + LOG.info("%s: Calling action plug-in module: %s" % + (self.session_id, ap_name)) + action_plugin = getattr(import_module(ap_name), 'ActionPlugin') + ap_db_instance = self._create_action_plugin_instance(ap.plugin, + hostname) + ap_instance = action_plugin(self, ap_db_instance) + ap_instance.run() + if ap_db_instance.state: + LOG.info('%s: %s finished with %s host %s' % + (self.session_id, ap.plugin, + ap_db_instance.state, hostname)) + if 'FAILED' in ap_db_instance.state: + raise Exception('%s: %s finished with %s host %s' % + (self.session_id, ap.plugin, + ap_db_instance.state, hostname)) + else: + raise Exception('%s: %s reported no state for host %s' % + (self.session_id, ap.plugin, hostname)) + # If ap_db_instance failed, we keep it for state + db_api.remove_action_plugin_instance(ap_db_instance) + else: + LOG.info("%s: No action plug-ins with type %s" % + (self.session_id, plugin_type)) + + def host_maintenance(self, hostname): + host = self.get_host_by_name(hostname) + LOG.info('%s: Maintaining host %s' % (self.session_id, hostname)) + for plugin_type in ["host", host.type]: + self.host_maintenance_by_plugin_type(hostname, plugin_type) + LOG.info('%s: Maintaining host %s complete' % (self.session_id, + hostname)) def maintenance(self): LOG.info("%s: maintenance called" % self.session_id)