x
/
fenix
Code Issues Proposed changes

Support download 

Add support for downloading software changes and plugins
Default download directory is /tmp/<session_id>
Download url with filename including 'workflow' or 'actions'
are automatically extracted to /tmp/<session_id>/<workflow|actions>
Plugins will be used straight from those directories is Fenix
did not have a plugin with the same name.
Download directory and content will be automatically removed
when session is deleted by the admin

story: 2004147
Task: #27620

Change-Id: Ia1cce13d268da1888f5d8f02d39099b3c113fb86
Signed-off-by: Tomi Juvonen <tomi.juvonen@nokia.com>
This commit is contained in:
Tomi Juvonen 2019-04-28 10:03:46 +03:00
parent 55d9de2b38
commit f189e9aeef
9 changed files with 251 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
fenix/db/api.py
View File

@ -137,6 +137,10 @@ def remove_action_plugin_instance(ap_instance):
return IMPL.remove_action_plugin_instance(ap_instance)
def create_downloads(download_dict_list):
return IMPL.create_downloads(download_dict_list)
def create_host(values):
"""Create a host from the values."""
return IMPL.create_host(values)

14
fenix/db/migration/alembic_migrations/versions/001_initial.py
View File

@ -128,6 +128,19 @@ def upgrade():
name='_session_plugin_instance_uc'),
sa.PrimaryKeyConstraint('id'))
op.create_table(
'downloads',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.String(36), primary_key=True,
default=_generate_unicode_uuid),
sa.Column('session_id', sa.String(36),
sa.ForeignKey('sessions.session_id')),
sa.Column('local_file', sa.String(160), nullable=False,),
sa.UniqueConstraint('session_id', 'local_file',
name='_session_local_file_uc'),
sa.PrimaryKeyConstraint('id'))
def downgrade():
op.drop_table('sessions')
@ -135,3 +148,4 @@ def downgrade():
op.drop_table('projects')
op.drop_table('instances')
op.drop_table('action_plugins')
op.drop_table('downloads')

43
fenix/db/sqlalchemy/api.py
View File

@ -56,6 +56,7 @@ def setup_db():
models.MaintenanceSession.metadata.create_all(engine)
models.MaintenanceActionPlugin.metadata.create_all(engine)
models.MaintenanceActionPluginInstance.metadata.create_all(engine)
models.MaintenanceDownload.metadata.create_all(engine)
models.MaintenanceHost.metadata.create_all(engine)
models.MaintenanceProject.metadata.create_all(engine)
models.MaintenanceInstance.metadata.create_all(engine)
@ -160,6 +161,11 @@ def remove_session(session_id):
for action in action_plugins:
session.delete(action)
downloads = _download_get_all(session, session_id)
if downloads:
for download in downloads:
download.delete(download)
hosts = _hosts_get(session, session_id)
if hosts:
for host in hosts:
@ -283,6 +289,43 @@ def remove_action_plugin_instance(ap_instance):
session.delete(ap_instance)
# Download
def _download_get(session, session_id, local_file):
query = model_query(models.MaintenanceDownload, session)
return query.filter_by(session_id=session_id,
local_file=local_file).first()
def download_get(session_id, plugin, download):
return _download_get(get_session(), session_id, download)
def _download_get_all(session, session_id):
query = model_query(models.MaintenanceDownload, session)
return query.filter_by(session_id=session_id).all()
def download_get_all(session_id):
return _download_get_all(get_session(), session_id)
def create_downloads(values_list):
for values in values_list:
vals = values.copy()
session = get_session()
with session.begin():
d = models.MaintenanceDownload()
d.update(vals)
try:
d.save(session=session)
except common_db_exc.DBDuplicateEntry as e:
# raise exception about duplicated columns (e.columns)
raise db_exc.FenixDBDuplicateEntry(
model=d.__class__.__name__, columns=e.columns)
return action_plugins_get_all(d.session_id)
# Host
def _host_get(session, session_id, hostname):
query = model_query(models.MaintenanceHost, session)

14
fenix/db/sqlalchemy/models.py
View File

@ -140,3 +140,17 @@ class MaintenanceInstance(mb.FenixBase):
def to_dict(self):
return super(MaintenanceInstance, self).to_dict()
class MaintenanceDownload(mb.FenixBase):
"""Maintenance project"""
__tablename__ = 'downloads'
id = _id_column()
session_id = sa.Column(sa.String(36), sa.ForeignKey('sessions.session_id'),
nullable=False)
local_file = sa.Column(sa.String(length=160), primary_key=True)
def to_dict(self):
return super(MaintenanceDownload, self).to_dict()

3
fenix/engine/__init__.py
View File

@ -24,6 +24,9 @@ opts = [
cfg.StrOpt('host',
default="127.0.0.1",
help="API host IP"),
cfg.StrOpt('local_cache_dir',
default="/tmp",
help="Local cache directory")
]
CONF = cfg.CONF

68
fenix/utils/download.py Normal file
View File

@ -0,0 +1,68 @@
# Copyright (c) 2019 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import requests
import tarfile
SUPPORTED_PACKAGE = {'.tar.gz': 'r:gz', '.tar.bz2': 'r:bz2', '.tar': 'r:'}
def check_url(url):
h = requests.head(url, allow_redirects=True)
header = h.headers
content_type = header.get('content-type')
if 'text' in content_type.lower():
return False
if 'html' in content_type.lower():
return False
return True
def url_to_filename(target_dir, url):
if url.find('/'):
return target_dir + '/' + url.rsplit('/', 1)[1]
else:
return None
def extract_plugin(target_dir, fname):
dest_dir = None
if "workflow" in fname.lower():
dest_dir = target_dir + '/workflow'
elif "action" in fname.lower():
dest_dir = target_dir + '/actions'
if dest_dir is not None:
if not os.path.isdir(dest_dir):
os.mkdir(dest_dir)
for postfix in SUPPORTED_PACKAGE:
if (fname.endswith(postfix)):
tar = tarfile.open(fname, SUPPORTED_PACKAGE[postfix])
tar.extractall(path=dest_dir)
tar.close()
break
def download_url(target_dir, url):
if not check_url(url):
raise Exception("%s is not downloadable file" % url)
fname = url_to_filename(target_dir, url)
if fname is None:
raise Exception("cannot make filename from: %s " % url)
r = requests.get(url, allow_redirects=True)
if not os.path.isdir(target_dir):
os.mkdir(target_dir)
open(fname, 'wb').write(r.content)
extract_plugin(target_dir, fname)

52
fenix/utils/service.py
View File

@ -14,14 +14,29 @@
# under the License.
from importlib import import_module
try:
from importlib.machinery import SourceFileLoader
def source_loader_workflow_instance(mname, mpath, conf, session_id, data):
mi = SourceFileLoader(mname, mpath).load_module()
return mi.Workflow(conf, session_id, data)
except ImportError:
from imp import load_source
def source_loader_workflow_instance(mname, mpath, conf, session_id, data):
mi = load_source(mname, mpath)
return mi.Workflow(conf, session_id, data)
import os
from oslo_config import cfg
from oslo_log import log as logging
import oslo_messaging as messaging
from oslo_service import service
from shutil import rmtree
from uuid import uuid1 as generate_uuid
from fenix import context
from fenix.utils.download import download_url
MAX_SESSIONS = 3
@ -51,6 +66,9 @@ opts = [
cfg.IntOpt('project_scale_in_reply',
default=60,
help="Project scale in reply confirmation time in seconds"),
cfg.StrOpt('local_cache_dir',
default="/tmp",
help="Local cache directory"),
]
CONF.register_opts(opts)
@ -100,11 +118,33 @@ class EngineEndpoint(object):
else:
workflow = "fenix.workflow.workflows.%s" % data["workflow"]
LOG.info("Workflow plugin module: %s" % workflow)
wf_plugin = getattr(import_module(workflow), 'Workflow')
self.workflow_sessions[session_id] = (
wf_plugin(CONF,
session_id,
data))
session_dir = "%s/%s" % (CONF.local_cache_dir, session_id)
os.mkdir(session_dir)
if "download" in data:
os.mkdir(session_dir + "/workflow")
os.mkdir(session_dir + "/actions")
for url in data["download"]:
download_url(session_dir, str(url))
try:
wf_plugin = getattr(import_module(workflow), 'Workflow')
self.workflow_sessions[session_id] = wf_plugin(CONF,
session_id,
data)
except ImportError:
download_plugin_dir = session_dir + "/workflow/"
download_plugin_file = "%s/%s.py" % (download_plugin_dir,
data["workflow"])
if os.path.isfile(download_plugin_file):
self.workflow_sessions[session_id] = (
source_loader_workflow_instance(workflow,
download_plugin_file,
CONF,
session_id,
data))
else:
raise Exception('%s: could not find workflow plugin %s' %
(self.session_id, data["workflow"]))
self.workflow_sessions[session_id].start()
return {'session_id': session_id}
@ -122,6 +162,8 @@ class EngineEndpoint(object):
self.workflow_sessions[session_id].cleanup()
self.workflow_sessions[session_id].stop()
self.workflow_sessions.pop(session_id)
session_dir = "%s/%s" % (CONF.local_cache_dir, session_id)
rmtree(session_dir)
return {}
def admin_update_session(self, ctx, session_id):

21
fenix/workflow/workflow.py
View File

@ -23,6 +23,7 @@ from threading import Thread
import time
from fenix.db import api as db_api
from fenix.utils.download import url_to_filename
from fenix.utils.identity_auth import get_identity_auth
from fenix.utils.identity_auth import get_session
@ -50,6 +51,11 @@ class BaseWorkflow(Thread):
self.actions = self._init_action_plugins(data["actions"])
else:
self.actions = []
if "download" in data:
self.downloads = self._init_downloads(data["download"])
else:
self.downloads = []
self.projects = []
self.instances = []
self.proj_instance_actions = {}
@ -98,6 +104,21 @@ class BaseWorkflow(Thread):
else:
return data
def _init_downloads(self, download_urls):
downloads = []
session_dir = "%s/%s" % (self.conf.engine.local_cache_dir,
self.session_id)
for url in download_urls:
LOG.info('%s: Download %s' % (self.session_id, url))
local_file = url_to_filename(session_dir, url)
if local_file is None:
raise Exception("cannot make filename from: %s " % url)
ddict = {
'session_id': self.session_id,
'local_file': local_file}
downloads.append(ddict)
return db_api.create_downloads(downloads)
def _init_session(self, data):
session = {
'session_id': self.session_id,

39
fenix/workflow/workflows/default.py
View File

@ -14,11 +14,26 @@
# under the License.
import datetime
from importlib import import_module
try:
from importlib.machinery import SourceFileLoader
def mod_loader_action_instance(mname, mpath, session_instance,
ap_db_instance):
mi = SourceFileLoader(mname, mpath).load_module()
return mi.ActionPlugin(session_instance, ap_db_instance)
except ImportError:
from imp import load_source
def mod_loader_action_instance(mname, mpath, session_instance,
ap_db_instance):
mi = load_source(mname, mpath)
return mi.ActionPlugin(session_instance, ap_db_instance)
from novaclient import API_MAX_VERSION as nova_max_version
import novaclient.client as novaclient
from novaclient.exceptions import BadRequest
import os
from oslo_log import log as logging
import time
@ -642,6 +657,9 @@ class Workflow(BaseWorkflow):
def maintenance_by_plugin_type(self, hostname, plugin_type):
aps = self.get_action_plugins_by_type(plugin_type)
session_dir = "%s/%s" % (self.conf.engine.local_cache_dir,
self.session_id)
download_plugin_dir = session_dir + "/actions/"
if aps:
LOG.info("%s: Calling action plug-ins with type %s" %
(self.session_id, plugin_type))
@ -649,10 +667,27 @@ class Workflow(BaseWorkflow):
ap_name = "fenix.workflow.actions.%s" % ap.plugin
LOG.info("%s: Calling action plug-in module: %s" %
(self.session_id, ap_name))
action_plugin = getattr(import_module(ap_name), 'ActionPlugin')
ap_db_instance = self._create_action_plugin_instance(ap.plugin,
hostname)
ap_instance = action_plugin(self, ap_db_instance)
try:
action_plugin = getattr(import_module(ap_name),
'ActionPlugin')
ap_instance = action_plugin(self, ap_db_instance)
except ImportError:
download_plugin_file = "%s/%s.py" % (download_plugin_dir,
ap.plugin)
LOG.info("%s: Trying from: %s" % (self.session_id,
download_plugin_file))
if os.path.isfile(download_plugin_file):
ap_instance = (
mod_loader_action_instance(ap_name,
download_plugin_file,
self,
ap_db_instance))
else:
raise Exception('%s: could not find action plugin %s' %
(self.session_id, ap.plugin))
ap_instance.run()
if ap_db_instance.state:
LOG.info('%s: %s finished with %s host %s' %