Support download
Add support for downloading software changes and plugins Default download directory is /tmp/<session_id> Download url with filename including 'workflow' or 'actions' are automatically extracted to /tmp/<session_id>/<workflow|actions> Plugins will be used straight from those directories is Fenix did not have a plugin with the same name. Download directory and content will be automatically removed when session is deleted by the admin story: 2004147 Task: #27620 Change-Id: Ia1cce13d268da1888f5d8f02d39099b3c113fb86 Signed-off-by: Tomi Juvonen <tomi.juvonen@nokia.com>
This commit is contained in:
parent
8a375d8417
commit
24bf1a2563
|
@ -137,6 +137,10 @@ def remove_action_plugin_instance(ap_instance):
|
|||
return IMPL.remove_action_plugin_instance(ap_instance)
|
||||
|
||||
|
||||
def create_downloads(download_dict_list):
|
||||
return IMPL.create_downloads(download_dict_list)
|
||||
|
||||
|
||||
def create_host(values):
|
||||
"""Create a host from the values."""
|
||||
return IMPL.create_host(values)
|
||||
|
|
|
@ -35,7 +35,7 @@ def MediumText():
|
|||
def upgrade():
|
||||
op.create_table(
|
||||
'sessions',
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('session_id', sa.String(36), primary_key=True),
|
||||
sa.Column('state', sa.String(length=32), nullable=True),
|
||||
|
@ -46,7 +46,7 @@ def upgrade():
|
|||
|
||||
op.create_table(
|
||||
'hosts',
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('id', sa.String(36), primary_key=True,
|
||||
default=_generate_unicode_uuid),
|
||||
|
@ -64,7 +64,7 @@ def upgrade():
|
|||
|
||||
op.create_table(
|
||||
'projects',
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('id', sa.String(36), primary_key=True,
|
||||
default=_generate_unicode_uuid),
|
||||
|
@ -78,7 +78,7 @@ def upgrade():
|
|||
|
||||
op.create_table(
|
||||
'instances',
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('id', sa.String(36), primary_key=True,
|
||||
default=_generate_unicode_uuid),
|
||||
|
@ -101,7 +101,7 @@ def upgrade():
|
|||
|
||||
op.create_table(
|
||||
'action_plugins',
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('id', sa.String(36), primary_key=True,
|
||||
default=_generate_unicode_uuid),
|
||||
|
@ -115,7 +115,7 @@ def upgrade():
|
|||
|
||||
op.create_table(
|
||||
'action_plugin_instances',
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('id', sa.String(36), primary_key=True,
|
||||
default=_generate_unicode_uuid),
|
||||
|
@ -128,6 +128,19 @@ def upgrade():
|
|||
name='_session_plugin_instance_uc'),
|
||||
sa.PrimaryKeyConstraint('id'))
|
||||
|
||||
op.create_table(
|
||||
'downloads',
|
||||
sa.Column('created_at', sa.DateTime(), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('id', sa.String(36), primary_key=True,
|
||||
default=_generate_unicode_uuid),
|
||||
sa.Column('session_id', sa.String(36),
|
||||
sa.ForeignKey('sessions.session_id')),
|
||||
sa.Column('local_file', sa.String(160), nullable=False,),
|
||||
sa.UniqueConstraint('session_id', 'local_file',
|
||||
name='_session_local_file_uc'),
|
||||
sa.PrimaryKeyConstraint('id'))
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table('sessions')
|
||||
|
@ -135,3 +148,4 @@ def downgrade():
|
|||
op.drop_table('projects')
|
||||
op.drop_table('instances')
|
||||
op.drop_table('action_plugins')
|
||||
op.drop_table('downloads')
|
||||
|
|
|
@ -56,6 +56,7 @@ def setup_db():
|
|||
models.MaintenanceSession.metadata.create_all(engine)
|
||||
models.MaintenanceActionPlugin.metadata.create_all(engine)
|
||||
models.MaintenanceActionPluginInstance.metadata.create_all(engine)
|
||||
models.MaintenanceDownload.metadata.create_all(engine)
|
||||
models.MaintenanceHost.metadata.create_all(engine)
|
||||
models.MaintenanceProject.metadata.create_all(engine)
|
||||
models.MaintenanceInstance.metadata.create_all(engine)
|
||||
|
@ -160,6 +161,11 @@ def remove_session(session_id):
|
|||
for action in action_plugins:
|
||||
session.delete(action)
|
||||
|
||||
downloads = _download_get_all(session, session_id)
|
||||
if downloads:
|
||||
for download in downloads:
|
||||
download.delete(download)
|
||||
|
||||
hosts = _hosts_get(session, session_id)
|
||||
if hosts:
|
||||
for host in hosts:
|
||||
|
@ -283,6 +289,43 @@ def remove_action_plugin_instance(ap_instance):
|
|||
session.delete(ap_instance)
|
||||
|
||||
|
||||
# Download
|
||||
def _download_get(session, session_id, local_file):
|
||||
query = model_query(models.MaintenanceDownload, session)
|
||||
return query.filter_by(session_id=session_id,
|
||||
local_file=local_file).first()
|
||||
|
||||
|
||||
def download_get(session_id, plugin, download):
|
||||
return _download_get(get_session(), session_id, download)
|
||||
|
||||
|
||||
def _download_get_all(session, session_id):
|
||||
query = model_query(models.MaintenanceDownload, session)
|
||||
return query.filter_by(session_id=session_id).all()
|
||||
|
||||
|
||||
def download_get_all(session_id):
|
||||
return _download_get_all(get_session(), session_id)
|
||||
|
||||
|
||||
def create_downloads(values_list):
|
||||
for values in values_list:
|
||||
vals = values.copy()
|
||||
session = get_session()
|
||||
with session.begin():
|
||||
d = models.MaintenanceDownload()
|
||||
d.update(vals)
|
||||
try:
|
||||
d.save(session=session)
|
||||
except common_db_exc.DBDuplicateEntry as e:
|
||||
# raise exception about duplicated columns (e.columns)
|
||||
raise db_exc.FenixDBDuplicateEntry(
|
||||
model=d.__class__.__name__, columns=e.columns)
|
||||
|
||||
return action_plugins_get_all(d.session_id)
|
||||
|
||||
|
||||
# Host
|
||||
def _host_get(session, session_id, hostname):
|
||||
query = model_query(models.MaintenanceHost, session)
|
||||
|
|
|
@ -140,3 +140,17 @@ class MaintenanceInstance(mb.FenixBase):
|
|||
|
||||
def to_dict(self):
|
||||
return super(MaintenanceInstance, self).to_dict()
|
||||
|
||||
|
||||
class MaintenanceDownload(mb.FenixBase):
|
||||
"""Maintenance project"""
|
||||
|
||||
__tablename__ = 'downloads'
|
||||
|
||||
id = _id_column()
|
||||
session_id = sa.Column(sa.String(36), sa.ForeignKey('sessions.session_id'),
|
||||
nullable=False)
|
||||
local_file = sa.Column(sa.String(length=160), primary_key=True)
|
||||
|
||||
def to_dict(self):
|
||||
return super(MaintenanceDownload, self).to_dict()
|
||||
|
|
|
@ -24,6 +24,9 @@ opts = [
|
|||
cfg.StrOpt('host',
|
||||
default="127.0.0.1",
|
||||
help="API host IP"),
|
||||
cfg.StrOpt('local_cache_dir',
|
||||
default="/tmp",
|
||||
help="Local cache directory")
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
|
|
@ -0,0 +1,66 @@
|
|||
# Copyright (c) 2019 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import os
|
||||
import requests
|
||||
import tarfile
|
||||
|
||||
|
||||
SUPPORTED_PACKAGE = {'.tar.gz': 'r:gz', '.tar.bz2': 'r:bz2', '.tar': 'r:'}
|
||||
|
||||
|
||||
def check_url(url):
|
||||
h = requests.head(url, allow_redirects=True)
|
||||
header = h.headers
|
||||
content_type = header.get('content-type')
|
||||
if any(ft in content_type.lower() for ft in ['text', 'html']):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def url_to_filename(target_dir, url):
|
||||
if url.find('/'):
|
||||
return target_dir + '/' + url.rsplit('/', 1)[1]
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def extract_plugin(target_dir, fname):
|
||||
dest_dir = None
|
||||
if "workflow" in fname.lower():
|
||||
dest_dir = target_dir + '/workflow'
|
||||
elif "action" in fname.lower():
|
||||
dest_dir = target_dir + '/actions'
|
||||
if dest_dir is not None:
|
||||
if not os.path.isdir(dest_dir):
|
||||
os.mkdir(dest_dir)
|
||||
for postfix in SUPPORTED_PACKAGE:
|
||||
if (fname.endswith(postfix)):
|
||||
tar = tarfile.open(fname, SUPPORTED_PACKAGE[postfix])
|
||||
tar.extractall(path=dest_dir)
|
||||
tar.close()
|
||||
break
|
||||
|
||||
|
||||
def download_url(target_dir, url):
|
||||
if not check_url(url):
|
||||
raise Exception("%s is not a downloadable file" % url)
|
||||
fname = url_to_filename(target_dir, url)
|
||||
if fname is None:
|
||||
raise Exception("Cannot make filename from: %s " % url)
|
||||
r = requests.get(url, allow_redirects=True)
|
||||
if not os.path.isdir(target_dir):
|
||||
os.mkdir(target_dir)
|
||||
open(fname, 'wb').write(r.content)
|
||||
extract_plugin(target_dir, fname)
|
|
@ -14,14 +14,29 @@
|
|||
# under the License.
|
||||
|
||||
from importlib import import_module
|
||||
try:
|
||||
from importlib.machinery import SourceFileLoader
|
||||
|
||||
def source_loader_workflow_instance(mname, mpath, conf, session_id, data):
|
||||
mi = SourceFileLoader(mname, mpath).load_module()
|
||||
return mi.Workflow(conf, session_id, data)
|
||||
except ImportError:
|
||||
from imp import load_source
|
||||
|
||||
def source_loader_workflow_instance(mname, mpath, conf, session_id, data):
|
||||
mi = load_source(mname, mpath)
|
||||
return mi.Workflow(conf, session_id, data)
|
||||
|
||||
import os
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log as logging
|
||||
import oslo_messaging as messaging
|
||||
from oslo_service import service
|
||||
from shutil import rmtree
|
||||
from uuid import uuid1 as generate_uuid
|
||||
|
||||
from fenix import context
|
||||
from fenix.utils.download import download_url
|
||||
|
||||
MAX_SESSIONS = 3
|
||||
|
||||
|
@ -51,6 +66,9 @@ opts = [
|
|||
cfg.IntOpt('project_scale_in_reply',
|
||||
default=60,
|
||||
help="Project scale in reply confirmation time in seconds"),
|
||||
cfg.StrOpt('local_cache_dir',
|
||||
default="/tmp",
|
||||
help="Local cache directory"),
|
||||
]
|
||||
|
||||
CONF.register_opts(opts)
|
||||
|
@ -100,11 +118,33 @@ class EngineEndpoint(object):
|
|||
else:
|
||||
workflow = "fenix.workflow.workflows.%s" % data["workflow"]
|
||||
LOG.info("Workflow plugin module: %s" % workflow)
|
||||
wf_plugin = getattr(import_module(workflow), 'Workflow')
|
||||
self.workflow_sessions[session_id] = (
|
||||
wf_plugin(CONF,
|
||||
session_id,
|
||||
data))
|
||||
session_dir = "%s/%s" % (CONF.local_cache_dir, session_id)
|
||||
os.mkdir(session_dir)
|
||||
if "download" in data:
|
||||
os.mkdir(session_dir + "/workflow")
|
||||
os.mkdir(session_dir + "/actions")
|
||||
for url in data["download"]:
|
||||
download_url(session_dir, str(url))
|
||||
try:
|
||||
wf_plugin = getattr(import_module(workflow), 'Workflow')
|
||||
self.workflow_sessions[session_id] = wf_plugin(CONF,
|
||||
session_id,
|
||||
data)
|
||||
except ImportError:
|
||||
download_plugin_dir = session_dir + "/workflow/"
|
||||
download_plugin_file = "%s/%s.py" % (download_plugin_dir,
|
||||
data["workflow"])
|
||||
if os.path.isfile(download_plugin_file):
|
||||
self.workflow_sessions[session_id] = (
|
||||
source_loader_workflow_instance(workflow,
|
||||
download_plugin_file,
|
||||
CONF,
|
||||
session_id,
|
||||
data))
|
||||
else:
|
||||
raise Exception('%s: could not find workflow plugin %s' %
|
||||
(self.session_id, data["workflow"]))
|
||||
|
||||
self.workflow_sessions[session_id].start()
|
||||
return {'session_id': session_id}
|
||||
|
||||
|
@ -122,6 +162,8 @@ class EngineEndpoint(object):
|
|||
self.workflow_sessions[session_id].cleanup()
|
||||
self.workflow_sessions[session_id].stop()
|
||||
self.workflow_sessions.pop(session_id)
|
||||
session_dir = "%s/%s" % (CONF.local_cache_dir, session_id)
|
||||
rmtree(session_dir)
|
||||
return {}
|
||||
|
||||
def admin_update_session(self, ctx, session_id):
|
||||
|
|
|
@ -23,6 +23,7 @@ from threading import Thread
|
|||
import time
|
||||
|
||||
from fenix.db import api as db_api
|
||||
from fenix.utils.download import url_to_filename
|
||||
from fenix.utils.identity_auth import get_identity_auth
|
||||
from fenix.utils.identity_auth import get_session
|
||||
|
||||
|
@ -50,6 +51,11 @@ class BaseWorkflow(Thread):
|
|||
self.actions = self._init_action_plugins(data["actions"])
|
||||
else:
|
||||
self.actions = []
|
||||
if "download" in data:
|
||||
self.downloads = self._init_downloads(data["download"])
|
||||
else:
|
||||
self.downloads = []
|
||||
|
||||
self.projects = []
|
||||
self.instances = []
|
||||
self.proj_instance_actions = {}
|
||||
|
@ -98,6 +104,21 @@ class BaseWorkflow(Thread):
|
|||
else:
|
||||
return data
|
||||
|
||||
def _init_downloads(self, download_urls):
|
||||
downloads = []
|
||||
session_dir = "%s/%s" % (self.conf.engine.local_cache_dir,
|
||||
self.session_id)
|
||||
for url in download_urls:
|
||||
LOG.info('%s: Download %s' % (self.session_id, url))
|
||||
local_file = url_to_filename(session_dir, url)
|
||||
if local_file is None:
|
||||
raise Exception("cannot make filename from: %s " % url)
|
||||
ddict = {
|
||||
'session_id': self.session_id,
|
||||
'local_file': local_file}
|
||||
downloads.append(ddict)
|
||||
return db_api.create_downloads(downloads)
|
||||
|
||||
def _init_session(self, data):
|
||||
session = {
|
||||
'session_id': self.session_id,
|
||||
|
|
|
@ -14,11 +14,26 @@
|
|||
# under the License.
|
||||
import datetime
|
||||
from importlib import import_module
|
||||
try:
|
||||
from importlib.machinery import SourceFileLoader
|
||||
|
||||
def mod_loader_action_instance(mname, mpath, session_instance,
|
||||
ap_db_instance):
|
||||
mi = SourceFileLoader(mname, mpath).load_module()
|
||||
return mi.ActionPlugin(session_instance, ap_db_instance)
|
||||
except ImportError:
|
||||
from imp import load_source
|
||||
|
||||
def mod_loader_action_instance(mname, mpath, session_instance,
|
||||
ap_db_instance):
|
||||
mi = load_source(mname, mpath)
|
||||
return mi.ActionPlugin(session_instance, ap_db_instance)
|
||||
|
||||
from novaclient import API_MAX_VERSION as nova_max_version
|
||||
import novaclient.client as novaclient
|
||||
from novaclient.exceptions import BadRequest
|
||||
|
||||
import os
|
||||
from oslo_log import log as logging
|
||||
import time
|
||||
|
||||
|
@ -642,6 +657,9 @@ class Workflow(BaseWorkflow):
|
|||
|
||||
def maintenance_by_plugin_type(self, hostname, plugin_type):
|
||||
aps = self.get_action_plugins_by_type(plugin_type)
|
||||
session_dir = "%s/%s" % (self.conf.engine.local_cache_dir,
|
||||
self.session_id)
|
||||
download_plugin_dir = session_dir + "/actions/"
|
||||
if aps:
|
||||
LOG.info("%s: Calling action plug-ins with type %s" %
|
||||
(self.session_id, plugin_type))
|
||||
|
@ -649,10 +667,27 @@ class Workflow(BaseWorkflow):
|
|||
ap_name = "fenix.workflow.actions.%s" % ap.plugin
|
||||
LOG.info("%s: Calling action plug-in module: %s" %
|
||||
(self.session_id, ap_name))
|
||||
action_plugin = getattr(import_module(ap_name), 'ActionPlugin')
|
||||
ap_db_instance = self._create_action_plugin_instance(ap.plugin,
|
||||
hostname)
|
||||
ap_instance = action_plugin(self, ap_db_instance)
|
||||
try:
|
||||
action_plugin = getattr(import_module(ap_name),
|
||||
'ActionPlugin')
|
||||
ap_instance = action_plugin(self, ap_db_instance)
|
||||
except ImportError:
|
||||
download_plugin_file = "%s/%s.py" % (download_plugin_dir,
|
||||
ap.plugin)
|
||||
LOG.info("%s: Trying from: %s" % (self.session_id,
|
||||
download_plugin_file))
|
||||
if os.path.isfile(download_plugin_file):
|
||||
ap_instance = (
|
||||
mod_loader_action_instance(ap_name,
|
||||
download_plugin_file,
|
||||
self,
|
||||
ap_db_instance))
|
||||
else:
|
||||
raise Exception('%s: could not find action plugin %s' %
|
||||
(self.session_id, ap.plugin))
|
||||
|
||||
ap_instance.run()
|
||||
if ap_db_instance.state:
|
||||
LOG.info('%s: %s finished with %s host %s' %
|
||||
|
|
Loading…
Reference in New Issue