diff --git a/savanna/cli.py b/savanna/cli.py index 0fcedc64a4..36939dae1f 100644 --- a/savanna/cli.py +++ b/savanna/cli.py @@ -18,7 +18,7 @@ from flask import Flask from oslo.config import cfg from savanna.openstack.common import log from savanna.storage.defaults import setup_defaults -from savanna.storage.storage import setup_storage, DB +from savanna.storage.db import setup_storage, DB CONF = cfg.CONF LOG = log.getLogger(__name__) diff --git a/savanna/main.py b/savanna/main.py index 1437cbca0e..00119551bc 100644 --- a/savanna/main.py +++ b/savanna/main.py @@ -24,7 +24,7 @@ from savanna.api import v02 as api_v02 from savanna.middleware.auth_valid import filter_factory as auth_valid from savanna.utils.scheduler import setup_scheduler from savanna.utils.api import render -from savanna.storage.storage import setup_storage +from savanna.storage.db import setup_storage from savanna.openstack.common import log diff --git a/savanna/service/api.py b/savanna/service/api.py index ef013894d3..b03a1be451 100644 --- a/savanna/service/api.py +++ b/savanna/service/api.py @@ -17,12 +17,10 @@ import eventlet from oslo.config import cfg from flask import request -from savanna.storage.models import NodeTemplate, NodeType, NodeProcess, \ - NodeTemplateConfig, Cluster, ClusterNodeCount -from savanna.storage.storage import DB from savanna.utils.api import abort_and_log from savanna.service import cluster_ops from savanna.openstack.common import log as logging +import savanna.storage.storage as storage LOG = logging.getLogger(__name__) @@ -30,6 +28,105 @@ CONF = cfg.CONF CONF.import_opt('allow_cluster_ops', 'savanna.config') +## Node Template ops: + +def get_node_template(**args): + return _node_template(storage.get_node_template(**args)) + + +def get_node_templates(**args): + return [_node_template(tmpl) for tmpl + in storage.get_node_templates(**args)] + + +def create_node_template(values): + """ + Creates new node template from values dict + :param values: dict + :return: created node template resource + """ + values = values.pop('node_template') + + name = values.pop('name') + node_type_id = storage.get_node_type(name=values.pop('node_type')).id + # todo(slukjanov): take tenant_id from headers + tenant_id = "tenant-01" + flavor_id = values.pop('flavor_id') + + nt = storage.create_node_template(name, node_type_id, tenant_id, + flavor_id, values) + + return get_node_template(id=nt.id) + + +def terminate_node_template(**args): + return storage.terminate_node_template(**args) + + +## Cluster ops: + +def get_cluster(**args): + return _cluster(storage.get_cluster(**args)) + + +def get_clusters(**args): + return [_cluster(cluster) for cluster in + storage.get_clusters(**args)] + + +def create_cluster(values): + values = values.pop('cluster') + + name = values.pop('name') + base_image_id = values.pop('base_image_id') + # todo(slukjanov): take tenant_id from headers + tenant_id = "tenant-01" + templates = values.pop('node_templates') + + # todo(slukjanov): check that we can create objects in the specified tenant + + cluster = storage.create_cluster(name, base_image_id, tenant_id, templates) + + eventlet.spawn(_cluster_creation_job, request.headers, cluster.id) + + return get_cluster(id=cluster.id) + + +def _cluster_creation_job(headers, cluster_id): + cluster = storage.get_cluster(id=cluster_id) + LOG.debug("Starting cluster '%s' creation: %s", cluster_id, + _cluster(cluster).dict) + + if CONF.allow_cluster_ops: + cluster_ops.launch_cluster(headers, cluster) + else: + LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag") + + # update cluster status + storage.update_cluster_status('Active', id=cluster.id) + + +def terminate_cluster(**args): + cluster = storage.update_cluster_status('Stoping', **args) + + eventlet.spawn(_cluster_termination_job, request.headers, cluster.id) + + +def _cluster_termination_job(headers, cluster_id): + cluster = storage.get_cluster(id=cluster_id) + LOG.debug("Stoping cluster '%s' creation: %s", cluster_id, + _cluster(cluster).dict) + + if CONF.allow_cluster_ops: + cluster_ops.stop_cluster(headers, cluster) + else: + LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag") + + storage.terminate_cluster(id=cluster.id) + + +## Utils and DB object to Resource converters + def _clean_nones(obj): d_type = type(obj) if d_type is not dict or d_type is not list: @@ -99,56 +196,6 @@ def _node_template(nt): return Resource('node_template', d) -def _template_id_by_name(template): - return NodeTemplate.query.filter_by(name=template).first().id - - -def _type_id_by_name(_type): - return NodeType.query.filter_by(name=_type).first().id - - -def get_node_template(**args): - return _node_template(NodeTemplate.query.filter_by(**args).first()) - - -def get_node_templates(**args): - return [_node_template(tmpl) for tmpl - in NodeTemplate.query.filter_by(**args).all()] - - -def create_node_template(values): - """ - Creates new node template from values dict - :param values: dict - :return: created node template resource - """ - values = values.pop('node_template') - - name = values.pop('name') - node_type_id = _type_id_by_name(values.pop('node_type')) - # todo(slukjanov): take tenant_id from headers - tenant_id = "tenant-01" - flavor_id = values.pop('flavor_id') - - nt = NodeTemplate(name, node_type_id, tenant_id, flavor_id) - DB.session.add(nt) - for process_name in values: - process = NodeProcess.query.filter_by(name=process_name).first() - conf = values.get(process_name) - for prop in process.node_process_properties: - val = conf.get(prop.name, None) - if not val and prop.required: - if not prop.default: - raise RuntimeError('Template \'%s\', value missed ' - 'for required param: %s %s' - % (name, process.name, prop.name)) - val = prop.default - DB.session.add(NodeTemplateConfig(nt.id, prop.id, val)) - DB.session.commit() - - return get_node_template(id=nt.id) - - def _cluster(cluster): if not cluster: abort_and_log(404, 'Cluster not found') @@ -173,94 +220,3 @@ def _cluster(cluster): d['service_urls'][service.name] = service.url return Resource('cluster', d) - - -def get_cluster(**args): - return _cluster(Cluster.query.filter_by(**args).first()) - - -def get_clusters(**args): - return [_cluster(cluster) for cluster in - Cluster.query.filter_by(**args).all()] - - -def create_cluster(values): - values = values.pop('cluster') - - name = values.pop('name') - base_image_id = values.pop('base_image_id') - # todo(slukjanov): take tenant_id from headers - tenant_id = "tenant-01" - templates = values.pop('node_templates') - - # todo(slukjanov): check that we can create objects in the specified tenant - - cluster = Cluster(name, base_image_id, tenant_id) - DB.session.add(cluster) - for template in templates: - count = templates.get(template) - template_id = _template_id_by_name(template) - cnc = ClusterNodeCount(cluster.id, template_id, int(count)) - DB.session.add(cnc) - DB.session.commit() - - eventlet.spawn(_cluster_creation_job, request.headers, cluster.id) - - return get_cluster(id=cluster.id) - - -def _cluster_creation_job(headers, cluster_id): - cluster = Cluster.query.filter_by(id=cluster_id).first() - LOG.debug("Starting cluster '%s' creation: %s", cluster_id, - _cluster(cluster).dict) - - if CONF.allow_cluster_ops: - cluster_ops.launch_cluster(headers, cluster) - else: - LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag") - - # update cluster status - cluster = Cluster.query.filter_by(id=cluster.id).first() - cluster.status = 'Active' - DB.session.add(cluster) - DB.session.commit() - - -def terminate_cluster(**args): - # update cluster status - cluster = Cluster.query.filter_by(**args).first() - cluster.status = 'Stoping' - DB.session.add(cluster) - DB.session.commit() - - eventlet.spawn(_cluster_termination_job, request.headers, cluster.id) - - -def _cluster_termination_job(headers, cluster_id): - cluster = Cluster.query.filter_by(id=cluster_id).first() - LOG.debug("Stoping cluster '%s' creation: %s", cluster_id, - _cluster(cluster).dict) - - if CONF.allow_cluster_ops: - cluster_ops.stop_cluster(headers, cluster) - else: - LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag") - - DB.session.delete(cluster) - DB.session.commit() - - -def terminate_node_template(**args): - template = NodeTemplate.query.filter_by(**args).first() - if template: - if len(template.nodes): - abort_and_log(500, "There are active nodes created using " - "template '%s' you trying to terminate" - % args) - else: - DB.session.delete(template) - DB.session.commit() - - return True - else: - return False diff --git a/savanna/service/cluster_ops.py b/savanna/service/cluster_ops.py index d2cb1aecb3..a5484eef0d 100644 --- a/savanna/service/cluster_ops.py +++ b/savanna/service/cluster_ops.py @@ -20,7 +20,7 @@ from jinja2 import PackageLoader from paramiko import SSHClient, AutoAddPolicy from oslo.config import cfg from savanna.storage.models import Node, ServiceUrl -from savanna.storage.storage import DB +from savanna.storage.db import DB from savanna.utils.openstack.nova import novaclient from savanna.openstack.common import log as logging diff --git a/savanna/storage/db.py b/savanna/storage/db.py new file mode 100644 index 0000000000..21144bc90d --- /dev/null +++ b/savanna/storage/db.py @@ -0,0 +1,40 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flask.ext.sqlalchemy import SQLAlchemy +from oslo.config import cfg + +DB = SQLAlchemy() + +opts = [ + cfg.StrOpt('database_uri', + default='sqlite:////tmp/savanna.db', + help='URL for sqlalchemy database'), + cfg.BoolOpt('echo', + default=False, + help='Sqlalchemy echo') +] + +CONF = cfg.CONF +CONF.register_opts(opts, group='sqlalchemy') + + +def setup_storage(app): + app.config['SQLALCHEMY_DATABASE_URI'] = CONF.sqlalchemy.database_uri + app.config['SQLALCHEMY_ECHO'] = CONF.sqlalchemy.echo + + DB.app = app + DB.init_app(app) + DB.create_all() diff --git a/savanna/storage/defaults.py b/savanna/storage/defaults.py index f92836cf3e..c278576373 100644 --- a/savanna/storage/defaults.py +++ b/savanna/storage/defaults.py @@ -13,96 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from savanna.storage.models import NodeProcess, NodeProcessProperty, \ - NodeType, NodeTemplate, NodeTemplateConfig, Cluster, ClusterNodeCount -from savanna.storage.storage import DB +from savanna.storage.storage import create_node_type, \ + create_node_template, create_node_process from savanna.openstack.common import log as logging LOG = logging.getLogger(__name__) -def create_node_process(name, properties): - """ - Creates new node process and node process properties - :param name: process name - :param properties: array of triples (name, required, default) - :return: created node process - """ - process = NodeProcess(name) - DB.session.add(process) - DB.session.commit() - for p in properties: - prop = NodeProcessProperty(process.id, p[0], p[1], p[2]) - DB.session.add(prop) - DB.session.commit() - return process - - -def create_node_type(name, processes): - """ - Creates new node type using specified list of processes - :param name: - :param processes: - :return: - """ - node_type = NodeType(name) - node_type.processes = processes - DB.session.add(node_type) - DB.session.commit() - return node_type - - -def create_node_template(name, node_type_id, tenant_id, flavor_id, configs): - """ - Creates new node templates - :param name: template name - :param node_type_id: node type - :param tenant_id: tenant - :param flavor_id: flavor - :param configs: dict of process->property->value - :return: created node template - """ - node_template = NodeTemplate(name, node_type_id, tenant_id, flavor_id) - DB.session.add(node_template) - for process_name in configs: - process = NodeProcess.query.filter_by(name=process_name).first() - conf = configs.get(process_name) - for prop in process.node_process_properties: - val = conf.get(prop.name, None) - if not val and prop.required: - if not prop.default: - raise RuntimeError('Template \'%s\', value missed ' - 'for required param: %s %s' - % (name, process.name, prop.name)) - val = prop.default - DB.session.add(NodeTemplateConfig(node_template.id, prop.id, val)) - DB.session.commit() - - return node_template - - -def create_cluster(name, base_image_id, tenant_id, templates): - """ - Creates new cluster - :param name: cluster name - :param base_image_id: base image - :param tenant_id: tenant - :param templates: dict of template->count - :return: created cluster - """ - cluster = Cluster(name, base_image_id, tenant_id) - DB.session.add(cluster) - for template in templates: - count = templates.get(template) - cnc = ClusterNodeCount(cluster.id, - NodeTemplate.query.filter_by(name=template) - .first().id, int(count)) - DB.session.add(cnc) - DB.session.commit() - - return cluster - - def setup_defaults(reset_db=False, gen_templates=False): nt_jt_nn = None nt_jt = None diff --git a/savanna/storage/models.py b/savanna/storage/models.py index b396707d01..805ce031a5 100644 --- a/savanna/storage/models.py +++ b/savanna/storage/models.py @@ -15,7 +15,7 @@ from uuid import uuid4 -from savanna.storage.storage import DB +from savanna.storage.db import DB class NodeTemplate(DB.Model): diff --git a/savanna/storage/storage.py b/savanna/storage/storage.py index 21144bc90d..462d2f7fd8 100644 --- a/savanna/storage/storage.py +++ b/savanna/storage/storage.py @@ -13,28 +13,150 @@ # See the License for the specific language governing permissions and # limitations under the License. -from flask.ext.sqlalchemy import SQLAlchemy -from oslo.config import cfg +from savanna.storage.db import DB -DB = SQLAlchemy() - -opts = [ - cfg.StrOpt('database_uri', - default='sqlite:////tmp/savanna.db', - help='URL for sqlalchemy database'), - cfg.BoolOpt('echo', - default=False, - help='Sqlalchemy echo') -] - -CONF = cfg.CONF -CONF.register_opts(opts, group='sqlalchemy') +from savanna.storage.models import NodeTemplate, NodeProcess, Cluster, \ + ClusterNodeCount, NodeTemplateConfig, NodeType, NodeProcessProperty +from savanna.utils.api import abort_and_log -def setup_storage(app): - app.config['SQLALCHEMY_DATABASE_URI'] = CONF.sqlalchemy.database_uri - app.config['SQLALCHEMY_ECHO'] = CONF.sqlalchemy.echo +## Node Template ops: - DB.app = app - DB.init_app(app) - DB.create_all() +def get_node_template(**args): + return NodeTemplate.query.filter_by(**args).first() + + +def get_node_templates(**args): + return NodeTemplate.query.filter_by(**args).all() + + +def create_node_template(name, node_type_id, tenant_id, flavor_id, configs): + """ + Creates new node templates + :param name: template name + :param node_type_id: node type + :param tenant_id: tenant + :param flavor_id: flavor + :param configs: dict of process->property->value + :return: created node template + """ + node_template = NodeTemplate(name, node_type_id, tenant_id, flavor_id) + DB.session.add(node_template) + for process_name in configs: + process = NodeProcess.query.filter_by(name=process_name).first() + conf = configs.get(process_name) + for prop in process.node_process_properties: + val = conf.get(prop.name, None) + if not val and prop.required: + if not prop.default: + raise RuntimeError('Template \'%s\', value missed ' + 'for required param: %s %s' + % (name, process.name, prop.name)) + val = prop.default + DB.session.add(NodeTemplateConfig(node_template.id, prop.id, val)) + DB.session.commit() + + return node_template + + +def terminate_node_template(**args): + template = NodeTemplate.query.filter_by(**args).first() + if template: + if len(template.nodes): + abort_and_log(500, "There are active nodes created using " + "template '%s' you trying to terminate" + % args) + else: + DB.session.delete(template) + DB.session.commit() + + return True + else: + return False + + +## Cluster ops: + +def get_cluster(**args): + return Cluster.query.filter_by(**args).first() + + +def get_clusters(**args): + return Cluster.query.filter_by(**args).all() + + +def create_cluster(name, base_image_id, tenant_id, templates): + """ + Creates new cluster + :param name: cluster name + :param base_image_id: base image + :param tenant_id: tenant + :param templates: dict of template->count + :return: created cluster + """ + cluster = Cluster(name, base_image_id, tenant_id) + DB.session.add(cluster) + for template in templates: + count = templates.get(template) + template_id = get_node_template(name=template).id + cnc = ClusterNodeCount(cluster.id, template_id, int(count)) + DB.session.add(cnc) + DB.session.commit() + + return cluster + + +def terminate_cluster(**args): + cluster = get_cluster(**args) + DB.session.delete(cluster) + DB.session.commit() + + +def update_cluster_status(new_status, **args): + cluster = Cluster.query.filter_by(**args).first() + cluster.status = new_status + DB.session.add(cluster) + DB.session.commit() + + return cluster + + +## Node Process ops: + +def create_node_process(name, properties): + """ + Creates new node process and node process properties + :param name: process name + :param properties: array of triples (name, required, default) + :return: created node process + """ + process = NodeProcess(name) + DB.session.add(process) + DB.session.commit() + for p in properties: + prop = NodeProcessProperty(process.id, p[0], p[1], p[2]) + DB.session.add(prop) + DB.session.commit() + + return process + + +## Node Type ops: + +def get_node_type(**args): + return NodeType.query.filter_by(**args).first() + + +def create_node_type(name, processes): + """ + Creates new node type using specified list of processes + :param name: + :param processes: + :return: + """ + node_type = NodeType(name) + node_type.processes = processes + DB.session.add(node_type) + DB.session.commit() + + return node_type diff --git a/savanna/tests/unit/test_api_v02.py b/savanna/tests/unit/test_api_v02.py index fbbb789067..dfa511cf22 100644 --- a/savanna/tests/unit/test_api_v02.py +++ b/savanna/tests/unit/test_api_v02.py @@ -26,7 +26,7 @@ from savanna.main import make_app from savanna.service import api from savanna.storage.defaults import setup_defaults from savanna.storage.models import Node, NodeTemplate -from savanna.storage.storage import DB +from savanna.storage.db import DB import savanna.main from savanna.utils import scheduler from savanna.openstack.common import log as logging @@ -85,8 +85,8 @@ def _stub_auth_valid(*args, **kwargs): CONF = cfg.CONF CONF.import_opt('debug', 'savanna.openstack.common.log') CONF.import_opt('allow_cluster_ops', 'savanna.config') -CONF.import_opt('database_uri', 'savanna.storage.storage', group='sqlalchemy') -CONF.import_opt('echo', 'savanna.storage.storage', group='sqlalchemy') +CONF.import_opt('database_uri', 'savanna.storage.db', group='sqlalchemy') +CONF.import_opt('echo', 'savanna.storage.db', group='sqlalchemy') class TestApi(unittest.TestCase):