openstack
/
sahara
Code Issues Proposed changes

small refactoring - service and storage (dao) layers has been created 

Change-Id: Ia3c106bfbe87a7e9c08659fd61308164b9afb2dc
This commit is contained in:
Sergey Lukjanov 2013-03-25 19:13:21 +04:00
parent 097f7c44a2
commit c67850ac63
9 changed files with 292 additions and 257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
savanna/cli.py
View File

@ -18,7 +18,7 @@ from flask import Flask
from oslo.config import cfg
from savanna.openstack.common import log
from savanna.storage.defaults import setup_defaults
from savanna.storage.storage import setup_storage, DB
from savanna.storage.db import setup_storage, DB
CONF = cfg.CONF
LOG = log.getLogger(__name__)

2
savanna/main.py
View File

@ -24,7 +24,7 @@ from savanna.api import v02 as api_v02
from savanna.middleware.auth_valid import filter_factory as auth_valid
from savanna.utils.scheduler import setup_scheduler
from savanna.utils.api import render
from savanna.storage.storage import setup_storage
from savanna.storage.db import setup_storage
from savanna.openstack.common import log

244
savanna/service/api.py
View File

@ -17,12 +17,10 @@ import eventlet
from oslo.config import cfg
from flask import request
from savanna.storage.models import NodeTemplate, NodeType, NodeProcess, \
NodeTemplateConfig, Cluster, ClusterNodeCount
from savanna.storage.storage import DB
from savanna.utils.api import abort_and_log
from savanna.service import cluster_ops
from savanna.openstack.common import log as logging
import savanna.storage.storage as storage
LOG = logging.getLogger(__name__)
@ -30,6 +28,105 @@ CONF = cfg.CONF
CONF.import_opt('allow_cluster_ops', 'savanna.config')
## Node Template ops:
def get_node_template(**args):
return _node_template(storage.get_node_template(**args))
def get_node_templates(**args):
return [_node_template(tmpl) for tmpl
in storage.get_node_templates(**args)]
def create_node_template(values):
"""
Creates new node template from values dict
:param values: dict
:return: created node template resource
"""
values = values.pop('node_template')
name = values.pop('name')
node_type_id = storage.get_node_type(name=values.pop('node_type')).id
# todo(slukjanov): take tenant_id from headers
tenant_id = "tenant-01"
flavor_id = values.pop('flavor_id')
nt = storage.create_node_template(name, node_type_id, tenant_id,
flavor_id, values)
return get_node_template(id=nt.id)
def terminate_node_template(**args):
return storage.terminate_node_template(**args)
## Cluster ops:
def get_cluster(**args):
return _cluster(storage.get_cluster(**args))
def get_clusters(**args):
return [_cluster(cluster) for cluster in
storage.get_clusters(**args)]
def create_cluster(values):
values = values.pop('cluster')
name = values.pop('name')
base_image_id = values.pop('base_image_id')
# todo(slukjanov): take tenant_id from headers
tenant_id = "tenant-01"
templates = values.pop('node_templates')
# todo(slukjanov): check that we can create objects in the specified tenant
cluster = storage.create_cluster(name, base_image_id, tenant_id, templates)
eventlet.spawn(_cluster_creation_job, request.headers, cluster.id)
return get_cluster(id=cluster.id)
def _cluster_creation_job(headers, cluster_id):
cluster = storage.get_cluster(id=cluster_id)
LOG.debug("Starting cluster '%s' creation: %s", cluster_id,
_cluster(cluster).dict)
if CONF.allow_cluster_ops:
cluster_ops.launch_cluster(headers, cluster)
else:
LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag")
# update cluster status
storage.update_cluster_status('Active', id=cluster.id)
def terminate_cluster(**args):
cluster = storage.update_cluster_status('Stoping', **args)
eventlet.spawn(_cluster_termination_job, request.headers, cluster.id)
def _cluster_termination_job(headers, cluster_id):
cluster = storage.get_cluster(id=cluster_id)
LOG.debug("Stoping cluster '%s' creation: %s", cluster_id,
_cluster(cluster).dict)
if CONF.allow_cluster_ops:
cluster_ops.stop_cluster(headers, cluster)
else:
LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag")
storage.terminate_cluster(id=cluster.id)
## Utils and DB object to Resource converters
def _clean_nones(obj):
d_type = type(obj)
if d_type is not dict or d_type is not list:
@ -99,56 +196,6 @@ def _node_template(nt):
return Resource('node_template', d)
def _template_id_by_name(template):
return NodeTemplate.query.filter_by(name=template).first().id
def _type_id_by_name(_type):
return NodeType.query.filter_by(name=_type).first().id
def get_node_template(**args):
return _node_template(NodeTemplate.query.filter_by(**args).first())
def get_node_templates(**args):
return [_node_template(tmpl) for tmpl
in NodeTemplate.query.filter_by(**args).all()]
def create_node_template(values):
"""
Creates new node template from values dict
:param values: dict
:return: created node template resource
"""
values = values.pop('node_template')
name = values.pop('name')
node_type_id = _type_id_by_name(values.pop('node_type'))
# todo(slukjanov): take tenant_id from headers
tenant_id = "tenant-01"
flavor_id = values.pop('flavor_id')
nt = NodeTemplate(name, node_type_id, tenant_id, flavor_id)
DB.session.add(nt)
for process_name in values:
process = NodeProcess.query.filter_by(name=process_name).first()
conf = values.get(process_name)
for prop in process.node_process_properties:
val = conf.get(prop.name, None)
if not val and prop.required:
if not prop.default:
raise RuntimeError('Template \'%s\', value missed '
'for required param: %s %s'
% (name, process.name, prop.name))
val = prop.default
DB.session.add(NodeTemplateConfig(nt.id, prop.id, val))
DB.session.commit()
return get_node_template(id=nt.id)
def _cluster(cluster):
if not cluster:
abort_and_log(404, 'Cluster not found')
@ -173,94 +220,3 @@ def _cluster(cluster):
d['service_urls'][service.name] = service.url
return Resource('cluster', d)
def get_cluster(**args):
return _cluster(Cluster.query.filter_by(**args).first())
def get_clusters(**args):
return [_cluster(cluster) for cluster in
Cluster.query.filter_by(**args).all()]
def create_cluster(values):
values = values.pop('cluster')
name = values.pop('name')
base_image_id = values.pop('base_image_id')
# todo(slukjanov): take tenant_id from headers
tenant_id = "tenant-01"
templates = values.pop('node_templates')
# todo(slukjanov): check that we can create objects in the specified tenant
cluster = Cluster(name, base_image_id, tenant_id)
DB.session.add(cluster)
for template in templates:
count = templates.get(template)
template_id = _template_id_by_name(template)
cnc = ClusterNodeCount(cluster.id, template_id, int(count))
DB.session.add(cnc)
DB.session.commit()
eventlet.spawn(_cluster_creation_job, request.headers, cluster.id)
return get_cluster(id=cluster.id)
def _cluster_creation_job(headers, cluster_id):
cluster = Cluster.query.filter_by(id=cluster_id).first()
LOG.debug("Starting cluster '%s' creation: %s", cluster_id,
_cluster(cluster).dict)
if CONF.allow_cluster_ops:
cluster_ops.launch_cluster(headers, cluster)
else:
LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag")
# update cluster status
cluster = Cluster.query.filter_by(id=cluster.id).first()
cluster.status = 'Active'
DB.session.add(cluster)
DB.session.commit()
def terminate_cluster(**args):
# update cluster status
cluster = Cluster.query.filter_by(**args).first()
cluster.status = 'Stoping'
DB.session.add(cluster)
DB.session.commit()
eventlet.spawn(_cluster_termination_job, request.headers, cluster.id)
def _cluster_termination_job(headers, cluster_id):
cluster = Cluster.query.filter_by(id=cluster_id).first()
LOG.debug("Stoping cluster '%s' creation: %s", cluster_id,
_cluster(cluster).dict)
if CONF.allow_cluster_ops:
cluster_ops.stop_cluster(headers, cluster)
else:
LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag")
DB.session.delete(cluster)
DB.session.commit()
def terminate_node_template(**args):
template = NodeTemplate.query.filter_by(**args).first()
if template:
if len(template.nodes):
abort_and_log(500, "There are active nodes created using "
"template '%s' you trying to terminate"
% args)
else:
DB.session.delete(template)
DB.session.commit()
return True
else:
return False

2
savanna/service/cluster_ops.py
View File

@ -20,7 +20,7 @@ from jinja2 import PackageLoader
from paramiko import SSHClient, AutoAddPolicy
from oslo.config import cfg
from savanna.storage.models import Node, ServiceUrl
from savanna.storage.storage import DB
from savanna.storage.db import DB
from savanna.utils.openstack.nova import novaclient
from savanna.openstack.common import log as logging

40
savanna/storage/db.py Normal file
View File

@ -0,0 +1,40 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask.ext.sqlalchemy import SQLAlchemy
from oslo.config import cfg
DB = SQLAlchemy()
opts = [
cfg.StrOpt('database_uri',
default='sqlite:////tmp/savanna.db',
help='URL for sqlalchemy database'),
cfg.BoolOpt('echo',
default=False,
help='Sqlalchemy echo')
]
CONF = cfg.CONF
CONF.register_opts(opts, group='sqlalchemy')
def setup_storage(app):
app.config['SQLALCHEMY_DATABASE_URI'] = CONF.sqlalchemy.database_uri
app.config['SQLALCHEMY_ECHO'] = CONF.sqlalchemy.echo
DB.app = app
DB.init_app(app)
DB.create_all()

87
savanna/storage/defaults.py
View File

@ -13,96 +13,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from savanna.storage.models import NodeProcess, NodeProcessProperty, \
NodeType, NodeTemplate, NodeTemplateConfig, Cluster, ClusterNodeCount
from savanna.storage.storage import DB
from savanna.storage.storage import create_node_type, \
create_node_template, create_node_process
from savanna.openstack.common import log as logging
LOG = logging.getLogger(__name__)
def create_node_process(name, properties):
"""
Creates new node process and node process properties
:param name: process name
:param properties: array of triples (name, required, default)
:return: created node process
"""
process = NodeProcess(name)
DB.session.add(process)
DB.session.commit()
for p in properties:
prop = NodeProcessProperty(process.id, p[0], p[1], p[2])
DB.session.add(prop)
DB.session.commit()
return process
def create_node_type(name, processes):
"""
Creates new node type using specified list of processes
:param name:
:param processes:
:return:
"""
node_type = NodeType(name)
node_type.processes = processes
DB.session.add(node_type)
DB.session.commit()
return node_type
def create_node_template(name, node_type_id, tenant_id, flavor_id, configs):
"""
Creates new node templates
:param name: template name
:param node_type_id: node type
:param tenant_id: tenant
:param flavor_id: flavor
:param configs: dict of process->property->value
:return: created node template
"""
node_template = NodeTemplate(name, node_type_id, tenant_id, flavor_id)
DB.session.add(node_template)
for process_name in configs:
process = NodeProcess.query.filter_by(name=process_name).first()
conf = configs.get(process_name)
for prop in process.node_process_properties:
val = conf.get(prop.name, None)
if not val and prop.required:
if not prop.default:
raise RuntimeError('Template \'%s\', value missed '
'for required param: %s %s'
% (name, process.name, prop.name))
val = prop.default
DB.session.add(NodeTemplateConfig(node_template.id, prop.id, val))
DB.session.commit()
return node_template
def create_cluster(name, base_image_id, tenant_id, templates):
"""
Creates new cluster
:param name: cluster name
:param base_image_id: base image
:param tenant_id: tenant
:param templates: dict of template->count
:return: created cluster
"""
cluster = Cluster(name, base_image_id, tenant_id)
DB.session.add(cluster)
for template in templates:
count = templates.get(template)
cnc = ClusterNodeCount(cluster.id,
NodeTemplate.query.filter_by(name=template)
.first().id, int(count))
DB.session.add(cnc)
DB.session.commit()
return cluster
def setup_defaults(reset_db=False, gen_templates=False):
nt_jt_nn = None
nt_jt = None

2
savanna/storage/models.py
View File

@ -15,7 +15,7 @@
from uuid import uuid4
from savanna.storage.storage import DB
from savanna.storage.db import DB
class NodeTemplate(DB.Model):

164
savanna/storage/storage.py
View File

@ -13,28 +13,150 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from flask.ext.sqlalchemy import SQLAlchemy
from oslo.config import cfg
from savanna.storage.db import DB
DB = SQLAlchemy()
opts = [
cfg.StrOpt('database_uri',
default='sqlite:////tmp/savanna.db',
help='URL for sqlalchemy database'),
cfg.BoolOpt('echo',
default=False,
help='Sqlalchemy echo')
]
CONF = cfg.CONF
CONF.register_opts(opts, group='sqlalchemy')
from savanna.storage.models import NodeTemplate, NodeProcess, Cluster, \
ClusterNodeCount, NodeTemplateConfig, NodeType, NodeProcessProperty
from savanna.utils.api import abort_and_log
def setup_storage(app):
app.config['SQLALCHEMY_DATABASE_URI'] = CONF.sqlalchemy.database_uri
app.config['SQLALCHEMY_ECHO'] = CONF.sqlalchemy.echo
## Node Template ops:
DB.app = app
DB.init_app(app)
DB.create_all()
def get_node_template(**args):
return NodeTemplate.query.filter_by(**args).first()
def get_node_templates(**args):
return NodeTemplate.query.filter_by(**args).all()
def create_node_template(name, node_type_id, tenant_id, flavor_id, configs):
"""
Creates new node templates
:param name: template name
:param node_type_id: node type
:param tenant_id: tenant
:param flavor_id: flavor
:param configs: dict of process->property->value
:return: created node template
"""
node_template = NodeTemplate(name, node_type_id, tenant_id, flavor_id)
DB.session.add(node_template)
for process_name in configs:
process = NodeProcess.query.filter_by(name=process_name).first()
conf = configs.get(process_name)
for prop in process.node_process_properties:
val = conf.get(prop.name, None)
if not val and prop.required:
if not prop.default:
raise RuntimeError('Template \'%s\', value missed '
'for required param: %s %s'
% (name, process.name, prop.name))
val = prop.default
DB.session.add(NodeTemplateConfig(node_template.id, prop.id, val))
DB.session.commit()
return node_template
def terminate_node_template(**args):
template = NodeTemplate.query.filter_by(**args).first()
if template:
if len(template.nodes):
abort_and_log(500, "There are active nodes created using "
"template '%s' you trying to terminate"
% args)
else:
DB.session.delete(template)
DB.session.commit()
return True
else:
return False
## Cluster ops:
def get_cluster(**args):
return Cluster.query.filter_by(**args).first()
def get_clusters(**args):
return Cluster.query.filter_by(**args).all()
def create_cluster(name, base_image_id, tenant_id, templates):
"""
Creates new cluster
:param name: cluster name
:param base_image_id: base image
:param tenant_id: tenant
:param templates: dict of template->count
:return: created cluster
"""
cluster = Cluster(name, base_image_id, tenant_id)
DB.session.add(cluster)
for template in templates:
count = templates.get(template)
template_id = get_node_template(name=template).id
cnc = ClusterNodeCount(cluster.id, template_id, int(count))
DB.session.add(cnc)
DB.session.commit()
return cluster
def terminate_cluster(**args):
cluster = get_cluster(**args)
DB.session.delete(cluster)
DB.session.commit()
def update_cluster_status(new_status, **args):
cluster = Cluster.query.filter_by(**args).first()
cluster.status = new_status
DB.session.add(cluster)
DB.session.commit()
return cluster
## Node Process ops:
def create_node_process(name, properties):
"""
Creates new node process and node process properties
:param name: process name
:param properties: array of triples (name, required, default)
:return: created node process
"""
process = NodeProcess(name)
DB.session.add(process)
DB.session.commit()
for p in properties:
prop = NodeProcessProperty(process.id, p[0], p[1], p[2])
DB.session.add(prop)
DB.session.commit()
return process
## Node Type ops:
def get_node_type(**args):
return NodeType.query.filter_by(**args).first()
def create_node_type(name, processes):
"""
Creates new node type using specified list of processes
:param name:
:param processes:
:return:
"""
node_type = NodeType(name)
node_type.processes = processes
DB.session.add(node_type)
DB.session.commit()
return node_type

6
savanna/tests/unit/test_api_v02.py
View File

@ -26,7 +26,7 @@ from savanna.main import make_app
from savanna.service import api
from savanna.storage.defaults import setup_defaults
from savanna.storage.models import Node, NodeTemplate
from savanna.storage.storage import DB
from savanna.storage.db import DB
import savanna.main
from savanna.utils import scheduler
from savanna.openstack.common import log as logging
@ -85,8 +85,8 @@ def _stub_auth_valid(*args, **kwargs):
CONF = cfg.CONF
CONF.import_opt('debug', 'savanna.openstack.common.log')
CONF.import_opt('allow_cluster_ops', 'savanna.config')
CONF.import_opt('database_uri', 'savanna.storage.storage', group='sqlalchemy')
CONF.import_opt('echo', 'savanna.storage.storage', group='sqlalchemy')
CONF.import_opt('database_uri', 'savanna.storage.db', group='sqlalchemy')
CONF.import_opt('echo', 'savanna.storage.db', group='sqlalchemy')
class TestApi(unittest.TestCase):