Some hacking.py fixes and fixes in validation and cluster_ops.

All code has been cleaned to pass hacking checks;
only N301 (one import per line) has been disabled now;
VMs creation rollback has been implemented for vms creation errors.

Change-Id: I8d973dfa0c01c8d595aff5fbaf47a9f81d55f74f

savanna/exceptions.py

@@ -17,8 +17,7 @@ import savanna.openstack.common.exception as ex
 
 
 class SavannaException(ex.ApiError):
-    """
-    Base Exception
+    """Base Exception for the project
 
     To correctly use this class, inherit from it and define
     a 'message' and 'code' properties.
@@ -40,7 +39,7 @@ class ClusterNameExistedException(SavannaException):
 
 class ImageNotFoundException(SavannaException):
     def __init__(self, value):
-        self.message = "Cannot find image with name '%s'" % value
+        self.message = "Cannot find image with id '%s'" % value
         self.code = "IMAGE_NOT_FOUND"
 
 

savanna/main.py

@@ -64,7 +64,8 @@ CONF.register_opts(opts)
 
 
 def make_app():
-    """
+    """App builder (wsgi)
+
     Entry point for Savanna REST API server
     """
     app = Flask('savanna.api')

savanna/middleware/auth_valid.py

@@ -22,16 +22,15 @@ LOG = logging.getLogger(__name__)
 
 
 class AuthValidator:
-    """
-    Auth Validation Middleware handles token auth results and tenants
-    """
+    """Handles token auth results and tenants."""
 
     def __init__(self, app, conf):
         self.app = app
         self.conf = conf
 
     def __call__(self, env, start_response):
-        """
+        """Ensures that tenants in url and token are equal.
+
         Handle incoming request by checking tenant info prom the headers and
         url ({tenant_id} url attribute).
 

savanna/openstack/commons.py

@@ -22,8 +22,7 @@
 
 
 def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False):
-    """
-    Validate and split the given HTTP request path.
+    """Validate and split the given HTTP request path.
 
     **Examples**::
 

savanna/service/api.py

@@ -39,8 +39,8 @@ def get_node_templates(**args):
 
 
 def create_node_template(values, headers):
-    """
-    Creates new node template from values dict
+    """Creates new node template from values dict.
+
     :param values: dict
     :return: created node template resource
     """
@@ -104,14 +104,14 @@ def _cluster_creation_job(headers, cluster_id):
 
 
 def terminate_cluster(headers, **args):
-    cluster = storage.update_cluster_status('Stoping', **args)
+    cluster = storage.update_cluster_status('Stopping', **args)
 
     eventlet.spawn(_cluster_termination_job, headers, cluster.id)
 
 
 def _cluster_termination_job(headers, cluster_id):
     cluster = storage.get_cluster(id=cluster_id)
-    LOG.debug("Stoping cluster '%s' creation: %s", cluster_id,
+    LOG.debug("Stopping cluster '%s' creation: %s", cluster_id,
               _cluster(cluster).dict)
 
     if CONF.allow_cluster_ops:

savanna/service/cluster_ops.py

@@ -23,6 +23,7 @@ import paramiko
 from savanna.openstack.common import log as logging
 from savanna.storage.db import DB
 from savanna.storage.models import Node, ServiceUrl
+from savanna.storage.storage import update_cluster_status
 from savanna.utils.openstack.nova import novaclient
 
 
@@ -121,6 +122,7 @@ def launch_cluster(headers, cluster):
 
         for _ in xrange(0, nc.count):
             node = dict()
+            node['id'] = None
             if ntype == 'JT+NN':
                 node['name'] = '%s-master' % cluster.name
             else:
@@ -133,10 +135,14 @@ def launch_cluster(headers, cluster):
             node['is_up'] = False
             clmap['nodes'].append(node)
 
-    for node in clmap['nodes']:
-        LOG.debug("Starting node for cluster '%s', node: %s, iamge: %s",
-                  cluster.name, node, clmap['image'])
-        _launch_node(nova, node, clmap['image'])
+    try:
+        for node in clmap['nodes']:
+            LOG.debug("Starting node for cluster '%s', node: %s, image: %s",
+                      cluster.name, node, clmap['image'])
+            _launch_node(nova, node, clmap['image'])
+    except Exception, e:
+        _rollback_cluster_creation(cluster, clmap, nova, e)
+        return
 
     all_set = False
 
@@ -170,10 +176,28 @@ def launch_cluster(headers, cluster):
 
 def _launch_node(nova, node, image):
     srv = nova.servers.create(node['name'], image, node['flavor'])
-    #srv = _find_by_name(nova_client.servers.list(), node['name'])
     node['id'] = srv.id
 
 
+def _rollback_cluster_creation(cluster, clmap, nova, error):
+    update_cluster_status("Error", id=cluster.id)
+
+    LOG.warn("Can't launch all vms for cluster '%s': %s", cluster.id, error)
+    for node in clmap['nodes']:
+        if node['id']:
+            _stop_node_silently(nova, cluster, node['id'])
+
+    LOG.info("All vms of cluster '%s' has been stopped", cluster.id)
+
+
+def _stop_node_silently(nova, cluster, vm_id):
+    LOG.debug("Stopping vm '%s' of cluster '%s'", vm_id, cluster.id)
+    try:
+        nova.servers.delete(vm_id)
+    except Exception, e:
+        LOG.error("Can't silently remove node '%s': %s", vm_id, e)
+
+
 def _check_if_up(nova, node):
     if node['is_up']:
         # all set

savanna/storage/storage.py

@@ -31,8 +31,8 @@ def get_node_templates(**args):
 
 
 def create_node_template(name, node_type_id, tenant_id, flavor_id, configs):
-    """
-    Creates new node templates
+    """Creates new node templates.
+
     :param name: template name
     :param node_type_id: node type
     :param tenant_id: tenant
@@ -86,8 +86,8 @@ def get_clusters(**args):
 
 
 def create_cluster(name, base_image_id, tenant_id, templates):
-    """
-    Creates new cluster
+    """Creates new cluster.
+
     :param name: cluster name
     :param base_image_id: base image
     :param tenant_id: tenant
@@ -124,8 +124,8 @@ def update_cluster_status(new_status, **args):
 ## Node Process ops:
 
 def create_node_process(name, properties):
-    """
-    Creates new node process and node process properties
+    """Creates new node process and node process properties.
+
     :param name: process name
     :param properties: array of triples (name, required, default)
     :return: created node process
@@ -152,8 +152,8 @@ def get_node_types(**args):
 
 
 def create_node_type(name, processes):
-    """
-    Creates new node type using specified list of processes
+    """Creates new node type using specified list of processes
+
     :param name:
     :param processes:
     :return:

savanna/tests/unit/test_service.py

@@ -238,7 +238,7 @@ class TestServiceLayer(unittest.TestCase):
 
         api.terminate_cluster({"X-Tenant-Id": "tenant-01"}, id="cluster-id")
 
-        update_status.assert_called_once_with('Stoping', id="cluster-id")
+        update_status.assert_called_once_with('Stopping', id="cluster-id")
         spawn.assert_called_once_with(api._cluster_termination_job,
                                       {"X-Tenant-Id": "tenant-01"},
                                       "cluster-id")

savanna/utils/openstack/nova.py

@@ -46,6 +46,6 @@ def get_flavors(headers):
 
 
 def get_images(headers):
-    images = [image.name for image
+    images = [image.id for image
               in novaclient(headers).images.list()]
     return images

tools/conf/extract_opts.py

@@ -122,7 +122,7 @@ def _guess_groups(opt, mod_obj):
 
     # is it in the DEFAULT group?
     if (opt.dest in cfg.CONF and
-        not isinstance(cfg.CONF[opt.dest], cfg.CONF.GroupAttr)):
+            not isinstance(cfg.CONF[opt.dest], cfg.CONF.GroupAttr)):
         groups.append('DEFAULT')
 
     # what other groups is it in?

tools/run_pep8

@@ -15,7 +15,7 @@ echo "=============================="
 # python tools/hacking.py --doctest
 
 # Until all these issues get fixed, ignore.
-PEP8='python tools/hacking.py --ignore=E12,E711,E721,E712,N301,N303,N403,N404'
+PEP8='python tools/hacking.py --ignore=N301'
 
 EXCLUDE='--exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*'
 EXCLUDE+=',*egg,build,*hacking.py'