Merge "Move cluster deletion to common engine module"

This commit is contained in:
Jenkins 2015-06-09 13:07:22 +00:00 committed by Gerrit Code Review
commit 7440497e4f
5 changed files with 268 additions and 228 deletions

View File

@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from novaclient import exceptions as nova_exceptions
from oslo_config import cfg
from oslo_log import log as logging
import six
@ -198,15 +197,6 @@ class DirectEngine(e.Engine):
policies=['anti-affinity'])
return server_group.id
def _delete_aa_server_group(self, cluster):
if cluster.anti_affinity:
server_group_name = g.generate_aa_group_name(cluster.name)
client = nova.client().server_groups
server_groups = client.findall(name=server_group_name)
if len(server_groups) == 1:
client.delete(server_groups[0].id)
def _find_aa_server_group(self, cluster):
server_group_name = g.generate_aa_group_name(cluster.name)
server_groups = nova.client().server_groups.findall(
@ -460,32 +450,6 @@ class DirectEngine(e.Engine):
LOG.info(_LI("All instances are active"))
@poll_utils.poll_status(
'delete_instances_timeout',
_("Wait for instances to be deleted"), sleep=1)
def _check_deleted(self, deleted_ids, cluster, instances):
if not g.check_cluster_exists(cluster):
return True
for instance in instances:
if instance.id not in deleted_ids:
with context.set_current_instance_id(instance.instance_id):
if self._check_if_deleted(instance):
LOG.debug("Instance is deleted")
deleted_ids.add(instance.id)
cpo.add_successful_event(instance)
return len(deleted_ids) == len(instances)
def _await_deleted(self, cluster, instances):
"""Await all instances are deleted."""
if not instances:
return
cpo.add_provisioning_step(
cluster.id, _("Wait for instances to be deleted"), len(instances))
deleted_ids = set()
self._check_deleted(deleted_ids, cluster, instances)
@cpo.event_wrapper(mark_successful_on_exit=False)
def _check_if_active(self, instance):
server = nova.get_instance_info(instance)
@ -494,15 +458,6 @@ class DirectEngine(e.Engine):
return server.status == 'ACTIVE'
@cpo.event_wrapper(mark_successful_on_exit=False)
def _check_if_deleted(self, instance):
try:
nova.get_instance_info(instance)
except nova_exceptions.NotFound:
return True
return False
def _rollback_cluster_creation(self, cluster, ex):
"""Shutdown all instances and update cluster status."""
@ -518,64 +473,9 @@ class DirectEngine(e.Engine):
cluster = conductor.cluster_get(context.ctx(), cluster)
g.clean_cluster_from_empty_ng(cluster)
def _shutdown_instances(self, cluster):
for node_group in cluster.node_groups:
for instance in node_group.instances:
with context.set_current_instance_id(instance.instance_id):
self._shutdown_instance(instance)
self._await_deleted(cluster, node_group.instances)
self._delete_auto_security_group(node_group)
def _delete_auto_security_group(self, node_group):
if not node_group.auto_security_group:
return
if not node_group.security_groups:
# node group has no security groups
# nothing to delete
return
name = node_group.security_groups[-1]
try:
client = nova.client().security_groups
security_group = client.get(name)
if (security_group.name !=
g.generate_auto_security_group_name(node_group)):
LOG.warning(_LW("Auto security group for node group {name} is "
"not found").format(name=node_group.name))
return
client.delete(name)
except Exception:
LOG.warning(_LW("Failed to delete security group {name}").format(
name=name))
def _shutdown_instance(self, instance):
ctx = context.ctx()
if instance.node_group.floating_ip_pool:
try:
networks.delete_floating_ip(instance.instance_id)
except nova_exceptions.NotFound:
LOG.warning(_LW("Attempted to delete non-existent floating IP "
"in pool {pool} from instance")
.format(pool=instance.node_group.floating_ip_pool))
try:
volumes.detach_from_instance(instance)
except Exception:
LOG.warning(_LW("Detaching volumes from instance failed"))
try:
nova.client().servers.delete(instance.instance_id)
except nova_exceptions.NotFound:
LOG.warning(_LW("Attempted to delete non-existent instance"))
conductor.instance_remove(ctx, instance)
def shutdown_cluster(self, cluster):
"""Shutdown specified cluster and all related resources."""
self._shutdown_instances(cluster)
self._clean_job_executions(cluster)
self._delete_aa_server_group(cluster)
self._remove_db_objects(cluster)

View File

@ -18,6 +18,7 @@ import abc
import datetime
import string
from novaclient import exceptions as nova_exceptions
from oslo_log import log as logging
import six
@ -25,7 +26,9 @@ from sahara import conductor as c
from sahara import context
from sahara.i18n import _
from sahara.i18n import _LI
from sahara.i18n import _LW
from sahara.service import networks
from sahara.service import volumes
from sahara.utils import cluster_progress_ops as cpo
from sahara.utils import edp
from sahara.utils import general as g
@ -186,6 +189,7 @@ sed '/^Defaults requiretty*/ s/^/#/' -i /etc/sudoers\n
user_home=user_home,
instance_name=instance_name)
# Deletion ops
def _clean_job_executions(self, cluster):
ctx = context.ctx()
for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id):
@ -196,3 +200,111 @@ sed '/^Defaults requiretty*/ s/^/#/' -i /etc/sudoers\n
update.update({"info": info,
"end_time": datetime.datetime.now()})
conductor.job_execution_update(ctx, je, update)
def _delete_auto_security_group(self, node_group):
if not node_group.auto_security_group:
return
if not node_group.security_groups:
# node group has no security groups
# nothing to delete
return
name = node_group.security_groups[-1]
try:
client = nova.client().security_groups
security_group = b.execute_with_retries(client.get, name)
if (security_group.name !=
g.generate_auto_security_group_name(node_group)):
LOG.warning(_LW("Auto security group for node group {name} is "
"not found").format(name=node_group.name))
return
b.execute_with_retries(client.delete, name)
except Exception:
LOG.warning(_LW("Failed to delete security group {name}").format(
name=name))
def _delete_aa_server_group(self, cluster):
if cluster.anti_affinity:
server_group_name = g.generate_aa_group_name(cluster.name)
client = nova.client().server_groups
server_groups = b.execute_with_retries(client.findall,
name=server_group_name)
if len(server_groups) == 1:
b.execute_with_retries(client.delete, server_groups[0].id)
def _shutdown_instance(self, instance):
if instance.node_group.floating_ip_pool:
try:
b.execute_with_retries(networks.delete_floating_ip,
instance.instance_id)
except nova_exceptions.NotFound:
LOG.warning(_LW("Attempted to delete non-existent floating IP "
"in pool {pool} from instance")
.format(pool=instance.node_group.floating_ip_pool))
try:
volumes.detach_from_instance(instance)
except Exception:
LOG.warning(_LW("Detaching volumes from instance failed"))
try:
b.execute_with_retries(nova.client().servers.delete,
instance.instance_id)
except nova_exceptions.NotFound:
LOG.warning(_LW("Attempted to delete non-existent instance"))
conductor.instance_remove(context.ctx(), instance)
@cpo.event_wrapper(mark_successful_on_exit=False)
def _check_if_deleted(self, instance):
try:
nova.get_instance_info(instance)
except nova_exceptions.NotFound:
return True
return False
@poll_utils.poll_status(
'delete_instances_timeout',
_("Wait for instances to be deleted"), sleep=1)
def _check_deleted(self, deleted_ids, cluster, instances):
if not g.check_cluster_exists(cluster):
return True
for instance in instances:
if instance.id not in deleted_ids:
with context.set_current_instance_id(instance.instance_id):
if self._check_if_deleted(instance):
LOG.debug("Instance is deleted")
deleted_ids.add(instance.id)
cpo.add_successful_event(instance)
return len(deleted_ids) == len(instances)
def _await_deleted(self, cluster, instances):
"""Await all instances are deleted."""
if not instances:
return
cpo.add_provisioning_step(
cluster.id, _("Wait for instances to be deleted"), len(instances))
deleted_ids = set()
self._check_deleted(deleted_ids, cluster, instances)
def _shutdown_instances(self, cluster):
for node_group in cluster.node_groups:
for instance in node_group.instances:
with context.set_current_instance_id(instance.instance_id):
self._shutdown_instance(instance)
self._await_deleted(cluster, node_group.instances)
self._delete_auto_security_group(node_group)
def _remove_db_objects(self, cluster):
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster)
instances = g.get_instances(cluster)
for inst in instances:
conductor.instance_remove(ctx, inst)

View File

@ -167,14 +167,16 @@ class HeatEngine(e.Engine):
stack = heat.get_stack(cluster.name)
heat.wait_stack_completion(stack)
except heat_exc.HTTPNotFound:
LOG.warning(_LW('Did not find stack for cluster'))
LOG.warning(_LW('Did not find stack for cluster. Trying to delete '
'cluster manually.'))
# Stack not found. Trying to delete cluster like direct engine
# do it
self._shutdown_instances(cluster)
self._delete_aa_server_group(cluster)
self._clean_job_executions(cluster)
ctx = context.ctx()
instances = g.get_instances(cluster)
for inst in instances:
conductor.instance_remove(ctx, inst)
self._remove_db_objects(cluster)
class _CreateLauncher(HeatEngine):

View File

@ -1,103 +0,0 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from sahara.service import direct_engine
from sahara.tests.unit import base
from sahara.utils import general as g
class TestDirectEngine(base.SaharaWithDbTestCase):
@mock.patch('sahara.utils.openstack.nova.client')
def test_delete_auto_security_group(self, nova_client):
engine = direct_engine.DirectEngine()
ng = mock.Mock(id="16fd2706-8baf-433b-82eb-8c7fada847da",
auto_security_group=True)
ng.name = "ngname"
ng.cluster.name = "cluster"
auto_name = g.generate_auto_security_group_name(ng)
ng.security_groups = [auto_name]
client = mock.Mock()
nova_client.return_value = client
client.security_groups.get.side_effect = lambda x: SecurityGroup(x)
engine._delete_auto_security_group(ng)
client.security_groups.delete.assert_called_once_with(auto_name)
@mock.patch('sahara.utils.openstack.nova.client')
def test_delete_auto_security_group_other_groups(self, nova_client):
engine = direct_engine.DirectEngine()
ng = mock.Mock(id="16fd2706-8baf-433b-82eb-8c7fada847da",
auto_security_group=True)
ng.name = "ngname"
ng.cluster.name = "cluster"
auto_name = g.generate_auto_security_group_name(ng)
ng.security_groups = ['1', '2', auto_name]
client = mock.Mock()
nova_client.return_value = client
client.security_groups.get.side_effect = lambda x: SecurityGroup(x)
engine._delete_auto_security_group(ng)
client.security_groups.delete.assert_called_once_with(auto_name)
@mock.patch('sahara.utils.openstack.nova.client')
def test_delete_auto_security_group_no_groups(self, nova_client):
engine = direct_engine.DirectEngine()
ng = mock.Mock(id="16fd2706-8baf-433b-82eb-8c7fada847da",
auto_security_group=True)
ng.name = "ngname"
ng.cluster.name = "cluster"
ng.security_groups = []
client = mock.Mock()
nova_client.return_value = client
client.security_groups.get.side_effect = lambda x: SecurityGroup(x)
engine._delete_auto_security_group(ng)
self.assertEqual(0, client.security_groups.delete.call_count)
@mock.patch('sahara.utils.openstack.nova.client')
def test_delete_auto_security_group_wrong_group(self, nova_client):
engine = direct_engine.DirectEngine()
ng = mock.Mock(id="16fd2706-8baf-433b-82eb-8c7fada847da",
auto_security_group=True)
ng.name = "ngname"
ng.cluster.name = "cluster"
ng.security_groups = ['1', '2']
client = mock.Mock()
nova_client.return_value = client
client.security_groups.get.side_effect = lambda x: SecurityGroup(x)
engine._delete_auto_security_group(ng)
self.assertEqual(0, client.security_groups.delete.call_count)
class SecurityGroup(object):
def __init__(self, name):
super(SecurityGroup, self).__init__()
self.name = name

View File

@ -13,34 +13,42 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from heatclient import exc as heat_exc
import mock
from sahara.service import engine
from sahara.service.heat import heat_engine
from sahara.tests.unit import base
from sahara.utils import general as g
class EngineTest(engine.Engine):
def __init__(self):
super(EngineTest, self).__init__()
self.order = []
def create_cluster(self, cluster):
pass
def get_type_and_version(self):
pass
def rollback_cluster(self, cluster, reason):
pass
def scale_cluster(self, cluster, node_group_id_map):
pass
def shutdown_cluster(self, cluster):
pass
class TestEngine(base.SaharaWithDbTestCase):
class EngineTest(engine.Engine):
def create_cluster(self, cluster):
pass
def get_type_and_version(self):
pass
def rollback_cluster(self, cluster, reason):
pass
def scale_cluster(self, cluster, node_group_id_map):
pass
def shutdown_cluster(self, cluster):
pass
def setUp(self):
super(TestEngine, self).setUp()
self.eng = self.EngineTest()
self.eng = EngineTest()
@mock.patch('sahara.utils.openstack.nova.client')
def test_get_node_group_image_username(self, nova_client):
@ -82,3 +90,124 @@ class TestEngine(base.SaharaWithDbTestCase):
'end_time': '28.04.2015'}
self.assertEqual(update, args[2])
class TestDeletion(base.SaharaTestCase):
def setUp(self):
super(TestDeletion, self).setUp()
self.engine = EngineTest()
@mock.patch('sahara.utils.openstack.nova.client')
def test_delete_auto_security_group(self, nova_client):
ng = mock.Mock(id="16fd2706-8baf-433b-82eb-8c7fada847da",
auto_security_group=True)
ng.name = "ngname"
ng.cluster.name = "cluster"
auto_name = g.generate_auto_security_group_name(ng)
ng.security_groups = [auto_name]
client = mock.Mock()
nova_client.return_value = client
client.security_groups.get.side_effect = lambda x: SecurityGroup(x)
self.engine._delete_auto_security_group(ng)
client.security_groups.delete.assert_called_once_with(auto_name)
@mock.patch('sahara.utils.openstack.nova.client')
def test_delete_auto_security_group_other_groups(self, nova_client):
ng = mock.Mock(id="16fd2706-8baf-433b-82eb-8c7fada847da",
auto_security_group=True)
ng.name = "ngname"
ng.cluster.name = "cluster"
auto_name = g.generate_auto_security_group_name(ng)
ng.security_groups = ['1', '2', auto_name]
client = mock.Mock()
nova_client.return_value = client
client.security_groups.get.side_effect = lambda x: SecurityGroup(x)
self.engine._delete_auto_security_group(ng)
client.security_groups.delete.assert_called_once_with(auto_name)
@mock.patch('sahara.utils.openstack.nova.client')
def test_delete_auto_security_group_no_groups(self, nova_client):
ng = mock.Mock(id="16fd2706-8baf-433b-82eb-8c7fada847da",
auto_security_group=True)
ng.name = "ngname"
ng.cluster.name = "cluster"
ng.security_groups = []
client = mock.Mock()
nova_client.return_value = client
client.security_groups.get.side_effect = lambda x: SecurityGroup(x)
self.engine._delete_auto_security_group(ng)
self.assertEqual(0, client.security_groups.delete.call_count)
@mock.patch('sahara.utils.openstack.nova.client')
def test_delete_auto_security_group_wrong_group(self, nova_client):
ng = mock.Mock(id="16fd2706-8baf-433b-82eb-8c7fada847da",
auto_security_group=True)
ng.name = "ngname"
ng.cluster.name = "cluster"
ng.security_groups = ['1', '2']
client = mock.Mock()
nova_client.return_value = client
client.security_groups.get.side_effect = lambda x: SecurityGroup(x)
self.engine._delete_auto_security_group(ng)
self.assertEqual(0, client.security_groups.delete.call_count)
@mock.patch('sahara.service.engine.Engine._delete_aa_server_group')
@mock.patch('sahara.service.engine.Engine._shutdown_instances')
@mock.patch('sahara.service.engine.Engine._remove_db_objects')
@mock.patch('sahara.service.engine.Engine._clean_job_executions')
@mock.patch('sahara.utils.openstack.heat.client')
@mock.patch('sahara.service.heat.heat_engine.LOG.warning')
def test_calls_order(self, logger, heat_client, _job_ex, _db_ob,
_shutdown, _delete_aa):
class FakeHeatEngine(heat_engine.HeatEngine):
def __init__(self):
super(FakeHeatEngine, self).__init__()
self.order = []
def _clean_job_executions(self, cluster):
self.order.append('clean_job_executions')
super(FakeHeatEngine, self)._clean_job_executions(cluster)
def _remove_db_objects(self, cluster):
self.order.append('remove_db_objects')
super(FakeHeatEngine, self)._remove_db_objects(cluster)
def _shutdown_instances(self, cluster):
self.order.append('shutdown_instances')
super(FakeHeatEngine, self)._shutdown_instances(cluster)
def _delete_aa_server_group(self, cluster):
self.order.append('delete_aa_server_group')
super(FakeHeatEngine, self)._delete_aa_server_group(cluster)
fake_cluster = mock.Mock()
heat_client.side_effect = heat_exc.HTTPNotFound()
engine = FakeHeatEngine()
engine.shutdown_cluster(fake_cluster)
self.assertEqual(['shutdown_instances', 'delete_aa_server_group',
'clean_job_executions', 'remove_db_objects'],
engine.order)
self.assertEqual(
[mock.call('Did not find stack for cluster. Trying to '
'delete cluster manually.')], logger.call_args_list)
class SecurityGroup(object):
def __init__(self, name):
self.name = name