Adding cleanup script

Consists of two parts:
1) OS cleanup:
Instances, subnets and networks from all OS
projects will be removed.
2) K8s objects cleanup

Usage:
ccp cleanup --auth-url <auth_url> --skip-os-cleanup

--auth-url parameter is optional, by default
it takes internal keystone url, whick is not
appropriate in some cases.
--skip-os-cleanup is optional

Note: do not use default namespace for deployment
because it could not be cleaned up by cleanup command

Change-Id: I53f59d9ca8de09ac33e37c90dacf821dda72afec

fuel_ccp/cleanup.py

@@ -0,0 +1,165 @@
+import time
+
+from k8sclient.client import rest
+from keystoneauth1 import exceptions as keystoneauth_exceptions
+from keystoneauth1.identity import v3
+from keystoneauth1 import session as keystone_session
+from neutronclient.v2_0 import client as neutron_client
+from novaclient import client as nova_client
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from fuel_ccp.common import utils
+from fuel_ccp import kubernetes
+
+
+CONF = cfg.CONF
+CONF.import_group('kubernetes', 'fuel_ccp.config.kubernetes')
+
+
+LOG = logging.getLogger(__name__)
+
+
+def _wait_until_empty(attempts, resource_path,
+                      command, *args, **kwargs):
+    while attempts > 0:
+        resources_list = command(*args, **kwargs)
+        if resource_path:
+            resources_list = resources_list[resource_path]
+        if not resources_list:
+            return
+        time.sleep(3)
+        attempts -= 1
+    return command(*args, **kwargs)
+
+
+def _get_session(auth_url, username, password, project_name,
+                 project_domain_name='default', user_domain_name='default'):
+    auth = v3.Password(auth_url=auth_url,
+                       username=username,
+                       password=password,
+                       project_name=project_name,
+                       project_domain_name=project_domain_name,
+                       user_domain_name=user_domain_name)
+
+    return keystone_session.Session(auth=auth)
+
+
+def _cleanup_servers(session):
+    LOG.info('Cleaning up instances')
+    nova = nova_client.Client("2", session=session)
+    server_list = nova.servers.list(search_opts={"all_tenants": True})
+    if not server_list:
+        return
+    for server in server_list:
+        LOG.info('Removing instance %s (%s)', server.name, server.id)
+        nova.servers.delete(server.id)
+    server_list = _wait_until_empty(
+        60, None, nova.servers.list, search_opts={"all_tenants": True})
+    if server_list:
+        LOG.warning("Some instances were not removed, trying to force delete")
+        for server in server_list:
+            LOG.info('Force deleting instance %s (%s)',
+                     (server.name, server.id))
+            nova.servers.force_delete(server.id)
+    server_list = _wait_until_empty(
+        60, None, nova.servers.list, search_opts={"all_tenants": True})
+    if server_list:
+        raise RuntimeError(
+            'Some instances were not removed after force delete: %s'
+            % ', '.join(['%s (%s)' % (server.name, server.id)
+                         for server in server_list]))
+
+
+def _cleanup_network_resources(session):
+    neutron = neutron_client.Client(session=session)
+    LOG.info('Cleaning up subnets')
+    for subnet in neutron.list_subnets()['subnets']:
+        LOG.info('Removing subnet %s (%s)', subnet['name'], subnet['id'])
+        neutron.delete_subnet(subnet['id'])
+    subnet_list = _wait_until_empty(10, 'subnets', neutron.list_subnets)
+    if subnet_list:
+        raise RuntimeError(
+            'Some subnets were not removed: %s'
+            % ', '.join(['%s (%s)' % (subnet['name'], subnet['id'])
+                         for subnet in subnet_list['subnets']]))
+
+    LOG.info('Cleaning up networks')
+    for network in neutron.list_networks()['networks']:
+        LOG.info('Removing network %s (%s)', network['name'], network['id'])
+        neutron.delete_network(network['id'])
+    network_list = _wait_until_empty(10, 'networks', neutron.list_networks)
+    if network_list:
+        raise RuntimeError(
+            'Some networks were not removed: %s'
+            % ', '.join(['%s (%s)' % (network['name'], network['id'])
+                         for network in network_list['networks']]))
+
+
+def _cleanup_openstack_environment(configs, auth_url=None):
+    if 'openstack_project_name' not in configs:
+        # Ensure that keystone configs are provided. Assume that it is not an
+        # OpenStack deployment otherwise
+        raise RuntimeError('There are no Keystone configs provided. '
+                           'Run with --skip-os-cleanup flag if OpenStack '
+                           'is not deployed')
+
+    configs['auth_url'] = auth_url or 'http://keystone:%s/v3' % configs[
+        'keystone_public_port']
+
+    session = _get_session(
+        configs['auth_url'], configs['openstack_user_name'],
+        configs['openstack_user_password'], configs['openstack_project_name'])
+
+    try:
+        session.get_project_id()
+    except (keystoneauth_exceptions.ConnectFailure,
+            keystoneauth_exceptions.EndpointNotFound):
+        LOG.error(
+            'Keystone is not deployed or %s is not accessible. '
+            'Cleanup is aborted. '
+            'Run with --skip-os-cleanup flag if OpenStack '
+            'is not deployed', configs['auth_url'])
+        raise
+    try:
+        _cleanup_servers(session)
+    except keystoneauth_exceptions.EndpointNotFound:
+        LOG.info('Nova is not present, skipping instances cleanup')
+        pass
+    try:
+        _cleanup_network_resources(session)
+    except keystoneauth_exceptions.EndpointNotFound:
+        LOG.info('Neutron is not present, skipping network resources '
+                 'cleanup')
+        pass
+    LOG.info('OpenStack cleanup has been finished successfully.')
+
+
+def _wait_for_namespace_delete(k8s_api):
+    attempts = 60
+    while attempts > 0:
+        try:
+            k8s_api.read_namespaced_namespace(CONF.kubernetes.namespace)
+        except rest.ApiException as e:
+            if e.status == 404:
+                return
+            raise e
+        time.sleep(3)
+        attempts -= 1
+    raise RuntimeError(
+        "Wasn't able to delete namespace %s" % CONF.kubernetes.namespace)
+
+
+def _cleanup_kubernetes_objects():
+    LOG.info('Starting Kubernetes objects cleanup')
+    k8s_api = kubernetes.get_v1_api(kubernetes.get_client())
+    k8s_api.delete_namespaced_namespace({}, CONF.kubernetes.namespace)
+    _wait_for_namespace_delete(k8s_api)
+    LOG.info('Kubernetes objects cleanup has been finished successfully.')
+
+
+def cleanup(auth_url=None, skip_os_cleanup=False):
+    configs = utils.get_global_parameters('configs')['configs']
+    if not skip_os_cleanup:
+        _cleanup_openstack_environment(configs, auth_url)
+    _cleanup_kubernetes_objects()

fuel_ccp/cli.py

@@ -5,6 +5,7 @@ from oslo_config import cfg
 from oslo_log import log as logging
 
 from fuel_ccp import build
+from fuel_ccp import cleanup
 from fuel_ccp import deploy
 from fuel_ccp import fetch
 
@@ -33,6 +34,11 @@ def do_fetch():
     fetch.fetch_repositories(CONF.repositories.names)
 
 
+def do_cleanup():
+    cleanup.cleanup(auth_url=CONF.action.auth_url,
+                    skip_os_cleanup=CONF.action.skip_os_cleanup)
+
+
 def signal_handler(signo, frame):
     sys.exit(-signo)
 

fuel_ccp/config/cli.py

@@ -21,6 +21,16 @@ def add_parsers(subparsers):
 
     subparsers.add_parser('fetch')
 
+    cleanup_action = subparsers.add_parser('cleanup')
+    # Making auth url configurable at least until Ingress/LB support will
+    # be implemented
+    cleanup_action.add_argument('--auth-url',
+                                help='The URL of Keystone authentication '
+                                     'server')
+    cleanup_action.add_argument('--skip-os-cleanup',
+                                action='store_true',
+                                help='Skip cleanup of OpenStack environment')
+
 
 CONF.register_cli_opt(cfg.SubCommandOpt('action',
                                         handler=add_parsers))

fuel_ccp/config/kubernetes.py

@@ -7,7 +7,7 @@ kubernetes_opts = [
                default='127.0.0.1:8080',
                help='Addres and port for kube-apiserver'),
     cfg.StrOpt('namespace',
-               default='default',
+               default='ccp',
                help='The name of the namespace'),
     cfg.StrOpt('ca-certs',
                help='The location of the CA certificate files'),

fuel_ccp/tests/test_cleanup.py

@@ -0,0 +1,82 @@
+import mock
+
+from k8sclient.client import rest
+
+from fuel_ccp import cleanup
+from fuel_ccp.tests import base
+
+
+class TestCleanup(base.TestCase):
+    @mock.patch('time.sleep')
+    def test_wait_until_empty(self, m_sleep):
+        # resources were deleted
+        test_command = mock.Mock(return_value=[])
+        res = cleanup._wait_until_empty(3, None, test_command)
+        m_sleep.assert_not_called()
+        self.assertIsNone(res)
+
+        # resources are still exist
+        m_sleep.reset_mock()
+        test_command = mock.Mock(return_value=['something'])
+        res = cleanup._wait_until_empty(3, None, test_command)
+        self.assertEqual(3, m_sleep.call_count)
+        self.assertEqual(['something'], res)
+
+    @mock.patch('time.sleep')
+    def test_wait_for_namespace_delete(self, m_sleep):
+        # namespace was deleted
+        k8s_api = mock.Mock()
+        k8s_api.read_namespaced_namespace.side_effect = [
+            'ns', 'ns', rest.ApiException(404)]
+        cleanup._wait_for_namespace_delete(k8s_api)
+        self.assertEqual(2, m_sleep.call_count)
+
+        # namespace is still exists
+        k8s_api = mock.Mock()
+        k8s_api.read_namespaced_namespace.return_value = 'ns'
+        self.assertRaisesRegexp(
+            RuntimeError, "Wasn't able to delete namespace ccp",
+            cleanup._wait_for_namespace_delete, k8s_api)
+
+    @mock.patch('time.sleep')
+    @mock.patch('neutronclient.v2_0.client.Client')
+    def test_cleanup_network_resources(self, m_client, m_sleep):
+        # subnets were not removed
+        neutron = mock.Mock()
+        neutron.list_subnets.return_value = {
+            'subnets': [{'id': 1, 'name': 'subnet1'}]}
+        m_client.return_value = neutron
+        self.assertRaisesRegexp(
+            RuntimeError, "Some subnets were not removed: subnet1 \(1\)",
+            cleanup._cleanup_network_resources, mock.Mock())
+
+        # subnets were removed but networks were not
+        neutron.list_subnets.return_value = {'subnets': []}
+        neutron.list_networks.return_value = {
+            'networks': [{'id': 1, 'name': 'net1'}]}
+        self.assertRaisesRegexp(
+            RuntimeError, "Some networks were not removed: net1 \(1\)",
+            cleanup._cleanup_network_resources, mock.Mock())
+
+        # subnets and networks were removed
+        neutron.list_networks.return_value = {'networks': []}
+        cleanup._cleanup_network_resources(mock.Mock())
+
+    @mock.patch('time.sleep')
+    @mock.patch('novaclient.client.Client')
+    def test_cleanup_servers(self, m_client, m_sleep):
+        # instances were not removed
+        nova = mock.Mock()
+        instance = mock.Mock(id=1)
+        instance.name = 'inst1'
+
+        nova.servers.list.return_value = [instance]
+        m_client.return_value = nova
+        self.assertRaisesRegexp(
+            RuntimeError, "Some instances were not removed "
+                          "after force delete: inst1 \(1\)",
+            cleanup._cleanup_servers, mock.Mock())
+
+        # instances were removed
+        nova.servers.list.return_value = []
+        cleanup._cleanup_servers(mock.Mock())

fuel_ccp/tests/test_kubernetes.py

@@ -45,7 +45,7 @@ class TestKubernetes(base.TestCase):
         kubernetes.create_object_from_definition(
             deployment_dict, client=mock.Mock())
         api.create_namespaced_deployment.assert_called_once_with(
-            body=deployment_dict, namespace='default')
+            body=deployment_dict, namespace='ccp')
 
     @mock.patch('k8sclient.client.apis.apiv_api.ApivApi')
     def test_create_service(self, api_v1):
@@ -59,4 +59,4 @@ class TestKubernetes(base.TestCase):
             service_dict, client=mock.Mock())
 
         api.create_namespaced_service.assert_called_once_with(
-            body=service_dict, namespace='default')
+            body=service_dict, namespace='ccp')

requirements.txt

@@ -13,3 +13,6 @@ oslo.log>=1.14.0  # Apache-2.0
 PyYAML>=3.1.0  # MIT
 six>=1.9.0  # MIT
 python-k8sclient
+keystoneauth1>=2.7.0  # Apache-2.0
+python-neutronclient>=4.2.0  # Apache-2.0
+python-novaclient>=2.29.0,!=2.33.0  # Apache-2.0