create apis for deploy, upgrade, host_destroy, host_precheck

- create apis for deploy, upgrade, host_destroy, host_precheck
- change CLI to use new APIs
- create new async api class for job-based apis
- add skeletal upgrade utest
- add some -v and -vv's to utests
- fix newly introduced bug in log_collector
- fix some pep8 warnings

Jira-Issue: OSTACKDEV-20

kollacli/api/async.py

@@ -0,0 +1,66 @@
+# Copyright(c) 2016, Oracle and/or its affiliates.  All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+import logging
+
+from kollacli.api.job import Job
+from kollacli.common.ansible import actions
+
+LOG = logging.getLogger(__name__)
+
+
+class AsyncApi(object):
+
+    # TODO(bmace) -- update this to only take host names
+    # and we will probably only support compute host individual deploys
+    def async_deploy(self, hostnames=[], groupnames=[], servicenames=[],
+                     serial_flag=False, verbose_level=1):
+        """Deploy.
+
+        Deploy containers to hosts.
+        """
+        ansible_job = actions.deploy(hostnames, groupnames, servicenames,
+                                     serial_flag, verbose_level)
+        return Job(ansible_job)
+
+    def async_upgrade(self, verbose_level=1):
+        """Upgrade.
+
+        Upgrade containers to new version specified by the property
+        "openstack_release."
+        """
+        ansible_job = actions.upgrade(verbose_level)
+        return Job(ansible_job)
+
+    def async_host_destroy(self, hostname, destroy_type, verbose_level=1,
+                           include_data=False):
+        """Destroy Hosts.
+
+        Stops and removes all kolla related docker containers on either the
+        specified host or all hosts if hostname is "all".
+        """
+        ansible_job = actions.destroy_hosts(hostname, destroy_type,
+                                            verbose_level, include_data)
+        return Job(ansible_job)
+
+    def async_host_precheck(self, hostname, verbose_level=1):
+        """Check pre-deployment configuration of host(s).
+
+        Check if host is ready for a new deployment. This will fail if
+        the host is not configured correctly or if it has already been
+        deployed to.
+
+        If hostname is "all", then check all hosts.
+        """
+        ansible_job = actions.precheck(hostname, verbose_level)
+        return Job(ansible_job)

kollacli/api/client.py

@@ -13,6 +13,7 @@
 #    under the License.
 import logging
 
+from kollacli.api.async import AsyncApi
 from kollacli.api.deploy import DeployApi
 from kollacli.api.host import HostApi
 
@@ -20,6 +21,7 @@ LOG = logging.getLogger(__name__)
 
 
 class ClientApi(
+        AsyncApi,
         DeployApi,
         HostApi
         ):

kollacli/api/deploy.py

@@ -13,21 +13,13 @@
 #    under the License.
 import logging
 
-LOG = logging.getLogger(__name__)
-
-from kollacli.common.ansible import actions
 from kollacli.common.inventory import Inventory
 
+LOG = logging.getLogger(__name__)
+
 
 class DeployApi(object):
 
-    # TODO(bmace) -- update this to only take host names
-    # and we will probably only support compute host individual deploys
-    def deploy(self, hostnames=[], groupnames=[], servicenames=[],
-               serial_flag=False, verbose_level=1):
-        actions.deploy(hostnames, groupnames, servicenames,
-                       serial_flag, verbose_level)
-
     def deploy_set_mode(self, remote_mode):
         inventory = Inventory.load()
         inventory.set_deploy_mode(remote_mode)

kollacli/api/job.py

@@ -0,0 +1,50 @@
+# Copyright(c) 2016, Oracle and/or its affiliates.  All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+class Job(object):
+    def __init__(self, ansible_job):
+        self._ansible_job = ansible_job
+
+    def wait(self):
+        """wait for job to complete
+
+        return status of job (see get_status() for status values)
+        """
+        return self._ansible_job.wait()
+
+    def get_status(self):
+        """get status of job
+
+        Status:
+        - None: still running
+        - 0: complete/success
+        - 1: complete/fail
+        """
+        return self._ansible_job.get_status()
+
+    def get_error_message(self):
+        """get error message
+
+        if job failed, this will return a string with the error message.
+        """
+        return self._ansible_job.get_error_message()
+
+    def get_console_output(self):
+        """get command output
+
+        get the console output from the job. Returns a string
+        containing the console output of the job.
+        """
+        return self._ansible_job.get_command_output()

kollacli/commands/deploy.py

@@ -62,7 +62,19 @@ class Deploy(Command):
             if parsed_args.serial:
                 serial_flag = True
 
-            CLIENT.deploy(hosts, groups, services, serial_flag, verbose_level)
+            job = CLIENT.async_deploy(hosts, groups, services, serial_flag,
+                                      verbose_level)
+            status = job.wait()
+            if verbose_level > 2:
+                LOG.info('\n\n' + 80 * '=')
+                LOG.info(u._('DEBUG command output:\n{out}')
+                         .format(out=job.get_console_output()))
+            if status == 0:
+                LOG.info(u._('Success'))
+            else:
+                raise CommandError(u._('Job failed:\n{msg}')
+                                   .format(msg=job.get_error_message()))
+
         except Exception:
             raise Exception(traceback.format_exc())
 

kollacli/commands/host.py

@@ -21,8 +21,6 @@ import yaml
 import kollacli.i18n as u
 
 from kollacli.api.client import ClientApi
-from kollacli.common.ansible.actions import destroy_hosts
-from kollacli.common.ansible.actions import precheck
 from kollacli.common.inventory import Inventory
 from kollacli.common.utils import convert_to_unicode
 from kollacli.common.utils import get_setup_user
@@ -99,7 +97,16 @@ class HostDestroy(Command):
 
             verbose_level = self.app.options.verbose_level
 
-            destroy_hosts(hostname, destroy_type, verbose_level, include_data)
+            job = CLIENT.async_host_destroy(hostname, destroy_type,
+                                            verbose_level, include_data)
+            status = job.wait()
+            if verbose_level > 2:
+                LOG.info('\n\n' + 80 * '=')
+                LOG.info(u._('DEBUG command output:\n{out}')
+                         .format(out=job.get_console_output()))
+            if status != 0:
+                raise CommandError(u._('Job failed:\n{msg}')
+                                   .format(msg=job.get_error_message()))
 
         except CommandError as e:
             raise e
@@ -193,7 +200,17 @@ class HostCheck(Command):
 
             if parsed_args.predeploy:
                 # run pre-deploy checks
-                precheck(hostname)
+                verbose_level = self.app.options.verbose_level
+                job = CLIENT.async_host_precheck(hostname, verbose_level)
+                status = job.wait()
+                if verbose_level > 2:
+                    LOG.info('\n\n' + 80 * '=')
+                    LOG.info(u._('DEBUG command output:\n{out}')
+                             .format(out=job.get_console_output()))
+                if status != 0:
+                    raise CommandError(u._('Job failed:\n{msg}')
+                                       .format(msg=job.get_error_message()))
+
             else:
                 # run ssh checks
                 all_ok = True

kollacli/commands/upgrade.py

@@ -14,10 +14,16 @@
 import logging
 import traceback
 
-from kollacli.common.ansible.actions import upgrade
-
 from cliff.command import Command
 
+import kollacli.i18n as u
+
+from kollacli.api.client import ClientApi
+from kollacli.exceptions import CommandError
+
+
+CLIENT = ClientApi()
+
 LOG = logging.getLogger(__name__)
 
 
@@ -28,9 +34,19 @@ class Upgrade(Command):
         return parser
 
     def take_action(self, parsed_args):
-        verbose_level = self.app.options.verbose_level
         try:
-            upgrade(verbose_level)
+            verbose_level = self.app.options.verbose_level
+            job = CLIENT.async_upgrade(verbose_level)
+            status = job.wait()
+            if verbose_level > 2:
+                LOG.info('\n\n' + 80 * '=')
+                LOG.info(u._('DEBUG command output:\n{out}')
+                         .format(out=job.get_console_output()))
+            if status == 0:
+                LOG.info(u._('Success'))
+            else:
+                raise CommandError(u._('Job failed:\n{msg}')
+                                   .format(msg=job.get_error_message()))
 
         except Exception:
             raise Exception(traceback.format_exc())

kollacli/common/ansible/actions.py

@@ -65,7 +65,7 @@ def destroy_hosts(hostname, destroy_type, verbose_level=1, include_data=False):
         playbook.print_output = False
     playbook.verbose_level = verbose_level
     job = playbook.run()
-    _process_job(job, verbose_level)
+    return job
 
 
 def deploy(hostnames=[], groupnames=[], servicenames=[],
@@ -84,7 +84,7 @@ def deploy(hostnames=[], groupnames=[], servicenames=[],
     _run_deploy_rules(playbook)
 
     job = playbook.run()
-    _process_job(job, verbose_level)
+    return job
 
 
 def precheck(hostname, verbose_level=1):
@@ -102,7 +102,7 @@ def precheck(hostname, verbose_level=1):
     playbook.print_output = True
     playbook.verbose_level = verbose_level
     job = playbook.run()
-    _process_job(job, verbose_level)
+    return job
 
 
 def upgrade(verbose_level=1):
@@ -114,19 +114,7 @@ def upgrade(verbose_level=1):
     playbook.print_output = True
     playbook.verbose_level = verbose_level
     job = playbook.run()
-    _process_job(job, verbose_level)
+    return job
-
-
-def _process_job(job, verbose_level):
-    job.wait()
-    status = job.get_status()
-    if status != 0:
-        if verbose_level > 2:
-            LOG.info('\n\n' + 80 * '=')
-            LOG.info('DEBUG command output:\n%s'
-                     % job.get_command_output())
-        raise CommandError(u._('Ansible command failed:\n{msg}')
-                           .format(msg=job.get_error_message()))
 
 
 def _run_deploy_rules(playbook):

kollacli/common/ansible/job.py

@@ -20,8 +20,8 @@ import subprocess  # nosec
 import tempfile
 import time
 
-from kollacli.common.utils import get_admin_uids
 from kollacli.common.inventory import remove_temp_inventory
+from kollacli.common.utils import get_admin_uids
 from kollacli.common.utils import safe_decode
 
 LOG = logging.getLogger(__name__)

tests/deploy.py

@@ -137,14 +137,19 @@ class TestFunctional(KollaCliTest):
                 self.run_cli_cmd('property set enable_%s no' % service)
 
             self.run_cli_cmd('deploy')
-            self.run_cli_cmd('deploy --serial')
+            self.run_cli_cmd('deploy --serial -v')
-            self.run_cli_cmd('deploy --groups=control')
+            self.run_cli_cmd('deploy --groups=control -vv')
 
         finally:
             # re-enable services after the test
             for service in ALL_SERVICES:
                 self.run_cli_cmd('property set enable_%s yes' % service)
 
+    def test_upgrade(self):
+        # test will upgrade an environment with no hosts, mostly a NOP,
+        # but it will go through the client code paths.
+        self.run_cli_cmd('upgrade -v')
+
     def check_json(self, msg, groups, hosts, included_groups, included_hosts):
         err_msg = ('included groups: %s\n' % included_groups +
                    'included hosts: %s\n' % included_hosts)

tests/destroy.py

@@ -112,7 +112,7 @@ class TestFunctional(KollaCliTest):
         # destroy non-data services (via --stop flag)
         # this should leave only data containers running
         try:
-            self.run_cli_cmd('host destroy %s --stop' % hostname)
+            self.run_cli_cmd('host destroy %s --stop -v' % hostname)
         except Exception as e:
             self.assertFalse(is_physical_host, '2nd destroy exception: %s' % e)
             self.assertIn(UNKNOWN_HOST, '%s' % e,
@@ -133,7 +133,8 @@ class TestFunctional(KollaCliTest):
                               'after no-data destroy.')
 
         try:
-            self.run_cli_cmd('host destroy %s --includedata --stop' % hostname)
+            self.run_cli_cmd('host destroy %s --includedata --stop -vv'
+                             % hostname)
         except Exception as e:
             self.assertFalse(is_physical_host, '3rd destroy exception: %s' % e)
             self.assertIn(UNKNOWN_HOST, '%s' % e,

tools/log_collector.py

@@ -20,10 +20,10 @@ import tarfile
 import tempfile
 
 from kollacli.common.inventory import Inventory
+from kollacli.common.inventory import remove_temp_inventory
 from kollacli.common import properties
 from kollacli.common.utils import get_admin_user
 from kollacli.common.utils import get_ansible_command
-from kollacli.common.utils import remove_temp_inventory
 from kollacli.common.utils import safe_decode
 
 tar_file_descr = None