Add tests for lifecycle hooks

* add API test for complete lifecycle
* change existing tests to use deletion-1.1 policy
* add functional test for deletion policy with hooks
* add integration test for deletion policy with hooks

Depends-On: I888a01c4f26959649121d6f82430017858a4c481

Change-Id: I5e2d4b0c073b515d9697cf91cd89ea0e6a3c81b8

senlin_tempest_plugin/common/constants.py

@@ -20,8 +20,7 @@ spec_nova_server = {
         "image": "cirros-0.3.5-x86_64-disk",
         "networks": [
             {"network": "private"}
-        ],
-        "key_name": "oskey"
+        ]
     }
 }
 
@@ -115,3 +114,28 @@ spec_batch_policy = {
         "pause_time": 3
     }
 }
+
+
+spec_deletion_policy = {
+    "type": "senlin.policy.deletion",
+    "version": "1.1",
+    "properties": {
+        "criteria": "OLDEST_FIRST"
+    }
+}
+
+
+spec_deletion_policy_with_hook = {
+    "type": "senlin.policy.deletion",
+    "version": "1.1",
+    "properties": {
+        "hooks": {
+            "type": "zaqar",
+            "timeout": 300,
+            "params": {
+                "queue": "test_queue"
+            }
+        },
+        "criteria": "OLDEST_FIRST"
+    }
+}

senlin_tempest_plugin/common/messaging_client.py

@@ -43,6 +43,25 @@ class V2MessagingClient(rest_client.RestClient):
         res['body'] = self._parse_resp(body)
         return res
 
+    def create_queue(self, queue_name):
+        uri = '{0}/queues/{1}'.format(self.uri_prefix, queue_name)
+        resp, body = self.put(uri, '', extra_headers=True,
+                              headers=self.headers)
+
+        return self.get_resp(resp, body)
+
+    def delete_queue(self, queue_name):
+        uri = '{0}/queues/{1}'.format(self.uri_prefix, queue_name)
+        resp, body = self.delete(uri, extra_headers=True, headers=self.headers)
+
+        return self.get_resp(resp, body)
+
+    def list_messages(self, queue_name):
+        uri = '{0}/queues/{1}/messages'.format(self.uri_prefix, queue_name)
+        resp, body = self.get(uri, extra_headers=True, headers=self.headers)
+
+        return self.get_resp(resp, body)
+
     def post_messages(self, queue_name, messages):
         uri = '{0}/queues/{1}/messages'.format(self.uri_prefix, queue_name)
         resp, body = self.post(uri, body=jsonutils.dumps(messages),

senlin_tempest_plugin/common/utils.py

@@ -129,9 +129,15 @@ def update_a_cluster(base, cluster_id, profile_id=None, name=None,
     return res['body']
 
 
-def get_a_cluster(base, cluster_id):
+def get_a_cluster(base, cluster_id, expected_status=None, wait_timeout=None):
     """Utility function that gets a Senlin cluster."""
-    res = base.client.get_obj('clusters', cluster_id)
+    if expected_status is None:
+        res = base.client.get_obj('clusters', cluster_id)
+    else:
+        base.client.wait_for_status('clusters', cluster_id, expected_status,
+                                    wait_timeout)
+        res = base.client.get_obj('clusters', cluster_id)
+
     return res['body']
 
 
@@ -254,6 +260,12 @@ def delete_a_policy(base, policy_id, ignore_missing=False):
     return
 
 
+def get_a_action(base, action_id):
+    """Utility function that gets a Senlin action."""
+    res = base.client.get_obj('actions', action_id)
+    return res['body']
+
+
 def cluster_attach_policy(base, cluster_id, policy_id,
                           expected_status='SUCCEEDED', wait_timeout=None):
     """Utility function that attach a policy to cluster."""
@@ -366,7 +378,7 @@ def cluster_scale_in(base, cluster_id, count=None,
     action_id = res['location'].split('/actions/')[1]
     res = base.client.wait_for_status('actions', action_id, expected_status,
                                       wait_timeout)
-    return res['body']['status_reason']
+    return res['body']['status_reason'], action_id
 
 
 def cluster_resize(base, cluster_id, adj_type=None, number=None, min_size=None,
@@ -391,6 +403,22 @@ def cluster_resize(base, cluster_id, adj_type=None, number=None, min_size=None,
     return res['body']['status_reason']
 
 
+def cluster_complete_lifecycle(base, cluster_id, lifecycle_action_token,
+                               expected_status='SUCCEEDED', wait_timeout=None):
+    """Utility function that completes lifecycle for a cluster."""
+
+    params = {
+        'complete_lifecycle': {
+            'lifecycle_action_token': lifecycle_action_token
+        }
+    }
+    res = base.client.trigger_action('clusters', cluster_id, params=params)
+    action_id = res['location'].split('/actions/')[1]
+    res = base.client.wait_for_status('actions', action_id, expected_status,
+                                      wait_timeout)
+    return res['body']['status_reason']
+
+
 def create_a_receiver(base, cluster_id, action, r_type=None, name=None,
                       params=None):
     """Utility function that generates a Senlin receiver."""
@@ -518,6 +546,35 @@ def delete_a_subnet(base, subnet_id, ignore_missing=False):
         raise exceptions.NotFound()
 
 
+def create_queue(base, queue_name):
+    """Utility function that creates Zaqar queue."""
+    res = base.messaging_client.create_queue(queue_name)
+
+    if res['status'] != 201 and res['status'] != 204:
+        msg = 'Failed in creating Zaqar queue %s' % queue_name
+        raise Exception(msg)
+
+
+def delete_queue(base, queue_name):
+    """Utility function that deletes Zaqar queue."""
+    res = base.messaging_client.delete_queue(queue_name)
+
+    if res['status'] != 204:
+        msg = 'Failed in deleting Zaqar queue %s' % queue_name
+        raise Exception(msg)
+
+
+def list_messages(base, queue_name):
+    """Utility function that lists messages in Zaqar queue."""
+    res = base.messaging_client.list_messages(queue_name)
+
+    if res['status'] != 200:
+        msg = 'Failed in listing messsages for Zaqar queue %s' % queue_name
+        raise Exception(msg)
+
+    return res['body']['messages']
+
+
 def post_messages(base, queue_name, messages):
     """Utility function that posts message(s) to Zaqar queue."""
     res = base.messaging_client.post_messages(queue_name,

senlin_tempest_plugin/tests/api/clusters/test_cluster_complete_lifecycle.py

@@ -0,0 +1,63 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from tempest.lib import decorators
+
+from senlin_tempest_plugin.common import utils
+from senlin_tempest_plugin.tests.api import base
+
+
+class TestClusterActionCompleteLifecycle(base.BaseSenlinAPITest):
+
+    def setUp(self):
+        super(TestClusterActionCompleteLifecycle, self).setUp()
+        profile_id = utils.create_a_profile(self)
+        self.addCleanup(utils.delete_a_profile, self, profile_id)
+        self.cluster_id = utils.create_a_cluster(self, profile_id,
+                                                 desired_capacity=1)
+        self.addCleanup(utils.delete_a_cluster, self, self.cluster_id)
+
+    @decorators.idempotent_id('d1338e7d-c954-4e1f-8601-dae0709e21fb')
+    def test_cluster_action_complete_lifecycle(self):
+        params = {
+            "scale_out": {
+                "count": "1"
+            }
+        }
+        # call cluster scale out to get a valid action id
+        res = self.client.trigger_action('clusters', self.cluster_id,
+                                         params=params)
+
+        # Verify resp code, body and location in headers
+        self.assertEqual(202, res['status'])
+        self.assertIn('actions', res['location'])
+
+        action_id = res['location'].split('/actions/')[1]
+        self.client.wait_for_status('actions', action_id, 'SUCCEEDED')
+
+        # use scale out action id to call complete_lifecycle
+        # since a valid action id is needed
+        params = {
+            "complete_lifecycle": {
+                "lifecycle_action_token": action_id
+            }
+        }
+        # Trigger cluster action
+        res = self.client.trigger_action('clusters', self.cluster_id,
+                                         params=params)
+
+        # Verify resp code, body and location in headers
+        self.assertEqual(202, res['status'])
+        self.assertIn('actions', res['location'])
+
+        action_id = res['location'].split('/actions/')[1]
+        self.client.wait_for_status('actions', action_id, 'SUCCEEDED')

senlin_tempest_plugin/tests/api/policy_types/test_policy_type_show.py

@@ -21,7 +21,7 @@ class TestPolicyTypeShow(base.BaseSenlinAPITest):
     @utils.api_microversion('1.4')
     @decorators.idempotent_id('57791ed7-7f57-4369-ba6e-7e039169ebdc')
     def test_policy_type_show(self):
-        res = self.client.get_obj('policy-types', 'senlin.policy.scaling-1.0')
+        res = self.client.get_obj('policy-types', 'senlin.policy.deletion-1.1')
 
         # Verify resp of policy type show API
         self.assertEqual(200, res['status'])
@@ -29,12 +29,12 @@ class TestPolicyTypeShow(base.BaseSenlinAPITest):
         policy_type = res['body']
         for key in ['name', 'schema']:
             self.assertIn(key, policy_type)
-        self.assertEqual('senlin.policy.scaling-1.0', policy_type['name'])
+        self.assertEqual('senlin.policy.deletion-1.1', policy_type['name'])
 
     @utils.api_microversion('1.5')
     @decorators.idempotent_id('1900b22a-012d-41f0-85a2-8aa6b65ec2ca')
     def test_policy_type_show_v1_5(self):
-        res = self.client.get_obj('policy-types', 'senlin.policy.scaling-1.0')
+        res = self.client.get_obj('policy-types', 'senlin.policy.deletion-1.1')
 
         # Verify resp of policy type show API
         self.assertEqual(200, res['status'])
@@ -42,5 +42,5 @@ class TestPolicyTypeShow(base.BaseSenlinAPITest):
         policy_type = res['body']
         for key in ['name', 'schema', 'support_status']:
             self.assertIn(key, policy_type)
-        self.assertEqual('senlin.policy.scaling-1.0', policy_type['name'])
+        self.assertEqual('senlin.policy.deletion-1.1', policy_type['name'])
         self.assertIsNotNone(policy_type['support_status'])

senlin_tempest_plugin/tests/functional/test_cluster_scaling.py

@@ -80,8 +80,8 @@ class TestClusterScaleInOut(base.BaseSenlinFunctionalTest):
         self.assertEqual(1, len(cluster['nodes']))
 
         # Keep scaling in cluster
-        res = utils.cluster_scale_in(self, self.cluster_id,
-                                     expected_status='FAILED')
+        res, action_id = utils.cluster_scale_in(self, self.cluster_id,
+                                                expected_status='FAILED')
 
         # Verify action result and action failure reason
         cluster = utils.get_a_cluster(self, self.cluster_id)

senlin_tempest_plugin/tests/functional/test_deletion_policy.py

@@ -0,0 +1,144 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from tempest.lib import decorators
+
+from senlin_tempest_plugin.common import constants
+from senlin_tempest_plugin.common import utils
+from senlin_tempest_plugin.tests.functional import base
+
+
+class TestDeletionPolicy(base.BaseSenlinFunctionalTest):
+
+    def setUp(self):
+        super(TestDeletionPolicy, self).setUp()
+        self.profile_id = utils.create_a_profile(self)
+        self.addCleanup(utils.delete_a_profile, self, self.profile_id)
+        self.cluster_id = utils.create_a_cluster(self, self.profile_id,
+                                                 min_size=0, max_size=5,
+                                                 desired_capacity=2)
+        self.addCleanup(utils.delete_a_cluster, self, self.cluster_id)
+
+    @decorators.attr(type=['functional'])
+    @decorators.idempotent_id('bc4af63f-c236-4e8c-a644-d6211c2ec160')
+    def test_deletion_policy(self):
+        # Create a deletion policy
+        spec = constants.spec_deletion_policy
+        policy_id = utils.create_a_policy(self, spec)
+        del_policy = utils.get_a_policy(self, policy_id)
+        self.addCleanup(utils.delete_a_policy, self, del_policy['id'])
+
+        # Attach deletion policy to cluster
+        utils.cluster_attach_policy(self, self.cluster_id, del_policy['id'])
+        self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id,
+                        del_policy['id'])
+
+        # Scale out cluster
+        utils.cluster_scale_out(self, self.cluster_id)
+
+        # Verify scale out result
+        cluster = utils.get_a_cluster(self, self.cluster_id)
+        self.assertEqual('ACTIVE', cluster['status'])
+        self.assertEqual(3, cluster['desired_capacity'])
+        self.assertEqual(3, len(cluster['nodes']))
+
+        # Scale in cluster
+        utils.cluster_scale_in(self, self.cluster_id)
+
+        # Verify scale in result
+        cluster = utils.get_a_cluster(self, self.cluster_id)
+        self.assertEqual('ACTIVE', cluster['status'])
+        self.assertEqual(2, cluster['desired_capacity'])
+        self.assertEqual(2, len(cluster['nodes']))
+
+    @decorators.attr(type=['functional'])
+    @decorators.idempotent_id('b08a229f-2cd4-496a-950a-1daff78e4e70')
+    def test_deletion_policy_with_hook(self):
+        # Create a deletion policy with hook
+        spec = constants.spec_deletion_policy_with_hook
+        policy_id = utils.create_a_policy(self, spec)
+        del_policy = utils.get_a_policy(self, policy_id)
+        self.addCleanup(utils.delete_a_policy, self, del_policy['id'])
+
+        # Attach deletion policy to cluster
+        utils.cluster_attach_policy(self, self.cluster_id, del_policy['id'])
+        self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id,
+                        del_policy['id'])
+
+        # Scale out cluster
+        utils.cluster_scale_out(self, self.cluster_id)
+
+        # Verify scale out result
+        cluster = utils.get_a_cluster(self, self.cluster_id)
+        self.assertEqual('ACTIVE', cluster['status'])
+        self.assertEqual(3, cluster['desired_capacity'])
+        self.assertEqual(3, len(cluster['nodes']))
+
+        # Scale in cluster
+        _, action_id = utils.cluster_scale_in(self, self.cluster_id,
+                                              expected_status='WAITING')
+
+        # get action details of scale in action
+        action = utils.get_a_action(self, action_id)
+        self.assertTrue(1, len(action['depends_on']))
+
+        # get dependent action and check status
+        dep_action_id = action['depends_on'][0]
+        dep_action = utils.get_a_action(self, dep_action_id)
+        self.assertEqual('WAITING_LIFECYCLE_COMPLETION', dep_action['status'])
+
+        # complete lifecycle
+        utils.cluster_complete_lifecycle(self, self.cluster_id,
+                                         dep_action_id, wait_timeout=10)
+
+        # verify cluster has been scaled in
+        cluster = utils.get_a_cluster(self, self.cluster_id)
+        self.assertEqual('ACTIVE', cluster['status'])
+        self.assertEqual(2, cluster['desired_capacity'])
+        self.assertEqual(2, len(cluster['nodes']))
+
+    @decorators.attr(type=['functional'])
+    @decorators.idempotent_id('88ea4617-10a6-4005-a641-b9459418661f')
+    @decorators.skip_because(bug="1746123")
+    def test_deletion_policy_with_hook_timeout(self):
+        # Create a deletion policy with hook
+        spec = constants.spec_deletion_policy_with_hook
+        spec['properties']['hooks']['timeout'] = 1
+
+        policy_id = utils.create_a_policy(self, spec)
+        del_policy = utils.get_a_policy(self, policy_id)
+        self.addCleanup(utils.delete_a_policy, self, del_policy['id'])
+
+        # Attach deletion policy to cluster
+        utils.cluster_attach_policy(self, self.cluster_id, del_policy['id'])
+        self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id,
+                        del_policy['id'])
+
+        # Scale out cluster
+        utils.cluster_scale_out(self, self.cluster_id)
+
+        # Verify scale out result
+        cluster = utils.get_a_cluster(self, self.cluster_id)
+        self.assertEqual('ACTIVE', cluster['status'])
+        self.assertEqual(3, cluster['desired_capacity'])
+        self.assertEqual(3, len(cluster['nodes']))
+
+        # Scale in cluster
+        _, action_id = utils.cluster_scale_in(self, self.cluster_id,
+                                              expected_status='SUCCEEDED',
+                                              wait_timeout=10)
+
+        # verify cluster has been scaled in
+        cluster = utils.get_a_cluster(self, self.cluster_id)
+        self.assertEqual('ACTIVE', cluster['status'])
+        self.assertEqual(2, cluster['desired_capacity'])
+        self.assertEqual(2, len(cluster['nodes']))

senlin_tempest_plugin/tests/functional/test_scaling_policy.py

@@ -115,8 +115,8 @@ class TestScalingPolicy(base.BaseSenlinFunctionalTest):
 
         # Keep scaling in cluster with count set to 2 to
         # verify best_effort parameter
-        res = utils.cluster_scale_in(self, self.cluster_id, count=2,
-                                     expected_status='FAILED')
+        res, action_id = utils.cluster_scale_in(self, self.cluster_id, count=2,
+                                                expected_status='FAILED')
 
         # Verify action result and action failure reason
         cluster = utils.get_a_cluster(self, self.cluster_id)

senlin_tempest_plugin/tests/integration/test_lifecycle_hook_message.py

@@ -0,0 +1,89 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from tempest.lib import decorators
+
+from senlin_tempest_plugin.common import constants
+from senlin_tempest_plugin.common import utils
+from senlin_tempest_plugin.tests.integration import base
+
+
+class TestLifecycleHookMessage(base.BaseSenlinIntegrationTest):
+
+    def setUp(self):
+        super(TestLifecycleHookMessage, self).setUp()
+        self.profile_id = utils.create_a_profile(self)
+        self.addCleanup(utils.delete_a_profile, self, self.profile_id)
+        self.cluster_id = utils.create_a_cluster(self, self.profile_id,
+                                                 min_size=0, max_size=5,
+                                                 desired_capacity=2)
+        self.addCleanup(utils.delete_a_cluster, self, self.cluster_id)
+
+    @decorators.attr(type=['integration'])
+    @decorators.idempotent_id('9ac7ed9d-7338-45fb-b749-f67ddeb6caa2')
+    def test_lifecycle_hook_message(self):
+        # Create a deletion policy with hook
+        spec = constants.spec_deletion_policy_with_hook
+        policy_id = utils.create_a_policy(self, spec)
+        del_policy = utils.get_a_policy(self, policy_id)
+        self.addCleanup(utils.delete_a_policy, self, del_policy['id'])
+
+        # Create zaqar queue
+        queue_name = spec['properties']['hooks']['params']['queue']
+        utils.create_queue(self, queue_name)
+        self.addCleanup(utils.delete_queue, self, queue_name)
+
+        # Attach deletion policy to cluster
+        utils.cluster_attach_policy(self, self.cluster_id, del_policy['id'])
+        self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id,
+                        del_policy['id'])
+
+        # Scale out cluster
+        utils.cluster_scale_out(self, self.cluster_id)
+
+        # Verify scale out result
+        cluster = utils.get_a_cluster(self, self.cluster_id)
+        self.assertEqual('ACTIVE', cluster['status'])
+        self.assertEqual(3, cluster['desired_capacity'])
+        self.assertEqual(3, len(cluster['nodes']))
+
+        # Scale in cluster
+        _, action_id = utils.cluster_scale_in(self, self.cluster_id,
+                                              expected_status='WAITING')
+
+        # get action details of scale in action
+        action = utils.get_a_action(self, action_id)
+        self.assertTrue(1, len(action['depends_on']))
+
+        # get lifecycle hook message from zaqar queue
+        messages = utils.list_messages(self, queue_name)
+        self.assertEqual(1, len(messages))
+        lifecycle_hook_message = messages[0]['body']
+
+        # get dependent action and check status
+        dep_action_id = action['depends_on'][0]
+        dep_action = utils.get_a_action(self, dep_action_id)
+        self.assertEqual('WAITING_LIFECYCLE_COMPLETION', dep_action['status'])
+        self.assertEqual(dep_action_id,
+                         lifecycle_hook_message['lifecycle_action_token'])
+        self.assertEqual(dep_action['target'],
+                         lifecycle_hook_message['node_id'])
+
+        # complete lifecycle
+        utils.cluster_complete_lifecycle(self, self.cluster_id,
+                                         dep_action_id, wait_timeout=10)
+
+        # verify cluster has been scaled in
+        cluster = utils.get_a_cluster(self, self.cluster_id,
+                                      expected_status='ACTIVE')
+        self.assertEqual(2, cluster['desired_capacity'])
+        self.assertEqual(2, len(cluster['nodes']))