Improve runtime functional tests

Change-Id: I5f7fd65592097cffd55ebab07d9d00d261aa8a44

devstack/plugin.sh

@@ -23,6 +23,9 @@ function install_k8s {
     pushd $QINLING_DIR
     source tools/gate/setup_gate.sh
     popd
+
+    # Pre-pull the default docker image for python runtime
+    sudo docker pull $QINLING_PYTHON_RUNTIME_IMAGE
 }
 
 

devstack/settings

@@ -21,3 +21,4 @@ QINLING_CONF_DIR=${QINLING_CONF_DIR:-/etc/qinling}
 QINLING_CONF_FILE=${QINLING_CONF_DIR}/qinling.conf
 QINLING_AUTH_CACHE_DIR=${QINLING_AUTH_CACHE_DIR:-/var/cache/qinling}
 QINLING_FUNCTION_STORAGE_DIR=${QINLING_FUNCTION_STORAGE_DIR:-/opt/qinling/funtion/packages}
+QINLING_PYTHON_RUNTIME_IMAGE=${QINLING_PYTHON_RUNTIME_IMAGE:-openstackqinling/python-runtime}

qinling/orchestrator/kubernetes/manager.py

@@ -20,6 +20,7 @@ import jinja2
 from kubernetes import client
 from oslo_log import log as logging
 import requests
+import tenacity
 import yaml
 
 from qinling import context
@@ -77,6 +78,22 @@ class KubernetesManager(base.OrchestratorBase):
 
             LOG.info('Namespace %s created.', self.conf.kubernetes.namespace)
 
+    @tenacity.retry(
+        wait=tenacity.wait_fixed(2),
+        stop=tenacity.stop_after_delay(600),
+        retry=tenacity.retry_if_result(lambda result: not result)
+    )
+    def _wait_deployment_available(self, name):
+        ret = self.v1extention.read_namespaced_deployment(
+            name,
+            self.conf.kubernetes.namespace
+        )
+
+        if not ret.status.replicas:
+            return False
+
+        return ret.status.replicas == ret.status.available_replicas
+
     def create_pool(self, name, image, labels=None):
         deployment_body = self.deployment_template.render(
             {
@@ -97,6 +114,8 @@ class KubernetesManager(base.OrchestratorBase):
             namespace=self.conf.kubernetes.namespace
         )
 
+        self._wait_deployment_available(name)
+
         LOG.info("Deployment for runtime %s created.", name)
 
     def delete_pool(self, name, labels=None):

qinling_tempest_plugin/tests/api/test_runtimes.py

@@ -11,9 +11,9 @@
 #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
-
 from tempest.lib.common.utils import data_utils
 from tempest.lib import decorators
+import tenacity
 
 from qinling_tempest_plugin.tests import base
 
@@ -21,8 +21,19 @@ from qinling_tempest_plugin.tests import base
 class RuntimesTest(base.BaseQinlingTest):
     name_prefix = 'RuntimesTest'
 
+    @tenacity.retry(
+        wait=tenacity.wait_fixed(2),
+        stop=tenacity.stop_after_delay(10),
+        retry=tenacity.retry_if_exception_type(AssertionError)
+    )
+    def _await_runtime_available(self, id):
+        resp, body = self.qinling_client.get_obj('runtimes', id)
+
+        self.assertEqual(200, resp.status)
+        self.assertEqual('available', body['status'])
+
     @decorators.idempotent_id('fdc2f07f-dd1d-4981-86d3-5bc7908d9a9b')
-    def test_create_delete_runtime(self):
+    def test_create_list_get_delete_runtime(self):
         name = data_utils.rand_name('runtime', prefix=self.name_prefix)
 
         req_body = {
@@ -35,6 +46,7 @@ class RuntimesTest(base.BaseQinlingTest):
         self.assertEqual(201, resp.status)
         self.assertEqual(name, body['name'])
 
+        # Get runtimes
         resp, body = self.qinling_client.get_list_objs('runtimes')
 
         self.assertEqual(200, resp.status)
@@ -43,13 +55,21 @@ class RuntimesTest(base.BaseQinlingTest):
             [runtime['id'] for runtime in body['runtimes']]
         )
 
+        # Wait for runtime to be available
+        self._await_runtime_available(runtime_id)
+
+        # Check k8s resource
         deploy = self.k8s_v1extention.read_namespaced_deployment(
             runtime_id,
             namespace=self.namespace
         )
 
         self.assertEqual(runtime_id, deploy.metadata.name)
+        self.assertEqual(
+            deploy.status.replicas, deploy.status.available_replicas
+        )
 
+        # Delete runtime
         resp, _ = self.qinling_client.delete_obj('runtimes', runtime_id)
 
         self.assertEqual(204, resp.status)

requirements.txt

@@ -28,3 +28,4 @@ PyYAML>=3.10.0 # MIT
 python-swiftclient>=3.2.0 # Apache-2.0
 croniter>=0.3.4 # MIT License
 python-dateutil>=2.4.2  # BSD
+tenacity>=3.2.1  # Apache-2.0