From 08ac9a6cbac5b324747c178c4cab36296c16b567 Mon Sep 17 00:00:00 2001
From: Corey O'Brien <corey.obrien@rackspace.com>
Date: Tue, 2 Feb 2016 15:57:40 -0500
Subject: [PATCH] Remove minion dependency on master

Make thhe minions work correctly without depending on the master.
Removing this dependency decreases provisioning time by provisioning
nodes in parallel.

Change-Id: I025dbf54ef0cdafc4fe26acc33a9faed8cacd1ca
Partial-Bug: 1536739
Partial-Bug: 1551824
---
 .testr.conf                                   |  2 +-
 .../fragments/configure-kubernetes-minion.sh  |  8 +++++
 magnum/templates/kubernetes/kubecluster.yaml  |  1 -
 magnum/tests/contrib/copy_instance_logs.sh    |  1 +
 .../functional/api/v1/clients/bay_client.py   |  6 ++--
 magnum/tests/functional/api/v1/test_bay.py    |  2 +-
 .../functional/k8s/test_k8s_python_client.py  | 12 +++++++
 magnum/tests/functional/python_client_base.py | 35 ++++++++++++-------
 8 files changed, 48 insertions(+), 19 deletions(-)

diff --git a/.testr.conf b/.testr.conf
index ae9ca6e90b..d39b6aa49e 100644
--- a/.testr.conf
+++ b/.testr.conf
@@ -1,7 +1,7 @@
 [DEFAULT]
 test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
              OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
-             OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-160} \
+             OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-45} \
              ${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./magnum/tests/unit} $LISTOPT $IDOPTION
 test_id_option=--load-list $IDFILE
 test_list_option=--list
diff --git a/magnum/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/templates/kubernetes/fragments/configure-kubernetes-minion.sh
index 015f5d398b..d5aaaf673c 100644
--- a/magnum/templates/kubernetes/fragments/configure-kubernetes-minion.sh
+++ b/magnum/templates/kubernetes/fragments/configure-kubernetes-minion.sh
@@ -36,6 +36,14 @@ if [ "$NETWORK_DRIVER" == "flannel" ]; then
     sed -i '
       /^FLANNEL_ETCD=/ s|=.*|="http://'"$ETCD_SERVER_IP"':2379"|
     ' /etc/sysconfig/flanneld
+
+    # Make sure etcd has a flannel configuration
+    . /etc/sysconfig/flanneld
+    until curl -sf "$FLANNEL_ETCD/v2/keys/coreos.com/network/config?quorum=false&recursive=false&sorted=false"
+    do
+        echo "Waiting for flannel configuration in etcd..."
+        sleep 5
+    done
 fi
 
 cat >> /etc/environment <<EOF
diff --git a/magnum/templates/kubernetes/kubecluster.yaml b/magnum/templates/kubernetes/kubecluster.yaml
index 16c32e9cb9..3d7fadd2e5 100644
--- a/magnum/templates/kubernetes/kubecluster.yaml
+++ b/magnum/templates/kubernetes/kubecluster.yaml
@@ -465,7 +465,6 @@ resources:
     type: OS::Heat::ResourceGroup
     depends_on:
       - extrouter_inside
-      - kube_masters
     properties:
       count: {get_param: number_of_minions}
       removal_policies: [{resource_list: {get_param: minions_to_remove}}]
diff --git a/magnum/tests/contrib/copy_instance_logs.sh b/magnum/tests/contrib/copy_instance_logs.sh
index 1c55bf4199..f9fcd20040 100755
--- a/magnum/tests/contrib/copy_instance_logs.sh
+++ b/magnum/tests/contrib/copy_instance_logs.sh
@@ -48,6 +48,7 @@ if [[ "$COE" == "kubernetes" ]]; then
     remote_exec $SSH_USER "sudo journalctl -u cloud-final --no-pager" cloud-final.log
     remote_exec $SSH_USER "sudo journalctl -u cloud-init-local --no-pager" cloud-init-local.log
     remote_exec $SSH_USER "sudo journalctl -u cloud-init --no-pager" cloud-init.log
+    remote_exec $SSH_USER "sudo cat /var/log/cloud-init-output.log" cloud-init-output.log
     remote_exec $SSH_USER "sudo journalctl -u kubelet --no-pager" kubelet.log
     remote_exec $SSH_USER "sudo journalctl -u kube-proxy --no-pager" kube-proxy.log
     remote_exec $SSH_USER "sudo journalctl -u etcd --no-pager" etcd.log
diff --git a/magnum/tests/functional/api/v1/clients/bay_client.py b/magnum/tests/functional/api/v1/clients/bay_client.py
index 7cd415949f..854ad05549 100644
--- a/magnum/tests/functional/api/v1/clients/bay_client.py
+++ b/magnum/tests/functional/api/v1/clients/bay_client.py
@@ -112,12 +112,12 @@ class BayClient(client.MagnumClient):
 
     def wait_for_bay_to_delete(self, bay_id):
         utils.wait_for_condition(
-            lambda: self.does_bay_not_exist(bay_id), 10, 3600)
+            lambda: self.does_bay_not_exist(bay_id), 10, 600)
 
     def wait_for_created_bay(self, bay_id, delete_on_error=True):
         try:
             utils.wait_for_condition(
-                lambda: self.does_bay_exist(bay_id), 10, 3600)
+                lambda: self.does_bay_exist(bay_id), 10, 1800)
         except Exception:
             # In error state.  Clean up the bay id if desired
             self.LOG.error('Bay %s entered an exception state.' % bay_id)
@@ -129,7 +129,7 @@ class BayClient(client.MagnumClient):
 
     def wait_for_final_state(self, bay_id):
         utils.wait_for_condition(
-            lambda: self.is_bay_in_final_state(bay_id), 10, 3600)
+            lambda: self.is_bay_in_final_state(bay_id), 10, 1800)
 
     def is_bay_in_final_state(self, bay_id):
         try:
diff --git a/magnum/tests/functional/api/v1/test_bay.py b/magnum/tests/functional/api/v1/test_bay.py
index a08a11a9a1..c73555acfb 100644
--- a/magnum/tests/functional/api/v1/test_bay.py
+++ b/magnum/tests/functional/api/v1/test_bay.py
@@ -61,7 +61,7 @@ class BayTest(base.BaseMagnumTest):
 
             # NOTE (dimtruck) by default tempest sets timeout to 20 mins.
             # We need more time.
-            test_timeout = 3600
+            test_timeout = 1800
             self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
         except Exception:
             self.tearDown()
diff --git a/magnum/tests/functional/k8s/test_k8s_python_client.py b/magnum/tests/functional/k8s/test_k8s_python_client.py
index b5c77ca8f2..dcd0162ac6 100644
--- a/magnum/tests/functional/k8s/test_k8s_python_client.py
+++ b/magnum/tests/functional/k8s/test_k8s_python_client.py
@@ -12,6 +12,7 @@
 
 from magnum.common.pythonk8sclient.swagger_client import api_client
 from magnum.common.pythonk8sclient.swagger_client.apis import apiv_api
+from magnum.tests.functional.common import utils
 from magnum.tests.functional.python_client_base import BayTest
 
 
@@ -32,6 +33,17 @@ class TestKubernetesAPIs(BayTest):
                                           cert_file=self.cert_file,
                                           ca_certs=self.ca_file)
         self.k8s_api = apiv_api.ApivApi(k8s_client)
+        # TODO(coreypobrien) https://bugs.launchpad.net/magnum/+bug/1551824
+        utils.wait_for_condition(self._is_api_ready, 5, 600)
+
+    def _is_api_ready(self):
+        try:
+            self.k8s_api.list_namespaced_node()
+            self.LOG.info("API is ready.")
+            return True
+        except Exception:
+            self.LOG.info("API is not ready yet.")
+            return False
 
     def test_pod_apis(self):
         pod_manifest = {'apiVersion': 'v1',
diff --git a/magnum/tests/functional/python_client_base.py b/magnum/tests/functional/python_client_base.py
index dcc4ac6090..5b688687ed 100644
--- a/magnum/tests/functional/python_client_base.py
+++ b/magnum/tests/functional/python_client_base.py
@@ -27,6 +27,7 @@ from six.moves import configparser
 
 from magnum.common.utils import rmtree_without_raise
 from magnum.tests.functional.common import base
+from magnum.tests.functional.common import utils
 from magnumclient.common.apiclient import exceptions
 from magnumclient.common import cliutils
 from magnumclient.v1 import client as v1client
@@ -91,19 +92,23 @@ class BaseMagnumClient(base.BaseMagnumTest):
                                  magnum_url=magnum_url)
 
     @classmethod
-    def _wait_on_status(cls, bay, wait_status, finish_status):
+    def _wait_on_status(cls, bay, wait_status, finish_status, timeout=6000):
         # Check status every 60 seconds for a total of 100 minutes
-        for i in range(100):
-            # sleep 1s to wait bay status changes, this will be useful for
-            # the first time we wait for the status, to avoid another 59s
-            time.sleep(1)
+
+        def _check_status():
             status = cls.cs.bays.get(bay.uuid).status
+            cls.LOG.debug("Bay status is %s" % status)
             if status in wait_status:
-                time.sleep(59)
+                return False
             elif status in finish_status:
-                break
+                return True
             else:
-                raise Exception("Unknown Status : %s" % status)
+                raise Exception("Unexpected Status: %s" % status)
+
+        # sleep 1s to wait bay status changes, this will be useful for
+        # the first time we wait for the status, to avoid another 59s
+        time.sleep(1)
+        utils.wait_for_condition(_check_status, interval=60, timeout=timeout)
 
     @classmethod
     def _create_baymodel(cls, name, **kwargs):
@@ -157,10 +162,12 @@ class BaseMagnumClient(base.BaseMagnumTest):
         cls.cs.bays.delete(bay_uuid)
 
         try:
-            cls._wait_on_status(cls.bay,
-                                ["CREATE_COMPLETE",
-                                 "DELETE_IN_PROGRESS", "CREATE_FAILED"],
-                                ["DELETE_FAILED", "DELETE_COMPLETE"])
+            cls._wait_on_status(
+                cls.bay,
+                ["CREATE_COMPLETE", "DELETE_IN_PROGRESS", "CREATE_FAILED"],
+                ["DELETE_FAILED", "DELETE_COMPLETE"],
+                timeout=600
+            )
         except exceptions.NotFound:
             pass
         else:
@@ -171,7 +178,9 @@ class BaseMagnumClient(base.BaseMagnumTest):
         self._wait_on_status(
             bay,
             [None, "CREATE_IN_PROGRESS"],
-            ["CREATE_FAILED", "CREATE_COMPLETE"])
+            ["CREATE_FAILED", "CREATE_COMPLETE"],
+            timeout=1800
+        )
 
         if self.cs.bays.get(bay.uuid).status == 'CREATE_FAILED':
             raise Exception("bay %s created failed" % bay.uuid)