Notify Heat only if kube-apiserver is running

Kubernetes bay needs to be fully functional before notifying Heat
to bring the stack to CREATE_COMPLETE. This patch added a systemd
unit that keeps pinging the kube-apiserver and notify Heat once
it goes through.

Change-Id: Ib4fb22f3040d76d488c6cac6873d64b4d973ebb1
Closes-Bug: #1551824

magnum/templates/kubernetes/fragments/wc-notify-master.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+. /etc/sysconfig/heat-params
+
+WC_NOTIFY_BIN=/usr/local/bin/wc-notify
+WC_NOTIFY_SERVICE=/etc/systemd/system/wc-notify.service
+
+cat > $WC_NOTIFY_BIN <<EOF
+#!/bin/bash -v
+until curl -sf "http://127.0.0.1:8080/healthz"; do
+    echo "Waiting for Kubernetes API..."
+    sleep 5
+done
+$WAIT_CURL --data-binary '{"status": "SUCCESS"}'
+EOF
+
+cat > $WC_NOTIFY_SERVICE <<EOF
+[Unit]
+Description=Notify Heat
+After=docker.service etcd.service
+Requires=docker.service etcd.service
+[Service]
+Type=oneshot
+ExecStart=$WC_NOTIFY_BIN
+[Install]
+WantedBy=multi-user.target
+EOF
+
+chown root:root $WC_NOTIFY_BIN
+chmod 0755 $WC_NOTIFY_BIN
+
+chown root:root $WC_NOTIFY_SERVICE
+chmod 0644 $WC_NOTIFY_SERVICE
+
+systemctl enable wc-notify
+systemctl start --no-block wc-notify

magnum/templates/kubernetes/kubemaster.yaml

@@ -325,13 +325,7 @@ resources:
     type: OS::Heat::SoftwareConfig
     properties:
       group: ungrouped
-      config:
-        str_replace:
-          template: |
-            #!/bin/bash -v
-            wc_notify --data-binary '{"status": "SUCCESS"}'
-          params:
-            wc_notify: {get_attr: [master_wait_handle, curl_cli]}
+      config: {get_file: fragments/wc-notify-master.sh}
 
   disable_selinux:
     type: OS::Heat::SoftwareConfig

magnum/tests/contrib/copy_instance_logs.sh

@@ -72,6 +72,7 @@ if [[ "$COE" == "kubernetes" ]]; then
     remote_exec $SSH_USER "sudo ip a" ipa.log
     remote_exec $SSH_USER "sudo netstat -an" netstat.log
     remote_exec $SSH_USER "sudo df -h" dfh.log
+    remote_exec $SSH_USER "sudo journalctl -u wc-notify --no-pager" wc-notify.log
 elif [[ "$COE" == "swarm" ]]; then
     SSH_USER=fedora
     remote_exec $SSH_USER "sudo systemctl --full list-units --no-pager" systemctl_list_units.log