Multinode-check: 2 Node via kubeadm with NFS

This PS introduces a 2 node check in zuul, it uses the KubeADM-AIO
container to stand up a master and worker node, before deploying
OpenStack-Helm using NFS as a PV backend. This check is a stopgap
untill a 3 node gate is deployed which will use ceph and enable
HA of all components to be tested.

Change-Id: I7bb70e6b103a483a0e1df1492fa2047d229eb86f

tools/gate/provision_gate_worker_node.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -ex
+
+cd ${WORK_DIR}
+source /etc/os-release
+export HOST_OS=${ID}
+source ${WORK_DIR}/tools/gate/funcs/network.sh
+source ${WORK_DIR}/tools/gate/funcs/kube.sh
+
+kubeadm_aio_reqs_install
+
+# Setup shared mounts for kubelet
+sudo mkdir -p /var/lib/kubelet
+sudo mount --bind /var/lib/kubelet /var/lib/kubelet
+sudo mount --make-shared /var/lib/kubelet
+
+# Cleanup any old deployment
+sudo docker rm -f kubeadm-aio || true
+sudo docker rm -f kubelet || true
+sudo docker ps -aq | xargs -r -l1 sudo docker rm -f
+sudo rm -rfv \
+    /etc/cni/net.d \
+    /etc/kubernetes \
+    /var/lib/etcd \
+    /var/etcd \
+    /var/lib/kubelet/* \
+    /run/openvswitch || true
+
+# Launch Container
+sudo docker run \
+    -dt \
+    --name=kubeadm-aio \
+    --net=host \
+    --security-opt=seccomp:unconfined \
+    --cap-add=SYS_ADMIN \
+    --tmpfs=/run \
+    --tmpfs=/run/lock \
+    --volume=/etc/machine-id:/etc/machine-id:ro \
+    --volume=${HOME}:${HOME}:rw \
+    --volume=/etc/kubernetes:/etc/kubernetes:rw \
+    --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro \
+    --volume=/var/run/docker.sock:/run/docker.sock \
+    --env KUBE_ROLE="worker" \
+    --env KUBELET_CONTAINER="${KUBEADM_IMAGE}" \
+    --env KUBEADM_JOIN_ARGS="--token=${KUBEADM_TOKEN} ${PRIMARY_NODE_IP}:6443" \
+    ${KUBEADM_IMAGE}

tools/gate/setup_gate.sh

@@ -38,14 +38,21 @@ fi
 net_resolv_pre_kube
 net_hosts_pre_kube
 
+# Setup helm
 helm_install
 helm_serve
 helm_lint
 
+# Setup the K8s Cluster
 if [ "x$INTEGRATION" == "xaio" ]; then
  bash ${WORK_DIR}/tools/gate/kubeadm_aio.sh
- bash ${WORK_DIR}/tools/gate/helm_dry_run.sh
- if [ "x$INTEGRATION_TYPE" == "xbasic" ]; then
-   bash ${WORK_DIR}/tools/gate/basic_launch.sh
- fi
+elif [ "x$INTEGRATION" == "xmulti" ]; then
+ bash ${WORK_DIR}/tools/gate/kubeadm_aio.sh
+ bash ${WORK_DIR}/tools/gate/setup_gate_worker_nodes.sh
+fi
+
+# Deploy OpenStack-Helm
+if [ "x$INTEGRATION_TYPE" == "xbasic" ]; then
+  bash ${WORK_DIR}/tools/gate/helm_dry_run.sh
+  bash ${WORK_DIR}/tools/gate/basic_launch.sh
 fi

tools/gate/setup_gate_worker_nodes.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -ex
+
+: ${SSH_PRIVATE_KEY:="/etc/nodepool/id_rsa"}
+sudo chown $(whoami) ${SSH_PRIVATE_KEY}
+sudo chmod 600 ${SSH_PRIVATE_KEY}
+
+
+PRIMARY_NODE_IP=$(cat /etc/nodepool/primary_node_private | tail -1)
+KUBEADM_TOKEN=$(sudo docker exec kubeadm-aio kubeadm token list | tail -n -1 | awk '{ print $1 }')
+
+NODE_2=$(cat /etc/nodepool/sub_nodes_private | tail -1)
+
+ssh-keyscan "${NODE_2}" >> ~/.ssh/known_hosts
+ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_2} mkdir -p ${WORK_DIR%/*}
+scp -i ${SSH_PRIVATE_KEY} -r ${WORK_DIR} $(whoami)@${NODE_2}:${WORK_DIR%/*}
+ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_2} "export WORK_DIR=${WORK_DIR}; export KUBEADM_TOKEN=${KUBEADM_TOKEN}; export PRIMARY_NODE_IP=${PRIMARY_NODE_IP}; export KUBEADM_IMAGE=${KUBEADM_IMAGE}; bash ${WORK_DIR}/tools/gate/provision_gate_worker_node.sh"
+
+sleep 120
+source ${WORK_DIR}/tools/gate/funcs/kube.sh
+kube_wait_for_pods kube-system 240
+kubectl get nodes --show-all
+kubectl get --all-namespaces all --show-all
+sudo docker exec kubeadm-aio openstack-helm-dev-prep