From 1ca8efa3a6063e86ade08b721ff42c77e4f230be Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Sun, 4 Jun 2017 11:49:17 -0500 Subject: [PATCH] KubeADM-AIO: Update Image to support Multinode Operation Natively This PS also pins the remaining components of the kubelet, moves the K8s version to 1.6.4 and restores CentOS 7 operation. Change-Id: Ia32c9e02dbf9451c13addce436f6e36f5a0a622f --- tools/gate/funcs/kube.sh | 15 +++-- tools/gate/kubeadm_aio.sh | 2 +- tools/gate/setup_gate.sh | 4 +- tools/kubeadm-aio/Dockerfile | 29 +++++--- tools/kubeadm-aio/README.rst | 8 +-- tools/kubeadm-aio/assets/etc/kube-role | 3 + .../assets/etc/kubeadm-join-command-args | 1 + tools/kubeadm-aio/assets/kubeadm-aio | 10 ++- tools/kubeadm-aio/assets/usr/bin/kubeadm-aio | 66 ++++++++++--------- 9 files changed, 84 insertions(+), 54 deletions(-) create mode 100644 tools/kubeadm-aio/assets/etc/kube-role create mode 100644 tools/kubeadm-aio/assets/etc/kubeadm-join-command-args diff --git a/tools/gate/funcs/kube.sh b/tools/gate/funcs/kube.sh index 17441f0b07..ee38107a19 100755 --- a/tools/gate/funcs/kube.sh +++ b/tools/gate/funcs/kube.sh @@ -56,11 +56,16 @@ function kubeadm_aio_reqs_install { sudo yum install -y \ epel-release sudo yum install -y \ - docker \ + docker-latest \ nfs-utils \ jq - sudo cp -f /usr/lib/systemd/system/docker.service /etc/systemd/system/docker.service + sudo cp -f /usr/lib/systemd/system/docker-latest.service /etc/systemd/system/docker.service + sudo sed -i "s|/var/lib/docker-latest|/var/lib/docker|g" /etc/systemd/system/docker.service + sudo sed -i 's/^OPTIONS/#OPTIONS/g' /etc/sysconfig/docker-latest sudo sed -i "s|^MountFlags=slave|MountFlags=share|g" /etc/systemd/system/docker.service + sudo sed -i "/--seccomp-profile/,+1 d" /etc/systemd/system/docker.service + echo "DOCKER_STORAGE_OPTIONS=--storage-driver=overlay" | sudo tee /etc/sysconfig/docker-latest-storage + sudo setenforce 0 || true sudo systemctl daemon-reload sudo systemctl restart docker elif [ "x$HOST_OS" == "xfedora" ]; then @@ -71,6 +76,7 @@ function kubeadm_aio_reqs_install { sudo cp -f /usr/lib/systemd/system/docker-latest.service /etc/systemd/system/docker.service sudo sed -i "s|/var/lib/docker-latest|/var/lib/docker|g" /etc/systemd/system/docker.service echo "DOCKER_STORAGE_OPTIONS=--storage-driver=overlay2" | sudo tee /etc/sysconfig/docker-latest-storage + sudo setenforce 0 || true sudo systemctl daemon-reload sudo systemctl restart docker fi @@ -87,11 +93,6 @@ function kubeadm_aio_build { } function kubeadm_aio_launch { - if [ "x$HOST_OS" == "xcentos" ]; then - sudo setenforce 0 || true - elif [ "x$HOST_OS" == "xfedora" ]; then - sudo setenforce 0 || true - fi ${WORK_DIR}/tools/kubeadm-aio/kubeadm-aio-launcher.sh mkdir -p ${HOME}/.kube cat ${KUBECONFIG} > ${HOME}/.kube/config diff --git a/tools/gate/kubeadm_aio.sh b/tools/gate/kubeadm_aio.sh index eb58958aa5..7dd8d4acb6 100755 --- a/tools/gate/kubeadm_aio.sh +++ b/tools/gate/kubeadm_aio.sh @@ -17,6 +17,6 @@ source ${WORK_DIR}/tools/gate/funcs/network.sh source ${WORK_DIR}/tools/gate/funcs/kube.sh kubeadm_aio_reqs_install -#kubeadm_aio_build +sudo docker pull ${KUBEADM_IMAGE} || kubeadm_aio_build kubeadm_aio_launch net_resolv_kube diff --git a/tools/gate/setup_gate.sh b/tools/gate/setup_gate.sh index c0a581a416..8872964ca5 100755 --- a/tools/gate/setup_gate.sh +++ b/tools/gate/setup_gate.sh @@ -14,9 +14,9 @@ set -ex export HELM_VERSION=${2:-v2.3.0} -export KUBE_VERSION=${3:-v1.6.2} +export KUBE_VERSION=${3:-v1.6.4} export KUBECONFIG=${HOME}/.kubeadm-aio/admin.conf -export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6 +export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:${KUBE_VERSION} export WORK_DIR=$(pwd) source /etc/os-release diff --git a/tools/kubeadm-aio/Dockerfile b/tools/kubeadm-aio/Dockerfile index 3ac2b87286..23a6694090 100644 --- a/tools/kubeadm-aio/Dockerfile +++ b/tools/kubeadm-aio/Dockerfile @@ -1,14 +1,15 @@ FROM ubuntu:16.04 MAINTAINER pete.birley@att.com -ARG HELM_VERSION=v2.3.0 -ARG KUBE_VERSION=v1.6.2 - -ENV container="docker" \ +ENV HELM_VERSION=v2.3.0 \ + KUBE_VERSION=v1.6.4 \ + CNI_VERSION=v0.5.2 \ + container="docker" \ DEBIAN_FRONTEND="noninteractive" RUN set -x \ - && cd /tmp \ + && TMP_DIR=$(mktemp --directory) \ + && cd ${TMP_DIR} \ && apt-get update \ && apt-get install -y \ apt-transport-https \ @@ -31,19 +32,29 @@ RUN set -x \ && dpkg --configure kubeadm \ && apt-get install -yf kubeadm \ && mkdir -p /etc/kubernetes/manifests \ +# Install kubectl: + && curl -sSL https://dl.k8s.io/${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz | tar -zxv --strip-components=1 \ + && mv ${TMP_DIR}/client/bin/kubectl /usr/bin/kubectl \ + && chmod +x /usr/bin/kubectl \ # Install kubelet & kubeadm binaries: # (portdirect) We do things in this weird way to let us use the deps and systemd # units from the packages in the .deb repo. && curl -sSL https://dl.k8s.io/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz | tar -zxv --strip-components=1 \ - && mv /tmp/server/bin/kubelet /usr/bin/kubelet \ + && mv ${TMP_DIR}/server/bin/kubelet /usr/bin/kubelet \ && chmod +x /usr/bin/kubelet \ - && mv /tmp/server/bin/kubeadm /usr/bin/kubeadm \ + && mv ${TMP_DIR}/server/bin/kubeadm /usr/bin/kubeadm \ && chmod +x /usr/bin/kubeadm \ +# Install CNI: + && CNI_BIN_DIR=/opt/cni/bin \ + && mkdir -p ${CNI_BIN_DIR} \ + && cd ${CNI_BIN_DIR} \ + && curl -sSL https://github.com/containernetworking/cni/releases/download/$CNI_VERSION/cni-amd64-$CNI_VERSION.tgz | tar -zxv --strip-components=1 \ + && cd ${TMP_DIR} \ # Move kubelet binary as we will run containerised && mv /usr/bin/kubelet /usr/bin/kubelet-real \ # Install helm binary && curl -sSL https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz | tar -zxv --strip-components=1 \ - && mv /tmp/helm /usr/bin/helm \ + && mv ${TMP_DIR}/helm /usr/bin/helm \ # Install openstack-helm dev utils && apt-get install -y \ make \ @@ -64,7 +75,7 @@ RUN set -x \ && rm -rf /var/lib/apt/lists/* \ # Clean up tmp dir && cd / \ - && rm -rf /tmp/* + && rm -rf ${TMP_DIR} # Load assets into place, setup startup target & units COPY ./assets/ / diff --git a/tools/kubeadm-aio/README.rst b/tools/kubeadm-aio/README.rst index e676bb4f8e..7301761b48 100644 --- a/tools/kubeadm-aio/README.rst +++ b/tools/kubeadm-aio/README.rst @@ -31,7 +31,7 @@ You should install the ``kubectl`` and ``helm`` binaries: .. code:: bash - KUBE_VERSION=v1.6.0 + KUBE_VERSION=v1.6.4 HELM_VERSION=v2.3.0 TMP_DIR=$(mktemp -d) @@ -57,7 +57,7 @@ repo run: .. code:: bash - export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6 + export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.4 sudo docker build --pull -t ${KUBEADM_IMAGE} tools/kubeadm-aio Deploy the AIO environment @@ -67,8 +67,8 @@ To launch the environment then run: .. code:: bash - export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6 - export KUBE_VERSION=v1.6.2 + export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.4 + export KUBE_VERSION=v1.6.4 ./tools/kubeadm-aio/kubeadm-aio-launcher.sh export KUBECONFIG=${HOME}/.kubeadm-aio/admin.conf diff --git a/tools/kubeadm-aio/assets/etc/kube-role b/tools/kubeadm-aio/assets/etc/kube-role new file mode 100644 index 0000000000..804a47a7f8 --- /dev/null +++ b/tools/kubeadm-aio/assets/etc/kube-role @@ -0,0 +1,3 @@ +# If KUBE_ROLE is set 'master' kubeadm-aio will set this node up to be a master +# node, otherwise if 'worker', will join an existing cluster. +KUBE_ROLE=master diff --git a/tools/kubeadm-aio/assets/etc/kubeadm-join-command-args b/tools/kubeadm-aio/assets/etc/kubeadm-join-command-args new file mode 100644 index 0000000000..c5813a89cc --- /dev/null +++ b/tools/kubeadm-aio/assets/etc/kubeadm-join-command-args @@ -0,0 +1 @@ +KUBEADM_JOIN_ARGS="no_command_supplied" diff --git a/tools/kubeadm-aio/assets/kubeadm-aio b/tools/kubeadm-aio/assets/kubeadm-aio index 7a695b558d..328726c0e8 100755 --- a/tools/kubeadm-aio/assets/kubeadm-aio +++ b/tools/kubeadm-aio/assets/kubeadm-aio @@ -25,7 +25,7 @@ echo 'Setting up K8s version to deploy' : ${KUBE_VERSION:="default"} sed -i "s|KUBE_VERSION=.*|KUBE_VERSION=${KUBE_VERSION}|g" /etc/kube-version -echo 'Setting up device image to use for kube-api' +echo 'Setting up device to use for kube-api' : ${KUBE_BIND_DEV:="autodetect"} sed -i "s|KUBE_BIND_DEV=.*|KUBE_BIND_DEV=${KUBE_BIND_DEV}|g" /etc/kubeapi-device @@ -33,5 +33,13 @@ echo 'Setting up container image to use for kubelet' : ${KUBELET_CONTAINER:="this_one"} sed -i "s|KUBELET_CONTAINER=.*|KUBELET_CONTAINER=${KUBELET_CONTAINER}|g" /etc/kubelet-container +echo 'Setting whether this node is a master, or slave, K8s node' +: ${KUBE_ROLE:="master"} +sed -i "s|KUBE_ROLE=.*|KUBE_ROLE=${KUBE_ROLE}|g" /etc/kube-role + +echo 'Setting any kubeadm join commands' +: ${KUBEADM_JOIN_ARGS:="no_command_supplied"} +sed -i "s|KUBEADM_JOIN_ARGS=.*|KUBEADM_JOIN_ARGS=\"${KUBEADM_JOIN_ARGS}\"|g" /etc/kubeadm-join-command-args + echo 'Starting Systemd' exec /bin/systemd --system diff --git a/tools/kubeadm-aio/assets/usr/bin/kubeadm-aio b/tools/kubeadm-aio/assets/usr/bin/kubeadm-aio index bd2079f33e..082240a4e8 100755 --- a/tools/kubeadm-aio/assets/usr/bin/kubeadm-aio +++ b/tools/kubeadm-aio/assets/usr/bin/kubeadm-aio @@ -15,34 +15,40 @@ # under the License. set -xe -# Define k8s version -source /etc/kube-version -if [[ "${KUBE_VERSION}" == "default" ]]; then - KUBE_VERSION_FLAG="" -else - KUBE_VERSION_FLAG="--kubernetes-version=${KUBE_VERSION}" - echo "We will use K8s ${KUBE_VERSION}" +source /etc/kube-role +if [[ "${KUBE_ROLE}" == "master" ]]; then + # Define k8s version + source /etc/kube-version + if [[ "${KUBE_VERSION}" == "default" ]]; then + KUBE_VERSION_FLAG="" + else + KUBE_VERSION_FLAG="--kubernetes-version=${KUBE_VERSION}" + echo "We will use K8s ${KUBE_VERSION}" + fi + + echo 'Setting up K8s' + source /etc/kubeapi-device + if [[ "$KUBE_BIND_DEV" != "autodetect" ]]; then + KUBE_BIND_IP=$(ip addr list ${KUBE_BIND_DEV} |grep "inet " |cut -d' ' -f6|cut -d/ -f1) + echo 'We are going to bind the K8s API to: ${KUBE_BIND_IP}' + kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} --api-advertise-addresses ${KUBE_BIND_IP} + else + kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} + fi + + echo 'Setting up K8s client' + cp /etc/kubernetes/admin.conf /root/ + export KUBECONFIG=/root/admin.conf + + echo 'Marking master node as schedulable' + kubectl taint nodes --all node-role.kubernetes.io/master- + + echo 'Installing Calico CNI' + kubectl apply -f /opt/cni-manifests/calico.yaml + + echo 'Setting Up Cluser for OpenStack-Helm dev use' + /usr/bin/openstack-helm-dev-prep +elif [[ "${KUBE_ROLE}" == "worker" ]]; then + source /etc/kubeadm-join-command-args + kubeadm join --skip-preflight-checks ${KUBEADM_JOIN_ARGS} fi - -echo 'Setting up K8s' -source /etc/kubeapi-device -if [[ "$KUBE_BIND_DEV" != "autodetect" ]]; then - KUBE_BIND_IP=$(ip addr list ${KUBE_BIND_DEV} |grep "inet " |cut -d' ' -f6|cut -d/ -f1) - echo 'We are going to bind the K8s API to: ${KUBE_BIND_IP}' - kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} --api-advertise-addresses ${KUBE_BIND_IP} -else - kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} -fi - -echo 'Setting up K8s client' -cp /etc/kubernetes/admin.conf /root/ -export KUBECONFIG=/root/admin.conf - -echo 'Marking master node as schedulable' -kubectl taint nodes --all node-role.kubernetes.io/master- - -echo 'Installing Calico CNI' -kubectl apply -f /opt/cni-manifests/calico.yaml - -echo 'Setting Up Cluser for OpenStack-Helm dev use' -/usr/bin/openstack-helm-dev-prep