diff --git a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh index 6eb2df0feb..4b477d3b52 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh @@ -1,24 +1,24 @@ #!/bin/sh +step="calico-service" +printf "Starting to run ${step}\n" + . /etc/sysconfig/heat-params -if [ "$NETWORK_DRIVER" != "calico" ]; then - exit 0 -fi +if [ "$NETWORK_DRIVER" = "calico" ]; then + _prefix=${CONTAINER_INFRA_PREFIX:-quay.io/calico/} + ETCD_SERVER_IP=${ETCD_LB_VIP:-$KUBE_NODE_IP} + CERT_DIR=/etc/kubernetes/certs + ETCD_CA=`cat ${CERT_DIR}/ca.crt | base64 | tr -d '\n'` + ETCD_CERT=`cat ${CERT_DIR}/server.crt | base64 | tr -d '\n'` + ETCD_KEY=`cat ${CERT_DIR}/server.key | base64 | tr -d '\n'` -_prefix=${CONTAINER_INFRA_PREFIX:-quay.io/calico/} -ETCD_SERVER_IP=${ETCD_LB_VIP:-$KUBE_NODE_IP} -CERT_DIR=/etc/kubernetes/certs -ETCD_CA=`cat ${CERT_DIR}/ca.crt | base64 | tr -d '\n'` -ETCD_CERT=`cat ${CERT_DIR}/server.crt | base64 | tr -d '\n'` -ETCD_KEY=`cat ${CERT_DIR}/server.key | base64 | tr -d '\n'` + CALICO_DEPLOY=/srv/magnum/kubernetes/manifests/calico-deploy.yaml -CALICO_DEPLOY=/srv/magnum/kubernetes/manifests/calico-deploy.yaml - -[ -f ${CALICO_DEPLOY} ] || { -echo "Writing File: $CALICO_DEPLOY" -mkdir -p $(dirname ${CALICO_DEPLOY}) -cat << EOF > ${CALICO_DEPLOY} + [ -f ${CALICO_DEPLOY} ] || { + echo "Writing File: $CALICO_DEPLOY" + mkdir -p $(dirname ${CALICO_DEPLOY}) + cat << EOF > ${CALICO_DEPLOY} # Calico Version v2.6.7 # https://docs.projectcalico.org/v2.6/releases#v2.6.7 # This manifest includes the following component versions: @@ -445,21 +445,15 @@ subjects: name: calico-node namespace: kube-system EOF -} + } -# NOTE(flwang): Let's keep the same addons yaml file on all masters, -# but if it's not the primary/bootstrapping master, don't try to -# create those resources to avoid race condition issue until the -# kubectl issue https://github.com/kubernetes/kubernetes/issues/44165 -# fixed. -if [ "$MASTER_INDEX" != "0" ]; then - exit 0 + until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] + do + echo "Waiting for Kubernetes API..." + sleep 5 + done + + /usr/bin/kubectl apply -f ${CALICO_DEPLOY} --namespace=kube-system fi -until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] -do - echo "Waiting for Kubernetes API..." - sleep 5 -done - -/usr/bin/kubectl apply -f ${CALICO_DEPLOY} --namespace=kube-system +printf "Finished running ${step}\n" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh index d05131e452..177ad9235f 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh @@ -2,6 +2,8 @@ . /etc/sysconfig/heat-params +set -x + if [ -n "$ETCD_VOLUME_SIZE" ] && [ "$ETCD_VOLUME_SIZE" -gt 0 ]; then attempts=60 diff --git a/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh index 084195a333..4c8c07dd71 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh @@ -1,5 +1,8 @@ #!/bin/sh +step="core-dns-service" +printf "Starting to run ${step}\n" + . /etc/sysconfig/heat-params _dns_prefix=${CONTAINER_INFRA_PREFIX:-docker.io/coredns/} @@ -245,15 +248,6 @@ spec: EOF } -# NOTE(flwang): Let's keep the same addons yaml file on all masters, -# but if it's not the primary/bootstrapping master, don't try to -# create those resources to avoid race condition issue until the -# kubectl issue https://github.com/kubernetes/kubernetes/issues/44165 -# fixed. -if [ "$MASTER_INDEX" != "0" ]; then - exit 0 -fi - echo "Waiting for Kubernetes API..." until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] do @@ -261,3 +255,5 @@ do done kubectl apply --validate=false -f $CORE_DNS + +printf "Finished running ${step}\n" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-cert-api-manager.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-cert-api-manager.sh index 8a96baeceb..48bc37b830 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/enable-cert-api-manager.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-cert-api-manager.sh @@ -1,15 +1,17 @@ -#!/bin/bash +#!/bin/sh + +step="enable-cert-api-manager" +printf "Starting to run ${step}\n" . /etc/sysconfig/heat-params -if [ "$(echo $CERT_MANAGER_API | tr '[:upper:]' '[:lower:]')" = "false" ]; then - exit 0 +if [ "$(echo $CERT_MANAGER_API | tr '[:upper:]' '[:lower:]')" != "false" ]; then + cert_dir=/etc/kubernetes/certs + + echo -e "$CA_KEY" > ${cert_dir}/ca.key + + chown kube.kube ${cert_dir}/ca.key + chmod 400 ${cert_dir}/ca.key fi -cert_dir=/etc/kubernetes/certs - -echo -e "$CA_KEY" > ${cert_dir}/ca.key - -chown kube.kube ${cert_dir}/ca.key -chmod 400 ${cert_dir}/ca.key - +printf "Finished running ${step}\n" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-controller.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-controller.sh index 52b8cf88b8..75518c52cd 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-controller.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-controller.sh @@ -1,4 +1,7 @@ -#!/bin/bash +#!/bin/sh + +step="enable-ingress-controller" +printf "Starting to run ${step}\n" # Enables the specified ingress controller. # @@ -21,3 +24,5 @@ EOF if [ "$(echo $INGRESS_CONTROLLER | tr '[:upper:]' '[:lower:]')" = "traefik" ]; then $enable-ingress-traefik fi + +printf "Finished running ${step}\n" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik.sh index 16c39de33d..6751b67701 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik.sh @@ -110,15 +110,6 @@ writeFile $INGRESS_TRAEFIK_MANIFEST "$INGRESS_TRAEFIK_MANIFEST_CONTENT" INGRESS_TRAEFIK_BIN="/srv/magnum/kubernetes/bin/ingress-traefik" INGRESS_TRAEFIK_SERVICE="/etc/systemd/system/ingress-traefik.service" -# NOTE(flwang): Let's keep the same addons yaml file on all masters, -# but if it's not the primary/bootstrapping master, don't try to -# create those resources to avoid race condition issue until the -# kubectl issue https://github.com/kubernetes/kubernetes/issues/44165 -# fixed. -if [ "$MASTER_INDEX" != "0" ]; then - exit 0 -fi - # Binary for ingress traefik INGRESS_TRAEFIK_BIN_CONTENT='''#!/bin/sh until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh index 5fdc915a25..b1aaf61090 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh @@ -1,4 +1,7 @@ -#!/bin/bash +#!/bin/sh + +step="enable-prometheus-monitoring" +printf "Starting to run ${step}\n" . /etc/sysconfig/heat-params @@ -361,144 +364,136 @@ writeFile $grafanaService_file "$grafanaService_content" . /etc/sysconfig/heat-params -# NOTE(flwang): Let's keep the same addons yaml file on all masters, -# but if it's not the primary/bootstrapping master, don't try to -# create those resources to avoid race condition issue until the -# kubectl issue https://github.com/kubernetes/kubernetes/issues/44165 -# fixed. -if [ "$MASTER_INDEX" != "0" ]; then - exit 0 -fi -if [ "$(echo $PROMETHEUS_MONITORING | tr '[:upper:]' '[:lower:]')" = "false" ]; then - exit 0 -fi +if [ "$(echo $PROMETHEUS_MONITORING | tr '[:upper:]' '[:lower:]')" = "true" ]; then + PROMETHEUS_MON_BASE_DIR="/srv/magnum/kubernetes/monitoring" + KUBE_MON_BIN=${PROMETHEUS_MON_BASE_DIR}"/bin/kube-enable-monitoring" + KUBE_MON_SERVICE="/etc/systemd/system/kube-enable-monitoring.service" + GRAFANA_DEF_DASHBOARDS=${PROMETHEUS_MON_BASE_DIR}"/dashboards" + GRAFANA_DEF_DASHBOARD_FILE=$GRAFANA_DEF_DASHBOARDS"/default.json" -PROMETHEUS_MON_BASE_DIR="/srv/magnum/kubernetes/monitoring" -KUBE_MON_BIN=${PROMETHEUS_MON_BASE_DIR}"/bin/kube-enable-monitoring" -KUBE_MON_SERVICE="/etc/systemd/system/kube-enable-monitoring.service" -GRAFANA_DEF_DASHBOARDS=${PROMETHEUS_MON_BASE_DIR}"/dashboards" -GRAFANA_DEF_DASHBOARD_FILE=$GRAFANA_DEF_DASHBOARDS"/default.json" + # Write the binary for enable-monitoring + KUBE_MON_BIN_CONTENT='''#!/bin/sh + until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] + do + echo "Waiting for Kubernetes API..." + sleep 5 + done -# Write the binary for enable-monitoring -KUBE_MON_BIN_CONTENT='''#!/bin/sh -until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] -do - echo "Waiting for Kubernetes API..." - sleep 5 -done - -# Check if prometheus-monitoring namespace exist already before creating the namespace -kubectl get namespace prometheus-monitoring -if [ "$?" != "0" ] && \ - [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusNamespace.yaml" ]; then - kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusNamespace.yaml -fi - -# Check if all resources exist already before creating them -# Check if configmap Prometheus exists -kubectl get configmap prometheus -n prometheus-monitoring -if [ "$?" != "0" ] && \ - [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusConfigMap.yaml" ]; then - kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusConfigMap.yaml -fi - -# Check if deployment and service Prometheus exist -kubectl get service prometheus -n prometheus-monitoring | kubectl get deployment prometheus -n prometheus-monitoring -if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \ - [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusService.yaml" ]; then - kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusService.yaml -fi - -# Check if configmap graf-dash exists -kubectl get configmap graf-dash -n prometheus-monitoring -if [ "$?" != "0" ] && \ - [ -f '''$GRAFANA_DEF_DASHBOARD_FILE''' ]; then - kubectl create configmap graf-dash --from-file='''$GRAFANA_DEF_DASHBOARD_FILE''' -n prometheus-monitoring -fi - -# Check if deployment and service Grafana exist -kubectl get service grafana -n prometheus-monitoring | kubectl get deployment grafana -n prometheus-monitoring -if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \ - [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/grafanaService.yaml" ]; then - kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/grafanaService.yaml -fi - -# Wait for Grafana pod and then inject data source -while true -do - echo "Waiting for Grafana pod to be up and Running" - if [ "$(kubectl get po -n prometheus-monitoring -l name=grafana -o jsonpath={..phase})" = "Running" ]; then - break + # Check if prometheus-monitoring namespace exist already before creating the namespace + kubectl get namespace prometheus-monitoring + if [ "$?" != "0" ] && \ + [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusNamespace.yaml" ]; then + kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusNamespace.yaml fi - sleep 2 -done -# Which node is running Grafana -NODE_IP=`kubectl get po -n prometheus-monitoring -o jsonpath={.items[0].status.hostIP} -l name=grafana` -PROM_SERVICE_IP=`kubectl get svc prometheus --namespace prometheus-monitoring -o jsonpath={..clusterIP}` + # Check if all resources exist already before creating them + # Check if configmap Prometheus exists + kubectl get configmap prometheus -n prometheus-monitoring + if [ "$?" != "0" ] && \ + [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusConfigMap.yaml" ]; then + kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusConfigMap.yaml + fi -# The Grafana pod might be running but the app might still be initiating -echo "Check if Grafana is ready..." -curl --user admin:$ADMIN_PASSWD -X GET http://$NODE_IP:3000/api/datasources/1 -until [ $? -eq 0 ] -do - sleep 2 + # Check if deployment and service Prometheus exist + kubectl get service prometheus -n prometheus-monitoring | kubectl get deployment prometheus -n prometheus-monitoring + if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \ + [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusService.yaml" ]; then + kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusService.yaml + fi + + # Check if configmap graf-dash exists + kubectl get configmap graf-dash -n prometheus-monitoring + if [ "$?" != "0" ] && \ + [ -f '''$GRAFANA_DEF_DASHBOARD_FILE''' ]; then + kubectl create configmap graf-dash --from-file='''$GRAFANA_DEF_DASHBOARD_FILE''' -n prometheus-monitoring + fi + + # Check if deployment and service Grafana exist + kubectl get service grafana -n prometheus-monitoring | kubectl get deployment grafana -n prometheus-monitoring + if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \ + [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/grafanaService.yaml" ]; then + kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/grafanaService.yaml + fi + + # Wait for Grafana pod and then inject data source + while true + do + echo "Waiting for Grafana pod to be up and Running" + if [ "$(kubectl get po -n prometheus-monitoring -l name=grafana -o jsonpath={..phase})" = "Running" ]; then + break + fi + sleep 2 + done + + # Which node is running Grafana + NODE_IP=`kubectl get po -n prometheus-monitoring -o jsonpath={.items[0].status.hostIP} -l name=grafana` + PROM_SERVICE_IP=`kubectl get svc prometheus --namespace prometheus-monitoring -o jsonpath={..clusterIP}` + + # The Grafana pod might be running but the app might still be initiating + echo "Check if Grafana is ready..." curl --user admin:$ADMIN_PASSWD -X GET http://$NODE_IP:3000/api/datasources/1 -done + until [ $? -eq 0 ] + do + sleep 2 + curl --user admin:$ADMIN_PASSWD -X GET http://$NODE_IP:3000/api/datasources/1 + done -# Inject Prometheus datasource into Grafana -while true -do - INJECT=`curl --user admin:$ADMIN_PASSWD -X POST \ - -H "Content-Type: application/json;charset=UTF-8" \ - --data-binary '''"'"'''{"name":"k8sPrometheus","isDefault":true, - "type":"prometheus","url":"http://'''"'"'''$PROM_SERVICE_IP'''"'"''':9090","access":"proxy"}'''"'"'''\ - "http://$NODE_IP:3000/api/datasources/"` + # Inject Prometheus datasource into Grafana + while true + do + INJECT=`curl --user admin:$ADMIN_PASSWD -X POST \ + -H "Content-Type: application/json;charset=UTF-8" \ + --data-binary '''"'"'''{"name":"k8sPrometheus","isDefault":true, + "type":"prometheus","url":"http://'''"'"'''$PROM_SERVICE_IP'''"'"''':9090","access":"proxy"}'''"'"'''\ + "http://$NODE_IP:3000/api/datasources/"` - if [[ "$INJECT" = *"Datasource added"* ]]; then - echo "Prometheus datasource injected into Grafana" - break + if [[ "$INJECT" = *"Datasource added"* ]]; then + echo "Prometheus datasource injected into Grafana" + break + fi + echo "Trying to inject Prometheus datasource into Grafana - "$INJECT + done + ''' + writeFile $KUBE_MON_BIN "$KUBE_MON_BIN_CONTENT" + + + # Write the monitoring service + KUBE_MON_SERVICE_CONTENT='''[Unit] + Description=Enable Prometheus monitoring stack + + [Service] + Type=oneshot + Environment=HOME=/root + EnvironmentFile=-/etc/kubernetes/config + ExecStart='''${KUBE_MON_BIN}''' + + [Install] + WantedBy=multi-user.target + ''' + writeFile $KUBE_MON_SERVICE "$KUBE_MON_SERVICE_CONTENT" + + chown root:root ${KUBE_MON_BIN} + chmod 0755 ${KUBE_MON_BIN} + + chown root:root ${KUBE_MON_SERVICE} + chmod 0644 ${KUBE_MON_SERVICE} + + # Download the default JSON Grafana dashboard + # Not a crucial step, so allow it to fail + # TODO: this JSON should be passed into the minions as gzip in cloud-init + GRAFANA_DASHB_URL="https://grafana.net/api/dashboards/1621/revisions/1/download" + mkdir -p $GRAFANA_DEF_DASHBOARDS + curl $GRAFANA_DASHB_URL -o $GRAFANA_DEF_DASHBOARD_FILE || echo "Failed to fetch default Grafana dashboard" + if [ -f $GRAFANA_DEF_DASHBOARD_FILE ]; then + sed -i -- 's|${DS_PROMETHEUS}|k8sPrometheus|g' $GRAFANA_DEF_DASHBOARD_FILE fi - echo "Trying to inject Prometheus datasource into Grafana - "$INJECT -done -''' -writeFile $KUBE_MON_BIN "$KUBE_MON_BIN_CONTENT" - -# Write the monitoring service -KUBE_MON_SERVICE_CONTENT='''[Unit] -Description=Enable Prometheus monitoring stack - -[Service] -Type=oneshot -Environment=HOME=/root -EnvironmentFile=-/etc/kubernetes/config -ExecStart='''${KUBE_MON_BIN}''' - -[Install] -WantedBy=multi-user.target -''' -writeFile $KUBE_MON_SERVICE "$KUBE_MON_SERVICE_CONTENT" - -chown root:root ${KUBE_MON_BIN} -chmod 0755 ${KUBE_MON_BIN} - -chown root:root ${KUBE_MON_SERVICE} -chmod 0644 ${KUBE_MON_SERVICE} - -# Download the default JSON Grafana dashboard -# Not a crucial step, so allow it to fail -# TODO: this JSON should be passed into the minions as gzip in cloud-init -GRAFANA_DASHB_URL="https://grafana.net/api/dashboards/1621/revisions/1/download" -mkdir -p $GRAFANA_DEF_DASHBOARDS -curl $GRAFANA_DASHB_URL -o $GRAFANA_DEF_DASHBOARD_FILE || echo "Failed to fetch default Grafana dashboard" -if [ -f $GRAFANA_DEF_DASHBOARD_FILE ]; then - sed -i -- 's|${DS_PROMETHEUS}|k8sPrometheus|g' $GRAFANA_DEF_DASHBOARD_FILE + # Launch the monitoring service + set -x + systemctl daemon-reload + systemctl enable kube-enable-monitoring.service + systemctl start --no-block kube-enable-monitoring.service fi -# Launch the monitoring service -set -x -systemctl daemon-reload -systemctl enable kube-enable-monitoring.service -systemctl start --no-block kube-enable-monitoring.service +printf "Finished running ${step}\n" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh index ad191e5a09..92ed125350 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh @@ -1,16 +1,10 @@ -#!/bin/sh -x +#!/bin/sh + +step="kube-apiserver-to-kubelet-role" +printf "Starting to run ${step}\n" . /etc/sysconfig/heat-params -# NOTE(flwang): Let's keep the same addons yaml file on all masters, -# but if it's not the primary/bootstrapping master, don't try to -# create those resources to avoid race condition issue until the -# kubectl issue https://github.com/kubernetes/kubernetes/issues/44165 -# fixed. -if [ "$MASTER_INDEX" != "0" ]; then - exit 0 -fi - echo "Waiting for Kubernetes API..." until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] do @@ -84,3 +78,5 @@ EOF } kubectl apply --validate=false -f ${ADMIN_RBAC} + +printf "Finished running ${step}\n" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh index a34e37ed4a..0a22f660c4 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh @@ -1,20 +1,20 @@ -#!/bin/bash -x +#!/bin/sh + +step="kube-dashboard-service" +printf "Starting to run ${step}\n" . /etc/sysconfig/heat-params -if [ "$(echo $KUBE_DASHBOARD_ENABLED | tr '[:upper:]' '[:lower:]')" == "false" ]; then - exit 0 -fi +if [ "$(echo $KUBE_DASHBOARD_ENABLED | tr '[:upper:]' '[:lower:]')" == "true" ]; then + KUBE_DASH_IMAGE="${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}kubernetes-dashboard-amd64:${KUBE_DASHBOARD_VERSION}" + HEAPSTER_IMAGE="${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}heapster-amd64:v1.4.2" -KUBE_DASH_IMAGE="${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}kubernetes-dashboard-amd64:${KUBE_DASHBOARD_VERSION}" -HEAPSTER_IMAGE="${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}heapster-amd64:v1.4.2" + KUBE_DASH_DEPLOY=/srv/magnum/kubernetes/kubernetes-dashboard.yaml -KUBE_DASH_DEPLOY=/srv/magnum/kubernetes/kubernetes-dashboard.yaml - -[ -f ${KUBE_DASH_DEPLOY} ] || { - echo "Writing File: $KUBE_DASH_DEPLOY" - mkdir -p $(dirname ${KUBE_DASH_DEPLOY}) - cat << EOF > ${KUBE_DASH_DEPLOY} + [ -f ${KUBE_DASH_DEPLOY} ] || { + echo "Writing File: $KUBE_DASH_DEPLOY" + mkdir -p $(dirname ${KUBE_DASH_DEPLOY}) + cat << EOF > ${KUBE_DASH_DEPLOY} # Copyright 2017 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -197,22 +197,22 @@ spec: selector: k8s-app: kubernetes-dashboard EOF -} + } -INFLUX_SINK="" -# Deploy INFLUX AND GRAFANA -if [ "$(echo $INFLUX_GRAFANA_DASHBOARD_ENABLED | tr '[:upper:]' '[:lower:]')" == "true" ]; then - INFLUX_SINK=" - --sink=influxdb:http://monitoring-influxdb.kube-system.svc:8086" - INFLUX_IMAGE="${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}heapster-influxdb-amd64:v1.3.3" - GRAFANA_IMAGE="${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}heapster-grafana-amd64:v4.4.3" + INFLUX_SINK="" + # Deploy INFLUX AND GRAFANA + if [ "$(echo $INFLUX_GRAFANA_DASHBOARD_ENABLED | tr '[:upper:]' '[:lower:]')" == "true" ]; then + INFLUX_SINK=" - --sink=influxdb:http://monitoring-influxdb.kube-system.svc:8086" + INFLUX_IMAGE="${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}heapster-influxdb-amd64:v1.3.3" + GRAFANA_IMAGE="${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}heapster-grafana-amd64:v4.4.3" - INFLUX_DEPLOY=/srv/magnum/kubernetes/influxdb.yaml - GRAFANA_DEPLOY=/srv/magnum/kubernetes/grafana.yaml + INFLUX_DEPLOY=/srv/magnum/kubernetes/influxdb.yaml + GRAFANA_DEPLOY=/srv/magnum/kubernetes/grafana.yaml - [ -f ${INFLUX_DEPLOY} ] || { - echo "Writing File: $INFLUX_DEPLOY" - mkdir -p $(dirname ${INFLUX_DEPLOY}) - cat << EOF > ${INFLUX_DEPLOY} + [ -f ${INFLUX_DEPLOY} ] || { + echo "Writing File: $INFLUX_DEPLOY" + mkdir -p $(dirname ${INFLUX_DEPLOY}) + cat << EOF > ${INFLUX_DEPLOY} apiVersion: extensions/v1beta1 kind: Deployment metadata: @@ -254,12 +254,12 @@ spec: selector: k8s-app: influxdb EOF - } + } - [ -f ${GRAFANA_DEPLOY} ] || { - echo "Writing File: $GRAFANA_DEPLOY" - mkdir -p $(dirname ${GRAFANA_DEPLOY}) - cat << EOF > ${GRAFANA_DEPLOY} + [ -f ${GRAFANA_DEPLOY} ] || { + echo "Writing File: $GRAFANA_DEPLOY" + mkdir -p $(dirname ${GRAFANA_DEPLOY}) + cat << EOF > ${GRAFANA_DEPLOY} apiVersion: extensions/v1beta1 kind: Deployment metadata: @@ -333,31 +333,25 @@ spec: selector: k8s-app: grafana EOF - } + } - if [ "$MASTER_INDEX" != "0" ]; then - exit 0 + echo "Waiting for Kubernetes API..." + until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] + do + sleep 5 + done + + kubectl apply --validate=false -f $INFLUX_DEPLOY + kubectl apply --validate=false -f $GRAFANA_DEPLOY fi + # Deploy Heapster + HEAPSTER_DEPLOY=/srv/magnum/kubernetes/heapster-controller.yaml - - echo "Waiting for Kubernetes API..." - until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] - do - sleep 5 - done - - kubectl apply --validate=false -f $INFLUX_DEPLOY - kubectl apply --validate=false -f $GRAFANA_DEPLOY -fi - -# Deploy Heapster -HEAPSTER_DEPLOY=/srv/magnum/kubernetes/heapster-controller.yaml - -[ -f ${HEAPSTER_DEPLOY} ] || { - echo "Writing File: $HEAPSTER_DEPLOY" - mkdir -p $(dirname ${HEAPSTER_DEPLOY}) - cat << EOF > ${HEAPSTER_DEPLOY} + [ -f ${HEAPSTER_DEPLOY} ] || { + echo "Writing File: $HEAPSTER_DEPLOY" + mkdir -p $(dirname ${HEAPSTER_DEPLOY}) + cat << EOF > ${HEAPSTER_DEPLOY} apiVersion: v1 kind: ServiceAccount metadata: @@ -452,23 +446,16 @@ subjects: name: heapster namespace: kube-system EOF -} + } -# NOTE(flwang): Let's keep the same addons yaml file on all masters, -# but if it's not the primary/bootstrapping master, don't try to -# create those resources to avoid race condition issue until the -# kubectl issue https://github.com/kubernetes/kubernetes/issues/44165 -# fixed. + echo "Waiting for Kubernetes API..." + until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] + do + sleep 5 + done -if [ "$MASTER_INDEX" != "0" ]; then - exit 0 + kubectl apply --validate=false -f $KUBE_DASH_DEPLOY + kubectl apply --validate=false -f $HEAPSTER_DEPLOY fi -echo "Waiting for Kubernetes API..." -until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] -do - sleep 5 -done - -kubectl apply --validate=false -f $KUBE_DASH_DEPLOY -kubectl apply --validate=false -f $HEAPSTER_DEPLOY +printf "Finished running ${step}\n" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml index a8d1f81339..42d88c7a05 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml @@ -5,7 +5,6 @@ write_files: owner: "root:root" permissions: "0600" content: | - MASTER_INDEX="$MASTER_INDEX" PROMETHEUS_MONITORING="$PROMETHEUS_MONITORING" KUBE_API_PUBLIC_ADDRESS="$KUBE_API_PUBLIC_ADDRESS" KUBE_API_PRIVATE_ADDRESS="$KUBE_API_PRIVATE_ADDRESS" @@ -42,7 +41,6 @@ write_files: HTTP_PROXY="$HTTP_PROXY" HTTPS_PROXY="$HTTPS_PROXY" NO_PROXY="$NO_PROXY" - WAIT_CURL="$WAIT_CURL" KUBE_TAG="$KUBE_TAG" ETCD_TAG="$ETCD_TAG" FLANNEL_TAG="$FLANNEL_TAG" diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index f4965dc115..62ff4bdb0e 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -655,7 +655,6 @@ resources: list_join: - '-' - [{ get_param: 'OS::stack_name' }, 'master', '%index%'] - master_index: '%index%' prometheus_monitoring: {get_param: prometheus_monitoring} grafana_admin_passwd: {get_param: grafana_admin_passwd} api_public_address: {get_attr: [api_lb, floating_address]} @@ -670,7 +669,6 @@ resources: docker_volume_type: {get_param: docker_volume_type} docker_storage_driver: {get_param: docker_storage_driver} cgroup_driver: {get_param: cgroup_driver} - wait_condition_timeout: {get_param: wait_condition_timeout} network_driver: {get_param: network_driver} flannel_network_cidr: {get_param: flannel_network_cidr} flannel_network_subnetlen: {get_param: flannel_network_subnetlen} @@ -738,6 +736,41 @@ resources: grafana_tag: {get_param: grafana_tag} heat_container_agent_tag: {get_param: heat_container_agent_tag} + kube_cluster_config: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: + list_join: + - "\n" + - + - get_file: ../../common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh + - str_replace: + template: {get_file: ../../common/templates/kubernetes/fragments/enable-cert-api-manager.sh} + params: + "$CA_KEY": {get_param: ca_key} + - get_file: ../../common/templates/kubernetes/fragments/core-dns-service.sh + - str_replace: + template: {get_file: ../../common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh} + params: + "$ADMIN_PASSWD": {get_param: grafana_admin_passwd} + - get_file: ../../common/templates/kubernetes/fragments/calico-service.sh + - str_replace: + params: + $enable-ingress-traefik: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-traefik.sh} + template: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-controller.sh} + - get_file: ../../common/templates/kubernetes/fragments/kube-dashboard-service.sh + + kube_cluster_deploy: + type: OS::Heat::SoftwareDeployment + properties: + actions: ['CREATE'] + signal_transport: HEAT_SIGNAL + config: + get_resource: kube_cluster_config + server: + get_attr: [kube_masters, resource.0] + ###################################################################### # diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 96607279a4..37c17ca00a 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -178,11 +178,6 @@ parameters: type: string description: network driver to use for instantiating container networks - wait_condition_timeout: - type: number - description : > - timeout for the Wait Conditions - secgroup_kube_master_id: type: string description: ID of the security group for kubernetes master. @@ -367,12 +362,6 @@ parameters: whether or not to use Octavia for LoadBalancer type service. default: False - master_index: - type: string - description: > - the index of master node, index 0 means the master node is the primary, - bootstrapping node. - kube_service_account_key: type: string hidden: true @@ -404,17 +393,6 @@ parameters: description: tag of the heat_container_agent system container resources: - - master_wait_handle: - type: OS::Heat::WaitConditionHandle - - master_wait_condition: - type: OS::Heat::WaitCondition - depends_on: kube-master - properties: - handle: {get_resource: master_wait_handle} - timeout: {get_param: wait_condition_timeout} - ###################################################################### # # resource that exposes the IPs of either the kube master or the API @@ -443,7 +421,6 @@ resources: str_replace: template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params-master.yaml} params: - "$MASTER_INDEX": {get_param: master_index} "$PROMETHEUS_MONITORING": {get_param: prometheus_monitoring} "$KUBE_API_PUBLIC_ADDRESS": {get_attr: [api_address_switch, public_ip]} "$KUBE_API_PRIVATE_ADDRESS": {get_attr: [api_address_switch, private_ip]} @@ -487,7 +464,6 @@ resources: "$FLANNEL_TAG": {get_param: flannel_tag} "$KUBE_VERSION": {get_param: kube_version} "$KUBE_DASHBOARD_VERSION": {get_param: kube_dashboard_version} - "$WAIT_CURL": {get_attr: [master_wait_handle, curl_cli]} "$TRUSTEE_USER_ID": {get_param: trustee_user_id} "$TRUSTEE_PASSWORD": {get_param: trustee_password} "$TRUST_ID": {get_param: trust_id} @@ -585,18 +561,6 @@ resources: group: ungrouped config: {get_file: ../../common/templates/kubernetes/fragments/enable-services-master.sh} - kube_apiserver_to_kubelet_role: - type: OS::Heat::SoftwareConfig - properties: - group: ungrouped - config: {get_file: ../../common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh} - - master_wc_notify: - type: OS::Heat::SoftwareConfig - properties: - group: ungrouped - config: {get_file: ../../common/templates/kubernetes/fragments/wc-notify-master.sh} - disable_selinux: type: OS::Heat::SoftwareConfig properties: @@ -633,109 +597,6 @@ resources: - config: {get_resource: write_flannel_config} - config: {get_resource: flannel_config_service} - config: {get_resource: flannel_service} - - config: {get_resource: kube_apiserver_to_kubelet_role} - - config: {get_resource: master_wc_notify} - - enable_cert_manager_api: - type: OS::Heat::SoftwareConfig - properties: - group: script - config: - str_replace: - template: {get_file: ../../common/templates/kubernetes/fragments/enable-cert-api-manager.sh} - params: - "$CA_KEY": {get_param: ca_key} - - enable_cert_manager_api_deployment: - type: OS::Heat::SoftwareDeployment - properties: - signal_transport: HEAT_SIGNAL - config: {get_resource: enable_cert_manager_api} - server: {get_resource: kube-master} - actions: ['CREATE'] - - core_dns_service: - type: OS::Heat::SoftwareConfig - properties: - group: script - config: {get_file: ../../common/templates/kubernetes/fragments/core-dns-service.sh} - - core_dns_service_deployment: - type: OS::Heat::SoftwareDeployment - depends_on: enable_cert_manager_api_deployment - properties: - signal_transport: HEAT_SIGNAL - config: {get_resource: core_dns_service} - server: {get_resource: kube-master} - actions: ['CREATE'] - - enable_prometheus_monitoring: - type: OS::Heat::SoftwareConfig - properties: - group: script - config: - str_replace: - template: {get_file: ../../common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh} - params: - "$ADMIN_PASSWD": {get_param: grafana_admin_passwd} - - enable_prometheus_monitoring_deployment: - type: OS::Heat::SoftwareDeployment - depends_on: core_dns_service_deployment - properties: - signal_transport: HEAT_SIGNAL - config: {get_resource: enable_prometheus_monitoring} - server: {get_resource: kube-master} - actions: ['CREATE'] - - calico_service: - type: OS::Heat::SoftwareConfig - properties: - group: script - config: {get_file: ../../common/templates/kubernetes/fragments/calico-service.sh} - - calico_service_deployment: - type: OS::Heat::SoftwareDeployment - depends_on: enable_prometheus_monitoring_deployment - properties: - signal_transport: HEAT_SIGNAL - config: {get_resource: calico_service} - server: {get_resource: kube-master} - actions: ['CREATE'] - - enable_ingress_controller: - type: OS::Heat::SoftwareConfig - properties: - group: script - config: - str_replace: - params: - $enable-ingress-traefik: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-traefik.sh} - template: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-controller.sh} - - enable_ingress_controller_deployment: - type: OS::Heat::SoftwareDeployment - depends_on: calico_service_deployment - properties: - signal_transport: HEAT_SIGNAL - config: {get_resource: enable_ingress_controller} - server: {get_resource: kube-master} - actions: ['CREATE'] - - kubernetes_dashboard: - type: OS::Heat::SoftwareConfig - properties: - group: script - config: {get_file: ../../common/templates/kubernetes/fragments/kube-dashboard-service.sh} - - kubernetes_dashboard_deployment: - type: OS::Heat::SoftwareDeployment - depends_on: enable_ingress_controller_deployment - properties: - signal_transport: HEAT_SIGNAL - config: {get_resource: kubernetes_dashboard} - server: {get_resource: kube-master} - actions: ['CREATE'] ###################################################################### # @@ -833,6 +694,9 @@ resources: outputs: + OS::stack_id: + value: { get_resource: kube-master } + kube_master_ip: value: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} description: > diff --git a/releasenotes/notes/k8s-cluster-creation-speedup-21b5b368184d7bf0.yaml b/releasenotes/notes/k8s-cluster-creation-speedup-21b5b368184d7bf0.yaml new file mode 100644 index 0000000000..4e4eea9987 --- /dev/null +++ b/releasenotes/notes/k8s-cluster-creation-speedup-21b5b368184d7bf0.yaml @@ -0,0 +1,5 @@ +features: + - | + Start Kubernetes workers installation right after the master instances are + created rather than waiting for all the services inside masters, which + could decrease the Kubernetes cluster launch time significantly.