From 1b72456e120c28526ef61aae9d176d613ee3ac82 Mon Sep 17 00:00:00 2001
From: Spyros Trigazis <spyridon.trigazis@cern.ch>
Date: Tue, 2 Feb 2021 09:10:25 +0000
Subject: [PATCH] k8s: Do not use insecure api port

* in 1.20 8080 is not supported anymore
** use only 6443
** change all probes for health to use kubectl and 6443
* configure the signing key in API

story: 2008524
task: 41731

Change-Id: Ibaf1840214016d2dd6ac15e2137eb3cd3d767889
Signed-off-by: Spyros Trigazis <spyridon.trigazis@cern.ch>
---
 .../fragments/calico-service-v3-3-x.sh        |  2 +-
 .../kubernetes/fragments/calico-service.sh    |  2 +-
 .../fragments/configure-kubernetes-master.sh  | 45 ++++++++-----------
 .../fragments/configure-kubernetes-minion.sh  |  1 -
 .../kubernetes/fragments/core-dns-service.sh  |  2 +-
 .../fragments/enable-auto-healing.sh          |  2 +-
 .../fragments/enable-auto-scaling.sh          |  2 +-
 .../kubernetes/fragments/enable-cinder-csi.sh |  2 +-
 .../fragments/enable-helm-tiller.sh           |  2 +-
 .../fragments/enable-ingress-octavia.sh       |  2 +-
 .../fragments/enable-ingress-traefik.sh       |  2 +-
 .../fragments/enable-keystone-auth.sh         |  2 +-
 .../fragments/enable-prometheus-monitoring.sh |  2 +-
 .../fragments/enable-services-master.sh       |  2 +-
 .../kubernetes/fragments/flannel-service.sh   |  2 +-
 .../fragments/install-helm-modules.sh         |  2 +-
 .../kube-apiserver-to-kubelet-role.sh         |  2 +-
 .../fragments/kube-dashboard-service.sh       |  2 +-
 .../kubernetes/fragments/wc-notify-master.sh  |  2 +-
 19 files changed, 36 insertions(+), 44 deletions(-)

diff --git a/magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-3-x.sh b/magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-3-x.sh
index 7ded4016ee..5e8202e461 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-3-x.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-3-x.sh
@@ -657,7 +657,7 @@ spec:
 EOF
     }
 
-    until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+    until  [ "ok" = "$(kubectl get --raw='/healthz')" ]
     do
         echo "Waiting for Kubernetes API..."
         sleep 5
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh
index ed916b5596..c297c62107 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh
@@ -837,7 +837,7 @@ EOF
 
 set -x
 
-    until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+    until  [ "ok" = "$(kubectl get --raw='/healthz')" ]
     do
         echo "Waiting for Kubernetes API..."
         sleep 5
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
index 12efbccf97..c9f7177dae 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
@@ -51,14 +51,12 @@ mkdir -p /srv/magnum/kubernetes/
 cat > /etc/kubernetes/config <<EOF
 KUBE_LOGTOSTDERR="--logtostderr=true"
 KUBE_LOG_LEVEL="--v=3"
-KUBE_MASTER="--master=http://127.0.0.1:8080"
 EOF
 cat > /etc/kubernetes/kubelet <<EOF
 KUBELET_ARGS="--fail-swap-on=false"
 EOF
 
 cat > /etc/kubernetes/apiserver <<EOF
-KUBE_API_ADDRESS="--insecure-bind-address=127.0.0.1"
 KUBE_ETCD_SERVERS="--etcd-servers=http://127.0.0.1:2379,http://127.0.0.1:4001"
 KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
 KUBE_ADMISSION_CONTROL="--admission-control=NodeRestriction,${ADMISSION_CONTROL_LIST}"
@@ -303,34 +301,29 @@ EOF
 
 sed -i '
     /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow-privileged='"$KUBE_ALLOW_PRIV"'"/
-    /^KUBE_MASTER=/ s|=.*|="--master=http://127.0.0.1:8080"|
 ' /etc/kubernetes/config
 
 KUBE_API_ARGS="--runtime-config=api/all=true"
 KUBE_API_ARGS="$KUBE_API_ARGS --allow-privileged=$KUBE_ALLOW_PRIV"
 KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP"
 KUBE_API_ARGS="$KUBE_API_ARGS $KUBEAPI_OPTIONS"
-if [ "$TLS_DISABLED" == "True" ]; then
-    KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0 --insecure-port=$KUBE_API_PORT"
-else
-    KUBE_API_ADDRESS="--bind-address=0.0.0.0 --secure-port=$KUBE_API_PORT"
-    # insecure port is used internaly
-    KUBE_API_ADDRESS="$KUBE_API_ADDRESS --insecure-bind-address=127.0.0.1 --insecure-port=8080"
-    KUBE_API_ARGS="$KUBE_API_ARGS --authorization-mode=Node,RBAC --tls-cert-file=$CERT_DIR/server.crt"
-    KUBE_API_ARGS="$KUBE_API_ARGS --tls-private-key-file=$CERT_DIR/server.key"
-    KUBE_API_ARGS="$KUBE_API_ARGS --client-ca-file=$CERT_DIR/ca.crt"
-    KUBE_API_ARGS="$KUBE_API_ARGS --service-account-key-file=${CERT_DIR}/service_account.key"
-    KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-certificate-authority=${CERT_DIR}/ca.crt --kubelet-client-certificate=${CERT_DIR}/server.crt --kubelet-client-key=${CERT_DIR}/server.key --kubelet-https=true"
-    # Allow for metrics-server/aggregator communication
-    KUBE_API_ARGS="${KUBE_API_ARGS} \
-        --proxy-client-cert-file=${CERT_DIR}/server.crt \
-        --proxy-client-key-file=${CERT_DIR}/server.key \
-        --requestheader-allowed-names=front-proxy-client,kube,kubernetes \
-        --requestheader-client-ca-file=${CERT_DIR}/ca.crt \
-        --requestheader-extra-headers-prefix=X-Remote-Extra- \
-        --requestheader-group-headers=X-Remote-Group \
-        --requestheader-username-headers=X-Remote-User"
-fi
+KUBE_API_ADDRESS="--bind-address=0.0.0.0 --secure-port=$KUBE_API_PORT"
+KUBE_API_ARGS="$KUBE_API_ARGS --authorization-mode=Node,RBAC --tls-cert-file=$CERT_DIR/server.crt"
+KUBE_API_ARGS="$KUBE_API_ARGS --tls-private-key-file=$CERT_DIR/server.key"
+KUBE_API_ARGS="$KUBE_API_ARGS --client-ca-file=$CERT_DIR/ca.crt"
+KUBE_API_ARGS="$KUBE_API_ARGS --service-account-key-file=${CERT_DIR}/service_account.key"
+KUBE_API_ARGS="$KUBE_API_ARGS --service-account-signing-key-file=${CERT_DIR}/service_account_private.key"
+KUBE_API_ARGS="$KUBE_API_ARGS --service-account-issuer=https://kubernetes.default.svc.cluster.local"
+KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-certificate-authority=${CERT_DIR}/ca.crt --kubelet-client-certificate=${CERT_DIR}/server.crt --kubelet-client-key=${CERT_DIR}/server.key --kubelet-https=true"
+# Allow for metrics-server/aggregator communication
+KUBE_API_ARGS="${KUBE_API_ARGS} \
+    --proxy-client-cert-file=${CERT_DIR}/server.crt \
+    --proxy-client-key-file=${CERT_DIR}/server.key \
+    --requestheader-allowed-names=front-proxy-client,kube,kubernetes \
+    --requestheader-client-ca-file=${CERT_DIR}/ca.crt \
+    --requestheader-extra-headers-prefix=X-Remote-Extra- \
+    --requestheader-group-headers=X-Remote-Group \
+    --requestheader-username-headers=X-Remote-User"
 
 KUBE_ADMISSION_CONTROL=""
 if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then
@@ -409,7 +402,7 @@ chmod 600 ${ADMIN_KUBECONFIG}
 export KUBECONFIG=${ADMIN_KUBECONFIG}
 
 # Add controller manager args
-KUBE_CONTROLLER_MANAGER_ARGS="--leader-elect=true"
+KUBE_CONTROLLER_MANAGER_ARGS="--leader-elect=true --kubeconfig=/etc/kubernetes/admin.conf"
 KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-name=${CLUSTER_UUID}"
 KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --allocate-node-cidrs=true"
 KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --cluster-cidr=${PODS_NETWORK_CIDR}"
@@ -435,7 +428,7 @@ sed -i '
     /^KUBE_CONTROLLER_MANAGER_ARGS=/ s#\(KUBE_CONTROLLER_MANAGER_ARGS\).*#\1="'"${KUBE_CONTROLLER_MANAGER_ARGS}"'"#
 ' /etc/kubernetes/controller-manager
 
-sed -i '/^KUBE_SCHEDULER_ARGS=/ s/=.*/="--leader-elect=true"/' /etc/kubernetes/scheduler
+sed -i '/^KUBE_SCHEDULER_ARGS=/ s#=.*#="--leader-elect=true --kubeconfig=/etc/kubernetes/admin.conf"#' /etc/kubernetes/scheduler
 
 $ssh_cmd mkdir -p /etc/kubernetes/manifests
 KUBELET_ARGS="--register-node=true --pod-manifest-path=/etc/kubernetes/manifests --hostname-override=${INSTANCE_NAME}"
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh
index 2f47c43ef2..8b788c01a5 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh
@@ -55,7 +55,6 @@ mkdir -p /srv/magnum/kubernetes/
 cat > /etc/kubernetes/config <<EOF
 KUBE_LOGTOSTDERR="--logtostderr=true"
 KUBE_LOG_LEVEL="--v=3"
-KUBE_MASTER="--master=http://127.0.0.1:8080"
 EOF
 cat > /etc/kubernetes/kubelet <<EOF
 KUBELET_ARGS="--fail-swap-on=false"
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh
index 2391fc11e8..affbc932d3 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh
@@ -298,7 +298,7 @@ EOF
 }
 
 echo "Waiting for Kubernetes API..."
-until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+until  [ "ok" = "$(kubectl get --raw='/healthz')" ]
 do
     sleep 5
 done
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-healing.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-healing.sh
index 4024fc0ea3..d14c449ed9 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-healing.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-healing.sh
@@ -125,7 +125,7 @@ EOF
     }
 
     echo "Waiting for Kubernetes API..."
-    until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+    until  [ "ok" = "$(kubectl get --raw='/healthz')" ]
     do
         sleep 5
     done
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-scaling.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-scaling.sh
index 6e3f1826e4..a19872f67f 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-scaling.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-scaling.sh
@@ -170,7 +170,7 @@ EOF
     }
 
     echo "Waiting for Kubernetes API..."
-    until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+    until  [ "ok" = "$(kubectl get --raw='/healthz')" ]
     do
         sleep 5
     done
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-cinder-csi.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-cinder-csi.sh
index 8559a93763..bf91bc88b5 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-cinder-csi.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-cinder-csi.sh
@@ -499,7 +499,7 @@ spec:
 EOF
 
     echo "Waiting for Kubernetes API..."
-    until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+    until  [ "ok" = "$(kubectl get --raw='/healthz')" ]
     do
         sleep 5
     done
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-helm-tiller.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-helm-tiller.sh
index 78c160458c..23bb96ff18 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-helm-tiller.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-helm-tiller.sh
@@ -222,7 +222,7 @@ data:
 EOF
     }
 
-    until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+    until  [ "ok" = "$(kubectl get --raw='/healthz')" ]
     do
         echo "Waiting for Kubernetes API..."
         sleep 5
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-octavia.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-octavia.sh
index d41fae449b..df4bbd18cc 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-octavia.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-octavia.sh
@@ -112,7 +112,7 @@ EOF
 writeFile $OCTAVIA_INGRESS_CONTROLLER "$OCTAVIA_INGRESS_CONTROLLER_CONTENT"
 
 echo "Waiting for Kubernetes API..."
-until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+until  [ "ok" = "$(kubectl get --raw='/healthz')" ]
 do
     sleep 5
 done
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik.sh
index 3f3e717c25..fba7ce9de5 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik.sh
@@ -169,7 +169,7 @@ EOF
 
 writeFile $INGRESS_TRAEFIK_MANIFEST "$INGRESS_TRAEFIK_MANIFEST_CONTENT"
 
-until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+until  [ "ok" = "$(kubectl get --raw='/healthz')" ]
 do
     echo "Waiting for Kubernetes API..."
     sleep 5
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-keystone-auth.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-keystone-auth.sh
index 73c108cb29..228cf76109 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-keystone-auth.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-keystone-auth.sh
@@ -166,7 +166,7 @@ spec:
 EOF
     }
 
-    until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+    until  [ "ok" = "$(kubectl get --raw='/healthz')" ]
     do
         echo "Waiting for Kubernetes API..."
         sleep 5
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh
index 5586a1fac8..e789c8592c 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh
@@ -462,7 +462,7 @@ if [ "$(echo $PROMETHEUS_MONITORING | tr '[:upper:]' '[:lower:]')" = "true" ]; t
 
     # Write the binary for enable-monitoring
     KUBE_MON_BIN_CONTENT='''#!/bin/sh
-until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+until  [ "ok" = "$(kubectl get --raw='/healthz')" ]
 do
     echo "Waiting for Kubernetes API..."
     sleep 5
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh
index 6d4e624c53..6ae5a15751 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh
@@ -27,7 +27,7 @@ for action in enable restart; do
 done
 
 # Label self as master
-until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] && \
+until  [ "ok" = "$(kubectl get --raw='/healthz')" ] && \
     kubectl patch node ${INSTANCE_NAME} \
         --patch '{"metadata": {"labels": {"node-role.kubernetes.io/master": ""}}}'
 do
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh
index ada71249c4..80c2281b5b 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh
@@ -260,7 +260,7 @@ EOF
 
     if [ "$MASTER_INDEX" = "0" ]; then
 
-        until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+        until  [ "ok" = "$(kubectl get --raw='/healthz')" ]
         do
             echo "Waiting for Kubernetes API..."
             sleep 5
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh b/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh
index a7c1fcc6c0..475e8dbf6c 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh
@@ -20,7 +20,7 @@ fi
 ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
 
 echo "Waiting for Kubernetes API..."
-until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]; do
+until  [ "ok" = "$(kubectl get --raw='/healthz')" ]; do
     sleep 5
 done
 
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh
index c53ee67021..34a66fcff6 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh
@@ -6,7 +6,7 @@ set +x
 set -x
 
 echo "Waiting for Kubernetes API..."
-until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+until  [ "ok" = "$(kubectl get --raw='/healthz')" ]
 do
     sleep 5
 done
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh
index 6fcd2c5bae..ecdd45069d 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh
@@ -4,7 +4,7 @@ printf "Starting to run ${step}\n"
 . /etc/sysconfig/heat-params
 
 echo "Waiting for Kubernetes API..."
-until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+until  [ "ok" = "$(kubectl get --raw='/healthz')" ]
 do
     sleep 5
 done
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/wc-notify-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/wc-notify-master.sh
index 4010d07f2b..7ac94a00c5 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/wc-notify-master.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/wc-notify-master.sh
@@ -11,7 +11,7 @@ WC_NOTIFY_SERVICE=/etc/systemd/system/wc-notify.service
 
 cat > $WC_NOTIFY_BIN <<EOF
 #!/bin/bash -v
-until  [ "ok" = "\$(curl --silent http://127.0.0.1:8080/healthz)" ]
+until  [ "ok" = "\$(kubectl get --raw='/healthz')" ]
 do
     echo "Waiting for Kubernetes API..."
     sleep 5