openstack
/
magnum
Code Issues Proposed changes

k8s: Do not use insecure api port 

* in 1.20 8080 is not supported anymore
** use only 6443
** change all probes for health to use kubectl and 6443
* configure the signing key in API

story: 2008524
task: 41731

Change-Id: Ibaf1840214016d2dd6ac15e2137eb3cd3d767889
Signed-off-by: Spyros Trigazis <spyridon.trigazis@cern.ch>
This commit is contained in:
Spyros Trigazis 2021-02-02 09:10:25 +00:00
parent d11f4e8393
commit 1b72456e12
19 changed files with 36 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-3-x.sh
View File

@ -657,7 +657,7 @@ spec:
EOF
}
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
until [ "ok" = "$(kubectl get --raw='/healthz')" ]
do
echo "Waiting for Kubernetes API..."
sleep 5

2
magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh
View File

@ -837,7 +837,7 @@ EOF
set -x
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
until [ "ok" = "$(kubectl get --raw='/healthz')" ]
do
echo "Waiting for Kubernetes API..."
sleep 5

45
magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
View File

@ -51,14 +51,12 @@ mkdir -p /srv/magnum/kubernetes/
cat > /etc/kubernetes/config <<EOF
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=3"
KUBE_MASTER="--master=http://127.0.0.1:8080"
EOF
cat > /etc/kubernetes/kubelet <<EOF
KUBELET_ARGS="--fail-swap-on=false"
EOF
cat > /etc/kubernetes/apiserver <<EOF
KUBE_API_ADDRESS="--insecure-bind-address=127.0.0.1"
KUBE_ETCD_SERVERS="--etcd-servers=http://127.0.0.1:2379,http://127.0.0.1:4001"
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
KUBE_ADMISSION_CONTROL="--admission-control=NodeRestriction,${ADMISSION_CONTROL_LIST}"
@ -303,34 +301,29 @@ EOF
sed -i '
/^KUBE_ALLOW_PRIV=/ s/=.*/="--allow-privileged='"$KUBE_ALLOW_PRIV"'"/
/^KUBE_MASTER=/ s|=.*|="--master=http://127.0.0.1:8080"|
' /etc/kubernetes/config
KUBE_API_ARGS="--runtime-config=api/all=true"
KUBE_API_ARGS="$KUBE_API_ARGS --allow-privileged=$KUBE_ALLOW_PRIV"
KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP"
KUBE_API_ARGS="$KUBE_API_ARGS $KUBEAPI_OPTIONS"
if [ "$TLS_DISABLED" == "True" ]; then
KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0 --insecure-port=$KUBE_API_PORT"
else
KUBE_API_ADDRESS="--bind-address=0.0.0.0 --secure-port=$KUBE_API_PORT"
# insecure port is used internaly
KUBE_API_ADDRESS="$KUBE_API_ADDRESS --insecure-bind-address=127.0.0.1 --insecure-port=8080"
KUBE_API_ARGS="$KUBE_API_ARGS --authorization-mode=Node,RBAC --tls-cert-file=$CERT_DIR/server.crt"
KUBE_API_ARGS="$KUBE_API_ARGS --tls-private-key-file=$CERT_DIR/server.key"
KUBE_API_ARGS="$KUBE_API_ARGS --client-ca-file=$CERT_DIR/ca.crt"
KUBE_API_ARGS="$KUBE_API_ARGS --service-account-key-file=${CERT_DIR}/service_account.key"
KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-certificate-authority=${CERT_DIR}/ca.crt --kubelet-client-certificate=${CERT_DIR}/server.crt --kubelet-client-key=${CERT_DIR}/server.key --kubelet-https=true"
# Allow for metrics-server/aggregator communication
KUBE_API_ARGS="${KUBE_API_ARGS} \
--proxy-client-cert-file=${CERT_DIR}/server.crt \
--proxy-client-key-file=${CERT_DIR}/server.key \
--requestheader-allowed-names=front-proxy-client,kube,kubernetes \
--requestheader-client-ca-file=${CERT_DIR}/ca.crt \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-group-headers=X-Remote-Group \
--requestheader-username-headers=X-Remote-User"
fi
KUBE_API_ADDRESS="--bind-address=0.0.0.0 --secure-port=$KUBE_API_PORT"
KUBE_API_ARGS="$KUBE_API_ARGS --authorization-mode=Node,RBAC --tls-cert-file=$CERT_DIR/server.crt"
KUBE_API_ARGS="$KUBE_API_ARGS --tls-private-key-file=$CERT_DIR/server.key"
KUBE_API_ARGS="$KUBE_API_ARGS --client-ca-file=$CERT_DIR/ca.crt"
KUBE_API_ARGS="$KUBE_API_ARGS --service-account-key-file=${CERT_DIR}/service_account.key"
KUBE_API_ARGS="$KUBE_API_ARGS --service-account-signing-key-file=${CERT_DIR}/service_account_private.key"
KUBE_API_ARGS="$KUBE_API_ARGS --service-account-issuer=https://kubernetes.default.svc.cluster.local"
KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-certificate-authority=${CERT_DIR}/ca.crt --kubelet-client-certificate=${CERT_DIR}/server.crt --kubelet-client-key=${CERT_DIR}/server.key --kubelet-https=true"
# Allow for metrics-server/aggregator communication
KUBE_API_ARGS="${KUBE_API_ARGS} \
--proxy-client-cert-file=${CERT_DIR}/server.crt \
--proxy-client-key-file=${CERT_DIR}/server.key \
--requestheader-allowed-names=front-proxy-client,kube,kubernetes \
--requestheader-client-ca-file=${CERT_DIR}/ca.crt \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-group-headers=X-Remote-Group \
--requestheader-username-headers=X-Remote-User"
KUBE_ADMISSION_CONTROL=""
if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then
@ -409,7 +402,7 @@ chmod 600 ${ADMIN_KUBECONFIG}
export KUBECONFIG=${ADMIN_KUBECONFIG}
# Add controller manager args
KUBE_CONTROLLER_MANAGER_ARGS="--leader-elect=true"
KUBE_CONTROLLER_MANAGER_ARGS="--leader-elect=true --kubeconfig=/etc/kubernetes/admin.conf"
KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-name=${CLUSTER_UUID}"
KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --allocate-node-cidrs=true"
KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --cluster-cidr=${PODS_NETWORK_CIDR}"
@ -435,7 +428,7 @@ sed -i '
/^KUBE_CONTROLLER_MANAGER_ARGS=/ s#\(KUBE_CONTROLLER_MANAGER_ARGS\).*#\1="'"${KUBE_CONTROLLER_MANAGER_ARGS}"'"#
' /etc/kubernetes/controller-manager
sed -i '/^KUBE_SCHEDULER_ARGS=/ s/=.*/="--leader-elect=true"/' /etc/kubernetes/scheduler
sed -i '/^KUBE_SCHEDULER_ARGS=/ s#=.*#="--leader-elect=true --kubeconfig=/etc/kubernetes/admin.conf"#' /etc/kubernetes/scheduler
$ssh_cmd mkdir -p /etc/kubernetes/manifests
KUBELET_ARGS="--register-node=true --pod-manifest-path=/etc/kubernetes/manifests --hostname-override=${INSTANCE_NAME}"

1
magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh
View File

@ -55,7 +55,6 @@ mkdir -p /srv/magnum/kubernetes/
cat > /etc/kubernetes/config <<EOF
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=3"
KUBE_MASTER="--master=http://127.0.0.1:8080"
EOF
cat > /etc/kubernetes/kubelet <<EOF
KUBELET_ARGS="--fail-swap-on=false"

2
magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh
View File

@ -298,7 +298,7 @@ EOF
}
echo "Waiting for Kubernetes API..."
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
until [ "ok" = "$(kubectl get --raw='/healthz')" ]
do
sleep 5
done

2
magnum/drivers/common/templates/kubernetes/fragments/enable-auto-healing.sh
View File

@ -125,7 +125,7 @@ EOF
}
echo "Waiting for Kubernetes API..."
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
until [ "ok" = "$(kubectl get --raw='/healthz')" ]
do
sleep 5
done

2
magnum/drivers/common/templates/kubernetes/fragments/enable-auto-scaling.sh
View File

@ -170,7 +170,7 @@ EOF
}
echo "Waiting for Kubernetes API..."
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
until [ "ok" = "$(kubectl get --raw='/healthz')" ]
do
sleep 5
done

2
magnum/drivers/common/templates/kubernetes/fragments/enable-cinder-csi.sh
View File

@ -499,7 +499,7 @@ spec:
EOF
echo "Waiting for Kubernetes API..."
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
until [ "ok" = "$(kubectl get --raw='/healthz')" ]
do
sleep 5
done

2
magnum/drivers/common/templates/kubernetes/fragments/enable-helm-tiller.sh
View File

@ -222,7 +222,7 @@ data:
EOF
}
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
until [ "ok" = "$(kubectl get --raw='/healthz')" ]
do
echo "Waiting for Kubernetes API..."
sleep 5

2
magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-octavia.sh
View File

@ -112,7 +112,7 @@ EOF
writeFile $OCTAVIA_INGRESS_CONTROLLER "$OCTAVIA_INGRESS_CONTROLLER_CONTENT"
echo "Waiting for Kubernetes API..."
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
until [ "ok" = "$(kubectl get --raw='/healthz')" ]
do
sleep 5
done

2
magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik.sh
View File

@ -169,7 +169,7 @@ EOF
writeFile $INGRESS_TRAEFIK_MANIFEST "$INGRESS_TRAEFIK_MANIFEST_CONTENT"
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
until [ "ok" = "$(kubectl get --raw='/healthz')" ]
do
echo "Waiting for Kubernetes API..."
sleep 5

2
magnum/drivers/common/templates/kubernetes/fragments/enable-keystone-auth.sh
View File

@ -166,7 +166,7 @@ spec:
EOF
}
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
until [ "ok" = "$(kubectl get --raw='/healthz')" ]
do
echo "Waiting for Kubernetes API..."
sleep 5

2
magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh
View File

@ -462,7 +462,7 @@ if [ "$(echo $PROMETHEUS_MONITORING | tr '[:upper:]' '[:lower:]')" = "true" ]; t
# Write the binary for enable-monitoring
KUBE_MON_BIN_CONTENT='''#!/bin/sh
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
until [ "ok" = "$(kubectl get --raw='/healthz')" ]
do
echo "Waiting for Kubernetes API..."
sleep 5

2
magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh
View File

@ -27,7 +27,7 @@ for action in enable restart; do
done
# Label self as master
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ] && \
until [ "ok" = "$(kubectl get --raw='/healthz')" ] && \
kubectl patch node ${INSTANCE_NAME} \
--patch '{"metadata": {"labels": {"node-role.kubernetes.io/master": ""}}}'
do

2
magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh
View File

@ -260,7 +260,7 @@ EOF
if [ "$MASTER_INDEX" = "0" ]; then
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
until [ "ok" = "$(kubectl get --raw='/healthz')" ]
do
echo "Waiting for Kubernetes API..."
sleep 5

2
magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh
View File

@ -20,7 +20,7 @@ fi
ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
echo "Waiting for Kubernetes API..."
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]; do
until [ "ok" = "$(kubectl get --raw='/healthz')" ]; do
sleep 5
done

2
magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh
View File

@ -6,7 +6,7 @@ set +x
set -x
echo "Waiting for Kubernetes API..."
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
until [ "ok" = "$(kubectl get --raw='/healthz')" ]
do
sleep 5
done

2
magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh
View File

@ -4,7 +4,7 @@ printf "Starting to run ${step}\n"
. /etc/sysconfig/heat-params
echo "Waiting for Kubernetes API..."
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
until [ "ok" = "$(kubectl get --raw='/healthz')" ]
do
sleep 5
done

2
magnum/drivers/common/templates/kubernetes/fragments/wc-notify-master.sh
View File

@ -11,7 +11,7 @@ WC_NOTIFY_SERVICE=/etc/systemd/system/wc-notify.service
cat > $WC_NOTIFY_BIN <<EOF
#!/bin/bash -v
until [ "ok" = "\$(curl --silent http://127.0.0.1:8080/healthz)" ]
until [ "ok" = "\$(kubectl get --raw='/healthz')" ]
do
echo "Waiting for Kubernetes API..."
sleep 5