k8s: stop introspecting instance name

We kept introspecting the name of the instance with the assumption
that the network always existed under .novalocal

This is not always the case, with certain variables changed inside
Neutron it is possible to control this, therefore, leading in failing
deploys.

With this change, we pass the instance name directly to the cluster
and therefore we always have the accurate name.

Task: 36160
Story: 2006371

Change-Id: I2ba32844b822ffc14da043e6ef7d071bb62a22ee
This commit is contained in:
Mohammed Naser 2019-06-27 17:35:50 -04:00 committed by Feilong Wang
parent 5823054627
commit 2f2d05c826
11 changed files with 17 additions and 25 deletions

View File

@ -188,8 +188,7 @@ sed -i '
sed -i '/^KUBE_SCHEDULER_ARGS=/ s/=.*/="--leader-elect=true"/' /etc/kubernetes/scheduler sed -i '/^KUBE_SCHEDULER_ARGS=/ s/=.*/="--leader-elect=true"/' /etc/kubernetes/scheduler
$ssh_cmd mkdir -p /etc/kubernetes/manifests $ssh_cmd mkdir -p /etc/kubernetes/manifests
HOSTNAME_OVERRIDE=$(cat /etc/hostname | head -1 | sed 's/\.novalocal//') KUBELET_ARGS="--register-node=true --pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=0 --hostname-override=${INSTANCE_NAME}"
KUBELET_ARGS="--register-node=true --pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=0 --hostname-override=${HOSTNAME_OVERRIDE}"
KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}pause:3.0" KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}pause:3.0"
KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}" KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"
KUBELET_ARGS="${KUBELET_ARGS} --volume-plugin-dir=/var/lib/kubelet/volumeplugins" KUBELET_ARGS="${KUBELET_ARGS} --volume-plugin-dir=/var/lib/kubelet/volumeplugins"
@ -211,7 +210,6 @@ KUBELET_ARGS="${KUBELET_ARGS} --register-with-taints=CriticalAddonsOnly=True:NoS
KUBELET_ARGS="${KUBELET_ARGS} --node-labels=node-role.kubernetes.io/master=\"\"" KUBELET_ARGS="${KUBELET_ARGS} --node-labels=node-role.kubernetes.io/master=\"\""
KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml
HOSTNAME_OVERRIDE=$(cat /etc/hostname | head -1 | sed 's/\.novalocal//')
cat << EOF >> ${KUBELET_KUBECONFIG} cat << EOF >> ${KUBELET_KUBECONFIG}
apiVersion: v1 apiVersion: v1
clusters: clusters:
@ -222,13 +220,13 @@ clusters:
contexts: contexts:
- context: - context:
cluster: kubernetes cluster: kubernetes
user: system:node:${HOSTNAME_OVERRIDE} user: system:node:${INSTANCE_NAME}
name: default name: default
current-context: default current-context: default
kind: Config kind: Config
preferences: {} preferences: {}
users: users:
- name: system:node:${HOSTNAME_OVERRIDE} - name: system:node:${INSTANCE_NAME}
user: user:
as-user-extra: {} as-user-extra: {}
client-certificate: ${CERT_DIR}/server.crt client-certificate: ${CERT_DIR}/server.crt

View File

@ -64,7 +64,6 @@ KUBE_MASTER_URI="$KUBE_PROTOCOL://$KUBE_MASTER_IP:$KUBE_API_PORT"
if [ -z "${KUBE_NODE_IP}" ]; then if [ -z "${KUBE_NODE_IP}" ]; then
KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4) KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
fi fi
HOSTNAME_OVERRIDE="$(cat /etc/hostname | head -1 | sed 's/\.novalocal//')"
cat << EOF >> ${KUBELET_KUBECONFIG} cat << EOF >> ${KUBELET_KUBECONFIG}
apiVersion: v1 apiVersion: v1
clusters: clusters:
@ -75,13 +74,13 @@ clusters:
contexts: contexts:
- context: - context:
cluster: kubernetes cluster: kubernetes
user: system:node:${HOSTNAME_OVERRIDE} user: system:node:${INSTANCE_NAME}
name: default name: default
current-context: default current-context: default
kind: Config kind: Config
preferences: {} preferences: {}
users: users:
- name: system:node:${HOSTNAME_OVERRIDE} - name: system:node:${INSTANCE_NAME}
user: user:
as-user-extra: {} as-user-extra: {}
client-certificate: ${CERT_DIR}/kubelet.crt client-certificate: ${CERT_DIR}/kubelet.crt
@ -133,7 +132,7 @@ sed -i '
# the option --hostname-override for kubelet uses the hostname to register the node. # the option --hostname-override for kubelet uses the hostname to register the node.
# Using any other name will break the load balancer and cinder volume features. # Using any other name will break the load balancer and cinder volume features.
mkdir -p /etc/kubernetes/manifests mkdir -p /etc/kubernetes/manifests
KUBELET_ARGS="--pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=0 --kubeconfig ${KUBELET_KUBECONFIG} --hostname-override=${HOSTNAME_OVERRIDE}" KUBELET_ARGS="--pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=0 --kubeconfig ${KUBELET_KUBECONFIG} --hostname-override=${INSTANCE_NAME}"
KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true" KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true"
KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}" KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"
KUBELET_ARGS="${KUBELET_ARGS} --volume-plugin-dir=/var/lib/kubelet/volumeplugins" KUBELET_ARGS="${KUBELET_ARGS} --volume-plugin-dir=/var/lib/kubelet/volumeplugins"
@ -201,6 +200,4 @@ EOF
cat >> /etc/environment <<EOF cat >> /etc/environment <<EOF
KUBERNETES_MASTER=$KUBE_MASTER_URI KUBERNETES_MASTER=$KUBE_MASTER_URI
EOF EOF
$ssh_cmd "hostname $(cat /etc/hostname | head -1 |sed 's/.novalocal//')"

View File

@ -101,7 +101,6 @@ EOF
} }
#Kubelet Certs #Kubelet Certs
INSTANCE_NAME=$(cat /etc/hostname | head -1 | sed 's/\.novalocal//')
HOSTNAME=$(cat /etc/hostname | head -1) HOSTNAME=$(cat /etc/hostname | head -1)
cat > ${cert_dir}/kubelet.conf <<EOF cat > ${cert_dir}/kubelet.conf <<EOF

View File

@ -149,7 +149,6 @@ extendedKeyUsage = clientAuth,serverAuth
EOF EOF
#Kubelet Certs #Kubelet Certs
INSTANCE_NAME=$(cat /etc/hostname | head -1 | sed 's/\.novalocal//')
cat > ${cert_dir}/kubelet.conf <<EOF cat > ${cert_dir}/kubelet.conf <<EOF
[req] [req]
distinguished_name = req_distinguished_name distinguished_name = req_distinguished_name

View File

@ -8,15 +8,14 @@ kubecontrol="/var/lib/containers/atomic/heat-container-agent.0/rootfs/usr/bin/ku
new_kube_tag="$kube_tag_input" new_kube_tag="$kube_tag_input"
if [ ${new_kube_tag}!=${KUBE_TAG} ]; then if [ ${new_kube_tag}!=${KUBE_TAG} ]; then
HOSTNAME_OVERRIDE="$(cat /etc/hostname | head -1 | sed 's/\.novalocal//')"
# If there is only one master and this is the master node, skip the drain, just cordon it # If there is only one master and this is the master node, skip the drain, just cordon it
# If there is only one worker and this is the worker node, skip the drain, just cordon it # If there is only one worker and this is the worker node, skip the drain, just cordon it
all_masters=$(${ssh_cmd} ${kubecontrol} get nodes --selector=node-role.kubernetes.io/master= -o name) all_masters=$(${ssh_cmd} ${kubecontrol} get nodes --selector=node-role.kubernetes.io/master= -o name)
all_workers=$(${ssh_cmd} ${kubecontrol} get nodes --selector=node-role.kubernetes.io/master!= -o name) all_workers=$(${ssh_cmd} ${kubecontrol} get nodes --selector=node-role.kubernetes.io/master!= -o name)
if [ "node/${HOSTNAME_OVERRIDE}" != "${all_masters}" ] && [ "node/${HOSTNAME_OVERRIDE}" != "${all_workers}" ]; then if [ "node/${INSTANCE_NAME}" != "${all_masters}" ] && [ "node/${INSTANCE_NAME}" != "${all_workers}" ]; then
${ssh_cmd} ${kubecontrol} drain ${HOSTNAME_OVERRIDE} --ignore-daemonsets --delete-local-data --force ${ssh_cmd} ${kubecontrol} drain ${INSTANCE_NAME} --ignore-daemonsets --delete-local-data --force
else else
${ssh_cmd} ${kubecontrol} cordon ${HOSTNAME_OVERRIDE} ${ssh_cmd} ${kubecontrol} cordon ${INSTANCE_NAME}
fi fi
declare -A service_image_mapping declare -A service_image_mapping
@ -40,7 +39,7 @@ if [ ${new_kube_tag}!=${KUBE_TAG} ]; then
systemctl restart ${service} systemctl restart ${service}
done done
${ssh_cmd} /var/lib/containers/atomic/heat-container-agent.0/rootfs/usr/bin/kubectl --kubeconfig /etc/kubernetes/kubelet-config.yaml uncordon ${HOSTNAME_OVERRIDE} ${ssh_cmd} /var/lib/containers/atomic/heat-container-agent.0/rootfs/usr/bin/kubectl --kubeconfig /etc/kubernetes/kubelet-config.yaml uncordon ${INSTANCE_NAME}
# FIXME(flwang): The KUBE_TAG could be out of date after a successful upgrade # FIXME(flwang): The KUBE_TAG could be out of date after a successful upgrade
for service in ${SERVICE_LIST}; do for service in ${SERVICE_LIST}; do

View File

@ -7,6 +7,7 @@ HEAT_PARAMS=/etc/sysconfig/heat-params
echo "Writing File: $HEAT_PARAMS" echo "Writing File: $HEAT_PARAMS"
mkdir -p "$(dirname ${HEAT_PARAMS})" mkdir -p "$(dirname ${HEAT_PARAMS})"
cat > ${HEAT_PARAMS} <<EOF cat > ${HEAT_PARAMS} <<EOF
INSTANCE_NAME="$INSTANCE_NAME"
PROMETHEUS_MONITORING="$PROMETHEUS_MONITORING" PROMETHEUS_MONITORING="$PROMETHEUS_MONITORING"
KUBE_API_PUBLIC_ADDRESS="$KUBE_API_PUBLIC_ADDRESS" KUBE_API_PUBLIC_ADDRESS="$KUBE_API_PUBLIC_ADDRESS"
KUBE_API_PRIVATE_ADDRESS="$KUBE_API_PRIVATE_ADDRESS" KUBE_API_PRIVATE_ADDRESS="$KUBE_API_PRIVATE_ADDRESS"

View File

@ -7,6 +7,7 @@ HEAT_PARAMS=/etc/sysconfig/heat-params
echo "Writing File: $HEAT_PARAMS" echo "Writing File: $HEAT_PARAMS"
mkdir -p "$(dirname ${HEAT_PARAMS})" mkdir -p "$(dirname ${HEAT_PARAMS})"
cat > ${HEAT_PARAMS} <<EOF cat > ${HEAT_PARAMS} <<EOF
INSTANCE_NAME="$INSTANCE_NAME"
PROMETHEUS_MONITORING="$PROMETHEUS_MONITORING" PROMETHEUS_MONITORING="$PROMETHEUS_MONITORING"
KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV" KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
KUBE_MASTER_IP="$KUBE_MASTER_IP" KUBE_MASTER_IP="$KUBE_MASTER_IP"

View File

@ -31,8 +31,6 @@ write_files:
INSECURE_REGISTRY_ARGS="" INSECURE_REGISTRY_ARGS=""
fi fi
HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//')
uuid_file="/var/run/kubelet-pod.uuid" uuid_file="/var/run/kubelet-pod.uuid"
CONF_FILE=/etc/systemd/system/kubelet.service CONF_FILE=/etc/systemd/system/kubelet.service
cat > $CONF_FILE <<EOF cat > $CONF_FILE <<EOF
@ -59,7 +57,7 @@ write_files:
--kubeconfig=/etc/kubernetes/master-kubeconfig.yaml \ --kubeconfig=/etc/kubernetes/master-kubeconfig.yaml \
--cni-conf-dir=/etc/kubernetes/cni/net.d \ --cni-conf-dir=/etc/kubernetes/cni/net.d \
--network-plugin=cni \ --network-plugin=cni \
--hostname-override=${HOSTNAME_OVERRIDE} \ --hostname-override=${INSTANCE_NAME} \
--container-runtime=${CONTAINER_RUNTIME} \ --container-runtime=${CONTAINER_RUNTIME} \
--register-with-taints=node.alpha.kubernetes.io/role=master:NoSchedule \ --register-with-taints=node.alpha.kubernetes.io/role=master:NoSchedule \
--allow-privileged=true \ --allow-privileged=true \

View File

@ -43,8 +43,6 @@ write_files:
fi fi
KUBE_MASTER_URI="$KUBE_PROTOCOL://$KUBE_MASTER_IP:$KUBE_API_PORT" KUBE_MASTER_URI="$KUBE_PROTOCOL://$KUBE_MASTER_IP:$KUBE_API_PORT"
HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//')
uuid_file="/var/run/kubelet-pod.uuid" uuid_file="/var/run/kubelet-pod.uuid"
CONF_FILE=/etc/systemd/system/kubelet.service CONF_FILE=/etc/systemd/system/kubelet.service
cat > $CONF_FILE <<EOF cat > $CONF_FILE <<EOF
@ -70,7 +68,7 @@ write_files:
ExecStart=/usr/lib/coreos/kubelet-wrapper \ ExecStart=/usr/lib/coreos/kubelet-wrapper \
--cni-conf-dir=/etc/kubernetes/cni/net.d \ --cni-conf-dir=/etc/kubernetes/cni/net.d \
--network-plugin=cni \ --network-plugin=cni \
--hostname-override=${HOSTNAME_OVERRIDE} \ --hostname-override=${INSTANCE_NAME} \
--container-runtime=${CONTAINER_RUNTIME} \ --container-runtime=${CONTAINER_RUNTIME} \
--allow-privileged=true \ --allow-privileged=true \
--pod-manifest-path=/etc/kubernetes/manifests \ --pod-manifest-path=/etc/kubernetes/manifests \

View File

@ -539,6 +539,7 @@ resources:
- str_replace: - str_replace:
template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params-master.sh} template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params-master.sh}
params: params:
"$INSTANCE_NAME": {get_param: name}
"$PROMETHEUS_MONITORING": {get_param: prometheus_monitoring} "$PROMETHEUS_MONITORING": {get_param: prometheus_monitoring}
"$KUBE_API_PUBLIC_ADDRESS": {get_attr: [api_address_switch, public_ip]} "$KUBE_API_PUBLIC_ADDRESS": {get_attr: [api_address_switch, public_ip]}
"$KUBE_API_PRIVATE_ADDRESS": {get_attr: [api_address_switch, private_ip]} "$KUBE_API_PRIVATE_ADDRESS": {get_attr: [api_address_switch, private_ip]}

View File

@ -319,6 +319,7 @@ resources:
- str_replace: - str_replace:
template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params.sh} template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params.sh}
params: params:
$INSTANCE_NAME: {get_param: name}
$PROMETHEUS_MONITORING: {get_param: prometheus_monitoring} $PROMETHEUS_MONITORING: {get_param: prometheus_monitoring}
$KUBE_ALLOW_PRIV: {get_param: kube_allow_priv} $KUBE_ALLOW_PRIV: {get_param: kube_allow_priv}
$KUBE_MASTER_IP: {get_param: kube_master_ip} $KUBE_MASTER_IP: {get_param: kube_master_ip}