Launch k8s scheduler & controller-manager as system containers

Following up of https://review.openstack.org/#/c/487357

Depends-On: I22918c0b06ca34d96ee68ac43fabcd5c0b281950
Implements: blueprint run-kube-as-container

Change-Id: I9a7d00cddb456b885b6de28cfb3d33d2e16cc348
This commit is contained in:
Mathieu Velten 2017-07-27 10:35:19 +02:00
parent 024f2c0241
commit d003e80a3a
5 changed files with 9 additions and 125 deletions

View File

@ -6,6 +6,8 @@ echo "configuring kubernetes (master)"
atomic install --storage ostree --system --system-package=no --name=kubelet docker.io/openstackmagnum/kubernetes-kubelet:${KUBE_VERSION}
atomic install --storage ostree --system --system-package=no --name=kube-apiserver docker.io/openstackmagnum/kubernetes-apiserver:${KUBE_VERSION}
atomic install --storage ostree --system --system-package=no --name=kube-controller-manager docker.io/openstackmagnum/kubernetes-controller-manager:${KUBE_VERSION}
atomic install --storage ostree --system --system-package=no --name=kube-scheduler docker.io/openstackmagnum/kubernetes-scheduler:${KUBE_VERSION}
sed -i '
/^KUBE_ALLOW_PRIV=/ s/=.*/="--allow-privileged='"$KUBE_ALLOW_PRIV"'"/
@ -45,9 +47,9 @@ sed -i '
# Add controller manager args
KUBE_CONTROLLER_MANAGER_ARGS=""
KUBE_CONTROLLER_MANAGER_ARGS="--leader-elect=true"
if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then
KUBE_CONTROLLER_MANAGER_ARGS="--service-account-private-key-file=$CERT_DIR/server.key --root-ca-file=$CERT_DIR/ca.crt"
KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --service-account-private-key-file=$CERT_DIR/server.key --root-ca-file=$CERT_DIR/ca.crt"
fi
if [ -n "$TRUST_ID" ]; then
@ -59,6 +61,8 @@ sed -i '
/^KUBE_CONTROLLER_MANAGER_ARGS=/ s#\(KUBE_CONTROLLER_MANAGER_ARGS\).*#\1="'"${KUBE_CONTROLLER_MANAGER_ARGS}"'"#
' /etc/kubernetes/controller-manager
sed -i '/^KUBE_SCHEDULER_ARGS=/ s/=.*/="--leader-elect=true"/' /etc/kubernetes/scheduler
HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//')
KUBELET_ARGS="--register-node=true --register-schedulable=false --pod-manifest-path=/etc/kubernetes/manifests --hostname-override=${HOSTNAME_OVERRIDE}"
KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"

View File

@ -1,113 +0,0 @@
#!/bin/sh
. /etc/sysconfig/heat-params
if [ -n "${INSECURE_REGISTRY_URL}" ]; then
HYPERKUBE_IMAGE="${INSECURE_REGISTRY_URL}/google_containers/hyperkube:${KUBE_VERSION}"
else
HYPERKUBE_IMAGE="gcr.io/google_containers/hyperkube:${KUBE_VERSION}"
fi
# vars also used by the Kubernetes config files
unset KUBE_API_PORT
unset KUBE_ALLOW_PRIV
# this function generate a list of args (one per line) from a list of possibly nested args
# the first parameter is the prefix to be added before each arg
# empty args are ignored
generate_pod_args() {
prefix=$1
for var in "${@:2}" ; do
for arg in "$var" ; do
echo "$prefix$arg"
done
done
}
init_templates () {
. /etc/kubernetes/config
. /etc/kubernetes/controller-manager
local TEMPLATE=/etc/kubernetes/manifests/kube-controller-manager.yaml
[ -f ${TEMPLATE} ] || {
echo "TEMPLATE: $TEMPLATE"
mkdir -p $(dirname ${TEMPLATE})
cat << EOF > ${TEMPLATE}
apiVersion: v1
kind: Pod
metadata:
name: kube-controller-manager
namespace: kube-system
spec:
hostNetwork: true
containers:
- name: kube-controller-manager
image: ${HYPERKUBE_IMAGE}
command:
- /hyperkube
- controller-manager
- --leader-elect=true
$(generate_pod_args " - " $KUBE_LOGTOSTDERR $KUBE_LOG_LEVEL $KUBE_MASTER $KUBE_CONTROLLER_MANAGER_ARGS)
livenessProbe:
httpGet:
host: 127.0.0.1
path: /healthz
port: 10252
initialDelaySeconds: ${SYSTEM_PODS_INITIAL_DELAY}
timeoutSeconds: ${SYSTEM_PODS_TIMEOUT}
volumeMounts:
- mountPath: /etc/kubernetes
name: kubernetes-config
readOnly: true
volumes:
- hostPath:
path: /etc/kubernetes
name: kubernetes-config
EOF
}
. /etc/kubernetes/scheduler
local TEMPLATE=/etc/kubernetes/manifests/kube-scheduler.yaml
[ -f ${TEMPLATE} ] || {
echo "TEMPLATE: $TEMPLATE"
mkdir -p $(dirname ${TEMPLATE})
cat << EOF > ${TEMPLATE}
apiVersion: v1
kind: Pod
metadata:
name: kube-scheduler
namespace: kube-system
spec:
hostNetwork: true
containers:
- name: kube-scheduler
image: ${HYPERKUBE_IMAGE}
command:
- /hyperkube
- scheduler
- --leader-elect=true
$(generate_pod_args " - " $KUBE_LOGTOSTDERR $KUBE_LOG_LEVEL $KUBE_MASTER $KUBE_SCHEDULER_ARGS)
livenessProbe:
httpGet:
host: 127.0.0.1
path: /healthz
port: 10251
initialDelaySeconds: ${SYSTEM_PODS_INITIAL_DELAY}
timeoutSeconds: ${SYSTEM_PODS_TIMEOUT}
volumeMounts:
- mountPath: /etc/kubernetes
name: kubernetes-config
readOnly: true
volumes:
- hostPath:
path: /etc/kubernetes
name: kubernetes-config
EOF
}
}
init_templates

View File

@ -4,7 +4,7 @@
systemctl daemon-reload
echo "starting services"
for service in etcd docker kube-apiserver kubelet; do
for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler kubelet; do
echo "activating service $service"
systemctl enable $service
systemctl --no-block start $service

View File

@ -424,12 +424,6 @@ resources:
group: ungrouped
config: {get_file: ../../common/templates/kubernetes/fragments/network-service.sh}
enable_kube_controller_manager_scheduler:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: ../../common/templates/kubernetes/fragments/enable-kube-controller-manager-scheduler.sh}
kube_system_namespace_service:
type: OS::Heat::SoftwareConfig
properties:
@ -493,7 +487,6 @@ resources:
- config: {get_resource: network_service}
- config: {get_resource: kube_system_namespace_service}
- config: {get_resource: core_dns_service}
- config: {get_resource: enable_kube_controller_manager_scheduler}
- config: {get_resource: enable_kube_proxy}
- config: {get_resource: kube_ui_service}
- config: {get_resource: enable_monitoring}

View File

@ -56,8 +56,8 @@ if [[ "$COE" == "kubernetes" ]]; then
remote_exec $SSH_USER "sudo journalctl -u kube-proxy --no-pager" kube-proxy.log
remote_exec $SSH_USER "sudo journalctl -u etcd --no-pager" etcd.log
remote_exec $SSH_USER "sudo journalctl -u kube-apiserver --no-pager" kube-apiserver.log
remote_exec $SSH_USER "kubectl logs --namespace=kube-system \$(kubectl --namespace=kube-system get pods | grep kube-scheduler | awk '{print \$1}')" kube-scheduler.log
remote_exec $SSH_USER "kubectl logs --namespace=kube-system \$(kubectl --namespace=kube-system get pods | grep kube-controller-manager | awk '{print \$1}')" kube-controller-manager.log
remote_exec $SSH_USER "sudo journalctl -u kube-scheduler --no-pager" kube-scheduler.log
remote_exec $SSH_USER "sudo journalctl -u kube-controller-manager --no-pager" kube-controller-manager.log
remote_exec $SSH_USER "sudo journalctl -u docker-storage-setup --no-pager" docker-storage-setup.log
remote_exec $SSH_USER "sudo systemctl status docker-storage-setup -l" docker-storage-setup.service.status.log
remote_exec $SSH_USER "sudo systemctl show docker-storage-setup --no-pager" docker-storage-setup.service.show.log