From 4f121e50c547abee195e30ce4aef588f71f509ee Mon Sep 17 00:00:00 2001 From: Spyros Trigazis Date: Thu, 16 Aug 2018 12:11:49 +0200 Subject: [PATCH] [k8s] Add proxy to master and set cluster-cidr 1. pods with host network can not reach coredns or any svc or resolve their own hostname 2. If webhooks are deployed in the cluster, the apiserver needs to contact them, which means kube-proxy is required in the master node with the cluster-cidr set. Change-Id: Icb8e7c3b8c75a3ab087c818c8580c0c8a9111d30 story: 2003460 task: 24719 --- .../fragments/configure-kubernetes-master.sh | 34 +++++++++++++++++-- .../fragments/configure-kubernetes-minion.sh | 6 ++-- .../fragments/enable-services-master.sh | 2 +- .../fragments/write-heat-params.yaml | 2 ++ .../templates/kubeminion.yaml | 2 ++ 5 files changed, 40 insertions(+), 6 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index 9bf97a4bd5..74e3474666 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -17,6 +17,38 @@ fi atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG} atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG} atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG} +atomic install --storage ostree --system --system-package=no --name=kube-proxy ${_prefix}kubernetes-proxy:${KUBE_TAG} + +CERT_DIR=/etc/kubernetes/certs + +# kube-proxy config +PROXY_KUBECONFIG=/etc/kubernetes/proxy-kubeconfig.yaml +cat > /etc/kubernetes/proxy << EOF +KUBE_PROXY_ARGS="--kubeconfig=${PROXY_KUBECONFIG} --cluster-cidr=${PODS_NETWORK_CIDR}" +EOF + +cat > ${PROXY_KUBECONFIG} << EOF +apiVersion: v1 +clusters: +- cluster: + certificate-authority: ${CERT_DIR}/ca.crt + server: http://127.0.0.1:8080 + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: kube-proxy + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: kube-proxy + user: + as-user-extra: {} +EOF + + if [ "$NETWORK_DRIVER" = "flannel" ]; then atomic install --storage ostree --system --system-package=no \ --name=flanneld ${_prefix}flannel:${FLANNEL_TAG} @@ -27,8 +59,6 @@ sed -i ' /^KUBE_MASTER=/ s|=.*|="--master=http://127.0.0.1:8080"| ' /etc/kubernetes/config -CERT_DIR=/etc/kubernetes/certs - KUBE_API_ARGS="--runtime-config=api/all=true" KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP" KUBE_API_ARGS="$KUBE_API_ARGS $KUBEAPI_OPTIONS" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh index b76cea8350..bc90e59bf6 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -179,9 +179,9 @@ sed -i ' /^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"| ' /etc/kubernetes/kubelet -sed -i ' - /^KUBE_PROXY_ARGS=/ s|=.*|=--kubeconfig='"$PROXY_KUBECONFIG"'| -' /etc/kubernetes/proxy +cat > /etc/kubernetes/proxy << EOF +KUBE_PROXY_ARGS="--kubeconfig=${PROXY_KUBECONFIG} --cluster-cidr=${PODS_NETWORK_CIDR}" +EOF if [ "$NETWORK_DRIVER" = "flannel" ]; then atomic install --storage ostree --system --system-package=no \ diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh index 0db1cef0da..94e0d46841 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh @@ -14,7 +14,7 @@ while [ ! -f /etc/kubernetes/certs/ca.key ] && \ done echo "starting services" -for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler; do +for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler kube-proxy; do echo "activating service $service" systemctl enable $service systemctl --no-block start $service diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml index 330f21c645..00f83a4d73 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml @@ -39,6 +39,8 @@ write_files: WAIT_CURL="$WAIT_CURL" KUBE_TAG="$KUBE_TAG" FLANNEL_TAG="$FLANNEL_TAG" + FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR" + PODS_NETWORK_CIDR="$PODS_NETWORK_CIDR" KUBE_VERSION="$KUBE_VERSION" TRUSTEE_USER_ID="$TRUSTEE_USER_ID" TRUSTEE_PASSWORD="$TRUSTEE_PASSWORD" diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml index e38a218ac1..0494f279bc 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml @@ -333,6 +333,8 @@ resources: $NO_PROXY: {get_param: no_proxy} $KUBE_TAG: {get_param: kube_tag} $FLANNEL_TAG: {get_param: flannel_tag} + $FLANNEL_NETWORK_CIDR: {get_param: flannel_network_cidr} + $PODS_NETWORK_CIDR: {get_param: pods_network_cidr} $KUBE_VERSION: {get_param: kube_version} $WAIT_CURL: {get_attr: [minion_wait_handle, curl_cli]} $TRUSTEE_USER_ID: {get_param: trustee_user_id}