From 6b27f02c55d488e982679b202e2560e3a26ef052 Mon Sep 17 00:00:00 2001 From: archyufa Date: Sun, 11 Sep 2016 13:09:33 -0400 Subject: [PATCH] [k8s] Add KubeDns addon to K8s Cluster This patch introduce kubeDns addon support in murano k8s app. The docs of how kubeDns works and how it can be checked are here: https://github.com/kubernetes/kubernetes/blob/master/build/kube-dns/README.md#how-do-i-test-if-it-is-working Co-Authored-By: ddovbii Change-Id: I77b400ffd059a326b8fa051e038706370cbf6aaf --- .../package/Classes/KubernetesCluster.yaml | 43 ++++- .../package/Classes/KubernetesMinionNode.yaml | 3 +- .../package/Resources/DeployKubeDns.template | 32 ++++ .../Resources/KubeMinionSetup.template | 3 +- .../scripts/addons/kube-dns-addon.yaml | 152 ++++++++++++++++++ .../Resources/scripts/deployKubeDns.sh | 5 + .../Resources/scripts/master-kube-setup.sh | 2 - .../Resources/scripts/minion-kube-setup.sh | 10 ++ .../Resources/scripts/systemd/kubelet.service | 2 + .../KubernetesCluster/package/UI/ui.yaml | 60 ++++--- 10 files changed, 282 insertions(+), 30 deletions(-) create mode 100644 Kubernetes/KubernetesCluster/package/Resources/DeployKubeDns.template create mode 100644 Kubernetes/KubernetesCluster/package/Resources/scripts/addons/kube-dns-addon.yaml create mode 100644 Kubernetes/KubernetesCluster/package/Resources/scripts/deployKubeDns.sh diff --git a/Kubernetes/KubernetesCluster/package/Classes/KubernetesCluster.yaml b/Kubernetes/KubernetesCluster/package/Classes/KubernetesCluster.yaml index 6af26b8..9930e67 100644 --- a/Kubernetes/KubernetesCluster/package/Classes/KubernetesCluster.yaml +++ b/Kubernetes/KubernetesCluster/package/Classes/KubernetesCluster.yaml @@ -51,6 +51,10 @@ Properties: Contract: $.bool().notNull() Default: false + enableKubeDns: + Contract: $.bool().notNull() + Default: true + dockerRegistry: Contract: $.string() @@ -144,6 +148,10 @@ Methods: - $.minionNodes.take($.nodeCount).pselect($.setupNode()) - $.gatewayNodes.take($.gatewayCount).pselect($.setupNode()) + - If: $.enableKubeDns + Then: + $._deployDns() + - $._environment.stack.push() - $._updateServicePublicIps() - $.setAttr(lastNodeCount, $.nodeCount) @@ -157,6 +165,40 @@ Methods: Return: $.masterNode.getIp() + _deployDns: + Body: + - If: not $.getAttr(dnsDeployed, false) + Then: + - $securityGroupIngress: + - ToPort: 8001 + FromPort: 8001 + IpProtocol: tcp + External: false + - ToPort: 10053 + FromPort: 10053 + IpProtocol: udp + External: false + - ToPort: 10053 + FromPort: 10053 + IpProtocol: tcp + External: false + - ToPort: 53 + FromPort: 53 + IpProtocol: udp + External: false + - ToPort: 53 + FromPort: 53 + IpProtocol: tcp + External: false + + - $._environment.securityGroupManager.addGroupIngress($securityGroupIngress) + + - $resources: new(sys:Resources) + - $template: $resources.yaml('DeployKubeDns.template') + - $.masterNode.instance.agent.call($template, $resources) + - $.setAttr(dnsDeployed, true) + + _deployContainersNetwork: Body: - If: $.useFlannel @@ -529,7 +571,6 @@ Methods: - $._environment.reporter.report($this, 'No gateway nodes that can be removed') - scaleRc: Arguments: - rcName: diff --git a/Kubernetes/KubernetesCluster/package/Classes/KubernetesMinionNode.yaml b/Kubernetes/KubernetesCluster/package/Classes/KubernetesMinionNode.yaml index 3d0b003..c49e014 100644 --- a/Kubernetes/KubernetesCluster/package/Classes/KubernetesMinionNode.yaml +++ b/Kubernetes/KubernetesCluster/package/Classes/KubernetesMinionNode.yaml @@ -103,7 +103,8 @@ Methods: dockerRegistry => $._cluster.dockerRegistry, dockerMirror => $._cluster.dockerMirror, gcloudKey => $._cluster.gcloudKey, - useFlannel => $._cluster.useFlannel + useFlannel => $._cluster.useFlannel, + enableKubeDns => $._cluster.enableKubeDns )) - $.instance.agent.call($template, $resources) - $.setAttr(nodeConfigured, true) diff --git a/Kubernetes/KubernetesCluster/package/Resources/DeployKubeDns.template b/Kubernetes/KubernetesCluster/package/Resources/DeployKubeDns.template new file mode 100644 index 0000000..8cf473e --- /dev/null +++ b/Kubernetes/KubernetesCluster/package/Resources/DeployKubeDns.template @@ -0,0 +1,32 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +FormatVersion: 2.0.0 +Version: 1.0.0 +Name: Deploy KubeDNS + +Parameters: + +Body: | + return deploy().stdout + +Scripts: + deploy: + Type: Application + Version: 1.0.0 + EntryPoint: deployKubeDns.sh + Files: + - addons/kube-dns-addon.yaml + Options: + captureStdout: true + captureStderr: true + diff --git a/Kubernetes/KubernetesCluster/package/Resources/KubeMinionSetup.template b/Kubernetes/KubernetesCluster/package/Resources/KubeMinionSetup.template index 82af461..23b8dd7 100644 --- a/Kubernetes/KubernetesCluster/package/Resources/KubeMinionSetup.template +++ b/Kubernetes/KubernetesCluster/package/Resources/KubeMinionSetup.template @@ -22,6 +22,7 @@ Parameters: dockerMirror: $dockerMirror gcloudKey: $gcloudKey useFlannel: $useFlannel + enableKubeDns: $enableKubeDns Body: | if args.dockerRegistry: @@ -32,7 +33,7 @@ Body: | loginToGoogleRegistry("'{0}'".format(args.gcloudKey)) restartDocker() - setup('{0} {1} {2} {3}'.format(args.name, args.ip, args.masterIp, args.useFlannel)) + setup('{0} {1} {2} {3} {4}'.format(args.name, args.ip, args.masterIp, args.useFlannel, args.enableKubeDns)) Scripts: setup: diff --git a/Kubernetes/KubernetesCluster/package/Resources/scripts/addons/kube-dns-addon.yaml b/Kubernetes/KubernetesCluster/package/Resources/scripts/addons/kube-dns-addon.yaml new file mode 100644 index 0000000..7a77f1e --- /dev/null +++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/addons/kube-dns-addon.yaml @@ -0,0 +1,152 @@ +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "KubeDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: 10.32.0.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + + +--- + + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: kube-dns-v20 + namespace: kube-system + labels: + k8s-app: kube-dns + version: v20 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: kube-dns + version: v20 + template: + metadata: + labels: + k8s-app: kube-dns + version: v20 + kubernetes.io/cluster-service: "true" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + spec: + volumes: + - name: kubeconfig + hostPath: + path: /etc/kubernetes/kubeconfig.yaml + containers: + - name: kubedns + image: gcr.io/google_containers/kubedns-amd64:1.8 + resources: + # TODO: Set memory limits when we've profiled the container for + # large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from + # restarting it. + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + livenessProbe: + httpGet: + path: /healthz-kubedns + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + # we poll on pod startup for the Kubernetes master service and + # only setup the /readiness HTTP server once that's available. + initialDelaySeconds: 3 + timeoutSeconds: 5 + args: + # command = "/kube-dns" + - --domain=kubernetes.local + - --dns-port=10053 + - --kubecfg-file=/etc/kubernetes/kubeconfig.yaml + volumeMounts: + - mountPath: /etc/kubernetes/kubeconfig.yaml + name: kubeconfig + readOnly: true + ports: + - containerPort: 10053 + name: dns-local + protocol: UDP + - containerPort: 10053 + name: dns-tcp-local + protocol: TCP + - name: dnsmasq + image: gcr.io/google_containers/kube-dnsmasq-amd64:1.4 + livenessProbe: + httpGet: + path: /healthz-dnsmasq + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + args: + - --cache-size=1000 + - --no-resolv + - --server=127.0.0.1#10053 + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - name: healthz + image: gcr.io/google_containers/exechealthz-amd64:1.2 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + memory: 50Mi + requests: + cpu: 10m + # Note that this container shouldn't really need 50Mi of memory. + # The + # limits are set higher than expected pending investigation on + # #29688. + # The extra memory was stolen from the kubedns container to keep + # the + # net memory requested by the pod constant. + memory: 50Mi + args: + - --cmd=nslookup kubernetes.default.svc.kubernetes.local 127.0.0.1 >/dev/null + - --url=/healthz-dnsmasq + - --cmd=nslookup kubernetes.default.svc.kubernetes.local 127.0.0.1:10053 >/dev/null + - --url=/healthz-kubedns + - --port=8080 + - --quiet + ports: + - containerPort: 8080 + protocol: TCP + dnsPolicy: Default # Don't use cluster DNS. diff --git a/Kubernetes/KubernetesCluster/package/Resources/scripts/deployKubeDns.sh b/Kubernetes/KubernetesCluster/package/Resources/scripts/deployKubeDns.sh new file mode 100644 index 0000000..37ef5dc --- /dev/null +++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/deployKubeDns.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cp -f addons/kube-dns-addon.yaml /etc/kubernetes/addons + +/opt/bin/kubectl create -f /etc/kubernetes/addons/kube-dns-addon.yaml >> /tmp/murano-kube.log \ No newline at end of file diff --git a/Kubernetes/KubernetesCluster/package/Resources/scripts/master-kube-setup.sh b/Kubernetes/KubernetesCluster/package/Resources/scripts/master-kube-setup.sh index 520bea0..90af7d8 100644 --- a/Kubernetes/KubernetesCluster/package/Resources/scripts/master-kube-setup.sh +++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/master-kube-setup.sh @@ -14,8 +14,6 @@ if [[ $(which systemctl) ]]; then sed -i.bak "s/%%MASTER_IP%%/$2/g" environ/apiserver sed -i.bak "s/%%MASTER_IP%%/$2/g" default_scripts/kube-apiserver - mkdir -p /etc/kubernetes/ - cp -f environ/apiserver /etc/kubernetes/apiserver cp -f environ/kube-config /etc/kubernetes/config diff --git a/Kubernetes/KubernetesCluster/package/Resources/scripts/minion-kube-setup.sh b/Kubernetes/KubernetesCluster/package/Resources/scripts/minion-kube-setup.sh index 8f2b89f..06107b6 100644 --- a/Kubernetes/KubernetesCluster/package/Resources/scripts/minion-kube-setup.sh +++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/minion-kube-setup.sh @@ -4,6 +4,7 @@ # $2 - IP # $3 - MASTER_IP # $4 - USE_CALICO +# $5 - ENABLE_DNS mkdir -p /var/run/murano-kubernetes mkdir -p /etc/kubernetes/ @@ -18,6 +19,15 @@ if [[ $(which systemctl) ]]; then echo KUBELET_ARGS=\"--network-plugin=cni --network-plugin-dir=/etc/cni/net.d\" >> environ/kubelet fi + if [ "$5" == "True" ]; then + echo KUBELET_CLUSTER_DNS=\"--cluster-dns=10.32.0.10\" >> environ/kubelet + echo KUBELET_DNS_DOMAIN=\"--cluster-domain=kubernetes.local\" >> environ/kubelet + + if [ "$4" == "False" ]; then + echo KUBE_PROXY_ARGS=\"--proxy-mode=iptables --masquerade-all=true\" > /etc/kubernetes/proxy + fi + fi + mkdir -p /etc/kubernetes/ cp -f environ/kubelet /etc/kubernetes/ diff --git a/Kubernetes/KubernetesCluster/package/Resources/scripts/systemd/kubelet.service b/Kubernetes/KubernetesCluster/package/Resources/scripts/systemd/kubelet.service index 5b1e291..1eab21b 100644 --- a/Kubernetes/KubernetesCluster/package/Resources/scripts/systemd/kubelet.service +++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/systemd/kubelet.service @@ -15,6 +15,8 @@ ExecStart=/opt/bin/hyperkube kubelet \ $KUBELET_PORT \ $KUBELET_HOSTNAME \ $KUBE_ALLOW_PRIV \ + $KUBELET_CLUSTER_DNS \ + $KUBELET_DNS_DOMAIN \ $KUBELET_ARGS Restart=on-failure KillMode=process diff --git a/Kubernetes/KubernetesCluster/package/UI/ui.yaml b/Kubernetes/KubernetesCluster/package/UI/ui.yaml index 68c7473..1e26afe 100644 --- a/Kubernetes/KubernetesCluster/package/UI/ui.yaml +++ b/Kubernetes/KubernetesCluster/package/UI/ui.yaml @@ -18,10 +18,10 @@ Templates: instance: ?: type: io.murano.resources.LinuxMuranoInstance - name: generateHostname($.appConfiguration.unitNamingPattern, 1) + name: generateHostname($.nodesConfiguration.unitNamingPattern, 1) flavor: $.instanceConfiguration.flavor image: $.instanceConfiguration.image - assignFloatingIp: $.appConfiguration.assignFloatingIP + assignFloatingIp: $.nodesConfiguration.assignFloatingIP keyname: $.instanceConfiguration.keyPair availabilityZone: $.instanceConfiguration.availabilityZone @@ -31,13 +31,13 @@ Templates: instance: ?: type: io.murano.resources.LinuxMuranoInstance - name: generateHostname($.appConfiguration.unitNamingPattern, $index + 1) + name: generateHostname($.nodesConfiguration.unitNamingPattern, $index + 1) flavor: $.instanceConfiguration.flavor image: $.instanceConfiguration.image - assignFloatingIp: $.appConfiguration.assignFloatingIP + assignFloatingIp: $.nodesConfiguration.assignFloatingIP keyname: $.instanceConfiguration.keyPair availabilityZone: $.instanceConfiguration.availabilityZone - exposeCAdvisor: $.appConfiguration.exposeCAdvisor + exposeCAdvisor: $.nodesConfiguration.exposeCAdvisor gatewayNode: @@ -46,30 +46,30 @@ Templates: instance: ?: type: io.murano.resources.LinuxMuranoInstance - name: generateHostname($.appConfiguration.gatewayNamingPattern, $index) + name: generateHostname($.nodesConfiguration.gatewayNamingPattern, $index) flavor: $.instanceConfiguration.flavor image: $.instanceConfiguration.image - assignFloatingIp: $.appConfiguration.assignGatewayFloatingIP + assignFloatingIp: $.nodesConfiguration.assignGatewayFloatingIP keyname: $.instanceConfiguration.keyPair availabilityZone: $.instanceConfiguration.availabilityZone Application: ?: type: com.mirantis.docker.kubernetes.KubernetesCluster - name: $.appConfiguration.name + name: $.nodesConfiguration.name masterNode: $masterNode - minionNodes: repeat($minionNode, $.appConfiguration.maxMinionCount) - nodeCount: $.appConfiguration.minionCount - gatewayCount: $.appConfiguration.gatewayCount - useFlannel: $.appConfiguration.useFlannel - gatewayNodes: repeat($gatewayNode, $.appConfiguration.maxGatewayCount) - dockerRegistry: $.appConfiguration.dockerRegistry - dockerMirror: $.appConfiguration.dockerMirror - gcloudKey: $.appConfiguration.gcloudKey - + minionNodes: repeat($minionNode, $.nodesConfiguration.maxMinionCount) + nodeCount: $.nodesConfiguration.minionCount + gatewayCount: $.nodesConfiguration.gatewayCount + useFlannel: $.kubeNetConfiguration.useFlannel + gatewayNodes: repeat($gatewayNode, $.nodesConfiguration.maxGatewayCount) + dockerRegistry: $.kubeNetConfiguration.dockerRegistry + dockerMirror: $.kubeNetConfiguration.dockerMirror + gcloudKey: $.kubeNetConfiguration.gcloudKey + enableKubeDns: $.kubeNetConfiguration.enableKubeDns Forms: - - appConfiguration: + - nodesConfiguration: fields: - name: license type: string @@ -106,13 +106,6 @@ Forms: description: >- Check to assign floating IP to Kubernetes nodes required: false - - name: useFlannel - type: boolean - initial: false - label: Use Flannel networking instead of Calico - description: >- - Check, if you are going use the Flannel networking instead of Calico - required: false - name: unitNamingPattern type: string initial: kube-# @@ -170,6 +163,23 @@ Forms: helpText: "# expands to gateway sequence number" description: >- Check to assign floating IP to gateway nodes + + - kubeNetConfiguration: + fields: + - name: useFlannel + type: boolean + initial: false + label: Use Flannel networking instead of Calico + description: >- + Check, if you are going to use the Flannel networking instead of Calico + required: false + - name: enableKubeDns + type: boolean + initial: true + label: Enable KubeDNS addon + description: >- + Check, if you are going to use KubeDNS feature in your cluster + required: false - name: dockerRegistry type: string label: Custom Docker registry URL