From 308f8a2121882aa5d8edddca19aa4d5008f85cf3 Mon Sep 17 00:00:00 2001 From: intlabs Date: Mon, 30 Oct 2017 19:05:14 -0500 Subject: [PATCH] Calico: Add basic dependency checks and rbac support This PS update the calico chart and deployment to use Kubernetes entrypoint, and apply appropriate RBAC rules to the pods. Change-Id: I9d875f50c4767b6714a4931b9ade0a6f94b533c2 --- calico/templates/configmap-calico-config.yaml | 2 +- calico/templates/daemonset-calico-etcd.yaml | 9 +- calico/templates/daemonset-calico-node.yaml | 4 + .../deployment-calico-policy-controller.yaml | 5 ++ calico/templates/rbac-entrypoint.yaml | 17 ++++ calico/templates/service-calico-etcd.yaml | 4 +- calico/values.yaml | 24 ++++++ .../_kubernetes_entrypoint_init_container.tpl | 5 +- .../snippets/_kubernetes_entrypoint_rbac.tpl | 86 +++++++++++++++++++ .../_kubernetes_entrypoint_secret_mount.tpl | 24 ++++++ .../templates/daemonset-registry-proxy.yaml | 3 +- registry/templates/deployment-registry.yaml | 3 +- registry/templates/job-bootstrap.yaml | 3 +- registry/templates/rbac-entrypoint.yaml | 17 ++++ .../deploy-kubeadm-master/tasks/helm-cni.yaml | 4 +- 15 files changed, 199 insertions(+), 11 deletions(-) create mode 100644 calico/templates/rbac-entrypoint.yaml create mode 100644 helm-toolkit/templates/snippets/_kubernetes_entrypoint_rbac.tpl create mode 100644 helm-toolkit/templates/snippets/_kubernetes_entrypoint_secret_mount.tpl create mode 100644 registry/templates/rbac-entrypoint.yaml diff --git a/calico/templates/configmap-calico-config.yaml b/calico/templates/configmap-calico-config.yaml index e20d9c619f..ef21fce711 100644 --- a/calico/templates/configmap-calico-config.yaml +++ b/calico/templates/configmap-calico-config.yaml @@ -24,7 +24,7 @@ metadata: data: # The location of your etcd cluster. This uses the Service clusterIP # defined below. - etcd_endpoints: "http://10.96.232.136:6666" + etcd_endpoints: http://10.96.232.136:{{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} # Configure the Calico backend to use. calico_backend: "bird" diff --git a/calico/templates/daemonset-calico-etcd.yaml b/calico/templates/daemonset-calico-etcd.yaml index 018e4c38cf..a9e0bfee2d 100644 --- a/calico/templates/daemonset-calico-etcd.yaml +++ b/calico/templates/daemonset-calico-etcd.yaml @@ -57,8 +57,13 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - command: ["/bin/sh","-c"] - args: ["/usr/local/bin/etcd --name=calico --data-dir=/var/etcd/calico-data --advertise-client-urls=http://$CALICO_ETCD_IP:6666 --listen-client-urls=http://0.0.0.0:6666 --listen-peer-urls=http://0.0.0.0:6667"] + command: + - /usr/local/bin/etcd + - --name=calico + - --data-dir=/var/etcd/calico-data + - --advertise-client-urls=http://$CALICO_ETCD_IP:{{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - --listen-client-urls=http://0.0.0.0:{{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - --listen-peer-urls=http://0.0.0.0:{{ tuple "etcd" "internal" "peer" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} volumeMounts: - name: var-etcd mountPath: /var/etcd diff --git a/calico/templates/daemonset-calico-node.yaml b/calico/templates/daemonset-calico-node.yaml index 02a94f8d56..e40f69b17b 100644 --- a/calico/templates/daemonset-calico-node.yaml +++ b/calico/templates/daemonset-calico-node.yaml @@ -15,6 +15,7 @@ limitations under the License. */}} {{- $envAll := . }} +{{- $dependencies := .Values.dependencies.calico_node }} --- # This manifest installs the calico/node container, as well # as the Calico CNI plugins and network config on @@ -52,6 +53,8 @@ spec: - key: CriticalAddonsOnly operator: Exists serviceAccountName: calico-cni-plugin + initContainers: +{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: # Runs calico/node container on each Kubernetes node. This # container programs network policy and routes on each @@ -166,3 +169,4 @@ spec: - name: cni-net-dir hostPath: path: /etc/cni/net.d +{{ tuple . | include "helm-toolkit.snippets.kubernetes_entrypoint_secret_mount" | indent 8 }} diff --git a/calico/templates/deployment-calico-policy-controller.yaml b/calico/templates/deployment-calico-policy-controller.yaml index 20431bf693..f993f8861d 100644 --- a/calico/templates/deployment-calico-policy-controller.yaml +++ b/calico/templates/deployment-calico-policy-controller.yaml @@ -15,6 +15,7 @@ limitations under the License. */}} {{- $envAll := . }} +{{- $dependencies := .Values.dependencies.calico_policy_controller }} --- # This manifest deploys the Calico policy controller on Kubernetes. # See https://github.com/projectcalico/k8s-policy @@ -53,6 +54,8 @@ spec: - key: CriticalAddonsOnly operator: Exists serviceAccountName: calico-policy-controller + initContainers: +{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: calico-policy-controller {{ tuple $envAll "calico_kube_policy_controller" | include "helm-toolkit.snippets.image" | indent 10 }} @@ -72,3 +75,5 @@ spec: # kubernetes.default to the correct service clusterIP. - name: CONFIGURE_ETC_HOSTS value: "true" + volumes: +{{ tuple . | include "helm-toolkit.snippets.kubernetes_entrypoint_secret_mount" | indent 8 }} diff --git a/calico/templates/rbac-entrypoint.yaml b/calico/templates/rbac-entrypoint.yaml new file mode 100644 index 0000000000..c05fe88896 --- /dev/null +++ b/calico/templates/rbac-entrypoint.yaml @@ -0,0 +1,17 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{ tuple . | include "helm-toolkit.snippets.kubernetes_entrypoint_rbac"}} diff --git a/calico/templates/service-calico-etcd.yaml b/calico/templates/service-calico-etcd.yaml index f0aa97b160..6a37c196a4 100644 --- a/calico/templates/service-calico-etcd.yaml +++ b/calico/templates/service-calico-etcd.yaml @@ -24,7 +24,7 @@ metadata: labels: k8s-app: calico-etcd {{ tuple $envAll "calico" "etcd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} - name: calico-etcd + name: {{ tuple "etcd" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} spec: # Select the calico-etcd pod running on the master. selector: @@ -34,4 +34,4 @@ spec: # on DNS to get access to etcd. clusterIP: 10.96.232.136 ports: - - port: 6666 + - port: {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} diff --git a/calico/values.yaml b/calico/values.yaml index 36f03aa67c..3b9ab0ee75 100644 --- a/calico/values.yaml +++ b/calico/values.yaml @@ -26,9 +26,33 @@ images: calico_node: quay.io/calico/node:v2.4.1 calico_cni: quay.io/calico/cni:v1.10.0 calico_kube_policy_controller: quay.io/calico/kube-policy-controller:v0.7.0 + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 pull_policy: IfNotPresent registry: prefix: null +dependencies: + calico_node: + services: + - service: etcd + endpoint: internal + calico_policy_controller: + services: + - service: etcd + endpoint: internal + +endpoints: + cluster_domain_suffix: cluster.local + etcd: + hosts: + default: calico-etcd + host_fqdn_override: + default: null + port: + client: + default: 6666 + peer: + default: 6667 + networking: podSubnet: 192.168.0.0/16 diff --git a/helm-toolkit/templates/snippets/_kubernetes_entrypoint_init_container.tpl b/helm-toolkit/templates/snippets/_kubernetes_entrypoint_init_container.tpl index 3c96d07184..30dff46bc7 100644 --- a/helm-toolkit/templates/snippets/_kubernetes_entrypoint_init_container.tpl +++ b/helm-toolkit/templates/snippets/_kubernetes_entrypoint_init_container.tpl @@ -18,6 +18,8 @@ limitations under the License. {{- $envAll := index . 0 -}} {{- $deps := index . 1 -}} {{- $mounts := index . 2 -}} +{{- $mountServiceAccount := dict "mountPath" "/var/run/secrets/kubernetes.io/serviceaccount" "name" "entrypoint-serviceaccount-secret" "readOnly" true -}} +{{- $mountsEntrypoint := append $mounts $mountServiceAccount -}} - name: init image: {{ $envAll.Values.images.tags.dep_check }} imagePullPolicy: {{ $envAll.Values.images.pull_policy }} @@ -46,5 +48,6 @@ limitations under the License. value: "echo done" command: - kubernetes-entrypoint - volumeMounts: {{ $mounts | default "[]"}} + volumeMounts: +{{ toYaml $mountsEntrypoint | indent 4 }} {{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_entrypoint_rbac.tpl b/helm-toolkit/templates/snippets/_kubernetes_entrypoint_rbac.tpl new file mode 100644 index 0000000000..6c65162461 --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_entrypoint_rbac.tpl @@ -0,0 +1,86 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_entrypoint_rbac" -}} +{{- $envAll := index . 0 -}} +{{- $component := $envAll.Release.Name -}} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: cluster-role-binding-entrypoint-{{ $component }} + annotations: + # Tiller sorts the execution of resources in the following order: + # Secret, ServiceAccount, Role, RoleBinding. The problem is that + # this Secret will not be created if ServiceAccount doesn't exist. + # The solution is to add pre-install hook so that these are created first. + helm.sh/hook: pre-install +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-role-entrypoint-{{ $component }} +subjects: + - kind: ServiceAccount + name: service-account-entrypoint-{{ $component }} + namespace: {{ $envAll.Release.Namespace }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: cluster-role-entrypoint-{{ $component }} + annotations: + # Tiller sorts the execution of resources in the following order: + # Secret, ServiceAccount, Role, RoleBinding. The problem is that + # this Secret will not be created if ServiceAccount doesn't exist. + # The solution is to add pre-install hook so that these are created first. + helm.sh/hook: pre-install +rules: + - apiGroups: + - "" + - extensions + - batch + - apps + resources: + - pods + - services + - jobs + - endpoints + - daemonsets + verbs: + - get + - list +--- +apiVersion: v1 +kind: Secret +metadata: + name: secret-entrypoint-{{ $component }} + namespace: {{ $envAll.Release.Namespace }} + annotations: + kubernetes.io/service-account.name: service-account-entrypoint-{{ $component }} +type: kubernetes.io/service-account-token +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: service-account-entrypoint-{{ $component }} + namespace: {{ $envAll.Release.Namespace }} + annotations: + # Tiller sorts the execution of resources in the following order: + # Secret, ServiceAccount, Role, RoleBinding. The problem is that + # this Secret will not be created if ServiceAccount doesn't exist. + # The solution is to add pre-install hook so that these are created first. + helm.sh/hook: pre-install +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_entrypoint_secret_mount.tpl b/helm-toolkit/templates/snippets/_kubernetes_entrypoint_secret_mount.tpl new file mode 100644 index 0000000000..405c4b206c --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_entrypoint_secret_mount.tpl @@ -0,0 +1,24 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_entrypoint_secret_mount" -}} +{{- $envAll := index . 0 -}} +{{- $component := $envAll.Release.Name -}} +- name: entrypoint-serviceaccount-secret + secret: + secretName: secret-entrypoint-{{ $component }} + defaultMode: 420 +{{- end -}} diff --git a/registry/templates/daemonset-registry-proxy.yaml b/registry/templates/daemonset-registry-proxy.yaml index 0212528cb9..5af130cc05 100644 --- a/registry/templates/daemonset-registry-proxy.yaml +++ b/registry/templates/daemonset-registry-proxy.yaml @@ -36,7 +36,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet hostNetwork: true initContainers: -{{ tuple $envAll $dependencies "[]" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: registry-proxy {{ tuple $envAll "registry_proxy" | include "helm-toolkit.snippets.image" | indent 8 }} @@ -61,4 +61,5 @@ spec: configMap: name: registry-etc defaultMode: 0444 +{{ tuple . | include "helm-toolkit.snippets.kubernetes_entrypoint_secret_mount" | indent 8 }} {{- end }} diff --git a/registry/templates/deployment-registry.yaml b/registry/templates/deployment-registry.yaml index 6f2b9da732..20bab7b0cd 100644 --- a/registry/templates/deployment-registry.yaml +++ b/registry/templates/deployment-registry.yaml @@ -38,7 +38,7 @@ spec: nodeSelector: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} initContainers: -{{ tuple $envAll $dependencies "[]" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: registry {{ tuple $envAll "registry" | include "helm-toolkit.snippets.image" | indent 10 }} @@ -71,4 +71,5 @@ spec: - name: docker-images persistentVolumeClaim: claimName: docker-images +{{ tuple . | include "helm-toolkit.snippets.kubernetes_entrypoint_secret_mount" | indent 8 }} {{- end }} diff --git a/registry/templates/job-bootstrap.yaml b/registry/templates/job-bootstrap.yaml index d873eb5d9c..64cfddd132 100644 --- a/registry/templates/job-bootstrap.yaml +++ b/registry/templates/job-bootstrap.yaml @@ -33,7 +33,7 @@ spec: nodeSelector: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} initContainers: -{{ tuple $envAll $dependencies "[]" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: docker-bootstrap {{ tuple $envAll "bootstrap" | include "helm-toolkit.snippets.image" | indent 10 }} @@ -60,5 +60,6 @@ spec: - name: docker-socket hostPath: path: /var/run/docker.sock +{{ tuple . | include "helm-toolkit.snippets.kubernetes_entrypoint_secret_mount" | indent 8 }} {{- end }} {{- end }} diff --git a/registry/templates/rbac-entrypoint.yaml b/registry/templates/rbac-entrypoint.yaml new file mode 100644 index 0000000000..c05fe88896 --- /dev/null +++ b/registry/templates/rbac-entrypoint.yaml @@ -0,0 +1,17 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{ tuple . | include "helm-toolkit.snippets.kubernetes_entrypoint_rbac"}} diff --git a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/helm-cni.yaml b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/helm-cni.yaml index 8377b6858e..614a3efc94 100644 --- a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/helm-cni.yaml +++ b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/helm-cni.yaml @@ -55,7 +55,7 @@ environment: KUBECONFIG: '/mnt/rootfs/etc/kubernetes/admin.conf' - name: kubeadm | cni | calico - command: helm install /opt/charts/calico --name calico --namespace kube-system --set networking.podSubnet="{{ k8s.networking.podSubnet }}" --wait + command: helm install /opt/charts/calico --name calico --namespace kube-system --set networking.podSubnet="{{ k8s.networking.podSubnet }}" --wait --timeout=600 environment: HELM_HOST: 'localhost:44134' - name: kubeadm | cni | calico @@ -72,7 +72,7 @@ delegate_to: 127.0.0.1 block: - name: kubeadm | cni | flannel - command: helm install /opt/charts/flannel --name flannel --namespace kube-system --set networking.podSubnet="{{ k8s.networking.podSubnet }}" --wait + command: helm install /opt/charts/flannel --name flannel --namespace kube-system --set networking.podSubnet="{{ k8s.networking.podSubnet }}" --wait --timeout=600 environment: HELM_HOST: 'localhost:44134' - name: kubeadm | cni | flannel