diff --git a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh index 4b477d3b52..2769c767c9 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh @@ -7,24 +7,112 @@ printf "Starting to run ${step}\n" if [ "$NETWORK_DRIVER" = "calico" ]; then _prefix=${CONTAINER_INFRA_PREFIX:-quay.io/calico/} - ETCD_SERVER_IP=${ETCD_LB_VIP:-$KUBE_NODE_IP} - CERT_DIR=/etc/kubernetes/certs - ETCD_CA=`cat ${CERT_DIR}/ca.crt | base64 | tr -d '\n'` - ETCD_CERT=`cat ${CERT_DIR}/server.crt | base64 | tr -d '\n'` - ETCD_KEY=`cat ${CERT_DIR}/server.key | base64 | tr -d '\n'` CALICO_DEPLOY=/srv/magnum/kubernetes/manifests/calico-deploy.yaml - [ -f ${CALICO_DEPLOY} ] || { echo "Writing File: $CALICO_DEPLOY" mkdir -p $(dirname ${CALICO_DEPLOY}) cat << EOF > ${CALICO_DEPLOY} -# Calico Version v2.6.7 -# https://docs.projectcalico.org/v2.6/releases#v2.6.7 +--- +# Calico Version v3.3.6 +# https://docs.projectcalico.org/v3.3/releases#v3.3.6 +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: calico-node +rules: + - apiGroups: [""] + resources: + - namespaces + - serviceaccounts + verbs: + - get + - list + - watch + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: [""] + resources: + - services + verbs: + - get + - apiGroups: [""] + resources: + - endpoints + verbs: + - get + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - update + - watch + - apiGroups: ["extensions"] + resources: + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - clusterinformations + - hostendpoints + verbs: + - create + - get + - list + - update + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: calico-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system + +--- +# Calico Version v3.3.6 +# https://docs.projectcalico.org/v3.3/releases#v3.3.6 # This manifest includes the following component versions: -# calico/node:v2.6.7 -# calico/cni:v1.11.2 -# calico/kube-controllers:v1.0.3 +# calico/node:v3.3.6 +# calico/cni:v3.3.6 # This ConfigMap is used to configure a self-hosted Calico installation. kind: ConfigMap @@ -33,60 +121,176 @@ metadata: name: calico-config namespace: kube-system data: - # Configure this with the location of your etcd cluster. - etcd_endpoints: "https://${ETCD_SERVER_IP}:2379" - + # To enable Typha, set this to "calico-typha" *and* set a non-zero value for Typha replicas + # below. We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is + # essential. + typha_service_name: "none" # Configure the Calico backend to use. calico_backend: "bird" - # The CNI network configuration to install on each node. + # Configure the MTU to use + veth_mtu: "1440" + + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. cni_network_config: |- { - "name": "k8s-pod-network", - "cniVersion": "0.1.0", - "type": "calico", - "etcd_endpoints": "__ETCD_ENDPOINTS__", - "etcd_key_file": "__ETCD_KEY_FILE__", - "etcd_cert_file": "__ETCD_CERT_FILE__", - "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__", - "log_level": "info", - "mtu": 1500, - "ipam": { - "type": "calico-ipam" + "name": "k8s-pod-network", + "cniVersion": "0.3.0", + "plugins": [ + { + "type": "calico", + "log_level": "info", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": { + "type": "host-local", + "subnet": "usePodCidr" + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } }, - "policy": { - "type": "k8s", - "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__", - "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__" - }, - "kubernetes": { - "kubeconfig": "__KUBECONFIG_FILEPATH__" + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} } + ] } - # If you're using TLS enabled etcd uncomment the following. - # You must also populate the Secret below with these files. - etcd_ca: "/calico-secrets/etcd-ca" - etcd_cert: "/calico-secrets/etcd-cert" - etcd_key: "/calico-secrets/etcd-key" --- -# The following contains k8s Secrets for use with a TLS enabled etcd cluster. -# For information on populating Secrets, see http://kubernetes.io/docs/user-guide/secrets/ + + +# This manifest creates a Service, which will be backed by Calico's Typha daemon. +# Typha sits in between Felix and the API server, reducing Calico's load on the API server. + apiVersion: v1 -kind: Secret -type: Opaque +kind: Service metadata: - name: calico-etcd-secrets + name: calico-typha namespace: kube-system -data: - # Populate the following files with etcd TLS configuration if desired, but leave blank if - # not using TLS for etcd. - # This self-hosted install expects three files with the following names. The values - # should be base64 encoded strings of the entire contents of each file. - etcd-key: ${ETCD_KEY} - etcd-cert: ${ETCD_CERT} - etcd-ca: ${ETCD_CA} + labels: + k8s-app: calico-typha +spec: + ports: + - port: 5473 + protocol: TCP + targetPort: calico-typha + name: calico-typha + selector: + k8s-app: calico-typha + --- + +# This manifest creates a Deployment of Typha to back the above service. + +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha +spec: + # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the + # typha_service_name variable in the calico-config ConfigMap above. + # + # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential + # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In + # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. + replicas: 0 + revisionHistoryLimit: 2 + template: + metadata: + labels: + k8s-app: calico-typha + annotations: + # This, along with the CriticalAddonsOnly toleration below, marks the pod as a critical + # add-on, ensuring it gets priority scheduling and that its resources are reserved + # if it ever gets evicted. + scheduler.alpha.kubernetes.io/critical-pod: '' + cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' + spec: + nodeSelector: + beta.kubernetes.io/os: linux + hostNetwork: true + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + # Since Calico can't network a pod until Typha is up, we need to run Typha itself + # as a host-networked pod. + serviceAccountName: calico-node + containers: + - image: "${CONTAINER_INFRA_PREFIX:-docker.io/calico/}typha:${CALICO_TAG}" + name: calico-typha + ports: + - containerPort: 5473 + name: calico-typha + protocol: TCP + env: + # Enable "info" logging by default. Can be set to "debug" to increase verbosity. + - name: TYPHA_LOGSEVERITYSCREEN + value: "info" + # Disable logging to file and syslog since those don't make sense in Kubernetes. + - name: TYPHA_LOGFILEPATH + value: "none" + - name: TYPHA_LOGSEVERITYSYS + value: "none" + # Monitor the Kubernetes API to find the number of running instances and rebalance + # connections. + - name: TYPHA_CONNECTIONREBALANCINGMODE + value: "kubernetes" + - name: TYPHA_DATASTORETYPE + value: "kubernetes" + - name: TYPHA_HEALTHENABLED + value: "true" + # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, + # this opens a port on the host, which may need to be secured. + #- name: TYPHA_PROMETHEUSMETRICSENABLED + # value: "true" + #- name: TYPHA_PROMETHEUSMETRICSPORT + # value: "9093" + livenessProbe: + exec: + command: + - calico-typha + - check + - liveness + periodSeconds: 30 + initialDelaySeconds: 30 + readinessProbe: + exec: + command: + - calico-typha + - check + - readiness + periodSeconds: 10 + +--- + +# This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict + +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-typha + +--- + # This manifest installs the calico/node container, as well # as the Calico CNI plugins and network config on # each master and worker node in a Kubernetes cluster. @@ -101,16 +305,26 @@ spec: selector: matchLabels: k8s-app: calico-node + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 template: metadata: labels: k8s-app: calico-node annotations: + # This, along with the CriticalAddonsOnly toleration below, + # marks the pod as a critical add-on, ensuring it gets + # priority scheduling and that its resources are reserved + # if it ever gets evicted. scheduler.alpha.kubernetes.io/critical-pod: '' spec: + nodeSelector: + beta.kubernetes.io/os: linux hostNetwork: true tolerations: - # Make sure calico/node gets scheduled on all nodes. + # Make sure calico-node gets scheduled on all nodes. - effect: NoSchedule operator: Exists # Mark the pod as a critical add-on for rescheduling. @@ -127,14 +341,25 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: ${_prefix}node:${CALICO_TAG} + image: "${CONTAINER_INFRA_PREFIX:-docker.io/calico/}node:${CALICO_TAG}" env: - # The location of the Calico etcd cluster. - - name: ETCD_ENDPOINTS + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Typha support: controlled by the ConfigMap. + - name: FELIX_TYPHAK8SSERVICENAME valueFrom: configMapKeyRef: name: calico-config - key: etcd_endpoints + key: typha_service_name + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName # Choose the backend to use. - name: CALICO_NETWORKING_BACKEND valueFrom: @@ -144,13 +369,12 @@ spec: # Cluster type to identify the deployment type - name: CLUSTER_TYPE value: "k8s,bgp" - # Disable file logging so 'kubectl logs' works. - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Configure the IP Pool from which Pod IPs will be chosen. + # Auto-detect the BGP IP address. + - name: IP + value: "autodetect" + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within '--cluster-cidr'. - name: CALICO_IPV4POOL_CIDR value: ${CALICO_IPV4POOL} - name: CALICO_IPV4POOL_IPIP @@ -162,36 +386,18 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + # Disable file logging so 'kubectl logs' works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" # Disable IPv6 on Kubernetes. - name: FELIX_IPV6SUPPORT value: "false" # Set Felix logging to "info" - name: FELIX_LOGSEVERITYSCREEN value: "info" - # Set MTU for tunnel device used if ipip is enabled - - name: FELIX_IPINIPMTU - value: "1440" - # Location of the CA certificate for etcd. - - name: ETCD_CA_CERT_FILE - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_ca - # Location of the client key for etcd. - - name: ETCD_KEY_FILE - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_key - # Location of the client certificate for etcd. - - name: ETCD_CERT_FILE - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_cert - # Auto-detect the BGP IP address. - - name: IP - value: "" - name: FELIX_HEALTHENABLED value: "true" securityContext: @@ -203,48 +409,61 @@ spec: httpGet: path: /liveness port: 9099 + host: localhost periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 readinessProbe: - httpGet: - path: /readiness - port: 9099 + exec: + command: + - /bin/calico-node + - -bird-ready + - -felix-ready periodSeconds: 10 volumeMounts: - mountPath: /lib/modules name: lib-modules readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false - mountPath: /var/run/calico name: var-run-calico readOnly: false - - mountPath: /calico-secrets - name: etcd-certs + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni - image: ${_prefix}cni:${CALICO_CNI_TAG} + image: "${CONTAINER_INFRA_PREFIX:-docker.io/calico/}cni:${CALICO_TAG}" command: ["/install-cni.sh"] env: - # The location of the Calico etcd cluster. - - name: ETCD_ENDPOINTS + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_endpoints + fieldRef: + fieldPath: spec.nodeName # The CNI network config to install on each node. - name: CNI_NETWORK_CONFIG valueFrom: configMapKeyRef: name: calico-config key: cni_network_config + # CNI MTU Config variable + - name: CNI_MTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu volumeMounts: - mountPath: /host/opt/cni/bin name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - mountPath: /calico-secrets - name: etcd-certs volumes: # Used by calico/node. - name: lib-modules @@ -253,6 +472,13 @@ spec: - name: var-run-calico hostPath: path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate # Used to install CNI. - name: cni-bin-dir hostPath: @@ -260,190 +486,151 @@ spec: - name: cni-net-dir hostPath: path: /etc/cni/net.d - # Mount in the etcd TLS secrets. - - name: etcd-certs - secret: - secretName: calico-etcd-secrets ---- -# This manifest deploys the Calico Kubernetes controllers. -# See https://github.com/projectcalico/kube-controllers -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - scheduler.alpha.kubernetes.io/tolerations: | - [{"key": "dedicated", "value": "master", "effect": "NoSchedule" }, - {"key":"CriticalAddonsOnly", "operator":"Exists"}] -spec: - # The controllers can only have a single active instance. - replicas: 1 - strategy: - type: Recreate - template: - metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers - spec: - # The controllers must run in the host network namespace so that - # it isn't governed by policy that would prevent it from working. - hostNetwork: true - serviceAccountName: calico-kube-controllers - containers: - - name: calico-kube-controllers - image: ${_prefix}kube-controllers:${CALICO_KUBE_CONTROLLERS_TAG} - env: - # The location of the Calico etcd cluster. - - name: ETCD_ENDPOINTS - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_endpoints - # Location of the CA certificate for etcd. - - name: ETCD_CA_CERT_FILE - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_ca - # Location of the client key for etcd. - - name: ETCD_KEY_FILE - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_key - # Location of the client certificate for etcd. - - name: ETCD_CERT_FILE - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_cert - # Choose which controllers to run. - - name: ENABLED_CONTROLLERS - value: policy,profile,workloadendpoint,node - volumeMounts: - # Mount in the etcd TLS secrets. - - mountPath: /calico-secrets - name: etcd-certs - volumes: - # Mount in the etcd TLS secrets. - - name: etcd-certs - secret: - secretName: calico-etcd-secrets ---- -# This deployment turns off the old "policy-controller". It should remain at 0 replicas, and then -# be removed entirely once the new kube-controllers deployment has been deployed above. -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: calico-policy-controller - namespace: kube-system - labels: - k8s-app: calico-policy -spec: - # Turn this deployment off in favor of the kube-controllers deployment above. - replicas: 0 - strategy: - type: Recreate - template: - metadata: - name: calico-policy-controller - namespace: kube-system - labels: - k8s-app: calico-policy - spec: - hostNetwork: true - serviceAccountName: calico-kube-controllers - containers: - - name: calico-policy-controller - image: ${_prefix}kube-controllers:${CALICO_KUBE_CONTROLLERS_TAG} - env: - # The location of the Calico etcd cluster. - - name: ETCD_ENDPOINTS - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_endpoints ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-kube-controllers - namespace: kube-system --- + apiVersion: v1 kind: ServiceAccount metadata: name: calico-node namespace: kube-system -# Calico Version v2.6.7 -# https://docs.projectcalico.org/v2.6/releases#v2.6.7 --- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 +# Create all the CustomResourceDefinitions needed for +# Calico policy and networking mode. + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition metadata: - name: calico-kube-controllers -rules: - - apiGroups: - - "" - - extensions - resources: - - pods - - namespaces - - networkpolicies - - nodes - verbs: - - watch - - list + name: felixconfigurations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: FelixConfiguration + plural: felixconfigurations + singular: felixconfiguration --- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition metadata: - name: calico-kube-controllers -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-kube-controllers -subjects: -- kind: ServiceAccount - name: calico-kube-controllers - namespace: kube-system + name: bgppeers.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BGPPeer + plural: bgppeers + singular: bgppeer --- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition metadata: - name: calico-node -rules: - - apiGroups: [""] - resources: - - pods - - nodes - verbs: - - get + name: bgpconfigurations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BGPConfiguration + plural: bgpconfigurations + singular: bgpconfiguration --- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition metadata: - name: calico-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system + name: ippools.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPPool + plural: ippools + singular: ippool + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: hostendpoints.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: HostEndpoint + plural: hostendpoints + singular: hostendpoint + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusterinformations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: ClusterInformation + plural: clusterinformations + singular: clusterinformation + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkPolicy + plural: globalnetworkpolicies + singular: globalnetworkpolicy + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: globalnetworksets.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkSet + plural: globalnetworksets + singular: globalnetworkset + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: networkpolicies.crd.projectcalico.org +spec: + scope: Namespaced + group: crd.projectcalico.org + version: v1 + names: + kind: NetworkPolicy + plural: networkpolicies + singular: networkpolicy + EOF } diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index 7644661f02..2d17ae9cdd 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -453,7 +453,7 @@ parameters: calico_tag: type: string description: tag of the calico containers used to provision the calico node - default: v2.6.7 + default: v3.3.6 calico_cni_tag: type: string