[k8s] Make flannel self-hosted

Similar to calico, deploy flannel as a DS.
Flannel can use the kubernetes API to store
data, so it doesn't need to contact the etcd
server directly anymore.

This patch drops to relatively large files for
flannel's config, flannel-config-service.sh and
write-flannel-config.sh. All required config is
in the manifests.

Additional options to the controller manager:
--allocate-node-cidrs=true and --cluster-cidr.

Change-Id: I4f1129e155e2602299394b5866165260f4ea0df8
story: 2002751
task: 24870

doc/source/user/index.rst

@@ -323,6 +323,8 @@ the table are linked to more details elsewhere in the user guide.
 +---------------------------------------+--------------------+---------------+
 | `flannel_tag`_                        | see below          | see below     |
 +---------------------------------------+--------------------+---------------+
+| `flannel_cni_tag`_                    | see below          | see below     |
++---------------------------------------+--------------------+---------------+
 | `heat_container_agent_tag`_           | see below          | see below     |
 +---------------------------------------+--------------------+---------------+
 | `kube_dashboard_enabled`_             | - true             | true          |
@@ -1132,10 +1134,20 @@ _`etcd_tag`
 
 _`flannel_tag`
   This label allows users to select `a specific flannel version,
-  based on its container tag
+  based on its container tag:
-  <https://hub.docker.com/r/openstackmagnum/flannel/tags/>`_.
+  Queens <https://hub.docker.com/r/openstackmagnum/flannel/tags/>`_
-  If unset, the current Magnum version's a default flannel version.
+  Rocky <https://quay.io/repository/coreos/flannel?tab=tags>`_
+  If unset, the default version will be used.
   For queens, v0.9.0
+  For stein, v0.10.0-amd64
+
+_`flannel_cni_tag`
+  This label allows users to select `a specific flannel_cni version,
+  based on its container tag. This container adds the cni plugins in
+  the host under /opt/cni/bin
+  <https://quay.io/repository/coreos/flannel-cni?tab=tags>`_.
+  If unset, the current Magnum version's a default flannel version.
+  For stein, v0.3.0
 
 _`heat_container_agent_tag`
   This label allows users to select `a specific heat_container_agent

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh

@@ -43,12 +43,6 @@ users:
     as-user-extra: {}
 EOF
 
-
-if [ "$NETWORK_DRIVER" = "flannel" ]; then
-    atomic install --storage ostree --system --system-package=no \
-    --name=flanneld ${_prefix}flannel:${FLANNEL_TAG}
-fi
-
 sed -i '
     /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow-privileged='"$KUBE_ALLOW_PRIV"'"/
     /^KUBE_MASTER=/ s|=.*|="--master=http://127.0.0.1:8080"|
@@ -131,6 +125,8 @@ sed -i '
 # Add controller manager args
 KUBE_CONTROLLER_MANAGER_ARGS="--leader-elect=true"
 KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-name=${CLUSTER_UUID}"
+KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --allocate-node-cidrs=true"
+KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --cluster-cidr=${PODS_NETWORK_CIDR}"
 KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS $KUBECONTROLLER_OPTIONS"
 if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then
     KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --service-account-private-key-file=$CERT_DIR/service_account_private.key --root-ca-file=$CERT_DIR/ca.crt"
@@ -172,9 +168,7 @@ if [ -n "${INSECURE_REGISTRY_URL}" ]; then
     echo "INSECURE_REGISTRY='--insecure-registry ${INSECURE_REGISTRY_URL}'" >> /etc/sysconfig/docker
 fi
 
-if [ "$NETWORK_DRIVER" = "calico" ]; then
+KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
-    KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
-fi
 KUBELET_ARGS="${KUBELET_ARGS} --register-with-taints=CriticalAddonsOnly=True:NoSchedule,dedicated=master:NoSchedule"
 KUBELET_ARGS="${KUBELET_ARGS} --node-labels=node-role.kubernetes.io/master=\"\""
 
@@ -245,3 +239,4 @@ sed -i '
 /^KUBELET_HOSTNAME=/ s/=.*/=""/
 /^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"|
 ' /etc/kubernetes/kubelet
+

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh

@@ -7,10 +7,10 @@ echo "configuring kubernetes (minion)"
 _prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
 
 _addtl_mounts=''
-if [ "$NETWORK_DRIVER" = "calico" ]; then
+mkdir -p /opt/cni
-    mkdir -p /opt/cni
+_addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}'
-    _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}'
 
+if [ "$NETWORK_DRIVER" = "calico" ]; then
     if [ "`systemctl status NetworkManager.service | grep -o "Active: active"`" = "Active: active" ]; then
         CALICO_NM=/etc/NetworkManager/conf.d/calico.conf
         [ -f ${CALICO_NM} ] || {
@@ -168,9 +168,7 @@ fi
 EOF
 chmod +x /etc/kubernetes/get_require_kubeconfig.sh
 
-if [ "$NETWORK_DRIVER" = "calico" ]; then
+KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
-    KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
-fi
 
 sed -i '
     /^KUBELET_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
@@ -183,37 +181,6 @@ cat > /etc/kubernetes/proxy << EOF
 KUBE_PROXY_ARGS="--kubeconfig=${PROXY_KUBECONFIG} --cluster-cidr=${PODS_NETWORK_CIDR}"
 EOF
 
-if [ "$NETWORK_DRIVER" = "flannel" ]; then
-    atomic install --storage ostree --system --system-package=no \
-    --name=flanneld ${_prefix}flannel:${FLANNEL_TAG}
-    if [ "$TLS_DISABLED" = "True" ]; then
-        FLANNEL_OPTIONS=""
-        ETCD_CURL_OPTIONS=""
-    else
-        FLANNEL_CERT_DIR=/etc/flanneld/certs
-        FLANNEL_OPTIONS="-etcd-cafile $FLANNEL_CERT_DIR/ca.crt"
-        FLANNEL_OPTIONS="$FLANNEL_OPTIONS -etcd-certfile $FLANNEL_CERT_DIR/proxy.crt"
-        FLANNEL_OPTIONS="$FLANNEL_OPTIONS -etcd-keyfile $FLANNEL_CERT_DIR/proxy.key"
-        ETCD_CURL_OPTIONS="--cacert $FLANNEL_CERT_DIR/ca.crt --cert $FLANNEL_CERT_DIR/proxy.crt --key $FLANNEL_CERT_DIR/proxy.key"
-    fi
-    FLANNELD_CONFIG=/etc/sysconfig/flanneld
-
-    cat >> $FLANNELD_CONFIG <<EOF
-    FLANNEL_ETCD_ENDPOINTS="$PROTOCOL://${ETCD_SERVER_IP}:2379"
-    FLANNEL_ETCD_PREFIX="/atomic.io/network"
-    FLANNEL_OPTIONS="$FLANNEL_OPTIONS"
-EOF
-
-    # Make sure etcd has a flannel configuration
-    . $FLANNELD_CONFIG
-    until curl -sf $ETCD_CURL_OPTIONS \
-        "$FLANNEL_ETCD_ENDPOINTS/v2/keys${FLANNEL_ETCD_PREFIX}/config?quorum=false&recursive=false&sorted=false"
-    do
-        echo "Waiting for flannel configuration in etcd..."
-        sleep 5
-    done
-fi
-
 cat >> /etc/environment <<EOF
 KUBERNETES_MASTER=$KUBE_MASTER_URI
 EOF

magnum/drivers/common/templates/kubernetes/fragments/flannel-config-service.sh

@@ -1,73 +0,0 @@
-#!/bin/sh
-
-. /etc/sysconfig/heat-params
-
-if [ "$NETWORK_DRIVER" != "flannel" ]; then
-    exit 0
-fi
-CERT_DIR=/etc/kubernetes/certs
-PROTOCOL=https
-ETCD_CURL_OPTIONS="--cacert $CERT_DIR/ca.crt \
---cert $CERT_DIR/server.crt --key $CERT_DIR/server.key"
-FLANNELD_CONFIG=/etc/sysconfig/flanneld
-
-if [ "$TLS_DISABLED" = "True" ]; then
-    PROTOCOL=http
-    ETCD_CURL_OPTIONS=""
-fi
-
-. $FLANNELD_CONFIG
-
-FLANNEL_CONFIG_BIN=/usr/local/bin/flannel-config
-FLANNEL_CONFIG_SERVICE=/etc/systemd/system/flannel-config.service
-FLANNEL_JSON=/etc/sysconfig/flannel-network.json
-
-echo "creating $FLANNEL_CONFIG_BIN"
-cat > $FLANNEL_CONFIG_BIN <<EOF
-#!/bin/sh
-
-if ! [ -f "$FLANNEL_JSON" ]; then
-  echo "ERROR: missing network configuration file" >&2
-  exit 1
-fi
-
-if [ -z "$FLANNEL_ETCD_ENDPOINTS" ] || [ -z "$FLANNEL_ETCD_PREFIX" ]; then
-  echo "ERROR: missing required configuration" >&2
-  exit 1
-fi
-
-echo "creating flanneld config in etcd"
-while ! curl -sf -L $ETCD_CURL_OPTIONS \
-        $FLANNEL_ETCD_ENDPOINTS/v2/keys${FLANNEL_ETCD_PREFIX}/config \
-        -X PUT --data-urlencode value@${FLANNEL_JSON}; do
-    echo "waiting for etcd"
-    sleep 1
-done
-EOF
-
-cat > $FLANNEL_CONFIG_SERVICE <<EOF
-[Unit]
-After=etcd.service
-Requires=etcd.service
-
-[Service]
-Type=oneshot
-EnvironmentFile=/etc/sysconfig/flanneld
-ExecStart=$FLANNEL_CONFIG_BIN
-
-[Install]
-WantedBy=multi-user.target
-EOF
-
-chown root:root $FLANNEL_CONFIG_BIN
-chmod 0755 $FLANNEL_CONFIG_BIN
-
-chown root:root $FLANNEL_CONFIG_SERVICE
-chmod 0644 $FLANNEL_CONFIG_SERVICE
-
-systemctl enable flannel-config
-systemctl start --no-block flannel-config
-
-echo "activating service flanneld"
-systemctl enable flanneld
-systemctl start --no-block flanneld

magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh

@@ -2,104 +2,213 @@
 
 . /etc/sysconfig/heat-params
 
-if [ "$NETWORK_DRIVER" != "flannel" ]; then
+set -x
-    exit 0
+
+if [ "$NETWORK_DRIVER" = "flannel" ]; then
+    _prefix=${CONTAINER_INFRA_PREFIX:-quay.io/coreos/}
+    FLANNEL_DEPLOY=/srv/magnum/kubernetes/manifests/flannel-deploy.yaml
+
+    [ -f ${FLANNEL_DEPLOY} ] || {
+    echo "Writing File: $FLANNEL_DEPLOY"
+    mkdir -p "$(dirname ${FLANNEL_DEPLOY})"
+    cat << EOF > ${FLANNEL_DEPLOY}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: flannel
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: flannel
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: flannel
+subjects:
+- kind: ServiceAccount
+  name: flannel
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: flannel
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-flannel-cfg
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+data:
+  cni-conf.json: |
+    {
+      "name": "cbr0",
+      "plugins": [
+        {
+          "type": "flannel",
+          "delegate": {
+            "hairpinMode": true,
+            "isDefaultGateway": true
+          }
+        },
+        {
+          "type": "portmap",
+          "capabilities": {
+            "portMappings": true
+          }
+        }
+      ]
+    }
+  net-conf.json: |
+    {
+      "Network": "$FLANNEL_NETWORK_CIDR",
+      "Subnetlen": $FLANNEL_NETWORK_SUBNETLEN,
+      "Backend": {
+        "Type": "$FLANNEL_BACKEND"
+      }
+    }
+  magnum-install-cni.sh: |
+    #!/bin/sh
+    set -e -x;
+    if [ -w "/host/opt/cni/bin/" ]; then
+      cp /opt/cni/bin/* /host/opt/cni/bin/;
+      echo "Wrote CNI binaries to /host/opt/cni/bin/";
+    fi;
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-flannel-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+spec:
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: flannel
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+      tolerations:
+        # Make sure flannel gets scheduled on all nodes.
+        - effect: NoSchedule
+          operator: Exists
+        # Mark the pod as a critical add-on for rescheduling.
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+      serviceAccountName: flannel
+      initContainers:
+      - name: install-cni-plugins
+        image: ${_prefix}flannel-cni:${FLANNEL_CNI_TAG}
+        command:
+        - sh
+        args:
+        - /etc/kube-flannel/magnum-install-cni.sh
+        volumeMounts:
+        - name: host-cni-bin
+          mountPath: /host/opt/cni/bin/
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      - name: install-cni
+        image: ${_prefix}flannel:${FLANNEL_TAG}
+        command:
+        - cp
+        args:
+        - -f
+        - /etc/kube-flannel/cni-conf.json
+        - /etc/cni/net.d/10-flannel.conflist
+        volumeMounts:
+        - name: cni
+          mountPath: /etc/cni/net.d
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      containers:
+      - name: kube-flannel
+        image: ${_prefix}flannel:${FLANNEL_TAG}
+        command:
+        - /opt/bin/flanneld
+        args:
+        - --ip-masq
+        - --kube-subnet-mgr
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        volumeMounts:
+        - name: run
+          mountPath: /run
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      volumes:
+        - name: host-cni-bin
+          hostPath:
+            path: /opt/cni/bin
+        - name: run
+          hostPath:
+            path: /run
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: flannel-cfg
+          configMap:
+            name: kube-flannel-cfg
+EOF
+    }
+
+    if [ "$MASTER_INDEX" = "0" ]; then
+
+        until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+        do
+            echo "Waiting for Kubernetes API..."
+            sleep 5
+        done
+    fi
+
+    /usr/bin/kubectl apply -f "${FLANNEL_DEPLOY}" --namespace=kube-system
 fi
-
-SYSTEMD_UNITS_DIR=/etc/systemd/system/
-FLANNEL_DOCKER_BRIDGE_BIN=/usr/local/bin/flannel-docker-bridge
-FLANNEL_DOCKER_BRIDGE_SERVICE=/etc/systemd/system/flannel-docker-bridge.service
-FLANNEL_IPTABLES_FORWARD_ACCEPT_SERVICE=flannel-iptables-forward-accept.service
-DOCKER_FLANNEL_CONF=/etc/systemd/system/docker.service.d/flannel.conf
-FLANNEL_DOCKER_BRIDGE_CONF=/etc/systemd/system/flanneld.service.d/flannel-docker-bridge.conf
-
-mkdir -p /etc/systemd/system/docker.service.d
-mkdir -p /etc/systemd/system/flanneld.service.d
-
-cat >> $FLANNEL_DOCKER_BRIDGE_BIN <<EOF1
-#!/bin/sh
-
-if ! [ "\$FLANNEL_SUBNET" ] && [ "\$FLANNEL_MTU" ] ; then
-  echo "ERROR: missing required environment variables." >&2
-  exit 1
-fi
-
-# NOTE(mnaser): Since Docker 1.13, it does not set the default forwarding
-#               policy to ACCEPT which will cause CNI networking to fail.
-iptables -P FORWARD ACCEPT
-
-mkdir -p /run/flannel/
-cat > /run/flannel/docker <<EOF2
-DOCKER_NETWORK_OPTIONS="--bip=\$FLANNEL_SUBNET --mtu=\$FLANNEL_MTU"
-EOF2
-EOF1
-
-chown root:root $FLANNEL_DOCKER_BRIDGE_BIN
-chmod 0755 $FLANNEL_DOCKER_BRIDGE_BIN
-
-cat >> $FLANNEL_DOCKER_BRIDGE_SERVICE <<EOF
-[Unit]
-After=flanneld.service
-Before=docker.service
-Requires=flanneld.service
-
-[Service]
-Type=oneshot
-EnvironmentFile=/run/flanneld/subnet.env
-ExecStart=/usr/local/bin/flannel-docker-bridge
-
-[Install]
-WantedBy=docker.service
-EOF
-
-chown root:root $FLANNEL_DOCKER_BRIDGE_SERVICE
-chmod 0644 $FLANNEL_DOCKER_BRIDGE_SERVICE
-
-cat >> $DOCKER_FLANNEL_CONF <<EOF
-[Unit]
-Requires=flannel-docker-bridge.service
-After=flannel-docker-bridge.service
-
-[Service]
-EnvironmentFile=/run/flannel/docker
-EOF
-
-chown root:root $DOCKER_FLANNEL_CONF
-chmod 0644 $DOCKER_FLANNEL_CONF
-
-cat >> $FLANNEL_DOCKER_BRIDGE_CONF <<EOF
-[Unit]
-Requires=flannel-docker-bridge.service
-Before=flannel-docker-bridge.service
-
-[Install]
-Also=flannel-docker-bridge.service
-EOF
-
-chown root:root $FLANNEL_DOCKER_BRIDGE_CONF
-chmod 0644 $FLANNEL_DOCKER_BRIDGE_CONF
-
-# Workaround for https://github.com/coreos/flannel/issues/799
-# Not solved upstream properly yet.
-cat >> "${SYSTEMD_UNITS_DIR}${FLANNEL_IPTABLES_FORWARD_ACCEPT_SERVICE}" <<EOF
-[Unit]
-After=flanneld.service docker.service kubelet.service kube-proxy.service
-Requires=flanneld.service
-
-[Service]
-Type=oneshot
-ExecStart=/usr/sbin/iptables -P FORWARD ACCEPT
-ExecStartPost=/usr/sbin/iptables -S
-
-[Install]
-WantedBy=flanneld.service
-EOF
-
-chown root:root "${SYSTEMD_UNITS_DIR}${FLANNEL_IPTABLES_FORWARD_ACCEPT_SERVICE}"
-chmod 0644 "${SYSTEMD_UNITS_DIR}${FLANNEL_IPTABLES_FORWARD_ACCEPT_SERVICE}"
-systemctl daemon-reload
-systemctl enable "${FLANNEL_IPTABLES_FORWARD_ACCEPT_SERVICE}"
-
-echo "activating service flanneld"
-systemctl enable flanneld
-systemctl start flanneld

magnum/drivers/common/templates/kubernetes/fragments/make-cert-client.sh

@@ -147,5 +147,3 @@ chmod 550 "${cert_dir}"
 chown -R kube:kube_etcd "${cert_dir}"
 chmod 440 ${cert_dir}/kubelet.key
 chmod 440 ${cert_dir}/proxy.key
-mkdir -p /etc/flanneld/certs
-cp ${cert_dir}/* /etc/flanneld/certs

magnum/drivers/common/templates/kubernetes/fragments/write-flannel-config.sh

@@ -1,28 +0,0 @@
-#!/bin/sh
-
-. /etc/sysconfig/heat-params
-
-if [ "$NETWORK_DRIVER" != "flannel" ]; then
-    exit 0
-fi
-
-FLANNEL_JSON=/etc/sysconfig/flannel-network.json
-FLANNELD_CONFIG=/etc/sysconfig/flanneld
-
-cat > /etc/sysconfig/flanneld <<EOF
-FLANNEL_ETCD_ENDPOINTS="http://127.0.0.1:2379"
-FLANNEL_ETCD_PREFIX="/atomic.io/network"
-FLANNEL_OPTIONS=
-EOF
-
-# Generate a flannel configuration that we will
-# store into etcd using curl.
-cat > $FLANNEL_JSON <<EOF
-{
-  "Network": "$FLANNEL_NETWORK_CIDR",
-  "Subnetlen": $FLANNEL_NETWORK_SUBNETLEN,
-  "Backend": {
-    "Type": "$FLANNEL_BACKEND"
-  }
-}
-EOF

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml

@@ -46,6 +46,7 @@ write_files:
       CLOUD_PROVIDER_ENABLED="$CLOUD_PROVIDER_ENABLED"
       ETCD_TAG="$ETCD_TAG"
       FLANNEL_TAG="$FLANNEL_TAG"
+      FLANNEL_CNI_TAG="$FLANNEL_CNI_TAG"
       KUBE_VERSION="$KUBE_VERSION"
       KUBE_DASHBOARD_VERSION="$KUBE_DASHBOARD_VERSION"
       TRUSTEE_USER_ID="$TRUSTEE_USER_ID"

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml

@@ -38,7 +38,6 @@ write_files:
       NO_PROXY="$NO_PROXY"
       WAIT_CURL="$WAIT_CURL"
       KUBE_TAG="$KUBE_TAG"
-      FLANNEL_TAG="$FLANNEL_TAG"
       FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR"
       PODS_NETWORK_CIDR="$PODS_NETWORK_CIDR"
       KUBE_VERSION="$KUBE_VERSION"

magnum/drivers/heat/k8s_fedora_template_def.py

@@ -109,7 +109,7 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
                       'cgroup_driver',
                       'calico_tag', 'calico_cni_tag',
                       'calico_kube_controllers_tag', 'calico_ipv4pool',
-                      'etcd_tag', 'flannel_tag',
+                      'etcd_tag', 'flannel_tag', 'flannel_cni_tag',
                       'cloud_provider_enabled',
                       'cloud_provider_tag',
                       'prometheus_tag',

magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

@@ -348,8 +348,13 @@ parameters:
 
   flannel_tag:
     type: string
-    description: tag of the flannel system containers
+    description: tag of the flannel container
-    default: v0.9.0
+    default: v0.10.0-amd64
+
+  flannel_cni_tag:
+    type: string
+    description: tag of the flannel cni container
+    default: v0.3.0
 
   kube_version:
     type: string
@@ -778,6 +783,7 @@ resources:
           kube_version: {get_param: kube_version}
           etcd_tag: {get_param: etcd_tag}
           flannel_tag: {get_param: flannel_tag}
+          flannel_cni_tag: {get_param: flannel_cni_tag}
           kube_dashboard_version: {get_param: kube_dashboard_version}
           trustee_user_id: {get_param: trustee_user_id}
           trustee_password: {get_param: trustee_password}
@@ -834,6 +840,7 @@ resources:
                   "$CA_KEY": {get_param: ca_key}
             - get_file: ../../common/templates/kubernetes/fragments/core-dns-service.sh
             - get_file: ../../common/templates/kubernetes/fragments/calico-service.sh
+            - get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh
             - get_file: ../../common/templates/kubernetes/fragments/enable-helm-tiller.sh
             - get_file: ../../common/templates/kubernetes/helm/metrics-server.sh
             - get_file: ../../common/templates/kubernetes/fragments/install-helm-modules.sh
@@ -919,7 +926,6 @@ resources:
           no_proxy: {get_param: no_proxy}
           kube_tag: {get_param: kube_tag}
           kube_version: {get_param: kube_version}
-          flannel_tag: {get_param: flannel_tag}
           trustee_user_id: {get_param: trustee_user_id}
           trustee_username: {get_param: trustee_username}
           trustee_password: {get_param: trustee_password}

magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml

@@ -239,6 +239,10 @@ parameters:
     type: string
     description: tag of the flannel system containers
 
+  flannel_cni_tag:
+    type: string
+    description: tag of the flannel cni container
+
   kube_version:
     type: string
     description: version of kubernetes used for kubernetes cluster
@@ -502,6 +506,7 @@ resources:
             "$CLOUD_PROVIDER_ENABLED": {get_param: cloud_provider_enabled}
             "$ETCD_TAG": {get_param: etcd_tag}
             "$FLANNEL_TAG": {get_param: flannel_tag}
+            "$FLANNEL_CNI_TAG": {get_param: flannel_cni_tag}
             "$KUBE_VERSION": {get_param: kube_version}
             "$KUBE_DASHBOARD_VERSION": {get_param: kube_dashboard_version}
             "$TRUSTEE_USER_ID": {get_param: trustee_user_id}
@@ -584,24 +589,6 @@ resources:
       group: ungrouped
       config: {get_file: ../../common/templates/kubernetes/fragments/configure-kubernetes-master.sh}
 
-  write_flannel_config:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/write-flannel-config.sh}
-
-  flannel_config_service:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/flannel-config-service.sh}
-
-  flannel_service:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh}
-
   enable_services:
     type: OS::Heat::SoftwareConfig
     properties:
@@ -641,9 +628,6 @@ resources:
         - config: {get_resource: add_proxy}
         - config: {get_resource: start_container_agent}
         - config: {get_resource: enable_services}
-        - config: {get_resource: write_flannel_config}
-        - config: {get_resource: flannel_config_service}
-        - config: {get_resource: flannel_service}
 
   ######################################################################
   #

magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml

@@ -182,10 +182,6 @@ parameters:
     type: string
     description: tag of the k8s containers used to provision the kubernetes cluster
 
-  flannel_tag:
-    type: string
-    description: tag of the flannel system containers
-
   kube_version:
     type: string
     description: version of kubernetes used for kubernetes cluster
@@ -342,7 +338,6 @@ resources:
             $HTTPS_PROXY: {get_param: https_proxy}
             $NO_PROXY: {get_param: no_proxy}
             $KUBE_TAG: {get_param: kube_tag}
-            $FLANNEL_TAG: {get_param: flannel_tag}
             $FLANNEL_NETWORK_CIDR: {get_param: flannel_network_cidr}
             $PODS_NETWORK_CIDR: {get_param: pods_network_cidr}
             $KUBE_VERSION: {get_param: kube_version}
@@ -405,12 +400,6 @@ resources:
       group: ungrouped
       config: {get_file: ../../common/templates/kubernetes/fragments/configure-kubernetes-minion.sh}
 
-  flannel_service:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh}
-
   enable_services:
     type: OS::Heat::SoftwareConfig
     properties:
@@ -466,7 +455,6 @@ resources:
         - config: {get_resource: configure_docker_storage}
         - config: {get_resource: configure_docker_registry}
         - config: {get_resource: configure_kubernetes_minion}
-        - config: {get_resource: flannel_service}
         - config: {get_resource: add_proxy}
         - config: {get_resource: enable_services}
         - config: {get_resource: enable_docker_registry}

magnum/tests/unit/drivers/test_template_definition.py

@@ -366,6 +366,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
         kube_tag = mock_cluster.labels.get('kube_tag')
         etcd_tag = mock_cluster.labels.get('etcd_tag')
         flannel_tag = mock_cluster.labels.get('flannel_tag')
+        flannel_cni_tag = mock_cluster.labels.get('flannel_cni_tag')
         container_infra_prefix = mock_cluster.labels.get(
             'container_infra_prefix')
         availability_zone = mock_cluster.labels.get(
@@ -457,6 +458,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'kube_tag': kube_tag,
             'etcd_tag': etcd_tag,
             'flannel_tag': flannel_tag,
+            'flannel_cni_tag': flannel_cni_tag,
             'container_infra_prefix': container_infra_prefix,
             'nodes_affinity_policy': 'soft-anti-affinity',
             'availability_zone': availability_zone,
@@ -732,6 +734,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
         kube_tag = mock_cluster.labels.get('kube_tag')
         etcd_tag = mock_cluster.labels.get('etcd_tag')
         flannel_tag = mock_cluster.labels.get('flannel_tag')
+        flannel_cni_tag = mock_cluster.labels.get('flannel_cni_tag')
         container_infra_prefix = mock_cluster.labels.get(
             'container_infra_prefix')
         availability_zone = mock_cluster.labels.get(
@@ -825,6 +828,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'kube_tag': kube_tag,
             'etcd_tag': etcd_tag,
             'flannel_tag': flannel_tag,
+            'flannel_cni_tag': flannel_cni_tag,
             'container_infra_prefix': container_infra_prefix,
             'nodes_affinity_policy': 'soft-anti-affinity',
             'availability_zone': availability_zone,

releasenotes/notes/flannel-cni-4a5c9f574325761e.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    For k8s_fedora_atomic, run flannel as a cni plugin. The deployment method
+    is taken from the flannel upstream documentation. One more label for the
+    cni tag is added `flannel_cni_tag` for the container,
+    quay.io/repository/coreos/flannel-cni. The flannel container is taken
+    from flannel upsteam as well quay.io/repository/coreos/flannel.