Merge "[k8s] Make flannel self-hosted"

doc/source/user/index.rst

@@ -323,6 +323,8 @@ the table are linked to more details elsewhere in the user guide.
 +---------------------------------------+--------------------+---------------+
 | `flannel_tag`_                        | see below          | see below     |
 +---------------------------------------+--------------------+---------------+
+| `flannel_cni_tag`_                    | see below          | see below     |
++---------------------------------------+--------------------+---------------+
 | `heat_container_agent_tag`_           | see below          | see below     |
 +---------------------------------------+--------------------+---------------+
 | `kube_dashboard_enabled`_             | - true             | true          |
@@ -1132,10 +1134,20 @@ _`etcd_tag`
 
 _`flannel_tag`
   This label allows users to select `a specific flannel version,
-  based on its container tag
-  <https://hub.docker.com/r/openstackmagnum/flannel/tags/>`_.
-  If unset, the current Magnum version's a default flannel version.
+  based on its container tag:
+  Queens <https://hub.docker.com/r/openstackmagnum/flannel/tags/>`_
+  Rocky <https://quay.io/repository/coreos/flannel?tab=tags>`_
+  If unset, the default version will be used.
   For queens, v0.9.0
+  For stein, v0.10.0-amd64
+
+_`flannel_cni_tag`
+  This label allows users to select `a specific flannel_cni version,
+  based on its container tag. This container adds the cni plugins in
+  the host under /opt/cni/bin
+  <https://quay.io/repository/coreos/flannel-cni?tab=tags>`_.
+  If unset, the current Magnum version's a default flannel version.
+  For stein, v0.3.0
 
 _`heat_container_agent_tag`
   This label allows users to select `a specific heat_container_agent

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh

@@ -43,12 +43,6 @@ users:
     as-user-extra: {}
 EOF
 
-
-if [ "$NETWORK_DRIVER" = "flannel" ]; then
-    atomic install --storage ostree --system --system-package=no \
-    --name=flanneld ${_prefix}flannel:${FLANNEL_TAG}
-fi
-
 sed -i '
     /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow-privileged='"$KUBE_ALLOW_PRIV"'"/
     /^KUBE_MASTER=/ s|=.*|="--master=http://127.0.0.1:8080"|
@@ -131,6 +125,8 @@ sed -i '
 # Add controller manager args
 KUBE_CONTROLLER_MANAGER_ARGS="--leader-elect=true"
 KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-name=${CLUSTER_UUID}"
+KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --allocate-node-cidrs=true"
+KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --cluster-cidr=${PODS_NETWORK_CIDR}"
 KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS $KUBECONTROLLER_OPTIONS"
 if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then
     KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --service-account-private-key-file=$CERT_DIR/service_account_private.key --root-ca-file=$CERT_DIR/ca.crt"
@@ -172,9 +168,7 @@ if [ -n "${INSECURE_REGISTRY_URL}" ]; then
     echo "INSECURE_REGISTRY='--insecure-registry ${INSECURE_REGISTRY_URL}'" >> /etc/sysconfig/docker
 fi
 
-if [ "$NETWORK_DRIVER" = "calico" ]; then
-    KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
-fi
+KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 KUBELET_ARGS="${KUBELET_ARGS} --register-with-taints=CriticalAddonsOnly=True:NoSchedule,dedicated=master:NoSchedule"
 KUBELET_ARGS="${KUBELET_ARGS} --node-labels=node-role.kubernetes.io/master=\"\""
 
@@ -245,3 +239,4 @@ sed -i '
 /^KUBELET_HOSTNAME=/ s/=.*/=""/
 /^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"|
 ' /etc/kubernetes/kubelet
+

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh

@@ -7,10 +7,10 @@ echo "configuring kubernetes (minion)"
 _prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
 
 _addtl_mounts=''
-if [ "$NETWORK_DRIVER" = "calico" ]; then
-    mkdir -p /opt/cni
-    _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}'
+mkdir -p /opt/cni
+_addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}'
 
+if [ "$NETWORK_DRIVER" = "calico" ]; then
     if [ "`systemctl status NetworkManager.service | grep -o "Active: active"`" = "Active: active" ]; then
         CALICO_NM=/etc/NetworkManager/conf.d/calico.conf
         [ -f ${CALICO_NM} ] || {
@@ -168,9 +168,7 @@ fi
 EOF
 chmod +x /etc/kubernetes/get_require_kubeconfig.sh
 
-if [ "$NETWORK_DRIVER" = "calico" ]; then
-    KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
-fi
+KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 
 sed -i '
     /^KUBELET_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
@@ -183,37 +181,6 @@ cat > /etc/kubernetes/proxy << EOF
 KUBE_PROXY_ARGS="--kubeconfig=${PROXY_KUBECONFIG} --cluster-cidr=${PODS_NETWORK_CIDR}"
 EOF
 
-if [ "$NETWORK_DRIVER" = "flannel" ]; then
-    atomic install --storage ostree --system --system-package=no \
-    --name=flanneld ${_prefix}flannel:${FLANNEL_TAG}
-    if [ "$TLS_DISABLED" = "True" ]; then
-        FLANNEL_OPTIONS=""
-        ETCD_CURL_OPTIONS=""
-    else
-        FLANNEL_CERT_DIR=/etc/flanneld/certs
-        FLANNEL_OPTIONS="-etcd-cafile $FLANNEL_CERT_DIR/ca.crt"
-        FLANNEL_OPTIONS="$FLANNEL_OPTIONS -etcd-certfile $FLANNEL_CERT_DIR/proxy.crt"
-        FLANNEL_OPTIONS="$FLANNEL_OPTIONS -etcd-keyfile $FLANNEL_CERT_DIR/proxy.key"
-        ETCD_CURL_OPTIONS="--cacert $FLANNEL_CERT_DIR/ca.crt --cert $FLANNEL_CERT_DIR/proxy.crt --key $FLANNEL_CERT_DIR/proxy.key"
-    fi
-    FLANNELD_CONFIG=/etc/sysconfig/flanneld
-
-    cat >> $FLANNELD_CONFIG <<EOF
-    FLANNEL_ETCD_ENDPOINTS="$PROTOCOL://${ETCD_SERVER_IP}:2379"
-    FLANNEL_ETCD_PREFIX="/atomic.io/network"
-    FLANNEL_OPTIONS="$FLANNEL_OPTIONS"
-EOF
-
-    # Make sure etcd has a flannel configuration
-    . $FLANNELD_CONFIG
-    until curl -sf $ETCD_CURL_OPTIONS \
-        "$FLANNEL_ETCD_ENDPOINTS/v2/keys${FLANNEL_ETCD_PREFIX}/config?quorum=false&recursive=false&sorted=false"
-    do
-        echo "Waiting for flannel configuration in etcd..."
-        sleep 5
-    done
-fi
-
 cat >> /etc/environment <<EOF
 KUBERNETES_MASTER=$KUBE_MASTER_URI
 EOF

magnum/drivers/common/templates/kubernetes/fragments/flannel-config-service.sh

@@ -1,73 +0,0 @@
-#!/bin/sh
-
-. /etc/sysconfig/heat-params
-
-if [ "$NETWORK_DRIVER" != "flannel" ]; then
-    exit 0
-fi
-CERT_DIR=/etc/kubernetes/certs
-PROTOCOL=https
-ETCD_CURL_OPTIONS="--cacert $CERT_DIR/ca.crt \
---cert $CERT_DIR/server.crt --key $CERT_DIR/server.key"
-FLANNELD_CONFIG=/etc/sysconfig/flanneld
-
-if [ "$TLS_DISABLED" = "True" ]; then
-    PROTOCOL=http
-    ETCD_CURL_OPTIONS=""
-fi
-
-. $FLANNELD_CONFIG
-
-FLANNEL_CONFIG_BIN=/usr/local/bin/flannel-config
-FLANNEL_CONFIG_SERVICE=/etc/systemd/system/flannel-config.service
-FLANNEL_JSON=/etc/sysconfig/flannel-network.json
-
-echo "creating $FLANNEL_CONFIG_BIN"
-cat > $FLANNEL_CONFIG_BIN <<EOF
-#!/bin/sh
-
-if ! [ -f "$FLANNEL_JSON" ]; then
-  echo "ERROR: missing network configuration file" >&2
-  exit 1
-fi
-
-if [ -z "$FLANNEL_ETCD_ENDPOINTS" ] || [ -z "$FLANNEL_ETCD_PREFIX" ]; then
-  echo "ERROR: missing required configuration" >&2
-  exit 1
-fi
-
-echo "creating flanneld config in etcd"
-while ! curl -sf -L $ETCD_CURL_OPTIONS \
-        $FLANNEL_ETCD_ENDPOINTS/v2/keys${FLANNEL_ETCD_PREFIX}/config \
-        -X PUT --data-urlencode value@${FLANNEL_JSON}; do
-    echo "waiting for etcd"
-    sleep 1
-done
-EOF
-
-cat > $FLANNEL_CONFIG_SERVICE <<EOF
-[Unit]
-After=etcd.service
-Requires=etcd.service
-
-[Service]
-Type=oneshot
-EnvironmentFile=/etc/sysconfig/flanneld
-ExecStart=$FLANNEL_CONFIG_BIN
-
-[Install]
-WantedBy=multi-user.target
-EOF
-
-chown root:root $FLANNEL_CONFIG_BIN
-chmod 0755 $FLANNEL_CONFIG_BIN
-
-chown root:root $FLANNEL_CONFIG_SERVICE
-chmod 0644 $FLANNEL_CONFIG_SERVICE
-
-systemctl enable flannel-config
-systemctl start --no-block flannel-config
-
-echo "activating service flanneld"
-systemctl enable flanneld
-systemctl start --no-block flanneld

magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh

@@ -2,104 +2,213 @@
 
 . /etc/sysconfig/heat-params
 
-if [ "$NETWORK_DRIVER" != "flannel" ]; then
-    exit 0
-fi
-
-SYSTEMD_UNITS_DIR=/etc/systemd/system/
-FLANNEL_DOCKER_BRIDGE_BIN=/usr/local/bin/flannel-docker-bridge
-FLANNEL_DOCKER_BRIDGE_SERVICE=/etc/systemd/system/flannel-docker-bridge.service
-FLANNEL_IPTABLES_FORWARD_ACCEPT_SERVICE=flannel-iptables-forward-accept.service
-DOCKER_FLANNEL_CONF=/etc/systemd/system/docker.service.d/flannel.conf
-FLANNEL_DOCKER_BRIDGE_CONF=/etc/systemd/system/flanneld.service.d/flannel-docker-bridge.conf
-
-mkdir -p /etc/systemd/system/docker.service.d
-mkdir -p /etc/systemd/system/flanneld.service.d
-
-cat >> $FLANNEL_DOCKER_BRIDGE_BIN <<EOF1
-#!/bin/sh
-
-if ! [ "\$FLANNEL_SUBNET" ] && [ "\$FLANNEL_MTU" ] ; then
-  echo "ERROR: missing required environment variables." >&2
-  exit 1
-fi
-
-# NOTE(mnaser): Since Docker 1.13, it does not set the default forwarding
-#               policy to ACCEPT which will cause CNI networking to fail.
-iptables -P FORWARD ACCEPT
-
-mkdir -p /run/flannel/
-cat > /run/flannel/docker <<EOF2
-DOCKER_NETWORK_OPTIONS="--bip=\$FLANNEL_SUBNET --mtu=\$FLANNEL_MTU"
-EOF2
-EOF1
-
-chown root:root $FLANNEL_DOCKER_BRIDGE_BIN
-chmod 0755 $FLANNEL_DOCKER_BRIDGE_BIN
-
-cat >> $FLANNEL_DOCKER_BRIDGE_SERVICE <<EOF
-[Unit]
-After=flanneld.service
-Before=docker.service
-Requires=flanneld.service
-
-[Service]
-Type=oneshot
-EnvironmentFile=/run/flanneld/subnet.env
-ExecStart=/usr/local/bin/flannel-docker-bridge
-
-[Install]
-WantedBy=docker.service
+set -x
+
+if [ "$NETWORK_DRIVER" = "flannel" ]; then
+    _prefix=${CONTAINER_INFRA_PREFIX:-quay.io/coreos/}
+    FLANNEL_DEPLOY=/srv/magnum/kubernetes/manifests/flannel-deploy.yaml
+
+    [ -f ${FLANNEL_DEPLOY} ] || {
+    echo "Writing File: $FLANNEL_DEPLOY"
+    mkdir -p "$(dirname ${FLANNEL_DEPLOY})"
+    cat << EOF > ${FLANNEL_DEPLOY}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: flannel
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: flannel
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: flannel
+subjects:
+- kind: ServiceAccount
+  name: flannel
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: flannel
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-flannel-cfg
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+data:
+  cni-conf.json: |
+    {
+      "name": "cbr0",
+      "plugins": [
+        {
+          "type": "flannel",
+          "delegate": {
+            "hairpinMode": true,
+            "isDefaultGateway": true
+          }
+        },
+        {
+          "type": "portmap",
+          "capabilities": {
+            "portMappings": true
+          }
+        }
+      ]
+    }
+  net-conf.json: |
+    {
+      "Network": "$FLANNEL_NETWORK_CIDR",
+      "Subnetlen": $FLANNEL_NETWORK_SUBNETLEN,
+      "Backend": {
+        "Type": "$FLANNEL_BACKEND"
+      }
+    }
+  magnum-install-cni.sh: |
+    #!/bin/sh
+    set -e -x;
+    if [ -w "/host/opt/cni/bin/" ]; then
+      cp /opt/cni/bin/* /host/opt/cni/bin/;
+      echo "Wrote CNI binaries to /host/opt/cni/bin/";
+    fi;
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-flannel-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+spec:
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: flannel
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+      tolerations:
+        # Make sure flannel gets scheduled on all nodes.
+        - effect: NoSchedule
+          operator: Exists
+        # Mark the pod as a critical add-on for rescheduling.
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+      serviceAccountName: flannel
+      initContainers:
+      - name: install-cni-plugins
+        image: ${_prefix}flannel-cni:${FLANNEL_CNI_TAG}
+        command:
+        - sh
+        args:
+        - /etc/kube-flannel/magnum-install-cni.sh
+        volumeMounts:
+        - name: host-cni-bin
+          mountPath: /host/opt/cni/bin/
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      - name: install-cni
+        image: ${_prefix}flannel:${FLANNEL_TAG}
+        command:
+        - cp
+        args:
+        - -f
+        - /etc/kube-flannel/cni-conf.json
+        - /etc/cni/net.d/10-flannel.conflist
+        volumeMounts:
+        - name: cni
+          mountPath: /etc/cni/net.d
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      containers:
+      - name: kube-flannel
+        image: ${_prefix}flannel:${FLANNEL_TAG}
+        command:
+        - /opt/bin/flanneld
+        args:
+        - --ip-masq
+        - --kube-subnet-mgr
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        volumeMounts:
+        - name: run
+          mountPath: /run
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      volumes:
+        - name: host-cni-bin
+          hostPath:
+            path: /opt/cni/bin
+        - name: run
+          hostPath:
+            path: /run
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: flannel-cfg
+          configMap:
+            name: kube-flannel-cfg
 EOF
+    }
 
-chown root:root $FLANNEL_DOCKER_BRIDGE_SERVICE
-chmod 0644 $FLANNEL_DOCKER_BRIDGE_SERVICE
+    if [ "$MASTER_INDEX" = "0" ]; then
 
-cat >> $DOCKER_FLANNEL_CONF <<EOF
-[Unit]
-Requires=flannel-docker-bridge.service
-After=flannel-docker-bridge.service
+        until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+        do
+            echo "Waiting for Kubernetes API..."
+            sleep 5
+        done
+    fi
 
-[Service]
-EnvironmentFile=/run/flannel/docker
-EOF
-
-chown root:root $DOCKER_FLANNEL_CONF
-chmod 0644 $DOCKER_FLANNEL_CONF
-
-cat >> $FLANNEL_DOCKER_BRIDGE_CONF <<EOF
-[Unit]
-Requires=flannel-docker-bridge.service
-Before=flannel-docker-bridge.service
-
-[Install]
-Also=flannel-docker-bridge.service
-EOF
-
-chown root:root $FLANNEL_DOCKER_BRIDGE_CONF
-chmod 0644 $FLANNEL_DOCKER_BRIDGE_CONF
-
-# Workaround for https://github.com/coreos/flannel/issues/799
-# Not solved upstream properly yet.
-cat >> "${SYSTEMD_UNITS_DIR}${FLANNEL_IPTABLES_FORWARD_ACCEPT_SERVICE}" <<EOF
-[Unit]
-After=flanneld.service docker.service kubelet.service kube-proxy.service
-Requires=flanneld.service
-
-[Service]
-Type=oneshot
-ExecStart=/usr/sbin/iptables -P FORWARD ACCEPT
-ExecStartPost=/usr/sbin/iptables -S
-
-[Install]
-WantedBy=flanneld.service
-EOF
-
-chown root:root "${SYSTEMD_UNITS_DIR}${FLANNEL_IPTABLES_FORWARD_ACCEPT_SERVICE}"
-chmod 0644 "${SYSTEMD_UNITS_DIR}${FLANNEL_IPTABLES_FORWARD_ACCEPT_SERVICE}"
-systemctl daemon-reload
-systemctl enable "${FLANNEL_IPTABLES_FORWARD_ACCEPT_SERVICE}"
-
-echo "activating service flanneld"
-systemctl enable flanneld
-systemctl start flanneld
+    /usr/bin/kubectl apply -f "${FLANNEL_DEPLOY}" --namespace=kube-system
+fi

magnum/drivers/common/templates/kubernetes/fragments/make-cert-client.sh

@@ -147,5 +147,3 @@ chmod 550 "${cert_dir}"
 chown -R kube:kube_etcd "${cert_dir}"
 chmod 440 ${cert_dir}/kubelet.key
 chmod 440 ${cert_dir}/proxy.key
-mkdir -p /etc/flanneld/certs
-cp ${cert_dir}/* /etc/flanneld/certs

magnum/drivers/common/templates/kubernetes/fragments/write-flannel-config.sh

@@ -1,28 +0,0 @@
-#!/bin/sh
-
-. /etc/sysconfig/heat-params
-
-if [ "$NETWORK_DRIVER" != "flannel" ]; then
-    exit 0
-fi
-
-FLANNEL_JSON=/etc/sysconfig/flannel-network.json
-FLANNELD_CONFIG=/etc/sysconfig/flanneld
-
-cat > /etc/sysconfig/flanneld <<EOF
-FLANNEL_ETCD_ENDPOINTS="http://127.0.0.1:2379"
-FLANNEL_ETCD_PREFIX="/atomic.io/network"
-FLANNEL_OPTIONS=
-EOF
-
-# Generate a flannel configuration that we will
-# store into etcd using curl.
-cat > $FLANNEL_JSON <<EOF
-{
-  "Network": "$FLANNEL_NETWORK_CIDR",
-  "Subnetlen": $FLANNEL_NETWORK_SUBNETLEN,
-  "Backend": {
-    "Type": "$FLANNEL_BACKEND"
-  }
-}
-EOF

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml

@@ -46,6 +46,7 @@ write_files:
       CLOUD_PROVIDER_ENABLED="$CLOUD_PROVIDER_ENABLED"
       ETCD_TAG="$ETCD_TAG"
       FLANNEL_TAG="$FLANNEL_TAG"
+      FLANNEL_CNI_TAG="$FLANNEL_CNI_TAG"
       KUBE_VERSION="$KUBE_VERSION"
       KUBE_DASHBOARD_VERSION="$KUBE_DASHBOARD_VERSION"
       TRUSTEE_USER_ID="$TRUSTEE_USER_ID"

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml

@@ -38,7 +38,6 @@ write_files:
       NO_PROXY="$NO_PROXY"
       WAIT_CURL="$WAIT_CURL"
       KUBE_TAG="$KUBE_TAG"
-      FLANNEL_TAG="$FLANNEL_TAG"
       FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR"
       PODS_NETWORK_CIDR="$PODS_NETWORK_CIDR"
       KUBE_VERSION="$KUBE_VERSION"

magnum/drivers/heat/k8s_fedora_template_def.py

@@ -109,7 +109,7 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
                       'cgroup_driver',
                       'calico_tag', 'calico_cni_tag',
                       'calico_kube_controllers_tag', 'calico_ipv4pool',
-                      'etcd_tag', 'flannel_tag',
+                      'etcd_tag', 'flannel_tag', 'flannel_cni_tag',
                       'cloud_provider_enabled',
                       'cloud_provider_tag',
                       'prometheus_tag',

magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

@@ -348,8 +348,13 @@ parameters:
 
   flannel_tag:
     type: string
-    description: tag of the flannel system containers
-    default: v0.9.0
+    description: tag of the flannel container
+    default: v0.10.0-amd64
+
+  flannel_cni_tag:
+    type: string
+    description: tag of the flannel cni container
+    default: v0.3.0
 
   kube_version:
     type: string
@@ -778,6 +783,7 @@ resources:
           kube_version: {get_param: kube_version}
           etcd_tag: {get_param: etcd_tag}
           flannel_tag: {get_param: flannel_tag}
+          flannel_cni_tag: {get_param: flannel_cni_tag}
           kube_dashboard_version: {get_param: kube_dashboard_version}
           trustee_user_id: {get_param: trustee_user_id}
           trustee_password: {get_param: trustee_password}
@@ -834,6 +840,7 @@ resources:
                   "$CA_KEY": {get_param: ca_key}
             - get_file: ../../common/templates/kubernetes/fragments/core-dns-service.sh
             - get_file: ../../common/templates/kubernetes/fragments/calico-service.sh
+            - get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh
             - get_file: ../../common/templates/kubernetes/fragments/enable-helm-tiller.sh
             - get_file: ../../common/templates/kubernetes/helm/metrics-server.sh
             - get_file: ../../common/templates/kubernetes/fragments/install-helm-modules.sh
@@ -919,7 +926,6 @@ resources:
           no_proxy: {get_param: no_proxy}
           kube_tag: {get_param: kube_tag}
           kube_version: {get_param: kube_version}
-          flannel_tag: {get_param: flannel_tag}
           trustee_user_id: {get_param: trustee_user_id}
           trustee_username: {get_param: trustee_username}
           trustee_password: {get_param: trustee_password}

magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml

@@ -239,6 +239,10 @@ parameters:
     type: string
     description: tag of the flannel system containers
 
+  flannel_cni_tag:
+    type: string
+    description: tag of the flannel cni container
+
   kube_version:
     type: string
     description: version of kubernetes used for kubernetes cluster
@@ -502,6 +506,7 @@ resources:
             "$CLOUD_PROVIDER_ENABLED": {get_param: cloud_provider_enabled}
             "$ETCD_TAG": {get_param: etcd_tag}
             "$FLANNEL_TAG": {get_param: flannel_tag}
+            "$FLANNEL_CNI_TAG": {get_param: flannel_cni_tag}
             "$KUBE_VERSION": {get_param: kube_version}
             "$KUBE_DASHBOARD_VERSION": {get_param: kube_dashboard_version}
             "$TRUSTEE_USER_ID": {get_param: trustee_user_id}
@@ -584,24 +589,6 @@ resources:
       group: ungrouped
       config: {get_file: ../../common/templates/kubernetes/fragments/configure-kubernetes-master.sh}
 
-  write_flannel_config:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/write-flannel-config.sh}
-
-  flannel_config_service:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/flannel-config-service.sh}
-
-  flannel_service:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh}
-
   enable_services:
     type: OS::Heat::SoftwareConfig
     properties:
@@ -641,9 +628,6 @@ resources:
         - config: {get_resource: add_proxy}
         - config: {get_resource: start_container_agent}
         - config: {get_resource: enable_services}
-        - config: {get_resource: write_flannel_config}
-        - config: {get_resource: flannel_config_service}
-        - config: {get_resource: flannel_service}
 
   ######################################################################
   #

magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml

@@ -182,10 +182,6 @@ parameters:
     type: string
     description: tag of the k8s containers used to provision the kubernetes cluster
 
-  flannel_tag:
-    type: string
-    description: tag of the flannel system containers
-
   kube_version:
     type: string
     description: version of kubernetes used for kubernetes cluster
@@ -342,7 +338,6 @@ resources:
             $HTTPS_PROXY: {get_param: https_proxy}
             $NO_PROXY: {get_param: no_proxy}
             $KUBE_TAG: {get_param: kube_tag}
-            $FLANNEL_TAG: {get_param: flannel_tag}
             $FLANNEL_NETWORK_CIDR: {get_param: flannel_network_cidr}
             $PODS_NETWORK_CIDR: {get_param: pods_network_cidr}
             $KUBE_VERSION: {get_param: kube_version}
@@ -405,12 +400,6 @@ resources:
       group: ungrouped
       config: {get_file: ../../common/templates/kubernetes/fragments/configure-kubernetes-minion.sh}
 
-  flannel_service:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh}
-
   enable_services:
     type: OS::Heat::SoftwareConfig
     properties:
@@ -466,7 +455,6 @@ resources:
         - config: {get_resource: configure_docker_storage}
         - config: {get_resource: configure_docker_registry}
         - config: {get_resource: configure_kubernetes_minion}
-        - config: {get_resource: flannel_service}
         - config: {get_resource: add_proxy}
         - config: {get_resource: enable_services}
         - config: {get_resource: enable_docker_registry}

magnum/tests/unit/drivers/test_template_definition.py

@@ -366,6 +366,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
         kube_tag = mock_cluster.labels.get('kube_tag')
         etcd_tag = mock_cluster.labels.get('etcd_tag')
         flannel_tag = mock_cluster.labels.get('flannel_tag')
+        flannel_cni_tag = mock_cluster.labels.get('flannel_cni_tag')
         container_infra_prefix = mock_cluster.labels.get(
             'container_infra_prefix')
         availability_zone = mock_cluster.labels.get(
@@ -457,6 +458,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'kube_tag': kube_tag,
             'etcd_tag': etcd_tag,
             'flannel_tag': flannel_tag,
+            'flannel_cni_tag': flannel_cni_tag,
             'container_infra_prefix': container_infra_prefix,
             'nodes_affinity_policy': 'soft-anti-affinity',
             'availability_zone': availability_zone,
@@ -732,6 +734,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
         kube_tag = mock_cluster.labels.get('kube_tag')
         etcd_tag = mock_cluster.labels.get('etcd_tag')
         flannel_tag = mock_cluster.labels.get('flannel_tag')
+        flannel_cni_tag = mock_cluster.labels.get('flannel_cni_tag')
         container_infra_prefix = mock_cluster.labels.get(
             'container_infra_prefix')
         availability_zone = mock_cluster.labels.get(
@@ -825,6 +828,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'kube_tag': kube_tag,
             'etcd_tag': etcd_tag,
             'flannel_tag': flannel_tag,
+            'flannel_cni_tag': flannel_cni_tag,
             'container_infra_prefix': container_infra_prefix,
             'nodes_affinity_policy': 'soft-anti-affinity',
             'availability_zone': availability_zone,

releasenotes/notes/flannel-cni-4a5c9f574325761e.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    For k8s_fedora_atomic, run flannel as a cni plugin. The deployment method
+    is taken from the flannel upstream documentation. One more label for the
+    cni tag is added `flannel_cni_tag` for the container,
+    quay.io/repository/coreos/flannel-cni. The flannel container is taken
+    from flannel upsteam as well quay.io/repository/coreos/flannel.