[kubernetes] add ingress controller

Add ingress controller configuration and backend to kubernetes clusters.

A new label 'ingress_controller' defines which backend should serve
ingress, with traefik added as the only option for now.

It is defined as a DaemonSet, with instances on all nodes defined with a
certain role. This role is set as an additional cluster label
'ingress_controller_role', with a default value of 'ingress'.

For now no node is automatically set with this role, with users or operators
having to do this manually after cluster creation.

Change-Id: I5175cf91f37e2988dc3d33042558d994810842f3
Closes-Bug: #1738808

doc/source/user/index.rst

@@ -353,6 +353,10 @@ the table are linked to more details elsewhere in the user guide.
 +---------------------------------------+--------------------+---------------+
 | `cert_manager_api`_                   | see below          | false         |
 +---------------------------------------+--------------------+---------------+
+| `ingress_controller`_                 | see below          | ""            |
++---------------------------------------+--------------------+---------------+
+| `ingress_controller_role`_            | see below          | "ingress"     |
++---------------------------------------+--------------------+---------------+
 
 Cluster
 -------
@@ -1144,6 +1148,29 @@ service is deleted.
 Refer to the `Kubernetes External Load Balancer`_ section
 for more details.
 
+Ingress Controller
+------------------
+
+In addition to the LoadBalancer described above, Kubernetes can also
+be configured with an Ingress Controller. Ingress can provide load
+balancing, SSL termination and name-based virtual hosting.
+
+Magnum allows selecting one of multiple controller options via the
+'ingress_controller' label. Check the Kubernetes documentation to define
+your own Ingress resources.
+
+_`ingress_controller`
+  This label sets the Ingress Controller to be used. Currently only traefik
+  is supported. The default is '', meaning no Ingress Controller configured.
+
+_`ingress_controller_role`
+  This label defines the role nodes should have to run an instance of the
+  Ingress Controller. This gives operators full control on which nodes should
+  be running an instance of the controller, and should be set in multiple nodes
+  for availability. Default is 'ingress'. An example of setting this in a
+  Kubernetes node would be::
+
+    kubectl label node <node-name> role=ingress
 
 Swarm
 =====

magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-controller

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Enables the specified ingress controller.
+#
+# Currently there is only support for traefik.
+. /etc/sysconfig/heat-params
+
+function writeFile {
+    # $1 is filename
+    # $2 is file content
+
+    [ -f ${1} ] || {
+        echo "Writing File: $1"
+        mkdir -p $(dirname ${1})
+        cat << EOF > ${1}
+$2
+EOF
+    }
+}
+
+if [ "$(echo $INGRESS_CONTROLLER | tr '[:upper:]' '[:lower:]')" = "traefik" ]; then
+    $enable-ingress-traefik
+fi

magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik

@@ -0,0 +1,147 @@
+INGRESS_TRAEFIK_MANIFEST=/srv/magnum/kubernetes/ingress-traefik.yaml
+INGRESS_TRAEFIK_MANIFEST_CONTENT=$(cat <<EOF
+---
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: ingress-traefik
+  namespace: kube-system
+  labels:
+    k8s-app: ingress-traefik-backend
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: ingress-traefik-backend
+        name: ingress-traefik-backend
+    spec:
+      terminationGracePeriodSeconds: 60
+      hostNetwork: true
+      containers:
+      - image: ${CONTAINER_INFRA_PREFIX:-docker.io/}traefik
+        name: ingress-traefik-backend
+        ports:
+        - name: http
+          containerPort: 80
+          hostPort: 80
+        - name: admin
+          containerPort: 8080
+        securityContext:
+          privileged: true
+        args:
+        - -d
+        - --web
+        - --kubernetes
+      nodeSelector:
+        role: ${INGRESS_CONTROLLER_ROLE}
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: ingress-traefik
+  namespace: kube-system
+spec:
+  selector:
+    k8s-app: ingress-traefik-backend
+  ports:
+    - name: http
+      protocol: TCP
+      port: 80
+    - name: admin
+      protocol: TCP
+      port: 8080
+  type: NodePort
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: ingress-traefik
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - services
+      - endpoints
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - extensions
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: ingress-traefik
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ingress-traefik
+subjects:
+- kind: ServiceAccount
+  name: ingress-traefik
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ingress-traefik
+  namespace: kube-system
+EOF
+)
+writeFile $INGRESS_TRAEFIK_MANIFEST "$INGRESS_TRAEFIK_MANIFEST_CONTENT"
+
+INGRESS_TRAEFIK_BIN="/srv/magnum/kubernetes/bin/ingress-traefik"
+INGRESS_TRAEFIK_SERVICE="/etc/systemd/system/ingress-traefik.service"
+
+# Binary for ingress traefik
+INGRESS_TRAEFIK_BIN_CONTENT='''#!/bin/sh
+until curl -sf "http://127.0.0.1:8080/healthz"
+do
+    echo "Waiting for Kubernetes API..."
+    sleep 5
+done
+
+# Check if all resources exist already before creating them
+kubectl -n kube-system get service ingress-traefik
+if [ "$?" != "0" ] && \
+        [ -f "'''${INGRESS_TRAEFIK_MANIFEST}'''" ]; then
+    kubectl create -f '''${INGRESS_TRAEFIK_MANIFEST}'''
+fi
+'''
+writeFile $INGRESS_TRAEFIK_BIN "$INGRESS_TRAEFIK_BIN_CONTENT"
+
+
+# Service for ingress traefik
+INGRESS_TRAEFIK_SERVICE_CONTENT='''[Unit]
+Requires=kube-apiserver.service
+
+[Service]
+Type=oneshot
+Environment=HOME=/root
+EnvironmentFile=-/etc/kubernetes/config
+ExecStart='''${INGRESS_TRAEFIK_BIN}'''
+
+[Install]
+WantedBy=multi-user.target
+'''
+writeFile $INGRESS_TRAEFIK_SERVICE "$INGRESS_TRAEFIK_SERVICE_CONTENT"
+
+chown root:root ${INGRESS_TRAEFIK_BIN}
+chmod 0755 ${INGRESS_TRAEFIK_BIN}
+
+chown root:root ${INGRESS_TRAEFIK_SERVICE}
+chmod 0644 ${INGRESS_TRAEFIK_SERVICE}
+
+# Launch the ingress traefik service
+set -x
+systemctl daemon-reload
+systemctl enable ingress-traefik.service
+systemctl start --no-block ingress-traefik.service

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml

@@ -59,3 +59,5 @@ write_files:
       CALICO_CNI_TAG="$CALICO_CNI_TAG"
       CALICO_KUBE_CONTROLLERS_TAG="$CALICO_KUBE_CONTROLLERS_TAG"
       CALICO_IPV4POOL="$CALICO_IPV4POOL"
+      INGRESS_CONTROLLER="$INGRESS_CONTROLLER"
+      INGRESS_CONTROLLER_ROLE="$INGRESS_CONTROLLER_ROLE"

magnum/drivers/heat/k8s_template_def.py

@@ -114,7 +114,9 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition):
                       'grafana_admin_passwd',
                       'kube_dashboard_enabled',
                       'etcd_volume_size',
-                      'cert_manager_api']
+                      'cert_manager_api',
+                      'ingress_controller',
+                      'ingress_controller_role']
 
         for label in label_list:
             extra_params[label] = cluster.labels.get(label)

magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

@@ -405,6 +405,18 @@ parameters:
     type: string
     description: Configure the IP pool/range from which pod IPs will be chosen
 
+  ingress_controller:
+    type: string
+    description: >
+      ingress controller backend to use
+    default: ""
+
+  ingress_controller_role:
+    type: string
+    description: >
+      node role where the ingress controller backend should run
+    default: "ingress"
+
 resources:
 
   ######################################################################
@@ -606,6 +618,8 @@ resources:
           calico_kube_controllers_tag: {get_param: calico_kube_controllers_tag}
           calico_ipv4pool: {get_param: calico_ipv4pool}
           pods_network_cidr: {get_param: pods_network_cidr}
+          ingress_controller: {get_param: ingress_controller}
+          ingress_controller_role: {get_param: ingress_controller_role}
 
   ######################################################################
   #

magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml

@@ -303,6 +303,16 @@ parameters:
     type: string
     description: Configure the IP pool/range from which pod IPs will be chosen
 
+  ingress_controller:
+    type: string
+    description: >
+      ingress controller backend to use
+
+  ingress_controller_role:
+    type: string
+    description: >
+      node role where the ingress controller should run
+
 resources:
 
   master_wait_handle:
@@ -396,6 +406,8 @@ resources:
             "$CALICO_CNI_TAG": {get_param: calico_cni_tag}
             "$CALICO_KUBE_CONTROLLERS_TAG": {get_param: calico_kube_controllers_tag}
             "$CALICO_IPV4POOL": {get_param: calico_ipv4pool}
+            "$INGRESS_CONTROLLER": {get_param: ingress_controller}
+            "$INGRESS_CONTROLLER_ROLE": {get_param: ingress_controller_role}
 
   install_openstack_ca:
     type: OS::Heat::SoftwareConfig
@@ -580,6 +592,24 @@ resources:
       server: {get_resource: kube-master}
       actions: ['CREATE']
 
+  enable_ingress_controller:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        str_replace:
+          params:
+            $enable-ingress-traefik: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-traefik}
+          template: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-controller}
+
+  enable_ingress_controller_deployment:
+    type: OS::Heat::SoftwareDeployment
+    properties:
+      signal_transport: HEAT_SIGNAL
+      config: {get_resource: enable_ingress_controller}
+      server: {get_resource: kube-master}
+      actions: ['CREATE']
+
   ######################################################################
   #
   # a single kubernetes master.

magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py

@@ -98,7 +98,9 @@ class TestClusterConductorWithK8s(base.TestCase):
                        'kube_dashboard_enabled': 'True',
                        'docker_volume_type': 'lvmdriver-1',
                        'availability_zone': 'az_1',
-                       'cert_manager_api': 'False'},
+                       'cert_manager_api': 'False',
+                       'ingress_controller': 'i-controller',
+                       'ingress_controller_role': 'i-controller-role'},
             'master_flavor_id': 'master_flavor_id',
             'flavor_id': 'flavor_id',
         }
@@ -179,7 +181,9 @@ class TestClusterConductorWithK8s(base.TestCase):
                        'docker_volume_type': 'lvmdriver-1',
                        'etcd_volume_size': None,
                        'availability_zone': 'az_1',
-                       'cert_manager_api': 'False'},
+                       'cert_manager_api': 'False',
+                       'ingress_controller': 'i-controller',
+                       'ingress_controller_role': 'i-controller-role'},
             'http_proxy': 'http_proxy',
             'https_proxy': 'https_proxy',
             'no_proxy': 'no_proxy',
@@ -237,6 +241,8 @@ class TestClusterConductorWithK8s(base.TestCase):
             "nodes_affinity_policy": "soft-anti-affinity",
             'availability_zone': 'az_1',
             'cert_manager_api': 'False',
+            'ingress_controller': 'i-controller',
+            'ingress_controller_role': 'i-controller-role',
         }
         if missing_attr is not None:
             expected.pop(mapping[missing_attr], None)
@@ -336,6 +342,8 @@ class TestClusterConductorWithK8s(base.TestCase):
             "nodes_affinity_policy": "soft-anti-affinity",
             'availability_zone': 'az_1',
             'cert_manager_api': 'False',
+            'ingress_controller': 'i-controller',
+            'ingress_controller_role': 'i-controller-role',
         }
 
         self.assertEqual(expected, definition)
@@ -422,6 +430,8 @@ class TestClusterConductorWithK8s(base.TestCase):
             "nodes_affinity_policy": "soft-anti-affinity",
             'availability_zone': 'az_1',
             'cert_manager_api': 'False',
+            'ingress_controller': 'i-controller',
+            'ingress_controller_role': 'i-controller-role',
         }
         self.assertEqual(expected, definition)
         self.assertEqual(
@@ -501,6 +511,8 @@ class TestClusterConductorWithK8s(base.TestCase):
             'verify_ca': True,
             'openstack_ca': '',
             'cert_manager_api': 'False',
+            'ingress_controller': 'i-controller',
+            'ingress_controller_role': 'i-controller-role',
         }
         self.assertEqual(expected, definition)
         self.assertEqual(
@@ -575,6 +587,8 @@ class TestClusterConductorWithK8s(base.TestCase):
             'verify_ca': True,
             'openstack_ca': '',
             'cert_manager_api': 'False',
+            'ingress_controller': 'i-controller',
+            'ingress_controller_role': 'i-controller-role',
         }
         self.assertEqual(expected, definition)
         self.assertEqual(
@@ -750,6 +764,8 @@ class TestClusterConductorWithK8s(base.TestCase):
             "nodes_affinity_policy": "soft-anti-affinity",
             'availability_zone': 'az_1',
             'cert_manager_api': 'False',
+            'ingress_controller': 'i-controller',
+            'ingress_controller_role': 'i-controller-role',
         }
         self.assertEqual(expected, definition)
         self.assertEqual(

magnum/tests/unit/drivers/test_template_definition.py

@@ -288,6 +288,10 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
             pods_network_cidr = flannel_cidr
         elif mock_cluster_template.network_driver == 'calico':
             pods_network_cidr = calico_ipv4pool
+        ingress_controller = mock_cluster.labels.get(
+            'ingress_controller')
+        ingress_controller_role = mock_cluster.labels.get(
+            'ingress_controller_role')
 
         k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
 
@@ -320,8 +324,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
             'calico_cni_tag': calico_cni_tag,
             'calico_kube_controllers_tag': calico_kube_controllers_tag,
             'calico_ipv4pool': calico_ipv4pool,
-            'pods_network_cidr': pods_network_cidr
+            'pods_network_cidr': pods_network_cidr,
-        }}
+            'ingress_controller': ingress_controller,
+            'ingress_controller_role': ingress_controller_role}}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,
                                                 mock_cluster,
@@ -396,6 +401,10 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
             pods_network_cidr = flannel_cidr
         elif mock_cluster_template.network_driver == 'calico':
             pods_network_cidr = calico_ipv4pool
+        ingress_controller = mock_cluster.labels.get(
+            'ingress_controller')
+        ingress_controller_role = mock_cluster.labels.get(
+            'ingress_controller_role')
 
         k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
 
@@ -430,8 +439,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
             'calico_cni_tag': calico_cni_tag,
             'calico_kube_controllers_tag': calico_kube_controllers_tag,
             'calico_ipv4pool': calico_ipv4pool,
-            'pods_network_cidr': pods_network_cidr
+            'pods_network_cidr': pods_network_cidr,
-        }}
+            'ingress_controller': ingress_controller,
+            'ingress_controller_role': ingress_controller_role}}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,
                                                 mock_cluster,

releasenotes/notes/ingress-controller-552ea956ceabdd25.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Add new labels 'ingress_controller' and 'ingress_controller_role' enabling
+    the deployment of a Kubernetes Ingress Controller backend for clusters.
+    Default for 'ingress_controller' is '' (meaning no controller deployed),
+    with possible values being 'traefik'.
+    Default for 'ingress_controller_role' is 'ingress'.