[kubernetes] add ingress controller

Add ingress controller configuration and backend to kubernetes clusters. A new label 'ingress_controller' defines which backend should serve ingress, with traefik added as the only option for now. It is defined as a DaemonSet, with instances on all nodes defined with a certain role. This role is set as an additional cluster label 'ingress_controller_role', with a default value of 'ingress'. For now no node is automatically set with this role, with users or operators having to do this manually after cluster creation. Change-Id: I5175cf91f37e2988dc3d33042558d994810842f3 Closes-Bug: #1738808
2017-12-18 16:31:30 +00:00 · 2017-12-18 16:31:30 +00:00 · 0b18989a50
commit 0b18989a50
parent 6ce434fa33
10 changed files with 286 additions and 7 deletions
--- a/doc/source/user/index.rst
+++ b/doc/source/user/index.rst
@ -353,6 +353,10 @@ the table are linked to more details elsewhere in the user guide.
 +---------------------------------------+--------------------+---------------+
 | `cert_manager_api`_                   | see below          | false         |
 +---------------------------------------+--------------------+---------------+
+| `ingress_controller`_                 | see below          | ""            |
+---------------------------------------+--------------------+---------------+
+| `ingress_controller_role`_            | see below          | "ingress"     |
+---------------------------------------+--------------------+---------------+

 Cluster
 -------
@ -1144,6 +1148,29 @@ service is deleted.
 Refer to the `Kubernetes External Load Balancer`_ section
 for more details.

+Ingress Controller
+------------------
+
+In addition to the LoadBalancer described above, Kubernetes can also
+be configured with an Ingress Controller. Ingress can provide load
+balancing, SSL termination and name-based virtual hosting.
+
+Magnum allows selecting one of multiple controller options via the
+'ingress_controller' label. Check the Kubernetes documentation to define
+your own Ingress resources.
+
+_`ingress_controller`
+  This label sets the Ingress Controller to be used. Currently only traefik
+  is supported. The default is '', meaning no Ingress Controller configured.
+
+_`ingress_controller_role`
+  This label defines the role nodes should have to run an instance of the
+  Ingress Controller. This gives operators full control on which nodes should
+  be running an instance of the controller, and should be set in multiple nodes
+  for availability. Default is 'ingress'. An example of setting this in a
+  Kubernetes node would be::
+
+    kubectl label node <node-name> role=ingress

 Swarm
 =====
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-controller
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-controller
@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Enables the specified ingress controller.
+#
+# Currently there is only support for traefik.
+. /etc/sysconfig/heat-params
+
+function writeFile {
+    # $1 is filename
+    # $2 is file content
+
+    [ -f ${1} ] || {
+        echo "Writing File: $1"
+        mkdir -p $(dirname ${1})
+        cat << EOF > ${1}
+$2
+EOF
+    }
+}
+
+if [ "$(echo $INGRESS_CONTROLLER | tr '[:upper:]' '[:lower:]')" = "traefik" ]; then
+    $enable-ingress-traefik
+fi
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-ingress-traefik
@ -0,0 +1,147 @@
+INGRESS_TRAEFIK_MANIFEST=/srv/magnum/kubernetes/ingress-traefik.yaml
+INGRESS_TRAEFIK_MANIFEST_CONTENT=$(cat <<EOF
+---
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: ingress-traefik
+  namespace: kube-system
+  labels:
+    k8s-app: ingress-traefik-backend
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: ingress-traefik-backend
+        name: ingress-traefik-backend
+    spec:
+      terminationGracePeriodSeconds: 60
+      hostNetwork: true
+      containers:
+      - image: ${CONTAINER_INFRA_PREFIX:-docker.io/}traefik
+        name: ingress-traefik-backend
+        ports:
+        - name: http
+          containerPort: 80
+          hostPort: 80
+        - name: admin
+          containerPort: 8080
+        securityContext:
+          privileged: true
+        args:
+        - -d
+        - --web
+        - --kubernetes
+      nodeSelector:
+        role: ${INGRESS_CONTROLLER_ROLE}
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: ingress-traefik
+  namespace: kube-system
+spec:
+  selector:
+    k8s-app: ingress-traefik-backend
+  ports:
+    - name: http
+      protocol: TCP
+      port: 80
+    - name: admin
+      protocol: TCP
+      port: 8080
+  type: NodePort
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: ingress-traefik
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - services
+      - endpoints
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - extensions
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: ingress-traefik
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ingress-traefik
+subjects:
+- kind: ServiceAccount
+  name: ingress-traefik
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ingress-traefik
+  namespace: kube-system
+EOF
+)
+writeFile $INGRESS_TRAEFIK_MANIFEST "$INGRESS_TRAEFIK_MANIFEST_CONTENT"
+
+INGRESS_TRAEFIK_BIN="/srv/magnum/kubernetes/bin/ingress-traefik"
+INGRESS_TRAEFIK_SERVICE="/etc/systemd/system/ingress-traefik.service"
+
+# Binary for ingress traefik
+INGRESS_TRAEFIK_BIN_CONTENT='''#!/bin/sh
+until curl -sf "http://127.0.0.1:8080/healthz"
+do
+    echo "Waiting for Kubernetes API..."
+    sleep 5
+done
+
+# Check if all resources exist already before creating them
+kubectl -n kube-system get service ingress-traefik
+if [ "$?" != "0" ] && \
+        [ -f "'''${INGRESS_TRAEFIK_MANIFEST}'''" ]; then
+    kubectl create -f '''${INGRESS_TRAEFIK_MANIFEST}'''
+fi
+'''
+writeFile $INGRESS_TRAEFIK_BIN "$INGRESS_TRAEFIK_BIN_CONTENT"
+
+
+# Service for ingress traefik
+INGRESS_TRAEFIK_SERVICE_CONTENT='''[Unit]
+Requires=kube-apiserver.service
+
+[Service]
+Type=oneshot
+Environment=HOME=/root
+EnvironmentFile=-/etc/kubernetes/config
+ExecStart='''${INGRESS_TRAEFIK_BIN}'''
+
+[Install]
+WantedBy=multi-user.target
+'''
+writeFile $INGRESS_TRAEFIK_SERVICE "$INGRESS_TRAEFIK_SERVICE_CONTENT"
+
+chown root:root ${INGRESS_TRAEFIK_BIN}
+chmod 0755 ${INGRESS_TRAEFIK_BIN}
+
+chown root:root ${INGRESS_TRAEFIK_SERVICE}
+chmod 0644 ${INGRESS_TRAEFIK_SERVICE}
+
+# Launch the ingress traefik service
+set -x
+systemctl daemon-reload
+systemctl enable ingress-traefik.service
+systemctl start --no-block ingress-traefik.service
--- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml
+++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml
@ -59,3 +59,5 @@ write_files:
      CALICO_CNI_TAG="$CALICO_CNI_TAG"
      CALICO_KUBE_CONTROLLERS_TAG="$CALICO_KUBE_CONTROLLERS_TAG"
      CALICO_IPV4POOL="$CALICO_IPV4POOL"
+      INGRESS_CONTROLLER="$INGRESS_CONTROLLER"
+      INGRESS_CONTROLLER_ROLE="$INGRESS_CONTROLLER_ROLE"
--- a/magnum/drivers/heat/k8s_template_def.py
+++ b/magnum/drivers/heat/k8s_template_def.py
@ -114,7 +114,9 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition):
                      'grafana_admin_passwd',
                      'kube_dashboard_enabled',
                      'etcd_volume_size',
-                      'cert_manager_api']
+                      'cert_manager_api',
+                      'ingress_controller',
+                      'ingress_controller_role']

        for label in label_list:
            extra_params[label] = cluster.labels.get(label)
--- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
+++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
@ -405,6 +405,18 @@ parameters:
    type: string
    description: Configure the IP pool/range from which pod IPs will be chosen

+  ingress_controller:
+    type: string
+    description: >
+      ingress controller backend to use
+    default: ""
+
+  ingress_controller_role:
+    type: string
+    description: >
+      node role where the ingress controller backend should run
+    default: "ingress"
+
 resources:

  ######################################################################
@ -606,6 +618,8 @@ resources:
          calico_kube_controllers_tag: {get_param: calico_kube_controllers_tag}
          calico_ipv4pool: {get_param: calico_ipv4pool}
          pods_network_cidr: {get_param: pods_network_cidr}
+          ingress_controller: {get_param: ingress_controller}
+          ingress_controller_role: {get_param: ingress_controller_role}

  ######################################################################
  #
--- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
+++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
@ -303,6 +303,16 @@ parameters:
    type: string
    description: Configure the IP pool/range from which pod IPs will be chosen

+  ingress_controller:
+    type: string
+    description: >
+      ingress controller backend to use
+
+  ingress_controller_role:
+    type: string
+    description: >
+      node role where the ingress controller should run
+
 resources:

  master_wait_handle:
@ -396,6 +406,8 @@ resources:
            "$CALICO_CNI_TAG": {get_param: calico_cni_tag}
            "$CALICO_KUBE_CONTROLLERS_TAG": {get_param: calico_kube_controllers_tag}
            "$CALICO_IPV4POOL": {get_param: calico_ipv4pool}
+            "$INGRESS_CONTROLLER": {get_param: ingress_controller}
+            "$INGRESS_CONTROLLER_ROLE": {get_param: ingress_controller_role}

  install_openstack_ca:
    type: OS::Heat::SoftwareConfig
@ -580,6 +592,24 @@ resources:
      server: {get_resource: kube-master}
      actions: ['CREATE']

+  enable_ingress_controller:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        str_replace:
+          params:
+            $enable-ingress-traefik: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-traefik}
+          template: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-controller}
+
+  enable_ingress_controller_deployment:
+    type: OS::Heat::SoftwareDeployment
+    properties:
+      signal_transport: HEAT_SIGNAL
+      config: {get_resource: enable_ingress_controller}
+      server: {get_resource: kube-master}
+      actions: ['CREATE']
+
  ######################################################################
  #
  # a single kubernetes master.
--- a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
+++ b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
@ -98,7 +98,9 @@ class TestClusterConductorWithK8s(base.TestCase):
                       'kube_dashboard_enabled': 'True',
                       'docker_volume_type': 'lvmdriver-1',
                       'availability_zone': 'az_1',
-                       'cert_manager_api': 'False'},
+                       'cert_manager_api': 'False',
+                       'ingress_controller': 'i-controller',
+                       'ingress_controller_role': 'i-controller-role'},
            'master_flavor_id': 'master_flavor_id',
            'flavor_id': 'flavor_id',
        }
@ -179,7 +181,9 @@ class TestClusterConductorWithK8s(base.TestCase):
                       'docker_volume_type': 'lvmdriver-1',
                       'etcd_volume_size': None,
                       'availability_zone': 'az_1',
-                       'cert_manager_api': 'False'},
+                       'cert_manager_api': 'False',
+                       'ingress_controller': 'i-controller',
+                       'ingress_controller_role': 'i-controller-role'},
            'http_proxy': 'http_proxy',
            'https_proxy': 'https_proxy',
            'no_proxy': 'no_proxy',
@ -237,6 +241,8 @@ class TestClusterConductorWithK8s(base.TestCase):
            "nodes_affinity_policy": "soft-anti-affinity",
            'availability_zone': 'az_1',
            'cert_manager_api': 'False',
+            'ingress_controller': 'i-controller',
+            'ingress_controller_role': 'i-controller-role',
        }
        if missing_attr is not None:
            expected.pop(mapping[missing_attr], None)
@ -336,6 +342,8 @@ class TestClusterConductorWithK8s(base.TestCase):
            "nodes_affinity_policy": "soft-anti-affinity",
            'availability_zone': 'az_1',
            'cert_manager_api': 'False',
+            'ingress_controller': 'i-controller',
+            'ingress_controller_role': 'i-controller-role',
        }

        self.assertEqual(expected, definition)
@ -422,6 +430,8 @@ class TestClusterConductorWithK8s(base.TestCase):
            "nodes_affinity_policy": "soft-anti-affinity",
            'availability_zone': 'az_1',
            'cert_manager_api': 'False',
+            'ingress_controller': 'i-controller',
+            'ingress_controller_role': 'i-controller-role',
        }
        self.assertEqual(expected, definition)
        self.assertEqual(
@ -501,6 +511,8 @@ class TestClusterConductorWithK8s(base.TestCase):
            'verify_ca': True,
            'openstack_ca': '',
            'cert_manager_api': 'False',
+            'ingress_controller': 'i-controller',
+            'ingress_controller_role': 'i-controller-role',
        }
        self.assertEqual(expected, definition)
        self.assertEqual(
@ -575,6 +587,8 @@ class TestClusterConductorWithK8s(base.TestCase):
            'verify_ca': True,
            'openstack_ca': '',
            'cert_manager_api': 'False',
+            'ingress_controller': 'i-controller',
+            'ingress_controller_role': 'i-controller-role',
        }
        self.assertEqual(expected, definition)
        self.assertEqual(
@ -750,6 +764,8 @@ class TestClusterConductorWithK8s(base.TestCase):
            "nodes_affinity_policy": "soft-anti-affinity",
            'availability_zone': 'az_1',
            'cert_manager_api': 'False',
+            'ingress_controller': 'i-controller',
+            'ingress_controller_role': 'i-controller-role',
        }
        self.assertEqual(expected, definition)
        self.assertEqual(
--- a/magnum/tests/unit/drivers/test_template_definition.py
+++ b/magnum/tests/unit/drivers/test_template_definition.py
@ -288,6 +288,10 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
            pods_network_cidr = flannel_cidr
        elif mock_cluster_template.network_driver == 'calico':
            pods_network_cidr = calico_ipv4pool
+        ingress_controller = mock_cluster.labels.get(
+            'ingress_controller')
+        ingress_controller_role = mock_cluster.labels.get(
+            'ingress_controller_role')

        k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()

@ -320,8 +324,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
            'calico_cni_tag': calico_cni_tag,
            'calico_kube_controllers_tag': calico_kube_controllers_tag,
            'calico_ipv4pool': calico_ipv4pool,
-            'pods_network_cidr': pods_network_cidr
-        }}
+            'pods_network_cidr': pods_network_cidr,
+            'ingress_controller': ingress_controller,
+            'ingress_controller_role': ingress_controller_role}}
        mock_get_params.assert_called_once_with(mock_context,
                                                mock_cluster_template,
                                                mock_cluster,
@ -396,6 +401,10 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
            pods_network_cidr = flannel_cidr
        elif mock_cluster_template.network_driver == 'calico':
            pods_network_cidr = calico_ipv4pool
+        ingress_controller = mock_cluster.labels.get(
+            'ingress_controller')
+        ingress_controller_role = mock_cluster.labels.get(
+            'ingress_controller_role')

        k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()

@ -430,8 +439,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
            'calico_cni_tag': calico_cni_tag,
            'calico_kube_controllers_tag': calico_kube_controllers_tag,
            'calico_ipv4pool': calico_ipv4pool,
-            'pods_network_cidr': pods_network_cidr
-        }}
+            'pods_network_cidr': pods_network_cidr,
+            'ingress_controller': ingress_controller,
+            'ingress_controller_role': ingress_controller_role}}
        mock_get_params.assert_called_once_with(mock_context,
                                                mock_cluster_template,
                                                mock_cluster,
--- a/releasenotes/notes/ingress-controller-552ea956ceabdd25.yaml
+++ b/releasenotes/notes/ingress-controller-552ea956ceabdd25.yaml
@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Add new labels 'ingress_controller' and 'ingress_controller_role' enabling
+    the deployment of a Kubernetes Ingress Controller backend for clusters.
+    Default for 'ingress_controller' is '' (meaning no controller deployed),
+    with possible values being 'traefik'.
+    Default for 'ingress_controller_role' is 'ingress'.