From 4b332cf3af0122af5e5cedb78fd7b618e62ca661 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Dulko?= <mdulko@redhat.com>
Date: Fri, 8 Mar 2019 15:51:28 +0100
Subject: [PATCH] Add option to deploy coredns

As a step to improve testing capabilities of our gates, this commit
enhances DevStack with support for deploying coredns in our K8s cluster.
The idea here is to be able to run any tests that are referring to
services by <namespace>.<service-name>, in particular upstream K8s
tests.

The tricky part here is that on gate VM's an instance of unbound DNS is
running on 127.0.0.1:53. As in DevStack-deployed Kuryr pods doesn't
support IPv6, we couldn't just take IPv6 addresses of upstream DNS from
unbound configuration and use them in coredns pods. Instead the coredns
instance is running on host networking and binds to $HOST_IP:53, which
is also used as value of kubelet's --cluster-dns option, while
forwarding any upstream DNS requests to the local unbound instance. This
isn't perfectly how it would be set up in production environment, but
should be close enough for our purposes.

This change only affects DevStack, so it's completely safe from release
point of view. coredns gets enabled only on gates running Kubernetes as
OpenShift gates run openshift-dns already.

Change-Id: Icdab52a6229b2209f58e26e4d885f551883727b5
Partial-Implements: blueprint k8s-upstream-tests
---
 .zuul.d/base.yaml    |  1 +
 .zuul.d/octavia.yaml |  1 +
 devstack/plugin.sh   | 92 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 94 insertions(+)

diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml
index 997c344d0..87e066e1c 100644
--- a/.zuul.d/base.yaml
+++ b/.zuul.d/base.yaml
@@ -76,6 +76,7 @@
         kubelet: true
         kuryr-kubernetes: true
         kuryr-daemon: true
+        coredns: true
       zuul_copy_output:
         '{{ devstack_log_dir }}/kubernetes': 'logs'
     irrelevant-files:
diff --git a/.zuul.d/octavia.yaml b/.zuul.d/octavia.yaml
index be726f47b..a05185eab 100644
--- a/.zuul.d/octavia.yaml
+++ b/.zuul.d/octavia.yaml
@@ -95,6 +95,7 @@
         kubernetes-controller-manager: false
         kubernetes-scheduler: false
         kubelet: false
+        coredns: false
         openshift-master: true
         openshift-node: true
         openshift-dnsmasq: true
diff --git a/devstack/plugin.sh b/devstack/plugin.sh
index 97bfe4c50..a74b3c6d3 100644
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@@ -754,6 +754,11 @@ function run_k8s_kubelet {
         command="$command --fail-swap-on=false"
     fi
 
+    if is_service_enabled coredns; then
+        local k8s_resolv_conf
+        command+=" --cluster-dns=${HOST_IP} --cluster-domain=cluster.local"
+    fi
+
     wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL"
     if [[ "$USE_SYSTEMD" = "True" ]]; then
         # If systemd is being used, proceed as normal
@@ -767,6 +772,86 @@ function run_k8s_kubelet {
     fi
 }
 
+function run_coredns {
+    local output_dir=$1
+    mkdir -p "$output_dir"
+    rm -f ${output_dir}/coredns.yml
+    cat >> "${output_dir}/coredns.yml" << EOF
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+data:
+  Corefile: |
+    .:53 {
+        bind ${HOST_IP}
+        errors
+        kubernetes cluster.local in-addr.arpa ip6.arpa {
+           pods insecure
+           upstream
+           fallthrough in-addr.arpa ip6.arpa
+        }
+        proxy . /etc/resolv.conf
+        cache 30
+        loop
+        reload
+        loadbalance
+EOF
+    if [[ "$ENABLE_DEBUG_LOG_LEVEL" == "True" ]]; then
+        cat >> "${output_dir}/coredns.yml" << EOF
+        debug
+        log
+EOF
+    fi
+    cat >> "${output_dir}/coredns.yml" << EOF
+    }
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+    k8s-app: coredns
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "CoreDNS"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: coredns
+  template:
+    metadata:
+      labels:
+        k8s-app: coredns
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
+    spec:
+      hostNetwork: true
+      containers:
+      - name: coredns
+        image: coredns/coredns
+        imagePullPolicy: Always
+        args: [ "-conf", "/etc/coredns/Corefile" ]
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/coredns
+      dnsPolicy: Default
+      volumes:
+        - name: config-volume
+          configMap:
+            name: coredns
+            items:
+            - key: Corefile
+              path: Corefile
+EOF
+
+    /usr/local/bin/kubectl apply -f ${output_dir}/coredns.yml
+}
+
+
 function run_kuryr_kubernetes {
     local python_bin=$(which python)
 
@@ -1071,6 +1156,13 @@ elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
             run_kuryr_daemon
         fi
 
+        if is_service_enabled coredns; then
+            #Open port 53 so pods can reach the DNS server
+            sudo iptables -I INPUT 1 -p udp -m udp --dport 53 -j ACCEPT
+
+            run_coredns "${DATA_DIR}/kuryr-kubernetes"
+        fi
+
         # Needs kuryr to be running
         if is_service_enabled openshift-dns; then
             configure_and_run_registry