Browse Source

Add option to deploy coredns

As a step to improve testing capabilities of our gates, this commit
enhances DevStack with support for deploying coredns in our K8s cluster.
The idea here is to be able to run any tests that are referring to
services by <namespace>.<service-name>, in particular upstream K8s
tests.

The tricky part here is that on gate VM's an instance of unbound DNS is
running on 127.0.0.1:53. As in DevStack-deployed Kuryr pods doesn't
support IPv6, we couldn't just take IPv6 addresses of upstream DNS from
unbound configuration and use them in coredns pods. Instead the coredns
instance is running on host networking and binds to $HOST_IP:53, which
is also used as value of kubelet's --cluster-dns option, while
forwarding any upstream DNS requests to the local unbound instance. This
isn't perfectly how it would be set up in production environment, but
should be close enough for our purposes.

This change only affects DevStack, so it's completely safe from release
point of view. coredns gets enabled only on gates running Kubernetes as
OpenShift gates run openshift-dns already.

Change-Id: Icdab52a6229b2209f58e26e4d885f551883727b5
Partial-Implements: blueprint k8s-upstream-tests
Michał Dulko 1 month ago
parent
commit
4b332cf3af
3 changed files with 94 additions and 0 deletions
  1. 1
    0
      .zuul.d/base.yaml
  2. 1
    0
      .zuul.d/octavia.yaml
  3. 92
    0
      devstack/plugin.sh

+ 1
- 0
.zuul.d/base.yaml View File

@@ -76,6 +76,7 @@
76 76
         kubelet: true
77 77
         kuryr-kubernetes: true
78 78
         kuryr-daemon: true
79
+        coredns: true
79 80
       zuul_copy_output:
80 81
         '{{ devstack_log_dir }}/kubernetes': 'logs'
81 82
     irrelevant-files:

+ 1
- 0
.zuul.d/octavia.yaml View File

@@ -95,6 +95,7 @@
95 95
         kubernetes-controller-manager: false
96 96
         kubernetes-scheduler: false
97 97
         kubelet: false
98
+        coredns: false
98 99
         openshift-master: true
99 100
         openshift-node: true
100 101
         openshift-dnsmasq: true

+ 92
- 0
devstack/plugin.sh View File

@@ -754,6 +754,11 @@ function run_k8s_kubelet {
754 754
         command="$command --fail-swap-on=false"
755 755
     fi
756 756
 
757
+    if is_service_enabled coredns; then
758
+        local k8s_resolv_conf
759
+        command+=" --cluster-dns=${HOST_IP} --cluster-domain=cluster.local"
760
+    fi
761
+
757 762
     wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL"
758 763
     if [[ "$USE_SYSTEMD" = "True" ]]; then
759 764
         # If systemd is being used, proceed as normal
@@ -767,6 +772,86 @@ function run_k8s_kubelet {
767 772
     fi
768 773
 }
769 774
 
775
+function run_coredns {
776
+    local output_dir=$1
777
+    mkdir -p "$output_dir"
778
+    rm -f ${output_dir}/coredns.yml
779
+    cat >> "${output_dir}/coredns.yml" << EOF
780
+apiVersion: v1
781
+kind: ConfigMap
782
+metadata:
783
+  name: coredns
784
+  namespace: kube-system
785
+data:
786
+  Corefile: |
787
+    .:53 {
788
+        bind ${HOST_IP}
789
+        errors
790
+        kubernetes cluster.local in-addr.arpa ip6.arpa {
791
+           pods insecure
792
+           upstream
793
+           fallthrough in-addr.arpa ip6.arpa
794
+        }
795
+        proxy . /etc/resolv.conf
796
+        cache 30
797
+        loop
798
+        reload
799
+        loadbalance
800
+EOF
801
+    if [[ "$ENABLE_DEBUG_LOG_LEVEL" == "True" ]]; then
802
+        cat >> "${output_dir}/coredns.yml" << EOF
803
+        debug
804
+        log
805
+EOF
806
+    fi
807
+    cat >> "${output_dir}/coredns.yml" << EOF
808
+    }
809
+---
810
+apiVersion: extensions/v1beta1
811
+kind: Deployment
812
+metadata:
813
+  name: coredns
814
+  namespace: kube-system
815
+  labels:
816
+    k8s-app: coredns
817
+    kubernetes.io/cluster-service: "true"
818
+    kubernetes.io/name: "CoreDNS"
819
+spec:
820
+  replicas: 1
821
+  selector:
822
+    matchLabels:
823
+      k8s-app: coredns
824
+  template:
825
+    metadata:
826
+      labels:
827
+        k8s-app: coredns
828
+      annotations:
829
+        scheduler.alpha.kubernetes.io/critical-pod: ''
830
+        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
831
+    spec:
832
+      hostNetwork: true
833
+      containers:
834
+      - name: coredns
835
+        image: coredns/coredns
836
+        imagePullPolicy: Always
837
+        args: [ "-conf", "/etc/coredns/Corefile" ]
838
+        volumeMounts:
839
+        - name: config-volume
840
+          mountPath: /etc/coredns
841
+      dnsPolicy: Default
842
+      volumes:
843
+        - name: config-volume
844
+          configMap:
845
+            name: coredns
846
+            items:
847
+            - key: Corefile
848
+              path: Corefile
849
+EOF
850
+
851
+    /usr/local/bin/kubectl apply -f ${output_dir}/coredns.yml
852
+}
853
+
854
+
770 855
 function run_kuryr_kubernetes {
771 856
     local python_bin=$(which python)
772 857
 
@@ -1071,6 +1156,13 @@ elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
1071 1156
             run_kuryr_daemon
1072 1157
         fi
1073 1158
 
1159
+        if is_service_enabled coredns; then
1160
+            #Open port 53 so pods can reach the DNS server
1161
+            sudo iptables -I INPUT 1 -p udp -m udp --dport 53 -j ACCEPT
1162
+
1163
+            run_coredns "${DATA_DIR}/kuryr-kubernetes"
1164
+        fi
1165
+
1074 1166
         # Needs kuryr to be running
1075 1167
         if is_service_enabled openshift-dns; then
1076 1168
             configure_and_run_registry

Loading…
Cancel
Save