Add option to deploy coredns

As a step to improve testing capabilities of our gates, this commit
enhances DevStack with support for deploying coredns in our K8s cluster.
The idea here is to be able to run any tests that are referring to
services by <namespace>.<service-name>, in particular upstream K8s
tests.

The tricky part here is that on gate VM's an instance of unbound DNS is
running on 127.0.0.1:53. As in DevStack-deployed Kuryr pods doesn't
support IPv6, we couldn't just take IPv6 addresses of upstream DNS from
unbound configuration and use them in coredns pods. Instead the coredns
instance is running on host networking and binds to $HOST_IP:53, which
is also used as value of kubelet's --cluster-dns option, while
forwarding any upstream DNS requests to the local unbound instance. This
isn't perfectly how it would be set up in production environment, but
should be close enough for our purposes.

This change only affects DevStack, so it's completely safe from release
point of view. coredns gets enabled only on gates running Kubernetes as
OpenShift gates run openshift-dns already.

Change-Id: Icdab52a6229b2209f58e26e4d885f551883727b5
Partial-Implements: blueprint k8s-upstream-tests
This commit is contained in:
Michał Dulko 2019-03-08 15:51:28 +01:00
parent 05795f8873
commit 4b332cf3af
3 changed files with 94 additions and 0 deletions

View File

@ -76,6 +76,7 @@
kubelet: true kubelet: true
kuryr-kubernetes: true kuryr-kubernetes: true
kuryr-daemon: true kuryr-daemon: true
coredns: true
zuul_copy_output: zuul_copy_output:
'{{ devstack_log_dir }}/kubernetes': 'logs' '{{ devstack_log_dir }}/kubernetes': 'logs'
irrelevant-files: irrelevant-files:

View File

@ -95,6 +95,7 @@
kubernetes-controller-manager: false kubernetes-controller-manager: false
kubernetes-scheduler: false kubernetes-scheduler: false
kubelet: false kubelet: false
coredns: false
openshift-master: true openshift-master: true
openshift-node: true openshift-node: true
openshift-dnsmasq: true openshift-dnsmasq: true

View File

@ -754,6 +754,11 @@ function run_k8s_kubelet {
command="$command --fail-swap-on=false" command="$command --fail-swap-on=false"
fi fi
if is_service_enabled coredns; then
local k8s_resolv_conf
command+=" --cluster-dns=${HOST_IP} --cluster-domain=cluster.local"
fi
wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL" wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL"
if [[ "$USE_SYSTEMD" = "True" ]]; then if [[ "$USE_SYSTEMD" = "True" ]]; then
# If systemd is being used, proceed as normal # If systemd is being used, proceed as normal
@ -767,6 +772,86 @@ function run_k8s_kubelet {
fi fi
} }
function run_coredns {
local output_dir=$1
mkdir -p "$output_dir"
rm -f ${output_dir}/coredns.yml
cat >> "${output_dir}/coredns.yml" << EOF
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns
namespace: kube-system
data:
Corefile: |
.:53 {
bind ${HOST_IP}
errors
kubernetes cluster.local in-addr.arpa ip6.arpa {
pods insecure
upstream
fallthrough in-addr.arpa ip6.arpa
}
proxy . /etc/resolv.conf
cache 30
loop
reload
loadbalance
EOF
if [[ "$ENABLE_DEBUG_LOG_LEVEL" == "True" ]]; then
cat >> "${output_dir}/coredns.yml" << EOF
debug
log
EOF
fi
cat >> "${output_dir}/coredns.yml" << EOF
}
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: coredns
namespace: kube-system
labels:
k8s-app: coredns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "CoreDNS"
spec:
replicas: 1
selector:
matchLabels:
k8s-app: coredns
template:
metadata:
labels:
k8s-app: coredns
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
spec:
hostNetwork: true
containers:
- name: coredns
image: coredns/coredns
imagePullPolicy: Always
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
dnsPolicy: Default
volumes:
- name: config-volume
configMap:
name: coredns
items:
- key: Corefile
path: Corefile
EOF
/usr/local/bin/kubectl apply -f ${output_dir}/coredns.yml
}
function run_kuryr_kubernetes { function run_kuryr_kubernetes {
local python_bin=$(which python) local python_bin=$(which python)
@ -1071,6 +1156,13 @@ elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
run_kuryr_daemon run_kuryr_daemon
fi fi
if is_service_enabled coredns; then
#Open port 53 so pods can reach the DNS server
sudo iptables -I INPUT 1 -p udp -m udp --dport 53 -j ACCEPT
run_coredns "${DATA_DIR}/kuryr-kubernetes"
fi
# Needs kuryr to be running # Needs kuryr to be running
if is_service_enabled openshift-dns; then if is_service_enabled openshift-dns; then
configure_and_run_registry configure_and_run_registry