Execute kuryr-cni as docker exec
This commit changes the way kuryr-cni is executed in containerized deployments. Now it'll use `docker exec` command to execute kuryr-cni inside the CNI container. This should make it easier to be consumed by deployers. To be able to do such changes I needed to stop mounting host's /etc directory. I believe this was unnecessary and was blocking curl from working in isolation from host OS. Closes-Bug: 1757531 Change-Id: I373d65536a43eab98f0fc708936b97637f82eaff
This commit is contained in:
Michał Dulko
parent
49470991ad
commit
206c158d37
@ -4,16 +4,13 @@ LABEL authors="Antoni Segura Puimedon<toni@kuryr.org>, Vikas Choudhary<vichoudh@
|
||||
ARG OSLO_LOCK_PATH=/var/kuryr-lock
|
||||
|
||||
RUN yum install -y epel-release https://rdoproject.org/repos/rdo-release.rpm \
|
||||
&& yum install -y --setopt=tsflags=nodocs python-pip iproute bridge-utils openvswitch sudo \
|
||||
&& yum install -y --setopt=tsflags=nodocs gcc python-devel git \
|
||||
&& pip install virtualenv \
|
||||
&& virtualenv /kuryr-kubernetes
|
||||
&& yum install -y --setopt=tsflags=nodocs python-pip iproute bridge-utils openvswitch sudo jq \
|
||||
&& yum install -y --setopt=tsflags=nodocs gcc python-devel git
|
||||
|
||||
COPY . /opt/kuryr-kubernetes
|
||||
|
||||
RUN cd /opt/kuryr-kubernetes \
|
||||
&& /kuryr-kubernetes/bin/pip install . \
|
||||
&& virtualenv --relocatable /kuryr-kubernetes \
|
||||
&& pip install . \
|
||||
&& rm -fr .git \
|
||||
&& yum -y history undo last \
|
||||
&& mkdir ${OSLO_LOCK_PATH}
|
||||
|
||||
39
cni_ds_init
39
cni_ds_init
@ -3,27 +3,44 @@
|
||||
function cleanup() {
|
||||
rm -f "/etc/cni/net.d/10-kuryr.conf"
|
||||
rm -f "/opt/cni/bin/kuryr-cni"
|
||||
rm -rf "/opt/cni/bin/kuryr-venv"
|
||||
rm -rf /etc/kuryr
|
||||
}
|
||||
|
||||
function deploy() {
|
||||
local serviceaccount_path
|
||||
serviceaccount_path="/var/run/secrets/kubernetes.io/serviceaccount"
|
||||
|
||||
mkdir -p /etc/kuryr
|
||||
cp "${serviceaccount_path}/token" /etc/kuryr/token
|
||||
cp "${serviceaccount_path}/ca.crt" /etc/kuryr/ca.crt
|
||||
# Prepare token.
|
||||
KUBE_TOKEN=$(<${serviceaccount_path}/token)
|
||||
POD_NAMESPACE=$(<${serviceaccount_path}/namespace)
|
||||
|
||||
cp /opt/kuryr-kubernetes/etc/cni/net.d/* /etc/cni/net.d
|
||||
cp -r /kuryr-kubernetes "/opt/cni/bin/kuryr-venv"
|
||||
CONTAINERID=""
|
||||
x=0
|
||||
while [ -z ${CONTAINERID} ] && [ $x -lt 9 ]; do
|
||||
sleep 5
|
||||
|
||||
CONTAINERID=$(curl -vvv -H "Authorization: Bearer $KUBE_TOKEN" --cacert ${serviceaccount_path}/ca.crt \
|
||||
https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}/api/v1/namespaces/${POD_NAMESPACE}/pods/${KURYR_CNI_POD_NAME} | jq -r '.["status"]["containerStatuses"][0]["containerID"]')
|
||||
|
||||
CONTAINERID=${CONTAINERID#*//}
|
||||
((x++)) || true
|
||||
done;
|
||||
|
||||
# There's no point to run if we cannot get CONTAINERID.
|
||||
if [ -z ${CONTAINERID} ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Write the script to a file.
|
||||
cat > /kuryr-cni << EOF
|
||||
#!/bin/bash
|
||||
${CNI_BIN_DIR_PATH}/kuryr-venv/bin/kuryr-cni
|
||||
#!/bin/bash -x
|
||||
envs=(\$(env | grep ^CNI_))
|
||||
docker exec \${envs[@]/#/--env } -i "${CONTAINERID}" kuryr-cni --config-file /etc/kuryr/kuryr.conf
|
||||
EOF
|
||||
|
||||
# Copy the script into the designated location
|
||||
cp /kuryr-cni "/opt/cni/bin/kuryr-cni"
|
||||
chmod +x /opt/cni/bin/kuryr-cni
|
||||
cat /tmp/kuryr/* > /etc/kuryr/kuryr.conf
|
||||
cp /opt/kuryr-kubernetes/etc/cni/net.d/* /etc/cni/net.d
|
||||
}
|
||||
|
||||
cleanup
|
||||
@ -31,7 +48,7 @@ deploy
|
||||
|
||||
# Start CNI daemon if required
|
||||
if [ "$CNI_DAEMON" == "True" ]; then
|
||||
/kuryr-kubernetes/bin/kuryr-daemon --config-file /etc/kuryr/kuryr.conf
|
||||
kuryr-daemon --config-file /etc/kuryr/kuryr.conf
|
||||
else
|
||||
sleep infinity
|
||||
fi
|
||||
|
||||
@ -541,6 +541,10 @@ spec:
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: KURYR_CNI_POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
@ -549,10 +553,8 @@ spec:
|
||||
- name: net-conf
|
||||
mountPath: /etc/cni/net.d
|
||||
- name: config-volume
|
||||
mountPath: /tmp/kuryr/kuryr.conf
|
||||
mountPath: /etc/kuryr/kuryr.conf
|
||||
subPath: kuryr-cni.conf
|
||||
- name: etc
|
||||
mountPath: /etc
|
||||
- name: proc
|
||||
mountPath: /host_proc
|
||||
- name: openvswitch
|
||||
@ -565,13 +567,13 @@ EOF
|
||||
path: /ready
|
||||
port: ${cni_health_server_port}
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 15
|
||||
timeoutSeconds: 5
|
||||
initialDelaySeconds: 60
|
||||
timeoutSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /alive
|
||||
port: ${cni_health_server_port}
|
||||
initialDelaySeconds: 15
|
||||
initialDelaySeconds: 60
|
||||
EOF
|
||||
fi
|
||||
cat >> "${output_dir}/cni_ds.yml" << EOF
|
||||
@ -585,9 +587,6 @@ EOF
|
||||
- name: config-volume
|
||||
configMap:
|
||||
name: kuryr-config
|
||||
- name: etc
|
||||
hostPath:
|
||||
path: /etc
|
||||
- name: proc
|
||||
hostPath:
|
||||
path: /proc
|
||||
|
||||
@ -79,17 +79,17 @@ function configure_kuryr {
|
||||
# process.
|
||||
iniset "$KURYR_CONFIG" vif_plug_ovs_privileged helper_command privsep-helper
|
||||
iniset "$KURYR_CONFIG" vif_plug_linux_bridge_privileged helper_command privsep-helper
|
||||
|
||||
# When running kuryr-daemon or CNI in container we need to set up
|
||||
# some configs.
|
||||
iniset "$KURYR_CONFIG" cni_daemon docker_mode True
|
||||
iniset "$KURYR_CONFIG" cni_daemon netns_proc_dir "/host_proc"
|
||||
fi
|
||||
|
||||
if is_service_enabled kuryr-daemon; then
|
||||
iniset "$KURYR_CONFIG" oslo_concurrency lock_path "$KURYR_LOCK_DIR"
|
||||
create_kuryr_lock_dir
|
||||
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then
|
||||
# When running kuryr-daemon in container we need to set up some
|
||||
# configs.
|
||||
iniset "$KURYR_CONFIG" cni_daemon docker_mode True
|
||||
iniset "$KURYR_CONFIG" cni_daemon netns_proc_dir "/host_proc"
|
||||
else
|
||||
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "False" ]; then
|
||||
iniset "$KURYR_CONFIG" cni_health_server cg_path \
|
||||
"/system.slice/system-devstack.slice/devstack@kuryr-daemon.service"
|
||||
fi
|
||||
@ -132,20 +132,15 @@ function generate_containerized_kuryr_resources {
|
||||
inicomment "$KURYR_CONFIG" kubernetes ssl_client_crt_file
|
||||
inicomment "$KURYR_CONFIG" kubernetes ssl_client_key_file
|
||||
|
||||
# kuryr-controller and kuryr-cni will have tokens in different dirs.
|
||||
KURYR_CNI_CONFIG=${KURYR_CONFIG}-cni
|
||||
cp $KURYR_CONFIG $KURYR_CNI_CONFIG
|
||||
# NOTE(dulek): In the container the CA bundle will be mounted in a standard
|
||||
# directory, so we need to modify that.
|
||||
iniset "$KURYR_CONFIG" neutron cafile /etc/ssl/certs/kuryr-ca-bundle.crt
|
||||
iniset "$KURYR_CONFIG" kubernetes token_file /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
iniset "$KURYR_CONFIG" kubernetes ssl_ca_crt_file /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
iniset "$KURYR_CNI_CONFIG" kubernetes token_file /etc/kuryr/token
|
||||
iniset "$KURYR_CNI_CONFIG" kubernetes ssl_ca_crt_file /etc/kuryr/ca.crt
|
||||
|
||||
# Generate kuryr resources in k8s formats.
|
||||
local output_dir="${DATA_DIR}/kuryr-kubernetes"
|
||||
generate_kuryr_configmap $output_dir $KURYR_CONFIG $KURYR_CNI_CONFIG
|
||||
generate_kuryr_configmap $output_dir $KURYR_CONFIG $KURYR_CONFIG
|
||||
generate_kuryr_certificates_secret $output_dir $SSL_BUNDLE_FILE
|
||||
generate_kuryr_service_account $output_dir
|
||||
generate_controller_deployment $output_dir $KURYR_HEALTH_SERVER_PORT
|
||||
@ -743,14 +738,25 @@ if [[ "$1" == "stack" && "$2" == "extra" ]]; then
|
||||
build_kuryr_containers $CNI_BIN_DIR $CNI_CONF_DIR False
|
||||
generate_containerized_kuryr_resources False
|
||||
fi
|
||||
run_containerized_kuryr_resources
|
||||
fi
|
||||
fi
|
||||
|
||||
elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
|
||||
if is_service_enabled kuryr-kubernetes; then
|
||||
# NOTE(dulek): This is so late, because Devstack's Octavia is unable
|
||||
# to create loadbalancers until test-config phase.
|
||||
create_k8s_router_fake_service
|
||||
create_k8s_api_service
|
||||
|
||||
# FIXME(dulek): This is a very late phase to start Kuryr services.
|
||||
# We're doing it here because we need K8s API LB to be
|
||||
# created in order to run kuryr-cni container. Thing is
|
||||
# Octavia is unable to create LB until test-config phase.
|
||||
# We can revisit this once Octavia's DevStack plugin will
|
||||
# get improved.
|
||||
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then
|
||||
run_containerized_kuryr_resources
|
||||
fi
|
||||
fi
|
||||
if is_service_enabled tempest && [[ "$KURYR_USE_PORT_POOLS" == "True" ]]; then
|
||||
iniset $TEMPEST_CONFIG kuryr_kubernetes port_pool_enabled True
|
||||
@ -763,19 +769,17 @@ fi
|
||||
if [[ "$1" == "unstack" ]]; then
|
||||
KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT)
|
||||
if is_service_enabled kuryr-kubernetes; then
|
||||
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "False" ]; then
|
||||
stop_process kuryr-kubernetes
|
||||
else
|
||||
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then
|
||||
$KURYR_HYPERKUBE_BINARY kubectl delete deploy/kuryr-controller
|
||||
fi
|
||||
stop_process kuryr-kubernetes
|
||||
elif is_service_enabled kubelet; then
|
||||
$KURYR_HYPERKUBE_BINARY kubectl delete nodes ${HOSTNAME}
|
||||
fi
|
||||
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "False" ]; then
|
||||
stop_process kuryr-daemon
|
||||
else
|
||||
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then
|
||||
$KURYR_HYPERKUBE_BINARY kubectl delete ds/kuryr-cni-ds
|
||||
fi
|
||||
stop_process kuryr-daemon
|
||||
|
||||
if is_service_enabled kubernetes-controller-manager; then
|
||||
stop_container kubernetes-controller-manager
|
||||
|
||||
Loading…
Reference in New Issue
Block a user