Merge "Add OVN Kubernetes support in image build"

This commit is contained in:
Zuul 2024-11-25 18:54:31 +00:00 committed by Gerrit Code Review
commit 386d5e2394
7 changed files with 412 additions and 4 deletions

48
ovn/Dockerfile.centos Normal file
View File

@ -0,0 +1,48 @@
FROM golang:1.20 AS ovn-kubernetes
ARG OVN_KUBERNETES_REF=5359e7d7f872058b6e5bf884c9f19d1922451f29
ADD https://github.com/ovn-org/ovn-kubernetes.git#${OVN_KUBERNETES_REF} /src
COPY patches/ovn-kubernetes /patches/ovn-kubernetes
RUN git -C /src apply --verbose /patches/ovn-kubernetes/*
RUN <<EOF bash -xe
cd /src/go-controller
go build -o /usr/bin/ovn-kube-util ./cmd/ovn-kube-util
EOF
FROM quay.io/centos/centos:stream9
ADD --chmod=755 https://github.com/krallin/tini/releases/download/v0.19.0/tini /tini
ARG OVS_SERIES=3.2
ARG OVS_VERSION=${OVS_SERIES}.0-80
ARG TARGETPLATFORM
ADD --chmod=755 https://dl.k8s.io/release/v1.29.3/bin/${TARGETPLATFORM}/kubectl /usr/local/bin/kubectl
ARG OVN_SERIES=24.03
ARG OVN_VERSION=${OVN_SERIES}.1-44
RUN <<EOF sh -xe
dnf -y --setopt=install_weak_deps=False --setopt=tsflags=nodocs install \
centos-release-nfv-openvswitch.noarch
dnf -y --setopt=install_weak_deps=False --setopt=tsflags=nodocs install \
iptables \
openvswitch${OVS_SERIES}-${OVS_VERSION}.el9s \
openvswitch${OVS_SERIES}-test-${OVS_VERSION}.el9s \
python3-netifaces \
tcpdump
dnf -y clean all
rm -rf /var/cache/dnf
EOF
RUN <<EOF sh -xe
dnf -y --setopt=install_weak_deps=False --setopt=tsflags=nodocs install \
firewalld-filesystem hostname ovn${OVN_SERIES}-${OVN_VERSION}.el9s procps-ng
dnf -y clean all
rm -rf /var/cache/dnf
EOF
ARG OVN_COMPONENT
RUN <<EOF sh -xe
dnf -y --setopt=install_weak_deps=False --setopt=tsflags=nodocs install \
ovn${OVN_SERIES}-central-${OVN_VERSION}.el9s \
ovn${OVN_SERIES}-host-${OVN_VERSION}.el9s
dnf -y clean all
rm -rf /var/cache/dnf
EOF
COPY --from=ovn-kubernetes --link /src/dist/images/ovndb-raft-functions.sh /root/ovndb-raft-functions.sh
COPY --from=ovn-kubernetes --link /src/dist/images/ovnkube.sh /root/ovnkube.sh
COPY --from=ovn-kubernetes --link /usr/bin/ovn-kube-util /usr/bin/ovn-kube-util

View File

@ -1,9 +1,31 @@
ARG FROM
FROM ${FROM}
FROM golang:1.20 AS ovn-kubernetes
ARG OVN_KUBERNETES_REF=5359e7d7f872058b6e5bf884c9f19d1922451f29
ADD https://github.com/ovn-org/ovn-kubernetes.git#${OVN_KUBERNETES_REF} /src
COPY patches/ovn-kubernetes /patches/ovn-kubernetes
RUN git -C /src apply --verbose /patches/ovn-kubernetes/*
RUN <<EOF bash -xe
cd /src/go-controller
go build -o /usr/bin/ovn-kube-util ./cmd/ovn-kube-util
EOF
FROM ${FROM} AS base
ARG TARGETPLATFORM=linux/amd64
ADD --chmod=755 https://dl.k8s.io/release/v1.29.3/bin/${TARGETPLATFORM}/kubectl /usr/local/bin/kubectl
ADD --chmod=755 https://github.com/krallin/tini/releases/download/v0.19.0/tini /tini
RUN apt-get update && apt-get install -y \
ovn-host \
ovn-central \
openvswitch-switch \
firewalld \
hostname \
procps \
dnsutils \
iproute2 \
python3-netifaces \
iptables \
&& rm -rf /var/lib/apt/lists/*
COPY --from=ovn-kubernetes --link /src/dist/images/ovndb-raft-functions.sh /root/ovndb-raft-functions.sh
COPY --from=ovn-kubernetes --link /src/dist/images/ovnkube.sh /root/ovnkube.sh
COPY --from=ovn-kubernetes --link /usr/bin/ovn-kube-util /usr/bin/ovn-kube-util

View File

@ -5,10 +5,10 @@ SCRIPT_DIR=`dirname ${SCRIPT}`
cd ${SCRIPT_DIR}/..
IMAGE="ovn"
VERSION=${VERSION:-latest}
DISTRO=${DISTRO:-ubuntu}
VERSION=${VERSION:-stream9}
DISTRO=${DISTRO:-centos}
REGISTRY_URI=${REGISTRY_URI:-"openstackhelm/"}
EXTRA_TAG_INFO=${EXTRA_TAG_INFO:-""}
docker build -f ${IMAGE}/Dockerfile.${DISTRO} --build-arg FROM=${DISTRO/_/:} --network=host -t ${REGISTRY_URI}${IMAGE}:${VERSION}-${DISTRO}${EXTRA_TAG_INFO} ${extra_build_args} ${IMAGE}
DOCKER_BUILDKIT=1 docker build -f ${IMAGE}/Dockerfile.${DISTRO} --build-arg FROM=${DISTRO/_/:} --network=host -t ${REGISTRY_URI}${IMAGE}:${VERSION}-${DISTRO}${EXTRA_TAG_INFO} ${extra_build_args} ${IMAGE}
cd -

View File

@ -0,0 +1,161 @@
From 0227559bb404c3d7d5f32737809c38024bbabef0 Mon Sep 17 00:00:00 2001
From: Mohammed Naser <mnaser@vexxhost.com>
Date: Thu, 18 Jan 2024 12:42:24 -0500
Subject: [PATCH 1/2] chore: refactor to using OVN_KUBERNETES_STATEFULSET
---
dist/images/ovndb-raft-functions.sh | 32 ++++++++++++++---------------
dist/images/ovnkube.sh | 5 ++++-
2 files changed, 20 insertions(+), 17 deletions(-)
diff --git a/dist/images/ovndb-raft-functions.sh b/dist/images/ovndb-raft-functions.sh
index dceb4ec2e..65e9b6d9b 100644
--- a/dist/images/ovndb-raft-functions.sh
+++ b/dist/images/ovndb-raft-functions.sh
@@ -10,7 +10,7 @@ verify-ovsdb-raft() {
fi
replicas=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \
- get statefulset -n ${ovn_kubernetes_namespace} ovnkube-db -o=jsonpath='{.spec.replicas}')
+ get statefulset -n ${ovn_kubernetes_namespace} ${ovn_kubernetes_statefulset} -o=jsonpath='{.spec.replicas}')
if [[ ${replicas} -lt 3 || $((${replicas} % 2)) -eq 0 ]]; then
echo "at least 3 nodes need to be configured, and it must be odd number of nodes"
exit 1
@@ -45,14 +45,14 @@ db_part_of_cluster() {
}
# Checks if cluster has already been initialized.
-# If not it returns false and sets init_ip to ovnkube-db-0
+# If not it returns false and sets init_ip to ${ovn_kubernetes_statefulset}-0
cluster_exists() {
# See if ep is available ...
local db=${1}
local port=${2}
db_pods=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \
- get pod -n ${ovn_kubernetes_namespace} -o=jsonpath='{.items[*].metadata.name}' | egrep -o 'ovnkube-db[^ ]+')
+ get pod -n ${ovn_kubernetes_namespace} -o=jsonpath='{.items[*].metadata.name}' | egrep -o "${ovn_kubernetes_statefulset}[^ ]+")
for db_pod in $db_pods; do
if db_part_of_cluster $db_pod $db $port; then
@@ -63,7 +63,7 @@ cluster_exists() {
# if we get here there is no cluster, set init_ip and get out
init_ip="$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \
- get pod -n ${ovn_kubernetes_namespace} ovnkube-db-0 -o=jsonpath='{.status.podIP}')"
+ get pod -n ${ovn_kubernetes_namespace} ${ovn_kubernetes_statefulset}-0 -o=jsonpath='{.status.podIP}')"
if [[ $? != 0 ]]; then
return 1
fi
@@ -90,17 +90,17 @@ check_and_apply_ovnkube_db_ep() {
# return if ovn db service endpoint already exists
result=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \
- get ep -n ${ovn_kubernetes_namespace} ovnkube-db 2>&1)
+ get ep -n ${ovn_kubernetes_namespace} ${ovn_kubernetes_statefulset} 2>&1)
test $? -eq 0 && return
if ! echo ${result} | grep -q "NotFound"; then
- echo "Failed to find ovnkube-db endpoint: ${result}, Exiting..."
+ echo "Failed to find ${ovn_kubernetes_statefulset} endpoint: ${result}, Exiting..."
exit 12
fi
- # Get IPs of all ovnkube-db PODs
+ # Get IPs of all ${ovn_kubernetes_statefulset} PODs
ips=()
for ((i = 0; i < ${replicas}; i++)); do
ip=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \
- get pod -n ${ovn_kubernetes_namespace} ovnkube-db-${i} -o=jsonpath='{.status.podIP}')
+ get pod -n ${ovn_kubernetes_namespace} ${ovn_kubernetes_statefulset}-${i} -o=jsonpath='{.status.podIP}')
if [[ ${ip} == "" ]]; then
break
fi
@@ -108,7 +108,7 @@ check_and_apply_ovnkube_db_ep() {
done
if [[ ${i} -eq ${replicas} ]]; then
- # Number of POD IPs is same as number of statefulset replicas. Now, if the number of ovnkube-db endpoints
+ # Number of POD IPs is same as number of statefulset replicas. Now, if the number of ${ovn_kubernetes_statefulset} endpoints
# is 0, then we are applying the endpoint for the first time. So, we need to make sure that each of the
# pod IP responds to the `ovsdb-client list-dbs` call before we set the endpoint. If they don't, retry several
# times and then give up.
@@ -170,7 +170,7 @@ set_election_timer() {
return 0
}
-# set_connection() will be called for ovnkube-db-0 pod when :
+# set_connection() will be called for ${ovn_kubernetes_statefulset}-0 pod when :
# 1. it is first started or
# 2. it restarts after the initial start has failed or
# 3. subsequent restarts during the lifetime of the pod
@@ -307,7 +307,7 @@ ovsdb-raft() {
--ovn-${db}-log="${ovn_loglevel_db}" &
else
# either we need to initialize a new cluster or wait for db-0 to create it
- if [[ "${POD_NAME}" == "ovnkube-db-0" ]]; then
+ if [[ "${POD_NAME}" == "${ovn_kubernetes_statefulset}-0" ]]; then
echo "Cluster does not exist for DB: ${db}, creating new raft cluster"
run_as_ovs_user_if_needed \
${OVNCTL_PATH} run_${db}_ovsdb --no-monitor \
@@ -317,7 +317,7 @@ ovsdb-raft() {
${db_ssl_opts} \
--ovn-${db}-log="${ovn_loglevel_db}" &
else
- echo "Cluster does not exist for DB: ${db}, waiting for ovnkube-db-0 pod to create it"
+ echo "Cluster does not exist for DB: ${db}, waiting for ${ovn_kubernetes_statefulset}-0 pod to create it"
# all non pod-0 pods will be blocked here till connection is set
wait_for_event cluster_exists ${db} ${port}
run_as_ovs_user_if_needed \
@@ -356,8 +356,8 @@ ovsdb-raft() {
fi
echo "=============== ${db}-ovsdb-raft ========== RUNNING"
- if [[ "${POD_NAME}" == "ovnkube-db-0" ]]; then
- # post raft create work has to be done only once and in ovnkube-db-0 while it is still
+ if [[ "${POD_NAME}" == "${ovn_kubernetes_statefulset}-0" ]]; then
+ # post raft create work has to be done only once and in ${ovn_kubernetes_statefulset}-0 while it is still
# a single-node cluster, additional protection against the case when pod-0 isn't a leader
# is needed in the cases of sudden pod-0 initialization logic restarts
current_raft_role=$(ovs-appctl -t ${OVN_RUNDIR}/ovn${db}_db.ctl cluster/status ${database} 2>&1 | grep "^Role")
@@ -381,9 +381,9 @@ ovsdb-raft() {
fi
last_node_index=$(expr ${replicas} - 1)
- # Create endpoints only if all ovnkube-db pods have started and are running. We do this
+ # Create endpoints only if all ${ovn_kubernetes_statefulset} pods have started and are running. We do this
# from the last pod of the statefulset.
- if [[ ${db} == "nb" && "${POD_NAME}" == "ovnkube-db-"${last_node_index} ]]; then
+ if [[ ${db} == "nb" && "${POD_NAME}" == "${ovn_kubernetes_statefulset}-"${last_node_index} ]]; then
check_and_apply_ovnkube_db_ep ${port}
fi
diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh
index bf3989e37..720b3e14d 100755
--- a/dist/images/ovnkube.sh
+++ b/dist/images/ovnkube.sh
@@ -40,6 +40,7 @@ fi
# OVN_NET_CIDR - the network cidr - v3
# OVN_SVC_CIDR - the cluster-service-cidr - v3
# OVN_KUBERNETES_NAMESPACE - k8s namespace - v3
+# OVN_KUBERNETES_STATEFULSET - k8s statefulset - v3
# K8S_NODE - hostname of the node - v3
#
# OVN_DAEMONSET_VERSION - version match daemonset and image - v3
@@ -198,12 +199,14 @@ metrics_bind_port=${OVN_METRICS_BIND_PORT:-9476}
metrics_exporter_port=${OVN_METRICS_EXPORTER_PORT:-9310}
ovn_kubernetes_namespace=${OVN_KUBERNETES_NAMESPACE:-ovn-kubernetes}
+ovn_kubernetes_statefulset=${OVN_KUBERNETES_STATEFULSET:-ovnkube-db}
+
# namespace used for classifying host network traffic
ovn_host_network_namespace=${OVN_HOST_NETWORK_NAMESPACE:-ovn-host-network}
# host on which ovnkube-db POD is running and this POD contains both
# OVN NB and SB DB running in their own container.
-ovn_db_host=${K8S_NODE_IP:-""}
+ovn_db_host=${K8S_NODE_IP:-$(hostname -f)}
# OVN_NB_PORT - ovn north db port (default 6641)
ovn_nb_port=${OVN_NB_PORT:-6641}
--
2.42.0

View File

@ -0,0 +1,144 @@
From 90851bd77718bc834446ebe2ddf34b8e9383dee8 Mon Sep 17 00:00:00 2001
From: Mohammed Naser <mnaser@vexxhost.com>
Date: Thu, 18 Jan 2024 16:16:11 -0500
Subject: [PATCH 2/2] chore(northd): refactor to being able to use split svcs
---
dist/images/ovnkube.sh | 75 +++++++++++++++++++++++++++++++++---------
1 file changed, 59 insertions(+), 16 deletions(-)
diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh
index 720b3e14d..1d3059cf8 100755
--- a/dist/images/ovnkube.sh
+++ b/dist/images/ovnkube.sh
@@ -200,6 +200,8 @@ metrics_exporter_port=${OVN_METRICS_EXPORTER_PORT:-9310}
ovn_kubernetes_namespace=${OVN_KUBERNETES_NAMESPACE:-ovn-kubernetes}
ovn_kubernetes_statefulset=${OVN_KUBERNETES_STATEFULSET:-ovnkube-db}
+ovn_kubernetes_nb_statefulset=${OVN_KUBERNETES_NB_STATEFULSET:-ovnkube-db}
+ovn_kubernetes_sb_statefulset=${OVN_KUBERNETES_SB_STATEFULSET:-ovnkube-db}
# namespace used for classifying host network traffic
ovn_host_network_namespace=${OVN_HOST_NETWORK_NAMESPACE:-ovn-host-network}
@@ -374,6 +376,24 @@ wait_for_event() {
done
}
+wait_for_db () {
+ local db=$1
+ local ep=$(get_ovnkube_zone_db_ep ${db})
+
+ echo "Getting the ${ep} ep"
+ # See if ep is available ...
+ IFS=" " read -a ep_hosts <<<"$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \
+ get endpointslice -n ${ovn_kubernetes_namespace} -l kubernetes.io/service-name=${ep} -o=jsonpath='{range .items[0].endpoints[*]}{.addresses[0]} ')"
+ if [[ ${#ep_hosts[@]} == 0 ]]; then
+ return 1
+ fi
+
+ ep_hosts_string="${ep_hosts[*]}"
+ declare -g -a "ovn_${db}db_hosts=($ep_hosts_string)"
+
+ return 0
+}
+
# The ovnkube-db kubernetes service must be populated with OVN DB service endpoints
# before various OVN K8s containers can come up. This functions checks for that.
# If OVN dbs are configured to listen only on unix sockets, then there will not be
@@ -384,15 +404,18 @@ ready_to_start_node() {
return 0
fi
- ovnkube_db_ep=$(get_ovnkube_zone_db_ep)
- echo "Getting the ${ovnkube_db_ep} ep"
- # See if ep is available ...
- IFS=" " read -a ovn_db_hosts <<<"$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \
- get ep -n ${ovn_kubernetes_namespace} ${ovnkube_db_ep} -o=jsonpath='{range .subsets[0].addresses[*]}{.ip}{" "}')"
- if [[ ${#ovn_db_hosts[@]} == 0 ]]; then
+ wait_for_db nb
+ if [[ $? != 0 ]]; then
+ return 1
+ fi
+
+ wait_for_db sb
+ if [[ $? != 0 ]]; then
return 1
fi
+
get_ovn_db_vars
+
return 0
}
# wait_for_event ready_to_start_node
@@ -410,17 +433,29 @@ check_ovn_daemonset_version() {
}
get_ovn_db_vars() {
+
+ index=0
ovn_nbdb_str=""
- ovn_sbdb_str=""
- for i in "${ovn_db_hosts[@]}"; do
+ for i in "${ovn_nbdb_hosts[@]}"; do
if [ -n "$ovn_nbdb_str" ]; then
ovn_nbdb_str=${ovn_nbdb_str}","
+ fi
+ host="${ovn_kubernetes_nb_statefulset}-${index}.${ovn_kubernetes_nb_statefulset}.${ovn_kubernetes_namespace}.svc.cluster.local"
+ ovn_nbdb_str=${ovn_nbdb_str}${transport}://${host}:${ovn_nb_port}
+ index=$((index + 1))
+ done
+
+ index=0
+ ovn_sbdb_str=""
+ for i in "${ovn_sbdb_hosts[@]}"; do
+ if [ -n "$ovn_sbdb_str" ]; then
ovn_sbdb_str=${ovn_sbdb_str}","
fi
- ip=$(bracketify $i)
- ovn_nbdb_str=${ovn_nbdb_str}${transport}://${ip}:${ovn_nb_port}
- ovn_sbdb_str=${ovn_sbdb_str}${transport}://${ip}:${ovn_sb_port}
+ host="${ovn_kubernetes_sb_statefulset}-${index}.${ovn_kubernetes_sb_statefulset}.${ovn_kubernetes_namespace}.svc.cluster.local"
+ ovn_sbdb_str=${ovn_sbdb_str}${transport}://${host}:${ovn_sb_port}
+ index=$((index + 1))
done
+
# OVN_NORTH and OVN_SOUTH override derived host
ovn_nbdb=${OVN_NORTH:-$ovn_nbdb_str}
ovn_sbdb=${OVN_SOUTH:-$ovn_sbdb_str}
@@ -730,7 +765,7 @@ set_ovnkube_db_ep() {
ips=("$@")
ovn_zone=$(get_node_zone)
- ovnkube_db_ep=$(get_ovnkube_zone_db_ep)
+ ovnkube_db_ep=$(get_ovnkube_zone_db_ep sb)
echo "=============== setting ${ovnkube_db_ep} endpoints to ${ips[@]}"
# create a new endpoint for the headless onvkube-db service without selectors
kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} apply -f - <<EOF
@@ -779,12 +814,20 @@ function get_node_zone() {
}
function get_ovnkube_zone_db_ep() {
- zone=$(get_node_zone)
- if [ "$zone" == "global" ]; then
- echo "ovnkube-db"
+ local db=$1
+
+ if [ "$db" == "nb" ]; then
+ ep="${ovn_kubernetes_nb_statefulset}"
else
- echo "ovnkube-db-$zone"
+ ep="${ovn_kubernetes_sb_statefulset}"
fi
+
+ zone=$(get_node_zone)
+ if [ "$zone" != "global" ]; then
+ ep="${ep}-${zone}"
+ fi
+
+ echo "${ep}"
}
# v3 - run nb_ovsdb in a separate container
--
2.42.0

View File

@ -0,0 +1,26 @@
From 71b19667c04fd114e49d3fda957e26e7fa595675 Mon Sep 17 00:00:00 2001
From: Mohammed Naser <mnaser@vexxhost.com>
Date: Sun, 4 Feb 2024 14:15:49 -0500
Subject: [PATCH] stop creating ovnkube eps
---
dist/images/ovnkube.sh | 3 ---
1 file changed, 3 deletions(-)
diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh
index 1d3059cf8..069d36648 100755
--- a/dist/images/ovnkube.sh
+++ b/dist/images/ovnkube.sh
@@ -908,9 +908,6 @@ sb-ovsdb() {
}
ovn-sbctl --inactivity-probe=0 set-connection p${transport}:${ovn_sb_port}:$(bracketify ${ovn_db_host})
- # create the ovnkube-db endpoints
- wait_for_event attempts=10 check_ovnkube_db_ep ${ovn_db_host} ${ovn_nb_port}
- set_ovnkube_db_ep ${ovn_db_host}
if memory_trim_on_compaction_supported "sbdb"
then
# Enable SBDB memory trimming on DB compaction, Every 10mins DBs are compacted
--
2.43.0

View File

@ -53,6 +53,13 @@
- "ubuntu_jammy-{{ currentdate }}"
build_args:
- FROM=ubuntu:jammy
- context: ovn
repository: openstackhelm/ovn
dockerfile: Dockerfile.centos
tags:
- centos_stream9
- "centos_stream9-{{ currentdate }}"
docker_use_buildkit: true
files: &ovn_files
- ovn/.*
- zuul.d/ovn.yaml