DevStack: Support dual stack

This commit makes sure DevStack plugin is able to run with dual stack
and create 2 subnetpools, 2 service subnets and 2 pod subnets. The K8s
is also configured with that.

Change-Id: I9c53bc4dd3529a48f5ba1ab77268d6a984a84808
This commit is contained in:
Michał Dulko 2021-02-16 09:54:39 +01:00
parent ed018fd688
commit daddd7a77b
5 changed files with 197 additions and 168 deletions

View File

@ -72,6 +72,17 @@
KURYR_IPV6: true
voting: false
- job:
name: kuryr-kubernetes-tempest-containerized-dual-stack
parent: kuryr-kubernetes-tempest-containerized
description: |
Kuryr-Kubernetes tempest job running kuryr containerized with dual stack
pod and service networks
vars:
devstack_localrc:
KURYR_DUAL_STACK: true
voting: false
- job:
name: kuryr-kubernetes-tempest-containerized-lower-constraints
parent: kuryr-kubernetes-tempest-containerized

View File

@ -43,6 +43,7 @@
- kuryr-kubernetes-tempest-openshift-multi-vif
- kuryr-kubernetes-tempest-multinode-ha
- kuryr-kubernetes-tempest-containerized-crio
- kuryr-kubernetes-tempest-containerized-dual-stack
- project-template:
name: kuryr-kubernetes-lower-constraints-bionic-jobs

View File

@ -36,27 +36,18 @@ function ovs_bind_for_kubelet() {
--device-owner compute:kuryr \
--project "$project_id" \
--security-group "$security_group" \
--security-group service_pod_access \
--host "${HOSTNAME}" \
--network "${KURYR_NEUTRON_DEFAULT_POD_NET}" \
-f value -c id \
kubelet-"${HOSTNAME}")
# Need to enable Amphorae subnet access to the kubelet iface for API
# access
openstack port set "$port_id" --security-group service_pod_access
ifname="kubelet${port_id}"
ifname="${ifname:0:14}"
service_subnet_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
subnet show "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" \
-c cidr -f value)
pod_subnet_gw=$(openstack subnet show "$KURYR_NEUTRON_DEFAULT_POD_SUBNET" \
-c gateway_ip -f value)
port_mac=$(openstack port show "$port_id" -c mac_address -f value)
fixed_ips=$(openstack port show "$port_id" -f value -c fixed_ips)
port_ips=($(python3 -c "print(${fixed_ips}[0]['ip_address'])"))
port_subnets=($(python3 -c "print(${fixed_ips}[0]['subnet_id'])"))
port_ips=($(python3 -c "print(' '.join([x['ip_address'] for x in ${fixed_ips}]))"))
port_subnets=($(python3 -c "print(' '.join([x['subnet_id'] for x in ${fixed_ips}]))"))
sudo ovs-vsctl -- --may-exist add-port $OVS_BRIDGE "$ifname" \
-- set Interface "$ifname" type=internal \
@ -66,36 +57,49 @@ function ovs_bind_for_kubelet() {
sudo ip link set dev "$ifname" address "$port_mac"
sudo ip link set dev "$ifname" up
for ((i=0; i < ${#port_ips[@]}; i++)); do
for i in "${!port_ips[@]}"; do
prefix=$(openstack subnet show "${port_subnets[$i]}" \
-c cidr -f value | \
cut -f2 -d/)
sudo ip addr add "${port_ips[$i]}/${prefix}" dev "$ifname"
done
if [[ "$KURYR_SUBNET_DRIVER" == "namespace" ]]; then
if [ "$KURYR_IPV6" == "False" ]; then
subnetpool_id=${KURYR_NEUTRON_DEFAULT_SUBNETPOOL_ID:-${SUBNETPOOL_V4_ID}}
else
subnetpool_id=${KURYR_NEUTRON_DEFAULT_SUBNETPOOL_ID:-${SUBNETPOOL_KURYR_V6_ID}}
fi
cidrs=$(openstack subnet pool show "${subnetpool_id}" -c prefixes -f value)
subnetpool_cidr=$(python3 -c "print(${cidrs}[0])")
sudo ip route add "$subnetpool_cidr" via "$pod_subnet_gw" dev "$ifname"
else
sudo ip route add "$service_subnet_cidr" via "$pod_subnet_gw" dev "$ifname"
# TODO(dulek): This hack is for compatibility with multinode job, we might
# want to do it better one day and actually support dual stack
# and NP here.
if [[ -z ${KURYR_SERVICE_SUBNETS_IDS} ]]; then
KURYR_SERVICE_SUBNETS_IDS=(${KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET}-IPv4)
KURYR_POD_SUBNETS_IDS=(${KURYR_NEUTRON_DEFAULT_POD_SUBNET}-IPv4)
fi
for i in "${!KURYR_SERVICE_SUBNETS_IDS[@]}"; do
pod_subnet_gw=$(openstack subnet show "${KURYR_POD_SUBNETS_IDS[$i]}" \
-c gateway_ip -f value)
if [[ "$KURYR_SUBNET_DRIVER" == "namespace" ]]; then
cidrs=$(openstack subnet pool show "${KURYR_SUBNETPOOLS_IDS[$i]}" -c prefixes -f value)
subnetpool_cidr=$(python3 -c "print(${cidrs}[0])")
sudo ip route add "$subnetpool_cidr" via "$pod_subnet_gw" dev "$ifname"
else
service_subnet_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
subnet show "${KURYR_SERVICE_SUBNETS_IDS[$i]}" \
-c cidr -f value)
sudo ip route add "$service_subnet_cidr" via "$pod_subnet_gw" dev "$ifname"
fi
done
if [ -n "$port_number" ]; then
# if openstack-INPUT chain doesn't exist we create it in INPUT (for
# local development envs since openstack-INPUT is usually only in gates)
if [ "$KURYR_IPV6" == "False" ]; then
if [[ "$KURYR_IPV6" == "False" || "$KURYR_DUAL_STACK" == "True" ]]; then
sudo iptables -I openstack-INPUT 1 \
-p tcp -s 0.0.0.0/0 -d 0.0.0.0/0 --dport $port_number -j ACCEPT || \
sudo iptables -I INPUT 1 \
-p tcp -m conntrack --ctstate NEW \
-m tcp --dport "$port_number" \
-m comment --comment "kuryr-devstack: Access to OpenShift API" -j ACCEPT
else
fi
if [[ "$KURYR_IPV6" == "True" || "$KURYR_DUAL_STACK" == "True" ]]; then
sudo ip6tables -I openstack-INPUT 1 \
-p tcp -s ::/0 -d ::/0 --dport $port_number -j ACCEPT || \
sudo ip6tables -I INPUT 1 \
@ -203,13 +207,15 @@ function create_k8s_icmp_sg_rules {
project_id=$(get_or_create_project \
"$KURYR_NEUTRON_DEFAULT_PROJECT" default)
icmp_sg_rules=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
security group rule create \
--project "$project_id" \
--protocol icmp \
--ethertype "$KURYR_ETHERTYPE" \
--"$direction" "$sg_id")
for ethertype in ${KURYR_ETHERTYPES[@]}; do
icmp_sg_rules=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
security group rule create \
--project "$project_id" \
--protocol icmp \
--ethertype "$ethertype" \
--"$direction" "$sg_id")
done
die_if_not_set $LINENO icmp_sg_rules \
"Failure creating icmp sg ${direction} rule for ${sg_id}"
}
@ -218,15 +224,16 @@ function create_k8s_icmp_sg_rules {
# Description: Creates a network and subnet for Kuryr-Kubernetes usage
# Params:
# project_id - Kuryr's project uuid
# net_name - Name of the network to create
# net_id - ID of the network where to create subnet in
# subnet_name - Name of the subnet to create
# subnetpool_id - uuid of the subnet pool to use
# router - name of the router to plug the subnet to
# split_allocation - Whether to allocate on all the subnet or only the
# latter half
# ip_version - IPv4 or IPv6
function create_k8s_subnet {
local project_id=$1
local net_name="$2"
local net_id="$2"
local subnet_name="$3"
local subnetpool_id="$4"
local router="$5"
@ -235,8 +242,9 @@ function create_k8s_subnet {
local split_allocation
split_allocation="${6:-False}"
local ip_version="${7:-IPv4}"
if [ "$KURYR_IPV6" == "False" ]; then
if [ "$ip_version" == "IPv4" ]; then
subnet_params+="--ip-version 4 "
else
# NOTE(dulek): K8s API won't accept subnets bigger than 20 bits.
@ -245,13 +253,6 @@ function create_k8s_subnet {
fi
subnet_params+="--no-dhcp --gateway none "
subnet_params+="--subnet-pool $subnetpool_id "
local net_id
net_id=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
network create --project "$project_id" \
"$net_name" \
-c id -f value)
subnet_params+="--network $net_id $subnet_name"
local subnet_id
@ -302,6 +303,7 @@ function create_k8s_subnet {
router add subnet "$router" "$subnet_id" \
|| die $LINENO \
"Failed to enable routing for K8s ${subnet_name} subnet"
echo $subnet_id
}
# create_k8s_fake_service

View File

@ -240,9 +240,12 @@ function create_k8s_api_service {
project_id=$(get_or_create_project \
"$KURYR_NEUTRON_DEFAULT_PROJECT" default)
lb_name='default/kubernetes'
# TODO(dulek): We only look at the first service subnet because kubernetes
# API service is only IPv4 in 1.20. It might be dual stack
# in the future.
service_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
subnet show "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" \
subnet show "${KURYR_SERVICE_SUBNETS_IDS[0]}" \
-c cidr -f value)
fixed_ips=$(openstack port show kubelet-"${HOSTNAME}" -c fixed_ips -f value)
@ -250,7 +253,7 @@ function create_k8s_api_service {
k8s_api_clusterip=$(_cidr_range "$service_cidr" | cut -f1)
create_load_balancer "$lb_name" "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET"\
create_load_balancer "$lb_name" "${KURYR_SERVICE_SUBNETS_IDS[0]}" \
"$project_id" "$k8s_api_clusterip"
create_load_balancer_listener default/kubernetes:${KURYR_K8S_API_LB_PORT} HTTPS ${KURYR_K8S_API_LB_PORT} "$lb_name" "$project_id" 3600000
create_load_balancer_pool default/kubernetes:${KURYR_K8S_API_LB_PORT} HTTPS ROUND_ROBIN \
@ -279,10 +282,7 @@ function create_k8s_api_service {
function configure_neutron_defaults {
local project_id
local pod_subnet_id
local sg_ids
local service_subnet_id
local subnetpool_id
local router
local router_id
local ext_svc_net_id
@ -297,11 +297,18 @@ function configure_neutron_defaults {
# If a subnetpool is not passed, we get the one created in devstack's
# Neutron module
KURYR_IPV6=$(trueorfalse False KURYR_IPV6)
if [ "$KURYR_IPV6" == "False" ]; then
KURYR_DUAL_STACK=$(trueorfalse False KURYR_DUAL_STACK)
export KURYR_SUBNETPOOLS_IDS=()
export KURYR_ETHERTYPES=()
if [[ "$KURYR_IPV6" == "False" || "$KURYR_DUAL_STACK" == "True" ]]; then
export KURYR_ETHERTYPE=IPv4
subnetpool_id=${KURYR_NEUTRON_DEFAULT_SUBNETPOOL_ID:-${SUBNETPOOL_V4_ID}}
else
KURYR_ETHERTYPES+=("IPv4")
KURYR_SUBNETPOOLS_IDS+=(${KURYR_NEUTRON_DEFAULT_SUBNETPOOL_ID:-${SUBNETPOOL_V4_ID}})
fi
if [[ "$KURYR_IPV6" == "True" || "$KURYR_DUAL_STACK" == "True" ]]; then
export KURYR_ETHERTYPE=IPv6
KURYR_ETHERTYPES+=("IPv6")
# NOTE(gryf): To not clash with subnets created by DevStack for IPv6,
# we create another subnetpool just for kuryr subnets.
# SUBNETPOOL_KURYR_V6_ID will be used in function configure_kuryr in
@ -314,15 +321,13 @@ function configure_neutron_defaults {
sed -e "s/\(..\)\(....\)\(....\)/\1:\2:\3/")
addrs_prefix="fd${IPV6_ID}::/56"
subnetpool_name=${SUBNETPOOL_KURYR_NAME_V6}
SUBNETPOOL_KURYR_V6_ID=$(openstack \
KURYR_SUBNETPOOLS_IDS+=($(openstack \
--os-cloud devstack-admin \
--os-region "${REGION_NAME}" \
subnet pool create "${subnetpool_name}" \
--default-prefix-length "${SUBNETPOOL_SIZE_V6}" \
--pool-prefix "${addrs_prefix}" \
--share -f value -c id)
export SUBNETPOOL_KURYR_V6_ID
subnetpool_id=${KURYR_NEUTRON_DEFAULT_SUBNETPOOL_ID:-${SUBNETPOOL_KURYR_V6_ID}}
--share -f value -c id))
fi
router=${KURYR_NEUTRON_DEFAULT_ROUTER:-$Q_ROUTER_NAME}
@ -332,29 +337,39 @@ function configure_neutron_defaults {
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
router set --external-gateway "$ext_svc_net_id" "$router"
fi
router_id="$(openstack router show -c id -f value \
"$router")"
router_id="$(openstack router show -c id -f value "$router")"
create_k8s_subnet "$project_id" \
"$KURYR_NEUTRON_DEFAULT_POD_NET" \
"$KURYR_NEUTRON_DEFAULT_POD_SUBNET" \
"$subnetpool_id" \
"$router"
pod_subnet_id="$(openstack subnet show -c id -f value \
"${KURYR_NEUTRON_DEFAULT_POD_SUBNET}")"
pod_net_id=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
network create --project "$project_id" \
"$KURYR_NEUTRON_DEFAULT_POD_NET" \
-c id -f value)
service_net_id=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
network create --project "$project_id" \
"$KURYR_NEUTRON_DEFAULT_SERVICE_NET" \
-c id -f value)
create_k8s_subnet "$project_id" \
"$KURYR_NEUTRON_DEFAULT_SERVICE_NET" \
"$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" \
"$subnetpool_id" \
"$router" \
"True"
service_subnet_id="$(openstack subnet show -c id -f value \
"${KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET}")"
export KURYR_POD_SUBNETS_IDS=()
export KURYR_SERVICE_SUBNETS_IDS=()
for i in "${!KURYR_SUBNETPOOLS_IDS[@]}"; do
KURYR_POD_SUBNETS_IDS+=($(create_k8s_subnet "$project_id" \
"$pod_net_id" \
"${KURYR_NEUTRON_DEFAULT_POD_SUBNET}-${KURYR_ETHERTYPES[$i]}" \
"${KURYR_SUBNETPOOLS_IDS[$i]}" \
"$router" "False" ${KURYR_ETHERTYPES[$i]}))
KURYR_SERVICE_SUBNETS_IDS+=($(create_k8s_subnet "$project_id" \
"$service_net_id" \
"${KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET}-${KURYR_ETHERTYPES[$i]}" \
"${KURYR_SUBNETPOOLS_IDS[$i]}" \
"$router" "True" ${KURYR_ETHERTYPES[$i]}))
done
sg_ids=()
if [[ "$KURYR_SG_DRIVER" == "default" ]]; then
sg_ids=$(echo $(openstack security group list \
--project "$project_id" -c ID -f value) | tr ' ' ',')
sg_ids+=($(echo $(openstack security group list \
--project "$project_id" -c ID -f value) | tr ' ' ','))
fi
# In order for the ports to allow service traffic under Octavia L3 mode,
@ -362,40 +377,39 @@ function configure_neutron_defaults {
# security groups. If L3 is used, then the pods created will include it.
# Otherwise it will be just used by the kubelet port used for the K8s API
# load balancer
local service_cidr
local service_pod_access_sg_id
service_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" subnet show \
"${KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET}" -f value -c cidr)
service_pod_access_sg_id=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
security group create --project "$project_id" \
service_pod_access -f value -c id)
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s service subnet allowed" \
--remote-ip "$service_cidr" --ethertype "$KURYR_ETHERTYPE" --protocol tcp \
"$service_pod_access_sg_id"
# Since Octavia supports also UDP load balancing, we need to allow
# also udp traffic
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s service subnet UDP allowed" \
--remote-ip "$service_cidr" --ethertype "$KURYR_ETHERTYPE" --protocol udp \
"$service_pod_access_sg_id"
# Octavia supports SCTP load balancing, we need to also allow SCTP traffic
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s service subnet SCTP allowed" \
--remote-ip "$service_cidr" --ethertype "$KURYR_ETHERTYPE" --protocol sctp \
"$service_pod_access_sg_id"
for i in "${!KURYR_SERVICE_SUBNETS_IDS[@]}"; do
local service_cidr
service_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" subnet show \
"${KURYR_SERVICE_SUBNETS_IDS[$i]}" -f value -c cidr)
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s service subnet allowed" \
--remote-ip "$service_cidr" --ethertype "${KURYR_ETHERTYPES[$i]}" --protocol tcp \
"$service_pod_access_sg_id"
# Since Octavia supports also UDP load balancing, we need to allow
# also udp traffic
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s service subnet UDP allowed" \
--remote-ip "$service_cidr" --ethertype "${KURYR_ETHERTYPES[$i]}" --protocol udp \
"$service_pod_access_sg_id"
# Octavia supports SCTP load balancing, we need to also allow SCTP traffic
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s service subnet SCTP allowed" \
--remote-ip "$service_cidr" --ethertype "${KURYR_ETHERTYPES[$i]}" --protocol sctp \
"$service_pod_access_sg_id"
done
if [[ "$KURYR_K8S_OCTAVIA_MEMBER_MODE" == "L3" ]]; then
if [ -n "$sg_ids" ]; then
sg_ids+=",${service_pod_access_sg_id}"
else
sg_ids="${service_pod_access_sg_id}"
fi
sg_ids+=(${service_pod_access_sg_id})
elif [[ "$KURYR_K8S_OCTAVIA_MEMBER_MODE" == "L2" ]]; then
# In case the member connectivity is L2, Octavia by default uses the
# admin 'default' sg to create a port for the amphora load balancer
@ -403,45 +417,46 @@ function configure_neutron_defaults {
# between the member ports and the octavia ports by allowing all
# access from the pod subnet range to the ports in that subnet, and
# include it into $sg_ids
local pod_cidr
local pod_pod_access_sg_id
pod_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" subnet show \
"${KURYR_NEUTRON_DEFAULT_POD_SUBNET}" -f value -c cidr)
local octavia_pod_access_sg_id
octavia_pod_access_sg_id=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
security group create --project "$project_id" \
octavia_pod_access -f value -c id)
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s pod subnet allowed from k8s-pod-subnet" \
--remote-ip "$pod_cidr" --ethertype "$KURYR_ETHERTYPE" --protocol tcp \
"$octavia_pod_access_sg_id"
# Since Octavia supports also UDP load balancing, we need to allow
# also udp traffic
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s pod subnet allowed from k8s-pod-subnet" \
--remote-ip "$pod_cidr" --ethertype "$KURYR_ETHERTYPE" --protocol udp \
"$octavia_pod_access_sg_id"
# Octavia supports SCTP load balancing, we need to also support SCTP traffic
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s pod subnet allowed from k8s-pod-subnet" \
--remote-ip "$pod_cidr" --ethertype "$KURYR_ETHERTYPE" --protocol sctp \
"$octavia_pod_access_sg_id"
if [ -n "$sg_ids" ]; then
sg_ids+=",${octavia_pod_access_sg_id}"
else
sg_ids="${octavia_pod_access_sg_id}"
fi
for i in "${!KURYR_POD_SUBNETS_IDS[@]}"; do
local pod_cidr
pod_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" subnet show \
"${KURYR_POD_SUBNETS_IDS[$i]}" -f value -c cidr)
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s pod subnet allowed from k8s-pod-subnet" \
--remote-ip "$pod_cidr" --ethertype "${KURYR_ETHERTYPES[$i]}" --protocol tcp \
"$octavia_pod_access_sg_id"
# Since Octavia supports also UDP load balancing, we need to allow
# also udp traffic
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s pod subnet allowed from k8s-pod-subnet" \
--remote-ip "$pod_cidr" --ethertype "${KURYR_ETHERTYPES[$i]}" --protocol udp \
"$octavia_pod_access_sg_id"
# Octavia supports SCTP load balancing, we need to also support SCTP traffic
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s pod subnet allowed from k8s-pod-subnet" \
--remote-ip "$pod_cidr" --ethertype "${KURYR_ETHERTYPES[$i]}" --protocol sctp \
"$octavia_pod_access_sg_id"
done
sg_ids+=(${octavia_pod_access_sg_id})
fi
iniset "$KURYR_CONFIG" neutron_defaults project "$project_id"
iniset "$KURYR_CONFIG" neutron_defaults pod_subnet "$pod_subnet_id"
iniset "$KURYR_CONFIG" neutron_defaults service_subnet "$service_subnet_id"
iniset "$KURYR_CONFIG" neutron_defaults pod_subnet "${KURYR_POD_SUBNETS_IDS[0]}"
iniset "$KURYR_CONFIG" neutron_defaults pod_subnets $(IFS=, ; echo "${KURYR_POD_SUBNETS_IDS[*]}")
iniset "$KURYR_CONFIG" neutron_defaults service_subnet "${KURYR_SERVICE_SUBNETS_IDS[0]}"
iniset "$KURYR_CONFIG" neutron_defaults service_subnets $(IFS=, ; echo "${KURYR_SERVICE_SUBNETS_IDS[*]}")
if [ "$KURYR_SUBNET_DRIVER" == "namespace" ]; then
iniset "$KURYR_CONFIG" namespace_subnet pod_subnet_pool "$subnetpool_id"
iniset "$KURYR_CONFIG" namespace_subnet pod_subnet_pool "${KURYR_SUBNETPOOLS_IDS[0]}"
iniset "$KURYR_CONFIG" namespace_subnet pod_subnet_pools $(IFS=, ; echo "${KURYR_SUBNETPOOLS_IDS[*]}")
iniset "$KURYR_CONFIG" namespace_subnet pod_router "$router_id"
fi
if [[ "$KURYR_SG_DRIVER" == "policy" ]]; then
@ -452,18 +467,16 @@ function configure_neutron_defaults {
--os-region "$REGION_NAME" \
security group create --project "$project_id" \
allow-all -f value -c id)
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "allow all ingress traffic" \
--ethertype "$KURYR_ETHERTYPE" --ingress --protocol any \
"$allow_all_sg_id"
if [ -n "$sg_ids" ]; then
sg_ids+=",${allow_all_sg_id}"
else
sg_ids="${allow_all_sg_id}"
fi
for ethertype in ${KURYR_ETHERTYPES[@]}; do
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "allow all ingress traffic" \
--ethertype "$ethertype" --ingress --protocol any \
"$allow_all_sg_id"
done
sg_ids+=(${allow_all_sg_id})
fi
iniset "$KURYR_CONFIG" neutron_defaults pod_security_groups "$sg_ids"
iniset "$KURYR_CONFIG" neutron_defaults pod_security_groups $(IFS=, ; echo "${sg_ids[*]}")
if [[ "$KURYR_SG_DRIVER" == "policy" ]]; then
# NOTE(ltomasbo): As more security groups and rules are created, there
@ -520,7 +533,7 @@ function prepare_kubernetes_files {
service_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
subnet show "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET"\
subnet show "${KURYR_SERVICE_SUBNETS_IDS[0]}"\
-c cidr -f value)
k8s_api_clusterip=$(_cidr_range "$service_cidr" | cut -f1)
@ -628,7 +641,7 @@ function setup_k8s_binaries() {
function run_k8s_api {
local service_cidr
local cluster_ip_range
local cluster_ip_ranges
local command
local tmp_kube_apiserver_path="/tmp/kube-apiserver"
local binary_name="kube-apiserver"
@ -638,18 +651,17 @@ function run_k8s_api {
# Runs Hyperkube's Kubernetes API Server
wait_for "etcd" "http://${SERVICE_HOST}:${ETCD_PORT}/v2/machines"
service_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
subnet show "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" \
-c cidr -f value)
if is_service_enabled octavia; then
cluster_ip_range=$(split_subnet "$service_cidr" | cut -f1)
else
cluster_ip_range="$service_cidr"
fi
cluster_ip_ranges=()
for service_subnet_id in ${KURYR_SERVICE_SUBNETS_IDS[@]}; do
service_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
subnet show "$service_subnet_id" \
-c cidr -f value)
cluster_ip_ranges+=($(split_subnet "$service_cidr" | cut -f1))
done
command="${KURYR_KUBE_APISERVER_BINARY} \
--service-cluster-ip-range=${cluster_ip_range} \
--service-cluster-ip-range=$(IFS=, ; echo "${cluster_ip_ranges[*]}") \
--insecure-bind-address=0.0.0.0 \
--insecure-port=${KURYR_K8S_API_PORT} \
--etcd-servers=http://${SERVICE_HOST}:${ETCD_PORT} \
@ -660,7 +672,7 @@ function run_k8s_api {
--tls-private-key-file=${KURYR_KUBERNETES_DATA_DIR}/server.key \
--token-auth-file=${KURYR_KUBERNETES_DATA_DIR}/known_tokens.csv \
--allow-privileged=true \
--feature-gates="SCTPSupport=true" \
--feature-gates="SCTPSupport=true,IPv6DualStack=true" \
--v=$(get_k8s_log_level) \
--logtostderr=true"
@ -684,7 +696,7 @@ function run_k8s_controller_manager {
--min-resync-period=3m \
--v=$(get_k8s_log_level) \
--logtostderr=true \
--feature-gates="SCTPSupport=true" \
--feature-gates="SCTPSupport=true,IPv6DualStack=true" \
--leader-elect=false"
run_process kubernetes-controller-manager "$command" root root
@ -760,7 +772,7 @@ function run_k8s_kubelet {
--address=0.0.0.0 \
--enable-server \
--network-plugin=cni \
--feature-gates="SCTPSupport=true" \
--feature-gates="SCTPSupport=true,IPv6DualStack=true" \
--cni-bin-dir=$CNI_BIN_DIR \
--cni-conf-dir=$CNI_CONF_DIR \
--cert-dir=${KURYR_KUBERNETES_DATA_DIR}/kubelet.cert \
@ -797,7 +809,7 @@ function run_k8s_kubelet {
if is_service_enabled coredns; then
service_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
subnet show "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" \
subnet show "${KURYR_SERVICE_SUBNETS_IDS[0]}" \
-c cidr -f value)
export KURYR_COREDNS_CLUSTER_IP=$(_cidr_range "$service_cidr" | cut -f2)
command+=" --cluster-dns=${KURYR_COREDNS_CLUSTER_IP} --cluster-domain=cluster.local"
@ -927,9 +939,11 @@ function configure_overcloud_vm_k8s_svc_sg {
security_group=$(openstack security group list \
--project "$project_id" -c ID -c Name -f value | \
awk '{if ($2=="default") print $1}')
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--dst-port "$dst_port" --ethertype "$KURYR_ETHERTYPE" "$security_group"
for ethertype in ${KURYR_ETHERTYPES[@]}; do
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--dst-port "$dst_port" --ethertype "$ethertype" "$security_group"
done
openstack port set "$KURYR_OVERCLOUD_VM_PORT" --security-group service_pod_access
}
@ -970,7 +984,7 @@ function update_tempest_conf_file {
if [[ "$KURYR_CONFIGMAP_MODIFIABLE" == "True" ]]; then
iniset $TEMPEST_CONFIG kuryr_kubernetes configmap_modifiable True
fi
if [[ "$KURYR_IPV6" == "True" ]]; then
if [[ "$KURYR_IPV6" == "True" || "$KURYR_DUAL_STACK" == "True" ]]; then
iniset $TEMPEST_CONFIG kuryr_kubernetes ipv6 True
fi
iniset $TEMPEST_CONFIG kuryr_kubernetes validate_crd True
@ -1007,7 +1021,7 @@ if [[ "$1" == "stack" && "$2" == "extra" ]]; then
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "False" ]; then
service_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
subnet show "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" \
subnet show "${KURYR_SERVICE_SUBNETS_IDS[0]}" \
-c cidr -f value)
k8s_api_clusterip=$(_cidr_range "$service_cidr" | cut -f1)
# NOTE(mrostecki): KURYR_K8S_API_ROOT will be a global to be used by next
@ -1068,11 +1082,11 @@ if [[ "$1" == "stack" && "$2" == "extra" ]]; then
fi
fi
if is_service_enabled kubernetes-api \
|| is_service_enabled kubernetes-controller-manager \
|| is_service_enabled kubernetes-scheduler \
|| is_service_enabled kubelet; then
if is_service_enabled kubernetes-api kubernetes-controller-manager kubernetes-scheduler kubelet; then
get_container "$KURYR_KUBERNETES_IMAGE" "$KURYR_KUBERNETES_VERSION"
fi
if is_service_enabled kubernetes-api kubernetes-controller-manager kubernetes-scheduler; then
prepare_kubernetes_files
fi

View File

@ -105,4 +105,5 @@ KURYR_CONTAINERS_USE_LOWER_CONSTRAINTS=${KURYR_CONTAINERS_USE_LOWER_CONSTRAINTS:
KURYR_OVERCLOUD_VM_PORT=${KURYR_OVERCLOUD_VM_PORT:-port0}
KURYR_IPV6=${KURYR_IPV6:-False}
KURYR_DUAL_STACK=${KURYR_DUAL_STACK:-False}
SUBNETPOOL_KURYR_NAME_V6=${SUBNETPOOL_KURYR_NAME_V6:-"shared-kuryr-subnetpool-v6"}