#!/bin/bash
#
# lib/kuryr
# Utilities for kuryr-kubernetes devstack
# bind_for_kubelet
# Description: Creates an OVS internal port so that baremetal kubelet will be
# able to make both liveness and readiness http/tcp probes.
# Params:
# project - Id or name of the project used for kuryr devstack
# port - Port to open for K8s API, relevant only for OpenStack infra
# Dependencies:
# (none)
KURYR_CONF_NEUTRON=$(trueorfalse True KURYR_CONFIGURE_NEUTRON_DEFAULTS)
KURYR_IPV6=$(trueorfalse False KURYR_IPV6)
KURYR_DUAL_STACK=$(trueorfalse False KURYR_DUAL_STACK)
KURYR_USE_LC=$(trueorfalse False KURYR_CONTAINERS_USE_LOWER_CONSTRAINTS)
function container_runtime {
# Ignore error at killing/removing a container doesn't running to avoid
# unstack is terminated.
# TODO: Support for CRI-O if it's required.
local regex_cmds_ignore="(kill|rm)\s+"
if [[ ${CONTAINER_ENGINE} == 'crio' ]]; then
sudo podman "$@" || die $LINENO "Error when running podman command"
else
if [[ $@ =~ $regex_cmds_ignore ]]; then
docker "$@"
else
docker "$@" || die $LINENO "Error when running docker command"
fi
fi
}
function ovs_bind_for_kubelet {
local port_id
local port_mac
local fixed_ips
local port_ips
local port_subnets
local prefix
local project_id
local port_number
local security_group
local ifname
local service_subnet_cidr
local pod_subnet_gw
local cidrs
local _sp_id=${KURYR_NEUTRON_DEFAULT_SUBNETPOOL_ID}
project_id="$1"
port_number="$2"
security_group=$(openstack security group list \
--project "$project_id" -c ID -c Name -f value | \
awk '{if ($2=="default") print $1}')
port_id=$(openstack port create \
--device-owner compute:kuryr \
--project "$project_id" \
--security-group "$security_group" \
--security-group service_pod_access \
--host "${HOSTNAME}" \
--network "${KURYR_NEUTRON_DEFAULT_POD_NET}" \
-f value -c id \
kubelet-"${HOSTNAME}")
ifname="kubelet${port_id}"
ifname="${ifname:0:14}"
port_mac=$(openstack port show "$port_id" -c mac_address -f value)
fixed_ips=$(openstack port show "$port_id" -f value -c fixed_ips)
port_ips=($(python3 -c "print(' '.join([x['ip_address'] for x in ${fixed_ips}]))"))
port_subnets=($(python3 -c "print(' '.join([x['subnet_id'] for x in ${fixed_ips}]))"))
sudo ovs-vsctl -- --may-exist add-port $OVS_BRIDGE "$ifname" \
-- set Interface "$ifname" type=internal \
-- set Interface "$ifname" external-ids:iface-status=active \
-- set Interface "$ifname" external-ids:attached-mac="$port_mac" \
-- set Interface "$ifname" external-ids:iface-id="$port_id"
sudo ip link set dev "$ifname" address "$port_mac"
sudo ip link set dev "$ifname" up
for i in "${!port_ips[@]}"; do
prefix=$(openstack subnet show "${port_subnets[$i]}" \
-c cidr -f value | \
cut -f2 -d/)
sudo ip addr add "${port_ips[$i]}/${prefix}" dev "$ifname"
done
# TODO(dulek): This hack is for compatibility with multinode job, we might
# want to do it better one day and actually support dual stack
# and NP here.
if [[ -z ${KURYR_SERVICE_SUBNETS_IDS} ]]; then
KURYR_SERVICE_SUBNETS_IDS=(${KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET}-IPv4)
KURYR_POD_SUBNETS_IDS=(${KURYR_NEUTRON_DEFAULT_POD_SUBNET}-IPv4)
fi
if [[ -z ${KURYR_SUBNETPOOLS_IDS} ]]; then
# NOTE(gryf): In case we are missing KURYR_SUBNETPOOLS_IDS variable
# populated, which probably means, that kuryr-kubernetes service is
# not enabled, but if kuryr-daemon service is enabled (which is the
# case for multi node setup, where worker nodes should have it
# enabled), we need to have it filled.
export KURYR_SUBNETPOOLS_IDS=()
export KURYR_ETHERTYPES=()
if [[ "$KURYR_IPV6" == "False" ]]; then
export KURYR_ETHERTYPE=IPv4
KURYR_ETHERTYPES+=("IPv4")
KURYR_SUBNETPOOLS_IDS+=(${_sp_id:-${SUBNETPOOL_V4_ID}})
else
KURYR_ETHERTYPES+=("IPv6")
KURYR_SUBNETPOOLS_IDS+=($(openstack \
--os-cloud devstack-admin \
--os-region "${REGION_NAME}" \
subnet pool show ${SUBNETPOOL_KURYR_NAME_V6} -c id -f value))
fi
fi
for i in "${!KURYR_SERVICE_SUBNETS_IDS[@]}"; do
pod_subnet_gw=$(openstack subnet show "${KURYR_POD_SUBNETS_IDS[$i]}" \
-c gateway_ip -f value)
if is_service_enabled kuryr-kubernetes && [[ "$KURYR_SUBNET_DRIVER" == "namespace" ]]; then
cidrs=$(openstack subnet pool show "${KURYR_SUBNETPOOLS_IDS[$i]}" -c prefixes -f value)
subnetpool_cidr=$(python3 -c "print(${cidrs}[0])")
sudo ip route add "$subnetpool_cidr" via "$pod_subnet_gw" dev "$ifname"
else
service_subnet_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
subnet show "${KURYR_SERVICE_SUBNETS_IDS[$i]}" \
-c cidr -f value)
sudo ip route add "$service_subnet_cidr" via "$pod_subnet_gw" dev "$ifname"
fi
done
if [ -n "$port_number" ]; then
# if openstack-INPUT chain doesn't exist we create it in INPUT (for
# local development envs since openstack-INPUT is usually only in gates)
if [[ "$KURYR_IPV6" == "False" || "$KURYR_DUAL_STACK" == "True" ]]; then
sudo iptables -I openstack-INPUT 1 \
-p tcp -s 0.0.0.0/0 -d 0.0.0.0/0 --dport $port_number -j ACCEPT || \
sudo iptables -I INPUT 1 \
-p tcp -m conntrack --ctstate NEW \
-m tcp --dport "$port_number" \
-m comment --comment "kuryr-devstack: Access to OpenShift API" -j ACCEPT
fi
if [[ "$KURYR_IPV6" == "True" || "$KURYR_DUAL_STACK" == "True" ]]; then
sudo ip6tables -I openstack-INPUT 1 \
-p tcp -s ::/0 -d ::/0 --dport $port_number -j ACCEPT || \
sudo ip6tables -I INPUT 1 \
-p tcp -m conntrack --ctstate NEW \
-m tcp --dport "$port_number" \
-m comment --comment "kuryr-devstack: Access to OpenShift API" -j ACCEPT
fi
fi
}
# _allocation_range
# Description: Writes out tab separated usable ip range for a CIDR
# Params:
# cidr - The cidr to get the range for
# gateway_position - Whether to reserve at 'beginning' or at 'end'
function _allocation_range {
python3 - <<EOF "$@"
import sys
from netaddr import IPNetwork
n = IPNetwork(str(sys.argv[1]))
gateway_position = sys.argv[2]
if gateway_position == 'beginning':
beg_offset = 2
end_offset = 2
elif gateway_position == 'end':
beg_offset = 1
end_offset = 3
else:
raise ValueError('Disallowed gateway position %s' % gateway_position)
print("%s\\t%s" % (n[beg_offset], n[-end_offset]))
EOF
}
# create_k8s_icmp_sg_rules
# Description: Creates icmp sg rules for Kuryr-Kubernetes pods
# Params:
# sg_id - Kuryr's security group id
# direction - egress or ingress direction
function create_k8s_icmp_sg_rules {
local sg_id=$1
local direction="$2"
local project_id
project_id=$(get_or_create_project \
"$KURYR_NEUTRON_DEFAULT_PROJECT" default)
for ethertype in "${KURYR_ETHERTYPES[@]}"; do
icmp_sg_rules=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
security group rule create \
--project "$project_id" \
--protocol icmp \
--ethertype "$ethertype" \
--"$direction" "$sg_id")
done
die_if_not_set $LINENO icmp_sg_rules \
"Failure creating icmp sg ${direction} rule for ${sg_id}"
}
# create_k8s_subnet
# Description: Creates a network and subnet for Kuryr-Kubernetes usage
# Params:
# project_id - Kuryr's project uuid
# net_id - ID of the network where to create subnet in
# subnet_name - Name of the subnet to create
# subnetpool_id - uuid of the subnet pool to use
# router - name of the router to plug the subnet to
# split_allocation - Whether to allocate on all the subnet or only the
# latter half
# ip_version - IPv4 or IPv6
function create_k8s_subnet {
local project_id=$1
local net_id="$2"
local subnet_name="$3"
local subnetpool_id="$4"
local router="$5"
local subnet_params="--project $project_id "
local subnet_cidr
local split_allocation
split_allocation="${6:-False}"
local ip_version="${7:-IPv4}"
if [ "$ip_version" == "IPv4" ]; then
subnet_params+="--ip-version 4 "
else
# NOTE(dulek): K8s API won't accept subnets bigger than 20 bits.
# And 20 will totally be fine for us.
subnet_params+="--ip-version 6 --prefix-length 108 "
fi
subnet_params+="--no-dhcp --gateway none "
subnet_params+="--subnet-pool $subnetpool_id "
subnet_params+="--network $net_id $subnet_name"
local subnet_id
subnet_id=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
subnet create $subnet_params \
--project "$project_id" \
-c id -f value)
die_if_not_set $LINENO subnet_id \
"Failure creating K8s ${subnet_name} IPv4 subnet for ${project_id}"
subnet_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
subnet show "$subnet_id" \
-c cidr -f value)
die_if_not_set $LINENO subnet_cidr \
"Failure getting K8s ${subnet_name} IPv4 subnet for $project_id"
# Since K8s has its own IPAM for services and allocates the first IP from
# service subnet CIDR to Kubernetes apiserver, we'll always put the router
# interface at the end of the range.
local router_ip
local allocation_start
local allocation_end
local allocation_subnet
router_ip=$(_cidr_range "$subnet_cidr" | cut -f3)
if [[ "$split_allocation" == "True" ]]; then
allocation_subnet=$(split_subnet "$subnet_cidr" | cut -f2)
allocation_start=$(_allocation_range "$allocation_subnet" end | cut -f1)
allocation_end=$(_allocation_range "$allocation_subnet" end | cut -f2)
else
allocation_start=$(_allocation_range "$subnet_cidr" end | cut -f1)
allocation_end=$(_allocation_range "$subnet_cidr" end | cut -f2)
fi
die_if_not_set $LINENO router_ip \
"Failed to determine K8s ${subnet_name} subnet router IP"
openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" subnet set \
--gateway "$router_ip" --no-allocation-pool "$subnet_id" \
|| die $LINENO "Failed to update K8s ${subnet_name} subnet"
# Set a new allocation pool for the subnet so ports can be created again
openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" subnet set \
--allocation-pool "start=${allocation_start},end=${allocation_end}" \
"$subnet_id" || die $LINENO "Failed to update K8s ${subnet_name} subnet"
openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
router add subnet "$router" "$subnet_id" \
|| die $LINENO \
"Failed to enable routing for K8s ${subnet_name} subnet"
echo "$subnet_id"
}
# build_kuryr_container_image
# Description: Generates a Kuryr controller or Kuryr CNI docker image in
# the local docker registry as kuryr/controller:latest for controller or
# kuryr/cni:latest for CNI.
function build_kuryr_container_image {
local target=$1 # controller or cni
local build_args
local build_dir
local tag="kuryr/${target}"
build_dir="${DEST}/kuryr-kubernetes"
pushd "$build_dir"
if [[ "$KURYR_USE_LC" == "True" ]]; then
build_args="--build-arg UPPER_CONSTRAINTS_FILE="`
`"/opt/kuryr-kubernetes/lower-constraints.txt"
fi
if [[ ${CONTAINER_ENGINE} == 'crio' ]]; then
# NOTE(gryf): for crio/podman we need to have it tagged with docker.io
# (or whatever registry), or we would need to setup one, otherwise the
# default tag would be 'localhost/kuryr/*' instead of 'kuryr/*' as in
# docker case (which by default becomes 'docker.io/kuryr/*' if no
# registry has been specified). Creating registry for just two images
# is a little bit of overkill, hence the trick with docker.io tag, and
# image pull policy set to "Never" on deployment definition, so that
# we assure for taking images that we built.
tag="docker.io/${tag}"
fi
container_runtime build -t "${tag}" -f "${target}.Dockerfile" \
${build_args} .
popd
}
function indent {
sed 's/^/ /';
}
function generate_kuryr_configmap {
local output_dir
local conf_path
output_dir=$1
conf_path=${2:-""}
mkdir -p "$output_dir"
rm -f "${output_dir}/config_map.yml"
cat >> "${output_dir}/config_map.yml" << EOF
apiVersion: v1
kind: ConfigMap
metadata:
name: kuryr-config
namespace: kube-system
data:
kuryr.conf: |
EOF
indent < "${conf_path}" >> "${output_dir}/config_map.yml"
}
function generate_kuryr_certificates_secret {
local output_dir
local certs_bundle_path
output_dir=$1
certs_bundle_path=${2:-""}
mkdir -p "$output_dir"
rm -f "${output_dir}/certificates_secret.yml"
CA_CERT=\"\" # It's a "" string that will be inserted into yaml file.
if [ "$certs_bundle_path" -a -f "$certs_bundle_path" ]; then
CA_CERT=$(base64 -w0 < "$certs_bundle_path")
fi
cat >> "${output_dir}/certificates_secret.yml" << EOF
apiVersion: v1
kind: Secret
metadata:
name: kuryr-certificates
namespace: kube-system
type: Opaque
data:
kuryr-ca-bundle.crt: $CA_CERT
EOF
}
# Generates kuryr-controller service account and kuryr-cni service account.
function generate_kuryr_service_account {
output_dir=$1
mkdir -p "$output_dir"
rm -f "${output_dir}/controller_service_account.yml"
rm -f "${output_dir}/cni_service_account.yml"
cat >> "${output_dir}/controller_service_account.yml" << EOF
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: kuryr-controller
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: kuryr-controller
rules:
- apiGroups:
- ""
verbs: ["*"]
resources:
- endpoints
- pods
- services
- services/status
- namespaces
- apiGroups:
- ""
verbs: ["get", "list", "watch"]
resources:
- nodes
- apiGroups:
- openstack.org
verbs: ["*"]
resources:
- kuryrnetworks
- kuryrnetworkpolicies
- kuryrloadbalancers
- kuryrports
- apiGroups: ["networking.k8s.io"]
resources:
- networkpolicies
verbs:
- get
- list
- watch
- update
- patch
- apiGroups: ["k8s.cni.cncf.io"]
resources:
- network-attachment-definitions
verbs:
- get
- apiGroups: ["", "events.k8s.io"]
resources:
- events
verbs:
- create
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: kuryr-controller-global
subjects:
- kind: ServiceAccount
name: kuryr-controller
namespace: kube-system
roleRef:
kind: ClusterRole
name: kuryr-controller
apiGroup: rbac.authorization.k8s.io
EOF
cat >> "${output_dir}/cni_service_account.yml" << EOF
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: kuryr-cni
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: kuryr-cni
rules:
- apiGroups:
- ""
verbs: ["*"]
resources:
- pods
- apiGroups:
- openstack.org
verbs: ["*"]
resources:
- kuryrports
- apiGroups: ["", "events.k8s.io"]
resources:
- events
verbs:
- create
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: kuryr-cni-global
subjects:
- kind: ServiceAccount
name: kuryr-cni
namespace: kube-system
roleRef:
kind: ClusterRole
name: kuryr-cni
apiGroup: rbac.authorization.k8s.io
EOF
}
function generate_controller_deployment {
output_dir=$1
health_server_port=$2
controller_ha=$3
mkdir -p "$output_dir"
rm -f "${output_dir}/controller_deployment.yml"
cat >> "${output_dir}/controller_deployment.yml" << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
name: kuryr-controller
name: kuryr-controller
namespace: kube-system
spec:
replicas: ${KURYR_CONTROLLER_REPLICAS:-1}
selector:
matchLabels:
name: kuryr-controller
EOF
# When running without HA we should make sure that we won't have more than
# one kuryr-controller pod in the deployment.
if [ "$controller_ha" == "False" ]; then
cat >> "${output_dir}/controller_deployment.yml" << EOF
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 0
maxUnavailable: 1
EOF
fi
cat >> "${output_dir}/controller_deployment.yml" << EOF
template:
metadata:
labels:
name: kuryr-controller
name: kuryr-controller
spec:
serviceAccountName: kuryr-controller
automountServiceAccountToken: true
hostNetwork: true
containers:
EOF
if [ "$controller_ha" == "True" ]; then
cat >> "${output_dir}/controller_deployment.yml" << EOF
- image: gcr.io/google_containers/leader-elector:0.5
name: leader-elector
args:
- "--election=kuryr-controller"
- "--http=0.0.0.0:${KURYR_CONTROLLER_HA_PORT:-16401}"
- "--election-namespace=kube-system"
- "--ttl=5s"
ports:
- containerPort: ${KURYR_CONTROLLER_HA_PORT:-16401}
protocol: TCP
EOF
fi
cat >> "${output_dir}/controller_deployment.yml" << EOF
- image: kuryr/controller:latest
imagePullPolicy: Never
name: controller
terminationMessagePath: "/dev/termination-log"
volumeMounts:
- name: config-volume
mountPath: "/etc/kuryr"
- name: certificates-volume
mountPath: "/etc/ssl/certs"
readOnly: true
readinessProbe:
httpGet:
path: /ready
port: ${health_server_port}
scheme: HTTP
timeoutSeconds: 5
livenessProbe:
httpGet:
path: /alive
port: ${health_server_port}
initialDelaySeconds: 15
EOF
cat >> "${output_dir}/controller_deployment.yml" << EOF
volumes:
- name: config-volume
configMap:
name: kuryr-config
- name: certificates-volume
secret:
secretName: kuryr-certificates
restartPolicy: Always
tolerations:
- key: "node-role.kubernetes.io/master"
operator: "Exists"
effect: "NoSchedule"
- key: "node.kubernetes.io/not-ready"
operator: "Exists"
effect: "NoSchedule"
EOF
}
function generate_cni_daemon_set {
output_dir=$1
cni_health_server_port=$2
local var_run=${VAR_RUN_PATH:-/var/run}
mkdir -p "$output_dir"
rm -f "${output_dir}/cni_ds.yml"
cat >> "${output_dir}/cni_ds.yml" << EOF
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kuryr-cni-ds
namespace: kube-system
labels:
tier: node
app: kuryr-cni
spec:
selector:
matchLabels:
app: kuryr-cni
template:
metadata:
labels:
tier: node
app: kuryr-cni
spec:
hostNetwork: true
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
- key: "node.kubernetes.io/not-ready"
operator: "Exists"
effect: "NoSchedule"
serviceAccountName: kuryr-cni
containers:
- name: kuryr-cni
image: kuryr/cni:latest
imagePullPolicy: Never
command: [ "cni_ds_init" ]
env:
- name: KUBERNETES_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: KURYR_CNI_POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
securityContext:
privileged: true
volumeMounts:
- name: bin
mountPath: /opt/cni/bin
- name: net-conf
mountPath: /etc/cni/net.d
- name: config-volume
mountPath: /etc/kuryr
EOF
if [ "$CONTAINER_ENGINE" != "crio" ]; then
cat >> "${output_dir}/cni_ds.yml" << EOF
- name: proc
mountPath: /host_proc
EOF
fi
cat >> "${output_dir}/cni_ds.yml" << EOF
- name: var-pci
mountPath: /var/pci_address
- name: var-run
mountPath: /var/run
mountPropagation: HostToContainer
EOF
# NOTE(gryf): assuming the --namespaces-dir parameter would not be used,
# otherwise /var/run/$crio_netns_path is all wrong
if [ "$CONTAINER_ENGINE" = "crio" ] && \
[ "${VAR_RUN_PATH}" != "/var/run" ]; then
cat >> "${output_dir}/cni_ds.yml" << EOF
- name: netns
mountPath: /var/run/netns
mountPropagation: HostToContainer
EOF
fi
cat >> "${output_dir}/cni_ds.yml" << EOF
readinessProbe:
httpGet:
path: /ready
port: ${cni_health_server_port}
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 10
livenessProbe:
httpGet:
path: /alive
port: ${cni_health_server_port}
initialDelaySeconds: 60
volumes:
- name: bin
hostPath:
path: ${CNI_PLUGIN_DIR}
- name: net-conf
hostPath:
path: ${CNI_CONF_DIR}
- name: config-volume
configMap:
name: kuryr-config
- name: var-run
hostPath:
path: ${var_run}
EOF
if [[ "$CONTAINER_ENGINE" != "crio" ]]; then
cat >> "${output_dir}/cni_ds.yml" << EOF
- name: proc
hostPath:
path: /proc
EOF
fi
cat >> "${output_dir}/cni_ds.yml" << EOF
- name: var-pci
hostPath:
path: /var/pci_address
EOF
if [ "${CONTAINER_ENGINE}" = "crio" ] && \
[ "${VAR_RUN_PATH}" != "/var/run" ]; then
cat >> "${output_dir}/cni_ds.yml" << EOF
- name: netns
hostPath:
path: /var/run/netns
EOF
fi
}
# lb_state
# Description: Returns the state of the load balancer
# Params:
# id - Id or name of the loadbalancer the state of which needs to be
# retrieved.
function lb_state {
local lb_id
lb_id="$1"
openstack loadbalancer show "$lb_id" | \
awk '/provisioning_status/ {print $4}'
}
function _wait_for_lb {
local lb_name
local curr_time
local time_diff
local start_time
lb_name="$1"
timeout=${2:-$KURYR_WAIT_TIMEOUT}
echo -n "Waiting for LB:$lb_name"
start_time=$(date +%s)
while [[ "$(lb_state "$lb_name")" != "ACTIVE" ]]; do
echo -n "Waiting till LB=$lb_name is ACTIVE."
curr_time=$(date +%s)
time_diff=$((curr_time - start_time))
[[ $time_diff -le $timeout ]] || die "Timed out waiting for $lb_name"
sleep 5
done
}
# create_load_balancer
# Description: Creates an OpenStack Load Balancer with either neutron LBaaS
# or Octavia
# Params:
# lb_name: Name to give to the load balancer.
# lb_vip_subnet: Id or name of the subnet where lb_vip should be
# allocated.
# project_id: Id of the project where the load balancer should be
# allocated.
# lb_vip: Virtual IP to give to the load balancer - optional.
function create_load_balancer {
local lb_name
local lb_vip_subnet
local lb_params
local project_id
lb_name="$1"
lb_vip_subnet="$2"
project_id="$3"
lb_params=" --name $lb_name "
if [ -z "$4" ]; then
echo -n "create_load_balancer LB=$lb_name, lb_vip not provided."
else
lb_params+=" --vip-address $4"
fi
lb_params+=" --project ${project_id} --vip-subnet-id $lb_vip_subnet"
openstack loadbalancer create $lb_params
}
# create_load_balancer_listener
# Description: Creates an OpenStack Load Balancer Listener for the specified
# Load Balancer with either neutron LBaaS or Octavia
# Params:
# name: Name to give to the load balancer listener.
# protocol: Whether it is HTTP, HTTPS, TCP, etc.
# port: The TCP port number to listen to.
# data_timeouts: Octavia's timeouts for client and server inactivity.
# lb: Id or name of the Load Balancer we want to add the Listener to.
# project_id: Id of the project where this listener belongs to.
function create_load_balancer_listener {
local name
local protocol
local port
local lb
local data_timeouts
local max_timeout
local project_id
name="$1"
protocol="$2"
port="$3"
lb="$4"
project_id="$5"
data_timeouts="$6"
max_timeout=1200
# Octavia needs the LB to be active for the listener
_wait_for_lb "$lb" "$max_timeout"
openstack loadbalancer listener create --name "$name" \
--protocol "$protocol" \
--protocol-port "$port" \
--timeout-client-data "$data_timeouts" \
--timeout-member-data "$data_timeouts" \
"$lb"
}
# create_load_balancer_pool
# Description: Creates an OpenStack Load Balancer Pool for the specified
# Load Balancer listener with either neutron LBaaS or Octavia
# Params:
# name: Name to give to the load balancer listener.
# protocol: Whether it is HTTP, HTTPS, TCP, etc.
# algorithm: Load Balancing algorithm to use.
# listener: Id or name of the Load Balancer Listener we want to add the
# pool to.
# project_id: Id of the project where this pool belongs to.
# lb: Id or name of the Load Balancer we want to add the pool to
# (optional).
function create_load_balancer_pool {
local name
local protocol
local algorithm
local listener
local lb
local project_id
name="$1"
protocol="$2"
algorithm="$3"
listener="$4"
project_id="$5"
lb="$6"
# We must wait for the LB to be active before we can put a Pool for it
_wait_for_lb "$lb"
openstack loadbalancer pool create --name "$name" \
--listener "$listener" \
--protocol "$protocol" \
--lb-algorithm "$algorithm"
}
# create_load_balancer_member
# Description: Creates an OpenStack load balancer pool member
# Params:
# name: Name to give to the load balancer pool member.
# address: Whether it is HTTP, HTTPS, TCP, etc.
# port: Port number the pool member is listening on.
# pool: Id or name of the Load Balancer pool this member belongs to.
# subnet: Id or name of the subnet the member address belongs to.
# lb: Id or name of the load balancer the member belongs to.
# project_id: Id of the project where this pool belongs to.
function create_load_balancer_member {
local name
local address
local port
local pool
local lb
local project_id
name="$1"
address="$2"
port="$3"
pool="$4"
lb="$5"
project_id="$6"
# We must wait for the pool creation update before we can add members
_wait_for_lb "$lb"
openstack loadbalancer member create --name "$name" \
--address "$address" \
--protocol-port "$port" \
"$pool"
}
# split_subnet
# Description: Splits a subnet in two subnets that constitute its halves
# Params:
# cidr: Subnet CIDR to split
# Returns: tab separated CIDRs of the two halves.
function split_subnet {
# precondition: The passed cidr must be of a prefix <= 30
python3 - <<EOF "$@"
import sys
from netaddr import IPNetwork
n = IPNetwork(str(sys.argv[1]))
first, last = n.subnet(n.prefixlen+1)
print("%s\\t%s" % (first, last))
EOF
}
# cleanup_kuryr_devstack_iptables
# Description: Fins all the iptables rules we set and deletes them
function cleanup_kuryr_devstack_iptables {
local chains
chains=( INPUT FORWARD OUTPUT )
for chain in "${chains[@]}"; do
sudo iptables -n -L "$chain" -v --line-numbers | \
awk -v chain="$chain" \
'/kuryr-devstack/ {print "sudo iptables -D " chain " " $1}' | \
tac | bash /dev/stdin
done
}
function build_install_kuryr_cni {
pushd "${KURYR_HOME}/kuryr_cni" || exit 1
hack/build-go.sh
sudo install -o "$STACK_USER" -m 0555 -D bin/kuryr-cni \
"${CNI_PLUGIN_DIR}/kuryr-cni"
popd
}
function create_kuryr_account {
create_service_user "kuryr" "admin"
get_or_create_service "kuryr-kubernetes" "kuryr-kubernetes" \
"Kuryr-Kubernetes Service"
}
function _create_kuryr_cache_dir {
# Create cache directory
sudo install -d -o "$STACK_USER" "$KURYR_AUTH_CACHE_DIR"
if [[ ! "$KURYR_AUTH_CACHE_DIR" == "" ]]; then
rm -f "$KURYR_AUTH_CACHE_DIR"/*
fi
}
function _create_kuryr_lock_dir {
# Create lock directory
sudo install -d -o "$STACK_USER" "$KURYR_LOCK_DIR"
}
function configure_kuryr {
local dir
if [[ ${CONTAINER_ENGINE} == 'crio' ]]; then
# According of the documentation we need those kernel modules for
# CRI-O. They might already be loaded by neutron, so don't fail on it.
# https://kubernetes.io/docs/setup/production-environment/container-runtimes/#cri-o
sudo modprobe overlay || true
sudo modprobe br_netfilter || true
fi
sudo install -d -o "$STACK_USER" "$KURYR_CONFIG_DIR"
"${KURYR_HOME}/tools/generate_config_file_samples.sh"
sudo install -o "$STACK_USER" -m 640 -D \
"${KURYR_HOME}/etc/kuryr.conf.sample" "$KURYR_CONFIG"
iniset "$KURYR_CONFIG" kubernetes ssl_client_crt_file "$KURYR_K8S_API_CERT"
iniset "$KURYR_CONFIG" kubernetes ssl_client_key_file "$KURYR_K8S_API_KEY"
if [ "$KURYR_K8S_API_CACERT" ]; then
iniset "$KURYR_CONFIG" kubernetes ssl_ca_crt_file "$KURYR_K8S_API_CACERT"
iniset "$KURYR_CONFIG" kubernetes ssl_verify_server_crt True
fi
if [ "$KURYR_MULTI_VIF_DRIVER" ]; then
iniset "$KURYR_CONFIG" kubernetes multi_vif_drivers "$KURYR_MULTI_VIF_DRIVER"
fi
# REVISIT(ivc): 'use_stderr' is required for current CNI driver. Once a
# daemon-based CNI driver is implemented, this could be removed.
iniset "$KURYR_CONFIG" DEFAULT use_stderr true
iniset "$KURYR_CONFIG" DEFAULT debug "$ENABLE_DEBUG_LOG_LEVEL"
iniset "$KURYR_CONFIG" kubernetes port_debug "$KURYR_PORT_DEBUG"
iniset "$KURYR_CONFIG" kubernetes pod_subnets_driver "$KURYR_SUBNET_DRIVER"
iniset "$KURYR_CONFIG" kubernetes pod_security_groups_driver "$KURYR_SG_DRIVER"
iniset "$KURYR_CONFIG" kubernetes service_security_groups_driver "$KURYR_SG_DRIVER"
iniset "$KURYR_CONFIG" kubernetes enabled_handlers "$KURYR_ENABLED_HANDLERS"
# Let Kuryr retry connections to K8s API for 20 minutes.
iniset "$KURYR_CONFIG" kubernetes watch_retry_timeout 1200
if [[ "$KURYR_PROJECT_DRIVER" == "annotation" ]]; then
iniset "$KURYR_CONFIG" kubernetes pod_project_driver annotation
iniset "$KURYR_CONFIG" kubernetes service_project_driver annotation
iniset "$KURYR_CONFIG" kubernetes namespace_project_driver annotation
iniset "$KURYR_CONFIG" kubernetes network_policy_project_driver annotation
fi
if [ "${KURYR_CONT}" == "True" ]; then
# This works around the issue of being unable to set oslo.privsep mode
# to FORK in os-vif. When running in a container we disable `sudo` that
# was prefixed before `privsep-helper` command. This let's us run in
# envs without sudo and keep the same python environment as the parent
# process.
iniset "$KURYR_CONFIG" vif_plug_ovs_privileged helper_command privsep-helper
iniset "$KURYR_CONFIG" vif_plug_linux_bridge_privileged helper_command privsep-helper
if [ "${CONTAINER_ENGINE}" = "docker" ]; then
# When running kuryr-daemon or CNI in container we need to set up
# some configs.
iniset "$KURYR_CONFIG" cni_daemon docker_mode True
iniset "$KURYR_CONFIG" cni_daemon netns_proc_dir "/host_proc"
fi
else
iniset "$KURYR_CONFIG" oslo_concurrency lock_path "$KURYR_LOCK_DIR"
_create_kuryr_lock_dir
iniset "$KURYR_CONFIG" cni_health_server cg_path \
"/system.slice/system-devstack.slice/devstack@kuryr-daemon.service"
fi
_create_kuryr_cache_dir
# Neutron API server & Neutron plugin
if is_service_enabled kuryr-kubernetes; then
configure_auth_token_middleware "$KURYR_CONFIG" kuryr \
"$KURYR_AUTH_CACHE_DIR" neutron
iniset "$KURYR_CONFIG" kubernetes pod_vif_driver "$KURYR_POD_VIF_DRIVER"
if [ "$KURYR_USE_PORTS_POOLS" ]; then
iniset "$KURYR_CONFIG" kubernetes vif_pool_driver "$KURYR_VIF_POOL_DRIVER"
iniset "$KURYR_CONFIG" vif_pool ports_pool_min "$KURYR_VIF_POOL_MIN"
iniset "$KURYR_CONFIG" vif_pool ports_pool_max "$KURYR_VIF_POOL_MAX"
iniset "$KURYR_CONFIG" vif_pool ports_pool_batch "$KURYR_VIF_POOL_BATCH"
iniset "$KURYR_CONFIG" vif_pool ports_pool_update_frequency "$KURYR_VIF_POOL_UPDATE_FREQ"
if [ "$KURYR_VIF_POOL_MANAGER" ]; then
iniset "$KURYR_CONFIG" kubernetes enable_manager "$KURYR_VIF_POOL_MANAGER"
dir=`iniget "$KURYR_CONFIG" vif_pool manager_sock_file`
if [[ -z $dir ]]; then
dir="/run/kuryr/kuryr_manage.sock"
fi
dir=`dirname $dir`
sudo mkdir -p $dir
fi
fi
fi
}
function copy_kuryr_certs {
# copy kubelet key and make ubuntu user owner of it
sudo cp /etc/kubernetes/pki/apiserver-kubelet-client.key \
/etc/kubernetes/pki/kuryr-client.key
sudo chown $(whoami) /etc/kubernetes/pki/kuryr-client.key
}
function _generate_containerized_kuryr_resources {
if [[ $KURYR_CONTROLLER_REPLICAS -eq 1 ]]; then
KURYR_CONTROLLER_HA="False"
else
KURYR_CONTROLLER_HA="True"
fi
# Containerized deployment will use tokens provided by k8s itself.
inicomment "$KURYR_CONFIG" kubernetes ssl_client_crt_file
inicomment "$KURYR_CONFIG" kubernetes ssl_client_key_file
iniset "$KURYR_CONFIG" kubernetes controller_ha ${KURYR_CONTROLLER_HA}
iniset "$KURYR_CONFIG" kubernetes controller_ha_port ${KURYR_CONTROLLER_HA_PORT}
# NOTE(dulek): In the container the CA bundle will be mounted in a standard
# directory
iniset "$KURYR_CONFIG" neutron cafile /etc/ssl/certs/kuryr-ca-bundle.crt
# Generate kuryr resources in k8s formats.
local output_dir="${DATA_DIR}/kuryr-kubernetes"
generate_kuryr_configmap $output_dir $KURYR_CONFIG
generate_kuryr_certificates_secret $output_dir $SSL_BUNDLE_FILE
generate_kuryr_service_account $output_dir
generate_controller_deployment $output_dir $KURYR_HEALTH_SERVER_PORT $KURYR_CONTROLLER_HA
generate_cni_daemon_set $output_dir $KURYR_CNI_HEALTH_SERVER_PORT
}
function run_containerized_kuryr_resources {
local k8s_data_dir="${DATA_DIR}/kuryr-kubernetes"
kubectl create -f \
"${k8s_data_dir}/config_map.yml" \
|| die $LINENO "Failed to create kuryr-kubernetes ConfigMap."
kubectl create -f \
"${k8s_data_dir}/certificates_secret.yml" \
|| die $LINENO "Failed to create kuryr-kubernetes certificates Secret."
kubectl create -f \
"${k8s_data_dir}/controller_service_account.yml" \
|| die $LINENO "Failed to create kuryr-controller ServiceAccount."
kubectl create -f \
"${k8s_data_dir}/cni_service_account.yml" \
|| die $LINENO "Failed to create kuryr-cni ServiceAccount."
kubectl create -f \
"${k8s_data_dir}/controller_deployment.yml" \
|| die $LINENO "Failed to create kuryr-kubernetes Deployment."
kubectl create -f \
"${k8s_data_dir}/cni_ds.yml" \
|| die $LINENO "Failed to create kuryr-kubernetes CNI DaemonSet."
}
function _cidr_range {
python3 - <<EOF "$1"
import sys
from netaddr import IPAddress, IPNetwork
n = IPNetwork(sys.argv[1])
print("%s\\t%s\\t%s" % (IPAddress(n.first + 1), IPAddress(n.first + 2), IPAddress(n.last - 1)))
EOF
}
function copy_tempest_kubeconfig {
local tempest_home
tempest_home='/home/tempest'
if [ -d "$tempest_home" ]; then
sudo cp -r "${HOME}/.kube" "$tempest_home"
sudo chown -R tempest "${tempest_home}/.kube"
fi
}
function create_lb_for_services {
# This allows pods that need access to kubernetes API (like the
# containerized kuryr controller or kube-dns) to talk to the K8s API
# service
local api_port=6443
local service_cidr
local kubelet_iface_ip
local lb_name
local use_octavia
local project_id
local fixed_ips
local address
project_id=$(get_or_create_project \
"$KURYR_NEUTRON_DEFAULT_PROJECT" default)
lb_name='default/kubernetes'
# TODO(dulek): We only look at the first service subnet because kubernetes
# API service is only IPv4 in 1.20. It might be dual stack
# in the future.
service_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
subnet show "${KURYR_SERVICE_SUBNETS_IDS[0]}" \
-c cidr -f value)
fixed_ips=$(openstack port show kubelet-"${HOSTNAME}" -c fixed_ips -f value)
kubelet_iface_ip=$(python3 -c "print(${fixed_ips}[0]['ip_address'])")
k8s_api_clusterip=$(_cidr_range "$service_cidr" | cut -f1)
echo "***********************************************************************"
echo "lbname: $lb_name subnet ${KURYR_SERVICE_SUBNETS_IDS[0]} pid: $project_id api-cluster: $k8s_api_clusteri"
echo "***********************************************************************"
create_load_balancer "$lb_name" "${KURYR_SERVICE_SUBNETS_IDS[0]}" \
"$project_id" "$k8s_api_clusterip"
create_load_balancer_listener default/kubernetes:${KURYR_K8S_API_LB_PORT} HTTPS ${KURYR_K8S_API_LB_PORT} "$lb_name" "$project_id" 3600000
create_load_balancer_pool default/kubernetes:${KURYR_K8S_API_LB_PORT} HTTPS ROUND_ROBIN \
default/kubernetes:${KURYR_K8S_API_LB_PORT} "$project_id" "$lb_name"
if [[ "${KURYR_OVS_BM}" == "True" ]]; then
address=${kubelet_iface_ip}
else
address="${HOST_IP}"
fi
# Regardless of the octavia mode, the k8s API will be behind an L3 mode
# amphora driver loadbalancer
create_load_balancer_member "$(hostname)" "$address" "$api_port" \
default/kubernetes:${KURYR_K8S_API_LB_PORT} "$lb_name" "$project_id"
}
function _configure_neutron_defaults {
local project_id
local sg_ids
local router
local router_id
local ext_svc_net_id
local addrs_prefix
local subnetpool_name
project_id=$(get_or_create_project \
"$KURYR_NEUTRON_DEFAULT_PROJECT" default)
ext_svc_net_id="$(openstack network show -c id -f value \
"${KURYR_NEUTRON_DEFAULT_EXT_SVC_NET}")"
# If a subnetpool is not passed, we get the one created in devstack's
# Neutron module
export KURYR_SUBNETPOOLS_IDS=()
export KURYR_ETHERTYPES=()
if [[ "$KURYR_IPV6" == "False" || "$KURYR_DUAL_STACK" == "True" ]]; then
export KURYR_ETHERTYPE=IPv4
KURYR_ETHERTYPES+=("IPv4")
KURYR_SUBNETPOOLS_IDS+=(${KURYR_NEUTRON_DEFAULT_SUBNETPOOL_ID:-${SUBNETPOOL_V4_ID}})
fi
if [[ "$KURYR_IPV6" == "True" || "$KURYR_DUAL_STACK" == "True" ]]; then
export KURYR_ETHERTYPE=IPv6
KURYR_ETHERTYPES+=("IPv6")
# NOTE(gryf): To not clash with subnets created by DevStack for IPv6,
# we create another subnetpool just for kuryr subnets.
# SUBNETPOOL_KURYR_V6_ID will be used in function configure_kuryr in
# case of namespace kuryr subnet driver.
# This is not required for IPv4, because DevStack is only adding a
# conflicting route for IPv6. On DevStack this route is opening public
# IPv6 network to be accessible from host, which doesn't have place in
# IPv4 net, because floating IPs are used instead.
IPV6_ID=$(uuidgen | sed s/-//g | cut -c 23- | \
sed -e "s/\(..\)\(....\)\(....\)/\1:\2:\3/")
addrs_prefix="fd${IPV6_ID}::/56"
subnetpool_name=${SUBNETPOOL_KURYR_NAME_V6}
KURYR_SUBNETPOOLS_IDS+=($(openstack \
--os-cloud devstack-admin \
--os-region "${REGION_NAME}" \
subnet pool create "${subnetpool_name}" \
--default-prefix-length "${SUBNETPOOL_SIZE_V6}" \
--pool-prefix "${addrs_prefix}" \
--share -f value -c id))
fi
router=${KURYR_NEUTRON_DEFAULT_ROUTER:-$Q_ROUTER_NAME}
if [ "$router" != "$Q_ROUTER_NAME" ]; then
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
router create --project "$project_id" "$router"
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
router set --external-gateway "$ext_svc_net_id" "$router"
fi
router_id="$(openstack router show -c id -f value "$router")"
pod_net_id=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
network create --project "$project_id" \
"$KURYR_NEUTRON_DEFAULT_POD_NET" \
-c id -f value)
service_net_id=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
network create --project "$project_id" \
"$KURYR_NEUTRON_DEFAULT_SERVICE_NET" \
-c id -f value)
export KURYR_POD_SUBNETS_IDS=()
export KURYR_SERVICE_SUBNETS_IDS=()
for i in "${!KURYR_SUBNETPOOLS_IDS[@]}"; do
KURYR_POD_SUBNETS_IDS+=($(create_k8s_subnet "$project_id" \
"$pod_net_id" \
"${KURYR_NEUTRON_DEFAULT_POD_SUBNET}-${KURYR_ETHERTYPES[$i]}" \
"${KURYR_SUBNETPOOLS_IDS[$i]}" \
"$router" "False" ${KURYR_ETHERTYPES[$i]}))
KURYR_SERVICE_SUBNETS_IDS+=($(create_k8s_subnet "$project_id" \
"$service_net_id" \
"${KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET}-${KURYR_ETHERTYPES[$i]}" \
"${KURYR_SUBNETPOOLS_IDS[$i]}" \
"$router" "True" ${KURYR_ETHERTYPES[$i]}))
done
sg_ids=()
if [[ "$KURYR_SG_DRIVER" == "default" ]]; then
sg_ids+=($(echo $(openstack security group list \
--project "$project_id" -c ID -f value) | tr ' ' ','))
fi
# In order for the ports to allow service traffic under Octavia L3 mode,
# it is necessary for the service subnet to be allowed into the port's
# security groups. If L3 is used, then the pods created will include it.
# Otherwise it will be just used by the kubelet port used for the K8s API
# load balancer
local service_pod_access_sg_id
service_pod_access_sg_id=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
security group create --project "$project_id" \
service_pod_access -f value -c id)
for i in "${!KURYR_SERVICE_SUBNETS_IDS[@]}"; do
local service_cidr
service_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" subnet show \
"${KURYR_SERVICE_SUBNETS_IDS[$i]}" -f value -c cidr)
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s service subnet allowed" \
--remote-ip "$service_cidr" --ethertype "${KURYR_ETHERTYPES[$i]}" --protocol tcp \
"$service_pod_access_sg_id"
# Since Octavia supports also UDP load balancing, we need to allow
# also udp traffic
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s service subnet UDP allowed" \
--remote-ip "$service_cidr" --ethertype "${KURYR_ETHERTYPES[$i]}" --protocol udp \
"$service_pod_access_sg_id"
# Octavia supports SCTP load balancing, we need to also allow SCTP traffic
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s service subnet SCTP allowed" \
--remote-ip "$service_cidr" --ethertype "${KURYR_ETHERTYPES[$i]}" --protocol sctp \
"$service_pod_access_sg_id"
done
if [[ "$KURYR_K8S_OCTAVIA_MEMBER_MODE" == "L3" ]]; then
sg_ids+=(${service_pod_access_sg_id})
elif [[ "$KURYR_K8S_OCTAVIA_MEMBER_MODE" == "L2" ]]; then
# In case the member connectivity is L2, Octavia by default uses the
# admin 'default' sg to create a port for the amphora load balancer
# at the member ports subnet. Thus we need to allow L2 communication
# between the member ports and the octavia ports by allowing all
# access from the pod subnet range to the ports in that subnet, and
# include it into $sg_ids
local octavia_pod_access_sg_id
octavia_pod_access_sg_id=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
security group create --project "$project_id" \
octavia_pod_access -f value -c id)
for i in "${!KURYR_POD_SUBNETS_IDS[@]}"; do
local pod_cidr
pod_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" subnet show \
"${KURYR_POD_SUBNETS_IDS[$i]}" -f value -c cidr)
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s pod subnet allowed from k8s-pod-subnet" \
--remote-ip "$pod_cidr" --ethertype "${KURYR_ETHERTYPES[$i]}" --protocol tcp \
"$octavia_pod_access_sg_id"
# Since Octavia supports also UDP load balancing, we need to allow
# also udp traffic
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s pod subnet allowed from k8s-pod-subnet" \
--remote-ip "$pod_cidr" --ethertype "${KURYR_ETHERTYPES[$i]}" --protocol udp \
"$octavia_pod_access_sg_id"
# Octavia supports SCTP load balancing, we need to also support SCTP traffic
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "k8s pod subnet allowed from k8s-pod-subnet" \
--remote-ip "$pod_cidr" --ethertype "${KURYR_ETHERTYPES[$i]}" --protocol sctp \
"$octavia_pod_access_sg_id"
done
sg_ids+=(${octavia_pod_access_sg_id})
fi
iniset "$KURYR_CONFIG" neutron_defaults project "$project_id"
iniset "$KURYR_CONFIG" neutron_defaults pod_subnet "${KURYR_POD_SUBNETS_IDS[0]}"
iniset "$KURYR_CONFIG" neutron_defaults pod_subnets $(IFS=, ; echo "${KURYR_POD_SUBNETS_IDS[*]}")
iniset "$KURYR_CONFIG" neutron_defaults service_subnet "${KURYR_SERVICE_SUBNETS_IDS[0]}"
iniset "$KURYR_CONFIG" neutron_defaults service_subnets $(IFS=, ; echo "${KURYR_SERVICE_SUBNETS_IDS[*]}")
if [ "$KURYR_SUBNET_DRIVER" == "namespace" ]; then
iniset "$KURYR_CONFIG" namespace_subnet pod_subnet_pool "${KURYR_SUBNETPOOLS_IDS[0]}"
iniset "$KURYR_CONFIG" namespace_subnet pod_subnet_pools $(IFS=, ; echo "${KURYR_SUBNETPOOLS_IDS[*]}")
iniset "$KURYR_CONFIG" namespace_subnet pod_router "$router_id"
fi
if [[ "$KURYR_SG_DRIVER" == "policy" ]]; then
# NOTE(dulek): Using the default DevStack's SG is not enough to match
# the NP specification. We need to open ingress to everywhere, so we
# create allow-all group.
allow_all_sg_id=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
security group create --project "$project_id" \
allow-all -f value -c id)
for ethertype in ${KURYR_ETHERTYPES[@]}; do
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--description "allow all ingress traffic" \
--ethertype "$ethertype" --ingress --protocol any \
"$allow_all_sg_id"
done
sg_ids+=(${allow_all_sg_id})
fi
iniset "$KURYR_CONFIG" neutron_defaults pod_security_groups $(IFS=, ; echo "${sg_ids[*]}")
if [[ "$KURYR_SG_DRIVER" == "policy" ]]; then
# NOTE(ltomasbo): As more security groups and rules are created, there
# is a need to increase the quota for it
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
quota set --secgroups 100 --secgroup-rules 300 "$project_id"
fi
# NOTE(dulek): DevStack's admin default for SG's and instances is 10, this
# is too little for our tests with Octavia configured to use
# amphora.
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
quota set --secgroups 100 --secgroup-rules 300 --instances 100 admin
if [ -n "$OVS_BRIDGE" ]; then
iniset "$KURYR_CONFIG" neutron_defaults ovs_bridge "$OVS_BRIDGE"
fi
iniset "$KURYR_CONFIG" neutron_defaults external_svc_net "$ext_svc_net_id"
iniset "$KURYR_CONFIG" octavia_defaults member_mode "$KURYR_K8S_OCTAVIA_MEMBER_MODE"
iniset "$KURYR_CONFIG" octavia_defaults enforce_sg_rules "$KURYR_ENFORCE_SG_RULES"
iniset "$KURYR_CONFIG" octavia_defaults lb_algorithm "$KURYR_LB_ALGORITHM"
iniset "$KURYR_CONFIG" octavia_defaults timeout_client_data "$KURYR_TIMEOUT_CLIENT_DATA"
iniset "$KURYR_CONFIG" octavia_defaults timeout_member_data "$KURYR_TIMEOUT_MEMBER_DATA"
# Octavia takes a very long time to start the LB in the gate. We need
# to tweak the timeout for the LB creation. Let's be generous and give
# it up to 20 minutes.
# FIXME(dulek): This might be removed when bug 1753653 is fixed and
# Kuryr restarts waiting for LB on timeouts.
iniset "$KURYR_CONFIG" neutron_defaults lbaas_activation_timeout 1200
iniset "$KURYR_CONFIG" kubernetes endpoints_driver_octavia_provider "$KURYR_EP_DRIVER_OCTAVIA_PROVIDER"
}
function configure_k8s_pod_sg_rules {
local project_id
local sg_id
project_id=$(get_or_create_project \
"$KURYR_NEUTRON_DEFAULT_PROJECT" default)
sg_id=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
security group list \
--project "$project_id" -c ID -c Name -f value | \
awk '{if ($2=="default") print $1}')
create_k8s_icmp_sg_rules "$sg_id" ingress
}
function prepare_kubernetes_files {
mkdir -p "${KURYR_KUBERNETES_DATA_DIR}"
# Copy certs for Kuryr services to use
sudo install -m 644 /etc/kubernetes/pki/apiserver.crt \
"${KURYR_KUBERNETES_DATA_DIR}/kuryr.crt"
sudo install -m 644 /etc/kubernetes/pki/apiserver.key \
"${KURYR_KUBERNETES_DATA_DIR}/kuryr.key"
sudo install -m 644 /etc/kubernetes/pki/ca.crt \
"${KURYR_KUBERNETES_DATA_DIR}/kuryr-ca.crt"
}
function wait_for {
local name
local url
local cacert_path
local start_time
local curr_time
local time_diff
name="$1"
url="$2"
cacert_path=${3:-}
timeout=${4:-$KURYR_WAIT_TIMEOUT}
echo -n "Waiting for $name to respond"
extra_flags=${cacert_path:+"--cacert ${cacert_path}"}
start_time=$(date +%s)
until curl -o /dev/null -s "$extra_flags" "$url"; do
echo -n "."
curr_time=$(date +%s)
time_diff=$((curr_time - start_time))
[[ $time_diff -le $timeout ]] || die "Timed out waiting for $name"
sleep 1
done
echo ""
}
function _wait_for_ok_ready {
local name
local start_time
local curr_time
local time_diff
name="$1"
timeout=${2:-$KURYR_WAIT_TIMEOUT}
start_time=$(date +%s)
echo -n "Waiting for ${name} to be ready"
until [[ "$(kubectl get --raw='/readyz')" == "ok" ]]; do
echo -n "."
curr_time=$(date +%s)
time_diff=$((curr_time - start_time))
[[ $time_diff -le $timeout ]] || die "Timed out waiting for $name"
sleep 1
done
echo ""
}
function prepare_kubeconfig {
kubectl config set-cluster devstack-cluster \
--server="${KURYR_K8S_API_URL}" --certificate-authority
kubectl config set-credentials stack
kubectl config set-context devstack --cluster=devstack-cluster --user=stack
kubectl config use-context devstack
}
function prepare_kubelet {
local kubelet_plugin_dir="/etc/cni/net.d/"
sudo install -o "$STACK_USER" -m 0664 -D \
"${KURYR_HOME}${kubelet_plugin_dir}/10-kuryr.conflist" \
"${CNI_CONF_DIR}/10-kuryr.conflist"
}
function run_kuryr_kubernetes {
local controller_bin
_wait_for_ok_ready "kubernetes" 1200
controller_bin=$(which kuryr-k8s-controller)
run_process kuryr-kubernetes "$controller_bin --config-file $KURYR_CONFIG"
}
function configure_overcloud_vm_k8s_svc_sg {
local dst_port
local project_id
local security_group
if is_service_enabled octavia; then
dst_port=${KURYR_K8S_API_LB_PORT}
else
dst_port=${KURYR_K8S_API_PORT}
fi
project_id=$(get_or_create_project \
"$KURYR_NEUTRON_DEFAULT_PROJECT" default)
security_group=$(openstack security group list \
--project "$project_id" -c ID -c Name -f value | \
awk '{if ($2=="default") print $1}')
for ethertype in "${KURYR_ETHERTYPES[@]}"; do
openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \
security group rule create --project "$project_id" \
--dst-port "$dst_port" --ethertype "$ethertype" "$security_group"
done
openstack port set "$KURYR_OVERCLOUD_VM_PORT" --security-group service_pod_access
}
function run_kuryr_daemon {
local daemon_bin
daemon_bin=$(which kuryr-daemon)
run_process kuryr-daemon \
"$daemon_bin --config-file $KURYR_CONFIG" root root
}
function update_tempest_conf_file {
if [[ "$KURYR_USE_PORT_POOLS" == "True" ]]; then
iniset "$TEMPEST_CONFIG" kuryr_kubernetes port_pool_enabled True
fi
if [[ "${KURYR_CONT}" == "True" ]]; then
iniset "$TEMPEST_CONFIG" kuryr_kubernetes containerized True
fi
if [[ "$KURYR_SUBNET_DRIVER" == "namespace" ]]; then
iniset "$TEMPEST_CONFIG" kuryr_kubernetes subnet_per_namespace True
iniset "$TEMPEST_CONFIG" kuryr_kubernetes kuryrnetworks True
iniset "$TEMPEST_CONFIG" kuryr_kubernetes trigger_namespace_upon_pod True
fi
if [[ "$KURYR_K8S_SERIAL_TESTS" == "True" ]]; then
iniset "$TEMPEST_CONFIG" kuryr_kubernetes run_tests_serial True
fi
if [[ "$KURYR_MULTI_VIF_DRIVER" == "npwg_multiple_interfaces" ]]; then
iniset "$TEMPEST_CONFIG" kuryr_kubernetes npwg_multi_vif_enabled True
fi
if [[ "$KURYR_ENABLED_HANDLERS" =~ .*policy.* ]]; then
iniset "$TEMPEST_CONFIG" kuryr_kubernetes network_policy_enabled True
iniset "$TEMPEST_CONFIG" kuryr_kubernetes new_kuryrnetworkpolicy_crd True
fi
# NOTE(yboaron): Services with protocol UDP are supported in Kuryr
# starting from Stein release
iniset "$TEMPEST_CONFIG" kuryr_kubernetes test_udp_services True
if [[ "$KURYR_CONTROLLER_HA" == "True" ]]; then
iniset "$TEMPEST_CONFIG" kuryr_kubernetes ap_ha True
fi
if [[ "$KURYR_K8S_MULTI_WORKER_TESTS" == "True" ]]; then
iniset "$TEMPEST_CONFIG" kuryr_kubernetes multi_worker_setup True
fi
if [[ "$KURYR_K8S_CLOUD_PROVIDER" == "True" ]]; then
iniset "$TEMPEST_CONFIG" kuryr_kubernetes cloud_provider True
fi
if [[ "$KURYR_CONFIGMAP_MODIFIABLE" == "True" ]]; then
iniset "$TEMPEST_CONFIG" kuryr_kubernetes configmap_modifiable True
fi
if [[ "$KURYR_IPV6" == "True" || "$KURYR_DUAL_STACK" == "True" ]]; then
iniset "$TEMPEST_CONFIG" kuryr_kubernetes ipv6 True
fi
# NOTE(digitalsimboja): Reconciliation tests create and delete LBs,
# so only enable them for OVN as it's faster when creating LBs
if [[ "$KURYR_EP_DRIVER_OCTAVIA_PROVIDER" == "ovn" ]]; then
iniset "$TEMPEST_CONFIG" kuryr_kubernetes enable_reconciliation True
fi
if [[ "$KURYR_PROJECT_DRIVER" == "annotation" ]]; then
iniset "$TEMPEST_CONFIG" kuryr_kubernetes annotation_project_driver True
fi
iniset "$TEMPEST_CONFIG" kuryr_kubernetes enable_listener_reconciliation True
iniset "$TEMPEST_CONFIG" kuryr_kubernetes validate_crd True
iniset "$TEMPEST_CONFIG" kuryr_kubernetes kuryrports True
iniset "$TEMPEST_CONFIG" kuryr_kubernetes kuryrloadbalancers True
iniset "$TEMPEST_CONFIG" kuryr_kubernetes test_services_without_selector True
iniset "$TEMPEST_CONFIG" kuryr_kubernetes test_sctp_services True
iniset "$TEMPEST_CONFIG" kuryr_kubernetes test_configurable_listener_timeouts True
if [[ "$KURYR_SUPPORT_POD_SECURITY" == "True" ]]; then
iniset "$TEMPEST_CONFIG" kuryr_kubernetes set_pod_security_context True
fi
}
function configure_neutron_defaults {
local k8s_api_clusterip
local service_cidr
if [ "${KURYR_CONF_NEUTRON}" == "False" ]; then
return
fi
if is_service_enabled kuryr-kubernetes; then
_configure_neutron_defaults
fi
if [ "${KURYR_CONT}" == "False" ]; then
KURYR_K8S_API_ROOT=${KURYR_K8S_API_URL}
iniset "$KURYR_CONFIG" kubernetes api_root "${KURYR_K8S_API_ROOT}"
iniset "$KURYR_CONFIG" kubernetes token_file '""'
else
iniset "$KURYR_CONFIG" kubernetes api_root '""'
fi
}
function uninstall_kuryr_cni {
sudo rm "${CNI_PLUGIN_DIR}/kuryr-cni"
if [ -z "$(ls -A ${CNI_PLUGIN_DIR})" ]; then
sudo rm -fr "${CNI_PLUGIN_DIR}"
fi
}
function rm_kuryr_conf {
sudo rm -fr /etc/kuryr
}