Use K8s API LB IP instead of HOST_IP in DevStack
This commit switches Kuryr configuration to use the IP of load balancer created for Kubernetes API instead of hardcoding HOST_IP there. This should be useful e.g. for multinode. Change-Id: I4535986fdac4d7d6b2eda0debdcaf6122a4f5b9f Closes-Bug: 1771801
This commit is contained in:
Michał Dulko
parent
719cc1cc28
commit
ce3305b9ef
@ -748,7 +748,6 @@ function run_openshift_master {
|
||||
"--write-config=${OPENSHIFT_DATA_DIR}"
|
||||
|
||||
# Reconfigure Kuryr-Kubernetes to use the certs generated
|
||||
iniset "$KURYR_CONFIG" kubernetes api_root "$OPENSHIFT_API_URL"
|
||||
iniset "$KURYR_CONFIG" kubernetes ssl_client_crt_file "${OPENSHIFT_DATA_DIR}/admin.crt"
|
||||
iniset "$KURYR_CONFIG" kubernetes ssl_client_key_file "${OPENSHIFT_DATA_DIR}/admin.key"
|
||||
iniset "$KURYR_CONFIG" kubernetes ssl_ca_crt_file "${OPENSHIFT_DATA_DIR}/ca.crt"
|
||||
|
||||
@ -52,7 +52,6 @@ function configure_kuryr {
|
||||
sudo install -o "$STACK_USER" -m 640 -D "${KURYR_HOME}/etc/kuryr.conf.sample" \
|
||||
"$KURYR_CONFIG"
|
||||
|
||||
iniset "$KURYR_CONFIG" kubernetes api_root "$KURYR_K8S_API_URL"
|
||||
if [ "$KURYR_K8S_API_CERT" ]; then
|
||||
iniset "$KURYR_CONFIG" kubernetes ssl_client_crt_file "$KURYR_K8S_API_CERT"
|
||||
fi
|
||||
@ -61,6 +60,7 @@ function configure_kuryr {
|
||||
fi
|
||||
if [ "$KURYR_K8S_API_CACERT" ]; then
|
||||
iniset "$KURYR_CONFIG" kubernetes ssl_ca_crt_file "$KURYR_K8S_API_CACERT"
|
||||
iniset "$KURYR_CONFIG" kubernetes ssl_verify_server_crt True
|
||||
fi
|
||||
# REVISIT(ivc): 'use_stderr' is required for current CNI driver. Once a
|
||||
# daemon-based CNI driver is implemented, this could be removed.
|
||||
@ -73,6 +73,9 @@ function configure_kuryr {
|
||||
iniset "$KURYR_CONFIG" kubernetes pod_subnets_driver "$KURYR_SUBNET_DRIVER"
|
||||
iniset "$KURYR_CONFIG" kubernetes enabled_handlers "$KURYR_ENABLED_HANDLERS"
|
||||
|
||||
# Let Kuryr retry connections to K8s API for 20 minutes.
|
||||
iniset "$KURYR_CONFIG" kubernetes watch_retry_timeout 1200
|
||||
|
||||
KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT)
|
||||
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then
|
||||
# This works around the issue of being unable to set oslo.privsep mode
|
||||
@ -237,9 +240,9 @@ function create_k8s_api_service {
|
||||
|
||||
create_load_balancer "$lb_name" "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET"\
|
||||
"$project_id" "$k8s_api_clusterip"
|
||||
create_load_balancer_listener default/kubernetes:443 HTTPS 443 "$lb_name" "$project_id" 3600000
|
||||
create_load_balancer_pool default/kubernetes:443 HTTPS ROUND_ROBIN \
|
||||
default/kubernetes:443 "$project_id" "$lb_name"
|
||||
create_load_balancer_listener default/kubernetes:${KURYR_K8S_API_LB_PORT} HTTPS ${KURYR_K8S_API_LB_PORT} "$lb_name" "$project_id" 3600000
|
||||
create_load_balancer_pool default/kubernetes:${KURYR_K8S_API_LB_PORT} HTTPS ROUND_ROBIN \
|
||||
default/kubernetes:${KURYR_K8S_API_LB_PORT} "$project_id" "$lb_name"
|
||||
|
||||
local api_port
|
||||
if is_service_enabled openshift-master; then
|
||||
@ -260,10 +263,10 @@ function create_k8s_api_service {
|
||||
if [[ "$use_octavia" == "True" && \
|
||||
"$KURYR_K8S_OCTAVIA_MEMBER_MODE" == "L2" ]]; then
|
||||
create_load_balancer_member "$(hostname)" "$address" "$api_port" \
|
||||
default/kubernetes:443 $KURYR_NEUTRON_DEFAULT_POD_SUBNET "$lb_name" "$project_id"
|
||||
default/kubernetes:${KURYR_K8S_API_LB_PORT} $KURYR_NEUTRON_DEFAULT_POD_SUBNET "$lb_name" "$project_id"
|
||||
else
|
||||
create_load_balancer_member "$(hostname)" "$address" "$api_port" \
|
||||
default/kubernetes:443 public-subnet "$lb_name" "$project_id"
|
||||
default/kubernetes:${KURYR_K8S_API_LB_PORT} public-subnet "$lb_name" "$project_id"
|
||||
fi
|
||||
}
|
||||
|
||||
@ -357,6 +360,22 @@ function configure_neutron_defaults {
|
||||
sg_ids+=",${octavia_pod_access_sg_id}"
|
||||
fi
|
||||
|
||||
KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT)
|
||||
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "False" ]; then
|
||||
local service_cidr
|
||||
local k8s_api_clusterip
|
||||
service_cidr=$(openstack --os-cloud devstack-admin \
|
||||
--os-region "$REGION_NAME" \
|
||||
subnet show "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" \
|
||||
-c cidr -f value)
|
||||
k8s_api_clusterip=$(_cidr_range "$service_cidr" | cut -f1)
|
||||
# NOTE(dulek): KURYR_K8S_API_LB_URL will be a global to be used by next
|
||||
# deployment phases.
|
||||
KURYR_K8S_API_LB_URL="https://${k8s_api_clusterip}:${KURYR_K8S_API_LB_PORT}"
|
||||
iniset "$KURYR_CONFIG" kubernetes api_root ${KURYR_K8S_API_LB_URL}
|
||||
else
|
||||
iniset "$KURYR_CONFIG" kubernetes api_root ""
|
||||
fi
|
||||
iniset "$KURYR_CONFIG" neutron_defaults project "$project_id"
|
||||
iniset "$KURYR_CONFIG" neutron_defaults pod_subnet "$pod_subnet_id"
|
||||
iniset "$KURYR_CONFIG" neutron_defaults pod_security_groups "$sg_ids"
|
||||
@ -435,6 +454,11 @@ function prepare_kubernetes_files {
|
||||
sudo bash -c "echo '$(create_token),kubelet,kubelet' >> ${KURYR_HYPERKUBE_DATA_DIR}/known_tokens.csv"
|
||||
sudo bash -c "echo '$(create_token),kube_proxy,kube_proxy' >> ${KURYR_HYPERKUBE_DATA_DIR}/known_tokens.csv"
|
||||
|
||||
# Copy certs for Kuryr services to use
|
||||
sudo install -m 644 "${KURYR_HYPERKUBE_DATA_DIR}/kubecfg.crt" "${KURYR_HYPERKUBE_DATA_DIR}/kuryr.crt"
|
||||
sudo install -m 644 "${KURYR_HYPERKUBE_DATA_DIR}/kubecfg.key" "${KURYR_HYPERKUBE_DATA_DIR}/kuryr.key"
|
||||
sudo install -m 644 "${KURYR_HYPERKUBE_DATA_DIR}/ca.crt" "${KURYR_HYPERKUBE_DATA_DIR}/kuryr-ca.crt"
|
||||
|
||||
# FIXME(ivc): replace 'sleep' with a strict check (e.g. wait_for_files)
|
||||
# 'kubernetes-api' fails if started before files are generated.
|
||||
# this is a workaround to prevent races.
|
||||
@ -456,7 +480,7 @@ function wait_for {
|
||||
extra_flags=${cacert_path:+"--cacert ${cacert_path}"}
|
||||
|
||||
local start_time=$(date +%s)
|
||||
until curl -o /dev/null -sf $extra_flags "$url"; do
|
||||
until curl -o /dev/null -s $extra_flags "$url"; do
|
||||
echo -n "."
|
||||
local curr_time=$(date +%s)
|
||||
local time_diff=$(($curr_time - $start_time))
|
||||
@ -631,13 +655,14 @@ function run_k8s_kubelet {
|
||||
|
||||
function run_kuryr_kubernetes {
|
||||
local python_bin=$(which python)
|
||||
|
||||
if is_service_enabled openshift-master; then
|
||||
wait_for "OpenShift API Server" "$OPENSHIFT_API_URL" \
|
||||
"${OPENSHIFT_DATA_DIR}/ca.crt"
|
||||
wait_for "OpenShift API Server" "$KURYR_K8S_API_LB_URL" \
|
||||
"${OPENSHIFT_DATA_DIR}/ca.crt" 1200
|
||||
else
|
||||
wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL"
|
||||
wait_for "Kubernetes API Server" "$KURYR_K8S_API_LB_URL" \
|
||||
"${KURYR_HYPERKUBE_DATA_DIR}/kuryr-ca.crt" 1200
|
||||
fi
|
||||
|
||||
run_process kuryr-kubernetes \
|
||||
"$python_bin ${KURYR_HOME}/scripts/run_server.py \
|
||||
--config-file $KURYR_CONFIG"
|
||||
@ -791,12 +816,6 @@ if [[ "$1" == "stack" && "$2" == "extra" ]]; then
|
||||
run_k8s_scheduler
|
||||
fi
|
||||
|
||||
KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT)
|
||||
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "False" ]; then
|
||||
# If running in containerized mode, we'll run the daemon as DaemonSet.
|
||||
run_kuryr_daemon
|
||||
fi
|
||||
|
||||
if is_service_enabled kubelet; then
|
||||
prepare_kubelet
|
||||
extract_hyperkube
|
||||
@ -813,6 +832,7 @@ if [[ "$1" == "stack" && "$2" == "extra" ]]; then
|
||||
configure_k8s_pod_sg_rules
|
||||
fi
|
||||
|
||||
KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT)
|
||||
if is_service_enabled kuryr-kubernetes; then
|
||||
/usr/local/bin/kubectl apply -f ${KURYR_HOME}/kubernetes_crds/kuryrnet.yaml
|
||||
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then
|
||||
@ -831,7 +851,6 @@ elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
|
||||
if is_service_enabled kuryr-kubernetes; then
|
||||
# NOTE(dulek): This is so late, because Devstack's Octavia is unable
|
||||
# to create loadbalancers until test-config phase.
|
||||
local use_octavia
|
||||
use_octavia=$(trueorfalse True KURYR_K8S_LBAAS_USE_OCTAVIA)
|
||||
if [[ "$use_octavia" == "False" ]]; then
|
||||
create_k8s_router_fake_service
|
||||
@ -846,7 +865,7 @@ elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
|
||||
|
||||
# FIXME(dulek): This is a very late phase to start Kuryr services.
|
||||
# We're doing it here because we need K8s API LB to be
|
||||
# created in order to run kuryr-cni container. Thing is
|
||||
# created in order to run kuryr services. Thing is
|
||||
# Octavia is unable to create LB until test-config phase.
|
||||
# We can revisit this once Octavia's DevStack plugin will
|
||||
# get improved.
|
||||
@ -854,6 +873,7 @@ elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
|
||||
run_containerized_kuryr_resources
|
||||
else
|
||||
run_kuryr_kubernetes
|
||||
run_kuryr_daemon
|
||||
fi
|
||||
fi
|
||||
if is_service_enabled tempest && [[ "$KURYR_USE_PORT_POOLS" == "True" ]]; then
|
||||
|
||||
@ -40,10 +40,12 @@ KURYR_HYPERKUBE_BINARY=${KURYR_HYPERKUBE_BINARY:-/usr/local/bin/hyperkube}
|
||||
|
||||
# Kubernetes
|
||||
KURYR_K8S_API_PORT=${KURYR_K8S_API_PORT:-8080}
|
||||
KURYR_K8S_API_URL=${KURYR_K8S_API_URL:-http://${HOST_IP}:${KURYR_K8S_API_PORT}}
|
||||
KURYR_K8S_API_CERT=${KURYR_K8S_API_CERT:-}
|
||||
KURYR_K8S_API_KEY=${KURYR_K8S_API_KEY:-}
|
||||
KURYR_K8S_API_CACERT=${KURYR_K8S_API_CACERT:-}
|
||||
# NOTE(dulek): [kubernetes]api_root option will use LB IP instead.
|
||||
KURYR_K8S_API_URL=${KURYR_K8S_API_URL:-"http://${HOST_IP}:${KURYR_K8S_API_PORT}"}
|
||||
KURYR_K8S_API_CERT=${KURYR_K8S_API_CERT:-"${KURYR_HYPERKUBE_DATA_DIR}/kuryr.crt"}
|
||||
KURYR_K8S_API_KEY=${KURYR_K8S_API_KEY:-"${KURYR_HYPERKUBE_DATA_DIR}/kuryr.key"}
|
||||
KURYR_K8S_API_CACERT=${KURYR_K8S_API_CACERT:-"${KURYR_HYPERKUBE_DATA_DIR}/kuryr-ca.crt"}
|
||||
KURYR_K8S_API_LB_PORT=${KURYR_K8S_API_LB_PORT:-443}
|
||||
KURYR_PORT_DEBUG=${KURYR_PORT_DEBUG:-True}
|
||||
KURYR_SUBNET_DRIVER=${KURYR_SUBNET_DRIVER:-default}
|
||||
KURYR_ENABLED_HANDLERS=${KURYR_ENABLED_HANDLERS:-vif,lb,lbaasspec}
|
||||
|
||||
@ -13,6 +13,8 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
|
||||
from kuryr.lib import utils
|
||||
|
||||
from kuryr_kubernetes import config
|
||||
@ -68,5 +70,12 @@ def setup_loadbalancer_client():
|
||||
|
||||
|
||||
def setup_kubernetes_client():
|
||||
_clients[_KUBERNETES_CLIENT] = k8s_client.K8sClient(
|
||||
config.CONF.kubernetes.api_root)
|
||||
if config.CONF.kubernetes.api_root:
|
||||
api_root = config.CONF.kubernetes.api_root
|
||||
else:
|
||||
# NOTE(dulek): This is for containerized deployments, i.e. running in
|
||||
# K8s Pods.
|
||||
host = os.environ['KUBERNETES_SERVICE_HOST']
|
||||
port = os.environ['KUBERNETES_SERVICE_PORT_HTTPS']
|
||||
api_root = "https://%s:%s" % (host, port)
|
||||
_clients[_KUBERNETES_CLIENT] = k8s_client.K8sClient(api_root)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user