diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml index e3d626c84..c7a691a6d 100644 --- a/.zuul.d/base.yaml +++ b/.zuul.d/base.yaml @@ -86,6 +86,7 @@ etcd3: true kubernetes-master: true kuryr-kubernetes: true + kuryr-daemon: true coredns: false neutron-tag-ports-during-bulk-creation: true zuul_copy_output: @@ -158,6 +159,7 @@ etcd3: true kubernetes-master: true kuryr-kubernetes: true + kuryr-daemon: true coredns: false neutron-tag-ports-during-bulk-creation: true zuul_copy_output: diff --git a/.zuul.d/multinode.yaml b/.zuul.d/multinode.yaml index 73b5062c3..6fbed9249 100644 --- a/.zuul.d/multinode.yaml +++ b/.zuul.d/multinode.yaml @@ -32,6 +32,7 @@ devstack_services: # Need to disable dstat due to bug https://github.com/dstat-real/dstat/pull/162 dstat: false + kuryr-daemon: true kuryr-kubernetes: false kubernetes-worker: true kubernetes-master: false @@ -43,11 +44,14 @@ USE_PYTHON3: true vars: devstack_localrc: - KURYR_K8S_API_URL: "http://${SERVICE_HOST}:${KURYR_K8S_API_PORT}" + KURYR_K8S_API_URL: "https://${SERVICE_HOST}:${KURYR_K8S_API_PORT}" KURYR_K8S_MULTI_WORKER_TESTS: True devstack_services: tls-proxy: false - kubernetes-worker: true + kubernetes-worker: false + kubernetes-master: true + kuryr-daemon: true + kuryr-kubernetes: true voting: false - job: diff --git a/devstack/lib/kubernetes b/devstack/lib/kubernetes index 08774a899..d89565502 100644 --- a/devstack/lib/kubernetes +++ b/devstack/lib/kubernetes @@ -124,6 +124,44 @@ EOF safe_chown $STACK_USER:$STACK_USER $kube_config_file } +function kubeadm_join { + local output_dir="${DATA_DIR}/kuryr-kubernetes" + + mkdir -p "${output_dir}" + + cluster_ip_ranges=() + for service_subnet_id in ${KURYR_SERVICE_SUBNETS_IDS[@]}; do + service_cidr=$(openstack --os-cloud devstack-admin \ + --os-region "$REGION_NAME" \ + subnet show "$service_subnet_id" \ + -c cidr -f value) + cluster_ip_ranges+=($(split_subnet "$service_cidr" | cut -f1)) + done + + # TODO(gryf): take care of cri-o case aswell + rm -f ${output_dir}/kubeadm-join.yaml + cat >> ${output_dir}/kubeadm-join.yaml << EOF +apiVersion: kubeadm.k8s.io/v1beta2 +discovery: + bootstrapToken: + apiServerEndpoint: ${SERVICE_HOST}:${KURYR_K8S_API_PORT} + token: "${KURYR_K8S_TOKEN}" + unsafeSkipCAVerification: true + tlsBootstrapToken: "${KURYR_K8S_TOKEN}" +kind: JoinConfiguration +nodeRegistration: + kubeletExtraArgs: + cgroup-driver: "$(docker info -f '{{.CgroupDriver}}')" + cni-bin-dir: "$CNI_BIN_DIR" + cni-conf-dir: "$CNI_CONF_DIR" + enable-server: "true" + taints: + [] +EOF + sudo -E kubeadm join --ignore-preflight-errors Swap \ + --config ${output_dir}/kubeadm-join.yaml +} + function get_k8s_apiserver { # assumption is, there is no other cluster, so there is only one API # server. diff --git a/devstack/lib/kuryr_kubernetes b/devstack/lib/kuryr_kubernetes index dddbc6f18..90fdcb74c 100644 --- a/devstack/lib/kuryr_kubernetes +++ b/devstack/lib/kuryr_kubernetes @@ -272,11 +272,12 @@ function create_k8s_subnet { echo "$subnet_id" } -# build_kuryr_containers -# Description: Generates a Kuryr controller and Kuryr CNI docker images in -# the local docker registry as kuryr/controller:latest and -# kuryr/cni:latest respectively -function build_kuryr_containers { +# build_kuryr_container_image +# Description: Generates a Kuryr controller or Kuryr CNI docker image in +# the local docker registry as kuryr/controller:latest for controller or +# kuryr/cni:latest for CNI. +function build_kuryr_container_image { + local target=$1 # controller or cni local build_args local build_dir @@ -293,11 +294,11 @@ function build_kuryr_containers { # resolved instead of podman we need to use buildah directly, # hence this awful if clause. if [[ ${CONTAINER_ENGINE} == 'crio' ]]; then - sudo buildah bud -t docker.io/kuryr/controller -f controller.Dockerfile . - sudo buildah bud -t docker.io/kuryr/cni -f cni.Dockerfile . + sudo buildah bud -t "docker.io/kuryr/${target}" \ + -f "${target}.Dockerfile" . else - container_runtime build -t kuryr/controller -f controller.Dockerfile ${build_args} . - container_runtime build -t kuryr/cni -f cni.Dockerfile ${build_args} . + container_runtime build -t "kuryr/${target}" \ + -f "${target}.Dockerfile" ${build_args} . fi popd } diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 77d67ae4d..98609015e 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -60,10 +60,9 @@ if is_service_enabled kuryr-kubernetes kuryr-daemon \ build_install_kuryr_cni fi - elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then echo_summary "Configure kuryr bits" - if is_service_enabled kuryr-kubernetes; then + if is_service_enabled kuryr-daemon; then create_kuryr_account configure_kuryr fi @@ -71,7 +70,7 @@ if is_service_enabled kuryr-kubernetes kuryr-daemon \ elif [[ "$1" == "stack" && "$2" == "extra" ]]; then echo_summary "Installing kubernetes and kuryr" # Initialize and start the template service - if is_service_enabled kubernetes-master; then + if is_service_enabled kuryr-kubernetes; then configure_neutron_defaults fi @@ -79,14 +78,23 @@ if is_service_enabled kuryr-kubernetes kuryr-daemon \ prepare_kubelet fi - if is_service_enabled kubernetes-master kubernetes-worker; then + if is_service_enabled kubernetes-master; then wait_for "etcd" "http://${SERVICE_HOST}:${ETCD_PORT}/v2/machines" kubeadm_init copy_kuryr_certs fi + if is_service_enabled kubernetes-worker; then + kubeadm_join + fi + if [ "${KURYR_CONT}" == "True" ]; then - build_kuryr_containers + if is_service_enabled kubernetes-master; then + build_kuryr_container_image "controller" + build_kuryr_container_image "cni" + else + build_kuryr_container_image "cni" + fi fi if is_service_enabled kubernetes-master; then @@ -120,21 +128,23 @@ if is_service_enabled kuryr-kubernetes kuryr-daemon \ elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then echo_summary "Run kuryr-kubernetes" - if is_service_enabled octavia; then - create_lb_for_services - fi + if is_service_enabled kuryr-kubernetes; then + if is_service_enabled octavia; then + create_lb_for_services + fi - # FIXME(dulek): This is a very late phase to start Kuryr services. - # We're doing it here because we need K8s API LB to be - # created in order to run kuryr services. Thing is - # Octavia is unable to create LB until test-config phase. - # We can revisit this once Octavia's DevStack plugin will - # get improved. - if [ "${KURYR_CONT}" == "True" ]; then - run_containerized_kuryr_resources - else - run_kuryr_kubernetes - run_kuryr_daemon + # FIXME(dulek): This is a very late phase to start Kuryr services. + # We're doing it here because we need K8s API LB to be created in + # order to run kuryr services. Thing is Octavia is unable to + # create LB until test-config phase. We can revisit this once + # Octavia's DevStack plugin will get improved. + + if [ "${KURYR_CONT}" == "True" ]; then + run_containerized_kuryr_resources + else + run_kuryr_kubernetes + run_kuryr_daemon + fi fi if is_service_enabled tempest; then