From fbea6d2d35f6ccb115f4d8d8c47af0f760b35694 Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Thu, 26 Jun 2025 14:58:16 -0500 Subject: [PATCH] [octavia] Add test case for load balancer We've been merging quite a few improvements for Ocatavia chart recently but we've been skipping testing them. This PS adds the Octavia test case which tests the simplest load balancing env with two workload instances and one amphora instance. The PS also brings some changes to the Octavia chart: - Run driver agent as a separate deployment on network nodes - Run worker as a daemonset (same as health manager) on network nodes. It creates an interface attached to the Octavia management network to get access to amophora instances. Change-Id: Id12e30eb7aac432e3f12b83e1f93d98e54c503cf Signed-off-by: Vladimir Kozhukalov --- .../bin/_octavia-worker-get-port.sh.tpl | 26 ++ .../bin/_octavia-worker-nic-init.sh.tpl | 31 ++ octavia/templates/bin/_octavia-worker.sh.tpl | 7 + octavia/templates/configmap-bin.yaml | 4 + ...ment-worker.yaml => daemonset-worker.yaml} | 72 +++- octavia/templates/deployment-api.yaml | 48 --- .../templates/deployment-driver-agent.yaml | 117 ++++++ octavia/values.yaml | 100 +++-- .../notes/octavia-73c0f7c8c13c00a1.yaml | 8 + roles/deploy-env/files/calico_patch.yaml | 2 +- roles/deploy-env/tasks/env_inventory.yaml | 31 ++ roles/deploy-env/tasks/main.yaml | 6 + .../tasks/openstack_provider_gateway.yaml | 4 + roles/deploy-env/tasks/prerequisites.yaml | 1 + tools/deployment/common/prepare-bashrc.sh | 24 ++ .../180-create-resource-for-octavia.sh | 84 ---- .../component/octavia/200-octavia.sh | 121 ------ .../octavia/create_dual_intermediate_CA.sh | 157 ++++++++ .../component/octavia/heat_octavia_env.yaml | 358 ++++++++++++++++++ tools/deployment/component/octavia/octavia.sh | 186 +++++++++ ...eate-octavia-certs.sh => octavia_certs.sh} | 21 +- .../component/octavia/octavia_resources.sh | 116 ++++++ .../component/octavia/octavia_test.sh | 48 +++ .../deployment/component/octavia/openssl.cnf | 144 +++++++ .../octavia/2025.1-ubuntu_jammy.yaml | 23 ++ .../octavia/2025.1-ubuntu_noble.yaml | 23 ++ zuul.d/2025.1.yaml | 11 + zuul.d/base.yaml | 41 +- zuul.d/nodesets.yaml | 35 ++ zuul.d/project.yaml | 1 + 30 files changed, 1545 insertions(+), 305 deletions(-) create mode 100644 octavia/templates/bin/_octavia-worker-get-port.sh.tpl create mode 100644 octavia/templates/bin/_octavia-worker-nic-init.sh.tpl rename octavia/templates/{deployment-worker.yaml => daemonset-worker.yaml} (60%) create mode 100644 octavia/templates/deployment-driver-agent.yaml create mode 100644 releasenotes/notes/octavia-73c0f7c8c13c00a1.yaml create mode 100644 roles/deploy-env/tasks/env_inventory.yaml create mode 100755 tools/deployment/common/prepare-bashrc.sh delete mode 100755 tools/deployment/component/octavia/180-create-resource-for-octavia.sh delete mode 100755 tools/deployment/component/octavia/200-octavia.sh create mode 100755 tools/deployment/component/octavia/create_dual_intermediate_CA.sh create mode 100644 tools/deployment/component/octavia/heat_octavia_env.yaml create mode 100755 tools/deployment/component/octavia/octavia.sh rename tools/deployment/component/octavia/{190-create-octavia-certs.sh => octavia_certs.sh} (64%) create mode 100755 tools/deployment/component/octavia/octavia_resources.sh create mode 100755 tools/deployment/component/octavia/octavia_test.sh create mode 100644 tools/deployment/component/octavia/openssl.cnf create mode 100644 values_overrides/octavia/2025.1-ubuntu_jammy.yaml create mode 100644 values_overrides/octavia/2025.1-ubuntu_noble.yaml diff --git a/octavia/templates/bin/_octavia-worker-get-port.sh.tpl b/octavia/templates/bin/_octavia-worker-get-port.sh.tpl new file mode 100644 index 0000000000..7d14a92700 --- /dev/null +++ b/octavia/templates/bin/_octavia-worker-get-port.sh.tpl @@ -0,0 +1,26 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +HOSTNAME=$(hostname -s) +PORTNAME=octavia-worker-port-$HOSTNAME + +HM_PORT_ID=$(openstack port show $PORTNAME -c id -f value) +HM_PORT_MAC=$(openstack port show $PORTNAME -c mac_address -f value) + +echo $HM_PORT_ID > /tmp/pod-shared/HM_PORT_ID +echo $HM_PORT_MAC > /tmp/pod-shared/HM_PORT_MAC diff --git a/octavia/templates/bin/_octavia-worker-nic-init.sh.tpl b/octavia/templates/bin/_octavia-worker-nic-init.sh.tpl new file mode 100644 index 0000000000..f1ac3c40cc --- /dev/null +++ b/octavia/templates/bin/_octavia-worker-nic-init.sh.tpl @@ -0,0 +1,31 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +HM_PORT_ID=$(cat /tmp/pod-shared/HM_PORT_ID) +HM_PORT_MAC=$(cat /tmp/pod-shared/HM_PORT_MAC) + +ovs-vsctl --no-wait show + +ovs-vsctl --may-exist add-port br-int o-w0 \ + -- set Interface o-w0 type=internal \ + -- set Interface o-w0 external-ids:iface-status=active \ + -- set Interface o-w0 external-ids:attached-mac=$HM_PORT_MAC \ + -- set Interface o-w0 external-ids:iface-id=$HM_PORT_ID \ + -- set Interface o-w0 external-ids:skip_cleanup=true + +ip link set dev o-w0 address $HM_PORT_MAC diff --git a/octavia/templates/bin/_octavia-worker.sh.tpl b/octavia/templates/bin/_octavia-worker.sh.tpl index f612ff38e9..db5a2a5f4c 100644 --- a/octavia/templates/bin/_octavia-worker.sh.tpl +++ b/octavia/templates/bin/_octavia-worker.sh.tpl @@ -20,6 +20,13 @@ set -ex COMMAND="${@:-start}" function start () { + cat > /tmp/dhclient.conf < /tmp/inventory_default_dev.txt + + echo -n > /tmp/inventory_k8s_control_plane.txt + {% for host in (groups['k8s_control_plane'] | default([])) %} + echo {{ hostvars[host].ansible_hostname }} >> /tmp/inventory_k8s_control_plane.txt + {% endfor %} + + echo -n > /tmp/inventory_k8s_nodes.txt + {% for host in (groups['k8s_nodes'] | default([])) %} + echo {{ hostvars[host].ansible_hostname }} >> /tmp/inventory_k8s_nodes.txt + {% endfor %} +... diff --git a/roles/deploy-env/tasks/main.yaml b/roles/deploy-env/tasks/main.yaml index 9705890694..21f0e8c2d8 100644 --- a/roles/deploy-env/tasks/main.yaml +++ b/roles/deploy-env/tasks/main.yaml @@ -108,4 +108,10 @@ when: - ingress_setup - inventory_hostname in (groups['primary'] | default([])) + +- name: Include env inventory tasks + include_tasks: + file: env_inventory.yaml + when: + - inventory_hostname in (groups['primary'] | default([])) ... diff --git a/roles/deploy-env/tasks/openstack_provider_gateway.yaml b/roles/deploy-env/tasks/openstack_provider_gateway.yaml index b1ab7b3cc6..67e480b38c 100644 --- a/roles/deploy-env/tasks/openstack_provider_gateway.yaml +++ b/roles/deploy-env/tasks/openstack_provider_gateway.yaml @@ -26,6 +26,10 @@ shell: | iptables -t nat -A POSTROUTING -o {{ cluster_default_dev }} -s {{ openstack_provider_network_cidr }} -j MASQUERADE +- name: Set up FORWARD for packets going from VMs + shell: | + iptables -t filter -I FORWARD -s {{ openstack_provider_network_cidr }} -j ACCEPT + # We use tcp proxy to forward traffic to make it possible to connect # to the Openstack public endpoint (managed by Metallb) from VMs. - name: Setup TCP proxy diff --git a/roles/deploy-env/tasks/prerequisites.yaml b/roles/deploy-env/tasks/prerequisites.yaml index f792336bb6..26295d1ff0 100644 --- a/roles/deploy-env/tasks/prerequisites.yaml +++ b/roles/deploy-env/tasks/prerequisites.yaml @@ -35,6 +35,7 @@ - git - git-review - gnupg2 + - htop - iptables - ipvsadm - jq diff --git a/tools/deployment/common/prepare-bashrc.sh b/tools/deployment/common/prepare-bashrc.sh new file mode 100755 index 0000000000..5f4137d48f --- /dev/null +++ b/tools/deployment/common/prepare-bashrc.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +cat >> ${HOME}/.bashrc < /tmp/octavia_hm_controller_ip_port_list - -# Create a flavor for amphora instance -openstack flavor create --id auto --ram 1024 --disk 2 --vcpus 1 --private m1.amphora - -# Create key pair to connect amphora instance via management network -ssh-keygen -b 2048 -t rsa -N '' -f ~/.ssh/octavia_ssh_key -openstack keypair create --public-key ~/.ssh/octavia_ssh_key.pub octavia_ssh_key - -# Create amphora image from file. Default is https://tarballs.openstack.org/octavia/test-images/ -if [ "$OSH_AMPHORA_IMAGE_FILE_PATH" == "" ]; then - curl https://tarballs.openstack.org/octavia/test-images/test-only-amphora-x64-haproxy-ubuntu-xenial.qcow2 \ - -o /tmp/test-only-amphora-x64-haproxy-ubuntu-xenial.qcow2 - - OSH_AMPHORA_IMAGE_FILE_PATH=/tmp/test-only-amphora-x64-haproxy-ubuntu-xenial.qcow2 -fi - -OSH_AMPHORA_IMAGE_ID=$(openstack image create -f value -c id \ - --public \ - --container-format=bare \ - --disk-format qcow2 < $OSH_AMPHORA_IMAGE_FILE_PATH \ - $OSH_LB_AMPHORA_IMAGE_NAME) -openstack image set --tag amphora $OSH_AMPHORA_IMAGE_ID diff --git a/tools/deployment/component/octavia/200-octavia.sh b/tools/deployment/component/octavia/200-octavia.sh deleted file mode 100755 index 64442527f5..0000000000 --- a/tools/deployment/component/octavia/200-octavia.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/bin/bash - -# Copyright 2019 Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -export OS_CLOUD=openstack_helm - -: ${OSH_LB_AMPHORA_IMAGE_NAME:="amphora-x64-haproxy"} -: ${OSH_LB_HM_HOST_PORT:="5555"} - -#NOTE: Deploy command -: ${OSH_EXTRA_HELM_ARGS:=""} -tee /tmp/octavia.yaml < serial + +# Create the client CA private key +openssl genpkey -algorithm RSA -out private/ca.key.pem -aes-128-cbc -pass pass:not-secure-passphrase +chmod 400 private/ca.key.pem + +# Create the client CA root certificate +openssl req -config ${OPENSSL_CONF} -key private/ca.key.pem -new -x509 -sha256 -extensions v3_ca -days 7300 -out certs/ca.cert.pem -subj "/C=US/ST=Oregon/L=Corvallis/O=OpenStack/OU=Octavia/CN=ClientRootCA" -passin pass:not-secure-passphrase + +###### Client Intermediate CA +mkdir intermediate_ca +mkdir intermediate_ca/certs intermediate_ca/crl intermediate_ca/newcerts intermediate_ca/private +chmod 700 intermediate_ca/private +touch intermediate_ca/index.txt +echo 1000 > intermediate_ca/serial + +# Create the client intermediate CA private key +openssl genpkey -algorithm RSA -out intermediate_ca/private/intermediate.ca.key.pem -aes-128-cbc -pass pass:not-secure-passphrase +chmod 400 intermediate_ca/private/intermediate.ca.key.pem + +# Create the client intermediate CA certificate signing request +openssl req -config ${OPENSSL_CONF} -key intermediate_ca/private/intermediate.ca.key.pem -new -sha256 -out intermediate_ca/client_intermediate.csr -subj "/C=US/ST=Oregon/L=Corvallis/O=OpenStack/OU=Octavia/CN=ClientIntermediateCA" -passin pass:not-secure-passphrase + +# Create the client intermediate CA certificate +openssl ca -config ${OPENSSL_CONF} -name CA_intermediate -extensions v3_intermediate_ca -days 3650 -notext -md sha256 -in intermediate_ca/client_intermediate.csr -out intermediate_ca/certs/intermediate.cert.pem -passin pass:not-secure-passphrase -batch + +# Create the client CA certificate chain +cat intermediate_ca/certs/intermediate.cert.pem certs/ca.cert.pem > intermediate_ca/ca-chain.cert.pem + +###### Create the client key and certificate +openssl genpkey -algorithm RSA -out intermediate_ca/private/controller.key.pem -aes-128-cbc -pass pass:not-secure-passphrase +chmod 400 intermediate_ca/private/controller.key.pem + +# Create the client controller certificate signing request +openssl req -config ${OPENSSL_CONF} -key intermediate_ca/private/controller.key.pem -new -sha256 -out intermediate_ca/controller.csr -subj "/C=US/ST=Oregon/L=Corvallis/O=OpenStack/OU=Octavia/CN=OctaviaController" -passin pass:not-secure-passphrase + +# Create the client controller certificate +openssl ca -config ${OPENSSL_CONF} -name CA_intermediate -extensions usr_cert -days 1825 -notext -md sha256 -in intermediate_ca/controller.csr -out intermediate_ca/certs/controller.cert.pem -passin pass:not-secure-passphrase -batch + +# Build the cancatenated client cert and key +openssl rsa -in intermediate_ca/private/controller.key.pem -out intermediate_ca/private/client.cert-and-key.pem -passin pass:not-secure-passphrase + +cat intermediate_ca/certs/controller.cert.pem >> intermediate_ca/private/client.cert-and-key.pem + +# We are done with the client CA +cd .. + +###### Stash the octavia default client CA cert files +cp client_ca/intermediate_ca/ca-chain.cert.pem etc/octavia/certs/client_ca.cert.pem +chmod 444 etc/octavia/certs/client_ca.cert.pem +cp client_ca/intermediate_ca/private/client.cert-and-key.pem etc/octavia/certs/client.cert-and-key.pem +chmod 600 etc/octavia/certs/client.cert-and-key.pem + +###### Server Root CA +mkdir server_ca +cd server_ca +mkdir certs crl newcerts private +chmod 700 private +touch index.txt +echo 1000 > serial + +# Create the server CA private key +openssl genpkey -algorithm RSA -out private/ca.key.pem -aes-128-cbc -pass pass:not-secure-passphrase +chmod 400 private/ca.key.pem + +# Create the server CA root certificate +openssl req -config ${OPENSSL_CONF} -key private/ca.key.pem -new -x509 -sha256 -extensions v3_ca -days 7300 -out certs/ca.cert.pem -subj "/C=US/ST=Oregon/L=Corvallis/O=OpenStack/OU=Octavia/CN=ServerRootCA" -passin pass:not-secure-passphrase + +###### Server Intermediate CA +mkdir intermediate_ca +mkdir intermediate_ca/certs intermediate_ca/crl intermediate_ca/newcerts intermediate_ca/private +chmod 700 intermediate_ca/private +touch intermediate_ca/index.txt +echo 1000 > intermediate_ca/serial + +# Create the server intermediate CA private key +openssl genpkey -algorithm RSA -out intermediate_ca/private/intermediate.ca.key.pem -aes-128-cbc -pass pass:not-secure-passphrase +chmod 400 intermediate_ca/private/intermediate.ca.key.pem + +# Create the server intermediate CA certificate signing request +openssl req -config ${OPENSSL_CONF} -key intermediate_ca/private/intermediate.ca.key.pem -new -sha256 -out intermediate_ca/server_intermediate.csr -subj "/C=US/ST=Oregon/L=Corvallis/O=OpenStack/OU=Octavia/CN=ServerIntermediateCA" -passin pass:not-secure-passphrase + +# Create the server intermediate CA certificate +openssl ca -config ${OPENSSL_CONF} -name CA_intermediate -extensions v3_intermediate_ca -days 3650 -notext -md sha256 -in intermediate_ca/server_intermediate.csr -out intermediate_ca/certs/intermediate.cert.pem -passin pass:not-secure-passphrase -batch + +# Create the server CA certificate chain +cat intermediate_ca/certs/intermediate.cert.pem certs/ca.cert.pem > intermediate_ca/ca-chain.cert.pem + +# We are done with the server CA +cd .. + +###### Stash the octavia default server CA cert files +cp server_ca/intermediate_ca/ca-chain.cert.pem etc/octavia/certs/server_ca-chain.cert.pem +chmod 444 etc/octavia/certs/server_ca-chain.cert.pem +cp server_ca/intermediate_ca/certs/intermediate.cert.pem etc/octavia/certs/server_ca.cert.pem +chmod 400 etc/octavia/certs/server_ca.cert.pem +cp server_ca/intermediate_ca/private/intermediate.ca.key.pem etc/octavia/certs/server_ca.key.pem +chmod 400 etc/octavia/certs/server_ca.key.pem + +##### Validate the Octavia PKI files +set +x +echo "################# Verifying the Octavia files ###########################" +openssl verify -CAfile etc/octavia/certs/client_ca.cert.pem etc/octavia/certs/client.cert-and-key.pem +openssl verify -CAfile etc/octavia/certs/server_ca-chain.cert.pem etc/octavia/certs/server_ca.cert.pem + +# We are done, stop enforcing shell errexit +set +e + +echo "!!!!!!!!!!!!!!!Do not use this script for deployments!!!!!!!!!!!!!" +echo "Please use the Octavia Certificate Configuration guide:" +echo "https://docs.openstack.org/octavia/latest/admin/guides/certificates.html" +echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" diff --git a/tools/deployment/component/octavia/heat_octavia_env.yaml b/tools/deployment/component/octavia/heat_octavia_env.yaml new file mode 100644 index 0000000000..d4bb4a05d3 --- /dev/null +++ b/tools/deployment/component/octavia/heat_octavia_env.yaml @@ -0,0 +1,358 @@ +--- +heat_template_version: 2021-04-16 + +parameters: + public_network_name: + type: string + default: public + + public_physical_network_name: + type: string + default: public + + public_subnet_name: + type: string + default: public + + public_subnet_cidr: + type: string + default: 172.24.4.0/24 + + public_subnet_gateway: + type: string + default: 172.24.4.1 + + public_allocation_pool_start: + type: string + default: 172.24.4.10 + + public_allocation_pool_end: + type: string + default: 172.24.4.254 + + private_subnet_cidr: + type: string + default: 192.168.128.0/24 + + dns_nameserver: + type: string + default: 172.24.4.1 + + image_name: + type: string + default: Ubuntu Jammy + + image_url: + type: string + default: "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img" + + ssh_key: + type: string + default: octavia-key + + compute_flavor_id: + type: string + + az_1: + type: string + + az_2: + type: string + +resources: + public_net: + type: OS::Neutron::ProviderNet + properties: + name: + get_param: public_network_name + router_external: true + physical_network: + get_param: public_physical_network_name + network_type: flat + + public_subnet: + type: OS::Neutron::Subnet + properties: + name: + get_param: public_subnet_name + network: + get_resource: public_net + cidr: + get_param: public_subnet_cidr + gateway_ip: + get_param: public_subnet_gateway + enable_dhcp: false + dns_nameservers: + - get_param: public_subnet_gateway + allocation_pools: + - start: {get_param: public_allocation_pool_start} + end: {get_param: public_allocation_pool_end} + + private_net: + type: OS::Neutron::Net + + private_subnet: + type: OS::Neutron::Subnet + properties: + network: + get_resource: private_net + cidr: + get_param: private_subnet_cidr + dns_nameservers: + - get_param: dns_nameserver + + image: + type: OS::Glance::WebImage + properties: + name: + get_param: image_name + location: + get_param: image_url + container_format: bare + disk_format: qcow2 + min_disk: 3 + visibility: public + + flavor_vm: + type: OS::Nova::Flavor + properties: + name: m1.test + disk: 3 + ram: 1024 + vcpus: 2 + + wait_handle_1: + type: OS::Heat::WaitConditionHandle + + wait_handle_2: + type: OS::Heat::WaitConditionHandle + + server_1: + type: OS::Nova::Server + properties: + image: + get_resource: image + flavor: + get_resource: flavor_vm + key_name: + get_param: ssh_key + networks: + - port: + get_resource: server_port_1 + user_data_format: RAW + user_data: + str_replace: + template: | + #!/bin/bash + echo "nameserver $nameserver" > /etc/resolv.conf + echo "127.0.0.1 $(hostname)" >> /etc/hosts + systemctl stop systemd-resolved + systemctl disable systemd-resolved + mkdir -p /var/www/html/ + echo "Hello from server_1: $(hostname)" > /var/www/html/index.html + nohup python3 -m http.server 8000 --directory /var/www/html > /dev/null 2>&1 & + $wc_notify --data-binary '{ "status": "SUCCESS" }' + params: + $nameserver: {get_param: dns_nameserver} + $wc_notify: {get_attr: ['wait_handle_1', 'curl_cli']} + availability_zone: {get_param: az_1} + + wait_server_1: + type: OS::Heat::WaitCondition + properties: + handle: {get_resource: wait_handle_1} + timeout: 1200 + + server_2: + type: OS::Nova::Server + properties: + image: + get_resource: image + flavor: + get_resource: flavor_vm + key_name: + get_param: ssh_key + networks: + - port: + get_resource: server_port_2 + user_data_format: RAW + user_data: + str_replace: + template: | + #!/bin/bash + echo "nameserver $nameserver" > /etc/resolv.conf + echo "127.0.0.1 $(hostname)" >> /etc/hosts + systemctl stop systemd-resolved + systemctl disable systemd-resolved + mkdir -p /var/www/html/ + echo "Hello from server_2: $(hostname)" > /var/www/html/index.html + nohup python3 -m http.server 8000 --directory /var/www/html > /dev/null 2>&1 & + $wc_notify --data-binary '{ "status": "SUCCESS" }' + params: + $nameserver: {get_param: dns_nameserver} + $wc_notify: {get_attr: ['wait_handle_2', 'curl_cli']} + availability_zone: {get_param: az_2} + + wait_server_2: + type: OS::Heat::WaitCondition + properties: + handle: {get_resource: wait_handle_2} + timeout: 1200 + + security_group: + type: OS::Neutron::SecurityGroup + properties: + name: default_port_security_group + rules: + - remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: 22 + port_range_max: 22 + - remote_ip_prefix: 0.0.0.0/0 + protocol: tcp + port_range_min: 8000 + port_range_max: 8000 + - remote_ip_prefix: 0.0.0.0/0 + protocol: icmp + + server_port_1: + type: OS::Neutron::Port + properties: + network: + get_resource: private_net + fixed_ips: + - subnet: + get_resource: private_subnet + security_groups: + - get_resource: security_group + + server_floating_ip_1: + type: OS::Neutron::FloatingIP + properties: + floating_network: + get_resource: public_net + port_id: + get_resource: server_port_1 + + server_port_2: + type: OS::Neutron::Port + properties: + network: + get_resource: private_net + fixed_ips: + - subnet: + get_resource: private_subnet + security_groups: + - get_resource: security_group + + server_floating_ip_2: + type: OS::Neutron::FloatingIP + properties: + floating_network: + get_resource: public_net + port_id: + get_resource: server_port_2 + + router: + type: OS::Neutron::Router + properties: + external_gateway_info: + network: + get_resource: public_net + + router_interface: + type: OS::Neutron::RouterInterface + properties: + router_id: + get_resource: router + subnet_id: + get_resource: private_subnet + + flavor_profile: + type: "OS::Octavia::FlavorProfile" + properties: + provider_name: amphora + flavor_data: + str_replace: + template: | + { + "loadbalancer_topology": "SINGLE", + "compute_flavor": "%compute_flavor%" + } + params: + "%compute_flavor%": {get_param: compute_flavor_id} + + flavor: + type: "OS::Octavia::Flavor" + properties: + flavor_profile: + get_resource: flavor_profile + + loadbalancer: + type: "OS::Octavia::LoadBalancer" + properties: + name: osh + provider: amphora + vip_subnet: + get_resource: private_subnet + flavor: + get_resource: flavor + + floating_ip: + type: OS::Neutron::FloatingIP + properties: + floating_network: {get_resource: public_net} + port_id: {get_attr: [loadbalancer, vip_port_id]} + + listener: + type: "OS::Octavia::Listener" + properties: + protocol_port: 80 + protocol: "HTTP" + loadbalancer: + get_resource: loadbalancer + + pool: + type: "OS::Octavia::Pool" + properties: + lb_algorithm: "ROUND_ROBIN" + listener: + get_resource: listener + protocol: "HTTP" + + monitor: + type: "OS::Octavia::HealthMonitor" + properties: + delay: 3 + max_retries: 9 + timeout: 3 + type: "PING" + pool: + get_resource: pool + + pool_member_1: + type: "OS::Octavia::PoolMember" + properties: + subnet: + get_resource: private_subnet + protocol_port: 8000 + pool: + get_resource: pool + address: + get_attr: + - "server_1" + - "first_address" + + pool_member_2: + type: "OS::Octavia::PoolMember" + properties: + subnet: + get_resource: private_subnet + protocol_port: 8000 + pool: + get_resource: pool + address: + get_attr: + - "server_2" + - "first_address" +... diff --git a/tools/deployment/component/octavia/octavia.sh b/tools/deployment/component/octavia/octavia.sh new file mode 100755 index 0000000000..1d2e9d8607 --- /dev/null +++ b/tools/deployment/component/octavia/octavia.sh @@ -0,0 +1,186 @@ +#!/bin/bash + +# Copyright 2019 Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Define variables +: ${OSH_HELM_REPO:="../openstack-helm"} +: ${OSH_VALUES_OVERRIDES_PATH:="../openstack-helm/values_overrides"} +: ${OSH_EXTRA_HELM_ARGS_OCTAVIA:="$(helm osh get-values-overrides ${DOWNLOAD_OVERRIDES:-} -p ${OSH_VALUES_OVERRIDES_PATH} -c octavia ${FEATURES})"} + +export OS_CLOUD=openstack_helm + +OSH_AMPHORA_IMAGE_NAME="amphora-x64-haproxy-ubuntu-jammy" +OSH_AMPHORA_IMAGE_OWNER_ID=$(openstack image show "${OSH_AMPHORA_IMAGE_NAME}" -f value -c owner) +OSH_AMPHORA_SECGROUP_LIST=$(openstack security group list -f value | grep lb-mgmt-sec-grp | awk '{print $1}') +OSH_AMPHORA_FLAVOR_ID=$(openstack flavor show m1.amphora -f value -c id) +OSH_AMPHORA_BOOT_NETWORK_LIST=$(openstack network list --name lb-mgmt-net -f value -c ID) +# Test nodes are quite small (usually 8Gb RAM) and for testing Octavia +# we need two worker VM instances and one amphora VM instance. +# We are going to run them all on different K8s nodes. +# The /tmp/inventory_k8s_nodes.txt file is created by the deploy-env role and contains the list +# of all K8s nodes. Amphora instance is run on the first K8s node from the list. +OSH_AMPHORA_TARGET_HOSTNAME=$(sed -n '1p' /tmp/inventory_k8s_nodes.txt) +CONTROLLER_IP_PORT_LIST=$(cat /tmp/octavia_hm_controller_ip_port_list) + +#NOTE: Deploy command +tee /tmp/octavia.yaml < /tmp/octavia_hm_controller_ip_port_list + +# Create a flavor for amphora instance +openstack flavor show m1.amphora || \ + openstack flavor create --ram 1024 --disk 3 --vcpus 1 m1.amphora + +# Create key pair to connect amphora instance via management network +mkdir -p ${SSH_DIR} +openstack keypair show octavia-key || \ + openstack keypair create --private-key ${SSH_DIR}/octavia_key octavia-key +sudo chown $(id -un) ${SSH_DIR}/octavia_key +chmod 600 ${SSH_DIR}/octavia_key + +# accept diffie-hellman-group1-sha1 algo for SSH (for compatibility with older images) +sudo tee -a /etc/ssh/ssh_config < /tmp/curl.txt +curl http://${LB_FLOATING_IP} >> /tmp/curl.txt +curl http://${LB_FLOATING_IP} >> /tmp/curl.txt +grep "Hello from server_1" /tmp/curl.txt +grep "Hello from server_2" /tmp/curl.txt diff --git a/tools/deployment/component/octavia/openssl.cnf b/tools/deployment/component/octavia/openssl.cnf new file mode 100644 index 0000000000..fe4cdb4334 --- /dev/null +++ b/tools/deployment/component/octavia/openssl.cnf @@ -0,0 +1,144 @@ +# OpenSSL root CA configuration file. + +[ ca ] +# `man ca` +default_ca = CA_default + +[ CA_default ] +# Directory and file locations. +dir = ./ +certs = $dir/certs +crl_dir = $dir/crl +new_certs_dir = $dir/newcerts +database = $dir/index.txt +serial = $dir/serial +RANDFILE = $dir/private/.rand + +# The root key and root certificate. +private_key = $dir/private/ca.key.pem +certificate = $dir/certs/ca.cert.pem + +# For certificate revocation lists. +crlnumber = $dir/crlnumber +crl = $dir/crl/ca.crl.pem +crl_extensions = crl_ext +default_crl_days = 30 + +# SHA-1 is deprecated, so use SHA-2 instead. +default_md = sha256 + +name_opt = ca_default +cert_opt = ca_default +# 10 years +default_days = 7300 +preserve = no +policy = policy_strict + +[ CA_intermediate ] +# Directory and file locations. +dir = ./intermediate_ca +certs = $dir/certs +crl_dir = $dir/crl +new_certs_dir = $dir/newcerts +database = $dir/index.txt +serial = $dir/serial +RANDFILE = $dir/private/.rand + +# The root key and root certificate. +private_key = ./private/ca.key.pem +certificate = ./certs/ca.cert.pem + +# For certificate revocation lists. +crlnumber = $dir/crlnumber +crl = $dir/crl/ca.crl.pem +crl_extensions = crl_ext +default_crl_days = 30 + +# SHA-1 is deprecated, so use SHA-2 instead. +default_md = sha256 + +name_opt = ca_default +cert_opt = ca_default +# 5 years +default_days = 3650 +preserve = no +policy = policy_strict + +[ policy_strict ] +# The root CA should only sign intermediate certificates that match. +# See the POLICY FORMAT section of `man ca`. +countryName = match +stateOrProvinceName = match +organizationName = match +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +[ req ] +# Options for the `req` tool (`man req`). +default_bits = 2048 +distinguished_name = req_distinguished_name +string_mask = utf8only + +# SHA-1 is deprecated, so use SHA-2 instead. +default_md = sha256 + +# Extension to add when the -x509 option is used. +x509_extensions = v3_ca + +[ req_distinguished_name ] +# See . +countryName = Country Name (2 letter code) +stateOrProvinceName = State or Province Name +localityName = Locality Name +0.organizationName = Organization Name +organizationalUnitName = Organizational Unit Name +commonName = Common Name +emailAddress = Email Address + +# Optionally, specify some defaults. +countryName_default = US +stateOrProvinceName_default = Oregon +localityName_default = Corvallis +0.organizationName_default = OpenStack +organizationalUnitName_default = Octavia +emailAddress_default = +commonName_default = example.org + +[ v3_ca ] +# Extensions for a typical CA (`man x509v3_config`). +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always,issuer +basicConstraints = critical, CA:true +keyUsage = critical, digitalSignature, cRLSign, keyCertSign + +[ v3_intermediate_ca ] +# Extensions for a typical intermediate CA (`man x509v3_config`). +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always,issuer +basicConstraints = critical, CA:true, pathlen:0 +keyUsage = critical, digitalSignature, cRLSign, keyCertSign + +[ usr_cert ] +# Extensions for client certificates (`man x509v3_config`). +basicConstraints = CA:FALSE +nsCertType = client, email +nsComment = "OpenSSL Generated Client Certificate" +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer +keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, emailProtection + +[ server_cert ] +# Extensions for server certificates (`man x509v3_config`). +basicConstraints = CA:FALSE +nsCertType = server +nsComment = "OpenSSL Generated Server Certificate" +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer:always +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth + +[ crl_ext ] +# Extension for CRLs (`man x509v3_config`). +authorityKeyIdentifier=keyid:always diff --git a/values_overrides/octavia/2025.1-ubuntu_jammy.yaml b/values_overrides/octavia/2025.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..bc5a0c2f7f --- /dev/null +++ b/values_overrides/octavia/2025.1-ubuntu_jammy.yaml @@ -0,0 +1,23 @@ +--- +images: + tags: + test: docker.io/xrally/xrally-openstack:2.0.0 + bootstrap: quay.io/airshipit/heat:2025.1-ubuntu_jammy + db_init: quay.io/airshipit/heat:2025.1-ubuntu_jammy + octavia_db_sync: quay.io/airshipit/octavia:2025.1-ubuntu_jammy + db_drop: quay.io/airshipit/heat:2025.1-ubuntu_jammy + rabbit_init: docker.io/rabbitmq:3.13-management + ks_user: quay.io/airshipit/heat:2025.1-ubuntu_jammy + ks_service: quay.io/airshipit/heat:2025.1-ubuntu_jammy + ks_endpoints: quay.io/airshipit/heat:2025.1-ubuntu_jammy + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_jammy + image_repo_sync: docker.io/docker:17.07.0 + octavia_api: quay.io/airshipit/octavia:2025.1-ubuntu_jammy + octavia_driver_agent: quay.io/airshipit/octavia:2025.1-ubuntu_jammy + octavia_worker: quay.io/airshipit/octavia:2025.1-ubuntu_jammy + octavia_worker_init: quay.io/airshipit/openstack-client:2025.1-ubuntu_jammy + octavia_housekeeping: quay.io/airshipit/octavia:2025.1-ubuntu_jammy + octavia_health_manager: quay.io/airshipit/octavia:2025.1-ubuntu_jammy + octavia_health_manager_init: quay.io/airshipit/openstack-client:2025.1-ubuntu_jammy + openvswitch_vswitchd: docker.io/openstackhelm/openvswitch:latest-ubuntu_jammy +... diff --git a/values_overrides/octavia/2025.1-ubuntu_noble.yaml b/values_overrides/octavia/2025.1-ubuntu_noble.yaml new file mode 100644 index 0000000000..5098da3d57 --- /dev/null +++ b/values_overrides/octavia/2025.1-ubuntu_noble.yaml @@ -0,0 +1,23 @@ +--- +images: + tags: + test: docker.io/xrally/xrally-openstack:2.0.0 + bootstrap: quay.io/airshipit/heat:2025.1-ubuntu_noble + db_init: quay.io/airshipit/heat:2025.1-ubuntu_noble + octavia_db_sync: quay.io/airshipit/octavia:2025.1-ubuntu_noble + db_drop: quay.io/airshipit/heat:2025.1-ubuntu_noble + rabbit_init: docker.io/rabbitmq:3.13-management + ks_user: quay.io/airshipit/heat:2025.1-ubuntu_noble + ks_service: quay.io/airshipit/heat:2025.1-ubuntu_noble + ks_endpoints: quay.io/airshipit/heat:2025.1-ubuntu_noble + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_noble + image_repo_sync: docker.io/docker:17.07.0 + octavia_api: quay.io/airshipit/octavia:2025.1-ubuntu_noble + octavia_driver_agent: quay.io/airshipit/octavia:2025.1-ubuntu_noble + octavia_worker: quay.io/airshipit/octavia:2025.1-ubuntu_noble + octavia_worker_init: quay.io/airshipit/openstack-client:2025.1-ubuntu_noble + octavia_housekeeping: quay.io/airshipit/octavia:2025.1-ubuntu_noble + octavia_health_manager: quay.io/airshipit/octavia:2025.1-ubuntu_noble + octavia_health_manager_init: quay.io/airshipit/openstack-client:2025.1-ubuntu_noble + openvswitch_vswitchd: docker.io/openstackhelm/openvswitch:latest-ubuntu_noble +... diff --git a/zuul.d/2025.1.yaml b/zuul.d/2025.1.yaml index 0de0cdb316..e38281fbe8 100644 --- a/zuul.d/2025.1.yaml +++ b/zuul.d/2025.1.yaml @@ -63,4 +63,15 @@ openstack_release: "2025.1" container_distro_name: ubuntu container_distro_version: noble + +- job: + name: openstack-helm-octavia-2025-1-ubuntu_jammy + parent: openstack-helm-octavia + nodeset: openstack-helm-4nodes-ubuntu_jammy + timeout: 10800 + vars: + osh_params: + openstack_release: "2025.1" + container_distro_name: ubuntu + container_distro_version: jammy ... diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml index 7563cf1931..99660f7a0a 100644 --- a/zuul.d/base.yaml +++ b/zuul.d/base.yaml @@ -126,6 +126,7 @@ abstract: true vars: gate_scripts: + - ./tools/deployment/common/prepare-bashrc.sh - ./tools/deployment/common/prepare-k8s.sh - ./tools/deployment/common/prepare-charts.sh - ./tools/deployment/common/setup-client.sh @@ -151,6 +152,7 @@ abstract: true vars: gate_scripts: + - ./tools/deployment/common/prepare-bashrc.sh - ./tools/deployment/common/prepare-k8s.sh - ./tools/deployment/common/prepare-charts.sh - ./tools/deployment/common/setup-client.sh @@ -177,6 +179,7 @@ abstract: true vars: gate_scripts: + - ./tools/deployment/common/prepare-bashrc.sh - ./tools/deployment/common/prepare-k8s.sh - ./tools/deployment/common/prepare-charts.sh - ./tools/deployment/common/setup-client.sh @@ -202,6 +205,7 @@ vars: osh_helm_repo: openstack-helm gate_scripts: + - ./tools/deployment/common/prepare-bashrc.sh - ./tools/deployment/common/prepare-k8s.sh - ./tools/deployment/common/prepare-charts.sh - ./tools/deployment/common/prepare-helm-repos-local.sh @@ -230,6 +234,7 @@ osh_helm_repo: openstack-helm download_overrides: "-d" gate_scripts: + - ./tools/deployment/common/prepare-bashrc.sh - ./tools/deployment/common/prepare-k8s.sh - ./tools/deployment/common/prepare-charts.sh - ./tools/deployment/common/prepare-helm-repos-public.sh @@ -256,6 +261,7 @@ abstract: true vars: gate_scripts: + - ./tools/deployment/common/prepare-bashrc.sh - ./tools/deployment/common/prepare-k8s.sh - ./tools/deployment/common/prepare-charts.sh - ./tools/deployment/common/setup-client.sh @@ -285,6 +291,7 @@ - ^zuul\.d/.*$ vars: gate_scripts: + - ./tools/deployment/common/prepare-bashrc.sh - ./tools/deployment/common/prepare-k8s.sh - ./tools/deployment/common/prepare-charts.sh - ./tools/deployment/common/setup-client.sh @@ -305,6 +312,7 @@ - ^tools/deployment/component/cinder/.*$ vars: gate_scripts: + - ./tools/deployment/common/prepare-bashrc.sh - ./tools/deployment/common/prepare-k8s.sh - ./tools/deployment/common/prepare-charts.sh - ./tools/deployment/common/setup-client.sh @@ -329,6 +337,7 @@ - ^tools/deployment/ceph/.*$ vars: gate_scripts: + - ./tools/deployment/common/prepare-bashrc.sh - ./tools/deployment/common/prepare-k8s.sh - ./tools/deployment/common/prepare-charts.sh - ./tools/deployment/common/setup-client.sh @@ -364,6 +373,7 @@ vars: run_helm_tests: "yes" gate_scripts: + - ./tools/deployment/common/prepare-bashrc.sh - ./tools/deployment/common/prepare-k8s.sh - ./tools/deployment/common/prepare-charts.sh - ./tools/deployment/common/setup-client.sh @@ -384,6 +394,7 @@ - ^tools/deployment/component/horizon/.*$ vars: gate_scripts: + - ./tools/deployment/common/prepare-bashrc.sh - ./tools/deployment/common/prepare-k8s.sh - ./tools/deployment/common/prepare-charts.sh - ./tools/deployment/common/setup-client.sh @@ -403,6 +414,7 @@ voting: false vars: gate_scripts: + - ./tools/deployment/common/prepare-bashrc.sh - ./tools/deployment/common/prepare-k8s.sh - ./tools/deployment/common/prepare-charts.sh - ./tools/deployment/common/setup-client.sh @@ -434,6 +446,7 @@ timeout: 7200 vars: gate_scripts: + - ./tools/deployment/common/prepare-bashrc.sh - ./tools/deployment/common/prepare-k8s.sh - ./tools/deployment/common/prepare-charts.sh - ./tools/deployment/common/setup-client.sh @@ -453,10 +466,11 @@ - job: name: openstack-helm-skyline - parent: openstack-helm-compute-kit + parent: openstack-helm-deploy timeout: 10800 vars: gate_scripts: + - ./tools/deployment/common/prepare-bashrc.sh - ./tools/deployment/common/prepare-k8s.sh - ./tools/deployment/common/prepare-charts.sh - ./tools/deployment/common/setup-client.sh @@ -471,4 +485,29 @@ - ./tools/deployment/component/compute-kit/compute-kit.sh - ./tools/deployment/component/skyline/skyline.sh - ./tools/gate/selenium/skyline-selenium.sh + +- job: + name: openstack-helm-octavia + parent: openstack-helm-deploy + timeout: 10800 + vars: + gate_scripts: + - ./tools/deployment/common/prepare-bashrc.sh + - ./tools/deployment/common/prepare-k8s.sh + - ./tools/deployment/common/prepare-charts.sh + - ./tools/deployment/common/setup-client.sh + - export VOLUME_HELM_ARGS="--set volume.enabled=false"; ./tools/deployment/component/common/rabbitmq.sh + - ./tools/deployment/db/mariadb.sh + - ./tools/deployment/component/common/memcached.sh + - ./tools/deployment/component/keystone/keystone.sh + - ./tools/deployment/component/heat/heat.sh + - export GLANCE_BACKEND=local; ./tools/deployment/component/glance/glance.sh + - ./tools/deployment/component/compute-kit/openvswitch.sh + - ./tools/deployment/component/compute-kit/libvirt.sh + - ./tools/deployment/component/compute-kit/compute-kit.sh + - ./tools/deployment/component/barbican/barbican.sh + - ./tools/deployment/component/octavia/octavia_resources.sh + - ./tools/deployment/component/octavia/octavia_certs.sh + - ./tools/deployment/component/octavia/octavia.sh + - ./tools/deployment/component/octavia/octavia_test.sh ... diff --git a/zuul.d/nodesets.yaml b/zuul.d/nodesets.yaml index 4f09233828..30c3672efb 100644 --- a/zuul.d/nodesets.yaml +++ b/zuul.d/nodesets.yaml @@ -193,6 +193,41 @@ nodes: - primary +- nodeset: + name: openstack-helm-4nodes-ubuntu_jammy + nodes: + - name: primary + label: ubuntu-jammy + - name: node-1 + label: ubuntu-jammy + - name: node-2 + label: ubuntu-jammy + - name: node-3 + label: ubuntu-jammy + groups: + - name: primary + nodes: + - primary + - name: nodes + nodes: + - node-1 + - node-2 + - node-3 + - name: k8s_cluster + nodes: + - primary + - node-1 + - node-2 + - node-3 + - name: k8s_control_plane + nodes: + - primary + - name: k8s_nodes + nodes: + - node-1 + - node-2 + - node-3 + - nodeset: name: openstack-helm-5nodes-ubuntu_jammy nodes: diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 188135174e..4f46ca432c 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -38,6 +38,7 @@ - openstack-helm-cinder-2025-1-ubuntu_jammy # 3 nodes rook - openstack-helm-compute-kit-2025-1-ubuntu_jammy # 1 node + 3 nodes - openstack-helm-skyline-2025-1-ubuntu_jammy # 3 nodes + - openstack-helm-octavia-2025-1-ubuntu_jammy # 4 nodes # 2025.1 Ubuntu Noble - openstack-helm-cinder-2025-1-ubuntu_noble # 5 nodes rook - openstack-helm-compute-kit-2025-1-ubuntu_noble # 1 node + 3 nodes