diff --git a/devstack/local.conf.sample b/devstack/local.conf.sample index 234030699..d7af1820e 100644 --- a/devstack/local.conf.sample +++ b/devstack/local.conf.sample @@ -35,11 +35,44 @@ enable_service q-dhcp enable_service q-l3 enable_service q-svc -# LBaaSv2 service and Haproxy agent -enable_plugin neutron-lbaas \ - git://git.openstack.org/openstack/neutron-lbaas -enable_service q-lbaasv2 -NEUTRON_LBAAS_SERVICE_PROVIDERV2="LOADBALANCERV2:Haproxy:neutron_lbaas.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default" +KURYR_K8S_LBAAS_USE_OCTAVIA=True + +if [[ "$KURYR_K8S_LBAAS_USE_OCTAVIA" == "True" ]]; then + # Octavia LBaaSv2 + LIBS_FROM_GIT+=python-octaviaclient + enable_plugin octavia https://git.openstack.org/openstack/octavia + enable_service octavia + enable_service o-api + enable_service o-cw + enable_service o-hm + enable_service o-hk + ## Octavia Deps + ### Image + ### Barbican + enable_plugin barbican https://git.openstack.org/openstack/barbican + ### Nova + enable_service n-api + enable_service n-api-meta + enable_service n-cpu + enable_service n-cond + enable_service n-sch + enable_service placement-api + enable_service placement-client + ### Glance + enable_service g-api + enable_service g-reg + ### Neutron-lbaas + #### In case Octavia is older than Pike, neutron-lbaas is needed + enable_plugin neutron-lbaas \ + git://git.openstack.org/openstack/neutron-lbaas + enable_service q-lbaasv2 +else + # LBaaSv2 service and Haproxy agent + enable_plugin neutron-lbaas \ + git://git.openstack.org/openstack/neutron-lbaas + enable_service q-lbaasv2 +fi + # Keystone enable_service key @@ -139,3 +172,15 @@ enable_service kubelet # part of the codebase that connects to the Kubernetes API server to read the # resource events and convert them to Neutron actions enable_service kuryr-kubernetes + +# Increase Octavia amphorae timeout so that the first LB amphora has time to +# build and boot +if [[ "$KURYR_K8S_LBAAS_USE_OCTAVIA" == "True" ]]; then + IMAGE_URLS+=",http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img" +else + NEUTRON_LBAAS_SERVICE_PROVIDERV2="LOADBALANCERV2:Haproxy:neutron_lbaas.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default" +fi + +[[post-config|$OCTAVIA_CONF_DIR/octavia.conf]] +[controller_worker] +amp_active_retries=9999 diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 428bf3422..603735d80 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -206,9 +206,6 @@ function configure_neutron_defaults { pod_subnet_id="$(neutron subnet-show -c id -f value \ "${KURYR_NEUTRON_DEFAULT_POD_SUBNET}")" - sg_ids=$(echo $(neutron security-group-list \ - --project-id "$project_id" -c id -f value) | tr ' ' ',') - create_k8s_subnet "$project_id" \ "$KURYR_NEUTRON_DEFAULT_SERVICE_NET" \ "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" \ @@ -217,6 +214,31 @@ function configure_neutron_defaults { service_subnet_id="$(neutron subnet-show -c id -f value \ "${KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET}")" + sg_ids=$(echo $(neutron security-group-list \ + --project-id "$project_id" -c id -f value) | tr ' ' ',') + + local use_octavia + use_octavia=$(trueorfalse True KURYR_K8S_LBAAS_USE_OCTAVIA) + if [[ "$use_octavia" == "True" ]]; then + # In order for the pods to allow service traffic under Octavia L3 mode, + #it is necessary for the service subnet to be allowed into the $sg_ids + local service_cidr + local service_pod_access_sg_id + service_cidr=$(openstack --os-cloud devstack-admin \ + --os-region "$REGION_NAME" subnet show \ + "${KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET}" -f value -c cidr) + service_pod_access_sg_id=$(openstack --os-cloud devstack-admin \ + --os-region "$REGION_NAME" \ + security group create --project "$project_id" \ + service_pod_access -f value -c id) + openstack --os-cloud devstack-admin --os-region "$REGION_NAME" \ + security group rule create --project "$project_id" \ + --description "k8s service subnet allowed" \ + --remote-ip "$service_cidr" --ethertype IPv4 --protocol tcp \ + "$service_pod_access_sg_id" + sg_ids+=",${service_pod_access_sg_id}" + fi + iniset "$KURYR_CONFIG" neutron_defaults project "$project_id" iniset "$KURYR_CONFIG" neutron_defaults pod_subnet "$pod_subnet_id" iniset "$KURYR_CONFIG" neutron_defaults pod_security_groups "$sg_ids" diff --git a/devstack/settings b/devstack/settings index b87a97297..a7015a497 100644 --- a/devstack/settings +++ b/devstack/settings @@ -41,5 +41,9 @@ KURYR_K8S_API_CERT=${KURYR_K8S_API_CERT:-} KURYR_K8S_API_KEY=${KURYR_K8S_API_KEY:-} KURYR_K8S_API_CACERT=${KURYR_K8S_API_CACERT:-} +# Octavia +KURYR_K8S_LBAAS_USE_OCTAVIA=${KURYR_K8S_LBAAS_USE_OCTAVIA:-True} +KURYR_K8S_OCTAVIA_MEMBER_MODE=${KURYR_K8S_OCTAVIA_MEMBER_MODE:-L3} + # Kuryr_ovs_baremetal KURYR_CONFIGURE_BAREMETAL_KUBELET_IFACE=${KURYR_CONFIGURE_BAREMETAL_KUBELET_IFACE:-True} diff --git a/doc/source/installation/manual.rst b/doc/source/installation/manual.rst index f62729a49..3456fac0a 100644 --- a/doc/source/installation/manual.rst +++ b/doc/source/installation/manual.rst @@ -51,6 +51,38 @@ Edit ``kuryr.conf``:: project = {id_of_project} service_subnet = {id_of_subnet_for_k8s_services} +Note that the service_subnet and the pod_subnet *should be routable* and that +the pods should allow service subnet access. + +Octavia supports two ways of performing the load balancing between the +Kubernetes load balancers and their members: + +* Layer2: Octavia, apart from the VIP port in the services subnet, creates a + Neutron port to the subnet of each of the members. This way the traffic from + the Service Haproxy to the members will not go through the router again, only + will have gone through the router to reach the service. +* Layer3: Octavia only creates the VIP port. The traffic from the service VIP to + the members will go back to the router to reach the pod subnet. It is + important to note that will have some performance impact depending on the SDN. + +At the moment Kuryr-Kubernetes supports only L3 mode (both for Octavia and for +the deprecated Neutron-LBaaSv2. + +This means that: + +* There should be a router between the two subnets. +* The pod_security_groups setting should include a security group with a rule + granting access to all the CIDR or the service subnet, e.g.:: + + openstack security group create --project k8s_cluster_project \ + service_pod_access_sg + openstack --project k8s_cluster_project security group rule create \ + --remote-ip cidr_of_service_subnet --ethertype IPv4 --protocol tcp \ + service_pod_access_sg + +* The uuid of this security group id should be added to the comma separated + list of pod security groups. *pod_security_groups* in *[neutron_defaults]*. + Run kuryr-k8s-controller:: $ kuryr-k8s-controller --config-file /etc/kuryr/kuryr.conf -d diff --git a/kuryr_kubernetes/controller/handlers/lbaas.py b/kuryr_kubernetes/controller/handlers/lbaas.py index 718395942..c08c5b8bd 100644 --- a/kuryr_kubernetes/controller/handlers/lbaas.py +++ b/kuryr_kubernetes/controller/handlers/lbaas.py @@ -300,13 +300,18 @@ class LoadBalancerHandler(k8s_base.ResourceEventHandler): continue port_name = subset_port.get('name') pool = pool_by_tgt_name[port_name] - target_subnet_id = self._get_pod_subnet(target_ref, - target_ip) + # We use the service subnet id so that the connectivity + # from VIP to pods happens in layer 3 mode, i.e., routed. + # TODO(apuimedo): Add L2 mode + # TODO(apuimedo): Do not pass subnet_id at all when in + # L3 mode once old neutron-lbaasv2 is not supported, as + # octavia does not require it + member_subnet_id = lbaas_state.loadbalancer.subnet_id member = self._drv_lbaas.ensure_member( endpoints=endpoints, loadbalancer=lbaas_state.loadbalancer, pool=pool, - subnet_id=target_subnet_id, + subnet_id=member_subnet_id, ip=target_ip, port=target_port, target_ref=target_ref) @@ -315,18 +320,6 @@ class LoadBalancerHandler(k8s_base.ResourceEventHandler): return changed - def _get_pod_subnet(self, target_ref, ip): - # REVISIT(ivc): consider using true pod object instead - pod = {'kind': target_ref['kind'], - 'metadata': {'name': target_ref['name'], - 'namespace': target_ref['namespace']}} - project_id = self._drv_pod_project.get_project(pod) - subnets_map = self._drv_pod_subnets.get_subnets(pod, project_id) - # FIXME(ivc): potentially unsafe [0] index - return [subnet_id for subnet_id, network in subnets_map.items() - for subnet in network.subnets.objects - if ip in subnet.cidr][0] - def _remove_unused_members(self, endpoints, lbaas_state, lbaas_spec): spec_port_names = {p.name for p in lbaas_spec.ports} current_targets = {(a['ip'], p['port']) diff --git a/kuryr_kubernetes/tests/unit/controller/handlers/test_lbaas.py b/kuryr_kubernetes/tests/unit/controller/handlers/test_lbaas.py index f47ca70c5..2894ece53 100644 --- a/kuryr_kubernetes/tests/unit/controller/handlers/test_lbaas.py +++ b/kuryr_kubernetes/tests/unit/controller/handlers/test_lbaas.py @@ -118,43 +118,6 @@ class TestLBaaSSpecHandler(test_base.TestCase): subnets_list = osv_subnet.SubnetList(objects=subnets) return osv_network.Network(subnets=subnets_list) - def test_get_subnet_id(self): - test_ip = '1.2.3.4' - test_cidr = '1.2.3.0/24' - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) - m_drv_subnets = mock.Mock(spec=drv_base.ServiceSubnetsDriver) - m_handler._drv_subnets = m_drv_subnets - m_drv_subnets.get_subnets.return_value = { - mock.sentinel.subnet_id: self._make_test_net_obj([test_cidr]) - } - - self.assertEqual(mock.sentinel.subnet_id, - h_lbaas.LBaaSSpecHandler._get_subnet_id( - m_handler, - mock.sentinel.service, - mock.sentinel.project_id, - test_ip)) - m_drv_subnets.get_subnets.assert_called_once_with( - mock.sentinel.service, mock.sentinel.project_id) - - def test_get_subnet_id_invalid(self): - test_ip = '1.2.3.4' - test_cidr = '3.2.1.0/24' - m_service = mock.MagicMock() - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) - m_drv_subnets = mock.Mock(spec=drv_base.ServiceSubnetsDriver) - m_handler._drv_subnets = m_drv_subnets - m_drv_subnets.get_subnets.return_value = { - mock.sentinel.subnet_id: self._make_test_net_obj([test_cidr]) - } - - self.assertRaises(k_exc.IntegrityError, - h_lbaas.LBaaSSpecHandler._get_subnet_id, - m_handler, - m_service, - mock.sentinel.project_id, - test_ip) - def test_generate_lbaas_spec(self): m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) @@ -494,28 +457,6 @@ class TestLoadBalancerHandler(test_base.TestCase): self.assertEqual(True, ret) - def test_get_pod_subnet(self): - subnet_id = mock.sentinel.subnet_id - project_id = mock.sentinel.project_id - target_ref = {'kind': k_const.K8S_OBJ_POD, - 'name': 'pod-name', - 'namespace': 'default'} - ip = '1.2.3.4' - m_handler = mock.Mock(spec=h_lbaas.LoadBalancerHandler) - m_drv_pod_project = mock.Mock() - m_drv_pod_project.get_project.return_value = project_id - m_handler._drv_pod_project = m_drv_pod_project - m_drv_pod_subnets = mock.Mock() - m_drv_pod_subnets.get_subnets.return_value = { - subnet_id: osv_network.Network(subnets=osv_subnet.SubnetList( - objects=[osv_subnet.Subnet(cidr='1.2.3.0/24')]))} - m_handler._drv_pod_subnets = m_drv_pod_subnets - - observed_subnet_id = h_lbaas.LoadBalancerHandler._get_pod_subnet( - m_handler, target_ref, ip) - - self.assertEqual(subnet_id, observed_subnet_id) - def _generate_lbaas_state(self, vip, targets, project_id, subnet_id): endpoints = mock.sentinel.endpoints drv = FakeLBaaSDriver() @@ -616,9 +557,7 @@ class TestLoadBalancerHandler(test_base.TestCase): handler = h_lbaas.LoadBalancerHandler() - with mock.patch.object(handler, '_get_pod_subnet') as m_get_pod_subnet: - m_get_pod_subnet.return_value = subnet_id - handler._sync_lbaas_members(endpoints, state, spec) + handler._sync_lbaas_members(endpoints, state, spec) lsnrs = {lsnr.id: lsnr for lsnr in state.listeners} pools = {pool.id: pool for pool in state.pools}