Use Octavia for LoadBalancer type service

In the OpenStack deployment with Octavia service enabled, the octavia
service should be used not only for master nodes high availability, but
also for k8s LoadBalancer type service implementation as well.

Change-Id: Ib61f59507510253794a4780a91e49aa6682c8039
Closes-Bug: #1770133

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml

@@ -69,3 +69,4 @@ write_files:
       KUBEAPI_OPTIONS="$KUBEAPI_OPTIONS"
       KUBEPROXY_OPTIONS="$KUBEPROXY_OPTIONS"
       KUBESCHEDULER_OPTIONS="$KUBESCHEDULER_OPTIONS"
+      OCTAVIA_ENABLED="$OCTAVIA_ENABLED"

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml

@@ -48,3 +48,4 @@ write_files:
       DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN"
       KUBELET_OPTIONS="$KUBELET_OPTIONS"
       KUBEPROXY_OPTIONS="$KUBEPROXY_OPTIONS"
+      OCTAVIA_ENABLED="$OCTAVIA_ENABLED"

magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh

@@ -16,6 +16,7 @@ password=$TRUSTEE_PASSWORD
 trust-id=$TRUST_ID
 ca-file=/etc/kubernetes/ca-bundle.crt
 [LoadBalancer]
+use-octavia=$OCTAVIA_ENABLED
 subnet-id=$CLUSTER_SUBNET
 create-monitor=yes
 monitor-delay=1m

magnum/drivers/heat/k8s_template_def.py

@@ -12,6 +12,7 @@
 
 from oslo_config import cfg
 
+from magnum.common import keystone
 from magnum.drivers.heat import template_def
 
 CONF = cfg.CONF
@@ -105,6 +106,8 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition):
             extra_params['loadbalancing_protocol'] = 'HTTP'
             extra_params['kubernetes_port'] = 8080
 
+        extra_params['octavia_enabled'] = keystone.is_octavia_enabled()
+
         label_list = ['flannel_network_cidr', 'flannel_backend',
                       'flannel_network_subnetlen',
                       'system_pods_initial_delay',

magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params-master.yaml

@@ -49,3 +49,4 @@ write_files:
       KUBE_DASHBOARD_VERSION="$KUBE_DASHBOARD_VERSION"
       DNS_SERVICE_IP="$DNS_SERVICE_IP"
       DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN"
+      OCTAVIA_ENABLED="$OCTAVIA_ENABLED"

magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params.yaml

@@ -47,3 +47,4 @@ write_files:
       CONTAINER_RUNTIME="$CONTAINER_RUNTIME"
       DNS_SERVICE_IP="$DNS_SERVICE_IP"
       DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN"
+      OCTAVIA_ENABLED="$OCTAVIA_ENABLED"

magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml

@@ -307,6 +307,12 @@ parameters:
       - allowed_values: ["affinity", "anti-affinity", "soft-affinity",
                          "soft-anti-affinity"]
 
+  octavia_enabled:
+    type: boolean
+    description: >
+      whether or not to use Octavia for LoadBalancer type service.
+    default: False
+
 resources:
 
   ######################################################################
@@ -491,6 +497,7 @@ resources:
           dns_cluster_domain: {get_param: dns_cluster_domain}
           openstack_ca: {get_param: openstack_ca}
           nodes_server_group_id: {get_resource: nodes_server_group}
+          octavia_enabled: {get_param: octavia_enabled}
 
   ######################################################################
   #
@@ -547,6 +554,7 @@ resources:
           dns_cluster_domain: {get_param: dns_cluster_domain}
           openstack_ca: {get_param: openstack_ca}
           nodes_server_group_id: {get_resource: nodes_server_group}
+          octavia_enabled: {get_param: octavia_enabled}
 
 outputs:
 

magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml

@@ -237,6 +237,12 @@ parameters:
     type: string
     description: ID of the server group for kubernetes cluster nodes.
 
+  octavia_enabled:
+    type: boolean
+    description: >
+      whether or not to use Octavia for LoadBalancer type service.
+    default: False
+
 resources:
 
   master_wait_handle:
@@ -322,6 +328,7 @@ resources:
             "$ETCD_LB_VIP": {get_param: etcd_lb_vip}
             "$DNS_SERVICE_IP": {get_param: dns_service_ip}
             "$DNS_CLUSTER_DOMAIN": {get_param: dns_cluster_domain}
+            "$OCTAVIA_ENABLED": {get_param: octavia_enabled}
 
   add_ext_ca_certs:
     type: OS::Heat::SoftwareConfig

magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml

@@ -164,6 +164,12 @@ parameters:
     type: string
     description: ID of the server group for kubernetes cluster nodes.
 
+  octavia_enabled:
+    type: boolean
+    description: >
+      whether or not to use Octavia for LoadBalancer type service.
+    default: False
+
 resources:
 
   minion_wait_handle:
@@ -222,6 +228,7 @@ resources:
             "$CONTAINER_RUNTIME": {get_param: container_runtime}
             "$DNS_SERVICE_IP": {get_param: dns_service_ip}
             "$DNS_CLUSTER_DOMAIN": {get_param: dns_cluster_domain}
+            "$OCTAVIA_ENABLED": {get_param: octavia_enabled}
 
   add_ext_ca_certs:
     type: OS::Heat::SoftwareConfig

magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

@@ -462,6 +462,12 @@ parameters:
       additional options to be passed to the scheduler
     default: ""
 
+  octavia_enabled:
+    type: boolean
+    description: >
+      whether or not to use Octavia for LoadBalancer type service.
+    default: False
+
 resources:
 
   ######################################################################
@@ -679,6 +685,7 @@ resources:
           kubeproxy_options: {get_param: kubeproxy_options}
           kubecontroller_options: {get_param: kubecontroller_options}
           kubescheduler_options: {get_param: kubescheduler_options}
+          octavia_enabled: {get_param: octavia_enabled}
 
   ######################################################################
   #
@@ -754,6 +761,7 @@ resources:
           pods_network_cidr: {get_param: pods_network_cidr}
           kubelet_options: {get_param: kubelet_options}
           kubeproxy_options: {get_param: kubeproxy_options}
+          octavia_enabled: {get_param: octavia_enabled}
 
 outputs:
 

magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml

@@ -350,6 +350,12 @@ parameters:
     description: >
       additional options to be passed to the scheduler
 
+  octavia_enabled:
+    type: boolean
+    description: >
+      whether or not to use Octavia for LoadBalancer type service.
+    default: False
+
 resources:
 
   master_wait_handle:
@@ -453,6 +459,7 @@ resources:
             "$KUBECONTROLLER_OPTIONS": {get_param: kubecontroller_options}
             "$KUBEPROXY_OPTIONS": {get_param: kubeproxy_options}
             "$KUBESCHEDULER_OPTIONS": {get_param: kubescheduler_options}
+            "$OCTAVIA_ENABLED": {get_param: octavia_enabled}
 
   install_openstack_ca:
     type: OS::Heat::SoftwareConfig

magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml

@@ -259,6 +259,12 @@ parameters:
     description: >
       additional options to be passed to the kube proxy
 
+  octavia_enabled:
+    type: boolean
+    description: >
+      whether or not to use Octavia for LoadBalancer type service.
+    default: False
+
 resources:
 
   minion_wait_handle:
@@ -327,6 +333,7 @@ resources:
             $DNS_CLUSTER_DOMAIN: {get_param: dns_cluster_domain}
             $KUBELET_OPTIONS: {get_param: kubelet_options}
             $KUBEPROXY_OPTIONS: {get_param: kubeproxy_options}
+            $OCTAVIA_ENABLED: {get_param: octavia_enabled}
 
   install_openstack_ca:
     type: OS::Heat::SoftwareConfig

magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml

@@ -360,6 +360,12 @@ parameters:
       - allowed_values: ["affinity", "anti-affinity", "soft-affinity",
                          "soft-anti-affinity"]
 
+  octavia_enabled:
+    type: boolean
+    description: >
+      whether or not to use Octavia for LoadBalancer type service.
+    default: False
+
 resources:
 
   api_lb:
@@ -535,6 +541,7 @@ resources:
           etcd_lb_vip: {get_attr: [etcd_lb, address]}
           openstack_ca: {get_param: openstack_ca}
           nodes_server_group_id: {get_resource: nodes_server_group}
+          octavia_enabled: {get_param: octavia_enabled}
 
   ######################################################################
   #
@@ -625,6 +632,7 @@ resources:
       container_infra_prefix: {get_param: container_infra_prefix}
       wc_curl_cli: {get_attr: [minion_wait_handle, curl_cli]}
       openstack_ca: {get_param: openstack_ca}
+      octavia_enabled: {get_param: octavia_enabled}
 
   ######################################################################
   #

magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml

@@ -250,6 +250,12 @@ parameters:
     type: string
     description: ID of the server group for kubernetes cluster nodes.
 
+  octavia_enabled:
+    type: boolean
+    description: >
+      whether or not to use Octavia for LoadBalancer type service.
+    default: False
+
 resources:
 
   ######################################################################
@@ -321,6 +327,7 @@ resources:
             "$CONTAINER_INFRA_PREFIX": {get_param: container_infra_prefix}
             "$ENABLE_CINDER": "False"
             "$ETCD_LB_VIP": {get_param: etcd_lb_vip}
+            "$OCTAVIA_ENABLED": {get_param: octavia_enabled}
 
   install_openstack_ca:
     type: OS::Heat::SoftwareConfig

magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion_software_configs.yaml

@@ -177,6 +177,12 @@ parameters:
     type: string
     description: The OpenStack CA certificate to install on the node.
 
+  octavia_enabled:
+    type: boolean
+    description: >
+      whether or not to use Octavia for LoadBalancer type service.
+    default: False
+
 resources:
 
   ######################################################################
@@ -231,6 +237,7 @@ resources:
             $INSECURE_REGISTRY_URL: {get_param: insecure_registry_url}
             $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix}
             $ENABLE_CINDER: "False"
+            $OCTAVIA_ENABLED: {get_param: octavia_enabled}
 
   install_openstack_ca:
     type: OS::Heat::SoftwareConfig

magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py

@@ -125,6 +125,13 @@ class TestClusterConductorWithK8s(base.TestCase):
         self.mock_osc.keystone.return_value = self.mock_keystone
         self.mock_osc_class.return_value = self.mock_osc
 
+        octavia_patcher = mock.patch(
+            'magnum.common.keystone.is_octavia_enabled'
+        )
+        self.mock_enable_octavia = octavia_patcher.start()
+        self.mock_enable_octavia.return_value = False
+        self.addCleanup(octavia_patcher.stop)
+
     @patch('requests.get')
     @patch('magnum.objects.ClusterTemplate.get_by_uuid')
     @patch('magnum.drivers.common.driver.Driver.get_driver')
@@ -263,6 +270,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'kubecontroller_options': '--kubecontroller',
             'kubescheduler_options': '--kubescheduler',
             'kubeproxy_options': '--kubeproxy',
+            'octavia_enabled': False,
         }
         if missing_attr is not None:
             expected.pop(mapping[missing_attr], None)
@@ -370,6 +378,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'kubecontroller_options': '--kubecontroller',
             'kubescheduler_options': '--kubescheduler',
             'kubeproxy_options': '--kubeproxy',
+            'octavia_enabled': False,
         }
 
         self.assertEqual(expected, definition)
@@ -464,6 +473,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'kubecontroller_options': '--kubecontroller',
             'kubescheduler_options': '--kubescheduler',
             'kubeproxy_options': '--kubeproxy',
+            'octavia_enabled': False,
         }
         self.assertEqual(expected, definition)
         self.assertEqual(
@@ -551,6 +561,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'kubecontroller_options': '--kubecontroller',
             'kubescheduler_options': '--kubescheduler',
             'kubeproxy_options': '--kubeproxy',
+            'octavia_enabled': False,
         }
         self.assertEqual(expected, definition)
         self.assertEqual(
@@ -633,6 +644,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'kubecontroller_options': '--kubecontroller',
             'kubescheduler_options': '--kubescheduler',
             'kubeproxy_options': '--kubeproxy',
+            'octavia_enabled': False,
         }
         self.assertEqual(expected, definition)
         self.assertEqual(
@@ -816,6 +828,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'kubecontroller_options': '--kubecontroller',
             'kubescheduler_options': '--kubescheduler',
             'kubeproxy_options': '--kubeproxy',
+            'octavia_enabled': False,
         }
         self.assertEqual(expected, definition)
         self.assertEqual(

magnum/tests/unit/drivers/test_template_definition.py

@@ -219,6 +219,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
     def get_definition(self):
         return k8sa_dr.Driver().get_template_definition()
 
+    @mock.patch('magnum.common.keystone.is_octavia_enabled')
     @mock.patch('magnum.common.clients.OpenStackClients')
     @mock.patch('magnum.drivers.k8s_fedora_atomic_v1.template_def'
                 '.AtomicK8sTemplateDefinition.get_discovery_url')
@@ -227,7 +228,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
     @mock.patch('magnum.drivers.heat.template_def.TemplateDefinition'
                 '.get_output')
     def test_k8s_get_params(self, mock_get_output, mock_get_params,
-                            mock_get_discovery_url, mock_osc_class):
+                            mock_get_discovery_url, mock_osc_class,
+                            mock_enable_octavia):
+        mock_enable_octavia.return_value = False
         mock_context = mock.MagicMock()
         mock_context.auth_token = 'AUTH_TOKEN'
         mock_cluster_template = mock.MagicMock()
@@ -349,12 +352,15 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
             'calico_ipv4pool': calico_ipv4pool,
             'pods_network_cidr': pods_network_cidr,
             'ingress_controller': ingress_controller,
-            'ingress_controller_role': ingress_controller_role}}
+            'ingress_controller_role': ingress_controller_role,
+            'octavia_enabled': False,
+        }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,
                                                 mock_cluster,
                                                 **expected_kwargs)
 
+    @mock.patch('magnum.common.keystone.is_octavia_enabled')
     @mock.patch('magnum.common.clients.OpenStackClients')
     @mock.patch('magnum.drivers.heat.template_def'
                 '.BaseTemplateDefinition.get_discovery_url')
@@ -363,7 +369,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
     @mock.patch('magnum.drivers.heat.template_def.TemplateDefinition'
                 '.get_output')
     def test_k8s_get_params_insecure(self, mock_get_output, mock_get_params,
-                                     mock_get_discovery_url, mock_osc_class):
+                                     mock_get_discovery_url, mock_osc_class,
+                                     mock_enable_octavia):
+        mock_enable_octavia.return_value = False
         mock_context = mock.MagicMock()
         mock_context.auth_token = 'AUTH_TOKEN'
         mock_cluster_template = mock.MagicMock()
@@ -487,7 +495,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
             'calico_ipv4pool': calico_ipv4pool,
             'pods_network_cidr': pods_network_cidr,
             'ingress_controller': ingress_controller,
-            'ingress_controller_role': ingress_controller_role}}
+            'ingress_controller_role': ingress_controller_role,
+            'octavia_enabled': False,
+        }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,
                                                 mock_cluster,