Allow k8s cluster without LBaaS

This patch adds an environment file and a couple of template resources
to allow the LBaaS resources to be conditionally enabled/disabled.

Change-Id: I40ef0839dca84f398efb02022fa7c1de821fb1a3
Partially-Implements: blueprint decouple-lbaas
Partially-Implements: blueprint bay-with-no-floating-ips

magnum/api/attr_validator.py

@@ -164,6 +164,12 @@ def validate_os_resources(context, baymodel):
                 validate_method(baymodel[attr])
 
 
+def validate_master_count(bay, baymodel):
+    if bay['master_count'] > 1 and not baymodel['master_lb_enabled']:
+        raise exception.InvalidParameterValue(_(
+            "master_count must be 1 when master_lb_enabled is False"))
+
+
 # Dictionary that maintains a list of validation functions
 validators = {'image_id': validate_image,
               'flavor_id': validate_flavor,

magnum/api/controllers/v1/bay.py

@@ -329,6 +329,7 @@ class BaysController(rest.RestController):
                        action='bay:create')
         baymodel = objects.BayModel.get_by_uuid(context, bay.baymodel_id)
         attr_validator.validate_os_resources(context, baymodel.as_dict())
+        attr_validator.validate_master_count(bay.as_dict(), baymodel.as_dict())
         bay_dict = bay.as_dict()
         bay_dict['project_id'] = context.project_id
         bay_dict['user_id'] = context.user_id

magnum/conductor/template_definition.py

@@ -161,6 +161,12 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition):
                                       extra_params=extra_params,
                                       **kwargs)
 
+    def get_env_files(self, baymodel):
+        if baymodel.master_lb_enabled:
+            return ['environments/with_master_lb.yaml']
+        else:
+            return ['environments/no_master_lb.yaml']
+
 
 class AtomicK8sTemplateDefinition(K8sTemplateDefinition):
     """Kubernetes template for a Fedora Atomic VM."""

magnum/templates/kubernetes/environments/no_master_lb.yaml

@@ -0,0 +1,12 @@
+# Environment file to disable LBaaS in a Kubernetes cluster by mapping
+# LBaaS-related resource types to OS::Heat::None
+resource_registry:
+  "Magnum::ApiGatewaySwitcher": ../fragments/api_gateway_switcher_master.yaml
+
+  # kubecluster.yaml
+  "Magnum::Optional::Neutron::Pool": "OS::Heat::None"
+  "Magnum::Optional::Neutron::Pool::FloatingIP": "OS::Heat::None"
+  "Magnum::Optional::Neutron::Pool::HealthMonitor": "OS::Heat::None"
+
+  # kubemaster.yaml
+  "Magnum::Optional::Neutron::PoolMember": "OS::Heat::None"

magnum/templates/kubernetes/environments/with_master_lb.yaml

@@ -0,0 +1,12 @@
+# Environment file to enable LBaaS in a Kubernetes cluster by mapping
+# LBaaS-related resource types to the real LBaaS resource types.
+resource_registry:
+  "Magnum::ApiGatewaySwitcher": ../fragments/api_gateway_switcher_pool.yaml
+
+  # kubecluster.yaml
+  "Magnum::Optional::Neutron::Pool": "OS::Neutron::Pool"
+  "Magnum::Optional::Neutron::Pool::FloatingIP": "OS::Neutron::FloatingIP"
+  "Magnum::Optional::Neutron::Pool::HealthMonitor": "OS::Neutron::HealthMonitor"
+
+  # kubemaster.yaml
+  "Magnum::Optional::Neutron::PoolMember": "OS::Neutron::PoolMember"

magnum/templates/kubernetes/fragments/api_gateway_switcher_master.yaml

@@ -0,0 +1,32 @@
+heat_template_version: 2013-05-23
+
+description: >
+  This is a template resource that accepts public and private IPs from both
+  a Neutron LBaaS Pool and a master node. It connects the master inputs
+  to its outputs, essentially acting as one state of a multiplexer.
+
+parameters:
+
+  pool_public_ip:
+    type: string
+    default: ""
+
+  pool_private_ip:
+    type: string
+    default: ""
+
+  master_public_ip:
+    type: string
+    default: ""
+
+  master_private_ip:
+    type: string
+    default: ""
+
+outputs:
+
+  public_ip:
+    value: {get_param: master_public_ip}
+
+  private_ip:
+    value: {get_param: master_private_ip}

magnum/templates/kubernetes/fragments/api_gateway_switcher_pool.yaml

@@ -0,0 +1,32 @@
+heat_template_version: 2013-05-23
+
+description: >
+  This is a template resource that accepts public and private IPs from both
+  a Neutron LBaaS Pool and a master node. It connects the pool inputs
+  to its outputs, essentially acting as one state of a multiplexer.
+
+parameters:
+
+  pool_public_ip:
+    type: string
+    default: ""
+
+  pool_private_ip:
+    type: string
+    default: ""
+
+  master_public_ip:
+    type: string
+    default: ""
+
+  master_private_ip:
+    type: string
+    default: ""
+
+outputs:
+
+  public_ip:
+    value: {get_param: pool_public_ip}
+
+  private_ip:
+    value: {get_param: pool_private_ip}

magnum/templates/kubernetes/kubecluster.yaml

@@ -350,7 +350,7 @@ resources:
   #
 
   api_monitor:
-    type: OS::Neutron::HealthMonitor
+    type: Magnum::Optional::Neutron::Pool::HealthMonitor
     properties:
       type: TCP
       delay: 5
@@ -358,7 +358,7 @@ resources:
       timeout: 5
 
   api_pool:
-    type: OS::Neutron::Pool
+    type: Magnum::Optional::Neutron::Pool
     properties:
       protocol: {get_param: loadbalancing_protocol}
       monitors: [{get_resource: api_monitor}]
@@ -368,7 +368,7 @@ resources:
         protocol_port: {get_param: kubernetes_port}
 
   api_pool_floating:
-    type: OS::Neutron::FloatingIP
+    type: Magnum::Optional::Neutron::Pool::FloatingIP
     depends_on:
       - extrouter_inside
     properties:
@@ -376,7 +376,7 @@ resources:
       port_id: {get_attr: [api_pool, vip, port_id]}
 
   etcd_monitor:
-    type: OS::Neutron::HealthMonitor
+    type: Magnum::Optional::Neutron::Pool::HealthMonitor
     properties:
       type: TCP
       delay: 5
@@ -384,7 +384,7 @@ resources:
       timeout: 5
 
   etcd_pool:
-    type: OS::Neutron::Pool
+    type: Magnum::Optional::Neutron::Pool
     properties:
       protocol: HTTP
       monitors: [{get_resource: etcd_monitor}]
@@ -393,6 +393,26 @@ resources:
       vip:
         protocol_port: 2379
 
+  ######################################################################
+  #
+  # resources that expose the IPs of either the kube master or a given
+  # LBaaS pool depending on whether LBaaS is enabled for the bay.
+  #
+
+  api_address_switch:
+    type: Magnum::ApiGatewaySwitcher
+    properties:
+      pool_public_ip: {get_attr: [api_pool_floating, floating_ip_address]}
+      pool_private_ip: {get_attr: [api_pool, vip, address]}
+      master_public_ip: {get_attr: [kube_masters, resource.0.kube_master_external_ip]}
+      master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]}
+
+  etcd_address_switch:
+    type: Magnum::ApiGatewaySwitcher
+    properties:
+      pool_private_ip: {get_attr: [etcd_pool, vip, address]}
+      master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]}
+
   ######################################################################
   #
   # kubernetes masters. This is a resource group that will create
@@ -470,8 +490,8 @@ resources:
           fixed_subnet: {get_resource: fixed_subnet}
           network_driver: {get_param: network_driver}
           flannel_network_cidr: {get_param: flannel_network_cidr}
-          kube_master_ip: {get_attr: [api_pool, vip, address]}
+          kube_master_ip: {get_attr: [api_address_switch, private_ip]}
-          etcd_server_ip: {get_attr: [etcd_pool, vip, address]}
+          etcd_server_ip: {get_attr: [etcd_address_switch, private_ip]}
           external_network: {get_param: external_network}
           kube_allow_priv: {get_param: kube_allow_priv}
           docker_volume_size: {get_param: docker_volume_size}
@@ -513,7 +533,7 @@ outputs:
       str_replace:
         template: api_ip_address
         params:
-          api_ip_address: {get_attr: [api_pool_floating, floating_ip_address]}
+          api_ip_address: {get_attr: [api_address_switch, public_ip]}
     description: >
       This is the API endpoint of the Kubernetes server. Use this to access
       the Kubernetes API from outside the cluster.

magnum/templates/kubernetes/kubemaster.yaml

@@ -89,10 +89,12 @@ parameters:
   api_public_address:
     type: string
     description: Public IP address of the Kubernetes master server.
+    default: ""
 
   api_private_address:
     type: string
     description: Private IP address of the Kubernetes master server.
+    default: ""
 
   fixed_network:
     type: string
@@ -193,6 +195,20 @@ resources:
       handle: {get_resource: master_wait_handle}
       timeout: {get_param: wait_condition_timeout}
 
+  ######################################################################
+  #
+  # resource that exposes the IPs of either the kube master or the API
+  # LBaaS pool depending on whether LBaaS is enabled for the bay.
+  #
+
+  api_address_switch:
+    type: Magnum::ApiGatewaySwitcher
+    properties:
+      pool_public_ip: {get_param: api_public_address}
+      pool_private_ip: {get_param: api_private_address}
+      master_public_ip: {get_attr: [kube_master_floating, floating_ip_address]}
+      master_private_ip: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
+
   ######################################################################
   #
   # software configs.  these are components that are combined into
@@ -207,8 +223,8 @@ resources:
         str_replace:
           template: {get_file: fragments/write-heat-params-master.yaml}
           params:
-            "$KUBE_API_PUBLIC_ADDRESS": {get_param: api_public_address}
+            "$KUBE_API_PUBLIC_ADDRESS": {get_attr: [api_address_switch, public_ip]}
-            "$KUBE_API_PRIVATE_ADDRESS": {get_param: api_private_address}
+            "$KUBE_API_PRIVATE_ADDRESS": {get_attr: [api_address_switch, private_ip]}
             "$KUBE_API_PORT": {get_param: kubernetes_port}
             "$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
             "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
@@ -410,14 +426,14 @@ resources:
       port_id: {get_resource: kube_master_eth0}
 
   api_pool_member:
-    type: OS::Neutron::PoolMember
+    type: Magnum::Optional::Neutron::PoolMember
     properties:
       pool_id: {get_param: api_pool_id}
       address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
       protocol_port: {get_param: kubernetes_port}
 
   etcd_pool_member:
-    type: OS::Neutron::PoolMember
+    type: Magnum::Optional::Neutron::PoolMember
     properties:
       pool_id: {get_param: etcd_pool_id}
       address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}

magnum/tests/unit/api/controllers/v1/test_bay.py

@@ -640,6 +640,24 @@ class TestPost(api_base.FunctionalTest):
         self.assertTrue(self.mock_valid_os_res.called)
         self.assertEqual(400, response.status_int)
 
+    def test_create_bay_with_no_lb_one_node(self):
+        baymodel = obj_utils.create_test_baymodel(
+            self.context, name='foo', uuid='foo', master_lb_enabled=False)
+        bdict = apiutils.bay_post_data(baymodel_id=baymodel.name,
+                                       master_count=1)
+        response = self.post_json('/bays', bdict, expect_errors=True)
+        self.assertEqual('application/json', response.content_type)
+        self.assertEqual(201, response.status_int)
+
+    def test_create_bay_with_no_lb_multi_node(self):
+        baymodel = obj_utils.create_test_baymodel(
+            self.context, name='foo', uuid='foo', master_lb_enabled=False)
+        bdict = apiutils.bay_post_data(baymodel_id=baymodel.name,
+                                       master_count=3)
+        response = self.post_json('/bays', bdict, expect_errors=True)
+        self.assertEqual('application/json', response.content_type)
+        self.assertEqual(400, response.status_int)
+
 
 class TestDelete(api_base.FunctionalTest):
 

magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py

@@ -48,6 +48,7 @@ class TestBayConductorWithK8s(base.TestCase):
             'server_type': 'vm',
             'registry_enabled': False,
             'insecure_registry': '10.0.0.1:5000',
+            'master_lb_enabled': False,
         }
         self.bay_dict = {
             'uuid': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52',
@@ -176,7 +177,7 @@ class TestBayConductorWithK8s(base.TestCase):
             expected.pop(mapping[missing_attr], None)
 
         self.assertEqual(expected, definition)
-        self.assertEqual([], env_files)
+        self.assertEqual(['environments/no_master_lb.yaml'], env_files)
 
     @patch('requests.get')
     @patch('magnum.objects.BayModel.get_by_uuid')
@@ -242,7 +243,7 @@ class TestBayConductorWithK8s(base.TestCase):
         }
 
         self.assertEqual(expected, definition)
-        self.assertEqual([], env_files)
+        self.assertEqual(['environments/no_master_lb.yaml'], env_files)
 
     @patch('requests.get')
     @patch('magnum.objects.BayModel.get_by_uuid')
@@ -296,7 +297,7 @@ class TestBayConductorWithK8s(base.TestCase):
             'insecure_registry_url': '10.0.0.1:5000',
         }
         self.assertEqual(expected, definition)
-        self.assertEqual([], env_files)
+        self.assertEqual(['environments/no_master_lb.yaml'], env_files)
 
     @patch('requests.get')
     @patch('magnum.objects.BayModel.get_by_uuid')
@@ -348,7 +349,7 @@ class TestBayConductorWithK8s(base.TestCase):
             'insecure_registry_url': '10.0.0.1:5000',
         }
         self.assertEqual(expected, definition)
-        self.assertEqual([], env_files)
+        self.assertEqual(['environments/no_master_lb.yaml'], env_files)
 
     @patch('requests.get')
     @patch('magnum.objects.BayModel.get_by_uuid')
@@ -508,7 +509,7 @@ class TestBayConductorWithK8s(base.TestCase):
             'insecure_registry_url': '10.0.0.1:5000',
         }
         self.assertEqual(expected, definition)
-        self.assertEqual([], env_files)
+        self.assertEqual(['environments/no_master_lb.yaml'], env_files)
         reqget.assert_called_once_with('http://etcd/test?size=1')
 
     @patch('magnum.common.short_id.generate_id')

magnum/tests/unit/db/utils.py

@@ -53,7 +53,7 @@ def get_test_baymodel(**kw):
         'public': kw.get('public', False),
         'server_type': kw.get('server_type', 'vm'),
         'insecure_registry': kw.get('insecure_registry', '10.0.0.1:5000'),
-        'master_lb_enabled': kw.get('master_lb_enabled', False),
+        'master_lb_enabled': kw.get('master_lb_enabled', True),
     }