Allow swarm cluster without LBaaS

This patch modifies the swarm bay driver/templates to allow its LBaaS
resources to be conditionally enabled/disabled.

Change-Id: Ic6ebec89514288837a7b540ce5517413e579f488
Partially-Implements: blueprint decouple-lbaas

magnum/drivers/swarm_fedora_atomic_v1/template_def.py

@@ -112,6 +112,12 @@ class AtomicSwarmTemplateDefinition(template_def.BaseTemplateDefinition):
                                       extra_params=extra_params,
                                       **kwargs)
 
+    def get_env_files(self, baymodel):
+        if baymodel.master_lb_enabled:
+            return ['environments/with_master_lb.yaml']
+        else:
+            return ['environments/no_master_lb.yaml']
+
     @property
     def template_path(self):
         return os.path.join(os.path.dirname(os.path.realpath(__file__)),

magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml

@@ -274,7 +274,7 @@ resources:
   #
 
   api_monitor:
-    type: OS::Neutron::HealthMonitor
+    type: Magnum::Optional::Neutron::Pool::HealthMonitor
     properties:
       type: TCP
       delay: 5
@@ -282,7 +282,7 @@ resources:
       timeout: 5
 
   api_pool:
-    type: OS::Neutron::Pool
+    type: Magnum::Optional::Neutron::Pool
     properties:
       protocol: {get_param: loadbalancing_protocol}
       monitors: [{get_resource: api_monitor}]
@@ -292,7 +292,7 @@ resources:
         protocol_port: {get_param: swarm_port}
 
   api_pool_floating:
-    type: OS::Neutron::FloatingIP
+    type: Magnum::Optional::Neutron::Pool::FloatingIP
     depends_on:
       - extrouter_inside
     properties:
@@ -300,7 +300,7 @@ resources:
       port_id: {get_attr: [api_pool, vip, port_id]}
 
   etcd_monitor:
-    type: OS::Neutron::HealthMonitor
+    type: Magnum::Optional::Neutron::Pool::HealthMonitor
     properties:
       type: TCP
       delay: 5
@@ -308,7 +308,7 @@ resources:
       timeout: 5
 
   etcd_pool:
-    type: OS::Neutron::Pool
+    type: Magnum::Optional::Neutron::Pool
     properties:
       protocol: HTTP
       monitors: [{get_resource: etcd_monitor}]
@@ -317,6 +317,26 @@ resources:
       vip:
         protocol_port: 2379
 
+  ######################################################################
+  #
+  # resources that expose the IPs of either the swarm master or a given
+  # LBaaS pool depending on whether LBaaS is enabled for the bay.
+  #
+
+  api_address_switch:
+    type: Magnum::ApiGatewaySwitcher
+    properties:
+      pool_public_ip: {get_attr: [api_pool_floating, floating_ip_address]}
+      pool_private_ip: {get_attr: [api_pool, vip, address]}
+      master_public_ip: {get_attr: [swarm_masters, resource.0.swarm_master_external_ip]}
+      master_private_ip: {get_attr: [swarm_masters, resource.0.swarm_master_ip]}
+
+  etcd_address_switch:
+    type: Magnum::ApiGatewaySwitcher
+    properties:
+      pool_private_ip: {get_attr: [etcd_pool, vip, address]}
+      master_private_ip: {get_attr: [swarm_masters, resource.0.swarm_master_ip]}
+
   ######################################################################
   #
   # Swarm manager is responsible for the entire cluster and manages the
@@ -386,15 +406,15 @@ resources:
           http_proxy: {get_param: http_proxy}
           https_proxy: {get_param: https_proxy}
           no_proxy: {get_param: no_proxy}
-          swarm_api_ip: {get_attr: [api_pool, vip, address]}
+          swarm_api_ip: {get_attr: [api_address_switch, private_ip]}
           bay_uuid: {get_param: bay_uuid}
           magnum_url: {get_param: magnum_url}
           tls_disabled: {get_param: tls_disabled}
           secgroup_swarm_node_id: {get_resource: secgroup_manager}
           flannel_network_cidr: {get_param: flannel_network_cidr}
           network_driver: {get_param: network_driver}
-          etcd_server_ip: {get_attr: [etcd_pool, vip, address]}
-          api_ip_address: {get_attr: [api_pool_floating, floating_ip_address]}
+          etcd_server_ip: {get_attr: [etcd_address_switch, private_ip]}
+          api_ip_address: {get_attr: [api_address_switch, public_ip]}
           swarm_version: {get_param: swarm_version}
           trustee_domain_id: {get_param: trustee_domain_id}
           trustee_user_id: {get_param: trustee_user_id}
@@ -416,7 +436,7 @@ outputs:
       str_replace:
         template: api_ip_address
         params:
-          api_ip_address: {get_attr: [api_pool_floating, floating_ip_address]}
+          api_ip_address: {get_attr: [api_address_switch, public_ip]}
     description: >
       This is the API endpoint of the Swarm masters. Use this to access
       the Swarm API server from outside the cluster.

magnum/drivers/swarm_fedora_atomic_v1/templates/environments/no_master_lb.yaml

@@ -0,0 +1,12 @@
+# Environment file to disable LBaaS in a Swarm cluster by mapping
+# LBaaS-related resource types to OS::Heat::None
+resource_registry:
+  "Magnum::ApiGatewaySwitcher": ../fragments/api_gateway_switcher_master.yaml
+
+  # swarmcluster.yaml
+  "Magnum::Optional::Neutron::Pool": "OS::Heat::None"
+  "Magnum::Optional::Neutron::Pool::FloatingIP": "OS::Heat::None"
+  "Magnum::Optional::Neutron::Pool::HealthMonitor": "OS::Heat::None"
+
+  # swarmmaster.yaml
+  "Magnum::Optional::Neutron::PoolMember": "OS::Heat::None"

magnum/drivers/swarm_fedora_atomic_v1/templates/environments/with_master_lb.yaml

@@ -0,0 +1,12 @@
+# Environment file to enable LBaaS in a Swarm cluster by mapping
+# LBaaS-related resource types to the real LBaaS resource types.
+resource_registry:
+  "Magnum::ApiGatewaySwitcher": ../fragments/api_gateway_switcher_pool.yaml
+
+  # swarmcluster.yaml
+  "Magnum::Optional::Neutron::Pool": "OS::Neutron::Pool"
+  "Magnum::Optional::Neutron::Pool::FloatingIP": "OS::Neutron::FloatingIP"
+  "Magnum::Optional::Neutron::Pool::HealthMonitor": "OS::Neutron::HealthMonitor"
+
+  # swarmmaster.yaml
+  "Magnum::Optional::Neutron::PoolMember": "OS::Neutron::PoolMember"

magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/api_gateway_switcher_master.yaml

@@ -0,0 +1,32 @@
+heat_template_version: 2013-05-23
+
+description: >
+  This is a template resource that accepts public and private IPs from both
+  a Neutron LBaaS Pool and a master node. It connects the master inputs
+  to its outputs, essentially acting as one state of a multiplexer.
+
+parameters:
+
+  pool_public_ip:
+    type: string
+    default: ""
+
+  pool_private_ip:
+    type: string
+    default: ""
+
+  master_public_ip:
+    type: string
+    default: ""
+
+  master_private_ip:
+    type: string
+    default: ""
+
+outputs:
+
+  public_ip:
+    value: {get_param: master_public_ip}
+
+  private_ip:
+    value: {get_param: master_private_ip}

magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/api_gateway_switcher_pool.yaml

@@ -0,0 +1,32 @@
+heat_template_version: 2013-05-23
+
+description: >
+  This is a template resource that accepts public and private IPs from both
+  a Neutron LBaaS Pool and a master node. It connects the pool inputs
+  to its outputs, essentially acting as one state of a multiplexer.
+
+parameters:
+
+  pool_public_ip:
+    type: string
+    default: ""
+
+  pool_private_ip:
+    type: string
+    default: ""
+
+  master_public_ip:
+    type: string
+    default: ""
+
+  master_private_ip:
+    type: string
+    default: ""
+
+outputs:
+
+  public_ip:
+    value: {get_param: pool_public_ip}
+
+  private_ip:
+    value: {get_param: pool_private_ip}

magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml

@@ -49,10 +49,12 @@ parameters:
   swarm_api_ip:
     type: string
     description: swarm master's api server ip address
+    default: ""
 
   api_ip_address:
     type: string
     description: swarm master's api server public ip address
+    default: ""
 
   server_image:
     type: string
@@ -121,6 +123,7 @@ parameters:
   etcd_server_ip:
     type: string
     description: ip address of the load balancer pool of etcd server.
+    default: ""
 
   trustee_user_id:
     type: string
@@ -162,6 +165,26 @@ resources:
       handle: {get_resource: master_wait_handle}
       timeout: 6000
 
+  ######################################################################
+  #
+  # resource that exposes the IPs of either the kube master or the API
+  # LBaaS pool depending on whether LBaaS is enabled for the bay.
+  #
+
+  api_address_switch:
+    type: Magnum::ApiGatewaySwitcher
+    properties:
+      pool_public_ip: {get_param: api_ip_address}
+      pool_private_ip: {get_param: swarm_api_ip}
+      master_public_ip: {get_attr: [swarm_master_floating, floating_ip_address]}
+      master_private_ip: {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]}
+
+  etcd_address_switch:
+    type: Magnum::ApiGatewaySwitcher
+    properties:
+      pool_private_ip: {get_param: etcd_server_ip}
+      master_private_ip: {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]}
+
   ######################################################################
   #
   # software configs.  these are components that are combined into
@@ -183,7 +206,7 @@ resources:
             "$HTTP_PROXY": {get_param: http_proxy}
             "$HTTPS_PROXY": {get_param: https_proxy}
             "$NO_PROXY": {get_param: no_proxy}
-            "$SWARM_API_IP": {get_param: swarm_api_ip}
+            "$SWARM_API_IP": {get_attr: [api_address_switch, private_ip]}
             "$SWARM_NODE_IP": {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]}
             "$BAY_UUID": {get_param: bay_uuid}
             "$MAGNUM_URL": {get_param: magnum_url}
@@ -192,8 +215,8 @@ resources:
             "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
             "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
             "$FLANNEL_BACKEND": {get_param: flannel_backend}
-            "$ETCD_SERVER_IP": {get_param: etcd_server_ip}
-            "$API_IP_ADDRESS": {get_param: api_ip_address}
+            "$ETCD_SERVER_IP": {get_attr: [etcd_address_switch, private_ip]}
+            "$API_IP_ADDRESS": {get_attr: [api_address_switch, public_ip]}
             "$SWARM_VERSION": {get_param: swarm_version}
             "$TRUSTEE_USER_ID": {get_param: trustee_user_id}
             "$TRUSTEE_PASSWORD": {get_param: trustee_password}
@@ -278,7 +301,7 @@ resources:
         str_replace:
           template: {get_file: fragments/write-swarm-master-service.sh}
           params:
-            "$ETCD_SERVER_IP": {get_param: etcd_server_ip}
+            "$ETCD_SERVER_IP": {get_attr: [etcd_address_switch, private_ip]}
             "$NODE_IP": {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]}
             "$DISCOVERY_URL": {get_param: discovery_url}
             "$WAIT_HANDLE_ENDPOINT": {get_attr: [master_wait_handle, endpoint]}
@@ -381,14 +404,14 @@ resources:
         get_resource: swarm_master_eth0
 
   api_pool_member:
-    type: OS::Neutron::PoolMember
+    type: Magnum::Optional::Neutron::PoolMember
     properties:
       pool_id: {get_param: api_pool_id}
       address: {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]}
       protocol_port: {get_param: swarm_port}
 
   etcd_pool_member:
-    type: OS::Neutron::PoolMember
+    type: Magnum::Optional::Neutron::PoolMember
     properties:
       pool_id: {get_param: etcd_pool_id}
       address: {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]}

magnum/tests/unit/conductor/handlers/test_swarm_bay_conductor.py

@@ -46,7 +46,8 @@ class TestBayConductorWithSwarm(base.TestCase):
             'network_driver': 'network_driver',
             'labels': {'flannel_network_cidr': '10.101.0.0/16',
                        'flannel_network_subnetlen': '26',
-                       'flannel_backend': 'vxlan'}
+                       'flannel_backend': 'vxlan'},
+            'master_lb_enabled': False,
         }
         self.bay_dict = {
             'id': 1,
@@ -126,7 +127,7 @@ class TestBayConductorWithSwarm(base.TestCase):
             'auth_url': 'http://192.168.10.10:5000/v3'
         }
         self.assertEqual(expected, definition)
-        self.assertEqual([], env_files)
+        self.assertEqual(['environments/no_master_lb.yaml'], env_files)
 
     @patch('requests.get')
     @patch('magnum.objects.BayModel.get_by_uuid')
@@ -186,7 +187,7 @@ class TestBayConductorWithSwarm(base.TestCase):
             'docker_storage_driver': 'devicemapper'
         }
         self.assertEqual(expected, definition)
-        self.assertEqual([], env_files)
+        self.assertEqual(['environments/no_master_lb.yaml'], env_files)
 
     @patch('requests.get')
     @patch('magnum.objects.BayModel.get_by_uuid')
@@ -238,7 +239,116 @@ class TestBayConductorWithSwarm(base.TestCase):
             'auth_url': 'http://192.168.10.10:5000/v3'
         }
         self.assertEqual(expected, definition)
-        self.assertEqual([], env_files)
+        self.assertEqual(['environments/no_master_lb.yaml'], env_files)
+
+    @patch('requests.get')
+    @patch('magnum.objects.BayModel.get_by_uuid')
+    def test_extract_template_definition_with_lb(
+            self,
+            mock_objects_baymodel_get_by_uuid,
+            mock_get):
+        self.baymodel_dict['master_lb_enabled'] = True
+        baymodel = objects.BayModel(self.context, **self.baymodel_dict)
+        mock_objects_baymodel_get_by_uuid.return_value = baymodel
+        expected_result = str('{"action":"get","node":{"key":"test","value":'
+                              '"1","modifiedIndex":10,"createdIndex":10}}')
+        mock_resp = mock.MagicMock()
+        mock_resp.text = expected_result
+        mock_get.return_value = mock_resp
+        bay = objects.Bay(self.context, **self.bay_dict)
+
+        (template_path,
+         definition,
+         env_files) = bay_conductor._extract_template_definition(self.context,
+                                                                 bay)
+
+        expected = {
+            'ssh_key_name': 'keypair_id',
+            'external_network': 'external_network_id',
+            'dns_nameserver': 'dns_nameserver',
+            'server_image': 'image_id',
+            'master_flavor': 'master_flavor_id',
+            'node_flavor': 'flavor_id',
+            'number_of_masters': 1,
+            'number_of_nodes': 1,
+            'docker_volume_size': 20,
+            'docker_storage_driver': 'devicemapper',
+            'discovery_url': 'https://discovery.test.io/123456789',
+            'http_proxy': 'http_proxy',
+            'https_proxy': 'https_proxy',
+            'no_proxy': 'no_proxy',
+            'bay_uuid': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52',
+            'magnum_url': self.mock_osc.magnum_url.return_value,
+            'tls_disabled': False,
+            'registry_enabled': False,
+            'network_driver': 'network_driver',
+            'flannel_network_cidr': '10.101.0.0/16',
+            'flannel_network_subnetlen': '26',
+            'flannel_backend': 'vxlan',
+            'trustee_domain_id': self.mock_keystone.trustee_domain_id,
+            'trustee_username': 'fake_trustee',
+            'trustee_password': 'fake_trustee_password',
+            'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656',
+            'trust_id': 'bd11efc5-d4e2-4dac-bbce-25e348ddf7de',
+            'auth_url': 'http://192.168.10.10:5000/v3'
+        }
+        self.assertEqual(expected, definition)
+        self.assertEqual(['environments/with_master_lb.yaml'], env_files)
+
+    @patch('requests.get')
+    @patch('magnum.objects.BayModel.get_by_uuid')
+    def test_extract_template_definition_multi_master(
+            self,
+            mock_objects_baymodel_get_by_uuid,
+            mock_get):
+        self.baymodel_dict['master_lb_enabled'] = True
+        self.bay_dict['master_count'] = 2
+        baymodel = objects.BayModel(self.context, **self.baymodel_dict)
+        mock_objects_baymodel_get_by_uuid.return_value = baymodel
+        expected_result = str('{"action":"get","node":{"key":"test","value":'
+                              '"2","modifiedIndex":10,"createdIndex":10}}')
+        mock_resp = mock.MagicMock()
+        mock_resp.text = expected_result
+        mock_get.return_value = mock_resp
+        bay = objects.Bay(self.context, **self.bay_dict)
+
+        (template_path,
+         definition,
+         env_files) = bay_conductor._extract_template_definition(self.context,
+                                                                 bay)
+
+        expected = {
+            'ssh_key_name': 'keypair_id',
+            'external_network': 'external_network_id',
+            'dns_nameserver': 'dns_nameserver',
+            'server_image': 'image_id',
+            'master_flavor': 'master_flavor_id',
+            'node_flavor': 'flavor_id',
+            'number_of_masters': 2,
+            'number_of_nodes': 1,
+            'docker_volume_size': 20,
+            'docker_storage_driver': 'devicemapper',
+            'discovery_url': 'https://discovery.test.io/123456789',
+            'http_proxy': 'http_proxy',
+            'https_proxy': 'https_proxy',
+            'no_proxy': 'no_proxy',
+            'bay_uuid': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52',
+            'magnum_url': self.mock_osc.magnum_url.return_value,
+            'tls_disabled': False,
+            'registry_enabled': False,
+            'network_driver': 'network_driver',
+            'flannel_network_cidr': '10.101.0.0/16',
+            'flannel_network_subnetlen': '26',
+            'flannel_backend': 'vxlan',
+            'trustee_domain_id': self.mock_keystone.trustee_domain_id,
+            'trustee_username': 'fake_trustee',
+            'trustee_password': 'fake_trustee_password',
+            'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656',
+            'trust_id': 'bd11efc5-d4e2-4dac-bbce-25e348ddf7de',
+            'auth_url': 'http://192.168.10.10:5000/v3'
+        }
+        self.assertEqual(expected, definition)
+        self.assertEqual(['environments/with_master_lb.yaml'], env_files)
 
     @patch('magnum.conductor.utils.retrieve_baymodel')
     @patch('oslo_config.cfg')