diff --git a/magnum/drivers/swarm_fedora_atomic_v1/template_def.py b/magnum/drivers/swarm_fedora_atomic_v1/template_def.py index 90a6fed462..e20db66fe4 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/template_def.py +++ b/magnum/drivers/swarm_fedora_atomic_v1/template_def.py @@ -112,6 +112,12 @@ class AtomicSwarmTemplateDefinition(template_def.BaseTemplateDefinition): extra_params=extra_params, **kwargs) + def get_env_files(self, baymodel): + if baymodel.master_lb_enabled: + return ['environments/with_master_lb.yaml'] + else: + return ['environments/no_master_lb.yaml'] + @property def template_path(self): return os.path.join(os.path.dirname(os.path.realpath(__file__)), diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml index 3c369d4d05..320c8b217a 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml @@ -274,7 +274,7 @@ resources: # api_monitor: - type: OS::Neutron::HealthMonitor + type: Magnum::Optional::Neutron::Pool::HealthMonitor properties: type: TCP delay: 5 @@ -282,7 +282,7 @@ resources: timeout: 5 api_pool: - type: OS::Neutron::Pool + type: Magnum::Optional::Neutron::Pool properties: protocol: {get_param: loadbalancing_protocol} monitors: [{get_resource: api_monitor}] @@ -292,7 +292,7 @@ resources: protocol_port: {get_param: swarm_port} api_pool_floating: - type: OS::Neutron::FloatingIP + type: Magnum::Optional::Neutron::Pool::FloatingIP depends_on: - extrouter_inside properties: @@ -300,7 +300,7 @@ resources: port_id: {get_attr: [api_pool, vip, port_id]} etcd_monitor: - type: OS::Neutron::HealthMonitor + type: Magnum::Optional::Neutron::Pool::HealthMonitor properties: type: TCP delay: 5 @@ -308,7 +308,7 @@ resources: timeout: 5 etcd_pool: - type: OS::Neutron::Pool + type: Magnum::Optional::Neutron::Pool properties: protocol: HTTP monitors: [{get_resource: etcd_monitor}] @@ -317,6 +317,26 @@ resources: vip: protocol_port: 2379 + ###################################################################### + # + # resources that expose the IPs of either the swarm master or a given + # LBaaS pool depending on whether LBaaS is enabled for the bay. + # + + api_address_switch: + type: Magnum::ApiGatewaySwitcher + properties: + pool_public_ip: {get_attr: [api_pool_floating, floating_ip_address]} + pool_private_ip: {get_attr: [api_pool, vip, address]} + master_public_ip: {get_attr: [swarm_masters, resource.0.swarm_master_external_ip]} + master_private_ip: {get_attr: [swarm_masters, resource.0.swarm_master_ip]} + + etcd_address_switch: + type: Magnum::ApiGatewaySwitcher + properties: + pool_private_ip: {get_attr: [etcd_pool, vip, address]} + master_private_ip: {get_attr: [swarm_masters, resource.0.swarm_master_ip]} + ###################################################################### # # Swarm manager is responsible for the entire cluster and manages the @@ -385,15 +405,15 @@ resources: http_proxy: {get_param: http_proxy} https_proxy: {get_param: https_proxy} no_proxy: {get_param: no_proxy} - swarm_api_ip: {get_attr: [api_pool, vip, address]} + swarm_api_ip: {get_attr: [api_address_switch, private_ip]} bay_uuid: {get_param: bay_uuid} magnum_url: {get_param: magnum_url} tls_disabled: {get_param: tls_disabled} secgroup_swarm_node_id: {get_resource: secgroup_manager} flannel_network_cidr: {get_param: flannel_network_cidr} network_driver: {get_param: network_driver} - etcd_server_ip: {get_attr: [etcd_pool, vip, address]} - api_ip_address: {get_attr: [api_pool_floating, floating_ip_address]} + etcd_server_ip: {get_attr: [etcd_address_switch, private_ip]} + api_ip_address: {get_attr: [api_address_switch, public_ip]} swarm_version: {get_param: swarm_version} trustee_domain_id: {get_param: trustee_domain_id} trustee_user_id: {get_param: trustee_user_id} @@ -415,7 +435,7 @@ outputs: str_replace: template: api_ip_address params: - api_ip_address: {get_attr: [api_pool_floating, floating_ip_address]} + api_ip_address: {get_attr: [api_address_switch, public_ip]} description: > This is the API endpoint of the Swarm masters. Use this to access the Swarm API server from outside the cluster. diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/environments/no_master_lb.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/environments/no_master_lb.yaml new file mode 100644 index 0000000000..63b35d0061 --- /dev/null +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/environments/no_master_lb.yaml @@ -0,0 +1,12 @@ +# Environment file to disable LBaaS in a Swarm cluster by mapping +# LBaaS-related resource types to OS::Heat::None +resource_registry: + "Magnum::ApiGatewaySwitcher": ../fragments/api_gateway_switcher_master.yaml + + # swarmcluster.yaml + "Magnum::Optional::Neutron::Pool": "OS::Heat::None" + "Magnum::Optional::Neutron::Pool::FloatingIP": "OS::Heat::None" + "Magnum::Optional::Neutron::Pool::HealthMonitor": "OS::Heat::None" + + # swarmmaster.yaml + "Magnum::Optional::Neutron::PoolMember": "OS::Heat::None" diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/environments/with_master_lb.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/environments/with_master_lb.yaml new file mode 100644 index 0000000000..becdab0a7d --- /dev/null +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/environments/with_master_lb.yaml @@ -0,0 +1,12 @@ +# Environment file to enable LBaaS in a Swarm cluster by mapping +# LBaaS-related resource types to the real LBaaS resource types. +resource_registry: + "Magnum::ApiGatewaySwitcher": ../fragments/api_gateway_switcher_pool.yaml + + # swarmcluster.yaml + "Magnum::Optional::Neutron::Pool": "OS::Neutron::Pool" + "Magnum::Optional::Neutron::Pool::FloatingIP": "OS::Neutron::FloatingIP" + "Magnum::Optional::Neutron::Pool::HealthMonitor": "OS::Neutron::HealthMonitor" + + # swarmmaster.yaml + "Magnum::Optional::Neutron::PoolMember": "OS::Neutron::PoolMember" diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/api_gateway_switcher_master.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/api_gateway_switcher_master.yaml new file mode 100644 index 0000000000..d10437827a --- /dev/null +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/api_gateway_switcher_master.yaml @@ -0,0 +1,32 @@ +heat_template_version: 2013-05-23 + +description: > + This is a template resource that accepts public and private IPs from both + a Neutron LBaaS Pool and a master node. It connects the master inputs + to its outputs, essentially acting as one state of a multiplexer. + +parameters: + + pool_public_ip: + type: string + default: "" + + pool_private_ip: + type: string + default: "" + + master_public_ip: + type: string + default: "" + + master_private_ip: + type: string + default: "" + +outputs: + + public_ip: + value: {get_param: master_public_ip} + + private_ip: + value: {get_param: master_private_ip} diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/api_gateway_switcher_pool.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/api_gateway_switcher_pool.yaml new file mode 100644 index 0000000000..7a1384a4cf --- /dev/null +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/api_gateway_switcher_pool.yaml @@ -0,0 +1,32 @@ +heat_template_version: 2013-05-23 + +description: > + This is a template resource that accepts public and private IPs from both + a Neutron LBaaS Pool and a master node. It connects the pool inputs + to its outputs, essentially acting as one state of a multiplexer. + +parameters: + + pool_public_ip: + type: string + default: "" + + pool_private_ip: + type: string + default: "" + + master_public_ip: + type: string + default: "" + + master_private_ip: + type: string + default: "" + +outputs: + + public_ip: + value: {get_param: pool_public_ip} + + private_ip: + value: {get_param: pool_private_ip} diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml index 452aa04e6d..93ca14ec33 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml @@ -49,10 +49,12 @@ parameters: swarm_api_ip: type: string description: swarm master's api server ip address + default: "" api_ip_address: type: string description: swarm master's api server public ip address + default: "" server_image: type: string @@ -121,6 +123,7 @@ parameters: etcd_server_ip: type: string description: ip address of the load balancer pool of etcd server. + default: "" trustee_user_id: type: string @@ -162,6 +165,26 @@ resources: handle: {get_resource: master_wait_handle} timeout: 6000 + ###################################################################### + # + # resource that exposes the IPs of either the kube master or the API + # LBaaS pool depending on whether LBaaS is enabled for the bay. + # + + api_address_switch: + type: Magnum::ApiGatewaySwitcher + properties: + pool_public_ip: {get_param: api_ip_address} + pool_private_ip: {get_param: swarm_api_ip} + master_public_ip: {get_attr: [swarm_master_floating, floating_ip_address]} + master_private_ip: {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]} + + etcd_address_switch: + type: Magnum::ApiGatewaySwitcher + properties: + pool_private_ip: {get_param: etcd_server_ip} + master_private_ip: {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]} + ###################################################################### # # software configs. these are components that are combined into @@ -184,7 +207,7 @@ resources: "$HTTP_PROXY": {get_param: http_proxy} "$HTTPS_PROXY": {get_param: https_proxy} "$NO_PROXY": {get_param: no_proxy} - "$SWARM_API_IP": {get_param: swarm_api_ip} + "$SWARM_API_IP": {get_attr: [api_address_switch, private_ip]} "$SWARM_NODE_IP": {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]} "$BAY_UUID": {get_param: bay_uuid} "$MAGNUM_URL": {get_param: magnum_url} @@ -193,8 +216,8 @@ resources: "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr} "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen} "$FLANNEL_BACKEND": {get_param: flannel_backend} - "$ETCD_SERVER_IP": {get_param: etcd_server_ip} - "$API_IP_ADDRESS": {get_param: api_ip_address} + "$ETCD_SERVER_IP": {get_attr: [etcd_address_switch, private_ip]} + "$API_IP_ADDRESS": {get_attr: [api_address_switch, public_ip]} "$SWARM_VERSION": {get_param: swarm_version} "$TRUSTEE_USER_ID": {get_param: trustee_user_id} "$TRUSTEE_PASSWORD": {get_param: trustee_password} @@ -279,7 +302,7 @@ resources: str_replace: template: {get_file: fragments/write-swarm-master-service.sh} params: - "$ETCD_SERVER_IP": {get_param: etcd_server_ip} + "$ETCD_SERVER_IP": {get_attr: [etcd_address_switch, private_ip]} "$NODE_IP": {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]} "$WAIT_HANDLE_ENDPOINT": {get_attr: [master_wait_handle, endpoint]} "$WAIT_HANDLE_TOKEN": {get_attr: [master_wait_handle, token]} @@ -381,14 +404,14 @@ resources: get_resource: swarm_master_eth0 api_pool_member: - type: OS::Neutron::PoolMember + type: Magnum::Optional::Neutron::PoolMember properties: pool_id: {get_param: api_pool_id} address: {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]} protocol_port: {get_param: swarm_port} etcd_pool_member: - type: OS::Neutron::PoolMember + type: Magnum::Optional::Neutron::PoolMember properties: pool_id: {get_param: etcd_pool_id} address: {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]} diff --git a/magnum/tests/unit/conductor/handlers/test_swarm_bay_conductor.py b/magnum/tests/unit/conductor/handlers/test_swarm_bay_conductor.py index 829883d687..959dbf9255 100644 --- a/magnum/tests/unit/conductor/handlers/test_swarm_bay_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_swarm_bay_conductor.py @@ -46,7 +46,8 @@ class TestBayConductorWithSwarm(base.TestCase): 'network_driver': 'network_driver', 'labels': {'flannel_network_cidr': '10.101.0.0/16', 'flannel_network_subnetlen': '26', - 'flannel_backend': 'vxlan'} + 'flannel_backend': 'vxlan'}, + 'master_lb_enabled': False, } self.bay_dict = { 'id': 1, @@ -126,7 +127,7 @@ class TestBayConductorWithSwarm(base.TestCase): 'auth_url': 'http://192.168.10.10:5000/v3' } self.assertEqual(expected, definition) - self.assertEqual([], env_files) + self.assertEqual(['environments/no_master_lb.yaml'], env_files) @patch('requests.get') @patch('magnum.objects.BayModel.get_by_uuid') @@ -186,7 +187,7 @@ class TestBayConductorWithSwarm(base.TestCase): 'docker_storage_driver': 'devicemapper' } self.assertEqual(expected, definition) - self.assertEqual([], env_files) + self.assertEqual(['environments/no_master_lb.yaml'], env_files) @patch('requests.get') @patch('magnum.objects.BayModel.get_by_uuid') @@ -238,7 +239,116 @@ class TestBayConductorWithSwarm(base.TestCase): 'auth_url': 'http://192.168.10.10:5000/v3' } self.assertEqual(expected, definition) - self.assertEqual([], env_files) + self.assertEqual(['environments/no_master_lb.yaml'], env_files) + + @patch('requests.get') + @patch('magnum.objects.BayModel.get_by_uuid') + def test_extract_template_definition_with_lb( + self, + mock_objects_baymodel_get_by_uuid, + mock_get): + self.baymodel_dict['master_lb_enabled'] = True + baymodel = objects.BayModel(self.context, **self.baymodel_dict) + mock_objects_baymodel_get_by_uuid.return_value = baymodel + expected_result = str('{"action":"get","node":{"key":"test","value":' + '"1","modifiedIndex":10,"createdIndex":10}}') + mock_resp = mock.MagicMock() + mock_resp.text = expected_result + mock_get.return_value = mock_resp + bay = objects.Bay(self.context, **self.bay_dict) + + (template_path, + definition, + env_files) = bay_conductor._extract_template_definition(self.context, + bay) + + expected = { + 'ssh_key_name': 'keypair_id', + 'external_network': 'external_network_id', + 'dns_nameserver': 'dns_nameserver', + 'server_image': 'image_id', + 'master_flavor': 'master_flavor_id', + 'node_flavor': 'flavor_id', + 'number_of_masters': 1, + 'number_of_nodes': 1, + 'docker_volume_size': 20, + 'docker_storage_driver': 'devicemapper', + 'discovery_url': 'https://discovery.test.io/123456789', + 'http_proxy': 'http_proxy', + 'https_proxy': 'https_proxy', + 'no_proxy': 'no_proxy', + 'bay_uuid': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52', + 'magnum_url': self.mock_osc.magnum_url.return_value, + 'tls_disabled': False, + 'registry_enabled': False, + 'network_driver': 'network_driver', + 'flannel_network_cidr': '10.101.0.0/16', + 'flannel_network_subnetlen': '26', + 'flannel_backend': 'vxlan', + 'trustee_domain_id': self.mock_keystone.trustee_domain_id, + 'trustee_username': 'fake_trustee', + 'trustee_password': 'fake_trustee_password', + 'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656', + 'trust_id': 'bd11efc5-d4e2-4dac-bbce-25e348ddf7de', + 'auth_url': 'http://192.168.10.10:5000/v3' + } + self.assertEqual(expected, definition) + self.assertEqual(['environments/with_master_lb.yaml'], env_files) + + @patch('requests.get') + @patch('magnum.objects.BayModel.get_by_uuid') + def test_extract_template_definition_multi_master( + self, + mock_objects_baymodel_get_by_uuid, + mock_get): + self.baymodel_dict['master_lb_enabled'] = True + self.bay_dict['master_count'] = 2 + baymodel = objects.BayModel(self.context, **self.baymodel_dict) + mock_objects_baymodel_get_by_uuid.return_value = baymodel + expected_result = str('{"action":"get","node":{"key":"test","value":' + '"2","modifiedIndex":10,"createdIndex":10}}') + mock_resp = mock.MagicMock() + mock_resp.text = expected_result + mock_get.return_value = mock_resp + bay = objects.Bay(self.context, **self.bay_dict) + + (template_path, + definition, + env_files) = bay_conductor._extract_template_definition(self.context, + bay) + + expected = { + 'ssh_key_name': 'keypair_id', + 'external_network': 'external_network_id', + 'dns_nameserver': 'dns_nameserver', + 'server_image': 'image_id', + 'master_flavor': 'master_flavor_id', + 'node_flavor': 'flavor_id', + 'number_of_masters': 2, + 'number_of_nodes': 1, + 'docker_volume_size': 20, + 'docker_storage_driver': 'devicemapper', + 'discovery_url': 'https://discovery.test.io/123456789', + 'http_proxy': 'http_proxy', + 'https_proxy': 'https_proxy', + 'no_proxy': 'no_proxy', + 'bay_uuid': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52', + 'magnum_url': self.mock_osc.magnum_url.return_value, + 'tls_disabled': False, + 'registry_enabled': False, + 'network_driver': 'network_driver', + 'flannel_network_cidr': '10.101.0.0/16', + 'flannel_network_subnetlen': '26', + 'flannel_backend': 'vxlan', + 'trustee_domain_id': self.mock_keystone.trustee_domain_id, + 'trustee_username': 'fake_trustee', + 'trustee_password': 'fake_trustee_password', + 'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656', + 'trust_id': 'bd11efc5-d4e2-4dac-bbce-25e348ddf7de', + 'auth_url': 'http://192.168.10.10:5000/v3' + } + self.assertEqual(expected, definition) + self.assertEqual(['environments/with_master_lb.yaml'], env_files) @patch('magnum.conductor.utils.retrieve_baymodel') @patch('oslo_config.cfg')