From 2ea72d739d115c6810ae86c776ef575deedfe7c3 Mon Sep 17 00:00:00 2001 From: Drago Rosson Date: Fri, 24 Jun 2016 12:25:39 -0500 Subject: [PATCH] Allow k8s cluster without LBaaS This patch adds an environment file and a couple of template resources to allow the LBaaS resources to be conditionally enabled/disabled. Change-Id: I40ef0839dca84f398efb02022fa7c1de821fb1a3 Partially-Implements: blueprint decouple-lbaas Partially-Implements: blueprint bay-with-no-floating-ips --- magnum/api/attr_validator.py | 6 ++++ magnum/api/controllers/v1/bay.py | 1 + magnum/conductor/template_definition.py | 6 ++++ .../kubernetes/environments/no_master_lb.yaml | 12 +++++++ .../environments/with_master_lb.yaml | 12 +++++++ .../api_gateway_switcher_master.yaml | 32 +++++++++++++++++ .../fragments/api_gateway_switcher_pool.yaml | 32 +++++++++++++++++ magnum/templates/kubernetes/kubecluster.yaml | 36 ++++++++++++++----- magnum/templates/kubernetes/kubemaster.yaml | 24 ++++++++++--- .../tests/unit/api/controllers/v1/test_bay.py | 18 ++++++++++ .../handlers/test_k8s_bay_conductor.py | 11 +++--- magnum/tests/unit/db/utils.py | 2 +- 12 files changed, 174 insertions(+), 18 deletions(-) create mode 100644 magnum/templates/kubernetes/environments/no_master_lb.yaml create mode 100644 magnum/templates/kubernetes/environments/with_master_lb.yaml create mode 100644 magnum/templates/kubernetes/fragments/api_gateway_switcher_master.yaml create mode 100644 magnum/templates/kubernetes/fragments/api_gateway_switcher_pool.yaml diff --git a/magnum/api/attr_validator.py b/magnum/api/attr_validator.py index 60f06f19fb..fc50815623 100644 --- a/magnum/api/attr_validator.py +++ b/magnum/api/attr_validator.py @@ -164,6 +164,12 @@ def validate_os_resources(context, baymodel): validate_method(baymodel[attr]) +def validate_master_count(bay, baymodel): + if bay['master_count'] > 1 and not baymodel['master_lb_enabled']: + raise exception.InvalidParameterValue(_( + "master_count must be 1 when master_lb_enabled is False")) + + # Dictionary that maintains a list of validation functions validators = {'image_id': validate_image, 'flavor_id': validate_flavor, diff --git a/magnum/api/controllers/v1/bay.py b/magnum/api/controllers/v1/bay.py index 83998df3a7..cfa80fc32f 100644 --- a/magnum/api/controllers/v1/bay.py +++ b/magnum/api/controllers/v1/bay.py @@ -329,6 +329,7 @@ class BaysController(rest.RestController): action='bay:create') baymodel = objects.BayModel.get_by_uuid(context, bay.baymodel_id) attr_validator.validate_os_resources(context, baymodel.as_dict()) + attr_validator.validate_master_count(bay.as_dict(), baymodel.as_dict()) bay_dict = bay.as_dict() bay_dict['project_id'] = context.project_id bay_dict['user_id'] = context.user_id diff --git a/magnum/conductor/template_definition.py b/magnum/conductor/template_definition.py index 530783572e..4c69610f4e 100644 --- a/magnum/conductor/template_definition.py +++ b/magnum/conductor/template_definition.py @@ -161,6 +161,12 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition): extra_params=extra_params, **kwargs) + def get_env_files(self, baymodel): + if baymodel.master_lb_enabled: + return ['environments/with_master_lb.yaml'] + else: + return ['environments/no_master_lb.yaml'] + class AtomicK8sTemplateDefinition(K8sTemplateDefinition): """Kubernetes template for a Fedora Atomic VM.""" diff --git a/magnum/templates/kubernetes/environments/no_master_lb.yaml b/magnum/templates/kubernetes/environments/no_master_lb.yaml new file mode 100644 index 0000000000..0d2a5bbc6f --- /dev/null +++ b/magnum/templates/kubernetes/environments/no_master_lb.yaml @@ -0,0 +1,12 @@ +# Environment file to disable LBaaS in a Kubernetes cluster by mapping +# LBaaS-related resource types to OS::Heat::None +resource_registry: + "Magnum::ApiGatewaySwitcher": ../fragments/api_gateway_switcher_master.yaml + + # kubecluster.yaml + "Magnum::Optional::Neutron::Pool": "OS::Heat::None" + "Magnum::Optional::Neutron::Pool::FloatingIP": "OS::Heat::None" + "Magnum::Optional::Neutron::Pool::HealthMonitor": "OS::Heat::None" + + # kubemaster.yaml + "Magnum::Optional::Neutron::PoolMember": "OS::Heat::None" diff --git a/magnum/templates/kubernetes/environments/with_master_lb.yaml b/magnum/templates/kubernetes/environments/with_master_lb.yaml new file mode 100644 index 0000000000..61d9695789 --- /dev/null +++ b/magnum/templates/kubernetes/environments/with_master_lb.yaml @@ -0,0 +1,12 @@ +# Environment file to enable LBaaS in a Kubernetes cluster by mapping +# LBaaS-related resource types to the real LBaaS resource types. +resource_registry: + "Magnum::ApiGatewaySwitcher": ../fragments/api_gateway_switcher_pool.yaml + + # kubecluster.yaml + "Magnum::Optional::Neutron::Pool": "OS::Neutron::Pool" + "Magnum::Optional::Neutron::Pool::FloatingIP": "OS::Neutron::FloatingIP" + "Magnum::Optional::Neutron::Pool::HealthMonitor": "OS::Neutron::HealthMonitor" + + # kubemaster.yaml + "Magnum::Optional::Neutron::PoolMember": "OS::Neutron::PoolMember" diff --git a/magnum/templates/kubernetes/fragments/api_gateway_switcher_master.yaml b/magnum/templates/kubernetes/fragments/api_gateway_switcher_master.yaml new file mode 100644 index 0000000000..d10437827a --- /dev/null +++ b/magnum/templates/kubernetes/fragments/api_gateway_switcher_master.yaml @@ -0,0 +1,32 @@ +heat_template_version: 2013-05-23 + +description: > + This is a template resource that accepts public and private IPs from both + a Neutron LBaaS Pool and a master node. It connects the master inputs + to its outputs, essentially acting as one state of a multiplexer. + +parameters: + + pool_public_ip: + type: string + default: "" + + pool_private_ip: + type: string + default: "" + + master_public_ip: + type: string + default: "" + + master_private_ip: + type: string + default: "" + +outputs: + + public_ip: + value: {get_param: master_public_ip} + + private_ip: + value: {get_param: master_private_ip} diff --git a/magnum/templates/kubernetes/fragments/api_gateway_switcher_pool.yaml b/magnum/templates/kubernetes/fragments/api_gateway_switcher_pool.yaml new file mode 100644 index 0000000000..7a1384a4cf --- /dev/null +++ b/magnum/templates/kubernetes/fragments/api_gateway_switcher_pool.yaml @@ -0,0 +1,32 @@ +heat_template_version: 2013-05-23 + +description: > + This is a template resource that accepts public and private IPs from both + a Neutron LBaaS Pool and a master node. It connects the pool inputs + to its outputs, essentially acting as one state of a multiplexer. + +parameters: + + pool_public_ip: + type: string + default: "" + + pool_private_ip: + type: string + default: "" + + master_public_ip: + type: string + default: "" + + master_private_ip: + type: string + default: "" + +outputs: + + public_ip: + value: {get_param: pool_public_ip} + + private_ip: + value: {get_param: pool_private_ip} diff --git a/magnum/templates/kubernetes/kubecluster.yaml b/magnum/templates/kubernetes/kubecluster.yaml index 47a3eafb5c..bb3992bc85 100644 --- a/magnum/templates/kubernetes/kubecluster.yaml +++ b/magnum/templates/kubernetes/kubecluster.yaml @@ -350,7 +350,7 @@ resources: # api_monitor: - type: OS::Neutron::HealthMonitor + type: Magnum::Optional::Neutron::Pool::HealthMonitor properties: type: TCP delay: 5 @@ -358,7 +358,7 @@ resources: timeout: 5 api_pool: - type: OS::Neutron::Pool + type: Magnum::Optional::Neutron::Pool properties: protocol: {get_param: loadbalancing_protocol} monitors: [{get_resource: api_monitor}] @@ -368,7 +368,7 @@ resources: protocol_port: {get_param: kubernetes_port} api_pool_floating: - type: OS::Neutron::FloatingIP + type: Magnum::Optional::Neutron::Pool::FloatingIP depends_on: - extrouter_inside properties: @@ -376,7 +376,7 @@ resources: port_id: {get_attr: [api_pool, vip, port_id]} etcd_monitor: - type: OS::Neutron::HealthMonitor + type: Magnum::Optional::Neutron::Pool::HealthMonitor properties: type: TCP delay: 5 @@ -384,7 +384,7 @@ resources: timeout: 5 etcd_pool: - type: OS::Neutron::Pool + type: Magnum::Optional::Neutron::Pool properties: protocol: HTTP monitors: [{get_resource: etcd_monitor}] @@ -393,6 +393,26 @@ resources: vip: protocol_port: 2379 + ###################################################################### + # + # resources that expose the IPs of either the kube master or a given + # LBaaS pool depending on whether LBaaS is enabled for the bay. + # + + api_address_switch: + type: Magnum::ApiGatewaySwitcher + properties: + pool_public_ip: {get_attr: [api_pool_floating, floating_ip_address]} + pool_private_ip: {get_attr: [api_pool, vip, address]} + master_public_ip: {get_attr: [kube_masters, resource.0.kube_master_external_ip]} + master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]} + + etcd_address_switch: + type: Magnum::ApiGatewaySwitcher + properties: + pool_private_ip: {get_attr: [etcd_pool, vip, address]} + master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]} + ###################################################################### # # kubernetes masters. This is a resource group that will create @@ -470,8 +490,8 @@ resources: fixed_subnet: {get_resource: fixed_subnet} network_driver: {get_param: network_driver} flannel_network_cidr: {get_param: flannel_network_cidr} - kube_master_ip: {get_attr: [api_pool, vip, address]} - etcd_server_ip: {get_attr: [etcd_pool, vip, address]} + kube_master_ip: {get_attr: [api_address_switch, private_ip]} + etcd_server_ip: {get_attr: [etcd_address_switch, private_ip]} external_network: {get_param: external_network} kube_allow_priv: {get_param: kube_allow_priv} docker_volume_size: {get_param: docker_volume_size} @@ -513,7 +533,7 @@ outputs: str_replace: template: api_ip_address params: - api_ip_address: {get_attr: [api_pool_floating, floating_ip_address]} + api_ip_address: {get_attr: [api_address_switch, public_ip]} description: > This is the API endpoint of the Kubernetes server. Use this to access the Kubernetes API from outside the cluster. diff --git a/magnum/templates/kubernetes/kubemaster.yaml b/magnum/templates/kubernetes/kubemaster.yaml index cebcac127a..7ce492616d 100644 --- a/magnum/templates/kubernetes/kubemaster.yaml +++ b/magnum/templates/kubernetes/kubemaster.yaml @@ -89,10 +89,12 @@ parameters: api_public_address: type: string description: Public IP address of the Kubernetes master server. + default: "" api_private_address: type: string description: Private IP address of the Kubernetes master server. + default: "" fixed_network: type: string @@ -193,6 +195,20 @@ resources: handle: {get_resource: master_wait_handle} timeout: {get_param: wait_condition_timeout} + ###################################################################### + # + # resource that exposes the IPs of either the kube master or the API + # LBaaS pool depending on whether LBaaS is enabled for the bay. + # + + api_address_switch: + type: Magnum::ApiGatewaySwitcher + properties: + pool_public_ip: {get_param: api_public_address} + pool_private_ip: {get_param: api_private_address} + master_public_ip: {get_attr: [kube_master_floating, floating_ip_address]} + master_private_ip: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} + ###################################################################### # # software configs. these are components that are combined into @@ -207,8 +223,8 @@ resources: str_replace: template: {get_file: fragments/write-heat-params-master.yaml} params: - "$KUBE_API_PUBLIC_ADDRESS": {get_param: api_public_address} - "$KUBE_API_PRIVATE_ADDRESS": {get_param: api_private_address} + "$KUBE_API_PUBLIC_ADDRESS": {get_attr: [api_address_switch, public_ip]} + "$KUBE_API_PRIVATE_ADDRESS": {get_attr: [api_address_switch, private_ip]} "$KUBE_API_PORT": {get_param: kubernetes_port} "$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv} @@ -410,14 +426,14 @@ resources: port_id: {get_resource: kube_master_eth0} api_pool_member: - type: OS::Neutron::PoolMember + type: Magnum::Optional::Neutron::PoolMember properties: pool_id: {get_param: api_pool_id} address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} protocol_port: {get_param: kubernetes_port} etcd_pool_member: - type: OS::Neutron::PoolMember + type: Magnum::Optional::Neutron::PoolMember properties: pool_id: {get_param: etcd_pool_id} address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} diff --git a/magnum/tests/unit/api/controllers/v1/test_bay.py b/magnum/tests/unit/api/controllers/v1/test_bay.py index a909d01e0a..3ac257cac0 100644 --- a/magnum/tests/unit/api/controllers/v1/test_bay.py +++ b/magnum/tests/unit/api/controllers/v1/test_bay.py @@ -640,6 +640,24 @@ class TestPost(api_base.FunctionalTest): self.assertTrue(self.mock_valid_os_res.called) self.assertEqual(400, response.status_int) + def test_create_bay_with_no_lb_one_node(self): + baymodel = obj_utils.create_test_baymodel( + self.context, name='foo', uuid='foo', master_lb_enabled=False) + bdict = apiutils.bay_post_data(baymodel_id=baymodel.name, + master_count=1) + response = self.post_json('/bays', bdict, expect_errors=True) + self.assertEqual('application/json', response.content_type) + self.assertEqual(201, response.status_int) + + def test_create_bay_with_no_lb_multi_node(self): + baymodel = obj_utils.create_test_baymodel( + self.context, name='foo', uuid='foo', master_lb_enabled=False) + bdict = apiutils.bay_post_data(baymodel_id=baymodel.name, + master_count=3) + response = self.post_json('/bays', bdict, expect_errors=True) + self.assertEqual('application/json', response.content_type) + self.assertEqual(400, response.status_int) + class TestDelete(api_base.FunctionalTest): diff --git a/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py b/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py index 9026f668e6..6af2e258bb 100644 --- a/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py @@ -48,6 +48,7 @@ class TestBayConductorWithK8s(base.TestCase): 'server_type': 'vm', 'registry_enabled': False, 'insecure_registry': '10.0.0.1:5000', + 'master_lb_enabled': False, } self.bay_dict = { 'uuid': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52', @@ -176,7 +177,7 @@ class TestBayConductorWithK8s(base.TestCase): expected.pop(mapping[missing_attr], None) self.assertEqual(expected, definition) - self.assertEqual([], env_files) + self.assertEqual(['environments/no_master_lb.yaml'], env_files) @patch('requests.get') @patch('magnum.objects.BayModel.get_by_uuid') @@ -242,7 +243,7 @@ class TestBayConductorWithK8s(base.TestCase): } self.assertEqual(expected, definition) - self.assertEqual([], env_files) + self.assertEqual(['environments/no_master_lb.yaml'], env_files) @patch('requests.get') @patch('magnum.objects.BayModel.get_by_uuid') @@ -296,7 +297,7 @@ class TestBayConductorWithK8s(base.TestCase): 'insecure_registry_url': '10.0.0.1:5000', } self.assertEqual(expected, definition) - self.assertEqual([], env_files) + self.assertEqual(['environments/no_master_lb.yaml'], env_files) @patch('requests.get') @patch('magnum.objects.BayModel.get_by_uuid') @@ -348,7 +349,7 @@ class TestBayConductorWithK8s(base.TestCase): 'insecure_registry_url': '10.0.0.1:5000', } self.assertEqual(expected, definition) - self.assertEqual([], env_files) + self.assertEqual(['environments/no_master_lb.yaml'], env_files) @patch('requests.get') @patch('magnum.objects.BayModel.get_by_uuid') @@ -508,7 +509,7 @@ class TestBayConductorWithK8s(base.TestCase): 'insecure_registry_url': '10.0.0.1:5000', } self.assertEqual(expected, definition) - self.assertEqual([], env_files) + self.assertEqual(['environments/no_master_lb.yaml'], env_files) reqget.assert_called_once_with('http://etcd/test?size=1') @patch('magnum.common.short_id.generate_id') diff --git a/magnum/tests/unit/db/utils.py b/magnum/tests/unit/db/utils.py index e0aa65d5e3..68c4b90933 100644 --- a/magnum/tests/unit/db/utils.py +++ b/magnum/tests/unit/db/utils.py @@ -53,7 +53,7 @@ def get_test_baymodel(**kw): 'public': kw.get('public', False), 'server_type': kw.get('server_type', 'vm'), 'insecure_registry': kw.get('insecure_registry', '10.0.0.1:5000'), - 'master_lb_enabled': kw.get('master_lb_enabled', False), + 'master_lb_enabled': kw.get('master_lb_enabled', True), }