From 2cc25abe49fd2f2a68f9584f3974ab030550739d Mon Sep 17 00:00:00 2001 From: "OTSUKA, Yuanying" Date: Fri, 5 Aug 2016 14:22:35 +0900 Subject: [PATCH] Allow k8s cluster without Floating IP This patch adds an environment file and a couple of template resources to allow the Floating IP resources to be conditionally enabled/disabled. Change-Id: I95025d39443165e8463a81f8f75e78f5aa3809a1 Partially-Implements: blueprint bay-with-no-floating-ips --- .../environments/disable_floating_ip.yaml | 13 +++ .../environments/enable_floating_ip.yaml | 10 +++ .../floating_ip_address_switcher_private.yaml | 21 +++++ .../floating_ip_address_switcher_public.yaml | 21 +++++ .../k8s_fedora_atomic_v1/template_def.py | 52 +++++++++++- .../templates/kubecluster.yaml | 22 +++-- .../templates/kubemaster.yaml | 2 +- .../templates/kubeminion.yaml | 2 +- .../handlers/test_k8s_bay_conductor.py | 10 ++- .../unit/drivers/test_template_definition.py | 83 ++++++++++++++++++- 10 files changed, 221 insertions(+), 15 deletions(-) create mode 100644 magnum/drivers/common/templates/environments/disable_floating_ip.yaml create mode 100644 magnum/drivers/common/templates/environments/enable_floating_ip.yaml create mode 100644 magnum/drivers/common/templates/fragments/floating_ip_address_switcher_private.yaml create mode 100644 magnum/drivers/common/templates/fragments/floating_ip_address_switcher_public.yaml diff --git a/magnum/drivers/common/templates/environments/disable_floating_ip.yaml b/magnum/drivers/common/templates/environments/disable_floating_ip.yaml new file mode 100644 index 0000000000..cf512734c1 --- /dev/null +++ b/magnum/drivers/common/templates/environments/disable_floating_ip.yaml @@ -0,0 +1,13 @@ +# Environment file to disable FloatingIP in a Kubernetes cluster by mapping +# FloatingIP-related resource types to OS::Heat::None +resource_registry: + "Magnum::FloatingIPAddressSwitcher": "../fragments/floating_ip_address_switcher_private.yaml" + + # with_master_lb.yaml + "Magnum::Optional::Neutron::Pool::FloatingIP": "OS::Heat::None" + + # kubemaster.yaml + "Magnum::Optional::KubeMaster::Neutron::FloatingIP": "OS::Heat::None" + + # kubeminion.yaml + "Magnum::Optional::KubeMinion::Neutron::FloatingIP": "OS::Heat::None" diff --git a/magnum/drivers/common/templates/environments/enable_floating_ip.yaml b/magnum/drivers/common/templates/environments/enable_floating_ip.yaml new file mode 100644 index 0000000000..19e2d741ca --- /dev/null +++ b/magnum/drivers/common/templates/environments/enable_floating_ip.yaml @@ -0,0 +1,10 @@ +# Environment file to disable FloatingIP in a Kubernetes cluster by mapping +# FloatingIP-related resource types to OS::Neutron::FloatingIP +resource_registry: + "Magnum::FloatingIPAddressSwitcher": "../fragments/floating_ip_address_switcher_public.yaml" + + # kubemaster.yaml + "Magnum::Optional::KubeMaster::Neutron::FloatingIP": "OS::Neutron::FloatingIP" + + # kubeminion.yaml + "Magnum::Optional::KubeMinion::Neutron::FloatingIP": "OS::Neutron::FloatingIP" diff --git a/magnum/drivers/common/templates/fragments/floating_ip_address_switcher_private.yaml b/magnum/drivers/common/templates/fragments/floating_ip_address_switcher_private.yaml new file mode 100644 index 0000000000..1b06731ec1 --- /dev/null +++ b/magnum/drivers/common/templates/fragments/floating_ip_address_switcher_private.yaml @@ -0,0 +1,21 @@ +heat_template_version: 2014-10-16 + +description: > + This is a template resource that accepts public and private IPs. + It connects private ip address to its outputs, essentially acting as + one state of a multiplexer. + +parameters: + + public_ip: + type: string + default: "" + + private_ip: + type: string + default: "" + +outputs: + + ip_address: + value: {get_param: private_ip} diff --git a/magnum/drivers/common/templates/fragments/floating_ip_address_switcher_public.yaml b/magnum/drivers/common/templates/fragments/floating_ip_address_switcher_public.yaml new file mode 100644 index 0000000000..d4b266c0a2 --- /dev/null +++ b/magnum/drivers/common/templates/fragments/floating_ip_address_switcher_public.yaml @@ -0,0 +1,21 @@ +heat_template_version: 2014-10-16 + +description: > + This is a template resource that accepts public and private IPs. + It connects public ip address to its outputs, essentially acting as + one state of a multiplexer. + +parameters: + + public_ip: + type: string + default: "" + + private_ip: + type: string + default: "" + +outputs: + + ip_address: + value: {get_param: public_ip} diff --git a/magnum/drivers/k8s_fedora_atomic_v1/template_def.py b/magnum/drivers/k8s_fedora_atomic_v1/template_def.py index 634a7af5a4..dedf9d9a53 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/template_def.py +++ b/magnum/drivers/k8s_fedora_atomic_v1/template_def.py @@ -15,6 +15,7 @@ from neutronclient.common import exceptions as n_exception from neutronclient.neutron import v2_0 as neutronV20 import os +from oslo_log import log as logging from magnum.common import exception from magnum.drivers.common import template_def @@ -24,6 +25,8 @@ CONF = cfg.CONF KUBE_SECURE_PORT = '6443' KUBE_INSECURE_PORT = '8080' +LOG = logging.getLogger(__name__) + class K8sApiAddressOutputMapping(template_def.OutputMapping): @@ -49,6 +52,34 @@ class K8sApiAddressOutputMapping(template_def.OutputMapping): setattr(bay, self.bay_attr, value) +class ServerAddressOutputMapping(template_def.OutputMapping): + + public_ip_output_key = None + private_ip_output_key = None + + def __init__(self, dummy_arg, bay_attr=None): + self.bay_attr = bay_attr + self.heat_output = self.public_ip_output_key + + def set_output(self, stack, baymodel, bay): + if not baymodel.floating_ip_enabled: + self.heat_output = self.private_ip_output_key + + LOG.debug("Using heat_output: %s", self.heat_output) + super(ServerAddressOutputMapping, + self).set_output(stack, baymodel, bay) + + +class MasterAddressOutputMapping(ServerAddressOutputMapping): + public_ip_output_key = 'kube_masters' + private_ip_output_key = 'kube_masters_private' + + +class NodeAddressOutputMapping(ServerAddressOutputMapping): + public_ip_output_key = 'kube_minions' + private_ip_output_key = 'kube_minions_private' + + class K8sTemplateDefinition(template_def.BaseTemplateDefinition): """Base Kubernetes template.""" @@ -84,11 +115,13 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition): self.add_output('kube_minions_private', bay_attr=None) self.add_output('kube_minions', - bay_attr='node_addresses') + bay_attr='node_addresses', + mapping_type=NodeAddressOutputMapping) self.add_output('kube_masters_private', bay_attr=None) self.add_output('kube_masters', - bay_attr='master_addresses') + bay_attr='master_addresses', + mapping_type=MasterAddressOutputMapping) def get_params(self, context, baymodel, bay, **kwargs): extra_params = kwargs.pop('extra_params', {}) @@ -152,10 +185,21 @@ class AtomicK8sTemplateDefinition(K8sTemplateDefinition): **kwargs) def get_env_files(self, baymodel): + env_files = [] if baymodel.master_lb_enabled: - return ['../../common/templates/environments/with_master_lb.yaml'] + env_files.append( + '../../common/templates/environments/with_master_lb.yaml') else: - return ['../../common/templates/environments/no_master_lb.yaml'] + env_files.append( + '../../common/templates/environments/no_master_lb.yaml') + if baymodel.floating_ip_enabled: + env_files.append( + '../../common/templates/environments/enable_floating_ip.yaml') + else: + env_files.append( + '../../common/templates/environments/disable_floating_ip.yaml') + + return env_files @property def template_path(self): diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index d256f809d5..dce2155dfd 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -399,7 +399,7 @@ resources: # LBaaS pool depending on whether LBaaS is enabled for the bay. # - api_address_switch: + api_address_lb_switch: type: Magnum::ApiGatewaySwitcher properties: pool_public_ip: {get_attr: [api_pool_floating, floating_ip_address]} @@ -407,12 +407,24 @@ resources: master_public_ip: {get_attr: [kube_masters, resource.0.kube_master_external_ip]} master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]} - etcd_address_switch: + etcd_address_lb_switch: type: Magnum::ApiGatewaySwitcher properties: pool_private_ip: {get_attr: [etcd_pool, vip, address]} master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]} + ###################################################################### + # + # resources that expose the IPs of either floating ip or a given + # fixed ip depending on whether FloatingIP is enabled for the bay. + # + + api_address_floating_switch: + type: Magnum::FloatingIPAddressSwitcher + properties: + public_ip: {get_attr: [api_address_lb_switch, public_ip]} + private_ip: {get_attr: [api_address_lb_switch, private_ip]} + ###################################################################### # # kubernetes masters. This is a resource group that will create @@ -490,8 +502,8 @@ resources: fixed_subnet: {get_resource: fixed_subnet} network_driver: {get_param: network_driver} flannel_network_cidr: {get_param: flannel_network_cidr} - kube_master_ip: {get_attr: [api_address_switch, private_ip]} - etcd_server_ip: {get_attr: [etcd_address_switch, private_ip]} + kube_master_ip: {get_attr: [api_address_lb_switch, private_ip]} + etcd_server_ip: {get_attr: [etcd_address_lb_switch, private_ip]} external_network: {get_param: external_network} kube_allow_priv: {get_param: kube_allow_priv} docker_volume_size: {get_param: docker_volume_size} @@ -533,7 +545,7 @@ outputs: str_replace: template: api_ip_address params: - api_ip_address: {get_attr: [api_address_switch, public_ip]} + api_ip_address: {get_attr: [api_address_floating_switch, ip_address]} description: > This is the API endpoint of the Kubernetes cluster. Use this to access the Kubernetes API. diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index eba16d4f39..bdf42ea2a9 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -414,7 +414,7 @@ resources: replacement_policy: AUTO kube_master_floating: - type: OS::Neutron::FloatingIP + type: Magnum::Optional::KubeMaster::Neutron::FloatingIP properties: floating_network: {get_param: external_network} port_id: {get_resource: kube_master_eth0} diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml index f2660fb743..18a28c0962 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml @@ -398,7 +398,7 @@ resources: replacement_policy: AUTO kube_minion_floating: - type: OS::Neutron::FloatingIP + type: Magnum::Optional::KubeMinion::Neutron::FloatingIP properties: floating_network: {get_param: external_network} port_id: {get_resource: kube_minion_eth0} diff --git a/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py b/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py index 34a80f288d..34906bee23 100644 --- a/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py @@ -49,6 +49,7 @@ class TestBayConductorWithK8s(base.TestCase): 'registry_enabled': False, 'insecure_registry': '10.0.0.1:5000', 'master_lb_enabled': False, + 'floating_ip_enabled': False, } self.bay_dict = { 'uuid': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52', @@ -178,7 +179,8 @@ class TestBayConductorWithK8s(base.TestCase): self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/no_master_lb.yaml'], + ['../../common/templates/environments/no_master_lb.yaml', + '../../common/templates/environments/disable_floating_ip.yaml'], env_files) @patch('requests.get') @@ -246,7 +248,8 @@ class TestBayConductorWithK8s(base.TestCase): self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/no_master_lb.yaml'], + ['../../common/templates/environments/no_master_lb.yaml', + '../../common/templates/environments/disable_floating_ip.yaml'], env_files) @patch('requests.get') @@ -518,7 +521,8 @@ class TestBayConductorWithK8s(base.TestCase): } self.assertEqual(expected, definition) self.assertEqual( - ['../../common/templates/environments/no_master_lb.yaml'], + ['../../common/templates/environments/no_master_lb.yaml', + '../../common/templates/environments/disable_floating_ip.yaml'], env_files) reqget.assert_called_once_with('http://etcd/test?size=1') diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py index 18e31625ce..6a3f1e624b 100644 --- a/magnum/tests/unit/drivers/test_template_definition.py +++ b/magnum/tests/unit/drivers/test_template_definition.py @@ -12,9 +12,11 @@ # License for the specific language governing permissions and limitations # under the License. +import abc import mock from neutronclient.common import exceptions as n_exception from oslo_config import cfg +import six from magnum.common import exception from magnum.drivers.common import template_def as cmn_tdef @@ -176,7 +178,56 @@ class TemplateDefinitionTestCase(base.TestCase): definition.output_mappings) -class AtomicK8sTemplateDefinitionTestCase(base.TestCase): +@six.add_metaclass(abc.ABCMeta) +class BaseTemplateDefinitionTestCase(base.TestCase): + + @abc.abstractmethod + def get_definition(self): + """Returns the template definition.""" + pass + + def _test_update_outputs_server_addrtess( + self, + floating_ip_enabled=True, + public_ip_output_key='kube_masters', + private_ip_output_key='kube_masters_private', + bay_attr='master_addresses', + ): + + definition = self.get_definition() + + expected_address = expected_public_address = ['public'] + expected_private_address = ['private'] + if not floating_ip_enabled: + expected_address = expected_private_address + + outputs = [ + {"output_value": expected_public_address, + "description": "No description given", + "output_key": public_ip_output_key}, + {"output_value": expected_private_address, + "description": "No description given", + "output_key": private_ip_output_key}, + ] + mock_stack = mock.MagicMock() + mock_stack.to_dict.return_value = {'outputs': outputs} + mock_bay = mock.MagicMock() + mock_baymodel = mock.MagicMock() + mock_baymodel.floating_ip_enabled = floating_ip_enabled + + definition.update_outputs(mock_stack, mock_baymodel, mock_bay) + + self.assertEqual(expected_address, getattr(mock_bay, bay_attr)) + + +class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): + + def get_definition(self): + return cmn_tdef.TemplateDefinition.get_template_definition( + 'vm', + 'fedora-atomic', + 'kubernetes', + ) @mock.patch('magnum.common.clients.OpenStackClients') @mock.patch('magnum.drivers.k8s_fedora_atomic_v1.template_def' @@ -495,6 +546,36 @@ class AtomicK8sTemplateDefinitionTestCase(base.TestCase): } self._test_update_outputs_none_api_address('swarm', params) + def test_update_outputs_master_address(self): + self._test_update_outputs_server_addrtess( + public_ip_output_key='kube_masters', + private_ip_output_key='kube_masters_private', + bay_attr='master_addresses', + ) + + def test_update_outputs_node_address(self): + self._test_update_outputs_server_addrtess( + public_ip_output_key='kube_minions', + private_ip_output_key='kube_minions_private', + bay_attr='node_addresses', + ) + + def test_update_outputs_master_address_fip_disabled(self): + self._test_update_outputs_server_addrtess( + floating_ip_enabled=False, + public_ip_output_key='kube_masters', + private_ip_output_key='kube_masters_private', + bay_attr='master_addresses', + ) + + def test_update_outputs_node_address_fip_disabled(self): + self._test_update_outputs_server_addrtess( + floating_ip_enabled=False, + public_ip_output_key='kube_minions', + private_ip_output_key='kube_minions_private', + bay_attr='node_addresses', + ) + class FedoraK8sIronicTemplateDefinitionTestCase(base.TestCase):