Support disabling floating IPs in swarm mode

We use the same technique that is used for kubernetes clusters, with a
custom heat resource that provides either a floating IP, or
OS::Heat::None when disabled. We also add coverage tests for swarm-mode.

Change-Id: I3b5877bcd89fc2436776f49e479ffadf72c00ea3
Story: 1772433
Task: 21662
Task: 22102
Co-authored-by: Mark Goddard <mark@stackhpc.com>
(cherry picked from commit b7bfee5d27)
This commit is contained in:
Bharat Kunwar 2018-06-12 14:33:42 +01:00 committed by Bharat Kunwar
parent 5560223d62
commit 06935c56a2
9 changed files with 361 additions and 40 deletions

View File

@ -14,3 +14,9 @@ resource_registry:
# dcosslave.yaml
"Magnum::Optional::DcosSlave::Neutron::FloatingIP": "OS::Heat::None"
# swarmmaster.yaml
"Magnum::Optional::SwarmMaster::Neutron::FloatingIP": "OS::Heat::None"
# swarmnode.yaml
"Magnum::Optional::SwarmNode::Neutron::FloatingIP": "OS::Heat::None"

View File

@ -14,3 +14,9 @@ resource_registry:
# dcosslave.yaml
"Magnum::Optional::DcosSlave::Neutron::FloatingIP": "OS::Neutron::FloatingIP"
# swarmmaster.yaml
"Magnum::Optional::SwarmMaster::Neutron::FloatingIP": "OS::Neutron::FloatingIP"
# swarmnode.yaml
"Magnum::Optional::SwarmNode::Neutron::FloatingIP": "OS::Neutron::FloatingIP"

View File

@ -71,13 +71,19 @@ def _get_public_ip():
def _build_subject_alt_names(config):
subject_alt_names = [
'IP:%s' % _get_public_ip(),
'IP:%s' % config['API_IP_ADDRESS'],
'IP:%s' % config['SWARM_NODE_IP'],
'IP:%s' % config['SWARM_API_IP'],
'IP:127.0.0.1'
]
ips = {
config['SWARM_NODE_IP'],
config['SWARM_API_IP'],
'127.0.0.1',
}
# NOTE(mgoddard): If floating IP is disabled, these can be empty.
public_ip = _get_public_ip()
if public_ip:
ips.add(public_ip)
api_ip = config['API_IP_ADDRESS']
if api_ip:
ips.add(api_ip)
subject_alt_names = ['IP:%s' % ip for ip in ips]
return ','.join(subject_alt_names)
@ -95,9 +101,10 @@ def write_ca_cert(config, verify_ca):
def write_server_key():
subprocess.call(['openssl', 'genrsa',
'-out', SERVER_KEY_PATH,
'4096'])
subprocess.check_call(
['openssl', 'genrsa',
'-out', SERVER_KEY_PATH,
'4096'])
def _write_csr_config(config):
@ -110,13 +117,14 @@ def _write_csr_config(config):
def create_server_csr(config):
_write_csr_config(config)
subprocess.call(['openssl', 'req', '-new',
'-days', '1000',
'-key', SERVER_KEY_PATH,
'-out', SERVER_CSR_PATH,
'-reqexts', 'req_ext',
'-extensions', 'req_ext',
'-config', SERVER_CONF_PATH])
subprocess.check_call(
['openssl', 'req', '-new',
'-days', '1000',
'-key', SERVER_KEY_PATH,
'-out', SERVER_CSR_PATH,
'-reqexts', 'req_ext',
'-extensions', 'req_ext',
'-config', SERVER_CONF_PATH])
with open(SERVER_CSR_PATH, 'r') as fp:
return {'cluster_uuid': config['CLUSTER_UUID'], 'csr': fp.read()}

View File

@ -9,12 +9,16 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from magnum.drivers.heat import template_def
from oslo_config import cfg
from oslo_log import log as logging
CONF = cfg.CONF
DOCKER_PORT = '2375'
LOG = logging.getLogger(__name__)
class SwarmModeApiAddressOutputMapping(template_def.OutputMapping):
@ -35,21 +39,46 @@ class SwarmModeApiAddressOutputMapping(template_def.OutputMapping):
setattr(cluster, self.cluster_attr, value)
class SwarmModeMasterAddressesOutputMapping(template_def.OutputMapping):
class ServerAddressOutputMapping(template_def.OutputMapping):
public_ip_output_key = None
private_ip_output_key = None
def __init__(self, dummy_arg, cluster_attr=None):
self.cluster_attr = cluster_attr
self.heat_output = self.public_ip_output_key
class MasterAddressOutputMapping(ServerAddressOutputMapping):
public_ip_output_key = ['swarm_primary_master',
'swarm_secondary_masters']
private_ip_output_key = ['swarm_primary_master_private',
'swarm_secondary_masters_private']
def set_output(self, stack, cluster_template, cluster):
if self.cluster_attr is None:
return
if not cluster_template.floating_ip_enabled:
self.heat_output = self.private_ip_output_key
LOG.debug("Using heat_output: %s", self.heat_output)
_master_addresses = []
for output in stack.to_dict().get('outputs', []):
if output['output_key'] == 'swarm_primary_master':
_master_addresses.append(output['output_value'][0])
elif output['output_key'] == 'swarm_secondary_masters':
if output['output_key'] in self.heat_output:
_master_addresses += output['output_value']
setattr(cluster, self.cluster_attr, _master_addresses)
class NodeAddressOutputMapping(ServerAddressOutputMapping):
public_ip_output_key = 'swarm_nodes'
private_ip_output_key = 'swarm_nodes_private'
def set_output(self, stack, cluster_template, cluster):
if not cluster_template.floating_ip_enabled:
self.heat_output = self.private_ip_output_key
LOG.debug("Using heat_output: %s", self.heat_output)
super(NodeAddressOutputMapping,
self).set_output(stack, cluster_template, cluster)
class SwarmModeTemplateDefinition(template_def.BaseTemplateDefinition):
"""Docker swarm mode template."""
@ -84,15 +113,12 @@ class SwarmModeTemplateDefinition(template_def.BaseTemplateDefinition):
self.add_output('api_address',
cluster_attr='api_address',
mapping_type=SwarmModeApiAddressOutputMapping)
self.add_output('swarm_primary_master_private',
cluster_attr=None)
self.add_output('swarm_primary_master',
self.add_output('swarm_masters',
cluster_attr='master_addresses',
mapping_type=SwarmModeMasterAddressesOutputMapping)
self.add_output('swarm_nodes_private',
cluster_attr=None)
mapping_type=MasterAddressOutputMapping)
self.add_output('swarm_nodes',
cluster_attr='node_addresses')
cluster_attr='node_addresses',
mapping_type=NodeAddressOutputMapping)
def get_params(self, context, cluster_template, cluster, **kwargs):
extra_params = kwargs.pop('extra_params', {})
@ -128,5 +154,6 @@ class SwarmModeTemplateDefinition(template_def.BaseTemplateDefinition):
template_def.add_priv_net_env_file(env_files, cluster_template)
template_def.add_volume_env_file(env_files, cluster)
template_def.add_lb_env_file(env_files, cluster_template)
template_def.add_fip_env_file(env_files, cluster_template)
return env_files

View File

@ -274,6 +274,18 @@ resources:
master_public_ip: {get_attr: [swarm_primary_master, resource.0.swarm_master_external_ip]}
master_private_ip: {get_attr: [swarm_primary_master, resource.0.swarm_master_ip]}
######################################################################
#
# resources that expose the IPs of either floating ip or a given
# fixed ip depending on whether FloatingIP is enabled for the cluster.
#
api_address_floating_switch:
type: Magnum::FloatingIPAddressSwitcher
properties:
public_ip: {get_attr: [api_address_lb_switch, public_ip]}
private_ip: {get_attr: [api_address_lb_switch, private_ip]}
######################################################################
#
# resources that expose the server group for all nodes include master
@ -434,7 +446,7 @@ outputs:
str_replace:
template: api_ip_address
params:
api_ip_address: {get_attr: [api_address_lb_switch, public_ip]}
api_ip_address: {get_attr: [api_address_floating_switch, ip_address]}
description: >
This is the API endpoint of the Swarm masters. Use this to access
the Swarm API server from outside the cluster.

View File

@ -346,7 +346,7 @@ resources:
get_param: fixed_subnet_id
swarm_master_floating:
type: "OS::Neutron::FloatingIP"
type: "Magnum::Optional::SwarmMaster::Neutron::FloatingIP"
properties:
floating_network:
get_param: external_network

View File

@ -318,7 +318,7 @@ resources:
get_param: fixed_subnet_id
swarm_node_floating:
type: "OS::Neutron::FloatingIP"
type: "Magnum::Optional::SwarmNode::Neutron::FloatingIP"
properties:
floating_network:
get_param: external_network

View File

@ -19,6 +19,7 @@ from oslo_serialization import jsonutils
from magnum.drivers.common import k8s_monitor
from magnum.drivers.mesos_ubuntu_v1 import monitor as mesos_monitor
from magnum.drivers.swarm_fedora_atomic_v1 import monitor as swarm_monitor
from magnum.drivers.swarm_fedora_atomic_v2 import monitor as swarm_v2_monitor
from magnum import objects
from magnum.tests import base
from magnum.tests.unit.db import utils
@ -45,6 +46,8 @@ class MonitorsTestCase(base.TestCase):
master_addresses=['10.0.0.6'])
self.cluster = objects.Cluster(self.context, **cluster)
self.monitor = swarm_monitor.SwarmMonitor(self.context, self.cluster)
self.v2_monitor = swarm_v2_monitor.SwarmMonitor(self.context,
self.cluster)
self.k8s_monitor = k8s_monitor.K8sMonitor(self.context, self.cluster)
self.mesos_monitor = mesos_monitor.MesosMonitor(self.context,
self.cluster)
@ -55,6 +58,13 @@ class MonitorsTestCase(base.TestCase):
self.mock_metrics_spec.return_value = self.test_metrics_spec
self.addCleanup(p.stop)
p2 = mock.patch('magnum.drivers.swarm_fedora_atomic_v2.monitor.'
'SwarmMonitor.metrics_spec',
new_callable=mock.PropertyMock)
self.mock_metrics_spec_v2 = p2.start()
self.mock_metrics_spec_v2.return_value = self.test_metrics_spec
self.addCleanup(p2.stop)
@mock.patch('magnum.common.docker_utils.docker_for_cluster')
def test_swarm_monitor_pull_data_success(self, mock_docker_cluster):
mock_docker = mock.MagicMock()
@ -70,6 +80,22 @@ class MonitorsTestCase(base.TestCase):
self.monitor.data['nodes'])
self.assertEqual(['test_container'], self.monitor.data['containers'])
@mock.patch('magnum.common.docker_utils.docker_for_cluster')
def test_swarm_v2_monitor_pull_data_success(self, mock_docker_cluster):
mock_docker = mock.MagicMock()
mock_docker.info.return_value = {'DriverStatus': [[
u' \u2514 Reserved Memory', u'0 B / 1 GiB']]}
mock_docker.containers.return_value = [mock.MagicMock()]
mock_docker.inspect_container.return_value = 'test_container'
mock_docker_cluster.return_value.__enter__.return_value = mock_docker
self.v2_monitor.pull_data()
self.assertEqual([{'MemTotal': 1073741824.0}],
self.v2_monitor.data['nodes'])
self.assertEqual(['test_container'],
self.v2_monitor.data['containers'])
@mock.patch('magnum.common.docker_utils.docker_for_cluster')
def test_swarm_monitor_pull_data_raise(self, mock_docker_cluster):
mock_container = mock.MagicMock()
@ -86,14 +112,38 @@ class MonitorsTestCase(base.TestCase):
self.monitor.data['nodes'])
self.assertEqual([mock_container], self.monitor.data['containers'])
@mock.patch('magnum.common.docker_utils.docker_for_cluster')
def test_swarm_v2_monitor_pull_data_raise(self, mock_docker_cluster):
mock_container = mock.MagicMock()
mock_docker = mock.MagicMock()
mock_docker.info.return_value = {'DriverStatus': [[
u' \u2514 Reserved Memory', u'0 B / 1 GiB']]}
mock_docker.containers.return_value = [mock_container]
mock_docker.inspect_container.side_effect = Exception("inspect error")
mock_docker_cluster.return_value.__enter__.return_value = mock_docker
self.v2_monitor.pull_data()
self.assertEqual([{'MemTotal': 1073741824.0}],
self.v2_monitor.data['nodes'])
self.assertEqual([mock_container], self.v2_monitor.data['containers'])
def test_swarm_monitor_get_metric_names(self):
names = self.monitor.get_metric_names()
self.assertEqual(sorted(['metric1', 'metric2']), sorted(names))
def test_swarm_v2_monitor_get_metric_names(self):
names = self.v2_monitor.get_metric_names()
self.assertEqual(sorted(['metric1', 'metric2']), sorted(names))
def test_swarm_monitor_get_metric_unit(self):
unit = self.monitor.get_metric_unit('metric1')
self.assertEqual('metric1_unit', unit)
def test_swarm_v2_monitor_get_metric_unit(self):
unit = self.v2_monitor.get_metric_unit('metric1')
self.assertEqual('metric1_unit', unit)
def test_swarm_monitor_compute_metric_value(self):
mock_func = mock.MagicMock()
mock_func.return_value = 'metric1_value'
@ -101,6 +151,13 @@ class MonitorsTestCase(base.TestCase):
value = self.monitor.compute_metric_value('metric1')
self.assertEqual('metric1_value', value)
def test_swarm_v2_monitor_compute_metric_value(self):
mock_func = mock.MagicMock()
mock_func.return_value = 'metric1_value'
self.v2_monitor.metric1_func = mock_func
value = self.v2_monitor.compute_metric_value('metric1')
self.assertEqual('metric1_value', value)
def test_swarm_monitor_compute_memory_util(self):
test_data = {
'nodes': [
@ -130,6 +187,35 @@ class MonitorsTestCase(base.TestCase):
mem_util = self.monitor.compute_memory_util()
self.assertEqual(0, mem_util)
def test_swarm_v2_monitor_compute_memory_util(self):
test_data = {
'nodes': [
{
'Name': 'node',
'MemTotal': 20,
},
],
'containers': [
{
'Name': 'container',
'HostConfig': {
'Memory': 10,
},
},
],
}
self.v2_monitor.data = test_data
mem_util = self.v2_monitor.compute_memory_util()
self.assertEqual(50, mem_util)
test_data = {
'nodes': [],
'containers': [],
}
self.v2_monitor.data = test_data
mem_util = self.v2_monitor.compute_memory_util()
self.assertEqual(0, mem_util)
@mock.patch('magnum.conductor.k8s_api.create_k8s_api')
def test_k8s_monitor_pull_data_success(self, mock_k8s_api):
mock_nodes = mock.MagicMock()

View File

@ -31,6 +31,8 @@ from magnum.drivers.mesos_ubuntu_v1 import driver as mesos_dr
from magnum.drivers.mesos_ubuntu_v1 import template_def as mesos_tdef
from magnum.drivers.swarm_fedora_atomic_v1 import driver as swarm_dr
from magnum.drivers.swarm_fedora_atomic_v1 import template_def as swarm_tdef
from magnum.drivers.swarm_fedora_atomic_v2 import driver as swarm_v2_dr
from magnum.drivers.swarm_fedora_atomic_v2 import template_def as swarm_v2_tdef
from magnum.tests import base
from requests import exceptions as req_exceptions
@ -96,6 +98,17 @@ class TemplateDefinitionTestCase(base.TestCase):
self.assertIsInstance(definition,
swarm_tdef.AtomicSwarmTemplateDefinition)
@mock.patch('magnum.drivers.common.driver.Driver.get_driver')
def test_get_vm_atomic_swarm_v2_definition(self, mock_driver):
mock_driver.return_value = swarm_v2_dr.Driver()
cluster_driver = driver.Driver.get_driver('vm',
'fedora-atomic',
'swarm-mode')
definition = cluster_driver.get_template_definition()
self.assertIsInstance(definition,
swarm_v2_tdef.AtomicSwarmTemplateDefinition)
@mock.patch('magnum.drivers.common.driver.Driver.get_driver')
def test_get_vm_ubuntu_mesos_definition(self, mock_driver):
mock_driver.return_value = mesos_dr.Driver()
@ -208,14 +221,14 @@ class TemplateDefinitionTestCase(base.TestCase):
@six.add_metaclass(abc.ABCMeta)
class BaseTemplateDefinitionTestCase(base.TestCase):
class BaseK8sTemplateDefinitionTestCase(base.TestCase):
@abc.abstractmethod
def get_definition(self):
"""Returns the template definition."""
pass
def _test_update_outputs_server_addrtess(
def _test_update_outputs_server_address(
self,
floating_ip_enabled=True,
public_ip_output_key='kube_masters',
@ -250,7 +263,7 @@ class BaseTemplateDefinitionTestCase(base.TestCase):
self.assertEqual(expected_address, getattr(mock_cluster, cluster_attr))
class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
def get_definition(self):
return k8sa_dr.Driver().get_template_definition()
@ -762,21 +775,21 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
self._test_update_outputs_none_api_address(template_definition, params)
def test_update_outputs_master_address(self):
self._test_update_outputs_server_addrtess(
self._test_update_outputs_server_address(
public_ip_output_key='kube_masters',
private_ip_output_key='kube_masters_private',
cluster_attr='master_addresses',
)
def test_update_outputs_node_address(self):
self._test_update_outputs_server_addrtess(
self._test_update_outputs_server_address(
public_ip_output_key='kube_minions',
private_ip_output_key='kube_minions_private',
cluster_attr='node_addresses',
)
def test_update_outputs_master_address_fip_disabled(self):
self._test_update_outputs_server_addrtess(
self._test_update_outputs_server_address(
floating_ip_enabled=False,
public_ip_output_key='kube_masters',
private_ip_output_key='kube_masters_private',
@ -784,7 +797,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
)
def test_update_outputs_node_address_fip_disabled(self):
self._test_update_outputs_server_addrtess(
self._test_update_outputs_server_address(
floating_ip_enabled=False,
public_ip_output_key='kube_minions',
private_ip_output_key='kube_minions_private',
@ -896,6 +909,169 @@ class FedoraK8sIronicTemplateDefinitionTestCase(base.TestCase):
)
class AtomicSwarmModeTemplateDefinitionTestCase(base.TestCase):
def get_definition(self):
return swarm_v2_dr.Driver().get_template_definition()
def _test_update_outputs_server_address(
self,
floating_ip_enabled=True,
public_ip_output_key='swarm_nodes',
private_ip_output_key='swarm_nodes_private',
cluster_attr='node_addresses',
):
definition = self.get_definition()
expected_address = expected_public_address = ['public']
expected_private_address = ['private']
if not floating_ip_enabled:
expected_address = expected_private_address
outputs = [
{"output_value": expected_public_address,
"description": "No description given",
"output_key": public_ip_output_key},
{"output_value": expected_private_address,
"description": "No description given",
"output_key": private_ip_output_key},
]
mock_stack = mock.MagicMock()
mock_stack.to_dict.return_value = {'outputs': outputs}
mock_cluster = mock.MagicMock()
mock_cluster_template = mock.MagicMock()
mock_cluster_template.floating_ip_enabled = floating_ip_enabled
definition.update_outputs(mock_stack, mock_cluster_template,
mock_cluster)
self.assertEqual(expected_address, getattr(mock_cluster, cluster_attr))
@mock.patch('magnum.common.clients.OpenStackClients')
@mock.patch('magnum.drivers.swarm_fedora_atomic_v2.template_def'
'.AtomicSwarmTemplateDefinition.get_discovery_url')
@mock.patch('magnum.drivers.heat.template_def.BaseTemplateDefinition'
'.get_params')
@mock.patch('magnum.drivers.heat.template_def.TemplateDefinition'
'.get_output')
def test_swarm_get_params(self, mock_get_output, mock_get_params,
mock_get_discovery_url, mock_osc_class):
mock_context = mock.MagicMock()
mock_context.auth_token = 'AUTH_TOKEN'
mock_cluster_template = mock.MagicMock()
mock_cluster_template.tls_disabled = False
mock_cluster_template.registry_enabled = False
mock_cluster = mock.MagicMock()
mock_cluster.uuid = '5d12f6fd-a196-4bf0-ae4c-1f639a523a52'
del mock_cluster.stack_id
mock_osc = mock.MagicMock()
mock_osc.magnum_url.return_value = 'http://127.0.0.1:9511/v1'
mock_osc_class.return_value = mock_osc
discovery_url = 'fake_discovery_url'
mock_get_discovery_url.return_value = discovery_url
mock_context.auth_url = 'http://192.168.10.10:5000/v3'
mock_context.user_name = 'fake_user'
mock_context.tenant = 'fake_tenant'
docker_volume_type = mock_cluster_template.labels.get(
'docker_volume_type')
rexray_preempt = mock_cluster_template.labels.get('rexray_preempt')
availability_zone = mock_cluster_template.labels.get(
'availability_zone')
number_of_secondary_masters = mock_cluster.master_count - 1
swarm_def = swarm_v2_tdef.AtomicSwarmTemplateDefinition()
swarm_def.get_params(mock_context, mock_cluster_template, mock_cluster)
expected_kwargs = {'extra_params': {
'magnum_url': mock_osc.magnum_url.return_value,
'auth_url': 'http://192.168.10.10:5000/v3',
'rexray_preempt': rexray_preempt,
'docker_volume_type': docker_volume_type,
'number_of_secondary_masters': number_of_secondary_masters,
'availability_zone': availability_zone,
'nodes_affinity_policy': 'soft-anti-affinity'}}
mock_get_params.assert_called_once_with(mock_context,
mock_cluster_template,
mock_cluster,
**expected_kwargs)
def test_swarm_get_heat_param(self):
swarm_def = swarm_v2_tdef.AtomicSwarmTemplateDefinition()
heat_param = swarm_def.get_heat_param(cluster_attr='node_count')
self.assertEqual('number_of_nodes', heat_param)
def test_update_outputs(self):
swarm_def = swarm_v2_tdef.AtomicSwarmTemplateDefinition()
expected_api_address = 'updated_address'
expected_node_addresses = ['ex_minion', 'address']
outputs = [
{"output_value": expected_api_address,
"description": "No description given",
"output_key": "api_address"},
{"output_value": ['any', 'output'],
"description": "No description given",
"output_key": "swarm_master_private"},
{"output_value": ['any', 'output'],
"description": "No description given",
"output_key": "swarm_master"},
{"output_value": ['any', 'output'],
"description": "No description given",
"output_key": "swarm_nodes_private"},
{"output_value": expected_node_addresses,
"description": "No description given",
"output_key": "swarm_nodes"},
]
mock_stack = mock.MagicMock()
mock_stack.to_dict.return_value = {'outputs': outputs}
mock_cluster = mock.MagicMock()
mock_cluster_template = mock.MagicMock()
swarm_def.update_outputs(mock_stack, mock_cluster_template,
mock_cluster)
expected_api_address = "tcp://%s:2375" % expected_api_address
self.assertEqual(expected_api_address, mock_cluster.api_address)
self.assertEqual(expected_node_addresses, mock_cluster.node_addresses)
def test_update_outputs_master_address(self):
self._test_update_outputs_server_address(
public_ip_output_key='swarm_primary_master',
private_ip_output_key='swarm_primary_master_private',
cluster_attr='master_addresses',
)
def test_update_outputs_node_address(self):
self._test_update_outputs_server_address(
public_ip_output_key='swarm_nodes',
private_ip_output_key='swarm_nodes_private',
cluster_attr='node_addresses',
)
def test_update_outputs_master_address_fip_disabled(self):
self._test_update_outputs_server_address(
floating_ip_enabled=False,
public_ip_output_key='swarm_primary_master',
private_ip_output_key='swarm_primary_master_private',
cluster_attr='master_addresses',
)
def test_update_outputs_node_address_fip_disabled(self):
self._test_update_outputs_server_address(
floating_ip_enabled=False,
public_ip_output_key='swarm_nodes',
private_ip_output_key='swarm_nodes_private',
cluster_attr='node_addresses',
)
class AtomicSwarmTemplateDefinitionTestCase(base.TestCase):
@mock.patch('magnum.common.clients.OpenStackClients')