openstack
/
magnum
Code Issues Proposed changes

Support soft-anti-affinity policy for nodes 

Currently, there is no guarantee to make sure all nodes of one cluster are
created on different compute hosts. So it would be nice if we can create
a server group and set it with anti-affinity policy to get a better HA
for cluster. This patch is proposing to create a server group for master
and minion nodes with soft-anti-affinity policy by default.

Closes-Bug: #1737802

Change-Id: Icc7a73ef55296a58bf00719ca4d1cdcc304fab86
This commit is contained in:
Feilong Wang 2017-12-14 11:22:13 +13:00
parent 65dfb2009f
commit be0609ce88
26 changed files with 219 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
magnum/conf/cluster.py
View File

@ -32,6 +32,12 @@ cluster_def_opts = [
'magnum_vm_ubuntu_mesos'],
help=_('Enabled cluster definition entry points.'),
deprecated_group='bay'),
cfg.StrOpt('nodes_affinity_policy',
default='soft-anti-affinity',
help=_('Affinity policy for server group of cluster nodes.'
'Possible values include "affinity", "anti-affinity",'
'"soft-affinity" and "soft-anti-affinity".')
),
]

3
magnum/drivers/heat/k8s_fedora_template_def.py
View File

@ -87,6 +87,9 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
if container_infra_prefix:
extra_params['container_infra_prefix'] = container_infra_prefix
extra_params['nodes_affinity_policy'] = \
CONF.cluster.nodes_affinity_policy
return super(K8sFedoraTemplateDefinition,
self).get_params(context, cluster_template, cluster,
extra_params=extra_params,

2
magnum/drivers/heat/swarm_fedora_template_def.py
View File

@ -102,6 +102,8 @@ class SwarmFedoraTemplateDefinition(template_def.BaseTemplateDefinition):
'swarm_strategy']
extra_params['auth_url'] = context.auth_url
extra_params['nodes_affinity_policy'] = \
CONF.cluster.nodes_affinity_policy
# set docker_volume_type
# use the configuration default if None provided

2
magnum/drivers/heat/swarm_mode_template_def.py
View File

@ -105,6 +105,8 @@ class SwarmModeTemplateDefinition(template_def.BaseTemplateDefinition):
label_list = ['rexray_preempt']
extra_params['auth_url'] = context.auth_url
extra_params['nodes_affinity_policy'] = \
CONF.cluster.nodes_affinity_policy
for label in label_list:
extra_params[label] = cluster_template.labels.get(label)

21
magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml
View File

@ -294,6 +294,14 @@ parameters:
hidden: true
description: The OpenStack CA certificate to install on the node.
nodes_affinity_policy:
type: string
description: >
affinity policy for nodes server group
constraints:
- allowed_values: ["affinity", "anti-affinity", "soft-affinity",
"soft-anti-affinity"]
resources:
######################################################################
@ -400,6 +408,17 @@ resources:
public_ip: {get_attr: [api_address_lb_switch, public_ip]}
private_ip: {get_attr: [api_address_lb_switch, private_ip]}
######################################################################
#
# resources that expose the server group for all nodes include master
# and minions.
#
nodes_server_group:
type: OS::Nova::ServerGroup
properties:
policies: [{get_param: nodes_affinity_policy}]
######################################################################
#
# kubernetes masters. This is a resource group that will create
@ -465,6 +484,7 @@ resources:
dns_service_ip: {get_param: dns_service_ip}
dns_cluster_domain: {get_param: dns_cluster_domain}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
######################################################################
#
@ -520,6 +540,7 @@ resources:
dns_service_ip: {get_param: dns_service_ip}
dns_cluster_domain: {get_param: dns_cluster_domain}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
outputs:

4
magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml
View File

@ -229,6 +229,9 @@ parameters:
openstack_ca:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
@ -511,6 +514,7 @@ resources:
user_data: {get_resource: kube_master_init}
networks:
- port: {get_resource: kube_master_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
kube_master_eth0:
type: OS::Neutron::Port

5
magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml
View File

@ -160,6 +160,10 @@ parameters:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
minion_wait_handle:
@ -338,6 +342,7 @@ resources:
user_data: {get_resource: kube_minion_init}
networks:
- port: {get_resource: kube_minion_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
kube_minion_eth0:
type: OS::Neutron::Port

21
magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
View File

@ -356,6 +356,14 @@ parameters:
hidden: true
description: The OpenStack CA certificate to install on the node.
nodes_affinity_policy:
type: string
description: >
affinity policy for nodes server group
constraints:
- allowed_values: ["affinity", "anti-affinity", "soft-affinity",
"soft-anti-affinity"]
resources:
######################################################################
@ -465,6 +473,17 @@ resources:
public_ip: {get_attr: [api_address_lb_switch, public_ip]}
private_ip: {get_attr: [api_address_lb_switch, private_ip]}
######################################################################
#
# resources that expose the server group for all nodes include master
# and minions.
#
nodes_server_group:
type: OS::Nova::ServerGroup
properties:
policies: [{get_param: nodes_affinity_policy}]
######################################################################
#
# kubernetes masters. This is a resource group that will create
@ -537,6 +556,7 @@ resources:
dns_service_ip: {get_param: dns_service_ip}
dns_cluster_domain: {get_param: dns_cluster_domain}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
######################################################################
#
@ -606,6 +626,7 @@ resources:
dns_service_ip: {get_param: dns_service_ip}
dns_cluster_domain: {get_param: dns_cluster_domain}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
outputs:

5
magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
View File

@ -263,6 +263,10 @@ parameters:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
master_wait_handle:
@ -521,6 +525,7 @@ resources:
user_data: {get_resource: kube_master_init}
networks:
- port: {get_resource: kube_master_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
kube_master_eth0:
type: OS::Neutron::Port

5
magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml
View File

@ -231,6 +231,10 @@ parameters:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
minion_wait_handle:
@ -433,6 +437,7 @@ resources:
user_data: {get_resource: kube_minion_init}
networks:
- port: {get_resource: kube_minion_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
kube_minion_eth0:
type: OS::Neutron::Port

21
magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml
View File

@ -347,6 +347,14 @@ parameters:
hidden: true
description: The OpenStack CA certificate to install on the node.
nodes_affinity_policy:
type: string
description: >
affinity policy for nodes server group
constraints:
- allowed_values: ["affinity", "anti-affinity", "soft-affinity",
"soft-anti-affinity"]
resources:
api_lb:
@ -443,6 +451,17 @@ resources:
public_ip: {get_attr: [api_address_lb_switch, public_ip]}
private_ip: {get_attr: [api_address_lb_switch, private_ip]}
######################################################################
#
# resources that expose the server group for all nodes include master
# and minions.
#
nodes_server_group:
type: OS::Nova::ServerGroup
properties:
policies: [{get_param: nodes_affinity_policy}]
######################################################################
#
# kubernetes masters. This is a resource group that will create
@ -509,6 +528,7 @@ resources:
wc_curl_cli: {get_attr: [master_wait_handle, curl_cli]}
etcd_lb_vip: {get_attr: [etcd_lb, address]}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
######################################################################
#
@ -552,6 +572,7 @@ resources:
flannel_network_cidr: {get_param: flannel_network_cidr}
external_network: {get_param: external_network}
kube_software_configs: {get_attr: [kubeminion_software_configs, kube_minion_init]}
nodes_server_group_id: {get_resource: nodes_server_group}
######################################################################
#

5
magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml
View File

@ -242,6 +242,10 @@ parameters:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
######################################################################
@ -491,6 +495,7 @@ resources:
user_data: {get_resource: kube_master_init}
networks:
- port: {get_resource: kube_master_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
kube_master_eth0:
type: "OS::Neutron::Port"

5
magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion.yaml
View File

@ -48,6 +48,10 @@ parameters:
description : >
ID of the multipart mime.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
######################################################################
@ -69,6 +73,7 @@ resources:
user_data: {get_param: kube_software_configs}
networks:
- port: {get_resource: kube_minion_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
kube_minion_eth0:
type: "OS::Neutron::Port"

21
magnum/drivers/mesos_ubuntu_v1/templates/mesoscluster.yaml
View File

@ -216,6 +216,14 @@ parameters:
hidden: true
description: The OpenStack CA certificate to install on the node.
nodes_affinity_policy:
type: string
description: >
affinity policy for nodes server group
constraints:
- allowed_values: ["affinity", "anti-affinity", "soft-affinity",
"soft-anti-affinity"]
resources:
######################################################################
@ -369,6 +377,17 @@ resources:
master_public_ip: {get_attr: [mesos_masters, resource.0.mesos_master_external_ip]}
master_private_ip: {get_attr: [mesos_masters, resource.0.mesos_master_ip]}
######################################################################
#
# resources that expose the server group for all nodes include master
# and minions.
#
nodes_server_group:
type: OS::Nova::ServerGroup
properties:
policies: [{get_param: nodes_affinity_policy}]
######################################################################
#
# Mesos masters. This is a resource group that will create
@ -397,6 +416,7 @@ resources:
secgroup_mesos_id: {get_resource: secgroup_master}
api_pool_id: {get_attr: [api_lb, pool_id]}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
######################################################################
#
@ -426,6 +446,7 @@ resources:
external_network: {get_param: external_network}
secgroup_slave_all_open_id: {get_resource: secgroup_slave_all_open}
mesos_slave_software_configs: {get_attr: [mesos_slave_software_configs, mesos_init]}
nodes_server_group_id: {get_resource: nodes_server_group}
######################################################################
#

5
magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml
View File

@ -48,6 +48,10 @@ parameters:
hidden: true
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
add_ext_ca_certs:
@ -85,6 +89,7 @@ resources:
user_data: {get_resource: mesos_master_init}
networks:
- port: {get_resource: mesos_master_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
mesos_master_eth0:
type: OS::Neutron::Port

5
magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml
View File

@ -43,6 +43,10 @@ parameters:
type: string
description: ID of the multipart mime.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
######################################################################
@ -64,6 +68,7 @@ resources:
user_data: {get_param: mesos_slave_software_configs}
networks:
- port: {get_resource: mesos_slave_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
mesos_slave_eth0:
type: OS::Neutron::Port

20
magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml
View File

@ -255,6 +255,13 @@ parameters:
hidden: true
description: The OpenStack CA certificate to install on the node.
nodes_affinity_policy:
type: string
description: >
affinity policy for nodes server group
constraints:
- allowed_values: ["affinity", "anti-affinity", "soft-affinity",
"soft-anti-affinity"]
resources:
@ -338,6 +345,17 @@ resources:
master_public_ip: {get_attr: [swarm_masters, resource.0.swarm_master_external_ip]}
master_private_ip: {get_attr: [swarm_masters, resource.0.swarm_master_ip]}
######################################################################
#
# resources that expose the server group for all nodes include master
# and minions.
#
nodes_server_group:
type: OS::Nova::ServerGroup
properties:
policies: [{get_param: nodes_affinity_policy}]
etcd_address_lb_switch:
type: Magnum::ApiGatewaySwitcher
properties:
@ -401,6 +419,7 @@ resources:
volume_driver: {get_param: volume_driver}
rexray_preempt: {get_param: rexray_preempt}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
swarm_nodes:
type: "OS::Heat::ResourceGroup"
@ -453,6 +472,7 @@ resources:
volume_driver: {get_param: volume_driver}
rexray_preempt: {get_param: rexray_preempt}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
outputs:

5
magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml
View File

@ -178,6 +178,10 @@ parameters:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
master_wait_handle:
@ -444,6 +448,7 @@ resources:
networks:
- port:
get_resource: swarm_master_eth0
scheduler_hints: { group: { get_param: nodes_server_group_id }}
swarm_master_eth0:
type: "OS::Neutron::Port"

5
magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml
View File

@ -177,6 +177,10 @@ parameters:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
node_wait_handle:
@ -400,6 +404,7 @@ resources:
networks:
- port:
get_resource: swarm_node_eth0
scheduler_hints: { group: { get_param: nodes_server_group_id }}
swarm_node_eth0:
type: "OS::Neutron::Port"

22
magnum/drivers/swarm_fedora_atomic_v2/templates/swarmcluster.yaml
View File

@ -186,6 +186,14 @@ parameters:
hidden: true
description: The OpenStack CA certificate to install on the node.
nodes_affinity_policy:
type: string
description: >
affinity policy for nodes server group
constraints:
- allowed_values: ["affinity", "anti-affinity", "soft-affinity",
"soft-anti-affinity"]
resources:
######################################################################
@ -260,6 +268,17 @@ resources:
master_public_ip: {get_attr: [swarm_primary_master, resource.0.swarm_master_external_ip]}
master_private_ip: {get_attr: [swarm_primary_master, resource.0.swarm_master_ip]}
######################################################################
#
# resources that expose the server group for all nodes include master
# and minions.
#
nodes_server_group:
type: OS::Nova::ServerGroup
properties:
policies: [{get_param: nodes_affinity_policy}]
######################################################################
#
# Swarm manager is responsible for the entire cluster and manages the
@ -309,6 +328,7 @@ resources:
rexray_preempt: {get_param: rexray_preempt}
verify_ca: {get_param: verify_ca}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
swarm_secondary_masters:
type: "OS::Heat::ResourceGroup"
@ -352,6 +372,7 @@ resources:
rexray_preempt: {get_param: rexray_preempt}
verify_ca: {get_param: verify_ca}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
swarm_nodes:
type: "OS::Heat::ResourceGroup"
@ -395,6 +416,7 @@ resources:
rexray_preempt: {get_param: rexray_preempt}
verify_ca: {get_param: verify_ca}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
outputs:

4
magnum/drivers/swarm_fedora_atomic_v2/templates/swarmmaster.yaml
View File

@ -140,6 +140,9 @@ parameters:
openstack_ca:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
@ -322,6 +325,7 @@ resources:
networks:
- port:
get_resource: swarm_master_eth0
scheduler_hints: { group: { get_param: nodes_server_group_id }}
swarm_master_eth0:
type: "OS::Neutron::Port"

5
magnum/drivers/swarm_fedora_atomic_v2/templates/swarmnode.yaml
View File

@ -133,6 +133,10 @@ parameters:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
node_wait_handle:
@ -293,6 +297,7 @@ resources:
networks:
- port:
get_resource: swarm_node_eth0
scheduler_hints: { group: { get_param: nodes_server_group_id }}
swarm_node_eth0:
type: "OS::Neutron::Port"

4
magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
View File

@ -229,6 +229,7 @@ class TestClusterConductorWithK8s(base.TestCase):
'kube_version': 'fake-version',
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity"
}
if missing_attr is not None:
expected.pop(mapping[missing_attr], None)
@ -325,6 +326,7 @@ class TestClusterConductorWithK8s(base.TestCase):
'kube_version': 'fake-version',
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity"
}
self.assertEqual(expected, definition)
@ -408,6 +410,7 @@ class TestClusterConductorWithK8s(base.TestCase):
'username': 'fake_user',
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity"
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -731,6 +734,7 @@ class TestClusterConductorWithK8s(base.TestCase):
'kube_version': 'fake-version',
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity"
}
self.assertEqual(expected, definition)
self.assertEqual(

5
magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py
View File

@ -164,6 +164,7 @@ class TestClusterConductorWithSwarm(base.TestCase):
'docker_volume_type': 'lvmdriver-1',
'verify_ca': True,
'openstack_ca': '',
'nodes_affinity_policy': 'soft-anti-affinity'
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -242,6 +243,7 @@ class TestClusterConductorWithSwarm(base.TestCase):
'docker_volume_type': 'lvmdriver-1',
'verify_ca': True,
'openstack_ca': '',
'nodes_affinity_policy': 'soft-anti-affinity'
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -314,6 +316,7 @@ class TestClusterConductorWithSwarm(base.TestCase):
'verify_ca': True,
'node_flavor': 'flavor_id',
'openstack_ca': '',
'nodes_affinity_policy': 'soft-anti-affinity'
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -386,6 +389,7 @@ class TestClusterConductorWithSwarm(base.TestCase):
'docker_volume_type': 'lvmdriver-1',
'verify_ca': True,
'openstack_ca': '',
'nodes_affinity_policy': 'soft-anti-affinity'
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -459,6 +463,7 @@ class TestClusterConductorWithSwarm(base.TestCase):
'docker_volume_type': 'lvmdriver-1',
'verify_ca': True,
'openstack_ca': '',
'nodes_affinity_policy': 'soft-anti-affinity'
}
self.assertEqual(expected, definition)
self.assertEqual(

9
magnum/tests/unit/drivers/test_template_definition.py
View File

@ -296,7 +296,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'magnum_url': mock_osc.magnum_url.return_value,
'region_name': mock_osc.cinder_region_name.return_value,
'kube_tag': kube_tag,
'container_infra_prefix': container_infra_prefix}}
'container_infra_prefix': container_infra_prefix,
'nodes_affinity_policy': 'soft-anti-affinity'}}
mock_get_params.assert_called_once_with(mock_context,
mock_cluster_template,
mock_cluster,
@ -381,7 +382,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'loadbalancing_protocol': 'HTTP',
'kubernetes_port': 8080,
'kube_tag': kube_tag,
'container_infra_prefix': container_infra_prefix}}
'container_infra_prefix': container_infra_prefix,
'nodes_affinity_policy': 'soft-anti-affinity'}}
mock_get_params.assert_called_once_with(mock_context,
mock_cluster_template,
mock_cluster,
@ -789,7 +791,8 @@ class AtomicSwarmTemplateDefinitionTestCase(base.TestCase):
'auth_url': 'http://192.168.10.10:5000/v3',
'rexray_preempt': rexray_preempt,
'swarm_strategy': swarm_strategy,
'docker_volume_type': docker_volume_type}}
'docker_volume_type': docker_volume_type,
'nodes_affinity_policy': 'soft-anti-affinity'}}
mock_get_params.assert_called_once_with(mock_context,
mock_cluster_template,
mock_cluster,

7
releasenotes/notes/support_nodes_affinity_policy-22253fb9cf6739ec.yaml Normal file
View File

@ -0,0 +1,7 @@
---
issues:
- |
Enhancement to support anfinity policy for cluster nodes. Before this patch,
There is no way to gurantee all nodes of a cluster created on different
compute hosts to get high availbility.