From be0609ce8805c71ccbe265ede772aa62976e2780 Mon Sep 17 00:00:00 2001 From: Feilong Wang Date: Thu, 14 Dec 2017 11:22:13 +1300 Subject: [PATCH] Support soft-anti-affinity policy for nodes Currently, there is no guarantee to make sure all nodes of one cluster are created on different compute hosts. So it would be nice if we can create a server group and set it with anti-affinity policy to get a better HA for cluster. This patch is proposing to create a server group for master and minion nodes with soft-anti-affinity policy by default. Closes-Bug: #1737802 Change-Id: Icc7a73ef55296a58bf00719ca4d1cdcc304fab86 --- magnum/conf/cluster.py | 6 +++++ .../drivers/heat/k8s_fedora_template_def.py | 3 +++ .../drivers/heat/swarm_fedora_template_def.py | 2 ++ .../drivers/heat/swarm_mode_template_def.py | 2 ++ .../k8s_coreos_v1/templates/kubecluster.yaml | 21 ++++++++++++++++++ .../k8s_coreos_v1/templates/kubemaster.yaml | 4 ++++ .../k8s_coreos_v1/templates/kubeminion.yaml | 5 +++++ .../templates/kubecluster.yaml | 21 ++++++++++++++++++ .../templates/kubemaster.yaml | 5 +++++ .../templates/kubeminion.yaml | 5 +++++ .../templates/kubecluster.yaml | 21 ++++++++++++++++++ .../templates/kubemaster.yaml | 5 +++++ .../templates/kubeminion.yaml | 5 +++++ .../templates/mesoscluster.yaml | 21 ++++++++++++++++++ .../templates/mesosmaster.yaml | 5 +++++ .../mesos_ubuntu_v1/templates/mesosslave.yaml | 5 +++++ .../templates/cluster.yaml | 20 +++++++++++++++++ .../templates/swarmmaster.yaml | 5 +++++ .../templates/swarmnode.yaml | 5 +++++ .../templates/swarmcluster.yaml | 22 +++++++++++++++++++ .../templates/swarmmaster.yaml | 4 ++++ .../templates/swarmnode.yaml | 5 +++++ .../handlers/test_k8s_cluster_conductor.py | 4 ++++ .../handlers/test_swarm_cluster_conductor.py | 5 +++++ .../unit/drivers/test_template_definition.py | 9 +++++--- ...odes_affinity_policy-22253fb9cf6739ec.yaml | 7 ++++++ 26 files changed, 219 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/support_nodes_affinity_policy-22253fb9cf6739ec.yaml diff --git a/magnum/conf/cluster.py b/magnum/conf/cluster.py index 83b38bc30d..c7444c6eb5 100644 --- a/magnum/conf/cluster.py +++ b/magnum/conf/cluster.py @@ -32,6 +32,12 @@ cluster_def_opts = [ 'magnum_vm_ubuntu_mesos'], help=_('Enabled cluster definition entry points.'), deprecated_group='bay'), + cfg.StrOpt('nodes_affinity_policy', + default='soft-anti-affinity', + help=_('Affinity policy for server group of cluster nodes.' + 'Possible values include "affinity", "anti-affinity",' + '"soft-affinity" and "soft-anti-affinity".') + ), ] diff --git a/magnum/drivers/heat/k8s_fedora_template_def.py b/magnum/drivers/heat/k8s_fedora_template_def.py index d9263d6336..3ba3433cf9 100644 --- a/magnum/drivers/heat/k8s_fedora_template_def.py +++ b/magnum/drivers/heat/k8s_fedora_template_def.py @@ -87,6 +87,9 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition): if container_infra_prefix: extra_params['container_infra_prefix'] = container_infra_prefix + extra_params['nodes_affinity_policy'] = \ + CONF.cluster.nodes_affinity_policy + return super(K8sFedoraTemplateDefinition, self).get_params(context, cluster_template, cluster, extra_params=extra_params, diff --git a/magnum/drivers/heat/swarm_fedora_template_def.py b/magnum/drivers/heat/swarm_fedora_template_def.py index a97af3814a..00fbc497b2 100644 --- a/magnum/drivers/heat/swarm_fedora_template_def.py +++ b/magnum/drivers/heat/swarm_fedora_template_def.py @@ -102,6 +102,8 @@ class SwarmFedoraTemplateDefinition(template_def.BaseTemplateDefinition): 'swarm_strategy'] extra_params['auth_url'] = context.auth_url + extra_params['nodes_affinity_policy'] = \ + CONF.cluster.nodes_affinity_policy # set docker_volume_type # use the configuration default if None provided diff --git a/magnum/drivers/heat/swarm_mode_template_def.py b/magnum/drivers/heat/swarm_mode_template_def.py index 33c632098d..a8c256db50 100644 --- a/magnum/drivers/heat/swarm_mode_template_def.py +++ b/magnum/drivers/heat/swarm_mode_template_def.py @@ -105,6 +105,8 @@ class SwarmModeTemplateDefinition(template_def.BaseTemplateDefinition): label_list = ['rexray_preempt'] extra_params['auth_url'] = context.auth_url + extra_params['nodes_affinity_policy'] = \ + CONF.cluster.nodes_affinity_policy for label in label_list: extra_params[label] = cluster_template.labels.get(label) diff --git a/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml index 8a9ad9fd0b..6517c2be0a 100644 --- a/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml @@ -294,6 +294,14 @@ parameters: hidden: true description: The OpenStack CA certificate to install on the node. + nodes_affinity_policy: + type: string + description: > + affinity policy for nodes server group + constraints: + - allowed_values: ["affinity", "anti-affinity", "soft-affinity", + "soft-anti-affinity"] + resources: ###################################################################### @@ -400,6 +408,17 @@ resources: public_ip: {get_attr: [api_address_lb_switch, public_ip]} private_ip: {get_attr: [api_address_lb_switch, private_ip]} + ###################################################################### + # + # resources that expose the server group for all nodes include master + # and minions. + # + + nodes_server_group: + type: OS::Nova::ServerGroup + properties: + policies: [{get_param: nodes_affinity_policy}] + ###################################################################### # # kubernetes masters. This is a resource group that will create @@ -465,6 +484,7 @@ resources: dns_service_ip: {get_param: dns_service_ip} dns_cluster_domain: {get_param: dns_cluster_domain} openstack_ca: {get_param: openstack_ca} + nodes_server_group_id: {get_resource: nodes_server_group} ###################################################################### # @@ -520,6 +540,7 @@ resources: dns_service_ip: {get_param: dns_service_ip} dns_cluster_domain: {get_param: dns_cluster_domain} openstack_ca: {get_param: openstack_ca} + nodes_server_group_id: {get_resource: nodes_server_group} outputs: diff --git a/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml index f6582e0a6b..157df46a3d 100644 --- a/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml @@ -229,6 +229,9 @@ parameters: openstack_ca: type: string description: The OpenStack CA certificate to install on the node. + nodes_server_group_id: + type: string + description: ID of the server group for kubernetes cluster nodes. resources: @@ -511,6 +514,7 @@ resources: user_data: {get_resource: kube_master_init} networks: - port: {get_resource: kube_master_eth0} + scheduler_hints: { group: { get_param: nodes_server_group_id }} kube_master_eth0: type: OS::Neutron::Port diff --git a/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml index 26fcbac7f3..8d720ca3cb 100644 --- a/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml @@ -160,6 +160,10 @@ parameters: type: string description: The OpenStack CA certificate to install on the node. + nodes_server_group_id: + type: string + description: ID of the server group for kubernetes cluster nodes. + resources: minion_wait_handle: @@ -338,6 +342,7 @@ resources: user_data: {get_resource: kube_minion_init} networks: - port: {get_resource: kube_minion_eth0} + scheduler_hints: { group: { get_param: nodes_server_group_id }} kube_minion_eth0: type: OS::Neutron::Port diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index 6793ecd114..0cc80d038a 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -356,6 +356,14 @@ parameters: hidden: true description: The OpenStack CA certificate to install on the node. + nodes_affinity_policy: + type: string + description: > + affinity policy for nodes server group + constraints: + - allowed_values: ["affinity", "anti-affinity", "soft-affinity", + "soft-anti-affinity"] + resources: ###################################################################### @@ -465,6 +473,17 @@ resources: public_ip: {get_attr: [api_address_lb_switch, public_ip]} private_ip: {get_attr: [api_address_lb_switch, private_ip]} + ###################################################################### + # + # resources that expose the server group for all nodes include master + # and minions. + # + + nodes_server_group: + type: OS::Nova::ServerGroup + properties: + policies: [{get_param: nodes_affinity_policy}] + ###################################################################### # # kubernetes masters. This is a resource group that will create @@ -537,6 +556,7 @@ resources: dns_service_ip: {get_param: dns_service_ip} dns_cluster_domain: {get_param: dns_cluster_domain} openstack_ca: {get_param: openstack_ca} + nodes_server_group_id: {get_resource: nodes_server_group} ###################################################################### # @@ -606,6 +626,7 @@ resources: dns_service_ip: {get_param: dns_service_ip} dns_cluster_domain: {get_param: dns_cluster_domain} openstack_ca: {get_param: openstack_ca} + nodes_server_group_id: {get_resource: nodes_server_group} outputs: diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 2737270a7c..85ec60f5bb 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -263,6 +263,10 @@ parameters: type: string description: The OpenStack CA certificate to install on the node. + nodes_server_group_id: + type: string + description: ID of the server group for kubernetes cluster nodes. + resources: master_wait_handle: @@ -521,6 +525,7 @@ resources: user_data: {get_resource: kube_master_init} networks: - port: {get_resource: kube_master_eth0} + scheduler_hints: { group: { get_param: nodes_server_group_id }} kube_master_eth0: type: OS::Neutron::Port diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml index a03975624a..183d5430ab 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml @@ -231,6 +231,10 @@ parameters: type: string description: The OpenStack CA certificate to install on the node. + nodes_server_group_id: + type: string + description: ID of the server group for kubernetes cluster nodes. + resources: minion_wait_handle: @@ -433,6 +437,7 @@ resources: user_data: {get_resource: kube_minion_init} networks: - port: {get_resource: kube_minion_eth0} + scheduler_hints: { group: { get_param: nodes_server_group_id }} kube_minion_eth0: type: OS::Neutron::Port diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml index e94cf39b37..3c7ae29632 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml @@ -347,6 +347,14 @@ parameters: hidden: true description: The OpenStack CA certificate to install on the node. + nodes_affinity_policy: + type: string + description: > + affinity policy for nodes server group + constraints: + - allowed_values: ["affinity", "anti-affinity", "soft-affinity", + "soft-anti-affinity"] + resources: api_lb: @@ -443,6 +451,17 @@ resources: public_ip: {get_attr: [api_address_lb_switch, public_ip]} private_ip: {get_attr: [api_address_lb_switch, private_ip]} + ###################################################################### + # + # resources that expose the server group for all nodes include master + # and minions. + # + + nodes_server_group: + type: OS::Nova::ServerGroup + properties: + policies: [{get_param: nodes_affinity_policy}] + ###################################################################### # # kubernetes masters. This is a resource group that will create @@ -509,6 +528,7 @@ resources: wc_curl_cli: {get_attr: [master_wait_handle, curl_cli]} etcd_lb_vip: {get_attr: [etcd_lb, address]} openstack_ca: {get_param: openstack_ca} + nodes_server_group_id: {get_resource: nodes_server_group} ###################################################################### # @@ -552,6 +572,7 @@ resources: flannel_network_cidr: {get_param: flannel_network_cidr} external_network: {get_param: external_network} kube_software_configs: {get_attr: [kubeminion_software_configs, kube_minion_init]} + nodes_server_group_id: {get_resource: nodes_server_group} ###################################################################### # diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml index 9db0bcd423..74fddf89cd 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml @@ -242,6 +242,10 @@ parameters: type: string description: The OpenStack CA certificate to install on the node. + nodes_server_group_id: + type: string + description: ID of the server group for kubernetes cluster nodes. + resources: ###################################################################### @@ -491,6 +495,7 @@ resources: user_data: {get_resource: kube_master_init} networks: - port: {get_resource: kube_master_eth0} + scheduler_hints: { group: { get_param: nodes_server_group_id }} kube_master_eth0: type: "OS::Neutron::Port" diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion.yaml index 815984867a..143b1bab96 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion.yaml @@ -48,6 +48,10 @@ parameters: description : > ID of the multipart mime. + nodes_server_group_id: + type: string + description: ID of the server group for kubernetes cluster nodes. + resources: ###################################################################### @@ -69,6 +73,7 @@ resources: user_data: {get_param: kube_software_configs} networks: - port: {get_resource: kube_minion_eth0} + scheduler_hints: { group: { get_param: nodes_server_group_id }} kube_minion_eth0: type: "OS::Neutron::Port" diff --git a/magnum/drivers/mesos_ubuntu_v1/templates/mesoscluster.yaml b/magnum/drivers/mesos_ubuntu_v1/templates/mesoscluster.yaml index b0e75b7fc7..e840ff1b4d 100644 --- a/magnum/drivers/mesos_ubuntu_v1/templates/mesoscluster.yaml +++ b/magnum/drivers/mesos_ubuntu_v1/templates/mesoscluster.yaml @@ -216,6 +216,14 @@ parameters: hidden: true description: The OpenStack CA certificate to install on the node. + nodes_affinity_policy: + type: string + description: > + affinity policy for nodes server group + constraints: + - allowed_values: ["affinity", "anti-affinity", "soft-affinity", + "soft-anti-affinity"] + resources: ###################################################################### @@ -369,6 +377,17 @@ resources: master_public_ip: {get_attr: [mesos_masters, resource.0.mesos_master_external_ip]} master_private_ip: {get_attr: [mesos_masters, resource.0.mesos_master_ip]} + ###################################################################### + # + # resources that expose the server group for all nodes include master + # and minions. + # + + nodes_server_group: + type: OS::Nova::ServerGroup + properties: + policies: [{get_param: nodes_affinity_policy}] + ###################################################################### # # Mesos masters. This is a resource group that will create @@ -397,6 +416,7 @@ resources: secgroup_mesos_id: {get_resource: secgroup_master} api_pool_id: {get_attr: [api_lb, pool_id]} openstack_ca: {get_param: openstack_ca} + nodes_server_group_id: {get_resource: nodes_server_group} ###################################################################### # @@ -426,6 +446,7 @@ resources: external_network: {get_param: external_network} secgroup_slave_all_open_id: {get_resource: secgroup_slave_all_open} mesos_slave_software_configs: {get_attr: [mesos_slave_software_configs, mesos_init]} + nodes_server_group_id: {get_resource: nodes_server_group} ###################################################################### # diff --git a/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml b/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml index c79233f5b3..5243e11e48 100644 --- a/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml +++ b/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml @@ -48,6 +48,10 @@ parameters: hidden: true description: The OpenStack CA certificate to install on the node. + nodes_server_group_id: + type: string + description: ID of the server group for kubernetes cluster nodes. + resources: add_ext_ca_certs: @@ -85,6 +89,7 @@ resources: user_data: {get_resource: mesos_master_init} networks: - port: {get_resource: mesos_master_eth0} + scheduler_hints: { group: { get_param: nodes_server_group_id }} mesos_master_eth0: type: OS::Neutron::Port diff --git a/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml b/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml index b6aaeacace..315171521f 100644 --- a/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml +++ b/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml @@ -43,6 +43,10 @@ parameters: type: string description: ID of the multipart mime. + nodes_server_group_id: + type: string + description: ID of the server group for kubernetes cluster nodes. + resources: ###################################################################### @@ -64,6 +68,7 @@ resources: user_data: {get_param: mesos_slave_software_configs} networks: - port: {get_resource: mesos_slave_eth0} + scheduler_hints: { group: { get_param: nodes_server_group_id }} mesos_slave_eth0: type: OS::Neutron::Port diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml index debd6c36df..c95c615593 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml @@ -255,6 +255,13 @@ parameters: hidden: true description: The OpenStack CA certificate to install on the node. + nodes_affinity_policy: + type: string + description: > + affinity policy for nodes server group + constraints: + - allowed_values: ["affinity", "anti-affinity", "soft-affinity", + "soft-anti-affinity"] resources: @@ -338,6 +345,17 @@ resources: master_public_ip: {get_attr: [swarm_masters, resource.0.swarm_master_external_ip]} master_private_ip: {get_attr: [swarm_masters, resource.0.swarm_master_ip]} + ###################################################################### + # + # resources that expose the server group for all nodes include master + # and minions. + # + + nodes_server_group: + type: OS::Nova::ServerGroup + properties: + policies: [{get_param: nodes_affinity_policy}] + etcd_address_lb_switch: type: Magnum::ApiGatewaySwitcher properties: @@ -401,6 +419,7 @@ resources: volume_driver: {get_param: volume_driver} rexray_preempt: {get_param: rexray_preempt} openstack_ca: {get_param: openstack_ca} + nodes_server_group_id: {get_resource: nodes_server_group} swarm_nodes: type: "OS::Heat::ResourceGroup" @@ -453,6 +472,7 @@ resources: volume_driver: {get_param: volume_driver} rexray_preempt: {get_param: rexray_preempt} openstack_ca: {get_param: openstack_ca} + nodes_server_group_id: {get_resource: nodes_server_group} outputs: diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml index 41d989e421..ba49f2fab2 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml @@ -178,6 +178,10 @@ parameters: type: string description: The OpenStack CA certificate to install on the node. + nodes_server_group_id: + type: string + description: ID of the server group for kubernetes cluster nodes. + resources: master_wait_handle: @@ -444,6 +448,7 @@ resources: networks: - port: get_resource: swarm_master_eth0 + scheduler_hints: { group: { get_param: nodes_server_group_id }} swarm_master_eth0: type: "OS::Neutron::Port" diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml index 5460621d19..03d77bb20b 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml @@ -177,6 +177,10 @@ parameters: type: string description: The OpenStack CA certificate to install on the node. + nodes_server_group_id: + type: string + description: ID of the server group for kubernetes cluster nodes. + resources: node_wait_handle: @@ -400,6 +404,7 @@ resources: networks: - port: get_resource: swarm_node_eth0 + scheduler_hints: { group: { get_param: nodes_server_group_id }} swarm_node_eth0: type: "OS::Neutron::Port" diff --git a/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmcluster.yaml b/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmcluster.yaml index 1a0bac1300..8a0aa63507 100644 --- a/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmcluster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmcluster.yaml @@ -186,6 +186,14 @@ parameters: hidden: true description: The OpenStack CA certificate to install on the node. + nodes_affinity_policy: + type: string + description: > + affinity policy for nodes server group + constraints: + - allowed_values: ["affinity", "anti-affinity", "soft-affinity", + "soft-anti-affinity"] + resources: ###################################################################### @@ -260,6 +268,17 @@ resources: master_public_ip: {get_attr: [swarm_primary_master, resource.0.swarm_master_external_ip]} master_private_ip: {get_attr: [swarm_primary_master, resource.0.swarm_master_ip]} + ###################################################################### + # + # resources that expose the server group for all nodes include master + # and minions. + # + + nodes_server_group: + type: OS::Nova::ServerGroup + properties: + policies: [{get_param: nodes_affinity_policy}] + ###################################################################### # # Swarm manager is responsible for the entire cluster and manages the @@ -309,6 +328,7 @@ resources: rexray_preempt: {get_param: rexray_preempt} verify_ca: {get_param: verify_ca} openstack_ca: {get_param: openstack_ca} + nodes_server_group_id: {get_resource: nodes_server_group} swarm_secondary_masters: type: "OS::Heat::ResourceGroup" @@ -352,6 +372,7 @@ resources: rexray_preempt: {get_param: rexray_preempt} verify_ca: {get_param: verify_ca} openstack_ca: {get_param: openstack_ca} + nodes_server_group_id: {get_resource: nodes_server_group} swarm_nodes: type: "OS::Heat::ResourceGroup" @@ -395,6 +416,7 @@ resources: rexray_preempt: {get_param: rexray_preempt} verify_ca: {get_param: verify_ca} openstack_ca: {get_param: openstack_ca} + nodes_server_group_id: {get_resource: nodes_server_group} outputs: diff --git a/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmmaster.yaml b/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmmaster.yaml index 4a9e0cbe8f..a9a62e4eb4 100644 --- a/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmmaster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmmaster.yaml @@ -140,6 +140,9 @@ parameters: openstack_ca: type: string description: The OpenStack CA certificate to install on the node. + nodes_server_group_id: + type: string + description: ID of the server group for kubernetes cluster nodes. resources: @@ -322,6 +325,7 @@ resources: networks: - port: get_resource: swarm_master_eth0 + scheduler_hints: { group: { get_param: nodes_server_group_id }} swarm_master_eth0: type: "OS::Neutron::Port" diff --git a/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmnode.yaml b/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmnode.yaml index 164ed28e04..adfbdaa6b9 100644 --- a/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmnode.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmnode.yaml @@ -133,6 +133,10 @@ parameters: type: string description: The OpenStack CA certificate to install on the node. + nodes_server_group_id: + type: string + description: ID of the server group for kubernetes cluster nodes. + resources: node_wait_handle: @@ -293,6 +297,7 @@ resources: networks: - port: get_resource: swarm_node_eth0 + scheduler_hints: { group: { get_param: nodes_server_group_id }} swarm_node_eth0: type: "OS::Neutron::Port" diff --git a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py index 9ea6ce724e..7b8195520c 100644 --- a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py @@ -229,6 +229,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'kube_version': 'fake-version', 'verify_ca': True, 'openstack_ca': '', + "nodes_affinity_policy": "soft-anti-affinity" } if missing_attr is not None: expected.pop(mapping[missing_attr], None) @@ -325,6 +326,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'kube_version': 'fake-version', 'verify_ca': True, 'openstack_ca': '', + "nodes_affinity_policy": "soft-anti-affinity" } self.assertEqual(expected, definition) @@ -408,6 +410,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'username': 'fake_user', 'verify_ca': True, 'openstack_ca': '', + "nodes_affinity_policy": "soft-anti-affinity" } self.assertEqual(expected, definition) self.assertEqual( @@ -731,6 +734,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'kube_version': 'fake-version', 'verify_ca': True, 'openstack_ca': '', + "nodes_affinity_policy": "soft-anti-affinity" } self.assertEqual(expected, definition) self.assertEqual( diff --git a/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py index 2d72b61f9f..a0a51a0c66 100644 --- a/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py @@ -164,6 +164,7 @@ class TestClusterConductorWithSwarm(base.TestCase): 'docker_volume_type': 'lvmdriver-1', 'verify_ca': True, 'openstack_ca': '', + 'nodes_affinity_policy': 'soft-anti-affinity' } self.assertEqual(expected, definition) self.assertEqual( @@ -242,6 +243,7 @@ class TestClusterConductorWithSwarm(base.TestCase): 'docker_volume_type': 'lvmdriver-1', 'verify_ca': True, 'openstack_ca': '', + 'nodes_affinity_policy': 'soft-anti-affinity' } self.assertEqual(expected, definition) self.assertEqual( @@ -314,6 +316,7 @@ class TestClusterConductorWithSwarm(base.TestCase): 'verify_ca': True, 'node_flavor': 'flavor_id', 'openstack_ca': '', + 'nodes_affinity_policy': 'soft-anti-affinity' } self.assertEqual(expected, definition) self.assertEqual( @@ -386,6 +389,7 @@ class TestClusterConductorWithSwarm(base.TestCase): 'docker_volume_type': 'lvmdriver-1', 'verify_ca': True, 'openstack_ca': '', + 'nodes_affinity_policy': 'soft-anti-affinity' } self.assertEqual(expected, definition) self.assertEqual( @@ -459,6 +463,7 @@ class TestClusterConductorWithSwarm(base.TestCase): 'docker_volume_type': 'lvmdriver-1', 'verify_ca': True, 'openstack_ca': '', + 'nodes_affinity_policy': 'soft-anti-affinity' } self.assertEqual(expected, definition) self.assertEqual( diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py index 2f308fba77..d4cf4f5d10 100644 --- a/magnum/tests/unit/drivers/test_template_definition.py +++ b/magnum/tests/unit/drivers/test_template_definition.py @@ -296,7 +296,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): 'magnum_url': mock_osc.magnum_url.return_value, 'region_name': mock_osc.cinder_region_name.return_value, 'kube_tag': kube_tag, - 'container_infra_prefix': container_infra_prefix}} + 'container_infra_prefix': container_infra_prefix, + 'nodes_affinity_policy': 'soft-anti-affinity'}} mock_get_params.assert_called_once_with(mock_context, mock_cluster_template, mock_cluster, @@ -381,7 +382,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): 'loadbalancing_protocol': 'HTTP', 'kubernetes_port': 8080, 'kube_tag': kube_tag, - 'container_infra_prefix': container_infra_prefix}} + 'container_infra_prefix': container_infra_prefix, + 'nodes_affinity_policy': 'soft-anti-affinity'}} mock_get_params.assert_called_once_with(mock_context, mock_cluster_template, mock_cluster, @@ -789,7 +791,8 @@ class AtomicSwarmTemplateDefinitionTestCase(base.TestCase): 'auth_url': 'http://192.168.10.10:5000/v3', 'rexray_preempt': rexray_preempt, 'swarm_strategy': swarm_strategy, - 'docker_volume_type': docker_volume_type}} + 'docker_volume_type': docker_volume_type, + 'nodes_affinity_policy': 'soft-anti-affinity'}} mock_get_params.assert_called_once_with(mock_context, mock_cluster_template, mock_cluster, diff --git a/releasenotes/notes/support_nodes_affinity_policy-22253fb9cf6739ec.yaml b/releasenotes/notes/support_nodes_affinity_policy-22253fb9cf6739ec.yaml new file mode 100644 index 0000000000..b5591660fe --- /dev/null +++ b/releasenotes/notes/support_nodes_affinity_policy-22253fb9cf6739ec.yaml @@ -0,0 +1,7 @@ +--- +issues: + - | + Enhancement to support anfinity policy for cluster nodes. Before this patch, + There is no way to gurantee all nodes of a cluster created on different + compute hosts to get high availbility. +