From 4f84c849f6aa83630d95e269f228753fbba85e8d Mon Sep 17 00:00:00 2001 From: Feilong Wang Date: Mon, 29 Oct 2018 14:45:57 +1300 Subject: [PATCH] Add server group for cluster worker nodes Now Magnums onlys has one server group for all master and worker nodes per cluster, which is not very flexible for small cloud scale. For a 3+ master clusters, it's easily meeting the capacity when using hard anti-affinity policy. This patch is proposing one server group for each master and worker nodes group to have better flexibility. story: 2004195 Change-Id: If11ba863a2aa538efe1e3e850084bdd33afd27d2 --- .../k8s_coreos_v1/templates/kubecluster.yaml | 15 ++++++--- .../templates/kubecluster.yaml | 15 ++++++--- .../templates/kubecluster.yaml | 15 ++++++--- .../templates/mesoscluster.yaml | 15 ++++++--- .../templates/cluster.yaml | 31 +++++++++++-------- .../templates/swarmcluster.yaml | 15 ++++++--- ...h-master-and-workder-bdd491e4323955d4.yaml | 9 ++++++ 7 files changed, 77 insertions(+), 38 deletions(-) create mode 100644 releasenotes/notes/server-groups-for-both-master-and-workder-bdd491e4323955d4.yaml diff --git a/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml index d23cff62e3..9e84549446 100644 --- a/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml @@ -605,11 +605,16 @@ resources: ###################################################################### # - # resources that expose the server group for all nodes include master - # and minions. + # resources that expose one server group for each master and worker nodes + # separately. # - nodes_server_group: + master_nodes_server_group: + type: OS::Nova::ServerGroup + properties: + policies: [{get_param: nodes_affinity_policy}] + + worker_nodes_server_group: type: OS::Nova::ServerGroup properties: policies: [{get_param: nodes_affinity_policy}] @@ -690,7 +695,7 @@ resources: dns_service_ip: {get_param: dns_service_ip} dns_cluster_domain: {get_param: dns_cluster_domain} openstack_ca: {get_param: openstack_ca_coreos} - nodes_server_group_id: {get_resource: nodes_server_group} + nodes_server_group_id: {get_resource: master_nodes_server_group} availability_zone: {get_param: availability_zone} ca_key: {get_param: ca_key} cert_manager_api: {get_param: cert_manager_api} @@ -778,7 +783,7 @@ resources: dns_service_ip: {get_param: dns_service_ip} dns_cluster_domain: {get_param: dns_cluster_domain} openstack_ca: {get_param: openstack_ca_coreos} - nodes_server_group_id: {get_resource: nodes_server_group} + nodes_server_group_id: {get_resource: worker_nodes_server_group} availability_zone: {get_param: availability_zone} pods_network_cidr: {get_param: pods_network_cidr} kubelet_options: {get_param: kubelet_options} diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index cd3ad8f3f0..66a1bac424 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -675,11 +675,16 @@ resources: ###################################################################### # - # resources that expose the server group for all nodes include master - # and minions. + # resources that expose one server group for each master and worker nodes + # separately. # - nodes_server_group: + master_nodes_server_group: + type: OS::Nova::ServerGroup + properties: + policies: [{get_param: nodes_affinity_policy}] + + worker_nodes_server_group: type: OS::Nova::ServerGroup properties: policies: [{get_param: nodes_affinity_policy}] @@ -762,7 +767,7 @@ resources: dns_service_ip: {get_param: dns_service_ip} dns_cluster_domain: {get_param: dns_cluster_domain} openstack_ca: {get_param: openstack_ca} - nodes_server_group_id: {get_resource: nodes_server_group} + nodes_server_group_id: {get_resource: master_nodes_server_group} availability_zone: {get_param: availability_zone} ca_key: {get_param: ca_key} cert_manager_api: {get_param: cert_manager_api} @@ -904,7 +909,7 @@ resources: dns_service_ip: {get_param: dns_service_ip} dns_cluster_domain: {get_param: dns_cluster_domain} openstack_ca: {get_param: openstack_ca} - nodes_server_group_id: {get_resource: nodes_server_group} + nodes_server_group_id: {get_resource: worker_nodes_server_group} availability_zone: {get_param: availability_zone} pods_network_cidr: {get_param: pods_network_cidr} kubelet_options: {get_param: kubelet_options} diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml index a22596f1da..49bfd0d037 100644 --- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml @@ -468,11 +468,16 @@ resources: ###################################################################### # - # resources that expose the server group for all nodes include master - # and minions. + # resources that expose one server group for each master and worker nodes + # separately. # - nodes_server_group: + master_nodes_server_group: + type: OS::Nova::ServerGroup + properties: + policies: [{get_param: nodes_affinity_policy}] + + worker_nodes_server_group: type: OS::Nova::ServerGroup properties: policies: [{get_param: nodes_affinity_policy}] @@ -544,7 +549,7 @@ resources: wc_curl_cli: {get_attr: [master_wait_handle, curl_cli]} etcd_lb_vip: {get_attr: [etcd_lb, address]} openstack_ca: {get_param: openstack_ca} - nodes_server_group_id: {get_resource: nodes_server_group} + nodes_server_group_id: {get_resource: master_nodes_server_group} octavia_enabled: {get_param: octavia_enabled} ###################################################################### @@ -589,7 +594,7 @@ resources: flannel_network_cidr: {get_param: flannel_network_cidr} external_network: {get_param: external_network} kube_software_configs: {get_attr: [kubeminion_software_configs, kube_minion_init]} - nodes_server_group_id: {get_resource: nodes_server_group} + nodes_server_group_id: {get_resource: worker_nodes_server_group} ###################################################################### # diff --git a/magnum/drivers/mesos_ubuntu_v1/templates/mesoscluster.yaml b/magnum/drivers/mesos_ubuntu_v1/templates/mesoscluster.yaml index 845509b015..d008af5540 100644 --- a/magnum/drivers/mesos_ubuntu_v1/templates/mesoscluster.yaml +++ b/magnum/drivers/mesos_ubuntu_v1/templates/mesoscluster.yaml @@ -380,11 +380,16 @@ resources: ###################################################################### # - # resources that expose the server group for all nodes include master - # and minions. + # resources that expose one server group for each master and worker nodes + # separately. # - nodes_server_group: + master_nodes_server_group: + type: OS::Nova::ServerGroup + properties: + policies: [{get_param: nodes_affinity_policy}] + + worker_nodes_server_group: type: OS::Nova::ServerGroup properties: policies: [{get_param: nodes_affinity_policy}] @@ -417,7 +422,7 @@ resources: secgroup_mesos_id: {get_resource: secgroup_master} api_pool_id: {get_attr: [api_lb, pool_id]} openstack_ca: {get_param: openstack_ca} - nodes_server_group_id: {get_resource: nodes_server_group} + nodes_server_group_id: {get_resource: master_nodes_server_group} ###################################################################### # @@ -447,7 +452,7 @@ resources: external_network: {get_param: external_network} secgroup_slave_all_open_id: {get_resource: secgroup_slave_all_open} mesos_slave_software_configs: {get_attr: [mesos_slave_software_configs, mesos_init]} - nodes_server_group_id: {get_resource: nodes_server_group} + nodes_server_group_id: {get_resource: worker_nodes_server_group} ###################################################################### # diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml index 4108512685..b9c6eaaf32 100644 --- a/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml @@ -346,23 +346,28 @@ resources: master_public_ip: {get_attr: [swarm_masters, resource.0.swarm_master_external_ip]} master_private_ip: {get_attr: [swarm_masters, resource.0.swarm_master_ip]} - ###################################################################### - # - # resources that expose the server group for all nodes include master - # and minions. - # - - nodes_server_group: - type: OS::Nova::ServerGroup - properties: - policies: [{get_param: nodes_affinity_policy}] - etcd_address_lb_switch: type: Magnum::ApiGatewaySwitcher properties: pool_private_ip: {get_attr: [etcd_lb, address]} master_private_ip: {get_attr: [swarm_masters, resource.0.swarm_master_ip]} + ###################################################################### + # + # resources that expose one server group for each master and worker nodes + # separately. + # + + master_nodes_server_group: + type: OS::Nova::ServerGroup + properties: + policies: [{get_param: nodes_affinity_policy}] + + worker_nodes_server_group: + type: OS::Nova::ServerGroup + properties: + policies: [{get_param: nodes_affinity_policy}] + ###################################################################### # # Swarm manager is responsible for the entire cluster and manages the @@ -420,7 +425,7 @@ resources: volume_driver: {get_param: volume_driver} rexray_preempt: {get_param: rexray_preempt} openstack_ca: {get_param: openstack_ca} - nodes_server_group_id: {get_resource: nodes_server_group} + nodes_server_group_id: {get_resource: master_nodes_server_group} swarm_nodes: type: "OS::Heat::ResourceGroup" @@ -473,7 +478,7 @@ resources: volume_driver: {get_param: volume_driver} rexray_preempt: {get_param: rexray_preempt} openstack_ca: {get_param: openstack_ca} - nodes_server_group_id: {get_resource: nodes_server_group} + nodes_server_group_id: {get_resource: worker_nodes_server_group} outputs: diff --git a/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmcluster.yaml b/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmcluster.yaml index 301a405e70..c71fefe5f8 100644 --- a/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmcluster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v2/templates/swarmcluster.yaml @@ -292,11 +292,16 @@ resources: ###################################################################### # - # resources that expose the server group for all nodes include master - # and minions. + # resources that expose one server group for each master and worker nodes + # separately. # - nodes_server_group: + master_nodes_server_group: + type: OS::Nova::ServerGroup + properties: + policies: [{get_param: nodes_affinity_policy}] + + worker_nodes_server_group: type: OS::Nova::ServerGroup properties: policies: [{get_param: nodes_affinity_policy}] @@ -350,7 +355,7 @@ resources: rexray_preempt: {get_param: rexray_preempt} verify_ca: {get_param: verify_ca} openstack_ca: {get_param: openstack_ca} - nodes_server_group_id: {get_resource: nodes_server_group} + nodes_server_group_id: {get_resource: master_nodes_server_group} availability_zone: {get_param: availability_zone} swarm_secondary_masters: @@ -395,7 +400,7 @@ resources: rexray_preempt: {get_param: rexray_preempt} verify_ca: {get_param: verify_ca} openstack_ca: {get_param: openstack_ca} - nodes_server_group_id: {get_resource: nodes_server_group} + nodes_server_group_id: {get_resource: worker_nodes_server_group} availability_zone: {get_param: availability_zone} swarm_nodes: diff --git a/releasenotes/notes/server-groups-for-both-master-and-workder-bdd491e4323955d4.yaml b/releasenotes/notes/server-groups-for-both-master-and-workder-bdd491e4323955d4.yaml new file mode 100644 index 0000000000..430bace0b4 --- /dev/null +++ b/releasenotes/notes/server-groups-for-both-master-and-workder-bdd491e4323955d4.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Magnums onlys has one server group for all master and worker nodes + per cluster, which is not very flexible for small cloud scale. For a + 3+ master clusters, it's easily meeting the capacity when using hard + anti-affinity policy. Now one server group is added for each + master and worker nodes group to have better flexibility. +