openstack
/
magnum
Code Issues Proposed changes

Put kube_master into a resource group 

With this patch, k8s template support more than one master nodes.
The number of master nodes can be specified through Heat parameter.

For fault tolerance, the cluster tolerants failure of no more than
half of the size. For example, a cluster with 3 master node tolerant
failure of 1 master node. Below is the table:

SIZE TOLERANCE
1    0
3    1
4    1
5    2
6    2
7    3
8    3
9    4

Change-Id: Ic28c13bdcadb39949f9c50be45e09f7627c5d627
Partially-Implements: blueprint make-master-ha
This commit is contained in:
Hongbin Lu 2015-07-28 15:21:24 -04:00
parent bcdd70cf1e
commit 63101497fc
2 changed files with 316 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

229
magnum/templates/heat-kubernetes/kubecluster.yaml
View File

@ -35,6 +35,11 @@ parameters:
description: address of a dns nameserver reachable in your environment
default: 8.8.8.8
number_of_masters:
type: string
description: how many kubernetes masters to spawn
default: 1
number_of_minions:
type: string
description: how many kubernetes minions to spawn
@ -107,16 +112,6 @@ parameters:
resources:
master_wait_handle:
type: OS::Heat::WaitConditionHandle
master_wait_condition:
type: OS::Heat::WaitCondition
depends_on: kube_master
properties:
handle: {get_resource: master_wait_handle}
timeout: {get_param: wait_condition_timeout}
######################################################################
#
# network resources. allocate a network and router for our server.
@ -145,169 +140,6 @@ resources:
router_id: {get_resource: extrouter}
subnet: {get_resource: fixed_subnet}
######################################################################
#
# security groups. we need to permit network traffic of various
# sorts.
#
secgroup_base:
type: OS::Neutron::SecurityGroup
properties:
rules:
- protocol: icmp
- protocol: tcp
port_range_min: 22
port_range_max: 22
secgroup_kubernetes:
type: OS::Neutron::SecurityGroup
properties:
rules:
- protocol: tcp
port_range_min: 7080
port_range_max: 7080
- protocol: tcp
port_range_min: 8080
port_range_max: 8080
- protocol: tcp
port_range_min: 4001
port_range_max: 4001
- protocol: tcp
port_range_min: 7001
port_range_max: 7001
######################################################################
#
# software configs. these are components that are combined into
# a multipart MIME user-data archive.
#
write_heat_params:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config:
str_replace:
template: {get_file: fragments/write-heat-params-master.yaml}
params:
"$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
"$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
"$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
"$FLANNEL_USE_VXLAN": {get_param: flannel_use_vxlan}
"$PORTAL_NETWORK_CIDR": {get_param: portal_network_cidr}
"$ETCD_DISCOVERY_URL": {get_param: discovery_url}
configure_etcd:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/configure-etcd.sh}
configure_kubernetes:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/configure-kubernetes-master.sh}
write_flannel_config:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/write-flannel-config.sh}
flannel_config_service:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/flannel-config.service.yaml}
enable_services:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/enable-services-master.sh}
kube_user:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/kube-user.yaml}
kube_examples:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/kube-examples.yaml}
master_wc_notify:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config:
str_replace:
template: |
#!/bin/bash -v
wc_notify --data-binary '{"status": "SUCCESS"}'
params:
wc_notify: {get_attr: [master_wait_handle, curl_cli]}
disable_selinux:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/disable-selinux.sh}
kube_master_init:
type: OS::Heat::MultipartMime
properties:
parts:
- config: {get_resource: disable_selinux}
- config: {get_resource: write_heat_params}
- config: {get_resource: configure_etcd}
- config: {get_resource: kube_user}
- config: {get_resource: configure_kubernetes}
- config: {get_resource: enable_services}
- config: {get_resource: write_flannel_config}
- config: {get_resource: flannel_config_service}
- config: {get_resource: kube_examples}
- config: {get_resource: master_wc_notify}
######################################################################
#
# kubernetes master server.
#
kube_master:
type: OS::Nova::Server
depends_on:
- extrouter_inside
properties:
image: {get_param: server_image}
flavor: {get_param: master_flavor}
key_name: {get_param: ssh_key_name}
user_data_format: RAW
user_data: {get_resource: kube_master_init}
networks:
- port: {get_resource: kube_master_eth0}
kube_master_eth0:
type: OS::Neutron::Port
properties:
network: {get_resource: fixed_network}
security_groups:
- {get_resource: secgroup_base}
- {get_resource: secgroup_kubernetes}
fixed_ips:
- subnet: {get_resource: fixed_subnet}
replacement_policy: AUTO
kube_master_floating:
type: OS::Neutron::FloatingIP
properties:
floating_network: {get_param: external_network}
port_id: {get_resource: kube_master_eth0}
######################################################################
#
# load balancers.
@ -331,13 +163,6 @@ resources:
vip:
protocol_port: 8080
api_pool_member:
type: OS::Neutron::PoolMember
properties:
pool_id: {get_resource: api_pool}
address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
protocol_port: 8080
api_pool_floating:
type: OS::Neutron::FloatingIP
properties:
@ -362,12 +187,36 @@ resources:
vip:
protocol_port: 4001
etcd_pool_member:
type: OS::Neutron::PoolMember
######################################################################
#
# kubernetes masters. This is a resource group that will create
# <number_of_masters> masters.
#
kube_masters:
type: "OS::Heat::ResourceGroup"
depends_on:
- extrouter_inside
properties:
pool_id: {get_resource: etcd_pool}
address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
protocol_port: 4001
count: {get_param: number_of_masters}
resource_def:
type: kubemaster.yaml
properties:
ssh_key_name: {get_param: ssh_key_name}
server_image: {get_param: server_image}
master_flavor: {get_param: master_flavor}
external_network: {get_param: external_network}
kube_allow_priv: {get_param: kube_allow_priv}
wait_condition_timeout: {get_param: wait_condition_timeout}
flannel_network_cidr: {get_param: flannel_network_cidr}
flannel_network_subnetlen: {get_param: flannel_network_subnetlen}
flannel_use_vxlan: {get_param: flannel_use_vxlan}
portal_network_cidr: {get_param: portal_network_cidr}
discovery_url: {get_param: discovery_url}
fixed_network: {get_resource: fixed_network}
fixed_subnet: {get_resource: fixed_subnet}
api_pool_id: {get_resource: api_pool}
etcd_pool_id: {get_resource: etcd_pool}
######################################################################
#
@ -380,7 +229,7 @@ resources:
type: "OS::Heat::ResourceGroup"
depends_on:
- extrouter_inside
- master_wait_condition
- kube_masters
properties:
count: {get_param: number_of_minions}
removal_policies: [{resource_list: {get_param: minions_to_remove}}]
@ -411,11 +260,11 @@ outputs:
This is the API endpoint of the Kubernetes server. Use this to access
the Kubernetes API from outside the cluster.
kube_master:
value: {get_attr: [kube_master_floating, floating_ip_address]}
kube_masters:
value: {get_attr: [kube_masters, kube_master_external_ip]}
description: >
This is the "public" ip address of the Kubernetes master server. Use this address to
log in to the Kubernetes master via ssh.
This is a list of "public" ip addresses of all Kubernetes master servers.
Use these addresses to log in to the Kubernetes masters via ssh.
kube_minions:
value: {get_attr: [kube_minions, kube_minion_ip]}

277
magnum/templates/heat-kubernetes/kubemaster.yaml Normal file
View File

@ -0,0 +1,277 @@
heat_template_version: 2013-05-23
description: >
This is a nested stack that defines a single Kubernetes master, This stack is
included by an ResourceGroup resource in the parent template
(kubecluster.yaml).
parameters:
server_image:
type: string
description: glance image used to boot the server
master_flavor:
type: string
default: m1.small
description: flavor to use when booting the server
ssh_key_name:
type: string
description: name of ssh key to be provisioned on our server
default: lars
external_network:
type: string
description: uuid/name of a network to use for floating ip addresses
portal_network_cidr:
type: string
description: >
address range used by kubernetes for service portals
default: 10.254.0.0/16
kube_allow_priv:
type: string
description: >
whether or not kubernetes should permit privileged containers.
default: false
constraints:
- allowed_values: [true, false]
flannel_network_cidr:
type: string
description: network range for flannel overlay network
default: 10.100.0.0/16
flannel_network_subnetlen:
type: string
description: size of subnet assigned to each master
default: 24
flannel_use_vxlan:
type: string
description: >
if true use the vxlan backend, otherwise use the default
udp backend
default: false
constraints:
- allowed_values: [true, false]
discovery_url:
type: string
description: >
Discovery URL used for bootstrapping the etcd cluster.
# The following are all generated in the parent template.
fixed_network:
type: string
description: Network from which to allocate fixed addresses.
fixed_subnet:
type: string
description: Subnet from which to allocate fixed addresses.
wait_condition_timeout:
type: number
description : >
timeout for the Wait Conditions
api_pool_id:
type: string
description: ID of the load balancer pool of k8s API server.
etcd_pool_id:
type: string
description: ID of the load balancer pool of etcd server.
resources:
master_wait_handle:
type: OS::Heat::WaitConditionHandle
master_wait_condition:
type: OS::Heat::WaitCondition
depends_on: kube_master
properties:
handle: {get_resource: master_wait_handle}
timeout: {get_param: wait_condition_timeout}
######################################################################
#
# security groups. we need to permit network traffic of various
# sorts.
#
secgroup_base:
type: OS::Neutron::SecurityGroup
properties:
rules:
- protocol: icmp
- protocol: tcp
port_range_min: 22
port_range_max: 22
secgroup_kubernetes:
type: OS::Neutron::SecurityGroup
properties:
rules:
- protocol: tcp
port_range_min: 7080
port_range_max: 7080
- protocol: tcp
port_range_min: 8080
port_range_max: 8080
- protocol: tcp
port_range_min: 4001
port_range_max: 4001
- protocol: tcp
port_range_min: 7001
port_range_max: 7001
######################################################################
#
# software configs. these are components that are combined into
# a multipart MIME user-data archive.
#
write_heat_params:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config:
str_replace:
template: {get_file: fragments/write-heat-params-master.yaml}
params:
"$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
"$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
"$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
"$FLANNEL_USE_VXLAN": {get_param: flannel_use_vxlan}
"$PORTAL_NETWORK_CIDR": {get_param: portal_network_cidr}
"$ETCD_DISCOVERY_URL": {get_param: discovery_url}
configure_etcd:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/configure-etcd.sh}
configure_kubernetes:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/configure-kubernetes-master.sh}
write_flannel_config:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/write-flannel-config.sh}
flannel_config_service:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/flannel-config.service.yaml}
enable_services:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/enable-services-master.sh}
kube_user:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/kube-user.yaml}
kube_examples:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/kube-examples.yaml}
master_wc_notify:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config:
str_replace:
template: |
#!/bin/bash -v
wc_notify --data-binary '{"status": "SUCCESS"}'
params:
wc_notify: {get_attr: [master_wait_handle, curl_cli]}
disable_selinux:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/disable-selinux.sh}
kube_master_init:
type: OS::Heat::MultipartMime
properties:
parts:
- config: {get_resource: disable_selinux}
- config: {get_resource: write_heat_params}
- config: {get_resource: configure_etcd}
- config: {get_resource: kube_user}
- config: {get_resource: configure_kubernetes}
- config: {get_resource: enable_services}
- config: {get_resource: write_flannel_config}
- config: {get_resource: flannel_config_service}
- config: {get_resource: kube_examples}
- config: {get_resource: master_wc_notify}
######################################################################
#
# a single kubernetes master.
#
kube_master:
type: OS::Nova::Server
properties:
image: {get_param: server_image}
flavor: {get_param: master_flavor}
key_name: {get_param: ssh_key_name}
user_data_format: RAW
user_data: {get_resource: kube_master_init}
networks:
- port: {get_resource: kube_master_eth0}
kube_master_eth0:
type: OS::Neutron::Port
properties:
network: {get_param: fixed_network}
security_groups:
- {get_resource: secgroup_base}
- {get_resource: secgroup_kubernetes}
fixed_ips:
- subnet: {get_param: fixed_subnet}
replacement_policy: AUTO
kube_master_floating:
type: OS::Neutron::FloatingIP
properties:
floating_network: {get_param: external_network}
port_id: {get_resource: kube_master_eth0}
api_pool_member:
type: OS::Neutron::PoolMember
properties:
pool_id: {get_param: api_pool_id}
address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
protocol_port: 8080
etcd_pool_member:
type: OS::Neutron::PoolMember
properties:
pool_id: {get_param: etcd_pool_id}
address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
protocol_port: 4001
outputs:
kube_master_ip:
value: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
kube_master_external_ip:
value: {get_attr: [kube_master_floating, floating_ip_address]}