Support HA for k8s coreos bay

Following things has been added to make core OS driver HA
  1) Created two pools for API and etcd.
  2) Added health monitoring for both API and etcd.
  3) Register masters into the pool created at step 1 to
     balance load among them.

Address switcher has been added to
  1) If LBaaS is not enabled(1 master) then master IPs are
     exposed.
  2) If LBaaS is enabled(more than 1 master) then LBaaS IPs
     are exposed.

Co-Author-By: Hongbin Lu <hongbin.lu@huawei.com>

Change-Id: I96391076f17bdb7161455cea7732c0d85cb72fe0
Closes-bug: #1580220

magnum/drivers/k8s_coreos_v1/template_def.py

@@ -127,6 +127,12 @@ class CoreOSK8sTemplateDefinition(K8sTemplateDefinition):
          'coe': 'kubernetes'},
     ]
 
+    def get_env_files(self, baymodel):
+        if baymodel.master_lb_enabled:
+            return ['../../common/templates/environments/with_master_lb.yaml']
+        else:
+            return ['../../common/templates/environments/no_master_lb.yaml']
+
     @property
     def template_path(self):
         return os.path.join(os.path.dirname(os.path.realpath(__file__)),

magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml

@@ -221,19 +221,90 @@ resources:
 
   ######################################################################
   #
-  # kubernetes masters. This is a resource group that will create
-  # 1 master.
+  # load balancers.
   #
 
-  kube_master:
+  api_monitor:
+    type: Magnum::Optional::Neutron::Pool::HealthMonitor
+    properties:
+      type: TCP
+      delay: 5
+      max_retries: 5
+      timeout: 5
+
+  api_pool:
+    type: Magnum::Optional::Neutron::Pool
+    properties:
+      protocol: {get_param: loadbalancing_protocol}
+      monitors: [{get_resource: api_monitor}]
+      subnet: {get_resource: fixed_subnet}
+      lb_method: ROUND_ROBIN
+      vip:
+        protocol_port: {get_param: kubernetes_port}
+
+  api_pool_floating:
+    type: Magnum::Optional::Neutron::Pool::FloatingIP
+    depends_on:
+      - extrouter_inside
+    properties:
+      floating_network: {get_param: external_network}
+      port_id: {get_attr: [api_pool, vip, port_id]}
+
+  etcd_monitor:
+    type: Magnum::Optional::Neutron::Pool::HealthMonitor
+    properties:
+      type: TCP
+      delay: 5
+      max_retries: 5
+      timeout: 5
+
+  etcd_pool:
+    type: Magnum::Optional::Neutron::Pool
+    properties:
+      protocol: HTTP
+      monitors: [{get_resource: etcd_monitor}]
+      subnet: {get_resource: fixed_subnet}
+      lb_method: ROUND_ROBIN
+      vip:
+        protocol_port: 2379
+
+  ######################################################################
+  #
+  # resources that expose the IPs of either the kube master or a given
+  # LBaaS pool depending on whether LBaaS is enabled for the bay.
+  #
+
+  api_address_switch:
+    type: Magnum::ApiGatewaySwitcher
+    properties:
+      pool_public_ip: {get_attr: [api_pool_floating, floating_ip_address]}
+      pool_private_ip: {get_attr: [api_pool, vip, address]}
+      master_public_ip: {get_attr: [kube_masters, resource.0.kube_master_external_ip]}
+      master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]}
+
+  etcd_address_switch:
+    type: Magnum::ApiGatewaySwitcher
+    properties:
+      pool_private_ip: {get_attr: [etcd_pool, vip, address]}
+      master_private_ip: {get_attr: [kube_masters, resource.0.kube_master_ip]}
+
+  ######################################################################
+  #
+  # kubernetes masters. This is a resource group that will create
+  # <number_of_masters> master.
+  #
+
+  kube_masters:
     type: OS::Heat::ResourceGroup
     depends_on:
       - extrouter_inside
     properties:
-      count: 1
+      count: {get_param: number_of_masters}
       resource_def:
         type: kubemaster.yaml
         properties:
+          api_public_address: {get_attr: [api_pool_floating, floating_ip_address]}
+          api_private_address: {get_attr: [api_pool, vip, address]}
           ssh_key_name: {get_param: ssh_key_name}
           server_image: {get_param: server_image}
           master_flavor: {get_param: master_flavor}
@@ -252,6 +323,8 @@ resources:
           kube_version: {get_param: kube_version}
           wait_condition_timeout: {get_param: wait_condition_timeout}
           bay_uuid: {get_param: bay_uuid}
+          api_pool_id: {get_resource: api_pool}
+          etcd_pool_id: {get_resource: etcd_pool}
           magnum_url: {get_param: magnum_url}
           trustee_user_id: {get_param: trustee_user_id}
           trustee_password: {get_param: trustee_password}
@@ -268,7 +341,7 @@ resources:
     type: OS::Heat::ResourceGroup
     depends_on:
       - extrouter_inside
-      - kube_master
+      - kube_masters
     properties:
       count: {get_param: number_of_minions}
       removal_policies: [{resource_list: {get_param: minions_to_remove}}]
@@ -281,14 +354,14 @@ resources:
           fixed_network: {get_resource: fixed_network}
           fixed_subnet: {get_resource: fixed_subnet}
           flannel_network_cidr: {get_param: flannel_network_cidr}
-          kube_master_ip: {"Fn::Select": [0, {get_attr: [kube_master, kube_master_ip]}]}
+          kube_master_ip: {get_attr: [api_address_switch, private_ip]}
           external_network: {get_param: external_network}
           kube_allow_priv: {get_param: kube_allow_priv}
           network_driver: {get_param: network_driver}
           kubernetes_port: {get_param: kubernetes_port}
           tls_disabled: {get_param: tls_disabled}
           kube_version: {get_param: kube_version}
-          etcd_server_ip: {"Fn::Select": [0, {get_attr: [kube_master, kube_master_ip]}]}
+          etcd_server_ip: {get_attr: [etcd_address_switch, private_ip]}
           wait_condition_timeout: {get_param: wait_condition_timeout}
           bay_uuid: {get_param: bay_uuid}
           magnum_url: {get_param: magnum_url}
@@ -300,18 +373,22 @@ resources:
 outputs:
 
   api_address:
-    value: {"Fn::Select": [0, {get_attr: [kube_master, kube_master_external_ip]}]}
+    value:
+      str_replace:
+        template: api_ip_address
+        params:
+          api_ip_address: {get_attr: [api_address_switch, public_ip]}
     description: >
       This is the API endpoint of the Kubernetes cluster. Use this to access
       the Kubernetes API.
 
   kube_masters_private:
-    value: {get_attr: [kube_master, kube_master_ip]}
+    value: {get_attr: [kube_masters, kube_master_ip]}
     description: >
       This is a list of the "private" IP addresses of all the Kubernetes masters.
 
   kube_masters:
-    value: {get_attr: [kube_master, kube_master_external_ip]}
+    value: {get_attr: [kube_masters, kube_master_external_ip]}
     description: >
       This is a list of the "public" IP addresses of all the Kubernetes masters.
       Use these IP addresses to log in to the Kubernetes masters via ssh or to access

magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml

@@ -28,6 +28,14 @@ parameters:
     description: >
       Discovery URL used for bootstrapping the etcd cluster.
 
+  api_pool_id:
+    type: string
+    description: ID of the load balancer pool of k8s API server.
+
+  etcd_pool_id:
+    type: string
+    description: ID of the load balancer pool of etcd server.
+
   portal_network_cidr:
     type: string
     description: >
@@ -95,6 +103,16 @@ parameters:
     type: string
     description: endpoint to retrieve TLS certs from
 
+  api_public_address:
+    type: string
+    description: Public IP address of the Kubernetes master server.
+    default: ""
+
+  api_private_address:
+    type: string
+    description: Private IP address of the Kubernetes master server.
+    default: ""
+
   trustee_user_id:
     type: string
     description: user id of the trustee
@@ -128,6 +146,20 @@ resources:
       handle: {get_resource: master_wait_handle}
       timeout: {get_param: wait_condition_timeout}
 
+  ######################################################################
+  #
+  # resource that exposes the IPs of either the kube master or the API
+  # LBaaS pool depending on whether LBaaS is enabled for the bay.
+  #
+
+  api_address_switch:
+    type: Magnum::ApiGatewaySwitcher
+    properties:
+      pool_public_ip: {get_param: api_public_address}
+      pool_private_ip: {get_param: api_private_address}
+      master_public_ip: {get_attr: [kube_master_floating, floating_ip_address]}
+      master_private_ip: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
+
   ######################################################################
   #
   # security groups.  we need to permit network traffic of various
@@ -177,6 +209,8 @@ resources:
         str_replace:
           template: {get_file: fragments/write-heat-params-master.yaml}
           params:
+            "$KUBE_API_PUBLIC_ADDRESS": {get_attr: [api_address_switch, public_ip]}
+            "$KUBE_API_PRIVATE_ADDRESS": {get_attr: [api_address_switch, private_ip]}
             "$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_master_floating, floating_ip_address]}
             "$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
             "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
@@ -366,6 +400,20 @@ resources:
       floating_network: {get_param: external_network}
       port_id: {get_resource: kube_master_eth0}
 
+  api_pool_member:
+    type: Magnum::Optional::Neutron::PoolMember
+    properties:
+      pool_id: {get_param: api_pool_id}
+      address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
+      protocol_port: {get_param: kubernetes_port}
+
+  etcd_pool_member:
+    type: Magnum::Optional::Neutron::PoolMember
+    properties:
+      pool_id: {get_param: etcd_pool_id}
+      address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
+      protocol_port: 2379
+
 outputs:
 
   kube_master_ip:

magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py

@@ -301,7 +301,9 @@ class TestBayConductorWithK8s(base.TestCase):
             'insecure_registry_url': '10.0.0.1:5000',
         }
         self.assertEqual(expected, definition)
-        self.assertEqual([], env_files)
+        self.assertEqual(
+            ['../../common/templates/environments/no_master_lb.yaml'],
+            env_files)
 
     @patch('requests.get')
     @patch('magnum.objects.BayModel.get_by_uuid')
@@ -353,7 +355,9 @@ class TestBayConductorWithK8s(base.TestCase):
             'insecure_registry_url': '10.0.0.1:5000',
         }
         self.assertEqual(expected, definition)
-        self.assertEqual([], env_files)
+        self.assertEqual(
+            ['../../common/templates/environments/no_master_lb.yaml'],
+            env_files)
 
     @patch('requests.get')
     @patch('magnum.objects.BayModel.get_by_uuid')