Fix CoreOS multi master with LB cluster creation

Cluster that uses ETCD like swarm and K8s failed with LB and TLS enable
because ETCD LB protocol is HTTP but SSL termination in on the ETCD
node. ETCD LB protocol should be the same as K8s with TLS enable

Partial-Bug: #1679724

Change-Id: Ie8c8a7e4609c0e2e63095d4c18af84cc653654e1

magnum/drivers/k8s_coreos_v1/templates/fragments/enable-network-service.yaml

@@ -29,29 +29,13 @@ write_files:
         KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
       fi
 
-      ETCD_SERVER_IP=${ETCD_SERVER_IP:-127.0.0.1}
-
-      PROTOCOL=https
-
-      if [ "$TLS_DISABLED" = "True" ]; then
-        PROTOCOL=http
-      fi
-
       ENV_FILE=/etc/flannel/options.env
       mkdir -p $(dirname $ENV_FILE)
       cat > $ENV_FILE <<EOF
       FLANNELD_IFACE=${KUBE_NODE_IP}
-      FLANNELD_ETCD_ENDPOINTS=${PROTOCOL}://${ETCD_SERVER_IP}:2379
+      FLANNELD_ETCD_ENDPOINTS=http://127.0.0.1:2379
       EOF
 
-      if [ "$TLS_DISABLED" = "False" ]; then
-        cat >> $ENV_FILE <<EOF
-      FLANNELD_ETCD_CAFILE=${KUBE_CERTS_PATH}/ca.pem
-      FLANNELD_ETCD_CERTFILE=${KUBE_CERTS_PATH}/apiserver.pem
-      FLANNELD_ETCD_KEYFILE=${KUBE_CERTS_PATH}/apiserver-key.pem
-      EOF
-      fi
-
       DROP_IN_FILE=/etc/systemd/system/flanneld.service.d/40-ExecStartPre-symlink.conf
       mkdir -p $(dirname $DROP_IN_FILE)
       cat > $DROP_IN_FILE <<EOF

magnum/drivers/k8s_coreos_v1/templates/fragments/make-cert.yaml

@@ -67,6 +67,10 @@ write_files:
 
       sans="${sans},IP:${KUBE_SERVICE_IP}"
 
+      if [[ -n "${ETCD_LB_VIP}" ]]; then
+        sans="${sans},IP:${ETCD_LB_VIP}"
+      fi
+
       cert_conf_dir=${KUBE_CERTS_PATH}/conf
 
       mkdir -p ${cert_conf_dir}

magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params-master.yaml

@@ -11,7 +11,6 @@ write_files:
       KUBE_NODE_PUBLIC_IP="$KUBE_NODE_PUBLIC_IP"
       KUBE_NODE_IP="$KUBE_NODE_IP"
       KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
-      ETCD_SERVER_IP="$ETCD_SERVER_IP"
       DOCKER_VOLUME="$DOCKER_VOLUME"
       DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER"
       NETWORK_DRIVER="$NETWORK_DRIVER"
@@ -44,3 +43,4 @@ write_files:
       HOST_CERTS_PATH="$HOST_CERTS_PATH"
       HYPERKUBE_IMAGE_REPO="$HYPERKUBE_IMAGE_REPO"
       CONTAINER_RUNTIME="$CONTAINER_RUNTIME"
+      ETCD_LB_VIP="$ETCD_LB_VIP"

magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml

@@ -296,7 +296,7 @@ resources:
     properties:
       fixed_subnet: {get_attr: [network, fixed_subnet]}
       external_network: {get_param: external_network}
-      protocol: HTTP
+      protocol: {get_param: loadbalancing_protocol}
       port: 2379
 
  ######################################################################
@@ -423,6 +423,7 @@ resources:
           container_runtime: {get_param: container_runtime}
           prometheus_monitoring: {get_param: prometheus_monitoring}
           grafana_admin_passwd: {get_param: grafana_admin_passwd}
+          etcd_lb_vip: {get_attr: [etcd_lb, address]}
 
   ######################################################################
   #

magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml

@@ -194,6 +194,12 @@ parameters:
     description: >
       Container runtime to use with Kubernetes.
 
+  etcd_lb_vip:
+    type: string
+    description: >
+      etcd lb vip private used to generate certs on master.
+    default: ""
+
 resources:
 
   master_wait_handle:
@@ -239,7 +245,6 @@ resources:
             "$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_master_floating, floating_ip_address]}
             "$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
             "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
-            "$ETCD_SERVER_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
             "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
             "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
             "$FLANNEL_BACKEND": {get_param: flannel_backend}
@@ -273,6 +278,7 @@ resources:
                   hyperkube_image: { get_param: hyperkube_image }
             "$INSECURE_REGISTRY_URL": {get_param: insecure_registry_url}
             "$CONTAINER_RUNTIME": {get_param: container_runtime}
+            "$ETCD_LB_VIP": {get_param: etcd_lb_vip}
 
   configure_etcd:
     type: OS::Heat::SoftwareConfig