[app][calico][kubernetes] Enable calico networking in k8s application

- binaries and scripts updated to use calico networking

Co-Authored-By: ddovbii <ddovbii@mirantis.com>
Change-Id: I3187b9cc171aa15caa4912a665c642ab057b4eae

Kubernetes/KubernetesCluster/package/Classes/KubernetesCluster.yaml

@@ -95,6 +95,18 @@ Methods:
                 FromPort: 7001
                 IpProtocol: tcp
                 External: false
+              - ToPort: 2379
+                FromPort: 2379
+                IpProtocol: tcp
+                External: false
+              - ToPort: 179
+                FromPort: 179
+                IpProtocol: tcp
+                External: false
+              - ToPort: 179
+                FromPort: 179
+                IpProtocol: udp
+                External: false
               - ToPort: 10250
                 FromPort: 10250
                 IpProtocol: tcp
@@ -132,12 +144,17 @@ Methods:
           - $.minionNodes.take($.nodeCount).select($.setupEtcd())
           - $.gatewayNodes.take($.gatewayCount).select($.setupEtcd())
 
+          - $.masterNode.setupFlannel()
           - $.masterNode.setupNode()
 
           - Parallel:
               - $.minionNodes.take($.nodeCount).pselect($.setupNode())
               - $.gatewayNodes.take($.gatewayCount).pselect($.setupNode())
 
+          - Parallel:
+              - $.minionNodes.take($.nodeCount).pselect($.setupFlannel())
+              - $.gatewayNodes.take($.gatewayCount).pselect($.setupFlannel())
+
           - $._environment.stack.push()
           - $._updateServicePublicIps()
           - $.setAttr(lastNodeCount, $.nodeCount)

Kubernetes/KubernetesCluster/package/Classes/KubernetesGatewayNode.yaml

@@ -55,15 +55,27 @@ Methods:
           - $.setAttr(etcdConfigured, true)
 
 
+  setupCalico:
+    Body:
+      - $._environment.reporter.report($this, 'Setup Calico network on gateway node {0}'.format($.instance.name))
+      - $resources: new(sys:Resources)
+      - $template: $resources.yaml('SetupCalicoNode.template').bind(dict(ip => $.getIp()))
+      - $.instance.agent.call($template, $resources)
+
+
+  setupFlannel:
+    Body:
+      - $._environment.reporter.report($this, 'Setup Flannel network on gateway node {0}'.format($.instance.name))
+      - $resources: new(sys:Resources)
+      - $template: $resources.yaml('SetupFlannelNode.template')
+      - $.instance.agent.call($template, $resources)
+
+
   setupNode:
     Body:
       - If: not $.getAttr(nodeConfigured, false)
         Then:
-          - $._environment.reporter.report($this, 'Setup Flannel network on {0}'.format($.instance.name))
           - $resources: new(sys:Resources)
-          - $template: $resources.yaml('SetupFlannelNode.template')
-          - $.instance.agent.call($template, $resources)
-
           - $._environment.reporter.report($, 'Setup Load Balancer on {0}'.format($.instance.name))
           - $template: $resources.yaml('HAProxySetup.template').bind(dict(
                 masterIp => $._cluster.masterNode.getIp()

Kubernetes/KubernetesCluster/package/Classes/KubernetesMasterNode.yaml

@@ -49,15 +49,27 @@ Methods:
           - $.setAttr(etcdConfigured, true)
 
 
+  setupCalico:
+    Body:
+      - $._environment.reporter.report($this, 'Setup Calico network on master node')
+      - $resources: new(sys:Resources)
+      - $template: $resources.yaml('SetupCalicoMaster.template').bind(dict(ip => $.getIp()))
+      - $.instance.agent.call($template, $resources)
+
+
+  setupFlannel:
+    Body:
+      - $._environment.reporter.report($this, 'Setup Flannel network on master node')
+      - $resources: new(sys:Resources)
+      - $template: $resources.yaml('SetupFlannelMaster.template')
+      - $.instance.agent.call($template, $resources)
+
+
   setupNode:
     Body:
       - If: not $.getAttr(nodeConfigured, false)
         Then:
           - $resources: new(sys:Resources)
-          - $._environment.reporter.report($this, 'Setup Flannel network on master node')
-          - $template: $resources.yaml('SetupFlannelMaster.template')
-          - $.instance.agent.call($template, $resources)
-
           - $._environment.reporter.report($this, 'Setup Kubernetes API host')
           - $resources: new(sys:Resources)
           - $template: $resources.yaml('KubeMasterSetup.template').bind(dict(

Kubernetes/KubernetesCluster/package/Classes/KubernetesMinionNode.yaml

@@ -63,15 +63,27 @@ Methods:
           - $.setAttr(etcdConfigured, true)
 
 
+  setupCalico:
+    Body:
+      - $._environment.reporter.report($this, 'Setup Calico network on node {0}'.format($.instance.name))
+      - $resources: new(sys:Resources)
+      - $template: $resources.yaml('SetupCalicoNode.template').bind(dict(ip => $.getIp()))
+      - $.instance.agent.call($template, $resources)
+
+
+  setupFlannel:
+    Body:
+      - $._environment.reporter.report($this, 'Setup Flannel network on node {0}'.format($.instance.name))
+      - $resources: new(sys:Resources)
+      - $template: $resources.yaml('SetupFlannelNode.template')
+      - $.instance.agent.call($template, $resources)
+
+
   setupNode:
     Body:
       - If: not $.getAttr(nodeConfigured, false)
         Then:
-          - $._environment.reporter.report($this, 'Setup Flannel network on {0}'.format($.instance.name))
           - $resources: new(sys:Resources)
-          - $template: $resources.yaml('SetupFlannelNode.template')
-          - $.instance.agent.call($template, $resources)
-
           - $securityGroupIngress:
               - ToPort: 4194
                 FromPort: 4194

Kubernetes/KubernetesCluster/package/Classes/KubernetesNode.yaml

@@ -44,6 +44,8 @@ Methods:
       - $.instance.deploy()
 
 
+  setupFlannel:
+  setupCalico:
   setupEtcd:
   setupNode:
   removeFromCluster:

Kubernetes/KubernetesCluster/package/Resources/SetupCalicoMaster.template

@@ -0,0 +1,33 @@
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+
+FormatVersion: 2.0.0
+Version: 1.0.0
+Name: Setup Calico on Master
+
+Parameters:
+  ip: $ip
+
+Body: |
+  return setupCalico('{0}'.format(args.ip)).stdout
+
+Scripts:
+  setupCalico:
+    Type: Application
+    Version: 1.0.0
+    EntryPoint: setupCalicoMaster.sh
+    Files:
+      - systemd/calico-node.service
+      - environ/network-environment
+    Options:
+      captureStdout: true
+      captureStderr: true

Kubernetes/KubernetesCluster/package/Resources/SetupCalicoNode.template

@@ -0,0 +1,34 @@
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+
+FormatVersion: 2.0.0
+Version: 1.0.0
+Name: Setup Calico on Node
+
+Parameters:
+  ip: $ip
+
+Body: |
+  return setupCalico('{0}'.format(args.ip)).stdout
+
+Scripts:
+  setupCalico:
+    Type: Application
+    Version: 1.0.0
+    EntryPoint: setupCalicoNode.sh
+    Files:
+      - systemd/calico-node.service
+      - environ/network-environment
+      - 10-calico.conf
+    Options:
+      captureStdout: true
+      captureStderr: true

Kubernetes/KubernetesCluster/package/Resources/scripts/10-calico.conf

@@ -0,0 +1,9 @@
+{
+    "name": "calico-k8s-network",
+    "type": "calico",
+    "etcd_authority": "%%MASTER_IP%%:2379",
+    "log_level": "info",
+    "ipam": {
+        "type": "calico-ipam"
+    }
+}

Kubernetes/KubernetesCluster/package/Resources/scripts/environ/etcd

@@ -4,9 +4,9 @@
 
 NAME="-name %%NAME%%"
 LISTEN_PEER_URLS="-listen-peer-urls http://%%IP%%:7001,http://127.0.0.1:7001"
-LISTEN_CLIENT_URLS="-listen-client-urls http://%%IP%%:4001,http://127.0.0.1:4001"
+LISTEN_CLIENT_URLS="-listen-client-urls http://%%IP%%:4001,http://127.0.0.1:4001,http://%%IP%%:2379,http://127.0.0.1:2379"
 INITIAL_ADV_PEER_URLS="-initial-advertise-peer-urls http://%%IP%%:7001"
 INITIAL_CLUSTER_STATE="-initial-cluster-state %%STATE%%"
 INITIAL_CLUSTER="-initial-cluster %%CLUSTER_CONFIG%%"
-ADV_CLIENT_URLS="-advertise-client-urls http://%%IP%%:4001,http://127.0.0.1:4001"
+ADV_CLIENT_URLS="-advertise-client-urls http://%%IP%%:4001,http://127.0.0.1:4001,http://%%IP%%:2379,http://127.0.0.1:2379"
 ETCD_OPTS="-snapshot-count 1000"

Kubernetes/KubernetesCluster/package/Resources/scripts/environ/kubelet

@@ -13,5 +13,3 @@ KUBELET_HOSTNAME="--hostname-override=%%IP%%"
 # location of the api-server
 KUBELET_API_SERVER="--api-servers=http://%%MASTER_IP%%:8080"
 
-# Add your own!
-KUBELET_ARGS=""

Kubernetes/KubernetesCluster/package/Resources/scripts/environ/network-environment

@@ -0,0 +1,6 @@
+# This host's IPv4 address (the source IP address used to reach other nodes
+# in the Kubernetes cluster).
+DEFAULT_IPV4=%%IP%%
+
+# IP and port of etcd instance used by Calico
+ETCD_AUTHORITY=%%MASTER_IP%%:2379

Kubernetes/KubernetesCluster/package/Resources/scripts/master-kube-setup.sh

@@ -61,6 +61,8 @@ else
   service kube-controller-manager start
 fi
 
+ln -s /opt/bin/hyperkube /opt/bin/kubectl
+
 mkdir /var/log/kubernetes
 /opt/bin/kubectl delete node 127.0.0.1
 sleep 1

Kubernetes/KubernetesCluster/package/Resources/scripts/setupCalicoMaster.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# $1 - IP
+
+# TODO(asilenlov): we need to refactor this script
+
+# Install Calico on master
+mkdir -p /opt/cni/bin
+cp /opt/copy/cni/bin/* /opt/cni/bin/
+ln -s /opt/bin/calicoctl /usr/bin/calicoctl
+docker load < /opt/copy/calico-node.tar
+
+sed -i.bak "s/%%MASTER_IP%%/$1/g" environ/network-environment
+sed -i.bak "s/%%IP%%/$1/g" environ/network-environment
+cp -f environ/network-environment /etc
+
+sed -i.bak "s/%%IP%%/$1/g" systemd/calico-node.service
+cp -f systemd/calico-node.service /etc/systemd/system/
+systemctl enable calico-node.service
+
+systemctl start calico-node

Kubernetes/KubernetesCluster/package/Resources/scripts/setupCalicoNode.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# $1 - IP
+
+# TODO(asilenlov): we need to refactor this script
+
+# Install Calico on worker
+mkdir -p /opt/cni/bin
+cp /opt/copy/cni/bin/* /opt/cni/bin/
+ln -s /opt/bin/calicoctl /usr/bin/calicoctl
+docker load < /opt/copy/calico-node.tar
+
+sed -i.bak "s/%%MASTER_IP%%/$1/g" environ/network-environment
+sed -i.bak "s/%%IP%%/$1/g" environ/network-environment
+cp -f environ/network-environment /etc
+
+sed -i.bak "s/%%IP%%/$1/g" systemd/calico-node.service
+cp -f systemd/calico-node.service /etc/systemd/system/
+systemctl enable calico-node.service
+
+mkdir -p /etc/cni/net.d
+sed -i.bak "s/%%MASTER_IP%%/$1/g" 10-calico.conf
+cp -f 10-calico.conf /etc/cni/net.d
+
+systemctl start calico-node

Kubernetes/KubernetesCluster/package/Resources/scripts/setupFlannelMaster.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+
 count=30
 
 echo "Adding flannel configuration to etcd"

Kubernetes/KubernetesCluster/package/Resources/scripts/systemd/calico-node.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Calico per-node agent
+Documentation=https://github.com/projectcalico/calico-docker
+Requires=docker.service
+After=docker.service
+
+[Service]
+User=root
+EnvironmentFile=/etc/network-environment
+PermissionsStartOnly=true
+ExecStart=/usr/bin/calicoctl node --ip=%%IP%% --detach=false
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target