From f49c8d68341747e5b6636bcf9c8aeb1ded40301e Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Thu, 6 Jul 2017 10:00:24 -0500 Subject: [PATCH 1/9] Use a separate etcd cluster for calico --- example/vagrant-input-config.yaml | 2 +- promenade/generator.py | 101 ++++++++- .../kubernetes/asset-loader/cni/calico.yaml | 16 +- .../auxiliary-calico-etcd-0/pki/client-ca.pem | 1 + .../pki/etcd-client-key.pem | 1 + .../pki/etcd-client.pem | 1 + .../pki/etcd-peer-key.pem | 1 + .../auxiliary-calico-etcd-0/pki/etcd-peer.pem | 1 + .../auxiliary-calico-etcd-0/pki/peer-ca.pem | 1 + .../auxiliary-calico-etcd-1/pki/client-ca.pem | 1 + .../pki/etcd-client-key.pem | 1 + .../pki/etcd-client.pem | 1 + .../pki/etcd-peer-key.pem | 1 + .../auxiliary-calico-etcd-1/pki/etcd-peer.pem | 1 + .../auxiliary-calico-etcd-1/pki/peer-ca.pem | 1 + .../manifests/auxiliary-calico-etcd.yaml | 194 ++++++++++++++++++ .../kubelet/manifests/auxiliary-etcd.yaml | 8 +- .../kubernetes/calico-etcd/pki/client-ca.pem | 1 + .../calico-etcd/pki/etcd-client-key.pem | 1 + .../calico-etcd/pki/etcd-client.pem | 1 + .../calico-etcd/pki/etcd-peer-key.pem | 1 + .../kubernetes/calico-etcd/pki/etcd-peer.pem | 1 + .../kubernetes/calico-etcd/pki/peer-ca.pem | 1 + .../kubelet/manifests/calico-etcd.yaml | 68 ++++++ .../kubelet/manifests/kube-etcd.yaml | 4 +- 25 files changed, 386 insertions(+), 25 deletions(-) create mode 100644 promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/client-ca.pem create mode 100644 promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client-key.pem create mode 100644 promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client.pem create mode 100644 promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer-key.pem create mode 100644 promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer.pem create mode 100644 promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/peer-ca.pem create mode 100644 promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/client-ca.pem create mode 100644 promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client-key.pem create mode 100644 promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client.pem create mode 100644 promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer-key.pem create mode 100644 promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer.pem create mode 100644 promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/peer-ca.pem create mode 100644 promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml create mode 100644 promenade/templates/master/etc/kubernetes/calico-etcd/pki/client-ca.pem create mode 100644 promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-client-key.pem create mode 100644 promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-client.pem create mode 100644 promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-peer-key.pem create mode 100644 promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-peer.pem create mode 100644 promenade/templates/master/etc/kubernetes/calico-etcd/pki/peer-ca.pem create mode 100644 promenade/templates/master/etc/kubernetes/kubelet/manifests/calico-etcd.yaml diff --git a/example/vagrant-input-config.yaml b/example/vagrant-input-config.yaml index f7794f7a..e6ca0c93 100644 --- a/example/vagrant-input-config.yaml +++ b/example/vagrant-input-config.yaml @@ -44,7 +44,7 @@ spec: kube_service_ip: 10.96.0.1 pod_ip_cidr: 10.97.0.0/16 service_ip_cidr: 10.96.0.0/16 - etcd_service_ip: 10.96.232.136 + calico_etcd_service_ip: 10.96.232.136 dns_servers: - 8.8.8.8 - 8.8.4.4 diff --git a/promenade/generator.py b/promenade/generator.py index fd8d0d90..e6e8cf4f 100644 --- a/promenade/generator.py +++ b/promenade/generator.py @@ -52,6 +52,14 @@ class Generator: ca_name='etcd-peer', cert_target='all', key_target='masters') + calico_etcd_client_ca, calico_etcd_client_ca_key = keys.generate_ca( + ca_name='calico-etcd-client', + cert_target='all', + key_target='masters') + calico_etcd_peer_ca, calico_etcd_peer_ca_key = keys.generate_ca( + ca_name='calico-etcd-peer', + cert_target='all', + key_target='masters') admin_cert, admin_cert_key = keys.generate_certificate( name='admin', @@ -68,19 +76,27 @@ class Generator: config.Configuration([ admin_cert, admin_cert_key, + calico_etcd_client_ca, + calico_etcd_client_ca_key, + calico_etcd_peer_ca, + calico_etcd_peer_ca_key, cluster_ca, cluster_ca_key, etcd_client_ca, etcd_client_ca_key, etcd_peer_ca, etcd_peer_ca_key, - sa_pub, sa_priv, + sa_pub, ]).write(os.path.join(output_dir, 'admin-bundle.yaml')) complete_configuration = [ admin_cert, admin_cert_key, + calico_etcd_client_ca, + calico_etcd_client_ca_key, + calico_etcd_peer_ca, + calico_etcd_peer_ca_key, cluster_ca, cluster_ca_key, etcd_client_ca, @@ -89,8 +105,8 @@ class Generator: etcd_peer_ca_key, masters, network, - sa_pub, sa_priv, + sa_pub, ] for hostname, data in cluster['nodes'].items(): @@ -149,6 +165,8 @@ class Generator: role_specific_documents.extend([ admin_cert, admin_cert_key, + calico_etcd_client_ca, + calico_etcd_peer_ca, cluster_ca_key, etcd_client_ca, etcd_peer_ca, @@ -158,8 +176,12 @@ class Generator: if 'genesis' not in data.get('roles', []): etcd_config = _master_etcd_config( cluster_name, genesis_hostname, hostname, masters) + calico_etcd_config = _master_calico_etcd_config( + cluster_name, genesis_hostname, hostname, masters) complete_configuration.append(etcd_config) + complete_configuration.append(calico_etcd_config) role_specific_documents.append(etcd_config) + role_specific_documents.append(calico_etcd_config) master_documents = _master_config(hostname, data, masters, network, keys) complete_configuration.extend(master_documents) @@ -169,6 +191,7 @@ class Generator: role_specific_documents.extend(_genesis_config(hostname, data, masters, network, keys)) role_specific_documents.append(_genesis_etcd_config(cluster_name, hostname)) + role_specific_documents.append(_genesis_calico_etcd_config(cluster_name, hostname)) node.data['spec']['is_genesis'] = True c = config.Configuration(common_documents + role_specific_documents) @@ -205,8 +228,23 @@ def _master_etcd_config(cluster_name, genesis_hostname, hostname, masters): 'auxiliary-etcd-0=https://%s:12380' % genesis_hostname, 'auxiliary-etcd-1=https://%s:22380' % genesis_hostname, ]) - return _etcd_config(cluster_name, alias='master-etcd', - name='master-etcd:%s' % hostname, + return _etcd_config(cluster_name, alias='kube-etcd', + name='master-kube-etcd:%s' % hostname, + target=hostname, + initial_cluster=initial_cluster, + initial_cluster_state='existing') + + +def _master_calico_etcd_config(cluster_name, genesis_hostname, hostname, masters): + initial_cluster = ['%s=https://%s:6667' % (m['hostname'], + m['hostname']) + for m in masters['nodes']] + initial_cluster.extend([ + 'auxiliary-calico-etcd-0=https://%s:16667' % genesis_hostname, + 'auxiliary-calico-etcd-1=https://%s:26667' % genesis_hostname, + ]) + return _etcd_config(cluster_name, alias='calico-etcd', + name='master-calico-etcd:%s' % hostname, target=hostname, initial_cluster=initial_cluster, initial_cluster_state='existing') @@ -218,8 +256,21 @@ def _genesis_etcd_config(cluster_name, hostname): 'auxiliary-etcd-0=https://%s:12380' % hostname, 'auxiliary-etcd-1=https://%s:22380' % hostname, ] - return _etcd_config(cluster_name, alias='genesis-etcd', - name='master-etcd:%s' % hostname, + return _etcd_config(cluster_name, alias='kube-etcd', + name='master-kube-etcd:%s' % hostname, + target=hostname, + initial_cluster=initial_cluster, + initial_cluster_state='new') + + +def _genesis_calico_etcd_config(cluster_name, hostname): + initial_cluster = [ + '%s=https://%s:6667' % (hostname, hostname), + 'auxiliary-calico-etcd-0=https://%s:16667' % hostname, + 'auxiliary-calico-etcd-1=https://%s:26667' % hostname, + ] + return _etcd_config(cluster_name, alias='calico-etcd', + name='master-calico-etcd:%s' % hostname, target=hostname, initial_cluster=initial_cluster, initial_cluster_state='new') @@ -256,7 +307,7 @@ def _master_config(hostname, host_data, masters, network, keys): 'calico-etcd.kube-system', 'calico-etcd.kube-system.svc', 'calico-etcd.kube-system.svc.cluster.local', - network['etcd_service_ip'], + network['calico_etcd_service_ip'], ] docs = [] @@ -284,6 +335,22 @@ def _master_config(hostname, host_data, masters, network, keys): target=hostname, )) + docs.extend(keys.generate_certificate( + alias='calico-etcd-client', + name='calico-etcd:client:%s' % hostname, + ca_name='calico-etcd-client', + hosts=kube_domains + calico_domains + [hostname, host_data['ip']], + target=hostname, + )) + + docs.extend(keys.generate_certificate( + alias='calico-etcd-peer', + name='calico-etcd:peer:%s' % hostname, + ca_name='calico-etcd-peer', + hosts=kube_domains + [hostname, host_data['ip']], + target=hostname, + )) + docs.extend(keys.generate_certificate( alias='apiserver', name='apiserver:%s' % hostname, @@ -341,10 +408,24 @@ def _genesis_config(hostname, host_data, masters, network, keys): target=hostname, )) + docs.extend(keys.generate_certificate( + name='auxiliary-calico-etcd-%d-client' % i, + ca_name='calico-etcd-client', + hosts=[hostname, host_data['ip']], + target=hostname, + )) + + docs.extend(keys.generate_certificate( + name='auxiliary-calico-etcd-%d-peer' % i, + ca_name='calico-etcd-peer', + hosts=[hostname, host_data['ip']], + target=hostname, + )) + docs.extend(keys.generate_certificate( - alias='etcd-calico-client', - name='etcd:client:calico', - ca_name='etcd-client', + alias='calico-etcd-node-client', + name='calico-etcd:node', + ca_name='calico-etcd-client', target=hostname, )) diff --git a/promenade/templates/genesis/etc/kubernetes/asset-loader/cni/calico.yaml b/promenade/templates/genesis/etc/kubernetes/asset-loader/cni/calico.yaml index 8281c2ce..f9be5c21 100644 --- a/promenade/templates/genesis/etc/kubernetes/asset-loader/cni/calico.yaml +++ b/promenade/templates/genesis/etc/kubernetes/asset-loader/cni/calico.yaml @@ -6,19 +6,19 @@ kind: Service metadata: labels: tier: control-plane - component: kube-etcd + component: calico-etcd name: calico-etcd namespace: kube-system spec: # Select the calico-etcd pod running on the master. selector: tier: control-plane - component: kube-etcd + component: calico-etcd # This ClusterIP needs to be known in advance, since we cannot rely # on DNS to get access to etcd. - clusterIP: {{ config['Network']['etcd_service_ip'] }} + clusterIP: {{ config['Network']['calico_etcd_service_ip'] }} ports: - - port: 2379 + - port: 6666 --- # Calico Version v2.2.1 @@ -37,7 +37,7 @@ metadata: data: # The location of your etcd cluster. This uses the Service clusterIP # defined below. - etcd_endpoints: https://{{ config['Network']['etcd_service_ip'] }}:2379 + etcd_endpoints: https://{{ config['Network']['calico_etcd_service_ip'] }}:6666 # Configure the Calico backend to use. calico_backend: "bird" @@ -96,9 +96,9 @@ data: # not using TLS for etcd. # This self-hosted install expects three files with the following names. The values # should be base64 encoded strings of the entire contents of each file. - etcd-key: {{ config.get(kind='CertificateKey', alias='etcd-calico-client')['data'] | b64enc }} - etcd-cert: {{ config.get(kind='Certificate', alias='etcd-calico-client')['data'] | b64enc }} - etcd-ca: {{ config.get(kind='CertificateAuthority', name='etcd-client')['data'] | b64enc }} + etcd-key: {{ config.get(kind='CertificateKey', alias='calico-etcd-node-client')['data'] | b64enc }} + etcd-cert: {{ config.get(kind='Certificate', alias='calico-etcd-node-client')['data'] | b64enc }} + etcd-ca: {{ config.get(kind='CertificateAuthority', name='calico-etcd-client')['data'] | b64enc }} --- # This manifest installs the calico/node container, as well diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/client-ca.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/client-ca.pem new file mode 100644 index 00000000..7491e20d --- /dev/null +++ b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/client-ca.pem @@ -0,0 +1 @@ +{{ config.get(kind='CertificateAuthority', name='calico-etcd-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client-key.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client-key.pem new file mode 100644 index 00000000..29b4f84e --- /dev/null +++ b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client-key.pem @@ -0,0 +1 @@ +{{ config.get(kind='CertificateKey', name='auxiliary-calico-etcd-0-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client.pem new file mode 100644 index 00000000..d0f2c764 --- /dev/null +++ b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client.pem @@ -0,0 +1 @@ +{{ config.get(kind='Certificate', name='auxiliary-calico-etcd-0-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer-key.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer-key.pem new file mode 100644 index 00000000..61b19257 --- /dev/null +++ b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer-key.pem @@ -0,0 +1 @@ +{{ config.get(kind='CertificateKey', name='auxiliary-calico-etcd-0-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer.pem new file mode 100644 index 00000000..beab792a --- /dev/null +++ b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer.pem @@ -0,0 +1 @@ +{{ config.get(kind='Certificate', name='auxiliary-calico-etcd-0-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/peer-ca.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/peer-ca.pem new file mode 100644 index 00000000..81c2a591 --- /dev/null +++ b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-0/pki/peer-ca.pem @@ -0,0 +1 @@ +{{ config.get(kind='CertificateAuthority', name='calico-etcd-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/client-ca.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/client-ca.pem new file mode 100644 index 00000000..7491e20d --- /dev/null +++ b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/client-ca.pem @@ -0,0 +1 @@ +{{ config.get(kind='CertificateAuthority', name='calico-etcd-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client-key.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client-key.pem new file mode 100644 index 00000000..cb44de87 --- /dev/null +++ b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client-key.pem @@ -0,0 +1 @@ +{{ config.get(kind='CertificateKey', name='auxiliary-calico-etcd-1-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client.pem new file mode 100644 index 00000000..90e1ce30 --- /dev/null +++ b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client.pem @@ -0,0 +1 @@ +{{ config.get(kind='Certificate', name='auxiliary-calico-etcd-1-client')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer-key.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer-key.pem new file mode 100644 index 00000000..e249cd95 --- /dev/null +++ b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer-key.pem @@ -0,0 +1 @@ +{{ config.get(kind='CertificateKey', name='auxiliary-calico-etcd-1-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer.pem new file mode 100644 index 00000000..a1fb1228 --- /dev/null +++ b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer.pem @@ -0,0 +1 @@ +{{ config.get(kind='Certificate', name='auxiliary-calico-etcd-1-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/peer-ca.pem b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/peer-ca.pem new file mode 100644 index 00000000..81c2a591 --- /dev/null +++ b/promenade/templates/genesis/etc/kubernetes/auxiliary-calico-etcd-1/pki/peer-ca.pem @@ -0,0 +1 @@ +{{ config.get(kind='CertificateAuthority', name='calico-etcd-peer')['data'] }} diff --git a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml new file mode 100644 index 00000000..b7b0e4a5 --- /dev/null +++ b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml @@ -0,0 +1,194 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: auxiliary-calico-etcd + namespace: kube-system + labels: + component: auxiliary-calico-etcd + promenade: genesis +spec: + hostNetwork: true + containers: + - name: auxiliary-calico-etcd-0 + image: quay.io/coreos/etcd:v3.0.17 + env: + - name: ETCD_NAME + value: auxiliary-calico-etcd-0 + - name: ETCD_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_PEER_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_DATA_DIR + value: /var/lib/auxiliary-calico-etcd-0 + - name: ETCD_TRUSTED_CA_FILE + value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/client-ca.pem + - name: ETCD_CERT_FILE + value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client.pem + - name: ETCD_KEY_FILE + value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-client-key.pem + - name: ETCD_PEER_TRUSTED_CA_FILE + value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/peer-ca.pem + - name: ETCD_PEER_CERT_FILE + value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer.pem + - name: ETCD_PEER_KEY_FILE + value: /etc/kubernetes/auxiliary-calico-etcd-0/pki/etcd-peer-key.pem + - name: ETCD_ADVERTISE_CLIENT_URLS + value: https://{{ config['Node']['hostname'] }}:16666 + - name: ETCD_INITIAL_ADVERTISE_PEER_URLS + value: https://{{ config['Node']['hostname'] }}:16667 + - name: ETCD_INITIAL_CLUSTER_TOKEN + value: promenade-calico-etcd-token + - name: ETCD_LISTEN_CLIENT_URLS + value: https://0.0.0.0:16666 + - name: ETCD_LISTEN_PEER_URLS + value: https://0.0.0.0:16667 + - name: ETCD_INITIAL_CLUSTER_STATE + value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster_state'] }} + - name: ETCD_INITIAL_CLUSTER + value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster'] | join(',') }} + ports: + - name: client + containerPort: 16666 + - name: peer + containerPort: 16667 + resources: + limits: + cpu: 100m + requests: + cpu: 100m + volumeMounts: + - name: data-0 + mountPath: /var/lib/auxiliary-calico-etcd-0 + - name: pki-0 + mountPath: /etc/kubernetes/auxiliary-calico-etcd-0/pki + readOnly: true + - name: auxiliary-calico-etcd-1 + image: quay.io/coreos/etcd:v3.0.17 + env: + - name: ETCD_NAME + value: auxiliary-calico-etcd-1 + - name: ETCD_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_PEER_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_DATA_DIR + value: /var/lib/auxiliary-calico-etcd-1 + - name: ETCD_TRUSTED_CA_FILE + value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/client-ca.pem + - name: ETCD_CERT_FILE + value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client.pem + - name: ETCD_KEY_FILE + value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-client-key.pem + - name: ETCD_PEER_TRUSTED_CA_FILE + value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/peer-ca.pem + - name: ETCD_PEER_CERT_FILE + value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer.pem + - name: ETCD_PEER_KEY_FILE + value: /etc/kubernetes/auxiliary-calico-etcd-1/pki/etcd-peer-key.pem + - name: ETCD_ADVERTISE_CLIENT_URLS + value: https://{{ config['Node']['hostname'] }}:26666 + - name: ETCD_INITIAL_ADVERTISE_PEER_URLS + value: https://{{ config['Node']['hostname'] }}:26667 + - name: ETCD_INITIAL_CLUSTER_TOKEN + value: promenade-calico-etcd-token + - name: ETCD_LISTEN_CLIENT_URLS + value: https://0.0.0.0:26666 + - name: ETCD_LISTEN_PEER_URLS + value: https://0.0.0.0:26667 + - name: ETCD_INITIAL_CLUSTER_STATE + value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster_state'] }} + - name: ETCD_INITIAL_CLUSTER + value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster'] | join(',') }} + ports: + - name: client + containerPort: 26666 + - name: peer + containerPort: 26667 + resources: + limits: + cpu: 100m + requests: + cpu: 100m + volumeMounts: + - name: data-1 + mountPath: /var/lib/auxiliary-calico-etcd-1 + - name: pki-1 + mountPath: /etc/kubernetes/auxiliary-calico-etcd-1/pki + readOnly: true + - name: cluster-monitor + image: quay.io/coreos/etcd:v3.0.17 + command: + - sh + - -c + - |- + set -x + while true; do + if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -ge {{ config['Masters']['nodes'] | length }} ]; then + {%- for master in config['Masters']['nodes'] %} + etcdctl member add {{ master['hostname'] }} --peer-urls https://{{ master['hostname'] }}:6667 + {%- endfor %} + break + fi + done + while true; do + sleep 5 + if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -eq {{ 2 + (config['Masters']['nodes'] | length) }} ]; then + etcdctl member remove $(etcdctl member list | grep auxiliary-calico-etcd-1 | cut -d , -f 1) + etcdctl member remove $(etcdctl member list | grep auxiliary-calico-etcd-0 | cut -d , -f 1) + sleep 60 + rm -rf \ + /var/lib/auxiliary-calico-etcd-0 \ + /var/lib/auxiliary-calico-etcd-1 \ + /etc/kubernetes/auxiliary-calico-etcd-0 \ + /etc/kubernetes/auxiliary-calico-etcd-1 \ + /etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml + sleep 10000 + fi + done + resources: + limits: + cpu: 100m + requests: + cpu: 100m + env: + - name: ETCDCTL_API + value: "3" + - name: ETCDCTL_CACERT + value: /etc/kubernetes/calico-etcd/pki/client-ca.pem + - name: ETCDCTL_CERT + value: /etc/kubernetes/calico-etcd/pki/etcd-client.pem + - name: ETCDCTL_ENDPOINTS + value: https://{{ config['Node']['ip'] }}:6666 + - name: ETCDCTL_KEY + value: /etc/kubernetes/calico-etcd/pki/etcd-client-key.pem + volumeMounts: + - name: pki + mountPath: /etc/kubernetes/calico-etcd/pki + readOnly: true + - name: manifests + mountPath: /etc/kubernetes/kubelet/manifests + - name: varlib + mountPath: /var/lib + volumes: + - name: data-0 + hostPath: + path: /var/lib/auxiliary-calico-etcd-0 + - name: data-1 + hostPath: + path: /var/lib/auxiliary-calico-etcd-1 + - name: pki + hostPath: + path: /etc/kubernetes/calico-etcd/pki + - name: pki-0 + hostPath: + path: /etc/kubernetes/auxiliary-calico-etcd-0/pki + - name: pki-1 + hostPath: + path: /etc/kubernetes/auxiliary-calico-etcd-1/pki + - name: manifests + hostPath: + path: /etc/kubernetes/kubelet/manifests + - name: varlib + hostPath: + path: /var/lib diff --git a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml index 7a8ed07a..1be76f0f 100644 --- a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml +++ b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml @@ -44,9 +44,9 @@ spec: - name: ETCD_LISTEN_PEER_URLS value: https://0.0.0.0:12380 - name: ETCD_INITIAL_CLUSTER_STATE - value: {{ config['Etcd']['initial_cluster_state'] }} + value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster_state'] }} - name: ETCD_INITIAL_CLUSTER - value: {{ config['Etcd']['initial_cluster'] | join(',') }} + value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster'] | join(',') }} ports: - name: client containerPort: 12379 @@ -97,9 +97,9 @@ spec: - name: ETCD_LISTEN_PEER_URLS value: https://0.0.0.0:22380 - name: ETCD_INITIAL_CLUSTER_STATE - value: {{ config['Etcd']['initial_cluster_state'] }} + value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster_state'] }} - name: ETCD_INITIAL_CLUSTER - value: {{ config['Etcd']['initial_cluster'] | join(',') }} + value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster'] | join(',') }} ports: - name: client containerPort: 22379 diff --git a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/client-ca.pem b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/client-ca.pem new file mode 100644 index 00000000..7491e20d --- /dev/null +++ b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/client-ca.pem @@ -0,0 +1 @@ +{{ config.get(kind='CertificateAuthority', name='calico-etcd-client')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-client-key.pem b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-client-key.pem new file mode 100644 index 00000000..cd8fd9e7 --- /dev/null +++ b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-client-key.pem @@ -0,0 +1 @@ +{{ config.get(kind='CertificateKey', alias='calico-etcd-client')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-client.pem b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-client.pem new file mode 100644 index 00000000..9122ec5d --- /dev/null +++ b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-client.pem @@ -0,0 +1 @@ +{{ config.get(kind='Certificate', alias='calico-etcd-client')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-peer-key.pem b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-peer-key.pem new file mode 100644 index 00000000..ce06abe2 --- /dev/null +++ b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-peer-key.pem @@ -0,0 +1 @@ +{{ config.get(kind='CertificateKey', alias='calico-etcd-peer')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-peer.pem b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-peer.pem new file mode 100644 index 00000000..c7e687fc --- /dev/null +++ b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/etcd-peer.pem @@ -0,0 +1 @@ +{{ config.get(kind='Certificate', alias='calico-etcd-peer')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/calico-etcd/pki/peer-ca.pem b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/peer-ca.pem new file mode 100644 index 00000000..81c2a591 --- /dev/null +++ b/promenade/templates/master/etc/kubernetes/calico-etcd/pki/peer-ca.pem @@ -0,0 +1 @@ +{{ config.get(kind='CertificateAuthority', name='calico-etcd-peer')['data'] }} diff --git a/promenade/templates/master/etc/kubernetes/kubelet/manifests/calico-etcd.yaml b/promenade/templates/master/etc/kubernetes/kubelet/manifests/calico-etcd.yaml new file mode 100644 index 00000000..47efa960 --- /dev/null +++ b/promenade/templates/master/etc/kubernetes/kubelet/manifests/calico-etcd.yaml @@ -0,0 +1,68 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: calico-etcd + namespace: kube-system + labels: + tier: control-plane + component: calico-etcd +spec: + hostNetwork: true + containers: + - name: k8s-etcd + image: quay.io/coreos/etcd:v3.0.17 + env: + - name: ETCD_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: ETCD_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_PEER_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_DATA_DIR + value: /var/lib/calico-etcd + - name: ETCD_TRUSTED_CA_FILE + value: /etc/kubernetes/calico-etcd/pki/client-ca.pem + - name: ETCD_CERT_FILE + value: /etc/kubernetes/calico-etcd/pki/etcd-client.pem + - name: ETCD_KEY_FILE + value: /etc/kubernetes/calico-etcd/pki/etcd-client-key.pem + - name: ETCD_PEER_TRUSTED_CA_FILE + value: /etc/kubernetes/calico-etcd/pki/peer-ca.pem + - name: ETCD_PEER_CERT_FILE + value: /etc/kubernetes/calico-etcd/pki/etcd-peer.pem + - name: ETCD_PEER_KEY_FILE + value: /etc/kubernetes/calico-etcd/pki/etcd-peer-key.pem + - name: ETCD_ADVERTISE_CLIENT_URLS + value: https://$(ETCD_NAME):6666 + - name: ETCD_INITIAL_ADVERTISE_PEER_URLS + value: https://$(ETCD_NAME):6667 + - name: ETCD_INITIAL_CLUSTER_TOKEN + value: promenade-calico-etcd-token + - name: ETCD_LISTEN_CLIENT_URLS + value: https://0.0.0.0:6666 + - name: ETCD_LISTEN_PEER_URLS + value: https://0.0.0.0:6667 + - name: ETCD_INITIAL_CLUSTER_STATE + value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster_state'] }} + - name: ETCD_INITIAL_CLUSTER + value: {{ config.get(kind='Etcd', alias='calico-etcd')['initial_cluster'] | join(',') }} + ports: + - name: client + containerPort: 6666 + - name: peer + containerPort: 6667 + volumeMounts: + - name: data + mountPath: /var/lib/calico-etcd + - name: pki + mountPath: /etc/kubernetes/calico-etcd/pki + volumes: + - name: data + hostPath: + path: /var/lib/calico-etcd + - name: pki + hostPath: + path: /etc/kubernetes/calico-etcd/pki diff --git a/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-etcd.yaml b/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-etcd.yaml index 6f862c62..4492115f 100644 --- a/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-etcd.yaml +++ b/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-etcd.yaml @@ -46,9 +46,9 @@ spec: - name: ETCD_LISTEN_PEER_URLS value: https://0.0.0.0:2380 - name: ETCD_INITIAL_CLUSTER_STATE - value: {{ config['Etcd']['initial_cluster_state'] }} + value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster_state'] }} - name: ETCD_INITIAL_CLUSTER - value: {{ config['Etcd']['initial_cluster'] | join(',') }} + value: {{ config.get(kind='Etcd', alias='kube-etcd')['initial_cluster'] | join(',') }} ports: - name: client containerPort: 2379 From 8221c21b8f420d02bdb1bca5f86805bcb289f2e5 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Thu, 6 Jul 2017 12:48:54 -0500 Subject: [PATCH 2/9] Increase etcd version for performance gains This was particularly useful for testing in a low power VirtualBox environment. Also, since etcd v3.1.0, it is necessary to disable strict host reconfig checking to allow expansion of the cluster by two nodes at once. --- .../kubelet/manifests/auxiliary-calico-etcd.yaml | 10 +++++++--- .../kubernetes/kubelet/manifests/auxiliary-etcd.yaml | 10 +++++++--- .../etc/kubernetes/kubelet/manifests/calico-etcd.yaml | 4 +++- .../etc/kubernetes/kubelet/manifests/kube-etcd.yaml | 4 +++- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml index b7b0e4a5..6eec42c3 100644 --- a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml +++ b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml @@ -11,10 +11,12 @@ spec: hostNetwork: true containers: - name: auxiliary-calico-etcd-0 - image: quay.io/coreos/etcd:v3.0.17 + image: quay.io/coreos/etcd:v3.2.1 env: - name: ETCD_NAME value: auxiliary-calico-etcd-0 + - name: ETCD_STRICT_RECONFIG_CHECK + value: "false" - name: ETCD_CLIENT_CERT_AUTH value: "true" - name: ETCD_PEER_CLIENT_CERT_AUTH @@ -64,10 +66,12 @@ spec: mountPath: /etc/kubernetes/auxiliary-calico-etcd-0/pki readOnly: true - name: auxiliary-calico-etcd-1 - image: quay.io/coreos/etcd:v3.0.17 + image: quay.io/coreos/etcd:v3.2.1 env: - name: ETCD_NAME value: auxiliary-calico-etcd-1 + - name: ETCD_STRICT_RECONFIG_CHECK + value: "false" - name: ETCD_CLIENT_CERT_AUTH value: "true" - name: ETCD_PEER_CLIENT_CERT_AUTH @@ -117,7 +121,7 @@ spec: mountPath: /etc/kubernetes/auxiliary-calico-etcd-1/pki readOnly: true - name: cluster-monitor - image: quay.io/coreos/etcd:v3.0.17 + image: quay.io/coreos/etcd:v3.2.1 command: - sh - -c diff --git a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml index 1be76f0f..8b436ca9 100644 --- a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml +++ b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml @@ -11,10 +11,12 @@ spec: hostNetwork: true containers: - name: auxiliary-etcd-0 - image: quay.io/coreos/etcd:v3.0.17 + image: quay.io/coreos/etcd:v3.2.1 env: - name: ETCD_NAME value: auxiliary-etcd-0 + - name: ETCD_STRICT_RECONFIG_CHECK + value: "false" - name: ETCD_CLIENT_CERT_AUTH value: "true" - name: ETCD_PEER_CLIENT_CERT_AUTH @@ -64,10 +66,12 @@ spec: mountPath: /etc/kubernetes/auxiliary-etcd-0/pki readOnly: true - name: auxiliary-etcd-1 - image: quay.io/coreos/etcd:v3.0.17 + image: quay.io/coreos/etcd:v3.2.1 env: - name: ETCD_NAME value: auxiliary-etcd-1 + - name: ETCD_STRICT_RECONFIG_CHECK + value: "false" - name: ETCD_CLIENT_CERT_AUTH value: "true" - name: ETCD_PEER_CLIENT_CERT_AUTH @@ -117,7 +121,7 @@ spec: mountPath: /etc/kubernetes/auxiliary-etcd-1/pki readOnly: true - name: cluster-monitor - image: quay.io/coreos/etcd:v3.0.17 + image: quay.io/coreos/etcd:v3.2.1 command: - sh - -c diff --git a/promenade/templates/master/etc/kubernetes/kubelet/manifests/calico-etcd.yaml b/promenade/templates/master/etc/kubernetes/kubelet/manifests/calico-etcd.yaml index 47efa960..a6a3d9b9 100644 --- a/promenade/templates/master/etc/kubernetes/kubelet/manifests/calico-etcd.yaml +++ b/promenade/templates/master/etc/kubernetes/kubelet/manifests/calico-etcd.yaml @@ -11,12 +11,14 @@ spec: hostNetwork: true containers: - name: k8s-etcd - image: quay.io/coreos/etcd:v3.0.17 + image: quay.io/coreos/etcd:v3.2.1 env: - name: ETCD_NAME valueFrom: fieldRef: fieldPath: spec.nodeName + - name: ETCD_STRICT_RECONFIG_CHECK + value: "false" - name: ETCD_CLIENT_CERT_AUTH value: "true" - name: ETCD_PEER_CLIENT_CERT_AUTH diff --git a/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-etcd.yaml b/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-etcd.yaml index 4492115f..cedc8ce0 100644 --- a/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-etcd.yaml +++ b/promenade/templates/master/etc/kubernetes/kubelet/manifests/kube-etcd.yaml @@ -11,12 +11,14 @@ spec: hostNetwork: true containers: - name: k8s-etcd - image: quay.io/coreos/etcd:v3.0.17 + image: quay.io/coreos/etcd:v3.2.1 env: - name: ETCD_NAME valueFrom: fieldRef: fieldPath: spec.nodeName + - name: ETCD_STRICT_RECONFIG_CHECK + value: "false" - name: ETCD_CLIENT_CERT_AUTH value: "true" - name: ETCD_PEER_CLIENT_CERT_AUTH From e36d2d864c409556e59726181e1efb66aebf2f87 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Thu, 6 Jul 2017 20:07:10 -0500 Subject: [PATCH 3/9] Only distribute etcd certificates to masters They are not needed on other nodes. --- promenade/generator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/promenade/generator.py b/promenade/generator.py index e6e8cf4f..e07a4c9e 100644 --- a/promenade/generator.py +++ b/promenade/generator.py @@ -46,19 +46,19 @@ class Generator: key_target='masters') etcd_client_ca, etcd_client_ca_key = keys.generate_ca( ca_name='etcd-client', - cert_target='all', + cert_target='masters', key_target='masters') etcd_peer_ca, etcd_peer_ca_key = keys.generate_ca( ca_name='etcd-peer', - cert_target='all', + cert_target='masters', key_target='masters') calico_etcd_client_ca, calico_etcd_client_ca_key = keys.generate_ca( ca_name='calico-etcd-client', - cert_target='all', + cert_target='masters', key_target='masters') calico_etcd_peer_ca, calico_etcd_peer_ca_key = keys.generate_ca( ca_name='calico-etcd-peer', - cert_target='all', + cert_target='masters', key_target='masters') admin_cert, admin_cert_key = keys.generate_certificate( From bee269802202cff8d402d2d58a9ee29bd7985da2 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Thu, 6 Jul 2017 20:27:58 -0500 Subject: [PATCH 4/9] Remove old calico code --- .../etc/kubernetes/asset-loader/cni/calico.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/promenade/templates/genesis/etc/kubernetes/asset-loader/cni/calico.yaml b/promenade/templates/genesis/etc/kubernetes/asset-loader/cni/calico.yaml index f9be5c21..f56c9329 100644 --- a/promenade/templates/genesis/etc/kubernetes/asset-loader/cni/calico.yaml +++ b/promenade/templates/genesis/etc/kubernetes/asset-loader/cni/calico.yaml @@ -72,16 +72,6 @@ data: etcd_cert: "/calico-secrets/etcd-cert" etcd_key: "/calico-secrets/etcd-key" -# ippool.yaml: |- -# apiVersion: v1 -# kind: ipPool -# metadata: -# cidr: {{ config['Network']['pod_ip_cidr'] }} -# spec: -# ipip: -# enabled: true -# nat-outgoing: true - --- # The following contains k8s Secrets for use with a TLS enabled etcd cluster. # For information on populating Secrets, see http://kubernetes.io/docs/user-guide/secrets/ From c2842f49c492b3dc5d04ebcdd38388c87b1d9692 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Thu, 6 Jul 2017 20:42:40 -0500 Subject: [PATCH 5/9] Vary CA name to improve debuggability --- promenade/pki.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/promenade/pki.py b/promenade/pki.py index cec73300..31266508 100644 --- a/promenade/pki.py +++ b/promenade/pki.py @@ -37,7 +37,7 @@ class PKI: result = self._cfssl(['gencert', '-initca', 'csr.json'], files={ 'csr.json': self.csr( - name='Kubernetes', + name=ca_name, groups=['Kubernetes']), }) LOG.debug('ca_cert=%r', result['cert']) From 7fe77f69055bfac260177d4d4fcf0f5f8ea7b2d1 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Mon, 10 Jul 2017 13:42:16 -0500 Subject: [PATCH 6/9] Make calico interface configurable This is useful in environments with multiple interfaces. It is even useful in the example Vagrant environment. --- example/vagrant-input-config.yaml | 4 ++++ .../genesis/etc/kubernetes/asset-loader/cni/calico.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/example/vagrant-input-config.yaml b/example/vagrant-input-config.yaml index e6ca0c93..8ee29d3c 100644 --- a/example/vagrant-input-config.yaml +++ b/example/vagrant-input-config.yaml @@ -8,6 +8,7 @@ spec: nodes: n0: ip: 192.168.77.10 + kubernetes_interface: enp0s8 roles: - master - genesis @@ -15,18 +16,21 @@ spec: - beta.kubernetes.io/arch=amd64 n1: ip: 192.168.77.11 + kubernetes_interface: enp0s8 roles: - master additional_labels: - beta.kubernetes.io/arch=amd64 n2: ip: 192.168.77.12 + kubernetes_interface: enp0s8 roles: - master additional_labels: - beta.kubernetes.io/arch=amd64 n3: ip: 192.168.77.13 + kubernetes_interface: enp0s8 roles: - worker additional_labels: diff --git a/promenade/templates/genesis/etc/kubernetes/asset-loader/cni/calico.yaml b/promenade/templates/genesis/etc/kubernetes/asset-loader/cni/calico.yaml index f56c9329..18be6633 100644 --- a/promenade/templates/genesis/etc/kubernetes/asset-loader/cni/calico.yaml +++ b/promenade/templates/genesis/etc/kubernetes/asset-loader/cni/calico.yaml @@ -187,6 +187,10 @@ spec: # Auto-detect the BGP IP address. - name: IP value: "" + {%- if config['Node']['kubernetes_interface'] is defined %} + - name: IP_AUTODETECTION_METHOD + value: interface={{ config['Node']['kubernetes_interface'] }} + {%- endif %} securityContext: privileged: true resources: From e0ab3f903cc66a8733f0b805bf51c19fd722d88e Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Mon, 10 Jul 2017 20:49:20 -0500 Subject: [PATCH 7/9] Use virtio to avoid VirtualBox issues under load On an Ubuntu 16.04 host, suring and after load spikes (which come naturally as part of spinning up kubernetes components, etcd, armada, etc.), the virtual hardware that VirtualBox provides in the provided Vagrant example would frequently and randomly become unstable. The host network that nodes communicate on remained stable, so nodes could speak to one another and even occasionally join successfully (if the joining node's public interface hadn't been disrupted so that it could download images). The additional load caused by the second etcd cluster for Calico made this issue occur so often that joining more than one node became quite unlikely Switching the VMs to use a paravirtualized network interface seems to have solved the issue. I never experienced this issue on Ubuntu 17.04, and it sometimes occurs without running calico on its own etcd. --- Vagrantfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Vagrantfile b/Vagrantfile index d1f7275f..1d688fc5 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -16,6 +16,7 @@ EOS config.vm.provider "virtualbox" do |vb| vb.cpus = 2 vb.memory = "2048" + vb.customize ["modifyvm", :id, "--nictype1", "virtio"] end config.vm.define "n0" do |c| From 3af4b2216b8165acfd6b0d603a0091315ed7b34e Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Tue, 18 Jul 2017 13:00:11 -0500 Subject: [PATCH 8/9] Update calico etcd image to be configurable --- .../kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml | 6 +++--- .../etc/kubernetes/kubelet/manifests/calico-etcd.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml index 6eec42c3..dc438eec 100644 --- a/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml +++ b/promenade/templates/genesis/etc/kubernetes/kubelet/manifests/auxiliary-calico-etcd.yaml @@ -11,7 +11,7 @@ spec: hostNetwork: true containers: - name: auxiliary-calico-etcd-0 - image: quay.io/coreos/etcd:v3.2.1 + image: {{ config['Versions']['images']['calico']['etcd'] }} env: - name: ETCD_NAME value: auxiliary-calico-etcd-0 @@ -66,7 +66,7 @@ spec: mountPath: /etc/kubernetes/auxiliary-calico-etcd-0/pki readOnly: true - name: auxiliary-calico-etcd-1 - image: quay.io/coreos/etcd:v3.2.1 + image: {{ config['Versions']['images']['calico']['etcd'] }} env: - name: ETCD_NAME value: auxiliary-calico-etcd-1 @@ -121,7 +121,7 @@ spec: mountPath: /etc/kubernetes/auxiliary-calico-etcd-1/pki readOnly: true - name: cluster-monitor - image: quay.io/coreos/etcd:v3.2.1 + image: {{ config['Versions']['images']['calico']['etcd'] }} command: - sh - -c diff --git a/promenade/templates/master/etc/kubernetes/kubelet/manifests/calico-etcd.yaml b/promenade/templates/master/etc/kubernetes/kubelet/manifests/calico-etcd.yaml index a6a3d9b9..3733334d 100644 --- a/promenade/templates/master/etc/kubernetes/kubelet/manifests/calico-etcd.yaml +++ b/promenade/templates/master/etc/kubernetes/kubelet/manifests/calico-etcd.yaml @@ -11,7 +11,7 @@ spec: hostNetwork: true containers: - name: k8s-etcd - image: quay.io/coreos/etcd:v3.2.1 + image: {{ config['Versions']['images']['calico']['etcd'] }} env: - name: ETCD_NAME valueFrom: From 2a5c057a5f7dbec3d1a9742f0792a07a8cf2e207 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Tue, 18 Jul 2017 13:00:53 -0500 Subject: [PATCH 9/9] Update example etcd version --- example/vagrant-input-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/vagrant-input-config.yaml b/example/vagrant-input-config.yaml index 87b1713e..07a1d2df 100644 --- a/example/vagrant-input-config.yaml +++ b/example/vagrant-input-config.yaml @@ -66,7 +66,7 @@ spec: armada: quay.io/attcomdev/armada:latest calico: cni: quay.io/calico/cni:v1.9.1 - etcd: quay.io/coreos/etcd:v3.0.17 + etcd: quay.io/coreos/etcd:v3.2.1 node: quay.io/calico/node:v1.3.0 policy-controller: quay.io/calico/kube-policy-controller:v0.6.0 kubernetes: @@ -76,7 +76,7 @@ spec: dnsmasq: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.2 kubedns: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.2 sidecar: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.2 - etcd: quay.io/coreos/etcd:v3.0.17 + etcd: quay.io/coreos/etcd:v3.2.1 kubectl: gcr.io/google_containers/hyperkube-amd64:v1.6.4 proxy: gcr.io/google_containers/hyperkube-amd64:v1.6.4 scheduler: gcr.io/google_containers/hyperkube-amd64:v1.6.4