Merge pull request #9 from mark-burnett/cleanup-etcd-join
Use auxiliary etcd instances to smooth join process
This commit is contained in:
Mark Burnett
GitHub
commit
fac36cec4f
@ -46,20 +46,27 @@ def _extract_etcd_data(hostname, genesis, masters):
|
||||
'env': {},
|
||||
}
|
||||
|
||||
peers = []
|
||||
for host in boot_order:
|
||||
peers.append(host)
|
||||
if host['hostname'] == hostname:
|
||||
break
|
||||
|
||||
result['env']['ETCD_INITIAL_CLUSTER'] = ','.join(
|
||||
'%s=https://%s:2380' % (p['hostname'], p['hostname'])
|
||||
for p in peers)
|
||||
peers = [
|
||||
{
|
||||
'hostname': 'auxiliary-etcd-%d' % i,
|
||||
'peer_port': 2380 + (i + 1) * 10000
|
||||
}
|
||||
for i in range(2)
|
||||
]
|
||||
peers.append({
|
||||
'hostname': genesis['hostname'],
|
||||
})
|
||||
|
||||
if hostname == genesis['hostname']:
|
||||
result['env']['ETCD_INITIAL_CLUSTER_STATE'] = 'new'
|
||||
else:
|
||||
result['env']['ETCD_INITIAL_CLUSTER_STATE'] = 'existing'
|
||||
for host in non_genesis_masters:
|
||||
peers.append({'hostname': host['hostname']})
|
||||
|
||||
result['env']['ETCD_INITIAL_CLUSTER'] = ','.join(
|
||||
'%s=https://%s:%d' % (p['hostname'], p['hostname'], p.get('peer_port', 2380))
|
||||
for p in peers)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@ -1,24 +0,0 @@
|
||||
from . import kube, logging
|
||||
|
||||
__all__ = ['add_member']
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def add_member(exec_pod, hostname, port):
|
||||
opts = ' '.join([
|
||||
'--cacert',
|
||||
'/etc/etcd-pki/cluster-ca.pem',
|
||||
'--cert',
|
||||
'/etc/etcd-pki/etcd.pem',
|
||||
'--key',
|
||||
'/etc/etcd-pki/etcd-key.pem',
|
||||
])
|
||||
result = kube.kc('exec', '-n', 'kube-system', '-t', exec_pod, '--', 'sh', '-c',
|
||||
'ETCDCTL_API=3 etcdctl %s member add %s --peer-urls https://%s:%d'
|
||||
% (opts, hostname, hostname, port))
|
||||
if result.returncode != 0:
|
||||
LOG.error('Failed to add etcd member. STDOUT: %r', result.stdout)
|
||||
LOG.error('Failed to add etcd member. STDERR: %r', result.stderr)
|
||||
result.check_returncode()
|
||||
@ -1,27 +0,0 @@
|
||||
from . import logging
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
__all__ = ['kc', 'wait_for_node']
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def wait_for_node(node):
|
||||
repeat = True
|
||||
while repeat:
|
||||
result = kc('get', 'nodes', node, '-o',
|
||||
r'jsonpath={.status.conditions[?(@.type=="Ready")].status}')
|
||||
if result.stdout == b'True':
|
||||
repeat = False
|
||||
else:
|
||||
LOG.debug('Node "%s" not ready, waiting. stdout=%r stderr=%r',
|
||||
node, result.stdout, result.stderr)
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
def kc(*args):
|
||||
return subprocess.run(['/target/usr/local/bin/kubectl',
|
||||
'--kubeconfig', '/target/etc/kubernetes/genesis/kubeconfig.yaml', *args],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
@ -1,4 +1,4 @@
|
||||
from . import config, etcd, logging, kube, pki, renderer
|
||||
from . import config, logging, pki, renderer
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
@ -26,7 +26,6 @@ class Operator:
|
||||
|
||||
def genesis(self, *, asset_dir=None):
|
||||
self.setup(asset_dir=asset_dir)
|
||||
self.expand_etcd_cluster()
|
||||
|
||||
def join(self, *, asset_dir=None):
|
||||
self.setup(asset_dir=asset_dir)
|
||||
@ -63,15 +62,3 @@ class Operator:
|
||||
self.target_dir,
|
||||
'/bin/bash', '/usr/local/bin/bootstrap'],
|
||||
check=True)
|
||||
|
||||
def expand_etcd_cluster(self):
|
||||
for node in self.node_data['etcd']['boot_order'][1:]:
|
||||
LOG.info('Waiting for Node "%s" to be Ready', node['hostname'])
|
||||
kube.wait_for_node(node['hostname'])
|
||||
LOG.info('Node "%s" Ready. Adding to etcd cluster.', node['hostname'])
|
||||
etcd.add_member(self.genesis_etcd_pod, node['hostname'], port=2380)
|
||||
LOG.info('Finished expanding etcd cluster.')
|
||||
|
||||
@property
|
||||
def genesis_etcd_pod(self):
|
||||
return 'kube-etcd-%s' % self.node_data['genesis']['hostname']
|
||||
|
||||
2
promenade/templates/common/etc/dnsmasq.d/auxilliary-etcd
Normal file
2
promenade/templates/common/etc/dnsmasq.d/auxilliary-etcd
Normal file
@ -0,0 +1,2 @@
|
||||
host-record=auxiliary-etcd-0,{{ genesis['ip'] }}
|
||||
host-record=auxiliary-etcd-1,{{ genesis['ip'] }}
|
||||
@ -0,0 +1,19 @@
|
||||
{
|
||||
"CN": "etcd:{{ current_node['hostname'] }}",
|
||||
"hosts": [
|
||||
"kubernetes",
|
||||
"kubernetes.default",
|
||||
"kubernetes.default.svc",
|
||||
"kubernetes.default.svc.cluster.local",
|
||||
"127.0.0.1",
|
||||
"{{ current_node['hostname'] }}",
|
||||
"auxiliary-etcd-0",
|
||||
"auxiliary-etcd-1",
|
||||
"{{ current_node['ip'] }}",
|
||||
"{{ network.kube_service_ip }}"
|
||||
],
|
||||
"key": {
|
||||
"algo": "rsa",
|
||||
"size": 2048
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,183 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: auxiliary-etcd
|
||||
namespace: kube-system
|
||||
labels:
|
||||
component: auxiliary-etcd
|
||||
promenade: genesis
|
||||
spec:
|
||||
hostNetwork: true
|
||||
containers:
|
||||
- name: auxiliary-etcd-0
|
||||
image: quay.io/coreos/etcd:v3.0.17
|
||||
env:
|
||||
- name: ETCD_NAME
|
||||
value: auxiliary-etcd-0
|
||||
- name: ETCD_CLIENT_CERT_AUTH
|
||||
value: "true"
|
||||
- name: ETCD_PEER_CLIENT_CERT_AUTH
|
||||
value: "true"
|
||||
- name: ETCD_DATA_DIR
|
||||
value: /var/lib/auxiliary-etcd-0
|
||||
- name: ETCD_TRUSTED_CA_FILE
|
||||
value: /etc/etcd-pki/cluster-ca.pem
|
||||
- name: ETCD_CERT_FILE
|
||||
value: /etc/etcd-pki/etcd.pem
|
||||
- name: ETCD_KEY_FILE
|
||||
value: /etc/etcd-pki/etcd-key.pem
|
||||
- name: ETCD_PEER_TRUSTED_CA_FILE
|
||||
value: /etc/etcd-pki/cluster-ca.pem
|
||||
- name: ETCD_PEER_CERT_FILE
|
||||
value: /etc/etcd-pki/etcd.pem
|
||||
- name: ETCD_PEER_KEY_FILE
|
||||
value: /etc/etcd-pki/etcd-key.pem
|
||||
- name: ETCD_ADVERTISE_CLIENT_URLS
|
||||
value: https://$(ETCD_NAME):12379
|
||||
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
|
||||
value: https://$(ETCD_NAME):12380
|
||||
- name: ETCD_INITIAL_CLUSTER_TOKEN
|
||||
value: promenade-kube-etcd-token
|
||||
- name: ETCD_LISTEN_CLIENT_URLS
|
||||
value: https://0.0.0.0:12379
|
||||
- name: ETCD_LISTEN_PEER_URLS
|
||||
value: https://0.0.0.0:12380
|
||||
{%- for env_name, env_value in etcd['env'].items() %}
|
||||
- name: {{ env_name }}
|
||||
value: {{ env_value }}
|
||||
{%- endfor %}
|
||||
ports:
|
||||
- name: client
|
||||
containerPort: 12379
|
||||
- name: peer
|
||||
containerPort: 12380
|
||||
resources:
|
||||
limits:
|
||||
cpu: 100m
|
||||
requests:
|
||||
cpu: 100m
|
||||
volumeMounts:
|
||||
- name: data-0
|
||||
mountPath: /var/lib/auxiliary-etcd-0
|
||||
- name: pki
|
||||
mountPath: /etc/etcd-pki
|
||||
readOnly: true
|
||||
- name: auxiliary-etcd-1
|
||||
image: quay.io/coreos/etcd:v3.0.17
|
||||
env:
|
||||
- name: ETCD_NAME
|
||||
value: auxiliary-etcd-1
|
||||
- name: ETCD_CLIENT_CERT_AUTH
|
||||
value: "true"
|
||||
- name: ETCD_PEER_CLIENT_CERT_AUTH
|
||||
value: "true"
|
||||
- name: ETCD_DATA_DIR
|
||||
value: /var/lib/auxiliary-etcd-1
|
||||
- name: ETCD_TRUSTED_CA_FILE
|
||||
value: /etc/etcd-pki/cluster-ca.pem
|
||||
- name: ETCD_CERT_FILE
|
||||
value: /etc/etcd-pki/etcd.pem
|
||||
- name: ETCD_KEY_FILE
|
||||
value: /etc/etcd-pki/etcd-key.pem
|
||||
- name: ETCD_PEER_TRUSTED_CA_FILE
|
||||
value: /etc/etcd-pki/cluster-ca.pem
|
||||
- name: ETCD_PEER_CERT_FILE
|
||||
value: /etc/etcd-pki/etcd.pem
|
||||
- name: ETCD_PEER_KEY_FILE
|
||||
value: /etc/etcd-pki/etcd-key.pem
|
||||
- name: ETCD_ADVERTISE_CLIENT_URLS
|
||||
value: https://$(ETCD_NAME):22379
|
||||
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
|
||||
value: https://$(ETCD_NAME):22380
|
||||
- name: ETCD_INITIAL_CLUSTER_TOKEN
|
||||
value: promenade-kube-etcd-token
|
||||
- name: ETCD_LISTEN_CLIENT_URLS
|
||||
value: https://0.0.0.0:22379
|
||||
- name: ETCD_LISTEN_PEER_URLS
|
||||
value: https://0.0.0.0:22380
|
||||
{%- for env_name, env_value in etcd['env'].items() %}
|
||||
- name: {{ env_name }}
|
||||
value: {{ env_value }}
|
||||
{%- endfor %}
|
||||
ports:
|
||||
- name: client
|
||||
containerPort: 22379
|
||||
- name: peer
|
||||
containerPort: 22380
|
||||
resources:
|
||||
limits:
|
||||
cpu: 100m
|
||||
requests:
|
||||
cpu: 100m
|
||||
volumeMounts:
|
||||
- name: data-1
|
||||
mountPath: /var/lib/auxiliary-etcd-1
|
||||
- name: pki
|
||||
mountPath: /etc/etcd-pki
|
||||
readOnly: true
|
||||
- name: cluster-monitor
|
||||
image: quay.io/coreos/etcd:v3.0.17
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |-
|
||||
set -x
|
||||
while true; do
|
||||
if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -ge {{ masters | length }} ]; then
|
||||
{%- for master in masters %}
|
||||
etcdctl member add {{ master['hostname'] }} --peer-urls https://{{ master['hostname'] }}:2380
|
||||
{%- endfor %}
|
||||
break
|
||||
fi
|
||||
done
|
||||
while true; do
|
||||
sleep 5
|
||||
if [ $(etcdctl member list | grep -v unstarted | wc -l || echo 0) -eq {{ 2 + (masters | length) }} ]; then
|
||||
etcdctl member remove $(etcdctl member list | grep auxiliary-etcd-1 | cut -d , -f 1)
|
||||
etcdctl member remove $(etcdctl member list | grep auxiliary-etcd-0 | cut -d , -f 1)
|
||||
sleep 60
|
||||
rm -rf /var/lib/auxiliary-etcd-0 /var/lib/auxiliary-etcd-1 /etc/kubernetes/kubelet/manifests/auxiliary-etcd.yaml
|
||||
sleep 10000
|
||||
fi
|
||||
done
|
||||
resources:
|
||||
limits:
|
||||
cpu: 100m
|
||||
requests:
|
||||
cpu: 100m
|
||||
env:
|
||||
- name: ETCDCTL_API
|
||||
value: "3"
|
||||
- name: ETCDCTL_CACERT
|
||||
value: /etc/etcd-pki/cluster-ca.pem
|
||||
- name: ETCDCTL_CERT
|
||||
value: /etc/etcd-pki/etcd.pem
|
||||
- name: ETCDCTL_ENDPOINTS
|
||||
value: https://127.0.0.1:12379
|
||||
- name: ETCDCTL_KEY
|
||||
value: /etc/etcd-pki/etcd-key.pem
|
||||
volumeMounts:
|
||||
- name: pki
|
||||
mountPath: /etc/etcd-pki
|
||||
readOnly: true
|
||||
- name: manifests
|
||||
mountPath: /etc/kubernetes/kubelet/manifests
|
||||
- name: varlib
|
||||
mountPath: /var/lib
|
||||
volumes:
|
||||
- name: data-0
|
||||
hostPath:
|
||||
path: /var/lib/auxiliary-etcd-0
|
||||
- name: data-1
|
||||
hostPath:
|
||||
path: /var/lib/auxiliary-etcd-1
|
||||
- name: pki
|
||||
hostPath:
|
||||
path: /etc/kubernetes/etcd/pki
|
||||
- name: manifests
|
||||
hostPath:
|
||||
path: /etc/kubernetes/kubelet/manifests
|
||||
- name: varlib
|
||||
hostPath:
|
||||
path: /var/lib
|
||||
Loading…
Reference in New Issue
Block a user