Add support for multi-controller HA deployments.
Change-Id: I819cec71cdbc8df7a85bd4f41f36493b34c8bdcc
This commit is contained in:
parent
8c16541bdf
commit
e6c8fab962
|
@ -6,29 +6,55 @@ import yaml
|
||||||
|
|
||||||
from pluginutils import NODES_CONFIG
|
from pluginutils import NODES_CONFIG
|
||||||
|
|
||||||
RECONFIGURE_ROUTE_REFLECTOR = "##REPLACE_ON_INSTALL##/calico_route_reflector.sh"
|
SCRIPTS_LOCATION="##REPLACE_ON_INSTALL##/"
|
||||||
|
RECONFIGURE_ROUTE_REFLECTOR = SCRIPTS_LOCATION + "calico_route_reflector.sh"
|
||||||
|
UPDATE_ETCD_CLUSTER = SCRIPTS_LOCATION + "update_etcd_cluster.sh"
|
||||||
|
|
||||||
|
|
||||||
def _get_configured_compute_nodes():
|
def _get_configured_nodes(roles):
|
||||||
with open(NODES_CONFIG, "r") as f:
|
with open(NODES_CONFIG, "r") as f:
|
||||||
config = yaml.safe_load(f)
|
config = yaml.safe_load(f)
|
||||||
|
|
||||||
compute_nodes = [node for node in config["nodes"]
|
return [node for node in config["nodes"] if node["role"] in roles]
|
||||||
if node["role"] == "compute"]
|
|
||||||
|
|
||||||
return compute_nodes
|
|
||||||
|
def _get_compute_nodes():
|
||||||
|
return _get_configured_nodes(["compute"])
|
||||||
|
|
||||||
|
|
||||||
|
def _get_control_nodes():
|
||||||
|
nodes = _get_configured_nodes(["controller", "primary-controller"])
|
||||||
|
|
||||||
|
for node in nodes:
|
||||||
|
# Note this does not change the node role in the Fuel deployment, just
|
||||||
|
# in the list of nodes internal to this script (where we are only
|
||||||
|
# concerned with the distinction between compute/control nodes, not
|
||||||
|
# whether a given control node is primary or not).
|
||||||
|
if node["role"] == "primary-controller":
|
||||||
|
node["role"] = "controller"
|
||||||
|
|
||||||
|
return nodes
|
||||||
|
|
||||||
|
|
||||||
class DeploymentChangeHandler(pyinotify.ProcessEvent):
|
class DeploymentChangeHandler(pyinotify.ProcessEvent):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super(DeploymentChangeHandler, self).__init__()
|
super(DeploymentChangeHandler, self).__init__()
|
||||||
self.compute_nodes = _get_configured_compute_nodes()
|
self.compute_nodes = _get_compute_nodes()
|
||||||
|
self.control_nodes = _get_control_nodes()
|
||||||
|
|
||||||
def process_IN_MODIFY(self, event):
|
def process_IN_MODIFY(self, event):
|
||||||
current_compute_nodes = _get_configured_compute_nodes()
|
current_compute_nodes = _get_compute_nodes()
|
||||||
if current_compute_nodes != self.compute_nodes:
|
current_control_nodes = _get_control_nodes()
|
||||||
|
|
||||||
|
if current_control_nodes != self.control_nodes:
|
||||||
subprocess.call(RECONFIGURE_ROUTE_REFLECTOR)
|
subprocess.call(RECONFIGURE_ROUTE_REFLECTOR)
|
||||||
self.compute_nodes = current_compute_nodes
|
subprocess.call(UPDATE_ETCD_CLUSTER)
|
||||||
|
|
||||||
|
elif current_compute_nodes != self.compute_nodes:
|
||||||
|
subprocess.call(RECONFIGURE_ROUTE_REFLECTOR)
|
||||||
|
|
||||||
|
self.compute_nodes = current_compute_nodes
|
||||||
|
self.control_nodes = current_control_nodes
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
@ -10,7 +10,7 @@ set -x
|
||||||
echo "Hi, I'm a compute node!"
|
echo "Hi, I'm a compute node!"
|
||||||
|
|
||||||
this_node_address=$(python get_node_ip.py `hostname`)
|
this_node_address=$(python get_node_ip.py `hostname`)
|
||||||
controller_node_address=$(python get_controller_ip.py)
|
controller_node_addresses=$(python get_node_ips_by_role.py controller)
|
||||||
|
|
||||||
# Get APT key for binaries.projectcalico.org.
|
# Get APT key for binaries.projectcalico.org.
|
||||||
|
|
||||||
|
@ -52,6 +52,12 @@ apt-get update
|
||||||
|
|
||||||
apt-get -y install etcd
|
apt-get -y install etcd
|
||||||
|
|
||||||
|
for controller_address in ${controller_node_addresses[@]}
|
||||||
|
do
|
||||||
|
initial_cluster+="${controller_address}=http://${controller_address}:2380,"
|
||||||
|
done
|
||||||
|
initial_cluster=${initial_cluster::-1} # remove trailing comma
|
||||||
|
|
||||||
service etcd stop
|
service etcd stop
|
||||||
rm -rf /var/lib/etcd/*
|
rm -rf /var/lib/etcd/*
|
||||||
awk '/exec \/usr\/bin\/etcd/{while(getline && $0 != ""){}}1' /etc/init/etcd.conf > tmp
|
awk '/exec \/usr\/bin\/etcd/{while(getline && $0 != ""){}}1' /etc/init/etcd.conf > tmp
|
||||||
|
@ -60,7 +66,7 @@ cat << EXEC_CMD >> /etc/init/etcd.conf
|
||||||
exec /usr/bin/etcd -proxy on \\
|
exec /usr/bin/etcd -proxy on \\
|
||||||
-listen-client-urls http://127.0.0.1:4001 \\
|
-listen-client-urls http://127.0.0.1:4001 \\
|
||||||
-advertise-client-urls http://127.0.0.1:7001 \\
|
-advertise-client-urls http://127.0.0.1:7001 \\
|
||||||
-initial-cluster controller=http://${controller_node_address}:2380
|
-initial-cluster ${initial_cluster}
|
||||||
EXEC_CMD
|
EXEC_CMD
|
||||||
service etcd start
|
service etcd start
|
||||||
|
|
||||||
|
@ -143,7 +149,7 @@ apt-get -y install calico-compute bird
|
||||||
# script. You should consult the relevant documentation for your chosen BGP
|
# script. You should consult the relevant documentation for your chosen BGP
|
||||||
# stack.
|
# stack.
|
||||||
|
|
||||||
calico-gen-bird-conf.sh $this_node_address $controller_node_address 64511
|
calico-gen-bird-mesh-conf.sh $this_node_address 64511 ${controller_node_addresses[@]}
|
||||||
|
|
||||||
# Edit the /etc/calico/felix.cfg file:
|
# Edit the /etc/calico/felix.cfg file:
|
||||||
# Change the MetadataAddr setting to 127.0.0.1.
|
# Change the MetadataAddr setting to 127.0.0.1.
|
||||||
|
|
|
@ -10,6 +10,7 @@ set -x
|
||||||
echo "Hi, I'm a controller node!"
|
echo "Hi, I'm a controller node!"
|
||||||
|
|
||||||
this_node_address=$(python get_node_ip.py `hostname`)
|
this_node_address=$(python get_node_ip.py `hostname`)
|
||||||
|
controller_node_addresses=$(python get_node_ips_by_role.py controller)
|
||||||
|
|
||||||
# Get APT key for binaries.projectcalico.org.
|
# Get APT key for binaries.projectcalico.org.
|
||||||
|
|
||||||
|
@ -51,19 +52,26 @@ apt-get update
|
||||||
|
|
||||||
apt-get -y install etcd
|
apt-get -y install etcd
|
||||||
|
|
||||||
|
for controller_address in ${controller_node_addresses[@]}
|
||||||
|
do
|
||||||
|
initial_cluster+="${controller_address}=http://${controller_address}:2380,"
|
||||||
|
done
|
||||||
|
initial_cluster=${initial_cluster::-1} # remove trailing comma
|
||||||
|
|
||||||
service etcd stop
|
service etcd stop
|
||||||
rm -rf /var/lib/etcd/*
|
rm -rf /var/lib/etcd/*
|
||||||
awk '/exec \/usr\/bin\/etcd/{while(getline && $0 != ""){}}1' /etc/init/etcd.conf > tmp
|
awk '/exec \/usr\/bin\/etcd/{while(getline && $0 != ""){}}1' /etc/init/etcd.conf > tmp
|
||||||
mv tmp /etc/init/etcd.conf
|
mv tmp /etc/init/etcd.conf
|
||||||
cat << EXEC_CMD >> /etc/init/etcd.conf
|
cat << EXEC_CMD >> /etc/init/etcd.conf
|
||||||
exec /usr/bin/etcd -name controller \\
|
exec /usr/bin/etcd -name ${this_node_address} \\
|
||||||
-advertise-client-urls "http://${this_node_address}:2379,http://${this_node_address}:4001" \\
|
-advertise-client-urls "http://${this_node_address}:2379,http://${this_node_address}:4001" \\
|
||||||
-listen-client-urls "http://0.0.0.0:2379,http://0.0.0.0:4001" \\
|
-listen-client-urls "http://0.0.0.0:2379,http://0.0.0.0:4001" \\
|
||||||
-listen-peer-urls "http://0.0.0.0:2380" \\
|
-listen-peer-urls "http://0.0.0.0:2380" \\
|
||||||
-initial-advertise-peer-urls "http://${this_node_address}:2380" \\
|
-initial-advertise-peer-urls "http://${this_node_address}:2380" \\
|
||||||
-initial-cluster-token fuel-cluster-1 \\
|
-initial-cluster-token fuel-cluster-1 \\
|
||||||
-initial-cluster controller=http://${this_node_address}:2380 \\
|
-initial-cluster ${initial_cluster} \\
|
||||||
-initial-cluster-state new
|
-initial-cluster-state new
|
||||||
|
|
||||||
EXEC_CMD
|
EXEC_CMD
|
||||||
|
|
||||||
service etcd start
|
service etcd start
|
||||||
|
|
|
@ -8,8 +8,10 @@ set -x
|
||||||
echo "Hi, I'm a route_reflector node!"
|
echo "Hi, I'm a route_reflector node!"
|
||||||
|
|
||||||
this_node_address=$(python get_node_ip.py `hostname`)
|
this_node_address=$(python get_node_ip.py `hostname`)
|
||||||
|
controller_node_addresses=$(python get_node_ips_by_role.py controller)
|
||||||
|
|
||||||
bgp_peers=$(python get_rr_peers.py)
|
client_peers=$(python get_node_ips_by_role.py compute)
|
||||||
|
route_reflector_peers=("${controller_node_addresses[@]/$this_node_address}")
|
||||||
|
|
||||||
# Generate basic config for a BIRD BGP route reflector.
|
# Generate basic config for a BIRD BGP route reflector.
|
||||||
cat > /etc/bird/bird.conf <<EOF
|
cat > /etc/bird/bird.conf <<EOF
|
||||||
|
@ -38,24 +40,33 @@ protocol device {
|
||||||
}
|
}
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# Add a BGP protocol stanza for each compute node.
|
# Add a BGP protocol stanza for all peers.
|
||||||
for node in $bgp_peers; do
|
for node in ${client_peers[@]} ${route_reflector_peers[@]}; do
|
||||||
if [ $node != $this_node_address ]; then
|
cat >> /etc/bird/bird.conf <<EOF
|
||||||
cat >> /etc/bird/bird.conf <<EOF
|
|
||||||
|
|
||||||
protocol bgp {
|
protocol bgp {
|
||||||
description "$node";
|
|
||||||
local as 64511;
|
local as 64511;
|
||||||
neighbor $node as 64511;
|
neighbor $node as 64511;
|
||||||
multihop;
|
multihop;
|
||||||
|
EOF
|
||||||
|
|
||||||
|
if [[ "${client_peers[@]}" =~ "${node}" ]]; then
|
||||||
|
cat >> /etc/bird/bird.conf <<EOF
|
||||||
|
description "Client $node";
|
||||||
rr client;
|
rr client;
|
||||||
|
EOF
|
||||||
|
else
|
||||||
|
cat >> /etc/bird/bird.conf <<EOF
|
||||||
|
description "Route Reflector $node";
|
||||||
|
EOF
|
||||||
|
fi
|
||||||
|
|
||||||
|
cat >> /etc/bird/bird.conf <<EOF
|
||||||
|
rr cluster id 1.2.3.4;
|
||||||
import all;
|
import all;
|
||||||
export all;
|
export all;
|
||||||
source address ${this_node_address};
|
source address ${this_node_address};
|
||||||
}
|
}
|
||||||
|
|
||||||
EOF
|
EOF
|
||||||
fi
|
|
||||||
done
|
done
|
||||||
|
|
||||||
# Restart BIRD with the new config.
|
# Restart BIRD with the new config.
|
||||||
|
|
|
@ -1,16 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
# Copyright 2015 Metaswitch Networks
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
with open("/etc/compute.yaml", "r") as f:
|
|
||||||
config = yaml.safe_load(f)
|
|
||||||
|
|
||||||
for node in config["nodes"]:
|
|
||||||
if node["role"] == "primary-controller":
|
|
||||||
controller_ip = node["internal_address"]
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
controller_ip = None
|
|
||||||
|
|
||||||
print controller_ip
|
|
|
@ -0,0 +1,32 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# Copyright 2015 Metaswitch Networks
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from pluginutils import NODES_CONFIG
|
||||||
|
|
||||||
|
|
||||||
|
def main(node_roles):
|
||||||
|
with open(NODES_CONFIG, "r") as f:
|
||||||
|
config = yaml.safe_load(f)
|
||||||
|
|
||||||
|
node_ips = [node["internal_address"] for node in config["nodes"]
|
||||||
|
if node["role"] in node_roles]
|
||||||
|
|
||||||
|
return node_ips
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("node_role", choices=["compute", "controller"])
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
args.node_role = [args.node_role]
|
||||||
|
if args.node_role == ["controller"]:
|
||||||
|
args.node_role.append("primary-controller")
|
||||||
|
|
||||||
|
node_ips = main(args.node_role)
|
||||||
|
if node_ips:
|
||||||
|
print " ".join(node_ips)
|
||||||
|
|
|
@ -1,22 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
# Copyright 2015 Metaswitch Networks
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
from pluginutils import NODES_CONFIG
|
|
||||||
|
|
||||||
def main():
|
|
||||||
with open(NODES_CONFIG, "r") as f:
|
|
||||||
config = yaml.safe_load(f)
|
|
||||||
|
|
||||||
# The route reflector should only peer with compute nodes.
|
|
||||||
peer_ips = [node["internal_address"] for node in config["nodes"]
|
|
||||||
if node["role"] == "compute"]
|
|
||||||
|
|
||||||
return peer_ips
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
peer_ips = main()
|
|
||||||
if peer_ips:
|
|
||||||
print " ".join(peer_ips)
|
|
||||||
|
|
|
@ -0,0 +1,42 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# Copyright 2015 Metaswitch Networks
|
||||||
|
|
||||||
|
this_node_address=$(python get_node_ip.py `hostname`)
|
||||||
|
controller_node_addresses=$(python get_node_ips_by_role.py controller)
|
||||||
|
|
||||||
|
for node_address in ${controller_node_addresses[@]}
|
||||||
|
do
|
||||||
|
initial_cluster+="${node_address}=http://${node_address}:2380,"
|
||||||
|
done
|
||||||
|
|
||||||
|
initial_cluster=${initial_cluster::-1} # remove trailing comma
|
||||||
|
|
||||||
|
service etcd stop
|
||||||
|
rm -rf /var/lib/etcd/*
|
||||||
|
awk '/exec \/usr\/bin\/etcd/{while(getline && $0 != ""){}}1' /etc/init/etcd.conf > tmp
|
||||||
|
mv tmp /etc/init/etcd.conf
|
||||||
|
cat << EXEC_CMD >> /etc/init/etcd.conf
|
||||||
|
exec /usr/bin/etcd -name ${this_node_address} \\
|
||||||
|
-advertise-client-urls "http://${this_node_address}:2379,http://${this_node_address}:4001" \\
|
||||||
|
-listen-client-urls "http://0.0.0.0:2379,http://0.0.0.0:4001" \\
|
||||||
|
-listen-peer-urls "http://0.0.0.0:2380" \\
|
||||||
|
-initial-advertise-peer-urls "http://${this_node_address}:2380" \\
|
||||||
|
-initial-cluster-token fuel-cluster-1 \\
|
||||||
|
-initial-cluster ${initial_cluster} \\
|
||||||
|
-initial-cluster-state new
|
||||||
|
|
||||||
|
EXEC_CMD
|
||||||
|
service etcd start
|
||||||
|
|
||||||
|
retry_count=0
|
||||||
|
while [[ $retry_count < 5 ]]; do
|
||||||
|
etcdctl cluster-health
|
||||||
|
if [[ $? == 0 ]]; then
|
||||||
|
break
|
||||||
|
else
|
||||||
|
((retry_count++))
|
||||||
|
service etcd restart
|
||||||
|
sleep 2
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
Loading…
Reference in New Issue