Refactors Heat templates for Container Networking Model

Refactors Heat templates to support multiple container network
implementations through the network_driver parameter. Keeps
Flannel and its associated parameters as-is for backwards
compatibility.

Partially-Implements: blueprint heat-network-refactor
Change-Id: If00adf5db9187b4ade33785bc19becedb4e688a0
This commit is contained in:
Daneyon Hansen 2015-08-19 18:53:27 +00:00
parent 4381c685b3
commit c18c10bed1
13 changed files with 174 additions and 119 deletions

View File

@ -21,7 +21,3 @@ sed -i '
sed -i '
/^KUBELET_ADDRESSES=/ s/=.*/="--machines='""'"/
' /etc/kubernetes/controller-manager
sed -i '
/^FLANNEL_ETCD=/ s/=.*/="http:\/\/127.0.0.1:2379"/
' /etc/sysconfig/flanneld

View File

@ -27,9 +27,11 @@ sed -i '
/^KUBE_MASTER=/ s/=.*/="--master='"$KUBE_MASTER_IP"':8080"/
' /etc/kubernetes/apiserver
if [ "$NETWORK_DRIVER" == "flannel" ]; then
sed -i '
/^FLANNEL_ETCD=/ s|=.*|="http://'"$ETCD_SERVER_IP"':2379"|
' /etc/sysconfig/flanneld
fi
cat >> /etc/environment <<EOF
KUBERNETES_MASTER=http://$KUBE_MASTER_IP:8080

View File

@ -1,58 +0,0 @@
#cloud-config
merge_how: dict(recurse_array)+list(append)
bootcmd:
- mkdir -p /etc/systemd/system/docker.service.d
- mkdir -p /etc/systemd/system/flanneld.service.d
write_files:
- path: /usr/local/bin/flannel-docker-bridge
owner: "root:root"
permissions: "0755"
content: |
#!/bin/sh
if ! [ "$FLANNEL_SUBNET" ] && [ "$FLANNEL_MTU" ] ; then
echo "ERROR: missing required environment variables." >&2
exit 1
fi
mkdir -p /run/flannel/
cat > /run/flannel/docker <<EOF
DOCKER_NETWORK_OPTIONS="--bip=$FLANNEL_SUBNET --mtu=$FLANNEL_MTU"
EOF
- path: /etc/systemd/system/flannel-docker-bridge.service
owner: "root:root"
permissions: "0644"
content: |
[Unit]
After=flanneld.service
Before=docker.service
Requires=flanneld.service
[Service]
Type=oneshot
EnvironmentFile=/run/flannel/subnet.env
ExecStart=/usr/local/bin/flannel-docker-bridge
[Install]
WantedBy=docker.service
- path: /etc/systemd/system/docker.service.d/flannel.conf
owner: "root:root"
permissions: "0644"
content: |
[Unit]
Requires=flannel-docker-bridge.service
After=flannel-docker-bridge.service
[Service]
EnvironmentFile=/run/flannel/docker
- path: /etc/systemd/system/flanneld.service.d/flannel-docker-bridge.conf
owner: "root:root"
permissions: "0644"
content: |
[Unit]
Requires=flannel-docker-bridge.service
Before=flannel-docker-bridge.service
[Install]
Also=flannel-docker-bridge.service

View File

@ -10,7 +10,7 @@ ip link del docker0
# make sure we pick up any modified unit files
systemctl daemon-reload
for service in flanneld docker kubelet kube-proxy; do
for service in docker kubelet kube-proxy; do
echo "activating service $service"
systemctl enable $service
systemctl --no-block start $service

View File

@ -1,46 +0,0 @@
#cloud-config
merge_how: dict(recurse_array)+list(append)
write_files:
- path: /usr/local/bin/flannel-config
owner: "root:root"
permissions: "0755"
content: |
#!/bin/sh
: ${FLANNEL_JSON:=/etc/sysconfig/flannel-network.json}
if ! [ -f "$FLANNEL_JSON" ]; then
echo "ERROR: missing network configuration file" >&2
exit 1
fi
if ! [ "$FLANNEL_ETCD" ] && [ "$FLANNEL_ETCD_KEY" ]; then
echo "ERROR: missing required configuration" >&2
exit 1
fi
echo "creating flanneld config in etcd"
while ! curl -sf -L $FLANNEL_ETCD/v2/keys${FLANNEL_ETCD_KEY}/config \
-X PUT --data-urlencode value@${FLANNEL_JSON}; do
echo "waiting for etcd"
sleep 1
done
- path: /etc/systemd/system/flannel-config.service
owner: "root:root"
permissions: "0644"
content: |
[Unit]
After=etcd.service
Requires=etcd.service
[Service]
Type=oneshot
EnvironmentFile=/etc/sysconfig/flanneld
ExecStart=/usr/local/bin/flannel-config
[Install]
WantedBy=multi-user.target
runcmd:
- systemctl enable flannel-config
- systemctl start --no-block flannel-config

View File

@ -0,0 +1,57 @@
#!/bin/sh
. /etc/sysconfig/heat-params
if [ "$NETWORK_DRIVER" == "flannel" ]; then
. /etc/sysconfig/flanneld
FLANNEL_CONFIG_BIN=/usr/local/bin/flannel-config
FLANNEL_CONFIG_SERVICE=/etc/systemd/system/flannel-config.service
FLANNEL_JSON=/etc/sysconfig/flannel-network.json
echo "creating $FLANNEL_CONFIG_BIN"
cat > $FLANNEL_CONFIG_BIN <<EOF
#!/bin/sh
if ! [ -f "$FLANNEL_JSON" ]; then
echo "ERROR: missing network configuration file" >&2
exit 1
fi
if ! [ "$FLANNEL_ETCD" ] && [ "$FLANNEL_ETCD_KEY" ]; then
echo "ERROR: missing required configuration" >&2
exit 1
fi
echo "creating flanneld config in etcd"
while ! curl -sf -L $FLANNEL_ETCD/v2/keys${FLANNEL_ETCD_KEY}/config \
-X PUT --data-urlencode value@${FLANNEL_JSON}; do
echo "waiting for etcd"
sleep 1
done
EOF
cat > $FLANNEL_CONFIG_SERVICE <<EOF
[Unit]
After=etcd.service
Requires=etcd.service
[Service]
Type=oneshot
EnvironmentFile=/etc/sysconfig/flanneld
ExecStart=$FLANNEL_CONFIG_BIN
[Install]
WantedBy=multi-user.target
EOF
chown root:root $FLANNEL_CONFIG_BIN
chmod 0755 $FLANNEL_CONFIG_BIN
chown root:root $FLANNEL_CONFIG_SERVICE
chmod 0644 $FLANNEL_CONFIG_SERVICE
systemctl enable flannel-config
systemctl start --no-block flannel-config
fi # end if [ "$NETWORK_DRIVER" == "flannel" ]

View File

@ -0,0 +1,77 @@
#!/bin/sh
. /etc/sysconfig/heat-params
if [ "$NETWORK_DRIVER" == "flannel" ]; then
FLANNEL_DOCKER_BRIDGE_BIN=/usr/local/bin/flannel-docker-bridge
FLANNEL_DOCKER_BRIDGE_SERVICE=/etc/systemd/system/flannel-docker-bridge.service
DOCKER_FLANNEL_CONF=/etc/systemd/system/docker.service.d/flannel.conf
FLANNEL_DOCKER_BRIDGE_CONF=/etc/systemd/system/flanneld.service.d/flannel-docker-bridge.conf
mkdir -p /etc/systemd/system/docker.service.d
mkdir -p /etc/systemd/system/flanneld.service.d
cat >> $FLANNEL_DOCKER_BRIDGE_BIN <<EOF
#!/bin/sh
if ! [ "\$FLANNEL_SUBNET" ] && [ "\$FLANNEL_MTU" ] ; then
echo "ERROR: missing required environment variables." >&2
exit 1
fi
mkdir -p /run/flannel/
cat > /run/flannel/docker <<EOF
DOCKER_NETWORK_OPTIONS="--bip=\$FLANNEL_SUBNET --mtu=\$FLANNEL_MTU"
EOF
chown root:root $FLANNEL_DOCKER_BRIDGE_BIN
chmod 0755 $FLANNEL_DOCKER_BRIDGE_BIN
cat >> $FLANNEL_DOCKER_BRIDGE_SERVICE <<EOF
[Unit]
After=flanneld.service
Before=docker.service
Requires=flanneld.service
[Service]
Type=oneshot
EnvironmentFile=/run/flannel/subnet.env
ExecStart=/usr/local/bin/flannel-docker-bridge
[Install]
WantedBy=docker.service
EOF
chown root:root $FLANNEL_DOCKER_BRIDGE_SERVICE
chmod 0644 $FLANNEL_DOCKER_BRIDGE_SERVICE
cat >> $DOCKER_FLANNEL_CONF <<EOF
[Unit]
Requires=flannel-docker-bridge.service
After=flannel-docker-bridge.service
[Service]
EnvironmentFile=/run/flannel/docker
EOF
chown root:root $DOCKER_FLANNEL_CONF
chmod 0644 $DOCKER_FLANNEL_CONF
cat >> $FLANNEL_DOCKER_BRIDGE_CONF <<EOF
[Unit]
Requires=flannel-docker-bridge.service
Before=flannel-docker-bridge.service
[Install]
Also=flannel-docker-bridge.service
EOF
chown root:root $FLANNEL_DOCKER_BRIDGE_CONF
chmod 0644 $FLANNEL_DOCKER_BRIDGE_CONF
echo "activating service flanneld"
systemctl enable flanneld
systemctl --no-block start flanneld
fi

View File

@ -6,6 +6,7 @@ write_files:
permissions: "0644"
content: |
KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
NETWORK_DRIVER="$NETWORK_DRIVER"
FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR"
FLANNEL_NETWORK_SUBNETLEN="$FLANNEL_NETWORK_SUBNETLEN"
FLANNEL_USE_VXLAN="$FLANNEL_USE_VXLAN"

View File

@ -9,4 +9,4 @@ write_files:
KUBE_MASTER_IP="$KUBE_MASTER_IP"
ETCD_SERVER_IP="$ETCD_SERVER_IP"
DOCKER_VOLUME="$DOCKER_VOLUME"
NETWORK_DRIVER="$NETWORK_DRIVER"

View File

@ -1,9 +1,19 @@
#!/bin/sh
. /etc/sysconfig/heat-params
if [ "$NETWORK_DRIVER" == "flannel" ]; then
. /etc/sysconfig/flanneld
FLANNEL_JSON=/etc/sysconfig/flannel-network.json
FLANNELD_CONFIG=/etc/sysconfig/flanneld
FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR"
FLANNEL_NETWORK_SUBNETLEN="$FLANNEL_NETWORK_SUBNETLEN"
FLANNEL_USE_VXLAN="$FLANNEL_USE_VXLAN"
sed -i '
/^FLANNEL_ETCD=/ s/=.*/="http:\/\/127.0.0.1:2379"/
' /etc/sysconfig/flanneld
if [ "$FLANNEL_USE_VXLAN" == "true" ]; then
use_vxlan=1
@ -30,3 +40,4 @@ cat >> $FLANNEL_JSON <<EOF
}
EOF
fi

View File

@ -56,6 +56,11 @@ parameters:
address range used by kubernetes for service portals
default: 10.254.0.0/16
network_driver:
type: string
description: network driver to use for instantiating container networks
default: flannel
flannel_network_cidr:
type: string
description: network range for flannel overlay network
@ -208,6 +213,7 @@ resources:
external_network: {get_param: external_network}
kube_allow_priv: {get_param: kube_allow_priv}
wait_condition_timeout: {get_param: wait_condition_timeout}
network_driver: {get_param: network_driver}
flannel_network_cidr: {get_param: flannel_network_cidr}
flannel_network_subnetlen: {get_param: flannel_network_subnetlen}
flannel_use_vxlan: {get_param: flannel_use_vxlan}
@ -240,6 +246,7 @@ resources:
minion_flavor: {get_param: minion_flavor}
fixed_network: {get_resource: fixed_network}
fixed_subnet: {get_resource: fixed_subnet}
network_driver: {get_param: network_driver}
kube_master_ip: {get_attr: [api_pool, vip, address]}
etcd_server_ip: {get_attr: [etcd_pool, vip, address]}
external_network: {get_param: external_network}

View File

@ -70,6 +70,9 @@ parameters:
fixed_subnet:
type: string
description: Subnet from which to allocate fixed addresses.
network_driver:
type: string
description: network driver to use for instantiating container networks
wait_condition_timeout:
type: number
description : >
@ -140,6 +143,7 @@ resources:
template: {get_file: fragments/write-heat-params-master.yaml}
params:
"$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
"$NETWORK_DRIVER": {get_param: network_driver}
"$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
"$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
"$FLANNEL_USE_VXLAN": {get_param: flannel_use_vxlan}
@ -158,17 +162,17 @@ resources:
group: ungrouped
config: {get_file: fragments/configure-kubernetes-master.sh}
write_flannel_config:
write_network_config:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/write-flannel-config.sh}
config: {get_file: fragments/write-network-config.sh}
flannel_config_service:
network_config_service:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/flannel-config.service.yaml}
config: {get_file: fragments/network-config-service.sh}
enable_services:
type: OS::Heat::SoftwareConfig
@ -216,8 +220,8 @@ resources:
- config: {get_resource: kube_user}
- config: {get_resource: configure_kubernetes}
- config: {get_resource: enable_services}
- config: {get_resource: write_flannel_config}
- config: {get_resource: flannel_config_service}
- config: {get_resource: write_network_config}
- config: {get_resource: network_config_service}
- config: {get_resource: kube_examples}
- config: {get_resource: master_wc_notify}

View File

@ -53,6 +53,9 @@ parameters:
fixed_subnet:
type: string
description: Subnet from which to allocate fixed addresses.
network_driver:
type: string
description: network driver to use for instantiating container networks
wait_condition_timeout:
type: number
description : >
@ -96,6 +99,7 @@ resources:
$KUBE_MASTER_IP: {get_param: kube_master_ip}
$ETCD_SERVER_IP: {get_param: etcd_server_ip}
$DOCKER_VOLUME: {get_resource: docker_volume}
$NETWORK_DRIVER: {get_param: network_driver}
configure_docker_storage:
type: OS::Heat::SoftwareConfig
@ -127,11 +131,11 @@ resources:
group: ungrouped
config: {get_file: fragments/kube-examples.yaml}
docker_service:
network_service:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/docker.service.yaml}
config: {get_file: fragments/network-service.sh}
enable_services:
type: OS::Heat::SoftwareConfig
@ -168,7 +172,7 @@ resources:
- config: {get_resource: configure_docker_storage}
- config: {get_resource: kube_register}
- config: {get_resource: configure_kubernetes_minion}
- config: {get_resource: docker_service}
- config: {get_resource: network_service}
- config: {get_resource: enable_services}
- config: {get_resource: minion_wc_notify}