#!/usr/bin/env bash
set -eux

### --start_docs

## Setup devstack-gate's vxlan networking for multinode jobs
## =========================================================

## .. note::
##   The following steps are needed:
##   * Source in common environment (deploy.env)
##   * Clone devstack and devstack-gate for necessary scripts
##   * Create the vxlan bridge
##   * SSH into subnodes, bring the bridge device up on them
##   * Perform a ping check to see communication is possible over the bridge

## Prepare Your Environment
## ------------------------

## * Source tripleo-ci's deploy.env environment variables (if available)
## ::

if [ -f "{{ tripleo_root }}/tripleo-ci/deploy.env" ]; then
    source {{ tripleo_root }}/tripleo-ci/deploy.env
fi

## * Use the clone() function from tripleo-ci/scripts/tripleo.sh
## ::

function clone {

    local repo=$1

    echo "$0 requires $repo to be cloned at \$TRIPLEO_ROOT ({{ tripleo_root }})"

    mkdir -p {{ tripleo_root }}
    if [ ! -d {{ tripleo_root }}/$(basename $repo) ]; then
        echo "$repo not found at {{ tripleo_root }}/$repo, git cloning."
        pushd {{ tripleo_root }}
        git clone https://opendev.org/$repo
        popd
    else
        echo "$repo found at {{ tripleo_root }}/$repo, nothing to do."
    fi

}

# This function creates an internal gre bridge to connect all external
# network bridges across the compute and network nodes.
# bridge_name: Bridge name on each host for logical l2 network
#              connectivity.
# host_ip: ip address of the bridge host which is reachable for all peer
#          the hub for all of our spokes.
# set_ips: Whether or not to set l3 addresses on our logical l2 network.
#          This can be helpful for setting up routing tables.
# offset: starting value for gre tunnel key and the ip addr suffix
# The next two parameters are only used if set_ips is "True".
# pub_addr_prefix: The IPv4 address three octet prefix used to give compute
#                  nodes non conflicting addresses on the pub_if_name'd
#                  network. Should be provided as X.Y.Z. Offset will be
#                  applied to this as well as the below mask to get the
#                  resulting address.
# pub_addr_mask: the CIDR mask less the '/' for the IPv4 addresses used
#                above.
# every additional parameter is considered as a peer host (spokes)
#
# For OVS troubleshooting needs:
#   http://www.yet.org/2014/09/openvswitch-troubleshooting/
#
function ovs_vxlan_bridge {
    if is_suse; then
        local ovs_package='openvswitch'
        local ovs_service='openvswitch'
    elif is_fedora; then
        local ovs_package='openvswitch openstack-selinux'
        local ovs_service='openvswitch'
    elif uses_debs; then
        local ovs_package='openvswitch-switch'
        local ovs_service='openvswitch-switch'
    else
        echo "Unsupported platform, can't determine openvswitch service"
        exit 1
    fi
    local install_ovs_deps="source {{ tripleo_root }}/devstack/functions-common; \
                            install_package ${ovs_package}; \
                            restart_service ${ovs_service}"
    local mtu=1450
    local bridge_name=$1
    local host_ip=$2
    local set_ips=$3
    local offset=$4
    if [[ "$set_ips" == "True" ]] ; then
        local pub_addr_prefix=$5
        local pub_addr_mask=$6
        shift 6
    else
        shift 4
    fi
    local peer_ips=$@
    # neutron uses 1:1000 with default devstack configuration, avoid overlap
    local additional_vni_offset=1000000
    eval $install_ovs_deps
    # create a bridge, just like you would with 'brctl addbr'
    # if the bridge exists, --may-exist prevents ovs from returning an error
    sudo ovs-vsctl --may-exist add-br $bridge_name
    # as for the mtu, look for notes on lp#1301958 in devstack-vm-gate.sh
    sudo ip link set mtu $mtu dev $bridge_name
    if [[ "$set_ips" == "True" ]] ; then
        echo "Set bridge: ${bridge_name}"
        if ! sudo ip addr show dev ${bridge_name} | grep -q \
            ${pub_addr_prefix}.${offset}/${pub_addr_mask} ; then
                sudo ip addr add ${pub_addr_prefix}.${offset}/${pub_addr_mask} \
                    dev ${bridge_name}
        fi
    fi
    sudo ip link set dev $bridge_name up
    for node_ip in $peer_ips; do
        offset=$(( offset+1 ))
        vni=$(( offset + additional_vni_offset ))
        # For reference on how to setup a tunnel using OVS see:
        #   http://openvswitch.org/support/config-cookbooks/port-tunneling/
        # The command below is equivalent to the sequence of ip/brctl commands
        # where an interface of vxlan type is created first, and then plugged into
        # the bridge; options are command specific configuration key-value pairs.
        #
        # Create the vxlan tunnel for the Controller/Network Node:
        #  This establishes a tunnel between remote $node_ip to local $host_ip
        #  uniquely identified by a key $offset
        sudo ovs-vsctl --may-exist add-port $bridge_name \
            ${bridge_name}_${node_ip} \
            -- set interface ${bridge_name}_${node_ip} type=vxlan \
            options:remote_ip=${node_ip} \
            options:key=${vni} \
            options:local_ip=${host_ip}
        # Now complete the vxlan tunnel setup for the Compute Node:
        #  Similarly this establishes the tunnel in the reverse direction
        remote_command $node_ip "$install_ovs_deps"
        remote_command $node_ip sudo ovs-vsctl --may-exist add-br $bridge_name
        remote_command $node_ip sudo ip link set mtu $mtu dev $bridge_name
        remote_command $node_ip sudo ovs-vsctl --may-exist add-port $bridge_name \
            ${bridge_name}_${host_ip} \
            -- set interface ${bridge_name}_${host_ip} type=vxlan \
            options:remote_ip=${host_ip} \
            options:key=${vni} \
            options:local_ip=${node_ip}
        if [[ "$set_ips" == "True" ]] ; then
            if ! remote_command $node_ip sudo ip addr show dev ${bridge_name} | \
                grep -q ${pub_addr_prefix}.${offset}/${pub_addr_mask} ; then
                    remote_command $node_ip sudo ip addr add \
                        ${pub_addr_prefix}.${offset}/${pub_addr_mask} \
                        dev ${bridge_name}
            fi
        fi
        remote_command $node_ip sudo ip link set dev $bridge_name up
    done
}

## Perform the Bridge Creation
## ---------------------------

## * Clone devstack and devstack-gate repositories for network setup scripts
## ::

clone openstack/devstack
clone openstack/devstack-gate

## * Source in devstack-gate's common utility functions
## ::

echo "Sourcing devstack-gate/functions.sh"
set +u
source {{ tripleo_root }}/devstack-gate/functions.sh
set -u

primary_node=$(cat /etc/nodepool/primary_node_private)
sub_nodes=$(cat /etc/nodepool/sub_nodes_private)

# Create OVS vxlan bridges
# If br-ctlplane already exists on this node, we need to bring it down
# first, then bring it back up after calling ovs_vxlan_bridge. This ensures
# that the route added to br-ex by ovs_vxlan_bridge will be preferred over
# the br-ctlplane route. If it's not preferred, multinode connectivity
# across the vxlan bridge will not work.
if [ -f /etc/sysconfig/network-scripts/ifcfg-br-ctlplane ]; then
    sudo ifdown br-ctlplane
fi

## * Set up the vxlan bridge to the subnode(s)
## ::

set +u
echo "Running ovs_vxlan_bridge"
ovs_vxlan_bridge {{ undercloud_local_interface }} $primary_node "True" 2 {{ vxlan_networking_addr_prefix }} {{ vxlan_networking_addr_netmask }} $sub_nodes
set -u

echo "Setting {{ undercloud_local_interface }} up on $primary_node"
sudo ip link set dev {{ undercloud_local_interface }} up
sudo ip link set dev {{ undercloud_local_interface }} mtu {{ vxlan_mtu }}
echo "MTU={{ vxlan_mtu }}" | sudo tee -a /etc/sysconfig/network-scripts/ifcfg-{{ undercloud_local_interface }}

if [ -f /etc/sysconfig/network-scripts/ifcfg-br-ctlplane ]; then
    sudo ifup br-ctlplane
    sudo ip link set dev br-ctlplane mtu {{ vxlan_mtu }}
    echo "MTU={{ vxlan_mtu }}" | sudo tee -a /etc/sysconfig/network-scripts/ifcfg-br-ctlplane
fi

## * Restart neutron-openvswitch-agent if it's enabled, since it may have
##   terminated when br-ctlplane was down
## ::

if [ "$(sudo systemctl is-enabled neutron-openvswitch-agent)" = 'enabled' ]; then
    sudo systemctl reset-failed neutron-openvswitch-agent
    sudo systemctl restart neutron-openvswitch-agent
fi

### --stop_docs