contrib: Add devstack-heat

Devstack heat is a convenience set of heat templates and cli to get
devstack deployments without hassles when you already have access to an
OpenStack cloud.

Change-Id: I2de8d12b2e8c805a86d03311857ca9be41ff0c75
Signed-off-by: Antoni Segura Puimedon <antonisp@celebdor.com>

contrib/devstack-heat/README.rst

@@ -0,0 +1,86 @@
+Kuryr Heat Templates
+====================
+
+This set of scripts and Heat templates are useful for deploying devstack
+scenarios. It handles the creation of an allinone devstack nova instance and its
+networking needs.
+
+Prerequisites
+~~~~~~~~~~~~~
+
+Packages to install on the host you run devstack-heat (not on the cloud server):
+
+* jq
+* openstack-cli
+
+If you want to run devstack from the master commit, this application requires a
+github token due to the github api rate limiting:
+
+You can generate one without any permissions at:
+
+    https://github.com/settings/tokens/new
+
+Then put it in your ~/.bashrc an ENV variable called DEVSTACK_HEAT_GH_TOKEN like
+so:
+
+echo "export DEVSTACK_HEAT_GH_TOKEN=my_token" >> ~/.bashrc
+
+After creating the instance, devstack-heat will immediately start creating a
+devstack `stack` user and using devstack to stack kuryr-kubernetes. When it is
+finished, there'll be a file names `/opt/stack/ready`.
+
+How to run
+~~~~~~~~~~
+
+In order to run it, make sure that you have sourced your OpenStack cloud
+provider openrc file and tweaked `hot/parameters.yml` to your liking then launch
+with::
+
+    ./devstack-heat stack
+
+This will deploy the latest master. You can also specify specific gerrit change
+numbers::
+
+    ./devstack-heat stack 465657
+
+To obtain this number, just look for example at the following change::
+
+    https://review.openstack.org/#/c/465657
+
+In this instance, the number to pass to the *stack* subcommand is 466291.
+
+This will create a stack named *gerrit_465657*. Further devstack-heat
+subcommands should be called with the whole name of the stack, i.e.,
+*gerrit_465657*.
+
+Getting inside the deployment
+-----------------------------
+
+You can then ssh into the deployment in two ways::
+
+    ./devstack-heat show name_of_my_stack
+
+Write down the FIP it tells you and then::
+
+    ./devstack-heat getkey name_of_my_stack > ~/name_of_my_stack.pem
+
+Finally to get in (use the default username for the distro of your chosen
+glance image, in the example below centos)::
+
+    ssh -i ~/name_of_my_stack.pem centos@obtained_fip
+
+Alternatively, if you wait a bit, devstack-heat will have set up the devstack
+stack user and you can just do::
+
+    ./devstack-heat ssh name_of_my_stack
+
+
+To delete the deployment::
+
+    ./devstack-heat unstack name_of_my_stack
+
+Supported images
+----------------
+
+It should work with the latest centos7 image. It is not tested with the latest
+ubuntu 16.04 cloud image but it will probably work.

contrib/devstack-heat/devstack-heat

@@ -0,0 +1,125 @@
+#!/bin/bash
+
+function sub_stack() {
+    local latest_commit
+    local deployment=${1:-master_}
+    local tmpdir
+
+    if [[ "${deployment}" == "master_" ]]; then
+        if [[ "$DEVSTACK_HEAT_GH_TOKEN" == "" ]]; then
+            set -e
+            echo "Didn't find a Github token in ENV var DEVSTACK_HEAT_GH_TOKEN. Falling back to cloning repo..."
+            tmpdir=$(mktemp -d)
+            git clone https://github.com/openstack/kuryr-kubernetes "${tmpdir}/kuryr-kubernetes"
+            pushd "${tmpdir}/kuryr-kubernetes"
+            latest_commit=$(git rev-parse HEAD)
+            popd
+            rm -fr "${tmpdir}"
+            set +e
+        else
+            latest_commit=$(curl -s -H "Authorization: token $DEVSTACK_HEAT_GH_TOKEN" https://api.github.com/repos/openstack/kuryr-kubernetes/commits/master | jq -r '.sha')
+        fi
+        if [[ "$latest_commit" == "null" ]]; then
+            echo "Couldn't get a valid master commit"
+            exit 1
+        fi
+        deployment="${deployment}${latest_commit}"
+    else
+        deployment="gerrit_${deployment}"
+    fi
+
+    # create stack
+    read -p "Deploying the stack ${deployment}[y/N]?" -n 1 -r
+    echo
+    if [[ ! "$REPLY" =~ ^[Yy]$ ]]; then
+        exit 1
+    fi
+    echo "Starting..."
+
+    openstack stack create -e hot/parameters.yml -t hot/devstack_heat_template.yml "$deployment"
+
+    sub_show "$deployment"
+}
+
+function sub_unstack()
+{
+    local deployment
+    deployment=${1:-master_}
+    if [[ "${deployment}" == "master_" ]]; then
+        echo "You must put the whole stack name for unstacking"
+        exit 1
+    fi
+    openstack stack delete "$deployment"
+
+}
+
+function sub_show() {
+    local deployment
+
+    deployment=${1:-master_}
+    if [[ "${deployment}" == "master_" ]]; then
+        echo "You must put the whole stack name for showing the stack resources"
+        exit 1
+    fi
+    echo "VM subnet:     $(openstack stack output show "${deployment}" vm_subnet -f json | jq -r '.output_value')"
+    echo "Nodes FIPs:    $(openstack stack output show "${deployment}" node_fips -f json | jq -r '.output_value' | jq -r '.[]?' | xargs echo)"
+    printf "\n"
+}
+
+function sub_getkey() {
+    local deployment
+    deployment=${1:-master_}
+    if [[ "${deployment}" == "master_" ]]; then
+        echo "You must put the whole stack name for getting the key"
+        exit 1
+    fi
+
+    openstack stack output show "${deployment}" master_key_priv -f json | jq -r '.output_value'
+}
+
+function sub_ssh() {
+    local deployment
+    local key
+    local fip
+
+    deployment=${1:-master_}
+    if [[ "${deployment}" == "master_" ]]; then
+        echo "You must put the whole stack name for getting the key"
+        exit 1
+    fi
+    key="${deployment}.pem"
+    fip=$(openstack stack output show "${deployment}" node_fips -f json | jq -r '.output_value' | jq -r '.[]?' | xargs echo)
+    sub_getkey "${deployment}" > "${key}"
+    chmod 0600 "${key}"
+    ssh -i "${deployment}.pem"  -o "StrictHostKeyChecking no" "stack@${fip}"
+    exit $?
+}
+
+
+function sub_help() {
+    local myname
+    myname=$(basename "$0")
+    printf "Usage: %s <subcommand> [options]\n" "$myname"
+    printf "Subcommands:\n"
+    printf "    stack gerrit_change_number     Create Heat stack\n"
+    printf "    unstack my_stack_name   Delete Heat stack\n"
+    printf "    show my_stack_name      Show important info about the deployed Heat stack\n"
+    printf "    getkey my_stack_name    Output the Heat stack instances privkey to stdout\n"
+    printf "    ssh my_stack_name       gets the key and sshs into the stack user of the Stack\n"
+}
+
+command=$1
+case $command in
+    "" | "-h" | "--help")
+        sub_help
+        ;;
+    *)
+        shift
+        "sub_${command}" "$@"
+        if [ $? = 127 ]; then
+            echo "Error: '$command' is not a known $(basename "$0") command." >&2
+            echo "       Run \'$(basename "$0")  --help\' for a list of known subcommands." >&2
+            exit 1
+        fi
+        ;;
+esac

contrib/devstack-heat/hot/devstack_heat_template.yml

@@ -0,0 +1,85 @@
+heat_template_version: 2015-10-15
+
+description: Simple template to deploy kuryr resources
+
+parameters:
+  image:
+    type: string
+    label: Image name or ID
+    description: Image to be used for the kuryr nodes
+    default: centos7
+  flavor:
+    type: string
+    label: Flavor
+    description: Flavor to be used for the masters
+    default: m1.small
+  worker_flavor:
+    type: string
+    label: Flavor
+    description: Flavor to be used for the workers
+    default: m1.small
+  public_net:
+    type: string
+    description: public network for the instances
+    default: public
+  vm_net_cidr:
+    type: string
+    description: vm_net network address (CIDR notation)
+    default: 10.11.0.0/24
+  vm_net_gateway:
+    type: string
+    description: vm_net network gateway address
+    default: 10.11.0.1
+  node_num:
+    type: number
+    description: Number of VMs
+    default: 1
+
+resources:
+  network:
+    type: OS::Kuryr::DevstackNetworking
+    properties:
+      public_net: { get_param: public_net }
+      vm_net_cidr: { get_param: vm_net_cidr }
+      vm_net_gateway: { get_param: vm_net_gateway }
+
+  master_key:
+    type: OS::Nova::KeyPair
+    properties:
+      name: { get_param: 'OS::stack_name' }
+      save_private_key: true
+
+  nodes:
+    type: OS::Heat::ResourceGroup
+    properties:
+      count: { get_param: node_num }
+      resource_def:
+        type: OS::Kuryr::DevstackNode
+        properties:
+          public_net: { get_param: public_net }
+          image: { get_param: image }
+          flavor: { get_param: flavor }
+          key: { get_resource: master_key }
+          private_key: { get_attr: [master_key, private_key] }
+          public_key: { get_attr: [master_key, public_key] }
+          vm_net: { get_attr: [network, vm_net_id] }
+          vm_subnet: { get_attr: [network, vm_subnet_id] }
+          k8s_api_sg: { get_attr: [network, k8s_api_sg_id] }
+          name:
+            str_replace:
+              template: "__stack__/vm-%index%"
+              params:
+                __stack__: { get_param: 'OS::stack_name' }
+          change_number: { get_param: 'OS::stack_name' }
+
+outputs:
+  node_fips:
+    value: { get_attr: [nodes, node_fip] }
+  vm_subnet:
+    value: { get_attr: [network, vm_subnet_id] }
+  k8s_api_sg:
+    value: { get_attr: [network, k8s_api_sg_id] }
+  master_key_pub:
+    value: { get_attr: [master_key, public_key] }
+  master_key_priv:
+    value: { get_attr: [master_key, private_key] }

contrib/devstack-heat/hot/distro_deps.sh

@@ -0,0 +1,16 @@
+distro=$(awk -F'=' '/^ID=/ {print $2}' /etc/os-release)
+distro="${distro%\"}"
+distro="${distro#\"}"
+
+if [[ "$distro" =~ centos|fedora ]]; then
+    yum install -y git python-devel
+    yum group install -y Development Tools
+    if [[ "$distro" == "centos" ]]; then
+        yum install -y epel-release
+        sed -i -e '/Defaults    requiretty/{ s/.*/# Defaults    requiretty/ }' /etc/sudoers
+    fi
+    yum install -y jq
+
+elif [[ "$distro" =~ ubuntu|debian ]]; then
+    apt-get install -y build-essential git python-dev jq
+fi

contrib/devstack-heat/hot/networking_deployment.yaml

@@ -0,0 +1,80 @@
+heat_template_version: 2014-10-16
+
+description: Simple template to deploy kuryr resources
+
+parameters:
+  public_net:
+    type: string
+    label: public net ID
+    description: Public network for the node FIPs
+  vm_net_cidr:
+    type: string
+    description: vm_net network address (CIDR notation)
+  vm_net_gateway:
+    type: string
+    description: vm_net network gateway address
+
+resources:
+  vm_net:
+    type: OS::Neutron::Net
+    properties:
+      name:
+        str_replace:
+          template: __stack__/vm_net
+          params:
+            __stack__: { get_param: 'OS::stack_name' }
+
+  vm_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      network_id: { get_resource: vm_net }
+      cidr: { get_param: vm_net_cidr }
+      gateway_ip: { get_param: vm_net_gateway }
+      name:
+        str_replace:
+          template: __stack__/vm_subnet
+          params:
+            __stack__: { get_param: 'OS::stack_name' }
+
+  kuryr_router:
+    type: OS::Neutron::Router
+    properties:
+      external_gateway_info:
+        network: { get_param: public_net }
+      name:
+        str_replace:
+          template: __stack__/router
+          params:
+            __stack__: { get_param: 'OS::stack_name' }
+
+  kr_vm_iface:
+    type: OS::Neutron::RouterInterface
+    properties:
+      router_id: { get_resource: kuryr_router }
+      subnet_id: { get_resource: vm_subnet }
+
+  k8s_api_sg:
+    type: OS::Neutron::SecurityGroup
+    properties:
+      name: K8s_api_sg
+      description: Ping and SSH
+      rules:
+      - protocol: icmp
+      - ethertype: IPv4
+        remote_mode: remote_group_id
+      - ethertype: IPv6
+        remote_mode: remote_group_id
+      - protocol: tcp
+        port_range_min: 22
+        port_range_max: 22
+      - protocol: tcp
+        port_range_min: 8080
+        port_range_max: 8080
+
+outputs:
+  vm_net_id:
+    value: { get_resource: vm_net }
+  vm_subnet_id:
+    value: { get_resource: vm_subnet }
+  k8s_api_sg_id:
+    value: { get_resource: k8s_api_sg }

contrib/devstack-heat/hot/node.yaml

@@ -0,0 +1,152 @@
+heat_template_version: 2015-10-15
+
+description: template to deploy devstack nodes
+
+parameters:
+  public_net:
+    type: string
+    label: public net ID
+    description: Public network for the node FIPs
+  image:
+    type: string
+    label: Image name or ID
+    description: Image to be used for the kuryr nodes
+  flavor:
+    type: string
+    label: Flavor
+    description: Flavor to be used for the image
+    default: m1.small
+  key:
+    type: string
+    label: key name
+    description: Keypair to be used for the instance
+  public_key:
+    type: string
+    label: key content for stack user authorized_keys
+    description: private key to configure all nodes
+  private_key:
+    type: string
+    label: key content to access other nodes
+    description: private key to configure all nodes
+  vm_net:
+    type: string
+    label: VM Network
+    description: Neutron network for VMs
+  vm_subnet:
+    type: string
+    label: VM Subnet
+    description: Neutron subnet for VMs
+  k8s_api_sg:
+    type: string
+    label: kubernetes API sg
+    description: Security Group for Kubernetes API
+  name:
+    type: string
+    label: Instance name
+    description: devstack node instance name
+  change_number:
+    type: string
+    label: Gerrit change number
+    description: Either the gerrit change number or master-sha_of_the_commit
+
+resources:
+  instance_port:
+    type: OS::Neutron::Port
+    properties:
+      network: { get_param: vm_net }
+      security_groups:
+        - default
+        - { get_param: k8s_api_sg }
+      fixed_ips:
+        - subnet: { get_param: vm_subnet }
+
+  instance_fip:
+    type: OS::Neutron::FloatingIP
+    properties:
+      floating_network: { get_param: public_net }
+      port_id: { get_resource: instance_port }
+
+  instance:
+    type: OS::Nova::Server
+    properties:
+      name: { get_param: name }
+      image: { get_param: image }
+      flavor: { get_param: flavor }
+      key_name: { get_param: key }
+      networks:
+        - port: { get_resource: instance_port }
+      user_data_format: RAW
+      user_data:
+        str_replace:
+          params:
+            __distro_deps__: { get_file: distro_deps.sh }
+            __change_number__: { get_param: change_number }
+            __pubkey__: { get_param: public_key }
+          template: |
+            #!/bin/bash
+            set -ex
+
+            # Wait a bit for connectivity
+            sleep 10
+
+            # Deps for devstack
+            __distro_deps__
+
+            # Stack user config
+            useradd -s /bin/bash -d /opt/stack -m stack
+            mkdir /opt/stack/.ssh
+            cat > /opt/stack/.ssh/authorized_keys << EOF
+            __pubkey__
+            EOF
+            echo "stack ALL=(ALL) NOPASSWD: ALL" | tee /etc/sudoers.d/stack
+
+            # Stacking
+            sudo -i -u stack /bin/bash - <<"EOF"
+            function get_from_gerrit() {
+                local stackname
+                local change_number
+                local ref
+
+                stackname="__change_number__"
+                change_number=${stackname#*_}
+                echo "Finding latest ref for change ${change_number}"
+                ref=$(curl -s "https://review.openstack.org/changes/${change_number}?o=CURRENT_REVISION" | tail -n +2 | jq -r '.revisions[].ref')
+                echo "Fetching ref ${ref}"
+                git fetch https://git.openstack.org/openstack/kuryr-kubernetes "${ref}" && git checkout FETCH_HEAD
+            }
+
+            function get_from_sha() {
+                local commit_sha
+
+                commit_sha=$(echo "__change_number__" | cut -d'_' -f2)
+                echo "Sha to fetch: ${commit_sha}"
+                git checkout "$commit_sha"
+            }
+
+            cd /opt/stack
+            git clone https://git.openstack.org/openstack-dev/devstack
+            git clone https://github.com/openstack/kuryr-kubernetes
+            pushd kuryr-kubernetes
+
+            if [[ "__change_number__" =~ ^master.* ]]; then
+                get_from_sha
+            elif [[ "__change_number__" =~ ^gerrit.* ]]; then
+                get_from_gerrit
+            else
+                echo "Unrecognized stack name. Quitting..."
+                exit 1
+            fi
+            popd
+            pushd devstack
+
+            # The change is already downloaded, do not reclone
+            sed -e 's/# RECLONE=/RECLONE=/' /opt/stack/kuryr-kubernetes/devstack/local.conf.sample > /opt/stack/devstack/local.conf
+            ./stack.sh
+            popd
+            touch ready
+            EOF
+
+outputs:
+  node_fip:
+    description: FIP address of the node
+    value: { get_attr: [instance_fip, floating_ip_address] }

contrib/devstack-heat/hot/parameters.yml

@@ -0,0 +1,11 @@
+parameters:
+  image: centos7
+  flavor: m1.medium
+  public_net: public
+  vm_net_cidr: 10.11.0.0/24
+  vm_net_gateway: 10.11.0.1
+  node_num: 1
+
+resource_registry:
+  OS::Kuryr::DevstackNetworking: networking_deployment.yaml
+  OS::Kuryr::DevstackNode: node.yaml