tripleo-common/scripts/upgrade-non-controller.sh

#!/bin/bash
#set -x
# Copyright 2016 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# Utility script that will be invoked by the operator as part of the documented
# upgrades workflow. Those roles that have disable_upgrade_deployment set to
# true will have had a /root/tripleo_upgrade_node.sh script delivered during
# the composable ansible upgrade steps. This tripleo_upgrade_node.sh is then
# invoked over ssh by the operator using the simple utility delivered in this
# file. See -h for options.
#
set -eu
set -o pipefail
SCRIPT_NAME=$(basename $0)

#can make the upgrade script overridable (if a different target will be used)
UPGRADE_SCRIPT=${UPGRADE_SCRIPT:-/root/tripleo_upgrade_node.sh}
#allow override incase the ssh user is not 'heat-admin' - must be able to sudo
STACK_NAME_DEFAULT="overcloud"
STACK_NAME=${STACK_NAME:-$STACK_NAME_DEFAULT}
UPGRADE_NODE_USER_DEFAULT="heat-admin"
UPGRADE_NODE_USER=${UPGRADE_NODE_USER:-$UPGRADE_NODE_USER_DEFAULT}
UPGRADE_NODE=""
QUERY_NODE=""
HOSTNAME=""
IP_ADDR=""
INVENTORY=""
ANSIBLE_OPTS="" # e.g. "--skip-tags validation"
function show_options {
    echo "Usage: $SCRIPT_NAME"
    echo
    echo "Options:"
    echo "  -h|--help                    -- print this help."
    echo "  -u|--upgrade <nova node>     -- nova node name or id or ctlplane IP"
    echo "                                  to upgrade."
    echo "  -q|--query <nova node>       -- check if the node is ACTIVE and tail"
    echo "                                  yum.log for any package update info."
    echo "  -s|--stack-name <stack name> -- the stack name, defaults to $STACK_NAME_DEFAULT"
    echo "  -I|--inventory <path>        -- use the specified tripleo ansible "
    echo "                                  inventory (yaml format)."
    echo "  -O|--ansible-opts \"opts\"   -- specify extra options to be passed "
    echo "                                  to ansible-playbook e.g. \"-vvv\" or "
    echo "                                  \"-vvv --skip-tags validation\"."
    echo "  -U|--overcloud-user <name>   -- the user with which to ssh to the"
    echo "                                  target upgrade node - defaults to"
    echo "                                  $UPGRADE_NODE_USER_DEFAULT."
    echo
    echo "Invoke the /root/tripleo_upgrade_node.sh script on roles that have"
    echo "the 'disable_upgrade_deployment' flag set true and then download and"
    echo "execute the upgrade and deployment steps ansible playbooks."
    echo
    echo "The tripleo_upgrade_node.sh is delivered to the 'disable_upgrade_deployment'"
    echo "nodes, when you execute the composable upgrade steps on the "
    echo "controlplane nodes (i.e. the first step of the upgrade process). "
    echo
    echo "This utility is then used by the operator to invoke the upgrade workflow"
    echo "by named node (nova name or uuid) on the 'disable_upgrade_deployment'"
    echo "nodes. You can use the nova UUID, name or an IP address on the provisioning"
    echo "network (e.g. for split stack deployments)."
    echo
    echo "Logfiles are generated in the"
    echo "current working directory by node name/UUID/IP as appropriate."
    echo
    echo "Example invocations:"
    echo
    echo "    upgrade-non-controller.sh --upgrade overcloud-compute-0 "
    echo "    upgrade-non-controller.sh -u 734eea90-087b-4f12-9cd9-4807da83ea78 "
    echo "    upgrade-non-controller.sh -u 192.168.24.15 "
    echo "    upgrade-non-controller.sh -U stack -u 192.168.24.15 -s overcloud"
    echo "    upgrade-non-controller.sh -U stack -u 192.168.24.16 \ "
    echo "                              -I /home/stack/tripleo-ansible-inventory.yaml"
    echo "    upgrade-non-controller.sh -U stack -u 192.168.24.16 \ "
    echo "                              -I /home/stack/tripleo-ansible-inventory.yaml \ "
    echo "                              -O \"-vvv --skip-tags validation\" "
    echo
    echo "You can run on multiple nodes in parallel: "

    echo "    for i in \$(seq 0 2); do "
    echo "      upgrade-non-controller.sh --upgrade overcloud-compute-\$i &"
    echo "      # Remove the '&' above to have these upgrade in sequence"
    echo "    done"
    echo
    exit $1
}

TEMP=`getopt -o h,u:,q:,s:,U:,I:,O: -l help,upgrade:,query:,stack-name:,overcloud-user:,inventory:,ansible-opts: -n $SCRIPT_NAME -- "$@"`

if [ $? != 0 ]; then
    echo "Terminating..." >&2
    exit 1
fi

# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"

while true ; do
    case "$1" in
        -h | --help ) show_options 0 >&2;;
        -u | --upgrade ) UPGRADE_NODE="$2" ; shift 2 ;;
        -q | --query ) QUERY_NODE="$2" ; shift 2 ;;
        -s | --stack-name ) STACK_NAME="$2" ; shift 2 ;;
        -U | --overcloud-user ) UPGRADE_NODE_USER="$2"; shift 2;;
        -I | --inventory ) INVENTORY="$2"; shift 2;;
        -O | --ansible-opts ) ANSIBLE_OPTS="$2"; shift 2;;
        -- ) shift ; break ;;
        * ) echo "Error: unsupported option $1." ; exit 1 ;;
    esac
done

LOGFILE="$SCRIPT_NAME"-$UPGRADE_NODE$QUERY_NODE
function log {
  echo "`date` $SCRIPT_NAME $1" 2>&1 | tee -a  $LOGFILE
}

if [[ -n $UPGRADE_NODE$QUERY_NODE ]]; then
    log "Logging to $LOGFILE"
fi

function reset_hostname_ip {
    HOSTNAME=""
    IP_ADDR=""
}

# find_node_by_name_id_or_ip expects one parameter that is the nova name,
# uuid or cltplane IP address of the node we want to upgrade.
# The function will try and determine the hostname and (ctlplane) IP address
# for that node and assign these values to the global HOSTNAME and IP_ADDR.
# These variables are thus reset at the outset.
function find_node_by_name_id_or_ip {
  reset_hostname_ip
  name_id_or_ip=$1
  # First try as a nova node name or UUID. Could also be an IP (split stack)
  set +e # dont want to fail if nova or ping does below
  nova_name=$(openstack server show $name_id_or_ip -f value -c name)
  if [[ -n $nova_name ]]; then
    set -e
    HOSTNAME=$nova_name
    addr_string=$(openstack server show $name_id_or_ip -f value -c addresses)
    IP_ADDR=${addr_string#*=} # remove the ctlpane from  ctlplane=192.168.24.11
    log "nova node $HOSTNAME found with IP $IP_ADDR "
  else
    log "$name_id_or_ip not known to nova. Trying it as an IP address"
    if ping -c1 $name_id_or_ip ; then
        set -e
        HOSTNAME=$(ssh $UPGRADE_NODE_USER@$name_id_or_ip hostname)
        IP_ADDR=$name_id_or_ip
        log "node $HOSTNAME found with address $IP_ADDR "
    else
        set -e
        log "ERROR $name_id_or_ip not known to nova or not responding to ping if it is an IP address. Exiting"
        exit 1
    fi
  fi
  set -e
}

# Generate static tripleo ansible inventory. This is mainly to deal with different
# ssh user for the ansible playbooks, e.g. in a split stack environment.
# $1 is the path for to write the tripleo-ansible-inventory to
function get_static_inventory {
  local config_dir=$1
  if [ -d "$config_dir" ] ;then
    local inventory_args=" --static-yaml-inventory $config_dir/tripleo-ansible-inventory.yaml"
    if [[ $UPGRADE_NODE_USER != $UPGRADE_NODE_USER_DEFAULT ]]; then
      inventory_args+=" --ansible_ssh_user $UPGRADE_NODE_USER"
    fi
    log "Generating static tripleo-ansible-inventory with these args: $inventory_args"
    /usr/bin/tripleo-ansible-inventory $inventory_args
  else
    log "ERROR can't generate tripleo-ansible-inventory - cannot find $config_dir"
    exit 1
  fi
}

function run_ansible_playbook {
    local playbook=$UPGRADE_NODE/$1
    full_args=" -b --limit $HOSTNAME --inventory $INVENTORY $ANSIBLE_OPTS $playbook"
    log "Running ansible-playbook with $full_args"
    ansible-playbook $full_args 2>&1 | tee -a  $LOGFILE
}

if [ -n "$UPGRADE_NODE" ]; then
  find_node_by_name_id_or_ip $UPGRADE_NODE
  log "Executing $UPGRADE_SCRIPT on $IP_ADDR"
  ssh $UPGRADE_NODE_USER@$IP_ADDR sudo $UPGRADE_SCRIPT 2>&1 | tee -a $LOGFILE
  log "Clearing any existing dir $UPGRADE_NODE and downloading config"
  rm -rf $UPGRADE_NODE
  openstack overcloud config download --name "$STACK_NAME" --config-dir "$UPGRADE_NODE"
  config_dir=$(ls -1 $UPGRADE_NODE)
  if [ -z "$INVENTORY" ]; then
    get_static_inventory $UPGRADE_NODE/$config_dir
    INVENTORY="$UPGRADE_NODE/$config_dir/tripleo-ansible-inventory.yaml"
  fi
  for book in upgrade_steps_playbook.yaml deploy_steps_playbook.yaml ; do
    run_ansible_playbook $config_dir/$book
  done
fi

if [ -n "$QUERY_NODE" ]; then
  # check node exists, check for upgrade script
  find_node_by_name_id_or_ip $QUERY_NODE
  log "We can't remotely tell if the upgrade has run on $QUERY_NODE."
  log "We can check for package updates... trying to tail yum.log on $QUERY_NODE:"
  ssh $UPGRADE_NODE_USER@$IP_ADDR "sudo tail /var/log/yum.log" 2>&1 | tee -a $LOGFILE
fi