#! /bin/bash
########################################################################
#
# Copyright (c) 2014-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
########################################################################
#
# Description: The collect tool is used to gather log, config and state
#              data from one or more hosts or subclouds for the purpose
#              of off box analysis.
#
# The collect tool is implemented as a bash script that executes inline
# expect scripts and collection commands, some that that require sudo
# priviledge.
#
# The collect tool can be run from any host to collect data for that host.
#
# The collect tool must be run from an active controller to collect data
# from its managed hosts or subclouds.
#
# Version 2.2 introduces the following behavioral changes.
#
# 1. Default to a 1 month date restricted collect. This only affects what
#    is collected from /var/log. Only collect log files that contain logs
#    with a date less than one month old are collected.
#    Use date options --start-date YYYYMMDD and/or --end-date YYYYMMDD to
#    specify a more precise date range if only older logs or only more
#    recent logs are required.
#
# 2. Collect for subclouds is added with the --subcloud or -sc option.
#    With this option specified collect will collect from all the hosts in
#    the specified subcloud(s).
#    All the typical scope and naming options like --list or --all or --name
#    options also apply to subcloud collections with the exception that
#    collection of a subcloud from the system controller includes all the
#    hosts in that subcloud.
#
# 3. Default to collecting from hosts or subclouds in parallel. Parallel
#    collect reduces the overall collect time for the specified system.
#    Collect now launchs host or subcloud collect requests as backgroud
#    threads and monitors for completion or error before moving on to
#    create the final tarball collect bundle.
#
#    The previous default one-by-one or one-after-the-other mode remains
#    supported with the introduction and use of the --inline or -in
#    command option.
#
# Typical Usages:
#
#  command line                  collect data for function
#  ---------------------------   -------------------------------------
# > collect                       - collect current host ; any host
# > collect <hostname>            - collect from specified host
# > collect --list host1 host2    - collect from a list of hosts
# > collect --all                 - collect all hosts in controller context
# > collect --all --subcloud      - collect all system controller subclouds
# > collect --subcloud --list ... - collect from a list of subclouds
# > collect --all --inline        - collect all hosts one after the other
#
#   See --help output for a complete list of full and abbreviated
#   command line options.
#
# Example Output for some typical usages:
#
# Any single host collect
#
# compute-0:~$ collect
# [sudo] password for sysadmin:
# collecting data from 1 host(s): compute-0
# collecting compute-0_20210806.145159 ... done (00:02:23   55M)
# creating single-node tarball /scratch/compute-0_20210806.145159.tar ... done (00:02:23   55M)
#
#
# An AIO-DX system collect
#
# controller-0:~$ collect -a
# [sudo] password for sysadmin:
# collecting data from 2 host(s): controller-0 controller-1
# collected controller-1_20210805.193726 ... done (00:01:35   87M)
# collected controller-0_20210805.193726 ... done (00:02:53  135M)
# creating all-nodes tarball /scratch/ALL_NODES_20210805.193726.tar ... done (00:02:53  221M)
#
#
# A parallel collect of a storage system
#
# controller-0:~$ collect --all
# [sudo] password for sysadmin:
# collecting data from 8 host(s): controller-0 compute-0 compute-1 compute-2 compute-3 controller-1 storage-0 storage-1
# collected    compute-1_20210714.195247 ... done (00:00:57   14M)
# collected    compute-2_20210714.195247 ... done (00:00:57   14M)
# collected controller-1_20210714.195247 ... done (00:01:02   16M)
# collected    storage-1_20210714.195247 ... done (00:01:05   13M)
# collected    storage-0_20210714.195247 ... done (00:01:06   13M)
# collected    compute-3_20210714.195247 ... done (00:02:07   14M)
# collected controller-0_20210714.195247 ... done (00:02:11   29M)
# collected    compute-0_20210714.195247 ... done (00:03:02   14M)
# creating all-nodes tarball /scratch/ALL_NODES_20210714.195247.tar ... done (00:03:02  124M)
#
#
# A parallel collect of all (3) subclouds in a system
#
# controller-0:~$ collect --all --subcloud
# [sudo] password for sysadmin:
# collecting data from 3 subcloud(s): subcloud1 subcloud2 subcloud3
# collected subcloud3_20210811.120100 ... done (00:01:47   64M)
# collected subcloud2_20210811.120100 ... done (00:02:50   71M)
# collected subcloud1_20210811.120100 ... done (00:03:46   75M)
# creating all-subclouds tarball /scratch/SUBCLOUDS_20210811.120100.tar ... done (00:03:47  209M)
#
#
# An inline collect of all (3) subclouds in a system
#
# controller-0:~$ collect --all --subcloud --inline
# [sudo] password for sysadmin:
# collecting data from 3 subcloud(s): subcloud1 subcloud2 subcloud3
# collecting subcloud1_20210811.140525 ... done (00:02:55   79M)
# collecting subcloud2_20210811.140525 ... done (00:02:59   74M)
# collecting subcloud3_20210811.140525 ... done (00:01:47   69M)
# creating all-subclouds tarball /scratch/SUBCLOUDS_20210811.140525.tar ... done (00:07:41  221M)
#
#
# Collect Output:
#
# Collect output is a tar file bundle containing compressed tarballs
# from each host or subcloud. A default named full system collect
# looks like this:
#
# /scratch/ALL_NODES_20210805.193726.tar
#
# or for subcloud(s) collect
#
# /scratch/SUBCLOUDS_20210805.192122.tar
#
# ssh the tarball bundle off box and extract the bundle to reveal its content.
#
# Extract the host tarballs with tar into that bundle's name dir
#
#    myhost~$ tar -xvf ALL_NODES_20210805.193726.tar
#    ALL_NODES_20210805.193726/controller-0_20210805.193726.tgz
#    ALL_NODES_20210805.193726/controller-1_20210805.193726.tgz
#
# For a subcloud tar bundle
#
#    myhost~ $ tar -xvf SUBCLOUDS_20210805.192122.tar
#    SUBCLOUDS_20210805.192122/subcloud1_20210805.192122.tar
#    SUBCLOUDS_20210805.192122/subcloud2_20210805.192122.tar
#    SUBCLOUDS_20210805.192122/subcloud3_20210805.192122.tar
#    SUBCLOUDS_20210805.192122/subcloud4_20210805.192122.tar
#
# The subcloud bundles have an additional tar level
#
#    myhost SUBCLOUDS_20210805.192122 $ sudo tar -xvf subcloud1_20210805.192122.tar
#    subcloud1_20210805.192122/controller-0_20210805.192122.tgz
#    subcloud1_20210805.192122/controller-1_20210805.192122.tgz
#    subcloud1_20210805.192122/compute-1_20210805.192122.tgz
#
# Host tarball content structure
#
#     - etc       ... config data
#     - root      ... root dir content
#     - var
#      |- crash   ... crash bundle summary files
#      |- lib/sm  ... sm flight recorder
#      |- log     ... the system logs
#      |- run     ... volatile run dir
#      |- extra   ... info files produced from /etc/collect.d plugins
#                 ... area specific configuration and data
#                 ... all databases in plain text ; except for keystone
#
# Exclusions from etc and /var/run, /var/log are in /etc/collect exclude files.
#
# Behavior   : See print_help below.
#
# Collect can be run to collect local hosts or it can be run to collect
# subclouds using the --subcloud or -sc option. The tool does not support
# collecting both in one command.
#
# Collect tool produces execution summary logs in /var/log/user.log and
# more detailed logs in /var/log/collect.log
#
# Collect cleans up after itself. Meaning that collected tarballs on
# remote hosts are removed after they are fetched by the active controller.
#
# The script first collects the process, host, memory, filesystem, interrupt
# and HA information. It then proceeds to calls run-parts against the
# /etc/collect.d direcory (plugins) which contains service level collectors.
# Additional plugins can be added to that collect.d directory and will be
# called automatically.
#
# The collector scripts must consider nodetype when deciding
# which commands to execute where.
#
##################################################################


TOOL_NAME="collect"
TOOL_VER=2
TOOL_REV=2

# only supported username
UN="sysadmin"
pw=""

# pull in common utils and environment
source /usr/local/sbin/collect_utils

declare -i RETVAL=${FAIL}
function collect_exit()
{
    # support accepting the exit code as arg1
    if [ ${#} -ne 0 ] ; then
        RETVAL=${1}
    fi
    exit ${RETVAL}
}

# collect must be run as sysadmin
if [ ${UID} -eq 0 ]; then
    elog "Cannot run collect as 'root' user"
    collect_exit
elif [ "${USER}" != "${UN}" ]; then
    elog "Can only run collect as '${UN}' user"
    collect_exit
fi

source_openrc_if_needed

# used to hold the name of the password file used to pass
# the sudo password to a subcloud
TEMPFILE=""

###########################################################################
#
#                            Trap Handling
#
###########################################################################
function cleanup()
{
    # kill all processes whose parent is this process
    pkill -P $$

    # remove the tempfile if it somehow still exists
    if [ "${TEMPFILE}" != "" ]; then
        rm -f ${TEMPFILE}
    fi
    collect_exit
}

TRAP_RESET_GATE=false
function cleanup_with_reset()
{
    # prevent reset from being called for every trap definition
    if [ "${TRAP_RESET_GATE}" = false ] ; then
        $(reset)
        TRAP_RESET_GATE=true
    fi
    cleanup
    collect_exit
}

# Handle exit signals
trap cleanup_with_reset SIGINT    # administrative process termination
trap cleanup_with_reset SIGTERM   # Control-C
trap cleanup EXIT                 # clean exit

############################################################################

# static expect log level control ;
# 0 = hide expect output
# 1 = show expect outout
USER_LOG_MODE=0

# limit scp bandwidth to 1MB/s
# increase limit of scp bandwidth from 1MB/s to 10MB/s
SCP_CMD="scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PreferredAuthentications=password -o PubkeyAuthentication=no -l $((10*8*1000))"
SCP_TIMEOUT="600"
SSH_CMD="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PreferredAuthentications=password -o PubkeyAuthentication=no"
NOWDATE=$(date +"%Y%m%d.%H%M%S")
COLLECT_BASE_DIR="/scratch"
collect_host="/usr/local/sbin/collect_host"
collect="/usr/local/sbin/collect"


# This is set true on the subcloud when doing an orchestrated collect
ORCHESTRATED_COLLECT=false

CURR_DIR=$(pwd)


# common permission error strings
pw_error="orry, try again"
ac_error="ermission denied"

function print_help()
{
    echo ""
    echo "StarlingX Log Collection Tool, version ${TOOL_VER}.${TOOL_REV}"
    echo ""
    echo "Usage: ${TOOL_NAME} [COMMANDS ...] {options}"
    echo ""
    echo "StarlingX 'collect' is used to gather system logs, configuration"
    echo "and state data for off system analysis."
    echo ""
    echo "Running collect will collect logs to /scratch/<prefix_date_time.tar>"
    echo "on the host collect is run from. Use host names to specify which"
    echo "hosts or subclouds to collect from."
    echo ""
    echo "Host data collection scope can be the current host or subcloud,"
    echo "any single specified hostname or subcloud, a --list of or --all"
    echo "hosts or subclouds in the system using a single command."
    echo ""
    echo "Hosts or subclouds are collected in parallel unless the --inline"
    echo "or -in option is specified forcing a one after the other collect."
    echo ""
    echo "Collect gathers /var/log files that contain logs that are dated"
    echo "less than a month old so as to limited the size of collect bundles."
    echo "Optionally specify --start-date and/or --end-date options to refine"
    echo "the collected date range. Only logs files in /var/log are affected"
    echo "by these date options."
    echo ""
    echo "Optionally specify a --name prefix to rename the final collected"
    echo "dated tar bundle."
    echo ""
    echo "With the command set specified, simply run collect as sysadmin and when"
    echo "prompted provide the sysadmin sudo password and let collect handle the rest."
    echo ""
    echo "Scope Options:"
    echo ""
    echo " collect                                        ... collect logs for current host"
    echo " collect host1                                  ... collect logs for single named host"
    echo " collect host1 host2 host3                      ... collect logs for stacked host list"
    echo " collect [--list | -l] host1 host2 host3        ... collect logs for list of named hosts"
    echo " collect [--all  | -a]                          ... collect logs for all hosts"
    echo " collect -a                                     ... collect logs for all hosts in parallel"
    echo " collect -a [--inline | -in]                    ... collect logs for all hosts one after the other"
    echo ""
    echo " collect [--subcloud | -sc ] <subcloud>         ... collect logs for subcloud"
    echo " collect [--subcloud | -sc ] -l subc1 subc2     ... collect logs for subclouds subc1 and subc2"
    echo " collect -a [--subcloud | -sc ]                 ... collect logs for all subclouds in parallel"
    echo " collect -a -sc [--inline | -in]                ... collect logs for all subclouds one after the other"
    echo " collect --subcloud --continue                  ... continue a suspended subcloud collect"
    echo ""
    echo "Dated Collect:"
    echo ""
    echo "collect [--start-date | -s] YYYYMMDD            ... collection of logs on and  after this date"
    echo "collect [--end-date   | -e] YYYYMMDD            ... collection of logs on and before this date"
    echo ""
    echo "Tarball Prefix:"
    echo ""
    echo "collect [--name | -n] name                      ... specify the name prefix of the collect tarball"
    echo ""
    echo "Detailed Display:"
    echo ""
    echo "collect [--verbose | -v]                        ... print details during collect"
    echo ""
    echo "collect [--inventory | -i]                      ... collect inventory by system cli command"
    echo ""
    echo "Avoid password and security masking:"
    echo ""
    echo "collect [--skip-mask]                           ... skip masking of collect data"
    echo ""
    echo "Examples:"
    echo ""
    echo "collect                                                      ... all logs for current host"
    echo "collect --all                                                ... all logs from all hosts in the system"
    echo "collect --all --subcloud                                     ... all logs from all hosts in all subclouds"
    echo "collect --all --start-date 20150101                          ... logs dated on and after Jan 1 2015 from all hosts"
    echo "collect --all --start-date 20151101 --end-date 20160201      ... logs dated between Nov 1, 2015 and Feb 1 2016 from all hosts"
    echo "collect --list controller-0 worker-0 storage-0               ... all logs from specified host list"
    echo "collect --list controller-0 worker-1 --end-date 20160201     ... only logs before Nov 1, 2015 for host list"
    echo "collect --list controller-1 storage-0 --start-date 20160101  ... only logs after Jan 1 2016 for controller-1 and storage-0"
    echo "collect --start-date 20151101 --end-date 20160201            ... only logs dated between Nov 1, 2015 and Feb 1 2016 for current host"
    echo "collect --subcloud subcloud1 subcloud2 subcloud3             ... only logs from a list of subclouds"
    echo ""
    exit 0
}

# command line arguement variables ; defaulted
DEBUG=false
CLEAN=false
VERBOSE=false
SKIP_MASK=false
INVENTORY=false
SUBCLOUD_COLLECT=false
SUBCLOUD_LOGIN_PROMPT="controller-"

# parallel collect mode as default
PARALLEL_COLLECT_MODE=true

# date variables - default to a 1 month dated collect
DATE_FORMAT="YYYYMMDD"
STARTDATE=$(date +%Y%m%d -d "-1 month")
STARTTIME="any"
ENDDATE="any"
ENDTIME="any"
GETSTARTDATE=false
GETENDDATE=false
DCROLE=""

# host selection variables
LISTING=false
ALLHOSTS=false

declare -i HOSTS=1
declare -a HOSTLIST=(${HOSTNAME})
declare -i SUBCLOUDS=0
declare -a SUBCLOUDLIST=()
declare -i DONE_COUNT=0
declare -i longest_name=0

PLEASE_STANDBY=false
COLLECT_CONTINUE_MSG_NEEDED=false
SUBCLOUD_COLLECT_CONTINUE=false
SUBCLOUD_COLLECT_CONTINUE_LIST_FILE="/tmp/collect_continue.lst"

# overall collect timeout
TIMEOUT=1000
SECONDS=0
let UNTIL=${SECONDS}+${TIMEOUT}

COLLECT_NAME=""

# clear multi option modes
function clear_variable_args()
{
    LISTING=false
    GETSTARTDATE=false
    GETENDDATE=false
}


############################################################################
#
# Name       : report_error
#
# Purpose    : Report error to console and logfile
#
# Assumptions: Handles specific cases of invalid password and permission errors
#              by exiting so as to avoid repeated errors during multi-host
#              collection.
#
# $1 - status string
# $2 - status code number
#
function report_error()
{
    local string=${1}
    local code=${2}

    if [[ "${PARALLEL_COLLECT_MODE}" = true && "${PLEASE_STANDBY}" = true && ${DONE_COUNT} -eq 0 ]] ; then
        DONE_COUNT=$((DONE_COUNT+1))
        # send new line to delineate '.' progress
        echo ""
        PLEASE_STANDBY=false
    fi

    if [ ${code} -eq ${FAIL_PASSWORD} ] ; then
        elog "Invalid password"
        collect_exit ${code}

    elif [ ${code} -eq ${FAIL_CONTINUE} ] ; then
        elog "${FAIL_CONTINUE_STR} ; ${string} (reason:${code})"
        collect_exit ${code}

    elif [ ${code} -eq ${FAIL_INACTIVE} ] ; then
        elog "${FAIL_INACTIVE_STR} ; ${string} (reason:${code})"
        collect_exit ${code}

    elif [ ${code} -eq ${FAIL_PERMISSION} ] ; then
        elog "Permission error ; exiting (${string})"

    elif [ ${code} -eq ${FAIL_UNREACHABLE} ] ; then
        elog "${string} (reason:${code}:unreachable)"

    elif [ ${code} -eq ${FAIL_PERMISSION_SKIP} ] ; then
        elog "${string} (reason:${code}:permission error)"

    elif [ ${code} -eq ${FAIL_OUT_OF_SPACE} ] ; then
        elog "${string} (reason:${code}:${FAIL_NOT_ENOUGH_SPACE_STR}) ; need to increase available space ${COLLECT_BASE_DIR}"

    elif [ ${code} -eq ${FAIL_INSUFFICIENT_SPACE} ] ; then
        elog "${string} (reason:${code}:${FAIL_NOT_ENOUGH_SPACE_STR}) ; ${COLLECT_BASE_DIR} usage must be below ${MIN_PERCENT_SPACE_REQUIRED}%"

    elif [ ${code} -ge ${FAIL_TIMEOUT} -a ${code} -le ${FAIL_TIMEOUT9} ] ; then
        elog "${FAIL_TIMEOUT_STR} ; ${string} (reason:${code})"

    elif [ ${code} -eq ${FAIL_SUBCLOUD_TIMEOUT} ] ; then
        elog "${FAIL_SUBCLOUD_TIMEOUT_STR} ; ${string} (reason:${code})"

    elif [ ${code} -eq ${FAIL_PASSWORD_PROMPT} ] ; then
        elog "${string} (reason:${code}:failed to learn password)"

    elif [ ${code} -eq ${FAIL_DATE_FORMAT} ] ; then
        elog "${FAIL_DATE_FORMAT_STR} ; ${string} (reason:${code})"

    elif [ ${code} -eq ${FAIL_NO_FILE_SPECIFIED} ] ; then
        elog "${FAIL_NO_FILE_SPECIFIED_STR} ; ${string} (reason:${code})"

    elif [ ${code} -eq ${FAIL_FILE_NOT_FOUND} ] ; then
        elog "${FAIL_FILE_NOT_FOUND_STR} ; ${string} (reason:${code})"

    elif [ ${code} -eq ${FAIL_FILE_EMPTY} ] ; then
        elog "${FAIL_FILE_EMPTY_STR} ; ${string} (reason:${code})"

    elif [ ${code} -eq ${FAIL_NO_HOSTS} ] ; then
        elog "${FAIL_NO_HOSTS_STR} ; ${string} (reason:${code})"

    elif [ ${code} -eq ${FAIL_NO_SUBCLOUDS} ] ; then
        elog "${FAIL_NO_SUBCLOUDS_STR} ; ${string} (reason:${code})"

    elif [ ${code} -eq ${FAIL_MISSING_PARAMETER} ] ; then
        elog "${FAIL_MISSING_PARAMETER_STR} ; ${string} (reason:${code})"

    else
        elog "${string} (reason:${code})"
    fi
}

###########################################################################
#
# Name      : is_valid_host
#
# Purpose   : Checks to see if the specified hostname is known
#             to inventory as a valid provisioned host
#
# Parameters: $1 check_hostname
#
# Return    : PASS              ... hostname is valid (success path)
#             FAIL_HOSTNAME     ... hostname is not valid
#             FAIL_INACTIVE     ... this host is not active
#
###########################################################################

function is_valid_host()
{
    local check_hostname=${1}

    if [ "${check_hostname}" == "None" ] ; then
        return ${FAIL_HOSTNAME}
    elif [ "${check_hostname}" == "${HOSTNAME}" ] ; then
        return ${PASS}
    elif [ "${ACTIVE}" = true ] ; then
        system host-show "${check_hostname}" 2>/dev/null 1>/dev/null
        if [ ${?} -ne 0 ] ; then
            return ${FAIL_HOSTNAME}
        else
            return ${PASS}
        fi
    else
        report_error "can only run collect for remote hosts on active controller" ${FAIL_INACTIVE}
        collect_exit ${FAIL_INACTIVE}
    fi
}

###########################################################################
#
# Name      : is_valid_subcloud
#
# Purpose   : Checks to see if the specified subcloud name is known
#             to dcmanager as a valid provisioned subcloud
#
# Parameters: $1 check_subcloudname
#
# Return    : PASS              ... subcloudname is valid (success path)
#             FAIL_SUBCLOUDNAME ... subcloudname is not valid
#             FAIL_INACTIVE     ... this host is not the active controller
#
###########################################################################

function is_valid_subcloud()
{
    local check_subcloudname=${1}

    if [ "${check_subcloudname}" == "None" ] ; then
        return ${FAIL_SUBCLOUDNAME}
    elif [ "${ACTIVE}" = true ] ; then
        dcmanager subcloud show "${check_subcloudname}" 2>/dev/null 1>/dev/null
        if [ ${?} -ne 0 ] ; then
            return ${FAIL_SUBCLOUDNAME}
        else
            return ${PASS}
        fi
    else
        report_error "can only run collect for subclouds from the active system controller" ${FAIL_INACTIVE}
        collect_exit ${FAIL_INACTIVE}
    fi
}

function query_and_update_dcrole ()
{
    DCROLE=$(system show | grep distributed_cloud_role | cut -d '|' -f 3 | tr -d ' ')
}

############################################################################
#                      Parse the command line                              #
############################################################################

# echo "`date` Debug: collect ${@}"

while [[ ${#} -gt 0 ]] ; do

    key="${1}"

    case $key in

        -h|--help)
        print_help
        collect_exit ${PASS}
        ;;

        -n|--name)
        if [ "${2}" == "" ] ; then
            report_error "need to specify a name with the --name option" ${FAIL_MISSING_PARAMETER}
            collect_exit ${FAIL_MISSING_PARAMETER}
        else
            name="${2}"
            len=${#name}
            max_length=$(expr 255 - 16 - 4) # Padding of timestamp and .tar

            if [ ${len} -gt ${max_length} ] ; then
                report_error "name must not be longer than ${max_length} characters" ${FAIL_NAME_TOO_LONG}
                collect_exit ${FAIL_NAME_TOO_LONG}
            fi
        fi
        COLLECT_NAME="${2}"
        clear_variable_args
        shift
        ;;

        -v|--verbose)
        USER_LOG_MODE=1
        VERBOSE=true
        ;;

        --clean)
        CLEAN=true
        ;;

        -c|--continue)
        SUBCLOUD_COLLECT_CONTINUE=true
        ;;

        -i|--inventory)
        INVENTORY=true
        ;;

        -l|--list)
        if [ "${ALLHOSTS}" = false ] ; then
            if [[ ${#} -lt  2 ]] ; then
                report_error "collect exit" ${FAIL_NO_HOSTS}
                collect_exit ${FAIL_NO_HOSTS}
            fi
            if [ "${ACTIVE}" = false ] ; then
                report_error "can only run collect for remote hosts on active controller" ${FAIL_INACTIVE}
                collect_exit ${FAIL_INACTIVE}
            fi
            HOSTLIST=(${2})
            HOSTS=1
            LISTING=true
            GETSTARTDATE=false
            GETENDDATE=false
            shift
        fi
        ;;

        -a|--all|all)
        if [ "${ACTIVE}" = false ] ; then
            wlog "collect with '${key}' option is only supported on an active controller ; defaulting to local collect"
        else
            ALLHOSTS=true
        fi

        HOSTLIST=(${HOSTNAME})
        HOSTS=1
        clear_variable_args
        ;;

        -s|--start-date)
        if [ "${2}" == "" ] ; then
            report_error "need to specify a date with the --start-date option" ${FAIL_MISSING_PARAMETER}
            collect_exit ${FAIL_MISSING_PARAMETER}
        elif [ "${2}" != "any" -a ${#2} -ne ${#DATE_FORMAT} ] ; then
            report_error "start date must be '${DATE_FORMAT}' format" ${FAIL_DATE_FORMAT}
            collect_exit ${FAIL_DATE_FORMAT}
        fi
        STARTDATE="${2}"
        LISTING=false
        GETSTARTDATE=true
        GETENDDATE=false
        shift
        ;;

        -e|--end-date)
        if [ "${2}" == "" ] ; then
            report_error "need to specify a date with the --end-date option" ${FAIL_MISSING_PARAMETER}
            collect_exit ${FAIL_MISSING_PARAMETER}
        elif [ "${2}" != "any" -a ${#2} -ne ${#DATE_FORMAT} ] ; then
            report_error "end date must be '${DATE_FORMAT}' format" ${FAIL_DATE_FORMAT}
            collect_exit ${FAIL_DATE_FORMAT}
        fi
        ENDDATE="${2}"
        LISTING=false
        GETSTARTDATE=false
        GETENDDATE=true
        shift
        ;;

        -sc|--subcloud)
        SUBCLOUD_COLLECT=true
        ;;

        -d|--debug)
        DEBUG=true
        clear_variable_args
        ;;

        --skip-mask)
        SKIP_MASK=true
        shift
        ;;

        -in|--inline)
        # switch to inline ; one-after-the-other (legacy) mode
        PARALLEL_COLLECT_MODE=false
        ;;

        -f|--file)
        TEMPFILE="${2}"
        if [ "${TEMPFILE}" == "" ]; then
            report_error "need file path/name to follow --file option" ${FAIL_NO_FILE_SPECIFIED}
            collect_exit ${FAIL_NO_FILE_SPECIFIED}
        elif [ ! -e "${TEMPFILE}" ]; then
            report_error "check path/file: ${TEMPFILE}" ${FAIL_NO_FILE_SPECIFIED}
            collect_exit ${FAIL_NO_FILE_SPECIFIED}
        elif [ ! -s "${TEMPFILE}" ] ; then
            report_error "file:${TEMPFILE}" ${FAIL_FILE_EMPTY}
            rm -f ${TEMPFILE}
            collect_exit ${FAIL_FILE_EMPTY}
        else
            # read first line in file
            pw=$(head -n 1 ${TEMPFILE})
            dlog "pw:${pw}"
            rm -f ${TEMPFILE}
            shift
        fi
        ;;

        *)
        if [ "${LISTING}" = true ] ; then
            HOSTS=$((HOSTS+1))
            HOSTLIST+=(${key})
        elif [ "${GETSTARTDATE}" = true ] ; then
            dlog "accepting but ignoring legacy starttime specification"
        elif [ "${GETENDDATE}" = true ] ; then
            dlog "accepting but ignoring legacy endtime specification"
        else
            HOSTLIST=(${key})
            HOSTS=1
            LISTING=true
        fi
        GETSTARTDATE=false
        GETENDDATE=false
        ;;
    esac
    shift # past argument or value
done


# startup state debug logs
dlog "${TOOL_NAME} ver ${TOOL_REV}.${TOOL_REV} (pid:$$)"
dlog "USERNAME  = ${USER}"
dlog "ACTIVE    = ${ACTIVE}"
dlog "HOSTNAME  = ${HOSTNAME}"
dlog "PARALLEL  = ${PARALLEL_COLLECT_MODE}"
dlog "INVENTORY = ${INVENTORY}"
dlog "STARTDATE = ${STARTDATE}"
dlog "ENDDATE   = ${ENDDATE}"
dlog "SKIPMASK  = ${SKIP_MASK}"
dlog "ALLHOSTS  = ${ALLHOSTS}"
dlog "LISTING   = ${LISTING}"
dlog "CLEAN     = ${CLEAN}"
dlog "TIMEOUT   = ${TIMEOUT}"
dlog "SECONDS   = ${SECONDS}"
dlog "UNTIL     = ${UNTIL}"

# the continue option is only supported for subcloud collect
if [[ "${SUBCLOUD_COLLECT_CONTINUE}" = true && "${SUBCLOUD_COLLECT}" = false ]] ; then
    report_error "collect continue is only supported for subclouds" ${FAIL_CONTINUE}
    collect_exit ${FAIL_CONTINUE}
fi

# subcloud option only on active SystemController
if [[ "${ACTIVE}" = false && "${SUBCLOUD_COLLECT}" = true ]] ; then
    report_error "subcloud collect can only be run from an active systemcontroller" ${FAIL_INACTIVE}
    collect_exit ${FAIL_INACTIVE}
fi

# Don't block the clean operation based on avalable space.
# That would defeat the purpose.
if [ "${CLEAN}" = false ] ; then
    space_precheck ${HOSTNAME} ${COLLECT_BASE_DIR}
fi

#
# If on the active controller load the DCROLE variable and
# handle subcloud collect from non SC
#
if [ "${ACTIVE}" = true ] ; then
    query_and_update_dcrole
    if [ "${SUBCLOUD_COLLECT}" = true ] ; then
        if [ "${DCROLE}" != "${DCROLE_SYSTEMCONTROLLER}" ] ; then
            report_error "must run subcloud collect from the systemcontroller" ${FAIL_NOT_SYSTEMCONTROLLER}
            collect_exit ${FAIL_NOT_SYSTEMCONTROLLER}
        fi
    fi
fi

#
# if the user specified the '--all' option then override
# the current list and add them all from inventory.
#
if [ "${ALLHOSTS}" = true ] ; then
    HOSTLIST=()
    HOSTS=0
    SUBCLOUDLIST=()
    SUBCLOUDS=0
    if [ "${SUBCLOUD_COLLECT}" = false ]; then
        HOSTLIST=(${HOSTNAME})
        HOSTS=1
        for foreign_host in $(system host-list | grep '[0-9]' | cut -d '|' -f 3 | tr -d ' ' | grep -v ${HOSTNAME}); do
            if [ "${foreign_host}" != "None" ] ; then
                HOSTS=$((HOSTS+1))
                HOSTLIST+=(${foreign_host})
            fi
        done

    else
        for foreign_host in $(dcmanager subcloud list | grep '[0-9]' | cut -d '|' -f 3 | tr -d ' '); do
            if [ "${foreign_host}" != "None" ] ; then
                SUBCLOUDS=$((SUBCLOUDS+1))
                SUBCLOUDLIST+=(${foreign_host})
            fi
        done
    fi
else
    # This host path
    # Filter default or user specified host list through temp_hostlist
    # This drops rather than deletes invalid or duplicate hosts.
    temp_hostlist=(${HOSTLIST[@]})
    temp_hosts=${HOSTS}
    HOSTLIST=()
    HOSTS=0
    SUBCLOUDLIST=()
    SUBCLOUDS=0

    # check for and handle collect --continue
    if [ "${SUBCLOUD_COLLECT_CONTINUE}" = true ] ; then
        if [ -f "${SUBCLOUD_COLLECT_CONTINUE_LIST_FILE}" ] && \
            [ -s "${SUBCLOUD_COLLECT_CONTINUE_LIST_FILE}" ] ; then
            SUBCLOUDLIST=($( cat ${SUBCLOUD_COLLECT_CONTINUE_LIST_FILE}))
            SUBCLOUDS=${#SUBCLOUDLIST[@]}
            dlog "continuing collect for remaining ${SUBCLOUDS} subclouds: ${SUBCLOUDLIST[@]}"
        else
            report_error "the ${SUBCLOUD_COLLECT_CONTINUE_LIST_FILE} file is empty or missing" ${FAIL_CONTINUE}
        fi

    elif [ "${SUBCLOUD_COLLECT}" = false ] ; then
        if [ ${temp_hosts} -eq 0 ] ; then
            report_error "no hosts specified" ${FAIL_NO_HOSTS}
            collect_exit ${FAIL_NO_HOSTS}
        else
            for host in "${temp_hostlist[@]}" ; do
                is_valid_host ${host}
                if [ ${?} -eq 0 ] ; then
                    # don't add duplicates
                    drop=false
                    for tmp in "${HOSTLIST[@]}" ; do
                        if [ "${host}" == "${tmp}" ] ; then
                            drop=true
                            break
                        fi
                    done
                    if [ "${drop}" = false ] ; then
                        # add this host
                        HOSTS=$((HOSTS+1))
                        HOSTLIST+=("${host}")
                    fi
                else
                    report_error "cannot collect data from unknown host '${host}'" ${WARN_HOSTNAME}
                fi
            done
        fi
    else
        if [ ${temp_hosts} -eq 0 ] ; then
            report_error "no subclouds specified" ${FAIL_NO_SUBCLOUDS}
            collect_exit ${FAIL_NO_SUBCLOUDS}
        # don't query a large number of subclouds individually,
        # that can take a long time. Instead get the full list and
        # validate the specified list from the full list
        elif [ ${temp_hosts} -gt 10 ] ; then
            SUBCLOUDLIST_TEMP=()
            # reuse HOSTS and HOSTLIST vars for this operation
            for foreign_host in $(dcmanager subcloud list | grep '[0-9]' | cut -d '|' -f 3 | tr -d ' '); do
                if [ "${foreign_host}" != "None" ] ; then
                    SUBCLOUDLIST_TEMP+=(${foreign_host})
                fi
            done
            # validate the subcloud names
            for subcloud in "${temp_hostlist[@]}" ; do
                for temp in "${SUBCLOUDLIST_TEMP[@]}" ; do
                    found=false
                    if [ "${temp}" == "${subcloud}" ] ; then
                        # don't add duplicates
                        drop=false
                        for tmp in "${SUBCLOUDLIST[@]}" ; do
                            if [ "${subcloud}" == "${tmp}" ] ; then
                                drop=true
                                break
                            fi
                        done
                        if [ "${drop}" = false ] ; then
                            SUBCLOUDS=$((SUBCLOUDS+1))
                            SUBCLOUDLIST+=(${subcloud})
                            found=true
                            break
                        fi
                    fi
                done
                if [ "${found}" = false ] ; then
                    is_valid_subcloud ${subcloud}
                    if [ ${?} -eq 0 ] ; then
                        # don't add duplicates
                        drop=false
                        for tmp in "${SUBCLOUDLIST[@]}" ; do
                            if [ "${subcloud}" == "${tmp}" ] ; then
                                drop=true
                                break
                            fi
                        done
                        if [ "${drop}" = false ] ; then
                            # add this subcloud
                            SUBCLOUDS=$((SUBCLOUDS+1))
                            SUBCLOUDLIST+=("${subcloud}")
                        fi
                    else
                        report_error "cannot collect data from unknown subcloud '${subcloud}'" ${WARN_SUBCLOUD}
                    fi
                fi
            done
        else
            # validate subclouds one by one through dcmanager
            for subcloud in "${temp_hostlist[@]}" ; do
                is_valid_subcloud ${subcloud}
                if [ ${?} -eq 0 ] ; then
                    # don't add duplicates
                    drop=false
                    for tmp in "${SUBCLOUDLIST[@]}" ; do
                        if [ "${subcloud}" == "${tmp}" ] ; then
                            drop=true
                            break
                        fi
                    done
                    if [ "${drop}" = false ] ; then
                        # add this subcloud
                        SUBCLOUDS=$((SUBCLOUDS+1))
                        SUBCLOUDLIST+=("${subcloud}")
                    fi
                else
                    report_error "cannot collect data from unknown subcloud '${subcloud}'" ${WARN_SUBCLOUD}
                fi
            done
        fi
    fi
fi

if [ ! -z ${COLLECT_NAME} ] ; then

    # User specified tarname
    #
    # This is the only case for system controller initiated subcloud collect
    COLLECT_TYPE="user-named"

    # Subcloud collect with a password at this point must be orchestrated
    # ... with collect date specified by the system controller.
    if [ "${DCROLE}" == "${DCROLE_SUBCLOUD}" -a "${pw}" != "" ] ; then
        dlog "date override ${NOWDATE} to ${COLLECT_NAME: -15}"
        NOWDATE=${COLLECT_NAME: -15}
        ilog "Orchestrated collect"
        ORCHESTRATED_COLLECT=true
    elif [ "${DCROLE}" == "" -a "${ACTIVE}" == false -a "${pw}" != "" ]; then
        wlog "Subcloud has not been properly configured."
        ERROR_DCROLE=$(cat /etc/platform/platform.conf | grep distributed_cloud_role | cut -d '=' -f 2)
        if [ "${ERROR_DCROLE}" = "subcloud" ]; then
            dlog "date override ${NOWDATE} to ${COLLECT_NAME: -15}"
            NOWDATE=${COLLECT_NAME: -15}
            ilog "Orchestrated Collect"
            ORCHESTRATED_COLLECT=true
        fi
    fi

elif [ "${ALLHOSTS}" = true ] ; then

    # All hosts/subclouds bundle
    if [ "${SUBCLOUD_COLLECT}" = true ] ; then
        COLLECT_NAME="ALL_SUBCLOUDS"
        COLLECT_TYPE="all-subclouds"
    else
        COLLECT_NAME="ALL_NODES"
        COLLECT_TYPE="all-nodes"
    fi

elif [ "${SUBCLOUD_COLLECT}" = false -a ${HOSTS} -eq 1 ] ; then

    # Single host bundle
    COLLECT_NAME="${HOSTLIST[0]}"
    COLLECT_TYPE="single-node"

elif [ "${SUBCLOUD_COLLECT}" = true -a ${SUBCLOUDS} -eq 1 ] ; then

    # Single host bundle
    COLLECT_NAME="${SUBCLOUDLIST[0]}"
    COLLECT_TYPE="single-subcloud"

else

    # Otherwise its a multi host bundle
    if [ "${SUBCLOUD_COLLECT}" = true ] ; then
        COLLECT_NAME="SELECT_SUBCLOUDS"
        COLLECT_TYPE="selected-subcloud"
    else
        COLLECT_NAME="SELECT_NODES"
        COLLECT_TYPE="selected-node"
    fi

fi

if [ "${ORCHESTRATED_COLLECT}" = false ] ; then
    COLLECT_NAME+="_${NOWDATE}"
fi
COLLECT_DIR="${COLLECT_BASE_DIR}/${COLLECT_NAME}"
TARBALL_NAME="${COLLECT_DIR}.tar"

# learned state debug logs
if [ "${SUBCLOUD_COLLECT}" = true ] ; then
    dlog "SUBCLOUDLIST = ${SUBCLOUDS}:${SUBCLOUDLIST[@]}"
else
    dlog "HOSTLIST     = ${HOSTS}:${HOSTLIST[@]}"
fi
if [ "${DCROLE}" != "" ] ;  then
    dlog "DCROLE       = ${DCROLE}"
fi
dlog "COLLECT_TYPE = ${COLLECT_TYPE}"
dlog "COLLECT_NAME = ${COLLECT_NAME}"
dlog "COLLECT_DIR  = ${COLLECT_DIR}"
dlog "TARBALL_NAME = ${TARBALL_NAME}"

############################################################################
#
# Password handling
#
# If the password is not learned by other means by this time
# then prompt the user to enter it.
#
# The password is used for expect driven requests.
#
############################################################################
# dlog "password coming in is:$pw"

if [ -z "${pw}" ] ; then
    read -s -p "[sudo] password for ${USER}:" pw
    echo ""
fi

# When the pw is used locally for expect requests ...
#
# Although bash 'read' will handle sanitizing the password
# input for the purposes of storing it in ${pw}, expect
# will need certain special characters to be backslash
# delimited
pw=${pw/\\/\\\\} # replace '\' with '\\'
pw=${pw/\]/\\\]} # replace ']' with '\]'
pw=${pw/\[/\\\[} # replace '[' with '\['
pw=${pw/$/\\$}   # replace '$' with '\$'
pw=${pw/\"/\\\"} # replace '"' with '\"'

###########################################################################
#
# Name       : passwordless_sudo_test
#
# Purpose    : Verify to detect passwordless sudo for getting password promptand
#               fails the collect if there is no password prompt
#
# Description: cat the content of the /usr/local/sbin/expect_done
#
###########################################################################

function passwordless_sudo_test()
{

/usr/bin/expect  << EOF
    log_user ${USER_LOG_MODE}
    spawn bash -i
    set timeout 60
    expect -re $
    send "sudo cat /usr/local/sbin/expect_done\n"
    expect {
        "assword:" {
            send "${pw}\r"
            expect {
                "${cmd_done_sig}" { exit ${PASS}           }
                "${pw_error}"     { exit ${FAIL_PASSWORD}  }
                timeout           { exit ${FAIL_TIMEOUT1}  }
            }
        }
        "${pw_error}"     { exit ${FAIL_PASSWORD} }
        timeout           { exit ${FAIL_TIMEOUT} }
    }
EOF
    local rc=${?}
    if [ ${rc} -ne ${PASS} ] ; then
        report_error "Timeout waiting for password prompt. Passwordless sudo may be enabled. Please disable and retry." ${rc}
        collect_exit ${rc}
    fi
}

passwordless_sudo_test

###########################################################################
#
# Name       : check_host_reachable
#
# Purpose    : Verify a host is reachable before trying to collect from it
#
# Description: ls the content of the scratch dir
# Parameters : $1 - remote hostname
#             $2 - dir or file with full path
#
###########################################################################

function check_host_reachable()
{
    local hostname=${1}

    if [ "${hostname}" == "${HOSTNAME}" ] ; then
        return ${PASS}
    fi

/usr/bin/expect << EOF
    log_user ${USER_LOG_MODE}
    spawn bash -i
    expect -re $
    set timeout 60
    send "${SSH_CMD} ${UN}@${hostname} cat ${cmd_done_file}\n"
    expect {
        "assword:" {
            expect -re $
            send "${pw}\r"
            expect {
                "assword:" { send -- "${pw}\r" ; exp_continue }
                "${cmd_done_sig}" { exit ${PASS} }
                "No such file or directory" { exit ${FAIL_FILE_NOT_FOUND} }
                "${pw_error}"     { exit ${FAIL_PASSWORD}  }
                "${ac_error}"     { exit ${FAIL_PERMISSION_SKIP}}
                timeout           { exit ${FAIL_TIMEOUT3}  }
            }
        }
        "(yes/no)?" {
            send "yes\r"
            exp_continue
        }
        "No route to host" {
            exit ${FAIL_UNREACHABLE}
        }
        "Could not resolve hostname" {
            exit ${FAIL_UNREACHABLE}
        }
        "Network is unreachable" {
            exit ${FAIL_UNREACHABLE}
        }
        "Connection refused" {
            exit ${FAIL_UNREACHABLE}
        }
        "Connection timed out" {
            exit ${FAIL_UNREACHABLE}
        }
        timeout { exit ${FAIL_TIMEOUT} }
    }
EOF
    return ${?}
}


###########################################################################
#
# Name      : clean_scratch_dir_local
#
# Purpose   : remove contents of the local /scratch directory
#
# Parameters: $1 - this hostname
#             $2 - specified directory (always $COLLECT_BASE_DIR)
#
###########################################################################

function clean_scratch_dir_local ()
{
    local this_hostname=${1}
    local directory=${2}

/usr/bin/expect << EOF
    log_user ${USER_LOG_MODE}
    spawn bash -i
    set timeout 60
    expect -re $
    send -- "sudo rm -rf ${directory}/*_????????.??????* ; cat ${cmd_done_file}\n"
    expect {
        "assword:"        { send "${pw}\r" ; exp_continue }
        "${cmd_done_sig}" { exit ${PASS} }
        "annot remove"    { exit ${FAIL_CLEANUP}    }
        "${pw_error}"     { exit ${FAIL_PASSWORD}   }
        "${ac_error}"     { exit ${FAIL_PERMISSION} }
        timeout           { exit ${FAIL_TIMEOUT}    }
    }
EOF
    local rc=${?}
    if [ ${rc} -ne ${PASS} ] ; then
        report_error "clean_scratch_dir_local ${this_hostname} failed" ${rc}
    fi
    return ${rc}
}

###########################################################################
#
# Name      : clean_scratch_dir_remote
#
# Purpose   : remove contents of the specified host's /scratch directory
#
# Parameters: $1 - host
#             $2 - specified directory (always $COLLECT_BASE_DIR)
#
###########################################################################

function clean_scratch_dir_remote()
{
    local this_hostname=${1}
    local directory=${2}

/usr/bin/expect << EOF
    log_user ${USER_LOG_MODE}
    spawn bash -i
    expect -re $
    set timeout 60
    send "${SSH_CMD} ${UN}@${this_hostname}\n"
    expect {
        "assword:" {
            send "${pw}\r"
            expect {
                "${this_hostname}" {
                    set timeout 30
                    expect -re $
                    send "sudo rm -rf ${directory}/*_????????.??????* ; cat ${cmd_done_file}\n"
                    expect {
                        "assword:" { send -- "${pw}\r" ; exp_continue }
                        "${cmd_done_sig}" { exit ${PASS} }
                        "${cmd_done_file}: No such file or directory" { exit ${PASS} }
                        "annot remove"    { exit ${FAIL_CLEANUP}   }
                        "${pw_error}"     { exit ${FAIL_PASSWORD}  }
                        "${ac_error}"     { exit ${FAIL_PERMISSION_SKIP}}
                        timeout           { exit ${FAIL_TIMEOUT3}  }
                    }
                }
                timeout { exit ${FAIL_TIMEOUT1} }
            }
        }
        "(yes/no)?" {
            send "yes\r"
            exp_continue
        }
        "No route to host" {
            exit ${FAIL_UNREACHABLE}
        }
        "Could not resolve hostname" {
            exit ${FAIL_UNREACHABLE}
        }
        timeout { exit ${FAIL_TIMEOUT} }
    }
EOF
    local rc=${?}
    if [ ${rc} -ne ${PASS} ] ; then
        report_error "failed to clean ${this_hostname}:${directory}" ${rc}
    fi
    return ${rc}
}

###########################################################################
#
# Name      : delete_remote_dir_or_file
#
# Purpose   : Deletes a remote directory or file
#
# Parameters: $1 - remote hostname
#             $2 - dir or file with full path
#             $3 - expected login prompt
#             $4 - alternative login prompt (optional)
#
###########################################################################

function delete_remote_dir_or_file()
{
    local remote_hostname=${1}
    local dir_or_file=${2}
    local login_prompt="${3}"
    
    # alt_login_prompt is optional. Used when the actual prompt does not
    # match the expected login_prompt (as contained in $login_prompt)
    local alt_login_prompt="${4}"
    
    # if ${4} is empty, use $login_prompt instead.
    if test -z "${4}";
    then 
        alt_login_prompt=${login_prompt};
    fi

/usr/bin/expect << EOF
    log_user ${USER_LOG_MODE}
    spawn bash -i
    expect -re $
    set timeout 60
    send "${SSH_CMD} ${UN}@${remote_hostname}\n"
    expect {
        "assword:" {
            send "${pw}\r"
            expect {
                timeout { exit ${FAIL_TIMEOUT1} }
                "${login_prompt}" {}
                "${alt_login_prompt}" {}
            }
            set timeout 10
            expect -re $
            send "sudo rm -rf ${dir_or_file} ; cat ${cmd_done_file}\n"
            expect {
                "assword:" { send -- "${pw}\r" ; exp_continue }
                "${cmd_done_sig}" { exit ${PASS} }
                "${cmd_done_file}: No such file or directory" { exit ${PASS} }
                "annot remove"    { exit ${FAIL_CLEANUP}   }
                "${pw_error}"     { exit ${FAIL_PASSWORD}  }
                "${ac_error}"     { exit ${FAIL_PERMISSION_SKIP}}
                timeout           { exit ${FAIL_TIMEOUT3}  }
            }
        }
        "(yes/no)?" {
            send "yes\r"
            exp_continue
        }
        "No route to host" {
            exit ${FAIL_UNREACHABLE}
        }
        "Could not resolve hostname" {
            exit ${FAIL_UNREACHABLE}
        }
        timeout { exit ${FAIL_TIMEOUT} }
    }
EOF
    local rc=${?}
    if [ ${rc} -ne ${PASS} ] ; then
        dlog "delete_remote_dir_or_file parms=${remote_hostname}:${login_prompt}:${dir_or_file}"
        report_error "failed to delete ${dir_or_file} on ${remote_hostname} (${login_prompt})" ${rc}
    fi
    return ${rc}
}

############################################################################
#
# Name      : get_file_from_host
#
# Purpose   : Fetch a file from a remote host
#
# Parameters: $1 - remote hostname
#             $2 - remote source path/filename
#             $3 - local path destination
#
############################################################################

function get_file_from_host()
{
    local remote_hostname=${1}
    local remote_file=${2}
    local local_dest=${3}

    remove_file_local ${HOST_COLLECT_ERROR_LOG}

    dlog "get_file_from_host: ${UN}@${remote_hostname}:${COLLECT_BASE_DIR}/${remote_file} ${local_dest}"

/usr/bin/expect << EOF
    log_user ${USER_LOG_MODE}
    spawn bash -i
    set timeout ${SCP_TIMEOUT}
    expect -re $
    send "${SCP_CMD} ${UN}@${remote_hostname}:${COLLECT_BASE_DIR}/${remote_file} ${local_dest} 2>>${HOST_COLLECT_ERROR_LOG}\n"
    expect {
        "assword:" {
            send "${pw}\r"
            expect {
                "100%"        { exit ${PASS} }
                "${pw_error}" { exit ${FAIL_PASSWORD}  }
                "${ac_error}" { exit ${FAIL_PERMISSION_SKIP}}
                timeout       { exit ${FAIL_TIMEOUT1}  }
            }
        }
        "No route to host" {
            exit ${FAIL_UNREACHABLE}
        }
        "Could not resolve hostname" {
            exit ${FAIL_UNREACHABLE}
        }
        timeout { exit ${FAIL_TIMEOUT} }
    }
EOF
    local rc=${?}
    if [ ${rc} -ne ${PASS} ] ; then
        report_error "failed to get file from ${remote_hostname}" ${rc}
    else
        # Look for "No space left on device" error
        grep -q "${FAIL_OUT_OF_SPACE_STR}" ${HOST_COLLECT_ERROR_LOG}
        if [ "$?" == "0" ] ; then
            remove_file_local "${local_dest}/${remote_file}"
            rc=${FAIL_OUT_OF_SPACE}
        fi
    fi

    remove_file_local ${HOST_COLLECT_ERROR_LOG}

    return ${rc}
}

############################################################################
#
# Name      : copy_file_to_host
#
# Purpose   : Copy a file to a remote host
#
# Parameters: $1 - local path/file
#             $2 - remote hostname
#             $3 - remote destination directory
#
############################################################################

function copy_file_to_host()
{
    local local_path_file_name="${1}"
    local remote_hostname="${2}"
    local remote_dir="${3}"

/usr/bin/expect << EOF
    log_user ${USER_LOG_MODE}
    spawn bash -i
    set timeout ${SCP_TIMEOUT}
    expect -re $
    send "${SCP_CMD} ${local_path_file_name} ${UN}@${remote_hostname}:${remote_dir} 2>>${HOST_COLLECT_ERROR_LOG}\n"
    expect {
        "assword:" {
            send "${pw}\r"
            expect {
                "100%"        { exit ${PASS} }
                "${pw_error}" { exit ${FAIL_PASSWORD}  }
                "${ac_error}" { exit ${FAIL_PERMISSION_SKIP}}
                timeout       { exit ${FAIL_TIMEOUT1}  }
            }
        }
        "No route to host" {
            exit ${FAIL_UNREACHABLE}
        }
        "Could not resolve hostname" {
            exit ${FAIL_UNREACHABLE}
        }
        timeout { exit ${FAIL_TIMEOUT} }
    }
EOF
    local rc=${?}
    if [ ${rc} -ne ${PASS} ] ; then
        report_error "${FAIL_FILE_COPY_STR} ${local_path_file_name} to ${remote_hostname}:${remote_dir}" ${rc}
    fi
    return ${rc}
}

###########################################################################
#
# Name       : create_collect_dir_local
#
# Purpose    : Create the local dated collect dir where all
#              the tarballs for this collect will get put.
#
# Assumptions: Permissions are set to make it easy to copy
#              tarballs from remote host into
#
# Parameters: $1 - the fill dir
#
###########################################################################

function create_collect_dir_local()
{
    local dir=${1}

/usr/bin/expect << EOF
    log_user ${USER_LOG_MODE}
    spawn bash -i
    set timeout 10
    expect -re $
    send "sudo mkdir -m 775 -p ${dir} ; cat ${cmd_done_file}\n"
    expect {
        "assword:" {
            send "${pw}\r"
            expect {
                "${cmd_done_sig}" { exit ${PASS}           }
                "${pw_error}"     { exit ${FAIL_PASSWORD}  }
                "${ac_error}"     { exit ${FAIL_PERMISSION}}
                timeout           { exit ${FAIL_TIMEOUT1}  }
            }
        }
        "${cmd_done_sig}" { exit ${PASS} }
        "${ac_error}"     { exit ${FAIL_PERMISSION}}
        timeout           { exit ${FAIL_TIMEOUT} }
    }
EOF
    local rc=${?}
    if [ ${rc} -ne ${PASS} ] ; then
        report_error "failed to create_collect_dir_local for ${dir}" ${rc}
        collect_exit ${rc}
    fi
    return ${rc}
}

############################################################################
#
# Create the local collect dir where the tarball(s) will temporarily stored
#
# Note: Catches the password error case
#
############################################################################

create_collect_dir_local "${COLLECT_DIR}"

##########################################################################
#
# Name      : remove_file_local
#
# Purpose   : Delete the specified file using sudo
#
# Parameters: $1 - the file to be delete with full path specified
#
###########################################################################

function remove_file_local()
{
    local local_file=${1}
    local rc=${PASS}

    if [ -e ${local_file} ] ; then

/usr/bin/expect << EOF
    log_user ${USER_LOG_MODE}
    spawn bash -i
    set timeout 10
    expect -re $
    send -- "sudo rm -f ${local_file} ; cat ${cmd_done_file}\n"
    expect {
        "assword:"        { send -- "${pw}\r" ; exp_continue }
        "${cmd_done_sig}" { exit ${PASS} }
        "annot remove"    { exit ${FAIL_CLEANUP}    }
        "${pw_error}"     { exit ${FAIL_PASSWORD}   }
        "${ac_error}"     { exit ${FAIL_PERMISSION} }
        timeout           { exit ${FAIL_TIMEOUT}    }
    }
EOF
        rc=${?}
        if [ ${rc} -ne ${PASS} ] ; then
            report_error "failed to remove_file_local ${local_file}" ${rc}
        fi
    fi
    return ${rc}
}

##########################################################################
#
# Name      : remove_dir_local
#
# Purpose   : Delete the specified file using sudo
#
# Parameters: $1 - the directory to be removed with full path specified
#
###########################################################################

function remove_dir_local()
{
    local dir=${1}

/usr/bin/expect << EOF
    log_user ${USER_LOG_MODE}
    spawn bash -i
    set timeout 10
    expect -re $
    send -- "sudo rm -rf ${dir} ; cat ${cmd_done_file}\n"
    expect {
        "assword:"        { send -- "${pw}\r" ; exp_continue }
        "${cmd_done_sig}" { exit ${PASS} }
        "annot remove"    { exit ${FAIL_CLEANUP}    }
        "${pw_error}"     { exit ${FAIL_PASSWORD}   }
        "${ac_error}"     { exit ${FAIL_PERMISSION} }
        timeout           { exit ${FAIL_TIMEOUT}    }
    }
EOF
    local rc=${?}
    if [ ${rc} -ne ${PASS} ] ; then
        report_error "failed to remove_dir_local ${dir}" ${rc}
        dlog "remove_dir_local failed: ${dir}"
    fi
    return ${rc}
}

###########################################################################
#
# Name      : move_file_local
#
# Purpose   : Move a file and change permissions using sudo
#
# Parameters: $1 - src  path/file
#             $2 - dest path/file
#
###########################################################################

function move_file_local()
{
    local src=${1}
    local dst=${2}

/usr/bin/expect << EOF
    log_user ${USER_LOG_MODE}
    spawn bash -i
    set timeout 10
    expect -re $
    send -- "sudo mv ${src} ${dst} ; cat ${cmd_done_file}\n"
    expect {
        "assword:"        { send -- "${pw}\r" ; exp_continue }
        "${cmd_done_sig}" { exit ${PASS} }
        "annot remove"    { exit ${FAIL_CLEANUP}    }
        "${pw_error}"     { exit ${FAIL_PASSWORD}   }
        "${ac_error}"     { exit ${FAIL_PERMISSION} }
        timeout           { exit ${FAIL_TIMEOUT}    }
    }
EOF
    local rc=${?}
    if [ ${rc} -ne ${PASS} ] ; then
        report_error "failed to move_file_local ${src} to ${dst}" ${rc}
    fi
    return ${rc}
}


###########################################################################


###########################################################################
function scratch_full()
{
    avail=$(df -k ${COLLECT_BASE_DIR} | grep -v Available | awk '{ print $4 }')
    if [ ${avail} -lt ${COLLECT_BASE_DIR_FULL_THRESHOLD} ] ; then
        return ${FAIL}
    else
        return ${PASS}
    fi
}

###########################################################################
#
# Name       : echo_stats
#
# Purpose    : print collect data and/or stats
#
# Description: Append the echoed collect done with collect stats
#              Produce a user log that duplicates the console output
#              in both parallel and inline collect modes.
#
# Parameters : $1 - seconds
#              $2 - label for control flow
#              $3 - path/file name to get the size of
#
##########################################################################

function echo_stats()
{
    local secs=${1}
    local label="${2}"
    local file="${3}"
    local MSG=""
    local stats=""

    MSG="collected "
    len=${#label}

    for ((i=len;i<longest_name+16;i++)) ; do
        MSG+=" "
    done

    if [ "${label}" == "stats-only" ] ; then
        MSG+="${file} ... done"
    else
        MSG+="${label} ... done"
    fi

    stats=" ($(date -d@${secs} -u +%H:%M:%S)"
    if [ -e ${file} ] ; then
        avail=$(df -h --output=pcent ${COLLECT_BASE_DIR}  | grep -v Use)
        size=$(du -h ${file} | cut -f 1 2>/dev/null)
        if [ $? -eq 0 ] ; then
            if [ "${label}" == "stats-only" ] ; then
                printf "%s %5s %3s)\n" "${stats}" "${size}" "${avail}"
                log "${MSG} $stats ${size} ${avail})"
            else
                if [ "${PARALLEL_COLLECT_MODE}" = true ] ; then
                    printf "%s %s %5s %3s)\n" "${MSG}" "${stats}" "${size}" "${avail}"
                else
                    printf "%s %5s %3s)\n" "${stats}" "${size}" "${avail}"
                fi
                log "${MSG} $stats ${size} ${avail})"
            fi
            return
        fi
    fi
    printf "stats error)\n"
}

############################################################################
#
# Name       : collect_host_run
#
# Purpose    : Run collect host in selected mode
#
# Description: Run collect_host as a background task for each host if
#              parallel option is specified. Otherwise, run collect in
#              forground (legacy mode) for each host one after the other.
#
############################################################################

function collect_host_run()
{
    local host="${1}"
    local rc=${PASS}

    if [ "${PARALLEL_COLLECT_MODE}" = false ] ; then
        local MSG="collecting"
        # line up the host names
        len=${#host}
        for ((i=len;i<longest_name;i++)) ; do
            MSG+=" "
        done
        MSG+=" ${TARNAME} ... "
        echo -n "$MSG"
    fi

    # Save current user log level
    local save=${USER_LOG_MODE}
    if [ "${VERBOSE}" = true ] ; then
        USER_LOG_MODE=1
    fi

    if [ "${host}" == "${HOSTNAME}" ] ;  then

/usr/bin/expect << EOF
        trap exit {SIGINT SIGTERM}
        log_user ${USER_LOG_MODE}
        spawn bash -i

        set timeout ${TIMEOUT}
        send "sudo SKIP_MASK=${SKIP_MASK} ${collect_host} ${TARNAME} ${STARTDATE_OPTION} ${STARTDATE} ${STARTTIME} ${ENDDATE_OPTION} ${ENDDATE} ${ENDTIME} ${VERBOSE} ${INVENTORY}\n"
        expect {
            "assword:" {
                send "${pw}\r"
                expect {
                    "${FAIL_INSUFFICIENT_SPACE_STR}" { exit ${FAIL_INSUFFICIENT_SPACE}}
                    "${FAIL_OUT_OF_SPACE_STR}" { exit ${FAIL_OUT_OF_SPACE}}
                    "${collect_done}"          { exit ${PASS}             }
                    "${pw_error}" { exit ${FAIL_PASSWORD}  }
                    "${ac_error}" { exit ${FAIL_PERMISSION}}
                    timeout       { exit ${FAIL_TIMEOUT}   }
                }
            }
            timeout { exit ${FAIL_TIMEOUT} }
        }
        exit { ${FAIL} }
EOF
        rc=${?}
    # otherwise the host is remote
    else

/usr/bin/expect << EOF
        trap exit {SIGINT SIGTERM}
        log_user ${USER_LOG_MODE}
        spawn bash -i
        set timeout 30
        expect -re $
        send "${SSH_CMD} ${UN}@${host}\n"
        expect {
            "assword:" {
                send "${pw}\r"
                expect {
                    "${host}:" {
                        set timeout 600
                        send "sudo SKIP_MASK=${SKIP_MASK} ${collect_host} ${TARNAME} ${STARTDATE_OPTION} ${STARTDATE} ${STARTTIME} ${ENDDATE_OPTION} ${ENDDATE} ${ENDTIME} ${VERBOSE} ${INVENTORY}\n"
                        expect {
                            "assword:" {
                                send "${pw}\r"
                                expect {
                                    "${FAIL_INSUFFICIENT_SPACE_STR}" {
                                       send "exit\r"
                                       exit ${FAIL_INSUFFICIENT_SPACE}
                                    }
                                    "${FAIL_OUT_OF_SPACE_STR}" {
                                       send "exit\r"
                                       exit ${FAIL_OUT_OF_SPACE}
                                    }
                                    "${collect_done}" {
                                        send "exit\r"
                                        exit ${PASS}
                                    }
                                    "${pw_error}"   { exit ${FAIL_PASSWORD} }
                                    "${ac_error}"   { exit ${FAIL_PERMISSION_SKIP}}
                                    timeout         { exit ${FAIL_TIMEOUT5} }
                                }
                            }
                            timeout { exit ${FAIL_TIMEOUT4} }
                        }
                    }
                    "${pw_error}" { exit ${FAIL_PASSWORD} }
                    "${ac_error}" { exit ${FAIL_PERMISSION_SKIP}}
                    timeout       { exit ${FAIL_TIMEOUT3} }
                }
            }
            "(yes/no)?" {
                send "yes\r"
                exp_continue
            }
            "No route to host" {
                exit ${FAIL_UNREACHABLE}
            }
            "Could not resolve hostname" {
                exit ${FAIL_UNREACHABLE}
            }
            "Host key verification failed" {
                send "rm -f /home/${UN}/.ssh/known_hosts\n"
                exit ${FAIL}
            }
            timeout { exit ${FAIL_TIMEOUT} }
        }
        exit { $FAIL }
EOF
        rc=${?}
    fi
    USER_LOG_MODE=${save}
    return ${rc}
}

############################################################################
#
# Name       : collect_subcloud_run
#
############################################################################

function collect_subcloud_run()
{
    local subcloud="${1}"
    local tarname="${2}"
    local -i rc=${PASS}

    if [ "${PARALLEL_COLLECT_MODE}" = false ] ; then
        local MSG="collecting"
        # line up the subclouds names
        len=${#subcloud}
        for ((i=len;i<longest_name;i++)) ; do
            MSG+=" "
        done
        MSG+=" ${tarname} ... "
        echo -n "$MSG"
    fi

    # build the command starting with the final tarball name prefix
    collect_cmd=("-n ${tarname}")

    # all hosts in a subcloud are collected
    collect_cmd+=("-a")

    # all subcloud hosts are collected in parallel unless legacy more is specified
    if [ "${PARALLEL_COLLECT_MODE}" = false ] ; then
        collect_cmd+=("-in") ;
    fi
    if [ "${DEBUG}" = true ] ; then
        collect_cmd+=("-d")
    fi
    if [ "${VERBOSE}" = true ] ; then
        collect_cmd+=("-v")
    fi

    collect_cmd+=("--start-date ${STARTDATE}")
    collect_cmd+=("--end-date $ENDDATE")

    # copy the pw file to the subcloud and then cleanup
    TEMPFILE=$(mktemp)
    echo "${pw}" > ${TEMPFILE}
    copy_file_to_host "${TEMPFILE}" "${subcloud}" "/tmp"
    rc=${?}
    remove_file_local ${TEMPFILE}
    if [ ${rc} -ne ${PASS} ] ; then
        report_error "failed to copy '${TEMPFILE}' to ${subcloud}/tmp" ${FAIL_FILE_COPY}
        collect_exit ${FAIL_FILE_COPY}
    fi

    # tell the remote subcloud the name of the password file
    collect_cmd+=("-f ${TEMPFILE}")

    # Save current user log level
    local save=${USER_LOG_MODE}
    if [ "${VERBOSE}" = true ] ; then
        USER_LOG_MODE=1
    fi

    # echo "Subcloud Collect: ${subcloud} ${collect_cmd[@]}"
/usr/bin/expect << EOF
    trap exit {SIGINT SIGTERM}
    log_user ${USER_LOG_MODE}
    spawn bash -i
    set timeout 30
    expect -re $
    send "${SSH_CMD} ${UN}@${subcloud}\n"
    expect {
        "assword:" {
            send "${pw}\r"
            expect {
                "${pw_error}" { exit ${FAIL_PASSWORD} }
                "${ac_error}" { exit ${FAIL_PERMISSION_SKIP}}
                timeout       { exit ${FAIL_TIMEOUT3} }
                "${SUBCLOUD_LOGIN_PROMPT}" {}
                "${subcloud}:" {}
            }
            set timeout ${TIMEOUT}
            send "${collect} ${collect_cmd[@]}\n"
            expect {
                "${collect_done}" {
                    send "exit\r"
                    exit ${PASS}
                }
                "${FAIL_INSUFFICIENT_SPACE_STR}" {
                   send "exit\r"
                   exit ${FAIL_INSUFFICIENT_SPACE}
                }
                "${FAIL_OUT_OF_SPACE_STR}" {
                   send "exit\r"
                   exit ${FAIL_OUT_OF_SPACE}
                }
                "${FAIL_PASSWORD_PROMPT_STR}" {
                   send "exit\r"
                   exit ${FAIL_PASSWORD_PROMPT}
                }
                "${COLLECT_ERROR} ${FAIL_FILE_EMPTY_STR}" {
                   send "exit\r"
                   exit ${FAIL_FILE_EMPTY}
                }
                "${COLLECT_ERROR} ${FAIL_FILE_NOT_FOUND_STR}" {
                   send "exit\r"
                   exit ${FAIL_FILE_NOT_FOUND}
                }
                "${COLLECT_ERROR} ${FAIL_DATE_FORMAT_STR}" {
                   send "exit\r"
                   exit ${FAIL_DATE_FORMAT}
                }
                "${COLLECT_ERROR} ${FAIL_INACTIVE_STR}" {
                   send "exit\r"
                   exit ${FAIL_INACTIVE}
                }
                "${COLLECT_ERROR} ${FAIL_NO_HOSTS_STR}" {
                   send "exit\r"
                   exit ${FAIL_NO_HOSTS}
                }
                "${COLLECT_ERROR} ${FAIL_NO_SUBCLOUDS_STR}" {
                   send "exit\r"
                   exit ${FAIL_NO_SUBCLOUDS}
                }
                "${COLLECT_ERROR} ${FAIL_MISSING_PARAMETER_STR}" {
                   send "exit\r"
                   exit ${FAIL_MISSING_PARAMETER}
                }
                "${COLLECT_ERROR} ${FAIL_NO_FILE_SPECIFIED_STR}" {
                   send "exit\r"
                   exit ${FAIL_NO_FILE_SPECIFIED}
                }
                "${COLLECT_ERROR} ${FAIL_SUBCLOUD_TIMEOUT_STR}" {
                   send "exit\r"
                   exit ${FAIL_SUBCLOUD_TIMEOUT}
                }
                "${COLLECT_ERROR}" {
                   send "exit\r"
                   exit ${FAIL}
                }
                timeout         { exit ${FAIL_TIMEOUT5} }
            }
        }
        "(yes/no)?" {
            send "yes\r"
            exp_continue
        }
        "No route to host" {
            exit ${FAIL_UNREACHABLE}
        }
        "Could not resolve" {
            exit ${FAIL_UNREACHABLE}
        }
        "Host key verification failed" {
            send "rm -f /home/${UN}/.ssh/known_hosts\n"
            exit ${FAIL}
        }
        timeout { exit ${FAIL_TIMEOUT} }
    }
    exit { $FAIL }
EOF
    rc=${?}
    USER_LOG_MODE=${save}
    return ${rc}
}

############################################################################
#
# Name       : collect_host_complete_local
#
# Description: Perform collect host complete operations for a
#              local collect host.
#
#              1. Get the tarball
#              2. Handle errors
#                 - report
#                 - cleanup
#
############################################################################

function collect_host_complete_local()
{
    local tarname="${1}"
    local rc=${PASS}

    # create the dir again just to handle the case where we are
    # collecting on ourself and have removed the collect_dir
    # directory in collect_host above.
    create_collect_dir_local "${COLLECT_DIR}"

    # move the tarball into the collect dir
    # only applies to the local collect since the remote
    # collect scp's it directly into the collect dir.
    move_file_local "${COLLECT_BASE_DIR}/${tarname}.tgz" "${COLLECT_DIR}"
    rc=${?}
    if [ ${rc} -eq ${PASS} ] ; then
        log "collect ${COLLECT_BASE_DIR}/${tarname}.tgz succeeded"
    else
        if [ ${rc} -eq ${FAIL_INSUFFICIENT_SPACE} ] ; then

            report_error "${FAIL_INSUFFICIENT_SPACE_STR}" ${rc}

            echo ""
            wlog "Increase available space in ${host}:${COLLECT_BASE_DIR} and retry operation."
            echo ""

            remove_dir_local ${COLLECT_DIR}

            collect_exit ${FAIL_INSUFFICIENT_SPACE}

        elif [ ${rc} -eq ${FAIL_OUT_OF_SPACE} ] ; then

            report_error "${FAIL_OUT_OF_SPACE_STR}" ${rc}

            echo ""
            wlog "Increase available space in ${host}:${COLLECT_BASE_DIR} and retry operation."
            echo ""

            # Remove the corrupt file and exit
            remove_file_local ${COLLECT_ERROR_LOG}
            remove_file_local ${COLLECT_BASE_DIR}/${tarname}.tgz
            remove_dir_local ${COLLECT_BASE_DIR}/${tarname}
            remove_dir_local ${COLLECT_BASE_DIR}/${COLLECT_NAME}

            collect_exit ${FAIL_OUT_OF_SPACE}

        else
            report_error "failed to collect from ${HOSTNAME} [host complete]" ${rc}
            dlog "collect_host_complete_local failure: ${COLLECT_DIR}:${tarname}:${rc}"
        fi
    fi
    return ${rc}
}

############################################################################
#
# Name       : collect_host_complete_remote
#
# Description: Perform collect host complete operations for a
#              remote host collect.
#
#              1. Fetch the tarball
#              2. Remove tarball from remote host
#              2. Handle errors
#                 - report
#                 - cleanup
#
############################################################################

function collect_host_complete_remote ()
{
    local host="${1}"
    local tarname="${2}"

    if [ "${SUBCLOUD_COLLECT}" == true ] ; then
        SUFFIX="tar"
    else
        SUFFIX="tgz"
    fi
    get_file_from_host "${host}" "${tarname}.${SUFFIX}" "${COLLECT_DIR}"

    local rc=${?}
    if [ ${rc} -eq ${PASS} ] ; then
        if [ "${SUBCLOUD_COLLECT}" == true ] ; then
            # login to subclouds does not show the subcloud name
            # in the login prompt. It will always be one of the controllers
            # so set login prompt to SUBCLOUD_LOGIN_PROMPT
            delete_remote_dir_or_file "${host}" "${COLLECT_BASE_DIR}/${tarname}*" "${SUBCLOUD_LOGIN_PROMPT}" "${host}:"
        else
            # hosts always login as host name, use that hostname as login prompt
            delete_remote_dir_or_file "${host}" "${COLLECT_BASE_DIR}/${tarname}*" "${host}"
        fi
        rc=$?
        if [ ${rc} -eq ${PASS} ] ; then
            log "collect ${COLLECT_BASE_DIR}/${tarname}.${SUFFIX} succeeded"
        else
            log "collect ${COLLECT_BASE_DIR}/${tarname}.${SUFFIX} succeeded but failed to cleanup"
            rc=${PASS}
        fi
    else
        report_error "failed to collect from ${host} [get file]" ${rc}
        dlog "get_file_from_host failure: ${host}:${tarname}.${SUFFIX}:${COLLECT_DIR}"
    fi
    return ${rc}
}

############################################################################
#
# Parallel Collect Support
#
# collect_host_run     - run collect_host as a background task
# collect_host_monitor - monitor for collect_host background task status
# collect_host_done    - mark collect_host done with status
# collect_host_stats   - print collect host stats
#
# collect_host_complete_local  - local collect complete operations
# collect_host_complete_remote - remote collect complete operations
#
# collect_host_ctrl_list_index_print - print collect host control list@index
#
# collect_host_ctrl_list is a structured host list used to track the state of
# collect_host run as a background task for each host.
#
# Structure members:
#
#  hostname - the name of the host being collected
#  stage    - the collect stage for this host ; RUN, MON, DONE
#  pid      - the pid of the background'ed collect host process
#  seconds  - the time in seconds of when the collect started
#  status   - the exit status of the remote collect 0..255
#  name     - the full path and name of the remote collected tarball
#
############################################################################
declare collect_host_ctrl_list=()

# The following index constants are used to access each field.
declare -r INDEX_HOST=0
declare -r INDEX_STAGE=1
declare -r INDEX_PID=2
declare -r INDEX_SECONDS=3
declare -r INDEX_STATUS=4
declare -r INDEX_TARBALL=5

# The stages each launched collect_host goes through
declare -r STAGE_RUN="run"
declare -r STAGE_MON="monitor"
declare -r STAGE_DONE="done"

# declare -r INVALID_PID=-1

###########################################################################
#
# Name       : collect_host_monitor
#
# Purpose    : Transition host into tjhe monitor stage
#
############################################################################

function collect_host_monitor()
{
    local index=${1}

    if [ "${SUBCLOUD_COLLECT}" = true ] ; then
        TARGETS=${SUBCLOUDS}
    else
        TARGETS=${HOSTS}
    fi
    if [ ${index} -lt ${TARGETS} ] ; then
        TARGET=${collect_host_ctrl_list[${index}]}
        info=(${TARGET//:/ })

        # Update collect host control structure for this host with
        #
        # collect_host_ctrl_list[index].stage = MONitor
        #
        collect_host_ctrl_list[${index}]="${info[${INDEX_HOST}]}:\
                                            ${STAGE_MON}:\
                                            ${info[${INDEX_PID}]}:\
                                            ${info[${INDEX_SECONDS}]}:\
                                            ${info[${INDEX_STATUS}]}:\
                                            ${info[${INDEX_TARBALL}]}"
        collect_host_ctrl_list_index_print ${index}
    else
        elog "collect_host_monitor ; invalid index:${index} ; must be smaller than ${TARGETS}"
        collect_exit ${FAIL_INTERNAL}
    fi
}

###########################################################################
#
# Name       : collect_host_done
#
# Purpose    : mark a host collect as done
#
############################################################################

function collect_host_done()
{
    local index=${1}
    local status=${2}

    if [ "${SUBCLOUD_COLLECT}" = true ] ; then
        TARGETS=${SUBCLOUDS}
    else
        TARGETS=${HOSTS}
    fi
    if [ ${index} -lt ${TARGETS} ] ; then
        TARGET=${collect_host_ctrl_list[${index}]}
        info=(${TARGET//:/ })

        # update struct for this pid/process with
        #
        # collect_host_ctrl_list[index].stage = DONE
        # collect_host_ctrl_list[index].seconds = script run time
        # collect_host_ctrl_list[index].status = status
        HOST_START_TIME=${info[${INDEX_SECONDS}]}
        collect_host_ctrl_list[${index}]="${info[${INDEX_HOST}]}:\
                                            ${STAGE_DONE}:\
                                            ${info[${INDEX_PID}]}:\
                                            $((SECONDS-HOST_START_TIME)):\
                                            ${status}:\
                                            ${info[${INDEX_TARBALL}]}"
        collect_host_ctrl_list_index_print ${index}
    else
        elog "collect_host_done ; invalid index:${index} ; must be smaller than ${TARGETS}"
        collect_exit ${FAIL_INTERNAL}
    fi
}

###########################################################################
#
# Name       : collect_host_stats
#
# Purpose    : call echo stats for specified collect_host_ctrl_list index
#
############################################################################

function collect_host_stats()
{
    local index=${1}

    if [ "${SUBCLOUD_COLLECT}" = true ] ; then
        SUFFIX="tar"
    else
        SUFFIX="tgz"
    fi

    if [[ "${PARALLEL_COLLECT_MODE}" = true && ${DONE_COUNT} -eq 0 ]] ; then
        # send new line to delineate '.' progress
        echo ""
        PLEASE_STANDBY=false
    fi

    HOST=${collect_host_ctrl_list[${index}]}
    info=(${HOST//:/ })
    echo_stats "${info[${INDEX_SECONDS}]}" \
               "${info[${INDEX_TARBALL}]}" \
               "${COLLECT_DIR}/${info[${INDEX_TARBALL}]}.${SUFFIX}"
}

###########################################################################
#
# Name       : collect_host_ctrl_list_index_print
#
# Purpose    : debug
#
# Description: print the structure for a specified index
#
############################################################################

collect_host_ctrl_list_index_print()
{
    local index=${1}

    if [ "${DEBUG}" = true ] ; then
        HOST=${collect_host_ctrl_list[${index}]}
        info=(${HOST//:/ })
        printf "%s Debug: %-12s %7s [%6s] | Secs:%3s | %3s | %s\n" \
            "$(date)" \
            "${info[${INDEX_HOST}]}" \
            "${info[${INDEX_STAGE}]}" \
            "${info[${INDEX_PID}]}" \
            "${info[${INDEX_SECONDS}]}" \
            "${info[${INDEX_STATUS}]}" \
            "${info[${INDEX_TARBALL}]}"
        dlog "${info[${INDEX_HOST}]} ${info[${INDEX_STAGE}]} [${info[${INDEX_PID}]}] | Secs:${info[${INDEX_SECONDS}]} | ${info[${INDEX_STATUS}]} | ${info[${INDEX_TARBALL}]}"
    fi
}

############################################################################
#
# Name      : collect_host_clean
#
# Purpose   : Clean collect content in /scratch on specified host
#
# Parameters: $1 - hostname
#
############################################################################

function collect_host_clean()
{
    local host="${1}"
    local rc=${FAIL}

    if [ "${host}" == "None" -o "${host}" == "" ] ; then
        report_error "invalid host (${host}) passed to collect_host_clean" ${FAIL_HOSTNAME}
        return
    fi

    echo -n "cleaning ${host}:${COLLECT_BASE_DIR} ... "
    if [ "${host}" == "${HOSTNAME}" ] ; then
        clean_scratch_dir_local ${host} ${COLLECT_BASE_DIR}
        rc=${?}
    else
        clean_scratch_dir_remote ${host} ${COLLECT_BASE_DIR}
        rc=${?}
    fi
    if [ ${rc} -eq ${PASS} ] ; then
        echo "done"
        log "user cleaned ${host}:${COLLECT_BASE_DIR} content"
    fi
}

############################################################################
#
# Name      : collect_subcloud_clean
#
# Purpose   : Clean collect content in /scratch on specified subcloud
#
# Parameters: $1 - subcloud
#
############################################################################

function collect_subcloud_clean()
{
    local subcloud="${1}"

    check_host_reachable "${subcloud}"
    if [ ${?} -ne ${PASS} ] ; then
        report_error "cannot clean ${subcloud}" ${FAIL_UNREACHABLE}
        return ${FAIL_UNREACHABLE}
    fi

    echo -n "cleaning subcloud $subcloud:${COLLECT_BASE_DIR} ... "

    # Save current user log level
    local save=${USER_LOG_MODE}
    if [ "${VERBOSE}" = true ] ; then
        USER_LOG_MODE=1
    fi

    # build the command
    collect_cmd=("--clean --all --name ${subcloud}")

    # copy the pw file to the subcloud and then cleanup
    TEMPFILE=$(mktemp)
    echo "${pw}" > ${TEMPFILE}
    copy_file_to_host "${TEMPFILE}" "${subcloud}" "/tmp"
    rc=${?}
    remove_file_local ${TEMPFILE}
    if [ ${rc} -ne ${PASS} ] ; then
        report_error "failed to copy '${TEMPFILE}' to ${subcloud}/tmp" ${FAIL_FILE_COPY}
        collect_exit ${FAIL_FILE_COPY}
    fi
    collect_cmd+=("-f ${TEMPFILE}")

    if [ "${DEBUG}"    = true ] ; then
        collect_cmd+=("-d")
    fi
    if [ "${VERBOSE}"  = true ] ; then
        collect_cmd+=("-v")
    fi

    # echo "Subcloud Collect Clean: ${subcloud} ${collect_cmd[@]}"

/usr/bin/expect << EOF
    trap exit {SIGINT SIGTERM}
    log_user ${USER_LOG_MODE}
    spawn bash -i
    set timeout 30
    expect -re $
    send "${SSH_CMD} ${UN}@${subcloud}\n"
    expect {
        "assword:" {
            send "${pw}\r"
            expect {
                "${pw_error}" { exit ${FAIL_PASSWORD} }
                "${ac_error}" { exit ${FAIL_PERMISSION_SKIP}}
                timeout       { exit ${FAIL_TIMEOUT3} }
                "${SUBCLOUD_LOGIN_PROMPT}" {}
                "${subcloud}:" {}
            }
            send "${collect} ${collect_cmd[@]}\n"
            expect {
                "${collect_done}" {
                    send "exit\r"
                    exit ${PASS}
                }
                "${COLLECT_ERROR} ${FAIL_INACTIVE_STR}" {
                   send "exit\r"
                   exit ${FAIL_INACTIVE}
                }
                "${COLLECT_ERROR} ${FAIL_NO_HOSTS_STR}" {
                   send "exit\r"
                   exit ${FAIL_NO_HOSTS}
                }
                "${COLLECT_ERROR} ${FAIL_MISSING_PARAMETER_STR}" {
                   send "exit\r"
                   exit ${FAIL_MISSING_PARAMETER}
                }
                "${COLLECT_ERROR} ${FAIL_NO_FILE_SPECIFIED_STR}" {
                   send "exit\r"
                   exit ${FAIL_NO_FILE_SPECIFIED}
                }
                "${COLLECT_ERROR}" {
                   send "exit\r"
                   exit ${FAIL}
                }
                timeout {
                    exit ${FAIL_TIMEOUT5}
                }
            }
        }
        "(yes/no)?" {
            send "yes\r"
            exp_continue
        }
        "No route to host" {
            exit ${FAIL_UNREACHABLE}
        }
        "Could not resolve" {
            exit ${FAIL_UNREACHABLE}
        }
        "Host key verification failed" {
            send "rm -f /home/${UN}/.ssh/known_hosts\n"
            exit ${FAIL}
        }
        timeout { exit ${FAIL_TIMEOUT} }
    }
    exit { $FAIL }
EOF
    rc=${?}
    if [ ${rc} -eq ${PASS} ] ; then
        log "clean of ${subcloud} hosts successful"
        echo "done"
    else
        echo "failed to clean ${subcloud} rc:${rc}"
    fi

    USER_LOG_MODE=${save}
    return ${rc}
}

############################################################################
#
#                    Handle clean command option
#
############################################################################

if [ "${CLEAN}" = true ] ; then

    if [ "${SUBCLOUD_COLLECT}" = true ] ; then
        if [ ${SUBCLOUDS} -eq 0 ] ; then
            report_error "no valid subclouds to clean" ${FAIL_NO_HOSTS}
            collect_exit ${FAIL_NO_HOSTS}
        fi
        dlog "cleaning scratch space on ${SUBCLOUDLIST[@]}"
        for subcloud in "${SUBCLOUDLIST[@]}" ; do
            collect_subcloud_clean "${subcloud}"
        done
    else
        if [ ${HOSTS} -eq 0 ] ; then
            report_error "no valid hosts to clean" ${FAIL_NO_HOSTS}
            collect_exit ${FAIL_NO_HOSTS}
        fi
        dlog "cleaning scratch space on ${HOSTLIST[@]}"
        for host in "${HOSTLIST[@]}" ; do
            collect_host_clean "$host"
        done
        if [ "${ORCHESTRATED_COLLECT}" = true ] ; then
            echo "${collect_done}"
        fi
    fi
    collect_exit ${PASS}
fi

############################################################################
#
#                         Handle collect
#
############################################################################

declare COLLECT_START_TIME=${SECONDS}

if [ "${SUBCLOUD_COLLECT}" = true ] ; then
    for subcloud in "${SUBCLOUDLIST[@]}" ; do
        len=${#subcloud}
        if [ $len -gt ${longest_name} ] ; then
            longest_name=$len
        fi
    done
else
    for host in "${HOSTLIST[@]}" ; do
        len=${#host}
        if [ $len -gt ${longest_name} ] ; then
            longest_name=$len
        fi
    done
fi

############################################################################
#
# Name       : collect_hosts
#
# Purpose    : Run collect for all hosts in HOSTLIST
#
# Description: Loop over all the targetted hosts and
#
# 1. run collect_host
#
#    if PARALLEL = true  - Collect all hosts in parallel (all at once).
#                          i.e. launch one background task per host.
#                          Default behavior.
#
#    if PARALLEL = false - Collect all hosts inline, one after the other.
#                          i.e. run collect for each host one after the other.
#                          Specify the -in or --inline command line option.
#
# 2. copy the tarball to $COLLECT_DIR
#
############################################################################

function collect_hosts()
{
    dlog "collect_hosts: [${HOSTS}] ${HOSTLIST[@]}"
    let NEXT_PROGRESS_TIME=${SECONDS}+${PROGRESS_INTERVAL}

    for host in "${HOSTLIST[@]}" ; do
        if [ "${host}" != " " ] ; then

            if [ "${host}" == "None" ] ; then
                continue
            elif [ "${host}" == "" ] ; then
                continue
            fi

            check_host_reachable "${host}"
            if [ ${?} -ne ${PASS} ] ; then
                report_error "cannot collect from ${host}" ${FAIL_UNREACHABLE}
                continue
            fi

            HOST_START_TIME=${SECONDS}
            TARNAME="${host}_${NOWDATE}"

            if [ "${PARALLEL_COLLECT_MODE}" = true ] ; then

                # run collect_host in the background
                (collect_host_run "${host}" "${TARNAME}")&

                # save the child process's pid
                CHILD_PID=${!}

                #################################################################
                #
                # Add this collect_host's background child process info
                # to the collect_host_ctrl_list
                #
                # collect_host_ctrl_list[index].hostname = host
                # collect_host_ctrl_list[index].stage    = RUN
                # collect_host_ctrl_list[index].pid      = invalid pid (-1)
                # collect_host_ctrl_list[index].seconds  = script time in secs
                # collect_host_ctrl_list[index].status   = default to FAIL
                # collect_host_ctrl_list[index].tarball  = host's tarball name
                #
                #################################################################
                collect_host_ctrl_list[${index}]="${host}:\
                                                    ${STAGE_RUN}:\
                                                    ${CHILD_PID}:\
                                                    ${SECONDS}:\
                                                    ${FAIL}:\
                                                    ${TARNAME}"
                collect_host_ctrl_list_index_print ${index}
                index=$((index+1))

            else

                collect_host_run "${host}" "${TARNAME}"
                rc=${?}
                if [ ${rc} -eq ${PASS} ] ; then

                    if [ "${host}" == "${HOSTNAME}" ] ; then
                        collect_host_complete_local "${TARNAME}"
                    else
                        collect_host_complete_remote "${host}" "${TARNAME}"
                    fi
                    rc=${?}
                    if [ ${rc} -ne ${PASS} ] ; then
                        # handle copy error here
                        report_error "failed to collect from ${host} [host file get]" ${rc}
                    else
                        secs=$((SECONDS-HOST_START_TIME))
                        echo -n "done"
                        echo_stats $secs "${TARNAME}" "${COLLECT_DIR}/${TARNAME}.tgz"
                    fi
                elif [ ${rc} -ge ${FAIL_TIMEOUT} -a ${rc} -le ${FAIL_TIMEOUT9} -a "${DCROLE}" == "${DCROLE_SUBCLOUD}" ] ; then
                    report_error "failed to collect from ${host} [subcloud host run timeout]" ${FAIL_SUBCLOUD_TIMEOUT}
                else
                    report_error "failed to collect from ${host} [host]" ${rc}
                fi
            fi
        fi
    done

    #############################################
    #
    # Parallel Collect Mode
    #
    #############################################
    monitoring=false
    if [ "${PARALLEL_COLLECT_MODE}" = true ] ; then

        echo -n "monitoring host collect ; please standby "
        PLEASE_STANDBY=true

        # All hosts collected overall timeout
        while [ ${UNTIL} -ge ${SECONDS} ] ; do
            index=0
            monitoring=false
            for HOST in "${collect_host_ctrl_list[@]}" ; do
                info=(${HOST//:/ })
                # collect_host_ctrl_list_index_print ${index}
                if [ "${info[${INDEX_STAGE}]}" == "${STAGE_MON}" ] ; then

                    # check to see if this collect_host pocess is done collecting
                    kill -0 "${info[${INDEX_PID}]}" 2>/dev/null
                    rc=${?}
                    if [ ${rc} -ne 0 ] ; then

                        # the process is done ; get its exit code
                        wait "${info[${INDEX_PID}]}"
                        rc=${?}
                        if [ ${rc} == ${PASS} ] ; then

                            # if it passed then fetch that host's tarball
                            if [ "${info[${INDEX_HOST}]}" == "${HOSTNAME}" ] ; then
                                collect_host_complete_local  "${info[${INDEX_TARBALL}]}"
                            else
                                collect_host_complete_remote "${info[${INDEX_HOST}]}" \
                                                             "${info[${INDEX_TARBALL}]}"
                            fi
                            rc=${?}
                            collect_host_done ${index} ${rc}
                            if [ ${rc} -eq ${PASS} ] ; then
                                collect_host_stats ${index} ${rc}
                            fi
                            DONE_COUNT=$((DONE_COUNT+1))
                        else
                            collect_host_done ${index} ${rc}
                            report_error "failed to collect from ${info[${INDEX_HOST}]} [target]" ${rc}
                        fi
                    else
                        if [ ${DONE_COUNT} -eq 0 ] ; then
                            if [ ${SECONDS} -gt ${NEXT_PROGRESS_TIME} ] ; then
                                echo -n "."
                                let NEXT_PROGRESS_TIME=${SECONDS}+${PROGRESS_INTERVAL}
                            fi
                        fi

                        monitoring=true
                    fi

                elif [ "${info[${INDEX_STAGE}]}" == "${STAGE_RUN}" ] ; then
                    monitoring=true
                    # update stage to Monitor
                    collect_host_monitor ${index}
                fi
                index=$((index+1))
            done

            if [ "${monitoring}" = false ] ; then
                ilog "collected from ${DONE_COUNT} hosts"
                break
            fi
        done
    fi

    # Report that the overall collect timed-out
    if [ "$monitoring" = true ]; then
        # there may be partial collect worth keeping
        report_error "collect operation timeout after $TIMEOUT secs" ${FAIL_TIMEOUT}
    fi
}

############################################################################
#
# Name       : collect_subclouds
#
# Purpose    : Run collect for all subclouds in SUBCLOUDLIST
#
# Description: Loop over all the specified subclouds and
#
# 1. run collect_subcloud
#
#    if PARALLEL = true  - Collect all subcloudss in parallel (all at once).
#                          i.e. launch one background task per subcloud.
#                          All hosts in subcloud also collected in parallel
#                          Default behavior.
#
#    if PARALLEL = false - Collect all hosts inline, one after the other.
#                          i.e. run collect for each host one after the other.
#                          All hosts in subcloud also collected inline
#                          Specify the -in or --inline command line option.
#
# 2. copy the tarball to $COLLECT_DIR
#
############################################################################

declare -i PROGRESS_INTERVAL=15  # seconds
collect_subclouds()
{
    dlog "collect_subclouds: [${SUBCLOUDS}] ${SUBCLOUDLIST[@]}"
    let NEXT_PROGRESS_TIME=${SECONDS}+${PROGRESS_INTERVAL}

    local -a DONE_LIST=()
    for subcloud in "${SUBCLOUDLIST[@]}" ; do
        if [ "${subcloud}" != " " ] ; then

            if [ "${subcloud}" == "None" ] ; then
                continue
            elif [ "${subcloud}" == "" ] ; then
                continue
            fi

            check_host_reachable "${subcloud}"
            if [ ${?} -ne ${PASS} ] ; then
                report_error "cannot collect from ${subcloud}" ${FAIL_UNREACHABLE}
                continue
            fi

            SUBCLOUD_START_TIME=${SECONDS}
            if [ "${PARALLEL_COLLECT_MODE}" = true ] ; then

                # Collect subclouds in parallel mode

                #Run collect_subcloud in the background
                (collect_subcloud_run "${subcloud}" "${subcloud}_${NOWDATE}")&

                # save the child process's pid
                CHILD_PID=${!}

                #################################################################
                #
                # Add this collect_subcloud_run's background child process info
                # to the collect_host_ctrl_list
                #
                # collect_host_ctrl_list[index].hostname = subcloud
                # collect_host_ctrl_list[index].stage    = RUN
                # collect_host_ctrl_list[index].pid      = invalid pid (-1)
                # collect_host_ctrl_list[index].seconds  = script time in secs
                # collect_host_ctrl_list[index].status   = default to FAIL
                # collect_host_ctrl_list[index].tarball  = host's tarball name
                #
                #################################################################
                collect_host_ctrl_list[${index}]="${subcloud}:\
                                                    ${STAGE_RUN}:\
                                                    ${CHILD_PID}:\
                                                    ${SECONDS}:\
                                                    ${FAIL}:\
                                                    ${subcloud}_${NOWDATE}"
                collect_host_ctrl_list_index_print ${index}
                index=$((index+1))

            else

                # Run collect subclouds one after the other (legacy) mode.

                # make the collected filename be the subcloud name it was
                # collected from with the date of this overall collect.
                collect_subcloud_run "${subcloud}" "${subcloud}_${NOWDATE}"
                rc=${?}
                if [ ${rc} -eq ${PASS} ] ; then

                    collect_host_complete_remote "${subcloud}" "${subcloud}_${NOWDATE}"
                    rc=${?}
                    if [ ${rc} -ne ${PASS} ] ; then
                        # handle copy error here
                        report_error "failed to collect from ${subcloud} [subcloud get]" ${rc}
                    else
                        secs=$((SECONDS-SUBCLOUD_START_TIME))
                        echo -n "done"
                        if [ "${SUBCLOUD_COLLECT}" = true ] ; then
                            SUFFIX="tar"
                        else
                            SUFFIX="tgz"
                        fi
                        echo_stats $secs "${COLLECT_NAME}" "${COLLECT_DIR}/${subcloud}_${NOWDATE}.${SUFFIX}"
                    fi
                else
                    report_error "failed to collect from ${subcloud} [subcloud run]" ${rc}
                fi
                DONE_COUNT=$((DONE_COUNT+1))
                DONE_LIST+=(${subcloud})

                #################################################
                # Check available space and stop collecting
                # if the scratch_full threshold is reached
                #################################################
                if [ ${DONE_COUNT} -lt ${SUBCLOUDS} ] ; then
                    scratch_full
                    if [ ${?} -eq ${FAIL} ] ; then
                        wlog "unable to collect more subclouds ; ${COLLECT_BASE_DIR} is almost full ; suspending subcloud collect"

                        TODO_LIST=()
                        for sc in "${SUBCLOUDLIST[@]}" ; do
                            local found=false
                            for done_sc in "${DONE_LIST[@]}" ; do
                                if [ "${done_sc}" == "${sc}" ] ; then
                                    found=true
                                    break
                                fi
                            done
                            if [ "${found}" = false ] ; then
                                TODO_LIST+=($sc)
                            fi
                        done
                        if [ ${#TODO_LIST[@]} -ne 0 ] ; then
                            log "the following ${#TODO_LIST[@]} subclouds were not collected: ${TODO_LIST[@]}"
                            echo "${TODO_LIST[@]}" > ${SUBCLOUD_COLLECT_CONTINUE_LIST_FILE}
                            COLLECT_CONTINUE_MSG_NEEDED=true
                        fi
                        monitoring=false
                        break
                    fi
                fi
            fi
        fi
    done

    #############################################
    #
    # Parallel Collect Mode - Monitoring
    #
    #############################################
    monitoring=false

    if [ "${PARALLEL_COLLECT_MODE}" = true ] ; then

        echo -n "monitoring subcloud collect ; please standby "
        PLEASE_STANDBY=true

        # All hosts collected overall timeout
        while [ ${UNTIL} -ge ${SECONDS} ] ; do
            index=0
            monitoring=false
            for subcloud in "${collect_host_ctrl_list[@]}" ; do
                info=(${subcloud//:/ })

                # collect_host_ctrl_list_index_print ${index}
                if [ "${info[${INDEX_STAGE}]}" == "${STAGE_MON}" ] ; then

                    # check to see if this collect_host pocess is done collecting
                    kill -0 "${info[${INDEX_PID}]}" 2>/dev/null
                    rc=${?}
                    if [ ${rc} -ne 0 ] ; then

                        # the process is done ; get its exit code
                        wait "${info[${INDEX_PID}]}"
                        rc=${?}
                        if [ ${rc} == ${PASS} ] ; then

                            # if it passed then fetch that host's tarball
                            if [ "${info[${INDEX_HOST}]}" == "${HOSTNAME}" ] ; then
                                collect_host_complete_local  "${info[${INDEX_TARBALL}]}"
                            else
                                collect_host_complete_remote "${info[${INDEX_HOST}]}" \
                                                             "${info[${INDEX_TARBALL}]}"
                            fi
                            rc=${?}
                            collect_host_done ${index} ${rc}
                            if [ ${rc} -eq ${PASS} ] ; then
                                collect_host_stats ${index} ${rc}
                            fi
                            DONE_COUNT=$((DONE_COUNT+1))

                            #################################################
                            # Check available space and stop collecting
                            # if the scratch_full threshold is reached
                            #################################################
                            if [ ${DONE_COUNT} -lt ${SUBCLOUDS} ] ; then
                                scratch_full
                                if [ ${?} -eq ${FAIL} ] ; then
                                    wlog "unable to collect more subclouds ; ${COLLECT_BASE_DIR} is almost full ; suspending subcloud collect"

                                    # search for subclouds in the MONitoring state
                                    # and add them to the TODO_LIST
                                    TODO_LIST=()
                                    for sc in "${collect_host_ctrl_list[@]}" ; do
                                        info=(${sc//:/ })
                                        if [ "${info[${INDEX_STAGE}]}" == "${STAGE_MON}" ] ; then
                                            TODO_LIST+=(${info[${INDEX_HOST}]})
                                        fi
                                    done
                                    if [ ${#TODO_LIST[@]} -ne 0 ] ; then
                                        log "the following ${#TODO_LIST[@]} subclouds were not collected: ${TODO_LIST[@]}"
                                        echo "${TODO_LIST[@]}" > ${SUBCLOUD_COLLECT_CONTINUE_LIST_FILE}
                                        COLLECT_CONTINUE_MSG_NEEDED=true
                                    fi
                                    monitoring=false
                                    break
                                fi
                            fi
                        else
                            collect_host_done ${index} ${rc}
                            report_error "failed to collect from ${info[${INDEX_HOST}]} [remote]" ${rc}
                        fi
                    else
                        if [ ${DONE_COUNT} -eq 0 ] ; then
                            if [ ${SECONDS} -gt ${NEXT_PROGRESS_TIME} ] ; then
                                echo -n "."
                                let NEXT_PROGRESS_TIME=${SECONDS}+${PROGRESS_INTERVAL}
                            fi
                        fi

                        monitoring=true
                    fi

                elif [ "${info[${INDEX_STAGE}]}" == "${STAGE_RUN}" ] ; then
                    monitoring=true
                    # update stage to Monitor
                    collect_host_monitor ${index}
                fi
                index=$((index+1))
            done

            if [ "${monitoring}" = false ] ; then
                ilog "collected from ${DONE_COUNT} subclouds"
                break
            fi
        done
    fi
    # Report that the overall collect timed-out
    if [ "$monitoring" = true ]; then
        if [ "${ORCHESTRATED_COLLECT}" = true ] ; then
            report_error "collect operation timeout after $TIMEOUT secs" ${FAIL_SUBCLOUD_TIMEOUT}
        else
            report_error "collect operation timeout after $TIMEOUT secs" ${FAIL_TIMEOUT}
        fi
    fi
}

############################################################################
#
#         Handle subcloud and system hosts batched collect
#
############################################################################

declare -i TIMEOUT_THRESHOLD_FACTOR=20
declare -i SUBCLOUDS_TIMEOUT_BOOST=20
declare -i HOSTS_TIMEOUT_BOOST=10
declare -i MAX_LIST_PRINT=6

if [ "${SUBCLOUD_COLLECT}" = true ] ; then
    if [ ${SUBCLOUDS} -eq 0 ] ; then
        report_error "no valid subclouds to collect" ${FAIL_NO_SUBCLOUDS}
        collect_exit ${FAIL_NO_SUBCLOUDS}
    fi
    if [ ${SUBCLOUDS} -gt ${TIMEOUT_THRESHOLD_FACTOR} -a "${PARALLEL_COLLECT_MODE}" = true ] ; then
        # adjust overall timeout to account for the large number of subclouds
        let UNTIL=$(((SUBCLOUDS*SUBCLOUDS_TIMEOUT_BOOST)+TIMEOUT))
        ilog "adjusted subcloud collect timout from ${TIMEOUT} to ${UNTIL} secs to account for ${SUBCLOUDS} subclouds"
    fi
    if [ "${ALLHOSTS}" = true ] ; then
        if [ ${SUBCLOUDS} -gt ${MAX_LIST_PRINT} ] ; then
            ilog "collecting data from all ${SUBCLOUDS} subcloud(s)"
        else
            ilog "collecting data from ${SUBCLOUDS} subcloud(s)"
        fi
    elif [ ${SUBCLOUDS} -gt ${MAX_LIST_PRINT} ] ; then
        ilog "collecting data from ${SUBCLOUDS} subcloud(s)"
    else
        ilog "collecting data from ${SUBCLOUDS} subcloud(s): ${SUBCLOUDLIST[@]}"
    fi
    collect_subclouds "$@"
else
    if [ ${HOSTS} -eq 0 ] ; then
        report_error "no valid hosts to collect" ${FAIL_NO_HOSTS}
        collect_exit ${FAIL_NO_HOSTS}
    fi
    if [ ${HOSTS} -gt ${TIMEOUT_THRESHOLD_FACTOR} -a "${PARALLEL_COLLECT_MODE}" = true ] ; then
        # adjust overall timeout to account for the large number of hosts
        let UNTIL=$(((HOSTS*HOSTS_TIMEOUT_BOOST)+TIMEOUT))
        ilog "adjusted hosts collect timout from ${TIMEOUT} to ${UNTIL} secs to account for ${HOSTS} hosts"
    fi
    if [ "${ALLHOSTS}" = true ] ; then
        if [ ${HOSTS} -gt ${MAX_LIST_PRINT} ] ; then
            ilog "collecting data from all ${HOSTS} host(s)"
        else
            ilog "collecting data from ${HOSTS} host(s)"
        fi
    elif [ ${HOSTS} -gt ${MAX_LIST_PRINT} ] ; then
        ilog "collecting data from ${HOSTS} host(s)"
    else
        ilog "collecting data from ${HOSTS} host(s): ${HOSTLIST[@]}"
    fi
    collect_hosts "$@"
fi

############################################################################
#
# Pre tar check. Don't try to create a tarball from an empty COLLECT_DIR
#
############################################################################

if [ -d ${COLLECT_DIR} ] ; then
    stat ${COLLECT_DIR}/* 2>/dev/null 1>/dev/null
    if [ $? -eq 0 ] ; then
        tarballs=(${COLLECT_DIR}/*)
        for tarball in "${tarballs[@]}" ; do
            dlog "collected $tarball"
        done
    else
        elog "No ${COLLECT_DIR} tarballs found ; refusing to create empty ${TARBALL_NAME}"
        collect_exit ${FAIL_NO_TARFILES}
    fi
else
    elog "${COLLECT_DIR} not present ; refusing to create empty ${TARBALL_NAME}"
    collect_exit ${FAIL_NO_TARDIR}
fi

############################################################################
#
# Proceed with the tar after cleaning up error files.
# These files are used to seach for tar failures due to out-of-space logs
#
############################################################################

echo -n "creating ${COLLECT_TYPE} tarball ${TARBALL_NAME} ... "

remove_file_local ${COLLECT_ERROR_LOG}
remove_file_local ${HOST_COLLECT_ERROR_LOG}

/usr/bin/expect << EOF
    log_user ${USER_LOG_MODE}
    spawn bash -i
    expect -re $
    set timeout 200
    send "(cd ${COLLECT_BASE_DIR} ; sudo ${IONICE_CMD} ${NICE_CMD} ${TAR_CMD_APPEND} ${TARBALL_NAME} --remove-files ${COLLECT_NAME}/* 2>>${COLLECT_ERROR_LOG} ; cat ${cmd_done_file})\n"
    expect {
        "assword:" {
            send "${pw}\r"
            expect {
                "${cmd_done_sig}" { exit ${PASS}            }
                "${pw_error}"     { exit ${FAIL_PASSWORD}   }
                "${ac_error}"     { exit ${FAIL_PERMISSION} }
                timeout           { exit ${FAIL_TIMEOUT1}   }
            }
        }
        timeout { exit ${FAIL_TIMEOUT} }
    }
EOF
rc=${?}
if [ ${rc} -ne ${PASS} ] ; then
    collect_errors ${HOSTNAME}
    report_error "failed to create ${TARBALL_NAME}" ${rc}
else
    collect_errors ${HOSTNAME}
    rc=$?
    if [ ${rc} -eq ${PASS} ] ; then
        secs=$((SECONDS-COLLECT_START_TIME))
        echo -n "done"
        echo_stats $secs "stats-only" "${TARBALL_NAME}"
        log "created ${COLLECT_TYPE} tarball ${TARBALL_NAME}"

        if [ "${ORCHESTRATED_COLLECT}" = true ] ; then
            echo "${collect_done}"
        fi
    else
        echo "removing incomplete collect: ${TARBALL_NAME}"
        remove_file_local "${TARBALL_NAME}"

        if [ "${COLLECT_CONTINUE_MSG_NEEDED}" = true ] ; then
            # collect continue is not supported if the previous collect fails
            remove_file_local "${SUBCLOUD_COLLECT_CONTINUE_LIST_FILE}"
            COLLECT_CONTINUE_MSG_NEEDED=false
        fi
    fi
fi
remove_file_local ${COLLECT_ERROR_LOG}
remove_dir_local "${COLLECT_DIR}"

if [ "${COLLECT_CONTINUE_MSG_NEEDED}" = true ] ; then
    echo "------------------------------------------------------------------------------------------"
    echo ""
    wlog "Unable to gather from all requested subclouds due to limited ${COLLECT_BASE_DIR} space."
    echo "... Successful subcloud collects stored in ${TARBALL_NAME}"
    echo "... List of uncollected subclouds is saved in ${SUBCLOUD_COLLECT_CONTINUE_LIST_FILE}"
    echo "... Copy ${TARBALL_NAME} off-system and then delete it from ${COLLECT_BASE_DIR}."
    echo "... Re-run collect subcloud with the --continue option to collect remaining subclouds:"
    echo ""
    echo "    ${HOSTNAME}:$ collect --subcloud --continue"
    echo ""
    echo "------------------------------------------------------------------------------------------"
fi

# return to callers dir
cd ${CURR_DIR}

collect_exit ${rc}