/*
 * Copyright (c) 2013-2016 Wind River Systems, Inc.
*
* SPDX-License-Identifier: Apache-2.0
*
 */

 /**
  * @file
  * Wind River CGCS Platform Process Monitor Service
  * Passive and Active Monitoring FSMs.
  */

#include "pmon.h"
#include "alarmUtil.h"     /* for ... alarmUtil_getSev_str                 */

#define WARNING_THROTTLE (10)

const char passiveStages_str [PMON_STAGE__STAGES][32] =
{
    "Start",
    "Manage",
    "Respawn",
    "Monitor-Wait",
    "Monitor-Process",
    "Restart-Wait",
    "Ignore-Process",
    "Handler-Finish",
    "Subfunction-Polling",
    "Monitor-Start-Wait",
} ;

const char statusStages_str [STATUS_STAGE__STAGES][32] =
{
    "Begin",
    "Execute-Status",
    "Execute-Status-Wait",
    "Execute-Start",
    "Execute-Start-Wait",
    "Interval-Wait"
} ;

int statusStageChange ( process_config_type * ptr , statusStage_enum newStage )
{
    if ((   newStage < STATUS_STAGE__STAGES ) &&
        ( ptr->status_stage < STATUS_STAGE__STAGES ))
    {
        clog ("%s %s -> %s (%d->%d)\n",
               ptr->process,
               statusStages_str[ptr->status_stage],
               statusStages_str[newStage],
               ptr->status_stage, newStage);
        ptr->status_stage = newStage ;
        return (PASS);
    }
    else
    {
        slog ("%s Invalid Stage (now:%d new:%d)\n", ptr->process, ptr->status_stage, newStage );
        ptr->status_stage = STATUS_STAGE__BEGIN ;
        return (FAIL);
    }
}

int passiveStageChange ( process_config_type * ptr , passiveStage_enum newStage )
{
    if ((   newStage < PMON_STAGE__STAGES ) &&
        ( ptr->stage < PMON_STAGE__STAGES ))
    {
        clog ("%s %s -> %s (%d->%d)\n",
               ptr->process,
               passiveStages_str[ptr->stage],
               passiveStages_str[newStage],
               ptr->stage, newStage);
        ptr->stage = newStage ;
        return (PASS);
    }
    else
    {
        slog ("%s Invalid Stage (now:%d new:%d)\n",
                  ptr->process, ptr->stage, newStage );
        ptr->stage = PMON_STAGE__FINISH ;
        return (FAIL);
    }
}

const char * get_pmonStage_str  ( process_config_type * ptr )
{
    if ( ptr->stage < PMON_STAGE__STAGES )
    {
        return ( &passiveStages_str[ptr->stage][0] ) ;
    }
    return (NULL);
}

const char activeStages_str [ACTIVE_STAGE__STAGES][32] =
{
    "Idle",           /*  0 */
    "Start",          /*  1 */
    "Request",        /*  2 */
    "Wait",           /*  3 */
    "Response",       /*  4 */
    "Gap-Setup",      /*  5 */
    "Gap",            /*  6 */
    "Failed",         /*  7 */
    "Debounce-Setup",
    "Debounce",
    "Finish",
} ;

int activeStageChange ( process_config_type * ptr , activeStage_enum newStage )
{
    if ((   newStage < ACTIVE_STAGE__STAGES ) &&
        ( ptr->active_stage < ACTIVE_STAGE__STAGES ))
    {
        clog ("%s %s -> %s (%d->%d)\n",
               ptr->process,
               activeStages_str[ptr->active_stage],
               activeStages_str[newStage],
               ptr->active_stage, newStage);
        ptr->active_stage = newStage ;
        return (PASS);
    }
    else
    {
        slog ("%s Invalid Stage (now:%d new:%d)\n",
                  ptr->process, ptr->active_stage, newStage );
        ptr->active_stage = ACTIVE_STAGE__PULSE_REQUEST ;
        return (FAIL);
    }
}

const char * get_amonStage_str  ( process_config_type * ptr )
{
    if ( ptr->active_stage < ACTIVE_STAGE__STAGES )
    {
        return ( &activeStages_str[ptr->active_stage][0] ) ;
    }
    return (NULL);
}


void clear_amon_counts (  process_config_type * ptr )
{
    if ( ptr->b2b_miss_count > ptr->b2b_miss_peak )
        ptr->b2b_miss_peak = ptr->b2b_miss_count ;

    if ( ptr->mesg_err_cnt > ptr->mesg_err_peak )
        ptr->mesg_err_peak = ptr->mesg_err_cnt ;

    ptr->b2b_miss_count = 0 ;
    ptr->send_err_cnt   = 0 ;
    ptr->recv_err_cnt   = 0 ;
    ptr->mesg_err_cnt   = 0 ;
}


/* Active Monitoring Handler
 * --------------------------
 * Verifies that the process has an active pid */
int pmon_active_handler ( process_config_type * ptr )
{
    int rc = PASS ;

    if ( ptr->active_stage < ACTIVE_STAGE__STAGES )
    {
        dlog2 ("%s Active-%s Stage %d\n",
                   ptr->process,
                   activeStages_str[ptr->active_stage],
                   ptr->active_stage );
    }
    else
    {
        activeStageChange ( ptr, ACTIVE_STAGE__PULSE_REQUEST );
    }

    switch ( ptr->active_stage )
    {
        case ACTIVE_STAGE__IDLE:
        {
           break ;
        }
        case ACTIVE_STAGE__START_MONITOR:
        {
            rc = open_process_socket ( ptr );
            if ( rc != PASS )
            {
                ptr->active_failed = true ;
                elog ("%s 'open_process_socket' failed (%d)\n", ptr->process , rc );
                manage_process_failure ( ptr );
            }

            activeStageChange ( ptr, ACTIVE_STAGE__GAP_SETUP );
            break ;
        }
        case ACTIVE_STAGE__GAP_SETUP:
        {
            mtcTimer_reset ( ptr->pt_ptr );
            mtcTimer_start ( ptr->pt_ptr, pmon_timer_handler, ptr->period );
            activeStageChange ( ptr, ACTIVE_STAGE__GAP_WAIT );
            break ;
        }
        case ACTIVE_STAGE__GAP_WAIT:
        {
            if ( ptr->pt_ptr->ring == true )
            {
                activeStageChange ( ptr, ACTIVE_STAGE__PULSE_REQUEST );
            }
            break ;
        }
        case ACTIVE_STAGE__FAILED:
        {
            ptr->active_response = false ;
            ptr->active_failed = true ;
            ptr->afailed_count++ ;
            ptr->b2b_miss_count = 0 ;
            mtcTimer_reset ( ptr->pt_ptr );

            manage_process_failure ( ptr );

            /* Stage change is handled else where */
            break ;
        }
        case ACTIVE_STAGE__REQUEST_WAIT:
        {
            if ( ptr->pt_ptr->ring == true )
            {
                activeStageChange ( ptr, ACTIVE_STAGE__PULSE_REQUEST );
            }
            break ;
        }
        case ACTIVE_STAGE__PULSE_REQUEST:
        {
            ptr->waiting = true ;
            if ( amon_send_request ( ptr ) != PASS )
            {
                ptr->waiting = false ;
                ptr->send_err_cnt++ ;
                wlog ("%s pulse request send failed (%d:%d)\n",
                          ptr->process,
                          ptr->b2b_miss_count,
                          ptr->send_err_cnt );
                if ( ++ptr->b2b_miss_count >= ptr->threshold )
                {
                    activeStageChange ( ptr, ACTIVE_STAGE__FAILED );
                }
                else
                {
                    activeStageChange ( ptr, ACTIVE_STAGE__GAP_SETUP );
                }
            }
            else
            {
                ptr->pulse_count++ ;
                mtcTimer_start ( ptr->pt_ptr, pmon_timer_handler, ptr->timeout );
                activeStageChange ( ptr, ACTIVE_STAGE__PULSE_RESPONSE );
            }
            break ;
        }
        case ACTIVE_STAGE__PULSE_RESPONSE:
        {
            if ( ptr->rx_sequence != 0 )
            {
                /* handle the first response */
                if ( ptr->active_response == false )
                {
                    ptr->active_response = true ;
                }

                if ( ptr->rx_sequence != ptr->tx_sequence )
                {
                    ptr->b2b_miss_count++ ;
                    ptr->mesg_err_cnt++ ;
                    wlog ( "%s out-of-sequence response (%d:%d)\n",
                               ptr->process ,
                               ptr->tx_sequence,
                               ptr->rx_sequence);

                    if ( ptr->b2b_miss_count >= ptr->threshold )
                    {
                        activeStageChange ( ptr, ACTIVE_STAGE__FAILED );
                    }
                }
                else
                {
                    if ( ptr->b2b_miss_count > ptr->b2b_miss_peak )
                         ptr->b2b_miss_peak = ptr->b2b_miss_count ;

                    ptr->b2b_miss_count = 0 ;
                    if ( ptr->active_debounce == true )
                    {
                        ilog ("%s is healthy (debouncing)\n", ptr->process );
                    }
                    else
                    {
                        mlog2 ("%s is healthy\n", ptr->process );
                    }
                }
                /* manage active monitoring debounce */
                if ( ptr->active_debounce == true )
                {
                     if ( ++ptr->adebounce_cnt >= ((ptr->period+1)) )
                     {
                         ilog ("%s Debounced  (%d)\n", ptr->process, ptr->pid );
                         ptr->active_debounce = false;
                         ptr->adebounce_cnt   = 0    ;
                         ptr->restarts_cnt = 0 ;
                         ptr->quorum_failure       = false;
                         ptr->quorum_unrecoverable = false;

                         clear_amon_counts ( ptr );
                         ptr->active_failed = false ;
                         manage_alarm ( ptr , PMON_CLEAR );
                     }
                }
                ptr->rx_sequence = 0 ;

                /* Wait out the remaining part of the period */
                ptr->waiting = false ;
            }

            if ( ptr->pt_ptr->ring == true )
            {
                /* Are we still waiting for a response ? */
                if ( ptr->waiting == true )
                {
                    /* handle case where response is delayed due to goenabled */
                    if ( ptr->full_init_reqd &&
                         !( is_goenabled ( get_ctrl_ptr()->nodetype, true ) )
                       )
                    {
                        /* we don't expect a resonse... do nothing to wait
                           another loop */
                    }
                    else
                    {
                        ptr->recv_err_cnt++ ;

                        /* don't log the first single pulse miss. */
                        if ( ptr->b2b_miss_count++ > 1 )
                        {
                            wlog ("%s missing pulse response (Miss:%d) (%d:%d)\n",
                                      ptr->process,
                                      ptr->b2b_miss_count,
                                      ptr->tx_sequence,
                                      ptr->rx_sequence);
                        }
                        if ( ptr->b2b_miss_count >= ptr->threshold )
                        {
                            /*****************************************************
                             * Only fail active heartbeating after MTC_MINS_3 of
                             * never having received a response.
                             *
                             * This condition is added to address an issue
                             * reported where the kernel takes a
                             * long time to timeout on external dns namservers
                             * after a DOR when the system is isolated from the
                             * external network.
                             ****************************************************/
                            if (( ptr->active_response == false ) &&
                                ( ptr->period < MTC_MINS_3 ) &&
                                ( ptr->b2b_miss_count < (MTC_MINS_3/ptr->period )))
                            {
                                ; /* more forgiving startup handling */
                            }
                            else
                            {
                                activeStageChange ( ptr, ACTIVE_STAGE__FAILED );
                                break ;
                            }
                        }
                    }
                }
                activeStageChange ( ptr, ACTIVE_STAGE__PULSE_REQUEST );
                break ;
            }
            else if ( ptr->waiting == false )
            {
                ; /* got the data ; just wait out the timer */
            }
            break ;
        }
        default:
        {
            activeStageChange ( ptr, ACTIVE_STAGE__GAP_SETUP );
            break ;
        }
    }
    return (rc);
}

/* Passive Monitoring Handler
 * --------------------------
 * Verifies that the process has an active pid */
int pmon_passive_handler ( process_config_type * ptr )
{
    int rc = RETRY ;

    if ( ptr->stage < PMON_STAGE__STAGES )
    {
        flog ("%s %s Stage %d\n", ptr->process, passiveStages_str[ptr->stage], ptr->stage );
    }
    else
    {
        slog ("%s Invalid stage (%d) ; correcting\n", ptr->process, ptr->stage );
        passiveStageChange ( ptr, PMON_STAGE__FINISH );
    }

    switch ( ptr->stage )
    {
        case PMON_STAGE__START:
        {
            dlog ( "%s failed:%d severity:%s restarts_cnt:%d debounce_cnt:%d\n",
                       ptr->process,
                       ptr->failed,
                       alarmUtil_getSev_str(ptr->alarm_severity).c_str(),
                       ptr->restarts_cnt,
                       ptr->debounce_cnt);

            ptr->stage_cnt = 0 ;

            break ;
        }
        /* Manage Restart Counts */
        case PMON_STAGE__MANAGE:
        {
            if ( ptr->restart == true )
            {
                pmon_ctrl_type * ctrl_ptr = get_ctrl_ptr() ;
                if ( ctrl_ptr->patching_in_progress == true )
                {
                    /* if patching is in progress and we get a process restart command
                     * then that means the rpms have all been installed already so we
                     * can exit patching in progress state */
                     ctrl_ptr->patching_in_progress = false ;
                }

                ; /* fall through and just change state at the bottom */
            }
            /* Handle Critical processes.
             * Critical with 0 restarts
             * Critical with # restarts
             * Req'ts:
             *   1. Avoid re-reporting the event
             *   2. Send minor for first occurance
             *   3. Try restarts if it supports it
             *   4. Stay in this stage once the max restarts has been reached.
             */
            else if ( ptr->sev == SEVERITY_CRITICAL )
            {
                /* handle the No-restarts case    */
                /* Go straight to event assertion */
                if ( ptr->restarts == 0 )
                {
                    manage_alarm ( ptr, PMON_ASSERT );

                    /* Send critical notification */
                    pmon_send_event ( MTC_EVENT_PMON_CRIT, ptr );

                    wlog ("%s auto-restart disabled\n", ptr->process );
                    passiveStageChange ( ptr, PMON_STAGE__IGNORE ) ;

                    /* if process is in quorum, and we're not trying to restart
                     * it, we declare the quorum failed */
                    if ( ptr->quorum )
                    {
                        quorum_process_failure ( ptr );
                    }
                    break ;
                }
                else if ( ptr->restarts_cnt >= ptr->restarts )
                {
                    manage_alarm ( ptr, PMON_ASSERT );

                    /* Send critical notification */
                    pmon_send_event ( MTC_EVENT_PMON_CRIT, ptr );

                    ptr->restarts_cnt = 0 ;
                    ilog ("%s allowing auto-restart of failed critical process\n", ptr->process);

                    /* if process is in quorum, and we haven't been able to
                     * restart it, we declare the quorum failed */
                    if ( ptr->quorum )
                    {
                        quorum_process_failure ( ptr );
                    }

                    /* Note: the above clear or restarts_cnt and commented break below
                     *       forces pmond to try and continue to recover the failing
                     *       critical process if for some reason the host does not
                     *       go through a reboot */
                    /* avoid stage change below and wait for the reboot */
                    // break ;
                }
                else
                {
                    /* Send a restart log to maintenance on the first restart only */
                    if ( ptr->restarts_cnt == 0 )
                    {
                        manage_alarm ( ptr, PMON_LOG );
                    }

                    /* Try and recover if the process is critical but
                     * supports some number of restart attempts first */
                }
            }
            /* Send a log on the first restart                     */
            /*                                                     */
            /* Note: This clause needs to be before the next one   */
            /*       to handle the restarts = 0 case               */
            else if (( ptr->restarts_cnt == 0 ) && ( ptr->restarts != 0 ))
            {
                ilog ("%s Sending Log Event to Maintenance\n", ptr->process );

                /* Send a log on the first one or every time
                 * we start a fresh restart cycle */
                manage_alarm ( ptr, PMON_LOG );
            }
            else if (( ptr->restarts_cnt == 0 ) && ( ptr->restarts == 0 ))
            {
                /* Auto recovery is disable, generate a log and raise a minor alarm */

                wlog ("%s Sending Log Event to Maintenance\n", ptr->process );
                manage_alarm ( ptr, PMON_LOG );

                manage_alarm ( ptr, PMON_ASSERT );

                wlog ("%s Auto-Restart Disabled ... but monitoring for recovery\n", ptr->process );

                /* if process is in quorum, and we're not trying to
                 * restart it, we declare the quorum failed */
                if ( ptr->quorum )
                {
                    ptr->quorum_failure = true;
                    quorum_process_failure ( ptr );
                }

                passiveStageChange ( ptr, PMON_STAGE__IGNORE ) ;
                break ;
            }
            /* Manage notification based on restart
             * threshold for non-critical processes */
            else if ( ptr->restarts_cnt >= ptr->restarts )
            {
                /* Restart threshold reached ; sending event to maintenance */
                manage_alarm ( ptr, PMON_ASSERT );

                /* Start the counts again */
                ptr->restarts_cnt = 0 ;
                ptr->debounce_cnt = 0 ;

                /* if process is in quorum, and we haven't been able to
                 * restart it, we declare the quorum failed */
                if ( ptr->quorum )
                {
                    quorum_process_failure ( ptr );
                }
            }
            passiveStageChange ( ptr, PMON_STAGE__RESPAWN ) ;
            break ;
        }

        /* Spawn the process */
        case PMON_STAGE__RESPAWN:
        {
            dlog ("%s stability period (%d secs)\n", ptr->process, ptr->debounce );

            /* Restart the process */
            respawn_process ( ptr ) ;

            /* Start the monitor debounce timer. */
            mtcTimer_reset ( ptr->pt_ptr );

            mtcTimer_start ( ptr->pt_ptr, pmon_timer_handler, ptr->startuptime );
            passiveStageChange ( ptr, PMON_STAGE__MONITOR_WAIT ) ;

            break ;
        }

        /* Give the process time to startup
         * before trying to monitor it */
        case PMON_STAGE__MONITOR_WAIT:
        {
            /* Give the process time to start */
            if ( ptr->pt_ptr->ring == true )
            {
                if (( !ptr->sigchld_rxed ) || ( !ptr->child_pid ) || ( ptr->status ))
                {
                    if ( ptr->child_pid == 0 )
                    {
                        elog ("%s spawn has null child pid\n", ptr->process );
                    }
                    else if ( ptr->sigchld_rxed == false )
                    {
                        elog ("%s spawn timeout (%d)\n", ptr->process, ptr->child_pid );
                    }
                    else if ( ptr->status != PASS )
                    {
                        elog ("%s spawn failed (rc:%d) (%d)\n", ptr->process, ptr->status, ptr->child_pid );
                    }
                    kill_running_child ( ptr ) ;

                    /* we had a startup timeout ; do restart */
                    mtcTimer_start( ptr->pt_ptr, pmon_timer_handler, ptr->interval );
                    passiveStageChange ( ptr, PMON_STAGE__RESTART_WAIT ) ;
                }
                else
                {
                    /* clear the monitor debounce counter */
                    ptr->debounce_cnt = 0 ;

                    if ( ptr->restart == true )
                    {
                        ptr->restart = false ;
                        ptr->registered = false ;
                        ptr->pid = get_process_pid ( ptr );
                        ilog ("%s Restarted  (%d)\n", ptr->process, ptr->pid );
                        passiveStageChange ( ptr, PMON_STAGE__MANAGE ) ;
                    }
                    else
                    {
                        /* Start debounce monitor phase */
                        passiveStageChange ( ptr, PMON_STAGE__MONITOR ) ;
                        process_running ( ptr );
                        ilog ("%s Monitor    (%d)\n", ptr->process, ptr->pid );
                    }
                }

                ptr->sigchld_rxed = false ;
            }
            break ;
        }
         /* Monitor the newly respawned process */
        case PMON_STAGE__MONITOR:
        {
            /* The process needs to stay running for x seconds before
             * clearing any assertion or declaring that this restart
             * attempt was successful */

            /* The process should be running.
             * If not then cancel the timer and start over through
             * the RESTART_WAIT stage which ensures that we manage
             * back to back restarts properly */
            if ( ! process_running ( ptr ) )
            {
                wlog ("%s Respawn Monitor Failed (%d of %d), retrying in (%d secs)\n",
                          ptr->process,
                          ptr->restarts_cnt,
                          ptr->restarts,
                          ptr->interval);

                passiveStageChange ( ptr, PMON_STAGE__TIMER_WAIT ) ;
            }
            else if ( ptr->pt_ptr->ring == true )
            {
                if ( ++ptr->debounce_cnt >= ptr->debounce )
                {
                    /* We made it through the monitor debounce
                     * period so lets finish up */
                    ilog ("%s Stable     (%d)\n", ptr->process, ptr->pid );
                    passiveStageChange ( ptr, PMON_STAGE__FINISH ) ;
                }
                /* else continue to monitor the freshly respawned process */
                else
                {
                    /* Start the monitor timer again since
                     * the debounce period is not over */
                    mtcTimer_start  ( ptr->pt_ptr, pmon_timer_handler, 1 );
                    dlog ("%s Debounce Monitor (TID:%p)\n", ptr->process, ptr->pt_ptr->tid );
                }
            }
            break ;
        }

        case PMON_STAGE__TIMER_WAIT:
        {
            if ( mtcTimer_expired ( ptr->pt_ptr ) )
            {
                /* if restart interval is zero then just ring the timer right away */
                if ( ptr->interval == 0 )
                {
                   ptr->pt_ptr->ring = true ;
                }
                else
                {
                    /* Now we are in the restart wait phase */
                    mtcTimer_start( ptr->pt_ptr, pmon_timer_handler, ptr->interval );
                }

                kill_running_child ( ptr ) ;
                passiveStageChange ( ptr, PMON_STAGE__RESTART_WAIT ) ;
            }
            else
            {
                dlog ("%s debounce timer wait\n", ptr->process);
            }
            break ;
        }
        /* Lets wait a bit before we try another restart */
        case PMON_STAGE__RESTART_WAIT:
        {
            if ( ptr->pt_ptr->ring == true )
            {
                /* Force the immediate (re)start */
                passiveStageChange ( ptr, PMON_STAGE__MANAGE) ;
            }
            break ;
        }
        /* A state that leaves a process failed but takes it out of
         * that failed state if it auto recovers on its own or
         * through external means */
        case PMON_STAGE__IGNORE:
        {
            int pid ;
            if ((pid = get_process_pid ( ptr )))
            {
                int result = kill (pid, 0 );
                if ( result == 0 )
                {
                    /* allow process recovery if it is started outside pmond */
                    if ( ptr->stopped == true )
                        ptr->stopped = false ;

                    passiveStageChange ( ptr, PMON_STAGE__FINISH );
                }
            }
            break ;
        }
        case PMON_STAGE__FINISH:
        {
            kill_running_child ( ptr ) ;
            ilog ("%s Recovered  (%d)\n", ptr->process, ptr->pid );

            /* Set all counts to default state ;
             * Even if they may have already been :) */
            ptr->failed = false ;
            ptr->debounce_cnt = 0 ;

            passiveStageChange ( ptr, PMON_STAGE__START ) ;

            /* Register the new process with the kernel */
            register_process ( ptr );

            if ( !ptr->active_monitoring )
            {
                ptr->restarts_cnt = 0 ;
                /* It's possible that a restart succeeded even after the
                 * max restarts threshold was reached (and we thought things
                 * were dead, so we marked quorum processes as unrecoverable)
                 */
                if ( ptr->quorum )
                {
                    ptr->quorum_failure       = false;
                    ptr->quorum_unrecoverable = false;
                }

                manage_alarm ( ptr, PMON_CLEAR );
            }
            /* Recover Active monitoring ,
             * event clear will occur in the active monitoring
             * FSM after it passes the debouce cycle */
            else
            {
                /* Open the process's active monitoring
                 * socket if it was or is closed */
                if ( ptr->msg.tx_sock == 0 )
                    open_process_socket ( ptr );

                /* Clear sequence*/
                ptr->tx_sequence       = 0     ;
                ptr->rx_sequence       = 0     ;

                /* Clear active monitoring state controls */
                // ptr->active_failed  = false ;
                ptr->waiting        = false ;

                /* Set the active monitor debounce flag and clear its counter */
                ptr->active_debounce = true ;
                ptr->adebounce_cnt   = 0    ;

                activeStageChange ( ptr, ACTIVE_STAGE__PULSE_REQUEST ) ;
            }
            rc = PASS ;
            break ;
        }
        /******************************************************************************
         *
         *  This polling stage was introduced for the introduction of the 2-Server
         *  configuration, aka combo blade/host.
         *
         *  /etc/pmon.d/<process.conf> files that declare
         *
         *  subfunction = worker
         *  or
         *  subfunction = storage
         *
         *  .. are not immediately monitored by pmond on startup.
         *
         *  Instead, pmond will wait for the specified subfunction config complete
         *  file to be present before starting to monitor that process.
         *
         *  This stage is here to manage that delayed monitoring startup of
         *  subfunction dependent processes.
         *
         *******************************************************************************/
        case PMON_STAGE__POLLING:
        {
            if ( ptr->pt_ptr->ring == false )
            {
                break ;
            }
            else
            {
                string config_filename = "" ;
                pmon_ctrl_type * ctrl_ptr = get_ctrl_ptr() ;
                if ( ptr->subfunction )
                {
                    if ( !strcmp (ptr->subfunction, "worker" ) )
                    {
                        config_filename = CONFIG_COMPLETE_WORKER ;
                    }
                    else if ( !strcmp (ptr->subfunction, "storage" ) )
                    {
                        config_filename = CONFIG_COMPLETE_STORAGE ;
                    }
                    /********************************************************
                     * issue: processes that set the subfunction to
                     *            'last-config' get a dependency override in
                     *            the AIO system. Such processes need to be
                     *            monitored only after the last configuration
                     *            step. Right now that is worker in aio.
                     *
                     ********************************************************/
                    else if (( ctrl_ptr->system_type != SYSTEM_TYPE__NORMAL ) &&
                             ( !strcmp (ptr->subfunction, "last-config" )))
                    {
                        config_filename = CONFIG_COMPLETE_WORKER ;
                        dlog ("%s dependency over-ride ; will wait for %s\n",
                                  ptr->process,
                                  config_filename.c_str());
                    }
                }

                if ( config_filename.empty() )
                {
                    passiveStageChange ( ptr, PMON_STAGE__IGNORE );
                    elog ("%s is subfunction polling with no subfunction ; ignoring\n", ptr->process );
                }
                else
                {
                    bool start_monitoring = true;
                    string waiting_for = "";

                    if ( daemon_is_file_present ( config_filename.data() ) != true )
                    {
                        start_monitoring = false;
                        waiting_for = config_filename;
                    }

                    mtcTimer_reset ( ptr->pt_ptr );
                    if ( start_monitoring == true )
                    {
                        ptr->passive_monitoring = true ;

                        /* check for startup failures from alarm query. */
                        if ( ptr->failed == true )
                        {
                            /* manage the process if its in the failed state */
                            passiveStageChange ( ptr, PMON_STAGE__MANAGE );
                        }
                        else
                        {
                            ilog ("monitor start of %s in %d seconds\n", ptr->process, daemon_get_cfg_ptr()->start_delay );
                            mtcTimer_start ( ptr->pt_ptr, pmon_timer_handler, daemon_get_cfg_ptr()->start_delay  );
                            passiveStageChange ( ptr, PMON_STAGE__START_WAIT );
                        }
                    }
                    else
                    {
                        mtcTimer_start ( ptr->pt_ptr, pmon_timer_handler, 3 );
                        wlog_throttled ( ptr->stage_cnt, 500, "%s monitoring is waiting on %s\n",
                                         ptr->process, waiting_for.c_str());
                    }
                }
            }
            break ;
        }
        case PMON_STAGE__START_WAIT:
        {
            if ( ptr->pt_ptr->ring == true )
            {
                ilog ("%s process monitoring started\n", ptr->process );
                register_process ( ptr );
                if ( ptr->active_monitoring == true )
                {
                    if ( open_process_socket ( ptr ) != PASS )
                    {
                        elog ("%s failed to open process socket\n",
                                 ptr->process );
                    }
                }
                passiveStageChange ( ptr, PMON_STAGE__MANAGE );
            }
            break ;
        }

        default:
        {
            slog ("%s Invalid stage (%d)\n", ptr->process, ptr->stage );

            /* Default to finish for invalid case.
             * If there is an issue then it will be detected */
            passiveStageChange ( ptr, PMON_STAGE__FINISH );
        }
    }
    return (rc);
}

/* Status Monitoring Handler
 * --------------------------
 * Monitors a process with status command */
int pmon_status_handler ( process_config_type * ptr )
{
    if ( ptr->status_stage >= STATUS_STAGE__STAGES )
    {
        wlog ("%s Invalid status_stage (%d) ; correcting\n", ptr->process, ptr->status_stage );
        statusStageChange ( ptr, STATUS_STAGE__BEGIN);
    }

    switch ( ptr->status_stage )
    {
        // First state
        case STATUS_STAGE__BEGIN:
        {
            mtcTimer_start  ( ptr->pt_ptr, pmon_timer_handler, ptr->period );
            dlog ("%s start period timer  %p\n", ptr->process,  ptr->pt_ptr->tid );
            statusStageChange ( ptr, STATUS_STAGE__EXECUTE_STATUS );
            break ;
        }

        // Execute the status command
        case STATUS_STAGE__EXECUTE_STATUS:
        {
            if ( ptr->pt_ptr->ring == true ) //wake up from period
            {
                ptr->status = PASS;
                mtcTimer_start  ( ptr->pt_ptr, pmon_timer_handler, ptr->timeout );
                dlog ("%s start the status command timer %p\n", ptr->process,  ptr->pt_ptr->tid );

                // Execute the status call
                int rc = execute_status_command(ptr);
                if (rc != PASS)
                {
                   elog ("%s execute_status_command returned a failure (%d)\n", ptr->process,  rc);
                   ptr->status = rc;
                }

                statusStageChange ( ptr, STATUS_STAGE__EXECUTE_STATUS_WAIT );
            }
            break ;
        }

        // Wait for the status command to finish and process results
        case STATUS_STAGE__EXECUTE_STATUS_WAIT:
        {
            // Give the command time to execute. The daemon_sigchld_hdlr will force
            // a ring when the command execute successfully or returns a failure
            if ( (ptr->pt_ptr->ring == true) || (ptr->status != PASS ) )
            {
                mtcTimer_reset( ptr->pt_ptr);
                ptr->pt_ptr->ring = false;

                if (( !ptr->sigchld_rxed ) || ( !ptr->child_pid ) || (ptr->status != PASS))
                {
                    if ( ptr->child_pid == 0 )
                    {
                        elog ("%s status command has null child pid\n", ptr->process );
                    }
                    else if ( ptr->sigchld_rxed == false )
                    {
                        elog ("%s status command execution timed out (%d)\n", ptr->process, ptr->child_pid );
                        kill_running_process ( ptr->child_pid );
                    }

                    elog ("%s status returned a failure (rc:%d) ; process(es) start pending\n", ptr->process, ptr->status );

                    // Go to execute start state since we do not know the status of the process
                    ptr->status_failed = true;
                    ptr->was_failed = true ;
                    statusStageChange ( ptr, STATUS_STAGE__EXECUTE_START );
                }
                else
                {
                    // Status reports everything is ok, reset variables
                    dlog ("%s status command was successful\n", ptr->process);
                    ptr->restarts_cnt = 0;

                    if ( ptr->failed == true )
                    {
                        manage_alarm ( ptr, PMON_CLEAR );
                    }
                    ptr->status_failed = false;
                    ptr->failed = false;
                    statusStageChange ( ptr, STATUS_STAGE__BEGIN );

                }

                ptr->child_pid = 0;
                ptr->sigchld_rxed = false;
            }
            break;
        }

        // Interval wait time before doing a start again if the start
        // had previously failed
        case STATUS_STAGE__INTERVAL_WAIT:
        {
            if (ptr->pt_ptr->ring == true)
            {
               statusStageChange ( ptr, STATUS_STAGE__EXECUTE_START );
            }
            break ;
        }

        // Execute the start command
        case STATUS_STAGE__EXECUTE_START:
        {
            ptr->status = PASS;
            mtcTimer_start  ( ptr->pt_ptr, pmon_timer_handler, ptr->timeout );
            dlog ("%s start the start command timer %p\n", ptr->process,  ptr->pt_ptr->tid );

            int rc = execute_start_command (ptr);
            if (rc != PASS)
            {
               elog ("%s execute_start_command returned a failure (%d)\n", ptr->process, rc);
               ptr->status = rc;
            }
            statusStageChange ( ptr, STATUS_STAGE__EXECUTE_START_WAIT );
            break;
        }

        // Wait for the start command to finish and process results
        case STATUS_STAGE__EXECUTE_START_WAIT:
        {
            // Give the command time to execute. The daemon_sigchld_hdlr will force
            // a ring when the command execute successfully or returns a failure
            if ( (ptr->pt_ptr->ring == true) || (ptr->status != PASS) )
            {
                mtcTimer_reset( ptr->pt_ptr);
                ptr->pt_ptr->ring = false;

                // If the status had failed then ptr->status_failed will be set to true. Status failure
                // will also cause restarts count increment, alarm and degrade state
                if (( !ptr->sigchld_rxed ) || ( !ptr->child_pid ) || ( ptr->status ) || (ptr->status_failed))
                {
                    if ( ptr->child_pid == 0 )
                    {
                        elog ("%s start command has null child pid\n", ptr->process );
                    }
                    else if ( ptr->sigchld_rxed == false )
                    {
                        elog ("%s start command execution timed out (%d)\n", ptr->process, ptr->child_pid );
                        kill_running_process ( ptr->child_pid );
                    }
                    else if ( ptr->status != PASS )
                    {
                        elog ("%s start command returned a failure (rc:%d)\n", ptr->process, ptr->status);
                    }

                    /* Send a log on the first failure                     */
                    if (( ptr->restarts_cnt == 0 ) && ( ptr->restarts != 0 ) )
                    {
                        wlog ("%s Sending Log Event to Maintenance\n", ptr->process );

                        /* Send a log on the first one or every time we start a fresh restart cycle */
                        manage_alarm ( ptr, PMON_LOG );
                    }

                    /* Manage notification based on restart */
                    else if ( ptr->restarts_cnt >= ptr->restarts )
                    {
                        wlog ("%s Failure threshold (%d) reached ; alarming\n", ptr->process, ptr->restarts );
                        manage_alarm ( ptr, PMON_ASSERT );
                        ptr->failed = true;   // this is used to degrade un-degrade the host
                        ptr->restarts_cnt = 0 ;
                    }
                    else
                    {
                        wlog ("%s has %d of %d failures ; retrying ...\n",
                                  ptr->process,
                                  ptr->restarts_cnt,
                                  ptr->restarts );
                    }

                    ptr->restarts_cnt++;
                    ptr->failed_cnt++ ;

                    //only want to check for status false on first restart iteration so reset the flag
                    ptr->status_failed = false;

                    // Go to interval state only if start failed otherwise we want to check
                    // the process status again
                    if (( !ptr->sigchld_rxed ) || ( !ptr->child_pid ) || ( ptr->status ))
                    {
                        // In here because the start failed

                        // Wait the interval time and then execute a start command again
                        mtcTimer_start  ( ptr->pt_ptr, pmon_timer_handler, ptr->interval );
                        dlog ("%s start interval timer %p\n", ptr->process,  ptr->pt_ptr->tid );
                        statusStageChange ( ptr, STATUS_STAGE__INTERVAL_WAIT );
                    }
                    else
                    {
                        // In here because status failed but start was successful

                        wlog ("%s start command was successful ; here because status had failed\n", ptr->process);
                        statusStageChange ( ptr, STATUS_STAGE__BEGIN );
                    }
                }
                else
                {
                    // Start was successful
                    wlog ("%s start command was successful\n", ptr->process);
                    statusStageChange ( ptr, STATUS_STAGE__BEGIN );
                }

                ptr->child_pid = 0;
                ptr->sigchld_rxed = false;
            }
            break;
        }

        default:
        {
            elog ("%s invalid status_stage (%d)\n", ptr->process, ptr->status_stage );

            /* Default to first state for invalid case. there is an issue then it will be detected */
            statusStageChange ( ptr, STATUS_STAGE__BEGIN );
        }
    }
    return (PASS);
}