ha/service-mgmt/sm/src/sm_service_audit.c

//
// Copyright (c) 2014 Wind River Systems, Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
#include "sm_service_audit.h"

#include <stdio.h>
#include <stdint.h>
#include <stdbool.h>
#include <stdlib.h>

#include "sm_types.h"
#include "sm_debug.h"
#include "sm_process_death.h"
#include "sm_timer.h"
#include "sm_service_fsm.h"
#include "sm_service_action.h"
#include "sm_service_go_active.h"

// ****************************************************************************
// Service Audit - Result Handler
// ==============================
static SmErrorT service_audit_result_handler( SmServiceT* service,
    SmServiceActionT action_running, SmServiceActionResultT action_result,
    SmServiceStateT service_state, SmServiceStatusT service_status,
    SmServiceConditionT service_condition, const char reason_text[] )
{
    bool retries_exhausted = true;
    int max_failure_retries;
    int max_timeout_retries;
    int max_total_retries;
    SmErrorT error;

    if( SM_SERVICE_ACTION_RESULT_SUCCESS == action_result )
    {
        DPRINTFD( "Action (%s) of service (%s) completed with success.",
                  sm_service_action_str( action_running ), service->name );
        goto REPORT;
    }

    error = sm_service_action_max_retries( service->name, action_running,
                                           &max_failure_retries,
                                           &max_timeout_retries,
                                           &max_total_retries );
    if( SM_OKAY != error )
    {
        DPRINTFE( "Failed to get max retries for action (%s) of service (%s), "
                  "error=%s.", sm_service_action_str( action_running ),
                  service->name, sm_error_str( error ) );
        goto REPORT;
    }

    if( SM_SERVICE_ACTION_RESULT_FAILED == action_result )
    {
        retries_exhausted =
            (( max_failure_retries <= service->action_attempts )||
             ( max_total_retries   <= service->action_attempts ));

    } else if( SM_SERVICE_ACTION_RESULT_TIMEOUT == action_result ) {
        retries_exhausted =
            (( max_timeout_retries <= service->action_attempts )||
             ( max_total_retries   <= service->action_attempts ));
    }

    if( retries_exhausted )
    {
        DPRINTFI( "Max retires met for action (%s) of service (%s), "
                  "attempts=%i.", sm_service_action_str( action_running ),
                  service->name, service->action_attempts );
        goto REPORT;
    } else {
        DPRINTFI( "Max retires not met for action (%s) of service (%s), "
                  "attempts=%i.", sm_service_action_str( action_running ),
                  service->name, service->action_attempts );

        if( SM_SERVICE_ACTION_AUDIT_ENABLED == action_running )
        {
            error = sm_service_audit_enabled( service );
            if( SM_OKAY != error )
            {
                DPRINTFE( "Failed to audit-enabled service (%s), error=%s.",
                          service->name, sm_error_str( error ) );
                goto REPORT;
            }
        } else {
            error = sm_service_audit_disabled( service );
            if( SM_OKAY != error )
            {
                DPRINTFE( "Failed to audit-disabled service (%s), error=%s.",
                          service->name, sm_error_str( error ) );
                goto REPORT;
            }
        }
    }

    return( SM_OKAY );

REPORT:
    service->action_attempts = 0;

    error = sm_service_fsm_action_complete_handler( service->name,
                        action_running, action_result, service_state,
                        service_status, service_condition, reason_text );
    if( SM_OKAY != error )
    {
        DPRINTFE( "Failed to notify service (%s) fsm action complete, "
                  "error=%s.", service->name, sm_error_str( error ) );
        abort();
    }

    return( SM_OKAY );
}
// ****************************************************************************

// ****************************************************************************
// Service Audit - Timeout
// =======================
static bool sm_service_audit_timeout( SmTimerIdT timer_id, int64_t user_data )
{
    int64_t id = user_data;
    SmServiceT* service;    
    SmServiceActionT action_running;
    SmServiceActionResultT action_result;
    SmServiceStateT service_state;
    SmServiceStatusT service_status;
    SmServiceConditionT service_condition;
    char reason_text[SM_SERVICE_ACTION_RESULT_REASON_TEXT_MAX_CHAR] = "";
    SmErrorT error;

    service = sm_service_table_read_by_id( id );
    if( NULL == service )
    {
        DPRINTFE( "Failed to read service, error=%s.",
                  sm_error_str(SM_NOT_FOUND) );
        return( false );
    }

    // timer to be disarmed after exit
    service->action_timer_id = SM_TIMER_ID_INVALID;

    if(( SM_SERVICE_ACTION_AUDIT_ENABLED  != service->action_running )&&
       ( SM_SERVICE_ACTION_AUDIT_DISABLED != service->action_running ))
    {
        DPRINTFE( "Service (%s) not running audit-enabled or audit-disabled "
                  "action.", service->name );
        return( false );
    }

    action_running = service->action_running;

    error = sm_service_action_result( SM_SERVICE_ACTION_PLUGIN_TIMEOUT,
                                      service->name, service->action_running,
                                      &action_result, &service_state,
                                      &service_status, &service_condition,
                                      reason_text );
    if( SM_OKAY != error )
    {
        DPRINTFE( "Failed to get action result for service (%s) defaulting "
                  "to failed, exit_code=%i, error=%s.", service->name,
                  SM_SERVICE_ACTION_PLUGIN_TIMEOUT, sm_error_str( error ) );
        action_result = SM_SERVICE_ACTION_RESULT_FAILED;
        service_state = SM_SERVICE_STATE_UNKNOWN;
        service_status = SM_SERVICE_STATUS_UNKNOWN;
        service_condition = SM_SERVICE_CONDITION_UNKNOWN;
    }

    DPRINTFI( "Action (%s) timeout with result (%s), state (%s), "
              "status (%s), and condition (%s) for service (%s), "
              "reason_text=%s, exit_code=%i.", 
              sm_service_action_str( action_running ),
              sm_service_action_result_str( action_result ),
              sm_service_state_str( service_state ),
              sm_service_status_str( service_status ),
              sm_service_condition_str( service_condition ),
              service->name, reason_text, SM_SERVICE_ACTION_PLUGIN_TIMEOUT );

    error = sm_service_action_abort( service->name, service->action_pid );
    if( SM_OKAY != error )
    {
        DPRINTFE( "Failed to abort action (%s) of service (%s), error=%s.",
                  sm_service_action_str( service->action_running ),
                  service->name, sm_error_str( error ) );
    }
   
    service->action_running = SM_SERVICE_ACTION_NONE;
    error = service_audit_result_handler( service, action_running,
                                          action_result, service_state,
                                          service_status, service_condition,
                                          reason_text );
    if( SM_OKAY != error )
    {
        DPRINTFE( "Failed to handle go-active result for service (%s), "
                  "error=%s.", service->name, sm_error_str( error ) );
        abort();
    }

    return( false );
}
// ****************************************************************************

// ****************************************************************************
// Service Audit - Complete
// ========================
static void sm_service_audit_complete( pid_t pid, int exit_code, 
    int64_t user_data )
{
    bool action_exists;
    SmServiceT* service;    
    SmServiceActionT action_running;
    SmServiceActionResultT action_result;
    SmServiceStateT service_state;
    SmServiceStatusT service_status;
    SmServiceConditionT service_condition;
    char reason_text[SM_SERVICE_ACTION_RESULT_REASON_TEXT_MAX_CHAR] = "";
    SmErrorT error;

    if( SM_PROCESS_FAILED == exit_code )
    {
        exit_code = SM_SERVICE_ACTION_PLUGIN_FAILURE;
    }

    service = sm_service_table_read_by_action_pid( (int) pid );
    if( NULL == service )
    {
        DPRINTFE( "Failed to query service based on pid (%i), error=%s.",
                  (int) pid, sm_error_str(SM_NOT_FOUND) );
        return;
    }
    
    if(( SM_SERVICE_ACTION_AUDIT_ENABLED  != service->action_running )&&
       ( SM_SERVICE_ACTION_AUDIT_DISABLED != service->action_running ))
    {
        DPRINTFE( "Service (%s) not running action (%s or %s).", service->name,
                  sm_service_action_str( SM_SERVICE_ACTION_AUDIT_ENABLED ),
                  sm_service_action_str( SM_SERVICE_ACTION_AUDIT_DISABLED ) );
        return;
    }

    if( SM_TIMER_ID_INVALID != service->action_timer_id )
    {
        error = sm_timer_deregister( service->action_timer_id );
        if( SM_OKAY == error )
        {
            DPRINTFD( "Timer stopped for action (%s) of service (%s).",
                      sm_service_action_str( service->action_running ),
                      service->name );
        } else {
            DPRINTFE( "Failed to stop timer for action (%s) of "
                      "service (%s), error=%s.",
                      sm_service_action_str( service->action_running ),
                      service->name, sm_error_str( error ) );
        } 
    }

    action_running = service->action_running;

    if( SM_SERVICE_ACTION_PLUGIN_FORCE_SUCCESS == exit_code )
    {
        DPRINTFI( "Action (%s) for service (%s) force-passed.",
                  sm_service_action_str( action_running ), service->name );

        action_result = SM_SERVICE_ACTION_RESULT_SUCCESS;

        if(( SM_SERVICE_STATE_INITIAL == service->state ) ||
           ( SM_SERVICE_STATE_UNKNOWN == service->state ))
        {
            service_state = SM_SERVICE_STATE_DISABLED;

        } else if( SM_SERVICE_STATE_ENABLED_GO_STANDBY == service->state ) {
            service_state = SM_SERVICE_STATE_ENABLED_ACTIVE;

        } else {
            service_state = service->state;
        }

        service_status = SM_SERVICE_STATUS_NONE;
        service_condition = SM_SERVICE_CONDITION_NONE;

    } else {
        error = sm_service_action_result( exit_code, service->name, 
                                          service->action_running,
                                          &action_result, &service_state,
                                          &service_status, &service_condition,
                                          reason_text );
        if( SM_OKAY != error )
        {
            DPRINTFE( "Failed to get action result for service (%s) "
                      "defaulting to failed, exit_code=%i, error=%s.",
                      service->name, exit_code, sm_error_str( error ) );
            action_result = SM_SERVICE_ACTION_RESULT_FAILED;
            service_state = SM_SERVICE_STATE_UNKNOWN;
            service_status = SM_SERVICE_STATUS_UNKNOWN;
            service_condition = SM_SERVICE_CONDITION_UNKNOWN;
        }
    }

    DPRINTFD( "Action (%s) completed with result (%s), state (%s), "
              "status (%s), and condition (%s) for service (%s), "
              "reason_text=%s, exit_code=%i.", 
              sm_service_action_str( action_running ),
              sm_service_action_result_str( action_result ),
              sm_service_state_str( service_state ),
              sm_service_status_str( service_status ),
              sm_service_condition_str( service_condition ),
              service->name, reason_text, exit_code );

    service->action_running = SM_SERVICE_ACTION_NONE;
    service->action_pid = -1;
    service->action_timer_id = SM_TIMER_ID_INVALID;

    if(( SM_SERVICE_STATE_UNKNOWN == service_state )&&
       ( SM_SERVICE_ACTION_RESULT_SUCCESS == action_result ))
    {
        error = sm_service_go_active_exists( service, &action_exists );
        if( SM_OKAY == error )
        {
            if( action_exists )
            {
                service_state = SM_SERVICE_STATE_ENABLED_STANDBY;
            } else {
                service_state = SM_SERVICE_STATE_ENABLED_ACTIVE;
            }

            DPRINTFD( "Action (%s) state (%s) result modified for service "
                      "(%s).", sm_service_action_str( action_running ),
                      sm_service_state_str( service_state ), service->name );
        } else {
            DPRINTFE( "Failed to determine if go-active action for "
                      "service (%s) exists, error=%s.", service->name,
                      sm_error_str( error ) );
            abort();
        }
    }

    error = service_audit_result_handler( service, action_running,
                                          action_result, service_state,
                                          service_status, service_condition,
                                          reason_text );
    if( SM_OKAY != error )
    {
        DPRINTFE( "Failed to handle audit result for service (%s), "
                  "error=%s.", service->name, sm_error_str( error ) );
        abort();
    }
}
// ****************************************************************************

// ****************************************************************************
// Service Audit
// =============
static SmErrorT sm_service_audit( SmServiceT* service, SmServiceActionT action )
{
    char timer_name[80] = "";
    int process_id = -1;
    int timeout = 0;
    SmTimerIdT timer_id = SM_TIMER_ID_INVALID;
    SmErrorT error;

    if( action == service->action_running )
    {
        DPRINTFI( "Service (%s) already running %s action.", service->name,
                  sm_service_action_str( service->action_running ) );
        return( SM_OKAY );
    }

    // Run action.
    error = sm_service_action_run( service->name, 
                                   service->instance_name,
                                   service->instance_params,
                                   action, &process_id, &timeout );
    if( SM_OKAY != error )
    {
        DPRINTFE( "Failed to run %s action for service (%s), "
                  "error=%s.", sm_service_action_str( action ),
                  service->name, sm_error_str( error ) );
        return( error );
    }
   
    // Register for action script exit notification.
    error = sm_process_death_register( process_id, true,
                                       sm_service_audit_complete, 0 );
    if( SM_OKAY != error )
    {
        DPRINTFE( "Failed to register for %s action completion for "
                  "service (%s), error=%s.", sm_service_action_str( action ),
                  service->name, sm_error_str( error ) );
        abort();
    }

    // Create timer for action completion.
    snprintf( timer_name, sizeof(timer_name), "%s %s action",
              service->name, sm_service_action_str( action ) );

    error = sm_timer_register( timer_name, timeout, sm_service_audit_timeout,
                               service->id, &timer_id );
    if( SM_OKAY != error )
    {
        DPRINTFE( "Failed to create a timer for action (%s) for service (%s), "
                  "error=%s.", sm_service_action_str( action ),
                  service->name, sm_error_str( error ) );
        abort();
    }

    // Write to database that we are running an action.
    service->action_running = action;
    service->action_pid = process_id;
    service->action_timer_id = timer_id;
    service->action_attempts += 1;

    DPRINTFD( "Started %s action (%i) for service (%s).",
              sm_service_action_str( action ), process_id, service->name );

    return( SM_OKAY );
}
// ****************************************************************************

// ****************************************************************************
// Service Audit - Abort
// =====================
static SmErrorT sm_service_audit_abort( SmServiceT* service )
{
    SmErrorT error;

    DPRINTFI( "Aborting action (%s) of service (%s).",
              sm_service_action_str( service->action_running ),
              service->name );

    error = sm_service_action_abort( service->name, service->action_pid );
    if( SM_OKAY != error )
    {
        DPRINTFE( "Failed to abort action (%s) of service (%s), error=%s.",
                  sm_service_action_str( service->action_running ),
                  service->name, sm_error_str( error ) );
        return( error );
    }

    service->action_running = SM_SERVICE_ACTION_NONE;
    service->action_pid = -1;

    if( SM_TIMER_ID_INVALID != service->action_timer_id )
    {
        error = sm_timer_deregister( service->action_timer_id );
        if( SM_OKAY != error )
        {
            DPRINTFE( "Cancel timer for action (%s) of service (%s) failed, "
                      "error=%s.", service->name,
                      sm_service_action_str( service->action_running ),
                      sm_error_str( error ) );
            return( error );
        }
    }

    service->action_timer_id = SM_TIMER_ID_INVALID;
    service->action_attempts = 0;

    return( SM_OKAY );
}
// ****************************************************************************

// ****************************************************************************
// Service Audit - Enabled
// =======================
SmErrorT sm_service_audit_enabled( SmServiceT* service )
{
    return( sm_service_audit( service, SM_SERVICE_ACTION_AUDIT_ENABLED ) );
}
// ****************************************************************************

// ****************************************************************************
// Service Audit - Disabled
// ========================
SmErrorT sm_service_audit_disabled( SmServiceT* service )
{
    return( sm_service_audit( service, SM_SERVICE_ACTION_AUDIT_DISABLED ) );
}
// ****************************************************************************

// ****************************************************************************
// Service Audit - Enabled Abort
// =============================
SmErrorT sm_service_audit_enabled_abort( SmServiceT* service )
{
    return( sm_service_audit_abort( service ) );
}
// ****************************************************************************

// ****************************************************************************
// Service Audit - Disabled Abort
// ==============================
SmErrorT sm_service_audit_disabled_abort( SmServiceT* service )
{
    return( sm_service_audit_abort( service ) );
}
// ****************************************************************************

// ****************************************************************************
// Service Audit - Enabled Interval
// ================================
SmErrorT sm_service_audit_enabled_interval( SmServiceT* service, int* interval )
{
    return( sm_service_action_interval( service->name, 
                                        SM_SERVICE_ACTION_AUDIT_ENABLED,
                                        interval ) );
}
// ****************************************************************************

// ****************************************************************************
// Service Audit - Disabled Interval
// ================================
SmErrorT sm_service_audit_disabled_interval( SmServiceT* service, int* interval )
{
    return( sm_service_action_interval( service->name,
                                        SM_SERVICE_ACTION_AUDIT_DISABLED,
                                        interval ) );
}
// ****************************************************************************

// ****************************************************************************
// Service Audit - Enabled Exists
// ==============================
SmErrorT sm_service_audit_enabled_exists( SmServiceT* service, bool* exists )
{
    SmErrorT error;

    error = sm_service_action_exist( service->name,
                                     SM_SERVICE_ACTION_AUDIT_ENABLED );
    if(( SM_OKAY != error )&&( SM_NOT_FOUND != error ))
    {
        DPRINTFE( "Failed to determine if action (%s) exists for "
                  "service (%s), error=%s.", 
                  sm_service_action_str( SM_SERVICE_ACTION_AUDIT_ENABLED ),
                  service->name, sm_error_str( error ) );
        return( error );
    }

    if( SM_OKAY == error )
    {
        *exists = true;
    } else {
        *exists = false;
    }

    return( SM_OKAY );
}
// ****************************************************************************

// ****************************************************************************
// Service Audit - Disabled Exists
// ===============================
SmErrorT sm_service_audit_disabled_exists( SmServiceT* service, bool* exists )
{
    SmErrorT error;

    error = sm_service_action_exist( service->name,
                                     SM_SERVICE_ACTION_AUDIT_DISABLED );
    if(( SM_OKAY != error )&&( SM_NOT_FOUND != error ))
    {
        DPRINTFE( "Failed to determine if action (%s) exists for "
                  "service (%s), error=%s.",
                  sm_service_action_str( SM_SERVICE_ACTION_AUDIT_DISABLED ),
                  service->name, sm_error_str( error ) );
        return( error );
    }

    if( SM_OKAY == error )
    {
        *exists = true;
    } else {
        *exists = false;
    }

    return( SM_OKAY );
}
// ****************************************************************************

// ****************************************************************************
// Service Audit - Initialized
// ===========================
SmErrorT sm_service_audit_initialize( void )
{
    return( SM_OKAY );
}
// ****************************************************************************

// ****************************************************************************
// Service Audit - Finalize
// ========================
SmErrorT sm_service_audit_finalize( void )
{
    return( SM_OKAY );
}
// ****************************************************************************