starlingx
/
ha
Code Issues Proposed changes

Detect peer SM failure 

This change is to detect SM failure/stall.

1. SM sends alive pulse to hbsAgent, (hbsAgent sends SM failed state
   along with hbs cluster info)
2. When SM lost heartbeat from peer, SM detects if peer has failed
   from hbs cluster info.
3. On standby controller, if peer SM is stalled, it will take over
   to become active after signaling mtce to fail peer node.

TCs passed:
   When SM detects peer failure from hbs cluster info, and signals
   mtce to fail peer node.

Depends-on: https://review.opendev.org/#/c/751558

Partial-bug: 1895350
Change-Id: Id51e9adb4ef30bf806159366e6fdf115e743fe97
Signed-off-by: Bin Qian <bin.qian@windriver.com>
This commit is contained in:
Bin Qian 2020-09-17 12:36:31 -04:00
parent cc692ac45a
commit 2d0fc9b611
11 changed files with 361 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

174
service-mgmt/sm/src/sm_cluster_hbs_info_msg.cpp
View File

@ -22,6 +22,7 @@
#include "sm_limits.h"
#include "sm_selobj.h"
#include "sm_worker_thread.h"
#include "sm_node_utils.h"
// uncomment when debugging this module to enabled DPRINTFD output to log file
// #define __DEBUG__MSG__
@ -44,6 +45,7 @@ static const unsigned int size_of_msg_header =
bool operator==(const SmClusterHbsInfoT& lhs, const SmClusterHbsInfoT& rhs)
{
return lhs.storage0_responding == rhs.storage0_responding &&
lhs.sm_heartbeat_fail == rhs.sm_heartbeat_fail &&
lhs.number_of_node_reachable == rhs.number_of_node_reachable;
}
@ -86,25 +88,29 @@ void log_cluster_hbs_state(const SmClusterHbsStateT& state)
if(state.storage0_enabled)
{
DPRINTFI("Cluster hbs last updated %d secs ago, storage-0 is provisioned, "
"from controller-0: %d nodes enabled, %d nodes reachable, storage-0 %s responding "
"from controller-1: %d nodes enabled, %d nodes reachable, storage-0 %s responding",
DPRINTFI("Cluster hbs last updated %d secs ago, storage-0 is provisioned,\n"
"from controller-0: SM %s, %d nodes enabled, %d nodes reachable, storage-0 %s responding\n"
"from controller-1: SM %s, %d nodes enabled, %d nodes reachable, storage-0 %s responding",
secs_since_update,
state.controllers[0].sm_heartbeat_fail ? "FAILED": "ok ",
state.controllers[0].number_of_node_enabled,
state.controllers[0].number_of_node_reachable,
state.controllers[0].storage0_responding ? "is" : "is not",
state.controllers[1].sm_heartbeat_fail ? "FAILED": "ok ",
state.controllers[1].number_of_node_enabled,
state.controllers[1].number_of_node_reachable,
state.controllers[1].storage0_responding ? "is" : "is not"
);
}else
{
DPRINTFI("Cluster hbs last updated %d secs ago, storage-0 is not provisioned, "
"from controller-0: %d nodes enabled, %d nodes reachable, "
"from controller-1: %d nodes enabled, %d nodes reachable",
DPRINTFI("Cluster hbs last updated %d secs ago, storage-0 is not provisioned,\n"
"from controller-0: SM %s, %d nodes enabled, %d nodes reachable,\n"
"from controller-1: SM %s, %d nodes enabled, %d nodes reachable",
secs_since_update,
state.controllers[0].sm_heartbeat_fail ? "FAILED": "ok ",
state.controllers[0].number_of_node_enabled,
state.controllers[0].number_of_node_reachable,
state.controllers[1].sm_heartbeat_fail ? "FAILED": "ok ",
state.controllers[1].number_of_node_enabled,
state.controllers[1].number_of_node_reachable
);
@ -117,6 +123,10 @@ int SmClusterHbsInfoMsg::_sock = -1;
SmClusterHbsStateT SmClusterHbsInfoMsg::_cluster_hbs_state_current;
SmClusterHbsStateT SmClusterHbsInfoMsg::_cluster_hbs_state_previous;
SmClusterHbsInfoMsg::hbs_query_respond_callback SmClusterHbsInfoMsg::_callbacks;
int SmClusterHbsInfoMsg::this_controller_index = -1;
int SmClusterHbsInfoMsg::peer_controller_index = -1;
char SmClusterHbsInfoMsg::server_port[SM_CONFIGURATION_VALUE_MAX_CHAR + 1] = {0};
char SmClusterHbsInfoMsg::client_port[SM_CONFIGURATION_VALUE_MAX_CHAR + 1] = {0};
const SmClusterHbsStateT& SmClusterHbsInfoMsg::get_current_state()
{
@ -128,6 +138,47 @@ const SmClusterHbsStateT& SmClusterHbsInfoMsg::get_previous_state()
return _cluster_hbs_state_previous;
}
int SmClusterHbsInfoMsg::get_peer_controller_index()
{
if(peer_controller_index == -1)
{
get_controller_index();
}
return peer_controller_index;
}
int SmClusterHbsInfoMsg::get_this_controller_index()
{
if(this_controller_index == -1)
{
get_controller_index();
}
return this_controller_index;
}
SmErrorT SmClusterHbsInfoMsg::get_controller_index()
{
char host_name[SM_NODE_NAME_MAX_CHAR];
SmErrorT error = sm_node_utils_get_hostname(host_name);
if( SM_OKAY != error )
{
DPRINTFE( "Failed to get hostname, error=%s.",
sm_error_str( error ) );
return SM_FAILED;
}
if(0 == strncmp(SM_NODE_CONTROLLER_0_NAME, host_name, sizeof(SM_NODE_CONTROLLER_0_NAME)))
{
this_controller_index = 0;
peer_controller_index = 1;
}else
{
this_controller_index = 1;
peer_controller_index = 0;
}
return SM_OKAY;
}
bool SmClusterHbsInfoMsg::_process_cluster_hbs_history(mtce_hbs_cluster_history_type history, SmClusterHbsStateT& state)
{
if(history.controller >= max_controllers)
@ -156,6 +207,13 @@ bool SmClusterHbsInfoMsg::_process_cluster_hbs_history(mtce_hbs_cluster_history_
SmClusterHbsInfoT& controller_state = state.controllers[history.controller];
controller_state.storage0_responding = history.storage0_responding;
controller_state.sm_heartbeat_fail = (history.sm_heartbeat_fail == 1);
if(controller_state.sm_heartbeat_fail)
{
const char* controllers[] = {"controller-0", "controller-1"};
DPRINTFI("%s SM to hbsAgent alive pulse failed.", controllers[history.controller]);
}
if(entry.hosts_responding > controller_state.number_of_node_reachable)
{
controller_state.number_of_node_reachable = entry.hosts_responding;
@ -241,7 +299,7 @@ void SmClusterHbsInfoMsg::_cluster_hbs_info_msg_received( int selobj, int64_t us
}
}
SmErrorT SmClusterHbsInfoMsg::_get_address(const char* port_key, struct sockaddr_in* addr)
SmErrorT SmClusterHbsInfoMsg::_get_address(struct sockaddr_in* addr)
{
struct addrinfo *address = NULL;
struct addrinfo hints;
@ -254,17 +312,10 @@ SmErrorT SmClusterHbsInfoMsg::_get_address(const char* port_key, struct sockaddr
hints.ai_addr = NULL;
hints.ai_next = NULL;
char port[SM_CONFIGURATION_VALUE_MAX_CHAR + 1];
if( SM_OKAY != sm_configuration_table_get(port_key, port, sizeof(port) - 1) )
{
DPRINTFE("Runtime error: system configuration %s undefined", port_key);
return SM_FAILED;
}
int result = getaddrinfo(LOOPBACK_IP, port, &hints, &address);
int result = getaddrinfo(LOOPBACK_IP, server_port, &hints, &address);
if(result != 0)
{
DPRINTFE("Failed to get addrinfo %s:%s", LOOPBACK_IP, port);
DPRINTFE("Failed to get addrinfo %s:%s", LOOPBACK_IP, server_port);
return SM_FAILED;
}
@ -285,15 +336,8 @@ static SmSimpleAction _query_hbs_cluster_info_action("send hbs-cluster query", s
// trigger a query of cluster hbs info.
// return true if request sent successfully, false otherwise.
// ========================
bool SmClusterHbsInfoMsg::cluster_hbs_info_query(cluster_hbs_query_ready_callback callback)
bool SmClusterHbsInfoMsg::cluster_hbs_info_query(cluster_hbs_query_ready_callback callback, bool alive_pulse)
{
char server_port[SM_CONFIGURATION_VALUE_MAX_CHAR + 1];
if( SM_OKAY != sm_configuration_table_get(SM_SERVER_PORT_KEY, server_port, sizeof(server_port) - 1) )
{
DPRINTFE("Runtime error: system configuration %s undefined", SM_SERVER_PORT_KEY);
return false;
}
int port = atoi(server_port);
if(0 > port)
{
@ -306,18 +350,24 @@ bool SmClusterHbsInfoMsg::cluster_hbs_info_query(cluster_hbs_query_ready_callbac
struct timespec ts;
{
mutex_holder holder(&_mutex);
if(0 != clock_gettime(CLOCK_REALTIME, &ts))
if(alive_pulse)
{
DPRINTFE("Failed to get realtime");
reqid = (unsigned short)1;
reqid = 0;
}else
{
unsigned short* v = (unsigned short*)(&ts.tv_nsec);
reqid = (*v) % 0xFFFE + 1;
if(0 != clock_gettime(CLOCK_REALTIME, &ts))
{
DPRINTFE("Failed to get realtime");
reqid = (unsigned short)1;
}else
{
unsigned short* v = (unsigned short*)(&ts.tv_nsec);
reqid = (*v) % 0xFFFE + 1;
}
}
struct sockaddr_in addr;
if(SM_OKAY != _get_address(SM_SERVER_PORT_KEY, &addr))
if(SM_OKAY != _get_address(&addr))
{
DPRINTFE("Failed to get address");
return false;
@ -325,7 +375,10 @@ bool SmClusterHbsInfoMsg::cluster_hbs_info_query(cluster_hbs_query_ready_callbac
int msg_size = snprintf(query, sizeof(query), json_fmt, reqid);
DPRINTFD("send %d bytes %s", msg_size, query);
if (reqid != 0)
{
DPRINTFI("send hbs cluster query [%d]", reqid);
}
if(0 > sendto(_sock, query, msg_size, 0, (sockaddr*)&addr, sizeof(addr)))
{
DPRINTFE("Failed to send msg. Error %s", strerror(errno));
@ -339,6 +392,11 @@ bool SmClusterHbsInfoMsg::cluster_hbs_info_query(cluster_hbs_query_ready_callbac
return true;
}
bool SmClusterHbsInfoMsg::send_alive_pulse()
{
return cluster_hbs_info_query(NULL, true);
}
SmErrorT SmClusterHbsInfoMsg::open_socket()
{
struct addrinfo *address = NULL;
@ -353,28 +411,13 @@ SmErrorT SmClusterHbsInfoMsg::open_socket()
hints.ai_next = NULL;
struct sockaddr_in addr;
char client_port[SM_CONFIGURATION_VALUE_MAX_CHAR + 1];
char server_port[SM_CONFIGURATION_VALUE_MAX_CHAR + 1];
if( SM_OKAY != sm_configuration_table_get(SM_CLIENT_PORT_KEY, client_port, sizeof(client_port) - 1) )
int port = atoi(server_port);
if(0 > port)
{
DPRINTFE("Runtime error: system configuration %s undefined", SM_CLIENT_PORT_KEY);
DPRINTFE("Invalid configuration %s: %s", SM_SERVER_PORT_KEY, server_port);
return SM_FAILED;
}
if( SM_OKAY != sm_configuration_table_get(SM_SERVER_PORT_KEY, server_port, sizeof(server_port) - 1) )
{
DPRINTFE("Runtime error: system configuration %s undefined", SM_SERVER_PORT_KEY);
return SM_FAILED;
}else
{
int port = atoi(server_port);
if(0 > port)
{
DPRINTFE("Invalid configuration %s: %s", SM_SERVER_PORT_KEY, server_port);
return SM_FAILED;
}
}
int result = getaddrinfo(LOOPBACK_IP, client_port, &hints, &address);
if(result != 0)
{
@ -441,6 +484,39 @@ SmErrorT SmClusterHbsInfoMsg::initialize()
return SM_FAILED;
}
error = get_controller_index();
if(SM_OKAY != error)
{
DPRINTFE("Failed to get controller index");
return SM_FAILED;
}
if( SM_OKAY != sm_configuration_table_get(SM_SERVER_PORT_KEY, server_port, sizeof(server_port) - 1) )
{
DPRINTFE("Runtime error: system configuration %s undefined", SM_SERVER_PORT_KEY);
return SM_FAILED;
}
int port = atoi(server_port);
if(0 > port)
{
DPRINTFE("Runtime error: Invalid configuration %s: %s", SM_SERVER_PORT_KEY, server_port);
return SM_FAILED;
}
if( SM_OKAY != sm_configuration_table_get(SM_CLIENT_PORT_KEY, client_port, sizeof(client_port) - 1) )
{
DPRINTFE("Runtime error: system configuration %s undefined", SM_CLIENT_PORT_KEY);
return SM_FAILED;
}
port = atoi(client_port);
if(0 > port)
{
DPRINTFE("Runtime error: Invalid configuration %s: %s", SM_CLIENT_PORT_KEY, client_port);
return SM_FAILED;
}
error = open_socket();
if(SM_OKAY != error)
{

18
service-mgmt/sm/src/sm_cluster_hbs_info_msg.h
View File

@ -12,6 +12,7 @@
#include "sm_types.h"
#include "sm_timer.h"
#include "sm_util_types.h"
#include "sm_limits.h"
// ****************************************************************************
// struct SmClusterHbsInfoT
@ -22,9 +23,11 @@ struct _SmClusterHbsInfoT
bool storage0_responding;
int number_of_node_reachable;
int number_of_node_enabled;
bool sm_heartbeat_fail;
_SmClusterHbsInfoT() : storage0_responding(false),
number_of_node_reachable(0),
number_of_node_enabled(0)
number_of_node_enabled(0),
sm_heartbeat_fail(false)
{
}
};
@ -74,8 +77,11 @@ class SmClusterHbsInfoMsg
static SmErrorT finalize();
static const SmClusterHbsStateT& get_current_state();
static const SmClusterHbsStateT& get_previous_state();
static bool cluster_hbs_info_query(cluster_hbs_query_ready_callback callback = NULL);
static bool cluster_hbs_info_query(cluster_hbs_query_ready_callback callback = NULL, bool alive_pulse = false);
static bool send_alive_pulse();
static void dump_hbs_record(FILE* fp);
static int get_peer_controller_index();
static int get_this_controller_index();
private:
static int _sock;
@ -85,11 +91,17 @@ class SmClusterHbsInfoMsg
static SmClusterHbsStateT _cluster_hbs_state_previous;
static hbs_query_respond_callback _callbacks;
static SmErrorT open_socket();
static SmErrorT get_controller_index();
static SmErrorT _get_address(const char* port_key, struct sockaddr_in* addr);
static SmErrorT _get_address(struct sockaddr_in* addr);
static void _cluster_hbs_info_msg_received( int selobj, int64_t user_data );
static bool _process_cluster_hbs_history(mtce_hbs_cluster_history_type history,
SmClusterHbsStateT& state);
static char server_port[SM_CONFIGURATION_VALUE_MAX_CHAR + 1];
static char client_port[SM_CONFIGURATION_VALUE_MAX_CHAR + 1];
static int peer_controller_index;
static int this_controller_index;
};
#endif // __SM_CLUSTER_HBS_INFO_MSG_H__

76
service-mgmt/sm/src/sm_failover.c
View File

@ -11,6 +11,9 @@
#include <time.h>
#include <stdio.h>
#include <json-c/json.h>
#include <errno.h>
#include <unistd.h>
#include <fcntl.h>
#include "sm_service_domain_interface_table.h"
#include "sm_debug.h"
@ -44,6 +47,8 @@
#define SM_FAILOVER_RECOVERY_INTERVAL_IN_SEC 100
#define SM_FAILOVER_INTERFACE_STATE_REPORT_INTERVAL_MS 20000
const char* RESET_PEER_NOW = "/var/run/.sm_reset_peer";
typedef enum
{
SM_FAILOVER_ACTION_RESULT_OK,
@ -816,12 +821,25 @@ static SmErrorT sm_ensure_leader_scheduler()
}
return error;
}
bool file_exist(const char* filename)
{
if( 0 != access(filename, F_OK ) )
{
if( ENOENT == errno )
{
return false;
}
}
return true;
}
// ****************************************************************************
// Failover - set system to scheduled status
// ==================
SmErrorT sm_failover_set_system(const SmSystemFailoverStatus& failover_status)
{
//valid combinations of target system scheduling are:
// valid combinations of target system scheduling are:
// active/standby
// active/failed
SmFailoverActionResultT result;
@ -829,6 +847,47 @@ SmErrorT sm_failover_set_system(const SmSystemFailoverStatus& failover_status)
host_target_state = failover_status.get_host_schedule_state();
peer_target_state = failover_status.get_peer_schedule_state();
SmHeartbeatStateT heartbeat_state = failover_status.get_heartbeat_state();
if(SM_NODE_STATE_ACTIVE == host_target_state &&
SM_NODE_STATE_FAILED == peer_target_state &&
failover_status.peer_stall())
{
int seconds_to_wait = sm_failover_get_reset_peer_wait_time();
int fd = open(RESET_PEER_NOW, O_RDWR | O_CREAT, S_IRUSR | S_IRGRP | S_IROTH);
if( 0 > fd )
{
DPRINTFE("Failed to create file (%s), error=%s.", RESET_PEER_NOW, strerror(errno) );
}else
{
close(fd);
}
for(int i = 0; i < seconds_to_wait * 10; i ++)
{
//wait up to 30 seconds for mtce to reset peer.
if(!file_exist(RESET_PEER_NOW))
{
DPRINTFI("%s is gone.", RESET_PEER_NOW);
break;
}else
{
usleep(100000); // 100ms
if(i % 10 == 0)
{
// log every second
DPRINTFI("Wait for %s to be removed", RESET_PEER_NOW);
}
}
}
if(file_exist(RESET_PEER_NOW))
{
DPRINTFE("%s still exists after %d seconds. Start activating the controller.",
RESET_PEER_NOW, seconds_to_wait);
}
}
if(SM_HEARTBEAT_OK != heartbeat_state)
{
if(SM_OKAY != sm_ensure_leader_scheduler())
@ -1102,6 +1161,9 @@ SmErrorT sm_failover_disable_peer()
{
DPRINTFE("Failed to disable peer %s", _peer_name);
return SM_FAILED;
}else
{
DPRINTFI("Request mtce to reset peer");
}
return SM_OKAY;
}
@ -1292,12 +1354,6 @@ SmErrorT sm_failover_initialize( void )
return SM_FAILED;
}
error = SmClusterHbsInfoMsg::initialize();
if(SM_OKAY != error)
{
DPRINTFE("Failed to initialize cluster hbs info messaging");
}
return SM_OKAY;
}
// ****************************************************************************
@ -1311,12 +1367,6 @@ SmErrorT sm_failover_finalize( void )
SmErrorT error;
error = SmClusterHbsInfoMsg::finalize();
if(SM_OKAY != error)
{
DPRINTFE("Failed to finalize cluster hbs info messaging");
}
sm_timer_deregister( failover_audit_timer_id );
if( NULL != _sm_db_handle )
{

33
service-mgmt/sm/src/sm_failover_fail_pending_state.cpp
View File

@ -6,6 +6,7 @@
#include "sm_failover_fail_pending_state.h"
#include <stdlib.h>
#include <unistd.h>
#include "sm_configuration_table.h"
#include "sm_cluster_hbs_info_msg.h"
#include "sm_types.h"
#include "sm_limits.h"
@ -18,8 +19,26 @@
#include "sm_node_api.h"
#include "sm_worker_thread.h"
static const int FAIL_PENDING_TIMEOUT = 2000; // 2seconds
static const int DELAY_QUERY_HBS_MS = FAIL_PENDING_TIMEOUT - 200; // give 200ms for hbs agent to respond
static const int FAIL_PENDING_TIMEOUT_DEFAULT = 2000; // 2 seconds
static const int FAIL_PENDING_TIMEOUT_MIN = 1200;
static inline int get_failpending_timeout()
{
int fail_pending_timeout =0;
char buf[SM_CONFIGURATION_VALUE_MAX_CHAR + 1];
if( SM_OKAY == sm_configuration_table_get("FAILPENDING_TIMEOUT_MS", buf, sizeof(buf) - 1) )
{
fail_pending_timeout = atoi(buf);
}
if(fail_pending_timeout < FAIL_PENDING_TIMEOUT_MIN)
{
fail_pending_timeout = FAIL_PENDING_TIMEOUT_DEFAULT;
}
return fail_pending_timeout;
}
static const int fail_pending_timeout = 0;
static SmTimerIdT action_timer_id = SM_TIMER_ID_INVALID;
static const int RESET_TIMEOUT = 10 * 1000; // 10 seconds for a reset command to reboot a node
@ -292,6 +311,10 @@ bool SmFailoverFailPendingState::_fail_pending_timeout(
SmErrorT SmFailoverFailPendingState::enter_state()
{
SmFSMState::enter_state();
DPRINTFI("Pre failure hbs cluster info:");
const SmClusterHbsStateT& cluster_hbs_state = SmClusterHbsInfoMsg::get_current_state();
log_cluster_hbs_state(cluster_hbs_state);
SmErrorT error = this->_register_timer();
if(SM_OKAY != error)
{
@ -303,6 +326,7 @@ SmErrorT SmFailoverFailPendingState::enter_state()
void _cluster_hbs_response_callback()
{
const SmClusterHbsStateT& cluster_hbs_state = SmClusterHbsInfoMsg::get_current_state();
DPRINTFI("Fail-pending timeout cluster info:");
log_cluster_hbs_state(cluster_hbs_state);
SmSystemFailoverStatus::get_status().set_cluster_hbs_state(cluster_hbs_state);
}
@ -323,13 +347,14 @@ SmErrorT SmFailoverFailPendingState::_register_timer()
this->_deregister_timer();
}
error = sm_timer_register(timer_name, FAIL_PENDING_TIMEOUT,
int fail_pending_timeout = get_failpending_timeout();
error = sm_timer_register(timer_name, fail_pending_timeout,
SmFailoverFailPendingState::_fail_pending_timeout,
0, &this->_pending_timer_id);
const char* delay_query_hbs_timer_name = "DELAY QUERY HBS";
error = sm_timer_register(delay_query_hbs_timer_name, DELAY_QUERY_HBS_MS,
error = sm_timer_register(delay_query_hbs_timer_name, fail_pending_timeout - 200,
SmFailoverFailPendingState::_delay_query_hbs_timeout,
0, &this->_delay_query_hbs_timer_id);

41
service-mgmt/sm/src/sm_failover_failed_state.cpp
View File

@ -24,6 +24,7 @@
#include "sm_failover_fsm.h"
#include "sm_failover_ss.h"
#include "sm_failover_utils.h"
#include "sm_cluster_hbs_info_msg.h"
extern bool is_cluster_host_interface_configured( void );
@ -38,6 +39,8 @@ static const int SM_FAILOVER_FAILED_LOG_THROTTLE_THLD = 12;
#define PROCESS_HBSAGENT ((const char *)("hbsAgent"))
#define PROCESS_SM ((const char *)("sm"))
static struct timespec start_time;
// Failover Failed state class constructor
SmFailoverFailedState::SmFailoverFailedState(SmFailoverFSM& fsm) : SmFSMState(fsm)
{
@ -61,6 +64,9 @@ SmErrorT SmFailoverFailedState::enter_state()
DPRINTFE("Entering Failover Failed state ; recovery audit started ");
DPRINTFE("********************************************************");
clock_gettime(CLOCK_MONOTONIC_RAW, &start_time);
DPRINTFI("Wait for %d seconds to start failed recovery audit",
sm_failover_get_reset_peer_wait_time());
SmErrorT error = this->_register_timer();
if(SM_OKAY != error)
{
@ -73,7 +79,13 @@ SmErrorT SmFailoverFailedState::enter_state()
bool SmFailoverFailedState::_failed_state_audit(
SmTimerIdT timer_id, int64_t user_data)
{
SmFailoverFSM::get_fsm().send_event(SM_FAILOVER_EVENT_FAILED_RECOVERY_AUDIT, NULL);
// wait enough time for peer to be reset to avoid conflict decision, i.e being reset
struct timespec now;
clock_gettime(CLOCK_REALTIME, &now);
if(now.tv_sec - start_time.tv_sec - 1 > sm_failover_get_reset_peer_wait_time())
{
SmFailoverFSM::get_fsm().send_event(SM_FAILOVER_EVENT_FAILED_RECOVERY_AUDIT, NULL);
}
return true ;
}
@ -139,21 +151,28 @@ static bool sm_failover_failed_recovery_criteria_met( void )
oam_state = sm_failover_get_interface_info(SM_INTERFACE_OAM);
mgmt_state = sm_failover_get_interface_info(SM_INTERFACE_MGMT);
if ( is_cluster_host_interface_configured() )
const SmClusterHbsStateT& cluster_hbs_state = SmClusterHbsInfoMsg::get_current_state();
int peer_controller_index = SmClusterHbsInfoMsg::get_peer_controller_index();
// If peer has stalled, do not recover until peer recovers from stall
if(!cluster_hbs_state.controllers[peer_controller_index].sm_heartbeat_fail)
{
cluster_host_state = sm_failover_get_interface_info(SM_INTERFACE_CLUSTER_HOST);
if ((( oam_state == SM_FAILOVER_INTERFACE_OK ) || ( oam_state == SM_FAILOVER_INTERFACE_MISSING_HEARTBEAT )) &&
(( mgmt_state == SM_FAILOVER_INTERFACE_OK ) || ( mgmt_state == SM_FAILOVER_INTERFACE_MISSING_HEARTBEAT )) &&
(( cluster_host_state == SM_FAILOVER_INTERFACE_OK ) || ( cluster_host_state == SM_FAILOVER_INTERFACE_MISSING_HEARTBEAT )))
if ( is_cluster_host_interface_configured() )
{
cluster_host_state = sm_failover_get_interface_info(SM_INTERFACE_CLUSTER_HOST);
if ((( oam_state == SM_FAILOVER_INTERFACE_OK ) || ( oam_state == SM_FAILOVER_INTERFACE_MISSING_HEARTBEAT )) &&
(( mgmt_state == SM_FAILOVER_INTERFACE_OK ) || ( mgmt_state == SM_FAILOVER_INTERFACE_MISSING_HEARTBEAT )) &&
(( cluster_host_state == SM_FAILOVER_INTERFACE_OK ) || ( cluster_host_state == SM_FAILOVER_INTERFACE_MISSING_HEARTBEAT )))
{
criteria_met = true ;
}
}
else if ((( oam_state == SM_FAILOVER_INTERFACE_OK ) || ( oam_state == SM_FAILOVER_INTERFACE_MISSING_HEARTBEAT )) &&
(( mgmt_state == SM_FAILOVER_INTERFACE_OK ) || ( mgmt_state == SM_FAILOVER_INTERFACE_MISSING_HEARTBEAT )))
{
criteria_met = true ;
}
}
else if ((( oam_state == SM_FAILOVER_INTERFACE_OK ) || ( oam_state == SM_FAILOVER_INTERFACE_MISSING_HEARTBEAT )) &&
(( mgmt_state == SM_FAILOVER_INTERFACE_OK ) || ( mgmt_state == SM_FAILOVER_INTERFACE_MISSING_HEARTBEAT )))
{
criteria_met = true ;
}
DPRINTFI("Oam:%s ; Mgmt:%s ; Cluster:%s ; recovery criteria met: %s",
sm_failover_interface_state_str(oam_state),

35
service-mgmt/sm/src/sm_failover_ss.c
View File

@ -380,32 +380,25 @@ SmErrorT sm_failover_ss_get_survivor(const SmSystemStatusT& system_status, SmSys
has_cluser_info = false;
}
DPRINTFI("cluster info (%s), maximum available nodes %d", has_cluser_info ? "yes": "no", max_nodes_available);
if(has_cluser_info && max_nodes_available > 1)
{
DPRINTFD("storage-0 is %s", expect_storage_0 ? "enabled":"not enabled");
int this_controller_index, peer_controller_index;
char host_name[SM_NODE_NAME_MAX_CHAR];
SmErrorT error = sm_node_utils_get_hostname(host_name);
if( SM_OKAY != error )
{
DPRINTFE( "Failed to get hostname, error=%s.",
sm_error_str( error ) );
return SM_FAILED;
}
if(0 == strncmp(SM_NODE_CONTROLLER_0_NAME, host_name, sizeof(SM_NODE_CONTROLLER_0_NAME)))
{
this_controller_index = 0;
peer_controller_index = 1;
}else
{
this_controller_index = 1;
peer_controller_index = 0;
}
int this_controller_index = SmClusterHbsInfoMsg::get_this_controller_index();
int peer_controller_index = SmClusterHbsInfoMsg::get_peer_controller_index();
bool survivor_selected = false;
if(expect_storage_0)
selection.set_peer_stall(false);
if(current_cluster_hbs_state.controllers[peer_controller_index].sm_heartbeat_fail)
{
// peer sm not sending alive pulse, failed
DPRINTFI("Peer controller has stalled.");
selection.set_host_schedule_state(SM_NODE_STATE_ACTIVE);
selection.set_peer_schedule_state(SM_NODE_STATE_FAILED);
survivor_selected = true;
selection.set_peer_stall(true);
}
else if(expect_storage_0)
{
if(current_cluster_hbs_state.controllers[this_controller_index].storage0_responding &&
!current_cluster_hbs_state.controllers[peer_controller_index].storage0_responding)

9
service-mgmt/sm/src/sm_failover_ss.h
View File

@ -69,6 +69,14 @@ class SmSystemFailoverStatus
inline SmHeartbeatStateT get_heartbeat_state() const {
return _heartbeat_state;
}
inline bool peer_stall() const {
return _peer_stall;
}
inline void set_peer_stall(bool stall) {
_peer_stall = stall;
}
void set_host_schedule_state(SmNodeScheduleStateT state);
void set_host_pre_failure_schedule_state(SmNodeScheduleStateT state);
void set_cluster_hbs_state(const SmClusterHbsStateT& state);
@ -93,6 +101,7 @@ class SmSystemFailoverStatus
SmNodeScheduleStateT _peer_schedule_state;
SmClusterHbsStateT _cluster_hbs_state;
SmHeartbeatStateT _heartbeat_state;
bool _peer_stall;
static const char filename[];
static const char file_format[];

25
service-mgmt/sm/src/sm_failover_utils.c
View File

@ -8,6 +8,7 @@
#include "sm_service_domain_member_table.h"
#include "sm_service_domain_assignment_table.h"
#include "sm_service_domain_table.h"
#include "sm_configuration_table.h"
#define SM_NODE_STAY_FAILED_FILE "/var/run/.sm_stay_fail"
@ -139,3 +140,27 @@ bool sm_failover_utils_is_stayfailed()
return false;
}
// ****************************************************************************
// ****************************************************************************
// Failover Utilities - get wait time for peer to be reset
// ==============================
int sm_failover_get_reset_peer_wait_time()
{
char buf[SM_CONFIGURATION_VALUE_MAX_CHAR + 1];
const int RESET_PEER_WAIT_TIME_MIN_SEC = 1;
const int RESET_PEER_WAIT_TIME_DEFAULT_SEC = 30;
int seconds_to_wait = 0;
if( SM_OKAY == sm_configuration_table_get("RESET_PEER_WAIT_TIMEOUT_SEC", buf, sizeof(buf) - 1) )
{
seconds_to_wait = atoi(buf);
}
if( seconds_to_wait < RESET_PEER_WAIT_TIME_MIN_SEC)
{
seconds_to_wait = RESET_PEER_WAIT_TIME_DEFAULT_SEC;
}
DPRINTFI("Peer reset wait time %d sec", seconds_to_wait);
return seconds_to_wait;
}
// ****************************************************************************

4
service-mgmt/sm/src/sm_failover_utils.h
View File

@ -36,6 +36,10 @@ extern SmErrorT sm_failover_utils_reset_stayfailed_flag();
extern bool sm_failover_utils_is_stayfailed();
// ****************************************************************************
// ****************************************************************************
// Failover Utilities - get wait time for peer to be reset
// ==============================
extern int sm_failover_get_reset_peer_wait_time();
#ifdef __cplusplus
}
#endif

34
service-mgmt/sm/src/sm_heartbeat_thread.c
View File

@ -35,6 +35,7 @@
#include "sm_alarm.h"
#include "sm_log.h"
#include "sm_failover.h"
#include "sm_cluster_hbs_info_msg.h"
#define SM_HEARTBEAT_THREAD_NAME "sm_heartbeat"
#define SM_HEARTBEAT_THREAD_TICK_INTERVAL_IN_MS 100
@ -106,6 +107,10 @@ static SmTimerIdT _alive_timer_id = SM_TIMER_ID_INVALID;
static char _node_name[SM_NODE_NAME_MAX_CHAR];
static SmHeartbeatMsgCallbacksT _callbacks;
static int alive_pulse_count = 0;
static int alive_pulse_fail_count = 0;
static SmTimeT alive_pulse_since = {0};
// ****************************************************************************
// Heartbeat Thread - SmNetworkAddressT ==
// =========================
@ -921,6 +926,35 @@ static bool sm_heartbeat_alive_timer( SmTimerIdT timer_id, int64_t user_data )
DPRINTFD( "Sent alive message for node (%s).", _node_name );
}
if(!SmClusterHbsInfoMsg::send_alive_pulse())
{
alive_pulse_fail_count ++;
DPRINTFE("Failed sending alive pulse to hbsAgent");
}else
{
alive_pulse_count ++;
}
if(alive_pulse_since.tv_sec == 0)
{
clock_gettime(CLOCK_MONOTONIC_RAW, &alive_pulse_since);
}else
{
SmTimeT now;
clock_gettime(CLOCK_MONOTONIC_RAW, &now);
int delta;
delta = (int)(now.tv_sec - alive_pulse_since.tv_sec);
if(delta > 60)
{
DPRINTFI("Record %d alive pulses (include %d failed) to hbsAgent since %d seconds ago",
alive_pulse_count + alive_pulse_fail_count, alive_pulse_fail_count, delta);
alive_pulse_count = 0;
alive_pulse_fail_count = 0;
alive_pulse_since = now;
}
}
DONE:
if( 0 != pthread_mutex_unlock( &_mutex ) )
{

13
service-mgmt/sm/src/sm_process.c
View File

@ -54,6 +54,7 @@
#include "sm_task_affining_thread.h"
#include "sm_worker_thread.h"
#include "sm_configuration_table.h"
#include "sm_cluster_hbs_info_msg.h"
#define SM_PROCESS_DB_CHECKPOINT_INTERVAL_IN_MS 30000
#define SM_PROCESS_TICK_INTERVAL_IN_MS 200
@ -233,6 +234,12 @@ static SmErrorT sm_process_initialize( void )
return( SM_FAILED );
}
error = SmClusterHbsInfoMsg::initialize();
if(SM_OKAY != error)
{
DPRINTFE("Failed to initialize cluster hbs info messaging");
}
if (_is_aio_simplex)
{
sm_heartbeat_thread_disable_heartbeat();
@ -518,6 +525,12 @@ static SmErrorT sm_process_finalize( void )
sm_error_str( error ) );
}
error = SmClusterHbsInfoMsg::finalize();
if(SM_OKAY != error)
{
DPRINTFE("Failed to finalize cluster hbs info messaging");
}
error = sm_log_finalize();
if( SM_OKAY != error )
{