Merge "Add admin network support to SM"

service-mgmt-tools/sm-tools/sm_tools/sm_api_msg_utils.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2016 Wind River Systems, Inc.
+# Copyright (c) 2016-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -23,6 +23,9 @@ SM_API_MSG_SKIP_DEP_CHECK = "skip-dep"
 SM_API_MSG_TYPE_PROVISION_SERVICE = "PROVISION_SERVICE"
 SM_API_MSG_TYPE_DEPROVISION_SERVICE = "DEPROVISION_SERVICE"
 
+SM_API_MSG_TYPE_PROVISION_SERVICE_DOMAIN_INTERFACE = "PROVISION_SERVICE_DOMAIN_INTERFACE"
+SM_API_MSG_TYPE_DEPROVISION_SERVICE_DOMAIN_INTERFACE = "DEPROVISION_SERVICE_DOMAIN_INTERFACE"
+
 SM_API_MSG_TYPE_RELOAD_DATA = "RELOAD_DATA"
 
 SM_API_MSG_TYPE_SDI_SET_STATE = "SERVICE_DOMAIN_INTERFACE_SET_STATE"
@@ -34,6 +37,10 @@ SM_API_MSG_SEQNO_FIELD = 2
 SM_API_MSG_TYPE_FIELD = 3
 SM_API_MSG_ORIGIN_FIELD = 4
 SM_API_MSG_SERVICE_NAME_FIELD = 5
+# For provisioning a service domain interface, the SM API
+# message expects the service domain name at the same offset
+# as the service name field (used for provisioning a service)
+SM_API_MSG_SERVICE_DOMAIN_NAME_FIELD = 5
 SM_API_MSG_PARAM = 6
 
 
@@ -98,3 +105,35 @@ def deprovision_service(service_name, service_group_name):
                      SM_API_MSG_TYPE_DEPROVISION_SERVICE, "sm-action",
                      service_name, service_group_name))
     _send_msg_to_sm(sm_api_msg)
+
+
+def provision_service_domain_interface(service_domain,
+                                       service_domain_interface_name):
+    """
+
+    :param service_domain:
+    :param service_domain_interface_name:
+    :return:
+    """
+    sm_api_msg = ("%s,%s,%i,%s,%s,%s,%s"
+                  % (SM_API_MSG_VERSION, SM_API_MSG_REVISION, 1,
+                     SM_API_MSG_TYPE_PROVISION_SERVICE_DOMAIN_INTERFACE,
+                     "sm-action", service_domain,
+                     service_domain_interface_name))
+    _send_msg_to_sm(sm_api_msg)
+
+
+def deprovision_service_domain_interface(service_domain,
+                                         service_domain_interface_name):
+    """
+
+    :param service_domain:
+    :param service_domain_interface_name:
+    :return:
+    """
+    sm_api_msg = ("%s,%s,%i,%s,%s,%s,%s"
+                  % (SM_API_MSG_VERSION, SM_API_MSG_REVISION, 1,
+                     SM_API_MSG_TYPE_DEPROVISION_SERVICE_DOMAIN_INTERFACE,
+                     "sm-action", service_domain,
+                     service_domain_interface_name))
+    _send_msg_to_sm(sm_api_msg)

service-mgmt-tools/sm-tools/sm_tools/sm_configure.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014-2015 Wind River Systems, Inc.
+# Copyright (c) 2014-2023 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -15,9 +15,12 @@ cpe_duplex = "duplex"
 cpe_duplex_direct = "duplex-direct"
 mgmt_if = 'management-interface'
 cluster_host_if = 'cluster-host-interface'
+admin_if = 'admin-interface'
+admin_ip = 'admin-ip'
 tor_connect = 'tor'
 dc_connect = 'dc'
 database_name = "/var/lib/sm/sm.db"
+runtime_db_name = "/var/run/sm/sm.db"
 
 
 def main():
@@ -41,6 +44,10 @@ def main():
         if_parser.add_argument('network_peer_port', help='network peer port')
         if_parser.add_argument('network_peer_heartbeat_port',
                                help='network peer heartbeat port')
+        if_parser.add_argument('--apply',
+                               help='Apply the new configuration '
+                                    'immediately',
+                               action='store_true')
 
         si_parser = subparsers.add_parser('service_instance',
                                           help='Service Instance '
@@ -49,6 +56,10 @@ def main():
         si_parser.add_argument('service', help='service name')
         si_parser.add_argument('instance', help='instance name')
         si_parser.add_argument('parameters', help='instance parameters')
+        si_parser.add_argument('--apply',
+                               help='Apply the new configuration '
+                                    'immediately',
+                               action='store_true')
 
         sys_parser = subparsers.add_parser('system',
                                            help='system Configuration')
@@ -110,6 +121,10 @@ def main():
                     print("Invalid sm_process_priority value. "
                           "Must be between -2 to -20")
                     sys.exit(-1)
+        elif args.which == 'interface':
+            _configure_interface(args)
+        elif args.which == 'service_instance':
+            _configure_service_instance(args)
         else:
             database = sqlite3.connect(database_name)
             _dispatch_config_action(args, database)
@@ -233,6 +248,37 @@ def _dispatch_config_action(args, database):
         database.commit()
 
 
+def _configure_interface(args):
+    database = sqlite3.connect(database_name)
+    _dispatch_config_action(args, database)
+    database.close()
+
+    if args.apply:
+        if args.service_domain_interface == admin_if:
+            database = sqlite3.connect(runtime_db_name)
+            _dispatch_config_action(args, database)
+            database.close()
+        else:
+            print("Apply operation on service "
+                  "domain interface %s is unsupported." %
+                  (args.service_domain_interface))
+
+
+def _configure_service_instance(args):
+    database = sqlite3.connect(database_name)
+    _dispatch_config_action(args, database)
+    database.close()
+    if args.apply:
+        if args.service == admin_ip:
+            database = sqlite3.connect(runtime_db_name)
+            _dispatch_config_action(args, database)
+            database.close()
+        else:
+            print("Apply operation on service "
+                  "instance %s is unsupported." %
+                  (args.service))
+
+
 def configure_cpe_duplex():
     configure_if_connect_type(mgmt_if, tor_connect)
     configure_if_connect_type(cluster_host_if, tor_connect)

service-mgmt-tools/sm-tools/sm_tools/sm_provision.py

@@ -11,6 +11,8 @@ import argparse
 import sqlite3
 from sm_tools.sm_api_msg_utils import provision_service
 from sm_tools.sm_api_msg_utils import deprovision_service
+from sm_tools.sm_api_msg_utils import provision_service_domain_interface
+from sm_tools.sm_api_msg_utils import deprovision_service_domain_interface
 
 database_name = "/var/lib/sm/sm.db"
 runtime_db_name = "/var/run/sm/sm.db"
@@ -64,6 +66,10 @@ def main():
                                       help='service domain name')
         sd_member_parser.add_argument('service_domain_interface',
                                       help='service domain interface name')
+        sd_member_parser.add_argument("--apply",
+                                      help="Apply the new configuration "
+                                           "immediately",
+                                      action="store_true")
 
         # Service-Group
         sg = subparsers.add_parser('service-group',
@@ -120,18 +126,25 @@ def main():
             database.close()
 
         elif args.which == 'service_domain_interface':
-            database = sqlite3.connect(database_name)
+            sql_update = "UPDATE SERVICE_DOMAIN_INTERFACES SET " \
+                         "PROVISIONED = '%s' WHERE SERVICE_DOMAIN = '%s' and " \
+                         "SERVICE_DOMAIN_INTERFACE = '%s';" \
+                         % (provision_str, args.service_domain,
+                            args.service_domain_interface)
+            sqls = [sql_update]
+            update_db(database_name, sqls)
 
-            cursor = database.cursor()
+            if args.apply and os.path.isfile(runtime_db_name):
+                # update runtime configuration
+                update_db(runtime_db_name, sqls)
 
-            cursor.execute("UPDATE SERVICE_DOMAIN_INTERFACES SET "
+                # tell SM to apply changes
-                           "PROVISIONED = '%s' WHERE SERVICE_DOMAIN = '%s' and "
+                if "yes" == provision_str:
-                           "SERVICE_DOMAIN_INTERFACE = '%s';"
+                    provision_service_domain_interface(args.service_domain,
-                           % (provision_str, args.service_domain,
+                                                       args.service_domain_interface)
-                              args.service_domain_interface))
+                elif "no" == provision_str:
-
+                    deprovision_service_domain_interface(args.service_domain,
-            database.commit()
+                                                         args.service_domain_interface)
-            database.close()
 
         elif args.which == 'service_group':
             database = sqlite3.connect(database_name)

service-mgmt/sm-common/src/sm_eru_process.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014-2017 Wind River Systems, Inc.
+// Copyright (c) 2014-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -115,6 +115,13 @@ void sm_eru_process_load_interfaces( void )
         DPRINTFE( "Failed to look up cluster-host interface, error=%s.",
                   sm_error_str(error) );
     }
+
+    error = sm_node_utils_get_admin_interface( &(_interfaces[3][0]) );
+    if(( SM_OKAY != error )&&( SM_NOT_FOUND != error ))
+    {
+        DPRINTFE( "Failed to look up admin interface, error=%s.",
+                  sm_error_str(error) );
+    }
 }
 // ****************************************************************************
 

service-mgmt/sm-common/src/sm_node_utils.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014-2020 Wind River Systems, Inc.
+// Copyright (c) 2014-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -222,6 +222,17 @@ SmErrorT sm_node_utils_get_cluster_host_interface( char interface_name[] )
 }
 // ****************************************************************************
 
+// ****************************************************************************
+// Node Utilities - Get Admin Interface
+// ===========================================
+SmErrorT sm_node_utils_get_admin_interface( char interface_name[] )
+{
+    return( sm_node_utils_read_platform_config( "admin_interface",
+                                                interface_name,
+                                                SM_INTERFACE_NAME_MAX_CHAR ) );
+}
+// ****************************************************************************
+
 // ****************************************************************************
 // Node Utilities - Get system mode string
 // =============================================

service-mgmt/sm-common/src/sm_node_utils.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014,2020 Wind River Systems, Inc.
+// Copyright (c) 2014,2020-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -50,6 +50,12 @@ extern SmErrorT sm_node_utils_get_oam_interface( char interface_name[] );
 extern SmErrorT sm_node_utils_get_cluster_host_interface( char interface_name[] );
 // ****************************************************************************
 
+// ****************************************************************************
+// Node Utilities - Get Admin Interface
+// ===========================================
+extern SmErrorT sm_node_utils_get_admin_interface( char interface_name[] );
+// ****************************************************************************
+
 // ****************************************************************************
 // Node Utilities - Get system mode
 // =============================================

service-mgmt/sm-common/src/sm_types.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014-2020 Wind River Systems, Inc.
+// Copyright (c) 2014-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -1872,6 +1872,9 @@ SmInterfaceTypeT sm_get_interface_type( const char* domain_interface )
     } else if ( 0 == strcmp ( SM_SERVICE_DOMAIN_CLUSTER_HOST_INTERFACE, domain_interface ) )
     {
         return SM_INTERFACE_CLUSTER_HOST;
+    } else if ( 0 == strcmp ( SM_SERVICE_DOMAIN_ADMIN_INTERFACE, domain_interface ) )
+    {
+        return SM_INTERFACE_ADMIN;
     }
 
     return SM_INTERFACE_UNKNOWN;

service-mgmt/sm-common/src/sm_types.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014-2020 Wind River Systems, Inc.
+// Copyright (c) 2014-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -96,10 +96,12 @@ extern "C" {
 #define SM_SERVICE_DOMAIN_MGMT_INTERFACE             "management-interface"
 #define SM_SERVICE_DOMAIN_OAM_INTERFACE              "oam-interface"
 #define SM_SERVICE_DOMAIN_CLUSTER_HOST_INTERFACE     "cluster-host-interface"
+#define SM_SERVICE_DOMAIN_ADMIN_INTERFACE            "admin-interface"
 
 #define SM_MGMT_INTERFACE_NAME                       "mgmt"
 #define SM_OAM_INTERFACE_NAME                        "oam"
 #define SM_CLUSTER_HOST_INTERFACE_NAME               "cluster-host"
+#define SM_ADMIN_INTERFACE_NAME                      "admin"
 #define SM_MAX_IF_NAME_LEN                           5
 
 #define SM_NODE_CONTROLLER_0_NAME                    "controller-0"
@@ -201,7 +203,8 @@ typedef enum
     SM_INTERFACE_UNKNOWN,
     SM_INTERFACE_MGMT,
     SM_INTERFACE_CLUSTER_HOST,
-    SM_INTERFACE_OAM
+    SM_INTERFACE_OAM,
+    SM_INTERFACE_ADMIN
 }SmInterfaceTypeT;
 
 typedef enum

service-mgmt/sm-db/database/create_sm_db.sql

@@ -6,6 +6,7 @@ CREATE TABLE SERVICE_DOMAIN_INTERFACES ( ID INTEGER PRIMARY KEY AUTOINCREMENT, P
 INSERT INTO "SERVICE_DOMAIN_INTERFACES" VALUES(1,'yes','controller','management-interface','secondary','none','','','','','','','','','','','','tor');
 INSERT INTO "SERVICE_DOMAIN_INTERFACES" VALUES(2,'yes','controller','oam-interface','secondary','hmac-sha512','titanium-server','','','','','','','','','','','tor');
 INSERT INTO "SERVICE_DOMAIN_INTERFACES" VALUES(3,'yes','controller','cluster-host-interface','secondary','none','','','','','','','','','','','','tor');
+INSERT INTO "SERVICE_DOMAIN_INTERFACES" VALUES(4,'no','controller','admin-interface','secondary','none','','','','','','','','','','','','tor');
 CREATE TABLE SERVICE_DOMAINS ( ID INTEGER PRIMARY KEY AUTOINCREMENT, PROVISIONED CHAR(32), NAME CHAR(32), ORCHESTRATION CHAR(32), DESIGNATION CHAR(32), PREEMPT CHAR(32), PRIORITY INT, HELLO_INTERVAL INT, DEAD_INTERVAL INT, WAIT_INTERVAL INT, EXCHANGE_INTERVAL INT, STATE CHAR(32), SPLIT_BRAIN_RECOVERY CHAR(32), LEADER CHAR(32), GENERATION INT);
 INSERT INTO "SERVICE_DOMAINS" VALUES(1,'yes','controller','regional','unknown','no',230,200,800,5000,2000,'initial','select-best-active','',1);
 CREATE TABLE SERVICE_DOMAIN_MEMBERS ( ID INTEGER PRIMARY KEY AUTOINCREMENT, PROVISIONED CHAR(32), NAME CHAR(32), SERVICE_GROUP_NAME CHAR(32), REDUNDANCY_MODEL CHAR(32), N_ACTIVE INT, M_STANDBY INT, SERVICE_GROUP_AGGREGATE CHAR(32), ACTIVE_ONLY_IF_ACTIVE CHAR(32) );
@@ -19,6 +20,7 @@ INSERT INTO "SERVICE_DOMAIN_MEMBERS" VALUES(7,'yes','controller','web-services',
 INSERT INTO "SERVICE_DOMAIN_MEMBERS" VALUES(8,'no','controller','storage-services','N',2,0,'','');
 INSERT INTO "SERVICE_DOMAIN_MEMBERS" VALUES(9,'no','controller','storage-monitoring-services','N + M',1,1,'','');
 INSERT INTO "SERVICE_DOMAIN_MEMBERS" VALUES(10,'no','controller','distributed-cloud-services','N + M',1,1,'controller-aggregate','');
+INSERT INTO "SERVICE_DOMAIN_MEMBERS" VALUES(11,'no','controller','admin-services','N + M',1,1,'controller-aggregate','directory-services');
 CREATE TABLE SERVICE_DOMAIN_NEIGHBORS ( ID INTEGER PRIMARY KEY AUTOINCREMENT, NAME CHAR(32), SERVICE_DOMAIN CHAR(32), ORCHESTRATION CHAR(32), DESIGNATION CHAR(32), PRIORITY INT, HELLO_INTERVAL INT, DEAD_INTERVAL INT, WAIT_INTERVAL INT, EXCHANGE_INTERVAL INT, STATE CHAR(32), GENERATION INT);
 CREATE TABLE SERVICE_DOMAIN_ASSIGNMENTS ( ID INTEGER PRIMARY KEY AUTOINCREMENT, UUID CHAR(42), NAME CHAR(32), NODE_NAME CHAR(32), SERVICE_GROUP_NAME CHAR(32), DESIRED_STATE CHAR(32), STATE CHAR(32), STATUS CHAR(32), CONDITION CHAR(32) );
 CREATE TABLE SERVICE_GROUPS ( ID INTEGER PRIMARY KEY AUTOINCREMENT, PROVISIONED CHAR(32), NAME CHAR(32), AUTO_RECOVER CHAR(32), CORE CHAR(32), DESIRED_STATE CHAR(32), STATE CHAR(32), STATUS CHAR(32), CONDITION CHAR(32), FAILURE_DEBOUNCE INT, FATAL_ERROR_REBOOT CHAR(32) );
@@ -32,6 +34,7 @@ INSERT INTO "SERVICE_GROUPS" VALUES(7,'no','storage-services','no','no','initial
 INSERT INTO "SERVICE_GROUPS" VALUES(8,'no','storage-monitoring-services','no','no','initial','initial','','',120000,'no');
 INSERT INTO "SERVICE_GROUPS" VALUES(9,'yes','vim-services','no','no','initial','initial','','',300000,'yes');
 INSERT INTO "SERVICE_GROUPS" VALUES(10,'no','distributed-cloud-services','no','no','initial','initial','','',300000,'yes');
+INSERT INTO "SERVICE_GROUPS" VALUES(11,'no','admin-services','no','no','initial','initial','','',300000,'no');
 CREATE TABLE SERVICE_GROUP_MEMBERS ( ID INTEGER PRIMARY KEY AUTOINCREMENT, PROVISIONED CHAR(32), NAME CHAR(32), SERVICE_NAME CHAR(32), SERVICE_FAILURE_IMPACT CHAR(32));
 INSERT INTO "SERVICE_GROUP_MEMBERS" VALUES(1,'yes','oam-services','oam-ip','critical');
 INSERT INTO "SERVICE_GROUP_MEMBERS" VALUES(2,'yes','controller-services','management-ip','critical');
@@ -95,6 +98,7 @@ INSERT INTO "SERVICE_GROUP_MEMBERS" SELECT MAX(id) + 1,'no','distributed-cloud-s
 INSERT INTO "SERVICE_GROUP_MEMBERS" SELECT MAX(id) + 1,'no','distributed-cloud-services','dcmanager-orchestrator','critical' FROM "SERVICE_GROUP_MEMBERS";
 INSERT INTO "SERVICE_GROUP_MEMBERS" SELECT MAX(id) + 1,'no','distributed-cloud-services','dcmanager-audit-worker','critical' FROM "SERVICE_GROUP_MEMBERS";
 INSERT INTO "SERVICE_GROUP_MEMBERS" SELECT MAX(id) + 1,'no','distributed-cloud-services','dcmanager-state','major' FROM "SERVICE_GROUP_MEMBERS";
+INSERT INTO "SERVICE_GROUP_MEMBERS" SELECT MAX(id) + 1,'no','admin-services','admin-ip','critical' FROM "SERVICE_GROUP_MEMBERS";
 CREATE TABLE SERVICES ( ID INTEGER PRIMARY KEY AUTOINCREMENT, PROVISIONED CHAR(32), NAME CHAR(32), DESIRED_STATE CHAR(32), STATE CHAR(32), STATUS CHAR(32), CONDITION CHAR(32), MAX_FAILURES INT, FAIL_COUNTDOWN INT, FAIL_COUNTDOWN_INTERVAL INT, MAX_ACTION_FAILURES INT, MAX_TRANSITION_FAILURES INT, PID_FILE CHAR(256) );
 INSERT INTO "SERVICES" VALUES(1,'yes','oam-ip','initial','initial','none','none',2,1,90000,4,16,'');
 INSERT INTO "SERVICES" VALUES(2,'yes','management-ip','initial','initial','none','none',2,1,90000,4,16,'');
@@ -158,6 +162,7 @@ INSERT INTO "SERVICES" SELECT MAX(id) + 1,'no','dcdbsync-openstack-api','initial
 INSERT INTO "SERVICES" SELECT MAX(id) + 1,'no','dcmanager-orchestrator','initial','initial','none','none',2,1,90000,4,16,'/var/run/resource-agents/dcmanager-orchestrator.pid' FROM "SERVICES";
 INSERT INTO "SERVICES" SELECT MAX(id) + 1,'no','dcmanager-audit-worker','initial','initial','none','none',2,1,90000,4,16,'/var/run/resource-agents/dcmanager-audit-worker.pid' FROM "SERVICES";
 INSERT INTO "SERVICES" SELECT MAX(id) + 1,'no','dcmanager-state','initial','initial','none','none',2,1,90000,4,16,'/var/run/resource-agents/dcmanager-state.pid' FROM "SERVICES";
+INSERT INTO "SERVICES" SELECT MAX(id) + 1,'no','admin-ip','initial','initial','none','none',2,1,90000,4,16,'' FROM "SERVICES";
 CREATE TABLE SERVICE_HEARTBEAT ( ID INTEGER PRIMARY KEY AUTOINCREMENT, PROVISIONED CHAR(32), NAME CHAR(32), TYPE CHAR(32), SRC_ADDRESS CHAR(108), SRC_PORT INT, DST_ADDRESS CHAR(108), DST_PORT INT, MESSAGE CHAR(256), INTERVAL_IN_MS INT, MISSED_WARN INT, MISSED_DEGRADE INT, MISSED_FAIL INT, STATE CHAR(32), MISSED INT, HEARTBEAT_TIMER_ID INT, HEARTBEAT_SOCKET INT );
 CREATE TABLE SERVICE_DEPENDENCY ( DEPENDENCY_TYPE CHAR(32), SERVICE_NAME CHAR(32), STATE CHAR(32), ACTION CHAR(32), DEPENDENT CHAR(32), DEPENDENT_STATE CHAR(32), PRIMARY KEY (DEPENDENCY_TYPE, SERVICE_NAME, STATE, ACTION, DEPENDENT));
 INSERT INTO "SERVICE_DEPENDENCY" VALUES('action','oam-ip','not-applicable','enable','management-ip','enabled-active');
@@ -576,6 +581,10 @@ INSERT INTO "SERVICE_ACTIONS" VALUES('dcmanager-state','enable','ocf-script','op
 INSERT INTO "SERVICE_ACTIONS" VALUES('dcmanager-state','disable','ocf-script','openstack','dcmanager-state','stop','',1,1,1,20,'');
 INSERT INTO "SERVICE_ACTIONS" VALUES('dcmanager-state','audit-enabled','ocf-script','openstack','dcmanager-state','monitor','',2,2,2,20,5);
 INSERT INTO "SERVICE_ACTIONS" VALUES('dcmanager-state','audit-disabled','ocf-script','openstack','dcmanager-state','monitor','',0,0,0,20,5);
+INSERT INTO "SERVICE_ACTIONS" VALUES('admin-ip','enable','ocf-script','heartbeat','IPaddr2','start','',2,2,2,20,'');
+INSERT INTO "SERVICE_ACTIONS" VALUES('admin-ip','disable','ocf-script','heartbeat','IPaddr2','stop','',1,1,1,20,'');
+INSERT INTO "SERVICE_ACTIONS" VALUES('admin-ip','audit-enabled','ocf-script','heartbeat','IPaddr2','monitor','',2,2,2,20,5);
+INSERT INTO "SERVICE_ACTIONS" VALUES('admin-ip','audit-disabled','ocf-script','heartbeat','IPaddr2','monitor','',0,0,0,20,5);
 CREATE TABLE SERVICE_ACTION_RESULTS ( PLUGIN_TYPE CHAR(32), PLUGIN_NAME CHAR(80), PLUGIN_COMMAND CHAR(80), PLUGIN_EXIT_CODE CHAR(32), ACTION_RESULT CHAR(32), SERVICE_STATE CHAR(32), SERVICE_STATUS CHAR(32), SERVICE_CONDITION CHAR(32), PRIMARY KEY (PLUGIN_TYPE, PLUGIN_NAME, PLUGIN_COMMAND, PLUGIN_EXIT_CODE));
 INSERT INTO "SERVICE_ACTION_RESULTS" VALUES('lsb-script','default','status','0','success','enabled-active','unknown','unknown');
 INSERT INTO "SERVICE_ACTION_RESULTS" VALUES('lsb-script','default','status','1','success','disabled','unknown','unknown');

service-mgmt/sm/src/sm_api.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014-2019 Wind River Systems, Inc.
+// Copyright (c) 2014-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -32,7 +32,8 @@
 #define SM_API_MSG_SKIP_DEP_CHECK                   "skip-dep"
 #define SM_API_MSG_TYPE_PROVISION_SERVICE           "PROVISION_SERVICE"
 #define SM_API_MSG_TYPE_DEPROVISION_SERVICE         "DEPROVISION_SERVICE"
-
+#define SM_API_MSG_TYPE_PROVISION_SERVICE_DOMAIN_INTERFACE "PROVISION_SERVICE_DOMAIN_INTERFACE"
+#define SM_API_MSG_TYPE_DEPROVISION_SERVICE_DOMAIN_INTERFACE "DEPROVISION_SERVICE_DOMAIN_INTERFACE"
 
 #define SM_API_MSG_NODE_ACTION_UNKNOWN              "unknown"
 #define SM_API_MSG_NODE_ACTION_LOCK                 "lock"
@@ -67,6 +68,9 @@
 #define SM_API_MSG_NODE_AVAIL_FIELD                 9
 
 #define SM_API_MSG_SERVICE_NAME_FIELD               5
+// The service domain interface provision / deprovision msg is at the same
+// message offset as the service name field.
+#define SM_API_MSG_SERVICE_DOMAIN_NAME_FIELD        5
 #define SM_API_MSG_PARAM                            6
 
 #define SM_API_MAX_MSG_SIZE                         2048
@@ -379,6 +383,8 @@ static void sm_api_dispatch( int selobj, int64_t user_data )
     char* node_name;
     char* service_name;
     char* service_group_name;
+    char* service_domain;
+    char* service_domain_interface;
     gchar** params;
     int bytes_read;
     SmNodeSetActionT action;
@@ -445,7 +451,7 @@ static void sm_api_dispatch( int selobj, int64_t user_data )
         DPRINTFE( "Missing message-type field in received message." );
         goto ERROR;
     }
-    
+
     if( 0 == strcmp( SM_API_MSG_TYPE_SET_NODE, params[SM_API_MSG_TYPE_FIELD] ) )
     {
         if( params[SM_API_MSG_ORIGIN_FIELD] == NULL )
@@ -598,6 +604,72 @@ static void sm_api_dispatch( int selobj, int64_t user_data )
         {
             _callbacks.deprovision_service( service_group_name, service_name, seqno);
         }
+    }
+    else if( 0 == strcmp( SM_API_MSG_TYPE_PROVISION_SERVICE_DOMAIN_INTERFACE,
+                          params[SM_API_MSG_TYPE_FIELD] ) )
+    {
+        if( params[SM_API_MSG_ORIGIN_FIELD] == NULL )
+        {
+            DPRINTFE( "Missing origin field in received message." );
+            goto ERROR;
+        }
+
+        if( params[SM_API_MSG_SERVICE_DOMAIN_NAME_FIELD] == NULL )
+        {
+            DPRINTFE( "Missing service-domain field in received message." );
+            goto ERROR;
+        }
+        service_domain  = (char*) params[SM_API_MSG_SERVICE_DOMAIN_NAME_FIELD];
+
+        service_domain_interface = (char*) params[SM_API_MSG_PARAM];
+        if ( NULL == service_domain_interface )
+        {
+            DPRINTFE( "Missing service domain interface parameter." );
+            goto ERROR;
+        }
+        else
+        {
+            DPRINTFI( "Provision service domain interface %s:%s",
+                      service_domain, service_domain_interface);
+        }
+
+        if( NULL != _callbacks.provision_service_domain_interface )
+        {
+            _callbacks.provision_service_domain_interface( service_domain, service_domain_interface, seqno);
+        }
+    }
+    else if( 0 == strcmp( SM_API_MSG_TYPE_DEPROVISION_SERVICE_DOMAIN_INTERFACE,
+                          params[SM_API_MSG_TYPE_FIELD] ) )
+    {
+        if( params[SM_API_MSG_ORIGIN_FIELD] == NULL )
+        {
+            DPRINTFE( "Missing origin field in received message." );
+            goto ERROR;
+        }
+
+        if( params[SM_API_MSG_SERVICE_DOMAIN_NAME_FIELD] == NULL )
+        {
+            DPRINTFE( "Missing service-domain field in received message." );
+            goto ERROR;
+        }
+        service_domain  = (char*) params[SM_API_MSG_SERVICE_DOMAIN_NAME_FIELD];
+
+        service_domain_interface = (char*) params[SM_API_MSG_PARAM];
+        if ( NULL == service_domain_interface )
+        {
+            DPRINTFE( "Missing service domain interface parameter." );
+            goto ERROR;
+        }
+        else
+        {
+            DPRINTFI( "Deprovision service domain interface %s:%s",
+                      service_domain, service_domain_interface);
+        }
+
+        if( NULL != _callbacks.deprovision_service_domain_interface )
+        {
+            _callbacks.deprovision_service_domain_interface( service_domain, service_domain_interface, seqno);
+        }
     } else {
         DPRINTFE( "Unknown/unsupported message-type (%s) received.",
                   params[SM_API_MSG_TYPE_FIELD] );

service-mgmt/sm/src/sm_api.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014-2019 Wind River Systems, Inc.
+// Copyright (c) 2014-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -37,12 +37,23 @@ typedef void (*SmApiProvisionServiceCallbackT) (char service_group_name[],
 typedef void (*SmApiDeprovisionServiceCallbackT) (char service_group_name[],
         char service_name[], int seqno);
 
+typedef void (*SmApiProvisionServiceDomainInterfaceCallbackT) (char service_domain[],
+        char interface_name[], int seqno);
+
+typedef void (*SmApiDeprovisionServiceDomainInterfaceCallbackT) (char service_domain[],
+        char interface_name[], int seqno);
+
+typedef void (*SmApiConfigureServiceDomainInterfaceCallbackT) (char service_domain[],
+        char service_domain_interface[], int seqno);
+
 typedef struct
 {
     SmApiNodeSetCallbackT node_set;
     SmApiServiceRestartCallbackT service_restart;
     SmApiProvisionServiceCallbackT provision_service;
     SmApiDeprovisionServiceCallbackT deprovision_service;
+    SmApiProvisionServiceDomainInterfaceCallbackT provision_service_domain_interface;
+    SmApiDeprovisionServiceDomainInterfaceCallbackT deprovision_service_domain_interface;
 } SmApiCallbacksT;
 
 // ****************************************************************************

service-mgmt/sm/src/sm_configure.cpp

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2019 Wind River Systems, Inc.
+// Copyright (c) 2019-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -20,6 +20,10 @@
 #include "sm_service_domain_assignment_table.h"
 #include "sm_timer.h"
 #include "sm_service_fsm.h"
+#include "sm_service_domain_interface_api.h"
+#include "sm_service_domain_interface_table.h"
+#include "sm_heartbeat.h"
+#include "sm_msg.h"
 
 
 // The behavior of a service is configured in multiple (>1) service groups has
@@ -184,3 +188,82 @@ SmErrorT sm_deprovision_service( char service_group_name[], char service_name[]
     return SM_OKAY;
 }
 // ****************************************************************************
+
+// ****************************************************************************
+// sm configure: provision service domain interface
+// ================================
+extern SmErrorT sm_provision_service_domain_interface( char service_domain_name[], char interface_name[] )
+{
+    SmErrorT error;
+
+    DPRINTFI("Start provisioning service domain interface %s:%s...", service_domain_name, interface_name);
+
+    error = sm_service_domain_interface_table_load();
+    if(SM_OKAY != error)
+    {
+        DPRINTFE("Failed to reload domain interface table");
+        return error;
+    }
+
+    SmServiceDomainInterfaceT* interface = sm_service_domain_interface_table_read(service_domain_name, interface_name);
+    if(NULL == interface)
+    {
+        DPRINTFI("Service domain interface %s not found", interface_name);
+        return SM_NOT_FOUND;
+    }
+
+    error = sm_service_domain_interface_api_provisioned(interface);
+    if(SM_OKAY != error)
+    {
+        DPRINTFE("Failed to provision service domain interface %s:%s", service_domain_name, interface_name);
+        return error;
+    }
+
+    error = sm_service_domain_interface_table_load();
+    if(SM_OKAY != error)
+    {
+        DPRINTFE("Failed to reload domain interface table");
+        return error;
+    }
+
+    error = sm_service_domain_interface_api_audit();
+    if(SM_OKAY != error)
+    {
+        DPRINTFE( "Failed to audit interfaces, error=%s.",
+                  sm_error_str( error ) );
+    }
+
+    DPRINTFI("%s:%s is provisioned successfully", service_domain_name, interface_name);
+    return SM_OKAY;
+
+}
+// ****************************************************************************
+
+// ****************************************************************************
+// sm configure: deprovision service domain interface
+// ===============================
+SmErrorT sm_deprovision_service_domain_interface( char service_domain_name[], char interface_name[] )
+{
+    SmErrorT error;
+    DPRINTFI("Start deprovisioning service domain interface %s:%s ...", service_domain_name, interface_name);
+
+    SmServiceDomainInterfaceT* interface = sm_service_domain_interface_table_read(service_domain_name, interface_name);
+    if(NULL == interface)
+    {
+        DPRINTFI("Service domain interface %s not found", interface_name);
+        return SM_NOT_FOUND;
+    }
+
+    error = sm_service_domain_interface_api_deprovisioned(interface);
+    if(SM_OKAY != error)
+    {
+        DPRINTFE("Failed to deprovision service domain interface %s:%s", service_domain_name, interface_name);
+        return error;
+    }
+
+    DPRINTFI("%s:%s is deprovisioned successfully", service_domain_name, interface_name);
+
+    return SM_OKAY;
+}
+// ****************************************************************************
+

service-mgmt/sm/src/sm_configure.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2019 Wind River Systems, Inc.
+// Copyright (c) 2019-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -24,6 +24,18 @@ extern SmErrorT sm_provision_service( char service_group_name[], char service_na
 extern SmErrorT sm_deprovision_service( char service_group_name[], char service_name[] );
 // ****************************************************************************
 
+// ****************************************************************************
+// system config: provision_service_domain_interface
+// ================================
+extern SmErrorT sm_provision_service_domain_interface( char service_domain_name[], char interface_name[] );
+// ****************************************************************************
+
+// ****************************************************************************
+// system config: deprovision_service_domain_interface
+// ================================
+extern SmErrorT sm_deprovision_service_domain_interface( char service_domain_name[], char interface_name[] );
+// ****************************************************************************
+
 
 #ifdef __cplusplus
 }

service-mgmt/sm/src/sm_failover.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2017 Wind River Systems, Inc.
+// Copyright (c) 2017-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -137,6 +137,7 @@ static unsigned int _total_interfaces;
 static SmFailoverInterfaceInfo* _oam_interface_info = NULL;
 static SmFailoverInterfaceInfo* _mgmt_interface_info = NULL;
 static SmFailoverInterfaceInfo* _cluster_host_interface_info = NULL;
+static SmFailoverInterfaceInfo* _admin_interface_info = NULL;
 static SmFailoverInterfaceInfo _peer_if_list[SM_INTERFACE_MAX];
 static pthread_mutex_t _mutex;
 static SmDbHandleT* _sm_db_handle = NULL;
@@ -193,6 +194,19 @@ static void sm_failover_interface_check( void* user_data[],
                 DPRINTFE( "Failed to look up cluster-host interface, error=%s.",
                       sm_error_str(error) );
             }
+        }else if( 0 == strcmp(SM_SERVICE_DOMAIN_ADMIN_INTERFACE, interface->service_domain_interface ))
+        {
+            SmErrorT error = sm_node_utils_get_admin_interface(interface->interface_name);
+            if(SM_OKAY == error)
+            {
+                _my_if_list[*count].set_interface(interface);
+                _admin_interface_info = _my_if_list + (*count);
+                (*count) ++;
+            } else if (SM_NOT_FOUND != error )
+            {
+                DPRINTFE( "Failed to look up admin interface, error=%s.",
+                      sm_error_str(error) );
+            }
         }else
         {
             DPRINTFE( "Unknown interface %s", interface->interface_name );
@@ -483,6 +497,21 @@ bool is_cluster_host_interface_configured()
 }
 // ****************************************************************************
 
+// ****************************************************************************
+// Failover - is admin interface configured
+// ===============================================
+bool is_admin_interface_configured()
+{
+    if (NULL == _admin_interface_info ||
+        _admin_interface_info->get_interface()->service_domain_interface[0] == '\0' )
+    {
+        return false;
+    }
+
+    return true;
+}
+// ****************************************************************************
+
 // ****************************************************************************
 // Failover - get interface state
 // ==================
@@ -491,6 +520,7 @@ int sm_failover_get_if_state()
     SmFailoverInterfaceStateT mgmt_state = _mgmt_interface_info->get_state();
     SmFailoverInterfaceStateT oam_state = _oam_interface_info->get_state();
     SmFailoverInterfaceStateT cluster_host_state;
+    SmFailoverInterfaceStateT admin_state;
     int if_state_flag  = 0;
     if ( is_cluster_host_interface_configured() )
     {
@@ -505,6 +535,19 @@ int sm_failover_get_if_state()
         }
     }
 
+    if ( is_admin_interface_configured() )
+    {
+        admin_state = _admin_interface_info->get_state();
+        if( SM_FAILOVER_INTERFACE_OK == admin_state )
+        {
+            if_state_flag |= SM_FAILOVER_HEARTBEAT_ALIVE;
+        }
+        else if ( SM_FAILOVER_INTERFACE_DOWN == admin_state )
+        {
+            if_state_flag |= SM_FAILOVER_ADMIN_DOWN;
+        }
+    }
+
     if( SM_FAILOVER_INTERFACE_OK == mgmt_state )
     {
         if_state_flag |= SM_FAILOVER_HEARTBEAT_ALIVE;
@@ -1018,6 +1061,13 @@ void sm_failover_audit()
         {
             event_data.set_interface_state(SM_INTERFACE_CLUSTER_HOST, SM_FAILOVER_INTERFACE_UNKNOWN);
         }
+        if( NULL != _admin_interface_info)
+        {
+            event_data.set_interface_state(SM_INTERFACE_ADMIN, _admin_interface_info->get_state());
+        }else
+        {
+            event_data.set_interface_state(SM_INTERFACE_ADMIN, SM_FAILOVER_INTERFACE_UNKNOWN);
+        }
         SmFailoverFSM::get_fsm().send_event(SM_FAILOVER_EVENT_IF_STATE_CHANGED, &event_data);
         _if_state_changed = false;
     }
@@ -1186,6 +1236,9 @@ SmFailoverInterfaceStateT sm_failover_get_interface_info(SmInterfaceTypeT interf
         case SM_INTERFACE_OAM:
             res = _oam_interface_info;
             break;
+        case SM_INTERFACE_ADMIN:
+            res = _admin_interface_info;
+            break;
         case SM_INTERFACE_UNKNOWN:
             break;
     }
@@ -1471,6 +1524,7 @@ void dump_interfaces_state(FILE* fp)
     dump_if_state(fp, _oam_interface_info,   "  OAM");
     dump_if_state(fp, _mgmt_interface_info,  " MGMT");
     dump_if_state(fp, _cluster_host_interface_info, "CLUSTER-HOST");
+    dump_if_state(fp, _cluster_host_interface_info, "ADMIN");
 }
 
 void dump_peer_if_state(FILE* fp)

service-mgmt/sm/src/sm_failover.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2017-2020 Wind River Systems, Inc.
+// Copyright (c) 2017-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -36,6 +36,7 @@ typedef enum
     SM_FAILOVER_OAM_DOWN = 4,
     SM_FAILOVER_HEARTBEAT_ALIVE = 8,
     SM_FAILOVER_HELLO_MSG_ALIVE = 16,
+    SM_FAILOVER_ADMIN_DOWN = 32,
     SM_FAILOVER_PEER_DISABLED = 0x4000,
 }SmFailoverCommFaultBitFlagT;
 

service-mgmt/sm/src/sm_failover_fail_pending_state.cpp

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2018 Wind River Systems, Inc.
+// Copyright (c) 2018-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -235,6 +235,7 @@ SmErrorT SmFailoverFailPendingState::event_handler(SmFailoverEventT event, const
                 {
                     interfaces_to_check[0] = SM_INTERFACE_MGMT;
                     interfaces_to_check[1] = SM_INTERFACE_CLUSTER_HOST;
+                    interfaces_to_check[2] = SM_INTERFACE_ADMIN;
                 }
                 for(int i = 0; interfaces_to_check[i] != SM_INTERFACE_UNKNOWN; i ++)
                 {

service-mgmt/sm/src/sm_failover_failed_state.cpp

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2018-2021 Wind River Systems, Inc.
+// Copyright (c) 2018-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -28,6 +28,7 @@
 #include "sm_cluster_hbs_info_msg.h"
 
 extern bool is_cluster_host_interface_configured( void );
+extern bool is_admin_interface_configured( void );
 
 // Failover Failed Recovery Audit period = 5 seconds
 static const int FAILED_STATE_AUDIT_PERIOD = 5000;
@@ -147,9 +148,10 @@ static bool sm_failover_failed_recovery_criteria_met( void )
 {
     bool criteria_met = false ;
 
-    SmFailoverInterfaceStateT oam_state, mgmt_state, cluster_host_state;
+    SmFailoverInterfaceStateT oam_state, mgmt_state, cluster_host_state, admin_state;
     oam_state = sm_failover_get_interface_info(SM_INTERFACE_OAM);
     mgmt_state = sm_failover_get_interface_info(SM_INTERFACE_MGMT);
+    admin_state = sm_failover_get_interface_info(SM_INTERFACE_ADMIN);
 
     const SmClusterHbsStateT& cluster_hbs_state = SmClusterHbsInfoMsg::get_current_state();
     int peer_controller_index = SmClusterHbsInfoMsg::get_peer_controller_index();
@@ -172,12 +174,23 @@ static bool sm_failover_failed_recovery_criteria_met( void )
         {
             criteria_met = true ;
         }
+
+        if ( criteria_met && is_admin_interface_configured() )
+        {
+            criteria_met = false ;
+            admin_state = sm_failover_get_interface_info(SM_INTERFACE_ADMIN);
+            if (( admin_state == SM_FAILOVER_INTERFACE_OK ) || ( admin_state == SM_FAILOVER_INTERFACE_MISSING_HEARTBEAT ))
+            {
+                criteria_met = true;
+            }
+        }
     }
 
-    DPRINTFI("Oam:%s ; Mgmt:%s ; Cluster:%s ; recovery criteria met: %s",
+    DPRINTFI("Oam:%s ; Mgmt:%s ; Cluster:%s ; Admin:%s recovery criteria met: %s",
               sm_failover_interface_state_str(oam_state),
               sm_failover_interface_state_str(mgmt_state),
               sm_failover_interface_state_str(cluster_host_state),
+              sm_failover_interface_state_str(admin_state),
                criteria_met ? "Yes" : "No");
 
     return (criteria_met);

service-mgmt/sm/src/sm_failover_fsm.cpp

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2018 Wind River Systems, Inc.
+// Copyright (c) 2018-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -36,6 +36,9 @@ void SmIFStateChangedEventData::set_interface_state(
         case SM_INTERFACE_CLUSTER_HOST:
             _cluster_host_state = interface_state;
             break;
+        case SM_INTERFACE_ADMIN:
+            _admin_state = interface_state;
+            break;
         default:
             DPRINTFE("Runtime error: invalid interface type %d", interface_type);
     }
@@ -51,6 +54,8 @@ SmFailoverInterfaceStateT SmIFStateChangedEventData::get_interface_state(SmInter
             return _mgmt_state;
         case SM_INTERFACE_CLUSTER_HOST:
             return _cluster_host_state;
+        case SM_INTERFACE_ADMIN:
+            return _admin_state;
         default:
             DPRINTFE("Runtime error: invalid interface type %d", interface_type);
             return SM_FAILOVER_INTERFACE_UNKNOWN;

service-mgmt/sm/src/sm_failover_fsm.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2018 Wind River Systems, Inc.
+// Copyright (c) 2018-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -73,6 +73,7 @@ class SmIFStateChangedEventData: public ISmFSMEventData
         SmFailoverInterfaceStateT _mgmt_state;
         SmFailoverInterfaceStateT _cluster_host_state;
         SmFailoverInterfaceStateT _oam_state;
+        SmFailoverInterfaceStateT _admin_state;
 };
 
 #endif //__SM_FAILOVER_FSM_H__

service-mgmt/sm/src/sm_failover_normal_state.cpp

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2018 Wind River Systems, Inc.
+// Copyright (c) 2018-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -25,13 +25,15 @@ SmErrorT SmFailoverNormalState::event_handler(SmFailoverEventT event, const ISmF
             {
                 const SmIFStateChangedEventData* data = (const SmIFStateChangedEventData*) event_data;
 
-                SmFailoverInterfaceStateT oam_state, mgmt_state, cluster_host_state;
+                SmFailoverInterfaceStateT oam_state, mgmt_state, cluster_host_state, admin_state;
                 oam_state = data->get_interface_state(SM_INTERFACE_OAM);
                 mgmt_state = data->get_interface_state(SM_INTERFACE_MGMT);
                 cluster_host_state = data->get_interface_state(SM_INTERFACE_CLUSTER_HOST);
+                admin_state = data->get_interface_state(SM_INTERFACE_ADMIN);
                 if(oam_state != SM_FAILOVER_INTERFACE_OK ||
                    mgmt_state != SM_FAILOVER_INTERFACE_OK ||
-                   (cluster_host_state != SM_FAILOVER_INTERFACE_OK && cluster_host_state != SM_FAILOVER_INTERFACE_UNKNOWN))
+                   (cluster_host_state != SM_FAILOVER_INTERFACE_OK && cluster_host_state != SM_FAILOVER_INTERFACE_UNKNOWN) ||
+                   (admin_state != SM_FAILOVER_INTERFACE_OK && admin_state != SM_FAILOVER_INTERFACE_UNKNOWN))
                 {
                     this->fsm.set_state(SM_FAILOVER_STATE_FAIL_PENDING);
                 }

service-mgmt/sm/src/sm_failover_ss.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2018 Wind River Systems, Inc.
+// Copyright (c) 2018-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -268,10 +268,12 @@ SmErrorT _get_system_status(SmSystemStatusT& sys_status, char host_name[], char
     sys_status.host_status.mgmt_state = sm_failover_get_interface_info(SM_INTERFACE_MGMT);
     sys_status.host_status.cluster_host_state = sm_failover_get_interface_info(SM_INTERFACE_CLUSTER_HOST);
     sys_status.host_status.oam_state = sm_failover_get_interface_info(SM_INTERFACE_OAM);
+    sys_status.host_status.admin_state = sm_failover_get_interface_info(SM_INTERFACE_ADMIN);
 
     if(SM_FAILOVER_INTERFACE_OK == sys_status.host_status.mgmt_state ||
        SM_FAILOVER_INTERFACE_OK == sys_status.host_status.oam_state ||
-       SM_FAILOVER_INTERFACE_OK == sys_status.host_status.cluster_host_state)
+       SM_FAILOVER_INTERFACE_OK == sys_status.host_status.cluster_host_state ||
+       SM_FAILOVER_INTERFACE_OK == sys_status.host_status.admin_state)
     {
         sys_status.heartbeat_state = SM_HEARTBEAT_OK;
     }else
@@ -545,7 +547,9 @@ SmErrorT _get_survivor_dc(const SmSystemStatusT& system_status, SmSystemFailover
     {
         if(system_status.host_status.mgmt_state == SM_FAILOVER_INTERFACE_DOWN &&
             (system_status.host_status.cluster_host_state == SM_FAILOVER_INTERFACE_DOWN ||
-            system_status.host_status.cluster_host_state == SM_FAILOVER_INTERFACE_UNKNOWN))
+            system_status.host_status.cluster_host_state == SM_FAILOVER_INTERFACE_UNKNOWN) &&
+            (system_status.host_status.admin_state == SM_FAILOVER_INTERFACE_DOWN ||
+            system_status.host_status.admin_state == SM_FAILOVER_INTERFACE_UNKNOWN))
         {
             if(SM_FAILOVER_INTERFACE_DOWN == system_status.host_status.oam_state)
             {

service-mgmt/sm/src/sm_failover_ss.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2018 Wind River Systems, Inc.
+// Copyright (c) 2018-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -18,6 +18,7 @@ typedef struct
     SmFailoverInterfaceStateT mgmt_state;
     SmFailoverInterfaceStateT cluster_host_state;
     SmFailoverInterfaceStateT oam_state;
+    SmFailoverInterfaceStateT admin_state;
     SmNodeScheduleStateT current_schedule_state;
 }SmNodeStatusT;
 

service-mgmt/sm/src/sm_heartbeat_thread.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014-2017 Wind River Systems, Inc.
+// Copyright (c) 2014-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -225,7 +225,8 @@ SmErrorT sm_heartbeat_thread_add_interface( const SmServiceDomainInterfaceT& dom
         interface->multicast_socket = -1;
 
         interface->interface_type = sm_get_interface_type(domain_interface.service_domain_interface);
-        if ( SM_INTERFACE_OAM == interface->interface_type )
+        if (( SM_INTERFACE_OAM == interface->interface_type ) ||
+            ( SM_INTERFACE_ADMIN == interface->interface_type ))
         {
             interface->method = SEND_UNICAST;
         } else
@@ -491,6 +492,11 @@ static bool sm_heartbeat_peer_alarm_on_interface( SmTimerIdT timer_id,
     {
         snprintf( network_type, sizeof(network_type), SM_CLUSTER_HOST_INTERFACE_NAME );
 
+    } else if( 0 == strcmp( SM_SERVICE_DOMAIN_ADMIN_INTERFACE,
+                            interface->service_domain_interface ) )
+    {
+        snprintf( network_type, sizeof(network_type), SM_ADMIN_INTERFACE_NAME );
+
     } else {
         snprintf( network_type, sizeof(network_type), "unknown" );
     }
@@ -829,7 +835,8 @@ static bool sm_heartbeat_alive_timer( SmTimerIdT timer_id, int64_t user_data )
                 continue;
             }
 
-            if  ( SM_INTERFACE_OAM == interface->interface_type )
+            if (( SM_INTERFACE_OAM == interface->interface_type ) ||
+                ( SM_INTERFACE_ADMIN == interface->interface_type ))
             {
                 error = sm_heartbeat_msg_close_sockets(
                                             &(interface->unicast_socket) );
@@ -847,7 +854,8 @@ static bool sm_heartbeat_alive_timer( SmTimerIdT timer_id, int64_t user_data )
                 continue;
             }
 
-            if  ( SM_INTERFACE_OAM == interface->interface_type )
+            if (( SM_INTERFACE_OAM == interface->interface_type ) ||
+                ( SM_INTERFACE_ADMIN == interface->interface_type ))
             {
                 error = sm_heartbeat_msg_open_sockets( interface->network_type,
                         &(interface->network_address),
@@ -906,7 +914,8 @@ static bool sm_heartbeat_alive_timer( SmTimerIdT timer_id, int64_t user_data )
             DPRINTFD( "Multicast not configured for interface %s", interface->interface_name );
         }
 
-        if  ( SM_INTERFACE_OAM == interface->interface_type )
+        if (( SM_INTERFACE_OAM == interface->interface_type ) ||
+            ( SM_INTERFACE_ADMIN == interface->interface_type ))
         {
             error = sm_heartbeat_msg_send_alive( interface->network_type, _node_name,
                         &(interface->network_address), &(interface->network_peer_address),

service-mgmt/sm/src/sm_main_event_handler.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014-2019 Wind River Systems, Inc.
+// Copyright (c) 2014-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -252,6 +252,26 @@ static void sm_main_event_handler_api_deprovision_service_callback(
     sm_deprovision_service(service_group_name, service_name);
 }
 
+// ****************************************************************************
+// Main Event Handler - Provision Service Domain Interface API Event Callback
+// ======================================================
+static void sm_main_event_handler_api_provision_service_domain_interface_callback(
+    char service_domain_name[], char interface_name[], int seqno )
+{
+    sm_provision_service_domain_interface(service_domain_name, interface_name);
+}
+// ****************************************************************************
+
+// ****************************************************************************
+// Main Event Handler - Provision Service Domain Interface API Event Callback
+// ======================================================
+static void sm_main_event_handler_api_deprovision_service_domain_interface_callback(
+    char service_domain_name[], char interface_name[], int seqno )
+{
+    sm_deprovision_service_domain_interface(service_domain_name, interface_name);
+}
+// ****************************************************************************
+
 // ****************************************************************************
 // Main Event Handler - Notify API Service Event Callback
 // ======================================================
@@ -382,6 +402,10 @@ SmErrorT sm_main_event_handler_initialize( void )
         = sm_main_event_handler_api_provision_service_callback;
     _api_callbacks.deprovision_service
         = sm_main_event_handler_api_deprovision_service_callback;
+    _api_callbacks.provision_service_domain_interface
+        = sm_main_event_handler_api_provision_service_domain_interface_callback;
+    _api_callbacks.deprovision_service_domain_interface
+        = sm_main_event_handler_api_deprovision_service_domain_interface_callback;
 
     error = sm_api_register_callbacks( &_api_callbacks );
     if( SM_OKAY != error )

service-mgmt/sm/src/sm_service_domain_interface_api.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014-2018 Wind River Systems, Inc.
+// Copyright (c) 2014-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -85,7 +85,24 @@ static void sm_service_domain_interface_api_get_hw_interface(
                       sm_error_str(error) );
             return;
         }
-    }  
+    } else if( 0 == strcmp( SM_SERVICE_DOMAIN_ADMIN_INTERFACE,
+                            interface->service_domain_interface ) )
+    {
+        error = sm_node_utils_get_admin_interface( interface->interface_name );
+        if( SM_OKAY == error )
+        {
+            DPRINTFI( "Network address (%s) maps to %s interface from config, "
+                      "type=%s.", net_addr_str, interface->interface_name,
+                      interface->service_domain_interface );
+            goto PERSIST;
+        }
+        else if( SM_NOT_FOUND != error )
+        {
+            DPRINTFE( "Failed to look up admin interface, error=%s.",
+                      sm_error_str(error) );
+            return;
+        }
+    }
 
     error = sm_hw_get_if_by_network_address( &(interface->network_address),
                                              interface->interface_name );
@@ -230,6 +247,46 @@ SmErrorT sm_service_domain_interface_api_node_disabled( void )
 }
 // ****************************************************************************
 
+// ****************************************************************************
+// Service Domain Interface API - Interface Provisioned
+// ============================================
+SmErrorT sm_service_domain_interface_api_provisioned( SmServiceDomainInterfaceT* interface )
+{
+    char reason_text[SM_LOG_REASON_TEXT_MAX_CHAR];
+    SmServiceDomainInterfaceEventT event;
+    void* user_data[] = { &event, reason_text };
+
+    event = SM_SERVICE_DOMAIN_INTERFACE_EVENT_UNKNOWN;
+
+    snprintf( reason_text, sizeof(reason_text), "%s interface is enabled",
+              interface->service_domain_interface );
+
+    sm_service_domain_interface_api_send_event( user_data, interface );
+
+    return( SM_OKAY );
+}
+// ****************************************************************************
+
+// ****************************************************************************
+// Service Domain Interface API - Interface Deprovisioned
+// ============================================
+SmErrorT sm_service_domain_interface_api_deprovisioned( SmServiceDomainInterfaceT* interface )
+{
+    char reason_text[SM_LOG_REASON_TEXT_MAX_CHAR];
+    SmServiceDomainInterfaceEventT event;
+    void* user_data[] = { &event, reason_text };
+
+    event = SM_SERVICE_DOMAIN_INTERFACE_EVENT_NOT_IN_USE;
+
+    snprintf( reason_text, sizeof(reason_text), "%s interface is disabled",
+              interface->service_domain_interface );
+
+    sm_service_domain_interface_api_send_event( user_data, interface );
+
+    return( SM_OKAY );
+}
+// ****************************************************************************
+
 // ****************************************************************************
 // Service Domain Interface API - Audit
 // ====================================

service-mgmt/sm/src/sm_service_domain_interface_api.h

@@ -7,6 +7,7 @@
 #define __SM_SERVICE_DOMAIN_INTERFACE_API_H__
 
 #include "sm_types.h"
+#include "sm_service_domain_interface_table.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -30,6 +31,18 @@ extern SmErrorT sm_service_domain_interface_api_node_disabled( void );
 extern SmErrorT sm_service_domain_interface_api_audit( void );
 // ****************************************************************************
 
+// ****************************************************************************
+// Service Domain Interface API - Provision
+// ====================================
+extern SmErrorT sm_service_domain_interface_api_provisioned( SmServiceDomainInterfaceT* interface );
+// ****************************************************************************
+
+// ****************************************************************************
+// Service Domain Interface API - Deprovision
+// ====================================
+extern SmErrorT sm_service_domain_interface_api_deprovisioned( SmServiceDomainInterfaceT* interface );
+// ****************************************************************************
+
 // ****************************************************************************
 // Service Domain Interface API - Initialize 
 // =========================================

service-mgmt/sm/src/sm_service_domain_interface_not_in_use_state.c

@@ -12,6 +12,7 @@
 #include "sm_debug.h"
 #include "sm_msg.h"
 #include "sm_log.h"
+#include "sm_hw.h"
 #include "sm_service_domain_interface_fsm.h"
 
 // ****************************************************************************
@@ -52,12 +53,49 @@ SmErrorT sm_service_domain_interface_niu_state_event_handler(
     SmServiceDomainInterfaceT* interface,
     SmServiceDomainInterfaceEventT event, void* event_data[] )
 {
+    char reason_text[SM_LOG_REASON_TEXT_MAX_CHAR] = "";
+    SmErrorT error;
+    bool enabled;
+
     switch( event )
     {
         case SM_SERVICE_DOMAIN_INTERFACE_EVENT_ENABLED:
             // undefined behavior
         break;
 
+        case SM_SERVICE_DOMAIN_INTERFACE_EVENT_UNKNOWN:
+            if (SM_SERVICE_DOMAIN_INTERFACE_EVENT_NOT_IN_USE != event)
+            {
+                error = sm_hw_get_if_state( interface->interface_name, &enabled );
+                if( SM_OKAY != error )
+                {
+                   DPRINTFE( "Failed to audit hardware state of interface (%s), "
+                      "error=%s", interface->interface_name,
+                      sm_error_str( error ) );
+                   return( error );
+                }
+            }
+
+            snprintf( reason_text, sizeof(reason_text), "node and %s "
+                      "enabled", interface->interface_name );
+
+            error = sm_service_domain_interface_fsm_set_state(
+                                            interface->service_domain,
+                                            interface->service_domain_interface,
+                                            SM_INTERFACE_STATE_UNKNOWN, 
+                                            reason_text );
+            if( SM_OKAY != error )
+            {
+                DPRINTFE( "Set state (%s) of service domain (%s) interface "
+                          "(%s) failed, error=%s.",
+                            sm_interface_state_str( SM_INTERFACE_STATE_UNKNOWN ),
+                            interface->service_domain,
+                            interface->service_domain_interface,
+                            sm_error_str( error ) );
+                return( error );
+            }
+        break;
+
         case SM_SERVICE_DOMAIN_INTERFACE_EVENT_NODE_DISABLED:
         case SM_SERVICE_DOMAIN_INTERFACE_EVENT_DISABLED:
         case SM_SERVICE_DOMAIN_INTERFACE_EVENT_NODE_ENABLED: