Add IPSec cert validation after system boot

This commit adds a new operation to IPSec server/client
to verify if local CA certificate is different from active
controller and renew it in case of failure. This operation
is executed everytime a system boot.

The operation is the comparison of the server cert serial
and the local cert serial. Because during initial authentication,
the server sends public CA certificate to each node to validade
the connection, but it's changed when the server update the
certificates.

Test Plan:
PASS: Full build, system install, bootstrap and unlock DX system w/
      unlocked enabled available status. Then, add a worker node and
      observe IPSec is enabled in all nodes and SAs are established.
PASS: Turn worker node off and update certificates in the controller
      nodes. Then, turn on the worker node again and observe that
      after reboot, the node will update the certificates and
      establish SAs with the other nodes.

Story: 2010940
Task: 50379

Change-Id: I1e765964797db9a35dc6fad00789b9c9c6232a49
Signed-off-by: Leonardo Mendes <Leonardo.MendesSantana@windriver.com>

controllerconfig/controllerconfig/scripts/controller_config

@@ -38,11 +38,12 @@ PUPPET_CACHE=/etc/puppet/cache
 PUPPET_CACHE_TMP=/etc/puppet/cache.tmp
 ACTIVE_CONTROLLER_NOT_FOUND_FLAG="/var/run/.active_controller_not_found"
 CERT_DIR=/etc/pki/ca-trust/source/anchors
-IPSEC_ENABLING_RETRIES=3
+IPSEC_RETRIES=3
-IPSEC_ENABLING_DELAY=5
+IPSEC_DELAY=5
 SWANCTL_CONF_FILE=/etc/swanctl/swanctl.conf
 SWANCTL_ACTIVE_CONF_FILE=/etc/swanctl/swanctl_active.conf
 SWANCTL_STANDBY_CONF_FILE=/etc/swanctl/swanctl_standby.conf
+IPSEC_SERVER_PORT=54724
 
 OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
 if [ "$OS_ID" == "debian" ]
@@ -243,6 +244,30 @@ umount_platform_dir()
     fi
 }
 
+validate_ca_cert_update()
+{
+    /usr/bin/ipsec-client -o 3 pxecontroller > /dev/null
+    if [ $? -eq 2 ]
+    then
+        ipsec_update_failed=1
+        for retry in $( seq 1 ${IPSEC_RETRIES} )
+        do
+            /usr/bin/ipsec-client -o 2 pxecontroller -f > /dev/null
+            if [ $? -eq 0 ]
+            then
+                ipsec_update_failed=0
+                break
+            fi
+            logger -t $0 -p warn "Updating IPsec cetificates failed (${retry}), retry in ${IPSEC_DELAY} seconds ..."
+            sleep ${IPSEC_DELAY}
+        done
+        if [ ${ipsec_update_failed} -ne 0 ]
+        then
+            warning_error "WARNING: Failed to update IPsec cetificates...."
+        fi
+    fi
+}
+
 start()
 {
     if [ -f /etc/platform/installation_failed ] ; then
@@ -281,7 +306,7 @@ start()
         logger -t $0 -p info "Config and enable IPsec ......"
 
         ipsec_enable_failed=1
-        for retry in $( seq 1 ${IPSEC_ENABLING_RETRIES} )
+        for retry in $( seq 1 ${IPSEC_RETRIES} )
         do
             /usr/bin/ipsec-client pxecontroller > /dev/null
             if [ $? -eq 0 ]
@@ -289,20 +314,34 @@ start()
                 ipsec_enable_failed=0
                 break
             fi
-            logger -t $0 -p warn "Enabling IPsec failed (${retry}), retry in ${IPSEC_ENABLING_DELAY} seconds ..."
+            logger -t $0 -p warn "Enabling IPsec failed (${retry}), retry in ${IPSEC_DELAY} seconds ..."
-            sleep ${IPSEC_ENABLING_DELAY}
+            sleep ${IPSEC_DELAY}
         done
         # Fail if retried maximum times
         if [ ${ipsec_enable_failed} -ne 0 ]
         then
             warning_error "WARNING: Failed to config and enable IPsec for the node"
         fi
+    elif [ ! -e ${FIRST_BOOT} ]
+    then
+        logger -t $0 -p info "Checking if IPsec Certificates is updated......"
+
+        #Verify if ipsec server is running
+        nc -z pxecontroller $IPSEC_SERVER_PORT
+        if [ $? -eq 0 ]
+        then
+            validate_ca_cert_update
+        else
+            warning_error "WARNING: IPSec server is not running...."
+        fi
+    fi
+
     # Link standby swanctl config file to swanctl.conf for initial boot.
     # This makes the symlink correct for only one controller reboot case (eg, active controller
     # is forcely reboot, but when it comes up it will be standby controller).
     # For rare cases such as reboot active controller while standby is locked, the symlink on the
     # active controller will be corrected by ipsec-config SM service when both controllers boots up.
-    elif [ -f ${SWANCTL_ACTIVE_CONF_FILE} ] && [ -f ${SWANCTL_STANDBY_CONF_FILE} ] && [ -f ${COMPLETED} ]
+    if [ -f ${SWANCTL_ACTIVE_CONF_FILE} ] && [ -f ${SWANCTL_STANDBY_CONF_FILE} ] && [ -f ${COMPLETED} ]
     then
         logger -t $0 -p info "Create IPsec configuration file symbolic link for the controller ..."
         ln -sf ${SWANCTL_STANDBY_CONF_FILE} ${SWANCTL_CONF_FILE}

storageconfig/storageconfig/storage_config

@@ -28,8 +28,9 @@ VOLATILE_CONFIG_FAIL="/var/run/.config_fail"
 DELAY_SEC=600
 IMA_POLICY=/etc/ima.policy
 FIRST_BOOT="/etc/platform/.first_boot"
-IPSEC_ENABLING_RETRIES=3
+IPSEC_RETRIES=3
-IPSEC_ENABLING_DELAY=5
+IPSEC_DELAY=5
+IPSEC_SERVER_PORT=54724
 
 fatal_error()
 {
@@ -100,6 +101,29 @@ get_ip()
     logger -t $0 -p warn "DNS query failed after max retries for $host (${DURATION} secs)"
 }
 
+validate_ca_cert_update()
+{
+    /usr/bin/ipsec-client -o 3 pxecontroller > /dev/null
+    if [ $? -eq 2 ]
+    then
+        ipsec_update_failed=1
+        for retry in $( seq 1 ${IPSEC_RETRIES} )
+        do
+            /usr/bin/ipsec-client -o 2 pxecontroller -f > /dev/null
+            if [ $? -eq 0 ]
+            then
+                ipsec_update_failed=0
+                break
+            fi
+            logger -t $0 -p warn "Updating IPsec cetificates failed (${retry}), retry in ${IPSEC_DELAY} seconds ..."
+            sleep ${IPSEC_DELAY}
+        done
+        if [ ${ipsec_update_failed} -ne 0 ]
+        then
+            warning_error "WARNING: Failed to update IPsec cetificates...."
+        fi
+    fi
+}
 
 start()
 {
@@ -139,7 +163,7 @@ start()
         logger -t $0 -p info "Config and enable IPsec ......"
 
         ipsec_enable_failed=1
-        for retry in $( seq 1 ${IPSEC_ENABLING_RETRIES} )
+        for retry in $( seq 1 ${IPSEC_RETRIES} )
         do
             /usr/bin/ipsec-client pxecontroller > /dev/null
             if [ $? -eq 0 ]
@@ -147,14 +171,25 @@ start()
                 ipsec_enable_failed=0
                 break
             fi
-            logger -t $0 -p warn "Enabling IPsec failed (${retry}), retry in ${IPSEC_ENABLING_DELAY} seconds ..."
+            logger -t $0 -p warn "Enabling IPsec failed (${retry}), retry in ${IPSEC_DELAY} seconds ..."
-            sleep ${IPSEC_ENABLING_DELAY}
+            sleep ${IPSEC_DELAY}
         done
         # Fail if retried maximum times
         if [ ${ipsec_enable_failed} -ne 0 ]
         then
             warning_error "WARNING: Failed to config and enable IPsec for the node"
         fi
+    else
+        logger -t $0 -p info "Checking if IPsec Certificates is updated......"
+
+        #Verify if ipsec server is running
+        nc -z pxecontroller $IPSEC_SERVER_PORT
+        if [ $? -eq 0 ]
+        then
+            validate_ca_cert_update
+        else
+            warning_error "WARNING: IPSec server is not running...."
+        fi
     fi
 
     HOST=$(hostname)

sysinv/sysinv/sysinv/scripts/ipsec-cert-renew.sh

@@ -18,7 +18,6 @@ NAME=$(basename $0)
 KUBE_CONFIG=/etc/kubernetes/admin.conf
 IPSEC_CERT_DIR=/etc/swanctl/x509
 IPSEC_CERT_PATH="$IPSEC_CERT_DIR/system-ipsec-certificate-${HOSTNAME}.crt"
-#IPSEC_CERT_PATH="$IPSEC_CERT_DIR/system-ipsec-certificate-${HOSTNAME}.crt-fake"
 ERR_CA=0
 ERR_CERT=0
 ERR_RENEW=0
@@ -47,39 +46,6 @@ time_left_s_by_openssl() {
     echo $time_left_s
 }
 
-# Check if the trusted CA cert is consistent with system-local-ca
-# in kubernetes. If it's not consistent, call ipsec-client to renew.
-# This is for cases such as a node is offline and misses a system-local-ca
-# update.
-
-# Retrieve the serial number of system-local-ca cert.
-if [ ${ERR_CA} -eq 0 ]; then
-    serial_in_secret=$(kubectl --kubeconfig=/etc/kubernetes/admin.conf get secret system-local-ca -n cert-manager -o jsonpath='{.data.tls\.crt}' | base64 --decode | openssl x509 -noout -serial)
-
-    if [ "x${serial_in_secret}" = "x"  ]; then
-        LOG_error "Failed to retrieve system-local-ca from secret."
-        ERR_CA=1
-    fi
-fi
-
-# Retrieve the serial number of the IPsec trusted CA cert.
-if [ ${ERR_CA} -eq 0 ]; then
-    serial_in_file=$(openssl x509 -in /etc/swanctl/x509ca/system-local-ca-1.crt -noout -serial)
-
-    if [ "x${serial_in_file}" = "x"  ]; then
-        LOG_error "Failed to retrieve serial number from CA cert file."
-        ERR_CA=1
-    fi
-fi
-
-# Compare to decide if they are consistent.
-if [ ${ERR_CA} -eq 0 ]; then
-    if [ "${serial_in_secret}" != "${serial_in_file}" ]; then
-        LOG_info "IPsec trusted CA is diverse from system-local-ca."
-        RENEWAL_REQUIRED=1
-    fi
-fi
-
 # Check if it's time to renew IPsec certificate.
 if [ ${ERR_CERT} -eq 0 ]; then
     time_left_s=$(time_left_s_by_openssl "${IPSEC_CERT_PATH}")

sysinv/sysinv/sysinv/sysinv/cmd/ipsec_client.py

@@ -24,9 +24,10 @@ def main():
 
     port = constants.DEFAULT_LISTEN_PORT
     opcode = 1
+    force_reload = False
 
     parser = argparse.ArgumentParser(
-        formatter_class=argparse.RawDescriptionHelpFormatter,
+        formatter_class=argparse.RawTextHelpFormatter,
         description=textwrap.dedent('''\
             Command line interface for IPsec Auth Client.
 
@@ -43,10 +44,17 @@ def main():
     parser.add_argument("-p", "--port", metavar='<port>', type=int,
                 help='Port number (Default: ' + str(port) + ')')
     parser.add_argument('-d', "--debug", action="store_true",
-                help="If enabled, the logging level will be set "
+                help=textwrap.dedent('''\
-                "to DEBUG instead of the default INFO level.")
+                If enabled, the logging level will be set
+                to DEBUG instead of the default INFO level.'''))
+    parser.add_argument('-f', "--force-reload", action="store_true",
+                help=textwrap.dedent('''\
+                If enabled, force to reload all configuration
+                files on StrongSwan during certificate renewal
+                procedure. This flag only applies to cert-renewal
+                operation (opcode 2)'''))
     parser.add_argument("-o", "--opcode", metavar='<opcode>',
-                type=int, choices=[1, 2],
+                type=int, choices=[1, 2, 3],
                 help='Operational code (Default: ' + str(opcode) + ')')
     args = parser.parse_args()
 
@@ -58,6 +66,9 @@ def main():
     if args.opcode:
         opcode = args.opcode
 
+    if args.force_reload:
+        force_reload = args.force_reload
+
     logging.register_options(CONF)
     logging.set_defaults()
 
@@ -72,5 +83,5 @@ def main():
     if not os.path.exists(constants.TMP_DIR_IPSEC_KEYS):
         os.makedirs(constants.TMP_DIR_IPSEC_KEYS)
 
-    client = Client(host, port, opcode)
+    client = Client(host, port, opcode, force_reload)
     client.run()

sysinv/sysinv/sysinv/sysinv/ipsec_auth/client/client.py

@@ -6,6 +6,7 @@
 import base64
 import json
 import os
+import sys
 import selectors
 import socket
 import subprocess
@@ -29,10 +30,11 @@ LOG = logging.getLogger(__name__)
 
 class Client(object):
 
-    def __init__(self, host, port, op_code):
+    def __init__(self, host, port, op_code, force_reload):
         self.host = host
         self.port = port
         self.op_code = str(op_code)
+        self.force_reload = force_reload
         self.state = State.STAGE_1
         self.ifname = utils.get_management_interface()
         self.personality = utils.get_personality()
@@ -141,6 +143,17 @@ class Client(object):
 
         if self.state == State.STAGE_2:
             LOG.info("Received IPSec Auth Response")
+
+            if self.op_code == constants.OP_CODE_CERT_VALIDATION:
+                server_cert_serial = msg['ca_cert_serial']
+                ca_cert = utils.load_data(constants.TRUSTED_CA_CERT_1_PATH)
+                local_cert = x509.load_pem_x509_certificate(ca_cert)
+                if server_cert_serial != local_cert.serial_number:
+                    return False
+
+                LOG.info("Cert Serial validation OK")
+                return True
+
             self.ots_token = msg['token']
             self.hostname = msg['hostname']
             key = base64.b64decode(msg['pub_key'])
@@ -209,23 +222,28 @@ class Client(object):
                     return False
 
             elif self.op_code == constants.OP_CODE_CERT_RENEWAL:
-                load_creds = subprocess.run(['swanctl', '--load-creds'], stdout=subprocess.PIPE,
+
+                if self.force_reload:
+                    load_all = subprocess.run(['swanctl', '--load-all'], stdout=subprocess.PIPE,
+                                    stderr=subprocess.PIPE, check=False)
+
+                    if load_all.returncode != 0:
+                        err = "Error: %s" % (load_all.stderr.decode("utf-8"))
+                        LOG.exception("Failed to load StrongSwan configuration: %s" % err)
+                        return False
+
+                    LOG.info('IPsec certificate renewed successfully')
+
+                    return True
+
+                load_conns = subprocess.run(['swanctl', '--load-conns'], stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE, check=False)
 
-                if load_creds.returncode != 0:
+                if load_conns.returncode != 0:
-                    err = "Error: %s" % (load_creds.stderr.decode("utf-8"))
+                    err = "Error: %s" % (load_conns.stderr.decode("utf-8"))
                     LOG.exception("Failed to load StrongSwan credentials: %s" % err)
                     return False
 
-                rekey = subprocess.run(['swanctl', '--rekey', '--ike', constants.IKE_SA_NAME,
-                                        '--reauth'], stdout=subprocess.PIPE,
-                                        stderr=subprocess.PIPE, check=False)
-
-                if rekey.returncode != 0:
-                    err = "Error: %s" % (rekey.stderr.decode("utf-8"))
-                    LOG.exception("Failed to rekey IKE SA with StrongSwan: %s" % err)
-                    return False
-
                 LOG.info('IPsec certificate renewed successfully')
 
         return True
@@ -258,6 +276,7 @@ class Client(object):
             selectors.EVENT_READ | selectors.EVENT_WRITE,
         )
 
+        exit_code = 0
         keep_running = True
         while keep_running:
             for key, mask in sel.select(timeout=1):
@@ -266,21 +285,34 @@ class Client(object):
                 LOG.debug("State{}".format(self.state))
                 if mask & selectors.EVENT_READ:
                     self.data = utils.socket_recv_all_json(sock, 8192)
-                    if not self._handle_rcvd_data(self.data):
+                    if not self.data:
-                        raise ConnectionAbortedError("Error receiving data from server")
+                        LOG.error("No data received from server")
+                        exit_code = 1
+                        self.state = State.END_STAGE
+                    elif not self._handle_rcvd_data(self.data):
+                        if self.op_code == constants.OP_CODE_CERT_VALIDATION:
+                            LOG.error("Cert Serial validation failed")
+                            exit_code = 2
+                        else:
+                            LOG.error("Error handling data from server")
+                            exit_code = 1
+
+                        self.state = State.END_STAGE
+
                     sel.modify(sock, selectors.EVENT_WRITE)
-                    self.state = State.get_next_state(self.state)
+                    self.state = State.get_next_state(self.state, self.op_code)
 
                 if mask & selectors.EVENT_WRITE:
                     msg = self._handle_send_data(self.data)
                     sock.sendall(bytes(msg, 'utf-8'))
                     sel.modify(sock, selectors.EVENT_READ)
-                    self.state = State.get_next_state(self.state)
+                    self.state = State.get_next_state(self.state, self.op_code)
 
-                if self.state == State.STAGE_5:
+                if self.state == State.END_STAGE:
                     keep_running = False
 
         LOG.info("Shutting down")
         sel.unregister(connection)
         connection.close()
         sel.close()
+        sys.exit(exit_code)

sysinv/sysinv/sysinv/sysinv/ipsec_auth/common/constants.py

@@ -60,8 +60,10 @@ PXECONTROLLER_URL = 'http://pxecontroller:6385'
 
 OP_CODE_INITIAL_AUTH = "1"
 OP_CODE_CERT_RENEWAL = "2"
+OP_CODE_CERT_VALIDATION = "3"
 SUPPORTED_OP_CODES = [OP_CODE_INITIAL_AUTH,
-                      OP_CODE_CERT_RENEWAL]
+                      OP_CODE_CERT_RENEWAL,
+                      OP_CODE_CERT_VALIDATION]
 
 MGMT_IPSEC_ENABLING = 'enabling'
 MGMT_IPSEC_ENABLED = 'enabled'

sysinv/sysinv/sysinv/sysinv/ipsec_auth/common/objects.py

@@ -12,6 +12,7 @@ import time
 import threading
 
 from oslo_log import log as logging
+from sysinv.ipsec_auth.common import constants
 
 LOG = logging.getLogger(__name__)
 
@@ -21,48 +22,56 @@ class State(enum.Enum):
     STAGE_2 = 2
     STAGE_3 = 3
     STAGE_4 = 4
-    STAGE_5 = 5
+    END_STAGE = 5
 
     @staticmethod
-    def get_next_state(state):
+    def get_next_state(state, op):
         '''Get the next IPsec Auth state whenever a Stage is finished.
 
         The IPsec Auth server-client interaction is separated into 5 work stages.
-        STAGE_1: represents the initial stage where IPsec Auth client send
+        STAGE_1:   represents the initial stage where IPsec Auth client send
-                the first message with OP code, mac address and a hash to
+                   the first message with OP code, mac address and a hash to
-                IPsec Auth server.
+                   IPsec Auth server.
-        STAGE_2: represents the stage of validation of the message 1 received
+        STAGE_2:   represents the stage of validation of the message 1 received
-                from the client and generation of a response message. If the
+                   from the client and generation of a response message. If the
-                validation is satisfied, the IPsec Auth server will encapsulate
+                   validation is satisfied, the IPsec Auth server will encapsulate
-                an OTS Token, client's hostname, generated public key,
+                   an OTS Token, client's hostname, generated public key,
-                system-local-ca's certificate and a signed hash of this payload
+                   system-local-ca's certificate and a signed hash of this payload
-                in the response message to send it to the client.
+                   in the response message to send it to the client.
-        STAGE_3: represents the stage of validation of the message 2 received
+        STAGE_3:   represents the stage of validation of the message 2 received
-                from the server and generation of a response message. if the
+                   from the server and generation of a response message. if the
-                validation is satisfied, the IPsec Auth Client will encapsulate
+                   validation is satisfied, the IPsec Auth Client will encapsulate
-                an OTS Token, an encrypted Initial Vector (eiv), an encrypted
+                   an OTS Token, an encrypted Initial Vector (eiv), an encrypted
-                symetric key (eak1), an encrypted certificate request (eCSR)
+                   symetric key (eak1), an encrypted certificate request (eCSR)
-                and a signed hash of this payload in the response message to
+                   and a signed hash of this payload in the response message to
-                send it to the server.
+                   send it to the server.
-        STAGE_4: represents the stage of validation of the message 3 from the
+        STAGE_4:   represents the stage of validation of the message 3 from the
-                client and generation of a final response message. If the
+                   client and generation of a final response message. If the
-                validation of the message is satisfied, the IPsec Auth server
+                   validation of the message is satisfied, the IPsec Auth server
-                will create a CertificateRequest resource with a CSR received
+                   will create a CertificateRequest resource with a CSR received
-                from client's message and will encapsulate the signed
+                   from client's message and will encapsulate the signed
-                Certificate, network info and a signed hash of this payload in
+                   Certificate, network info and a signed hash of this payload in
-                the response message to send it to the client.
+                   the response message to send it to the client.
-        STAGE_5: represents the final stage of IPsec PKI Auth procedure and demands
+        END_STAGE: represents the final stage of IPsec PKI Auth procedure and demands
-                that IPsec Auth server and client close the connection that
+                   that IPsec Auth server and client close the connection that
-                finished STAGE_4.
+                   finished STAGE_4.
         '''
-        if state == State.STAGE_1:
+
-            state = State.STAGE_2
+        if op == constants.OP_CODE_CERT_VALIDATION:
-        elif state == State.STAGE_2:
+            if state == State.STAGE_1:
-            state = State.STAGE_3
+                state = State.STAGE_2
-        elif state == State.STAGE_3:
+            elif state == State.STAGE_2:
-            state = State.STAGE_4
+                state = State.END_STAGE
-        elif state == State.STAGE_4:
+        else:
-            state = State.STAGE_5
+            if state == State.STAGE_1:
+                state = State.STAGE_2
+            elif state == State.STAGE_2:
+                state = State.STAGE_3
+            elif state == State.STAGE_3:
+                state = State.STAGE_4
+            elif state == State.STAGE_4:
+                state = State.END_STAGE
+
         return state
 
 

sysinv/sysinv/sysinv/sysinv/ipsec_auth/server/server.py

@@ -18,6 +18,7 @@ from sysinv.ipsec_auth.common import utils
 from sysinv.ipsec_auth.common.objects import State
 from sysinv.ipsec_auth.common.objects import Token
 
+from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.asymmetric import rsa
@@ -115,7 +116,7 @@ class IPsecConnection(object):
             if data:
                 # A readable client socket has data
                 LOG.debug("Received {!r}".format(data))
-                self.state = State.get_next_state(self.state)
+                self.state = State.get_next_state(self.state, self.op_code)
 
                 LOG.debug("Preparing payload")
                 msg = self._handle_write(data)
@@ -123,14 +124,15 @@ class IPsecConnection(object):
 
                 if self.state == State.STAGE_2:
                     self.ots_token.activate()
-                self.state = State.get_next_state(self.state)
+                self.state = State.get_next_state(self.state, self.op_code)
-            elif self.state == State.STAGE_5:
+            elif self.state == State.END_STAGE:
                 self.ots_token.purge()
-                self._update_mgmt_ipsec_state(constants.MGMT_IPSEC_ENABLED)
+                if self.op_code != constants.OP_CODE_CERT_VALIDATION:
+                    self._update_mgmt_ipsec_state(constants.MGMT_IPSEC_ENABLED)
                 self.keep_connection = False
             elif not data:
                 # Interpret empty result as closed connection
-                LOG.error('Failed to receive data from client or empty buffer provided.')
+                LOG.info('No Data received from client or empty buffer provided.')
                 self._cleanup_connection_data()
                 self.keep_connection = False
         except Exception as e:
@@ -163,24 +165,29 @@ class IPsecConnection(object):
                            "received in payload.")
                     raise ConnectionRefusedError(msg)
 
-                client_data = utils.get_client_host_info_by_mac(mac_addr)
+                if self.op_code == constants.OP_CODE_CERT_VALIDATION:
-                self.hostname = client_data['hostname']
+                    cert = x509.load_pem_x509_certificate(base64.b64decode(self.ca_crt))
-                self.mgmt_subnet = client_data['mgmt_subnet']
+                    LOG.debug("Cert Serial:{}".format(cert.serial_number))
-                self.unit_ip = client_data['unit_ip']
+                    payload["ca_cert_serial"] = cert.serial_number
-                self.floating_ip = client_data['floating_ip']
+                else:
+                    client_data = utils.get_client_host_info_by_mac(mac_addr)
+                    self.hostname = client_data['hostname']
+                    self.mgmt_subnet = client_data['mgmt_subnet']
+                    self.unit_ip = client_data['unit_ip']
+                    self.floating_ip = client_data['floating_ip']
 
-                pub_key = self._generate_tmp_key_pair()
+                    pub_key = self._generate_tmp_key_pair()
-                token = self.ots_token.get_content()
+                    token = self.ots_token.get_content()
-                hash_payload = utils.hash_and_sign_payload(self.ca_key, token + pub_key)
+                    hash_payload = utils.hash_and_sign_payload(self.ca_key, token + pub_key)
 
-                payload["token"] = repr(self.ots_token)
+                    payload["token"] = repr(self.ots_token)
-                payload["hostname"] = self.hostname
+                    payload["hostname"] = self.hostname
-                payload["pub_key"] = pub_key.decode("utf-8")
+                    payload["pub_key"] = pub_key.decode("utf-8")
-                payload["ca_cert"] = self.ca_crt.decode("utf-8")
+                    payload["ca_cert"] = self.ca_crt.decode("utf-8")
-                payload["root_ca_cert"] = self.root_ca_crt.decode("utf-8")
+                    payload["root_ca_cert"] = self.root_ca_crt.decode("utf-8")
-                payload["hash"] = hash_payload.decode("utf-8")
+                    payload["hash"] = hash_payload.decode("utf-8")
 
-                LOG.info("Sending IPSec Auth Response")
+                    LOG.info("Sending IPSec Auth Response")
 
             if self.state == State.STAGE_4:
                 LOG.info("Received IPSec Auth CSR request")
@@ -287,7 +294,8 @@ class IPsecConnection(object):
                 return False
 
         # Certificate renewal request
-        elif self.op_code == constants.OP_CODE_CERT_RENEWAL:
+        elif (self.op_code == constants.OP_CODE_CERT_RENEWAL or
+                self.op_code == constants.OP_CODE_CERT_VALIDATION):
             if self.uuid and self.mgmt_ipsec == constants.MGMT_IPSEC_ENABLED and mgmt_mac:
                 # Valid so do nothing
                 pass

workerconfig/workerconfig/worker_config

@@ -28,8 +28,9 @@ VOLATILE_CONFIG_FAIL="/var/run/.config_fail"
 LOGFILE="/var/log/worker_config.log"
 IMA_POLICY=/etc/ima.policy
 FIRST_BOOT="/etc/platform/.first_boot"
-IPSEC_ENABLING_RETRIES=3
+IPSEC_RETRIES=3
-IPSEC_ENABLING_DELAY=5
+IPSEC_DELAY=5
+IPSEC_SERVER_PORT=54724
 
 # Copy of /opt/platform required for worker_services
 VOLATILE_PLATFORM_PATH=$VOLATILE_PATH/cpe_upgrade_opt_platform
@@ -135,6 +136,30 @@ wait_for_controller_services()
    return 1
 }
 
+validate_ca_cert_update()
+{
+    /usr/bin/ipsec-client -o 3 pxecontroller > /dev/null
+    if [ $? -eq 2 ]
+    then
+        ipsec_update_failed=1
+        for retry in $( seq 1 ${IPSEC_RETRIES} )
+        do
+            /usr/bin/ipsec-client -o 2 pxecontroller -f > /dev/null
+            if [ $? -eq 0 ]
+            then
+                ipsec_update_failed=0
+                break
+            fi
+            logger -t $0 -p warn "Updating IPsec cetificates failed (${retry}), retry in ${IPSEC_DELAY} seconds ..."
+            sleep ${IPSEC_DELAY}
+        done
+        if [ ${ipsec_update_failed} -ne 0 ]
+        then
+            warning_error "WARNING: Failed to update IPsec cetificates...."
+        fi
+    fi
+}
+
 start()
 {
     if [ -f /etc/platform/installation_failed ] ; then
@@ -206,7 +231,7 @@ start()
         logger -t $0 -p info "Config and enable IPsec ......"
 
         ipsec_enable_failed=1
-        for retry in $( seq 1 ${IPSEC_ENABLING_RETRIES} )
+        for retry in $( seq 1 ${IPSEC_RETRIES} )
         do
             /usr/bin/ipsec-client pxecontroller > /dev/null
             if [ $? -eq 0 ]
@@ -214,14 +239,25 @@ start()
                 ipsec_enable_failed=0
                 break
             fi
-            logger -t $0 -p warn "Enabling IPsec failed (${retry}), retry in ${IPSEC_ENABLING_DELAY} seconds ..."
+            logger -t $0 -p warn "Enabling IPsec failed (${retry}), retry in ${IPSEC_DELAY} seconds ..."
-            sleep ${IPSEC_ENABLING_DELAY}
+            sleep ${IPSEC_DELAY}
         done
         # Fail if retried maximum times
         if [ ${ipsec_enable_failed} -ne 0 ]
         then
             warning_error "WARNING: Failed to config and enable IPsec for the node"
         fi
+    else
+        logger -t $0 -p info "Checking if IPsec Certificates is updated......"
+
+        #Verify if ipsec server is running
+        nc -z pxecontroller $IPSEC_SERVER_PORT
+        if [ $? -eq 0 ]
+        then
+            validate_ca_cert_update
+        else
+            warning_error "WARNING: IPSec server is not running...."
+        fi
     fi
 
     HOST=$(hostname)