Verify cert chain after adminep update

This commit checks the validity of the certificate chain
after admin-ep certificate is renewed on subclouds. If
the check fails, the update_admin_ep_cert deletes the secret
and subsequent audits will recreate. This check fixes the
scenario where old ICA was used when updating &
renewing rootCA.

Closes-bug: #1923071

Signed-off-by: Sabeel Ansari <Sabeel.Ansari@windriver.com>
Change-Id: I112dd52e220dcc8bdb72c2c772ede72fbb786c7b

sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py

@@ -11,18 +11,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Copyright (c) 2020 Wind River Systems, Inc.
+# Copyright (c) 2020-2021 Wind River Systems, Inc.
 #
 # The right to copy, distribute, modify, or otherwise make use
 # of this software may be licensed only pursuant to the terms
 # of an applicable Wind River license agreement.
 #
+import base64
 import json
 import os
 import re
 import requests
 import ssl
 import tempfile
+from eventlet.green import subprocess
 from six.moves.urllib.parse import urlparse
 from keystoneclient.v3 import client as keystone_client
 from keystoneauth1 import session
@@ -79,6 +81,65 @@ def update_admin_ep_cert(token, ca_crt, tls_crt, tls_key):
         raise Exception('Update admin endpoint certificate failed')
 
 
+def verify_adminep_cert_chain():
+    """
+    Verify admin endpoint certificate chain & delete if invalid
+    :param context: an admin context.
+    :return: True/False if chain is valid
+    """
+    """
+    * Retrieve ICA & AdminEP cert secrets from k8s
+    * base64 decode ICA cert (tls.crt from SC_INTERMEDIATE_CA_SECRET_NAME)
+    *   & adminep (tls.crt from SC_ADMIN_ENDPOINT_SECRET_NAME)
+    *   & store the crts in tempfiles
+    * Run openssl verify against RootCA to verify the chain
+    """
+    kube_op = sys_kube.KubeOperator()
+
+    secret_ica = kube_op.kube_get_secret(constants.SC_INTERMEDIATE_CA_SECRET_NAME,
+                                         CERT_NAMESPACE_SUBCLOUD_CONTROLLER)
+    if 'tls.crt' not in secret_ica.data:
+        raise Exception('%s tls.crt (ICA) data missing'
+                        % (constants.SC_INTERMEDIATE_CA_SECRET_NAME))
+
+    secret_adminep = kube_op.kube_get_secret(constants.SC_ADMIN_ENDPOINT_SECRET_NAME,
+                                             CERT_NAMESPACE_SUBCLOUD_CONTROLLER)
+    if 'tls.crt' not in secret_adminep.data:
+        raise Exception('%s tls.crt data missing'
+                        % (constants.SC_ADMIN_ENDPOINT_SECRET_NAME))
+
+    txt_ca_crt = base64.b64decode(secret_ica.data['tls.crt'])
+    txt_tls_crt = base64.b64decode(secret_adminep.data['tls.crt'])
+
+    with tempfile.NamedTemporaryFile() as ca_tmpfile:
+        ca_tmpfile.write(txt_ca_crt)
+        ca_tmpfile.flush()
+        with tempfile.NamedTemporaryFile() as adminep_tmpfile:
+            adminep_tmpfile.write(txt_tls_crt)
+            adminep_tmpfile.flush()
+
+            cmd = ['openssl', 'verify', '-CAfile', constants.DC_ROOT_CA_CERT_PATH,
+                   '-untrusted', ca_tmpfile.name, adminep_tmpfile.name]
+            proc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
+                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            stdout, stderr = proc.communicate()
+            proc.wait()
+            if 0 == proc.returncode:
+                LOG.info('verify_adminep_cert_chain passed. Valid chain')
+                return True
+            else:
+                LOG.info('verify_adminep_cert_chain: Chain is invalid\n%s\n%s'
+                         % (stdout, stderr))
+
+                res = kube_op.kube_delete_secret(constants.SC_ADMIN_ENDPOINT_SECRET_NAME,
+                                                 CERT_NAMESPACE_SUBCLOUD_CONTROLLER)
+                LOG.info('Deleting AdminEP secret due to invalid chain. %s:%s, result %s, msg %s'
+                         % (CERT_NAMESPACE_SUBCLOUD_CONTROLLER,
+                         constants.SC_ADMIN_ENDPOINT_SECRET_NAME,
+                         res.status, res.message))
+                return False
+
+
 def dc_get_subcloud_sysinv_url(subcloud_name):
     user_auth = v3.Password(
         auth_url=CONF.endpoint_cache.auth_uri,

sysinv/sysinv/sysinv/sysinv/cert_mon/watcher.py

@@ -11,7 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Copyright (c) 2020 Wind River Systems, Inc.
+# Copyright (c) 2020-2021 Wind River Systems, Inc.
 #
 # The right to copy, distribute, modify, or otherwise make use
 # of this software may be licensed only pursuant to the terms
@@ -365,8 +365,16 @@ class AdminEndpointRenew(CertificateRenew):
 
     def update_certificate(self, event_data):
         token = self.context.get_token()
+
+        role = self.context.dc_role
         utils.update_admin_ep_cert(token, event_data.ca_crt, event_data.tls_crt,
-                                   event_data.tls_key)
+                                event_data.tls_key)
+
+        # In subclouds, it was observed that sometimes old ICA was used
+        # to sign adminep-cert. Here we run a verification to confirm that
+        # the chain is valid & delete secret if chain fails
+        if role == constants.DISTRIBUTED_CLOUD_ROLE_SUBCLOUD:
+            utils.verify_adminep_cert_chain()
 
 
 class DCIntermediateCertRenew(CertificateRenew):

sysinv/sysinv/sysinv/sysinv/common/constants.py

@@ -1800,6 +1800,7 @@ CERTIFICATE_TYPE_ADMIN_ENDPOINT_INTERMEDIATE_CA = 'intermediate-ca-cert'
 
 DC_ADMIN_ENDPOINT_SECRET_NAME = 'dc-adminep-certificate'
 SC_ADMIN_ENDPOINT_SECRET_NAME = 'sc-adminep-certificate'
+SC_INTERMEDIATE_CA_SECRET_NAME = 'sc-adminep-ca-certificate'
 
 DC_ADMIN_ROOT_CA_SECRET_NAME = 'dc-adminep-root-ca-certificate'