Fix PKCS#11 reinitialization after failure

This patch fixes an error condition where the PCKS#11 object is lost
afte multiple failures from the PKCS#11 device.

Without this patch Barbican can enter an error state where the PKCS11
object is deallocated and is unable to recover returning 500s until the
process is restarted.

Story: 2008649
Task: 41913
Change-Id: Ieab0f152ef307fb9311b92f0ad8bd0032e78269e
(cherry picked from commit fa7722bfc5)
(cherry picked from commit 3d3e00a92a)
(cherry picked from commit 475074a47c)

barbican/plugin/crypto/p11_crypto.py

@@ -184,6 +184,8 @@ class P11CryptoPlugin(plugin.CryptoPluginBase):
         # Wrap pkcs11 calls to enable a single retry when exceptions are raised
         # that can be fixed by reinitializing the pkcs11 library
         try:
+            if self.pkcs11 is None:
+                self._reinitialize_pkcs11()
             return func(*args, **kwargs)
         except (exception.PKCS11Exception) as pe:
             LOG.warning("Reinitializing PKCS#11 library: %s", pe)
@@ -335,7 +337,8 @@ class P11CryptoPlugin(plugin.CryptoPluginBase):
         )
 
     def _reinitialize_pkcs11(self):
-        self.pkcs11.finalize()
+        if self.pkcs11 is not None:
+            self.pkcs11.finalize()
         self.pkcs11 = None
 
         with self.caching_session_lock:

barbican/plugin/crypto/pkcs11.py

@@ -31,6 +31,7 @@ CKMechanism = collections.namedtuple("CKMechanism", ["mech", "cffivals"])
 Token = collections.namedtuple("Token", ["slot_id", "label", "serial_number"])
 
 CKR_OK = 0
+CKR_CRYPTOKI_ALREADY_INITIALIZED = 0x00000191
 CK_TRUE = 1
 CKF_RW_SESSION = (1 << 1)
 CKF_SERIAL_SESSION = (1 << 2)
@@ -867,7 +868,7 @@ class PKCS11(object):
         self._check_error(rv)
 
     def _check_error(self, value):
-        if value != CKR_OK:
+        if value != CKR_OK and value != CKR_CRYPTOKI_ALREADY_INITIALIZED:
             code = ERROR_CODES.get(value, 'CKR_????')
             hex_code = "{hex} {code}".format(hex=hex(value), code=code)
 

barbican/tests/plugin/crypto/test_p11_crypto.py

@@ -342,6 +342,16 @@ class WhenTestingP11CryptoPlugin(utils.BaseTestCase):
         self.assertEqual(0, self.pkcs11.return_session.call_count)
         self.assertEqual(2, self.plugin._encrypt.call_count)
 
+    def test_call_pkcs11_reinitializes_pkcs11_object(self):
+        self.plugin._reinitialize_pkcs11 = mock.Mock()
+        self.plugin.pkcs11 = None
+
+        def test_func(*args, **kwargs):
+            pass
+
+        self.plugin._call_pkcs11(test_func)
+        self.plugin._reinitialize_pkcs11.assert_called_once()
+
     def test_reinitialize_pkcs11(self):
         pkcs11 = self.pkcs11
         self.plugin._create_pkcs11 = mock.Mock()

releasenotes/notes/fix-story-2008649-reinitialize-pkcs11-object-4c0dc51c83288c21.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixed Story #2008649: Correctly reinitialize PKCS11 object after secondary
+    failures.