Replace md5 with oslo version

md5 is not an approved algorithm in FIPS mode, and trying to
instantiate a hashlib.md5() will fail when the system is running in
FIPS mode.

md5 is allowed when in a non-security context.  There is a plan to
add a keyword parameter (usedforsecurity) to hashlib.md5() to annotate
whether or not the instance is being used in a security context.

In the case where it is not, the instantiation of md5 will be allowed.
See https://bugs.python.org/issue9216 for more details.

Some downstream python versions already support this parameter.  To
support these versions, a new encapsulation of md5() has been added to
oslo_utils.  See https://review.opendev.org/#/c/750031/

This patch is to replace the instances of hashlib.md5() with this new
encapsulation, adding an annotation indicating whether the usage is
a security context or not.

Reviewers need to pay particular attention as to whether the keyword
parameter (usedforsecurity) is set correctly.  Almost all instances
of md5 usage appear to be to refer to etags, to do checksums, or to
generate uuids for paths.

I had hoped to update the bandit config to enable scanning for instances
of md5 and bad algorithms, so that instances would not creep in in future,
but I couldn't find the bandit config.

With this patch (and the corresponding os-brick and oslo-versioned_object
dependent changes) all the functional tests and alnmost all the unit tests
pass on a FIPS enabled system.

Issues I found were as follows:

- Cinder appears to be using md5 in a security context in
  cinder/volume/drivers/synology/synology_common.py.  If this is really
  the case, then we'll need to consider how to replace md5 in this usage.
  This case did not appear to exercised in the unit or functional tests I ran.

- Cinder appears to use md5 in a security context in
  cinder/volume/drivers/stx/client.py, which resulted in the failed unit test
  cinder.tests.unit.volume.drivers.test_seagate.TestSeagateClient.test_login
  This was the only unit test that failed.

Change-Id: I57ec3e7e99c78535fa8051d011d970adb7fb89ab
Depends-On: https://review.opendev.org/#/c/756151

cinder/api/v3/volume_metadata.py

@@ -15,10 +15,10 @@
 
 """The volume metadata V3 api."""
 
-import hashlib
 from http import HTTPStatus
 
 from oslo_serialization import jsonutils
+from oslo_utils.secretutils import md5
 import webob
 
 from cinder.api import microversions as mv
@@ -37,7 +37,7 @@ class Controller(volume_meta_v2.Controller):
         metadata = self._get_metadata(context, volume_id)
         data = jsonutils.dumps({"metadata": metadata})
         data = data.encode('utf-8')
-        checksum = hashlib.md5(data).hexdigest()
+        checksum = md5(data, usedforsecurity=False).hexdigest()
         return checksum in req.if_match.etags
 
     @wsgi.extends
@@ -48,7 +48,7 @@ class Controller(volume_meta_v2.Controller):
             data = jsonutils.dumps(metadata)
             data = data.encode('utf-8')
             resp = webob.Response()
-            resp.headers['Etag'] = hashlib.md5(data).hexdigest()
+            resp.headers['Etag'] = md5(data, usedforsecurity=False).hexdigest()
             resp.body = data
             return resp
         return metadata

cinder/backup/chunkeddriver.py

@@ -32,6 +32,7 @@ from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_service import loopingcall
 from oslo_utils import excutils
+from oslo_utils import secretutils
 from oslo_utils import units
 
 from cinder.backup import driver
@@ -371,7 +372,8 @@ class ChunkedBackupDriver(driver.BackupDriver, metaclass=abc.ABCMeta):
                 container, object_name, extra_metadata=extra_metadata
         ) as writer:
             writer.write(output_data)
-        md5 = eventlet.tpool.execute(hashlib.md5, data).hexdigest()
+        md5 = eventlet.tpool.execute(
+            secretutils.md5, data, usedforsecurity=False).hexdigest()
         obj[object_name]['md5'] = md5
         LOG.debug('backup MD5 for %(object_name)s: %(md5)s',
                   {'object_name': object_name, 'md5': md5})

cinder/backup/drivers/gcs.py

@@ -27,7 +27,6 @@ Server-centric flow is used for authentication.
 """
 
 import base64
-import hashlib
 import io
 import os
 
@@ -55,6 +54,7 @@ from googleapiclient import errors
 from googleapiclient import http
 from oslo_config import cfg
 from oslo_log import log as logging
+from oslo_utils import secretutils
 from oslo_utils import timeutils
 from packaging import version
 
@@ -338,7 +338,7 @@ class GoogleObjectWriter(object):
             body={},
             media_body=media).execute(num_retries=self.num_retries)
         etag = resp['md5Hash']
-        md5 = hashlib.md5(self.data).digest()
+        md5 = secretutils.md5(self.data, usedforsecurity=False).digest()
         md5 = md5.encode('utf-8')
         etag = bytes(etag, 'utf-8')
         md5 = base64.b64encode(md5)

cinder/backup/drivers/swift.py

@@ -43,12 +43,12 @@
                              certificate for SSL connections (default: False)
 """
 
-import hashlib
 import io
 import socket
 
 from oslo_config import cfg
 from oslo_log import log as logging
+from oslo_utils import secretutils
 from oslo_utils import timeutils
 from swiftclient import client as swift
 
@@ -289,7 +289,7 @@ class SwiftBackupDriver(chunkeddriver.ChunkedBackupDriver):
                                             content_length=len(self.data))
             except socket.error as err:
                 raise exception.SwiftConnectionFailed(reason=err)
-            md5 = hashlib.md5(self.data).hexdigest()
+            md5 = secretutils.md5(self.data, usedforsecurity=False).hexdigest()
             if etag != md5:
                 err = _('error writing object to swift, MD5 of object in '
                         'swift %(etag)s is not the same as MD5 of object sent '

cinder/tests/unit/backup/drivers/test_backup_nfs.py

@@ -175,7 +175,7 @@ class BackupNFSShareTestCase(test.TestCase):
                          mock_remotefsclient.mount.call_args_list)
 
 
-def fake_md5(arg):
+def fake_md5(arg, usedforsecurity=False):
     class result(object):
         def hexdigest(self):
             return 'fake-md5-sum'

cinder/tests/unit/backup/drivers/test_backup_swift.py

@@ -48,7 +48,7 @@ CONF = cfg.CONF
 ANY = mock.ANY
 
 
-def fake_md5(arg):
+def fake_md5(arg, usedforsecurity=False):
     class result(object):
         def hexdigest(self):
             return 'fake-md5-sum'

cinder/tests/unit/backup/fake_swift_client2.py

@@ -14,12 +14,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import hashlib
 from http import client as http_client
 import os
 import socket
 import tempfile
 
+from oslo_utils.secretutils import md5
 from swiftclient import client as swift
 
 
@@ -78,7 +78,7 @@ class FakeSwiftConnection2(object):
         object_path = tempfile.gettempdir() + '/' + container + '/' + name
         with open(object_path, 'wb') as object_file:
             object_file.write(reader.read())
-        return hashlib.md5(reader.read()).hexdigest()
+        return md5(reader.read(), usedforsecurity=False).hexdigest()
 
     def delete_object(self, container, name):
         pass

cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_cmode.py

@@ -14,12 +14,12 @@
 #    under the License.
 """Mock unit tests for the NetApp cmode nfs storage driver."""
 
-import hashlib
 from unittest import mock
 import uuid
 
 import ddt
 from os_brick.remotefs import remotefs as remotefs_brick
+from oslo_utils.secretutils import md5
 from oslo_utils import units
 
 from cinder import exception
@@ -880,8 +880,9 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
         drv = self.driver
         cinder_mount_point_base = '/opt/stack/data/cinder/mnt/'
         # To get the cinder mount point directory, we use:
-        mount_dir = hashlib.md5(
-            '203.0.113.122:/cinder-flexvol1'.encode('utf-8')).hexdigest()
+        mount_dir = md5(
+            '203.0.113.122:/cinder-flexvol1'.encode('utf-8'),
+            usedforsecurity=False).hexdigest()
         cinder_mount_point = cinder_mount_point_base + mount_dir
         destination_copied_file = (
             '/cinder-flexvol1/a155308c-0290-497b-b278-4cdd01de0253'

cinder/tests/unit/volume/drivers/nexenta/test_nexenta5_jsonrpc.py

@@ -15,12 +15,12 @@
 """Unit tests for NexentaStor 5 REST API helper."""
 
 import copy
-import hashlib
 import json
 import posixpath
 from unittest import mock
 import uuid
 
+from oslo_utils.secretutils import md5
 import requests
 import six
 
@@ -1187,7 +1187,7 @@ class TestNefProxy(test.TestCase):
         path = '%s:%s' % (guid, self.proxy.path)
         if isinstance(path, six.text_type):
             path = path.encode('utf-8')
-        expected = hashlib.md5(path).hexdigest()
+        expected = md5(path, usedforsecurity=False).hexdigest()
         self.assertEqual(expected, self.proxy.lock)
 
     def test_url(self):

cinder/tests/unit/volume/drivers/nexenta/test_nexenta5_nfs.py

@@ -13,10 +13,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 """Unit tests for OpenStack Cinder volume driver."""
-import hashlib
 import os
 from unittest import mock
 
+from oslo_utils.secretutils import md5
 from oslo_utils import units
 
 from cinder import context
@@ -802,7 +802,7 @@ class TestNexentaNfsDriver(test.TestCase):
         result = self.drv._local_volume_dir(volume)
         get_share.assert_called_with(volume)
         share = share.encode('utf-8')
-        digest = hashlib.md5(share).hexdigest()
+        digest = md5(share, usedforsecurity=False).hexdigest()
         expected = os.path.join(self.cfg.nexenta_mount_point_base, digest)
         self.assertEqual(expected, result)
 

cinder/tests/unit/volume/drivers/veritas_access/test_veritas_iscsi.py

@@ -14,13 +14,13 @@
 """
 Unit tests for Veritas Access cinder driver.
 """
-import hashlib
 import json
 import tempfile
 from unittest import mock
 from xml.dom.minidom import Document
 
 from oslo_config import cfg
+from oslo_utils.secretutils import md5
 import requests
 
 from cinder import context
@@ -225,8 +225,10 @@ class ACCESSIscsiDriverTestCase(test.TestCase):
         index = int(length / 2)
         name1 = self.volume.id[:index]
         name2 = self.volume.id[index:]
-        crc1 = hashlib.md5(name1.encode('utf-8')).hexdigest()[:5]
-        crc2 = hashlib.md5(name2.encode('utf-8')).hexdigest()[:5]
+        crc1 = md5(name1.encode('utf-8'),
+                   usedforsecurity=False).hexdigest()[:5]
+        crc2 = md5(name2.encode('utf-8'),
+                   usedforsecurity=False).hexdigest()[:5]
 
         volume_name_to_ret = 'cinder' + '-' + crc1 + '-' + crc2
 

cinder/volume/drivers/dell_emc/powermax/utils.py

@@ -15,10 +15,10 @@
 
 from copy import deepcopy
 import datetime
-import hashlib
 import re
 
 from oslo_log import log as logging
+from oslo_utils.secretutils import md5
 from oslo_utils import strutils
 from oslo_utils import units
 import packaging.version
@@ -431,7 +431,7 @@ class PowerMaxUtils(object):
         :returns: uuid
         """
         input_str = input_str.lower()
-        m = hashlib.md5()
+        m = md5(usedforsecurity=False)
         m.update(input_str.encode('utf-8'))
         return m.hexdigest()
 

cinder/volume/drivers/fujitsu/eternus_dx/eternus_dx_common.py

@@ -20,7 +20,6 @@
 
 import ast
 import base64
-import hashlib
 import time
 
 from lxml import etree as ET
@@ -28,6 +27,7 @@ from oslo_concurrency import lockutils
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_service import loopingcall
+from oslo_utils.secretutils import md5
 from oslo_utils import units
 import six
 
@@ -1102,7 +1102,7 @@ class FJDXCommon(object):
             LOG.error(msg)
             raise exception.VolumeBackendAPIException(data=msg)
 
-        m = hashlib.md5()
+        m = md5(usedforsecurity=False)
         m.update(id_code.encode('utf-8'))
 
         # pylint: disable=E1121

cinder/volume/drivers/huawei/huawei_utils.py

@@ -13,11 +13,11 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import hashlib
 import json
 import math
 
 from oslo_log import log as logging
+from oslo_utils.secretutils import md5
 from oslo_utils import strutils
 import six
 
@@ -36,7 +36,8 @@ LOG = logging.getLogger(__name__)
 
 
 def encode_name(name):
-    encoded_name = hashlib.md5(name.encode('utf-8')).hexdigest()
+    encoded_name = md5(name.encode('utf-8'),
+                       usedforsecurity=False).hexdigest()
     prefix = name.split('-')[0] + '-'
     postfix = encoded_name[:constants.MAX_NAME_LENGTH - len(prefix)]
     return prefix + postfix
@@ -54,7 +55,8 @@ def old_encode_name(name):
 
 def encode_host_name(name):
     if name and len(name) > constants.MAX_NAME_LENGTH:
-        encoded_name = hashlib.md5(name.encode('utf-8')).hexdigest()
+        encoded_name = md5(name.encode('utf-8'),
+                           usedforsecurity=False).hexdigest()
         return encoded_name[:constants.MAX_NAME_LENGTH]
     return name
 

cinder/volume/drivers/nexenta/nfs.py

@@ -13,13 +13,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import hashlib
 import os
 import re
 
 from eventlet import greenthread
 from oslo_log import log as logging
 from oslo_utils import fileutils
+from oslo_utils.secretutils import md5
 from oslo_utils import units
 import six
 
@@ -613,7 +613,7 @@ class NexentaNfsDriver(nfs.NfsDriver):  # pylint: disable=R0921
         """
         nfs_share = nfs_share.encode('utf-8')
         return os.path.join(self.configuration.nexenta_mount_point_base,
-                            hashlib.md5(nfs_share).hexdigest())
+                            md5(nfs_share, usedforsecurity=False).hexdigest())
 
     def remote_path(self, volume):
         """Get volume path (mounted remotely fs path) for given volume.

cinder/volume/drivers/nexenta/ns5/jsonrpc.py

@@ -13,12 +13,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import hashlib
 import json
 import posixpath
 
 from eventlet import greenthread
 from oslo_log import log as logging
+from oslo_utils.secretutils import md5
 import requests
 import six
 
@@ -601,7 +601,7 @@ class NefProxy(object):
         path = '%s:%s' % (guid, self.path)
         if isinstance(path, six.text_type):
             path = path.encode('utf-8')
-        self.lock = hashlib.md5(path).hexdigest()
+        self.lock = md5(path, usedforsecurity=False).hexdigest()
 
     def url(self, path):
         netloc = '%s:%d' % (self.host, int(self.port))

cinder/volume/drivers/nexenta/ns5/nfs.py

@@ -14,12 +14,12 @@
 #    under the License.
 
 import errno
-import hashlib
 import os
 import posixpath
 import uuid
 
 from oslo_log import log as logging
+from oslo_utils.secretutils import md5
 from oslo_utils import units
 import six
 
@@ -770,7 +770,7 @@ class NexentaNfsDriver(nfs.NfsDriver):
         share = self._get_volume_share(volume)
         if isinstance(share, six.text_type):
             share = share.encode('utf-8')
-        path = hashlib.md5(share).hexdigest()
+        path = md5(share, usedforsecurity=False).hexdigest()
         return os.path.join(self.mount_point_base, path)
 
     def local_path(self, volume):

cinder/volume/drivers/remotefs.py

@@ -17,7 +17,6 @@
 import binascii
 import collections
 import errno
-import hashlib
 import inspect
 import json
 import math
@@ -32,6 +31,7 @@ from castellan import key_manager
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_serialization import jsonutils
+from oslo_utils.secretutils import md5
 from oslo_utils import units
 import six
 
@@ -990,7 +990,7 @@ class RemoteFSSnapDriverBase(RemoteFSDriver):
         """
         if isinstance(base_str, six.text_type):
             base_str = base_str.encode('utf-8')
-        return hashlib.md5(base_str).hexdigest()
+        return md5(base_str, usedforsecurity=False).hexdigest()
 
     def _get_mount_point_for_share(self, share):
         """Return mount point for share.

cinder/volume/drivers/stx/client.py

@@ -113,6 +113,10 @@ class STXClient(object):
     def _get_session_key(self):
         """Retrieve a session key from the array."""
 
+        # TODO(alee): This appears to use md5 in a security related
+        # context in providing a session key and hashing a login and
+        # password.  This should likely be replaced by a version that
+        # does not use md5 here.
         self._session_key = None
         hash_ = "%s_%s" % (self._login, self._password)
         if six.PY3:

cinder/volume/drivers/synology/synology_common.py

@@ -15,7 +15,6 @@
 
 import base64
 import functools
-import hashlib
 import json
 import math
 from os import urandom
@@ -32,6 +31,7 @@ import eventlet
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_utils import excutils
+from oslo_utils.secretutils import md5
 from oslo_utils import units
 import requests
 from six.moves import urllib
@@ -112,11 +112,13 @@ class AESCipher(object):
         bs = self._bs
         return (s + (bs - len(s) % bs) * chr(bs - len(s) % bs)).encode('utf-8')
 
+    # TODO(alee): This probably needs to be replaced with a version that
+    # does not use md5, as this will be disallowed on a FIPS enabled system
     def _derive_key_and_iv(self, password, salt, key_length, iv_length):
         d = d_i = b''
         while len(d) < key_length + iv_length:
             md5_str = d_i + password + salt
-            d_i = hashlib.md5(md5_str).digest()
+            d_i = md5(md5_str, usedforsecurity=True).digest()
             d += d_i
         return d[:key_length], d[key_length:key_length + iv_length]
 

cinder/volume/drivers/veritas_access/veritas_iscsi.py

@@ -16,7 +16,6 @@ Veritas Access Driver for ISCSI.
 
 """
 import ast
-import hashlib
 import json
 from random import randint
 from xml.dom import minidom
@@ -25,6 +24,7 @@ from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_service import loopingcall
 from oslo_utils import netutils
+from oslo_utils.secretutils import md5
 from oslo_utils import strutils
 from oslo_utils import units
 import requests
@@ -164,8 +164,10 @@ class ACCESSIscsiDriver(driver.ISCSIDriver):
         index = int(length / 2)
         name1 = name[:index]
         name2 = name[index:]
-        crc1 = hashlib.md5(name1.encode('utf-8')).hexdigest()[:5]
-        crc2 = hashlib.md5(name2.encode('utf-8')).hexdigest()[:5]
+        crc1 = md5(name1.encode('utf-8'),
+                   usedforsecurity=False).hexdigest()[:5]
+        crc2 = md5(name2.encode('utf-8'),
+                   usedforsecurity=False).hexdigest()[:5]
         return 'cinder' + '-' + crc1 + '-' + crc2
 
     def check_for_setup_error(self):

lower-constraints.txt

@@ -76,7 +76,7 @@ oslo.reports==1.18.0
 oslo.rootwrap==5.8.0
 oslo.serialization==2.25.0
 oslo.service==2.0.0
-oslo.utils==3.40.2
+oslo.utils==4.7.0
 oslo.versionedobjects==1.31.2
 oslo.vmware==2.35.0
 oslotest==3.2.0

requirements.txt

@@ -28,7 +28,7 @@ oslo.rootwrap>=5.8.0 # Apache-2.0
 oslo.serialization>=2.25.0 # Apache-2.0
 oslo.service>=2.0.0 # Apache-2.0
 oslo.upgradecheck>=0.1.0 # Apache-2.0
-oslo.utils>=3.40.2 # Apache-2.0
+oslo.utils>=4.7.0 # Apache-2.0
 oslo.versionedobjects>=1.31.2 # Apache-2.0
 osprofiler>=1.4.0 # Apache-2.0
 packaging>=20.4