Reduce deprecation warnings

This makes several types of changes to get rid of all of the deprecation
warnings that are spewed during our unit test runs.

Context changes:
    The user_domain and project_domain properties have been deprecated
    and user_domain_id and project_domain_id should be used instead. We
    have used the newer correct properties for some time now, but for
    compatibility we were still setting the old properties as well.

Collections.abc:
    Between py2 and py3, some abstract classes were moved from
    collections to collections.abc. We had some compatibility code to
    try to handle this in most cases, but also had a few cases of using
    the classed directly from the old, deprecated location.

assertRaisesRegexp:
    This was deprecated in favor of assertRaisesRegex.

DefusedXML:
    DefusedXML has deprecated several things now that lxml handles
    things correctly. The project now recommends just using lxml.

Change-Id: Ie2e46b45248d6a06edbe6cffc2eb8e2d341cd4c3
Signed-off-by: Sean McGinnis <sean.mcginnis@gmail.com>

cinder/api/middleware/auth.py

@@ -111,13 +111,13 @@ class CinderKeystoneContext(base_wsgi.Middleware):
             return webob.exc.HTTPUnauthorized()
 
         if req.environ.get('X_PROJECT_DOMAIN_ID'):
-            ctx.project_domain = req.environ['X_PROJECT_DOMAIN_ID']
+            ctx.project_domain_id = req.environ['X_PROJECT_DOMAIN_ID']
 
         if req.environ.get('X_PROJECT_DOMAIN_NAME'):
             ctx.project_domain_name = req.environ['X_PROJECT_DOMAIN_NAME']
 
         if req.environ.get('X_USER_DOMAIN_ID'):
-            ctx.user_domain = req.environ['X_USER_DOMAIN_ID']
+            ctx.user_domain_id = req.environ['X_USER_DOMAIN_ID']
 
         if req.environ.get('X_USER_DOMAIN_NAME'):
             ctx.user_domain_name = req.environ['X_USER_DOMAIN_NAME']

cinder/context.py

@@ -180,10 +180,8 @@ class RequestContext(context.RequestContext):
                    is_admin=values.get('is_admin'),
                    roles=values.get('roles'),
                    auth_token=values.get('auth_token'),
-                   user_domain_id=values.get('user_domain'),
+                   user_domain_id=values.get('user_domain_id'),
-                   project_domain_id=values.get('project_domain'),
+                   project_domain_id=values.get('project_domain_id'),
-                   user_domain=values.get('user_domain'),
-                   project_domain=values.get('project_domain'),
                    )
 
     def authorize(self, action, target=None, target_obj=None, fatal=True):

cinder/scheduler/host_manager.py

@@ -15,7 +15,7 @@
 
 """Manage backends in the current zone."""
 
-import collections
+from collections import abc
 import random
 
 from oslo_config import cfg
@@ -67,7 +67,7 @@ CONF.import_opt('max_over_subscription_ratio', 'cinder.volume.driver')
 LOG = logging.getLogger(__name__)
 
 
-class ReadOnlyDict(collections.Mapping):
+class ReadOnlyDict(abc.Mapping):
     """A read-only dict."""
     def __init__(self, source=None):
         if source is not None:

cinder/tests/unit/volume/drivers/dell_emc/unity/test_replication.py

@@ -123,11 +123,11 @@ class UnityReplicationDeviceTest(unittest.TestCase):
     @ddt.unpack
     def test_init_raise(self, conf_dict):
         self.driver.configuration.replication_device = conf_dict
-        self.assertRaisesRegexp(exception.InvalidConfigurationValue,
+        self.assertRaisesRegex(exception.InvalidConfigurationValue,
-                                'Value .* is not valid for configuration '
+                               'Value .* is not valid for configuration '
-                                'option "unity-backend.replication_device"',
+                               'option "unity-backend.replication_device"',
-                                replication.ReplicationDevice,
+                               replication.ReplicationDevice,
-                                conf_dict, self.driver)
+                               conf_dict, self.driver)
 
     @ddt.data(
         {

cinder/tests/unit/volume/drivers/fusionstorage/test_fs_conf.py

@@ -51,7 +51,8 @@ class FusionStorageConfTestCase(test.TestCase):
 
         config.add_section('manager_ip')
         config.set('manager_ip', 'fake_host', 'fake_ip')
-        config.write(open(self.conf.cinder_fusionstorage_conf_file, 'w'))
+        with open(self.conf.cinder_fusionstorage_conf_file, 'w') as conf_file:
+            config.write(conf_file)
 
     @mock.patch.object(fs_conf.FusionStorageConf, '_encode_authentication')
     @mock.patch.object(fs_conf.FusionStorageConf, '_pools_name')

cinder/tests/unit/volume/drivers/test_qnap.py

@@ -19,8 +19,8 @@ from unittest import mock
 from ddt import data
 from ddt import ddt
 from ddt import unpack
-from defusedxml import cElementTree as ET
 import eventlet
+from lxml import etree as ET
 from oslo_config import cfg
 from oslo_utils import units
 import requests

cinder/tests/unit/volume/drivers/test_seagate.py

@@ -18,7 +18,7 @@
 
 from unittest import mock
 
-from defusedxml import lxml as etree
+from lxml import etree
 import requests
 
 from cinder import exception
@@ -215,7 +215,7 @@ class TestSeagateClient(test.TestCase):
                                      RequestException("error")]
         mock_requests_get.return_value = m
         ret = self.client._api_request('/path')
-        self.assertTrue(type(ret) == etree.RestrictedElement)
+        self.assertTrue(type(ret) == etree._Element)
         self.assertRaises(stx_exception.ConnectionError,
                           self.client._api_request,
                           '/path')

cinder/volume/drivers/fujitsu/eternus_dx/eternus_dx_common.py

@@ -23,7 +23,7 @@ import base64
 import hashlib
 import time
 
-from defusedxml import ElementTree as ET
+from lxml import etree as ET
 from oslo_concurrency import lockutils
 from oslo_config import cfg
 from oslo_log import log as logging

cinder/volume/drivers/huawei/huawei_conf.py

@@ -23,7 +23,7 @@ import base64
 import os
 import re
 
-from defusedxml import ElementTree as ET
+from lxml import etree as ET
 from oslo_log import log as logging
 import six
 

cinder/volume/drivers/nec/volume_common.py

@@ -19,7 +19,7 @@ import os
 import re
 import traceback
 
-from defusedxml import ElementTree
+from lxml import etree
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_utils import excutils
@@ -291,7 +291,7 @@ class MStorageVolumeCommon(object):
         try:
             with open(product, 'r') as f:
                 xml = f.read()
-                root = ElementTree.fromstring(xml)
+                root = etree.fromstring(xml)
                 vendor_name = root.findall('./VendorName')[0].text
 
                 product_dict = {}
@@ -785,7 +785,7 @@ class MStorageVolumeCommon(object):
         return hostports
 
     def configs(self, xml):
-        root = ElementTree.fromstring(xml)
+        root = etree.fromstring(xml)
         pools = self.get_pool_config(xml, root)
         lds, used_ldns = self.get_ld_config(xml, root, pools)
         iscsi_ldsets = self.get_iscsi_ldset_config(xml, root)

cinder/volume/drivers/qnap.py

@@ -23,8 +23,8 @@ import re
 import threading
 import time
 
-from defusedxml import cElementTree as ET
 import eventlet
+from lxml import etree as ET
 from oslo_concurrency import lockutils
 from oslo_config import cfg
 from oslo_log import log as logging

cinder/volume/drivers/quobyte.py

@@ -649,10 +649,11 @@ class QuobyteDriver(remotefs_drv.RemoteFSSnapDriverDistributed):
     def _mount_quobyte(self, quobyte_volume, mount_path, ensure=False):
         """Mount Quobyte volume to mount path."""
         mounted = False
-        for l in QuobyteDriver.read_proc_mount():
+        with QuobyteDriver.read_proc_mount() as proc_mount:
-            if l.split()[1] == mount_path:
+            for l in proc_mount:
-                mounted = True
+                if l.split()[1] == mount_path:
-                break
+                    mounted = True
+                    break
 
         if mounted:
             try:

cinder/volume/drivers/stx/client.py

@@ -19,7 +19,7 @@ import hashlib
 import math
 import time
 
-from defusedxml import lxml as etree
+from lxml import etree
 from oslo_log import log as logging
 from oslo_utils import strutils
 from oslo_utils import units

cinder/volume/drivers/veritas_access/veritas_iscsi.py

@@ -19,8 +19,8 @@ import ast
 import hashlib
 import json
 from random import randint
+from xml.dom import minidom
 
-from defusedxml import minidom
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_service import loopingcall

cinder/volume/drivers/zadara.py

@@ -17,7 +17,7 @@
 This driver requires VPSA with API version 15.07 or higher.
 """
 
-from defusedxml import lxml
+from lxml import etree
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_utils import strutils
@@ -287,7 +287,7 @@ class ZadaraVPSAConnection(object):
             raise exception.BadHTTPResponseStatus(status=response.status_code)
 
         data = response.content
-        xml_tree = lxml.fromstring(data)
+        xml_tree = etree.fromstring(data)
         status = xml_tree.findtext('status')
         if status == '5':
             # Invalid Credentials

lower-constraints.txt

@@ -20,7 +20,6 @@ cursive==0.2.1
 ddt==1.2.1
 debtcollector==1.19.0
 decorator==3.4.0
-defusedxml==0.5.0
 doc8==0.6.0
 docutils==0.14
 dogpile.cache==0.6.5

requirements.txt

@@ -4,7 +4,6 @@
 
 pbr!=2.1.0,>=2.0.0 # Apache-2.0
 decorator>=3.4.0 # BSD
-defusedxml>=0.5.0 # PSF
 eventlet!=0.23.0,!=0.25.0,>=0.22.0 # MIT
 greenlet>=0.4.10 # MIT
 httplib2>=0.9.1 # MIT