diff --git a/cinder/api/api_utils.py b/cinder/api/api_utils.py index 1d700502b76..d60bfd40033 100644 --- a/cinder/api/api_utils.py +++ b/cinder/api/api_utils.py @@ -130,3 +130,16 @@ def validate_integer(value, name, min_value=None, max_value=None): return value except ValueError as e: raise webob.exc.HTTPBadRequest(explanation=six.text_type(e)) + + +def walk_class_hierarchy(clazz, encountered=None): + """Walk class hierarchy, yielding most derived classes first.""" + if not encountered: + encountered = [] + for subclass in clazz.__subclasses__(): + if subclass not in encountered: + encountered.append(subclass) + # drill down to leaves first + for subsubclass in walk_class_hierarchy(subclass, encountered): + yield subsubclass + yield subclass diff --git a/cinder/api/middleware/fault.py b/cinder/api/middleware/fault.py index 862fe25e481..463e8579263 100644 --- a/cinder/api/middleware/fault.py +++ b/cinder/api/middleware/fault.py @@ -20,10 +20,10 @@ from six.moves import http_client import webob.dec import webob.exc +from cinder.api import api_utils from cinder.api.openstack import wsgi from cinder import exception from cinder.i18n import _ -from cinder import utils from cinder.wsgi import common as base_wsgi @@ -38,7 +38,7 @@ class FaultWrapper(base_wsgi.Middleware): @staticmethod def status_to_type(status): if not FaultWrapper._status_to_type: - for clazz in utils.walk_class_hierarchy(webob.exc.HTTPError): + for clazz in api_utils.walk_class_hierarchy(webob.exc.HTTPError): FaultWrapper._status_to_type[clazz.code] = clazz return FaultWrapper._status_to_type.get( status, webob.exc.HTTPInternalServerError)() diff --git a/cinder/tests/unit/test_qos_specs.py b/cinder/tests/unit/test_qos_specs.py index 2722756793e..38f38d9f7f3 100644 --- a/cinder/tests/unit/test_qos_specs.py +++ b/cinder/tests/unit/test_qos_specs.py @@ -29,7 +29,7 @@ from cinder import db from cinder import exception from cinder import test from cinder.tests.unit import fake_constants as fake -from cinder import utils +from cinder.tests.unit import utils as test_utils from cinder.volume import qos_specs from cinder.volume import volume_types @@ -374,7 +374,7 @@ class QoSSpecsTestCase(test.TestCase): qos_specs_dict['specs']) qos_specs_dict['id'] = qos_specs_id specs = db.qos_specs_get(self.ctxt, qos_specs_id) - qos_specs_list[index]['created_at'] = utils.time_format( + qos_specs_list[index]['created_at'] = test_utils.time_format( specs['created_at']) res = qos_specs.get_all_specs(self.ctxt) diff --git a/cinder/tests/unit/test_utils.py b/cinder/tests/unit/test_utils.py index 3d83cb07200..57b6e3c477f 100644 --- a/cinder/tests/unit/test_utils.py +++ b/cinder/tests/unit/test_utils.py @@ -33,6 +33,7 @@ from cinder import exception from cinder import test from cinder.tests.unit import fake_constants as fake from cinder import utils +from cinder.volume import utils as volume_utils POOL_CAPS = {'total_capacity_gb': 0, 'free_capacity_gb': 0, @@ -124,26 +125,26 @@ class GenericUtilsTestCase(test.TestCase): def test_hostname_unicode_sanitization(self): hostname = u"\u7684.test.example.com" self.assertEqual("test.example.com", - utils.sanitize_hostname(hostname)) + volume_utils.sanitize_hostname(hostname)) def test_hostname_sanitize_periods(self): hostname = "....test.example.com..." self.assertEqual("test.example.com", - utils.sanitize_hostname(hostname)) + volume_utils.sanitize_hostname(hostname)) def test_hostname_sanitize_dashes(self): hostname = "----test.example.com---" self.assertEqual("test.example.com", - utils.sanitize_hostname(hostname)) + volume_utils.sanitize_hostname(hostname)) def test_hostname_sanitize_characters(self): hostname = "(#@&$!(@*--#&91)(__=+--test-host.example!!.com-0+" self.assertEqual("91----test-host.example.com-0", - utils.sanitize_hostname(hostname)) + volume_utils.sanitize_hostname(hostname)) def test_hostname_translate(self): hostname = "<}\x1fh\x10e\x08l\x02l\x05o\x12!{>" - self.assertEqual("hello", utils.sanitize_hostname(hostname)) + self.assertEqual("hello", volume_utils.sanitize_hostname(hostname)) @mock.patch('os.path.join', side_effect=lambda x, y: '/'.join((x, y))) def test_make_dev_path(self, mock_join): @@ -385,11 +386,11 @@ class WalkClassHierarchyTestCase(test.TestCase): pass class_pairs = zip((D, B, E), - utils.walk_class_hierarchy(A, encountered=[C])) + api_utils.walk_class_hierarchy(A, encountered=[C])) for actual, expected in class_pairs: self.assertEqual(expected, actual) - class_pairs = zip((D, B, C, E), utils.walk_class_hierarchy(A)) + class_pairs = zip((D, B, C, E), api_utils.walk_class_hierarchy(A)) for actual, expected in class_pairs: self.assertEqual(expected, actual) diff --git a/cinder/tests/unit/utils.py b/cinder/tests/unit/utils.py index 4b849e77c29..6949e968ea2 100644 --- a/cinder/tests/unit/utils.py +++ b/cinder/tests/unit/utils.py @@ -556,3 +556,19 @@ def set_timeout(timeout): return _wrapper return _decorator + + +def time_format(at=None): + """Format datetime string to date. + + :param at: Type is datetime.datetime (example + 'datetime.datetime(2017, 12, 24, 22, 11, 32, 6086)') + :returns: Format date (example '2017-12-24T22:11:32Z'). + """ + if not at: + at = timeutils.utcnow() + date_string = at.strftime("%Y-%m-%dT%H:%M:%S") + tz = at.tzname(None) if at.tzinfo else 'UTC' + # Need to handle either iso8601 or python UTC format + date_string += ('Z' if tz in ['UTC', 'UTC+00:00'] else tz) + return date_string diff --git a/cinder/tests/unit/volume/drivers/netapp/dataontap/client/test_client_cmode.py b/cinder/tests/unit/volume/drivers/netapp/dataontap/client/test_client_cmode.py index fe0a5e89f2e..12fb2c8ea22 100644 --- a/cinder/tests/unit/volume/drivers/netapp/dataontap/client/test_client_cmode.py +++ b/cinder/tests/unit/volume/drivers/netapp/dataontap/client/test_client_cmode.py @@ -1055,7 +1055,7 @@ class NetAppCmodeClientTestCase(test.TestCase): mock.call('qos-policy-group-delete-iter', api_args, False)]) self.assertEqual(1, mock_log.call_count) - @mock.patch('cinder.utils.resolve_hostname', + @mock.patch('cinder.volume.utils.resolve_hostname', return_value='192.168.1.101') def test_get_if_info_by_ip_not_found(self, mock_resolve_hostname): fake_ip = '192.168.1.101' @@ -1070,7 +1070,7 @@ class NetAppCmodeClientTestCase(test.TestCase): self.assertRaises(exception.NotFound, self.client.get_if_info_by_ip, fake_ip) - @mock.patch('cinder.utils.resolve_hostname', + @mock.patch('cinder.volume.utils.resolve_hostname', return_value='192.168.1.101') def test_get_if_info_by_ip(self, mock_resolve_hostname): fake_ip = '192.168.1.101' diff --git a/cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_base.py b/cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_base.py index 4f0bdf4d897..cc5b4ff1640 100644 --- a/cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_base.py +++ b/cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_base.py @@ -41,6 +41,7 @@ from cinder.volume.drivers.netapp.dataontap.utils import loopingcalls from cinder.volume.drivers.netapp import utils as na_utils from cinder.volume.drivers import nfs from cinder.volume.drivers import remotefs +from cinder.volume import utils as volume_utils @ddt.ddt @@ -651,7 +652,7 @@ class NetAppNfsDriverTestCase(test.TestCase): fake.NFS_SHARE) def test_get_share_mount_and_vol_from_vol_ref(self): - self.mock_object(utils, 'resolve_hostname', + self.mock_object(volume_utils, 'resolve_hostname', return_value='10.12.142.11') self.mock_object(os.path, 'isfile', return_value=True) self.driver._mounted_shares = [self.fake_nfs_export_1] @@ -669,7 +670,7 @@ class NetAppNfsDriverTestCase(test.TestCase): self.assertEqual('test_file_name', file_path) def test_get_share_mount_and_vol_from_vol_ref_with_bad_ref(self): - self.mock_object(utils, 'resolve_hostname', + self.mock_object(volume_utils, 'resolve_hostname', return_value='10.12.142.11') self.driver._mounted_shares = [self.fake_nfs_export_1] vol_ref = {'source-id': '1234546'} @@ -683,7 +684,7 @@ class NetAppNfsDriverTestCase(test.TestCase): vol_ref) def test_get_share_mount_and_vol_from_vol_ref_where_not_found(self): - self.mock_object(utils, 'resolve_hostname', + self.mock_object(volume_utils, 'resolve_hostname', return_value='10.12.142.11') self.driver._mounted_shares = [self.fake_nfs_export_1] vol_path = "%s/%s" % (self.fake_nfs_export_2, 'test_file_name') @@ -698,7 +699,7 @@ class NetAppNfsDriverTestCase(test.TestCase): vol_ref) def test_get_share_mount_and_vol_from_vol_ref_where_is_dir(self): - self.mock_object(utils, 'resolve_hostname', + self.mock_object(volume_utils, 'resolve_hostname', return_value='10.12.142.11') self.driver._mounted_shares = [self.fake_nfs_export_1] vol_ref = {'source-name': self.fake_nfs_export_2} diff --git a/cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_cmode.py b/cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_cmode.py index b3b6f3defcd..133bf968462 100644 --- a/cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_cmode.py +++ b/cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_cmode.py @@ -307,7 +307,7 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase): self.driver.zapi_client, 'get_operational_lif_addresses', return_value=[fake.SHARE_IP]) mock_resolve_hostname = self.mock_object( - utils, 'resolve_hostname', return_value=fake.SHARE_IP) + volume_utils, 'resolve_hostname', return_value=fake.SHARE_IP) mock_get_flexvol = self.mock_object( self.driver.zapi_client, 'get_flexvol', return_value={'name': fake.NETAPP_VOLUME}) @@ -330,7 +330,7 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase): self.mock_object(self.driver.zapi_client, 'get_operational_lif_addresses', return_value=[]) - self.mock_object(utils, + self.mock_object(volume_utils, 'resolve_hostname', return_value=fake.SHARE_IP) @@ -344,7 +344,7 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase): self.mock_object(self.driver.zapi_client, 'get_operational_lif_addresses', return_value=[fake.SHARE_IP]) - self.mock_object(utils, + self.mock_object(volume_utils, 'resolve_hostname', return_value=fake.SHARE_IP) side_effect = exception.VolumeBackendAPIException(data='fake_data') diff --git a/cinder/utils.py b/cinder/utils.py index b7cb9ae9095..b601083eade 100644 --- a/cinder/utils.py +++ b/cinder/utils.py @@ -32,7 +32,6 @@ import pyclbr import random import re import shutil -import socket import stat import sys import tempfile @@ -275,22 +274,6 @@ def last_completed_audit_period(unit=None): return (begin, end) -def time_format(at=None): - """Format datetime string to date. - - :param at: Type is datetime.datetime (example - 'datetime.datetime(2017, 12, 24, 22, 11, 32, 6086)') - :returns: Format date (example '2017-12-24T22:11:32Z'). - """ - if not at: - at = timeutils.utcnow() - date_string = at.strftime("%Y-%m-%dT%H:%M:%S") - tz = at.tzname(None) if at.tzinfo else 'UTC' - # Need to handle either iso8601 or python UTC format - date_string += ('Z' if tz in ['UTC', 'UTC+00:00'] else tz) - return date_string - - def monkey_patch(): """Patches decorators for all functions in a specified module. @@ -353,23 +336,6 @@ def make_dev_path(dev, partition=None, base='/dev'): return path -def sanitize_hostname(hostname): - """Return a hostname which conforms to RFC-952 and RFC-1123 specs.""" - if six.PY3: - hostname = hostname.encode('latin-1', 'ignore') - hostname = hostname.decode('latin-1') - else: - if isinstance(hostname, six.text_type): - hostname = hostname.encode('latin-1', 'ignore') - - hostname = re.sub(r'[ _]', '-', hostname) - hostname = re.sub(r'[^\w.-]+', '', hostname) - hostname = hostname.lower() - hostname = hostname.strip('.-') - - return hostname - - def robust_file_write(directory, filename, data): """Robust file write. @@ -453,19 +419,6 @@ def tempdir(**kwargs): six.text_type(e)) -def walk_class_hierarchy(clazz, encountered=None): - """Walk class hierarchy, yielding most derived classes first.""" - if not encountered: - encountered = [] - for subclass in clazz.__subclasses__(): - if subclass not in encountered: - encountered.append(subclass) - # drill down to leaves first - for subsubclass in walk_class_hierarchy(subclass, encountered): - yield subsubclass - yield subclass - - def get_root_helper(): return 'sudo cinder-rootwrap %s' % CONF.rootwrap_config @@ -922,22 +875,6 @@ def setup_tracing(trace_flags): TRACE_API = 'api' in trace_flags -def resolve_hostname(hostname): - """Resolves host name to IP address. - - Resolves a host name (my.data.point.com) to an IP address (10.12.143.11). - This routine also works if the data passed in hostname is already an IP. - In this case, the same IP address will be returned. - - :param hostname: Host name to resolve. - :returns: IP Address for Host name. - """ - ip = socket.getaddrinfo(hostname, None)[0][4][0] - LOG.debug('Asked to resolve hostname %(host)s and got IP %(ip)s.', - {'host': hostname, 'ip': ip}) - return ip - - def build_or_str(elements, str_format=None): """Builds a string of elements joined by 'or'. diff --git a/cinder/volume/drivers/netapp/dataontap/client/client_cmode.py b/cinder/volume/drivers/netapp/dataontap/client/client_cmode.py index e7de3ee22d0..b676aadd8eb 100644 --- a/cinder/volume/drivers/netapp/dataontap/client/client_cmode.py +++ b/cinder/volume/drivers/netapp/dataontap/client/client_cmode.py @@ -30,6 +30,7 @@ from cinder import utils from cinder.volume.drivers.netapp.dataontap.client import api as netapp_api from cinder.volume.drivers.netapp.dataontap.client import client_base from cinder.volume.drivers.netapp import utils as na_utils +from cinder.volume import utils as volume_utils from oslo_utils import strutils @@ -668,7 +669,7 @@ class Client(client_base.Client): net_if_iter.add_child_elem(query) query.add_node_with_children( 'net-interface-info', - **{'address': utils.resolve_hostname(ip)}) + **{'address': volume_utils.resolve_hostname(ip)}) result = self.connection.invoke_successfully(net_if_iter, True) num_records = result.get_child_content('num-records') if num_records and int(num_records) >= 1: diff --git a/cinder/volume/drivers/netapp/dataontap/nfs_base.py b/cinder/volume/drivers/netapp/dataontap/nfs_base.py index df768d1b95f..4016c2b7f7c 100644 --- a/cinder/volume/drivers/netapp/dataontap/nfs_base.py +++ b/cinder/volume/drivers/netapp/dataontap/nfs_base.py @@ -906,7 +906,7 @@ class NetAppNfsDriver(driver.ManageableVD, # First strip out share and convert to IP format. share_split = vol_ref.rsplit(':', 1) - vol_ref_share_ip = utils.resolve_hostname(share_split[0]) + vol_ref_share_ip = volume_utils.resolve_hostname(share_split[0]) # Now place back into volume reference. vol_ref_share = vol_ref_share_ip + ':' + share_split[1] diff --git a/cinder/volume/drivers/netapp/dataontap/nfs_cmode.py b/cinder/volume/drivers/netapp/dataontap/nfs_cmode.py index fbb50e06cc1..d135cb8f28a 100644 --- a/cinder/volume/drivers/netapp/dataontap/nfs_cmode.py +++ b/cinder/volume/drivers/netapp/dataontap/nfs_cmode.py @@ -327,7 +327,7 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver, for share in self._mounted_shares: host, junction_path = na_utils.get_export_host_junction_path(share) - address = utils.resolve_hostname(host) + address = volume_utils.resolve_hostname(host) if address not in vserver_addresses: LOG.warning('Address not found for NFS share %s.', share) @@ -463,7 +463,7 @@ class NetAppCmodeNfsDriver(nfs_base.NetAppNfsDriver, def _get_ip_verify_on_cluster(self, host): """Verifies if host on same cluster and returns ip.""" - ip = utils.resolve_hostname(host) + ip = volume_utils.resolve_hostname(host) vserver = self._get_vserver_for_ip(ip) if not vserver: raise exception.NotFound(_("Unable to locate an SVM that is " diff --git a/cinder/volume/drivers/zfssa/zfssaiscsi.py b/cinder/volume/drivers/zfssa/zfssaiscsi.py index a46d465adc9..5a8d540ef39 100644 --- a/cinder/volume/drivers/zfssa/zfssaiscsi.py +++ b/cinder/volume/drivers/zfssa/zfssaiscsi.py @@ -263,7 +263,7 @@ class ZFSSAISCSIDriver(driver.ISCSIDriver): # Lookup the zfssa_target_portal DNS name to an IP address host, port = lcfg.zfssa_target_portal.split(':') - host_ip_addr = utils.resolve_hostname(host) + host_ip_addr = volume_utils.resolve_hostname(host) self.zfssa_target_portal = host_ip_addr + ':' + port def check_for_setup_error(self): diff --git a/cinder/volume/manager.py b/cinder/volume/manager.py index fc7fee7ab57..7c4b91deaa6 100644 --- a/cinder/volume/manager.py +++ b/cinder/volume/manager.py @@ -1305,7 +1305,7 @@ class VolumeManager(manager.CleanableManager, raise exception.InvalidVolume( reason=_("being attached by different mode")) - host_name_sanitized = utils.sanitize_hostname( + host_name_sanitized = vol_utils.sanitize_hostname( host_name) if host_name else None if instance_uuid: attachments = ( diff --git a/cinder/volume/utils.py b/cinder/volume/utils.py index 19df5de56bc..4f1bfb66575 100644 --- a/cinder/volume/utils.py +++ b/cinder/volume/utils.py @@ -23,6 +23,7 @@ import operator import os from os import urandom import re +import socket import tempfile import time import uuid @@ -1213,3 +1214,36 @@ def sanitize_host(host): if netutils.is_valid_ipv6(host): return '[%s]' % host return host + + +def sanitize_hostname(hostname): + """Return a hostname which conforms to RFC-952 and RFC-1123 specs.""" + if six.PY3: + hostname = hostname.encode('latin-1', 'ignore') + hostname = hostname.decode('latin-1') + else: + if isinstance(hostname, six.text_type): + hostname = hostname.encode('latin-1', 'ignore') + + hostname = re.sub(r'[ _]', '-', hostname) + hostname = re.sub(r'[^\w.-]+', '', hostname) + hostname = hostname.lower() + hostname = hostname.strip('.-') + + return hostname + + +def resolve_hostname(hostname): + """Resolves host name to IP address. + + Resolves a host name (my.data.point.com) to an IP address (10.12.143.11). + This routine also works if the data passed in hostname is already an IP. + In this case, the same IP address will be returned. + + :param hostname: Host name to resolve. + :returns: IP Address for Host name. + """ + ip = socket.getaddrinfo(hostname, None)[0][4][0] + LOG.debug('Asked to resolve hostname %(host)s and got IP %(ip)s.', + {'host': hostname, 'ip': ip}) + return ip