From 91243e3e100bc9c2477cf637b07fb30a93015cbf Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Thu, 17 Nov 2016 08:43:22 -0500 Subject: [PATCH] Move to tooz hash ring implementation This changes the ironic driver to use the hash ring implementation from tooz, which is nearly identical to nova.hash_ring. Change-Id: I95716057cb1860b1357a61f3396f27ed878964e8 Depends-On: Ic1f8b89b819ace8df9b15c61eaf9bf136ad3166b --- nova/hash_ring.py | 134 ------------------- nova/tests/unit/test_hash_ring.py | 145 --------------------- nova/tests/unit/virt/ironic/test_driver.py | 6 +- nova/virt/ironic/driver.py | 12 +- requirements.txt | 1 + 5 files changed, 13 insertions(+), 285 deletions(-) delete mode 100644 nova/hash_ring.py delete mode 100644 nova/tests/unit/test_hash_ring.py diff --git a/nova/hash_ring.py b/nova/hash_ring.py deleted file mode 100644 index f107d5eda721..000000000000 --- a/nova/hash_ring.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import bisect -import hashlib - -import six - -from nova import exception -from nova.i18n import _ - - -# NOTE(jroll) these constants will be config options in Ocata, when the hash -# ring code is in oslo. -# Number of partitions per service is 2^PARTITION_EXPONENT. -# 5 should be fine for most deployments, as an experimental feature. -PARTITION_EXPONENT = 5 -# This should always be 1 in nova, as two compute daemons handling the same -# node should not be possible. -DISTRIBUTION_REPLICAS = 1 - - -class HashRing(object): - """A stable hash ring. - - We map item N to a host Y based on the closest lower hash: - - - hash(item) -> partition - - hash(host) -> divider - - closest lower divider is the host to use - - we hash each host many times to spread load more finely - as otherwise adding a host gets (on average) 50% of the load of - just one other host assigned to it. - """ - - def __init__(self, hosts): - """Create a new hash ring across the specified hosts. - - :param hosts: an iterable of hosts which will be mapped. - """ - replicas = DISTRIBUTION_REPLICAS - - try: - self.hosts = set(hosts) - self.replicas = replicas if replicas <= len(hosts) else len(hosts) - except TypeError: - raise exception.Invalid( - _("Invalid hosts supplied when building HashRing.")) - - self._host_hashes = {} - for host in hosts: - key = str(host).encode('utf8') - key_hash = hashlib.md5(key) - for p in range(2 ** PARTITION_EXPONENT): - key_hash.update(key) - hashed_key = self._hash2int(key_hash) - self._host_hashes[hashed_key] = host - # Gather the (possibly colliding) resulting hashes into a bisectable - # list. - self._partitions = sorted(self._host_hashes.keys()) - - def _hash2int(self, key_hash): - """Convert the given hash's digest to a numerical value for the ring. - - :returns: An integer equivalent value of the digest. - """ - return int(key_hash.hexdigest(), 16) - - def _get_partition(self, data): - try: - if six.PY3 and data is not None: - data = data.encode('utf-8') - key_hash = hashlib.md5(data) - hashed_key = self._hash2int(key_hash) - position = bisect.bisect(self._partitions, hashed_key) - return position if position < len(self._partitions) else 0 - except TypeError: - raise exception.Invalid( - _("Invalid data supplied to HashRing.get_hosts.")) - - def get_hosts(self, data, ignore_hosts=None): - """Get the list of hosts which the supplied data maps onto. - - :param data: A string identifier to be mapped across the ring. - :param ignore_hosts: A list of hosts to skip when performing the hash. - Useful to temporarily skip down hosts without - performing a full rebalance. - Default: None. - :returns: a list of hosts. - The length of this list depends on the number of replicas - this `HashRing` was created with. It may be less than this - if ignore_hosts is not None. - """ - hosts = [] - if ignore_hosts is None: - ignore_hosts = set() - else: - ignore_hosts = set(ignore_hosts) - ignore_hosts.intersection_update(self.hosts) - partition = self._get_partition(data) - for replica in range(0, self.replicas): - if len(hosts) + len(ignore_hosts) == len(self.hosts): - # prevent infinite loop - cannot allocate more fallbacks. - break - # Linear probing: partition N, then N+1 etc. - host = self._get_host(partition) - while host in hosts or host in ignore_hosts: - partition += 1 - if partition >= len(self._partitions): - partition = 0 - host = self._get_host(partition) - hosts.append(host) - return hosts - - def _get_host(self, partition): - """Find what host is serving a partition. - - :param partition: The index of the partition in the partition map. - e.g. 0 is the first partition, 1 is the second. - :return: The host object the ring was constructed with. - """ - return self._host_hashes[self._partitions[partition]] diff --git a/nova/tests/unit/test_hash_ring.py b/nova/tests/unit/test_hash_ring.py deleted file mode 100644 index 0a84c11bb992..000000000000 --- a/nova/tests/unit/test_hash_ring.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import hashlib - -import mock -from testtools import matchers - -from nova import exception -from nova import hash_ring -from nova import test - - -class HashRingTestCase(test.TestCase): - - # NOTE(deva): the mapping used in these tests is as follows: - # if hosts = [foo, bar]: - # fake -> foo, bar - # if hosts = [foo, bar, baz]: - # fake -> foo, bar, baz - # fake-again -> bar, baz, foo - - @mock.patch.object(hashlib, 'md5', autospec=True) - def test__hash2int_returns_int(self, mock_md5): - r1 = 32 * 'a' - r2 = 32 * 'b' - # 2**PARTITION_EXPONENT calls to md5.update per host - # PARTITION_EXPONENT is currently always 5, so 32 calls each here - mock_md5.return_value.hexdigest.side_effect = [r1] * 32 + [r2] * 32 - - hosts = ['foo', 'bar'] - ring = hash_ring.HashRing(hosts) - - self.assertIn(int(r1, 16), ring._host_hashes) - self.assertIn(int(r2, 16), ring._host_hashes) - - def test_create_ring(self): - hosts = ['foo', 'bar'] - ring = hash_ring.HashRing(hosts) - self.assertEqual(set(hosts), ring.hosts) - self.assertEqual(1, ring.replicas) - self.assertEqual(2 ** 5 * 2, len(ring._partitions)) - - def test_distribution_one_replica(self): - hosts = ['foo', 'bar', 'baz'] - ring = hash_ring.HashRing(hosts) - fake_1_hosts = ring.get_hosts('fake') - fake_2_hosts = ring.get_hosts('fake-again') - # We should have one hosts for each thing - self.assertThat(fake_1_hosts, matchers.HasLength(1)) - self.assertThat(fake_2_hosts, matchers.HasLength(1)) - # And they must not be the same answers even on this simple data. - self.assertNotEqual(fake_1_hosts, fake_2_hosts) - - def test_ignore_hosts(self): - hosts = ['foo', 'bar', 'baz'] - ring = hash_ring.HashRing(hosts) - equals_bar_or_baz = matchers.MatchesAny( - matchers.Equals(['bar']), - matchers.Equals(['baz'])) - self.assertThat( - ring.get_hosts('fake', ignore_hosts=['foo']), - equals_bar_or_baz) - self.assertThat( - ring.get_hosts('fake', ignore_hosts=['foo', 'bar']), - equals_bar_or_baz) - self.assertEqual([], ring.get_hosts('fake', ignore_hosts=hosts)) - - def _compare_rings(self, nodes, conductors, ring, - new_conductors, new_ring): - delta = {} - mapping = {'node': ring.get_hosts(node)[0] for node in nodes} - new_mapping = {'node': new_ring.get_hosts(node)[0] for node in nodes} - - for key, old in mapping.items(): - new = new_mapping.get(key, None) - if new != old: - delta[key] = (old, new) - return delta - - def test_rebalance_stability_join(self): - num_services = 10 - num_nodes = 10000 - # Adding 1 service to a set of N should move 1/(N+1) of all nodes - # Eg, for a cluster of 10 nodes, adding one should move 1/11, or 9% - # We allow for 1/N to allow for rounding in tests. - redistribution_factor = 1.0 / num_services - - nodes = [str(x) for x in range(num_nodes)] - services = [str(x) for x in range(num_services)] - new_services = services + ['new'] - delta = self._compare_rings( - nodes, services, hash_ring.HashRing(services), - new_services, hash_ring.HashRing(new_services)) - - self.assertLess(len(delta), num_nodes * redistribution_factor) - - def test_rebalance_stability_leave(self): - num_services = 10 - num_nodes = 10000 - # Removing 1 service from a set of N should move 1/(N) of all nodes - # Eg, for a cluster of 10 nodes, removing one should move 1/10, or 10% - # We allow for 1/(N-1) to allow for rounding in tests. - redistribution_factor = 1.0 / (num_services - 1) - - nodes = [str(x) for x in range(num_nodes)] - services = [str(x) for x in range(num_services)] - new_services = services[:] - new_services.pop() - delta = self._compare_rings( - nodes, services, hash_ring.HashRing(services), - new_services, hash_ring.HashRing(new_services)) - - self.assertLess(len(delta), num_nodes * redistribution_factor) - - def test_ignore_non_existent_host(self): - hosts = ['foo', 'bar'] - ring = hash_ring.HashRing(hosts) - self.assertEqual(['foo'], ring.get_hosts('fake', - ignore_hosts=['baz'])) - - def test_create_ring_invalid_data(self): - hosts = None - self.assertRaises(exception.Invalid, - hash_ring.HashRing, - hosts) - - def test_get_hosts_invalid_data(self): - hosts = ['foo', 'bar'] - ring = hash_ring.HashRing(hosts) - self.assertRaises(exception.Invalid, - ring.get_hosts, - None) diff --git a/nova/tests/unit/virt/ironic/test_driver.py b/nova/tests/unit/virt/ironic/test_driver.py index 23e1e23814cd..a25e4a143e5b 100644 --- a/nova/tests/unit/virt/ironic/test_driver.py +++ b/nova/tests/unit/virt/ironic/test_driver.py @@ -22,6 +22,7 @@ from oslo_service import loopingcall from oslo_utils import uuidutils import six from testtools import matchers +from tooz import hashring as hash_ring from nova.api.metadata import base as instance_metadata from nova.compute import power_state as nova_states @@ -30,7 +31,6 @@ from nova.compute import vm_states from nova.console import type as console_type from nova import context as nova_context from nova import exception -from nova import hash_ring from nova import objects from nova import servicegroup from nova import test @@ -1805,7 +1805,7 @@ class HashRingTestCase(test.NoDBTestCase): mock_services.assert_called_once_with( mock.ANY, self.driver._get_hypervisor_type()) - mock_hash_ring.assert_called_once_with(expected_hosts) + mock_hash_ring.assert_called_once_with(expected_hosts, partitions=32) self.assertEqual(SENTINEL, self.driver.hash_ring) self.mock_is_up.assert_has_calls(is_up_calls) @@ -1854,7 +1854,7 @@ class NodeCacheTestCase(test.NoDBTestCase): self.flags(host=self.host) @mock.patch.object(ironic_driver.IronicDriver, '_refresh_hash_ring') - @mock.patch.object(hash_ring.HashRing, 'get_hosts') + @mock.patch.object(hash_ring.HashRing, 'get_nodes') @mock.patch.object(ironic_driver.IronicDriver, '_get_node_list') @mock.patch.object(objects.InstanceList, 'get_uuids_by_host') def _test__refresh_cache(self, instances, nodes, hosts, mock_instances, diff --git a/nova/virt/ironic/driver.py b/nova/virt/ironic/driver.py index dfcd067240a3..92b864aaa6ec 100644 --- a/nova/virt/ironic/driver.py +++ b/nova/virt/ironic/driver.py @@ -32,6 +32,7 @@ from oslo_utils import excutils from oslo_utils import importutils import six import six.moves.urllib.parse as urlparse +from tooz import hashring as hash_ring from nova.api.metadata import base as instance_metadata from nova.compute import power_state @@ -41,7 +42,6 @@ import nova.conf from nova.console import type as console_type from nova import context as nova_context from nova import exception -from nova import hash_ring from nova.i18n import _ from nova.i18n import _LE from nova.i18n import _LI @@ -82,6 +82,10 @@ _NODE_FIELDS = ('uuid', 'power_state', 'target_power_state', 'provision_state', # Console state checking interval in seconds _CONSOLE_STATE_CHECKING_INTERVAL = 1 +# Number of hash ring partitions per service +# 5 should be fine for most deployments, as an experimental feature. +_HASH_RING_PARTITIONS = 2 ** 5 + def map_power_state(state): try: @@ -549,7 +553,8 @@ class IronicDriver(virt_driver.ComputeDriver): # table will be here so far, and we might be brand new. services.add(CONF.host) - self.hash_ring = hash_ring.HashRing(services) + self.hash_ring = hash_ring.HashRing(services, + partitions=_HASH_RING_PARTITIONS) def _refresh_cache(self): # NOTE(lucasagomes): limit == 0 is an indicator to continue @@ -571,7 +576,8 @@ class IronicDriver(virt_driver.ComputeDriver): # nova while the service was down, and not yet reaped, will not be # reported until the periodic task cleans it up. elif (node.instance_uuid is None and - CONF.host in self.hash_ring.get_hosts(node.uuid)): + CONF.host in + self.hash_ring.get_nodes(node.uuid.encode('utf-8'))): node_cache[node.uuid] = node self.node_cache = node_cache diff --git a/requirements.txt b/requirements.txt index 4ef92e206a23..3c141bee350d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -59,3 +59,4 @@ os-win>=1.3.0 # Apache-2.0 castellan>=0.4.0 # Apache-2.0 microversion-parse>=0.1.2 # Apache-2.0 os-xenapi>=0.1.1 # Apache-2.0 +tooz>=1.47.0 # Apache-2.0