openstack/nova
Code Issues Proposed changes

Merge "Move to tooz hash ring implementation"

Browse Source
This commit is contained in:
Jenkins 2017-03-20 16:18:53 +00:00 committed by Gerrit Code Review
commit efdc75fcb9
5 changed files with 13 additions and 285 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
nova/hash_ring.py
View File

@ -1,134 +0,0 @@
# Copyright 2013 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import bisect
import hashlib
import six
from nova import exception
from nova.i18n import _
# NOTE(jroll) these constants will be config options in Ocata, when the hash
# ring code is in oslo.
# Number of partitions per service is 2^PARTITION_EXPONENT.
# 5 should be fine for most deployments, as an experimental feature.
PARTITION_EXPONENT = 5
# This should always be 1 in nova, as two compute daemons handling the same
# node should not be possible.
DISTRIBUTION_REPLICAS = 1
class HashRing(object):
"""A stable hash ring.
We map item N to a host Y based on the closest lower hash:
- hash(item) -> partition
- hash(host) -> divider
- closest lower divider is the host to use
- we hash each host many times to spread load more finely
as otherwise adding a host gets (on average) 50% of the load of
just one other host assigned to it.
"""
def __init__(self, hosts):
"""Create a new hash ring across the specified hosts.
:param hosts: an iterable of hosts which will be mapped.
"""
replicas = DISTRIBUTION_REPLICAS
try:
self.hosts = set(hosts)
self.replicas = replicas if replicas <= len(hosts) else len(hosts)
except TypeError:
raise exception.Invalid(
_("Invalid hosts supplied when building HashRing."))
self._host_hashes = {}
for host in hosts:
key = str(host).encode('utf8')
key_hash = hashlib.md5(key)
for p in range(2 ** PARTITION_EXPONENT):
key_hash.update(key)
hashed_key = self._hash2int(key_hash)
self._host_hashes[hashed_key] = host
# Gather the (possibly colliding) resulting hashes into a bisectable
# list.
self._partitions = sorted(self._host_hashes.keys())
def _hash2int(self, key_hash):
"""Convert the given hash's digest to a numerical value for the ring.
:returns: An integer equivalent value of the digest.
"""
return int(key_hash.hexdigest(), 16)
def _get_partition(self, data):
try:
if six.PY3 and data is not None:
data = data.encode('utf-8')
key_hash = hashlib.md5(data)
hashed_key = self._hash2int(key_hash)
position = bisect.bisect(self._partitions, hashed_key)
return position if position < len(self._partitions) else 0
except TypeError:
raise exception.Invalid(
_("Invalid data supplied to HashRing.get_hosts."))
def get_hosts(self, data, ignore_hosts=None):
"""Get the list of hosts which the supplied data maps onto.
:param data: A string identifier to be mapped across the ring.
:param ignore_hosts: A list of hosts to skip when performing the hash.
Useful to temporarily skip down hosts without
performing a full rebalance.
Default: None.
:returns: a list of hosts.
The length of this list depends on the number of replicas
this `HashRing` was created with. It may be less than this
if ignore_hosts is not None.
"""
hosts = []
if ignore_hosts is None:
ignore_hosts = set()
else:
ignore_hosts = set(ignore_hosts)
ignore_hosts.intersection_update(self.hosts)
partition = self._get_partition(data)
for replica in range(0, self.replicas):
if len(hosts) + len(ignore_hosts) == len(self.hosts):
# prevent infinite loop - cannot allocate more fallbacks.
break
# Linear probing: partition N, then N+1 etc.
host = self._get_host(partition)
while host in hosts or host in ignore_hosts:
partition += 1
if partition >= len(self._partitions):
partition = 0
host = self._get_host(partition)
hosts.append(host)
return hosts
def _get_host(self, partition):
"""Find what host is serving a partition.
:param partition: The index of the partition in the partition map.
e.g. 0 is the first partition, 1 is the second.
:return: The host object the ring was constructed with.
"""
return self._host_hashes[self._partitions[partition]]

145
nova/tests/unit/test_hash_ring.py
View File

@ -1,145 +0,0 @@
# Copyright 2013 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import hashlib
import mock
from testtools import matchers
from nova import exception
from nova import hash_ring
from nova import test
class HashRingTestCase(test.TestCase):
# NOTE(deva): the mapping used in these tests is as follows:
# if hosts = [foo, bar]:
# fake -> foo, bar
# if hosts = [foo, bar, baz]:
# fake -> foo, bar, baz
# fake-again -> bar, baz, foo
@mock.patch.object(hashlib, 'md5', autospec=True)
def test__hash2int_returns_int(self, mock_md5):
r1 = 32 * 'a'
r2 = 32 * 'b'
# 2**PARTITION_EXPONENT calls to md5.update per host
# PARTITION_EXPONENT is currently always 5, so 32 calls each here
mock_md5.return_value.hexdigest.side_effect = [r1] * 32 + [r2] * 32
hosts = ['foo', 'bar']
ring = hash_ring.HashRing(hosts)
self.assertIn(int(r1, 16), ring._host_hashes)
self.assertIn(int(r2, 16), ring._host_hashes)
def test_create_ring(self):
hosts = ['foo', 'bar']
ring = hash_ring.HashRing(hosts)
self.assertEqual(set(hosts), ring.hosts)
self.assertEqual(1, ring.replicas)
self.assertEqual(2 ** 5 * 2, len(ring._partitions))
def test_distribution_one_replica(self):
hosts = ['foo', 'bar', 'baz']
ring = hash_ring.HashRing(hosts)
fake_1_hosts = ring.get_hosts('fake')
fake_2_hosts = ring.get_hosts('fake-again')
# We should have one hosts for each thing
self.assertThat(fake_1_hosts, matchers.HasLength(1))
self.assertThat(fake_2_hosts, matchers.HasLength(1))
# And they must not be the same answers even on this simple data.
self.assertNotEqual(fake_1_hosts, fake_2_hosts)
def test_ignore_hosts(self):
hosts = ['foo', 'bar', 'baz']
ring = hash_ring.HashRing(hosts)
equals_bar_or_baz = matchers.MatchesAny(
matchers.Equals(['bar']),
matchers.Equals(['baz']))
self.assertThat(
ring.get_hosts('fake', ignore_hosts=['foo']),
equals_bar_or_baz)
self.assertThat(
ring.get_hosts('fake', ignore_hosts=['foo', 'bar']),
equals_bar_or_baz)
self.assertEqual([], ring.get_hosts('fake', ignore_hosts=hosts))
def _compare_rings(self, nodes, conductors, ring,
new_conductors, new_ring):
delta = {}
mapping = {'node': ring.get_hosts(node)[0] for node in nodes}
new_mapping = {'node': new_ring.get_hosts(node)[0] for node in nodes}
for key, old in mapping.items():
new = new_mapping.get(key, None)
if new != old:
delta[key] = (old, new)
return delta
def test_rebalance_stability_join(self):
num_services = 10
num_nodes = 10000
# Adding 1 service to a set of N should move 1/(N+1) of all nodes
# Eg, for a cluster of 10 nodes, adding one should move 1/11, or 9%
# We allow for 1/N to allow for rounding in tests.
redistribution_factor = 1.0 / num_services
nodes = [str(x) for x in range(num_nodes)]
services = [str(x) for x in range(num_services)]
new_services = services + ['new']
delta = self._compare_rings(
nodes, services, hash_ring.HashRing(services),
new_services, hash_ring.HashRing(new_services))
self.assertLess(len(delta), num_nodes * redistribution_factor)
def test_rebalance_stability_leave(self):
num_services = 10
num_nodes = 10000
# Removing 1 service from a set of N should move 1/(N) of all nodes
# Eg, for a cluster of 10 nodes, removing one should move 1/10, or 10%
# We allow for 1/(N-1) to allow for rounding in tests.
redistribution_factor = 1.0 / (num_services - 1)
nodes = [str(x) for x in range(num_nodes)]
services = [str(x) for x in range(num_services)]
new_services = services[:]
new_services.pop()
delta = self._compare_rings(
nodes, services, hash_ring.HashRing(services),
new_services, hash_ring.HashRing(new_services))
self.assertLess(len(delta), num_nodes * redistribution_factor)
def test_ignore_non_existent_host(self):
hosts = ['foo', 'bar']
ring = hash_ring.HashRing(hosts)
self.assertEqual(['foo'], ring.get_hosts('fake',
ignore_hosts=['baz']))
def test_create_ring_invalid_data(self):
hosts = None
self.assertRaises(exception.Invalid,
hash_ring.HashRing,
hosts)
def test_get_hosts_invalid_data(self):
hosts = ['foo', 'bar']
ring = hash_ring.HashRing(hosts)
self.assertRaises(exception.Invalid,
ring.get_hosts,
None)

6
nova/tests/unit/virt/ironic/test_driver.py
View File

@ -22,6 +22,7 @@ from oslo_service import loopingcall
from oslo_utils import uuidutils
import six
from testtools import matchers
from tooz import hashring as hash_ring
from nova.api.metadata import base as instance_metadata
from nova.compute import power_state as nova_states
@ -30,7 +31,6 @@ from nova.compute import vm_states
from nova.console import type as console_type
from nova import context as nova_context
from nova import exception
from nova import hash_ring
from nova import objects
from nova.objects import fields
from nova import servicegroup
@ -2029,7 +2029,7 @@ class HashRingTestCase(test.NoDBTestCase):
mock_services.assert_called_once_with(
mock.ANY, self.driver._get_hypervisor_type())
mock_hash_ring.assert_called_once_with(expected_hosts)
mock_hash_ring.assert_called_once_with(expected_hosts, partitions=32)
self.assertEqual(SENTINEL, self.driver.hash_ring)
self.mock_is_up.assert_has_calls(is_up_calls)
@ -2078,7 +2078,7 @@ class NodeCacheTestCase(test.NoDBTestCase):
self.flags(host=self.host)
@mock.patch.object(ironic_driver.IronicDriver, '_refresh_hash_ring')
@mock.patch.object(hash_ring.HashRing, 'get_hosts')
@mock.patch.object(hash_ring.HashRing, 'get_nodes')
@mock.patch.object(ironic_driver.IronicDriver, '_get_node_list')
@mock.patch.object(objects.InstanceList, 'get_uuids_by_host')
def _test__refresh_cache(self, instances, nodes, hosts, mock_instances,

12
nova/virt/ironic/driver.py
View File

@ -30,6 +30,7 @@ from oslo_utils import excutils
from oslo_utils import importutils
import six
import six.moves.urllib.parse as urlparse
from tooz import hashring as hash_ring
from nova.api.metadata import base as instance_metadata
from nova.compute import power_state
@ -39,7 +40,6 @@ import nova.conf
from nova.console import type as console_type
from nova import context as nova_context
from nova import exception
from nova import hash_ring
from nova.i18n import _
from nova.i18n import _LE
from nova.i18n import _LI
@ -81,6 +81,10 @@ _NODE_FIELDS = ('uuid', 'power_state', 'target_power_state', 'provision_state',
# Console state checking interval in seconds
_CONSOLE_STATE_CHECKING_INTERVAL = 1
# Number of hash ring partitions per service
# 5 should be fine for most deployments, as an experimental feature.
_HASH_RING_PARTITIONS = 2 ** 5
def map_power_state(state):
try:
@ -548,7 +552,8 @@ class IronicDriver(virt_driver.ComputeDriver):
# table will be here so far, and we might be brand new.
services.add(CONF.host)
self.hash_ring = hash_ring.HashRing(services)
self.hash_ring = hash_ring.HashRing(services,
partitions=_HASH_RING_PARTITIONS)
def _refresh_cache(self):
# NOTE(lucasagomes): limit == 0 is an indicator to continue
@ -570,7 +575,8 @@ class IronicDriver(virt_driver.ComputeDriver):
# nova while the service was down, and not yet reaped, will not be
# reported until the periodic task cleans it up.
elif (node.instance_uuid is None and
CONF.host in self.hash_ring.get_hosts(node.uuid)):
CONF.host in
self.hash_ring.get_nodes(node.uuid.encode('utf-8'))):
node_cache[node.uuid] = node
self.node_cache = node_cache

1
requirements.txt
View File

@ -59,3 +59,4 @@ os-win>=1.4.0 # Apache-2.0
castellan>=0.4.0 # Apache-2.0
microversion-parse>=0.1.2 # Apache-2.0
os-xenapi>=0.1.1 # Apache-2.0
tooz>=1.47.0 # Apache-2.0