Initial implementation of the hash client

docs/getting_started.rst

@@ -7,12 +7,29 @@ Basic Usage
 
 .. code-block:: python
 
-    from pymemcache.client import Client
+    from pymemcache.client.base import Client
 
     client = Client(('localhost', 11211))
     client.set('some_key', 'some_value')
     result = client.get('some_key')
 
+Using a memcached cluster
+-------------------------
+This will use a consistent hashing algorithm to choose which server to
+set/get the values from. It will also automatically rebalance depending
+on if a server goes down.
+
+.. code-block:: python
+
+    from pymemcache.client.hash import HashClient
+
+    client = HashClient([
+        ('127.0.0.1', 11211),
+        ('127.0.0.1', 11212)
+    ])
+    client.set('some_key', 'some value')
+    result = client.get('some_key')
+
 
 Serialization
 --------------
@@ -20,7 +37,7 @@ Serialization
 .. code-block:: python
 
      import json
-     from pymemcache.client import Client
+     from pymemcache.client.base import Client
 
      def json_serializer(key, value):
          if type(value) == str:

pymemcache/client/base.py

@@ -20,6 +20,16 @@ import six
 
 from pymemcache import pool
 
+from pymemcache.exceptions import (
+    MemcacheError,
+    MemcacheClientError,
+    MemcacheUnknownCommandError,
+    MemcacheIllegalInputError,
+    MemcacheServerError,
+    MemcacheUnknownError,
+    MemcacheUnexpectedCloseError
+)
+
 
 RECV_SIZE = 4096
 VALID_STORE_RESULTS = {
@@ -56,49 +66,6 @@ STAT_TYPES = {
     b'slab_automove': lambda value: int(value) != 0,
 }
 
-
-class MemcacheError(Exception):
-    "Base exception class"
-    pass
-
-
-class MemcacheClientError(MemcacheError):
-    """Raised when memcached fails to parse the arguments to a request, likely
-    due to a malformed key and/or value, a bug in this library, or a version
-    mismatch with memcached."""
-    pass
-
-
-class MemcacheUnknownCommandError(MemcacheClientError):
-    """Raised when memcached fails to parse a request, likely due to a bug in
-    this library or a version mismatch with memcached."""
-    pass
-
-
-class MemcacheIllegalInputError(MemcacheClientError):
-    """Raised when a key or value is not legal for Memcache (see the class docs
-    for Client for more details)."""
-    pass
-
-
-class MemcacheServerError(MemcacheError):
-    """Raised when memcached reports a failure while processing a request,
-    likely due to a bug or transient issue in memcached."""
-    pass
-
-
-class MemcacheUnknownError(MemcacheError):
-    """Raised when this library receives a response from memcached that it
-    cannot parse, likely due to a bug in this library or a version mismatch
-    with memcached."""
-    pass
-
-
-class MemcacheUnexpectedCloseError(MemcacheServerError):
-    "Raised when the connection with memcached closes unexpectedly."
-    pass
-
-
 # Common helper functions.
 
 def _check_key(key, key_prefix=b''):
@@ -131,7 +98,7 @@ class Client(object):
      Values must have a __str__() method to convert themselves to a byte
      string. Unicode objects can be a problem since str() on a Unicode object
      will attempt to encode it as ASCII (which will fail if the value contains
-     code points larger than U+127). You can fix this will a serializer or by
+     code points larger than U+127). You can fix this with a serializer or by
      just calling encode on the string (using UTF-8, for instance).
 
      If you intend to use anything but str as a value, it is a good idea to use

pymemcache/client/hash.py

@@ -0,0 +1,95 @@
+import socket
+import zlib
+from pymemcache.client.base import Client, PooledClient
+from clandestined import RendezvousHash as RH
+
+
+class RendezvousHash(RH):
+    def get_node(self, key):
+        return self.find_node(key)
+
+
+class HashClient(object):
+    """
+    A client for communicating with a cluster of memcached servers
+    """
+    def __init__(
+        self,
+        servers,
+        hasher=None,
+        serializer=None,
+        connect_timeout=None,
+        timeout=None,
+        no_delay=False,
+        ignore_exc=False,
+        socket_module=socket,
+        key_prefix=b'',
+        max_pool_size=None,
+        lock_generator=None,
+        use_pooling=False,
+    ):
+        """
+        Args:
+          servers: list(tuple(hostname, port))
+          serializer: optional class with ``serialize`` and ``deserialize``
+                      functions.
+          hasher: optional class three functions ``get_node``, ``add_node``, and
+                  ``remove_node``
+
+                  defaults to crc32 hash.
+          use_pooling: use py:class:`.PooledClient` as the default underlying
+                       class. ``max_pool_size`` and ``lock_generator`` can
+                       be used with this. default: False
+
+        Further arguments are interpreted as for :py:class:`.Client`
+        constructor.
+        """
+        self.clients = {}
+
+        if hasher is None:
+            self.hasher = RendezvousHash()
+
+        for server, port in servers:
+            key = '%s:%s' % (server, port)
+            kwargs = {
+                'connect_timeout': connect_timeout,
+                'timeout': timeout,
+                'no_delay': no_delay,
+                'ignore_exc': ignore_exc,
+                'socket_module': socket_module,
+                'key_prefix': key_prefix,
+            }
+
+            if serializer is not None:
+                kwargs['serializer'] = serializer.serialize
+                kwargs['deserializer'] = serializer.deserialize
+
+            if use_pooling is True:
+                kwargs.update({
+                    'max_pool_size': max_pool_size,
+                    'lock_generator': lock_generator
+                })
+
+                client = PooledClient(
+                    (server, port),
+                    **kwargs
+                )
+            else:
+                client = Client((server, port))
+
+            self.clients[key] = client
+            self.hasher.add_node(key)
+
+    def _get_client(self, key):
+        server = self.hasher.get_node(key)
+        print('got server %s' % server)
+        client = self.clients[server]
+        return client
+
+    def set(self, key, *args, **kwargs):
+        client = self._get_client(key)
+        return client.set(key, *args, **kwargs)
+
+    def get(self, key, *args, **kwargs):
+        client = self._get_client(key)
+        return client.get(key, *args, **kwargs)

pymemcache/exceptions.py

@@ -0,0 +1,40 @@
+class MemcacheError(Exception):
+    "Base exception class"
+    pass
+
+
+class MemcacheClientError(MemcacheError):
+    """Raised when memcached fails to parse the arguments to a request, likely
+    due to a malformed key and/or value, a bug in this library, or a version
+    mismatch with memcached."""
+    pass
+
+
+class MemcacheUnknownCommandError(MemcacheClientError):
+    """Raised when memcached fails to parse a request, likely due to a bug in
+    this library or a version mismatch with memcached."""
+    pass
+
+
+class MemcacheIllegalInputError(MemcacheClientError):
+    """Raised when a key or value is not legal for Memcache (see the class docs
+    for Client for more details)."""
+    pass
+
+
+class MemcacheServerError(MemcacheError):
+    """Raised when memcached reports a failure while processing a request,
+    likely due to a bug or transient issue in memcached."""
+    pass
+
+
+class MemcacheUnknownError(MemcacheError):
+    """Raised when this library receives a response from memcached that it
+    cannot parse, likely due to a bug in this library or a version mismatch
+    with memcached."""
+    pass
+
+
+class MemcacheUnexpectedCloseError(MemcacheServerError):
+    "Raised when the connection with memcached closes unexpectedly."
+    pass

pymemcache/test/test_client.py

@@ -19,10 +19,15 @@ import socket
 import unittest
 import pytest
 
-from pymemcache.client import PooledClient
+from pymemcache.client.base import PooledClient, Client
-from pymemcache.client import Client, MemcacheUnknownCommandError
+from pymemcache.exceptions import (
-from pymemcache.client import MemcacheClientError, MemcacheServerError
+    MemcacheClientError,
-from pymemcache.client import MemcacheUnknownError, MemcacheIllegalInputError
+    MemcacheServerError,
+    MemcacheUnknownCommandError,
+    MemcacheUnknownError,
+    MemcacheIllegalInputError
+)
+
 from pymemcache import pool
 from pymemcache.test.utils import MockMemcacheClient
 

pymemcache/test/test_integration.py

@@ -16,8 +16,11 @@ import json
 import pytest
 import six
 
-from pymemcache.client import (Client, MemcacheClientError)
+from pymemcache.client.base import Client
-from pymemcache.client import MemcacheIllegalInputError
+from pymemcache.exceptions import (
+    MemcacheIllegalInputError,
+    MemcacheClientError
+)
 
 
 @pytest.mark.integration()

pymemcache/test/utils.py

@@ -9,7 +9,7 @@ import time
 
 import six
 
-from pymemcache.client import MemcacheIllegalInputError
+from pymemcache.exceptions import MemcacheIllegalInputError
 
 
 class MockMemcacheClient(object):

setup.py

@@ -10,7 +10,7 @@ setup(
     author='Charles Gordon',
     author_email='charles@pinterest.com',
     packages=find_packages(),
-    install_requires=['six'],
+    install_requires=['six', 'clandestined'],
     description='A comprehensive, fast, pure Python memcached client',
     long_description=open('README.md').read(),
     license='Apache License 2.0',