From f394d42966f4c98e9e74a62def7f6a98f0dfcb10 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Sat, 20 Jun 2015 13:21:21 -0700 Subject: [PATCH] Initial implementation of the hash client --- docs/getting_started.rst | 21 +++++- pymemcache/client/__init__.py | 0 pymemcache/{client.py => client/base.py} | 55 +++----------- pymemcache/client/hash.py | 95 ++++++++++++++++++++++++ pymemcache/exceptions.py | 40 ++++++++++ pymemcache/test/test_client.py | 13 +++- pymemcache/test/test_integration.py | 7 +- pymemcache/test/utils.py | 2 +- setup.py | 2 +- 9 files changed, 181 insertions(+), 54 deletions(-) create mode 100644 pymemcache/client/__init__.py rename pymemcache/{client.py => client/base.py} (96%) create mode 100644 pymemcache/client/hash.py create mode 100644 pymemcache/exceptions.py diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 68125fc..10b66c1 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -7,12 +7,29 @@ Basic Usage .. code-block:: python - from pymemcache.client import Client + from pymemcache.client.base import Client client = Client(('localhost', 11211)) client.set('some_key', 'some_value') result = client.get('some_key') +Using a memcached cluster +------------------------- +This will use a consistent hashing algorithm to choose which server to +set/get the values from. It will also automatically rebalance depending +on if a server goes down. + +.. code-block:: python + + from pymemcache.client.hash import HashClient + + client = HashClient([ + ('127.0.0.1', 11211), + ('127.0.0.1', 11212) + ]) + client.set('some_key', 'some value') + result = client.get('some_key') + Serialization -------------- @@ -20,7 +37,7 @@ Serialization .. code-block:: python import json - from pymemcache.client import Client + from pymemcache.client.base import Client def json_serializer(key, value): if type(value) == str: diff --git a/pymemcache/client/__init__.py b/pymemcache/client/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pymemcache/client.py b/pymemcache/client/base.py similarity index 96% rename from pymemcache/client.py rename to pymemcache/client/base.py index 9b5ae2c..8fe7d9b 100644 --- a/pymemcache/client.py +++ b/pymemcache/client/base.py @@ -20,6 +20,16 @@ import six from pymemcache import pool +from pymemcache.exceptions import ( + MemcacheError, + MemcacheClientError, + MemcacheUnknownCommandError, + MemcacheIllegalInputError, + MemcacheServerError, + MemcacheUnknownError, + MemcacheUnexpectedCloseError +) + RECV_SIZE = 4096 VALID_STORE_RESULTS = { @@ -56,49 +66,6 @@ STAT_TYPES = { b'slab_automove': lambda value: int(value) != 0, } - -class MemcacheError(Exception): - "Base exception class" - pass - - -class MemcacheClientError(MemcacheError): - """Raised when memcached fails to parse the arguments to a request, likely - due to a malformed key and/or value, a bug in this library, or a version - mismatch with memcached.""" - pass - - -class MemcacheUnknownCommandError(MemcacheClientError): - """Raised when memcached fails to parse a request, likely due to a bug in - this library or a version mismatch with memcached.""" - pass - - -class MemcacheIllegalInputError(MemcacheClientError): - """Raised when a key or value is not legal for Memcache (see the class docs - for Client for more details).""" - pass - - -class MemcacheServerError(MemcacheError): - """Raised when memcached reports a failure while processing a request, - likely due to a bug or transient issue in memcached.""" - pass - - -class MemcacheUnknownError(MemcacheError): - """Raised when this library receives a response from memcached that it - cannot parse, likely due to a bug in this library or a version mismatch - with memcached.""" - pass - - -class MemcacheUnexpectedCloseError(MemcacheServerError): - "Raised when the connection with memcached closes unexpectedly." - pass - - # Common helper functions. def _check_key(key, key_prefix=b''): @@ -131,7 +98,7 @@ class Client(object): Values must have a __str__() method to convert themselves to a byte string. Unicode objects can be a problem since str() on a Unicode object will attempt to encode it as ASCII (which will fail if the value contains - code points larger than U+127). You can fix this will a serializer or by + code points larger than U+127). You can fix this with a serializer or by just calling encode on the string (using UTF-8, for instance). If you intend to use anything but str as a value, it is a good idea to use diff --git a/pymemcache/client/hash.py b/pymemcache/client/hash.py new file mode 100644 index 0000000..bc92e70 --- /dev/null +++ b/pymemcache/client/hash.py @@ -0,0 +1,95 @@ +import socket +import zlib +from pymemcache.client.base import Client, PooledClient +from clandestined import RendezvousHash as RH + + +class RendezvousHash(RH): + def get_node(self, key): + return self.find_node(key) + + +class HashClient(object): + """ + A client for communicating with a cluster of memcached servers + """ + def __init__( + self, + servers, + hasher=None, + serializer=None, + connect_timeout=None, + timeout=None, + no_delay=False, + ignore_exc=False, + socket_module=socket, + key_prefix=b'', + max_pool_size=None, + lock_generator=None, + use_pooling=False, + ): + """ + Args: + servers: list(tuple(hostname, port)) + serializer: optional class with ``serialize`` and ``deserialize`` + functions. + hasher: optional class three functions ``get_node``, ``add_node``, and + ``remove_node`` + + defaults to crc32 hash. + use_pooling: use py:class:`.PooledClient` as the default underlying + class. ``max_pool_size`` and ``lock_generator`` can + be used with this. default: False + + Further arguments are interpreted as for :py:class:`.Client` + constructor. + """ + self.clients = {} + + if hasher is None: + self.hasher = RendezvousHash() + + for server, port in servers: + key = '%s:%s' % (server, port) + kwargs = { + 'connect_timeout': connect_timeout, + 'timeout': timeout, + 'no_delay': no_delay, + 'ignore_exc': ignore_exc, + 'socket_module': socket_module, + 'key_prefix': key_prefix, + } + + if serializer is not None: + kwargs['serializer'] = serializer.serialize + kwargs['deserializer'] = serializer.deserialize + + if use_pooling is True: + kwargs.update({ + 'max_pool_size': max_pool_size, + 'lock_generator': lock_generator + }) + + client = PooledClient( + (server, port), + **kwargs + ) + else: + client = Client((server, port)) + + self.clients[key] = client + self.hasher.add_node(key) + + def _get_client(self, key): + server = self.hasher.get_node(key) + print('got server %s' % server) + client = self.clients[server] + return client + + def set(self, key, *args, **kwargs): + client = self._get_client(key) + return client.set(key, *args, **kwargs) + + def get(self, key, *args, **kwargs): + client = self._get_client(key) + return client.get(key, *args, **kwargs) diff --git a/pymemcache/exceptions.py b/pymemcache/exceptions.py new file mode 100644 index 0000000..416fa0a --- /dev/null +++ b/pymemcache/exceptions.py @@ -0,0 +1,40 @@ +class MemcacheError(Exception): + "Base exception class" + pass + + +class MemcacheClientError(MemcacheError): + """Raised when memcached fails to parse the arguments to a request, likely + due to a malformed key and/or value, a bug in this library, or a version + mismatch with memcached.""" + pass + + +class MemcacheUnknownCommandError(MemcacheClientError): + """Raised when memcached fails to parse a request, likely due to a bug in + this library or a version mismatch with memcached.""" + pass + + +class MemcacheIllegalInputError(MemcacheClientError): + """Raised when a key or value is not legal for Memcache (see the class docs + for Client for more details).""" + pass + + +class MemcacheServerError(MemcacheError): + """Raised when memcached reports a failure while processing a request, + likely due to a bug or transient issue in memcached.""" + pass + + +class MemcacheUnknownError(MemcacheError): + """Raised when this library receives a response from memcached that it + cannot parse, likely due to a bug in this library or a version mismatch + with memcached.""" + pass + + +class MemcacheUnexpectedCloseError(MemcacheServerError): + "Raised when the connection with memcached closes unexpectedly." + pass diff --git a/pymemcache/test/test_client.py b/pymemcache/test/test_client.py index 1efb030..29a71b1 100644 --- a/pymemcache/test/test_client.py +++ b/pymemcache/test/test_client.py @@ -19,10 +19,15 @@ import socket import unittest import pytest -from pymemcache.client import PooledClient -from pymemcache.client import Client, MemcacheUnknownCommandError -from pymemcache.client import MemcacheClientError, MemcacheServerError -from pymemcache.client import MemcacheUnknownError, MemcacheIllegalInputError +from pymemcache.client.base import PooledClient, Client +from pymemcache.exceptions import ( + MemcacheClientError, + MemcacheServerError, + MemcacheUnknownCommandError, + MemcacheUnknownError, + MemcacheIllegalInputError +) + from pymemcache import pool from pymemcache.test.utils import MockMemcacheClient diff --git a/pymemcache/test/test_integration.py b/pymemcache/test/test_integration.py index 52576fe..7ac80a9 100644 --- a/pymemcache/test/test_integration.py +++ b/pymemcache/test/test_integration.py @@ -16,8 +16,11 @@ import json import pytest import six -from pymemcache.client import (Client, MemcacheClientError) -from pymemcache.client import MemcacheIllegalInputError +from pymemcache.client.base import Client +from pymemcache.exceptions import ( + MemcacheIllegalInputError, + MemcacheClientError +) @pytest.mark.integration() diff --git a/pymemcache/test/utils.py b/pymemcache/test/utils.py index 77b7c22..9661426 100644 --- a/pymemcache/test/utils.py +++ b/pymemcache/test/utils.py @@ -9,7 +9,7 @@ import time import six -from pymemcache.client import MemcacheIllegalInputError +from pymemcache.exceptions import MemcacheIllegalInputError class MockMemcacheClient(object): diff --git a/setup.py b/setup.py index b1f2404..f053090 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( author='Charles Gordon', author_email='charles@pinterest.com', packages=find_packages(), - install_requires=['six'], + install_requires=['six', 'clandestined'], description='A comprehensive, fast, pure Python memcached client', long_description=open('README.md').read(), license='Apache License 2.0',