From e13f99b61921438bde5d049b3fd7d6e11a95f99e Mon Sep 17 00:00:00 2001 From: Tony Breeds Date: Tue, 12 Sep 2017 16:12:51 -0600 Subject: [PATCH] Retire Packaging Deb project repos This commit is part of a series to retire the Packaging Deb project. Step 2 is to remove all content from the project repos, replacing it with a README notification where to find ongoing work, and how to recover the repo if needed at some future point (as in https://docs.openstack.org/infra/manual/drivers.html#retiring-a-project). Change-Id: I00c02762c4c6dfa3a0cfa1dc2262f2ed98256ace --- .coveragerc | 8 - .gitignore | 24 - .gitreview | 4 - .testr.conf | 9 - CONTRIBUTING.rst | 17 - LICENSE | 176 --- README | 14 + README.rst | 24 - bindep.txt | 11 - doc/source/conf.py | 263 ----- doc/source/index.rst | 25 - doc/source/install/index.rst | 43 - doc/source/reference/index.rst | 82 -- doc/source/user/compatibility.rst | 128 --- doc/source/user/drivers.rst | 233 ---- doc/source/user/history.rst | 1 - doc/source/user/index.rst | 15 - doc/source/user/tutorial/coordinator.rst | 42 - doc/source/user/tutorial/group_membership.rst | 41 - doc/source/user/tutorial/hashring.rst | 10 - doc/source/user/tutorial/index.rst | 16 - doc/source/user/tutorial/leader_election.rst | 25 - doc/source/user/tutorial/lock.rst | 14 - doc/source/user/tutorial/partitioner.rst | 11 - examples/coordinator.py | 5 - examples/coordinator_heartbeat.py | 5 - examples/group_membership.py | 19 - examples/group_membership_watch.py | 22 - examples/hashring.py | 15 - examples/leader_election.py | 36 - examples/lock.py | 11 - examples/partitioner.py | 11 - .../notes/add-reno-996dd44974d53238.yaml | 3 - .../notes/hashring-0470f9119ef63d49.yaml | 3 - .../join_group_create-5095ec02e20c7242.yaml | 4 - .../notes/partitioner-4005767d287dc7c9.yaml | 6 - releasenotes/source/_static/.placeholder | 0 releasenotes/source/_templates/.placeholder | 0 releasenotes/source/conf.py | 281 ----- releasenotes/source/index.rst | 9 - releasenotes/source/ocata.rst | 6 - releasenotes/source/unreleased.rst | 5 - requirements.txt | 15 - run-examples.sh | 10 - run-tests.sh | 22 - setup-consul-env.sh | 27 - setup-etcd-env.sh | 31 - setup.cfg | 86 -- setup.py | 29 - tools/compat-matrix.py | 150 --- tools/pretty_tox.sh | 16 - tools/tox_install.sh | 30 - tooz/__init__.py | 36 - tooz/_retry.py | 31 - tooz/coordination.py | 897 --------------- tooz/drivers/__init__.py | 0 tooz/drivers/consul.py | 172 --- tooz/drivers/etcd.py | 258 ----- tooz/drivers/etcd3.py | 148 --- tooz/drivers/etcd3gw.py | 204 ---- tooz/drivers/file.py | 532 --------- tooz/drivers/ipc.py | 243 ---- tooz/drivers/memcached.py | 516 --------- tooz/drivers/mysql.py | 198 ---- tooz/drivers/pgsql.py | 249 ---- tooz/drivers/redis.py | 753 ------------ tooz/drivers/zake.py | 58 - tooz/drivers/zookeeper.py | 547 --------- tooz/hashring.py | 142 --- tooz/locking.py | 109 -- tooz/partitioner.py | 96 -- tooz/tests/__init__.py | 73 -- tooz/tests/drivers/__init__.py | 0 tooz/tests/drivers/test_file.py | 73 -- tooz/tests/test_coordination.py | 1022 ----------------- tooz/tests/test_etcd.py | 44 - tooz/tests/test_hashring.py | 243 ---- tooz/tests/test_memcache.py | 85 -- tooz/tests/test_mysql.py | 54 - tooz/tests/test_partitioner.py | 103 -- tooz/tests/test_postgresql.py | 114 -- tooz/tests/test_utils.py | 136 --- tooz/utils.py | 225 ---- tox.ini | 63 - 84 files changed, 14 insertions(+), 9503 deletions(-) delete mode 100644 .coveragerc delete mode 100644 .gitignore delete mode 100644 .gitreview delete mode 100644 .testr.conf delete mode 100644 CONTRIBUTING.rst delete mode 100644 LICENSE create mode 100644 README delete mode 100644 README.rst delete mode 100644 bindep.txt delete mode 100644 doc/source/conf.py delete mode 100644 doc/source/index.rst delete mode 100644 doc/source/install/index.rst delete mode 100644 doc/source/reference/index.rst delete mode 100644 doc/source/user/compatibility.rst delete mode 100644 doc/source/user/drivers.rst delete mode 100644 doc/source/user/history.rst delete mode 100644 doc/source/user/index.rst delete mode 100644 doc/source/user/tutorial/coordinator.rst delete mode 100644 doc/source/user/tutorial/group_membership.rst delete mode 100644 doc/source/user/tutorial/hashring.rst delete mode 100644 doc/source/user/tutorial/index.rst delete mode 100644 doc/source/user/tutorial/leader_election.rst delete mode 100644 doc/source/user/tutorial/lock.rst delete mode 100644 doc/source/user/tutorial/partitioner.rst delete mode 100644 examples/coordinator.py delete mode 100644 examples/coordinator_heartbeat.py delete mode 100644 examples/group_membership.py delete mode 100644 examples/group_membership_watch.py delete mode 100644 examples/hashring.py delete mode 100644 examples/leader_election.py delete mode 100644 examples/lock.py delete mode 100644 examples/partitioner.py delete mode 100644 releasenotes/notes/add-reno-996dd44974d53238.yaml delete mode 100644 releasenotes/notes/hashring-0470f9119ef63d49.yaml delete mode 100644 releasenotes/notes/join_group_create-5095ec02e20c7242.yaml delete mode 100644 releasenotes/notes/partitioner-4005767d287dc7c9.yaml delete mode 100644 releasenotes/source/_static/.placeholder delete mode 100644 releasenotes/source/_templates/.placeholder delete mode 100644 releasenotes/source/conf.py delete mode 100644 releasenotes/source/index.rst delete mode 100644 releasenotes/source/ocata.rst delete mode 100644 releasenotes/source/unreleased.rst delete mode 100644 requirements.txt delete mode 100755 run-examples.sh delete mode 100755 run-tests.sh delete mode 100755 setup-consul-env.sh delete mode 100755 setup-etcd-env.sh delete mode 100644 setup.cfg delete mode 100644 setup.py delete mode 100644 tools/compat-matrix.py delete mode 100755 tools/pretty_tox.sh delete mode 100755 tools/tox_install.sh delete mode 100644 tooz/__init__.py delete mode 100644 tooz/_retry.py delete mode 100755 tooz/coordination.py delete mode 100644 tooz/drivers/__init__.py delete mode 100644 tooz/drivers/consul.py delete mode 100644 tooz/drivers/etcd.py delete mode 100644 tooz/drivers/etcd3.py delete mode 100644 tooz/drivers/etcd3gw.py delete mode 100644 tooz/drivers/file.py delete mode 100644 tooz/drivers/ipc.py delete mode 100644 tooz/drivers/memcached.py delete mode 100644 tooz/drivers/mysql.py delete mode 100644 tooz/drivers/pgsql.py delete mode 100644 tooz/drivers/redis.py delete mode 100644 tooz/drivers/zake.py delete mode 100644 tooz/drivers/zookeeper.py delete mode 100644 tooz/hashring.py delete mode 100644 tooz/locking.py delete mode 100644 tooz/partitioner.py delete mode 100644 tooz/tests/__init__.py delete mode 100644 tooz/tests/drivers/__init__.py delete mode 100644 tooz/tests/drivers/test_file.py delete mode 100644 tooz/tests/test_coordination.py delete mode 100644 tooz/tests/test_etcd.py delete mode 100644 tooz/tests/test_hashring.py delete mode 100644 tooz/tests/test_memcache.py delete mode 100644 tooz/tests/test_mysql.py delete mode 100644 tooz/tests/test_partitioner.py delete mode 100644 tooz/tests/test_postgresql.py delete mode 100644 tooz/tests/test_utils.py delete mode 100644 tooz/utils.py delete mode 100644 tox.ini diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 41e5648..0000000 --- a/.coveragerc +++ /dev/null @@ -1,8 +0,0 @@ -[run] -branch = True -source = tooz -omit = tooz/tests/*,tooz/openstack/* - -[report] -ignore_errors = True -precision = 2 diff --git a/.gitignore b/.gitignore deleted file mode 100644 index f1e7fde..0000000 --- a/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -*.py[co] -*.egg -*.egg-info -build -/.* -!.coveragerc -!.gitignore -!.mailmap -!.testr.conf -.*.sw? -cover/* -covhtml -dist -.tox -# Generated by pbr -AUTHORS -ChangeLog -# Generated by testrepository -.testrepository -# Generated by etcd -etcd-v* -default.etcd -# reno build -releasenotes/build diff --git a/.gitreview b/.gitreview deleted file mode 100644 index f3feb8a..0000000 --- a/.gitreview +++ /dev/null @@ -1,4 +0,0 @@ -[gerrit] -host=review.openstack.org -port=29418 -project=openstack/tooz.git diff --git a/.testr.conf b/.testr.conf deleted file mode 100644 index 142a752..0000000 --- a/.testr.conf +++ /dev/null @@ -1,9 +0,0 @@ -[DEFAULT] -test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ - OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ - OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-160} \ - OS_DEBUG=${OS_DEBUG:-TRACE} \ - OS_LOG_CAPTURE=${OS_LOG_CAPTURE:-1} \ - ${PYTHON:-python} -m subunit.run discover tooz $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst deleted file mode 100644 index 6f624c8..0000000 --- a/CONTRIBUTING.rst +++ /dev/null @@ -1,17 +0,0 @@ -If you would like to contribute to the development of OpenStack, you must -follow the steps in this page: - - http://docs.openstack.org/infra/manual/developers.html - -If you already have a good understanding of how the system works and your -OpenStack accounts are set up, you can skip to the development workflow -section of this documentation to learn how changes to OpenStack should be -submitted for review via the Gerrit tool: - - http://docs.openstack.org/infra/manual/developers.html#development-workflow - -Pull requests submitted through GitHub will be ignored. - -Bugs should be filed on Launchpad, not GitHub: - - https://bugs.launchpad.net/python-tooz/ diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 68c771a..0000000 --- a/LICENSE +++ /dev/null @@ -1,176 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - diff --git a/README b/README new file mode 100644 index 0000000..8fcd2b2 --- /dev/null +++ b/README @@ -0,0 +1,14 @@ +This project is no longer maintained. + +The contents of this repository are still available in the Git +source code management system. To see the contents of this +repository before it reached its end of life, please check out the +previous commit with "git checkout HEAD^1". + +For ongoing work on maintaining OpenStack packages in the Debian +distribution, please see the Debian OpenStack packaging team at +https://wiki.debian.org/OpenStack/. + +For any further questions, please email +openstack-dev@lists.openstack.org or join #openstack-dev on +Freenode. diff --git a/README.rst b/README.rst deleted file mode 100644 index 96b4ed2..0000000 --- a/README.rst +++ /dev/null @@ -1,24 +0,0 @@ -Tooz -==== - -.. image:: https://img.shields.io/pypi/v/tooz.svg - :target: https://pypi.python.org/pypi/tooz/ - :alt: Latest Version - -.. image:: https://img.shields.io/pypi/dm/tooz.svg - :target: https://pypi.python.org/pypi/tooz/ - :alt: Downloads - -The Tooz project aims at centralizing the most common distributed primitives -like group membership protocol, lock service and leader election by providing -a coordination API helping developers to build distributed applications. - -* Free software: Apache license -* Documentation: https://docs.openstack.org/tooz/latest/ -* Source: https://git.openstack.org/cgit/openstack/tooz -* Bugs: https://bugs.launchpad.net/python-tooz/ - -Join us -------- - -- https://launchpad.net/python-tooz diff --git a/bindep.txt b/bindep.txt deleted file mode 100644 index d3d87e1..0000000 --- a/bindep.txt +++ /dev/null @@ -1,11 +0,0 @@ -redis-sentinel [platform:ubuntu !platform:ubuntu-trusty] -redis-server [platform:dpkg] -libpq-dev [platform:dpkg] -postgresql [platform:dpkg] -mysql-client [platform:dpkg] -mysql-server [platform:dpkg] -build-essential [platform:dpkg] -libffi-dev [platform:dpkg] -zookeeperd [platform:dpkg] -memcached [platform:dpkg] -unzip [platform:dpkg] diff --git a/doc/source/conf.py b/doc/source/conf.py deleted file mode 100644 index 2d26f1f..0000000 --- a/doc/source/conf.py +++ /dev/null @@ -1,263 +0,0 @@ -# -*- coding: utf-8 -*- -# -# tooz documentation build configuration file -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import datetime -import subprocess - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ----------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.todo', - 'sphinx.ext.graphviz', - 'sphinx.ext.extlinks', - 'openstackdocstheme', - 'sphinx.ext.inheritance_diagram', - 'sphinx.ext.viewcode', - 'stevedore.sphinxext', -] - -# openstackdocstheme options -repository_name = 'openstack/tooz' -bug_project = 'tooz' -bug_tag = '' - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'tooz' -copyright = u'%s, OpenStack Foundation' % datetime.date.today().year - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = subprocess.Popen(['sh', '-c', 'cd ../..; python setup.py --version'], - stdout=subprocess.PIPE).stdout.read() -version = version.strip() -# The full version, including alpha/beta/rc tags. -release = version - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = [] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'openstackdocs' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -html_last_updated_fmt = '%Y-%m-%d %H:%M' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'toozdoc' - - -# -- Options for LaTeX output -------------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'tooz.tex', u'tooz Documentation', - u'eNovance', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -# man_pages = [ -# ('index', 'tooz', u'tooz Documentation', -# [u'eNovance'], 1) -# ] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------------ - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'tooz', u'tooz Documentation', - u'OpenStack Foundation', 'tooz', 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -# extlinks = { -# } - -autodoc_default_flags = ['members', 'special-members', 'show-inheritance'] diff --git a/doc/source/index.rst b/doc/source/index.rst deleted file mode 100644 index af18503..0000000 --- a/doc/source/index.rst +++ /dev/null @@ -1,25 +0,0 @@ -============================================================= - Tooz -- Distributed System Helper Library -============================================================= - -The Tooz project aims at centralizing the most common distributed primitives -like group membership protocol, lock service and leader election by providing -a coordination API helping developers to build distributed applications. [#f1]_ - -.. toctree:: - :maxdepth: 2 - - install/index - user/index - reference/index - -.. rubric:: Indices and tables - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - -.. [#f1] It should be noted that even though it is designed with OpenStack - integration in mind, and that is where most of its *current* - integration is it aims to be generally usable and useful in any - project. diff --git a/doc/source/install/index.rst b/doc/source/install/index.rst deleted file mode 100644 index 3070b4a..0000000 --- a/doc/source/install/index.rst +++ /dev/null @@ -1,43 +0,0 @@ -============ -Installation -============ - -Python Versions -=============== - -Tooz is tested under Python 2.7 and 3.4. - -.. _install-basic: - -Basic Installation -================== - -Tooz should be installed into the same site-packages area where -the application and extensions are installed (either a virtualenv or -the global site-packages). You may need administrative privileges to -do that. The easiest way to install it is using pip_:: - - $ pip install tooz - -or:: - - $ sudo pip install tooz - -.. _pip: http://pypi.python.org/pypi/pip - -Download -======== - -Tooz releases are hosted on PyPI and can be downloaded from: -http://pypi.python.org/pypi/tooz - -Source Code -=========== - -The source is hosted on the OpenStack infrastructure: https://git.openstack.org/cgit/openstack/tooz/ - -Reporting Bugs -============== - -Please report bugs through the launchpad project: -https://launchpad.net/python-tooz diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst deleted file mode 100644 index 15b7d57..0000000 --- a/doc/source/reference/index.rst +++ /dev/null @@ -1,82 +0,0 @@ -================ -Module Reference -================ - -Interfaces ----------- - -.. autoclass:: tooz.coordination.CoordinationDriver - :members: - -Consul -~~~~~~ - -.. autoclass:: tooz.drivers.consul.ConsulDriver - :members: - -Etcd -~~~~ - -.. autoclass:: tooz.drivers.etcd.EtcdDriver - :members: - -File -~~~~ - -.. autoclass:: tooz.drivers.file.FileDriver - :members: - -IPC -~~~ - -.. autoclass:: tooz.drivers.ipc.IPCDriver - :members: - -Memcached -~~~~~~~~~ - -.. autoclass:: tooz.drivers.memcached.MemcachedDriver - :members: - -Mysql -~~~~~ - -.. autoclass:: tooz.drivers.mysql.MySQLDriver - :members: - -PostgreSQL -~~~~~~~~~~ - -.. autoclass:: tooz.drivers.pgsql.PostgresDriver - :members: - -Redis -~~~~~ - -.. autoclass:: tooz.drivers.redis.RedisDriver - :members: - -Zake -~~~~ - -.. autoclass:: tooz.drivers.zake.ZakeDriver - :members: - -Zookeeper -~~~~~~~~~ - -.. autoclass:: tooz.drivers.zookeeper.KazooDriver - :members: - -Exceptions ----------- - -.. autoclass:: tooz.ToozError -.. autoclass:: tooz.coordination.ToozConnectionError -.. autoclass:: tooz.coordination.OperationTimedOut -.. autoclass:: tooz.coordination.GroupNotCreated -.. autoclass:: tooz.coordination.GroupAlreadyExist -.. autoclass:: tooz.coordination.MemberAlreadyExist -.. autoclass:: tooz.coordination.MemberNotJoined -.. autoclass:: tooz.coordination.GroupNotEmpty -.. autofunction:: tooz.utils.raise_with_cause diff --git a/doc/source/user/compatibility.rst b/doc/source/user/compatibility.rst deleted file mode 100644 index 6a38c29..0000000 --- a/doc/source/user/compatibility.rst +++ /dev/null @@ -1,128 +0,0 @@ -============= -Compatibility -============= - -Grouping -======== - -APIs ----- - -* :py:meth:`~tooz.coordination.CoordinationDriver.watch_join_group` -* :py:meth:`~tooz.coordination.CoordinationDriver.unwatch_join_group` -* :py:meth:`~tooz.coordination.CoordinationDriver.watch_leave_group` -* :py:meth:`~tooz.coordination.CoordinationDriver.unwatch_leave_group` -* :py:meth:`~tooz.coordination.CoordinationDriver.create_group` -* :py:meth:`~tooz.coordination.CoordinationDriver.get_groups` -* :py:meth:`~tooz.coordination.CoordinationDriver.join_group` -* :py:meth:`~tooz.coordination.CoordinationDriver.leave_group` -* :py:meth:`~tooz.coordination.CoordinationDriver.delete_group` -* :py:meth:`~tooz.coordination.CoordinationDriver.get_members` -* :py:meth:`~tooz.coordination.CoordinationDriver.get_member_capabilities` -* :py:meth:`~tooz.coordination.CoordinationDriver.update_capabilities` - -Driver support --------------- - -.. list-table:: - :header-rows: 1 - - * - Driver - - Supported - * - :py:class:`~tooz.drivers.consul.ConsulDriver` - - No - * - :py:class:`~tooz.drivers.etcd.EtcdDriver` - - No - * - :py:class:`~tooz.drivers.file.FileDriver` - - Yes - * - :py:class:`~tooz.drivers.ipc.IPCDriver` - - No - * - :py:class:`~tooz.drivers.memcached.MemcachedDriver` - - Yes - * - :py:class:`~tooz.drivers.mysql.MySQLDriver` - - No - * - :py:class:`~tooz.drivers.pgsql.PostgresDriver` - - No - * - :py:class:`~tooz.drivers.redis.RedisDriver` - - Yes - * - :py:class:`~tooz.drivers.zake.ZakeDriver` - - Yes - * - :py:class:`~tooz.drivers.zookeeper.KazooDriver` - - Yes - -Leaders -======= - -APIs ----- - -* :py:meth:`~tooz.coordination.CoordinationDriver.watch_elected_as_leader` -* :py:meth:`~tooz.coordination.CoordinationDriver.unwatch_elected_as_leader` -* :py:meth:`~tooz.coordination.CoordinationDriver.stand_down_group_leader` -* :py:meth:`~tooz.coordination.CoordinationDriver.get_leader` - -Driver support --------------- - -.. list-table:: - :header-rows: 1 - - * - Driver - - Supported - * - :py:class:`~tooz.drivers.consul.ConsulDriver` - - No - * - :py:class:`~tooz.drivers.etcd.EtcdDriver` - - No - * - :py:class:`~tooz.drivers.file.FileDriver` - - No - * - :py:class:`~tooz.drivers.ipc.IPCDriver` - - No - * - :py:class:`~tooz.drivers.memcached.MemcachedDriver` - - Yes - * - :py:class:`~tooz.drivers.mysql.MySQLDriver` - - No - * - :py:class:`~tooz.drivers.pgsql.PostgresDriver` - - No - * - :py:class:`~tooz.drivers.redis.RedisDriver` - - Yes - * - :py:class:`~tooz.drivers.zake.ZakeDriver` - - Yes - * - :py:class:`~tooz.drivers.zookeeper.KazooDriver` - - Yes - -Locking -======= - -APIs ----- - -* :py:meth:`~tooz.coordination.CoordinationDriver.get_lock` - -Driver support --------------- - -.. list-table:: - :header-rows: 1 - - * - Driver - - Supported - * - :py:class:`~tooz.drivers.consul.ConsulDriver` - - Yes - * - :py:class:`~tooz.drivers.etcd.EtcdDriver` - - Yes - * - :py:class:`~tooz.drivers.file.FileDriver` - - Yes - * - :py:class:`~tooz.drivers.ipc.IPCDriver` - - Yes - * - :py:class:`~tooz.drivers.memcached.MemcachedDriver` - - Yes - * - :py:class:`~tooz.drivers.mysql.MySQLDriver` - - Yes - * - :py:class:`~tooz.drivers.pgsql.PostgresDriver` - - Yes - * - :py:class:`~tooz.drivers.redis.RedisDriver` - - Yes - * - :py:class:`~tooz.drivers.zake.ZakeDriver` - - Yes - * - :py:class:`~tooz.drivers.zookeeper.KazooDriver` - - Yes diff --git a/doc/source/user/drivers.rst b/doc/source/user/drivers.rst deleted file mode 100644 index 3d3fdd3..0000000 --- a/doc/source/user/drivers.rst +++ /dev/null @@ -1,233 +0,0 @@ -======= -Drivers -======= - -Tooz is provided with several drivers implementing the provided coordination -API. While all drivers provides the same set of features with respect to the -API, some of them have different characteristics: - -Zookeeper ---------- - -**Driver:** :py:class:`tooz.drivers.zookeeper.KazooDriver` - -**Characteristics:** - -:py:attr:`tooz.drivers.zookeeper.KazooDriver.CHARACTERISTICS` - -**Entrypoint name:** ``zookeeper`` or ``kazoo`` - -**Summary:** - -The zookeeper is the reference implementation and provides the most solid -features as it's possible to build a cluster of zookeeper servers that is -resilient towards network partitions for example. - -**Test driver:** :py:class:`tooz.drivers.zake.ZakeDriver` - -**Characteristics:** - -:py:attr:`tooz.drivers.zake.ZakeDriver.CHARACTERISTICS` - -**Test driver entrypoint name:** ``zake`` - -Considerations -~~~~~~~~~~~~~~ - -- Primitives are based on sessions (and typically require careful selection - of session heartbeat periodicity and server side configuration of session - expiry). - -Memcached ---------- - -**Driver:** :py:class:`tooz.drivers.memcached.MemcachedDriver` - -**Characteristics:** - -:py:attr:`tooz.drivers.memcached.MemcachedDriver.CHARACTERISTICS` - -**Entrypoint name:** ``memcached`` - -**Summary:** - -The memcached driver is a basic implementation and provides little -resiliency, though it's much simpler to setup. A lot of the features provided -in tooz are based on timeout (heartbeats, locks, etc) so are less resilient -than other backends. - -Considerations -~~~~~~~~~~~~~~ - -- Less resilient than other backends such as zookeeper and redis. -- Primitives are often based on TTL(s) that may expire before - being renewed. -- Lacks certain primitives (compare and delete) so certain functionality - is fragile and/or broken due to this. - -Redis ------ - -**Driver:** :py:class:`tooz.drivers.redis.RedisDriver` - -**Characteristics:** - -:py:attr:`tooz.drivers.redis.RedisDriver.CHARACTERISTICS` - -**Entrypoint name:** ``redis`` - -**Summary:** - -The redis driver is a basic implementation and provides reasonable resiliency -when used with `redis-sentinel`_. A lot of the features provided in tooz are -based on timeout (heartbeats, locks, etc) so are less resilient than other -backends. - -Considerations -~~~~~~~~~~~~~~ - -- Less resilient than other backends such as zookeeper. -- Primitives are often based on TTL(s) that may expire before - being renewed. - -IPC ---- - -**Driver:** :py:class:`tooz.drivers.ipc.IPCDriver` - -**Characteristics:** :py:attr:`tooz.drivers.ipc.IPCDriver.CHARACTERISTICS` - -**Entrypoint name:** ``ipc`` - -**Summary:** - -The IPC driver is based on Posix IPC API and implements a lock -mechanism and some basic group primitives (with **huge** -limitations). - -Considerations -~~~~~~~~~~~~~~ - -- The lock can **only** be distributed locally to a computer - processes. - -File ----- - -**Driver:** :py:class:`tooz.drivers.file.FileDriver` - -**Characteristics:** :py:attr:`tooz.drivers.file.FileDriver.CHARACTERISTICS` - -**Entrypoint name:** ``file`` - -**Summary:** - -The file driver is a **simple** driver based on files and directories. It -implements a lock based on POSIX or Window file level locking -mechanism and some basic group primitives (with **huge** -limitations). - -Considerations -~~~~~~~~~~~~~~ - -- The lock can **only** be distributed locally to a computer processes. -- Certain concepts provided by it are **not** crash tolerant. - -PostgreSQL ----------- - -**Driver:** :py:class:`tooz.drivers.pgsql.PostgresDriver` - -**Characteristics:** - -:py:attr:`tooz.drivers.pgsql.PostgresDriver.CHARACTERISTICS` - -**Entrypoint name:** ``postgresql`` - -**Summary:** - -The postgresql driver is a driver providing only a distributed lock (for now) -and is based on the `PostgreSQL database server`_ and its API(s) that provide -for `advisory locks`_ to be created and used by applications. When a lock is -acquired it will release either when explicitly released or automatically when -the database session ends (for example if the program using the lock crashes). - -Considerations -~~~~~~~~~~~~~~ - -- Lock that may be acquired restricted by - ``max_locks_per_transaction * (max_connections + max_prepared_transactions)`` - upper bound (PostgreSQL server configuration settings). - -MySQL ------ - -**Driver:** :py:class:`tooz.drivers.mysql.MySQLDriver` - -**Characteristics:** :py:attr:`tooz.drivers.mysql.MySQLDriver.CHARACTERISTICS` - -**Entrypoint name:** ``mysql`` - -**Summary:** - -The MySQL driver is a driver providing only distributed locks (for now) -and is based on the `MySQL database server`_ supported `get_lock`_ -primitives. When a lock is acquired it will release either when explicitly -released or automatically when the database session ends (for example if -the program using the lock crashes). - -Considerations -~~~~~~~~~~~~~~ - -- Does **not** work correctly on some MySQL versions. -- Does **not** work when MySQL replicates from one server to another (locks - are local to the server that they were created from). - -Etcd ----- - -**Driver:** :py:class:`tooz.drivers.etcd.EtcdDriver` - -**Characteristics:** :py:attr:`tooz.drivers.etcd.EtcdDriver.CHARACTERISTICS` - -**Entrypoint name:** ``etcd`` - -**Summary:** - -The etcd driver is a driver providing only distributed locks (for now) -and is based on the `etcd server`_ supported key/value storage and -associated primitives. - -Consul ------- - -**Driver:** :py:class:`tooz.drivers.consul.ConsulDriver` - -**Characteristics:** - -:py:attr:`tooz.drivers.consul.ConsulDriver.CHARACTERISTICS` - -**Entrypoint name:** ``consul`` - -**Summary:** - -The `consul`_ driver is a driver providing only distributed locks (for now) -and is based on the consul server key/value storage and/or -primitives. When a lock is acquired it will release either when explicitly -released or automatically when the consul session ends (for example if -the program using the lock crashes). - -Characteristics ---------------- - -.. autoclass:: tooz.coordination.Characteristics - -.. _etcd server: https://coreos.com/etcd/ -.. _consul: https://www.consul.io/ -.. _advisory locks: http://www.postgresql.org/docs/8.2/interactive/\ - explicit-locking.html#ADVISORY-LOCKS -.. _get_lock: http://dev.mysql.com/doc/refman/5.5/en/\ - miscellaneous-functions.html#function_get-lock -.. _PostgreSQL database server: http://postgresql.org -.. _MySQL database server: http://mysql.org -.. _redis-sentinel: http://redis.io/topics/sentinel diff --git a/doc/source/user/history.rst b/doc/source/user/history.rst deleted file mode 100644 index f69be70..0000000 --- a/doc/source/user/history.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../../ChangeLog diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst deleted file mode 100644 index 7ab2c1a..0000000 --- a/doc/source/user/index.rst +++ /dev/null @@ -1,15 +0,0 @@ -================== -User Documentation -================== - -.. toctree:: - :maxdepth: 2 - - tutorial/index - drivers - compatibility - -.. toctree:: - :maxdepth: 1 - - history diff --git a/doc/source/user/tutorial/coordinator.rst b/doc/source/user/tutorial/coordinator.rst deleted file mode 100644 index 0131dee..0000000 --- a/doc/source/user/tutorial/coordinator.rst +++ /dev/null @@ -1,42 +0,0 @@ -======================= - Creating A Coordinator -======================= - -The principal object provided by tooz is the *coordinator*. It allows you to -use various features, such as group membership, leader election or -distributed locking. - -The features provided by tooz coordinator are implemented using different -drivers. When creating a coordinator, you need to specify which back-end -driver you want it to use. Different drivers may provide different set of -capabilities. - -If a driver does not support a feature, it will raise a -:class:`~tooz.NotImplemented` exception. - -This example program loads a basic coordinator using the ZooKeeper based -driver. - -.. literalinclude:: ../../../../examples/coordinator.py - :language: python - -The second argument passed to the coordinator must be a unique identifier -identifying the running program. - -After the coordinator is created, it can be used to use the various features -provided. - -In order to keep the connection to the coordination server active, the method -:meth:`~tooz.coordination.CoordinationDriver.heartbeat` method must be called -regularly. This will ensure that the coordinator is not considered dead by -other program participating in the coordination. Unless you want to call it -manually, you can use tooz builtin heartbeat manager by passing the -`start_heart` argument. - -.. literalinclude:: ../../../../examples/coordinator_heartbeat.py - :language: python - -heartbeat at different moment or intervals. - -Note that certain drivers, such as `memcached` are heavily based on timeout, -so the interval used to run the heartbeat is important. diff --git a/doc/source/user/tutorial/group_membership.rst b/doc/source/user/tutorial/group_membership.rst deleted file mode 100644 index 502d211..0000000 --- a/doc/source/user/tutorial/group_membership.rst +++ /dev/null @@ -1,41 +0,0 @@ -===================== - Group Membership -===================== - -Basic operations -=================== - -One of the feature provided by the coordinator is the ability to handle -group membership. Once a group is created, any coordinator can join the -group and become a member of it. Any coordinator can be notified when a -members joins or leaves the group. - -.. literalinclude:: ../../../../examples/group_membership.py - :language: python - -Note that all the operation are asynchronous. That means you cannot be sure -that your group has been created or joined before you call the -:meth:`tooz.coordination.CoordAsyncResult.get` method. - -You can also leave a group using the -:meth:`tooz.coordination.CoordinationDriver.leave_group` method. The list of -all available groups is retrievable via the -:meth:`tooz.coordination.CoordinationDriver.get_groups` method. - -Watching Group Changes -====================== -It's possible to watch and get notified when the member list of a group -changes. That's useful to run callback functions whenever something happens -in that group. - - -.. literalinclude:: ../../../../examples/group_membership_watch.py - :language: python - -Using :meth:`tooz.coordination.CoordinationDriver.watch_join_group` and -:meth:`tooz.coordination.CoordinationDriver.watch_leave_group` your -application can be notified each time a member join or leave a group. To -stop watching an event, the two methods -:meth:`tooz.coordination.CoordinationDriver.unwatch_join_group` and -:meth:`tooz.coordination.CoordinationDriver.unwatch_leave_group` allow to -unregister a particular callback. diff --git a/doc/source/user/tutorial/hashring.rst b/doc/source/user/tutorial/hashring.rst deleted file mode 100644 index 1e00d1f..0000000 --- a/doc/source/user/tutorial/hashring.rst +++ /dev/null @@ -1,10 +0,0 @@ -=========== - Hash ring -=========== - -Tooz provides a consistent hash ring implementation. It can be used to map -objects (represented via binary keys) to one or several nodes. When the node -list changes, the rebalancing of objects across the ring is kept minimal. - -.. literalinclude:: ../../../../examples/hashring.py - :language: python diff --git a/doc/source/user/tutorial/index.rst b/doc/source/user/tutorial/index.rst deleted file mode 100644 index c2d398c..0000000 --- a/doc/source/user/tutorial/index.rst +++ /dev/null @@ -1,16 +0,0 @@ -===================================== - Using Tooz in Your Application -===================================== - -This tutorial is a step-by-step walk-through demonstrating how to -use tooz in your application. - -.. toctree:: - :maxdepth: 2 - - coordinator - group_membership - leader_election - lock - hashring - partitioner diff --git a/doc/source/user/tutorial/leader_election.rst b/doc/source/user/tutorial/leader_election.rst deleted file mode 100644 index df2a77b..0000000 --- a/doc/source/user/tutorial/leader_election.rst +++ /dev/null @@ -1,25 +0,0 @@ -================= - Leader Election -================= - -Each group can elect its own leader. There can be only one leader at a time -in a group. Only members that are running for the election can be elected. -As soon as one of leader steps down or dies, a new member that was running -for the election will be elected. - -.. literalinclude:: ../../../../examples/leader_election.py - :language: python - -The method -:meth:`tooz.coordination.CoordinationDriver.watch_elected_as_leader` allows -to register for a function to be called back when the member is elected as a -leader. Using this function indicates that the run is therefore running for -the election. The member can stop running by unregistering all its callbacks -with :meth:`tooz.coordination.CoordinationDriver.unwatch_elected_as_leader`. -It can also temporarily try to step down as a leader with -:meth:`tooz.coordination.CoordinationDriver.stand_down_group_leader`. If -another member is in the run for election, it may be elected instead. - -To retrieve the leader of a group, even when not being part of the group, -the method :meth:`tooz.coordination.CoordinationDriver.get_leader()` can be -used. diff --git a/doc/source/user/tutorial/lock.rst b/doc/source/user/tutorial/lock.rst deleted file mode 100644 index e3aaaae..0000000 --- a/doc/source/user/tutorial/lock.rst +++ /dev/null @@ -1,14 +0,0 @@ -====== - Lock -====== - -Tooz provides distributed locks. A lock is identified by a name, and a lock can -only be acquired by one coordinator at a time. - -.. literalinclude:: ../../../../examples/lock.py - :language: python - -The method :meth:`tooz.coordination.CoordinationDriver.get_lock` allows -to create a lock identified by a name. Once you retrieve this lock, you can -use it as a context manager or use the :meth:`tooz.locking.Lock.acquire` and -:meth:`tooz.locking.Lock.release` methods to acquire and release the lock. diff --git a/doc/source/user/tutorial/partitioner.rst b/doc/source/user/tutorial/partitioner.rst deleted file mode 100644 index 24e12cf..0000000 --- a/doc/source/user/tutorial/partitioner.rst +++ /dev/null @@ -1,11 +0,0 @@ -============= - Partitioner -============= - -Tooz provides a partitioner object based on its consistent hash ring -implementation. It can be used to map Python objects to one or several nodes. -The partitioner object automatically keeps track of nodes joining and leaving -the group, so the rebalancing is managed. - -.. literalinclude:: ../../../../examples/partitioner.py - :language: python diff --git a/examples/coordinator.py b/examples/coordinator.py deleted file mode 100644 index f0efef4..0000000 --- a/examples/coordinator.py +++ /dev/null @@ -1,5 +0,0 @@ -from tooz import coordination - -coordinator = coordination.get_coordinator('zake://', b'host-1') -coordinator.start() -coordinator.stop() diff --git a/examples/coordinator_heartbeat.py b/examples/coordinator_heartbeat.py deleted file mode 100644 index 2a6534a..0000000 --- a/examples/coordinator_heartbeat.py +++ /dev/null @@ -1,5 +0,0 @@ -from tooz import coordination - -coordinator = coordination.get_coordinator('zake://', b'host-1') -coordinator.start(start_heart=True) -coordinator.stop() diff --git a/examples/group_membership.py b/examples/group_membership.py deleted file mode 100644 index 7b85e6c..0000000 --- a/examples/group_membership.py +++ /dev/null @@ -1,19 +0,0 @@ -import uuid - -import six - -from tooz import coordination - -coordinator = coordination.get_coordinator('zake://', b'host-1') -coordinator.start() - -# Create a group -group = six.binary_type(six.text_type(uuid.uuid4()).encode('ascii')) -request = coordinator.create_group(group) -request.get() - -# Join a group -request = coordinator.join_group(group) -request.get() - -coordinator.stop() diff --git a/examples/group_membership_watch.py b/examples/group_membership_watch.py deleted file mode 100644 index 51836c2..0000000 --- a/examples/group_membership_watch.py +++ /dev/null @@ -1,22 +0,0 @@ -import uuid - -import six - -from tooz import coordination - -coordinator = coordination.get_coordinator('zake://', b'host-1') -coordinator.start() - -# Create a group -group = six.binary_type(six.text_type(uuid.uuid4()).encode('ascii')) -request = coordinator.create_group(group) -request.get() - - -def group_joined(event): - # Event is an instance of tooz.coordination.MemberJoinedGroup - print(event.group_id, event.member_id) - - -coordinator.watch_join_group(group, group_joined) -coordinator.stop() diff --git a/examples/hashring.py b/examples/hashring.py deleted file mode 100644 index 31ab1d4..0000000 --- a/examples/hashring.py +++ /dev/null @@ -1,15 +0,0 @@ -from tooz import hashring - -hashring = hashring.HashRing({'node1', 'node2', 'node3'}) - -# Returns set(['node2']) -nodes_for_foo = hashring[b'foo'] - -# Returns set(['node2', 'node3']) -nodes_for_foo_with_replicas = hashring.get_nodes(b'foo', - replicas=2) - -# Returns set(['node1', 'node3']) -nodes_for_foo_with_replicas = hashring.get_nodes(b'foo', - replicas=2, - ignore_nodes={'node2'}) diff --git a/examples/leader_election.py b/examples/leader_election.py deleted file mode 100644 index 661c314..0000000 --- a/examples/leader_election.py +++ /dev/null @@ -1,36 +0,0 @@ -import time -import uuid - -import six - -from tooz import coordination - -ALIVE_TIME = 1 -coordinator = coordination.get_coordinator('zake://', b'host-1') -coordinator.start() - -# Create a group -group = six.binary_type(six.text_type(uuid.uuid4()).encode('ascii')) -request = coordinator.create_group(group) -request.get() - -# Join a group -request = coordinator.join_group(group) -request.get() - - -def when_i_am_elected_leader(event): - # event is a LeaderElected event - print(event.group_id, event.member_id) - - -# Propose to be a leader for the group -coordinator.watch_elected_as_leader(group, when_i_am_elected_leader) - -start = time.time() -while time.time() - start < ALIVE_TIME: - coordinator.heartbeat() - coordinator.run_watchers() - time.sleep(0.1) - -coordinator.stop() diff --git a/examples/lock.py b/examples/lock.py deleted file mode 100644 index 42a1d03..0000000 --- a/examples/lock.py +++ /dev/null @@ -1,11 +0,0 @@ -from tooz import coordination - -coordinator = coordination.get_coordinator('zake://', b'host-1') -coordinator.start() - -# Create a lock -lock = coordinator.get_lock("foobar") -with lock: - print("Do something that is distributed") - -coordinator.stop() diff --git a/examples/partitioner.py b/examples/partitioner.py deleted file mode 100644 index 6bf3aa2..0000000 --- a/examples/partitioner.py +++ /dev/null @@ -1,11 +0,0 @@ -from tooz import coordination - -coordinator = coordination.get_coordinator('zake://', b'host-1') -coordinator.start() -partitioner = coordinator.join_partitioned_group("group1") - -# Returns {'host-1'} -member = partitioner.members_for_object(object()) - -coordinator.leave_partitioned_group(partitioner) -coordinator.stop() diff --git a/releasenotes/notes/add-reno-996dd44974d53238.yaml b/releasenotes/notes/add-reno-996dd44974d53238.yaml deleted file mode 100644 index 2234c38..0000000 --- a/releasenotes/notes/add-reno-996dd44974d53238.yaml +++ /dev/null @@ -1,3 +0,0 @@ ---- -other: - - Introduce reno for deployer release notes. diff --git a/releasenotes/notes/hashring-0470f9119ef63d49.yaml b/releasenotes/notes/hashring-0470f9119ef63d49.yaml deleted file mode 100644 index de7b7cf..0000000 --- a/releasenotes/notes/hashring-0470f9119ef63d49.yaml +++ /dev/null @@ -1,3 +0,0 @@ ---- -features: - - Add `tooz.hashring`, a consistent hash ring implementation. diff --git a/releasenotes/notes/join_group_create-5095ec02e20c7242.yaml b/releasenotes/notes/join_group_create-5095ec02e20c7242.yaml deleted file mode 100644 index a831321..0000000 --- a/releasenotes/notes/join_group_create-5095ec02e20c7242.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- -features: - - Coordination drivers now have a method `join_group_create` that is able to - create a group before joining it if it does not exist yet. diff --git a/releasenotes/notes/partitioner-4005767d287dc7c9.yaml b/releasenotes/notes/partitioner-4005767d287dc7c9.yaml deleted file mode 100644 index 6cf1f45..0000000 --- a/releasenotes/notes/partitioner-4005767d287dc7c9.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -features: - - >- - Introduce a new partitioner object. This object is synchronized within a - group of nodes and exposes a way to distribute object management across - several nodes. diff --git a/releasenotes/source/_static/.placeholder b/releasenotes/source/_static/.placeholder deleted file mode 100644 index e69de29..0000000 diff --git a/releasenotes/source/_templates/.placeholder b/releasenotes/source/_templates/.placeholder deleted file mode 100644 index e69de29..0000000 diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py deleted file mode 100644 index f575b4d..0000000 --- a/releasenotes/source/conf.py +++ /dev/null @@ -1,281 +0,0 @@ -# -*- coding: utf-8 -*- -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'openstackdocstheme', - 'reno.sphinxext', -] - -# openstackdocstheme options -repository_name = 'openstack/tooz' -bug_project = 'tooz' -bug_tag = '' - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'tooz Release Notes' -copyright = u'2016, tooz Developers' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -# The full version, including alpha/beta/rc tags. -import pkg_resources -release = pkg_resources.get_distribution('tooz').version -# The short X.Y version. -version = release - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = [] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -# keep_warnings = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'openstackdocs' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -# html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -html_last_updated_fmt = '%Y-%m-%d %H:%M' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_domain_indices = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'toozReleaseNotesDoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # 'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ('index', 'toozReleaseNotes.tex', - u'tooz Release Notes Documentation', - u'tooz Developers', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False - -# If true, show page references after internal links. -# latex_show_pagerefs = False - -# If true, show URL addresses after external links. -# latex_show_urls = False - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'toozReleaseNotes', - u'tooz Release Notes Documentation', - [u'tooz Developers'], 1) -] - -# If true, show URL addresses after external links. -# man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'toozReleaseNotes', - u'tooz Release Notes Documentation', - u'tooz Developers', 'toozReleaseNotes', - 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -# texinfo_appendices = [] - -# If false, no module index is generated. -# texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -# texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -# texinfo_no_detailmenu = False - -# -- Options for Internationalization output ------------------------------ -locale_dirs = ['locale/'] diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst deleted file mode 100644 index d6392aa..0000000 --- a/releasenotes/source/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -==================== - tooz Release Notes -==================== - - .. toctree:: - :maxdepth: 1 - - unreleased - ocata diff --git a/releasenotes/source/ocata.rst b/releasenotes/source/ocata.rst deleted file mode 100644 index ebe62f4..0000000 --- a/releasenotes/source/ocata.rst +++ /dev/null @@ -1,6 +0,0 @@ -=================================== - Ocata Series Release Notes -=================================== - -.. release-notes:: - :branch: origin/stable/ocata diff --git a/releasenotes/source/unreleased.rst b/releasenotes/source/unreleased.rst deleted file mode 100644 index cd22aab..0000000 --- a/releasenotes/source/unreleased.rst +++ /dev/null @@ -1,5 +0,0 @@ -============================== - Current Series Release Notes -============================== - -.. release-notes:: diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 893cdc5..0000000 --- a/requirements.txt +++ /dev/null @@ -1,15 +0,0 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. -pbr>=1.6 # Apache-2.0 -stevedore>=1.16.0 # Apache-2.0 -six>=1.9.0 # MIT -enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD -voluptuous>=0.8.9 # BSD License -msgpack-python>=0.4.0 # Apache-2.0 -fasteners>=0.7 # Apache-2.0 -tenacity>=3.2.1 # Apache-2.0 -futures>=3.0;python_version=='2.7' or python_version=='2.6' # BSD -futurist!=0.15.0,>=0.11.0 # Apache-2.0 -oslo.utils>=3.15.0 # Apache-2.0 -oslo.serialization>=1.10.0 # Apache-2.0 diff --git a/run-examples.sh b/run-examples.sh deleted file mode 100755 index 0f13fd7..0000000 --- a/run-examples.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -e - -python_version=$(python --version 2>&1) -echo "Running using '$python_version'" -for filename in examples/*.py; do - echo "Activating '$filename'" - python $filename -done diff --git a/run-tests.sh b/run-tests.sh deleted file mode 100755 index 56fa755..0000000 --- a/run-tests.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -e -set -x - -if [ -n "$TOOZ_TEST_DRIVERS" ] -then - IFS="," - for TOOZ_TEST_DRIVER in $TOOZ_TEST_DRIVERS - do - IFS=" " - TOOZ_TEST_DRIVER=(${TOOZ_TEST_DRIVER}) - SETUP_ENV_SCRIPT="./setup-${TOOZ_TEST_DRIVER[0]}-env.sh" - [ -x $SETUP_ENV_SCRIPT ] || unset SETUP_ENV_SCRIPT - $SETUP_ENV_SCRIPT pifpaf -e TOOZ_TEST run "${TOOZ_TEST_DRIVER[@]}" -- $* - done - unset IFS -else - for d in $TOOZ_TEST_URLS - do - TOOZ_TEST_URL=$d $* - done -fi diff --git a/setup-consul-env.sh b/setup-consul-env.sh deleted file mode 100755 index 21ecef8..0000000 --- a/setup-consul-env.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -eux - -if [ -z "$(which consul)" ]; then - CONSUL_VERSION=0.6.3 - CONSUL_RELEASE_URL=https://releases.hashicorp.com/consul - case `uname -s` in - Darwin) - consul_file="consul_${CONSUL_VERSION}_darwin_amd64.zip" - ;; - Linux) - consul_file="consul_${CONSUL_VERSION}_linux_amd64.zip" - ;; - *) - echo "Unknown operating system" - exit 1 - ;; - esac - consul_dir=`basename $consul_file .zip` - mkdir -p $consul_dir - curl -L $CONSUL_RELEASE_URL/$CONSUL_VERSION/$consul_file > $consul_dir/$consul_file - unzip $consul_dir/$consul_file -d $consul_dir - export PATH=$PATH:$consul_dir -fi - -# Yield execution to venv command -$* diff --git a/setup-etcd-env.sh b/setup-etcd-env.sh deleted file mode 100755 index cfdbaac..0000000 --- a/setup-etcd-env.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -set -eux -if [ -z "$(which etcd)" ]; then - ETCD_VERSION=3.1.3 - case `uname -s` in - Darwin) - OS=darwin - SUFFIX=zip - ;; - Linux) - OS=linux - SUFFIX=tar.gz - ;; - *) - echo "Unsupported OS" - exit 1 - esac - case `uname -m` in - x86_64) - MACHINE=amd64 - ;; - *) - echo "Unsupported machine" - exit 1 - esac - TARBALL_NAME=etcd-v${ETCD_VERSION}-$OS-$MACHINE - test ! -d "$TARBALL_NAME" && curl -L https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/${TARBALL_NAME}.${SUFFIX} | tar xz - export PATH=$PATH:$TARBALL_NAME -fi - -$* diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index c997a87..0000000 --- a/setup.cfg +++ /dev/null @@ -1,86 +0,0 @@ -[metadata] -name = tooz -author = OpenStack -author-email = openstack-dev@lists.openstack.org -summary = Coordination library for distributed systems. -description-file = README.rst -license = Apache-2 -home-page = https://docs.openstack.org/tooz/latest/ -classifier = - Environment :: OpenStack - Intended Audience :: Developers - Intended Audience :: Information Technology - License :: OSI Approved :: Apache Software License - Operating System :: POSIX :: Linux - Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 - Topic :: System :: Distributed Computing - -[files] -packages = - tooz - -[entry_points] -tooz.backends = - etcd = tooz.drivers.etcd:EtcdDriver - etcd3 = tooz.drivers.etcd3:Etcd3Driver - etcd3+http = tooz.drivers.etcd3gw:Etcd3Driver - kazoo = tooz.drivers.zookeeper:KazooDriver - zake = tooz.drivers.zake:ZakeDriver - memcached = tooz.drivers.memcached:MemcachedDriver - ipc = tooz.drivers.ipc:IPCDriver - redis = tooz.drivers.redis:RedisDriver - postgresql = tooz.drivers.pgsql:PostgresDriver - mysql = tooz.drivers.mysql:MySQLDriver - file = tooz.drivers.file:FileDriver - zookeeper = tooz.drivers.zookeeper:KazooDriver - consul = tooz.drivers.consul:ConsulDriver - -[extras] -consul = - python-consul>=0.4.7 # MIT License -etcd = - requests>=2.10.0 # Apache-2.0 -etcd3 = - etcd3>=0.6.2 # Apache-2.0 -etcd3gw = - etcd3gw>=0.1.0 # Apache-2.0 -zake = - zake>=0.1.6 # Apache-2.0 -redis = - redis>=2.10.0 # MIT -postgresql = - psycopg2>=2.5 # LGPL/ZPL -mysql = - PyMySQL>=0.6.2 # MIT License -zookeeper = - kazoo>=2.2 # Apache-2.0 -memcached = - pymemcache!=1.3.0,>=1.2.9 # Apache 2.0 License -ipc = - sysv-ipc>=0.6.8 # BSD License -test = - mock>=2.0 # BSD - python-subunit>=0.0.18 # Apache-2.0/BSD - testrepository>=0.0.18 # Apache-2.0/BSD - testtools>=1.4.0 # MIT - coverage>=3.6 # Apache-2.0 - fixtures>=3.0.0 # Apache-2.0/BSD - pifpaf>=0.10.0 # Apache-2.0 - os-testr>=0.8.0 # Apache-2.0 -doc = - sphinx>=1.6.2 # BSD - openstackdocstheme>=1.11.0 # Apache-2.0 - reno>=1.8.0 # Apache-2.0 - -[wheel] -universal = 1 - -[build_sphinx] -all_files = 1 -build-dir = doc/build -source-dir = doc/source -warning-is-error = 1 diff --git a/setup.py b/setup.py deleted file mode 100644 index 782bb21..0000000 --- a/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT -import setuptools - -# In python < 2.7.4, a lazy loading of package `pbr` will break -# setuptools if some other modules registered functions in `atexit`. -# solution from: http://bugs.python.org/issue15881#msg170215 -try: - import multiprocessing # noqa -except ImportError: - pass - -setuptools.setup( - setup_requires=['pbr>=1.8'], - pbr=True) diff --git a/tools/compat-matrix.py b/tools/compat-matrix.py deleted file mode 100644 index 53574b8..0000000 --- a/tools/compat-matrix.py +++ /dev/null @@ -1,150 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from tabulate import tabulate - - -def print_header(txt, delim="="): - print(txt) - print(delim * len(txt)) - - -def print_methods(methods): - driver_tpl = ":py:meth:`~tooz.coordination.CoordinationDriver.%s`" - for api_name in methods: - method_name = driver_tpl % api_name - print("* %s" % method_name) - if methods: - print("") - - -driver_tpl = ":py:class:`~tooz.drivers.%s`" -driver_class_names = [ - "consul.ConsulDriver", - "etcd.EtcdDriver", - "file.FileDriver", - "ipc.IPCDriver", - "memcached.MemcachedDriver", - "mysql.MySQLDriver", - "pgsql.PostgresDriver", - "redis.RedisDriver", - "zake.ZakeDriver", - "zookeeper.KazooDriver", -] -driver_headers = [] -for n in driver_class_names: - driver_headers.append(driver_tpl % (n)) - -print_header("Grouping") -print("") - -print_header("APIs", delim="-") -print("") -grouping_methods = [ - 'watch_join_group', - 'unwatch_join_group', - 'watch_leave_group', - 'unwatch_leave_group', - 'create_group', - 'get_groups', - 'join_group', - 'leave_group', - 'delete_group', - 'get_members', - 'get_member_capabilities', - 'update_capabilities', -] -print_methods(grouping_methods) - -print_header("Driver support", delim="-") -print("") -grouping_table = [ - [ - "No", # Consul - "No", # Etcd - "Yes", # File - "No", # IPC - "Yes", # Memcached - "No", # MySQL - "No", # PostgreSQL - "Yes", # Redis - "Yes", # Zake - "Yes", # Zookeeper - ], -] -print(tabulate(grouping_table, driver_headers, tablefmt="rst")) -print("") - -print_header("Leaders") -print("") - -print_header("APIs", delim="-") -print("") -leader_methods = [ - 'watch_elected_as_leader', - 'unwatch_elected_as_leader', - 'stand_down_group_leader', - 'get_leader', -] -print_methods(leader_methods) - -print_header("Driver support", delim="-") -print("") -leader_table = [ - [ - "No", # Consul - "No", # Etcd - "No", # File - "No", # IPC - "Yes", # Memcached - "No", # MySQL - "No", # PostgreSQL - "Yes", # Redis - "Yes", # Zake - "Yes", # Zookeeper - ], -] -print(tabulate(leader_table, driver_headers, tablefmt="rst")) -print("") - -print_header("Locking") -print("") - -print_header("APIs", delim="-") -print("") -lock_methods = [ - 'get_lock', -] -print_methods(lock_methods) - -print_header("Driver support", delim="-") -print("") -lock_table = [ - [ - "Yes", # Consul - "Yes", # Etcd - "Yes", # File - "Yes", # IPC - "Yes", # Memcached - "Yes", # MySQL - "Yes", # PostgreSQL - "Yes", # Redis - "Yes", # Zake - "Yes", # Zookeeper - ], -] -print(tabulate(lock_table, driver_headers, tablefmt="rst")) -print("") diff --git a/tools/pretty_tox.sh b/tools/pretty_tox.sh deleted file mode 100755 index 799ac18..0000000 --- a/tools/pretty_tox.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -o pipefail - -TESTRARGS=$1 - -# --until-failure is not compatible with --subunit see: -# -# https://bugs.launchpad.net/testrepository/+bug/1411804 -# -# this work around exists until that is addressed -if [[ "$TESTARGS" =~ "until-failure" ]]; then - python setup.py testr --slowest --testr-args="$TESTRARGS" -else - python setup.py testr --slowest --testr-args="--subunit $TESTRARGS" | subunit-trace -f -fi diff --git a/tools/tox_install.sh b/tools/tox_install.sh deleted file mode 100755 index e61b63a..0000000 --- a/tools/tox_install.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# Client constraint file contains this client version pin that is in conflict -# with installing the client from source. We should remove the version pin in -# the constraints file before applying it for from-source installation. - -CONSTRAINTS_FILE="$1" -shift 1 - -set -e - -# NOTE(tonyb): Place this in the tox enviroment's log dir so it will get -# published to logs.openstack.org for easy debugging. -localfile="$VIRTUAL_ENV/log/upper-constraints.txt" - -if [[ "$CONSTRAINTS_FILE" != http* ]]; then - CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE" -fi -# NOTE(tonyb): need to add curl to bindep.txt if the project supports bindep -curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile" - -pip install -c"$localfile" openstack-requirements - -# This is the main purpose of the script: Allow local installation of -# the current repo. It is listed in constraints file and thus any -# install will be constrained and we need to unconstrain it. -edit-constraints "$localfile" -- "$CLIENT_NAME" - -pip install -c"$localfile" -U "$@" -exit $? diff --git a/tooz/__init__.py b/tooz/__init__.py deleted file mode 100644 index f265759..0000000 --- a/tooz/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2014 eNovance Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -class ToozError(Exception): - """Exception raised when an internal error occurs. - - Raised for instance in case of server internal error. - - :ivar cause: the cause of the exception being raised, when not none this - will itself be an exception instance, this is useful for - creating a chain of exceptions for versions of python where - this is not yet implemented/supported natively. - - """ - - def __init__(self, message, cause=None): - super(ToozError, self).__init__(message) - self.cause = cause - - -class NotImplemented(NotImplementedError, ToozError): - pass diff --git a/tooz/_retry.py b/tooz/_retry.py deleted file mode 100644 index 0fa7e95..0000000 --- a/tooz/_retry.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2016 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import tenacity -from tenacity import stop -from tenacity import wait - - -_default_wait = wait.wait_exponential(max=1) - - -def retry(stop_max_delay=None, **kwargs): - k = {"wait": _default_wait, "retry": lambda x: False} - if stop_max_delay not in (True, False, None): - k['stop'] = stop.stop_after_delay(stop_max_delay) - return tenacity.retry(**k) - - -TryAgain = tenacity.TryAgain diff --git a/tooz/coordination.py b/tooz/coordination.py deleted file mode 100755 index f118792..0000000 --- a/tooz/coordination.py +++ /dev/null @@ -1,897 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2016 Red Hat, Inc. -# Copyright (C) 2013-2014 eNovance Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import collections -from concurrent import futures -import enum -import logging -import threading - -from oslo_utils import encodeutils -from oslo_utils import excutils -from oslo_utils import netutils -from oslo_utils import timeutils -import six -from stevedore import driver - -import tooz -from tooz import _retry -from tooz import partitioner -from tooz import utils - -LOG = logging.getLogger(__name__) - - -TOOZ_BACKENDS_NAMESPACE = "tooz.backends" - - -class Characteristics(enum.Enum): - """Attempts to describe the characteristic that a driver supports.""" - - DISTRIBUTED_ACROSS_THREADS = 'DISTRIBUTED_ACROSS_THREADS' - """Coordinator components when used by multiple **threads** work - the same as if those components were only used by a single thread.""" - - DISTRIBUTED_ACROSS_PROCESSES = 'DISTRIBUTED_ACROSS_PROCESSES' - """Coordinator components when used by multiple **processes** work - the same as if those components were only used by a single thread.""" - - DISTRIBUTED_ACROSS_HOSTS = 'DISTRIBUTED_ACROSS_HOSTS' - """Coordinator components when used by multiple **hosts** work - the same as if those components were only used by a single thread.""" - - NON_TIMEOUT_BASED = 'NON_TIMEOUT_BASED' - """The driver has the following property: - - * Its operations are not based on the timeout of other clients, but on some - other more robust mechanisms. - """ - - LINEARIZABLE = 'LINEARIZABLE' - """The driver has the following properties: - - * Ensures each operation must take place before its - completion time. - * Any operation invoked subsequently must take place - after the invocation and by extension, after the original operation - itself. - """ - - SEQUENTIAL = 'SEQUENTIAL' - """The driver has the following properties: - - * Operations can take effect before or after completion – but all - operations retain the constraint that operations from any given process - must take place in that processes order. - """ - - CAUSAL = 'CAUSAL' - """The driver has the following properties: - - * Does **not** have to enforce the order of every - operation from a process, perhaps, only causally related operations - must occur in order. - """ - - SERIALIZABLE = 'SERIALIZABLE' - """The driver has the following properties: - - * The history of **all** operations is equivalent to - one that took place in some single atomic order but with unknown - invocation and completion times - it places no bounds on - time or order. - """ - - SAME_VIEW_UNDER_PARTITIONS = 'SAME_VIEW_UNDER_PARTITIONS' - """When a client is connected to a server and that server is partitioned - from a group of other servers it will (somehow) have the same view of - data as a client connected to a server on the other side of the - partition (typically this is accomplished by write availability being - lost and therefore nothing can change). - """ - - SAME_VIEW_ACROSS_CLIENTS = 'SAME_VIEW_ACROSS_CLIENTS' - """A client connected to one server will *always* have the same view - every other client will have (no matter what server those other - clients are connected to). Typically this is a sacrifice in - write availability because before a write can be acknowledged it must - be acknowledged by *all* servers in a cluster (so that all clients - that are connected to those servers read the exact *same* thing). - """ - - -class Hooks(list): - def run(self, *args, **kwargs): - return list(map(lambda cb: cb(*args, **kwargs), self)) - - -class Event(object): - """Base class for events.""" - - -class MemberJoinedGroup(Event): - """A member joined a group event.""" - - def __init__(self, group_id, member_id): - self.group_id = group_id - self.member_id = member_id - - def __repr__(self): - return "<%s: group %s: +member %s>" % (self.__class__.__name__, - self.group_id, - self.member_id) - - -class MemberLeftGroup(Event): - """A member left a group event.""" - - def __init__(self, group_id, member_id): - self.group_id = group_id - self.member_id = member_id - - def __repr__(self): - return "<%s: group %s: -member %s>" % (self.__class__.__name__, - self.group_id, - self.member_id) - - -class LeaderElected(Event): - """A leader as been elected.""" - - def __init__(self, group_id, member_id): - self.group_id = group_id - self.member_id = member_id - - -class Heart(object): - """Coordination drivers main liveness pump (its heart).""" - - def __init__(self, driver, thread_cls=threading.Thread, - event_cls=threading.Event): - self._thread_cls = thread_cls - self._dead = event_cls() - self._runner = None - self._driver = driver - self._beats = 0 - - @property - def beats(self): - """How many times the heart has beaten.""" - return self._beats - - def is_alive(self): - """Returns if the heart is beating.""" - return not (self._runner is None - or not self._runner.is_alive()) - - @excutils.forever_retry_uncaught_exceptions - def _beat_forever_until_stopped(self): - """Inner beating loop.""" - while not self._dead.is_set(): - with timeutils.StopWatch() as w: - wait_until_next_beat = self._driver.heartbeat() - ran_for = w.elapsed() - has_to_sleep_for = wait_until_next_beat - ran_for - if has_to_sleep_for < 0: - LOG.warning( - "Heartbeating took too long to execute (it ran for" - " %0.2f seconds which is %0.2f seconds longer than" - " the next heartbeat idle time). This may cause" - " timeouts (in locks, leadership, ...) to" - " happen (which will not end well).", ran_for, - ran_for - wait_until_next_beat) - self._beats += 1 - # NOTE(harlowja): use the event object for waiting and - # not a sleep function since doing that will allow this code - # to terminate early if stopped via the stop() method vs - # having to wait until the sleep function returns. - # NOTE(jd): Wait for only the half time of what we should. - # This is a measure of safety, better be too soon than too late. - self._dead.wait(has_to_sleep_for / 2.0) - - def start(self, thread_cls=None): - """Starts the heart beating thread (noop if already started).""" - if not self.is_alive(): - self._dead.clear() - self._beats = 0 - if thread_cls is None: - thread_cls = self._thread_cls - self._runner = thread_cls(target=self._beat_forever_until_stopped) - self._runner.daemon = True - self._runner.start() - - def stop(self): - """Requests the heart beating thread to stop beating.""" - self._dead.set() - - def wait(self, timeout=None): - """Wait up to given timeout for the heart beating thread to stop.""" - self._runner.join(timeout) - return self._runner.is_alive() - - -class CoordinationDriver(object): - - requires_beating = False - """ - Usage requirement that if true requires that the :py:meth:`~.heartbeat` - be called periodically (at a given rate) to avoid locks, sessions and - other from being automatically closed/discarded by the coordinators - backing store. - """ - - CHARACTERISTICS = () - """ - Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable - enum member(s) that can be used to interogate how this driver works. - """ - - def __init__(self, member_id, parsed_url, options): - super(CoordinationDriver, self).__init__() - self._member_id = member_id - self._started = False - self._hooks_join_group = collections.defaultdict(Hooks) - self._hooks_leave_group = collections.defaultdict(Hooks) - self._hooks_elected_leader = collections.defaultdict(Hooks) - self.requires_beating = ( - CoordinationDriver.heartbeat != self.__class__.heartbeat - ) - self.heart = Heart(self) - - def _has_hooks_for_group(self, group_id): - return (group_id in self._hooks_join_group or - group_id in self._hooks_leave_group) - - def join_partitioned_group( - self, group_id, - weight=1, - partitions=partitioner.Partitioner.DEFAULT_PARTITION_NUMBER): - """Join a group and get a partitioner. - - A partitioner allows to distribute a bunch of objects across several - members using a consistent hash ring. Each object gets assigned (at - least) one member responsible for it. It's then possible to check which - object is owned by any member of the group. - - This method also creates if necessary, and joins the group with the - selected weight. - - :param group_id: The group to create a partitioner for. - :param weight: The weight to use in the hashring for this node. - :param partitions: The number of partitions to create. - :return: A :py:class:`~tooz.partitioner.Partitioner` object. - - """ - self.join_group_create(group_id, capabilities={'weight': weight}) - return partitioner.Partitioner(self, group_id, partitions=partitions) - - def leave_partitioned_group(self, partitioner): - """Leave a partitioned group. - - This leaves the partitioned group and stop the partitioner. - :param group_id: The group to create a partitioner for. - """ - leave = self.leave_group(partitioner.group_id) - partitioner.stop() - return leave.get() - - @staticmethod - def run_watchers(timeout=None): - """Run the watchers callback. - - This may also activate :py:meth:`.run_elect_coordinator` (depending - on driver implementation). - """ - raise tooz.NotImplemented - - @staticmethod - def run_elect_coordinator(): - """Try to leader elect this coordinator & activate hooks on success.""" - raise tooz.NotImplemented - - def watch_join_group(self, group_id, callback): - """Call a function when group_id sees a new member joined. - - The callback functions will be executed when `run_watchers` is - called. - - :param group_id: The group id to watch - :param callback: The function to execute when a member joins this group - - """ - self._hooks_join_group[group_id].append(callback) - - def unwatch_join_group(self, group_id, callback): - """Stop executing a function when a group_id sees a new member joined. - - :param group_id: The group id to unwatch - :param callback: The function that was executed when a member joined - this group - """ - try: - # Check if group_id is in hooks to avoid creating a default empty - # entry in hooks list. - if group_id not in self._hooks_join_group: - raise ValueError - self._hooks_join_group[group_id].remove(callback) - except ValueError: - raise WatchCallbackNotFound(group_id, callback) - - if not self._hooks_join_group[group_id]: - del self._hooks_join_group[group_id] - - def watch_leave_group(self, group_id, callback): - """Call a function when group_id sees a new member leaving. - - The callback functions will be executed when `run_watchers` is - called. - - :param group_id: The group id to watch - :param callback: The function to execute when a member leaves this - group - - """ - self._hooks_leave_group[group_id].append(callback) - - def unwatch_leave_group(self, group_id, callback): - """Stop executing a function when a group_id sees a new member leaving. - - :param group_id: The group id to unwatch - :param callback: The function that was executed when a member left - this group - """ - try: - # Check if group_id is in hooks to avoid creating a default empty - # entry in hooks list. - if group_id not in self._hooks_leave_group: - raise ValueError - self._hooks_leave_group[group_id].remove(callback) - except ValueError: - raise WatchCallbackNotFound(group_id, callback) - - if not self._hooks_leave_group[group_id]: - del self._hooks_leave_group[group_id] - - def watch_elected_as_leader(self, group_id, callback): - """Call a function when member gets elected as leader. - - The callback functions will be executed when `run_watchers` is - called. - - :param group_id: The group id to watch - :param callback: The function to execute when a member leaves this - group - - """ - self._hooks_elected_leader[group_id].append(callback) - - def unwatch_elected_as_leader(self, group_id, callback): - """Call a function when member gets elected as leader. - - The callback functions will be executed when `run_watchers` is - called. - - :param group_id: The group id to watch - :param callback: The function to execute when a member leaves this - group - - """ - try: - self._hooks_elected_leader[group_id].remove(callback) - except ValueError: - raise WatchCallbackNotFound(group_id, callback) - - if not self._hooks_elected_leader[group_id]: - del self._hooks_elected_leader[group_id] - - @staticmethod - def stand_down_group_leader(group_id): - """Stand down as the group leader if we are. - - :param group_id: The group where we don't want to be a leader anymore - """ - raise tooz.NotImplemented - - @property - def is_started(self): - return self._started - - def start(self, start_heart=False): - """Start the service engine. - - If needed, the establishment of a connection to the servers - is initiated. - """ - if self._started: - raise tooz.ToozError( - "Can not start a driver which has not been stopped") - self._start() - if self.requires_beating and start_heart: - self.heart.start() - self._started = True - # Tracks which group are joined - self._joined_groups = set() - - def _start(self): - pass - - def stop(self): - """Stop the service engine. - - If needed, the connection to servers is closed and the client will - disappear from all joined groups. - """ - if not self._started: - raise tooz.ToozError( - "Can not stop a driver which has not been started") - if self.heart.is_alive(): - self.heart.stop() - self.heart.wait() - # Some of the drivers modify joined_groups when being called to leave - # so clone it so that we aren't modifying something while iterating. - joined_groups = self._joined_groups.copy() - leaving = [self.leave_group(group) for group in joined_groups] - for fut in leaving: - try: - fut.get() - except tooz.ToozError: - # Whatever happens, ignore. Maybe we got booted out/never - # existed in the first place, or something is down, but we just - # want to call _stop after whatever happens to not leak any - # connection. - pass - self._stop() - self._started = False - - def _stop(self): - pass - - @staticmethod - def create_group(group_id): - """Request the creation of a group asynchronously. - - :param group_id: the id of the group to create - :type group_id: ascii bytes - :returns: None - :rtype: CoordAsyncResult - """ - raise tooz.NotImplemented - - @staticmethod - def get_groups(): - """Return the list composed by all groups ids asynchronously. - - :returns: the list of all created group ids - :rtype: CoordAsyncResult - """ - raise tooz.NotImplemented - - @staticmethod - def join_group(group_id, capabilities=b""): - """Join a group and establish group membership asynchronously. - - :param group_id: the id of the group to join - :type group_id: ascii bytes - :param capabilities: the capabilities of the joined member - :type capabilities: object - :returns: None - :rtype: CoordAsyncResult - """ - raise tooz.NotImplemented - - @_retry.retry() - def join_group_create(self, group_id, capabilities=b""): - """Join a group and create it if necessary. - - If the group cannot be joined because it does not exist, it is created - before being joined. - - This function will keep retrying until it can create the group and join - it. Since nothing is transactional, it may have to retry several times - if another member is creating/deleting the group at the same time. - - :param group_id: Identifier of the group to join and create - :param capabilities: the capabilities of the joined member - """ - req = self.join_group(group_id, capabilities) - try: - req.get() - except GroupNotCreated: - req = self.create_group(group_id) - try: - req.get() - except GroupAlreadyExist: - # The group might have been created in the meantime, ignore - pass - # Now retry to join the group - raise _retry.TryAgain - - @staticmethod - def leave_group(group_id): - """Leave a group asynchronously. - - :param group_id: the id of the group to leave - :type group_id: ascii bytes - :returns: None - :rtype: CoordAsyncResult - """ - raise tooz.NotImplemented - - @staticmethod - def delete_group(group_id): - """Delete a group asynchronously. - - :param group_id: the id of the group to leave - :type group_id: ascii bytes - :returns: Result - :rtype: CoordAsyncResult - """ - raise tooz.NotImplemented - - @staticmethod - def get_members(group_id): - """Return the set of all members ids of the specified group. - - :returns: set of all created group ids - :rtype: CoordAsyncResult - """ - raise tooz.NotImplemented - - @staticmethod - def get_member_capabilities(group_id, member_id): - """Return the capabilities of a member asynchronously. - - :param group_id: the id of the group of the member - :type group_id: ascii bytes - :param member_id: the id of the member - :type member_id: ascii bytes - :returns: capabilities of a member - :rtype: CoordAsyncResult - """ - raise tooz.NotImplemented - - @staticmethod - def get_member_info(group_id, member_id): - """Return the statistics and capabilities of a member asynchronously. - - :param group_id: the id of the group of the member - :type group_id: ascii bytes - :param member_id: the id of the member - :type member_id: ascii bytes - :returns: capabilities and statistics of a member - :rtype: CoordAsyncResult - """ - raise tooz.NotImplemented - - @staticmethod - def update_capabilities(group_id, capabilities): - """Update member capabilities in the specified group. - - :param group_id: the id of the group of the current member - :type group_id: ascii bytes - :param capabilities: the capabilities of the updated member - :type capabilities: object - :returns: None - :rtype: CoordAsyncResult - """ - raise tooz.NotImplemented - - @staticmethod - def get_leader(group_id): - """Return the leader for a group. - - :param group_id: the id of the group: - :returns: the leader - :rtype: CoordAsyncResult - """ - raise tooz.NotImplemented - - @staticmethod - def get_lock(name): - """Return a distributed lock. - - This is a exclusive lock, a second call to acquire() will block or - return False. - - :param name: The lock name that is used to identify it across all - nodes. - - """ - raise tooz.NotImplemented - - @staticmethod - def heartbeat(): - """Update member status to indicate it is still alive. - - Method to run once in a while to be sure that the member is not dead - and is still an active member of a group. - - :return: The number of seconds to wait before sending a new heartbeat. - """ - pass - - -@six.add_metaclass(abc.ABCMeta) -class CoordAsyncResult(object): - """Representation of an asynchronous task. - - Every call API returns an CoordAsyncResult object on which the result or - the status of the task can be requested. - - """ - - @abc.abstractmethod - def get(self, timeout=10): - """Retrieve the result of the corresponding asynchronous call. - - :param timeout: block until the timeout expire. - :type timeout: float - """ - - @abc.abstractmethod - def done(self): - """Returns True if the task is done, False otherwise.""" - - -class CoordinatorResult(CoordAsyncResult): - """Asynchronous result that references a future.""" - - def __init__(self, fut, failure_translator=None): - self._fut = fut - self._failure_translator = failure_translator - - def get(self, timeout=None): - try: - if self._failure_translator: - with self._failure_translator(): - return self._fut.result(timeout=timeout) - else: - return self._fut.result(timeout=timeout) - except futures.TimeoutError as e: - utils.raise_with_cause(OperationTimedOut, - encodeutils.exception_to_unicode(e), - cause=e) - - def done(self): - return self._fut.done() - - -class CoordinationDriverWithExecutor(CoordinationDriver): - - EXCLUDE_OPTIONS = None - - def __init__(self, member_id, parsed_url, options): - self._options = utils.collapse(options, exclude=self.EXCLUDE_OPTIONS) - self._executor = utils.ProxyExecutor.build( - self.__class__.__name__, self._options) - super(CoordinationDriverWithExecutor, self).__init__( - member_id, parsed_url, options) - - def start(self, start_heart=False): - self._executor.start() - super(CoordinationDriverWithExecutor, self).start(start_heart) - - def stop(self): - super(CoordinationDriverWithExecutor, self).stop() - self._executor.stop() - - -class CoordinationDriverCachedRunWatchers(CoordinationDriver): - """Coordination driver with a `run_watchers` implementation. - - This implementation of `run_watchers` is based on a cache of the group - members between each run of `run_watchers` that is being updated between - each run. - - """ - - def __init__(self, member_id, parsed_url, options): - super(CoordinationDriverCachedRunWatchers, self).__init__( - member_id, parsed_url, options) - # A cache for group members - self._group_members = collections.defaultdict(set) - self._joined_groups = set() - - def _init_watch_group(self, group_id): - if group_id not in self._group_members: - members = self.get_members(group_id) - self._group_members[group_id] = members.get() - - def watch_join_group(self, group_id, callback): - self._init_watch_group(group_id) - super(CoordinationDriverCachedRunWatchers, self).watch_join_group( - group_id, callback) - - def unwatch_join_group(self, group_id, callback): - super(CoordinationDriverCachedRunWatchers, self).unwatch_join_group( - group_id, callback) - - if (not self._has_hooks_for_group(group_id) and - group_id in self._group_members): - del self._group_members[group_id] - - def watch_leave_group(self, group_id, callback): - self._init_watch_group(group_id) - super(CoordinationDriverCachedRunWatchers, self).watch_leave_group( - group_id, callback) - - def unwatch_leave_group(self, group_id, callback): - super(CoordinationDriverCachedRunWatchers, self).unwatch_leave_group( - group_id, callback) - - if (not self._has_hooks_for_group(group_id) and - group_id in self._group_members): - del self._group_members[group_id] - - def run_watchers(self, timeout=None): - with timeutils.StopWatch(duration=timeout) as w: - result = [] - group_with_hooks = set(self._hooks_join_group.keys()).union( - set(self._hooks_leave_group.keys())) - for group_id in group_with_hooks: - try: - group_members = self.get_members(group_id).get( - timeout=w.leftover(return_none=True)) - except GroupNotCreated: - group_members = set() - if (group_id in self._joined_groups and - self._member_id not in group_members): - self._joined_groups.discard(group_id) - old_group_members = self._group_members.get(group_id, set()) - for member_id in (group_members - old_group_members): - result.extend( - self._hooks_join_group[group_id].run( - MemberJoinedGroup(group_id, member_id))) - for member_id in (old_group_members - group_members): - result.extend( - self._hooks_leave_group[group_id].run( - MemberLeftGroup(group_id, member_id))) - self._group_members[group_id] = group_members - return result - - -def get_coordinator(backend_url, member_id, - characteristics=frozenset(), **kwargs): - """Initialize and load the backend. - - :param backend_url: the backend URL to use - :type backend: str - :param member_id: the id of the member - :type member_id: ascii bytes - :param characteristics: set - :type characteristics: set of :py:class:`.Characteristics` that will - be matched to the requested driver (this **will** - become a **required** parameter in a future tooz - version) - :param kwargs: additional coordinator options (these take precedence over - options of the **same** name found in the ``backend_url`` - arguments query string) - """ - parsed_url = netutils.urlsplit(backend_url) - parsed_qs = six.moves.urllib.parse.parse_qs(parsed_url.query) - if kwargs: - options = {} - for (k, v) in six.iteritems(kwargs): - options[k] = [v] - for (k, v) in six.iteritems(parsed_qs): - if k not in options: - options[k] = v - else: - options = parsed_qs - d = driver.DriverManager( - namespace=TOOZ_BACKENDS_NAMESPACE, - name=parsed_url.scheme, - invoke_on_load=True, - invoke_args=(member_id, parsed_url, options)).driver - characteristics = set(characteristics) - driver_characteristics = set(getattr(d, 'CHARACTERISTICS', set())) - missing_characteristics = characteristics - driver_characteristics - if missing_characteristics: - raise ToozDriverChosenPoorly("Desired characteristics %s" - " is not a strict subset of driver" - " characteristics %s, %s" - " characteristics were not found" - % (characteristics, - driver_characteristics, - missing_characteristics)) - return d - - -# TODO(harlowja): We'll have to figure out a way to remove this 'alias' at -# some point in the future (when we have a better way to tell people it has -# moved without messing up their exception catching hierarchy). -ToozError = tooz.ToozError - - -class ToozDriverChosenPoorly(tooz.ToozError): - """Raised when a driver does not match desired characteristics.""" - - -class ToozConnectionError(tooz.ToozError): - """Exception raised when the client cannot connect to the server.""" - - -class OperationTimedOut(tooz.ToozError): - """Exception raised when an operation times out.""" - - -class LockAcquireFailed(tooz.ToozError): - """Exception raised when a lock acquire fails in a context manager.""" - - -class GroupNotCreated(tooz.ToozError): - """Exception raised when the caller request an nonexistent group.""" - def __init__(self, group_id): - self.group_id = group_id - super(GroupNotCreated, self).__init__( - "Group %s does not exist" % group_id) - - -class GroupAlreadyExist(tooz.ToozError): - """Exception raised trying to create an already existing group.""" - def __init__(self, group_id): - self.group_id = group_id - super(GroupAlreadyExist, self).__init__( - "Group %s already exists" % group_id) - - -class MemberAlreadyExist(tooz.ToozError): - """Exception raised trying to join a group already joined.""" - def __init__(self, group_id, member_id): - self.group_id = group_id - self.member_id = member_id - super(MemberAlreadyExist, self).__init__( - "Member %s has already joined %s" % - (member_id, group_id)) - - -class MemberNotJoined(tooz.ToozError): - """Exception raised trying to access a member not in a group.""" - def __init__(self, group_id, member_id): - self.group_id = group_id - self.member_id = member_id - super(MemberNotJoined, self).__init__("Member %s has not joined %s" % - (member_id, group_id)) - - -class GroupNotEmpty(tooz.ToozError): - "Exception raised when the caller try to delete a group with members." - def __init__(self, group_id): - self.group_id = group_id - super(GroupNotEmpty, self).__init__("Group %s is not empty" % group_id) - - -class WatchCallbackNotFound(tooz.ToozError): - """Exception raised when unwatching a group. - - Raised when the caller tries to unwatch a group with a callback that - does not exist. - - """ - def __init__(self, group_id, callback): - self.group_id = group_id - self.callback = callback - super(WatchCallbackNotFound, self).__init__( - 'Callback %s is not registered on group %s' % - (callback.__name__, group_id)) - - -# TODO(harlowja,jd): We'll have to figure out a way to remove this 'alias' at -# some point in the future (when we have a better way to tell people it has -# moved without messing up their exception catching hierarchy). -SerializationError = utils.SerializationError diff --git a/tooz/drivers/__init__.py b/tooz/drivers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tooz/drivers/consul.py b/tooz/drivers/consul.py deleted file mode 100644 index f9fd559..0000000 --- a/tooz/drivers/consul.py +++ /dev/null @@ -1,172 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2015 Yahoo! Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from __future__ import absolute_import - -import consul -from oslo_utils import encodeutils - -import tooz -from tooz import _retry -from tooz import coordination -from tooz import locking -from tooz import utils - - -class ConsulLock(locking.Lock): - def __init__(self, name, node, address, session_id, client): - super(ConsulLock, self).__init__(name) - self._name = name - self._node = node - self._address = address - self._session_id = session_id - self._client = client - self.acquired = False - - def acquire(self, blocking=True, shared=False): - if shared: - raise tooz.NotImplemented - - @_retry.retry(stop_max_delay=blocking) - def _acquire(): - # Check if we are the owner and if we are simulate - # blocking (because consul will not block a second - # acquisition attempt by the same owner). - _index, value = self._client.kv.get(key=self._name) - if value and value.get('Session') == self._session_id: - if blocking is False: - return False - else: - raise _retry.TryAgain - else: - # The value can be anything. - gotten = self._client.kv.put(key=self._name, - value=u"I got it!", - acquire=self._session_id) - if gotten: - self.acquired = True - return True - if blocking is False: - return False - else: - raise _retry.TryAgain - - return _acquire() - - def release(self): - if not self.acquired: - return False - # Get the lock to verify the session ID's are same - _index, contents = self._client.kv.get(key=self._name) - if not contents: - return False - owner = contents.get('Session') - if owner == self._session_id: - removed = self._client.kv.put(key=self._name, - value=self._session_id, - release=self._session_id) - if removed: - self.acquired = False - return True - return False - - -class ConsulDriver(coordination.CoordinationDriver): - """This driver uses `python-consul`_ client against `consul`_ servers. - - The ConsulDriver implements a minimal set of coordination driver API(s) - needed to make Consul being used as an option for Distributed Locking. The - data is stored in Consul's key-value store. - - To configure the client to your liking please refer - http://python-consul.readthedocs.org/en/latest/. Few options like 'ttl' - and 'namespace' will be passed as part of the options. 'ttl' governs the - duration till when the session holding the lock will be active. - - .. _python-consul: http://python-consul.readthedocs.org/ - .. _consul: https://consul.io/ - """ - - #: Default namespace when none is provided - TOOZ_NAMESPACE = u"tooz" - - #: Default TTL - DEFAULT_TTL = 15 - - #: Default consul port if not provided. - DEFAULT_PORT = 8500 - - def __init__(self, member_id, parsed_url, options): - super(ConsulDriver, self).__init__(member_id, parsed_url, options) - options = utils.collapse(options) - self._host = parsed_url.hostname - self._port = parsed_url.port or self.DEFAULT_PORT - self._session_id = None - self._session_name = encodeutils.safe_decode(member_id) - self._ttl = int(options.get('ttl', self.DEFAULT_TTL)) - namespace = options.get('namespace', self.TOOZ_NAMESPACE) - self._namespace = encodeutils.safe_decode(namespace) - self._client = None - - def _start(self): - """Create a client, register a node and create a session.""" - # Create a consul client - if self._client is None: - self._client = consul.Consul(host=self._host, port=self._port) - - local_agent = self._client.agent.self() - self._node = local_agent['Member']['Name'] - self._address = local_agent['Member']['Addr'] - - # Register a Node - self._client.catalog.register(node=self._node, - address=self._address) - - # Create a session - self._session_id = self._client.session.create( - name=self._session_name, node=self._node, ttl=self._ttl) - - def _stop(self): - if self._client is not None: - if self._session_id is not None: - self._client.session.destroy(self._session_id) - self._session_id = None - self._client = None - - def get_lock(self, name): - real_name = self._paths_join(self._namespace, u"locks", name) - return ConsulLock(real_name, self._node, self._address, - session_id=self._session_id, - client=self._client) - - @staticmethod - def _paths_join(*args): - pieces = [] - for arg in args: - pieces.append(encodeutils.safe_decode(arg)) - return u"/".join(pieces) - - def watch_join_group(self, group_id, callback): - raise tooz.NotImplemented - - def unwatch_join_group(self, group_id, callback): - raise tooz.NotImplemented - - def watch_leave_group(self, group_id, callback): - raise tooz.NotImplemented - - def unwatch_leave_group(self, group_id, callback): - raise tooz.NotImplemented diff --git a/tooz/drivers/etcd.py b/tooz/drivers/etcd.py deleted file mode 100644 index 87c77fe..0000000 --- a/tooz/drivers/etcd.py +++ /dev/null @@ -1,258 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import threading - -import fasteners -from oslo_utils import encodeutils -from oslo_utils import timeutils -import requests -import six - -import tooz -from tooz import coordination -from tooz import locking -from tooz import utils - - -LOG = logging.getLogger(__name__) - - -def _translate_failures(func): - """Translates common requests exceptions into tooz exceptions.""" - - @six.wraps(func) - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except ValueError as e: - # Typically json decoding failed for some reason. - utils.raise_with_cause(tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - except requests.exceptions.RequestException as e: - utils.raise_with_cause(coordination.ToozConnectionError, - encodeutils.exception_to_unicode(e), - cause=e) - - return wrapper - - -class _Client(object): - def __init__(self, host, port, protocol): - self.host = host - self.port = port - self.protocol = protocol - self.session = requests.Session() - - @property - def base_url(self): - return self.protocol + '://' + self.host + ':' + str(self.port) - - def get_url(self, path): - return self.base_url + '/v2/' + path.lstrip("/") - - def get(self, url, **kwargs): - if kwargs.pop('make_url', True): - url = self.get_url(url) - return self.session.get(url, **kwargs).json() - - def put(self, url, **kwargs): - if kwargs.pop('make_url', True): - url = self.get_url(url) - return self.session.put(url, **kwargs).json() - - def delete(self, url, **kwargs): - if kwargs.pop('make_url', True): - url = self.get_url(url) - return self.session.delete(url, **kwargs).json() - - def self_stats(self): - return self.session.get(self.get_url("/stats/self")) - - -class EtcdLock(locking.Lock): - - _TOOZ_LOCK_PREFIX = "tooz_locks" - - def __init__(self, lock_url, name, coord, client, ttl=60): - super(EtcdLock, self).__init__(name) - self.client = client - self.coord = coord - self.ttl = ttl - self._lock_url = lock_url - self._node = None - - # NOTE(jschwarz): this lock is mainly used to prevent concurrent runs - # of hearthbeat() with another function. For more details, see - # https://bugs.launchpad.net/python-tooz/+bug/1603005. - self._lock = threading.Lock() - - @_translate_failures - @fasteners.locked - def break_(self): - reply = self.client.delete(self._lock_url, make_url=False) - return reply.get('errorCode') is None - - def acquire(self, blocking=True, shared=False): - if shared: - raise tooz.NotImplemented - - blocking, timeout = utils.convert_blocking(blocking) - if timeout is not None: - watch = timeutils.StopWatch(duration=timeout) - watch.start() - else: - watch = None - - while True: - if self.acquired: - # We already acquired the lock. Just go ahead and wait for ever - # if blocking != False using the last index. - lastindex = self._node['modifiedIndex'] - else: - try: - reply = self.client.put( - self._lock_url, - make_url=False, - timeout=watch.leftover() if watch else None, - data={"ttl": self.ttl, - "prevExist": "false"}) - except requests.exceptions.RequestException: - if not watch or watch.leftover() == 0: - return False - - # We got the lock! - if reply.get("errorCode") is None: - with self._lock: - self._node = reply['node'] - self.coord._acquired_locks.add(self) - return True - - # No lock, somebody got it, wait for it to be released - lastindex = reply['index'] + 1 - - # We didn't get the lock and we don't want to wait - if not blocking: - return False - - # Ok, so let's wait a bit (or forever!) - try: - reply = self.client.get( - self._lock_url + - "?wait=true&waitIndex=%d" % lastindex, - make_url=False, - timeout=watch.leftover() if watch else None) - except requests.exceptions.RequestException: - if not watch or watch.expired(): - return False - - @_translate_failures - @fasteners.locked - def release(self): - if self.acquired: - lock_url = self._lock_url - lock_url += "?prevIndex=%s" % self._node['modifiedIndex'] - reply = self.client.delete(lock_url, make_url=False) - errorcode = reply.get("errorCode") - if errorcode is None: - self.coord._acquired_locks.discard(self) - self._node = None - return True - else: - LOG.warning("Unable to release '%s' due to %d, %s", - self.name, errorcode, reply.get('message')) - return False - - @property - def acquired(self): - return self in self.coord._acquired_locks - - @_translate_failures - @fasteners.locked - def heartbeat(self): - """Keep the lock alive.""" - if self.acquired: - poked = self.client.put(self._lock_url, - data={"ttl": self.ttl, - "prevExist": "true"}, make_url=False) - self._node = poked['node'] - errorcode = poked.get("errorCode") - if not errorcode: - return True - LOG.warning("Unable to heartbeat by updating key '%s' with " - "extended expiry of %s seconds: %d, %s", self.name, - self.ttl, errorcode, poked.get("message")) - return False - - -class EtcdDriver(coordination.CoordinationDriver): - """An etcd based driver. - - This driver uses etcd provide the coordination driver semantics and - required API(s). - """ - - #: Default socket/lock/member/leader timeout used when none is provided. - DEFAULT_TIMEOUT = 30 - - #: Default hostname used when none is provided. - DEFAULT_HOST = "localhost" - - #: Default port used if none provided (4001 or 2379 are the common ones). - DEFAULT_PORT = 2379 - - #: Class that will be used to encode lock names into a valid etcd url. - lock_encoder_cls = utils.Base64LockEncoder - - def __init__(self, member_id, parsed_url, options): - super(EtcdDriver, self).__init__(member_id, parsed_url, options) - host = parsed_url.hostname or self.DEFAULT_HOST - port = parsed_url.port or self.DEFAULT_PORT - options = utils.collapse(options) - self.client = _Client(host=host, port=port, - protocol=options.get('protocol', 'http')) - default_timeout = options.get('timeout', self.DEFAULT_TIMEOUT) - self.lock_encoder = self.lock_encoder_cls(self.client.get_url("keys")) - self.lock_timeout = int(options.get('lock_timeout', default_timeout)) - self._acquired_locks = set() - - def _start(self): - try: - self.client.self_stats() - except requests.exceptions.ConnectionError as e: - raise coordination.ToozConnectionError( - encodeutils.exception_to_unicode(e)) - - def get_lock(self, name): - return EtcdLock(self.lock_encoder.check_and_encode(name), name, - self, self.client, self.lock_timeout) - - def heartbeat(self): - for lock in self._acquired_locks.copy(): - lock.heartbeat() - return self.lock_timeout - - def watch_join_group(self, group_id, callback): - raise tooz.NotImplemented - - def unwatch_join_group(self, group_id, callback): - raise tooz.NotImplemented - - def watch_leave_group(self, group_id, callback): - raise tooz.NotImplemented - - def unwatch_leave_group(self, group_id, callback): - raise tooz.NotImplemented diff --git a/tooz/drivers/etcd3.py b/tooz/drivers/etcd3.py deleted file mode 100644 index 7c19673..0000000 --- a/tooz/drivers/etcd3.py +++ /dev/null @@ -1,148 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from __future__ import absolute_import -import threading - -import etcd3 -from etcd3 import exceptions as etcd3_exc -from oslo_utils import encodeutils -import six - -import tooz -from tooz import coordination -from tooz import locking -from tooz import utils - - -def _translate_failures(func): - """Translates common requests exceptions into tooz exceptions.""" - - @six.wraps(func) - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except etcd3_exc.ConnectionFailedError as e: - utils.raise_with_cause(coordination.ToozConnectionError, - encodeutils.exception_to_unicode(e), - cause=e) - except etcd3_exc.ConnectionTimeoutError as e: - utils.raise_with_cause(coordination.OperationTimedOut, - encodeutils.exception_to_unicode(e), - cause=e) - except etcd3_exc.Etcd3Exception as e: - utils.raise_with_cause(coordination.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - - return wrapper - - -class Etcd3Lock(locking.Lock): - """An etcd3-specific lock. - - Thin wrapper over etcd3's lock object basically to provide the heartbeat() - semantics for the coordination driver. - """ - - LOCK_PREFIX = b"/tooz/locks" - - def __init__(self, coord, name, timeout): - super(Etcd3Lock, self).__init__(name) - self._coord = coord - self._lock = coord.client.lock(name.decode(), timeout) - self._exclusive_access = threading.Lock() - - @_translate_failures - def acquire(self, blocking=True, shared=False): - if shared: - raise tooz.NotImplemented - - blocking, timeout = utils.convert_blocking(blocking) - if blocking is False: - timeout = 0 - - if self._lock.acquire(timeout): - self._coord._acquired_locks.add(self) - return True - - return False - - @property - def acquired(self): - return self in self._coord._acquired_locks - - @_translate_failures - def release(self): - with self._exclusive_access: - if self.acquired and self._lock.release(): - self._coord._acquired_locks.discard(self) - return True - return False - - @_translate_failures - def heartbeat(self): - with self._exclusive_access: - if self.acquired: - self._lock.refresh() - return True - return False - - -class Etcd3Driver(coordination.CoordinationDriver): - """An etcd based driver. - - This driver uses etcd provide the coordination driver semantics and - required API(s). - """ - - #: Default socket/lock/member/leader timeout used when none is provided. - DEFAULT_TIMEOUT = 30 - - #: Default hostname used when none is provided. - DEFAULT_HOST = "localhost" - - #: Default port used if none provided (4001 or 2379 are the common ones). - DEFAULT_PORT = 2379 - - def __init__(self, member_id, parsed_url, options): - super(Etcd3Driver, self).__init__(member_id, parsed_url, options) - host = parsed_url.hostname or self.DEFAULT_HOST - port = parsed_url.port or self.DEFAULT_PORT - options = utils.collapse(options) - timeout = int(options.get('timeout', self.DEFAULT_TIMEOUT)) - self.client = etcd3.client(host=host, port=port, timeout=timeout) - self.lock_timeout = int(options.get('lock_timeout', timeout)) - self._acquired_locks = set() - - def get_lock(self, name): - return Etcd3Lock(self, name, self.lock_timeout) - - def heartbeat(self): - # NOTE(jaypipes): Copying because set can mutate during iteration - for lock in self._acquired_locks.copy(): - lock.heartbeat() - return self.lock_timeout - - def watch_join_group(self, group_id, callback): - raise tooz.NotImplemented - - def unwatch_join_group(self, group_id, callback): - raise tooz.NotImplemented - - def watch_leave_group(self, group_id, callback): - raise tooz.NotImplemented - - def unwatch_leave_group(self, group_id, callback): - raise tooz.NotImplemented diff --git a/tooz/drivers/etcd3gw.py b/tooz/drivers/etcd3gw.py deleted file mode 100644 index c3367ca..0000000 --- a/tooz/drivers/etcd3gw.py +++ /dev/null @@ -1,204 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from __future__ import absolute_import -import base64 -import threading -import uuid - -import etcd3gw -from etcd3gw import exceptions as etcd3_exc -from oslo_utils import encodeutils -import six - -import tooz -from tooz import _retry -from tooz import coordination -from tooz import locking -from tooz import utils - - -def _translate_failures(func): - """Translates common requests exceptions into tooz exceptions.""" - - @six.wraps(func) - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except etcd3_exc.ConnectionFailedError as e: - utils.raise_with_cause(coordination.ToozConnectionError, - encodeutils.exception_to_unicode(e), - cause=e) - except etcd3_exc.ConnectionTimeoutError as e: - utils.raise_with_cause(coordination.OperationTimedOut, - encodeutils.exception_to_unicode(e), - cause=e) - except etcd3_exc.Etcd3Exception as e: - utils.raise_with_cause(coordination.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - - return wrapper - - -class Etcd3Lock(locking.Lock): - """An etcd3-specific lock. - - Thin wrapper over etcd3's lock object basically to provide the heartbeat() - semantics for the coordination driver. - """ - - LOCK_PREFIX = b"/tooz/locks" - - def __init__(self, coord, name, timeout): - super(Etcd3Lock, self).__init__(name) - self._timeout = timeout - self._coord = coord - self._key = self.LOCK_PREFIX + name - self._key_b64 = base64.b64encode(self._key).decode("ascii") - self._uuid = base64.b64encode(uuid.uuid4().bytes).decode("ascii") - self._lease = self._coord.client.lease(self._timeout) - self._exclusive_access = threading.Lock() - - @_translate_failures - def acquire(self, blocking=True, shared=False): - if shared: - raise tooz.NotImplemented - - @_retry.retry(stop_max_delay=blocking) - def _acquire(): - # TODO(jd): save the created revision so we can check it later to - # make sure we still have the lock - txn = { - 'compare': [{ - 'key': self._key_b64, - 'result': 'EQUAL', - 'target': 'CREATE', - 'create_revision': 0 - }], - 'success': [{ - 'request_put': { - 'key': self._key_b64, - 'value': self._uuid, - 'lease': self._lease.id - } - }], - 'failure': [{ - 'request_range': { - 'key': self._key_b64 - } - }] - } - result = self._coord.client.transaction(txn) - success = result.get('succeeded', False) - - if success is not True: - if blocking is False: - return False - raise _retry.TryAgain - self._coord._acquired_locks.add(self) - return True - - return _acquire() - - @_translate_failures - def release(self): - txn = { - 'compare': [{ - 'key': self._key_b64, - 'result': 'EQUAL', - 'target': 'VALUE', - 'value': self._uuid - }], - 'success': [{ - 'request_delete_range': { - 'key': self._key_b64 - } - }] - } - - with self._exclusive_access: - result = self._coord.client.transaction(txn) - success = result.get('succeeded', False) - if success: - self._coord._acquired_locks.remove(self) - return True - return False - - @_translate_failures - def break_(self): - if self._coord.client.delete(self._key): - self._coord._acquired_locks.discard(self) - return True - return False - - @property - def acquired(self): - return self in self._coord._acquired_locks - - @_translate_failures - def heartbeat(self): - with self._exclusive_access: - if self.acquired: - self._lease.refresh() - return True - return False - - -class Etcd3Driver(coordination.CoordinationDriver): - """An etcd based driver. - - This driver uses etcd provide the coordination driver semantics and - required API(s). - """ - - #: Default socket/lock/member/leader timeout used when none is provided. - DEFAULT_TIMEOUT = 30 - - #: Default hostname used when none is provided. - DEFAULT_HOST = "localhost" - - #: Default port used if none provided (4001 or 2379 are the common ones). - DEFAULT_PORT = 2379 - - def __init__(self, member_id, parsed_url, options): - super(Etcd3Driver, self).__init__(member_id, parsed_url, options) - host = parsed_url.hostname or self.DEFAULT_HOST - port = parsed_url.port or self.DEFAULT_PORT - options = utils.collapse(options) - timeout = int(options.get('timeout', self.DEFAULT_TIMEOUT)) - self.client = etcd3gw.client(host=host, port=port, timeout=timeout) - self.lock_timeout = int(options.get('lock_timeout', timeout)) - self._acquired_locks = set() - - def get_lock(self, name): - return Etcd3Lock(self, name, self.lock_timeout) - - def heartbeat(self): - # NOTE(jaypipes): Copying because set can mutate during iteration - for lock in self._acquired_locks.copy(): - lock.heartbeat() - return self.lock_timeout - - def watch_join_group(self, group_id, callback): - raise tooz.NotImplemented - - def unwatch_join_group(self, group_id, callback): - raise tooz.NotImplemented - - def watch_leave_group(self, group_id, callback): - raise tooz.NotImplemented - - def unwatch_leave_group(self, group_id, callback): - raise tooz.NotImplemented diff --git a/tooz/drivers/file.py b/tooz/drivers/file.py deleted file mode 100644 index aaef57b..0000000 --- a/tooz/drivers/file.py +++ /dev/null @@ -1,532 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2015 eNovance -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import contextlib -import datetime -import errno -import functools -import hashlib -import logging -import os -import re -import shutil -import sys -import tempfile -import threading -import weakref - -import fasteners -from oslo_utils import encodeutils -from oslo_utils import fileutils -from oslo_utils import timeutils -import six -import voluptuous - -import tooz -from tooz import coordination -from tooz import locking -from tooz import utils - -LOG = logging.getLogger(__name__) - - -class _Barrier(object): - def __init__(self): - self.cond = threading.Condition() - self.owner = None - self.shared = False - self.ref = 0 - - -@contextlib.contextmanager -def _translate_failures(): - try: - yield - except (EnvironmentError, voluptuous.Invalid) as e: - utils.raise_with_cause(tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - - -def _convert_from_old_format(data): - # NOTE(sileht): previous version of the driver was storing str as-is - # making impossible to read from python3 something written with python2 - # version of the lib. - # Now everything is stored with explicit type bytes or unicode. This - # convert the old format to the new one to maintain compat of already - # deployed file. - # example of potential old python2 payload: - # {b"member_id": b"member"} - # {b"member_id": u"member"} - # example of potential old python3 payload: - # {u"member_id": b"member"} - # {u"member_id": u"member"} - if six.PY3 and b"member_id" in data or b"group_id" in data: - data = dict((k.decode("utf8"), v) for k, v in data.items()) - # About member_id and group_id valuse if the file have been written - # with python2 and in the old format, we can't known with python3 - # if we need to decode the value or not. Python3 see bytes blob - # We keep it as-is and pray, this have a good change to break if - # the application was using str in python2 and unicode in python3 - # The member file is often overridden so it's should be fine - # But the group file can be very old, so we - # now have to update it each time create_group is called - return data - - -def _lock_me(lock): - - def wrapper(func): - - @six.wraps(func) - def decorator(*args, **kwargs): - with lock: - return func(*args, **kwargs) - - return decorator - - return wrapper - - -class FileLock(locking.Lock): - """A file based lock.""" - - def __init__(self, path, barrier, member_id): - super(FileLock, self).__init__(path) - self.acquired = False - self._lock = fasteners.InterProcessLock(path) - self._barrier = barrier - self._member_id = member_id - self.ref = 0 - - def is_still_owner(self): - return self.acquired - - def acquire(self, blocking=True, shared=False): - blocking, timeout = utils.convert_blocking(blocking) - watch = timeutils.StopWatch(duration=timeout) - watch.start() - - # Make the shared barrier ours first. - with self._barrier.cond: - while self._barrier.owner is not None: - if (shared and self._barrier.shared): - break - if not blocking or watch.expired(): - return False - self._barrier.cond.wait(watch.leftover(return_none=True)) - self._barrier.owner = (threading.current_thread().ident, - os.getpid(), self._member_id) - self._barrier.shared = shared - self._barrier.ref += 1 - self.ref += 1 - - # Ok at this point we are now working in a thread safe manner, - # and now we can try to get the actual lock... - gotten = False - try: - gotten = self._lock.acquire( - blocking=blocking, - # Since the barrier waiting may have - # taken a long time, we have to use - # the leftover (and not the original). - timeout=watch.leftover(return_none=True)) - finally: - # NOTE(harlowja): do this in a finally block to **ensure** that - # we release the barrier if something bad happens... - if not gotten: - # Release the barrier to let someone else have a go at it... - with self._barrier.cond: - self._barrier.owner = None - self._barrier.ref = 0 - self._barrier.shared = False - self._barrier.cond.notify_all() - - self.acquired = gotten - return gotten - - def release(self): - if not self.acquired: - return False - with self._barrier.cond: - self._barrier.ref -= 1 - self.ref -= 1 - if not self.ref: - self.acquired = False - if not self._barrier.ref: - self._barrier.owner = None - self._lock.release() - self._barrier.cond.notify_all() - return True - - def __del__(self): - if self.acquired: - LOG.warning("Unreleased lock %s garbage collected", self.name) - - -class FileDriver(coordination.CoordinationDriverCachedRunWatchers, - coordination.CoordinationDriverWithExecutor): - """A file based driver. - - This driver uses files and directories (and associated file locks) to - provide the coordination driver semantics and required API(s). It **is** - missing some functionality but in the future these not implemented API(s) - will be filled in. - - General recommendations/usage considerations: - - - It does **not** automatically delete members from - groups of processes that have died, manual cleanup will be needed - for those types of failures. - - - It is **not** distributed (or recommended to be used in those - situations, so the developer using this should really take that into - account when applying this driver in there app). - """ - - CHARACTERISTICS = ( - coordination.Characteristics.NON_TIMEOUT_BASED, - coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, - coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES, - ) - """ - Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable - enum member(s) that can be used to interogate how this driver works. - """ - - HASH_ROUTINE = 'sha1' - """This routine is used to hash a member (or group) id into a filesystem - safe name that can be used for member lookup and group joining.""" - - _barriers = weakref.WeakValueDictionary() - """ - Barriers shared among all file driver locks, this is required - since interprocess locking is not thread aware, so we must add the - thread awareness on-top of it instead. - """ - - def __init__(self, member_id, parsed_url, options): - """Initialize the file driver.""" - super(FileDriver, self).__init__(member_id, parsed_url, options) - self._dir = self._normalize_path(parsed_url.path) - self._group_dir = os.path.join(self._dir, 'groups') - self._tmpdir = os.path.join(self._dir, 'tmp') - self._driver_lock_path = os.path.join(self._dir, '.driver_lock') - self._driver_lock = self._get_raw_lock(self._driver_lock_path, - self._member_id) - self._reserved_dirs = [self._dir, self._group_dir, self._tmpdir] - self._reserved_paths = list(self._reserved_dirs) - self._reserved_paths.append(self._driver_lock_path) - self._safe_member_id = self._make_filesystem_safe(member_id) - self._timeout = int(self._options.get('timeout', 10)) - - @staticmethod - def _normalize_path(path): - if sys.platform == 'win32': - # Replace slashes with backslashes and make sure we don't - # have any at the beginning of paths that include drive letters. - # - # Expected url format: - # file:////share_address/share_name - # file:///C:/path - return re.sub(r'\\(?=\w:\\)', '', - os.path.normpath(path)) - return path - - @classmethod - def _get_raw_lock(cls, path, member_id): - lock_barrier = cls._barriers.setdefault(path, _Barrier()) - return FileLock(path, lock_barrier, member_id) - - def get_lock(self, name): - path = utils.safe_abs_path(self._dir, name.decode()) - if path in self._reserved_paths: - raise ValueError("Unable to create a lock using" - " reserved path '%s' for lock" - " with name '%s'" % (path, name)) - return self._get_raw_lock(path, self._member_id) - - @classmethod - def _make_filesystem_safe(cls, item): - item = utils.to_binary(item, encoding="utf8") - return hashlib.new(cls.HASH_ROUTINE, item).hexdigest() - - def _start(self): - super(FileDriver, self)._start() - for a_dir in self._reserved_dirs: - try: - fileutils.ensure_tree(a_dir) - except OSError as e: - raise coordination.ToozConnectionError(e) - - def _update_group_metadata(self, path, group_id): - details = { - u'group_id': utils.to_binary(group_id, encoding="utf8") - } - details[u'encoded'] = details[u"group_id"] != group_id - details_blob = utils.dumps(details) - fd, name = tempfile.mkstemp("tooz", dir=self._tmpdir) - with os.fdopen(fd, "wb") as fh: - fh.write(details_blob) - os.rename(name, path) - - def create_group(self, group_id): - safe_group_id = self._make_filesystem_safe(group_id) - group_dir = os.path.join(self._group_dir, safe_group_id) - group_meta_path = os.path.join(group_dir, '.metadata') - - def _do_create_group(): - if os.path.exists(os.path.join(group_dir, ".metadata")): - # NOTE(sileht): We update the group metadata even - # they are already good, so ensure dict key are convert - # to unicode in case of the file have been written with - # tooz < 1.36 - self._update_group_metadata(group_meta_path, group_id) - raise coordination.GroupAlreadyExist(group_id) - else: - fileutils.ensure_tree(group_dir) - self._update_group_metadata(group_meta_path, group_id) - fut = self._executor.submit(_do_create_group) - return FileFutureResult(fut) - - def join_group(self, group_id, capabilities=b""): - safe_group_id = self._make_filesystem_safe(group_id) - group_dir = os.path.join(self._group_dir, safe_group_id) - me_path = os.path.join(group_dir, "%s.raw" % self._safe_member_id) - - @_lock_me(self._driver_lock) - def _do_join_group(): - if not os.path.exists(os.path.join(group_dir, ".metadata")): - raise coordination.GroupNotCreated(group_id) - if os.path.isfile(me_path): - raise coordination.MemberAlreadyExist(group_id, - self._member_id) - details = { - u'capabilities': capabilities, - u'joined_on': datetime.datetime.now(), - u'member_id': utils.to_binary(self._member_id, - encoding="utf-8") - } - details[u'encoded'] = details[u"member_id"] != self._member_id - details_blob = utils.dumps(details) - with open(me_path, "wb") as fh: - fh.write(details_blob) - self._joined_groups.add(group_id) - - fut = self._executor.submit(_do_join_group) - return FileFutureResult(fut) - - def leave_group(self, group_id): - safe_group_id = self._make_filesystem_safe(group_id) - group_dir = os.path.join(self._group_dir, safe_group_id) - me_path = os.path.join(group_dir, "%s.raw" % self._safe_member_id) - - @_lock_me(self._driver_lock) - def _do_leave_group(): - if not os.path.exists(os.path.join(group_dir, ".metadata")): - raise coordination.GroupNotCreated(group_id) - try: - os.unlink(me_path) - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - else: - raise coordination.MemberNotJoined(group_id, - self._member_id) - else: - self._joined_groups.discard(group_id) - - fut = self._executor.submit(_do_leave_group) - return FileFutureResult(fut) - - _SCHEMAS = { - 'group': voluptuous.Schema({ - voluptuous.Required('group_id'): voluptuous.Any(six.text_type, - six.binary_type), - # NOTE(sileht): tooz <1.36 was creating file without this - voluptuous.Optional('encoded'): bool, - }), - 'member': voluptuous.Schema({ - voluptuous.Required('member_id'): voluptuous.Any(six.text_type, - six.binary_type), - voluptuous.Required('joined_on'): datetime.datetime, - # NOTE(sileht): tooz <1.36 was creating file without this - voluptuous.Optional('encoded'): bool, - }, extra=voluptuous.ALLOW_EXTRA), - } - - def _load_and_validate(self, blob, schema_key): - data = utils.loads(blob) - data = _convert_from_old_format(data) - schema = self._SCHEMAS[schema_key] - return schema(data) - - def _read_member_id(self, path): - with open(path, 'rb') as fh: - details = self._load_and_validate(fh.read(), 'member') - if details.get("encoded"): - return details[u'member_id'].decode("utf-8") - return details[u'member_id'] - - def get_members(self, group_id): - safe_group_id = self._make_filesystem_safe(group_id) - group_dir = os.path.join(self._group_dir, safe_group_id) - - @_lock_me(self._driver_lock) - def _do_get_members(): - if not os.path.isdir(group_dir): - raise coordination.GroupNotCreated(group_id) - members = set() - try: - entries = os.listdir(group_dir) - except EnvironmentError as e: - # Did someone manage to remove it before we got here... - if e.errno != errno.ENOENT: - raise - else: - for entry in entries: - if not entry.endswith('.raw'): - continue - entry_path = os.path.join(group_dir, entry) - try: - m_time = datetime.datetime.fromtimestamp( - os.stat(entry_path).st_mtime) - current_time = datetime.datetime.now() - delta_time = timeutils.delta_seconds(m_time, - current_time) - if delta_time >= 0 and delta_time <= self._timeout: - member_id = self._read_member_id(entry_path) - else: - continue - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - else: - members.add(member_id) - return members - - fut = self._executor.submit(_do_get_members) - return FileFutureResult(fut) - - def get_member_capabilities(self, group_id, member_id): - safe_group_id = self._make_filesystem_safe(group_id) - group_dir = os.path.join(self._group_dir, safe_group_id) - safe_member_id = self._make_filesystem_safe(member_id) - member_path = os.path.join(group_dir, "%s.raw" % safe_member_id) - - @_lock_me(self._driver_lock) - def _do_get_member_capabilities(): - try: - with open(member_path, "rb") as fh: - contents = fh.read() - except EnvironmentError as e: - if e.errno == errno.ENOENT: - if not os.path.isdir(group_dir): - raise coordination.GroupNotCreated(group_id) - else: - raise coordination.MemberNotJoined(group_id, - member_id) - else: - raise - else: - details = self._load_and_validate(contents, 'member') - return details.get(u"capabilities") - - fut = self._executor.submit(_do_get_member_capabilities) - return FileFutureResult(fut) - - def delete_group(self, group_id): - safe_group_id = self._make_filesystem_safe(group_id) - group_dir = os.path.join(self._group_dir, safe_group_id) - - @_lock_me(self._driver_lock) - def _do_delete_group(): - try: - entries = os.listdir(group_dir) - except EnvironmentError as e: - if e.errno == errno.ENOENT: - raise coordination.GroupNotCreated(group_id) - else: - raise - else: - if len(entries) > 1: - raise coordination.GroupNotEmpty(group_id) - elif len(entries) == 1 and entries != ['.metadata']: - raise tooz.ToozError( - "Unexpected path '%s' found in" - " group directory '%s' (expected to only find" - " a '.metadata' path)" % (entries[0], group_dir)) - else: - try: - shutil.rmtree(group_dir) - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - - fut = self._executor.submit(_do_delete_group) - return FileFutureResult(fut) - - def _read_group_id(self, path): - with open(path, 'rb') as fh: - details = self._load_and_validate(fh.read(), 'group') - if details.get("encoded"): - return details[u'group_id'].decode("utf-8") - return details[u'group_id'] - - def get_groups(self): - - def _do_get_groups(): - groups = [] - for entry in os.listdir(self._group_dir): - path = os.path.join(self._group_dir, entry, '.metadata') - try: - groups.append(self._read_group_id(path)) - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - return groups - - fut = self._executor.submit(_do_get_groups) - return FileFutureResult(fut) - - def heartbeat(self): - for group_id in self._joined_groups: - safe_group_id = self._make_filesystem_safe(group_id) - group_dir = os.path.join(self._group_dir, safe_group_id) - member_path = os.path.join(group_dir, "%s.raw" % - self._safe_member_id) - - @_lock_me(self._driver_lock) - def _do_heartbeat(): - try: - os.utime(member_path, None) - except EnvironmentError as err: - if err.errno != errno.ENOENT: - raise - _do_heartbeat() - return self._timeout - - @staticmethod - def watch_elected_as_leader(group_id, callback): - raise tooz.NotImplemented - - @staticmethod - def unwatch_elected_as_leader(group_id, callback): - raise tooz.NotImplemented - - -FileFutureResult = functools.partial(coordination.CoordinatorResult, - failure_translator=_translate_failures) diff --git a/tooz/drivers/ipc.py b/tooz/drivers/ipc.py deleted file mode 100644 index f5d67a3..0000000 --- a/tooz/drivers/ipc.py +++ /dev/null @@ -1,243 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2014 eNovance -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import hashlib -import struct -import time - -import msgpack -import six -import sysv_ipc - -import tooz -from tooz import coordination -from tooz import locking -from tooz import utils - -if sysv_ipc.KEY_MIN <= 0: - _KEY_RANGE = abs(sysv_ipc.KEY_MIN) + sysv_ipc.KEY_MAX -else: - _KEY_RANGE = sysv_ipc.KEY_MAX - sysv_ipc.KEY_MIN - - -def ftok(name, project): - # Similar to ftok & http://semanchuk.com/philip/sysv_ipc/#ftok_weakness - # but hopefully without as many weaknesses... - h = hashlib.md5() - if not isinstance(project, six.binary_type): - project = project.encode('ascii') - h.update(project) - if not isinstance(name, six.binary_type): - name = name.encode('ascii') - h.update(name) - return (int(h.hexdigest(), 16) % _KEY_RANGE) + sysv_ipc.KEY_MIN - - -class IPCLock(locking.Lock): - """A sysv IPC based lock. - - Please ensure you have read over (and understand) the limitations of sysv - IPC locks, and especially have tried and used $ ipcs -l (note the maximum - number of semaphores system wide field that command outputs). To ensure - that you do not reach that limit it is recommended to use destroy() at - the correct program exit/entry points. - """ - _LOCK_PROJECT = b'__TOOZ_LOCK_' - - def __init__(self, name): - super(IPCLock, self).__init__(name) - self.key = ftok(name, self._LOCK_PROJECT) - self._lock = None - - def break_(self): - try: - lock = sysv_ipc.Semaphore(key=self.key) - lock.remove() - except sysv_ipc.ExistentialError: - return False - else: - return True - - def acquire(self, blocking=True, shared=False): - if shared: - raise tooz.NotImplemented - - if (blocking is not True and - sysv_ipc.SEMAPHORE_TIMEOUT_SUPPORTED is False): - raise tooz.NotImplemented("This system does not support" - " semaphore timeouts") - blocking, timeout = utils.convert_blocking(blocking) - start_time = None - if not blocking: - timeout = 0 - elif blocking and timeout is not None: - start_time = time.time() - while True: - tmplock = None - try: - tmplock = sysv_ipc.Semaphore(self.key, - flags=sysv_ipc.IPC_CREX, - initial_value=1) - tmplock.undo = True - except sysv_ipc.ExistentialError: - # We failed to create it because it already exists, then try to - # grab the existing one. - try: - tmplock = sysv_ipc.Semaphore(self.key) - tmplock.undo = True - except sysv_ipc.ExistentialError: - # Semaphore has been deleted in the mean time, retry from - # the beginning! - continue - - if start_time is not None: - elapsed = max(0.0, time.time() - start_time) - if elapsed >= timeout: - # Ran out of time... - return False - adjusted_timeout = timeout - elapsed - else: - adjusted_timeout = timeout - try: - tmplock.acquire(timeout=adjusted_timeout) - except sysv_ipc.BusyError: - tmplock = None - return False - except sysv_ipc.ExistentialError: - # Likely the lock has been deleted in the meantime, retry - continue - else: - self._lock = tmplock - return True - - def release(self): - if self._lock is not None: - try: - self._lock.remove() - self._lock = None - except sysv_ipc.ExistentialError: - return False - return True - return False - - -class IPCDriver(coordination.CoordinationDriverWithExecutor): - """A `IPC`_ based driver. - - This driver uses `IPC`_ concepts to provide the coordination driver - semantics and required API(s). It **is** missing some functionality but - in the future these not implemented API(s) will be filled in. - - General recommendations/usage considerations: - - - It is **not** distributed (or recommended to be used in those - situations, so the developer using this should really take that into - account when applying this driver in there app). - - .. _IPC: http://en.wikipedia.org/wiki/Inter-process_communication - """ - - CHARACTERISTICS = ( - coordination.Characteristics.NON_TIMEOUT_BASED, - coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, - coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES, - ) - """ - Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable - enum member(s) that can be used to interogate how this driver works. - """ - - _SEGMENT_SIZE = 1024 - _GROUP_LIST_KEY = "GROUP_LIST" - _GROUP_PROJECT = "_TOOZ_INTERNAL" - _INTERNAL_LOCK_NAME = "TOOZ_INTERNAL_LOCK" - - def _start(self): - super(IPCDriver, self)._start() - self._group_list = sysv_ipc.SharedMemory( - ftok(self._GROUP_LIST_KEY, self._GROUP_PROJECT), - sysv_ipc.IPC_CREAT, - size=self._SEGMENT_SIZE) - self._lock = self.get_lock(self._INTERNAL_LOCK_NAME) - - def _stop(self): - super(IPCDriver, self)._stop() - try: - self._group_list.detach() - self._group_list.remove() - except sysv_ipc.ExistentialError: - pass - - def _read_group_list(self): - data = self._group_list.read(byte_count=2) - length = struct.unpack("H", data)[0] - if length == 0: - return set() - data = self._group_list.read(byte_count=length, offset=2) - return set(msgpack.loads(data)) - - def _write_group_list(self, group_list): - data = msgpack.dumps(list(group_list)) - if len(data) >= self._SEGMENT_SIZE - 2: - raise tooz.ToozError("Group list is too big") - self._group_list.write(struct.pack('H', len(data))) - self._group_list.write(data, offset=2) - - def create_group(self, group_id): - def _create_group(): - with self._lock: - group_list = self._read_group_list() - if group_id in group_list: - raise coordination.GroupAlreadyExist(group_id) - group_list.add(group_id) - self._write_group_list(group_list) - - return coordination.CoordinatorResult( - self._executor.submit(_create_group)) - - def delete_group(self, group_id): - def _delete_group(): - with self._lock: - group_list = self._read_group_list() - if group_id not in group_list: - raise coordination.GroupNotCreated(group_id) - group_list.remove(group_id) - self._write_group_list(group_list) - - return coordination.CoordinatorResult( - self._executor.submit(_delete_group)) - - def watch_join_group(self, group_id, callback): - # Check the group exist - self.get_members(group_id).get() - super(IPCDriver, self).watch_join_group(group_id, callback) - - def watch_leave_group(self, group_id, callback): - # Check the group exist - self.get_members(group_id).get() - super(IPCDriver, self).watch_leave_group(group_id, callback) - - def _get_groups_handler(self): - with self._lock: - return self._read_group_list() - - def get_groups(self): - return coordination.CoordinatorResult(self._executor.submit( - self._get_groups_handler)) - - @staticmethod - def get_lock(name): - return IPCLock(name) diff --git a/tooz/drivers/memcached.py b/tooz/drivers/memcached.py deleted file mode 100644 index 99fbcd4..0000000 --- a/tooz/drivers/memcached.py +++ /dev/null @@ -1,516 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2014 eNovance -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import contextlib -import errno -import functools -import logging -import socket - -from oslo_utils import encodeutils -from pymemcache import client as pymemcache_client -import six - -import tooz -from tooz import _retry -from tooz import coordination -from tooz import locking -from tooz import utils - - -LOG = logging.getLogger(__name__) - - -@contextlib.contextmanager -def _failure_translator(): - """Translates common pymemcache exceptions into tooz exceptions. - - https://github.com/pinterest/pymemcache/blob/d995/pymemcache/client.py#L202 - """ - try: - yield - except pymemcache_client.MemcacheUnexpectedCloseError as e: - utils.raise_with_cause(coordination.ToozConnectionError, - encodeutils.exception_to_unicode(e), - cause=e) - except (socket.timeout, socket.error, - socket.gaierror, socket.herror) as e: - # TODO(harlowja): get upstream pymemcache to produce a better - # exception for these, using socket (vs. a memcache specific - # error) seems sorta not right and/or the best approach... - msg = encodeutils.exception_to_unicode(e) - if e.errno is not None: - msg += " (with errno %s [%s])" % (errno.errorcode[e.errno], - e.errno) - utils.raise_with_cause(coordination.ToozConnectionError, - msg, cause=e) - except pymemcache_client.MemcacheError as e: - utils.raise_with_cause(tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - - -def _translate_failures(func): - - @six.wraps(func) - def wrapper(*args, **kwargs): - with _failure_translator(): - return func(*args, **kwargs) - - return wrapper - - -class MemcachedLock(locking.Lock): - _LOCK_PREFIX = b'__TOOZ_LOCK_' - - def __init__(self, coord, name, timeout): - super(MemcachedLock, self).__init__(self._LOCK_PREFIX + name) - self.coord = coord - self.timeout = timeout - - def is_still_owner(self): - if not self.acquired: - return False - else: - owner = self.get_owner() - if owner is None: - return False - return owner == self.coord._member_id - - def acquire(self, blocking=True, shared=False): - if shared: - raise tooz.NotImplemented - - @_retry.retry(stop_max_delay=blocking) - @_translate_failures - def _acquire(): - if self.coord.client.add( - self.name, - self.coord._member_id, - expire=self.timeout, - noreply=False): - self.coord._acquired_locks.append(self) - return True - if blocking is False: - return False - raise _retry.TryAgain - - return _acquire() - - @_translate_failures - def break_(self): - return bool(self.coord.client.delete(self.name, noreply=False)) - - @_translate_failures - def release(self): - if not self.acquired: - return False - # NOTE(harlowja): this has the potential to delete others locks - # especially if this key expired before the delete/release call is - # triggered. - # - # For example: - # - # 1. App #1 with coordinator 'A' acquires lock "b" - # 2. App #1 heartbeats every 10 seconds, expiry for lock let's - # say is 11 seconds. - # 3. App #2 with coordinator also named 'A' blocks trying to get - # lock "b" (let's say it retries attempts every 0.5 seconds) - # 4. App #1 is running behind a little bit, tries to heartbeat but - # key has expired (log message is written); at this point app #1 - # doesn't own the lock anymore but it doesn't know that. - # 5. App #2 now retries and adds the key, and now it believes it - # has the lock. - # 6. App #1 (still believing it has the lock) calls release, and - # deletes app #2 lock, app #2 now doesn't own the lock anymore - # but it doesn't know that and now app #(X + 1) can get it. - # 7. App #2 calls release (repeat #6 as many times as desired) - # - # Sadly I don't think memcache has the primitives to actually make - # this work, redis does because it has lua which can check a session - # id and then do the delete and bail out if the session id is not - # as expected but memcache doesn't seem to have any equivalent - # capability. - if self not in self.coord._acquired_locks: - return False - # Do a ghetto test to see what the value is... (see above note), - # and how this really can't be done safely with memcache due to - # it being done in the client side (non-atomic). - value = self.coord.client.get(self.name) - if value != self.coord._member_id: - return False - else: - was_deleted = self.coord.client.delete(self.name, noreply=False) - if was_deleted: - self.coord._acquired_locks.remove(self) - return was_deleted - - @_translate_failures - def heartbeat(self): - """Keep the lock alive.""" - if self.acquired: - poked = self.coord.client.touch(self.name, - expire=self.timeout, - noreply=False) - if poked: - return True - LOG.warning("Unable to heartbeat by updating key '%s' with " - "extended expiry of %s seconds", self.name, - self.timeout) - return False - - @_translate_failures - def get_owner(self): - return self.coord.client.get(self.name) - - @property - def acquired(self): - return self in self.coord._acquired_locks - - -class MemcachedDriver(coordination.CoordinationDriverCachedRunWatchers, - coordination.CoordinationDriverWithExecutor): - """A `memcached`_ based driver. - - This driver users `memcached`_ concepts to provide the coordination driver - semantics and required API(s). It **is** fully functional and implements - all of the coordination driver API(s). It stores data into memcache - using expiries and `msgpack`_ encoded values. - - General recommendations/usage considerations: - - - Memcache (without different backend technology) is a **cache** enough - said. - - .. _memcached: http://memcached.org/ - .. _msgpack: http://msgpack.org/ - """ - - CHARACTERISTICS = ( - coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, - coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES, - coordination.Characteristics.DISTRIBUTED_ACROSS_HOSTS, - coordination.Characteristics.CAUSAL, - ) - """ - Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable - enum member(s) that can be used to interogate how this driver works. - """ - - #: Key prefix attached to groups (used in name-spacing keys) - GROUP_PREFIX = b'_TOOZ_GROUP_' - - #: Key prefix attached to leaders of groups (used in name-spacing keys) - GROUP_LEADER_PREFIX = b'_TOOZ_GROUP_LEADER_' - - #: Key prefix attached to members of groups (used in name-spacing keys) - MEMBER_PREFIX = b'_TOOZ_MEMBER_' - - #: Key where all groups 'known' are stored. - GROUP_LIST_KEY = b'_TOOZ_GROUP_LIST' - - #: Default socket/lock/member/leader timeout used when none is provided. - DEFAULT_TIMEOUT = 30 - - #: String used to keep a key/member alive (until it next expires). - STILL_ALIVE = b"It's alive!" - - def __init__(self, member_id, parsed_url, options): - super(MemcachedDriver, self).__init__(member_id, parsed_url, options) - self.host = (parsed_url.hostname or "localhost", - parsed_url.port or 11211) - default_timeout = self._options.get('timeout', self.DEFAULT_TIMEOUT) - self.timeout = int(default_timeout) - self.membership_timeout = int(self._options.get( - 'membership_timeout', default_timeout)) - self.lock_timeout = int(self._options.get( - 'lock_timeout', default_timeout)) - self.leader_timeout = int(self._options.get( - 'leader_timeout', default_timeout)) - max_pool_size = self._options.get('max_pool_size', None) - if max_pool_size is not None: - self.max_pool_size = int(max_pool_size) - else: - self.max_pool_size = None - self._acquired_locks = [] - - @staticmethod - def _msgpack_serializer(key, value): - if isinstance(value, six.binary_type): - return value, 1 - return utils.dumps(value), 2 - - @staticmethod - def _msgpack_deserializer(key, value, flags): - if flags == 1: - return value - if flags == 2: - return utils.loads(value) - raise coordination.SerializationError("Unknown serialization" - " format '%s'" % flags) - - @_translate_failures - def _start(self): - super(MemcachedDriver, self)._start() - self.client = pymemcache_client.PooledClient( - self.host, - serializer=self._msgpack_serializer, - deserializer=self._msgpack_deserializer, - timeout=self.timeout, - connect_timeout=self.timeout, - max_pool_size=self.max_pool_size) - # Run heartbeat here because pymemcache use a lazy connection - # method and only connect once you do an operation. - self.heartbeat() - - @_translate_failures - def _stop(self): - super(MemcachedDriver, self)._stop() - for lock in list(self._acquired_locks): - lock.release() - self.client.delete(self._encode_member_id(self._member_id)) - self.client.close() - - def _encode_group_id(self, group_id): - return self.GROUP_PREFIX + group_id - - def _encode_member_id(self, member_id): - return self.MEMBER_PREFIX + member_id - - def _encode_group_leader(self, group_id): - return self.GROUP_LEADER_PREFIX + group_id - - @_retry.retry() - def _add_group_to_group_list(self, group_id): - """Add group to the group list. - - :param group_id: The group id - """ - group_list, cas = self.client.gets(self.GROUP_LIST_KEY) - if cas: - group_list = set(group_list) - group_list.add(group_id) - if not self.client.cas(self.GROUP_LIST_KEY, - list(group_list), cas): - # Someone updated the group list before us, try again! - raise _retry.TryAgain - else: - if not self.client.add(self.GROUP_LIST_KEY, - [group_id], noreply=False): - # Someone updated the group list before us, try again! - raise _retry.TryAgain - - @_retry.retry() - def _remove_from_group_list(self, group_id): - """Remove group from the group list. - - :param group_id: The group id - """ - group_list, cas = self.client.gets(self.GROUP_LIST_KEY) - group_list = set(group_list) - group_list.remove(group_id) - if not self.client.cas(self.GROUP_LIST_KEY, - list(group_list), cas): - # Someone updated the group list before us, try again! - raise _retry.TryAgain - - def create_group(self, group_id): - encoded_group = self._encode_group_id(group_id) - - @_translate_failures - def _create_group(): - if not self.client.add(encoded_group, {}, noreply=False): - raise coordination.GroupAlreadyExist(group_id) - self._add_group_to_group_list(group_id) - - return MemcachedFutureResult(self._executor.submit(_create_group)) - - def get_groups(self): - - @_translate_failures - def _get_groups(): - return self.client.get(self.GROUP_LIST_KEY) or [] - - return MemcachedFutureResult(self._executor.submit(_get_groups)) - - def join_group(self, group_id, capabilities=b""): - encoded_group = self._encode_group_id(group_id) - - @_retry.retry() - @_translate_failures - def _join_group(): - group_members, cas = self.client.gets(encoded_group) - if group_members is None: - raise coordination.GroupNotCreated(group_id) - if self._member_id in group_members: - raise coordination.MemberAlreadyExist(group_id, - self._member_id) - group_members[self._member_id] = { - b"capabilities": capabilities, - } - if not self.client.cas(encoded_group, group_members, cas): - # It changed, let's try again - raise _retry.TryAgain - self._joined_groups.add(group_id) - - return MemcachedFutureResult(self._executor.submit(_join_group)) - - def leave_group(self, group_id): - encoded_group = self._encode_group_id(group_id) - - @_retry.retry() - @_translate_failures - def _leave_group(): - group_members, cas = self.client.gets(encoded_group) - if group_members is None: - raise coordination.GroupNotCreated(group_id) - if self._member_id not in group_members: - raise coordination.MemberNotJoined(group_id, self._member_id) - del group_members[self._member_id] - if not self.client.cas(encoded_group, group_members, cas): - # It changed, let's try again - raise _retry.TryAgain - self._joined_groups.discard(group_id) - - return MemcachedFutureResult(self._executor.submit(_leave_group)) - - def _destroy_group(self, group_id): - self.client.delete(self._encode_group_id(group_id)) - - def delete_group(self, group_id): - encoded_group = self._encode_group_id(group_id) - - @_retry.retry() - @_translate_failures - def _delete_group(): - group_members, cas = self.client.gets(encoded_group) - if group_members is None: - raise coordination.GroupNotCreated(group_id) - if group_members != {}: - raise coordination.GroupNotEmpty(group_id) - # Delete is not atomic, so we first set the group to - # using CAS, and then we delete it, to avoid race conditions. - if not self.client.cas(encoded_group, None, cas): - raise _retry.TryAgain - self.client.delete(encoded_group) - self._remove_from_group_list(group_id) - - return MemcachedFutureResult(self._executor.submit(_delete_group)) - - @_retry.retry() - @_translate_failures - def _get_members(self, group_id): - encoded_group = self._encode_group_id(group_id) - group_members, cas = self.client.gets(encoded_group) - if group_members is None: - raise coordination.GroupNotCreated(group_id) - actual_group_members = {} - for m, v in six.iteritems(group_members): - # Never kick self from the group, we know we're alive - if (m == self._member_id or - self.client.get(self._encode_member_id(m))): - actual_group_members[m] = v - if group_members != actual_group_members: - # There are some dead members, update the group - if not self.client.cas(encoded_group, actual_group_members, cas): - # It changed, let's try again - raise _retry.TryAgain - return actual_group_members - - def get_members(self, group_id): - - def _get_members(): - return set(self._get_members(group_id).keys()) - - return MemcachedFutureResult(self._executor.submit(_get_members)) - - def get_member_capabilities(self, group_id, member_id): - - def _get_member_capabilities(): - group_members = self._get_members(group_id) - if member_id not in group_members: - raise coordination.MemberNotJoined(group_id, member_id) - return group_members[member_id][b'capabilities'] - - return MemcachedFutureResult( - self._executor.submit(_get_member_capabilities)) - - def update_capabilities(self, group_id, capabilities): - encoded_group = self._encode_group_id(group_id) - - @_retry.retry() - @_translate_failures - def _update_capabilities(): - group_members, cas = self.client.gets(encoded_group) - if group_members is None: - raise coordination.GroupNotCreated(group_id) - if self._member_id not in group_members: - raise coordination.MemberNotJoined(group_id, self._member_id) - group_members[self._member_id][b'capabilities'] = capabilities - if not self.client.cas(encoded_group, group_members, cas): - # It changed, try again - raise _retry.TryAgain - - return MemcachedFutureResult( - self._executor.submit(_update_capabilities)) - - def get_leader(self, group_id): - - def _get_leader(): - return self._get_leader_lock(group_id).get_owner() - - return MemcachedFutureResult(self._executor.submit(_get_leader)) - - @_translate_failures - def heartbeat(self): - self.client.set(self._encode_member_id(self._member_id), - self.STILL_ALIVE, - expire=self.membership_timeout) - # Reset the acquired locks - for lock in self._acquired_locks: - lock.heartbeat() - return min(self.membership_timeout, - self.leader_timeout, - self.lock_timeout) - - def get_lock(self, name): - return MemcachedLock(self, name, self.lock_timeout) - - def _get_leader_lock(self, group_id): - return MemcachedLock(self, self._encode_group_leader(group_id), - self.leader_timeout) - - @_translate_failures - def run_elect_coordinator(self): - for group_id, hooks in six.iteritems(self._hooks_elected_leader): - # Try to grab the lock, if that fails, that means someone has it - # already. - leader_lock = self._get_leader_lock(group_id) - if leader_lock.acquire(blocking=False): - # We got the lock - hooks.run(coordination.LeaderElected( - group_id, - self._member_id)) - - def run_watchers(self, timeout=None): - result = super(MemcachedDriver, self).run_watchers(timeout=timeout) - self.run_elect_coordinator() - return result - - -MemcachedFutureResult = functools.partial( - coordination.CoordinatorResult, - failure_translator=_failure_translator) diff --git a/tooz/drivers/mysql.py b/tooz/drivers/mysql.py deleted file mode 100644 index cbeea8d..0000000 --- a/tooz/drivers/mysql.py +++ /dev/null @@ -1,198 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2014 eNovance -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import logging - -from oslo_utils import encodeutils -import pymysql - -import tooz -from tooz import _retry -from tooz import coordination -from tooz import locking -from tooz import utils - -LOG = logging.getLogger(__name__) - - -class MySQLLock(locking.Lock): - """A MySQL based lock.""" - - MYSQL_DEFAULT_PORT = 3306 - - def __init__(self, name, parsed_url, options): - super(MySQLLock, self).__init__(name) - self.acquired = False - self._conn = MySQLDriver.get_connection(parsed_url, options, True) - - def acquire(self, blocking=True, shared=False): - - if shared: - raise tooz.NotImplemented - - @_retry.retry(stop_max_delay=blocking) - def _lock(): - # NOTE(sileht): mysql-server (<5.7.5) allows only one lock per - # connection at a time: - # select GET_LOCK("a", 0); - # select GET_LOCK("b", 0); <-- this release lock "a" ... - # Or - # select GET_LOCK("a", 0); - # select GET_LOCK("a", 0); release and lock again "a" - # - # So, we track locally the lock status with self.acquired - if self.acquired is True: - if blocking: - raise _retry.TryAgain - return False - - try: - if not self._conn.open: - self._conn.connect() - with self._conn as cur: - cur.execute("SELECT GET_LOCK(%s, 0);", self.name) - # Can return NULL on error - if cur.fetchone()[0] is 1: - self.acquired = True - return True - except pymysql.MySQLError as e: - utils.raise_with_cause( - tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - - if blocking: - raise _retry.TryAgain - self._conn.close() - return False - - try: - return _lock() - except Exception: - # Close the connection if we tried too much and finally failed, or - # anything else bad happened. - self._conn.close() - raise - - def release(self): - if not self.acquired: - return False - try: - with self._conn as cur: - cur.execute("SELECT RELEASE_LOCK(%s);", self.name) - cur.fetchone() - self.acquired = False - self._conn.close() - return True - except pymysql.MySQLError as e: - utils.raise_with_cause(tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - - def __del__(self): - if self.acquired: - LOG.warning("unreleased lock %s garbage collected", self.name) - - -class MySQLDriver(coordination.CoordinationDriver): - """A `MySQL`_ based driver. - - This driver users `MySQL`_ database tables to - provide the coordination driver semantics and required API(s). It **is** - missing some functionality but in the future these not implemented API(s) - will be filled in. - - .. _MySQL: http://dev.mysql.com/ - """ - - CHARACTERISTICS = ( - coordination.Characteristics.NON_TIMEOUT_BASED, - coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, - coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES, - coordination.Characteristics.DISTRIBUTED_ACROSS_HOSTS, - ) - """ - Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable - enum member(s) that can be used to interogate how this driver works. - """ - - def __init__(self, member_id, parsed_url, options): - """Initialize the MySQL driver.""" - super(MySQLDriver, self).__init__(member_id, parsed_url, options) - self._parsed_url = parsed_url - self._options = utils.collapse(options) - - def _start(self): - self._conn = MySQLDriver.get_connection(self._parsed_url, - self._options) - - def _stop(self): - self._conn.close() - - def get_lock(self, name): - return MySQLLock(name, self._parsed_url, self._options) - - @staticmethod - def watch_join_group(group_id, callback): - raise tooz.NotImplemented - - @staticmethod - def unwatch_join_group(group_id, callback): - raise tooz.NotImplemented - - @staticmethod - def watch_leave_group(group_id, callback): - raise tooz.NotImplemented - - @staticmethod - def unwatch_leave_group(group_id, callback): - raise tooz.NotImplemented - - @staticmethod - def watch_elected_as_leader(group_id, callback): - raise tooz.NotImplemented - - @staticmethod - def unwatch_elected_as_leader(group_id, callback): - raise tooz.NotImplemented - - @staticmethod - def get_connection(parsed_url, options, defer_connect=False): - host = parsed_url.hostname - port = parsed_url.port or MySQLLock.MYSQL_DEFAULT_PORT - dbname = parsed_url.path[1:] - username = parsed_url.username - password = parsed_url.password - unix_socket = options.get("unix_socket") - - try: - if unix_socket: - return pymysql.Connect(unix_socket=unix_socket, - port=port, - user=username, - passwd=password, - database=dbname, - defer_connect=defer_connect) - else: - return pymysql.Connect(host=host, - port=port, - user=username, - passwd=password, - database=dbname, - defer_connect=defer_connect) - except (pymysql.err.OperationalError, pymysql.err.InternalError) as e: - utils.raise_with_cause(coordination.ToozConnectionError, - encodeutils.exception_to_unicode(e), - cause=e) diff --git a/tooz/drivers/pgsql.py b/tooz/drivers/pgsql.py deleted file mode 100644 index 1ad83df..0000000 --- a/tooz/drivers/pgsql.py +++ /dev/null @@ -1,249 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2014 eNovance -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import contextlib -import hashlib -import logging - -from oslo_utils import encodeutils -import psycopg2 -import six - -import tooz -from tooz import _retry -from tooz import coordination -from tooz import locking -from tooz import utils - -LOG = logging.getLogger(__name__) - -# See: psycopg/diagnostics_type.c for what kind of fields these -# objects may have (things like 'schema_name', 'internal_query' -# and so-on which are useful for figuring out what went wrong...) -_DIAGNOSTICS_ATTRS = tuple([ - 'column_name', - 'constraint_name', - 'context', - 'datatype_name', - 'internal_position', - 'internal_query', - 'message_detail', - 'message_hint', - 'message_primary', - 'schema_name', - 'severity', - 'source_file', - 'source_function', - 'source_line', - 'sqlstate', - 'statement_position', - 'table_name', -]) - - -def _format_exception(e): - lines = [ - "%s: %s" % (type(e).__name__, - encodeutils.exception_to_unicode(e).strip()), - ] - if hasattr(e, 'pgcode') and e.pgcode is not None: - lines.append("Error code: %s" % e.pgcode) - # The reason this hasattr check is done is that the 'diag' may not always - # be present, depending on how new of a psycopg is installed... so better - # to be safe than sorry... - if hasattr(e, 'diag') and e.diag is not None: - diagnostic_lines = [] - for attr_name in _DIAGNOSTICS_ATTRS: - if not hasattr(e.diag, attr_name): - continue - attr_value = getattr(e.diag, attr_name) - if attr_value is None: - continue - diagnostic_lines.append(" %s = %s" (attr_name, attr_value)) - if diagnostic_lines: - lines.append('Diagnostics:') - lines.extend(diagnostic_lines) - return "\n".join(lines) - - -@contextlib.contextmanager -def _translating_cursor(conn): - try: - with conn.cursor() as cur: - yield cur - except psycopg2.Error as e: - utils.raise_with_cause(tooz.ToozError, - _format_exception(e), - cause=e) - - -class PostgresLock(locking.Lock): - """A PostgreSQL based lock.""" - - def __init__(self, name, parsed_url, options): - super(PostgresLock, self).__init__(name) - self.acquired = False - self._conn = None - self._parsed_url = parsed_url - self._options = options - h = hashlib.md5() - h.update(name) - if six.PY2: - self.key = list(map(ord, h.digest()[0:2])) - else: - self.key = h.digest()[0:2] - - def acquire(self, blocking=True, shared=False): - - if shared: - raise tooz.NotImplemented - - @_retry.retry(stop_max_delay=blocking) - def _lock(): - # NOTE(sileht) One the same session the lock is not exclusive - # so we track it internally if the process already has the lock. - if self.acquired is True: - if blocking: - raise _retry.TryAgain - return False - - if not self._conn or self._conn.closed: - self._conn = PostgresDriver.get_connection(self._parsed_url, - self._options) - - with _translating_cursor(self._conn) as cur: - if blocking is True: - cur.execute("SELECT pg_advisory_lock(%s, %s);", - self.key) - cur.fetchone() - self.acquired = True - return True - else: - cur.execute("SELECT pg_try_advisory_lock(%s, %s);", - self.key) - if cur.fetchone()[0] is True: - self.acquired = True - return True - elif blocking is False: - self._conn.close() - return False - else: - raise _retry.TryAgain - - try: - return _lock() - except Exception: - if self._conn: - self._conn.close() - raise - - def release(self): - if not self.acquired: - return False - - with _translating_cursor(self._conn) as cur: - cur.execute("SELECT pg_advisory_unlock(%s, %s);", self.key) - cur.fetchone() - self.acquired = False - self._conn.close() - return True - - def __del__(self): - if self.acquired: - LOG.warning("unreleased lock %s garbage collected", self.name) - - -class PostgresDriver(coordination.CoordinationDriver): - """A `PostgreSQL`_ based driver. - - This driver users `PostgreSQL`_ database tables to - provide the coordination driver semantics and required API(s). It **is** - missing some functionality but in the future these not implemented API(s) - will be filled in. - - .. _PostgreSQL: http://www.postgresql.org/ - """ - - CHARACTERISTICS = ( - coordination.Characteristics.NON_TIMEOUT_BASED, - coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, - coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES, - coordination.Characteristics.DISTRIBUTED_ACROSS_HOSTS, - ) - """ - Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable - enum member(s) that can be used to interogate how this driver works. - """ - - def __init__(self, member_id, parsed_url, options): - """Initialize the PostgreSQL driver.""" - super(PostgresDriver, self).__init__(member_id, parsed_url, options) - self._parsed_url = parsed_url - self._options = utils.collapse(options) - - def _start(self): - self._conn = self.get_connection(self._parsed_url, self._options) - - def _stop(self): - self._conn.close() - - def get_lock(self, name): - return PostgresLock(name, self._parsed_url, self._options) - - @staticmethod - def watch_join_group(group_id, callback): - raise tooz.NotImplemented - - @staticmethod - def unwatch_join_group(group_id, callback): - raise tooz.NotImplemented - - @staticmethod - def watch_leave_group(group_id, callback): - raise tooz.NotImplemented - - @staticmethod - def unwatch_leave_group(group_id, callback): - raise tooz.NotImplemented - - @staticmethod - def watch_elected_as_leader(group_id, callback): - raise tooz.NotImplemented - - @staticmethod - def unwatch_elected_as_leader(group_id, callback): - raise tooz.NotImplemented - - @staticmethod - def get_connection(parsed_url, options): - host = options.get("host") or parsed_url.hostname - port = options.get("port") or parsed_url.port - dbname = options.get("dbname") or parsed_url.path[1:] - kwargs = {} - if parsed_url.username is not None: - kwargs["user"] = parsed_url.username - if parsed_url.password is not None: - kwargs["password"] = parsed_url.password - - try: - return psycopg2.connect(host=host, - port=port, - database=dbname, - **kwargs) - except psycopg2.Error as e: - utils.raise_with_cause(coordination.ToozConnectionError, - _format_exception(e), - cause=e) diff --git a/tooz/drivers/redis.py b/tooz/drivers/redis.py deleted file mode 100644 index 8d6be82..0000000 --- a/tooz/drivers/redis.py +++ /dev/null @@ -1,753 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from __future__ import absolute_import - -import contextlib -from distutils import version -import functools -import logging -import string -import threading - -from oslo_utils import encodeutils -from oslo_utils import strutils -import redis -from redis import exceptions -from redis import sentinel -import six -from six.moves import map as compat_map -from six.moves import zip as compat_zip - -import tooz -from tooz import coordination -from tooz import locking -from tooz import utils - -LOG = logging.getLogger(__name__) - - -@contextlib.contextmanager -def _translate_failures(): - """Translates common redis exceptions into tooz exceptions.""" - try: - yield - except (exceptions.ConnectionError, exceptions.TimeoutError) as e: - utils.raise_with_cause(coordination.ToozConnectionError, - encodeutils.exception_to_unicode(e), - cause=e) - except exceptions.RedisError as e: - utils.raise_with_cause(tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - - -class RedisLock(locking.Lock): - def __init__(self, coord, client, name, timeout): - name = "%s_%s_lock" % (coord.namespace, six.text_type(name)) - super(RedisLock, self).__init__(name) - # NOTE(jd) Make sure we don't release and heartbeat at the same time by - # using a exclusive access lock (LP#1557593) - self._exclusive_access = threading.Lock() - self._lock = client.lock(name, - timeout=timeout, - thread_local=False) - self._coord = coord - self._client = client - - def is_still_owner(self): - with _translate_failures(): - lock_tok = self._lock.local.token - if not lock_tok: - return False - owner_tok = self._client.get(self.name) - return owner_tok == lock_tok - - def break_(self): - with _translate_failures(): - return bool(self._client.delete(self.name)) - - def acquire(self, blocking=True, shared=False): - if shared: - raise tooz.NotImplemented - blocking, timeout = utils.convert_blocking(blocking) - with _translate_failures(): - acquired = self._lock.acquire( - blocking=blocking, blocking_timeout=timeout) - if acquired: - with self._exclusive_access: - self._coord._acquired_locks.add(self) - return acquired - - def release(self): - with self._exclusive_access: - if not self.acquired: - return False - with _translate_failures(): - try: - self._lock.release() - except exceptions.LockError: - return False - self._coord._acquired_locks.discard(self) - return True - - def heartbeat(self): - with self._exclusive_access: - if self.acquired: - with _translate_failures(): - self._lock.extend(self._lock.timeout) - return True - return False - - @property - def acquired(self): - return self in self._coord._acquired_locks - - -class RedisDriver(coordination.CoordinationDriverCachedRunWatchers, - coordination.CoordinationDriverWithExecutor): - """Redis provides a few nice benefits that act as a poormans zookeeper. - - It **is** fully functional and implements all of the coordination - driver API(s). It stores data into `redis`_ using the provided `redis`_ - API(s) using `msgpack`_ encoded values as needed. - - - Durability (when setup with `AOF`_ mode). - - Consistent, note that this is still restricted to only - one redis server, without the recently released redis (alpha) - clustering > 1 server will not be consistent when partitions - or failures occur (even redis clustering docs state it is - not a fully AP or CP solution, which means even with it there - will still be *potential* inconsistencies). - - Master/slave failover (when setup with redis `sentinel`_), giving - some notion of HA (values *can* be lost when a failover transition - occurs). - - To use a `sentinel`_ the connection URI must point to the sentinel server. - At connection time the sentinel will be asked for the current IP and port - of the master and then connect there. The connection URI for sentinel - should be written as follows:: - - redis://:?sentinel= - - Additional sentinel hosts are listed with multiple ``sentinel_fallback`` - parameters as follows:: - - redis://:?sentinel=& - sentinel_fallback=:& - sentinel_fallback=:& - sentinel_fallback=: - - Further resources/links: - - - http://redis.io/ - - http://redis.io/topics/sentinel - - http://redis.io/topics/cluster-spec - - Note that this client will itself retry on transaction failure (when they - keys being watched have changed underneath the current transaction). - Currently the number of attempts that are tried is infinite (this might - be addressed in https://github.com/andymccurdy/redis-py/issues/566 when - that gets worked on). See http://redis.io/topics/transactions for more - information on this topic. - - General recommendations/usage considerations: - - - When used for locks, run in AOF mode and think carefully about how - your redis deployment handles losing a server (the clustering support - is supposed to aid in losing servers, but it is also of unknown - reliablity and is relatively new, so use at your own risk). - - .. _redis: http://redis.io/ - .. _msgpack: http://msgpack.org/ - .. _sentinel: http://redis.io/topics/sentinel - .. _AOF: http://redis.io/topics/persistence - """ - - CHARACTERISTICS = ( - coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, - coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES, - coordination.Characteristics.DISTRIBUTED_ACROSS_HOSTS, - coordination.Characteristics.CAUSAL, - ) - """ - Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable - enum member(s) that can be used to interogate how this driver works. - """ - - MIN_VERSION = version.LooseVersion("2.6.0") - """ - The min redis version that this driver requires to operate with... - """ - - GROUP_EXISTS = b'__created__' - """ - Redis deletes dictionaries that have no keys in them, which means the - key will disappear which means we can't tell the difference between - a group not existing and a group being empty without this key being - saved... - """ - - #: Value used (with group exists key) to keep a group from disappearing. - GROUP_EXISTS_VALUE = b'1' - - #: Default namespace for keys when none is provided. - DEFAULT_NAMESPACE = b'_tooz' - - NAMESPACE_SEP = b':' - """ - Separator that is used to combine a key with the namespace (to get - the **actual** key that will be used). - """ - - DEFAULT_ENCODING = 'utf8' - """ - This is for python3.x; which will behave differently when returned - binary types or unicode types (redis uses binary internally it appears), - so to just stick with a common way of doing this, make all the things - binary (with this default encoding if one is not given and a unicode - string is provided). - """ - - CLIENT_ARGS = frozenset([ - 'db', - 'encoding', - 'retry_on_timeout', - 'socket_keepalive', - 'socket_timeout', - 'ssl', - 'ssl_certfile', - 'ssl_keyfile', - 'sentinel', - 'sentinel_fallback', - ]) - """ - Keys that we allow to proxy from the coordinator configuration into the - redis client (used to configure the redis client internals so that - it works as you expect/want it to). - - See: http://redis-py.readthedocs.org/en/latest/#redis.Redis - - See: https://github.com/andymccurdy/redis-py/blob/2.10.3/redis/client.py - """ - - #: Client arguments that are expected/allowed to be lists. - CLIENT_LIST_ARGS = frozenset([ - 'sentinel_fallback', - ]) - - #: Client arguments that are expected to be boolean convertible. - CLIENT_BOOL_ARGS = frozenset([ - 'retry_on_timeout', - 'ssl', - ]) - - #: Client arguments that are expected to be int convertible. - CLIENT_INT_ARGS = frozenset([ - 'db', - 'socket_keepalive', - 'socket_timeout', - ]) - - #: Default socket timeout to use when none is provided. - CLIENT_DEFAULT_SOCKET_TO = 30 - - #: String used to keep a key/member alive (until it next expires). - STILL_ALIVE = b"Not dead!" - - SCRIPTS = { - 'create_group': """ --- Extract *all* the variables (so we can easily know what they are)... -local namespaced_group_key = KEYS[1] -local all_groups_key = KEYS[2] -local no_namespaced_group_key = ARGV[1] -if redis.call("exists", namespaced_group_key) == 1 then - return 0 -end -redis.call("sadd", all_groups_key, no_namespaced_group_key) -redis.call("hset", namespaced_group_key, - "${group_existence_key}", "${group_existence_value}") -return 1 -""", - 'delete_group': """ --- Extract *all* the variables (so we can easily know what they are)... -local namespaced_group_key = KEYS[1] -local all_groups_key = KEYS[2] -local no_namespaced_group_key = ARGV[1] -if redis.call("exists", namespaced_group_key) == 0 then - return -1 -end -if redis.call("sismember", all_groups_key, no_namespaced_group_key) == 0 then - return -2 -end -if redis.call("hlen", namespaced_group_key) > 1 then - return -3 -end --- First remove from the set (then delete the group); if the set removal --- fails, at least the group will still exist (and can be fixed manually)... -if redis.call("srem", all_groups_key, no_namespaced_group_key) == 0 then - return -4 -end -redis.call("del", namespaced_group_key) -return 1 -""", - 'update_capabilities': """ --- Extract *all* the variables (so we can easily know what they are)... -local group_key = KEYS[1] -local member_id = ARGV[1] -local caps = ARGV[2] -if redis.call("exists", group_key) == 0 then - return -1 -end -if redis.call("hexists", group_key, member_id) == 0 then - return -2 -end -redis.call("hset", group_key, member_id, caps) -return 1 -""", - } - """`Lua`_ **template** scripts that will be used by various methods (they - are turned into real scripts and loaded on call into the :func:`.start` - method). - - .. _Lua: http://www.lua.org - """ - - EXCLUDE_OPTIONS = CLIENT_LIST_ARGS - - def __init__(self, member_id, parsed_url, options): - super(RedisDriver, self).__init__(member_id, parsed_url, options) - self._parsed_url = parsed_url - self._encoding = self._options.get('encoding', self.DEFAULT_ENCODING) - timeout = self._options.get('timeout', self.CLIENT_DEFAULT_SOCKET_TO) - self.timeout = int(timeout) - self.membership_timeout = float(self._options.get( - 'membership_timeout', timeout)) - lock_timeout = self._options.get('lock_timeout', self.timeout) - self.lock_timeout = int(lock_timeout) - namespace = self._options.get('namespace', self.DEFAULT_NAMESPACE) - self._namespace = utils.to_binary(namespace, encoding=self._encoding) - self._group_prefix = self._namespace + b"_group" - self._beat_prefix = self._namespace + b"_beats" - self._groups = self._namespace + b"_groups" - self._client = None - self._acquired_locks = set() - self._started = False - self._server_info = {} - self._scripts = {} - - def _check_fetch_redis_version(self, geq_version, not_existent=True): - if isinstance(geq_version, six.string_types): - desired_version = version.LooseVersion(geq_version) - elif isinstance(geq_version, version.LooseVersion): - desired_version = geq_version - else: - raise TypeError("Version check expects a string/version type") - try: - redis_version = version.LooseVersion( - self._server_info['redis_version']) - except KeyError: - return (not_existent, None) - else: - if redis_version < desired_version: - return (False, redis_version) - else: - return (True, redis_version) - - @property - def namespace(self): - return self._namespace - - @property - def running(self): - return self._started - - def get_lock(self, name): - return RedisLock(self, self._client, name, self.lock_timeout) - - _dumps = staticmethod(utils.dumps) - _loads = staticmethod(utils.loads) - - @classmethod - def _make_client(cls, parsed_url, options, default_socket_timeout): - kwargs = {} - if parsed_url.hostname: - kwargs['host'] = parsed_url.hostname - if parsed_url.port: - kwargs['port'] = parsed_url.port - else: - if not parsed_url.path: - raise ValueError("Expected socket path in parsed urls path") - kwargs['unix_socket_path'] = parsed_url.path - if parsed_url.password: - kwargs['password'] = parsed_url.password - for a in cls.CLIENT_ARGS: - if a not in options: - continue - if a in cls.CLIENT_BOOL_ARGS: - v = strutils.bool_from_string(options[a]) - elif a in cls.CLIENT_LIST_ARGS: - v = options[a] - elif a in cls.CLIENT_INT_ARGS: - v = int(options[a]) - else: - v = options[a] - kwargs[a] = v - if 'socket_timeout' not in kwargs: - kwargs['socket_timeout'] = default_socket_timeout - - # Ask the sentinel for the current master if there is a - # sentinel arg. - if 'sentinel' in kwargs: - sentinel_hosts = [ - tuple(fallback.split(':')) - for fallback in kwargs.get('sentinel_fallback', []) - ] - sentinel_hosts.insert(0, (kwargs['host'], kwargs['port'])) - sentinel_server = sentinel.Sentinel( - sentinel_hosts, - socket_timeout=kwargs['socket_timeout']) - sentinel_name = kwargs['sentinel'] - del kwargs['sentinel'] - if 'sentinel_fallback' in kwargs: - del kwargs['sentinel_fallback'] - master_client = sentinel_server.master_for(sentinel_name, **kwargs) - # The master_client is a redis.StrictRedis using a - # Sentinel managed connection pool. - return master_client - return redis.StrictRedis(**kwargs) - - def _start(self): - super(RedisDriver, self)._start() - try: - self._client = self._make_client(self._parsed_url, self._options, - self.timeout) - except exceptions.RedisError as e: - utils.raise_with_cause(coordination.ToozConnectionError, - encodeutils.exception_to_unicode(e), - cause=e) - else: - # Ensure that the server is alive and not dead, this does not - # ensure the server will always be alive, but does insure that it - # at least is alive once... - with _translate_failures(): - self._server_info = self._client.info() - # Validate we have a good enough redis version we are connected - # to so that the basic set of features we support will actually - # work (instead of blowing up). - new_enough, redis_version = self._check_fetch_redis_version( - self.MIN_VERSION) - if not new_enough: - raise tooz.NotImplemented("Redis version greater than or" - " equal to '%s' is required" - " to use this driver; '%s' is" - " being used which is not new" - " enough" % (self.MIN_VERSION, - redis_version)) - tpl_params = { - 'group_existence_value': self.GROUP_EXISTS_VALUE, - 'group_existence_key': self.GROUP_EXISTS, - } - # For py3.x ensure these are unicode since the string template - # replacement will expect unicode (and we don't want b'' as a - # prefix which will happen in py3.x if this is not done). - for (k, v) in six.iteritems(tpl_params.copy()): - if isinstance(v, six.binary_type): - v = v.decode('ascii') - tpl_params[k] = v - prepared_scripts = {} - for name, raw_script_tpl in six.iteritems(self.SCRIPTS): - script_tpl = string.Template(raw_script_tpl) - script = script_tpl.substitute(**tpl_params) - prepared_scripts[name] = self._client.register_script(script) - self._scripts = prepared_scripts - self.heartbeat() - self._started = True - - def _encode_beat_id(self, member_id): - member_id = utils.to_binary(member_id, encoding=self._encoding) - return self.NAMESPACE_SEP.join([self._beat_prefix, member_id]) - - def _encode_member_id(self, member_id): - member_id = utils.to_binary(member_id, encoding=self._encoding) - if member_id == self.GROUP_EXISTS: - raise ValueError("Not allowed to use private keys as a member id") - return member_id - - def _decode_member_id(self, member_id): - return utils.to_binary(member_id, encoding=self._encoding) - - def _encode_group_leader(self, group_id): - group_id = utils.to_binary(group_id, encoding=self._encoding) - return b"leader_of_" + group_id - - def _encode_group_id(self, group_id, apply_namespace=True): - group_id = utils.to_binary(group_id, encoding=self._encoding) - if not apply_namespace: - return group_id - return self.NAMESPACE_SEP.join([self._group_prefix, group_id]) - - def _decode_group_id(self, group_id): - return utils.to_binary(group_id, encoding=self._encoding) - - def heartbeat(self): - with _translate_failures(): - beat_id = self._encode_beat_id(self._member_id) - expiry_ms = max(0, int(self.membership_timeout * 1000.0)) - self._client.psetex(beat_id, time_ms=expiry_ms, - value=self.STILL_ALIVE) - for lock in self._acquired_locks.copy(): - try: - lock.heartbeat() - except tooz.ToozError: - LOG.warning("Unable to heartbeat lock '%s'", lock, - exc_info=True) - return min(self.lock_timeout, self.membership_timeout) - - def _stop(self): - while self._acquired_locks: - lock = self._acquired_locks.pop() - try: - lock.release() - except tooz.ToozError: - LOG.warning("Unable to release lock '%s'", lock, exc_info=True) - super(RedisDriver, self)._stop() - if self._client is not None: - # Make sure we no longer exist... - beat_id = self._encode_beat_id(self._member_id) - try: - # NOTE(harlowja): this will delete nothing if the key doesn't - # exist in the first place, which is fine/expected/desired... - with _translate_failures(): - self._client.delete(beat_id) - except tooz.ToozError: - LOG.warning("Unable to delete heartbeat key '%s'", beat_id, - exc_info=True) - self._client = None - self._server_info = {} - self._scripts.clear() - self._started = False - - def _submit(self, cb, *args, **kwargs): - if not self._started: - raise tooz.ToozError("Redis driver has not been started") - return self._executor.submit(cb, *args, **kwargs) - - def _get_script(self, script_key): - try: - return self._scripts[script_key] - except KeyError: - raise tooz.ToozError("Redis driver has not been started") - - def create_group(self, group_id): - script = self._get_script('create_group') - - def _create_group(script): - encoded_group = self._encode_group_id(group_id) - keys = [ - encoded_group, - self._groups, - ] - args = [ - self._encode_group_id(group_id, apply_namespace=False), - ] - result = script(keys=keys, args=args) - result = strutils.bool_from_string(result) - if not result: - raise coordination.GroupAlreadyExist(group_id) - - return RedisFutureResult(self._submit(_create_group, script)) - - def update_capabilities(self, group_id, capabilities): - script = self._get_script('update_capabilities') - - def _update_capabilities(script): - keys = [ - self._encode_group_id(group_id), - ] - args = [ - self._encode_member_id(self._member_id), - self._dumps(capabilities), - ] - result = int(script(keys=keys, args=args)) - if result == -1: - raise coordination.GroupNotCreated(group_id) - if result == -2: - raise coordination.MemberNotJoined(group_id, self._member_id) - - return RedisFutureResult(self._submit(_update_capabilities, script)) - - def leave_group(self, group_id): - encoded_group = self._encode_group_id(group_id) - encoded_member_id = self._encode_member_id(self._member_id) - - def _leave_group(p): - if not p.exists(encoded_group): - raise coordination.GroupNotCreated(group_id) - p.multi() - p.hdel(encoded_group, encoded_member_id) - c = p.execute()[0] - if c == 0: - raise coordination.MemberNotJoined(group_id, self._member_id) - else: - self._joined_groups.discard(group_id) - - return RedisFutureResult(self._submit(self._client.transaction, - _leave_group, encoded_group, - value_from_callable=True)) - - def get_members(self, group_id): - encoded_group = self._encode_group_id(group_id) - - def _get_members(p): - if not p.exists(encoded_group): - raise coordination.GroupNotCreated(group_id) - potential_members = set() - for m in p.hkeys(encoded_group): - m = self._decode_member_id(m) - if m != self.GROUP_EXISTS: - potential_members.add(m) - if not potential_members: - return set() - # Ok now we need to see which members have passed away... - gone_members = set() - member_values = p.mget(compat_map(self._encode_beat_id, - potential_members)) - for (potential_member, value) in compat_zip(potential_members, - member_values): - # Always preserve self (just incase we haven't heartbeated - # while this call/s was being made...), this does *not* prevent - # another client from removing this though... - if potential_member == self._member_id: - continue - if not value: - gone_members.add(potential_member) - # Trash all the members that no longer are with us... RIP... - if gone_members: - p.multi() - encoded_gone_members = list(self._encode_member_id(m) - for m in gone_members) - p.hdel(encoded_group, *encoded_gone_members) - p.execute() - return set(m for m in potential_members - if m not in gone_members) - return potential_members - - return RedisFutureResult(self._submit(self._client.transaction, - _get_members, encoded_group, - value_from_callable=True)) - - def get_member_capabilities(self, group_id, member_id): - encoded_group = self._encode_group_id(group_id) - encoded_member_id = self._encode_member_id(member_id) - - def _get_member_capabilities(p): - if not p.exists(encoded_group): - raise coordination.GroupNotCreated(group_id) - capabilities = p.hget(encoded_group, encoded_member_id) - if capabilities is None: - raise coordination.MemberNotJoined(group_id, member_id) - return self._loads(capabilities) - - return RedisFutureResult(self._submit(self._client.transaction, - _get_member_capabilities, - encoded_group, - value_from_callable=True)) - - def join_group(self, group_id, capabilities=b""): - encoded_group = self._encode_group_id(group_id) - encoded_member_id = self._encode_member_id(self._member_id) - - def _join_group(p): - if not p.exists(encoded_group): - raise coordination.GroupNotCreated(group_id) - p.multi() - p.hset(encoded_group, encoded_member_id, - self._dumps(capabilities)) - c = p.execute()[0] - if c == 0: - # Field already exists... - raise coordination.MemberAlreadyExist(group_id, - self._member_id) - else: - self._joined_groups.add(group_id) - - return RedisFutureResult(self._submit(self._client.transaction, - _join_group, - encoded_group, - value_from_callable=True)) - - def delete_group(self, group_id): - script = self._get_script('delete_group') - - def _delete_group(script): - keys = [ - self._encode_group_id(group_id), - self._groups, - ] - args = [ - self._encode_group_id(group_id, apply_namespace=False), - ] - result = int(script(keys=keys, args=args)) - if result in (-1, -2): - raise coordination.GroupNotCreated(group_id) - if result == -3: - raise coordination.GroupNotEmpty(group_id) - if result == -4: - raise tooz.ToozError("Unable to remove '%s' key" - " from set located at '%s'" - % (args[0], keys[-1])) - if result != 1: - raise tooz.ToozError("Internal error, unable" - " to complete group '%s' removal" - % (group_id)) - - return RedisFutureResult(self._submit(_delete_group, script)) - - def _destroy_group(self, group_id): - """Should only be used in tests...""" - self._client.delete(self._encode_group_id(group_id)) - - def get_groups(self): - - def _get_groups(): - results = [] - for g in self._client.smembers(self._groups): - results.append(self._decode_group_id(g)) - return results - - return RedisFutureResult(self._submit(_get_groups)) - - def _get_leader_lock(self, group_id): - name = self._encode_group_leader(group_id) - return self.get_lock(name) - - def run_elect_coordinator(self): - for group_id, hooks in six.iteritems(self._hooks_elected_leader): - leader_lock = self._get_leader_lock(group_id) - if leader_lock.acquire(blocking=False): - # We got the lock - hooks.run(coordination.LeaderElected(group_id, - self._member_id)) - - def run_watchers(self, timeout=None): - result = super(RedisDriver, self).run_watchers(timeout=timeout) - self.run_elect_coordinator() - return result - - -RedisFutureResult = functools.partial(coordination.CoordinatorResult, - failure_translator=_translate_failures) diff --git a/tooz/drivers/zake.py b/tooz/drivers/zake.py deleted file mode 100644 index 4a010be..0000000 --- a/tooz/drivers/zake.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2013-2014 Mirantis Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from __future__ import absolute_import - -from zake import fake_client -from zake import fake_storage - -from tooz import coordination -from tooz.drivers import zookeeper - - -class ZakeDriver(zookeeper.KazooDriver): - """This driver uses the `zake`_ client to mimic real `zookeeper`_ servers. - - It **should** be mainly used (and **is** really only intended to be used in - this manner) for testing and integration (where real `zookeeper`_ servers - are typically not available). - - .. _zake: https://pypi.python.org/pypi/zake - .. _zookeeper: http://zookeeper.apache.org/ - """ - - CHARACTERISTICS = ( - coordination.Characteristics.NON_TIMEOUT_BASED, - coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, - ) - """ - Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable - enum member(s) that can be used to interogate how this driver works. - """ - - # NOTE(harlowja): this creates a shared backend 'storage' layer that - # would typically exist inside a zookeeper server, but since zake has - # no concept of a 'real' zookeeper server we create a fake one and share - # it among active clients to simulate zookeeper's consistent storage in - # a thread-safe manner. - fake_storage = fake_storage.FakeStorage( - fake_client.k_threading.SequentialThreadingHandler()) - - @classmethod - def _make_client(cls, parsed_url, options): - if 'storage' in options: - storage = options['storage'] - else: - storage = cls.fake_storage - return fake_client.FakeClient(storage=storage) diff --git a/tooz/drivers/zookeeper.py b/tooz/drivers/zookeeper.py deleted file mode 100644 index 52f151e..0000000 --- a/tooz/drivers/zookeeper.py +++ /dev/null @@ -1,547 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2013-2014 eNovance Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from kazoo import client -from kazoo import exceptions -from kazoo import security -try: - from kazoo.handlers import eventlet as eventlet_handler -except ImportError: - eventlet_handler = None -from kazoo.handlers import threading as threading_handler -from kazoo.protocol import paths -from oslo_utils import encodeutils -from oslo_utils import strutils -import six -from six.moves import filter as compat_filter - -import tooz -from tooz import coordination -from tooz import locking -from tooz import utils - - -class ZooKeeperLock(locking.Lock): - def __init__(self, name, lock): - super(ZooKeeperLock, self).__init__(name) - self._lock = lock - self._client = lock.client - - def is_still_owner(self): - if not self.acquired: - return False - try: - data, _znode = self._client.get( - paths.join(self._lock.path, self._lock.node)) - return data == self._lock.data - except (self._client.handler.timeout_exception, - exceptions.ConnectionLoss, - exceptions.ConnectionDropped, - exceptions.NoNodeError): - return False - except exceptions.KazooException as e: - utils.raise_with_cause(tooz.ToozError, - "operation error: %s" % (e), - cause=e) - - def acquire(self, blocking=True, shared=False): - if shared: - raise tooz.NotImplemented - blocking, timeout = utils.convert_blocking(blocking) - return self._lock.acquire(blocking=blocking, - timeout=timeout) - - def release(self): - if self.acquired: - self._lock.release() - return True - else: - return False - - @property - def acquired(self): - return self._lock.is_acquired - - -class KazooDriver(coordination.CoordinationDriverCachedRunWatchers): - """This driver uses the `kazoo`_ client against real `zookeeper`_ servers. - - It **is** fully functional and implements all of the coordination - driver API(s). It stores data into `zookeeper`_ using znodes - and `msgpack`_ encoded values. - - To configure the client to your liking a subset of the options defined at - http://kazoo.readthedocs.org/en/latest/api/client.html - will be extracted from the coordinator url (or any provided options), - so that a specific coordinator can be created that will work for you. - - Currently the following options will be proxied to the contained client: - - ================ =============================== ==================== - Name Source Default - ================ =============================== ==================== - hosts url netloc + 'hosts' option key localhost:2181 - timeout 'timeout' options key 10.0 (kazoo default) - connection_retry 'connection_retry' options key None - command_retry 'command_retry' options key None - randomize_hosts 'randomize_hosts' options key True - ================ =============================== ==================== - - .. _kazoo: http://kazoo.readthedocs.org/ - .. _zookeeper: http://zookeeper.apache.org/ - .. _msgpack: http://msgpack.org/ - """ - #: Default namespace when none is provided. - TOOZ_NAMESPACE = b"tooz" - - HANDLERS = { - 'threading': threading_handler.SequentialThreadingHandler, - } - - if eventlet_handler: - HANDLERS['eventlet'] = eventlet_handler.SequentialEventletHandler - - """ - Restricted immutable dict of handler 'kinds' -> handler classes that - this driver can accept via 'handler' option key (the expected value for - this option is one of the keys in this dictionary). - """ - - CHARACTERISTICS = ( - coordination.Characteristics.NON_TIMEOUT_BASED, - coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, - coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES, - coordination.Characteristics.DISTRIBUTED_ACROSS_HOSTS, - # Writes *always* go through a single leader process, but it may - # take a while for those writes to propagate to followers (and = - # during this time clients can read older values)... - coordination.Characteristics.SEQUENTIAL, - ) - """ - Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable - enum member(s) that can be used to interogate how this driver works. - """ - - def __init__(self, member_id, parsed_url, options): - super(KazooDriver, self).__init__(member_id, parsed_url, options) - options = utils.collapse(options, exclude=['hosts']) - self.timeout = int(options.get('timeout', '10')) - self._namespace = options.get('namespace', self.TOOZ_NAMESPACE) - self._coord = self._make_client(parsed_url, options) - self._timeout_exception = self._coord.handler.timeout_exception - - def _start(self): - try: - self._coord.start(timeout=self.timeout) - except self._coord.handler.timeout_exception as e: - e_msg = encodeutils.exception_to_unicode(e) - utils.raise_with_cause(coordination.ToozConnectionError, - "Operational error: %s" % e_msg, - cause=e) - try: - self._coord.ensure_path(self._paths_join("/", self._namespace)) - except exceptions.KazooException as e: - e_msg = encodeutils.exception_to_unicode(e) - utils.raise_with_cause(tooz.ToozError, - "Operational error: %s" % e_msg, - cause=e) - self._leader_locks = {} - - def _stop(self): - self._coord.stop() - - @staticmethod - def _dumps(data): - return utils.dumps(data) - - @staticmethod - def _loads(blob): - return utils.loads(blob) - - def _create_group_handler(self, async_result, timeout, - timeout_exception, group_id): - try: - async_result.get(block=True, timeout=timeout) - except timeout_exception as e: - utils.raise_with_cause(coordination.OperationTimedOut, - encodeutils.exception_to_unicode(e), - cause=e) - except exceptions.NodeExistsError: - raise coordination.GroupAlreadyExist(group_id) - except exceptions.NoNodeError as e: - utils.raise_with_cause(tooz.ToozError, - "Tooz namespace '%s' has not" - " been created" % self._namespace, - cause=e) - except exceptions.ZookeeperError as e: - utils.raise_with_cause(tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - - def create_group(self, group_id): - group_path = self._path_group(group_id) - async_result = self._coord.create_async(group_path) - return ZooAsyncResult(async_result, self._create_group_handler, - timeout_exception=self._timeout_exception, - group_id=group_id) - - @staticmethod - def _delete_group_handler(async_result, timeout, - timeout_exception, group_id): - try: - async_result.get(block=True, timeout=timeout) - except timeout_exception as e: - utils.raise_with_cause(coordination.OperationTimedOut, - encodeutils.exception_to_unicode(e), - cause=e) - except exceptions.NoNodeError: - raise coordination.GroupNotCreated(group_id) - except exceptions.NotEmptyError: - raise coordination.GroupNotEmpty(group_id) - except exceptions.ZookeeperError as e: - utils.raise_with_cause(tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - - def delete_group(self, group_id): - group_path = self._path_group(group_id) - async_result = self._coord.delete_async(group_path) - return ZooAsyncResult(async_result, self._delete_group_handler, - timeout_exception=self._timeout_exception, - group_id=group_id) - - @staticmethod - def _join_group_handler(async_result, timeout, - timeout_exception, group_id, member_id): - try: - async_result.get(block=True, timeout=timeout) - except timeout_exception as e: - utils.raise_with_cause(coordination.OperationTimedOut, - encodeutils.exception_to_unicode(e), - cause=e) - except exceptions.NodeExistsError: - raise coordination.MemberAlreadyExist(group_id, member_id) - except exceptions.NoNodeError: - raise coordination.GroupNotCreated(group_id) - except exceptions.ZookeeperError as e: - utils.raise_with_cause(tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - - def join_group(self, group_id, capabilities=b""): - member_path = self._path_member(group_id, self._member_id) - capabilities = self._dumps(capabilities) - async_result = self._coord.create_async(member_path, - value=capabilities, - ephemeral=True) - return ZooAsyncResult(async_result, self._join_group_handler, - timeout_exception=self._timeout_exception, - group_id=group_id, member_id=self._member_id) - - @staticmethod - def _leave_group_handler(async_result, timeout, - timeout_exception, group_id, member_id): - try: - async_result.get(block=True, timeout=timeout) - except timeout_exception as e: - utils.raise_with_cause(coordination.OperationTimedOut, - encodeutils.exception_to_unicode(e), - cause=e) - except exceptions.NoNodeError: - raise coordination.MemberNotJoined(group_id, member_id) - except exceptions.ZookeeperError as e: - utils.raise_with_cause(tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - - def heartbeat(self): - # Just fetch the base path (and do nothing with it); this will - # force any waiting heartbeat responses to be flushed, and also - # ensures that the connection still works as expected... - base_path = self._paths_join("/", self._namespace) - try: - self._coord.get(base_path) - except self._timeout_exception as e: - utils.raise_with_cause(coordination.OperationTimedOut, - encodeutils.exception_to_unicode(e), - cause=e) - except exceptions.NoNodeError: - pass - except exceptions.ZookeeperError as e: - utils.raise_with_cause(tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - return self.timeout - - def leave_group(self, group_id): - member_path = self._path_member(group_id, self._member_id) - async_result = self._coord.delete_async(member_path) - return ZooAsyncResult(async_result, self._leave_group_handler, - timeout_exception=self._timeout_exception, - group_id=group_id, member_id=self._member_id) - - @staticmethod - def _get_members_handler(async_result, timeout, - timeout_exception, group_id): - try: - members_ids = async_result.get(block=True, timeout=timeout) - except timeout_exception as e: - utils.raise_with_cause(coordination.OperationTimedOut, - encodeutils.exception_to_unicode(e), - cause=e) - except exceptions.NoNodeError: - raise coordination.GroupNotCreated(group_id) - except exceptions.ZookeeperError as e: - utils.raise_with_cause(tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - else: - return set(m.encode('ascii') for m in members_ids) - - def get_members(self, group_id): - group_path = self._paths_join("/", self._namespace, group_id) - async_result = self._coord.get_children_async(group_path) - return ZooAsyncResult(async_result, self._get_members_handler, - timeout_exception=self._timeout_exception, - group_id=group_id) - - @staticmethod - def _update_capabilities_handler(async_result, timeout, - timeout_exception, group_id, member_id): - try: - async_result.get(block=True, timeout=timeout) - except timeout_exception as e: - utils.raise_with_cause(coordination.OperationTimedOut, - encodeutils.exception_to_unicode(e), - cause=e) - except exceptions.NoNodeError: - raise coordination.MemberNotJoined(group_id, member_id) - except exceptions.ZookeeperError as e: - utils.raise_with_cause(tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - - def update_capabilities(self, group_id, capabilities): - member_path = self._path_member(group_id, self._member_id) - capabilities = self._dumps(capabilities) - async_result = self._coord.set_async(member_path, capabilities) - return ZooAsyncResult(async_result, self._update_capabilities_handler, - timeout_exception=self._timeout_exception, - group_id=group_id, member_id=self._member_id) - - @classmethod - def _get_member_capabilities_handler(cls, async_result, timeout, - timeout_exception, group_id, - member_id): - try: - capabilities = async_result.get(block=True, timeout=timeout)[0] - except timeout_exception as e: - utils.raise_with_cause(coordination.OperationTimedOut, - encodeutils.exception_to_unicode(e), - cause=e) - except exceptions.NoNodeError: - raise coordination.MemberNotJoined(group_id, member_id) - except exceptions.ZookeeperError as e: - utils.raise_with_cause(tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - else: - return cls._loads(capabilities) - - def get_member_capabilities(self, group_id, member_id): - member_path = self._path_member(group_id, member_id) - async_result = self._coord.get_async(member_path) - return ZooAsyncResult(async_result, - self._get_member_capabilities_handler, - timeout_exception=self._timeout_exception, - group_id=group_id, member_id=self._member_id) - - @classmethod - def _get_member_info_handler(cls, async_result, timeout, - timeout_exception, group_id, - member_id): - try: - capabilities, znode_stats = async_result.get(block=True, - timeout=timeout) - except timeout_exception as e: - utils.raise_with_cause(coordination.OperationTimedOut, - encodeutils.exception_to_unicode(e), - cause=e) - except exceptions.NoNodeError: - raise coordination.MemberNotJoined(group_id, member_id) - except exceptions.ZookeeperError as e: - utils.raise_with_cause(tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - else: - member_info = { - 'capabilities': cls._loads(capabilities), - 'created_at': utils.millis_to_datetime(znode_stats.ctime), - 'updated_at': utils.millis_to_datetime(znode_stats.mtime) - } - return member_info - - def get_member_info(self, group_id, member_id): - member_path = self._path_member(group_id, member_id) - async_result = self._coord.get_async(member_path) - return ZooAsyncResult(async_result, - self._get_member_info_handler, - timeout_exception=self._timeout_exception, - group_id=group_id, member_id=self._member_id) - - def _get_groups_handler(self, async_result, timeout, timeout_exception): - try: - group_ids = async_result.get(block=True, timeout=timeout) - except timeout_exception as e: - utils.raise_with_cause(coordination.OperationTimedOut, - encodeutils.exception_to_unicode(e), - cause=e) - except exceptions.NoNodeError as e: - utils.raise_with_cause(tooz.ToozError, - "Tooz namespace '%s' has not" - " been created" % self._namespace, - cause=e) - except exceptions.ZookeeperError as e: - utils.raise_with_cause(tooz.ToozError, - encodeutils.exception_to_unicode(e), - cause=e) - else: - return set(g.encode('ascii') for g in group_ids) - - def get_groups(self): - tooz_namespace = self._paths_join("/", self._namespace) - async_result = self._coord.get_children_async(tooz_namespace) - return ZooAsyncResult(async_result, self._get_groups_handler, - timeout_exception=self._timeout_exception) - - def _path_group(self, group_id): - return self._paths_join("/", self._namespace, group_id) - - def _path_member(self, group_id, member_id): - return self._paths_join("/", self._namespace, group_id, member_id) - - @staticmethod - def _paths_join(arg, *more_args): - """Converts paths into a string (unicode).""" - args = [arg] - args.extend(more_args) - cleaned_args = [] - for arg in args: - if isinstance(arg, six.binary_type): - cleaned_args.append(arg.decode('ascii')) - else: - cleaned_args.append(arg) - return paths.join(*cleaned_args) - - def _make_client(self, parsed_url, options): - # Creates a kazoo client, - # See: https://github.com/python-zk/kazoo/blob/2.2.1/kazoo/client.py - # for what options a client takes... - if parsed_url.username and parsed_url.password: - username = parsed_url.username - password = parsed_url.password - - digest_auth = "%s:%s" % (username, password) - digest_acl = security.make_digest_acl(username, password, all=True) - default_acl = (digest_acl,) - auth_data = [('digest', digest_auth)] - else: - default_acl = None - auth_data = None - - maybe_hosts = [parsed_url.netloc] + list(options.get('hosts', [])) - hosts = list(compat_filter(None, maybe_hosts)) - if not hosts: - hosts = ['localhost:2181'] - randomize_hosts = options.get('randomize_hosts', True) - client_kwargs = { - 'hosts': ",".join(hosts), - 'timeout': float(options.get('timeout', self.timeout)), - 'connection_retry': options.get('connection_retry'), - 'command_retry': options.get('command_retry'), - 'randomize_hosts': strutils.bool_from_string(randomize_hosts), - 'auth_data': auth_data, - 'default_acl': default_acl, - } - handler_kind = options.get('handler') - if handler_kind: - try: - handler_cls = self.HANDLERS[handler_kind] - except KeyError: - raise ValueError("Unknown handler '%s' requested" - " valid handlers are %s" - % (handler_kind, - sorted(self.HANDLERS.keys()))) - client_kwargs['handler'] = handler_cls() - return client.KazooClient(**client_kwargs) - - def stand_down_group_leader(self, group_id): - if group_id in self._leader_locks: - self._leader_locks[group_id].release() - return True - return False - - def _get_group_leader_lock(self, group_id): - if group_id not in self._leader_locks: - self._leader_locks[group_id] = self._coord.Lock( - self._path_group(group_id) + "/leader", - self._member_id.decode('ascii')) - return self._leader_locks[group_id] - - def get_leader(self, group_id): - contenders = self._get_group_leader_lock(group_id).contenders() - if contenders and contenders[0]: - leader = contenders[0].encode('ascii') - else: - leader = None - return ZooAsyncResult(None, lambda *args: leader) - - def get_lock(self, name): - z_lock = self._coord.Lock( - self._paths_join(b"/", self._namespace, b"locks", name), - self._member_id.decode('ascii')) - return ZooKeeperLock(name, z_lock) - - def run_elect_coordinator(self): - for group_id in six.iterkeys(self._hooks_elected_leader): - leader_lock = self._get_group_leader_lock(group_id) - if leader_lock.is_acquired: - # Previously acquired/still leader, leave it be... - continue - if leader_lock.acquire(blocking=False): - # We are now leader for this group - self._hooks_elected_leader[group_id].run( - coordination.LeaderElected( - group_id, - self._member_id)) - - def run_watchers(self, timeout=None): - results = super(KazooDriver, self).run_watchers(timeout) - self.run_elect_coordinator() - return results - - -class ZooAsyncResult(coordination.CoordAsyncResult): - def __init__(self, kazoo_async_result, handler, **kwargs): - self._kazoo_async_result = kazoo_async_result - self._handler = handler - self._kwargs = kwargs - - def get(self, timeout=10): - return self._handler(self._kazoo_async_result, timeout, **self._kwargs) - - def done(self): - return self._kazoo_async_result.ready() diff --git a/tooz/hashring.py b/tooz/hashring.py deleted file mode 100644 index bfe8638..0000000 --- a/tooz/hashring.py +++ /dev/null @@ -1,142 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2016 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import bisect -import hashlib - -import six - -import tooz -from tooz import utils - - -class UnknownNode(tooz.ToozError): - """Node is unknown.""" - def __init__(self, node): - super(UnknownNode, self).__init__("Unknown node `%s'" % node) - self.node = node - - -class HashRing(object): - """Map objects onto nodes based on their consistent hash.""" - - DEFAULT_PARTITION_NUMBER = 2**5 - - def __init__(self, nodes, partitions=DEFAULT_PARTITION_NUMBER): - """Create a new hashring. - - :param nodes: List of nodes where objects will be mapped onto. - :param partitions: Number of partitions to spread objects onto. - """ - self.nodes = {} - self._ring = dict() - self._partitions = [] - self._partition_number = partitions - - self.add_nodes(set(nodes)) - - def add_node(self, node, weight=1): - """Add a node to the hashring. - - :param node: Node to add. - :param weight: How many resource instances this node should manage - compared to the other nodes (default 1). Higher weights will be - assigned more resources. Three nodes A, B and C with weights 1, 2 and 3 - will each handle 1/6, 1/3 and 1/2 of the resources, respectively. - - """ - return self.add_nodes((node,), weight) - - def add_nodes(self, nodes, weight=1): - """Add nodes to the hashring with equal weight - - :param nodes: Nodes to add. - :param weight: How many resource instances this node should manage - compared to the other nodes (default 1). Higher weights will be - assigned more resources. Three nodes A, B and C with weights 1, 2 and 3 - will each handle 1/6, 1/3 and 1/2 of the resources, respectively. - """ - for node in nodes: - key = utils.to_binary(node, 'utf-8') - key_hash = hashlib.md5(key) - for r in six.moves.range(self._partition_number * weight): - key_hash.update(key) - self._ring[self._hash2int(key_hash)] = node - - self.nodes[node] = weight - - self._partitions = sorted(self._ring.keys()) - - def remove_node(self, node): - """Remove a node from the hashring. - - Raises py:exc:`UnknownNode` - - :param node: Node to remove. - """ - try: - weight = self.nodes.pop(node) - except KeyError: - raise UnknownNode(node) - - key = utils.to_binary(node, 'utf-8') - key_hash = hashlib.md5(key) - for r in six.moves.range(self._partition_number * weight): - key_hash.update(key) - del self._ring[self._hash2int(key_hash)] - - self._partitions = sorted(self._ring.keys()) - - @staticmethod - def _hash2int(key): - return int(key.hexdigest(), 16) - - def _get_partition(self, data): - hashed_key = self._hash2int(hashlib.md5(data)) - position = bisect.bisect(self._partitions, hashed_key) - return position if position < len(self._partitions) else 0 - - def _get_node(self, partition): - return self._ring[self._partitions[partition]] - - def get_nodes(self, data, ignore_nodes=None, replicas=1): - """Get the set of nodes which the supplied data map onto. - - :param data: A byte identifier to be mapped across the ring. - :param ignore_nodes: Set of nodes to ignore. - :param replicas: Number of replicas to use. - :return: A set of nodes whose length depends on the number of replicas. - """ - partition = self._get_partition(data) - - ignore_nodes = set(ignore_nodes) if ignore_nodes else set() - candidates = set(self.nodes.keys()) - ignore_nodes - - replicas = min(replicas, len(candidates)) - - nodes = set() - while len(nodes) < replicas: - node = self._get_node(partition) - if node not in ignore_nodes: - nodes.add(node) - partition = (partition + 1 - if partition + 1 < len(self._partitions) else 0) - return nodes - - def __getitem__(self, key): - return self.get_nodes(key) - - def __len__(self): - return len(self._partitions) diff --git a/tooz/locking.py b/tooz/locking.py deleted file mode 100644 index 7d7ec81..0000000 --- a/tooz/locking.py +++ /dev/null @@ -1,109 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2014 eNovance Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import abc - -import six - -import tooz -from tooz import coordination - - -class _LockProxy(object): - def __init__(self, lock, *args, **kwargs): - self.lock = lock - self.args = args - self.kwargs = kwargs - - def __enter__(self): - return self.lock.__enter__(*self.args, **self.kwargs) - - def __exit__(self, exc_type, exc_val, exc_tb): - self.lock.__exit__(exc_type, exc_val, exc_tb) - - -@six.add_metaclass(abc.ABCMeta) -class Lock(object): - def __init__(self, name): - if not name: - raise ValueError("Locks must be provided a name") - self._name = name - - @property - def name(self): - return self._name - - def __call__(self, *args, **kwargs): - return _LockProxy(self, *args, **kwargs) - - def __enter__(self, *args, **kwargs): - acquired = self.acquire(*args, **kwargs) - if not acquired: - msg = u'Acquiring lock %s failed' % self.name - raise coordination.LockAcquireFailed(msg) - - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.release() - - def is_still_owner(self): - """Checks if the lock is still owned by the acquiree. - - :returns: returns true if still acquired (false if not) and - false if the lock was never acquired in the first place - or raises ``NotImplemented`` if not implemented. - """ - raise tooz.NotImplemented - - @abc.abstractmethod - def release(self): - """Attempts to release the lock, returns true if released. - - The behavior of releasing a lock which was not acquired in the first - place is undefined (it can range from harmless to releasing some other - users lock).. - - :returns: returns true if released (false if not) - :rtype: bool - """ - - def break_(self): - """Forcefully release the lock. - - This is mostly used for testing purposes, to simulate an out of - band operation that breaks the lock. Backends may allow waiters to - acquire immediately if a lock is broken, or they should raise an - exception. Releasing should be successful for objects that believe - they hold the lock but do not have the lock anymore. However, - they should be careful not to re-break the lock by releasing it, - since they may not be the holder anymore. - - :returns: returns true if forcefully broken (false if not) - or raises ``NotImplemented`` if not implemented. - """ - raise tooz.NotImplemented - - @abc.abstractmethod - def acquire(self, blocking=True): - """Attempts to acquire the lock. - - :param blocking: If True, blocks until the lock is acquired. If False, - returns right away. Otherwise, the value is used as a - timeout value and the call returns maximum after this - number of seconds. - :returns: returns true if acquired (false if not) - :rtype: bool - """ diff --git a/tooz/partitioner.py b/tooz/partitioner.py deleted file mode 100644 index dd5fb05..0000000 --- a/tooz/partitioner.py +++ /dev/null @@ -1,96 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2016 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from tooz import hashring - - -class Partitioner(object): - """Partition set of objects across several members. - - Objects to be partitioned should implement the __tooz_hash__ method to - identify themselves across the consistent hashring. This should method - return bytes. - - """ - - DEFAULT_PARTITION_NUMBER = hashring.HashRing.DEFAULT_PARTITION_NUMBER - - def __init__(self, coordinator, group_id, - partitions=DEFAULT_PARTITION_NUMBER): - members = coordinator.get_members(group_id) - self.partitions = partitions - self.group_id = group_id - self._coord = coordinator - caps = [(m, self._coord.get_member_capabilities(self.group_id, m)) - for m in members.get()] - self._coord.watch_join_group(self.group_id, self._on_member_join) - self._coord.watch_leave_group(self.group_id, self._on_member_leave) - self.ring = hashring.HashRing([], partitions=self.partitions) - for m_id, cap in caps: - self.ring.add_node(m_id, cap.get().get("weight", 1)) - - def _on_member_join(self, event): - weight = self._coord.get_member_capabilities( - self.group_id, event.member_id).get().get("weight", 1) - self.ring.add_node(event.member_id, weight) - - def _on_member_leave(self, event): - self.ring.remove_node(event.member_id) - - @staticmethod - def _hash_object(obj): - if hasattr(obj, "__tooz_hash__"): - return obj.__tooz_hash__() - return str(hash(obj)).encode('ascii') - - def members_for_object(self, obj, ignore_members=None, replicas=1): - """Return the members responsible for an object. - - :param obj: The object to check owning for. - :param member_id: The member to check if it owns the object. - :param ignore_members: Group members to ignore. - :param replicas: Number of replicas for the object. - """ - return self.ring.get_nodes(self._hash_object(obj), - ignore_nodes=ignore_members, - replicas=replicas) - - def belongs_to_member(self, obj, member_id, - ignore_members=None, replicas=1): - """Return whether an object belongs to a member. - - :param obj: The object to check owning for. - :param member_id: The member to check if it owns the object. - :param ignore_members: Group members to ignore. - :param replicas: Number of replicas for the object. - """ - return member_id in self.members_for_object( - obj, ignore_members=ignore_members, replicas=replicas) - - def belongs_to_self(self, obj, ignore_members=None, replicas=1): - """Return whether an object belongs to this coordinator. - - :param obj: The object to check owning for. - :param ignore_members: Group members to ignore. - :param replicas: Number of replicas for the object. - """ - return self.belongs_to_member(obj, self._coord._member_id, - ignore_members=ignore_members, - replicas=replicas) - - def stop(self): - """Stop the partitioner.""" - self._coord.unwatch_join_group(self.group_id, self._on_member_join) - self._coord.unwatch_leave_group(self.group_id, self._on_member_leave) diff --git a/tooz/tests/__init__.py b/tooz/tests/__init__.py deleted file mode 100644 index 505ef5f..0000000 --- a/tooz/tests/__init__.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2014 eNovance Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import functools -import os - -import fixtures -from oslo_utils import uuidutils -import six -from testtools import testcase - -import tooz - - -def get_random_uuid(): - return uuidutils.generate_uuid().encode('ascii') - - -def _skip_decorator(func): - @functools.wraps(func) - def skip_if_not_implemented(*args, **kwargs): - try: - return func(*args, **kwargs) - except tooz.NotImplemented as e: - raise testcase.TestSkipped(str(e)) - return skip_if_not_implemented - - -class SkipNotImplementedMeta(type): - def __new__(cls, name, bases, local): - for attr in local: - value = local[attr] - if callable(value) and ( - attr.startswith('test_') or attr == 'setUp'): - local[attr] = _skip_decorator(value) - return type.__new__(cls, name, bases, local) - - -@six.add_metaclass(SkipNotImplementedMeta) -class TestWithCoordinator(testcase.TestCase): - url = os.getenv("TOOZ_TEST_URL") - - def setUp(self): - super(TestWithCoordinator, self).setUp() - if self.url is None: - raise RuntimeError("No URL set for this driver") - if os.getenv("TOOZ_TEST_ETCD3"): - self.url = self.url.replace("etcd://", "etcd3://") - if os.getenv("TOOZ_TEST_ETCD3GW"): - self.url = self.url.replace("etcd://", "etcd3+http://") - self.useFixture(fixtures.NestedTempfile()) - self.group_id = get_random_uuid() - self.member_id = get_random_uuid() - self._coord = tooz.coordination.get_coordinator(self.url, - self.member_id) - self._coord.start() - - def tearDown(self): - self._coord.stop() - super(TestWithCoordinator, self).tearDown() diff --git a/tooz/tests/drivers/__init__.py b/tooz/tests/drivers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tooz/tests/drivers/test_file.py b/tooz/tests/drivers/test_file.py deleted file mode 100644 index 149c371..0000000 --- a/tooz/tests/drivers/test_file.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2016 Cloudbase Solutions Srl -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os - -import fixtures -import mock -from testtools import testcase - -import tooz -from tooz import coordination -from tooz import tests - - -class TestFileDriver(testcase.TestCase): - _FAKE_MEMBER_ID = tests.get_random_uuid() - - def test_base_dir(self): - file_path = '/fake/file/path' - url = 'file://%s' % file_path - - coord = coordination.get_coordinator(url, self._FAKE_MEMBER_ID) - self.assertEqual(file_path, coord._dir) - - def test_leftover_file(self): - fixture = self.useFixture(fixtures.TempDir()) - - file_path = fixture.path - url = 'file://%s' % file_path - - coord = coordination.get_coordinator(url, self._FAKE_MEMBER_ID) - coord.start() - self.addCleanup(coord.stop) - - coord.create_group(b"my_group").get() - safe_group_id = coord._make_filesystem_safe(b"my_group") - with open(os.path.join(file_path, 'groups', - safe_group_id, "junk.txt"), "wb"): - pass - os.unlink(os.path.join(file_path, 'groups', - safe_group_id, '.metadata')) - self.assertRaises(tooz.ToozError, - coord.delete_group(b"my_group").get) - - @mock.patch('os.path.normpath', lambda x: x.replace('/', '\\')) - @mock.patch('sys.platform', 'win32') - def test_base_dir_win32(self): - coord = coordination.get_coordinator( - 'file:///C:/path/', self._FAKE_MEMBER_ID) - self.assertEqual('C:\\path\\', coord._dir) - - coord = coordination.get_coordinator( - 'file:////share_addr/share_path/', self._FAKE_MEMBER_ID) - self.assertEqual('\\\\share_addr\\share_path\\', coord._dir) - - # Administrative shares should be handled properly. - coord = coordination.get_coordinator( - 'file:////c$/path/', self._FAKE_MEMBER_ID) - self.assertEqual('\\\\c$\\path\\', coord._dir) diff --git a/tooz/tests/test_coordination.py b/tooz/tests/test_coordination.py deleted file mode 100644 index d5db96d..0000000 --- a/tooz/tests/test_coordination.py +++ /dev/null @@ -1,1022 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013-2015 eNovance Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import threading -import time - -from concurrent import futures -import mock -from six.moves.urllib import parse -from testtools import matchers -from testtools import testcase - -import tooz -import tooz.coordination -from tooz import tests - - -def try_to_lock_job(name, coord, url, member_id): - if not coord: - coord = tooz.coordination.get_coordinator( - url, member_id) - coord.start() - lock2 = coord.get_lock(name) - return lock2.acquire(blocking=False) - - -class TestAPI(tests.TestWithCoordinator): - def assertRaisesAny(self, exc_classes, callable_obj, *args, **kwargs): - checkers = [matchers.MatchesException(exc_class) - for exc_class in exc_classes] - matcher = matchers.Raises(matchers.MatchesAny(*checkers)) - callable_obj = testcase.Nullary(callable_obj, *args, **kwargs) - self.assertThat(callable_obj, matcher) - - def test_connection_error_bad_host(self): - if (tooz.coordination.Characteristics.DISTRIBUTED_ACROSS_HOSTS - not in self._coord.CHARACTERISTICS): - self.skipTest("This driver is not distributed across hosts") - scheme = parse.urlparse(self.url).scheme - coord = tooz.coordination.get_coordinator( - "%s://localhost:1/f00" % scheme, - self.member_id) - self.assertRaises(tooz.coordination.ToozConnectionError, - coord.start) - - def test_stop_first(self): - c = tooz.coordination.get_coordinator(self.url, - self.member_id) - self.assertRaises(tooz.ToozError, - c.stop) - - def test_create_group(self): - self._coord.create_group(self.group_id).get() - all_group_ids = self._coord.get_groups().get() - self.assertIn(self.group_id, all_group_ids) - - def test_get_lock_release_broken(self): - name = tests.get_random_uuid() - memberid2 = tests.get_random_uuid() - coord2 = tooz.coordination.get_coordinator(self.url, - memberid2) - coord2.start() - lock1 = self._coord.get_lock(name) - lock2 = coord2.get_lock(name) - self.assertTrue(lock1.acquire(blocking=False)) - self.assertFalse(lock2.acquire(blocking=False)) - self.assertTrue(lock2.break_()) - self.assertTrue(lock2.acquire(blocking=False)) - self.assertFalse(lock1.release()) - # Assert lock is not accidentally broken now - memberid3 = tests.get_random_uuid() - coord3 = tooz.coordination.get_coordinator(self.url, - memberid3) - coord3.start() - lock3 = coord3.get_lock(name) - self.assertFalse(lock3.acquire(blocking=False)) - - def test_create_group_already_exist(self): - self._coord.create_group(self.group_id).get() - create_group = self._coord.create_group(self.group_id) - self.assertRaises(tooz.coordination.GroupAlreadyExist, - create_group.get) - - def test_get_groups(self): - groups_ids = [tests.get_random_uuid() for _ in range(0, 5)] - for group_id in groups_ids: - self._coord.create_group(group_id).get() - created_groups = self._coord.get_groups().get() - for group_id in groups_ids: - self.assertIn(group_id, created_groups) - - def test_delete_group(self): - self._coord.create_group(self.group_id).get() - all_group_ids = self._coord.get_groups().get() - self.assertIn(self.group_id, all_group_ids) - self._coord.delete_group(self.group_id).get() - all_group_ids = self._coord.get_groups().get() - self.assertNotIn(self.group_id, all_group_ids) - join_group = self._coord.join_group(self.group_id) - self.assertRaises(tooz.coordination.GroupNotCreated, - join_group.get) - - def test_delete_group_non_existent(self): - delete = self._coord.delete_group(self.group_id) - self.assertRaises(tooz.coordination.GroupNotCreated, - delete.get) - - def test_delete_group_non_empty(self): - self._coord.create_group(self.group_id).get() - self._coord.join_group(self.group_id).get() - delete = self._coord.delete_group(self.group_id) - self.assertRaises(tooz.coordination.GroupNotEmpty, - delete.get) - self._coord.leave_group(self.group_id).get() - self._coord.delete_group(self.group_id).get() - - def test_join_group(self): - self._coord.create_group(self.group_id).get() - self._coord.join_group(self.group_id).get() - member_list = self._coord.get_members(self.group_id).get() - self.assertIn(self.member_id, member_list) - - def test_join_nonexistent_group(self): - join_group = self._coord.join_group(self.group_id) - self.assertRaises(tooz.coordination.GroupNotCreated, - join_group.get) - - def test_join_group_create(self): - self._coord.join_group_create(self.group_id) - member_list = self._coord.get_members(self.group_id).get() - self.assertIn(self.member_id, member_list) - - def test_join_group_with_member_id_already_exists(self): - self._coord.create_group(self.group_id).get() - self._coord.join_group(self.group_id).get() - client = tooz.coordination.get_coordinator(self.url, - self.member_id) - client.start() - join_group = client.join_group(self.group_id) - self.assertRaises(tooz.coordination.MemberAlreadyExist, - join_group.get) - - def test_leave_group(self): - self._coord.create_group(self.group_id).get() - all_group_ids = self._coord.get_groups().get() - self.assertIn(self.group_id, all_group_ids) - self._coord.join_group(self.group_id).get() - member_list = self._coord.get_members(self.group_id).get() - self.assertIn(self.member_id, member_list) - member_ids = self._coord.get_members(self.group_id).get() - self.assertIn(self.member_id, member_ids) - self._coord.leave_group(self.group_id).get() - new_member_objects = self._coord.get_members(self.group_id).get() - new_member_list = [member.member_id for member in new_member_objects] - self.assertNotIn(self.member_id, new_member_list) - - def test_leave_nonexistent_group(self): - all_group_ids = self._coord.get_groups().get() - self.assertNotIn(self.group_id, all_group_ids) - leave_group = self._coord.leave_group(self.group_id) - # Drivers raise one of those depending on their capability - self.assertRaisesAny([tooz.coordination.MemberNotJoined, - tooz.coordination.GroupNotCreated], - leave_group.get) - - def test_leave_group_not_joined_by_member(self): - self._coord.create_group(self.group_id).get() - all_group_ids = self._coord.get_groups().get() - self.assertIn(self.group_id, all_group_ids) - leave_group = self._coord.leave_group(self.group_id) - self.assertRaises(tooz.coordination.MemberNotJoined, - leave_group.get) - - def test_get_lock_twice_locked_one_released_two(self): - name = tests.get_random_uuid() - lock1 = self._coord.get_lock(name) - lock2 = self._coord.get_lock(name) - self.assertTrue(lock1.acquire()) - self.assertFalse(lock2.acquire(blocking=False)) - self.assertFalse(lock2.release()) - self.assertTrue(lock1.release()) - self.assertFalse(lock2.release()) - - def test_get_members(self): - group_id_test2 = tests.get_random_uuid() - member_id_test2 = tests.get_random_uuid() - client2 = tooz.coordination.get_coordinator(self.url, - member_id_test2) - client2.start() - - self._coord.create_group(group_id_test2).get() - self._coord.join_group(group_id_test2).get() - client2.join_group(group_id_test2).get() - members_ids = self._coord.get_members(group_id_test2).get() - self.assertEqual({self.member_id, member_id_test2}, members_ids) - - def test_get_member_capabilities(self): - self._coord.create_group(self.group_id).get() - self._coord.join_group(self.group_id, b"test_capabilities") - - capa = self._coord.get_member_capabilities(self.group_id, - self.member_id).get() - self.assertEqual(capa, b"test_capabilities") - - def test_get_member_capabilities_complex(self): - self._coord.create_group(self.group_id).get() - caps = { - 'type': 'warrior', - 'abilities': ['fight', 'flight', 'double-hit-damage'], - } - self._coord.join_group(self.group_id, caps).get() - capa = self._coord.get_member_capabilities(self.group_id, - self.member_id).get() - self.assertEqual(capa, caps) - self.assertEqual(capa['type'], caps['type']) - - def test_get_member_capabilities_nonexistent_group(self): - capa = self._coord.get_member_capabilities(self.group_id, - self.member_id) - # Drivers raise one of those depending on their capability - self.assertRaisesAny([tooz.coordination.MemberNotJoined, - tooz.coordination.GroupNotCreated], - capa.get) - - def test_get_member_capabilities_nonjoined_member(self): - self._coord.create_group(self.group_id).get() - capa = self._coord.get_member_capabilities(self.group_id, - self.member_id) - self.assertRaises(tooz.coordination.MemberNotJoined, - capa.get) - - def test_get_member_info(self): - self._coord.create_group(self.group_id).get() - self._coord.join_group(self.group_id, b"test_capabilities").get() - - member_info = self._coord.get_member_info(self.group_id, - self.member_id).get() - self.assertEqual(member_info['capabilities'], b"test_capabilities") - - def test_get_member_info_complex(self): - self._coord.create_group(self.group_id).get() - caps = { - 'type': 'warrior', - 'abilities': ['fight', 'flight', 'double-hit-damage'], - } - member_info = {'capabilities': 'caps', - 'created_at': '0', - 'updated_at': '0'} - self._coord.join_group(self.group_id, caps).get() - member_info = self._coord.get_member_info(self.group_id, - self.member_id).get() - self.assertEqual(member_info['capabilities'], caps) - - def test_get_member_info_nonexistent_group(self): - member_info = self._coord.get_member_info(self.group_id, - self.member_id) - # Drivers raise one of those depending on their capability - self.assertRaisesAny([tooz.coordination.MemberNotJoined, - tooz.coordination.GroupNotCreated], - member_info.get) - - def test_get_member_info_nonjoined_member(self): - self._coord.create_group(self.group_id).get() - member_id = tests.get_random_uuid() - member_info = self._coord.get_member_info(self.group_id, - member_id) - self.assertRaises(tooz.coordination.MemberNotJoined, - member_info.get) - - def test_update_capabilities(self): - self._coord.create_group(self.group_id).get() - self._coord.join_group(self.group_id, b"test_capabilities1").get() - - capa = self._coord.get_member_capabilities(self.group_id, - self.member_id).get() - self.assertEqual(capa, b"test_capabilities1") - self._coord.update_capabilities(self.group_id, - b"test_capabilities2").get() - - capa2 = self._coord.get_member_capabilities(self.group_id, - self.member_id).get() - self.assertEqual(capa2, b"test_capabilities2") - - def test_update_capabilities_with_group_id_nonexistent(self): - update_cap = self._coord.update_capabilities(self.group_id, - b'test_capabilities') - # Drivers raise one of those depending on their capability - self.assertRaisesAny([tooz.coordination.MemberNotJoined, - tooz.coordination.GroupNotCreated], - update_cap.get) - - def test_heartbeat(self): - if not self._coord.requires_beating: - raise testcase.TestSkipped("Test not applicable (heartbeating" - " not required)") - self._coord.heartbeat() - - def test_heartbeat_loop(self): - if not self._coord.requires_beating: - raise testcase.TestSkipped("Test not applicable (heartbeating" - " not required)") - - heart = self._coord.heart - self.assertFalse(heart.is_alive()) - heart.start() - - # This will timeout if nothing ever is done... - try: - while not heart.beats: - time.sleep(1) - finally: - heart.stop() - heart.wait() - - def test_disconnect_leave_group(self): - member_id_test2 = tests.get_random_uuid() - client2 = tooz.coordination.get_coordinator(self.url, - member_id_test2) - client2.start() - self._coord.create_group(self.group_id).get() - self._coord.join_group(self.group_id).get() - client2.join_group(self.group_id).get() - members_ids = self._coord.get_members(self.group_id).get() - self.assertIn(self.member_id, members_ids) - self.assertIn(member_id_test2, members_ids) - client2.stop() - members_ids = self._coord.get_members(self.group_id).get() - self.assertIn(self.member_id, members_ids) - self.assertNotIn(member_id_test2, members_ids) - - def test_timeout(self): - if (tooz.coordination.Characteristics.NON_TIMEOUT_BASED - in self._coord.CHARACTERISTICS): - self.skipTest("This driver is not based on timeout") - self._coord.stop() - if "?" in self.url: - sep = "&" - else: - sep = "?" - url = self.url + sep + "timeout=5" - self._coord = tooz.coordination.get_coordinator(url, self.member_id) - self._coord.start() - - member_id_test2 = tests.get_random_uuid() - client2 = tooz.coordination.get_coordinator(url, member_id_test2) - client2.start() - self._coord.create_group(self.group_id).get() - self._coord.join_group(self.group_id).get() - client2.join_group(self.group_id).get() - members_ids = self._coord.get_members(self.group_id).get() - self.assertIn(self.member_id, members_ids) - self.assertIn(member_id_test2, members_ids) - - # Watch the group, we want to be sure that when client2 is kicked out - # we get an event. - self._coord.watch_leave_group(self.group_id, self._set_event) - - # Run watchers to be sure we initialize the member cache and we *know* - # client2 is a member now - self._coord.run_watchers() - - time.sleep(3) - self._coord.heartbeat() - time.sleep(3) - - # Now client2 has timed out! - - members_ids = self._coord.get_members(self.group_id).get() - while True: - if self._coord.run_watchers(): - break - self.assertIn(self.member_id, members_ids) - self.assertNotIn(member_id_test2, members_ids) - # Check that the event has been triggered - self.assertEqual(1, len(self.events)) - event = self.events[0] - self.assertIsInstance(event, tooz.coordination.MemberLeftGroup) - self.assertEqual(member_id_test2, event.member_id) - self.assertEqual(self.group_id, event.group_id) - - def _set_event(self, event): - if not hasattr(self, "events"): - self.events = [event] - else: - self.events.append(event) - return 42 - - def test_watch_group_join(self): - member_id_test2 = tests.get_random_uuid() - client2 = tooz.coordination.get_coordinator(self.url, - member_id_test2) - client2.start() - self._coord.create_group(self.group_id).get() - - # Watch the group - self._coord.watch_join_group(self.group_id, self._set_event) - - # Join the group - client2.join_group(self.group_id).get() - members_ids = self._coord.get_members(self.group_id).get() - self.assertIn(member_id_test2, members_ids) - while True: - if self._coord.run_watchers(): - break - self.assertEqual(1, len(self.events)) - event = self.events[0] - self.assertIsInstance(event, tooz.coordination.MemberJoinedGroup) - self.assertEqual(member_id_test2, event.member_id) - self.assertEqual(self.group_id, event.group_id) - - # Stop watching - self._coord.unwatch_join_group(self.group_id, self._set_event) - self.events = [] - - # Leave and rejoin group - client2.leave_group(self.group_id).get() - client2.join_group(self.group_id).get() - self._coord.run_watchers() - self.assertEqual([], self.events) - - def test_watch_leave_group(self): - member_id_test2 = tests.get_random_uuid() - client2 = tooz.coordination.get_coordinator(self.url, - member_id_test2) - client2.start() - self._coord.create_group(self.group_id).get() - - # Watch the group: this can leads to race conditions in certain - # driver that are not able to see all events, so we join, wait for - # the join to be seen, and then we leave, and wait for the leave to - # be seen. - self._coord.watch_join_group(self.group_id, lambda children: True) - self._coord.watch_leave_group(self.group_id, self._set_event) - - # Join and leave the group - client2.join_group(self.group_id).get() - # Consumes join event - while True: - if self._coord.run_watchers(): - break - client2.leave_group(self.group_id).get() - # Consumes leave event - while True: - if self._coord.run_watchers(): - break - - self.assertEqual(1, len(self.events)) - event = self.events[0] - self.assertIsInstance(event, tooz.coordination.MemberLeftGroup) - self.assertEqual(member_id_test2, event.member_id) - self.assertEqual(self.group_id, event.group_id) - - # Stop watching - self._coord.unwatch_leave_group(self.group_id, self._set_event) - self.events = [] - - # Rejoin and releave group - client2.join_group(self.group_id).get() - client2.leave_group(self.group_id).get() - self._coord.run_watchers() - self.assertEqual([], self.events) - - def test_watch_join_group_disappear(self): - if not hasattr(self._coord, '_destroy_group'): - self.skipTest("This test only works with coordinators" - " that have the ability to destroy groups.") - - self._coord.create_group(self.group_id).get() - self._coord.watch_join_group(self.group_id, self._set_event) - self._coord.watch_leave_group(self.group_id, self._set_event) - - member_id_test2 = tests.get_random_uuid() - client2 = tooz.coordination.get_coordinator(self.url, - member_id_test2) - client2.start() - client2.join_group(self.group_id).get() - - while True: - if self._coord.run_watchers(): - break - self.assertEqual(1, len(self.events)) - event = self.events[0] - self.assertIsInstance(event, tooz.coordination.MemberJoinedGroup) - self.events = [] - - # Force the group to disappear... - self._coord._destroy_group(self.group_id) - - while True: - if self._coord.run_watchers(): - break - - self.assertEqual(1, len(self.events)) - event = self.events[0] - self.assertIsInstance(event, tooz.coordination.MemberLeftGroup) - - def test_watch_join_group_non_existent(self): - self.assertRaises(tooz.coordination.GroupNotCreated, - self._coord.watch_join_group, - self.group_id, - lambda: None) - self.assertEqual(0, len(self._coord._hooks_join_group[self.group_id])) - - def test_watch_join_group_booted_out(self): - self._coord.create_group(self.group_id).get() - self._coord.join_group(self.group_id).get() - self._coord.watch_join_group(self.group_id, self._set_event) - self._coord.watch_leave_group(self.group_id, self._set_event) - - member_id_test2 = tests.get_random_uuid() - client2 = tooz.coordination.get_coordinator(self.url, - member_id_test2) - client2.start() - client2.join_group(self.group_id).get() - - while True: - if self._coord.run_watchers(): - break - - client3 = tooz.coordination.get_coordinator(self.url, self.member_id) - client3.start() - client3.leave_group(self.group_id).get() - - # Only works for clients that have access to the groups they are part - # of, to ensure that after we got booted out by client3 that this - # client now no longer believes its part of the group. - if (hasattr(self._coord, '_joined_groups') - and (self._coord.run_watchers - == tooz.coordination.CoordinationDriverCachedRunWatchers.run_watchers)): # noqa - self.assertIn(self.group_id, self._coord._joined_groups) - self._coord.run_watchers() - self.assertNotIn(self.group_id, self._coord._joined_groups) - - def test_watch_leave_group_non_existent(self): - self.assertRaises(tooz.coordination.GroupNotCreated, - self._coord.watch_leave_group, - self.group_id, - lambda: None) - self.assertEqual(0, len(self._coord._hooks_leave_group[self.group_id])) - - def test_run_for_election(self): - self._coord.create_group(self.group_id).get() - self._coord.watch_elected_as_leader(self.group_id, self._set_event) - self._coord.run_watchers() - - self.assertEqual(1, len(self.events)) - event = self.events[0] - self.assertIsInstance(event, tooz.coordination.LeaderElected) - self.assertEqual(self.member_id, event.member_id) - self.assertEqual(self.group_id, event.group_id) - - def test_run_for_election_multiple_clients(self): - self._coord.create_group(self.group_id).get() - self._coord.watch_elected_as_leader(self.group_id, self._set_event) - self._coord.run_watchers() - - member_id_test2 = tests.get_random_uuid() - client2 = tooz.coordination.get_coordinator(self.url, - member_id_test2) - client2.start() - client2.watch_elected_as_leader(self.group_id, self._set_event) - client2.run_watchers() - - self.assertEqual(1, len(self.events)) - event = self.events[0] - self.assertIsInstance(event, tooz.coordination.LeaderElected) - self.assertEqual(self.member_id, event.member_id) - self.assertEqual(self.group_id, event.group_id) - self.assertEqual(self._coord.get_leader(self.group_id).get(), - self.member_id) - - self.events = [] - - self._coord.stop() - client2.run_watchers() - - self.assertEqual(1, len(self.events)) - event = self.events[0] - - self.assertIsInstance(event, tooz.coordination.LeaderElected) - self.assertEqual(member_id_test2, - event.member_id) - self.assertEqual(self.group_id, event.group_id) - self.assertEqual(client2.get_leader(self.group_id).get(), - member_id_test2) - - # Restart the coord because tearDown stops it - self._coord.start() - - def test_get_leader(self): - self._coord.create_group(self.group_id).get() - - leader = self._coord.get_leader(self.group_id).get() - self.assertIsNone(leader) - - self._coord.join_group(self.group_id).get() - - leader = self._coord.get_leader(self.group_id).get() - self.assertIsNone(leader) - - # Let's get elected - self._coord.watch_elected_as_leader(self.group_id, self._set_event) - self._coord.run_watchers() - - leader = self._coord.get_leader(self.group_id).get() - self.assertEqual(leader, self.member_id) - - def test_run_for_election_multiple_clients_stand_down(self): - self._coord.create_group(self.group_id).get() - self._coord.watch_elected_as_leader(self.group_id, self._set_event) - self._coord.run_watchers() - - member_id_test2 = tests.get_random_uuid() - client2 = tooz.coordination.get_coordinator(self.url, - member_id_test2) - client2.start() - client2.watch_elected_as_leader(self.group_id, self._set_event) - client2.run_watchers() - - self.assertEqual(1, len(self.events)) - event = self.events[0] - - self.assertIsInstance(event, tooz.coordination.LeaderElected) - self.assertEqual(self.member_id, event.member_id) - self.assertEqual(self.group_id, event.group_id) - - self.events = [] - - self._coord.stand_down_group_leader(self.group_id) - client2.run_watchers() - - self.assertEqual(1, len(self.events)) - event = self.events[0] - - self.assertIsInstance(event, tooz.coordination.LeaderElected) - self.assertEqual(member_id_test2, event.member_id) - self.assertEqual(self.group_id, event.group_id) - - self.events = [] - - client2.stand_down_group_leader(self.group_id) - self._coord.run_watchers() - - self.assertEqual(1, len(self.events)) - event = self.events[0] - - self.assertIsInstance(event, tooz.coordination.LeaderElected) - self.assertEqual(self.member_id, event.member_id) - self.assertEqual(self.group_id, event.group_id) - - def test_unwatch_elected_as_leader(self): - # Create a group and add a elected_as_leader callback - self._coord.create_group(self.group_id).get() - self._coord.watch_elected_as_leader(self.group_id, self._set_event) - - # Ensure exactly one leader election hook exists - self.assertEqual(1, - len(self._coord._hooks_elected_leader[self.group_id])) - - # Unwatch, and ensure no leader election hooks exist - self._coord.unwatch_elected_as_leader(self.group_id, self._set_event) - self.assertEqual(0, len(self._coord._hooks_elected_leader)) - - def test_unwatch_elected_as_leader_callback_not_found(self): - self._coord.create_group(self.group_id).get() - self.assertRaises(tooz.coordination.WatchCallbackNotFound, - self._coord.unwatch_elected_as_leader, - self.group_id, lambda x: None) - - def test_unwatch_join_group_callback_not_found(self): - self._coord.create_group(self.group_id).get() - self.assertRaises(tooz.coordination.WatchCallbackNotFound, - self._coord.unwatch_join_group, - self.group_id, lambda x: None) - - def test_unwatch_leave_group(self): - # Create a group and add a leave_group callback - self._coord.create_group(self.group_id).get() - self.assertEqual(0, len(self._coord._hooks_leave_group)) - self._coord.watch_leave_group(self.group_id, self._set_event) - - # Ensure exactly one leave group hook exists - self.assertEqual(1, len(self._coord._hooks_leave_group[self.group_id])) - - # Unwatch, and ensure no leave group hooks exist - self._coord.unwatch_leave_group(self.group_id, self._set_event) - self.assertEqual(0, len(self._coord._hooks_leave_group)) - - def test_unwatch_leave_group_callback_not_found(self): - self._coord.create_group(self.group_id).get() - self.assertRaises(tooz.coordination.WatchCallbackNotFound, - self._coord.unwatch_leave_group, - self.group_id, lambda x: None) - - def test_get_lock(self): - lock = self._coord.get_lock(tests.get_random_uuid()) - self.assertTrue(lock.acquire()) - self.assertTrue(lock.release()) - with lock: - pass - - def test_heartbeat_lock_not_acquired(self): - lock = self._coord.get_lock(tests.get_random_uuid()) - # Not all locks need heartbeat - if hasattr(lock, "heartbeat"): - self.assertFalse(lock.heartbeat()) - - def test_get_shared_lock(self): - lock = self._coord.get_lock(tests.get_random_uuid()) - self.assertTrue(lock.acquire(shared=True)) - self.assertTrue(lock.release()) - with lock(shared=True): - pass - - def test_get_shared_lock_locking_same_lock_twice(self): - lock = self._coord.get_lock(tests.get_random_uuid()) - self.assertTrue(lock.acquire(shared=True)) - self.assertTrue(lock.acquire(shared=True)) - self.assertTrue(lock.release()) - self.assertTrue(lock.release()) - self.assertFalse(lock.release()) - with lock(shared=True): - pass - - def test_get_shared_lock_locking_two_lock(self): - name = tests.get_random_uuid() - lock1 = self._coord.get_lock(name) - coord = tooz.coordination.get_coordinator( - self.url, tests.get_random_uuid()) - coord.start() - lock2 = coord.get_lock(name) - - self.assertTrue(lock1.acquire(shared=True)) - self.assertTrue(lock2.acquire(shared=True)) - self.assertTrue(lock1.release()) - self.assertTrue(lock2.release()) - - def test_get_lock_locking_shared_and_exclusive(self): - name = tests.get_random_uuid() - lock1 = self._coord.get_lock(name) - coord = tooz.coordination.get_coordinator( - self.url, tests.get_random_uuid()) - coord.start() - lock2 = coord.get_lock(name) - - self.assertTrue(lock1.acquire(shared=True)) - self.assertFalse(lock2.acquire(blocking=False)) - self.assertTrue(lock1.release()) - self.assertFalse(lock2.release()) - - def test_get_lock_locking_exclusive_and_shared(self): - name = tests.get_random_uuid() - lock1 = self._coord.get_lock(name) - coord = tooz.coordination.get_coordinator( - self.url, tests.get_random_uuid()) - coord.start() - lock2 = coord.get_lock(name) - - self.assertTrue(lock1.acquire()) - self.assertFalse(lock2.acquire(shared=True, blocking=False)) - self.assertTrue(lock1.release()) - self.assertFalse(lock2.release()) - - def test_get_lock_concurrency_locking_same_lock(self): - lock = self._coord.get_lock(tests.get_random_uuid()) - - graceful_ending = threading.Event() - - def thread(): - self.assertTrue(lock.acquire()) - self.assertTrue(lock.release()) - graceful_ending.set() - - t = threading.Thread(target=thread) - t.daemon = True - with lock: - t.start() - # Ensure the thread try to get the lock - time.sleep(.1) - t.join() - graceful_ending.wait(.2) - self.assertTrue(graceful_ending.is_set()) - - def _do_test_get_lock_concurrency_locking_two_lock(self, executor, - use_same_coord): - name = tests.get_random_uuid() - lock1 = self._coord.get_lock(name) - with lock1: - with executor(max_workers=1) as e: - coord = self._coord if use_same_coord else None - f = e.submit(try_to_lock_job, name, coord, self.url, - tests.get_random_uuid()) - self.assertFalse(f.result()) - - def _do_test_get_lock_serial_locking_two_lock(self, executor, - use_same_coord): - name = tests.get_random_uuid() - lock1 = self._coord.get_lock(name) - lock1.acquire() - lock1.release() - with executor(max_workers=1) as e: - coord = self._coord if use_same_coord else None - f = e.submit(try_to_lock_job, name, coord, self.url, - tests.get_random_uuid()) - self.assertTrue(f.result()) - - def test_get_lock_concurrency_locking_two_lock_process(self): - # NOTE(jd) Using gRPC and forking is not supported so this test might - # very likely hang forever or crash. See - # https://github.com/grpc/grpc/issues/10140#issuecomment-297548714 for - # more info. - if self.url.startswith("etcd3://"): - self.skipTest("Unable to use etcd3 with fork()") - self._do_test_get_lock_concurrency_locking_two_lock( - futures.ProcessPoolExecutor, False) - - def test_get_lock_serial_locking_two_lock_process(self): - # NOTE(jd) Using gRPC and forking is not supported so this test might - # very likely hang forever or crash. See - # https://github.com/grpc/grpc/issues/10140#issuecomment-297548714 for - # more info. - if self.url.startswith("etcd3://"): - self.skipTest("Unable to use etcd3 with fork()") - self._do_test_get_lock_serial_locking_two_lock( - futures.ProcessPoolExecutor, False) - - def test_get_lock_concurrency_locking_two_lock_thread1(self): - self._do_test_get_lock_concurrency_locking_two_lock( - futures.ThreadPoolExecutor, False) - - def test_get_lock_concurrency_locking_two_lock_thread2(self): - self._do_test_get_lock_concurrency_locking_two_lock( - futures.ThreadPoolExecutor, True) - - def test_get_lock_concurrency_locking2(self): - # NOTE(sileht): some database based lock can have only - # one lock per connection, this test ensures acquiring a - # second lock doesn't release the first one. - lock1 = self._coord.get_lock(tests.get_random_uuid()) - lock2 = self._coord.get_lock(tests.get_random_uuid()) - - graceful_ending = threading.Event() - thread_locked = threading.Event() - - def thread(): - with lock2: - try: - self.assertFalse(lock1.acquire(blocking=False)) - except tooz.NotImplemented: - pass - thread_locked.set() - graceful_ending.set() - - t = threading.Thread(target=thread) - t.daemon = True - - with lock1: - t.start() - thread_locked.wait() - self.assertTrue(thread_locked.is_set()) - t.join() - graceful_ending.wait() - self.assertTrue(graceful_ending.is_set()) - - def test_get_lock_twice_locked_twice(self): - name = tests.get_random_uuid() - lock1 = self._coord.get_lock(name) - lock2 = self._coord.get_lock(name) - with lock1: - self.assertFalse(lock2.acquire(blocking=False)) - - def test_get_lock_context_fails(self): - name = tests.get_random_uuid() - lock1 = self._coord.get_lock(name) - lock2 = self._coord.get_lock(name) - with mock.patch.object(lock2, 'acquire', return_value=False): - with lock1: - self.assertRaises( - tooz.coordination.LockAcquireFailed, - lock2.__enter__) - - def test_get_lock_context_check_value(self): - name = tests.get_random_uuid() - lock = self._coord.get_lock(name) - with lock as returned_lock: - self.assertEqual(lock, returned_lock) - - def test_lock_context_manager_acquire_no_argument(self): - name = tests.get_random_uuid() - lock1 = self._coord.get_lock(name) - lock2 = self._coord.get_lock(name) - with lock1(): - self.assertFalse(lock2.acquire(blocking=False)) - - def test_lock_context_manager_acquire_argument_return_value(self): - name = tests.get_random_uuid() - blocking_value = 10.12 - lock = self._coord.get_lock(name) - with lock(blocking_value) as returned_lock: - self.assertEqual(lock, returned_lock) - - def test_lock_context_manager_acquire_argument_release_within(self): - name = tests.get_random_uuid() - blocking_value = 10.12 - lock = self._coord.get_lock(name) - with lock(blocking_value) as returned_lock: - self.assertTrue(returned_lock.release()) - - def test_lock_context_manager_acquire_argument(self): - name = tests.get_random_uuid() - blocking_value = 10.12 - lock = self._coord.get_lock(name) - with mock.patch.object(lock, 'acquire', wraps=True, autospec=True) as \ - mock_acquire: - with lock(blocking_value): - mock_acquire.assert_called_once_with(blocking_value) - - def test_lock_context_manager_acquire_argument_timeout(self): - name = tests.get_random_uuid() - lock1 = self._coord.get_lock(name) - lock2 = self._coord.get_lock(name) - with lock1: - try: - with lock2(False): - self.fail('Lock acquire should have failed') - except tooz.coordination.LockAcquireFailed: - pass - - def test_get_lock_locked_twice(self): - name = tests.get_random_uuid() - lock = self._coord.get_lock(name) - with lock: - self.assertFalse(lock.acquire(blocking=False)) - - def test_get_multiple_locks_with_same_coord(self): - name = tests.get_random_uuid() - lock1 = self._coord.get_lock(name) - lock2 = self._coord.get_lock(name) - self.assertTrue(lock1.acquire()) - self.assertFalse(lock2.acquire(blocking=False)) - self.assertFalse(self._coord.get_lock(name).acquire(blocking=False)) - self.assertTrue(lock1.release()) - - def test_ensure_acquire_release_return(self): - name = tests.get_random_uuid() - lock1 = self._coord.get_lock(name) - self.assertTrue(lock1.acquire()) - self.assertTrue(lock1.release()) - self.assertFalse(lock1.release()) - - def test_get_lock_multiple_coords(self): - member_id2 = tests.get_random_uuid() - client2 = tooz.coordination.get_coordinator(self.url, - member_id2) - client2.start() - - lock_name = tests.get_random_uuid() - lock = self._coord.get_lock(lock_name) - self.assertTrue(lock.acquire()) - - lock2 = client2.get_lock(lock_name) - self.assertFalse(lock2.acquire(blocking=False)) - self.assertTrue(lock.release()) - self.assertTrue(lock2.acquire(blocking=True)) - self.assertTrue(lock2.release()) - - def test_get_started_status(self): - self.assertTrue(self._coord.is_started) - self._coord.stop() - self.assertFalse(self._coord.is_started) - self._coord.start() - - def do_test_name_property(self): - name = tests.get_random_uuid() - lock = self._coord.get_lock(name) - self.assertEqual(name, lock.name) - - def test_acquire_twice_no_deadlock_releasing(self): - name = tests.get_random_uuid() - lock = self._coord.get_lock(name) - self.assertTrue(lock.acquire(blocking=False)) - self.assertFalse(lock.acquire(blocking=False)) - self.assertTrue(lock.release()) - - -class TestHook(testcase.TestCase): - def setUp(self): - super(TestHook, self).setUp() - self.hooks = tooz.coordination.Hooks() - self.triggered = False - - def _trigger(self): - self.triggered = True - - def test_register_hook(self): - self.assertEqual(self.hooks.run(), []) - self.assertFalse(self.triggered) - self.hooks.append(self._trigger) - self.assertEqual(self.hooks.run(), [None]) - self.assertTrue(self.triggered) - - def test_unregister_hook(self): - self.hooks.append(self._trigger) - self.assertEqual(self.hooks.run(), [None]) - self.assertTrue(self.triggered) - self.triggered = False - self.hooks.remove(self._trigger) - self.assertEqual(self.hooks.run(), []) - self.assertFalse(self.triggered) diff --git a/tooz/tests/test_etcd.py b/tooz/tests/test_etcd.py deleted file mode 100644 index bbc99e5..0000000 --- a/tooz/tests/test_etcd.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2016 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import mock -from testtools import testcase - -import tooz.coordination - - -class TestEtcd(testcase.TestCase): - FAKE_URL = "etcd://mocked-not-really-localhost:2379" - FAKE_MEMBER_ID = "mocked-not-really-member" - - def setUp(self): - super(TestEtcd, self).setUp() - self._coord = tooz.coordination.get_coordinator(self.FAKE_URL, - self.FAKE_MEMBER_ID) - - def test_multiple_locks_etcd_wait_index(self): - lock = self._coord.get_lock('mocked-not-really-random') - - return_values = [ - {'errorCode': {}, 'node': {}, 'index': 10}, - {'errorCode': None, 'node': {}, 'index': 10} - ] - with mock.patch.object(lock.client, 'put', side_effect=return_values): - with mock.patch.object(lock.client, 'get') as mocked_get: - self.assertTrue(lock.acquire()) - mocked_get.assert_called_once() - call = str(mocked_get.call_args) - self.assertIn("waitIndex=11", call) diff --git a/tooz/tests/test_hashring.py b/tooz/tests/test_hashring.py deleted file mode 100644 index c1407bd..0000000 --- a/tooz/tests/test_hashring.py +++ /dev/null @@ -1,243 +0,0 @@ -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import hashlib - -import mock -from testtools import matchers -from testtools import testcase - -from tooz import hashring - - -class HashRingTestCase(testcase.TestCase): - - # NOTE(deva): the mapping used in these tests is as follows: - # if nodes = [foo, bar]: - # fake -> foo, bar - # if nodes = [foo, bar, baz]: - # fake -> foo, bar, baz - # fake-again -> bar, baz, foo - - @mock.patch.object(hashlib, 'md5', autospec=True) - def test_hash2int_returns_int(self, mock_md5): - r1 = 32 * 'a' - r2 = 32 * 'b' - # 2**PARTITION_EXPONENT calls to md5.update per node - # PARTITION_EXPONENT is currently always 5, so 32 calls each here - mock_md5.return_value.hexdigest.side_effect = [r1] * 32 + [r2] * 32 - - nodes = ['foo', 'bar'] - ring = hashring.HashRing(nodes) - - self.assertIn(int(r1, 16), ring._ring) - self.assertIn(int(r2, 16), ring._ring) - - def test_create_ring(self): - nodes = {'foo', 'bar'} - ring = hashring.HashRing(nodes) - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * 2, len(ring)) - - def test_add_node(self): - nodes = {'foo', 'bar'} - ring = hashring.HashRing(nodes) - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * len(nodes), len(ring)) - nodes.add('baz') - ring.add_node('baz') - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * len(nodes), len(ring)) - - def test_add_node_bytes(self): - nodes = {'foo', 'bar'} - ring = hashring.HashRing(nodes) - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * len(nodes), len(ring)) - nodes.add(b'Z\xe2\xfa\x90\x17EC\xac\xae\x88\xa7[\xa1}:E') - ring.add_node(b'Z\xe2\xfa\x90\x17EC\xac\xae\x88\xa7[\xa1}:E') - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * len(nodes), len(ring)) - - def test_add_node_unicode(self): - nodes = {'foo', 'bar'} - ring = hashring.HashRing(nodes) - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * len(nodes), len(ring)) - nodes.add(u'\u0634\u0628\u06a9\u0647') - ring.add_node(u'\u0634\u0628\u06a9\u0647') - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * len(nodes), len(ring)) - - def test_add_node_weight(self): - nodes = {'foo', 'bar'} - ring = hashring.HashRing(nodes) - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * len(nodes), len(ring)) - nodes.add('baz') - ring.add_node('baz', weight=10) - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * 12, len(ring)) - - def test_add_nodes_weight(self): - nodes = {'foo', 'bar'} - ring = hashring.HashRing(nodes) - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * len(nodes), len(ring)) - nodes.add('baz') - nodes.add('baz2') - ring.add_nodes(set(['baz', 'baz2']), weight=10) - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * 22, len(ring)) - - def test_remove_node(self): - nodes = {'foo', 'bar'} - ring = hashring.HashRing(nodes) - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * len(nodes), len(ring)) - nodes.discard('bar') - ring.remove_node('bar') - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * len(nodes), len(ring)) - - def test_remove_node_bytes(self): - nodes = {'foo', b'Z\xe2\xfa\x90\x17EC\xac\xae\x88\xa7[\xa1}:E'} - ring = hashring.HashRing(nodes) - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * len(nodes), len(ring)) - nodes.discard(b'Z\xe2\xfa\x90\x17EC\xac\xae\x88\xa7[\xa1}:E') - ring.remove_node(b'Z\xe2\xfa\x90\x17EC\xac\xae\x88\xa7[\xa1}:E') - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * len(nodes), len(ring)) - - def test_remove_node_unknown(self): - nodes = ['foo', 'bar'] - ring = hashring.HashRing(nodes) - self.assertRaises( - hashring.UnknownNode, - ring.remove_node, 'biz') - - def test_add_then_removenode(self): - nodes = {'foo', 'bar'} - ring = hashring.HashRing(nodes) - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * len(nodes), len(ring)) - nodes.add('baz') - ring.add_node('baz') - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * len(nodes), len(ring)) - nodes.discard('bar') - ring.remove_node('bar') - self.assertEqual(nodes, set(ring.nodes.keys())) - self.assertEqual(2 ** 5 * len(nodes), len(ring)) - - def test_distribution_one_replica(self): - nodes = ['foo', 'bar', 'baz'] - ring = hashring.HashRing(nodes) - fake_1_nodes = ring.get_nodes(b'fake') - fake_2_nodes = ring.get_nodes(b'fake-again') - # We should have one nodes for each thing - self.assertEqual(1, len(fake_1_nodes)) - self.assertEqual(1, len(fake_2_nodes)) - # And they must not be the same answers even on this simple data. - self.assertNotEqual(fake_1_nodes, fake_2_nodes) - - def test_distribution_more_replica(self): - nodes = ['foo', 'bar', 'baz'] - ring = hashring.HashRing(nodes) - fake_1_nodes = ring.get_nodes(b'fake', replicas=2) - fake_2_nodes = ring.get_nodes(b'fake-again', replicas=2) - # We should have one nodes for each thing - self.assertEqual(2, len(fake_1_nodes)) - self.assertEqual(2, len(fake_2_nodes)) - fake_1_nodes = ring.get_nodes(b'fake', replicas=3) - fake_2_nodes = ring.get_nodes(b'fake-again', replicas=3) - # We should have one nodes for each thing - self.assertEqual(3, len(fake_1_nodes)) - self.assertEqual(3, len(fake_2_nodes)) - self.assertEqual(fake_1_nodes, fake_2_nodes) - - def test_ignore_nodes(self): - nodes = ['foo', 'bar', 'baz'] - ring = hashring.HashRing(nodes) - equals_bar_or_baz = matchers.MatchesAny( - matchers.Equals({'bar'}), - matchers.Equals({'baz'})) - self.assertThat( - ring.get_nodes(b'fake', ignore_nodes=['foo']), - equals_bar_or_baz) - self.assertThat( - ring.get_nodes(b'fake', ignore_nodes=['foo', 'bar']), - equals_bar_or_baz) - self.assertEqual(set(), ring.get_nodes(b'fake', ignore_nodes=nodes)) - - @staticmethod - def _compare_rings(nodes, conductors, ring, new_conductors, new_ring): - delta = {} - mapping = { - 'node': list(ring.get_nodes(node.encode('ascii')))[0] - for node in nodes - } - new_mapping = { - 'node': list(new_ring.get_nodes(node.encode('ascii')))[0] - for node in nodes - } - - for key, old in mapping.items(): - new = new_mapping.get(key, None) - if new != old: - delta[key] = (old, new) - return delta - - def test_rebalance_stability_join(self): - num_services = 10 - num_nodes = 10000 - # Adding 1 service to a set of N should move 1/(N+1) of all nodes - # Eg, for a cluster of 10 nodes, adding one should move 1/11, or 9% - # We allow for 1/N to allow for rounding in tests. - redistribution_factor = 1.0 / num_services - - nodes = [str(x) for x in range(num_nodes)] - services = [str(x) for x in range(num_services)] - new_services = services + ['new'] - delta = self._compare_rings( - nodes, services, hashring.HashRing(services), - new_services, hashring.HashRing(new_services)) - - self.assertLess(len(delta), num_nodes * redistribution_factor) - - def test_rebalance_stability_leave(self): - num_services = 10 - num_nodes = 10000 - # Removing 1 service from a set of N should move 1/(N) of all nodes - # Eg, for a cluster of 10 nodes, removing one should move 1/10, or 10% - # We allow for 1/(N-1) to allow for rounding in tests. - redistribution_factor = 1.0 / (num_services - 1) - - nodes = [str(x) for x in range(num_nodes)] - services = [str(x) for x in range(num_services)] - new_services = services[:] - new_services.pop() - delta = self._compare_rings( - nodes, services, hashring.HashRing(services), - new_services, hashring.HashRing(new_services)) - - self.assertLess(len(delta), num_nodes * redistribution_factor) - - def test_ignore_non_existent_node(self): - nodes = ['foo', 'bar'] - ring = hashring.HashRing(nodes) - self.assertEqual({'foo'}, ring.get_nodes(b'fake', - ignore_nodes=['baz'])) diff --git a/tooz/tests/test_memcache.py b/tooz/tests/test_memcache.py deleted file mode 100644 index 07bb815..0000000 --- a/tooz/tests/test_memcache.py +++ /dev/null @@ -1,85 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (c) 2015 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import socket - -try: - from unittest import mock -except ImportError: - import mock - -from testtools import testcase - -from tooz import coordination -from tooz import tests - - -class TestMemcacheDriverFailures(testcase.TestCase): - FAKE_URL = "memcached://mocked-not-really-localhost" - - @mock.patch('pymemcache.client.PooledClient') - def test_client_failure_start(self, mock_client_cls): - mock_client_cls.side_effect = socket.timeout('timed-out') - member_id = tests.get_random_uuid() - coord = coordination.get_coordinator(self.FAKE_URL, member_id) - self.assertRaises(coordination.ToozConnectionError, coord.start) - - @mock.patch('pymemcache.client.PooledClient') - def test_client_failure_join(self, mock_client_cls): - mock_client = mock.MagicMock() - mock_client_cls.return_value = mock_client - member_id = tests.get_random_uuid() - coord = coordination.get_coordinator(self.FAKE_URL, member_id) - coord.start() - mock_client.gets.side_effect = socket.timeout('timed-out') - fut = coord.join_group(tests.get_random_uuid()) - self.assertRaises(coordination.ToozConnectionError, fut.get) - - @mock.patch('pymemcache.client.PooledClient') - def test_client_failure_leave(self, mock_client_cls): - mock_client = mock.MagicMock() - mock_client_cls.return_value = mock_client - member_id = tests.get_random_uuid() - coord = coordination.get_coordinator(self.FAKE_URL, member_id) - coord.start() - mock_client.gets.side_effect = socket.timeout('timed-out') - fut = coord.leave_group(tests.get_random_uuid()) - self.assertRaises(coordination.ToozConnectionError, fut.get) - - @mock.patch('pymemcache.client.PooledClient') - def test_client_failure_heartbeat(self, mock_client_cls): - mock_client = mock.MagicMock() - mock_client_cls.return_value = mock_client - member_id = tests.get_random_uuid() - coord = coordination.get_coordinator(self.FAKE_URL, member_id) - coord.start() - mock_client.set.side_effect = socket.timeout('timed-out') - self.assertRaises(coordination.ToozConnectionError, coord.heartbeat) - - @mock.patch( - 'tooz.coordination.CoordinationDriverCachedRunWatchers.run_watchers', - autospec=True) - @mock.patch('pymemcache.client.PooledClient') - def test_client_run_watchers_mixin(self, mock_client_cls, - mock_run_watchers): - mock_client = mock.MagicMock() - mock_client_cls.return_value = mock_client - member_id = tests.get_random_uuid() - coord = coordination.get_coordinator(self.FAKE_URL, member_id) - coord.start() - coord.run_watchers() - self.assertTrue(mock_run_watchers.called) diff --git a/tooz/tests/test_mysql.py b/tooz/tests/test_mysql.py deleted file mode 100644 index 291f44a..0000000 --- a/tooz/tests/test_mysql.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (c) 2015 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_utils import encodeutils -from testtools import testcase - -import tooz -from tooz import coordination -from tooz import tests - - -class TestMySQLDriver(testcase.TestCase): - - def _create_coordinator(self, url): - - def _safe_stop(coord): - try: - coord.stop() - except tooz.ToozError as e: - message = encodeutils.exception_to_unicode(e) - if (message != 'Can not stop a driver which has not' - ' been started'): - raise - - coord = coordination.get_coordinator(url, - tests.get_random_uuid()) - self.addCleanup(_safe_stop, coord) - return coord - - def test_connect_failure_invalid_hostname_provided(self): - c = self._create_coordinator("mysql://invalidhost/test") - self.assertRaises(coordination.ToozConnectionError, c.start) - - def test_connect_failure_invalid_port_provided(self): - c = self._create_coordinator("mysql://localhost:54/test") - self.assertRaises(coordination.ToozConnectionError, c.start) - - def test_connect_failure_invalid_hostname_and_port_provided(self): - c = self._create_coordinator("mysql://invalidhost:54/test") - self.assertRaises(coordination.ToozConnectionError, c.start) diff --git a/tooz/tests/test_partitioner.py b/tooz/tests/test_partitioner.py deleted file mode 100644 index 10fa6b9..0000000 --- a/tooz/tests/test_partitioner.py +++ /dev/null @@ -1,103 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2016 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import six - -from tooz import coordination -from tooz import tests - - -class TestPartitioner(tests.TestWithCoordinator): - - def setUp(self): - super(TestPartitioner, self).setUp() - self._extra_coords = [] - - def tearDown(self): - for c in self._extra_coords: - c.stop() - super(TestPartitioner, self).tearDown() - - def _add_members(self, number_of_members, weight=1): - groups = [] - for _ in six.moves.range(number_of_members): - m = tests.get_random_uuid() - coord = coordination.get_coordinator(self.url, m) - coord.start() - groups.append(coord.join_partitioned_group( - self.group_id, weight=weight)) - self._extra_coords.append(coord) - self._coord.run_watchers() - return groups - - def _remove_members(self, number_of_members): - for _ in six.moves.range(number_of_members): - c = self._extra_coords.pop() - c.stop() - self._coord.run_watchers() - - def test_join_partitioned_group(self): - group_id = tests.get_random_uuid() - self._coord.join_partitioned_group(group_id) - - def test_hashring_size(self): - p = self._coord.join_partitioned_group(self.group_id) - self.assertEqual(1, len(p.ring.nodes)) - self._add_members(1) - self.assertEqual(2, len(p.ring.nodes)) - self._add_members(2) - self.assertEqual(4, len(p.ring.nodes)) - self._remove_members(3) - self.assertEqual(1, len(p.ring.nodes)) - p.stop() - - def test_hashring_weight(self): - p = self._coord.join_partitioned_group(self.group_id, weight=5) - self.assertEqual([5], list(p.ring.nodes.values())) - p2 = self._add_members(1, weight=10)[0] - self.assertEqual(set([5, 10]), set(p.ring.nodes.values())) - self.assertEqual(set([5, 10]), set(p2.ring.nodes.values())) - p.stop() - - def test_stop(self): - p = self._coord.join_partitioned_group(self.group_id) - p.stop() - self.assertEqual(0, len(self._coord._hooks_join_group)) - self.assertEqual(0, len(self._coord._hooks_leave_group)) - - def test_members_of_object_and_others(self): - p = self._coord.join_partitioned_group(self.group_id) - self._add_members(3) - o = object() - m = p.members_for_object(o) - self.assertEqual(1, len(m)) - m = m.pop() - self.assertTrue(p.belongs_to_member(o, m)) - self.assertFalse(p.belongs_to_member(o, b"chupacabra")) - maybe = self.assertTrue if m == self.member_id else self.assertFalse - maybe(p.belongs_to_self(o)) - p.stop() - - -class ZakeTestPartitioner(TestPartitioner): - url = "zake://" - - -class IPCTestPartitioner(TestPartitioner): - url = "ipc://" - - -class FileTestPartitioner(TestPartitioner): - url = "file:///tmp" diff --git a/tooz/tests/test_postgresql.py b/tooz/tests/test_postgresql.py deleted file mode 100644 index 8556d94..0000000 --- a/tooz/tests/test_postgresql.py +++ /dev/null @@ -1,114 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -try: - # Added in python 3.3+ - from unittest import mock -except ImportError: - import mock - -from oslo_utils import encodeutils -import testtools -from testtools import testcase - -import tooz -from tooz import coordination -from tooz import tests - -# Handle the case gracefully where the driver is not installed. -try: - import psycopg2 - PGSQL_AVAILABLE = True -except ImportError: - PGSQL_AVAILABLE = False - - -@testtools.skipUnless(PGSQL_AVAILABLE, 'psycopg2 is not available') -class TestPostgreSQLFailures(testcase.TestCase): - - # Not actually used (but required none the less), since we mock out - # the connect() method... - FAKE_URL = "postgresql://localhost:1" - - def _create_coordinator(self): - - def _safe_stop(coord): - try: - coord.stop() - except tooz.ToozError as e: - # TODO(harlowja): make this better, so that we don't have to - # do string checking... - message = encodeutils.exception_to_unicode(e) - if (message != 'Can not stop a driver which has not' - ' been started'): - raise - - coord = coordination.get_coordinator(self.FAKE_URL, - tests.get_random_uuid()) - self.addCleanup(_safe_stop, coord) - return coord - - @mock.patch("tooz.drivers.pgsql.psycopg2.connect") - def test_connect_failure(self, psycopg2_connector): - psycopg2_connector.side_effect = psycopg2.Error("Broken") - c = self._create_coordinator() - self.assertRaises(coordination.ToozConnectionError, c.start) - - @mock.patch("tooz.drivers.pgsql.psycopg2.connect") - def test_connect_failure_operational(self, psycopg2_connector): - psycopg2_connector.side_effect = psycopg2.OperationalError("Broken") - c = self._create_coordinator() - self.assertRaises(coordination.ToozConnectionError, c.start) - - @mock.patch("tooz.drivers.pgsql.psycopg2.connect") - def test_failure_acquire_lock(self, psycopg2_connector): - execute_mock = mock.MagicMock() - execute_mock.execute.side_effect = psycopg2.OperationalError("Broken") - - cursor_mock = mock.MagicMock() - cursor_mock.__enter__ = mock.MagicMock(return_value=execute_mock) - cursor_mock.__exit__ = mock.MagicMock(return_value=False) - - conn_mock = mock.MagicMock() - conn_mock.cursor.return_value = cursor_mock - psycopg2_connector.return_value = conn_mock - - c = self._create_coordinator() - c.start() - test_lock = c.get_lock(b'test-lock') - self.assertRaises(tooz.ToozError, test_lock.acquire) - - @mock.patch("tooz.drivers.pgsql.psycopg2.connect") - def test_failure_release_lock(self, psycopg2_connector): - execute_mock = mock.MagicMock() - execute_mock.execute.side_effect = [ - True, - psycopg2.OperationalError("Broken"), - ] - - cursor_mock = mock.MagicMock() - cursor_mock.__enter__ = mock.MagicMock(return_value=execute_mock) - cursor_mock.__exit__ = mock.MagicMock(return_value=False) - - conn_mock = mock.MagicMock() - conn_mock.cursor.return_value = cursor_mock - psycopg2_connector.return_value = conn_mock - - c = self._create_coordinator() - c.start() - test_lock = c.get_lock(b'test-lock') - self.assertTrue(test_lock.acquire()) - self.assertRaises(tooz.ToozError, test_lock.release) diff --git a/tooz/tests/test_utils.py b/tooz/tests/test_utils.py deleted file mode 100644 index 39f83fb..0000000 --- a/tooz/tests/test_utils.py +++ /dev/null @@ -1,136 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (c) 2015 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import tempfile - -import futurist -import six -from testtools import testcase - -import tooz -from tooz import utils - - -class TestProxyExecutor(testcase.TestCase): - def test_fetch_check_executor(self): - try_options = [ - ({'executor': 'sync'}, futurist.SynchronousExecutor), - ({'executor': 'thread'}, futurist.ThreadPoolExecutor), - ] - for options, expected_cls in try_options: - executor = utils.ProxyExecutor.build("test", options) - self.assertTrue(executor.internally_owned) - - executor.start() - self.assertTrue(executor.started) - self.assertIsInstance(executor.executor, expected_cls) - - executor.stop() - self.assertFalse(executor.started) - - def test_fetch_default_executor(self): - executor = utils.ProxyExecutor.build("test", {}) - executor.start() - try: - self.assertIsInstance(executor.executor, - futurist.ThreadPoolExecutor) - finally: - executor.stop() - - def test_fetch_unknown_executor(self): - options = {'executor': 'huh'} - self.assertRaises(tooz.ToozError, - utils.ProxyExecutor.build, 'test', - options) - - def test_no_submit_stopped(self): - executor = utils.ProxyExecutor.build("test", {}) - self.assertRaises(tooz.ToozError, - executor.submit, lambda: None) - - -class TestUtilsSafePath(testcase.TestCase): - base = tempfile.gettempdir() - - def test_join(self): - self.assertEqual(os.path.join(self.base, 'b'), - utils.safe_abs_path(self.base, "b")) - self.assertEqual(os.path.join(self.base, 'b', 'c'), - utils.safe_abs_path(self.base, "b", 'c')) - self.assertEqual(self.base, - utils.safe_abs_path(self.base, "b", 'c', '../..')) - - def test_unsafe_join(self): - self.assertRaises(ValueError, utils.safe_abs_path, - self.base, "../b") - self.assertRaises(ValueError, utils.safe_abs_path, - self.base, "b", 'c', '../../../') - - -class TestUtilsCollapse(testcase.TestCase): - - def test_bad_type(self): - self.assertRaises(TypeError, utils.collapse, "") - self.assertRaises(TypeError, utils.collapse, []) - self.assertRaises(TypeError, utils.collapse, 2) - - def test_collapse_simple(self): - ex = { - 'a': [1], - 'b': 2, - 'c': (1, 2, 3), - } - c_ex = utils.collapse(ex) - self.assertEqual({'a': 1, 'c': 3, 'b': 2}, c_ex) - - def test_collapse_exclusions(self): - ex = { - 'a': [1], - 'b': 2, - 'c': (1, 2, 3), - } - c_ex = utils.collapse(ex, exclude=['a']) - self.assertEqual({'a': [1], 'c': 3, 'b': 2}, c_ex) - - def test_no_collapse(self): - ex = { - 'a': [1], - 'b': [2], - 'c': (1, 2, 3), - } - c_ex = utils.collapse(ex, exclude=set(six.iterkeys(ex))) - self.assertEqual(ex, c_ex) - - def test_custom_selector(self): - ex = { - 'a': [1, 2, 3], - } - c_ex = utils.collapse(ex, - item_selector=lambda items: items[0]) - self.assertEqual({'a': 1}, c_ex) - - def test_empty_lists(self): - ex = { - 'a': [], - 'b': (), - 'c': [1], - } - c_ex = utils.collapse(ex) - self.assertNotIn('b', c_ex) - self.assertNotIn('a', c_ex) - self.assertIn('c', c_ex) diff --git a/tooz/utils.py b/tooz/utils.py deleted file mode 100644 index 1d7b959..0000000 --- a/tooz/utils.py +++ /dev/null @@ -1,225 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import base64 -import datetime -import operator -import os - -import futurist -import msgpack -from oslo_serialization import msgpackutils -from oslo_utils import encodeutils -from oslo_utils import excutils -import six - -import tooz - - -class Base64LockEncoder(object): - def __init__(self, keyspace_url, prefix=''): - self.keyspace_url = keyspace_url - if prefix: - self.keyspace_url += prefix - - def check_and_encode(self, name): - if not isinstance(name, (six.text_type, six.binary_type)): - raise TypeError("Provided lock name is expected to be a string" - " or binary type and not %s" % type(name)) - try: - return self.encode(name) - except (UnicodeDecodeError, UnicodeEncodeError) as e: - raise ValueError("Invalid lock name due to encoding/decoding " - " issue: %s" - % encodeutils.exception_to_unicode(e)) - - def encode(self, name): - if isinstance(name, six.text_type): - name = name.encode("ascii") - enc_name = base64.urlsafe_b64encode(name) - return self.keyspace_url + "/" + enc_name.decode("ascii") - - -class ProxyExecutor(object): - KIND_TO_FACTORY = { - 'threaded': (lambda: - futurist.ThreadPoolExecutor(max_workers=1)), - 'synchronous': lambda: futurist.SynchronousExecutor(), - } - - # Provide a few common aliases... - KIND_TO_FACTORY['thread'] = KIND_TO_FACTORY['threaded'] - KIND_TO_FACTORY['threading'] = KIND_TO_FACTORY['threaded'] - KIND_TO_FACTORY['sync'] = KIND_TO_FACTORY['synchronous'] - - DEFAULT_KIND = 'threaded' - - def __init__(self, driver_name, default_executor_factory): - self.default_executor_factory = default_executor_factory - self.driver_name = driver_name - self.started = False - self.executor = None - self.internally_owned = True - - @classmethod - def build(cls, driver_name, options): - default_executor_fact = cls.KIND_TO_FACTORY[cls.DEFAULT_KIND] - if 'executor' in options: - executor_kind = options['executor'] - try: - default_executor_fact = cls.KIND_TO_FACTORY[executor_kind] - except KeyError: - executors_known = sorted(list(cls.KIND_TO_FACTORY)) - raise tooz.ToozError("Unknown executor" - " '%s' provided, accepted values" - " are %s" % (executor_kind, - executors_known)) - return cls(driver_name, default_executor_fact) - - def start(self): - if self.started: - return - self.executor = self.default_executor_factory() - self.started = True - - def stop(self): - executor = self.executor - self.executor = None - if executor is not None: - executor.shutdown() - self.started = False - - def submit(self, cb, *args, **kwargs): - if not self.started: - raise tooz.ToozError("%s driver asynchronous executor" - " has not been started" - % self.driver_name) - try: - return self.executor.submit(cb, *args, **kwargs) - except RuntimeError: - raise tooz.ToozError("%s driver asynchronous executor has" - " been shutdown" % self.driver_name) - - -def safe_abs_path(rooted_at, *pieces): - # Avoids the following junk... - # - # >>> import os - # >>> os.path.join("/b", "..") - # '/b/..' - # >>> os.path.abspath(os.path.join("/b", "..")) - # '/' - path = os.path.abspath(os.path.join(rooted_at, *pieces)) - if not path.startswith(rooted_at): - raise ValueError("Unable to create path that is outside of" - " parent directory '%s' using segments %s" - % (rooted_at, list(pieces))) - return path - - -def convert_blocking(blocking): - """Converts a multi-type blocking variable into its derivatives.""" - timeout = None - if not isinstance(blocking, bool): - timeout = float(blocking) - blocking = True - return blocking, timeout - - -def collapse(config, exclude=None, item_selector=operator.itemgetter(-1)): - """Collapses config with keys and **list/tuple** values. - - NOTE(harlowja): The last item/index from the list/tuple value is selected - be default as the new value (values that are not lists/tuples are left - alone). If the list/tuple value is empty (zero length), then no value - is set. - """ - if not isinstance(config, dict): - raise TypeError("Unexpected config type, dict expected") - if not config: - return {} - if exclude is None: - exclude = set() - collapsed = {} - for (k, v) in six.iteritems(config): - if isinstance(v, (tuple, list)): - if k in exclude: - collapsed[k] = v - else: - if len(v): - collapsed[k] = item_selector(v) - else: - collapsed[k] = v - return collapsed - - -def to_binary(text, encoding='ascii'): - """Return the binary representation of string (if not already binary).""" - if not isinstance(text, six.binary_type): - text = text.encode(encoding) - return text - - -class SerializationError(tooz.ToozError): - "Exception raised when serialization or deserialization breaks." - - -def dumps(data, excp_cls=SerializationError): - """Serializes provided data using msgpack into a byte string.""" - try: - return msgpackutils.dumps(data) - except (msgpack.PackException, ValueError) as e: - raise_with_cause(excp_cls, - encodeutils.exception_to_unicode(e), - cause=e) - - -def loads(blob, excp_cls=SerializationError): - """Deserializes provided data using msgpack (from a prior byte string).""" - try: - return msgpackutils.loads(blob) - except (msgpack.UnpackException, ValueError) as e: - raise_with_cause(excp_cls, - encodeutils.exception_to_unicode(e), - cause=e) - - -def millis_to_datetime(milliseconds): - """Converts number of milliseconds (from epoch) into a datetime object.""" - return datetime.datetime.fromtimestamp(float(milliseconds) / 1000) - - -def raise_with_cause(exc_cls, message, *args, **kwargs): - """Helper to raise + chain exceptions (when able) and associate a *cause*. - - **For internal usage only.** - - NOTE(harlowja): Since in py3.x exceptions can be chained (due to - :pep:`3134`) we should try to raise the desired exception with the given - *cause*. - - :param exc_cls: the :py:class:`~tooz.ToozError` class to raise. - :param message: the text/str message that will be passed to - the exceptions constructor as its first positional - argument. - :param args: any additional positional arguments to pass to the - exceptions constructor. - :param kwargs: any additional keyword arguments to pass to the - exceptions constructor. - """ - if not issubclass(exc_cls, tooz.ToozError): - raise ValueError("Subclass of tooz error is required") - excutils.raise_with_cause(exc_cls, message, *args, **kwargs) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 48adc5a..0000000 --- a/tox.ini +++ /dev/null @@ -1,63 +0,0 @@ -[tox] -minversion = 1.8 -envlist = py27,py35,py{27,35}-{zookeeper,redis,sentinel,memcached,postgresql,mysql,consul,etcd,etcd3,etcd3gw},pep8 - -[testenv] -# We need to install a bit more than just `test' because those drivers have -# custom tests that we always run -deps = .[test,zake,ipc,memcached,mysql,etcd,etcd3,etcd3gw] - zookeeper: .[zookeeper] - redis: .[redis] - sentinel: .[redis] - memcached: .[memcached] - postgresql: .[postgresql] - mysql: .[mysql] - etcd: .[etcd] - etcd3: .[etcd3] - etcd3gw: .[etcd3gw] - consul: .[consul] -setenv = - TOOZ_TEST_URLS = file:///tmp zake:// ipc:// - zookeeper: TOOZ_TEST_DRIVERS = zookeeper - redis: TOOZ_TEST_DRIVERS = redis - sentinel: TOOZ_TEST_DRIVERS = redis --sentinel - memcached: TOOZ_TEST_DRIVERS = memcached - mysql: TOOZ_TEST_DRIVERS = mysql - postgresql: TOOZ_TEST_DRIVERS = postgresql - etcd: TOOZ_TEST_DRIVERS = etcd,etcd --cluster - etcd3: TOOZ_TEST_DRIVERS = etcd - etcd3: TOOZ_TEST_ETCD3 = 1 - etcd3gw: TOOZ_TEST_DRIVERS = etcd - etcd3gw: TOOZ_TEST_ETCD3GW = 1 - consul: TOOZ_TEST_DRIVERS = consul -# NOTE(tonyb): This project has chosen to *NOT* consume upper-constraints.txt -commands = - {toxinidir}/run-tests.sh {toxinidir}/tools/pretty_tox.sh "{posargs}" - {toxinidir}/run-examples.sh - -[testenv:venv] -# This target is used by the gate go run Sphinx to build the doc -deps = {[testenv:docs]deps} -commands = {posargs} - -[testenv:cover] -commands = python setup.py testr --slowest --coverage --testr-args="{posargs}" - -[testenv:docs] -deps = .[doc,zake,ipc,zookeeper,redis,memcached,mysql,postgresql,consul] -commands = python setup.py build_sphinx - -[testenv:pep8] -deps = hacking<0.13,>=0.12 - doc8 -commands = - flake8 - doc8 doc/source - -[flake8] -exclude=.venv,.git,.tox,dist,*egg,*.egg-info,build,examples,doc -show-source = True - -[testenv:releasenotes] -deps = .[doc] -commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html