openstack/etcd3gw
Code Issues Proposed changes

Apply ruff

Browse Source 
Change-Id: I279d4cbb640f8b5d034805671140e7d268e852eb
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>
This commit is contained in:
Stephen Finucane committed 2026-03-09 13:32:18 +00:00
1 parent 22bf783f62
commit f55fa6e8b1
13 files changed
+374 -310
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.pre-commit-config.yaml
+6 -5
View File
@@ -12,14 +12,15 @@ repos:
- id: debug-statements
- id: check-yaml
files: .*\.(yaml|yml)$
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.5
hooks:
- id: ruff-check
args: ['--fix', '--unsafe-fixes']
- id: ruff-format
- repo: https://opendev.org/openstack/hacking
rev: 8.0.0
hooks:
- id: hacking
additional_dependencies: []
exclude: '^(doc|releasenotes|tools)/.*$'
- repo: https://github.com/asottile/pyupgrade
rev: v3.21.2
hooks:
- id: pyupgrade
args: [--py3-only]
doc/source/conf.py
+8 -4
View File
@@ -73,8 +73,13 @@ html_static_path = []
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'etcd3-gateway.tex', 'etcd3-gateway Documentation',
'Davanum Srinivas', 'manual'),
(
master_doc,
'etcd3-gateway.tex',
'etcd3-gateway Documentation',
'Davanum Srinivas',
'manual',
),
]
# -- Options for manual page output ---------------------------------------
@@ -82,8 +87,7 @@ latex_documents = [
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'etcd3-gateway', 'etcd3-gateway Documentation',
[author], 1)
(master_doc, 'etcd3-gateway', 'etcd3-gateway Documentation', [author], 1)
]
# -- Options for intersphinx extension ------------------------------------
etcd3gw/__init__.py
+1 -2
View File
@@ -18,8 +18,7 @@ from etcd3gw.lease import Lease
from etcd3gw.lock import Lock
from etcd3gw import utils
__version__ = pbr.version.VersionInfo(
'etcd3gw').version_string()
__version__ = pbr.version.VersionInfo('etcd3gw').version_string()
__all__ = (
'Etcd3Client',
etcd3gw/client.py
+105 -79
View File
@@ -42,9 +42,18 @@ DEFAULT_API_PATH = os.getenv('ETCD3GW_API_PATH')
class Etcd3Client:
def __init__(self, host='localhost', port=2379, protocol="http",
ca_cert=None, cert_key=None, cert_cert=None, timeout=None,
api_path=DEFAULT_API_PATH, session=None):
def __init__(
self,
host='localhost',
port=2379,
protocol="http",
ca_cert=None,
cert_key=None,
cert_cert=None,
timeout=None,
api_path=DEFAULT_API_PATH,
session=None,
):
"""Construct an client to talk to etcd3's grpc-gateway's /v3 HTTP API
:param host: etcd host
@@ -83,26 +92,27 @@ class Etcd3Client:
@property
def base_url(self):
host = ('[' + self.host + ']' if (self.host.find(':') != -1)
else self.host)
host = (
'[' + self.host + ']' if (self.host.find(':') != -1) else self.host
)
return self.protocol + '://' + host + ':' + str(self.port)
def _discover_api_path(self):
"""Discover api version and set api_path
"""
"""Discover api version and set api_path"""
resp = self._request('get', self.base_url + '/version')
try:
version_str = resp['etcdserver']
except KeyError:
raise exceptions.ApiVersionDiscoveryFailedError(
'Malformed response from version API')
'Malformed response from version API'
)
try:
version = tuple(int(part) for part in version_str.split('.', 2))
except ValueError:
raise exceptions.ApiVersionDiscoveryFailedError(
'Failed to parse etcd cluster version: %s' % version_str)
f'Failed to parse etcd cluster version: {version_str}'
)
# NOTE(tkajinam): https://etcd.io/docs/v3.5/dev-guide/api_grpc_gateway/
# explains mapping between etcd version and available
@@ -131,12 +141,12 @@ class Etcd3Client:
:return: json response
"""
try:
resp = getattr(self.session, method)(*args, timeout=self.timeout,
**kwargs)
resp = getattr(self.session, method)(
*args, timeout=self.timeout, **kwargs
)
if resp.status_code in _EXCEPTIONS_BY_CODE:
raise _EXCEPTIONS_BY_CODE[resp.status_code](
resp.text,
resp.reason
resp.text, resp.reason
)
if resp.status_code != requests.codes['ok']:
raise exceptions.Etcd3Exception(resp.text, resp.reason)
@@ -160,16 +170,14 @@ class Etcd3Client:
:return: json response
"""
return self.post(self.get_url("/maintenance/status"),
json={})
return self.post(self.get_url("/maintenance/status"), json={})
def members(self):
"""Lists all the members in the cluster.
:return: json response
"""
result = self.post(self.get_url("/cluster/member/list"),
json={})
result = self.post(self.get_url("/cluster/member/list"), json={})
return result['members']
def lease(self, ttl=DEFAULT_TIMEOUT):
@@ -178,8 +186,9 @@ class Etcd3Client:
:param ttl: timeout
:return: Lease object
"""
result = self.post(self.get_url("/lease/grant"),
json={"TTL": ttl, "ID": 0})
result = self.post(
self.get_url("/lease/grant"), json={"TTL": ttl, "ID": 0}
)
return Lease(int(result['ID']), client=self)
def lock(self, id=None, ttl=DEFAULT_TIMEOUT):
@@ -211,19 +220,23 @@ class Etcd3Client:
base64_key = _encode(key)
base64_value = _encode(value)
txn = {
'compare': [{
'key': base64_key,
'result': 'EQUAL',
'target': 'CREATE',
'create_revision': 0
}],
'success': [{
'request_put': {
'compare': [
{
'key': base64_key,
'value': base64_value,
'result': 'EQUAL',
'target': 'CREATE',
'create_revision': 0,
}
}],
'failure': []
],
'success': [
{
'request_put': {
'key': base64_key,
'value': base64_value,
}
}
],
'failure': [],
}
if lease:
txn['success'][0]['request_put']['lease'] = lease.id
@@ -243,17 +256,15 @@ class Etcd3Client:
:param lease:
:return: boolean
"""
payload = {
"key": _encode(key),
"value": _encode(value)
}
payload = {"key": _encode(key), "value": _encode(value)}
if lease:
payload['lease'] = lease.id
self.post(self.get_url("/kv/put"), json=payload)
return True
def get(self, key, metadata=False, sort_order=None,
sort_target=None, **kwargs):
def get(
self, key, metadata=False, sort_order=None, sort_target=None, **kwargs
):
"""Range gets the keys in the range from the key-value store.
:param key:
@@ -275,8 +286,10 @@ class Etcd3Client:
if sort_target:
target = _SORT_TARGET.index(sort_target)
except ValueError:
raise ValueError('sort_target must be one of "key", '
'"version", "create", "mod" or "value"')
raise ValueError(
'sort_target must be one of "key", '
'"version", "create", "mod" or "value"'
)
payload = {
"key": _encode(key),
@@ -284,16 +297,17 @@ class Etcd3Client:
"sort_target": target,
}
payload.update(kwargs)
result = self.post(self.get_url("/kv/range"),
json=payload)
result = self.post(self.get_url("/kv/range"), json=payload)
if 'kvs' not in result:
return []
if metadata:
def value_with_metadata(item):
item['key'] = _decode(item['key'])
value = _decode(item.pop('value', ''))
return value, item
return [value_with_metadata(item) for item in result['kvs']]
return [_decode(item.get('value', '')) for item in result['kvs']]
@@ -319,11 +333,13 @@ class Etcd3Client:
:returns: sequence of (value, metadata) tuples
"""
return self.get(key_prefix,
metadata=True,
range_end=_encode(_increment_last_byte(key_prefix)),
sort_order=sort_order,
sort_target=sort_target)
return self.get(
key_prefix,
metadata=True,
range_end=_encode(_increment_last_byte(key_prefix)),
sort_order=sort_order,
sort_target=sort_target,
)
def replace(self, key, initial_value, new_value):
"""Atomically replace the value of a key with a new value.
@@ -345,19 +361,23 @@ class Etcd3Client:
base64_initial_value = _encode(initial_value)
base64_new_value = _encode(new_value)
txn = {
'compare': [{
'key': base64_key,
'result': 'EQUAL',
'target': 'VALUE',
'value': base64_initial_value
}],
'success': [{
'request_put': {
'compare': [
{
'key': base64_key,
'value': base64_new_value,
'result': 'EQUAL',
'target': 'VALUE',
'value': base64_initial_value,
}
}],
'failure': []
],
'success': [
{
'request_put': {
'key': base64_key,
'value': base64_new_value,
}
}
],
'failure': [],
}
result = self.transaction(txn)
if 'succeeded' in result:
@@ -379,8 +399,7 @@ class Etcd3Client:
}
payload.update(kwargs)
result = self.post(self.get_url("/kv/deleterange"),
json=payload)
result = self.post(self.get_url("/kv/deleterange"), json=payload)
if 'deleted' in result:
return True
return False
@@ -388,7 +407,8 @@ class Etcd3Client:
def delete_prefix(self, key_prefix):
"""Delete a range of keys with a prefix in etcd."""
return self.delete(
key_prefix, range_end=_encode(_increment_last_byte(key_prefix)))
key_prefix, range_end=_encode(_increment_last_byte(key_prefix))
)
def transaction(self, txn):
"""Txn processes multiple requests in a single transaction.
@@ -400,8 +420,7 @@ class Etcd3Client:
:param txn:
:return:
"""
return self.post(self.get_url("/kv/txn"),
data=json.dumps(txn))
return self.post(self.get_url("/kv/txn"), data=json.dumps(txn))
def watch(self, key, **kwargs):
"""Watch a key.
@@ -437,8 +456,7 @@ class Etcd3Client:
def watch_prefix(self, key_prefix, **kwargs):
"""The same as ``watch``, but watches a range of keys with a prefix."""
kwargs['range_end'] = \
_increment_last_byte(key_prefix)
kwargs['range_end'] = _increment_last_byte(key_prefix)
return self.watch(key_prefix, **kwargs)
def watch_once(self, key, timeout=None, **kwargs):
@@ -463,22 +481,30 @@ class Etcd3Client:
def watch_prefix_once(self, key_prefix, timeout=None, **kwargs):
"""Watches a range of keys with a prefix, similar to watch_once"""
kwargs['range_end'] = \
_increment_last_byte(key_prefix)
kwargs['range_end'] = _increment_last_byte(key_prefix)
return self.watch_once(key_prefix, timeout=timeout, **kwargs)
def client(host='localhost', port=2379,
ca_cert=None, cert_key=None, cert_cert=None,
timeout=None, protocol="http", api_path=DEFAULT_API_PATH,
session=None):
def client(
host='localhost',
port=2379,
ca_cert=None,
cert_key=None,
cert_cert=None,
timeout=None,
protocol="http",
api_path=DEFAULT_API_PATH,
session=None,
):
"""Return an instance of an Etcd3Client."""
return Etcd3Client(host=host,
port=port,
ca_cert=ca_cert,
cert_key=cert_key,
cert_cert=cert_cert,
timeout=timeout,
api_path=api_path,
protocol=protocol,
session=session)
return Etcd3Client(
host=host,
port=port,
ca_cert=ca_cert,
cert_key=cert_key,
cert_cert=cert_cert,
timeout=timeout,
api_path=api_path,
protocol=protocol,
session=session,
)
etcd3gw/examples/etcd.py
+20 -20
View File
@@ -21,50 +21,50 @@ def main():
print('>>>> Status')
result = client.status()
print("cluster id : %r" % result['header']['cluster_id'])
print("cluster id : {!r}".format(result['header']['cluster_id']))
result = client.members()
print("first member info : %r" % result[0])
print(f"first member info : {result[0]!r}")
print('>>>> Lease')
lease = client.lease()
print("Lease id : %r" % lease.id)
print("Lease ttl : %r" % lease.ttl())
print("Lease refresh : %r" % lease.refresh())
print(f"Lease id : {lease.id!r}")
print(f"Lease ttl : {lease.ttl()!r}")
print(f"Lease refresh : {lease.refresh()!r}")
result = client.put('foo2', 'bar2', lease)
print("Key put foo2 : %r" % result)
print(f"Key put foo2 : {result!r}")
result = client.put('foo3', 'bar3', lease)
print("Key put foo3 : %r" % result)
print("Lease Keys : %r" % lease.keys())
print(f"Key put foo3 : {result!r}")
print(f"Lease Keys : {lease.keys()!r}")
result = lease.revoke()
print("Lease Revoke : %r" % result)
print(f"Lease Revoke : {result!r}")
result = client.get('foox')
print("Key get foox : %r" % result)
print(f"Key get foox : {result!r}")
result = client.put('foo', 'bar')
print("Key put foo : %r" % result)
print(f"Key put foo : {result!r}")
result = client.get('foo')
print("Key get foo : %r" % result)
print(f"Key get foo : {result!r}")
result = client.delete('foo')
print("Key delete foo : %r" % result)
print(f"Key delete foo : {result!r}")
result = client.delete('foo-unknown')
print("Key delete foo-unknown : %r" % result)
print(f"Key delete foo-unknown : {result!r}")
print('>>>> Lock')
lock = Lock('xyz-%s' % time.perf_counter(), ttl=10000, client=client)
lock = Lock(f'xyz-{time.perf_counter()}', ttl=10000, client=client)
result = lock.acquire()
print("acquire : %r" % result)
print(f"acquire : {result!r}")
result = lock.refresh()
print("refresh : %r" % result)
print(f"refresh : {result!r}")
result = lock.is_acquired()
print("is_acquired : %r" % result)
print(f"is_acquired : {result!r}")
result = lock.release()
print("release : %r" % result)
print(f"release : {result!r}")
result = lock.is_acquired()
print("is_acquired : %r" % result)
print(f"is_acquired : {result!r}")
if __name__ == "__main__":
etcd3gw/lease.py
+13 -9
View File
@@ -33,8 +33,9 @@ class Lease:
:return:
"""
self.client.post(self.client.get_url("/kv/lease/revoke"),
json={"ID": self.id})
self.client.post(
self.client.get_url("/kv/lease/revoke"), json={"ID": self.id}
)
return True
def ttl(self):
@@ -46,8 +47,9 @@ class Lease:
:return:
"""
result = self.client.post(self.client.get_url("/kv/lease/timetolive"),
json={"ID": self.id})
result = self.client.post(
self.client.get_url("/kv/lease/timetolive"), json={"ID": self.id}
)
return int(result['TTL'])
def refresh(self):
@@ -62,8 +64,9 @@ class Lease:
according to etcd documentation.
https://etcd.io/docs/v3.5/dev-guide/apispec/swagger/rpc.swagger.json
"""
result = self.client.post(self.client.get_url("/lease/keepalive"),
json={"ID": self.id})
result = self.client.post(
self.client.get_url("/lease/keepalive"), json={"ID": self.id}
)
return int(result['result'].get('TTL', -1))
def keys(self):
@@ -71,8 +74,9 @@ class Lease:
:return:
"""
result = self.client.post(self.client.get_url("/kv/lease/timetolive"),
json={"ID": self.id,
"keys": True})
result = self.client.post(
self.client.get_url("/kv/lease/timetolive"),
json={"ID": self.id, "keys": True},
)
keys = result['keys'] if 'keys' in result else []
return [_decode(key) for key in keys]
etcd3gw/lock.py
+23 -25
View File
@@ -44,24 +44,24 @@ class Lock:
base64_key = _encode(self.key)
base64_value = _encode(self._uuid)
txn = {
'compare': [{
'key': base64_key,
'result': 'EQUAL',
'target': 'CREATE',
'create_revision': 0
}],
'success': [{
'request_put': {
'compare': [
{
'key': base64_key,
'value': base64_value,
'lease': self.lease.id
'result': 'EQUAL',
'target': 'CREATE',
'create_revision': 0,
}
}],
'failure': [{
'request_range': {
'key': base64_key
],
'success': [
{
'request_put': {
'key': base64_key,
'value': base64_value,
'lease': self.lease.id,
}
}
}]
],
'failure': [{'request_range': {'key': base64_key}}],
}
result = self.client.transaction(txn)
if 'succeeded' in result:
@@ -74,17 +74,15 @@ class Lock:
base64_value = _encode(self._uuid)
txn = {
'compare': [{
'key': base64_key,
'result': 'EQUAL',
'target': 'VALUE',
'value': base64_value
}],
'success': [{
'request_delete_range': {
'key': base64_key
'compare': [
{
'key': base64_key,
'result': 'EQUAL',
'target': 'VALUE',
'value': base64_value,
}
}]
],
'success': [{'request_delete_range': {'key': base64_key}}],
}
result = self.client.transaction(txn)
etcd3gw/tests/test_client.py
+91 -55
View File
@@ -19,7 +19,6 @@ from etcd3gw.tests import base
class TestEtcd3Gateway(base.TestCase):
def test_client_version_discovery(self):
client = Etcd3Client()
with mock.patch.object(client, "session") as mock_session:
@@ -28,13 +27,14 @@ class TestEtcd3Gateway(base.TestCase):
mock_response.reason = "OK"
mock_response.json.return_value = {
"etcdserver": "3.4.0",
"etcdcluster": "3.0.0"
"etcdcluster": "3.0.0",
}
mock_session.get.return_value = mock_response
self.assertEqual("http://localhost:2379",
client.base_url)
self.assertEqual("http://localhost:2379/v3/lease/grant",
client.get_url("/lease/grant"))
self.assertEqual("http://localhost:2379", client.base_url)
self.assertEqual(
"http://localhost:2379/v3/lease/grant",
client.get_url("/lease/grant"),
)
def test_client_version_discovery_v3beta(self):
client = Etcd3Client()
@@ -44,13 +44,14 @@ class TestEtcd3Gateway(base.TestCase):
mock_response.reason = "OK"
mock_response.json.return_value = {
"etcdserver": "3.3.0",
"etcdcluster": "3.0.0"
"etcdcluster": "3.0.0",
}
mock_session.get.return_value = mock_response
self.assertEqual("http://localhost:2379",
client.base_url)
self.assertEqual("http://localhost:2379/v3beta/lease/grant",
client.get_url("/lease/grant"))
self.assertEqual("http://localhost:2379", client.base_url)
self.assertEqual(
"http://localhost:2379/v3beta/lease/grant",
client.get_url("/lease/grant"),
)
def test_client_version_discovery_v3alpha(self):
client = Etcd3Client()
@@ -60,13 +61,14 @@ class TestEtcd3Gateway(base.TestCase):
mock_response.reason = "OK"
mock_response.json.return_value = {
"etcdserver": "3.2.0",
"etcdcluster": "3.0.0"
"etcdcluster": "3.0.0",
}
mock_session.get.return_value = mock_response
self.assertEqual("http://localhost:2379",
client.base_url)
self.assertEqual("http://localhost:2379/v3alpha/lease/grant",
client.get_url("/lease/grant"))
self.assertEqual("http://localhost:2379", client.base_url)
self.assertEqual(
"http://localhost:2379/v3alpha/lease/grant",
client.get_url("/lease/grant"),
)
def test_client_version_discovery_fail(self):
client = Etcd3Client()
@@ -76,8 +78,8 @@ class TestEtcd3Gateway(base.TestCase):
mock_response.reason = "Internal Server Error"
mock_session.get.return_value = mock_response
self.assertRaises(
exc.Etcd3Exception,
client.get_url, "/lease/grant")
exc.Etcd3Exception, client.get_url, "/lease/grant"
)
def test_client_version_discovery_version_absent(self):
client = Etcd3Client()
@@ -89,7 +91,9 @@ class TestEtcd3Gateway(base.TestCase):
mock_session.get.return_value = mock_response
self.assertRaises(
exc.ApiVersionDiscoveryFailedError,
client.get_url, "/lease/grant")
client.get_url,
"/lease/grant",
)
def test_client_version_discovery_version_malformed(self):
client = Etcd3Client()
@@ -99,33 +103,37 @@ class TestEtcd3Gateway(base.TestCase):
mock_response.reason = "OK"
mock_response.json.return_value = {
"etcdserver": "3.2.a",
"etcdcluster": "3.0.0"
"etcdcluster": "3.0.0",
}
mock_session.get.return_value = mock_response
self.assertRaises(
exc.ApiVersionDiscoveryFailedError,
client.get_url, "/lease/grant")
client.get_url,
"/lease/grant",
)
def test_client_api_path(self):
client = Etcd3Client(api_path='/v3/')
self.assertEqual("http://localhost:2379",
client.base_url)
self.assertEqual("http://localhost:2379/v3/lease/grant",
client.get_url("/lease/grant"))
self.assertEqual("http://localhost:2379", client.base_url)
self.assertEqual(
"http://localhost:2379/v3/lease/grant",
client.get_url("/lease/grant"),
)
def test_client_ipv4(self):
client = Etcd3Client(host="127.0.0.1", api_path='/v3/')
self.assertEqual("http://127.0.0.1:2379",
client.base_url)
self.assertEqual("http://127.0.0.1:2379/v3/lease/grant",
client.get_url("/lease/grant"))
self.assertEqual("http://127.0.0.1:2379", client.base_url)
self.assertEqual(
"http://127.0.0.1:2379/v3/lease/grant",
client.get_url("/lease/grant"),
)
def test_client_ipv6(self):
client = Etcd3Client(host="::1", api_path='/v3/')
self.assertEqual("http://[::1]:2379",
client.base_url)
self.assertEqual("http://[::1]:2379/v3/lease/grant",
client.get_url("/lease/grant"))
self.assertEqual("http://[::1]:2379", client.base_url)
self.assertEqual(
"http://[::1]:2379/v3/lease/grant", client.get_url("/lease/grant")
)
def test_client_status(self):
client = Etcd3Client(host="127.0.0.1", api_path='/v3/')
@@ -136,10 +144,15 @@ class TestEtcd3Gateway(base.TestCase):
mock_response.text = "{}"
mock_session.post.return_value = mock_response
client.status()
mock_session.post.assert_has_calls([
mock.call('http://127.0.0.1:2379/v3/maintenance/status',
timeout=None, json={})
])
mock_session.post.assert_has_calls(
[
mock.call(
'http://127.0.0.1:2379/v3/maintenance/status',
timeout=None,
json={},
)
]
)
def test_client_with_timeout(self):
client = Etcd3Client(host="127.0.0.1", api_path='/v3/', timeout=60)
@@ -150,31 +163,47 @@ class TestEtcd3Gateway(base.TestCase):
mock_response.text = "{}"
mock_session.post.return_value = mock_response
client.status()
mock_session.post.assert_has_calls([
mock.call('http://127.0.0.1:2379/v3/maintenance/status',
timeout=60, json={})
])
mock_session.post.assert_has_calls(
[
mock.call(
'http://127.0.0.1:2379/v3/maintenance/status',
timeout=60,
json={},
)
]
)
def test_client_timed_out(self):
client = Etcd3Client(host="127.0.0.1", api_path='/v3/', timeout=60)
with mock.patch.object(client, "session") as mock_session:
mock_session.post.side_effect = requests.exceptions.Timeout()
self.assertRaises(exc.ConnectionTimeoutError, client.status)
mock_session.post.assert_has_calls([
mock.call('http://127.0.0.1:2379/v3/maintenance/status',
timeout=60, json={})
])
mock_session.post.assert_has_calls(
[
mock.call(
'http://127.0.0.1:2379/v3/maintenance/status',
timeout=60,
json={},
)
]
)
def test_client_connection_error(self):
client = Etcd3Client(host="127.0.0.1", api_path='/v3/')
with mock.patch.object(client, "session") as mock_session:
mock_session.post.side_effect = \
mock_session.post.side_effect = (
requests.exceptions.ConnectionError()
)
self.assertRaises(exc.ConnectionFailedError, client.status)
mock_session.post.assert_has_calls([
mock.call('http://127.0.0.1:2379/v3/maintenance/status',
timeout=None, json={})
])
mock_session.post.assert_has_calls(
[
mock.call(
'http://127.0.0.1:2379/v3/maintenance/status',
timeout=None,
json={},
)
]
)
def test_client_bad_request(self):
client = Etcd3Client(host="127.0.0.1", api_path='/v3/')
@@ -192,10 +221,13 @@ class TestEtcd3Gateway(base.TestCase):
self.assertFalse(True)
except exc.Etcd3Exception as e:
self.assertEqual(str(e), "Bad Request")
self.assertEqual(e.detail_text, '''{
self.assertEqual(
e.detail_text,
'''{
"error": "etcdserver: mvcc: required revision has been compacted",
"code": 11
}''')
}''',
)
def test_client_exceptions_by_code(self):
client = Etcd3Client(host="127.0.0.1", api_path='/v3/')
@@ -212,9 +244,12 @@ class TestEtcd3Gateway(base.TestCase):
self.assertFalse(True)
except exc.InternalServerError as e:
self.assertEqual(str(e), "Internal Server Error")
self.assertEqual(e.detail_text, '''{
self.assertEqual(
e.detail_text,
'''{
"error": "etcdserver: unable to reach quorum"
}''')
}''',
)
def test_client_custom_session_with_pool_configuration(self):
from requests.adapters import HTTPAdapter
@@ -238,4 +273,5 @@ class TestEtcd3Gateway(base.TestCase):
http_adapter = client.session.get_adapter('http://localhost:2379')
self.assertEqual(
http_adapter.poolmanager.connection_pool_kw['maxsize'], 20)
http_adapter.poolmanager.connection_pool_kw['maxsize'], 20
)
etcd3gw/tests/test_etcd3gw.py
+73 -83
View File
@@ -52,8 +52,7 @@ class TestEtcd3Gateway(base.TestCase):
def setUpClass(cls):
cls.client = Etcd3Client(port=ETCD_PORT)
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_client_status(self):
response = self.client.status()
self.assertIsNotNone(response)
@@ -61,16 +60,14 @@ class TestEtcd3Gateway(base.TestCase):
self.assertIn('header', response)
self.assertIn('cluster_id', response['header'])
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_client_members(self):
response = self.client.members()
self.assertTrue(len(response) > 0)
self.assertIn('clientURLs', response[0])
self.assertIn('peerURLs', response[0])
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_client_with_keys_and_values(self):
self.assertTrue(self.client.put('foo0', 'bar0'))
self.assertTrue(self.client.put('foo1', 2001))
@@ -86,11 +83,10 @@ class TestEtcd3Gateway(base.TestCase):
self.assertEqual(False, self.client.delete('foo0'))
self.assertTrue(len(self.client.get_all()) > 0)
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_get_and_delete_prefix(self):
for i in range(20):
self.client.put('/doot1/range{}'.format(i), 'i am a range')
self.client.put(f'/doot1/range{i}', 'i am a range')
values = list(self.client.get_prefix('/doot1/range'))
assert len(values) == 20
@@ -102,93 +98,95 @@ class TestEtcd3Gateway(base.TestCase):
values = list(self.client.get_prefix('/doot1/range'))
assert len(values) == 0
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_get_prefix_sort_order(self):
def remove_prefix(string, prefix):
return string[len(prefix):]
return string[len(prefix) :]
initial_keys = 'abcde'
initial_values = 'qwert'
for k, v in zip(initial_keys, initial_values):
self.client.put('/doot2/{}'.format(k), v)
self.client.put(f'/doot2/{k}', v)
keys = b''
for value, meta in self.client.get_prefix(
'/doot2', sort_order='ascend'):
'/doot2', sort_order='ascend'
):
keys += remove_prefix(meta['key'], '/doot2/')
assert keys == initial_keys.encode("latin-1")
reverse_keys = b''
for value, meta in self.client.get_prefix(
'/doot2', sort_order='descend'):
'/doot2', sort_order='descend'
):
reverse_keys += remove_prefix(meta['key'], '/doot2/')
assert reverse_keys == ''.join(
reversed(initial_keys)
).encode("latin-1")
assert reverse_keys == ''.join(reversed(initial_keys)).encode(
"latin-1"
)
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_get_prefix_sort_order_explicit_sort_target_key(self):
def remove_prefix(string, prefix):
return string[len(prefix):]
return string[len(prefix) :]
initial_keys_ordered = 'abcde'
initial_keys = 'aebdc'
initial_values = 'qwert'
for k, v in zip(initial_keys, initial_values):
self.client.put('/doot2/{}'.format(k), v)
self.client.put(f'/doot2/{k}', v)
keys = b''
for value, meta in self.client.get_prefix(
'/doot2', sort_order='ascend', sort_target='key'):
'/doot2', sort_order='ascend', sort_target='key'
):
keys += remove_prefix(meta['key'], '/doot2/')
assert keys == initial_keys_ordered.encode("latin-1")
reverse_keys = b''
for value, meta in self.client.get_prefix(
'/doot2', sort_order='descend', sort_target='key'):
'/doot2', sort_order='descend', sort_target='key'
):
reverse_keys += remove_prefix(meta['key'], '/doot2/')
assert reverse_keys == ''.join(
reversed(initial_keys_ordered)
).encode("latin-1")
assert reverse_keys == ''.join(reversed(initial_keys_ordered)).encode(
"latin-1"
)
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_get_prefix_sort_order_explicit_sort_target_rev(self):
def remove_prefix(string, prefix):
return string[len(prefix):]
return string[len(prefix) :]
initial_keys = 'aebdc'
initial_values = 'qwert'
for k, v in zip(initial_keys, initial_values):
self.client.put('/expsortmod/{}'.format(k), v)
self.client.put(f'/expsortmod/{k}', v)
keys = b''
for value, meta in self.client.get_prefix(
'/expsortmod', sort_order='ascend', sort_target='mod'):
'/expsortmod', sort_order='ascend', sort_target='mod'
):
keys += remove_prefix(meta['key'], '/expsortmod/')
assert keys == initial_keys.encode("latin-1")
reverse_keys = b''
for value, meta in self.client.get_prefix(
'/expsortmod', sort_order='descend', sort_target='mod'):
'/expsortmod', sort_order='descend', sort_target='mod'
):
reverse_keys += remove_prefix(meta['key'], '/expsortmod/')
assert reverse_keys == ''.join(
reversed(initial_keys)
).encode("latin-1")
assert reverse_keys == ''.join(reversed(initial_keys)).encode(
"latin-1"
)
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_replace_success(self):
key = '/doot/thing' + str(uuid.uuid4())
self.client.put(key, 'toot')
@@ -197,8 +195,7 @@ class TestEtcd3Gateway(base.TestCase):
self.assertEqual([b'doot'], v)
self.assertTrue(status)
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_replace_fail(self):
key = '/doot/thing' + str(uuid.uuid4())
self.client.put(key, 'boot')
@@ -207,8 +204,7 @@ class TestEtcd3Gateway(base.TestCase):
self.assertEqual([b'boot'], v)
self.assertFalse(status)
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_client_lease(self):
lease = self.client.lease(ttl=60)
self.assertIsNotNone(lease)
@@ -224,8 +220,7 @@ class TestEtcd3Gateway(base.TestCase):
self.assertTrue(lease.revoke())
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_client_lease_with_keys(self):
lease = self.client.lease(ttl=60)
self.assertIsNotNone(lease)
@@ -243,10 +238,9 @@ class TestEtcd3Gateway(base.TestCase):
self.assertTrue(lease.revoke())
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_watch_key(self):
key = '/%s-watch_key/watch' % str(uuid.uuid4())
key = f'/{str(uuid.uuid4())}-watch_key/watch'
def update_etcd(v):
self.client.put(key, v)
@@ -287,10 +281,9 @@ class TestEtcd3Gateway(base.TestCase):
t.join()
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_watch_huge_payload(self):
key = '/%s-watch_key/watch/huge_payload' % str(uuid.uuid4())
key = f'/{str(uuid.uuid4())}-watch_key/watch/huge_payload'
def update_etcd(v):
print(f"put({key}, {v}")
@@ -332,10 +325,9 @@ class TestEtcd3Gateway(base.TestCase):
t.join()
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_watch_prefix(self):
key = '/%s-watch_prefix/watch/prefix/' % str(uuid.uuid4())
key = f'/{str(uuid.uuid4())}-watch_prefix/watch/prefix/'
def update_etcd(v):
self.client.put(key + v, v)
@@ -365,7 +357,7 @@ class TestEtcd3Gateway(base.TestCase):
self.assertEqual(
event['kv']['key'],
('{}{}'.format(key, change_count)).encode("latin-1"),
(f'{key}{change_count}').encode("latin-1"),
)
self.assertEqual(
event['kv']['value'],
@@ -382,8 +374,7 @@ class TestEtcd3Gateway(base.TestCase):
t.join()
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_sequential_watch_prefix_once(self):
try:
self.client.watch_prefix_once('/doot/', 1)
@@ -398,8 +389,7 @@ class TestEtcd3Gateway(base.TestCase):
except exceptions.WatchTimedOut:
pass
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_client_lock_acquire_release(self):
with self.client.lock(ttl=60) as lock:
ttl = lock.refresh()
@@ -409,10 +399,9 @@ class TestEtcd3Gateway(base.TestCase):
with self.client.lock(ttl=60) as lock:
self.assertFalse(lock.acquire())
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_client_locks(self):
lock = self.client.lock(id='xyz-%s' % time.perf_counter(), ttl=60)
lock = self.client.lock(id=f'xyz-{time.perf_counter()}', ttl=60)
self.assertIsNotNone(lock)
self.assertTrue(lock.acquire())
@@ -426,8 +415,7 @@ class TestEtcd3Gateway(base.TestCase):
self.assertFalse(lock.release())
self.assertFalse(lock.is_acquired())
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_create_success(self):
key = '/foo/unique' + str(uuid.uuid4())
# Verify that key is empty
@@ -438,8 +426,7 @@ class TestEtcd3Gateway(base.TestCase):
self.assertEqual([b'bar'], self.client.get(key))
self.assertTrue(status)
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_create_fail(self):
key = '/foo/' + str(uuid.uuid4())
# Assign value to the key
@@ -451,8 +438,7 @@ class TestEtcd3Gateway(base.TestCase):
self.assertEqual([b'bar'], self.client.get(key))
self.assertFalse(status)
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_create_with_lease_success(self):
key = '/foo/unique' + str(uuid.uuid4())
# Verify that key is empty
@@ -468,13 +454,21 @@ class TestEtcd3Gateway(base.TestCase):
self.assertIn(key.encode('latin-1'), keys)
def my_iter_content(self, *args, **kwargs):
payload = json.dumps({
'result': {
'events': [{
'kv': {'key': base64.b64encode(b'value').decode('utf-8')},
}]
payload = json.dumps(
{
'result': {
'events': [
{
'kv': {
'key': base64.b64encode(b'value').decode(
'utf-8'
)
},
}
]
}
}
})
)
if not kwargs.get('decode_unicode', False):
payload = payload.encode()
@@ -503,37 +497,33 @@ class TestEtcd3Gateway(base.TestCase):
payload["value"] = utils._encode(key_name)
self.client.post(self.client.get_url("/kv/put"), json=payload)
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_client_keys_with_metadata_and_value(self):
test_key_value = b"some_key"
self._post_key(test_key_value)
result = self.client.get(test_key_value, metadata=True)
self.assertTrue(
len(result) > 0,
str(test_key_value) + " key is not found in etcd"
len(result) > 0, str(test_key_value) + " key is not found in etcd"
)
value, metadata = result[0]
self.assertEqual(
value,
test_key_value,
"unable to get value for " + str(test_key_value)
"unable to get value for " + str(test_key_value),
)
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")
@unittest.skipUnless(_is_etcd3_running(), "etcd3 is not available")
def test_client_keys_with_metadata_and_no_value(self):
value_is_not_set_default = b""
test_key = b"some_key"
self._post_key(test_key, provide_value=False)
result = self.client.get(test_key, metadata=True)
self.assertTrue(
len(result) > 0,
str(test_key) + " key is not found in etcd"
len(result) > 0, str(test_key) + " key is not found in etcd"
)
value, metadata = result[0]
self.assertEqual(
value,
value_is_not_set_default,
"unable to get value for " + str(test_key)
"unable to get value for " + str(test_key),
)
etcd3gw/watch.py
+11 -14
View File
@@ -40,14 +40,11 @@ def _watch(resp, callback):
class Watcher:
KW_ARGS = ['start_revision', 'progress_notify', 'filters', 'prev_kv']
KW_ENCODED_ARGS = ['range_end']
def __init__(self, client, key, callback, **kwargs):
create_watch = {
'key': _encode(key)
}
create_watch = {'key': _encode(key)}
for arg in kwargs:
if arg in self.KW_ARGS:
@@ -55,12 +52,10 @@ class Watcher:
elif arg in self.KW_ENCODED_ARGS:
create_watch[arg] = _encode(kwargs[arg])
create_request = {
"create_request": create_watch
}
self._response = client.session.post(client.get_url('/watch'),
json=create_request,
stream=True)
create_request = {"create_request": create_watch}
self._response = client.session.post(
client.get_url('/watch'), json=create_request, stream=True
)
clazz = _get_threadpool_executor()
self._executor = clazz(max_workers=2)
@@ -68,12 +63,14 @@ class Watcher:
def stop(self):
try:
s = socket.fromfd(self._response.raw._fp.fileno(),
socket.AF_INET,
socket.SOCK_STREAM)
s = socket.fromfd(
self._response.raw._fp.fileno(),
socket.AF_INET,
socket.SOCK_STREAM,
)
s.shutdown(socket.SHUT_RDWR)
s.close()
except Exception:
except Exception: # noqa: S110
pass
self._response.connection.close()
self._executor.shutdown(wait=False)
pyproject.toml
+14
View File
@@ -35,3 +35,17 @@ Repository = "https://opendev.org/openstack/etcd3gw"
packages = [
"etcd3gw"
]
[tool.ruff]
line-length = 79
[tool.ruff.format]
quote-style = "preserve"
docstring-code-format = true
[tool.ruff.lint]
select = ["E4", "E5", "E7", "E9", "F", "G", "LOG", "S", "UP"]
external = ["H"]
[tool.ruff.lint.per-file-ignores]
"etcd3gw/tests/*" = ["S"]
setup.py
+1 -3
View File
@@ -15,6 +15,4 @@
import setuptools
setuptools.setup(
setup_requires=['pbr>=2.0'],
pbr=True)
setuptools.setup(setup_requires=['pbr>=2.0'], pbr=True)
tox.ini
+8 -11
View File
@@ -39,6 +39,11 @@ commands =
allowlist_externals =
{toxinidir}/setup-etcd-env.sh
[testenv:examples]
commands = {toxinidir}/setup-etcd-env.sh pifpaf -e TOOZ_TEST run etcd -- python {toxinidir}/etcd3gw/examples/etcd.py
allowlist_externals =
{toxinidir}/setup-etcd-env.sh
[testenv:docs]
deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
-r{toxinidir}/doc/requirements.txt
@@ -49,11 +54,6 @@ commands = {toxinidir}/setup-etcd-env.sh pifpaf -e TOOZ_TEST run etcd -- stestr
allowlist_externals =
{toxinidir}/setup-etcd-env.sh
[testenv:examples]
commands = {toxinidir}/setup-etcd-env.sh pifpaf -e TOOZ_TEST run etcd -- python {toxinidir}/etcd3gw/examples/etcd.py
allowlist_externals =
{toxinidir}/setup-etcd-env.sh
[testenv:releasenotes]
deps = {[testenv:docs]deps}
commands =
@@ -63,10 +63,7 @@ commands =
commands = oslo_debug_helper {posargs}
[flake8]
# We only enable the hacking (H) checks
select = H
show-source = True
# H904: Delay string interpolations at logging calls
enable-extensions = H904
# E123, E125 skipped as they are invalid PEP-8.
ignore = E123,E125
builtins = _
exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build
exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build