openstack
/
etcd3gw
Code Issues Proposed changes

Discover API version automatically 

etcd changed API version from v3alpha to v3beta, then v3, and because
of transition available api versions are different according to
the etcd server available.

This change implements the mechanism to detect the current api version
according to the etcd version obtained via the version API. Discovery
is executed at the first request and the result is stored.

Closes-Bug: #2043810
Change-Id: I08bc429dd8c59e5d2b6e62f8f48afd78a116fbea
This commit is contained in:
Takashi Kajinami 2023-11-17 22:53:59 +09:00
parent 823914edcf
commit e5faa53ead
6 changed files with 177 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
etcd3gw/client.py
View File

@ -38,7 +38,7 @@ _EXCEPTIONS_BY_CODE = {
requests.codes['precondition_failed']: exceptions.PreconditionFailedError,
}
DEFAULT_API_PATH = os.getenv('ETCD3GW_API_PATH', '/v3alpha/')
DEFAULT_API_PATH = os.getenv('ETCD3GW_API_PATH')
class Etcd3Client(object):
@ -62,7 +62,47 @@ class Etcd3Client(object):
self.session.verify = ca_cert
if cert_cert is not None and cert_key is not None:
self.session.cert = (cert_cert, cert_key)
self.api_path = api_path
self._api_path = api_path
@property
def api_path(self):
if self._api_path is not None:
return self._api_path
self._discover_api_path()
return self._api_path
@property
def base_url(self):
host = ('[' + self.host + ']' if (self.host.find(':') != -1)
else self.host)
return self.protocol + '://' + host + ':' + str(self.port)
def _discover_api_path(self):
"""Discover api version and set api_path
"""
resp = self._request('get', self.base_url + '/version')
try:
version_str = resp['etcdserver']
except KeyError:
raise exceptions.ApiVersionDiscoveryFailedError(
'Malformed response from version API')
try:
version = tuple(int(part) for part in version_str.split('.', 2))
except ValueError:
raise exceptions.ApiVersionDiscoveryFailedError(
'Failed to parse etcd cluster version: %s' % version_str)
# NOTE(tkajinam): https://etcd.io/docs/v3.5/dev-guide/api_grpc_gateway/
# explains mapping between etcd version and available
# api versions
if version >= (3, 4):
self._api_path = '/v3/'
elif version >= (3, 3):
self._api_path = '/v3beta/'
else:
self._api_path = '/v3alpha/'
def get_url(self, path):
"""Construct a full url to the v3 API given a specific path
@ -70,20 +110,18 @@ class Etcd3Client(object):
:param path:
:return: url
"""
host = ('[' + self.host + ']' if (self.host.find(':') != -1)
else self.host)
base_url = self.protocol + '://' + host + ':' + str(self.port)
return base_url + self.api_path + path.lstrip("/")
def post(self, *args, **kwargs):
"""helper method for HTTP POST
return self.base_url + self.api_path + path.lstrip("/")
def _request(self, method, *args, **kwargs):
"""helper method for HTTP requests
:param args:
:param kwargs:
:return: json response
"""
try:
resp = self.session.post(*args, **kwargs)
resp = getattr(self.session, method)(*args, **kwargs)
if resp.status_code in _EXCEPTIONS_BY_CODE:
raise _EXCEPTIONS_BY_CODE[resp.status_code](
resp.text,
@ -97,6 +135,15 @@ class Etcd3Client(object):
raise exceptions.ConnectionFailedError(str(ex))
return resp.json()
def post(self, *args, **kwargs):
"""helper method for HTTP POST
:param args:
:param kwargs:
:return: json response
"""
return self._request('post', *args, **kwargs)
def status(self):
"""Status gets the status of the etcd cluster member.

4
etcd3gw/exceptions.py
View File

@ -35,3 +35,7 @@ class ConnectionTimeoutError(Etcd3Exception):
class PreconditionFailedError(Etcd3Exception):
pass
class ApiVersionDiscoveryFailedError(Etcd3Exception):
pass

117
etcd3gw/tests/test_client.py
View File

@ -13,6 +13,7 @@
from unittest import mock
from etcd3gw.client import Etcd3Client
from etcd3gw.exceptions import ApiVersionDiscoveryFailedError
from etcd3gw.exceptions import Etcd3Exception
from etcd3gw.exceptions import InternalServerError
from etcd3gw.tests import base
@ -20,26 +21,115 @@ from etcd3gw.tests import base
class TestEtcd3Gateway(base.TestCase):
def test_client_default(self):
def test_client_version_discovery(self):
client = Etcd3Client()
self.assertEqual("http://localhost:2379%slease/grant" %
client.api_path,
with mock.patch.object(client, "session") as mock_session:
mock_response = mock.Mock()
mock_response.status_code = 200
mock_response.reason = "OK"
mock_response.json.return_value = {
"etcdserver": "3.4.0",
"etcdcluster": "3.0.0"
}
mock_session.get.return_value = mock_response
self.assertEqual("http://localhost:2379",
client.base_url)
self.assertEqual("http://localhost:2379/v3/lease/grant",
client.get_url("/lease/grant"))
def test_client_version_discovery_v3beta(self):
client = Etcd3Client()
with mock.patch.object(client, "session") as mock_session:
mock_response = mock.Mock()
mock_response.status_code = 200
mock_response.reason = "OK"
mock_response.json.return_value = {
"etcdserver": "3.3.0",
"etcdcluster": "3.0.0"
}
mock_session.get.return_value = mock_response
self.assertEqual("http://localhost:2379",
client.base_url)
self.assertEqual("http://localhost:2379/v3beta/lease/grant",
client.get_url("/lease/grant"))
def test_client_version_discovery_v3alpha(self):
client = Etcd3Client()
with mock.patch.object(client, "session") as mock_session:
mock_response = mock.Mock()
mock_response.status_code = 200
mock_response.reason = "OK"
mock_response.json.return_value = {
"etcdserver": "3.2.0",
"etcdcluster": "3.0.0"
}
mock_session.get.return_value = mock_response
self.assertEqual("http://localhost:2379",
client.base_url)
self.assertEqual("http://localhost:2379/v3alpha/lease/grant",
client.get_url("/lease/grant"))
def test_client_version_discovery_fail(self):
client = Etcd3Client()
with mock.patch.object(client, "session") as mock_session:
mock_response = mock.Mock()
mock_response.status_code = 500
mock_response.reason = "Internal Server Error"
mock_session.get.return_value = mock_response
self.assertRaises(
Etcd3Exception,
client.get_url, "/lease/grant")
def test_client_version_discovery_version_absent(self):
client = Etcd3Client()
with mock.patch.object(client, "session") as mock_session:
mock_response = mock.Mock()
mock_response.status_code = 200
mock_response.reason = "OK"
mock_response.json.return_value = {}
mock_session.get.return_value = mock_response
self.assertRaises(
ApiVersionDiscoveryFailedError,
client.get_url, "/lease/grant")
def test_client_version_discovery_version_malformed(self):
client = Etcd3Client()
with mock.patch.object(client, "session") as mock_session:
mock_response = mock.Mock()
mock_response.status_code = 200
mock_response.reason = "OK"
mock_response.json.return_value = {
"etcdserver": "3.2.a",
"etcdcluster": "3.0.0"
}
mock_session.get.return_value = mock_response
self.assertRaises(
ApiVersionDiscoveryFailedError,
client.get_url, "/lease/grant")
def test_client_api_path(self):
client = Etcd3Client(api_path='/v3/')
self.assertEqual("http://localhost:2379",
client.base_url)
self.assertEqual("http://localhost:2379/v3/lease/grant",
client.get_url("/lease/grant"))
def test_client_ipv4(self):
client = Etcd3Client(host="127.0.0.1")
self.assertEqual("http://127.0.0.1:2379%slease/grant" %
client.api_path,
client = Etcd3Client(host="127.0.0.1", api_path='/v3/')
self.assertEqual("http://127.0.0.1:2379",
client.base_url)
self.assertEqual("http://127.0.0.1:2379/v3/lease/grant",
client.get_url("/lease/grant"))
def test_client_ipv6(self):
client = Etcd3Client(host="::1")
self.assertEqual("http://[::1]:2379%slease/grant" %
client.api_path,
client = Etcd3Client(host="::1", api_path='/v3/')
self.assertEqual("http://[::1]:2379",
client.base_url)
self.assertEqual("http://[::1]:2379/v3/lease/grant",
client.get_url("/lease/grant"))
def test_client_bad_request(self):
client = Etcd3Client(host="127.0.0.1")
client = Etcd3Client(host="127.0.0.1", api_path='/v3/')
with mock.patch.object(client, "session") as mock_session:
mock_response = mock.Mock()
mock_response.status_code = 400
@ -60,7 +150,7 @@ class TestEtcd3Gateway(base.TestCase):
}''')
def test_client_exceptions_by_code(self):
client = Etcd3Client(host="127.0.0.1")
client = Etcd3Client(host="127.0.0.1", api_path='/v3/')
with mock.patch.object(client, "session") as mock_session:
mock_response = mock.Mock()
mock_response.status_code = 500
@ -77,8 +167,3 @@ class TestEtcd3Gateway(base.TestCase):
self.assertEqual(e.detail_text, '''{
"error": "etcdserver: unable to reach quorum"
}''')
def test_client_api_path(self):
client = Etcd3Client(host="127.0.0.1", api_path='/v3/')
self.assertEqual("http://127.0.0.1:2379/v3/lease/grant",
client.get_url("/lease/grant"))

2
etcd3gw/tests/test_etcd3gw.py
View File

@ -51,7 +51,7 @@ def _is_etcd3_running():
class TestEtcd3Gateway(base.TestCase):
@classmethod
def setUpClass(cls):
cls.client = Etcd3Client()
cls.client = Etcd3Client(api_path='/v3/')
@unittest.skipUnless(
_is_etcd3_running(), "etcd3 is not available")

11
releasenotes/notes/api-version-discovery-2acf4ffb64f1faa7.yaml Normal file
View File

@ -0,0 +1,11 @@
---
features:
- |
The ``Etcd3Client`` class now automatically discovers available API version
and determines the api path. It detects the appropriate api path before
sending its first request, and the api path is reused for its subsequent
requests. The client instance needs to be recreated, or the service using
the instance needs to be restarted, after its backend etcd server is
upgraded, so that the new api path is detected. The detection is skipped
if the `api_path` argument is set when creating a class or
the `ETCD3GW_API_PATH` environment is set.

4
test-requirements.txt
View File

@ -14,3 +14,7 @@ pifpaf>=0.10.0 # Apache-2.0
nose>=1.3.7 # GNU LGPL
pytest>=3.0.0 # MIT
urllib3>=1.15.1 # MIT
# TODO(tkajinam): Remove this once the following change is released
# https://github.com/testing-cabal/testrepository/pull/48
extras>=1.0.0 # MIT