From e5faa53eada8b17795b2e68fdffb226aaceb0f92 Mon Sep 17 00:00:00 2001 From: Takashi Kajinami Date: Fri, 17 Nov 2023 22:53:59 +0900 Subject: [PATCH] Discover API version automatically etcd changed API version from v3alpha to v3beta, then v3, and because of transition available api versions are different according to the etcd server available. This change implements the mechanism to detect the current api version according to the etcd version obtained via the version API. Discovery is executed at the first request and the result is stored. Closes-Bug: #2043810 Change-Id: I08bc429dd8c59e5d2b6e62f8f48afd78a116fbea --- etcd3gw/client.py | 65 ++++++++-- etcd3gw/exceptions.py | 4 + etcd3gw/tests/test_client.py | 117 +++++++++++++++--- etcd3gw/tests/test_etcd3gw.py | 2 +- ...pi-version-discovery-2acf4ffb64f1faa7.yaml | 11 ++ test-requirements.txt | 4 + 6 files changed, 177 insertions(+), 26 deletions(-) create mode 100644 releasenotes/notes/api-version-discovery-2acf4ffb64f1faa7.yaml diff --git a/etcd3gw/client.py b/etcd3gw/client.py index 43690ce..8e2d233 100644 --- a/etcd3gw/client.py +++ b/etcd3gw/client.py @@ -38,7 +38,7 @@ _EXCEPTIONS_BY_CODE = { requests.codes['precondition_failed']: exceptions.PreconditionFailedError, } -DEFAULT_API_PATH = os.getenv('ETCD3GW_API_PATH', '/v3alpha/') +DEFAULT_API_PATH = os.getenv('ETCD3GW_API_PATH') class Etcd3Client(object): @@ -62,7 +62,47 @@ class Etcd3Client(object): self.session.verify = ca_cert if cert_cert is not None and cert_key is not None: self.session.cert = (cert_cert, cert_key) - self.api_path = api_path + self._api_path = api_path + + @property + def api_path(self): + if self._api_path is not None: + return self._api_path + self._discover_api_path() + return self._api_path + + @property + def base_url(self): + host = ('[' + self.host + ']' if (self.host.find(':') != -1) + else self.host) + return self.protocol + '://' + host + ':' + str(self.port) + + def _discover_api_path(self): + """Discover api version and set api_path + + """ + resp = self._request('get', self.base_url + '/version') + try: + version_str = resp['etcdserver'] + except KeyError: + raise exceptions.ApiVersionDiscoveryFailedError( + 'Malformed response from version API') + + try: + version = tuple(int(part) for part in version_str.split('.', 2)) + except ValueError: + raise exceptions.ApiVersionDiscoveryFailedError( + 'Failed to parse etcd cluster version: %s' % version_str) + + # NOTE(tkajinam): https://etcd.io/docs/v3.5/dev-guide/api_grpc_gateway/ + # explains mapping between etcd version and available + # api versions + if version >= (3, 4): + self._api_path = '/v3/' + elif version >= (3, 3): + self._api_path = '/v3beta/' + else: + self._api_path = '/v3alpha/' def get_url(self, path): """Construct a full url to the v3 API given a specific path @@ -70,20 +110,18 @@ class Etcd3Client(object): :param path: :return: url """ - host = ('[' + self.host + ']' if (self.host.find(':') != -1) - else self.host) - base_url = self.protocol + '://' + host + ':' + str(self.port) - return base_url + self.api_path + path.lstrip("/") - def post(self, *args, **kwargs): - """helper method for HTTP POST + return self.base_url + self.api_path + path.lstrip("/") + + def _request(self, method, *args, **kwargs): + """helper method for HTTP requests :param args: :param kwargs: :return: json response """ try: - resp = self.session.post(*args, **kwargs) + resp = getattr(self.session, method)(*args, **kwargs) if resp.status_code in _EXCEPTIONS_BY_CODE: raise _EXCEPTIONS_BY_CODE[resp.status_code]( resp.text, @@ -97,6 +135,15 @@ class Etcd3Client(object): raise exceptions.ConnectionFailedError(str(ex)) return resp.json() + def post(self, *args, **kwargs): + """helper method for HTTP POST + + :param args: + :param kwargs: + :return: json response + """ + return self._request('post', *args, **kwargs) + def status(self): """Status gets the status of the etcd cluster member. diff --git a/etcd3gw/exceptions.py b/etcd3gw/exceptions.py index cd9134a..4294ff1 100644 --- a/etcd3gw/exceptions.py +++ b/etcd3gw/exceptions.py @@ -35,3 +35,7 @@ class ConnectionTimeoutError(Etcd3Exception): class PreconditionFailedError(Etcd3Exception): pass + + +class ApiVersionDiscoveryFailedError(Etcd3Exception): + pass diff --git a/etcd3gw/tests/test_client.py b/etcd3gw/tests/test_client.py index 7c36c94..2290f48 100644 --- a/etcd3gw/tests/test_client.py +++ b/etcd3gw/tests/test_client.py @@ -13,6 +13,7 @@ from unittest import mock from etcd3gw.client import Etcd3Client +from etcd3gw.exceptions import ApiVersionDiscoveryFailedError from etcd3gw.exceptions import Etcd3Exception from etcd3gw.exceptions import InternalServerError from etcd3gw.tests import base @@ -20,26 +21,115 @@ from etcd3gw.tests import base class TestEtcd3Gateway(base.TestCase): - def test_client_default(self): + def test_client_version_discovery(self): client = Etcd3Client() - self.assertEqual("http://localhost:2379%slease/grant" % - client.api_path, + with mock.patch.object(client, "session") as mock_session: + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.reason = "OK" + mock_response.json.return_value = { + "etcdserver": "3.4.0", + "etcdcluster": "3.0.0" + } + mock_session.get.return_value = mock_response + self.assertEqual("http://localhost:2379", + client.base_url) + self.assertEqual("http://localhost:2379/v3/lease/grant", + client.get_url("/lease/grant")) + + def test_client_version_discovery_v3beta(self): + client = Etcd3Client() + with mock.patch.object(client, "session") as mock_session: + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.reason = "OK" + mock_response.json.return_value = { + "etcdserver": "3.3.0", + "etcdcluster": "3.0.0" + } + mock_session.get.return_value = mock_response + self.assertEqual("http://localhost:2379", + client.base_url) + self.assertEqual("http://localhost:2379/v3beta/lease/grant", + client.get_url("/lease/grant")) + + def test_client_version_discovery_v3alpha(self): + client = Etcd3Client() + with mock.patch.object(client, "session") as mock_session: + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.reason = "OK" + mock_response.json.return_value = { + "etcdserver": "3.2.0", + "etcdcluster": "3.0.0" + } + mock_session.get.return_value = mock_response + self.assertEqual("http://localhost:2379", + client.base_url) + self.assertEqual("http://localhost:2379/v3alpha/lease/grant", + client.get_url("/lease/grant")) + + def test_client_version_discovery_fail(self): + client = Etcd3Client() + with mock.patch.object(client, "session") as mock_session: + mock_response = mock.Mock() + mock_response.status_code = 500 + mock_response.reason = "Internal Server Error" + mock_session.get.return_value = mock_response + self.assertRaises( + Etcd3Exception, + client.get_url, "/lease/grant") + + def test_client_version_discovery_version_absent(self): + client = Etcd3Client() + with mock.patch.object(client, "session") as mock_session: + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.reason = "OK" + mock_response.json.return_value = {} + mock_session.get.return_value = mock_response + self.assertRaises( + ApiVersionDiscoveryFailedError, + client.get_url, "/lease/grant") + + def test_client_version_discovery_version_malformed(self): + client = Etcd3Client() + with mock.patch.object(client, "session") as mock_session: + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.reason = "OK" + mock_response.json.return_value = { + "etcdserver": "3.2.a", + "etcdcluster": "3.0.0" + } + mock_session.get.return_value = mock_response + self.assertRaises( + ApiVersionDiscoveryFailedError, + client.get_url, "/lease/grant") + + def test_client_api_path(self): + client = Etcd3Client(api_path='/v3/') + self.assertEqual("http://localhost:2379", + client.base_url) + self.assertEqual("http://localhost:2379/v3/lease/grant", client.get_url("/lease/grant")) def test_client_ipv4(self): - client = Etcd3Client(host="127.0.0.1") - self.assertEqual("http://127.0.0.1:2379%slease/grant" % - client.api_path, + client = Etcd3Client(host="127.0.0.1", api_path='/v3/') + self.assertEqual("http://127.0.0.1:2379", + client.base_url) + self.assertEqual("http://127.0.0.1:2379/v3/lease/grant", client.get_url("/lease/grant")) def test_client_ipv6(self): - client = Etcd3Client(host="::1") - self.assertEqual("http://[::1]:2379%slease/grant" % - client.api_path, + client = Etcd3Client(host="::1", api_path='/v3/') + self.assertEqual("http://[::1]:2379", + client.base_url) + self.assertEqual("http://[::1]:2379/v3/lease/grant", client.get_url("/lease/grant")) def test_client_bad_request(self): - client = Etcd3Client(host="127.0.0.1") + client = Etcd3Client(host="127.0.0.1", api_path='/v3/') with mock.patch.object(client, "session") as mock_session: mock_response = mock.Mock() mock_response.status_code = 400 @@ -60,7 +150,7 @@ class TestEtcd3Gateway(base.TestCase): }''') def test_client_exceptions_by_code(self): - client = Etcd3Client(host="127.0.0.1") + client = Etcd3Client(host="127.0.0.1", api_path='/v3/') with mock.patch.object(client, "session") as mock_session: mock_response = mock.Mock() mock_response.status_code = 500 @@ -77,8 +167,3 @@ class TestEtcd3Gateway(base.TestCase): self.assertEqual(e.detail_text, '''{ "error": "etcdserver: unable to reach quorum" }''') - - def test_client_api_path(self): - client = Etcd3Client(host="127.0.0.1", api_path='/v3/') - self.assertEqual("http://127.0.0.1:2379/v3/lease/grant", - client.get_url("/lease/grant")) diff --git a/etcd3gw/tests/test_etcd3gw.py b/etcd3gw/tests/test_etcd3gw.py index 175157f..65a2785 100644 --- a/etcd3gw/tests/test_etcd3gw.py +++ b/etcd3gw/tests/test_etcd3gw.py @@ -51,7 +51,7 @@ def _is_etcd3_running(): class TestEtcd3Gateway(base.TestCase): @classmethod def setUpClass(cls): - cls.client = Etcd3Client() + cls.client = Etcd3Client(api_path='/v3/') @unittest.skipUnless( _is_etcd3_running(), "etcd3 is not available") diff --git a/releasenotes/notes/api-version-discovery-2acf4ffb64f1faa7.yaml b/releasenotes/notes/api-version-discovery-2acf4ffb64f1faa7.yaml new file mode 100644 index 0000000..4e6e5dd --- /dev/null +++ b/releasenotes/notes/api-version-discovery-2acf4ffb64f1faa7.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + The ``Etcd3Client`` class now automatically discovers available API version + and determines the api path. It detects the appropriate api path before + sending its first request, and the api path is reused for its subsequent + requests. The client instance needs to be recreated, or the service using + the instance needs to be restarted, after its backend etcd server is + upgraded, so that the new api path is detected. The detection is skipped + if the `api_path` argument is set when creating a class or + the `ETCD3GW_API_PATH` environment is set. diff --git a/test-requirements.txt b/test-requirements.txt index 26be395..962a6c4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -14,3 +14,7 @@ pifpaf>=0.10.0 # Apache-2.0 nose>=1.3.7 # GNU LGPL pytest>=3.0.0 # MIT urllib3>=1.15.1 # MIT + +# TODO(tkajinam): Remove this once the following change is released +# https://github.com/testing-cabal/testrepository/pull/48 +extras>=1.0.0 # MIT