From 9cc097f06e5563ab88b6c5baca0d4e0cf8fc7e23 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 17 Sep 2019 17:06:26 +0200 Subject: [PATCH] Shared-DB Provides Enable the provides side of the interface with an Endpoint implementation. Change-Id: If218a23445709543426de0501765ed24eb8af308 --- .gitignore | 2 + .stestr.conf | 3 + .zuul.yaml | 3 +- README.md | 28 ++++ interface.yaml | 9 ++ provides.py | 94 +++++++++++++ test-requirements.txt | 5 +- tox.ini | 31 ++--- unit_tests/__init__.py | 22 +++ unit_tests/test_provides.py | 160 ++++++++++++++++++++++ unit_tests/test_requires.py | 260 ++++++++++++++++++++++++++++++++++++ 11 files changed, 600 insertions(+), 17 deletions(-) create mode 100644 .stestr.conf create mode 100644 unit_tests/__init__.py create mode 100644 unit_tests/test_provides.py create mode 100644 unit_tests/test_requires.py diff --git a/.gitignore b/.gitignore index 9dd3eb8..eda88d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .tox .testrepository +.stestr/ +__pycache__ diff --git a/.stestr.conf b/.stestr.conf new file mode 100644 index 0000000..5fcccac --- /dev/null +++ b/.stestr.conf @@ -0,0 +1,3 @@ +[DEFAULT] +test_path=./unit_tests +top_dir=./ diff --git a/.zuul.yaml b/.zuul.yaml index 23d3066..18efca1 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,3 +1,4 @@ - project: templates: - - python-charm-interface-jobs + - python35-charm-jobs + - openstack-python3-train-jobs diff --git a/README.md b/README.md index fc9df15..1c5b324 100644 --- a/README.md +++ b/README.md @@ -91,3 +91,31 @@ parameter of the configure method. def setup_database(database): database.configure('mydatabase', 'myusername', hostname='hostname.override') ``` + +## Provides + +The interface layer will set the following states, as appropriate: + + * `{relation_name}.connected` The relation is established, but the client + has not provided the database information yet. + * `{relation_name}.available` The requested information is complete. The DB, + user and hostname can be created. + * connection information is passed back to the client with the following method: + * `set_db_connection_info()` + +For example: + +```python +@reactive.when('leadership.is_leader') +@reactive.when('leadership.set.cluster-instances-clustered') +@reactive.when('shared-db.available') +def shared_db_respond(shared_db): + with charm.provide_charm_instance() as instance: + instance.create_databases_and_users(shared_db) + instance.assess_status() +``` + +The interface will automatically determine the network space binding on the +local unit to present to the remote mysql-shared client based on the name of +the relation. This can be overridden using the db_host parameter of the +set_db_connection_info method. diff --git a/interface.yaml b/interface.yaml index bc67cac..dcc71ed 100644 --- a/interface.yaml +++ b/interface.yaml @@ -1,3 +1,12 @@ name: mysql-shared summary: Interface for intergrating with MySQL maintainer: OpenStack Charmers +ignore: + - 'unit_tests' + - '.stestr.conf' + - 'test-requirements.txt' + - 'tox.ini' + - '.gitignore' + - '.travis.yml' + - '.zuul.yaml' + - '.tox' diff --git a/provides.py b/provides.py index e69de29..05125b3 100644 --- a/provides.py +++ b/provides.py @@ -0,0 +1,94 @@ +# Copyright 2019 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from charms import reactive +import charmhelpers.contrib.network.ip as ch_net_ip + + +class MySQLSharedProvides(reactive.Endpoint): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.ingress_address = ch_net_ip.get_relation_ip(self.endpoint_name) + + def relation_ids(self): + return [x.relation_id for x in self.relations] + + def set_ingress_address(self): + for relation in self.relations: + relation.to_publish_raw["ingress-address"] = self.ingress_address + relation.to_publish_raw["private-address"] = self.ingress_address + + def available(self): + for unit in self.all_joined_units: + if unit.received['username']: + return True + for key in unit.received.keys(): + if "_username" in key: + return True + return False + + @reactive.when('endpoint.{endpoint_name}.joined') + def joined(self): + reactive.clear_flag(self.expand_name('{endpoint_name}.available')) + reactive.set_flag(self.expand_name('{endpoint_name}.connected')) + self.set_ingress_address() + + @reactive.when('endpoint.{endpoint_name}.changed') + def changed(self): + flags = ( + self.expand_name( + 'endpoint.{endpoint_name}.changed.database'), + self.expand_name( + 'endpoint.{endpoint_name}.changed.username'), + self.expand_name( + 'endpoint.{endpoint_name}.changed.hostname'), + ) + if reactive.all_flags_set(*flags): + for flag in flags: + reactive.clear_flag(flag) + + if self.available(): + reactive.set_flag(self.expand_name('{endpoint_name}.available')) + else: + reactive.clear_flag(self.expand_name('{endpoint_name}.available')) + + @reactive.when_any('endpoint.{endpoint_name}.broken', + 'endpoint.{endpoint_name}.departed') + def departed(self): + flags = ( + self.expand_name('{endpoint_name}.connected'), + self.expand_name('{endpoint_name}.available'), + ) + for flag in flags: + reactive.clear_flag(flag) + + def set_db_connection_info( + self, relation_id, db_host, password, + allowed_units=None, prefix=None): + # Implementations of shared-db pre-date the json encoded era of + # interface layers. In order not to have to update dozens of charms, + # publish in raw data + + # Everyone gets db_host + self.relations[relation_id].to_publish_raw["db_host"] = db_host + if not prefix: + self.relations[relation_id].to_publish_raw["password"] = password + self.relations[relation_id].to_publish_raw[ + "allowed_units"] = allowed_units + else: + self.relations[relation_id].to_publish_raw[ + "{}_password".format(prefix)] = password + self.relations[relation_id].to_publish_raw[ + "{}_allowed_units".format(prefix)] = allowed_units diff --git a/test-requirements.txt b/test-requirements.txt index 095ec9c..3900c51 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,5 @@ +charms.reactive flake8>=2.2.4,<=2.4.1 -os-testr>=0.4.1 +mock>=1.2 +stestr>=2.2.0 +git+https://github.com/openstack/charms.openstack.git#egg=charms.openstack diff --git a/tox.ini b/tox.ini index f877d2c..07ad7a8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,36 +1,37 @@ [tox] -envlist = pep8,py27,py34,py35,py36 +envlist = pep8,py3 skipsdist = True -skip_missing_interpreters = True +# NOTE(beisner): Avoid build/test env pollution by not enabling sitepackages. +sitepackages = False +# NOTE(beisner): Avoid false positives by not skipping missing interpreters. +skip_missing_interpreters = False [testenv] setenv = VIRTUAL_ENV={envdir} PYTHONHASHSEED=0 install_command = pip install {opts} {packages} -commands = ostestr {posargs} - -[testenv:py27] -basepython = python2.7 +commands = stestr run {posargs} +passenv = HOME TERM deps = -r{toxinidir}/test-requirements.txt -# TODO: Need to write unit tests then remove the following command. -commands = /bin/true [testenv:py34] basepython = python3.4 -deps = -r{toxinidir}/test-requirements.txt -# TODO: Need to write unit tests then remove the following command. -commands = /bin/true [testenv:py35] basepython = python3.5 -deps = -r{toxinidir}/test-requirements.txt -# TODO: Need to write unit tests then remove the following command. -commands = /bin/true + +[testenv:py36] +basepython = python3.6 + +[testenv:py37] +basepython = python3.7 + +[testenv:py3] +basepython = python3 [testenv:pep8] basepython = python3 -deps = -r{toxinidir}/test-requirements.txt commands = flake8 {posargs} [testenv:venv] diff --git a/unit_tests/__init__.py b/unit_tests/__init__.py new file mode 100644 index 0000000..0df0b6a --- /dev/null +++ b/unit_tests/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2019 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +sys.path.append('src') +sys.path.append('src/lib') + +# Mock out charmhelpers so that we can test without it. +import charms_openstack.test_mocks # noqa +charms_openstack.test_mocks.mock_charmhelpers() diff --git a/unit_tests/test_provides.py b/unit_tests/test_provides.py new file mode 100644 index 0000000..393ea46 --- /dev/null +++ b/unit_tests/test_provides.py @@ -0,0 +1,160 @@ +# Copyright 2019 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import charms_openstack.test_utils as test_utils +import mock +import provides + + +class TestRegisteredHooks(test_utils.TestRegisteredHooks): + + def test_hooks(self): + defaults = [] + hook_set = { + "when": { + "joined": ( + "endpoint.{endpoint_name}.joined",), + + "changed": ( + "endpoint.{endpoint_name}.changed",), + "departed": ("endpoint.{endpoint_name}.broken", + "endpoint.{endpoint_name}.departed",), + }, + } + # test that the hooks were registered + self.registered_hooks_test_helper(provides, hook_set, defaults) + + +class TestMySQLSharedProvides(test_utils.PatchHelper): + + def setUp(self): + super().setUp() + self._patches = {} + self._patches_start = {} + self.patch_object(provides.reactive, "clear_flag") + self.patch_object(provides.reactive, "set_flag") + + self.fake_unit = mock.MagicMock() + self.fake_unit.unit_name = "myunit/4" + self.fake_unit.received = {"username": None} + + self.fake_relation_id = "shared-db:19" + self.fake_relation = mock.MagicMock() + self.fake_relation.relation_id = self.fake_relation_id + self.fake_relation.units = [self.fake_unit] + + self.ep_name = "ep" + self.ep = provides.MySQLSharedProvides( + self.ep_name, [self.fake_relation_id]) + self.ep.ingress_address = "10.10.10.10" + self.ep.relations[0] = self.fake_relation + + def tearDown(self): + self.ep = None + for k, v in self._patches.items(): + v.stop() + setattr(self, k, None) + self._patches = None + self._patches_start = None + + def test_joined(self): + self.ep.set_ingress_address = mock.MagicMock() + self.ep.joined() + self.clear_flag.assert_called_once_with( + "{}.available".format(self.ep_name)) + self.set_flag.assert_called_once_with( + "{}.connected".format(self.ep_name)) + self.ep.set_ingress_address.assert_called_once() + + def test_changed_not_available(self): + self.ep.available = mock.MagicMock() + self.ep.available.return_value = False + self.ep.changed() + + _calls = [ + mock.call("{}.available".format(self.ep_name)), + mock.call("endpoint.{}.changed.database".format(self.ep_name)), + mock.call("endpoint.{}.changed.username".format(self.ep_name)), + mock.call("endpoint.{}.changed.hostname".format(self.ep_name))] + self.clear_flag.assert_has_calls(_calls, any_order=True) + self.set_flag.assert_not_called() + + def test_changed_available(self): + self.ep.available = mock.MagicMock() + self.ep.available.return_value = True + self.ep.changed() + + _calls = [ + mock.call("endpoint.{}.changed.database".format(self.ep_name)), + mock.call("endpoint.{}.changed.username".format(self.ep_name)), + mock.call("endpoint.{}.changed.hostname".format(self.ep_name))] + self.clear_flag.assert_has_calls(_calls, any_order=True) + self.set_flag.assert_called_once_with( + "{}.available".format(self.ep_name)) + + def test_departed(self): + self.ep.departed() + _calls = [ + mock.call("{}.available".format(self.ep_name)), + mock.call("{}.connected".format(self.ep_name))] + self.clear_flag.assert_has_calls(_calls, any_order=True) + + def test_relation_ids(self): + self.assertEqual([self.fake_relation_id], self.ep.relation_ids()) + + def test_set_ingress_address(self): + _calls = [ + mock.call("ingress-address", self.ep.ingress_address), + mock.call("private-address", self.ep.ingress_address)] + self.ep.set_ingress_address() + self.fake_relation.to_publish_raw.__setitem__.assert_has_calls(_calls) + + def test_available_not_available(self): + self.assertFalse(self.ep.available()) + + def test_available_simple_available(self): + self.fake_unit.received = {"username": "user"} + self.assertTrue(self.ep.available()) + + def test_available_prefixed_available(self): + self.fake_unit.received["prefix_username"] = "user" + self.assertTrue(self.ep.available()) + + def test_set_db_connection_info_no_prefix(self): + _pw = "fakepassword" + self.ep.set_db_connection_info( + self.fake_relation_id, + self.ep.ingress_address, + _pw, + allowed_units=self.fake_unit.unit_name) + _calls = [ + mock.call("db_host", self.ep.ingress_address), + mock.call("password", _pw), + mock.call("allowed_units", self.fake_unit.unit_name)] + self.fake_relation.to_publish_raw.__setitem__.assert_has_calls(_calls) + + def test_set_db_connection_info_prefixed(self): + _p = "prefix" + _pw = "fakepassword" + self.ep.set_db_connection_info( + self.fake_relation_id, + self.ep.ingress_address, + _pw, + allowed_units=self.fake_unit.unit_name, + prefix=_p) + _calls = [ + mock.call("db_host", self.ep.ingress_address), + mock.call("{}_password".format(_p), _pw), + mock.call("{}_allowed_units".format(_p), self.fake_unit.unit_name)] + self.fake_relation.to_publish_raw.__setitem__.assert_has_calls(_calls) diff --git a/unit_tests/test_requires.py b/unit_tests/test_requires.py new file mode 100644 index 0000000..c74c098 --- /dev/null +++ b/unit_tests/test_requires.py @@ -0,0 +1,260 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest +import mock + +import requires + + +_hook_args = {} + + +def mock_hook(*args, **kwargs): + + def inner(f): + # remember what we were passed. Note that we can't actually determine + # the class we're attached to, as the decorator only gets the function. + _hook_args[f.__name__] = dict(args=args, kwargs=kwargs) + return f + return inner + + +class TestMySQLSharedRequires(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls._patched_hook = mock.patch('charms.reactive.hook', mock_hook) + cls._patched_hook_started = cls._patched_hook.start() + # force requires to rerun the mock_hook decorator: + # try except is Python2/Python3 compatibility as Python3 has moved + # reload to importlib. + try: + reload(requires) + except NameError: + import importlib + importlib.reload(requires) + + @classmethod + def tearDownClass(cls): + cls._patched_hook.stop() + cls._patched_hook_started = None + cls._patched_hook = None + # and fix any breakage we did to the module + try: + reload(requires) + except NameError: + import importlib + importlib.reload(requires) + + def setUp(self): + self._patches = {} + self._patches_start = {} + + self._rel_ids = ["mysql-shared:3"] + self._remote_data = {} + self._local_data = {} + + self._conversation = mock.MagicMock() + self._conversation.relation_ids = self._rel_ids + self._conversation.scope = requires.scopes.GLOBAL + self._conversation.get_remote.side_effect = self.get_fake_remote_data + self._conversation.get_local.side_effect = self.get_fake_local_data + + # The Relation object + self.mysql_shared = requires.MySQLSharedRequires( + 'mysql-shared', [self._conversation]) + self.patch_mysql_shared('conversations', [self._conversation]) + self.patch_mysql_shared('set_remote') + self.patch_mysql_shared('set_local') + self.patch_mysql_shared('set_state') + self.patch_mysql_shared('remove_state') + self.patch_mysql_shared('db_host', "10.5.0.21") + + def tearDown(self): + self.mysql_shared = None + for k, v in self._patches.items(): + v.stop() + setattr(self, k, None) + self._patches = None + self._patches_start = None + + def patch_mysql_shared(self, attr, return_value=None): + mocked = mock.patch.object(self.mysql_shared, attr) + self._patches[attr] = mocked + started = mocked.start() + started.return_value = return_value + self._patches_start[attr] = started + setattr(self, attr, started) + + def get_fake_remote_data(self, key, default=None): + return self._remote_data.get(key) or default + + def get_fake_local_data(self, key, default=None): + return self._local_data.get(key) or default + + def test_registered_hooks(self): + # test that the hooks actually registered the relation expressions that + # are meaningful for this interface: this is to handle regressions. + # The keys are the function names that the hook attaches to. + hook_patterns = { + 'joined': ('{requires:mysql-shared}-relation-joined',), + 'changed': ('{requires:mysql-shared}-relation-changed',), + 'departed': ( + '{requires:mysql-shared}-relation-{broken,departed}',)} + for k, v in _hook_args.items(): + self.assertEqual(hook_patterns[k], v['args']) + + def test_changed_available(self): + self.patch_mysql_shared('base_data_complete', True) + self.patch_mysql_shared('access_network_data_complete', True) + self.patch_mysql_shared('ssl_data_complete', True) + _calls = [ + mock.call("{relation_name}.available"), + mock.call("{relation_name}.available.access_network"), + mock.call("{relation_name}.available.ssl")] + self.mysql_shared.changed() + self.set_state.assert_has_calls(_calls) + + def test_changed_not_available(self): + self.patch_mysql_shared('base_data_complete', False) + self.patch_mysql_shared('access_network_data_complete', False) + self.patch_mysql_shared('ssl_data_complete', False) + self.mysql_shared.changed() + self.set_state.assert_not_called() + + def test_joined(self): + self.mysql_shared.joined() + self.set_state.assert_called_once_with('{relation_name}.connected') + + def test_departed(self): + self.mysql_shared.departed() + _calls = [ + mock.call("{relation_name}.available"), + mock.call("{relation_name}.available.access_network"), + mock.call("{relation_name}.available.ssl")] + self.remove_state.assert_has_calls(_calls) + + def test_base_data_complete(self): + self._remote_data = {"password": "1234", + "allowed_units": "unit/1"} + assert self.mysql_shared.base_data_complete() is True + self.db_host.return_value = None + assert self.mysql_shared.base_data_complete() is False + + def test_base_data_complete_prefixed(self): + self._local_data = {"prefixes": ["myprefix"]} + self._remote_data = {"myprefix_password": "1234", + "myprefix_allowed_units": "unit/1"} + assert self.mysql_shared.base_data_complete() is True + self.db_host.return_value = None + assert self.mysql_shared.base_data_complete() is False + + def test_base_data_incomplete(self): + assert self.mysql_shared.base_data_complete() is False + + def test_access_network_data_incomplete(self): + self.patch_mysql_shared('access_network', "10.92.3.0/24") + assert self.mysql_shared.access_network_data_complete() is True + self.access_network.return_value = None + assert self.mysql_shared.access_network_data_complete() is False + + def test_ssl_data_incomplete(self): + self.patch_mysql_shared('ssl_cert', "somecert") + self.patch_mysql_shared('ssl_key', "somekey") + assert self.mysql_shared.ssl_data_complete() is True + self.ssl_key.return_value = None + assert self.mysql_shared.ssl_data_complete() is False + + def test_local_accessors(self): + _prefix = "myprefix" + _value = "value" + _tests = { + "database": self.mysql_shared.database, + "username": self.mysql_shared.username, + "hostname": self.mysql_shared.hostname} + # Not set + for key, test in _tests.items(): + self.assertEqual(test(), None) + # Unprefixed + for key, test in _tests.items(): + self._local_data = {key: _value} + self.assertEqual(test(), _value) + # Prefixed + self._local_data = {"prefixes": [_prefix]} + for key, test in _tests.items(): + self._local_data["{}_{}".format(_prefix, key)] = _value + self.assertEqual(test(prefix=_prefix), _value) + + def test_remote_accessors(self): + _prefix = "myprefix" + _value = "value" + _tests = { + "password": self.mysql_shared.password, + "allowed_units": self.mysql_shared.allowed_units} + # Not set + for key, test in _tests.items(): + self.assertEqual(test(), None) + # Unprefixed + for key, test in _tests.items(): + self._remote_data = {key: _value} + self.assertEqual(test(), _value) + # Prefixed + self._local_data = {"prefixes": [_prefix]} + for key, test in _tests.items(): + self._remote_data = {"{}_{}".format(_prefix, key): _value} + self.assertEqual(test(prefix=_prefix), _value) + + def test_configure(self): + _db = "db" + _user = "user" + _host = "host" + _prefix = None + self.mysql_shared.configure(_db, _user, _host, prefix=_prefix) + self.set_remote.assert_called_once_with( + database=_db, username=_user, hostname=_host) + self.set_local.assert_called_once_with( + database=_db, username=_user, hostname=_host) + + def test_configure_prefixed(self): + self.patch_mysql_shared('set_prefix') + _db = "db" + _user = "user" + _host = "host" + _prefix = "prefix" + _expected = { + "{}_database".format(_prefix): _db, + "{}_username".format(_prefix): _user, + "{}_hostname".format(_prefix): _host} + self.mysql_shared.configure(_db, _user, _host, prefix=_prefix) + self.set_remote.assert_called_once_with(**_expected) + self.set_local.assert_called_once_with(**_expected) + self.set_prefix.assert_called_once() + + def test_get_prefix(self): + _prefix = "prefix" + self._local_data = {"prefixes": [_prefix]} + self.assertEqual( + self.mysql_shared.get_prefixes(), [_prefix]) + + def test_set_prefix(self): + # First + _prefix = "prefix" + self.mysql_shared.set_prefix(_prefix) + self.set_local.assert_called_once_with("prefixes", [_prefix]) + # More than one + self.set_local.reset_mock() + self._local_data = {"prefixes": [_prefix]} + _second = "secondprefix" + self.mysql_shared.set_prefix(_second) + self.set_local.assert_called_once_with("prefixes", [_prefix, _second])