From 8e068099deed0863cd83987dc856e81ef196b7eb Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Fri, 18 Oct 2019 10:01:19 +0000 Subject: [PATCH] Add unit tests for Nova Cells filters This moves the Nova Cells filters alongside the service filters for ease of testing. Partially Implements: blueprint support-nova-cells Change-Id: I32d35c065812c6b46c64bacdf283a0bdad0f8a0f --- .../roles/nova-cell/filter_plugins/filters.py | 81 +------- kolla_ansible/nova_filters.py | 93 +++++++++ .../nova_manage_cli_output_duplicate_cells | 7 + .../nova_manage_cli_output_multiple_cells | 8 + kolla_ansible/tests/unit/test_nova_filters.py | 189 ++++++++++++++++++ 5 files changed, 303 insertions(+), 75 deletions(-) create mode 100644 kolla_ansible/nova_filters.py create mode 100644 kolla_ansible/tests/unit/data/nova_manage_cli_output_duplicate_cells create mode 100644 kolla_ansible/tests/unit/data/nova_manage_cli_output_multiple_cells create mode 100644 kolla_ansible/tests/unit/test_nova_filters.py diff --git a/ansible/roles/nova-cell/filter_plugins/filters.py b/ansible/roles/nova-cell/filter_plugins/filters.py index 95f2f07cff..48c9ce8f3f 100644 --- a/ansible/roles/nova-cell/filter_plugins/filters.py +++ b/ansible/roles/nova-cell/filter_plugins/filters.py @@ -1,5 +1,7 @@ #!/usr/bin/python +# Copyright (c) 2019 StackHPC Ltd. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -12,82 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from jinja2.exceptions import TemplateRuntimeError -import re +from kolla_ansible import nova_filters as filters class FilterModule(object): + """Nova cell filters.""" + def filters(self): - return { - 'extract_cell': self._extract_cell, - 'namespace_haproxy_for_cell': self._namespace_haproxy_for_cell, - } - - def _extract_cell(self, list_cells_cli_output, cell_name): - """Extract cell settings from nova_manage CLI - - This filter tries to extract the cell settings for the specified cell - from the output of the command: - nova-manage cell_v2 list_cells --verbose - If the cell is not registered, nothing is returned. - - An example line from this command for a cell with no name looks like this: - - | | 68a3f49e-27ec-422f-9e2e-2a4e5dc8291b | rabbit://openstack:password@1.2.3.4:5672 | mysql+pymysql://nova:password@1.2.3.4:3306/nova | False | # noqa - - And for a cell with a name: - - | cell1 | 68a3f49e-27ec-422f-9e2e-2a4e5dc8291b | rabbit://openstack:password@1.2.3.4:5672 | mysql+pymysql://nova:password@1.2.3.4:3306/nova | False | # noqa - - """ - # NOTE(priteau): regexp doesn't support passwords containing spaces - p = re.compile( - r'\| +(?P[^ ]+)? +' - r'\| +(?!00000000-0000-0000-0000-000000000000)' - r'(?P[0-9a-f\-]+) +' - r'\| +(?P[^ ]+) +' - r'\| +(?P[^ ]+) +' - r'\| +(?P[^ ]+) +' - r'\|$') - cells = [] - for line in list_cells_cli_output['stdout_lines']: - match = p.match(line) - if match: - # If there is no cell name, we get None in the cell_name match - # group. Use an empty string to match the default cell. - match_cell_name = match.group('cell_name') or "" - if match_cell_name == cell_name: - cells.append(match.groupdict()) - if len(cells) > 1: - raise TemplateRuntimeError( - "Cell: {} has duplicates. " - "Manual cleanup required.".format(cell_name)) - return cells[0] if cells else None - - def _namespace_haproxy_for_cell(self, services, cell_name): - """Add namespacing to HAProxy configuration for a cell. - - :param services: dict defining service configuration. - :param cell_name: name of the cell, or empty if cell has no name. - :returns: the services dict, with haproxy configuration modified to - provide namespacing between cells. - """ - - def _namespace(name): - # Backwards compatibility - no cell name suffix for cells without a - # name. - return "{}_{}".format(name, cell_name) if cell_name else name - - # Service name must be namespaced as haproxy-config uses this as the - # config file name. - services = { - _namespace(service_name): service - for service_name, service in services.items() - } - for service in services.values(): - if service.get('haproxy'): - service['haproxy'] = { - _namespace(name): service['haproxy'][name] - for name in service['haproxy'] - } - return services + return filters.get_filters() diff --git a/kolla_ansible/nova_filters.py b/kolla_ansible/nova_filters.py new file mode 100644 index 0000000000..fc2be76f45 --- /dev/null +++ b/kolla_ansible/nova_filters.py @@ -0,0 +1,93 @@ +# Copyright (c) 2019 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import jinja2 +import re + + +def extract_cell(list_cells_cli_output, cell_name): + """Extract cell settings from nova_manage CLI + + This filter tries to extract the cell settings for the specified cell + from the output of the command: + nova-manage cell_v2 list_cells --verbose + If the cell is not registered, nothing is returned. + + An example line from this command for a cell with no name looks like this: + + | | 68a3f49e-27ec-422f-9e2e-2a4e5dc8291b | rabbit://openstack:password@1.2.3.4:5672 | mysql+pymysql://nova:password@1.2.3.4:3306/nova | False | # noqa + + And for a cell with a name: + + | cell1 | 68a3f49e-27ec-422f-9e2e-2a4e5dc8291b | rabbit://openstack:password@1.2.3.4:5672 | mysql+pymysql://nova:password@1.2.3.4:3306/nova | False | # noqa + + """ + # NOTE(priteau): regexp doesn't support passwords containing spaces + p = re.compile( + r'\| +(?P[^ ]+)? +' + r'\| +(?!00000000-0000-0000-0000-000000000000)' + r'(?P[0-9a-f\-]+) +' + r'\| +(?P[^ ]+) +' + r'\| +(?P[^ ]+) +' + r'\| +(?P[^ ]+) +' + r'\|$') + cells = [] + for line in list_cells_cli_output['stdout_lines']: + match = p.match(line) + if match: + # If there is no cell name, we get None in the cell_name match + # group. Use an empty string to match the default cell. + match_cell_name = match.group('cell_name') or "" + if match_cell_name == cell_name: + cells.append(match.groupdict()) + if len(cells) > 1: + raise jinja2.TemplateRuntimeError( + "Cell: {} has duplicates. " + "Manual cleanup required.".format(cell_name)) + return cells[0] if cells else None + + +def namespace_haproxy_for_cell(services, cell_name): + """Add namespacing to HAProxy configuration for a cell. + + :param services: dict defining service configuration. + :param cell_name: name of the cell, or empty if cell has no name. + :returns: the services dict, with haproxy configuration modified to + provide namespacing between cells. + """ + def _namespace(name): + # Backwards compatibility - no cell name suffix for cells without a + # name. + return "{}_{}".format(name, cell_name) if cell_name else name + + # Service name must be namespaced as haproxy-config uses this as the + # config file name. + services = { + _namespace(service_name): service + for service_name, service in services.items() + } + for service in services.values(): + if service.get('haproxy'): + service['haproxy'] = { + _namespace(name): service['haproxy'][name] + for name in service['haproxy'] + } + return services + + +def get_filters(): + return { + "extract_cell": extract_cell, + "namespace_haproxy_for_cell": namespace_haproxy_for_cell, + } diff --git a/kolla_ansible/tests/unit/data/nova_manage_cli_output_duplicate_cells b/kolla_ansible/tests/unit/data/nova_manage_cli_output_duplicate_cells new file mode 100644 index 0000000000..dde0899cb9 --- /dev/null +++ b/kolla_ansible/tests/unit/data/nova_manage_cli_output_duplicate_cells @@ -0,0 +1,7 @@ ++----------+--------------------------------------+----------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------+----------+ +| Name | UUID | Transport URL | Database Connection | Disabled | ++----------+--------------------------------------+----------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------+----------+ +| cell0 | 00000000-0000-0000-0000-000000000000 | none:/// | mysql+pymysql://nova:gTEDJEJU24ZXJjhhEt6Xu4nP3fnvPAoQ5oOb8OnS@172.28.128.253:3306/nova_cell0 | False | +| cell0001 | 5af50f3f-82ed-47b2-9de8-7dd4eafa7648 | rabbit://openstack:0flZc3qqwczsisbaT94uCcArVNXEJnhhj4JAAAMI@172.28.128.32:5673/nova_cell0001 | mysql+pymysql://nova:mpyerUgpbQeXjaDV1nhsVDLWjsFSoCss6dHCXK7G@172.28.128.252:3305/nova_cell0001 | False | +| cell0001 | 5af50f3f-82ed-47b2-9de8-7dd4eafa7648 | rabbit://openstack:0flZc3qqwczsisbaT94uCcArVNXEJnhhj4JAAAMI@172.28.128.32:5673/nova_cell0001 | mysql+pymysql://nova:mpyerUgpbQeXjaDV1nhsVDLWjsFSoCss6dHCXK7G@172.28.128.252:3305/nova_cell0001 | False | ++----------+--------------------------------------+----------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------+----------+ diff --git a/kolla_ansible/tests/unit/data/nova_manage_cli_output_multiple_cells b/kolla_ansible/tests/unit/data/nova_manage_cli_output_multiple_cells new file mode 100644 index 0000000000..061399a067 --- /dev/null +++ b/kolla_ansible/tests/unit/data/nova_manage_cli_output_multiple_cells @@ -0,0 +1,8 @@ ++----------+--------------------------------------+----------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------+----------+ +| Name | UUID | Transport URL | Database Connection | Disabled | ++----------+--------------------------------------+----------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------+----------+ +| cell0 | 00000000-0000-0000-0000-000000000000 | none:/// | mysql+pymysql://nova:gTEDJEJU24ZXJjhhEt6Xu4nP3fnvPAoQ5oOb8OnS@172.28.128.253:3306/nova_cell0 | False | +| | 68a3f49e-27ec-422f-9e2e-2a4e5dc8291b | rabbit://openstack:94uCcArVNXEJnhhj4JAAAMI432h5j3k2bb3bbjkB@172.28.128.30:5672 | mysql+pymysql://nova:1nhsVDLWjsFSoCsda453bJBsdsbjkabf77sadsdD@172.28.128.252:3305/nova | False | +| cell0001 | 5af50f3f-82ed-47b2-9de8-7dd4eafa7648 | rabbit://openstack:0flZc3qqwczsisbaT94uCcArVNXEJnhhj4JAAAMI@172.28.128.32:5673/nova_cell0001 | mysql+pymysql://nova:mpyerUgpbQeXjaDV1nhsVDLWjsFSoCss6dHCXK7G@172.28.128.252:3305/nova_cell0001 | False | +| cell0002 | 087697bb-bfc2-471d-befb-0c0fcc7630e4 | rabbit://openstack:d9LaCxGrQX9Lla9aMZcS7fQ5xLH8S5HwHFrl6jdJ@172.28.128.31:5672/nova_cell0002 | mysql+pymysql://nova:Dj3f3l4yv2SuhbsJyv3ahGIwUMa9TKchw6EXtQfp@172.28.128.253:3306/nova_cell0002 | True | ++----------+--------------------------------------+----------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------+----------+ diff --git a/kolla_ansible/tests/unit/test_nova_filters.py b/kolla_ansible/tests/unit/test_nova_filters.py new file mode 100644 index 0000000000..128ee67c57 --- /dev/null +++ b/kolla_ansible/tests/unit/test_nova_filters.py @@ -0,0 +1,189 @@ +# Copyright (c) 2019 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + +import jinja2 +import os + +from kolla_ansible import nova_filters as filters + + +class TestFilters(unittest.TestCase): + + def _test_extract_cell(self, test_data, cell_name): + nova_manage_output = {} + with open(test_data, 'r') as f: + nova_manage_output['stdout_lines'] = f.readlines() + return filters.extract_cell(nova_manage_output, cell_name) + + def test_extract_with_no_name(self): + test_data = os.path.join(os.path.dirname( + __file__), 'data', 'nova_manage_cli_output_multiple_cells') + actual = self._test_extract_cell(test_data, '') + expected = {'cell_name': None, + 'cell_uuid': '68a3f49e-27ec-422f-9e2e-2a4e5dc8291b', + 'cell_message_queue': 'rabbit://openstack:94uCcArVNXEJnhhj4JAAAMI432h5j3k2bb3bbjkB@172.28.128.30:5672', # noqa + 'cell_database': 'mysql+pymysql://nova:1nhsVDLWjsFSoCsda453bJBsdsbjkabf77sadsdD@172.28.128.252:3305/nova', # noqa + 'cell_disabled': 'False'} + self.assertDictEqual(expected, actual) + + def test_extract_cell0001(self): + test_data = os.path.join(os.path.dirname( + __file__), 'data', 'nova_manage_cli_output_multiple_cells') + actual = self._test_extract_cell(test_data, 'cell0001') + expected = {'cell_name': 'cell0001', + 'cell_uuid': '5af50f3f-82ed-47b2-9de8-7dd4eafa7648', + 'cell_message_queue': 'rabbit://openstack:0flZc3qqwczsisbaT94uCcArVNXEJnhhj4JAAAMI@172.28.128.32:5673/nova_cell0001', # noqa + 'cell_database': 'mysql+pymysql://nova:mpyerUgpbQeXjaDV1nhsVDLWjsFSoCss6dHCXK7G@172.28.128.252:3305/nova_cell0001', # noqa + 'cell_disabled': 'False'} + self.assertDictEqual(expected, actual) + + def test_extract_cell0002(self): + test_data = os.path.join(os.path.dirname( + __file__), 'data', 'nova_manage_cli_output_multiple_cells') + actual = self._test_extract_cell(test_data, 'cell0002') + expected = {'cell_name': 'cell0002', + 'cell_uuid': '087697bb-bfc2-471d-befb-0c0fcc7630e4', + 'cell_message_queue': 'rabbit://openstack:d9LaCxGrQX9Lla9aMZcS7fQ5xLH8S5HwHFrl6jdJ@172.28.128.31:5672/nova_cell0002', # noqa + 'cell_database': 'mysql+pymysql://nova:Dj3f3l4yv2SuhbsJyv3ahGIwUMa9TKchw6EXtQfp@172.28.128.253:3306/nova_cell0002', # noqa + 'cell_disabled': 'True'} + self.assertDictEqual(expected, actual) + + def test_extract_missing_cell(self): + test_data = os.path.join(os.path.dirname( + __file__), 'data', 'nova_manage_cli_output_multiple_cells') + actual = self._test_extract_cell(test_data, 'cell0003') + self.assertIsNone(actual) + + def test_extract_duplicate_cell(self): + test_data = os.path.join(os.path.dirname( + __file__), 'data', 'nova_manage_cli_output_duplicate_cells') + self.assertRaisesRegexp(jinja2.TemplateRuntimeError, 'duplicates', + self._test_extract_cell, test_data, 'cell0001') + + def test_namespace_haproxy_for_cell_with_empty_name(self): + example_services = { + 'nova-novncproxy': { + 'group': 'some_group', + 'enabled': True, + 'haproxy': { + 'nova_novncproxy': { + 'enabled': True, + 'mode': 'http', + 'external': False, + 'port': '1232', + 'listen_port': '1233', + 'backend_http_extra': ['timeout tunnel 1h'], + }, + 'nova_novncproxy_external': { + 'enabled': True, + 'mode': 'http', + 'external': True, + 'port': '1234', + 'listen_port': '1235', + 'backend_http_extra': ['timeout tunnel 1h'], + } + } + } + } + actual = filters.namespace_haproxy_for_cell(example_services, '') + # No change + self.assertDictEqual(example_services, actual) + + def test_namespace_haproxy_for_cell_with_single_proxy(self): + example_services = { + 'nova-novncproxy': { + 'group': 'some_group', + 'enabled': True, + 'haproxy': { + 'nova_novncproxy': { + 'enabled': True, + 'mode': 'http', + 'external': False, + 'port': '1232', + 'listen_port': '1233', + 'backend_http_extra': ['timeout tunnel 1h'], + }, + 'nova_novncproxy_external': { + 'enabled': True, + 'mode': 'http', + 'external': True, + 'port': '1234', + 'listen_port': '1235', + 'backend_http_extra': ['timeout tunnel 1h'], + } + } + } + } + actual = filters.namespace_haproxy_for_cell( + example_services, 'cell0001') + expected = { + 'nova-novncproxy_cell0001': { + 'group': 'some_group', + 'enabled': True, + 'haproxy': { + 'nova_novncproxy_cell0001': { + 'enabled': True, + 'mode': 'http', + 'external': False, + 'port': '1232', + 'listen_port': '1233', + 'backend_http_extra': ['timeout tunnel 1h'], + }, + 'nova_novncproxy_external_cell0001': { + 'enabled': True, + 'mode': 'http', + 'external': True, + 'port': '1234', + 'listen_port': '1235', + 'backend_http_extra': ['timeout tunnel 1h'], + } + } + } + } + self.assertDictEqual(expected, actual) + + def test_namespace_haproxy_for_cell_with_multiple_proxies(self): + example_services = { + 'nova-novncproxy': { + 'haproxy': { + 'nova_novncproxy': {}, + 'nova_novncproxy_external': {} + } + }, + 'nova-spicehtml5proxy': { + 'haproxy': { + 'nova_spicehtml5proxy': {}, + 'nova_spicehtml5proxy_external': {} + } + } + } + actual = filters.namespace_haproxy_for_cell( + example_services, 'cell0002') + expected = { + 'nova-novncproxy_cell0002': { + 'haproxy': { + 'nova_novncproxy_cell0002': {}, + 'nova_novncproxy_external_cell0002': {} + } + }, + 'nova-spicehtml5proxy_cell0002': { + 'haproxy': { + 'nova_spicehtml5proxy_cell0002': {}, + 'nova_spicehtml5proxy_external_cell0002': {} + } + } + } + self.assertDictEqual(expected, actual)