Merge "Add custom filters for checking services"

.stestr.conf

@@ -1,4 +1,4 @@
 [DEFAULT]
-test_path=./tests
+test_path=./
 top_dir=./
 

ansible/filter_plugins/services.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2019 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from kolla_ansible import filters
+
+
+class FilterModule(object):
+    """Service filters."""
+
+    def filters(self):
+        return filters.get_filters()

doc/source/user/quickstart.rst

@@ -220,15 +220,15 @@ Install Kolla for development
 
    .. code-block:: console
 
-      pip install -r kolla/requirements.txt
-      pip install -r kolla-ansible/requirements.txt
+      pip install ./kolla
+      pip install ./kolla-ansible
 
    If not using a virtual environment:
 
    .. code-block:: console
 
-      sudo pip install -r kolla/requirements.txt
-      sudo pip install -r kolla-ansible/requirements.txt
+      sudo pip install ./kolla
+      sudo pip install ./kolla-ansible
 
 #. Create the ``/etc/kolla`` directory.
 

kolla_ansible/exception.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2019 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+try:
+    from ansible.errors import AnsibleFilterError
+except ImportError:
+    # NOTE(mgoddard): For unit testing we don't depend on Ansible since it is
+    # not in global requirements.
+    AnsibleFilterError = Exception
+
+
+class FilterError(AnsibleFilterError):
+    """Error during execution of a jinja2 filter."""

kolla_ansible/filters.py

@@ -0,0 +1,107 @@
+# Copyright (c) 2019 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import jinja2
+
+from kolla_ansible import exception
+
+
+def _call_bool_filter(context, value):
+    """Pass a value through the 'bool' filter.
+
+    :param context: Jinja2 Context object.
+    :param value: Value to pass through bool filter.
+    :returns: A boolean.
+    """
+    return context.environment.call_filter("bool", value, context=context)
+
+
+@jinja2.contextfilter
+def service_enabled(context, service):
+    """Return whether a service is enabled.
+
+    :param context: Jinja2 Context object.
+    :param service: Service definition, dict.
+    :returns: A boolean.
+    """
+    enabled = service.get('enabled')
+    if enabled is None:
+        raise exception.FilterError(
+            "Service definition for '%s' does not have an 'enabled' attribute"
+            % service.get("container_name", "<unknown>"))
+    return _call_bool_filter(context, enabled)
+
+
+@jinja2.contextfilter
+def service_mapped_to_host(context, service):
+    """Return whether a service is mapped to this host.
+
+    There are two ways to describe the service to host mapping. The most common
+    is via a 'group' attribute, where the service is mapped to all hosts in the
+    group. The second approach is via a 'host_in_groups' attribute, which is a
+    boolean expression which should be evaluated for every host. The latter
+    approach takes precedence over the first.
+
+    :param context: Jinja2 Context object.
+    :param service: Service definition, dict.
+    :returns: A boolean.
+    """
+    host_in_groups = service.get("host_in_groups")
+    if host_in_groups is not None:
+        return _call_bool_filter(context, host_in_groups)
+
+    group = service.get("group")
+    if group is not None:
+        return group in context.get("groups")
+
+    raise exception.FilterError(
+        "Service definition for '%s' does not have a 'group' or "
+        "'host_in_groups' attribute" %
+        service.get("container_name", "<unknown>"))
+
+
+@jinja2.contextfilter
+def service_enabled_and_mapped_to_host(context, service):
+    """Return whether a service is enabled and mapped to this host.
+
+    :param context: Jinja2 Context object.
+    :param service: Service definition, dict.
+    :returns: A boolean.
+    """
+    return (service_enabled(context, service) and
+            service_mapped_to_host(context, service))
+
+
+@jinja2.contextfilter
+def select_services_enabled_and_mapped_to_host(context, services):
+    """Select services that are enabled and mapped to this host.
+
+    :param context: Jinja2 Context object.
+    :param services: Service definitions, dict.
+    :returns: A dict containing enabled services mapped to this host.
+    """
+    return {service_name: service
+            for service_name, service in services.items()
+            if service_enabled_and_mapped_to_host(context, service)}
+
+
+def get_filters():
+    return {
+        "service_enabled": service_enabled,
+        "service_mapped_to_host": service_mapped_to_host,
+        "service_enabled_and_mapped_to_host": (
+            service_enabled_and_mapped_to_host),
+        "select_services_enabled_and_mapped_to_host": (
+            select_services_enabled_and_mapped_to_host),
+    }

kolla_ansible/tests/unit/test_filters.py

@@ -0,0 +1,173 @@
+# Copyright (c) 2019 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+import jinja2
+import mock
+
+from kolla_ansible import exception
+from kolla_ansible import filters
+
+
+def _to_bool(value):
+    """Simplified version of the bool filter.
+
+    Avoids having a dependency on Ansible in unit tests.
+    """
+    if value == 'yes':
+        return True
+    if value == 'no':
+        return False
+    return bool(value)
+
+
+class TestFilters(unittest.TestCase):
+
+    def setUp(self):
+        # Bandit complains about Jinja2 autoescaping without nosec.
+        self.env = jinja2.Environment()  # nosec
+        self.env.filters['bool'] = _to_bool
+        self.context = self._make_context()
+
+    def _make_context(self, parent=None):
+        if parent is None:
+            parent = {}
+        return self.env.context_class(
+            self.env, parent=parent, name='dummy', blocks={})
+
+    def test_service_enabled_true(self):
+        service = {
+            'enabled': True
+        }
+        self.assertTrue(filters.service_enabled(self.context, service))
+
+    def test_service_enabled_yes(self):
+        service = {
+            'enabled': 'yes'
+        }
+        self.assertTrue(filters.service_enabled(self.context, service))
+
+    def test_service_enabled_false(self):
+        service = {
+            'enabled': False
+        }
+        self.assertFalse(filters.service_enabled(self.context, service))
+
+    def test_service_enabled_no(self):
+        service = {
+            'enabled': 'no'
+        }
+        self.assertFalse(filters.service_enabled(self.context, service))
+
+    def test_service_enabled_no_attr(self):
+        service = {}
+        self.assertRaises(exception.FilterError,
+                          filters.service_enabled, self.context, service)
+
+    def test_service_mapped_to_host_host_in_groups_true(self):
+        service = {
+            'host_in_groups': True
+        }
+        self.assertTrue(filters.service_mapped_to_host(self.context, service))
+
+    def test_service_mapped_to_host_host_in_groups_yes(self):
+        service = {
+            'host_in_groups': 'yes'
+        }
+        self.assertTrue(filters.service_mapped_to_host(self.context, service))
+
+    def test_service_mapped_to_host_host_in_groups_false(self):
+        service = {
+            'host_in_groups': False
+        }
+        self.assertFalse(filters.service_mapped_to_host(self.context, service))
+
+    def test_service_mapped_to_host_host_in_groups_no(self):
+        service = {
+            'host_in_groups': 'no'
+        }
+        self.assertFalse(filters.service_mapped_to_host(self.context, service))
+
+    def test_service_mapped_to_host_in_group(self):
+        service = {
+            'group': 'foo'
+        }
+        context = self._make_context({'groups': ['foo', 'bar']})
+        self.assertTrue(filters.service_mapped_to_host(context, service))
+
+    def test_service_mapped_to_host_not_in_group(self):
+        service = {
+            'group': 'foo'
+        }
+        context = self._make_context({'groups': ['bar']})
+        self.assertFalse(filters.service_mapped_to_host(context, service))
+
+    def test_service_mapped_to_host_no_attr(self):
+        service = {}
+        self.assertRaises(exception.FilterError,
+                          filters.service_mapped_to_host, self.context,
+                          service)
+
+    @mock.patch.object(filters, 'service_enabled')
+    @mock.patch.object(filters, 'service_mapped_to_host')
+    def test_service_enabled_and_mapped_to_host(self,  mock_mapped,
+                                                mock_enabled):
+        service = {}
+        mock_enabled.return_value = True
+        mock_mapped.return_value = True
+        self.assertTrue(filters.service_enabled_and_mapped_to_host(
+            self.context, service))
+        mock_enabled.assert_called_once_with(self.context, service)
+        mock_mapped.assert_called_once_with(self.context, service)
+
+    @mock.patch.object(filters, 'service_enabled')
+    @mock.patch.object(filters, 'service_mapped_to_host')
+    def test_service_enabled_and_mapped_to_host_disabled(self,  mock_mapped,
+                                                         mock_enabled):
+        service = {}
+        mock_enabled.return_value = False
+        mock_mapped.return_value = True
+        self.assertFalse(filters.service_enabled_and_mapped_to_host(
+            self.context, service))
+        mock_enabled.assert_called_once_with(self.context, service)
+        self.assertFalse(mock_mapped.called)
+
+    @mock.patch.object(filters, 'service_enabled')
+    @mock.patch.object(filters, 'service_mapped_to_host')
+    def test_service_enabled_and_mapped_to_host_not_mapped(self,  mock_mapped,
+                                                           mock_enabled):
+        service = {}
+        mock_enabled.return_value = True
+        mock_mapped.return_value = False
+        self.assertFalse(filters.service_enabled_and_mapped_to_host(
+            self.context, service))
+        mock_enabled.assert_called_once_with(self.context, service)
+        mock_mapped.assert_called_once_with(self.context, service)
+
+    @mock.patch.object(filters, 'service_enabled_and_mapped_to_host')
+    def test_select_services_enabled_and_mapped_to_host(self, mock_seamth):
+        services = {
+            'foo': object(),
+            'bar': object(),
+            'baz': object(),
+        }
+        mock_seamth.side_effect = lambda _, s: s != services['bar']
+        result = filters.select_services_enabled_and_mapped_to_host(
+            self.context, services)
+        expected = {
+            'foo': services['foo'],
+            'baz': services['baz'],
+        }
+        self.assertEqual(expected, result)

releasenotes/notes/jinja-filters-818d5bb97ddc75c6.yaml

@@ -0,0 +1,13 @@
+---
+upgrade:
+  - |
+    When installing ``kolla-ansible`` from source, the ``kolla_ansible`` python
+    module must now be installed in addition to the python dependencies listed
+    in ``requirements.txt``. This is done via::
+
+        pip install /path/to/kolla-ansible
+
+    If the git repository is in the current directory, use the following
+    to avoid installing the package from PyPI::
+
+        pip install ./kolla-ansible

tests/run.yml

@@ -137,9 +137,9 @@
               dest: ironic-agent.kernel
       when: scenario == "ironic"
 
-    - name: install kolla-ansible requirements
+    - name: install kolla-ansible
       pip:
-        requirements: "{{ kolla_ansible_src_dir }}/requirements.txt"
+        name: "{{ kolla_ansible_src_dir }}"
       become: true
 
     - name: copy passwords.yml file
@@ -312,9 +312,9 @@
               when: "{{ is_ceph }}"
           when: item.when | default(true)
 
-        - name: upgrade kolla-ansible requirements
+        - name: upgrade kolla-ansible
           pip:
-            requirements: "{{ kolla_ansible_src_dir }}/requirements.txt"
+            name: "{{ kolla_ansible_src_dir }}"
           become: true
 
         # Update passwords.yml to include any new passwords added in this