Use environment variables for plugin paths

Kayobe provides various roles and plugins in the ansible directory.
These are accessible to Kayobe playbooks in the same directory.

In some cases it can be useful to use these items from Kayobe custom
playbooks, however they cannot since they reside in a different
directory. Typically we work around this by symlinking to the relevant
directory from the directory containing the custom playbook. This is not
an elegant workaround, and has assumptions about the relative paths of
the Kayobe configuration and virtual environment in which Kayobe is
installed.

This change adds the Kayobe role, collection, and plugin paths to the
relevant Ansible lookup paths using environment variables. This allows
custom playbooks to use these items. Also added to the lookup paths are
roles, collections and plugins in Kayobe configuration in the
etc/kayobe/ansible/ directory. This removes the limitation of playbooks
needing to reside in the same directory as those items in order to use
them.

We import the Ansible configuration settings module directly, since it
avoids replicating the configuration logic.

Story: 2010280
Task: 46234

Change-Id: I2fb2b4d7ed937e0184a62b0f119659569448f8df

ansible/inventory/hosts

@@ -0,0 +1 @@
+# Dummy inventory file to allow Ansible to consume this inventory directory.

doc/source/custom-ansible-playbooks.rst

@@ -49,27 +49,19 @@ playbooks in this repository makes a lot of sense, and kayobe has special
 support for this.
 
 It is recommended to store custom playbooks in
-``$KAYOBE_CONFIG_PATH/ansible/``.  Roles located in
-``$KAYOBE_CONFIG_PATH/ansible/roles/`` will be automatically available to
-playbooks in this directory.
+``$KAYOBE_CONFIG_PATH/ansible/``.  It is also possible to use the following
+subdirectories, and since the Zed 13.0.0 release these will be available to all
+Kayobe playbook executions.
 
-With this directory layout, the following commands could be used to create
-symlinks that allow access to Kayobe's filter plugins, group variables and test
-plugins:
+* ``roles``
+* ``collections``
+* ``action_plugins``
+* ``filter_plugins``
+* ``test_plugins``
 
-.. code-block:: console
-
-   cd ${KAYOBE_CONFIG_PATH}/ansible/
-   ln -s ../../../../kayobe/ansible/filter_plugins/ filter_plugins
-   ln -s ../../../../kayobe/ansible/test_plugins/ test_plugins
-
-These symlinks can even be committed to the kayobe-config Git repository.
-
-.. note::
-
-   These symlinks rely on having a kayobe source checkout at the same level as
-   the kayobe-config repository checkout, as described in
-   :ref:`installation-source`.
+Note that since the Zed 13.0.0 release, it is no longer necessary to create
+symlinks in order to use Kayobe's roles, collections or plugins. Existing
+symlinks may be removed.
 
 Ansible Galaxy
 --------------

kayobe/ansible.py

@@ -21,6 +21,8 @@ import subprocess
 import sys
 import tempfile
 
+import ansible.constants
+
 from kayobe import exception
 from kayobe import utils
 from kayobe import vault
@@ -231,6 +233,40 @@ def _get_environment(parsed_args):
     ansible_cfg_path = os.path.join(parsed_args.config_path, "ansible.cfg")
     if utils.is_readable_file(ansible_cfg_path)["result"]:
         env.setdefault("ANSIBLE_CONFIG", ansible_cfg_path)
+
+    # Update various role, collection and plugin paths to include the Kayobe
+    # roles, collections and plugins. This allows custom playbooks to use these
+    # resources.
+    roles_paths = [
+        os.path.join(parsed_args.config_path, "ansible", "roles"),
+        utils.get_data_files_path("ansible", "roles"),
+    ] + ansible.constants.DEFAULT_ROLES_PATH
+    env.setdefault("ANSIBLE_ROLES_PATH", ":".join(roles_paths))
+
+    collections_paths = [
+        os.path.join(parsed_args.config_path, "ansible", "collections"),
+        utils.get_data_files_path("ansible", "collections"),
+    ] + ansible.constants.COLLECTIONS_PATHS
+    env.setdefault("ANSIBLE_COLLECTIONS_PATH", ":".join(collections_paths))
+
+    action_plugins = [
+        os.path.join(parsed_args.config_path, "ansible", "action_plugins"),
+        utils.get_data_files_path("ansible", "action_plugins"),
+    ] + ansible.constants.DEFAULT_ACTION_PLUGIN_PATH
+    env.setdefault("ANSIBLE_ACTION_PLUGINS", ":".join(action_plugins))
+
+    filter_plugins = [
+        os.path.join(parsed_args.config_path, "ansible", "filter_plugins"),
+        utils.get_data_files_path("ansible", "filter_plugins"),
+    ] + ansible.constants.DEFAULT_FILTER_PLUGIN_PATH
+    env.setdefault("ANSIBLE_FILTER_PLUGINS", ":".join(filter_plugins))
+
+    test_plugins = [
+        os.path.join(parsed_args.config_path, "ansible", "test_plugins"),
+        utils.get_data_files_path("ansible", "test_plugins"),
+    ] + ansible.constants.DEFAULT_TEST_PLUGIN_PATH
+    env.setdefault("ANSIBLE_TEST_PLUGINS", ":".join(test_plugins))
+
     return env
 
 

kayobe/tests/unit/test_ansible.py

@@ -15,6 +15,7 @@
 import argparse
 import errno
 import os
+import os.path
 import shutil
 import subprocess
 import tempfile
@@ -52,7 +53,41 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe"}
+        home = os.path.expanduser("~")
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": ":".join([
+                "/etc/kayobe/ansible/roles",
+                utils.get_data_files_path("ansible", "roles"),
+                home + "/.ansible/roles",
+                "/usr/share/ansible/roles",
+                "/etc/ansible/roles",
+            ]),
+            "ANSIBLE_COLLECTIONS_PATH": ":".join([
+                "/etc/kayobe/ansible/collections",
+                utils.get_data_files_path("ansible", "collections"),
+                home + "/.ansible/collections",
+                "/usr/share/ansible/collections",
+            ]),
+            "ANSIBLE_ACTION_PLUGINS": ":".join([
+                "/etc/kayobe/ansible/action_plugins",
+                utils.get_data_files_path("ansible", "action_plugins"),
+                home + "/.ansible/plugins/action",
+                "/usr/share/ansible/plugins/action",
+            ]),
+            "ANSIBLE_FILTER_PLUGINS": ":".join([
+                "/etc/kayobe/ansible/filter_plugins",
+                utils.get_data_files_path("ansible", "filter_plugins"),
+                home + "/.ansible/plugins/filter",
+                "/usr/share/ansible/plugins/filter",
+            ]),
+            "ANSIBLE_TEST_PLUGINS": ":".join([
+                "/etc/kayobe/ansible/test_plugins",
+                utils.get_data_files_path("ansible", "test_plugins"),
+                home + "/.ansible/plugins/test",
+                "/usr/share/ansible/plugins/test",
+            ]),
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
         mock_vars.assert_called_once_with(["/etc/kayobe"])
@@ -99,8 +134,42 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/path/to/config",
-                        "KAYOBE_ENVIRONMENT": "test-env"}
+        home = os.path.expanduser("~")
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/path/to/config",
+            "KAYOBE_ENVIRONMENT": "test-env",
+            "ANSIBLE_ROLES_PATH": ":".join([
+                "/path/to/config/ansible/roles",
+                utils.get_data_files_path("ansible", "roles"),
+                home + "/.ansible/roles",
+                "/usr/share/ansible/roles",
+                "/etc/ansible/roles",
+            ]),
+            "ANSIBLE_COLLECTIONS_PATH": ":".join([
+                "/path/to/config/ansible/collections",
+                utils.get_data_files_path("ansible", "collections"),
+                home + "/.ansible/collections",
+                "/usr/share/ansible/collections",
+            ]),
+            "ANSIBLE_ACTION_PLUGINS": ":".join([
+                "/path/to/config/ansible/action_plugins",
+                utils.get_data_files_path("ansible", "action_plugins"),
+                home + "/.ansible/plugins/action",
+                "/usr/share/ansible/plugins/action",
+            ]),
+            "ANSIBLE_FILTER_PLUGINS": ":".join([
+                "/path/to/config/ansible/filter_plugins",
+                utils.get_data_files_path("ansible", "filter_plugins"),
+                home + "/.ansible/plugins/filter",
+                "/usr/share/ansible/plugins/filter",
+            ]),
+            "ANSIBLE_TEST_PLUGINS": ":".join([
+                "/path/to/config/ansible/test_plugins",
+                utils.get_data_files_path("ansible", "test_plugins"),
+                home + "/.ansible/plugins/test",
+                "/usr/share/ansible/plugins/test",
+            ]),
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
         mock_vars.assert_called_once_with(
@@ -153,9 +222,16 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/path/to/config",
-                        "KAYOBE_ENVIRONMENT": "test-env",
-                        "KAYOBE_VAULT_PASSWORD": "test-pass"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/path/to/config",
+            "KAYOBE_ENVIRONMENT": "test-env",
+            "KAYOBE_VAULT_PASSWORD": "test-pass",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         expected_calls = [
             mock.call(["which", "kayobe-vault-password-helper"],
                       check_output=True, universal_newlines=True),
@@ -189,7 +265,14 @@ class TestCase(unittest.TestCase):
             "--inventory", "/etc/kayobe/inventory",
             "playbook1.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
         mock_update.assert_called_once_with(mock.ANY, expected_env)
@@ -219,8 +302,15 @@ class TestCase(unittest.TestCase):
             "--inventory", "/etc/kayobe/inventory",
             "playbook1.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe",
-                        "KAYOBE_VAULT_PASSWORD": "test-pass"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "KAYOBE_VAULT_PASSWORD": "test-pass",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
 
@@ -279,7 +369,14 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
         mock_vars.assert_called_once_with(["/etc/kayobe"])
@@ -309,7 +406,14 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
         mock_vars.assert_called_once_with(["/etc/kayobe"])
@@ -339,7 +443,14 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
         mock_vars.assert_called_once_with(["/etc/kayobe"])
@@ -365,7 +476,12 @@ class TestCase(unittest.TestCase):
         ]
         expected_env = {
             "ANSIBLE_CONFIG": "/etc/kayobe/ansible.cfg",
-            "KAYOBE_CONFIG_PATH": "/etc/kayobe"
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
         }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
@@ -394,7 +510,12 @@ class TestCase(unittest.TestCase):
         ]
         expected_env = {
             "ANSIBLE_CONFIG": "/path/to/ansible.cfg",
-            "KAYOBE_CONFIG_PATH": "/etc/kayobe"
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
         }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
@@ -689,7 +810,14 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
         mock_vars.assert_called_once_with(["/etc/kayobe"])
@@ -722,8 +850,15 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe",
-                        "KAYOBE_ENVIRONMENT": "test-env"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "KAYOBE_ENVIRONMENT": "test-env",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         expected_calls = [
             mock.call("/etc/kayobe/inventory"),
             mock.call("/etc/kayobe/environments/test-env/inventory"),
@@ -762,8 +897,15 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe",
-                        "KAYOBE_ENVIRONMENT": "test-env"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "KAYOBE_ENVIRONMENT": "test-env",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         expected_calls = [
             mock.call("/etc/kayobe/inventory"),
             mock.call("/etc/kayobe/environments/test-env/inventory"),
@@ -802,8 +944,15 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe",
-                        "KAYOBE_ENVIRONMENT": "test-env"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "KAYOBE_ENVIRONMENT": "test-env",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         expected_calls = [
             mock.call("/etc/kayobe/inventory"),
             mock.call("/etc/kayobe/environments/test-env/inventory"),

releasenotes/notes/plugin-env-vars-e6265564b3441dfe.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    Roles, collections and plugins included with Kayobe configuration are now
+    accessible to all Kayobe playbook executions.
+upgrade:
+  - |
+    Changes the environment used during Kayobe playbook execution to include
+    Kayobe's collections, roles and plugins in the Ansible lookup paths.
+    This allows custom playbooks to use these items, without the requirement to
+    symlink into the Kayobe installation. Existing symlinks may be removed.