Manage ansible installations within zuul

As a first step towards supporting multiple ansible versions we need
tooling to manage ansible installations. This moves the installation
of ansible from the requirements.txt into zuul. This is called as a
setup hook to install the ansible versions into
<prefix>/lib/zuul/ansible. Further this tooling abstracts knowledge
that the executor must know in order to actually run the correct
version of ansible.

The actual usage of multiple ansible versions will be done in
follow-ups.

For better maintainability the ansible plugins live in
zuul/ansible/base where plugins can be kept in different versions if
necessary. For each supported ansible version there is a specific
folder that symlinks the according plugins.

Change-Id: I5ce1385245c76818777aa34230786a9dbaf723e5
Depends-On: https://review.openstack.org/623927

zuul/ansible/base/action/add_host.py

@@ -0,0 +1,43 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software.  If not, see <http://www.gnu.org/licenses/>.
+
+from zuul.ansible import paths
+add_host = paths._import_ansible_action_plugin("add_host")
+
+
+class ActionModule(add_host.ActionModule):
+
+    def run(self, tmp=None, task_vars=None):
+        safe_args = set((
+            'ansible_connection',
+            'ansible_host',
+            'ansible_port',
+            'ansible_user',
+            'ansible_password',
+            'ansible_ssh_host',
+            'ansible_ssh_port',
+            'ansible_ssh_user',
+            'ansible_ssh_pass',
+        ))
+        args = set(filter(
+            lambda x: x.startswith('ansible_'), self._task.args.keys()))
+        conn = self._task.args.get('ansible_connection', 'ssh')
+        if args.issubset(safe_args) and conn in ('kubectl', 'ssh'):
+            return super(ActionModule, self).run(tmp, task_vars)
+
+        return dict(
+            failed=True,
+            msg="Adding hosts %s with %s to the inventory is prohibited" % (
+                conn, " ".join(args.difference(safe_args))))

zuul/ansible/base/action/aireos.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/aireos_config.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/aruba.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/aruba_config.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/asa.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/asa_config.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/asa_template.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/assemble.py

@@ -0,0 +1,35 @@
+# Copyright 2016 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from zuul.ansible import paths
+assemble = paths._import_ansible_action_plugin("assemble")
+
+
+class ActionModule(assemble.ActionModule):
+
+    def _find_needle(self, dirname, needle):
+        return paths._safe_find_needle(
+            super(ActionModule, self), dirname, needle)
+
+    def run(self, tmp=None, task_vars=None):
+
+        if not paths._is_official_module(self):
+            return paths._fail_module_dict(self._task.action)
+
+        if paths._is_localhost_task(self):
+            paths._fail_if_unsafe(self._task.args['dest'])
+
+        return super(ActionModule, self).run(tmp, task_vars)

zuul/ansible/base/action/ce.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/ce_config.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/ce_template.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/copy.py

@@ -0,0 +1,35 @@
+# Copyright 2016 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from zuul.ansible import paths
+copy = paths._import_ansible_action_plugin("copy")
+
+
+class ActionModule(copy.ActionModule):
+
+    def _find_needle(self, dirname, needle):
+        return paths._safe_find_needle(
+            super(ActionModule, self), dirname, needle)
+
+    def run(self, tmp=None, task_vars=None):
+
+        if not paths._is_official_module(self):
+            return paths._fail_module_dict(self._task.action)
+
+        if paths._is_localhost_task(self):
+            paths._fail_if_unsafe(self._task.args['dest'])
+
+        return super(ActionModule, self).run(tmp, task_vars)

zuul/ansible/base/action/dellos10_config.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/dellos6_config.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/dellos9_config.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/eos_config.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/eos_template.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/fetch.py

@@ -0,0 +1,31 @@
+# Copyright 2016 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from zuul.ansible import paths
+fetch = paths._import_ansible_action_plugin("fetch")
+
+
+class ActionModule(fetch.ActionModule):
+
+    def run(self, tmp=None, task_vars=None):
+        if not paths._is_official_module(self):
+            return paths._fail_module_dict(self._task.action)
+
+        dest = self._task.args.get('dest', None)
+
+        if dest and not paths._is_safe_path(dest):
+            return paths._fail_dict(dest)
+        return super(ActionModule, self).run(tmp, task_vars)

zuul/ansible/base/action/fortios_config.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/include_vars.py

@@ -0,0 +1,40 @@
+# Copyright 2016 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from zuul.ansible import paths
+include_vars = paths._import_ansible_action_plugin("include_vars")
+
+
+class ActionModule(include_vars.ActionModule):
+
+    def _find_needle(self, dirname, needle):
+        return paths._safe_find_needle(
+            super(ActionModule, self), dirname, needle)
+
+    def run(self, tmp=None, task_vars=None):
+        if not paths._is_official_module(self):
+            return paths._fail_module_dict(self._task.action)
+
+        source_dir = self._task.args.get('dir', None)
+
+        # This is the handling for source_dir. The source_file is handled by
+        # the _find_needle override.
+        if source_dir:
+            self._set_args()
+            self._set_root_dir()
+            if not paths._is_safe_path(self.source_dir, allow_trusted=True):
+                return paths._fail_dict(self.source_dir)
+        return super(ActionModule, self).run(tmp, task_vars)

zuul/ansible/base/action/ios_config.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/ios_template.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/iosxr_config.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/iosxr_template.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/junos_config.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/junos_template.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_banner.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_base.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_config.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_interface.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_l2_interface.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_l3_interface.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_linkagg.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_lldp.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_lldp_interface.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_logging.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_ping.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_static_route.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_system.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_template.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_user.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_vlan.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/net_vrf.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/netconf_config.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/network.py

@@ -0,0 +1,25 @@
+# Copyright 2016 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from zuul.ansible import paths
+network = paths._import_ansible_action_plugin("network")
+
+
+class ActionModule(network.ActionModule):
+
+    def run(self, tmp=None, task_vars=None):
+
+        return dict(failed=True, msg='Use of network modules is prohibited')

zuul/ansible/base/action/normal.py

@@ -0,0 +1,118 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software.  If not, see <http://www.gnu.org/licenses/>.
+
+from ansible.module_utils.six.moves.urllib.parse import urlparse
+from ansible.errors import AnsibleError
+
+from zuul.ansible import paths
+normal = paths._import_ansible_action_plugin('normal')
+
+ALLOWED_URL_SCHEMES = ('https', 'http', 'ftp')
+
+
+class ActionModule(normal.ActionModule):
+    '''Override the normal action plugin
+
+    :py:class:`ansible.plugins.normal.ActionModule` is run for every
+    module that does not have a more specific matching action plugin.
+
+    Our overridden version of it wraps the execution with checks to block
+    undesired actions on localhost.
+    '''
+
+    def run(self, tmp=None, task_vars=None):
+        '''Overridden primary method from the base class.'''
+
+        if paths._is_localhost_task(self):
+            if not self.dispatch_handler():
+                raise AnsibleError("Executing local code is prohibited")
+        return super(ActionModule, self).run(tmp, task_vars)
+
+    def dispatch_handler(self):
+        '''Run per-action handler if one exists.'''
+        handler_name = 'handle_{action}'.format(action=self._task.action)
+        handler = getattr(self, handler_name, None)
+        if handler:
+            paths._fail_if_local_module(self)
+            handler()
+            return True
+        return False
+
+    def handle_zuul_return(self):
+        '''Allow zuul_return module on localhost.'''
+        pass
+
+    def handle_stat(self):
+        '''Allow stat module on localhost if it doesn't touch unsafe files.
+
+        The :ansible:module:`stat` can be useful in jobs for manipulating logs
+        and artifacts.
+
+        Block any access of files outside the zuul work dir.
+        '''
+        if self._task.args.get('get_mime'):
+            raise AnsibleError("get_mime on localhost is forbidden")
+        paths._fail_if_unsafe(self._task.args['path'])
+
+    def handle_file(self):
+        '''Allow file module on localhost if it doesn't touch unsafe files.
+
+        The :ansible:module:`file` can be useful in jobs for manipulating logs
+        and artifacts.
+
+        Block any access of files outside the zuul work dir.
+        '''
+        for arg in ('path', 'dest', 'name'):
+            dest = self._task.args.get(arg)
+            if dest:
+                paths._fail_if_unsafe(dest)
+
+    def handle_known_hosts(self):
+        '''Allow known_hosts on localhost
+
+        The :ansible:module:`known_hosts` can be used to add SSH host keys of
+        a remote system. When run from a executor it can be used with the
+        add_host task to access remote servers. This is needed because ansible
+        on the executor is configured to check host keys by default.
+
+        Block any access of files outside the zuul work dir.
+        '''
+        if paths._is_localhost_task(self):
+            path = self._task.args.get('path')
+            if path:
+                paths._fail_if_unsafe(path)
+
+    def handle_uri(self):
+        '''Allow uri module on localhost if it doesn't touch unsafe files.
+
+        The :ansible:module:`uri` can be used from the executor to do
+        things like pinging readthedocs.org that otherwise don't need a node.
+        However, it can also download content to a local file, or be used to
+        read from file:/// urls.
+
+        Block any use of url schemes other than https, http and ftp. Further,
+        block any local file interaction that falls outside of the zuul
+        work dir.
+        '''
+        # uri takes all the file arguments, so just let handle_file validate
+        # them for us.
+        self.handle_file()
+        scheme = urlparse(self._task.args['url']).scheme
+        if scheme not in ALLOWED_URL_SCHEMES:
+            raise AnsibleError(
+                "{scheme} urls are not allowed from localhost."
+                " Only {allowed_schemes} are allowed".format(
+                    scheme=scheme,
+                    allowed_schemes=ALLOWED_URL_SCHEMES))

zuul/ansible/base/action/nxos_config.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/nxos_template.py

@@ -0,0 +1 @@
+network.py

zuul/ansible/base/action/ops_config.py

@@ -0,0 +1 @@
+network.py