diff --git a/zuul/ansible/lookup/__init__.py b/zuul/ansible/lookup/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/zuul/ansible/lookup/_banned.py b/zuul/ansible/lookup/_banned.py new file mode 100644 index 0000000000..65708f80d7 --- /dev/null +++ b/zuul/ansible/lookup/_banned.py @@ -0,0 +1,25 @@ +# Copyright 2017 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see . + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase + + +class LookupModule(LookupBase): + + def run(self, *args, **kwargs): + raise AnsibleError( + "Use of lookup modules that perform local actions on the executor" + " is forbidden.") diff --git a/zuul/ansible/lookup/consul_kv.py b/zuul/ansible/lookup/consul_kv.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/consul_kv.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/credstash.py b/zuul/ansible/lookup/credstash.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/credstash.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/csvfile.py b/zuul/ansible/lookup/csvfile.py new file mode 100644 index 0000000000..6506aa2d21 --- /dev/null +++ b/zuul/ansible/lookup/csvfile.py @@ -0,0 +1,25 @@ +# Copyright 2017 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see . + + +from zuul.ansible import paths +csvfile = paths._import_ansible_lookup_plugin("csvfile") + + +class LookupModule(csvfile.LookupModule): + + def read_csv(self, filename, *args, **kwargs): + paths._fail_if_unsafe(filename) + return super(LookupModule, self).read_csv(filename, *args, **kwargs) diff --git a/zuul/ansible/lookup/dig.py b/zuul/ansible/lookup/dig.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/dig.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/dnstxt.py b/zuul/ansible/lookup/dnstxt.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/dnstxt.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/env.py b/zuul/ansible/lookup/env.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/env.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/etcd.py b/zuul/ansible/lookup/etcd.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/etcd.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/file.py b/zuul/ansible/lookup/file.py new file mode 100644 index 0000000000..7403535b25 --- /dev/null +++ b/zuul/ansible/lookup/file.py @@ -0,0 +1,28 @@ +# Copyright 2017 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see . + + +from zuul.ansible import paths +file_mod = paths._import_ansible_lookup_plugin("file") + + +class LookupModule(file_mod.LookupModule): + + def run(self, terms, variables=None, **kwargs): + for term in terms: + lookupfile = self.find_file_in_search_path( + variables, 'files', term) + paths._fail_if_unsafe(lookupfile) + return super(LookupModule, self).run(terms, variables, **kwargs) diff --git a/zuul/ansible/lookup/fileglob.py b/zuul/ansible/lookup/fileglob.py new file mode 100644 index 0000000000..4b9b449425 --- /dev/null +++ b/zuul/ansible/lookup/fileglob.py @@ -0,0 +1,45 @@ +# (c) 2012, Michael DeHaan +# Copyright 2017 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see . + +# Forked from lib/ansible/plugins/lookup/fileglob.py in ansible + +import os +import glob + +from zuul.ansible import paths + +from ansible.plugins.lookup import LookupBase +from ansible.module_utils._text import to_bytes, to_text + + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + + ret = [] + for term in terms: + term_file = os.path.basename(term) + dwimmed_path = self.find_file_in_search_path( + variables, 'files', os.path.dirname(term)) + if dwimmed_path: + paths._fail_if_unsafe(dwimmed_path) + globbed = glob.glob(to_bytes( + os.path.join(dwimmed_path, term_file), + errors='surrogate_or_strict')) + ret.extend( + to_text(g, errors='surrogate_or_strict') + for g in globbed if os.path.isfile(g)) + return ret diff --git a/zuul/ansible/lookup/filetree.py b/zuul/ansible/lookup/filetree.py new file mode 100644 index 0000000000..0c054a335c --- /dev/null +++ b/zuul/ansible/lookup/filetree.py @@ -0,0 +1,32 @@ +# Copyright 2017 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see . + +import os + +from zuul.ansible import paths +filetree = paths._import_ansible_lookup_plugin("filetree") + + +class LookupModule(filetree.LookupModule): + + def run(self, terms, variables=None, **kwargs): + basedir = self.get_basedir(variables) + for term in terms: + term_file = os.path.basename(term) + dwimmed_path = self._loader.path_dwim_relative( + basedir, 'files', os.path.dirname(term)) + path = os.path.join(dwimmed_path, term_file) + paths._fail_if_unsafe(path) + return super(LookupModule, self).run(terms, variables, **kwargs) diff --git a/zuul/ansible/lookup/first_found.py b/zuul/ansible/lookup/first_found.py new file mode 100644 index 0000000000..d741df0f87 --- /dev/null +++ b/zuul/ansible/lookup/first_found.py @@ -0,0 +1,201 @@ +# (c) 2013, seth vidal red hat, inc +# Copyright 2017 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see . + +# take a list of files and (optionally) a list of paths +# return the first existing file found in the paths +# [file1, file2, file3], [path1, path2, path3] +# search order is: +# path1/file1 +# path1/file2 +# path1/file3 +# path2/file1 +# path2/file2 +# path2/file3 +# path3/file1 +# path3/file2 +# path3/file3 + +# first file found with os.path.exists() is returned +# no file matches raises ansibleerror +# EXAMPLES +# - name: copy first existing file found to /some/file +# action: copy src=$item dest=/some/file +# with_first_found: +# - files: foo ${inventory_hostname} bar +# paths: /tmp/production /tmp/staging + +# that will look for files in this order: +# /tmp/production/foo +# ${inventory_hostname} +# bar +# /tmp/staging/foo +# ${inventory_hostname} +# bar + +# - name: copy first existing file found to /some/file +# action: copy src=$item dest=/some/file +# with_first_found: +# - files: /some/place/foo ${inventory_hostname} /some/place/else + +# that will look for files in this order: +# /some/place/foo +# $relative_path/${inventory_hostname} +# /some/place/else + +# example - including tasks: +# tasks: +# - include: $item +# with_first_found: +# - files: generic +# paths: tasks/staging tasks/production +# this will include the tasks in the file generic where it is found first +# (staging or production) + +# example simple file lists +# tasks: +# - name: first found file +# action: copy src=$item dest=/etc/file.cfg +# with_first_found: +# - files: foo.${inventory_hostname} foo + + +# example skipping if no matched files +# First_found also offers the ability to control whether or not failing +# to find a file returns an error or not +# +# - name: first found file - or skip +# action: copy src=$item dest=/etc/file.cfg +# with_first_found: +# - files: foo.${inventory_hostname} +# skip: true + +# example a role with default configuration and configuration per host +# you can set multiple terms with their own files and paths to look through. +# consider a role that sets some configuration per host falling back on a +# default config. +# +# - name: some configuration template +# template: src={{ item }} dest=/etc/file.cfg mode=0444 owner=root group=root +# with_first_found: +# - files: +# - ${inventory_hostname}/etc/file.cfg +# paths: +# - ../../../templates.overwrites +# - ../../../templates +# - files: +# - etc/file.cfg +# paths: +# - templates + +# the above will return an empty list if the files cannot be found at all +# if skip is unspecificed or if it is set to false then it will return a list +# error which can be caught bye ignore_errors: true for that action. + +# finally - if you want you can use it, in place to replace +# first_available_file: +# you simply cannot use the - files, path or skip options. simply replace +# first_available_file with with_first_found and leave the file listing in +# place +# +# +# - name: with_first_found like first_available_file +# action: copy src=$item dest=/tmp/faftest +# with_first_found: +# - ../files/foo +# - ../files/bar +# - ../files/baz +# ignore_errors: true + +import os + +from jinja2.exceptions import UndefinedError + +from ansible.constants import mk_boolean as boolean +from ansible.errors import AnsibleLookupError +from ansible.errors import AnsibleUndefinedVariable +from ansible.module_utils.six import string_types +from ansible.plugins.lookup import LookupBase + +from zuul.ansible import paths as zuul_paths + + +class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): + + anydict = False + skip = False + + for term in terms: + if isinstance(term, dict): + anydict = True + + total_search = [] + if anydict: + for term in terms: + if isinstance(term, dict): + files = term.get('files', []) + paths = term.get('paths', []) + skip = boolean(term.get('skip', False)) + + filelist = files + if isinstance(files, string_types): + files = files.replace(',', ' ') + files = files.replace(';', ' ') + filelist = files.split(' ') + + pathlist = paths + if paths: + if isinstance(paths, string_types): + paths = paths.replace(',', ' ') + paths = paths.replace(':', ' ') + paths = paths.replace(';', ' ') + pathlist = paths.split(' ') + + if not pathlist: + total_search = filelist + else: + for path in pathlist: + for fn in filelist: + f = os.path.join(path, fn) + total_search.append(f) + else: + total_search.append(term) + else: + total_search = self._flatten(terms) + + for fn in total_search: + zuul_paths._fail_if_unsafe(fn) + try: + fn = self._templar.template(fn) + except (AnsibleUndefinedVariable, UndefinedError): + continue + + # get subdir if set by task executor, default to files otherwise + subdir = getattr(self, '_subdir', 'files') + path = None + path = self.find_file_in_search_path( + variables, subdir, fn, ignore_missing=True) + if path is not None: + return [path] + else: + if skip: + return [] + else: + raise AnsibleLookupError( + "No file was found when using with_first_found. Use the" + " 'skip: true' option to allow this task to be skipped if" + " no files are found") diff --git a/zuul/ansible/lookup/hashi_valut.py b/zuul/ansible/lookup/hashi_valut.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/hashi_valut.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/ini.py b/zuul/ansible/lookup/ini.py new file mode 100644 index 0000000000..51127ff1b3 --- /dev/null +++ b/zuul/ansible/lookup/ini.py @@ -0,0 +1,31 @@ +# Copyright 2017 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see . + + +from zuul.ansible import paths +ini = paths._import_ansible_lookup_plugin("ini") + + +class LookupModule(ini.LookupModule): + + def read_properties(self, filename, *args, **kwargs): + paths._fail_if_unsafe(filename) + return super(LookupModule, self).read_properties( + filename, *args, **kwargs) + + def read_ini(self, filename, *args, **kwargs): + paths._fail_if_unsafe(filename) + return super(LookupModule, self).read_ini( + filename, *args, **kwargs) diff --git a/zuul/ansible/lookup/keyring.py b/zuul/ansible/lookup/keyring.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/keyring.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/lastpass.py b/zuul/ansible/lookup/lastpass.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/lastpass.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/lines.py b/zuul/ansible/lookup/lines.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/lines.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/mongodb.py b/zuul/ansible/lookup/mongodb.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/mongodb.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/password.py b/zuul/ansible/lookup/password.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/password.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/passwordstore.py b/zuul/ansible/lookup/passwordstore.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/passwordstore.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/pipe.py b/zuul/ansible/lookup/pipe.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/pipe.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/redis_kv.py b/zuul/ansible/lookup/redis_kv.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/redis_kv.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/shelvefile.py b/zuul/ansible/lookup/shelvefile.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/shelvefile.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/template.py b/zuul/ansible/lookup/template.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/template.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/lookup/url.py b/zuul/ansible/lookup/url.py new file mode 120000 index 0000000000..d45b9c405d --- /dev/null +++ b/zuul/ansible/lookup/url.py @@ -0,0 +1 @@ +_banned.py \ No newline at end of file diff --git a/zuul/ansible/paths.py b/zuul/ansible/paths.py index e387732a19..bc619753d0 100644 --- a/zuul/ansible/paths.py +++ b/zuul/ansible/paths.py @@ -16,7 +16,9 @@ import imp import os +from ansible.errors import AnsibleError import ansible.plugins.action +import ansible.plugins.lookup def _is_safe_path(path): @@ -35,6 +37,12 @@ def _fail_dict(path, prefix='Accessing files from'): curdir=os.path.abspath(os.path.curdir))) +def _fail_if_unsafe(path): + if not _is_safe_path(path): + msg_dict = _fail_dict(path) + raise AnsibleError(msg_dict['msg']) + + def _import_ansible_action_plugin(name): # Ansible forces the import of our action plugins # (zuul.ansible.action.foo) as ansible.plugins.action.foo, which @@ -51,3 +59,11 @@ def _import_ansible_action_plugin(name): return imp.load_module( 'zuul.ansible.protected.action.' + name, *imp.find_module(name, ansible.plugins.action.__path__)) + + +def _import_ansible_lookup_plugin(name): + # See _import_ansible_action_plugin + + return imp.load_module( + 'zuul.ansible.protected.lookup.' + name, + *imp.find_module(name, ansible.plugins.lookup.__path__)) diff --git a/zuul/executor/server.py b/zuul/executor/server.py index 67fc5e6b14..d6daa2a13d 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -33,6 +33,7 @@ import zuul.merger.merger import zuul.ansible.action import zuul.ansible.callback import zuul.ansible.library +import zuul.ansible.lookup from zuul.lib import commandsocket COMMANDS = ['stop', 'pause', 'unpause', 'graceful', 'verbose', @@ -274,6 +275,10 @@ class ExecutorServer(object): if not os.path.exists(self.callback_dir): os.makedirs(self.callback_dir) + self.lookup_dir = os.path.join(ansible_dir, 'lookup') + if not os.path.exists(self.lookup_dir): + os.makedirs(self.lookup_dir) + library_path = os.path.dirname(os.path.abspath( zuul.ansible.library.__file__)) for fn in os.listdir(library_path): @@ -289,6 +294,11 @@ class ExecutorServer(object): for fn in os.listdir(callback_path): shutil.copy(os.path.join(callback_path, fn), self.callback_dir) + lookup_path = os.path.dirname(os.path.abspath( + zuul.ansible.lookup.__file__)) + for fn in os.listdir(lookup_path): + shutil.copy(os.path.join(lookup_path, fn), self.lookup_dir) + self.job_workers = {} def _getMerger(self, root): @@ -867,6 +877,8 @@ class AnsibleJob(object): if not trusted: config.write('action_plugins = %s\n' % self.executor_server.action_dir) + config.write('lookup_plugins = %s\n' + % self.executor_server.lookup_dir) # On trusted jobs, we want to prevent the printing of args, # since trusted jobs might have access to secrets that they may