diff --git a/zuul/ansible/lookup/__init__.py b/zuul/ansible/lookup/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/zuul/ansible/lookup/_banned.py b/zuul/ansible/lookup/_banned.py
new file mode 100644
index 0000000000..65708f80d7
--- /dev/null
+++ b/zuul/ansible/lookup/_banned.py
@@ -0,0 +1,25 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see .
+
+from ansible.errors import AnsibleError
+from ansible.plugins.lookup import LookupBase
+
+
+class LookupModule(LookupBase):
+
+ def run(self, *args, **kwargs):
+ raise AnsibleError(
+ "Use of lookup modules that perform local actions on the executor"
+ " is forbidden.")
diff --git a/zuul/ansible/lookup/consul_kv.py b/zuul/ansible/lookup/consul_kv.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/consul_kv.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/credstash.py b/zuul/ansible/lookup/credstash.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/credstash.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/csvfile.py b/zuul/ansible/lookup/csvfile.py
new file mode 100644
index 0000000000..6506aa2d21
--- /dev/null
+++ b/zuul/ansible/lookup/csvfile.py
@@ -0,0 +1,25 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see .
+
+
+from zuul.ansible import paths
+csvfile = paths._import_ansible_lookup_plugin("csvfile")
+
+
+class LookupModule(csvfile.LookupModule):
+
+ def read_csv(self, filename, *args, **kwargs):
+ paths._fail_if_unsafe(filename)
+ return super(LookupModule, self).read_csv(filename, *args, **kwargs)
diff --git a/zuul/ansible/lookup/dig.py b/zuul/ansible/lookup/dig.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/dig.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/dnstxt.py b/zuul/ansible/lookup/dnstxt.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/dnstxt.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/env.py b/zuul/ansible/lookup/env.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/env.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/etcd.py b/zuul/ansible/lookup/etcd.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/etcd.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/file.py b/zuul/ansible/lookup/file.py
new file mode 100644
index 0000000000..7403535b25
--- /dev/null
+++ b/zuul/ansible/lookup/file.py
@@ -0,0 +1,28 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see .
+
+
+from zuul.ansible import paths
+file_mod = paths._import_ansible_lookup_plugin("file")
+
+
+class LookupModule(file_mod.LookupModule):
+
+ def run(self, terms, variables=None, **kwargs):
+ for term in terms:
+ lookupfile = self.find_file_in_search_path(
+ variables, 'files', term)
+ paths._fail_if_unsafe(lookupfile)
+ return super(LookupModule, self).run(terms, variables, **kwargs)
diff --git a/zuul/ansible/lookup/fileglob.py b/zuul/ansible/lookup/fileglob.py
new file mode 100644
index 0000000000..4b9b449425
--- /dev/null
+++ b/zuul/ansible/lookup/fileglob.py
@@ -0,0 +1,45 @@
+# (c) 2012, Michael DeHaan
+# Copyright 2017 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see .
+
+# Forked from lib/ansible/plugins/lookup/fileglob.py in ansible
+
+import os
+import glob
+
+from zuul.ansible import paths
+
+from ansible.plugins.lookup import LookupBase
+from ansible.module_utils._text import to_bytes, to_text
+
+
+class LookupModule(LookupBase):
+
+ def run(self, terms, variables=None, **kwargs):
+
+ ret = []
+ for term in terms:
+ term_file = os.path.basename(term)
+ dwimmed_path = self.find_file_in_search_path(
+ variables, 'files', os.path.dirname(term))
+ if dwimmed_path:
+ paths._fail_if_unsafe(dwimmed_path)
+ globbed = glob.glob(to_bytes(
+ os.path.join(dwimmed_path, term_file),
+ errors='surrogate_or_strict'))
+ ret.extend(
+ to_text(g, errors='surrogate_or_strict')
+ for g in globbed if os.path.isfile(g))
+ return ret
diff --git a/zuul/ansible/lookup/filetree.py b/zuul/ansible/lookup/filetree.py
new file mode 100644
index 0000000000..0c054a335c
--- /dev/null
+++ b/zuul/ansible/lookup/filetree.py
@@ -0,0 +1,32 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see .
+
+import os
+
+from zuul.ansible import paths
+filetree = paths._import_ansible_lookup_plugin("filetree")
+
+
+class LookupModule(filetree.LookupModule):
+
+ def run(self, terms, variables=None, **kwargs):
+ basedir = self.get_basedir(variables)
+ for term in terms:
+ term_file = os.path.basename(term)
+ dwimmed_path = self._loader.path_dwim_relative(
+ basedir, 'files', os.path.dirname(term))
+ path = os.path.join(dwimmed_path, term_file)
+ paths._fail_if_unsafe(path)
+ return super(LookupModule, self).run(terms, variables, **kwargs)
diff --git a/zuul/ansible/lookup/first_found.py b/zuul/ansible/lookup/first_found.py
new file mode 100644
index 0000000000..d741df0f87
--- /dev/null
+++ b/zuul/ansible/lookup/first_found.py
@@ -0,0 +1,201 @@
+# (c) 2013, seth vidal red hat, inc
+# Copyright 2017 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see .
+
+# take a list of files and (optionally) a list of paths
+# return the first existing file found in the paths
+# [file1, file2, file3], [path1, path2, path3]
+# search order is:
+# path1/file1
+# path1/file2
+# path1/file3
+# path2/file1
+# path2/file2
+# path2/file3
+# path3/file1
+# path3/file2
+# path3/file3
+
+# first file found with os.path.exists() is returned
+# no file matches raises ansibleerror
+# EXAMPLES
+# - name: copy first existing file found to /some/file
+# action: copy src=$item dest=/some/file
+# with_first_found:
+# - files: foo ${inventory_hostname} bar
+# paths: /tmp/production /tmp/staging
+
+# that will look for files in this order:
+# /tmp/production/foo
+# ${inventory_hostname}
+# bar
+# /tmp/staging/foo
+# ${inventory_hostname}
+# bar
+
+# - name: copy first existing file found to /some/file
+# action: copy src=$item dest=/some/file
+# with_first_found:
+# - files: /some/place/foo ${inventory_hostname} /some/place/else
+
+# that will look for files in this order:
+# /some/place/foo
+# $relative_path/${inventory_hostname}
+# /some/place/else
+
+# example - including tasks:
+# tasks:
+# - include: $item
+# with_first_found:
+# - files: generic
+# paths: tasks/staging tasks/production
+# this will include the tasks in the file generic where it is found first
+# (staging or production)
+
+# example simple file lists
+# tasks:
+# - name: first found file
+# action: copy src=$item dest=/etc/file.cfg
+# with_first_found:
+# - files: foo.${inventory_hostname} foo
+
+
+# example skipping if no matched files
+# First_found also offers the ability to control whether or not failing
+# to find a file returns an error or not
+#
+# - name: first found file - or skip
+# action: copy src=$item dest=/etc/file.cfg
+# with_first_found:
+# - files: foo.${inventory_hostname}
+# skip: true
+
+# example a role with default configuration and configuration per host
+# you can set multiple terms with their own files and paths to look through.
+# consider a role that sets some configuration per host falling back on a
+# default config.
+#
+# - name: some configuration template
+# template: src={{ item }} dest=/etc/file.cfg mode=0444 owner=root group=root
+# with_first_found:
+# - files:
+# - ${inventory_hostname}/etc/file.cfg
+# paths:
+# - ../../../templates.overwrites
+# - ../../../templates
+# - files:
+# - etc/file.cfg
+# paths:
+# - templates
+
+# the above will return an empty list if the files cannot be found at all
+# if skip is unspecificed or if it is set to false then it will return a list
+# error which can be caught bye ignore_errors: true for that action.
+
+# finally - if you want you can use it, in place to replace
+# first_available_file:
+# you simply cannot use the - files, path or skip options. simply replace
+# first_available_file with with_first_found and leave the file listing in
+# place
+#
+#
+# - name: with_first_found like first_available_file
+# action: copy src=$item dest=/tmp/faftest
+# with_first_found:
+# - ../files/foo
+# - ../files/bar
+# - ../files/baz
+# ignore_errors: true
+
+import os
+
+from jinja2.exceptions import UndefinedError
+
+from ansible.constants import mk_boolean as boolean
+from ansible.errors import AnsibleLookupError
+from ansible.errors import AnsibleUndefinedVariable
+from ansible.module_utils.six import string_types
+from ansible.plugins.lookup import LookupBase
+
+from zuul.ansible import paths as zuul_paths
+
+
+class LookupModule(LookupBase):
+
+ def run(self, terms, variables, **kwargs):
+
+ anydict = False
+ skip = False
+
+ for term in terms:
+ if isinstance(term, dict):
+ anydict = True
+
+ total_search = []
+ if anydict:
+ for term in terms:
+ if isinstance(term, dict):
+ files = term.get('files', [])
+ paths = term.get('paths', [])
+ skip = boolean(term.get('skip', False))
+
+ filelist = files
+ if isinstance(files, string_types):
+ files = files.replace(',', ' ')
+ files = files.replace(';', ' ')
+ filelist = files.split(' ')
+
+ pathlist = paths
+ if paths:
+ if isinstance(paths, string_types):
+ paths = paths.replace(',', ' ')
+ paths = paths.replace(':', ' ')
+ paths = paths.replace(';', ' ')
+ pathlist = paths.split(' ')
+
+ if not pathlist:
+ total_search = filelist
+ else:
+ for path in pathlist:
+ for fn in filelist:
+ f = os.path.join(path, fn)
+ total_search.append(f)
+ else:
+ total_search.append(term)
+ else:
+ total_search = self._flatten(terms)
+
+ for fn in total_search:
+ zuul_paths._fail_if_unsafe(fn)
+ try:
+ fn = self._templar.template(fn)
+ except (AnsibleUndefinedVariable, UndefinedError):
+ continue
+
+ # get subdir if set by task executor, default to files otherwise
+ subdir = getattr(self, '_subdir', 'files')
+ path = None
+ path = self.find_file_in_search_path(
+ variables, subdir, fn, ignore_missing=True)
+ if path is not None:
+ return [path]
+ else:
+ if skip:
+ return []
+ else:
+ raise AnsibleLookupError(
+ "No file was found when using with_first_found. Use the"
+ " 'skip: true' option to allow this task to be skipped if"
+ " no files are found")
diff --git a/zuul/ansible/lookup/hashi_valut.py b/zuul/ansible/lookup/hashi_valut.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/hashi_valut.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/ini.py b/zuul/ansible/lookup/ini.py
new file mode 100644
index 0000000000..51127ff1b3
--- /dev/null
+++ b/zuul/ansible/lookup/ini.py
@@ -0,0 +1,31 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see .
+
+
+from zuul.ansible import paths
+ini = paths._import_ansible_lookup_plugin("ini")
+
+
+class LookupModule(ini.LookupModule):
+
+ def read_properties(self, filename, *args, **kwargs):
+ paths._fail_if_unsafe(filename)
+ return super(LookupModule, self).read_properties(
+ filename, *args, **kwargs)
+
+ def read_ini(self, filename, *args, **kwargs):
+ paths._fail_if_unsafe(filename)
+ return super(LookupModule, self).read_ini(
+ filename, *args, **kwargs)
diff --git a/zuul/ansible/lookup/keyring.py b/zuul/ansible/lookup/keyring.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/keyring.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/lastpass.py b/zuul/ansible/lookup/lastpass.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/lastpass.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/lines.py b/zuul/ansible/lookup/lines.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/lines.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/mongodb.py b/zuul/ansible/lookup/mongodb.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/mongodb.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/password.py b/zuul/ansible/lookup/password.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/password.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/passwordstore.py b/zuul/ansible/lookup/passwordstore.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/passwordstore.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/pipe.py b/zuul/ansible/lookup/pipe.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/pipe.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/redis_kv.py b/zuul/ansible/lookup/redis_kv.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/redis_kv.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/shelvefile.py b/zuul/ansible/lookup/shelvefile.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/shelvefile.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/template.py b/zuul/ansible/lookup/template.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/template.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/lookup/url.py b/zuul/ansible/lookup/url.py
new file mode 120000
index 0000000000..d45b9c405d
--- /dev/null
+++ b/zuul/ansible/lookup/url.py
@@ -0,0 +1 @@
+_banned.py
\ No newline at end of file
diff --git a/zuul/ansible/paths.py b/zuul/ansible/paths.py
index e387732a19..bc619753d0 100644
--- a/zuul/ansible/paths.py
+++ b/zuul/ansible/paths.py
@@ -16,7 +16,9 @@
import imp
import os
+from ansible.errors import AnsibleError
import ansible.plugins.action
+import ansible.plugins.lookup
def _is_safe_path(path):
@@ -35,6 +37,12 @@ def _fail_dict(path, prefix='Accessing files from'):
curdir=os.path.abspath(os.path.curdir)))
+def _fail_if_unsafe(path):
+ if not _is_safe_path(path):
+ msg_dict = _fail_dict(path)
+ raise AnsibleError(msg_dict['msg'])
+
+
def _import_ansible_action_plugin(name):
# Ansible forces the import of our action plugins
# (zuul.ansible.action.foo) as ansible.plugins.action.foo, which
@@ -51,3 +59,11 @@ def _import_ansible_action_plugin(name):
return imp.load_module(
'zuul.ansible.protected.action.' + name,
*imp.find_module(name, ansible.plugins.action.__path__))
+
+
+def _import_ansible_lookup_plugin(name):
+ # See _import_ansible_action_plugin
+
+ return imp.load_module(
+ 'zuul.ansible.protected.lookup.' + name,
+ *imp.find_module(name, ansible.plugins.lookup.__path__))
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 67fc5e6b14..d6daa2a13d 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -33,6 +33,7 @@ import zuul.merger.merger
import zuul.ansible.action
import zuul.ansible.callback
import zuul.ansible.library
+import zuul.ansible.lookup
from zuul.lib import commandsocket
COMMANDS = ['stop', 'pause', 'unpause', 'graceful', 'verbose',
@@ -274,6 +275,10 @@ class ExecutorServer(object):
if not os.path.exists(self.callback_dir):
os.makedirs(self.callback_dir)
+ self.lookup_dir = os.path.join(ansible_dir, 'lookup')
+ if not os.path.exists(self.lookup_dir):
+ os.makedirs(self.lookup_dir)
+
library_path = os.path.dirname(os.path.abspath(
zuul.ansible.library.__file__))
for fn in os.listdir(library_path):
@@ -289,6 +294,11 @@ class ExecutorServer(object):
for fn in os.listdir(callback_path):
shutil.copy(os.path.join(callback_path, fn), self.callback_dir)
+ lookup_path = os.path.dirname(os.path.abspath(
+ zuul.ansible.lookup.__file__))
+ for fn in os.listdir(lookup_path):
+ shutil.copy(os.path.join(lookup_path, fn), self.lookup_dir)
+
self.job_workers = {}
def _getMerger(self, root):
@@ -867,6 +877,8 @@ class AnsibleJob(object):
if not trusted:
config.write('action_plugins = %s\n'
% self.executor_server.action_dir)
+ config.write('lookup_plugins = %s\n'
+ % self.executor_server.lookup_dir)
# On trusted jobs, we want to prevent the printing of args,
# since trusted jobs might have access to secrets that they may