diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst index cc9d1810ef..99912d7e96 100644 --- a/doc/source/admin/components.rst +++ b/doc/source/admin/components.rst @@ -317,6 +317,24 @@ executor user=zuul +.. _admin_sitewide_variables: + +**variables** + Path to an Ansible variables file to supply site-wide variables. + This should be a YAML-formatted file consisting of a single + dictionary. The contents will be made available to all jobs as + Ansible variables. These variables take precedence over all other + forms (job variables and secrets). Care should be taken when naming + these variables to avoid potential collisions with those used by + jobs. Prefixing variable names with a site-specific identifier is + recommended. The default is not to add any site-wide variables. + See the :ref:`User's Guide ` for more + information. + + Example:: + + variables=/etc/zuul/variables.yaml + merger """""" diff --git a/doc/source/user/jobs.rst b/doc/source/user/jobs.rst index a367aa0fe8..1812bc57d9 100644 --- a/doc/source/user/jobs.rst +++ b/doc/source/user/jobs.rst @@ -73,6 +73,24 @@ behavior. Variables --------- +There are several sources of variables which are available to Ansible: +variables defined in jobs, secrets, and site-wide variables. The +order of precedence is: + +* Site-wide variables + +* Secrets + +* Job variables + +Meaning that a site-wide variable with the same name as any other will +override its value, and similarly, secrets override job variables of +the same name. Each of the three sources is described below. + + +Job Variables +~~~~~~~~~~~~~ + Any variables specified in the job definition are available as Ansible host variables. They are added to the `vars` section of the inventory file under the `all` hosts group, so they are available to all hosts. @@ -293,6 +311,19 @@ executor running the job is available: **zuul.executor.log_root** The path to the logs directory. + +.. _user_sitewide_variables: + +Site-wide Variables +~~~~~~~~~~~~~~~~~~~ + +The Zuul administrator may define variables which will be available to +all jobs running in the system. These are statically defined and may +not be altered by jobs. See the :ref:`Administrator's Guide +` for information on how a site +administrator may define these variables. + + SSH Keys -------- diff --git a/tests/base.py b/tests/base.py index 2c478ad433..98f4c7f25d 100755 --- a/tests/base.py +++ b/tests/base.py @@ -16,6 +16,7 @@ # under the License. import configparser +from contextlib import contextmanager import datetime import gc import hashlib @@ -2769,6 +2770,24 @@ class AnsibleZuulTestCase(ZuulTestCase): """ZuulTestCase but with an actual ansible executor running""" run_ansible = True + @contextmanager + def jobLog(self, build): + """Print job logs on assertion errors + + This method is a context manager which, if it encounters an + ecxeption, adds the build log to the debug output. + + :arg Build build: The build that's being asserted. + """ + try: + yield + except Exception: + path = os.path.join(self.test_root, build.uuid, + 'work', 'logs', 'job-output.txt') + with open(path) as f: + self.log.debug(f.read()) + raise + class SSLZuulTestCase(ZuulTestCase): """ZuulTestCase but using SSL when possible""" diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml index ce392a4017..3f62c4cfd1 100644 --- a/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml +++ b/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml @@ -20,3 +20,13 @@ - zuul.project.name == 'org/project' - zuul.project.canonical_hostname == 'review.example.com' - zuul.project.canonical_name == 'review.example.com/org/project' + + - debug: + msg: "vartest secret {{ vartest_secret }}" + + - name: Assert variable precedence. + assert: + that: + - vartest_job == 'vartest_job' + - vartest_secret.value == 'vartest_secret' + - vartest_site == 'vartest_site' \ No newline at end of file diff --git a/tests/fixtures/config/ansible/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml index 1a1b22f364..f050c6fc01 100644 --- a/tests/fixtures/config/ansible/git/common-config/zuul.yaml +++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml @@ -48,6 +48,27 @@ Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs= +# This is used by the check-vars job to evaluate variable precedence. +# The name of this secret conflicts with a site variable. +- secret: + name: vartest_site + data: + value: vartest_secret + +# This is used by the check-vars job to evaluate variable precedence. +# The name of this secret conflicts with a job variable. +- secret: + name: vartest_job + data: + value: vartest_secret + +# This is used by the check-vars job to evaluate variable precedence. +# The name of this secret should not conflict. +- secret: + name: vartest_secret + data: + value: vartest_secret + - job: name: base-urls success-url: https://success.example.com/zuul-logs/{build.uuid}/ @@ -77,6 +98,14 @@ nodes: - name: ubuntu-xenial label: ubuntu-xenial + vars: + vartest_job: vartest_job + vartest_secret: vartest_job + vartest_site: vartest_job + auth: + secrets: + - vartest_site + - vartest_secret - job: parent: base-urls diff --git a/tests/fixtures/config/ansible/variables.yaml b/tests/fixtures/config/ansible/variables.yaml new file mode 100644 index 0000000000..692eb7fff6 --- /dev/null +++ b/tests/fixtures/config/ansible/variables.yaml @@ -0,0 +1 @@ +vartest_site: vartest_site diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py index fb80660818..ad0b2cc578 100644 --- a/tests/unit/test_v3.py +++ b/tests/unit/test_v3.py @@ -595,54 +595,70 @@ class TestAnsible(AnsibleZuulTestCase): tenant_config_file = 'config/ansible/main.yaml' def test_playbook(self): + # Keep the jobdir around so we can inspect contents if an + # assert fails. + self.executor_server.keep_jobdir = True + # Output extra ansible info so we might see errors. + self.executor_server.verbose = True + # Add a site variables file, used by check-vars + path = os.path.join(FIXTURE_DIR, 'config', 'ansible', + 'variables.yaml') + self.config.set('executor', 'variables', path) A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() build_timeout = self.getJobFromHistory('timeout') - self.assertEqual(build_timeout.result, 'TIMED_OUT') + with self.jobLog(build_timeout): + self.assertEqual(build_timeout.result, 'TIMED_OUT') build_faillocal = self.getJobFromHistory('faillocal') - self.assertEqual(build_faillocal.result, 'FAILURE') + with self.jobLog(build_faillocal): + self.assertEqual(build_faillocal.result, 'FAILURE') build_failpost = self.getJobFromHistory('failpost') - self.assertEqual(build_failpost.result, 'POST_FAILURE') + with self.jobLog(build_failpost): + self.assertEqual(build_failpost.result, 'POST_FAILURE') build_check_vars = self.getJobFromHistory('check-vars') - self.assertEqual(build_check_vars.result, 'SUCCESS') + with self.jobLog(build_check_vars): + self.assertEqual(build_check_vars.result, 'SUCCESS') build_hello = self.getJobFromHistory('hello-world') - self.assertEqual(build_hello.result, 'SUCCESS') + with self.jobLog(build_hello): + self.assertEqual(build_hello.result, 'SUCCESS') build_python27 = self.getJobFromHistory('python27') - self.assertEqual(build_python27.result, 'SUCCESS') - flag_path = os.path.join(self.test_root, build_python27.uuid + '.flag') - self.assertTrue(os.path.exists(flag_path)) - copied_path = os.path.join(self.test_root, build_python27.uuid + - '.copied') - self.assertTrue(os.path.exists(copied_path)) - failed_path = os.path.join(self.test_root, build_python27.uuid + - '.failed') - self.assertFalse(os.path.exists(failed_path)) - pre_flag_path = os.path.join(self.test_root, build_python27.uuid + - '.pre.flag') - self.assertTrue(os.path.exists(pre_flag_path)) - post_flag_path = os.path.join(self.test_root, build_python27.uuid + - '.post.flag') - self.assertTrue(os.path.exists(post_flag_path)) - bare_role_flag_path = os.path.join(self.test_root, - build_python27.uuid + - '.bare-role.flag') - self.assertTrue(os.path.exists(bare_role_flag_path)) + with self.jobLog(build_python27): + self.assertEqual(build_python27.result, 'SUCCESS') + flag_path = os.path.join(self.test_root, + build_python27.uuid + '.flag') + self.assertTrue(os.path.exists(flag_path)) + copied_path = os.path.join(self.test_root, build_python27.uuid + + '.copied') + self.assertTrue(os.path.exists(copied_path)) + failed_path = os.path.join(self.test_root, build_python27.uuid + + '.failed') + self.assertFalse(os.path.exists(failed_path)) + pre_flag_path = os.path.join(self.test_root, build_python27.uuid + + '.pre.flag') + self.assertTrue(os.path.exists(pre_flag_path)) + post_flag_path = os.path.join(self.test_root, build_python27.uuid + + '.post.flag') + self.assertTrue(os.path.exists(post_flag_path)) + bare_role_flag_path = os.path.join(self.test_root, + build_python27.uuid + + '.bare-role.flag') + self.assertTrue(os.path.exists(bare_role_flag_path)) + secrets_path = os.path.join(self.test_root, + build_python27.uuid + '.secrets') + with open(secrets_path) as f: + self.assertEqual(f.read(), "test-username test-password") - secrets_path = os.path.join(self.test_root, - build_python27.uuid + '.secrets') - with open(secrets_path) as f: - self.assertEqual(f.read(), "test-username test-password") - - msg = A.messages[0] - success = "{} https://success.example.com/zuul-logs/{}" - fail = "{} https://failure.example.com/zuul-logs/{}" - self.assertIn(success.format("python27", build_python27.uuid), msg) - self.assertIn(fail.format("faillocal", build_faillocal.uuid), msg) - self.assertIn(success.format("check-vars", build_check_vars.uuid), msg) - self.assertIn(success.format("hello-world", build_hello.uuid), msg) - self.assertIn(fail.format("timeout", build_timeout.uuid), msg) - self.assertIn(fail.format("failpost", build_failpost.uuid), msg) + msg = A.messages[0] + success = "{} https://success.example.com/zuul-logs/{}" + fail = "{} https://failure.example.com/zuul-logs/{}" + self.assertIn(success.format("python27", build_python27.uuid), msg) + self.assertIn(fail.format("faillocal", build_faillocal.uuid), msg) + self.assertIn(success.format("check-vars", + build_check_vars.uuid), msg) + self.assertIn(success.format("hello-world", build_hello.uuid), msg) + self.assertIn(fail.format("timeout", build_timeout.uuid), msg) + self.assertIn(fail.format("failpost", build_failpost.uuid), msg) class TestPrePlaybooks(AnsibleZuulTestCase): diff --git a/zuul/driver/bubblewrap/__init__.py b/zuul/driver/bubblewrap/__init__.py index ee81bb14b7..ec94d4b74a 100644 --- a/zuul/driver/bubblewrap/__init__.py +++ b/zuul/driver/bubblewrap/__init__.py @@ -82,7 +82,7 @@ class BubblewrapDriver(Driver, WrapperInterface): pass def setMountsMap(self, ro_dirs=[], rw_dirs=[]): - self.mounts_map = {'ro': ro_dirs, 'rw': [] + rw_dirs} + self.mounts_map = {'ro': ro_dirs, 'rw': rw_dirs} def getPopen(self, **kwargs): # Set zuul_dir if it was not passed in diff --git a/zuul/executor/server.py b/zuul/executor/server.py index 9a07b5ad77..ea5a37c3c8 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -757,6 +757,12 @@ class AnsibleJob(object): '~/.ssh/id_rsa') self.ssh_agent = SshAgent() + self.executor_variables_file = None + + if self.executor_server.config.has_option('executor', 'variables'): + self.executor_variables_file = self.executor_server.config.get( + 'executor', 'variables') + def run(self): self.running = True self.thread = threading.Thread(target=self.execute) @@ -1375,6 +1381,9 @@ class AnsibleJob(object): ro_dirs.append(self.executor_server.ansible_dir) + if self.executor_variables_file: + ro_dirs.append(self.executor_variables_file) + self.executor_server.execution_wrapper.setMountsMap(ro_dirs, rw_dirs) @@ -1477,6 +1486,9 @@ class AnsibleJob(object): % playbook.canonical_name_and_path]) cmd.extend(['-e', 'zuul_execution_branch=%s' % str(playbook.branch)]) + if self.executor_variables_file is not None: + cmd.extend(['-e@%s' % self.executor_variables_file]) + result, code = self.runAnsible( cmd=cmd, timeout=timeout, config_file=playbook.ansible_config,