zuul
/
zuul
Code Issues Proposed changes

Fix deletion of stale build dirs on startup 

This code had a bug -- it didn't build the full path.
This code was not tested.

These two things are related.

Change-Id: I7881fb30017cedc12435e0fcbfda321bdf20d611
This commit is contained in:
James E. Blair 2018-11-28 13:05:32 -08:00 committed by Tobias Henkel
parent 03290d8b2a
commit fdb1a5ce50
No known key found for this signature in database
GPG Key ID: 03750DEC158E5FA2
25 changed files with 69 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
tests/base.py
View File

@ -3177,6 +3177,7 @@ class ZuulTestCase(BaseTestCase):
self.state_root = os.path.join(self.test_root, "lib")
self.merger_state_root = os.path.join(self.test_root, "merger-lib")
self.executor_state_root = os.path.join(self.test_root, "executor-lib")
self.jobdir_root = os.path.join(self.test_root, "builds")
if os.path.exists(self.test_root):
shutil.rmtree(self.test_root)
@ -3185,6 +3186,7 @@ class ZuulTestCase(BaseTestCase):
os.makedirs(self.state_root)
os.makedirs(self.merger_state_root)
os.makedirs(self.executor_state_root)
os.makedirs(self.jobdir_root)
# Make per test copy of Configuration.
self.setup_config()
@ -3272,7 +3274,7 @@ class ZuulTestCase(BaseTestCase):
self.executor_server = RecordingExecutorServer(
self.config, self.connections,
jobdir_root=self.test_root,
jobdir_root=self.jobdir_root,
_run_ansible=self.run_ansible,
_test_root=self.test_root,
keep_jobdir=KEEP_TEMPDIRS,
@ -4408,11 +4410,11 @@ class AnsibleZuulTestCase(ZuulTestCase):
try:
yield
except Exception:
path = os.path.join(self.test_root, build.uuid,
path = os.path.join(self.jobdir_root, build.uuid,
'work', 'logs', 'job-output.txt')
with open(path) as f:
self.log.debug(f.read())
path = os.path.join(self.test_root, build.uuid,
path = os.path.join(self.jobdir_root, build.uuid,
'work', 'logs', 'job-output.json')
with open(path) as f:
self.log.debug(f.read())

2
tests/fixtures/config/ansible/git/bare-role/tasks/main.yaml vendored
View File

@ -1,3 +1,3 @@
- file:
path: "{{zuul._test.test_root}}/{{zuul.build}}.bare-role.flag"
path: "{{zuul._test.test_root}}/builds/{{zuul.build}}.bare-role.flag"
state: touch

2
tests/fixtures/config/ansible/git/common-config/playbooks/post.yaml vendored
View File

@ -1,5 +1,5 @@
- hosts: all
tasks:
- file:
path: "{{zuul._test.test_root}}/{{zuul.build}}.post.flag"
path: "{{zuul._test.test_root}}/builds/{{zuul.build}}.post.flag"
state: touch

2
tests/fixtures/config/ansible/git/common-config/playbooks/pre.yaml vendored
View File

@ -1,5 +1,5 @@
- hosts: all
tasks:
- file:
path: "{{zuul._test.test_root}}/{{zuul.build}}.pre.flag"
path: "{{zuul._test.test_root}}/builds/{{zuul.build}}.pre.flag"
state: touch

6
tests/fixtures/config/ansible/git/common-config/playbooks/python27.yaml vendored
View File

@ -4,10 +4,10 @@
path: "{{flagpath}}"
state: touch
- copy:
src: "{{zuul._test.test_root}}/{{zuul.build}}.flag"
dest: "{{zuul._test.test_root}}/{{zuul.build}}.copied"
src: "{{zuul._test.test_root}}/builds/{{zuul.build}}.flag"
dest: "{{zuul._test.test_root}}/builds/{{zuul.build}}.copied"
- copy:
content: "{{test_secret.username}} {{test_secret.password}}"
dest: "{{zuul._test.test_root}}/{{zuul.build}}.secrets"
dest: "{{zuul._test.test_root}}/builds/{{zuul.build}}.secrets"
roles:
- bare-role

2
tests/fixtures/config/ansible/git/common-config/zuul.yaml vendored
View File

@ -98,7 +98,7 @@
pre-run: playbooks/pre.yaml
post-run: playbooks/post.yaml
vars:
flagpath: '{{zuul._test.test_root}}/{{zuul.build}}.flag'
flagpath: '{{zuul._test.test_root}}/builds/{{zuul.build}}.flag'
roles:
- zuul: bare-role
secrets:

4
tests/fixtures/config/ansible/git/org_project/playbooks/faillocal.yaml vendored
View File

@ -1,5 +1,5 @@
- hosts: all
tasks:
- copy:
src: "{{zuul._test.test_root}}/{{zuul.build}}.flag"
dest: "{{zuul._test.test_root}}/{{zuul.build}}.failed"
src: "{{zuul._test.test_root}}/builds/{{zuul.build}}.flag"
dest: "{{zuul._test.test_root}}/builds/{{zuul.build}}.failed"

2
tests/fixtures/config/cleanup-playbook/git/common-config/playbooks/cleanup.yaml vendored
View File

@ -2,4 +2,4 @@
tasks:
- copy:
content: "{{ zuul_success }}"
dest: "{{ zuul._test.test_root }}/{{ zuul.build }}.cleanup.flag"
dest: "{{ zuul._test.test_root }}/builds/{{ zuul.build }}.cleanup.flag"

4
tests/fixtures/config/cleanup-playbook/git/common-config/playbooks/post.yaml vendored
View File

@ -2,12 +2,12 @@
tasks:
- debug: var=waitpath
- file:
path: "{{zuul._test.test_root}}/{{zuul.build}}.post_start.flag"
path: "{{zuul._test.test_root}}/builds/{{zuul.build}}.post_start.flag"
state: touch
# Do not finish until test creates the flag file
- wait_for:
state: present
path: "{{waitpath}}"
- file:
path: "{{zuul._test.test_root}}/{{zuul.build}}.post_end.flag"
path: "{{zuul._test.test_root}}/builds/{{zuul.build}}.post_end.flag"
state: touch

2
tests/fixtures/config/cleanup-playbook/git/common-config/zuul.yaml vendored
View File

@ -26,7 +26,7 @@
post-run: playbooks/post.yaml
cleanup-run: playbooks/cleanup.yaml
vars:
waitpath: '{{zuul._test.test_root}}/{{zuul.build}}/test_wait'
waitpath: '{{zuul._test.test_root}}/builds/{{zuul.build}}/test_wait'
run: playbooks/python27.yaml
- job:

4
tests/fixtures/config/post-playbook/git/common-config/playbooks/post.yaml vendored
View File

@ -2,12 +2,12 @@
tasks:
- debug: var=waitpath
- file:
path: "{{zuul._test.test_root}}/{{zuul.build}}.post_start.flag"
path: "{{zuul._test.test_root}}/builds/{{zuul.build}}.post_start.flag"
state: touch
# Do not finish until test creates the flag file
- wait_for:
state: present
path: "{{waitpath}}"
- file:
path: "{{zuul._test.test_root}}/{{zuul.build}}.post_end.flag"
path: "{{zuul._test.test_root}}/builds/{{zuul.build}}.post_end.flag"
state: touch

2
tests/fixtures/config/post-playbook/git/common-config/zuul.yaml vendored
View File

@ -25,5 +25,5 @@
pre-run: playbooks/pre.yaml
post-run: playbooks/post.yaml
vars:
waitpath: '{{zuul._test.test_root}}/{{zuul.build}}/test_wait'
waitpath: '{{zuul._test.test_root}}/builds/{{zuul.build}}/test_wait'
run: playbooks/python27.yaml

2
tests/fixtures/config/pre-playbook/git/common-config/playbooks/post-fail.yaml vendored
View File

@ -1,5 +1,5 @@
- hosts: all
tasks:
- invalid-task:
path: "{{zuul._test.test_root}}/{{zuul.build}}.post.flag"
path: "{{zuul._test.test_root}}/builds/{{zuul.build}}.post.flag"
state: touch

2
tests/fixtures/config/pre-playbook/git/common-config/playbooks/post.yaml vendored
View File

@ -1,5 +1,5 @@
- hosts: all
tasks:
- file:
path: "{{zuul._test.test_root}}/{{zuul.build}}.post.flag"
path: "{{zuul._test.test_root}}/builds/{{zuul.build}}.post.flag"
state: touch

6
tests/fixtures/config/pre-playbook/git/common-config/playbooks/pre.yaml vendored
View File

@ -1,8 +1,8 @@
- hosts: all
tasks:
- copy:
src: "{{zuul._test.test_root}}/{{zuul.build}}.flag"
dest: "{{zuul._test.test_root}}/{{zuul.build}}.failed"
src: "{{zuul._test.test_root}}/builds/{{zuul.build}}.flag"
dest: "{{zuul._test.test_root}}/builds/{{zuul.build}}.failed"
- file:
path: "{{zuul._test.test_root}}/{{zuul.build}}.pre.flag"
path: "{{zuul._test.test_root}}/builds/{{zuul.build}}.pre.flag"
state: touch

2
tests/fixtures/config/pre-playbook/git/common-config/playbooks/python27.yaml vendored
View File

@ -1,5 +1,5 @@
- hosts: all
tasks:
- file:
path: "{{zuul._test.test_root}}/{{zuul.build}}.main.flag"
path: "{{zuul._test.test_root}}/builds/{{zuul.build}}.main.flag"
state: touch

2
tests/fixtures/config/roles/git/bare-role/tasks/main.yaml vendored
View File

@ -1,3 +1,3 @@
- file:
path: "{{zuul._test.test_root}}/{{zuul.build}}.bare-role.flag"
path: "{{zuul._test.test_root}}/builds/{{zuul.build}}.bare-role.flag"
state: touch

2
tests/fixtures/config/streamer/git/common-config/zuul.yaml vendored
View File

@ -22,5 +22,5 @@
- job:
name: python27
vars:
waitpath: '{{zuul._test.test_root}}/{{zuul.build}}/test_wait'
waitpath: '{{zuul._test.test_root}}/builds/{{zuul.build}}/test_wait'
run: playbooks/python27.yaml

2
tests/remote/test_remote_action_modules.py
View File

@ -78,7 +78,7 @@ class TestActionModules25(AnsibleZuulTestCase):
self.assertEqual(build.result, result)
if expect_error:
path = os.path.join(self.test_root, build.uuid,
path = os.path.join(self.jobdir_root, build.uuid,
'work', 'logs', 'job-output.txt')
with open(path, 'r') as f:
self.assertIn(expect_error, f.read())

2
tests/remote/test_remote_hostvars.py
View File

@ -73,7 +73,7 @@ class TestZuulHostVars(AnsibleZuulTestCase):
with self.jobLog(job):
build = self.history[-1]
inventory = yaml.safe_load(open(os.path.join(
self.test_root, build.uuid, 'ansible', 'inventory.yaml')))
self.jobdir_root, build.uuid, 'ansible', 'inventory.yaml')))
self.assertEqual(
"python2",
inventory["all"]["hosts"]["controller"][

2
tests/remote/test_remote_zuul_json.py
View File

@ -68,7 +68,7 @@ class TestZuulJSON25(AnsibleZuulTestCase):
return job
def _get_json_as_text(self, build):
path = os.path.join(self.test_root, build.uuid,
path = os.path.join(self.jobdir_root, build.uuid,
'work', 'logs', 'job-output.json')
with open(path) as f:
return f.read()

2
tests/remote/test_remote_zuul_stream.py
View File

@ -75,7 +75,7 @@ class TestZuulStream25(AnsibleZuulTestCase):
return job
def _get_job_output(self, build):
path = os.path.join(self.test_root, build.uuid,
path = os.path.join(self.jobdir_root, build.uuid,
'work', 'logs', 'job-output.txt')
with open(path) as f:
return f.read()

12
tests/unit/test_executor.py
View File

@ -789,3 +789,15 @@ class TestExecutorFacts(AnsibleZuulTestCase):
date_time = \
j[0]['plays'][0]['tasks'][0]['hosts']['localhost']['date_time']
self.assertEqual(18, len(date_time))
class TestExecutorStart(ZuulTestCase):
tenant_config_file = 'config/single-tenant/main.yaml'
def setup_config(self):
super(TestExecutorStart, self).setup_config()
self.junk_dir = os.path.join(self.jobdir_root, 'junk')
os.makedirs(self.junk_dir)
def test_executor_start(self):
self.assertFalse(os.path.exists(self.junk_dir))

46
tests/unit/test_v3.py
View File

@ -2591,8 +2591,8 @@ class TestAnsible25(AnsibleZuulTestCase):
self.waitUntilSettled()
build_timeout = self.getJobFromHistory('timeout', result='TIMED_OUT')
with self.jobLog(build_timeout):
post_flag_path = os.path.join(self.test_root, build_timeout.uuid +
'.post.flag')
post_flag_path = os.path.join(
self.jobdir_root, build_timeout.uuid + '.post.flag')
self.assertTrue(os.path.exists(post_flag_path))
build_post_timeout = self.getJobFromHistory('post-timeout')
with self.jobLog(build_post_timeout):
@ -2635,26 +2635,26 @@ class TestAnsible25(AnsibleZuulTestCase):
build_python27 = self.getJobFromHistory('python27')
with self.jobLog(build_python27):
self.assertEqual(build_python27.result, 'SUCCESS')
flag_path = os.path.join(self.test_root,
flag_path = os.path.join(self.jobdir_root,
build_python27.uuid + '.flag')
self.assertTrue(os.path.exists(flag_path))
copied_path = os.path.join(self.test_root, build_python27.uuid +
copied_path = os.path.join(self.jobdir_root, build_python27.uuid +
'.copied')
self.assertTrue(os.path.exists(copied_path))
failed_path = os.path.join(self.test_root, build_python27.uuid +
failed_path = os.path.join(self.jobdir_root, build_python27.uuid +
'.failed')
self.assertFalse(os.path.exists(failed_path))
pre_flag_path = os.path.join(self.test_root, build_python27.uuid +
'.pre.flag')
pre_flag_path = os.path.join(
self.jobdir_root, build_python27.uuid + '.pre.flag')
self.assertTrue(os.path.exists(pre_flag_path))
post_flag_path = os.path.join(self.test_root, build_python27.uuid +
'.post.flag')
post_flag_path = os.path.join(
self.jobdir_root, build_python27.uuid + '.post.flag')
self.assertTrue(os.path.exists(post_flag_path))
bare_role_flag_path = os.path.join(self.test_root,
bare_role_flag_path = os.path.join(self.jobdir_root,
build_python27.uuid +
'.bare-role.flag')
self.assertTrue(os.path.exists(bare_role_flag_path))
secrets_path = os.path.join(self.test_root,
secrets_path = os.path.join(self.jobdir_root,
build_python27.uuid + '.secrets')
with open(secrets_path) as f:
self.assertEqual(f.read(), "test-username test-password")
@ -2796,8 +2796,8 @@ class TestPrePlaybooks(AnsibleZuulTestCase):
pre_flag_path = os.path.join(self.test_root, build.uuid +
'.pre.flag')
self.assertFalse(os.path.exists(pre_flag_path))
post_flag_path = os.path.join(self.test_root, build.uuid +
'.post.flag')
post_flag_path = os.path.join(
self.jobdir_root, build.uuid + '.post.flag')
self.assertTrue(os.path.exists(post_flag_path),
"The file %s should exist" % post_flag_path)
@ -2878,7 +2878,7 @@ class TestPostPlaybooks(AnsibleZuulTestCase):
break
build = self.builds[0]
post_start = os.path.join(self.test_root, build.uuid +
post_start = os.path.join(self.jobdir_root, build.uuid +
'.post_start.flag')
for _ in iterate_timeout(60, 'job post running'):
if os.path.exists(post_start):
@ -2890,7 +2890,7 @@ class TestPostPlaybooks(AnsibleZuulTestCase):
build = self.getJobFromHistory('python27')
self.assertEqual('ABORTED', build.result)
post_end = os.path.join(self.test_root, build.uuid +
post_end = os.path.join(self.jobdir_root, build.uuid +
'.post_end.flag')
self.assertTrue(os.path.exists(post_start))
self.assertFalse(os.path.exists(post_end))
@ -2910,19 +2910,19 @@ class TestCleanupPlaybooks(AnsibleZuulTestCase):
break
build = self.builds[0]
post_start = os.path.join(self.test_root, build.uuid +
post_start = os.path.join(self.jobdir_root, build.uuid +
'.post_start.flag')
for _ in iterate_timeout(60, 'job post running'):
if os.path.exists(post_start):
break
with open(os.path.join(self.test_root, build.uuid, 'test_wait'),
with open(os.path.join(self.jobdir_root, build.uuid, 'test_wait'),
"w") as of:
of.write("continue")
self.waitUntilSettled()
build = self.getJobFromHistory('python27')
self.assertEqual('SUCCESS', build.result)
cleanup_flag = os.path.join(self.test_root, build.uuid +
cleanup_flag = os.path.join(self.jobdir_root, build.uuid +
'.cleanup.flag')
self.assertTrue(os.path.exists(cleanup_flag))
with open(cleanup_flag) as f:
@ -2949,7 +2949,7 @@ class TestCleanupPlaybooks(AnsibleZuulTestCase):
build = self.getJobFromHistory('python27-failure')
self.assertEqual('FAILURE', build.result)
cleanup_flag = os.path.join(self.test_root, build.uuid +
cleanup_flag = os.path.join(self.jobdir_root, build.uuid +
'.cleanup.flag')
self.assertTrue(os.path.exists(cleanup_flag))
with open(cleanup_flag) as f:
@ -2966,7 +2966,7 @@ class TestCleanupPlaybooks(AnsibleZuulTestCase):
break
build = self.builds[0]
post_start = os.path.join(self.test_root, build.uuid +
post_start = os.path.join(self.jobdir_root, build.uuid +
'.post_start.flag')
for _ in iterate_timeout(60, 'job post running'):
if os.path.exists(post_start):
@ -2978,9 +2978,9 @@ class TestCleanupPlaybooks(AnsibleZuulTestCase):
build = self.getJobFromHistory('python27')
self.assertEqual('ABORTED', build.result)
post_end = os.path.join(self.test_root, build.uuid +
post_end = os.path.join(self.jobdir_root, build.uuid +
'.post_end.flag')
cleanup_flag = os.path.join(self.test_root, build.uuid +
cleanup_flag = os.path.join(self.jobdir_root, build.uuid +
'.cleanup.flag')
self.assertTrue(os.path.exists(cleanup_flag))
self.assertTrue(os.path.exists(post_start))
@ -3320,7 +3320,7 @@ class TestProjectKeys(ZuulTestCase):
class RoleTestCase(ZuulTestCase):
def _getRolesPaths(self, build, playbook):
path = os.path.join(self.test_root, build.uuid,
path = os.path.join(self.jobdir_root, build.uuid,
'ansible', playbook, 'ansible.cfg')
roles_paths = []
with open(path) as f:

1
zuul/executor/server.py
View File

@ -2390,6 +2390,7 @@ class ExecutorServer(object):
# in case we were uncleanly shut down.
if not self.keep_jobdir:
for fn in os.listdir(self.jobdir_root):
fn = os.path.join(self.jobdir_root, fn)
if not os.path.isdir(fn):
continue
self.log.info("Deleting stale jobdir %s", fn)