diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst index 04de33e014..c2053b3a27 100644 --- a/doc/source/user/config.rst +++ b/doc/source/user/config.rst @@ -686,6 +686,57 @@ Here is an example of two job definitions: tags from all the jobs and variants used in constructing the frozen job, with no duplication. + .. attr:: provides + + A list of free-form strings which identifies resources provided + by this job which may be used by other jobs for other changes + using the :attr:`job.requires` attribute. + + .. attr:: requires + + A list of free-form strings which identify resources which may + be provided by other jobs for other changes (via the + :attr:`job.provides` attribute) that are used by this job. + + When Zuul encounters a job with a `requires` attribute, it + searches for those values in the `provides` attributes of any + jobs associated with any queue items ahead of the current + change. In this way, if a change uses either git dependencies + or a `Depends-On` header to indicate a dependency on another + change, Zuul will be able to determine that the parent change + affects the run-time environment of the child change. If such a + relationship is found, the job with `requires` will not start + until all of the jobs with matching `provides` have completed or + paused. Additionally, the :ref:`artifacts ` + returned by the `provides` jobs will be made available to the + `requires` job. + + For example, a job which produces a builder container image in + one project that is then consumed by a container image build job + in another project might look like this: + + .. code-block:: yaml + + - job: + name: build-builder-image + provides: images + + - job: + name: build-final-image + requires: images + + - project: + name: builder-project + check: + jobs: + - build-builder-image + + - project: + name: final-project + check: + jobs: + - build-final-image + .. attr:: secrets A list of secrets which may be used by the job. A diff --git a/doc/source/user/jobs.rst b/doc/source/user/jobs.rst index cc195e7a69..826ff348d0 100644 --- a/doc/source/user/jobs.rst +++ b/doc/source/user/jobs.rst @@ -228,6 +228,41 @@ of item. All items provide the following information as Ansible variables under the ``zuul`` key: + .. var:: artifacts + :type: list + + If the job has a :attr:`job.requires` attribute, and Zuul has + found changes ahead of this change in the pipeline with matching + :attr:`job.provides` attributes, then information about any + :ref:`artifacts returned ` from those jobs + will appear here. + + This value is a list of dictionaries with the following format: + + .. var:: project + + The name of the project which supplied this artifact. + + .. var:: change + + The change number which supplied this artifact. + + .. var:: patchset + + The patchset of the change. + + .. var:: job + + The name of the job which produced the artifact. + + .. var:: name + + The name of the artifact (as supplied to :ref:`return_artifacts`). + + .. var:: url + + The URL of the artifact (as supplied to :ref:`return_artifacts`). + .. var:: build The UUID of the build. A build is a single execution of a job. diff --git a/releasenotes/notes/provides_requires-4c6b54ede999e86c.yaml b/releasenotes/notes/provides_requires-4c6b54ede999e86c.yaml new file mode 100644 index 0000000000..1990065915 --- /dev/null +++ b/releasenotes/notes/provides_requires-4c6b54ede999e86c.yaml @@ -0,0 +1,7 @@ +--- +features: + - Support for expressing artifact or other resource dependencies + between jobs running on different changes with a dependency + relationship (e.g., a container image built in one project and + consumed in a second project) has been added via the + :attr:`job.provides` and :attr:`job.requires` job attributes. diff --git a/tests/base.py b/tests/base.py index a2f7c931b0..a8803c79cc 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1355,6 +1355,11 @@ class FakeBuild(object): items = self.parameters['zuul']['items'] self.changes = ' '.join(['%s,%s' % (x['change'], x['patchset']) for x in items if 'change' in x]) + if 'change' in items[-1]: + self.change = ' '.join((items[-1]['change'], + items[-1]['patchset'])) + else: + self.change = None def __repr__(self): waiting = '' @@ -1401,6 +1406,8 @@ class FakeBuild(object): self._wait() self.log.debug("Build %s continuing" % self.unique) + self.writeReturnData() + result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success if self.shouldFail(): result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure @@ -1418,6 +1425,14 @@ class FakeBuild(object): return True return False + def writeReturnData(self): + changes = self.executor_server.return_data.get(self.name, {}) + data = changes.get(self.change) + if data is None: + return + with open(self.jobdir.result_data_file, 'w') as f: + f.write(json.dumps(data)) + def hasChanges(self, *changes): """Return whether this build has certain changes in its git repos. @@ -1554,6 +1569,7 @@ class RecordingExecutorServer(zuul.executor.server.ExecutorServer): self.running_builds = [] self.build_history = [] self.fail_tests = {} + self.return_data = {} self.job_builds = {} def failJob(self, name, change): @@ -1569,6 +1585,19 @@ class RecordingExecutorServer(zuul.executor.server.ExecutorServer): l.append(change) self.fail_tests[name] = l + def returnData(self, name, change, data): + """Instruct the executor to return data for this build. + + :arg str name: The name of the job to return data. + :arg Change change: The :py:class:`~tests.base.FakeChange` + instance which should cause the job to return data. + :arg dict data: The data to return + + """ + changes = self.return_data.setdefault(name, {}) + cid = ' '.join((str(change.number), str(change.latest_patchset))) + changes[cid] = data + def release(self, regex=None): """Release a held build. diff --git a/tests/fixtures/config/provides-requires-pause/git/common-config/zuul.yaml b/tests/fixtures/config/provides-requires-pause/git/common-config/zuul.yaml new file mode 100644 index 0000000000..12901dd09a --- /dev/null +++ b/tests/fixtures/config/provides-requires-pause/git/common-config/zuul.yaml @@ -0,0 +1,38 @@ +- pipeline: + name: check + manager: independent + post-review: true + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + Verified: 1 + failure: + gerrit: + Verified: -1 + +- pipeline: + name: gate + manager: dependent + post-review: True + trigger: + gerrit: + - event: comment-added + approval: + - Approved: 1 + success: + gerrit: + Verified: 2 + submit: true + failure: + gerrit: + Verified: -2 + start: + gerrit: + Verified: 0 + precedence: high + +- job: + name: base + parent: null diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project1/README b/tests/fixtures/config/provides-requires-pause/git/org_project1/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/provides-requires-pause/git/org_project1/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-builder.yaml b/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-builder.yaml new file mode 100644 index 0000000000..7d773b612b --- /dev/null +++ b/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-builder.yaml @@ -0,0 +1,10 @@ +- hosts: all + tasks: + - name: Pause and let child run + zuul_return: + data: + zuul: + pause: true + artifacts: + - name: image + url: http://example.com/image diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-user.yaml b/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-user.yaml new file mode 100644 index 0000000000..583279c71a --- /dev/null +++ b/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-user.yaml @@ -0,0 +1,4 @@ +- hosts: all + tasks: + - debug: + var: zuul.artifacts diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml b/tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml new file mode 100644 index 0000000000..412fe2c18c --- /dev/null +++ b/tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml @@ -0,0 +1,26 @@ +- job: + name: image-builder + provides: + - image + run: playbooks/image-builder.yaml + +- job: + name: image-user + requires: + - image + run: playbooks/image-user.yaml + +- project: + check: + jobs: + - image-builder + - image-user: + dependencies: + - image-builder + gate: + queue: integrated + jobs: + - image-builder + - image-user: + dependencies: + - image-builder diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml b/tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml new file mode 100644 index 0000000000..e9e6b58679 --- /dev/null +++ b/tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml @@ -0,0 +1,8 @@ +- project: + check: + jobs: + - image-user + gate: + queue: integrated + jobs: + - image-user diff --git a/tests/fixtures/config/provides-requires-pause/main.yaml b/tests/fixtures/config/provides-requires-pause/main.yaml new file mode 100644 index 0000000000..3a7415582b --- /dev/null +++ b/tests/fixtures/config/provides-requires-pause/main.yaml @@ -0,0 +1,8 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config + - org/project1 + - org/project2 diff --git a/tests/fixtures/layouts/provides-requires-two-jobs.yaml b/tests/fixtures/layouts/provides-requires-two-jobs.yaml new file mode 100644 index 0000000000..6d1cc77984 --- /dev/null +++ b/tests/fixtures/layouts/provides-requires-two-jobs.yaml @@ -0,0 +1,72 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + Verified: 1 + resultsdb_mysql: null + resultsdb_postgresql: null + failure: + gerrit: + Verified: -1 + resultsdb_mysql: null + resultsdb_postgresql: null + +- pipeline: + name: gate + manager: dependent + success-message: Build succeeded (gate). + trigger: + gerrit: + - event: comment-added + approval: + - Approved: 1 + success: + gerrit: + Verified: 2 + submit: true + failure: + gerrit: + Verified: -2 + start: + gerrit: + Verified: 0 + precedence: high + +- job: + name: base + parent: null + run: playbooks/base.yaml + +- job: + name: image-builder + provides: images + +- job: + name: image-user + requires: images + +- project: + name: org/project1 + check: + jobs: + - image-builder + gate: + queue: integrated + jobs: + - image-builder + - image-user: + dependencies: image-builder + +- project: + name: org/project2 + check: + jobs: + - image-user + gate: + queue: integrated + jobs: + - image-user diff --git a/tests/fixtures/layouts/provides-requires-unshared.yaml b/tests/fixtures/layouts/provides-requires-unshared.yaml new file mode 100644 index 0000000000..65df1bc59c --- /dev/null +++ b/tests/fixtures/layouts/provides-requires-unshared.yaml @@ -0,0 +1,58 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + Verified: 1 + failure: + gerrit: + Verified: -1 + +- pipeline: + name: gate + manager: dependent + success-message: Build succeeded (gate). + trigger: + gerrit: + - event: comment-added + approval: + - Approved: 1 + success: + gerrit: + Verified: 2 + submit: true + failure: + gerrit: + Verified: -2 + start: + gerrit: + Verified: 0 + precedence: high + +- job: + name: base + parent: null + run: playbooks/base.yaml + +- job: + name: image-builder + provides: images + +- job: + name: image-user + requires: images + +- project: + name: org/project1 + gate: + jobs: + - image-builder + +- project: + name: org/project2 + gate: + jobs: + - image-user diff --git a/tests/fixtures/layouts/provides-requires.yaml b/tests/fixtures/layouts/provides-requires.yaml new file mode 100644 index 0000000000..39473451c6 --- /dev/null +++ b/tests/fixtures/layouts/provides-requires.yaml @@ -0,0 +1,70 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + Verified: 1 + resultsdb_mysql: null + resultsdb_postgresql: null + failure: + gerrit: + Verified: -1 + resultsdb_mysql: null + resultsdb_postgresql: null + +- pipeline: + name: gate + manager: dependent + success-message: Build succeeded (gate). + trigger: + gerrit: + - event: comment-added + approval: + - Approved: 1 + success: + gerrit: + Verified: 2 + submit: true + failure: + gerrit: + Verified: -2 + start: + gerrit: + Verified: 0 + precedence: high + +- job: + name: base + parent: null + run: playbooks/base.yaml + +- job: + name: image-builder + provides: images + +- job: + name: image-user + requires: images + +- project: + name: org/project1 + check: + jobs: + - image-builder + gate: + queue: integrated + jobs: + - image-builder + +- project: + name: org/project2 + check: + jobs: + - image-user + gate: + queue: integrated + jobs: + - image-user diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py index d6554a08a7..4ae89fd291 100644 --- a/tests/unit/test_v3.py +++ b/tests/unit/test_v3.py @@ -28,6 +28,7 @@ from zuul.lib import encryption from tests.base import ( AnsibleZuulTestCase, ZuulTestCase, + ZuulDBTestCase, FIXTURE_DIR, simple_layout, ) @@ -4714,3 +4715,290 @@ class TestContainerJobs(AnsibleZuulTestCase): dict(name='container-machine', result='SUCCESS', changes='1,1'), dict(name='container-native', result='SUCCESS', changes='1,1'), ]) + + +class TestProvidesRequiresPause(AnsibleZuulTestCase): + tenant_config_file = "config/provides-requires-pause/main.yaml" + + def test_provides_requires_pause(self): + # Changes share a queue, with both running at the same time. + self.executor_server.hold_jobs_in_build = True + A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') + A.addApproval('Code-Review', 2) + self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) + self.waitUntilSettled() + + self.assertEqual(len(self.builds), 1) + + B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') + B.addApproval('Code-Review', 2) + self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) + self.waitUntilSettled() + + self.assertEqual(len(self.builds), 1) + + # Release image-build, it should cause both instances of + # image-user to run. + self.executor_server.hold_jobs_in_build = False + self.executor_server.release() + self.waitUntilSettled() + + self.assertHistory([ + dict(name='image-builder', result='SUCCESS', changes='1,1'), + dict(name='image-user', result='SUCCESS', changes='1,1'), + dict(name='image-user', result='SUCCESS', changes='1,1 2,1'), + ], ordered=False) + build = self.getJobFromHistory('image-user', project='org/project2') + self.assertEqual( + build.parameters['zuul']['artifacts'], + [{ + 'project': 'org/project1', + 'change': '1', + 'patchset': '1', + 'job': 'image-builder', + 'url': 'http://example.com/image', + 'name': 'image', + }]) + + +class TestProvidesRequires(ZuulDBTestCase): + config_file = "zuul-sql-driver.conf" + + @simple_layout('layouts/provides-requires.yaml') + def test_provides_requires_shared_queue_fast(self): + # Changes share a queue, but with only one job, the first + # merges before the second starts. + self.executor_server.hold_jobs_in_build = True + A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') + self.executor_server.returnData( + 'image-builder', A, + {'zuul': + {'artifacts': [ + {'name': 'image', 'url': 'http://example.com/image'}, + ]}} + ) + A.addApproval('Code-Review', 2) + self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) + self.waitUntilSettled() + + self.assertEqual(len(self.builds), 1) + + B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') + B.addApproval('Code-Review', 2) + self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) + self.waitUntilSettled() + + self.assertEqual(len(self.builds), 1) + + self.executor_server.hold_jobs_in_build = False + self.executor_server.release() + self.waitUntilSettled() + + self.assertHistory([ + dict(name='image-builder', result='SUCCESS', changes='1,1'), + dict(name='image-user', result='SUCCESS', changes='1,1 2,1'), + ]) + # Data are not passed in this instance because the builder + # change merges before the user job runs. + self.assertFalse('artifacts' in self.history[-1].parameters['zuul']) + + @simple_layout('layouts/provides-requires-two-jobs.yaml') + def test_provides_requires_shared_queue_slow(self): + # Changes share a queue, with both running at the same time. + self.executor_server.hold_jobs_in_build = True + A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') + self.executor_server.returnData( + 'image-builder', A, + {'zuul': + {'artifacts': [ + {'name': 'image', 'url': 'http://example.com/image'}, + ]}} + ) + A.addApproval('Code-Review', 2) + self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) + self.waitUntilSettled() + + self.assertEqual(len(self.builds), 1) + + B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') + B.addApproval('Code-Review', 2) + self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) + self.waitUntilSettled() + + self.assertEqual(len(self.builds), 1) + + # Release image-build, it should cause both instances of + # image-user to run. + self.executor_server.release() + self.waitUntilSettled() + self.assertEqual(len(self.builds), 2) + self.assertHistory([ + dict(name='image-builder', result='SUCCESS', changes='1,1'), + ]) + + self.orderedRelease() + self.waitUntilSettled() + + self.assertHistory([ + dict(name='image-builder', result='SUCCESS', changes='1,1'), + dict(name='image-user', result='SUCCESS', changes='1,1'), + dict(name='image-user', result='SUCCESS', changes='1,1 2,1'), + ]) + self.assertEqual( + self.history[-1].parameters['zuul']['artifacts'], + [{ + 'project': 'org/project1', + 'change': '1', + 'patchset': '1', + 'job': 'image-builder', + 'url': 'http://example.com/image', + 'name': 'image', + }]) + + @simple_layout('layouts/provides-requires-unshared.yaml') + def test_provides_requires_unshared_queue(self): + self.executor_server.hold_jobs_in_build = True + A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') + self.executor_server.returnData( + 'image-builder', A, + {'zuul': + {'artifacts': [ + {'name': 'image', 'url': 'http://example.com/image'}, + ]}} + ) + A.addApproval('Code-Review', 2) + self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) + self.waitUntilSettled() + + self.assertEqual(len(self.builds), 1) + + B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') + B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % ( + B.subject, A.data['id']) + B.addApproval('Code-Review', 2) + self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) + self.waitUntilSettled() + + self.assertEqual(len(self.builds), 1) + + self.executor_server.hold_jobs_in_build = False + self.executor_server.release() + self.waitUntilSettled() + + self.assertHistory([ + dict(name='image-builder', result='SUCCESS', changes='1,1'), + ]) + + self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) + self.waitUntilSettled() + + self.assertHistory([ + dict(name='image-builder', result='SUCCESS', changes='1,1'), + dict(name='image-user', result='SUCCESS', changes='2,1'), + ]) + # Data are not passed in this instance because the builder + # change merges before the user job runs. + self.assertFalse('artifacts' in self.history[-1].parameters['zuul']) + + @simple_layout('layouts/provides-requires.yaml') + def test_provides_requires_check_current(self): + self.executor_server.hold_jobs_in_build = True + A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') + self.executor_server.returnData( + 'image-builder', A, + {'zuul': + {'artifacts': [ + {'name': 'image', 'url': 'http://example.com/image'}, + ]}} + ) + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertEqual(len(self.builds), 1) + + B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') + B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % ( + B.subject, A.data['id']) + self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertEqual(len(self.builds), 1) + + self.executor_server.hold_jobs_in_build = False + self.executor_server.release() + self.waitUntilSettled() + + self.assertHistory([ + dict(name='image-builder', result='SUCCESS', changes='1,1'), + dict(name='image-user', result='SUCCESS', changes='1,1 2,1'), + ]) + self.assertEqual( + self.history[-1].parameters['zuul']['artifacts'], + [{ + 'project': 'org/project1', + 'change': '1', + 'patchset': '1', + 'job': 'image-builder', + 'url': 'http://example.com/image', + 'name': 'image', + }]) + + @simple_layout('layouts/provides-requires.yaml') + def test_provides_requires_check_old_success(self): + A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') + self.executor_server.returnData( + 'image-builder', A, + {'zuul': + {'artifacts': [ + {'name': 'image', 'url': 'http://example.com/image'}, + ]}} + ) + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + self.assertHistory([ + dict(name='image-builder', result='SUCCESS', changes='1,1'), + ]) + + B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') + B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % ( + B.subject, A.data['id']) + self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertHistory([ + dict(name='image-builder', result='SUCCESS', changes='1,1'), + dict(name='image-user', result='SUCCESS', changes='1,1 2,1'), + ]) + self.assertEqual( + self.history[-1].parameters['zuul']['artifacts'], + [{ + 'project': 'org/project1', + 'change': '1', + 'patchset': '1', + 'job': 'image-builder', + 'url': 'http://example.com/image', + 'name': 'image', + }]) + + @simple_layout('layouts/provides-requires.yaml') + def test_provides_requires_check_old_failure(self): + A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') + self.executor_server.failJob('image-builder', A) + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertHistory([ + dict(name='image-builder', result='FAILURE', changes='1,1'), + ]) + + B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') + B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % ( + B.subject, A.data['id']) + self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertHistory([ + dict(name='image-builder', result='FAILURE', changes='1,1'), + ]) + self.assertIn('image-user : SKIPPED', B.messages[0]) + self.assertIn('not met by build', B.messages[0]) diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py index 05701a6537..e267734958 100755 --- a/tests/unit/test_web.py +++ b/tests/unit/test_web.py @@ -305,10 +305,13 @@ class TestWeb(BaseTestWeb): 'parent': 'base', 'post_review': None, 'protected': None, + 'provides': [], 'required_projects': [], + 'requires': [], 'roles': [common_config_role], 'semaphore': None, 'source_context': source_ctx, + 'tags': [], 'timeout': None, 'variables': {}, 'variant_description': '', @@ -337,10 +340,13 @@ class TestWeb(BaseTestWeb): 'parent': 'base', 'post_review': None, 'protected': None, + 'provides': [], 'required_projects': [], + 'requires': [], 'roles': [common_config_role], 'semaphore': None, 'source_context': source_ctx, + 'tags': [], 'timeout': None, 'variables': {}, 'variant_description': 'stable', @@ -363,13 +369,16 @@ class TestWeb(BaseTestWeb): 'parent': 'base', 'post_review': None, 'protected': None, + 'provides': [], 'required_projects': [ {'override_branch': None, 'override_checkout': None, 'project_name': 'review.example.com/org/project'}], + 'requires': [], 'roles': [common_config_role], 'semaphore': None, 'source_context': source_ctx, + 'tags': [], 'timeout': None, 'variables': {}, 'variant_description': '', @@ -434,13 +443,16 @@ class TestWeb(BaseTestWeb): 'parent': 'base', 'post_review': None, 'protected': None, + 'provides': [], 'required_projects': [], + 'requires': [], 'roles': [], 'semaphore': None, 'source_context': { 'branch': 'master', 'path': 'zuul.yaml', 'project': 'common-config'}, + 'tags': [], 'timeout': None, 'variables': {}, 'variant_description': '', @@ -458,13 +470,16 @@ class TestWeb(BaseTestWeb): 'parent': 'base', 'post_review': None, 'protected': None, + 'provides': [], 'required_projects': [], + 'requires': [], 'roles': [], 'semaphore': None, 'source_context': { 'branch': 'master', 'path': 'zuul.yaml', 'project': 'common-config'}, + 'tags': [], 'timeout': None, 'variables': {}, 'variant_description': '', @@ -482,13 +497,16 @@ class TestWeb(BaseTestWeb): 'parent': 'base', 'post_review': None, 'protected': None, + 'provides': [], 'required_projects': [], + 'requires': [], 'roles': [], 'semaphore': None, 'source_context': { 'branch': 'master', 'path': 'zuul.yaml', 'project': 'common-config'}, + 'tags': [], 'timeout': None, 'variables': {}, 'variant_description': '', @@ -506,13 +524,16 @@ class TestWeb(BaseTestWeb): 'parent': 'base', 'post_review': None, 'protected': None, + 'provides': [], 'required_projects': [], + 'requires': [], 'roles': [], 'semaphore': None, 'source_context': { 'branch': 'master', 'path': 'zuul.yaml', 'project': 'common-config'}, + 'tags': [], 'timeout': None, 'variables': {}, 'variant_description': '', diff --git a/zuul/configloader.py b/zuul/configloader.py index 64318e2658..5e4afc0e8d 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -545,6 +545,8 @@ class JobParser(object): 'final': bool, 'abstract': bool, 'protected': bool, + 'requires': to_list(str), + 'provides': to_list(str), 'failure-message': str, 'success-message': str, 'failure-url': str, @@ -769,11 +771,10 @@ class JobParser(object): semaphore.get('name'), semaphore.get('resources-first', False)) - tags = conf.get('tags') - if tags: - job.tags = set(tags) - - job.dependencies = frozenset(as_list(conf.get('dependencies'))) + for k in ('tags', 'requires', 'provides', 'dependencies'): + v = frozenset(as_list(conf.get(k))) + if v: + setattr(job, k, v) variables = conf.get('vars', None) if variables: diff --git a/zuul/driver/sql/alembic/versions/39d302d34d38_add_provides.py b/zuul/driver/sql/alembic/versions/39d302d34d38_add_provides.py new file mode 100644 index 0000000000..043b0c83d2 --- /dev/null +++ b/zuul/driver/sql/alembic/versions/39d302d34d38_add_provides.py @@ -0,0 +1,46 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""add_provides + +Revision ID: 39d302d34d38 +Revises: 649ce63b5fe5 +Create Date: 2019-01-28 15:01:07.408072 + +""" + +# revision identifiers, used by Alembic. +revision = '39d302d34d38' +down_revision = '649ce63b5fe5' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +PROVIDES_TABLE = 'zuul_provides' +BUILD_TABLE = 'zuul_build' + + +def upgrade(table_prefix=''): + op.create_table( + table_prefix + PROVIDES_TABLE, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('build_id', sa.Integer, + sa.ForeignKey(table_prefix + BUILD_TABLE + ".id")), + sa.Column('name', sa.String(255)), + ) + + +def downgrade(): + raise Exception("Downgrades not supported") diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py index 1a0efb559c..5cf79e8ca6 100644 --- a/zuul/driver/sql/sqlconnection.py +++ b/zuul/driver/sql/sqlconnection.py @@ -28,6 +28,7 @@ from zuul.connection import BaseConnection BUILDSET_TABLE = 'zuul_buildset' BUILD_TABLE = 'zuul_build' ARTIFACT_TABLE = 'zuul_artifact' +PROVIDES_TABLE = 'zuul_provides' class DatabaseSession(object): @@ -56,17 +57,21 @@ class DatabaseSession(object): def getBuilds(self, tenant=None, project=None, pipeline=None, change=None, branch=None, patchset=None, ref=None, newrev=None, uuid=None, job_name=None, voting=None, - node_name=None, result=None, limit=50, offset=0): + node_name=None, result=None, provides=None, + limit=50, offset=0): build_table = self.connection.zuul_build_table buildset_table = self.connection.zuul_buildset_table + provides_table = self.connection.zuul_provides_table # contains_eager allows us to perform eager loading on the # buildset *and* use that table in filters (unlike # joinedload). q = self.session().query(self.connection.buildModel).\ join(self.connection.buildSetModel).\ + outerjoin(self.connection.providesModel).\ options(orm.contains_eager(self.connection.buildModel.buildset), + orm.selectinload(self.connection.buildModel.provides), orm.selectinload(self.connection.buildModel.artifacts)).\ with_hint(build_table, 'USE INDEX (PRIMARY)', 'mysql') @@ -83,6 +88,7 @@ class DatabaseSession(object): q = self.listFilter(q, build_table.c.voting, voting) q = self.listFilter(q, build_table.c.node_name, node_name) q = self.listFilter(q, build_table.c.result, result) + q = self.listFilter(q, provides_table.c.name, provides) q = q.order_by(build_table.c.id.desc()).\ limit(limit).\ @@ -224,6 +230,15 @@ class SQLConnection(BaseConnection): session.flush() return a + def createProvides(self, *args, **kw): + session = orm.session.Session.object_session(self) + p = ProvidesModel(*args, **kw) + p.build_id = self.id + self.provides.append(p) + session.add(p) + session.flush() + return p + class ArtifactModel(Base): __tablename__ = self.table_prefix + ARTIFACT_TABLE id = sa.Column(sa.Integer, primary_key=True) @@ -233,6 +248,17 @@ class SQLConnection(BaseConnection): url = sa.Column(sa.TEXT()) build = orm.relationship(BuildModel, backref="artifacts") + class ProvidesModel(Base): + __tablename__ = self.table_prefix + PROVIDES_TABLE + id = sa.Column(sa.Integer, primary_key=True) + build_id = sa.Column(sa.Integer, sa.ForeignKey( + self.table_prefix + BUILD_TABLE + ".id")) + name = sa.Column(sa.String(255)) + build = orm.relationship(BuildModel, backref="provides") + + self.providesModel = ProvidesModel + self.zuul_provides_table = self.providesModel.__table__ + self.artifactModel = ArtifactModel self.zuul_artifact_table = self.artifactModel.__table__ diff --git a/zuul/driver/sql/sqlreporter.py b/zuul/driver/sql/sqlreporter.py index 1e148d623f..16651e4b4d 100644 --- a/zuul/driver/sql/sqlreporter.py +++ b/zuul/driver/sql/sqlreporter.py @@ -16,9 +16,9 @@ import datetime import logging import time import voluptuous as v -import urllib.parse from zuul.reporter import BaseReporter +from zuul.lib.artifacts import get_artifacts_from_result_data class SQLReporter(BaseReporter): @@ -27,26 +27,6 @@ class SQLReporter(BaseReporter): name = 'sql' log = logging.getLogger("zuul.SQLReporter") - artifact = { - 'name': str, - 'url': str, - } - zuul_data = { - 'zuul': { - 'log_url': str, - 'artifacts': [artifact], - v.Extra: object, - } - } - artifact_schema = v.Schema(zuul_data) - - def validateArtifactSchema(self, data): - try: - self.artifact_schema(data) - except Exception: - return False - return True - def report(self, item): """Create an entry into a database.""" @@ -104,32 +84,13 @@ class SQLReporter(BaseReporter): node_name=build.node_name, ) - if self.validateArtifactSchema(build.result_data): - artifacts = build.result_data.get('zuul', {}).get( - 'artifacts', []) - default_url = build.result_data.get('zuul', {}).get( - 'log_url') - if default_url: - if default_url[-1] != '/': - default_url += '/' - for artifact in artifacts: - url = artifact['url'] - if default_url: - # If the artifact url is relative, it will - # be combined with the log_url; if it is - # absolute, it will replace it. - try: - url = urllib.parse.urljoin(default_url, url) - except Exception: - self.log.debug("Error parsing URL:", - exc_info=1) - db_build.createArtifact( - name=artifact['name'], - url=url, - ) - else: - self.log.debug("Result data did not pass artifact schema " - "validation: %s", build.result_data) + for provides in job.provides: + db_build.createProvides(name=provides) + + for artifact in get_artifacts_from_result_data( + build.result_data, + logger=self.log): + db_build.createArtifact(**artifact) def getSchema(): diff --git a/zuul/executor/client.py b/zuul/executor/client.py index 00ea8eb6be..801726d10a 100644 --- a/zuul/executor/client.py +++ b/zuul/executor/client.py @@ -165,6 +165,8 @@ class ExecutorClient(object): timeout=job.timeout, jobtags=sorted(job.tags), _inheritance_path=list(job.inheritance_path)) + if job.artifact_data: + zuul_params['artifacts'] = job.artifact_data if job.override_checkout: zuul_params['override_checkout'] = job.override_checkout if hasattr(item.change, 'branch'): diff --git a/zuul/lib/artifacts.py b/zuul/lib/artifacts.py new file mode 100644 index 0000000000..c7c2fe0ec7 --- /dev/null +++ b/zuul/lib/artifacts.py @@ -0,0 +1,69 @@ +# Copyright 2018-2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import voluptuous as v +import urllib.parse + +artifact = { + 'name': str, + 'url': str, +} + +zuul_data = { + 'zuul': { + 'log_url': str, + 'artifacts': [artifact], + v.Extra: object, + } +} + +artifact_schema = v.Schema(zuul_data) + + +def validate_artifact_schema(data): + try: + artifact_schema(data) + except Exception: + return False + return True + + +def get_artifacts_from_result_data(result_data, logger=None): + ret = [] + if validate_artifact_schema(result_data): + artifacts = result_data.get('zuul', {}).get( + 'artifacts', []) + default_url = result_data.get('zuul', {}).get( + 'log_url') + if default_url: + if default_url[-1] != '/': + default_url += '/' + for artifact in artifacts: + url = artifact['url'] + if default_url: + # If the artifact url is relative, it will be combined + # with the log_url; if it is absolute, it will replace + # it. + try: + url = urllib.parse.urljoin(default_url, url) + except Exception: + if logger: + logger.debug("Error parsing URL:", + exc_info=1) + ret.append({'name': artifact['name'], + 'url': url}) + else: + logger.debug("Result data did not pass artifact schema " + "validation: %s", result_data) + return ret diff --git a/zuul/model.py b/zuul/model.py index ac589c2228..5a842df6fb 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -28,6 +28,7 @@ import itertools from zuul import change_matcher from zuul.lib.config import get_default +from zuul.lib.artifacts import get_artifacts_from_result_data MERGER_MERGE = 1 # "git merge" MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve" @@ -164,6 +165,11 @@ class TemplateNotFoundError(Exception): pass +class RequirementsError(Exception): + """A job's requirements were not met.""" + pass + + class Attributes(object): """A class to hold attributes for string formatting.""" @@ -1070,6 +1076,8 @@ class Job(ConfigObject): file_matcher=None, irrelevant_file_matcher=None, # skip-if tags=frozenset(), + provides=frozenset(), + requires=frozenset(), dependencies=frozenset(), ) @@ -1111,6 +1119,7 @@ class Job(ConfigObject): start_mark=None, inheritance_path=(), parent_data=None, + artifact_data=None, description=None, variant_description=None, protected_origin=None, @@ -1161,6 +1170,10 @@ class Job(ConfigObject): d['protected'] = self.protected d['voting'] = self.voting d['timeout'] = self.timeout + d['tags'] = list(self.tags) + d['provides'] = list(self.provides) + d['requires'] = list(self.requires) + d['dependencies'] = list(self.dependencies) d['attempts'] = self.attempts d['roles'] = list(map(lambda x: x.toDict(), self.roles)) d['post_review'] = self.post_review @@ -1170,9 +1183,6 @@ class Job(ConfigObject): d['parent'] = self.parent else: d['parent'] = tenant.default_base_job - d['dependencies'] = [] - for dependency in self.dependencies: - d['dependencies'].append(dependency) if isinstance(self.nodeset, str): ns = tenant.layout.nodesets.get(self.nodeset) else: @@ -1343,6 +1353,9 @@ class Job(ConfigObject): self.parent_data = v self.variables = Job._deepUpdate(self.parent_data, self.variables) + def updateArtifactData(self, artifact_data): + self.artifact_data = artifact_data + def updateProjectVariables(self, project_vars): # Merge project/template variables directly into the job # variables. Job variables override project variables. @@ -1499,11 +1512,12 @@ class Job(ConfigObject): for k in self.context_attributes: if (other._get(k) is not None and - k not in set(['tags'])): + k not in set(['tags', 'requires', 'provides'])): setattr(self, k, other._get(k)) - if other._get('tags') is not None: - self.tags = frozenset(self.tags.union(other.tags)) + for k in ('tags', 'requires', 'provides'): + if other._get(k) is not None: + setattr(self, k, getattr(self, k).union(other._get(k))) self.inheritance_path = self.inheritance_path + (repr(other),) @@ -1924,6 +1938,7 @@ class BuildSet(object): class QueueItem(object): + """Represents the position of a Change in a ChangeQueue. All Changes are enqueued into ChangeQueue in a QueueItem. The QueueItem @@ -1950,6 +1965,7 @@ class QueueItem(object): self.layout = None self.project_pipeline_config = None self.job_graph = None + self._cached_sql_results = None def __repr__(self): if self.pipeline: @@ -2146,6 +2162,110 @@ class QueueItem(object): return False return self.item_ahead.isHoldingFollowingChanges() + def _getRequirementsResultFromSQL(self, requirements): + # This either returns data or raises an exception + if self._cached_sql_results is None: + sql_driver = self.pipeline.manager.sched.connections.drivers['sql'] + conn = sql_driver.tenant_connections.get(self.pipeline.tenant.name) + if conn: + builds = conn.getBuilds( + tenant=self.pipeline.tenant.name, + project=self.change.project.name, + pipeline=self.pipeline.name, + change=self.change.number, + branch=self.change.branch, + patchset=self.change.patchset, + provides=list(requirements)) + else: + builds = [] + # Just look at the most recent buildset. + # TODO: query for a buildset instead of filtering. + builds = [b for b in builds + if b.buildset.uuid == builds[0].buildset.uuid] + self._cached_sql_results = builds + + builds = self._cached_sql_results + data = [] + if not builds: + return data + + for build in builds: + if build.result != 'SUCCESS': + provides = [x.name for x in build.provides] + requirement = list(requirements.intersection(set(provides))) + raise RequirementsError( + "Requirements %s not met by build %s" % ( + requirement, build.uuid)) + else: + artifacts = [{'name': a.name, + 'url': a.url, + 'project': build.buildset.project, + 'change': str(build.buildset.change), + 'patchset': build.buildset.patchset, + 'job': build.job_name} + for a in build.artifacts] + data += artifacts + return data + + def providesRequirements(self, requirements, data): + # Mutates data and returns true/false if requirements + # satisfied. + if not requirements: + return True + if not self.live: + # Look for this item in other queues in the pipeline. + item = None + found = False + for item in self.pipeline.getAllItems(): + if item.live and item.change == self.change: + found = True + break + if found: + if not item.providesRequirements(requirements, data): + return False + else: + # Look for this item in the SQL DB. + data += self._getRequirementsResultFromSQL(requirements) + if self.hasJobGraph(): + for job in self.getJobs(): + if job.provides.intersection(requirements): + build = self.current_build_set.getBuild(job.name) + if not build: + return False + if build.result and build.result != 'SUCCESS': + return False + if not build.result and not build.paused: + return False + artifacts = get_artifacts_from_result_data( + build.result_data, + logger=self.log) + artifacts = [{'name': a['name'], + 'url': a['url'], + 'project': self.change.project.name, + 'change': self.change.number, + 'patchset': self.change.patchset, + 'job': build.job.name} + for a in artifacts] + data += artifacts + if not self.item_ahead: + return True + return self.item_ahead.providesRequirements(requirements, data) + + def jobRequirementsReady(self, job): + if not self.item_ahead: + return True + try: + data = [] + ret = self.item_ahead.providesRequirements(job.requires, data) + job.updateArtifactData(data) + except RequirementsError as e: + self.warning(str(e)) + fakebuild = Build(job, None) + fakebuild.result = 'SKIPPED' + self.addBuild(fakebuild) + ret = True + return ret + def findJobsToRun(self, semaphore_handler): torun = [] if not self.live: @@ -2173,6 +2293,8 @@ class QueueItem(object): for job in self.job_graph.getJobs(): if job not in jobs_not_started: continue + if not self.jobRequirementsReady(job): + continue all_parent_jobs_successful = True parent_builds_with_data = {} for parent_job in self.job_graph.getParentJobsRecursively( @@ -2237,6 +2359,8 @@ class QueueItem(object): for job in self.job_graph.getJobs(): if job not in jobs_not_requested: continue + if not self.jobRequirementsReady(job): + continue all_parent_jobs_successful = True for parent_job in self.job_graph.getParentJobsRecursively( job.name): diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py index fcdae361ac..dbbb707390 100755 --- a/zuul/web/__init__.py +++ b/zuul/web/__init__.py @@ -440,6 +440,7 @@ class ZuulWebAPI(object): 'newrev': buildset.newrev, 'ref_url': buildset.ref_url, 'artifacts': [], + 'provides': [], } for artifact in build.artifacts: @@ -447,6 +448,10 @@ class ZuulWebAPI(object): 'name': artifact.name, 'url': artifact.url, }) + for provides in build.provides: + ret['provides'].append({ + 'name': artifact.name, + }) return ret @cherrypy.expose