Browse Source

Merge "Add provides/requires support"

tags/3.6.0
Zuul 5 months ago
parent
commit
b44b6c532c

+ 51
- 0
doc/source/user/config.rst View File

@@ -686,6 +686,57 @@ Here is an example of two job definitions:
686 686
       tags from all the jobs and variants used in constructing the
687 687
       frozen job, with no duplication.
688 688
 
689
+   .. attr:: provides
690
+
691
+      A list of free-form strings which identifies resources provided
692
+      by this job which may be used by other jobs for other changes
693
+      using the :attr:`job.requires` attribute.
694
+
695
+   .. attr:: requires
696
+
697
+      A list of free-form strings which identify resources which may
698
+      be provided by other jobs for other changes (via the
699
+      :attr:`job.provides` attribute) that are used by this job.
700
+
701
+      When Zuul encounters a job with a `requires` attribute, it
702
+      searches for those values in the `provides` attributes of any
703
+      jobs associated with any queue items ahead of the current
704
+      change.  In this way, if a change uses either git dependencies
705
+      or a `Depends-On` header to indicate a dependency on another
706
+      change, Zuul will be able to determine that the parent change
707
+      affects the run-time environment of the child change.  If such a
708
+      relationship is found, the job with `requires` will not start
709
+      until all of the jobs with matching `provides` have completed or
710
+      paused.  Additionally, the :ref:`artifacts <return_artifacts>`
711
+      returned by the `provides` jobs will be made available to the
712
+      `requires` job.
713
+
714
+      For example, a job which produces a builder container image in
715
+      one project that is then consumed by a container image build job
716
+      in another project might look like this:
717
+
718
+      .. code-block:: yaml
719
+
720
+         - job:
721
+             name: build-builder-image
722
+             provides: images
723
+
724
+         - job:
725
+             name: build-final-image
726
+             requires: images
727
+
728
+         - project:
729
+             name: builder-project
730
+             check:
731
+               jobs:
732
+                 - build-builder-image
733
+
734
+         - project:
735
+             name: final-project
736
+             check:
737
+               jobs:
738
+                 - build-final-image
739
+
689 740
    .. attr:: secrets
690 741
 
691 742
       A list of secrets which may be used by the job.  A

+ 35
- 0
doc/source/user/jobs.rst View File

@@ -228,6 +228,41 @@ of item.
228 228
    All items provide the following information as Ansible variables
229 229
    under the ``zuul`` key:
230 230
 
231
+   .. var:: artifacts
232
+      :type: list
233
+
234
+      If the job has a :attr:`job.requires` attribute, and Zuul has
235
+      found changes ahead of this change in the pipeline with matching
236
+      :attr:`job.provides` attributes, then information about any
237
+      :ref:`artifacts returned <return_artifacts>` from those jobs
238
+      will appear here.
239
+
240
+      This value is a list of dictionaries with the following format:
241
+
242
+      .. var:: project
243
+
244
+         The name of the project which supplied this artifact.
245
+
246
+      .. var:: change
247
+
248
+         The change number which supplied this artifact.
249
+
250
+      .. var:: patchset
251
+
252
+         The patchset of the change.
253
+
254
+      .. var:: job
255
+
256
+         The name of the job which produced the artifact.
257
+
258
+      .. var:: name
259
+
260
+         The name of the artifact (as supplied to :ref:`return_artifacts`).
261
+
262
+      .. var:: url
263
+
264
+         The URL of the artifact (as supplied to :ref:`return_artifacts`).
265
+
231 266
    .. var:: build
232 267
 
233 268
       The UUID of the build.  A build is a single execution of a job.

+ 7
- 0
releasenotes/notes/provides_requires-4c6b54ede999e86c.yaml View File

@@ -0,0 +1,7 @@
1
+---
2
+features:
3
+  - Support for expressing artifact or other resource dependencies
4
+    between jobs running on different changes with a dependency
5
+    relationship (e.g., a container image built in one project and
6
+    consumed in a second project) has been added via the
7
+    :attr:`job.provides` and :attr:`job.requires` job attributes.

+ 29
- 0
tests/base.py View File

@@ -1355,6 +1355,11 @@ class FakeBuild(object):
1355 1355
         items = self.parameters['zuul']['items']
1356 1356
         self.changes = ' '.join(['%s,%s' % (x['change'], x['patchset'])
1357 1357
                                 for x in items if 'change' in x])
1358
+        if 'change' in items[-1]:
1359
+            self.change = ' '.join((items[-1]['change'],
1360
+                                    items[-1]['patchset']))
1361
+        else:
1362
+            self.change = None
1358 1363
 
1359 1364
     def __repr__(self):
1360 1365
         waiting = ''
@@ -1401,6 +1406,8 @@ class FakeBuild(object):
1401 1406
             self._wait()
1402 1407
         self.log.debug("Build %s continuing" % self.unique)
1403 1408
 
1409
+        self.writeReturnData()
1410
+
1404 1411
         result = (RecordingAnsibleJob.RESULT_NORMAL, 0)  # Success
1405 1412
         if self.shouldFail():
1406 1413
             result = (RecordingAnsibleJob.RESULT_NORMAL, 1)  # Failure
@@ -1418,6 +1425,14 @@ class FakeBuild(object):
1418 1425
                 return True
1419 1426
         return False
1420 1427
 
1428
+    def writeReturnData(self):
1429
+        changes = self.executor_server.return_data.get(self.name, {})
1430
+        data = changes.get(self.change)
1431
+        if data is None:
1432
+            return
1433
+        with open(self.jobdir.result_data_file, 'w') as f:
1434
+            f.write(json.dumps(data))
1435
+
1421 1436
     def hasChanges(self, *changes):
1422 1437
         """Return whether this build has certain changes in its git repos.
1423 1438
 
@@ -1554,6 +1569,7 @@ class RecordingExecutorServer(zuul.executor.server.ExecutorServer):
1554 1569
         self.running_builds = []
1555 1570
         self.build_history = []
1556 1571
         self.fail_tests = {}
1572
+        self.return_data = {}
1557 1573
         self.job_builds = {}
1558 1574
 
1559 1575
     def failJob(self, name, change):
@@ -1569,6 +1585,19 @@ class RecordingExecutorServer(zuul.executor.server.ExecutorServer):
1569 1585
         l.append(change)
1570 1586
         self.fail_tests[name] = l
1571 1587
 
1588
+    def returnData(self, name, change, data):
1589
+        """Instruct the executor to return data for this build.
1590
+
1591
+        :arg str name: The name of the job to return data.
1592
+        :arg Change change: The :py:class:`~tests.base.FakeChange`
1593
+            instance which should cause the job to return data.
1594
+        :arg dict data: The data to return
1595
+
1596
+        """
1597
+        changes = self.return_data.setdefault(name, {})
1598
+        cid = ' '.join((str(change.number), str(change.latest_patchset)))
1599
+        changes[cid] = data
1600
+
1572 1601
     def release(self, regex=None):
1573 1602
         """Release a held build.
1574 1603
 

+ 38
- 0
tests/fixtures/config/provides-requires-pause/git/common-config/zuul.yaml View File

@@ -0,0 +1,38 @@
1
+- pipeline:
2
+    name: check
3
+    manager: independent
4
+    post-review: true
5
+    trigger:
6
+      gerrit:
7
+        - event: patchset-created
8
+    success:
9
+      gerrit:
10
+        Verified: 1
11
+    failure:
12
+      gerrit:
13
+        Verified: -1
14
+
15
+- pipeline:
16
+    name: gate
17
+    manager: dependent
18
+    post-review: True
19
+    trigger:
20
+      gerrit:
21
+        - event: comment-added
22
+          approval:
23
+            - Approved: 1
24
+    success:
25
+      gerrit:
26
+        Verified: 2
27
+        submit: true
28
+    failure:
29
+      gerrit:
30
+        Verified: -2
31
+    start:
32
+      gerrit:
33
+        Verified: 0
34
+    precedence: high
35
+
36
+- job:
37
+    name: base
38
+    parent: null

+ 1
- 0
tests/fixtures/config/provides-requires-pause/git/org_project1/README View File

@@ -0,0 +1 @@
1
+test

+ 10
- 0
tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-builder.yaml View File

@@ -0,0 +1,10 @@
1
+- hosts: all
2
+  tasks:
3
+    - name: Pause and let child run
4
+      zuul_return:
5
+        data:
6
+          zuul:
7
+            pause: true
8
+            artifacts:
9
+              - name: image
10
+                url: http://example.com/image

+ 4
- 0
tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-user.yaml View File

@@ -0,0 +1,4 @@
1
+- hosts: all
2
+  tasks:
3
+    - debug:
4
+        var: zuul.artifacts

+ 26
- 0
tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml View File

@@ -0,0 +1,26 @@
1
+- job:
2
+    name: image-builder
3
+    provides:
4
+      - image
5
+    run: playbooks/image-builder.yaml
6
+
7
+- job:
8
+    name: image-user
9
+    requires:
10
+      - image
11
+    run: playbooks/image-user.yaml
12
+
13
+- project:
14
+    check:
15
+      jobs:
16
+        - image-builder
17
+        - image-user:
18
+            dependencies:
19
+              - image-builder
20
+    gate:
21
+      queue: integrated
22
+      jobs:
23
+        - image-builder
24
+        - image-user:
25
+            dependencies:
26
+              - image-builder

+ 8
- 0
tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml View File

@@ -0,0 +1,8 @@
1
+- project:
2
+    check:
3
+      jobs:
4
+        - image-user
5
+    gate:
6
+      queue: integrated
7
+      jobs:
8
+        - image-user

+ 8
- 0
tests/fixtures/config/provides-requires-pause/main.yaml View File

@@ -0,0 +1,8 @@
1
+- tenant:
2
+    name: tenant-one
3
+    source:
4
+      gerrit:
5
+        config-projects:
6
+          - common-config
7
+          - org/project1
8
+          - org/project2

+ 72
- 0
tests/fixtures/layouts/provides-requires-two-jobs.yaml View File

@@ -0,0 +1,72 @@
1
+- pipeline:
2
+    name: check
3
+    manager: independent
4
+    trigger:
5
+      gerrit:
6
+        - event: patchset-created
7
+    success:
8
+      gerrit:
9
+        Verified: 1
10
+      resultsdb_mysql: null
11
+      resultsdb_postgresql: null
12
+    failure:
13
+      gerrit:
14
+        Verified: -1
15
+      resultsdb_mysql: null
16
+      resultsdb_postgresql: null
17
+
18
+- pipeline:
19
+    name: gate
20
+    manager: dependent
21
+    success-message: Build succeeded (gate).
22
+    trigger:
23
+      gerrit:
24
+        - event: comment-added
25
+          approval:
26
+            - Approved: 1
27
+    success:
28
+      gerrit:
29
+        Verified: 2
30
+        submit: true
31
+    failure:
32
+      gerrit:
33
+        Verified: -2
34
+    start:
35
+      gerrit:
36
+        Verified: 0
37
+    precedence: high
38
+
39
+- job:
40
+    name: base
41
+    parent: null
42
+    run: playbooks/base.yaml
43
+
44
+- job:
45
+    name: image-builder
46
+    provides: images
47
+
48
+- job:
49
+    name: image-user
50
+    requires: images
51
+
52
+- project:
53
+    name: org/project1
54
+    check:
55
+      jobs:
56
+        - image-builder
57
+    gate:
58
+      queue: integrated
59
+      jobs:
60
+        - image-builder
61
+        - image-user:
62
+            dependencies: image-builder
63
+
64
+- project:
65
+    name: org/project2
66
+    check:
67
+      jobs:
68
+        - image-user
69
+    gate:
70
+      queue: integrated
71
+      jobs:
72
+        - image-user

+ 58
- 0
tests/fixtures/layouts/provides-requires-unshared.yaml View File

@@ -0,0 +1,58 @@
1
+- pipeline:
2
+    name: check
3
+    manager: independent
4
+    trigger:
5
+      gerrit:
6
+        - event: patchset-created
7
+    success:
8
+      gerrit:
9
+        Verified: 1
10
+    failure:
11
+      gerrit:
12
+        Verified: -1
13
+
14
+- pipeline:
15
+    name: gate
16
+    manager: dependent
17
+    success-message: Build succeeded (gate).
18
+    trigger:
19
+      gerrit:
20
+        - event: comment-added
21
+          approval:
22
+            - Approved: 1
23
+    success:
24
+      gerrit:
25
+        Verified: 2
26
+        submit: true
27
+    failure:
28
+      gerrit:
29
+        Verified: -2
30
+    start:
31
+      gerrit:
32
+        Verified: 0
33
+    precedence: high
34
+
35
+- job:
36
+    name: base
37
+    parent: null
38
+    run: playbooks/base.yaml
39
+
40
+- job:
41
+    name: image-builder
42
+    provides: images
43
+
44
+- job:
45
+    name: image-user
46
+    requires: images
47
+
48
+- project:
49
+    name: org/project1
50
+    gate:
51
+      jobs:
52
+        - image-builder
53
+
54
+- project:
55
+    name: org/project2
56
+    gate:
57
+      jobs:
58
+        - image-user

+ 70
- 0
tests/fixtures/layouts/provides-requires.yaml View File

@@ -0,0 +1,70 @@
1
+- pipeline:
2
+    name: check
3
+    manager: independent
4
+    trigger:
5
+      gerrit:
6
+        - event: patchset-created
7
+    success:
8
+      gerrit:
9
+        Verified: 1
10
+      resultsdb_mysql: null
11
+      resultsdb_postgresql: null
12
+    failure:
13
+      gerrit:
14
+        Verified: -1
15
+      resultsdb_mysql: null
16
+      resultsdb_postgresql: null
17
+
18
+- pipeline:
19
+    name: gate
20
+    manager: dependent
21
+    success-message: Build succeeded (gate).
22
+    trigger:
23
+      gerrit:
24
+        - event: comment-added
25
+          approval:
26
+            - Approved: 1
27
+    success:
28
+      gerrit:
29
+        Verified: 2
30
+        submit: true
31
+    failure:
32
+      gerrit:
33
+        Verified: -2
34
+    start:
35
+      gerrit:
36
+        Verified: 0
37
+    precedence: high
38
+
39
+- job:
40
+    name: base
41
+    parent: null
42
+    run: playbooks/base.yaml
43
+
44
+- job:
45
+    name: image-builder
46
+    provides: images
47
+
48
+- job:
49
+    name: image-user
50
+    requires: images
51
+
52
+- project:
53
+    name: org/project1
54
+    check:
55
+      jobs:
56
+        - image-builder
57
+    gate:
58
+      queue: integrated
59
+      jobs:
60
+        - image-builder
61
+
62
+- project:
63
+    name: org/project2
64
+    check:
65
+      jobs:
66
+        - image-user
67
+    gate:
68
+      queue: integrated
69
+      jobs:
70
+        - image-user

+ 288
- 0
tests/unit/test_v3.py View File

@@ -28,6 +28,7 @@ from zuul.lib import encryption
28 28
 from tests.base import (
29 29
     AnsibleZuulTestCase,
30 30
     ZuulTestCase,
31
+    ZuulDBTestCase,
31 32
     FIXTURE_DIR,
32 33
     simple_layout,
33 34
 )
@@ -4714,3 +4715,290 @@ class TestContainerJobs(AnsibleZuulTestCase):
4714 4715
             dict(name='container-machine', result='SUCCESS', changes='1,1'),
4715 4716
             dict(name='container-native', result='SUCCESS', changes='1,1'),
4716 4717
         ])
4718
+
4719
+
4720
+class TestProvidesRequiresPause(AnsibleZuulTestCase):
4721
+    tenant_config_file = "config/provides-requires-pause/main.yaml"
4722
+
4723
+    def test_provides_requires_pause(self):
4724
+        # Changes share a queue, with both running at the same time.
4725
+        self.executor_server.hold_jobs_in_build = True
4726
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4727
+        A.addApproval('Code-Review', 2)
4728
+        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
4729
+        self.waitUntilSettled()
4730
+
4731
+        self.assertEqual(len(self.builds), 1)
4732
+
4733
+        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4734
+        B.addApproval('Code-Review', 2)
4735
+        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4736
+        self.waitUntilSettled()
4737
+
4738
+        self.assertEqual(len(self.builds), 1)
4739
+
4740
+        # Release image-build, it should cause both instances of
4741
+        # image-user to run.
4742
+        self.executor_server.hold_jobs_in_build = False
4743
+        self.executor_server.release()
4744
+        self.waitUntilSettled()
4745
+
4746
+        self.assertHistory([
4747
+            dict(name='image-builder', result='SUCCESS', changes='1,1'),
4748
+            dict(name='image-user', result='SUCCESS', changes='1,1'),
4749
+            dict(name='image-user', result='SUCCESS', changes='1,1 2,1'),
4750
+        ], ordered=False)
4751
+        build = self.getJobFromHistory('image-user', project='org/project2')
4752
+        self.assertEqual(
4753
+            build.parameters['zuul']['artifacts'],
4754
+            [{
4755
+                'project': 'org/project1',
4756
+                'change': '1',
4757
+                'patchset': '1',
4758
+                'job': 'image-builder',
4759
+                'url': 'http://example.com/image',
4760
+                'name': 'image',
4761
+            }])
4762
+
4763
+
4764
+class TestProvidesRequires(ZuulDBTestCase):
4765
+    config_file = "zuul-sql-driver.conf"
4766
+
4767
+    @simple_layout('layouts/provides-requires.yaml')
4768
+    def test_provides_requires_shared_queue_fast(self):
4769
+        # Changes share a queue, but with only one job, the first
4770
+        # merges before the second starts.
4771
+        self.executor_server.hold_jobs_in_build = True
4772
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4773
+        self.executor_server.returnData(
4774
+            'image-builder', A,
4775
+            {'zuul':
4776
+             {'artifacts': [
4777
+                 {'name': 'image', 'url': 'http://example.com/image'},
4778
+             ]}}
4779
+        )
4780
+        A.addApproval('Code-Review', 2)
4781
+        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
4782
+        self.waitUntilSettled()
4783
+
4784
+        self.assertEqual(len(self.builds), 1)
4785
+
4786
+        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4787
+        B.addApproval('Code-Review', 2)
4788
+        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4789
+        self.waitUntilSettled()
4790
+
4791
+        self.assertEqual(len(self.builds), 1)
4792
+
4793
+        self.executor_server.hold_jobs_in_build = False
4794
+        self.executor_server.release()
4795
+        self.waitUntilSettled()
4796
+
4797
+        self.assertHistory([
4798
+            dict(name='image-builder', result='SUCCESS', changes='1,1'),
4799
+            dict(name='image-user', result='SUCCESS', changes='1,1 2,1'),
4800
+        ])
4801
+        # Data are not passed in this instance because the builder
4802
+        # change merges before the user job runs.
4803
+        self.assertFalse('artifacts' in self.history[-1].parameters['zuul'])
4804
+
4805
+    @simple_layout('layouts/provides-requires-two-jobs.yaml')
4806
+    def test_provides_requires_shared_queue_slow(self):
4807
+        # Changes share a queue, with both running at the same time.
4808
+        self.executor_server.hold_jobs_in_build = True
4809
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4810
+        self.executor_server.returnData(
4811
+            'image-builder', A,
4812
+            {'zuul':
4813
+             {'artifacts': [
4814
+                 {'name': 'image', 'url': 'http://example.com/image'},
4815
+             ]}}
4816
+        )
4817
+        A.addApproval('Code-Review', 2)
4818
+        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
4819
+        self.waitUntilSettled()
4820
+
4821
+        self.assertEqual(len(self.builds), 1)
4822
+
4823
+        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4824
+        B.addApproval('Code-Review', 2)
4825
+        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4826
+        self.waitUntilSettled()
4827
+
4828
+        self.assertEqual(len(self.builds), 1)
4829
+
4830
+        # Release image-build, it should cause both instances of
4831
+        # image-user to run.
4832
+        self.executor_server.release()
4833
+        self.waitUntilSettled()
4834
+        self.assertEqual(len(self.builds), 2)
4835
+        self.assertHistory([
4836
+            dict(name='image-builder', result='SUCCESS', changes='1,1'),
4837
+        ])
4838
+
4839
+        self.orderedRelease()
4840
+        self.waitUntilSettled()
4841
+
4842
+        self.assertHistory([
4843
+            dict(name='image-builder', result='SUCCESS', changes='1,1'),
4844
+            dict(name='image-user', result='SUCCESS', changes='1,1'),
4845
+            dict(name='image-user', result='SUCCESS', changes='1,1 2,1'),
4846
+        ])
4847
+        self.assertEqual(
4848
+            self.history[-1].parameters['zuul']['artifacts'],
4849
+            [{
4850
+                'project': 'org/project1',
4851
+                'change': '1',
4852
+                'patchset': '1',
4853
+                'job': 'image-builder',
4854
+                'url': 'http://example.com/image',
4855
+                'name': 'image',
4856
+            }])
4857
+
4858
+    @simple_layout('layouts/provides-requires-unshared.yaml')
4859
+    def test_provides_requires_unshared_queue(self):
4860
+        self.executor_server.hold_jobs_in_build = True
4861
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4862
+        self.executor_server.returnData(
4863
+            'image-builder', A,
4864
+            {'zuul':
4865
+             {'artifacts': [
4866
+                 {'name': 'image', 'url': 'http://example.com/image'},
4867
+             ]}}
4868
+        )
4869
+        A.addApproval('Code-Review', 2)
4870
+        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
4871
+        self.waitUntilSettled()
4872
+
4873
+        self.assertEqual(len(self.builds), 1)
4874
+
4875
+        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4876
+        B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4877
+            B.subject, A.data['id'])
4878
+        B.addApproval('Code-Review', 2)
4879
+        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4880
+        self.waitUntilSettled()
4881
+
4882
+        self.assertEqual(len(self.builds), 1)
4883
+
4884
+        self.executor_server.hold_jobs_in_build = False
4885
+        self.executor_server.release()
4886
+        self.waitUntilSettled()
4887
+
4888
+        self.assertHistory([
4889
+            dict(name='image-builder', result='SUCCESS', changes='1,1'),
4890
+        ])
4891
+
4892
+        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4893
+        self.waitUntilSettled()
4894
+
4895
+        self.assertHistory([
4896
+            dict(name='image-builder', result='SUCCESS', changes='1,1'),
4897
+            dict(name='image-user', result='SUCCESS', changes='2,1'),
4898
+        ])
4899
+        # Data are not passed in this instance because the builder
4900
+        # change merges before the user job runs.
4901
+        self.assertFalse('artifacts' in self.history[-1].parameters['zuul'])
4902
+
4903
+    @simple_layout('layouts/provides-requires.yaml')
4904
+    def test_provides_requires_check_current(self):
4905
+        self.executor_server.hold_jobs_in_build = True
4906
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4907
+        self.executor_server.returnData(
4908
+            'image-builder', A,
4909
+            {'zuul':
4910
+             {'artifacts': [
4911
+                 {'name': 'image', 'url': 'http://example.com/image'},
4912
+             ]}}
4913
+        )
4914
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4915
+        self.waitUntilSettled()
4916
+
4917
+        self.assertEqual(len(self.builds), 1)
4918
+
4919
+        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4920
+        B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4921
+            B.subject, A.data['id'])
4922
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4923
+        self.waitUntilSettled()
4924
+
4925
+        self.assertEqual(len(self.builds), 1)
4926
+
4927
+        self.executor_server.hold_jobs_in_build = False
4928
+        self.executor_server.release()
4929
+        self.waitUntilSettled()
4930
+
4931
+        self.assertHistory([
4932
+            dict(name='image-builder', result='SUCCESS', changes='1,1'),
4933
+            dict(name='image-user', result='SUCCESS', changes='1,1 2,1'),
4934
+        ])
4935
+        self.assertEqual(
4936
+            self.history[-1].parameters['zuul']['artifacts'],
4937
+            [{
4938
+                'project': 'org/project1',
4939
+                'change': '1',
4940
+                'patchset': '1',
4941
+                'job': 'image-builder',
4942
+                'url': 'http://example.com/image',
4943
+                'name': 'image',
4944
+            }])
4945
+
4946
+    @simple_layout('layouts/provides-requires.yaml')
4947
+    def test_provides_requires_check_old_success(self):
4948
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4949
+        self.executor_server.returnData(
4950
+            'image-builder', A,
4951
+            {'zuul':
4952
+             {'artifacts': [
4953
+                 {'name': 'image', 'url': 'http://example.com/image'},
4954
+             ]}}
4955
+        )
4956
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4957
+        self.waitUntilSettled()
4958
+        self.assertHistory([
4959
+            dict(name='image-builder', result='SUCCESS', changes='1,1'),
4960
+        ])
4961
+
4962
+        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4963
+        B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4964
+            B.subject, A.data['id'])
4965
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4966
+        self.waitUntilSettled()
4967
+
4968
+        self.assertHistory([
4969
+            dict(name='image-builder', result='SUCCESS', changes='1,1'),
4970
+            dict(name='image-user', result='SUCCESS', changes='1,1 2,1'),
4971
+        ])
4972
+        self.assertEqual(
4973
+            self.history[-1].parameters['zuul']['artifacts'],
4974
+            [{
4975
+                'project': 'org/project1',
4976
+                'change': '1',
4977
+                'patchset': '1',
4978
+                'job': 'image-builder',
4979
+                'url': 'http://example.com/image',
4980
+                'name': 'image',
4981
+            }])
4982
+
4983
+    @simple_layout('layouts/provides-requires.yaml')
4984
+    def test_provides_requires_check_old_failure(self):
4985
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4986
+        self.executor_server.failJob('image-builder', A)
4987
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4988
+        self.waitUntilSettled()
4989
+
4990
+        self.assertHistory([
4991
+            dict(name='image-builder', result='FAILURE', changes='1,1'),
4992
+        ])
4993
+
4994
+        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4995
+        B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4996
+            B.subject, A.data['id'])
4997
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4998
+        self.waitUntilSettled()
4999
+
5000
+        self.assertHistory([
5001
+            dict(name='image-builder', result='FAILURE', changes='1,1'),
5002
+        ])
5003
+        self.assertIn('image-user : SKIPPED', B.messages[0])
5004
+        self.assertIn('not met by build', B.messages[0])

+ 21
- 0
tests/unit/test_web.py View File

@@ -305,10 +305,13 @@ class TestWeb(BaseTestWeb):
305 305
                 'parent': 'base',
306 306
                 'post_review': None,
307 307
                 'protected': None,
308
+                'provides': [],
308 309
                 'required_projects': [],
310
+                'requires': [],
309 311
                 'roles': [common_config_role],
310 312
                 'semaphore': None,
311 313
                 'source_context': source_ctx,
314
+                'tags': [],
312 315
                 'timeout': None,
313 316
                 'variables': {},
314 317
                 'variant_description': '',
@@ -337,10 +340,13 @@ class TestWeb(BaseTestWeb):
337 340
                 'parent': 'base',
338 341
                 'post_review': None,
339 342
                 'protected': None,
343
+                'provides': [],
340 344
                 'required_projects': [],
345
+                'requires': [],
341 346
                 'roles': [common_config_role],
342 347
                 'semaphore': None,
343 348
                 'source_context': source_ctx,
349
+                'tags': [],
344 350
                 'timeout': None,
345 351
                 'variables': {},
346 352
                 'variant_description': 'stable',
@@ -363,13 +369,16 @@ class TestWeb(BaseTestWeb):
363 369
                 'parent': 'base',
364 370
                 'post_review': None,
365 371
                 'protected': None,
372
+                'provides': [],
366 373
                 'required_projects': [
367 374
                     {'override_branch': None,
368 375
                      'override_checkout': None,
369 376
                      'project_name': 'review.example.com/org/project'}],
377
+                'requires': [],
370 378
                 'roles': [common_config_role],
371 379
                 'semaphore': None,
372 380
                 'source_context': source_ctx,
381
+                'tags': [],
373 382
                 'timeout': None,
374 383
                 'variables': {},
375 384
                 'variant_description': '',
@@ -434,13 +443,16 @@ class TestWeb(BaseTestWeb):
434 443
                   'parent': 'base',
435 444
                   'post_review': None,
436 445
                   'protected': None,
446
+                  'provides': [],
437 447
                   'required_projects': [],
448
+                  'requires': [],
438 449
                   'roles': [],
439 450
                   'semaphore': None,
440 451
                   'source_context': {
441 452
                       'branch': 'master',
442 453
                       'path': 'zuul.yaml',
443 454
                       'project': 'common-config'},
455
+                  'tags': [],
444 456
                   'timeout': None,
445 457
                   'variables': {},
446 458
                   'variant_description': '',
@@ -458,13 +470,16 @@ class TestWeb(BaseTestWeb):
458 470
                   'parent': 'base',
459 471
                   'post_review': None,
460 472
                   'protected': None,
473
+                  'provides': [],
461 474
                   'required_projects': [],
475
+                  'requires': [],
462 476
                   'roles': [],
463 477
                   'semaphore': None,
464 478
                   'source_context': {
465 479
                       'branch': 'master',
466 480
                       'path': 'zuul.yaml',
467 481
                       'project': 'common-config'},
482
+                  'tags': [],
468 483
                   'timeout': None,
469 484
                   'variables': {},
470 485
                   'variant_description': '',
@@ -482,13 +497,16 @@ class TestWeb(BaseTestWeb):
482 497
                   'parent': 'base',
483 498
                   'post_review': None,
484 499
                   'protected': None,
500
+                  'provides': [],
485 501
                   'required_projects': [],
502
+                  'requires': [],
486 503
                   'roles': [],
487 504
                   'semaphore': None,
488 505
                   'source_context': {
489 506
                       'branch': 'master',
490 507
                       'path': 'zuul.yaml',
491 508
                       'project': 'common-config'},
509
+                  'tags': [],
492 510
                   'timeout': None,
493 511
                   'variables': {},
494 512
                   'variant_description': '',
@@ -506,13 +524,16 @@ class TestWeb(BaseTestWeb):
506 524
                   'parent': 'base',
507 525
                   'post_review': None,
508 526
                   'protected': None,
527
+                  'provides': [],
509 528
                   'required_projects': [],
529
+                  'requires': [],
510 530
                   'roles': [],
511 531
                   'semaphore': None,
512 532
                   'source_context': {
513 533
                       'branch': 'master',
514 534
                       'path': 'zuul.yaml',
515 535
                       'project': 'common-config'},
536
+                  'tags': [],
516 537
                   'timeout': None,
517 538
                   'variables': {},
518 539
                   'variant_description': '',

+ 6
- 5
zuul/configloader.py View File

@@ -545,6 +545,8 @@ class JobParser(object):
545 545
                       'final': bool,
546 546
                       'abstract': bool,
547 547
                       'protected': bool,
548
+                      'requires': to_list(str),
549
+                      'provides': to_list(str),
548 550
                       'failure-message': str,
549 551
                       'success-message': str,
550 552
                       'failure-url': str,
@@ -769,11 +771,10 @@ class JobParser(object):
769 771
                     semaphore.get('name'),
770 772
                     semaphore.get('resources-first', False))
771 773
 
772
-        tags = conf.get('tags')
773
-        if tags:
774
-            job.tags = set(tags)
775
-
776
-        job.dependencies = frozenset(as_list(conf.get('dependencies')))
774
+        for k in ('tags', 'requires', 'provides', 'dependencies'):
775
+            v = frozenset(as_list(conf.get(k)))
776
+            if v:
777
+                setattr(job, k, v)
777 778
 
778 779
         variables = conf.get('vars', None)
779 780
         if variables:

+ 46
- 0
zuul/driver/sql/alembic/versions/39d302d34d38_add_provides.py View File

@@ -0,0 +1,46 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+#      http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+"""add_provides
14
+
15
+Revision ID: 39d302d34d38
16
+Revises: 649ce63b5fe5
17
+Create Date: 2019-01-28 15:01:07.408072
18
+
19
+"""
20
+
21
+# revision identifiers, used by Alembic.
22
+revision = '39d302d34d38'
23
+down_revision = '649ce63b5fe5'
24
+branch_labels = None
25
+depends_on = None
26
+
27
+from alembic import op
28
+import sqlalchemy as sa
29
+
30
+
31
+PROVIDES_TABLE = 'zuul_provides'
32
+BUILD_TABLE = 'zuul_build'
33
+
34
+
35
+def upgrade(table_prefix=''):
36
+    op.create_table(
37
+        table_prefix + PROVIDES_TABLE,
38
+        sa.Column('id', sa.Integer, primary_key=True),
39
+        sa.Column('build_id', sa.Integer,
40
+                  sa.ForeignKey(table_prefix + BUILD_TABLE + ".id")),
41
+        sa.Column('name', sa.String(255)),
42
+    )
43
+
44
+
45
+def downgrade():
46
+    raise Exception("Downgrades not supported")

+ 27
- 1
zuul/driver/sql/sqlconnection.py View File

@@ -28,6 +28,7 @@ from zuul.connection import BaseConnection
28 28
 BUILDSET_TABLE = 'zuul_buildset'
29 29
 BUILD_TABLE = 'zuul_build'
30 30
 ARTIFACT_TABLE = 'zuul_artifact'
31
+PROVIDES_TABLE = 'zuul_provides'
31 32
 
32 33
 
33 34
 class DatabaseSession(object):
@@ -56,17 +57,21 @@ class DatabaseSession(object):
56 57
     def getBuilds(self, tenant=None, project=None, pipeline=None,
57 58
                   change=None, branch=None, patchset=None, ref=None,
58 59
                   newrev=None, uuid=None, job_name=None, voting=None,
59
-                  node_name=None, result=None, limit=50, offset=0):
60
+                  node_name=None, result=None, provides=None,
61
+                  limit=50, offset=0):
60 62
 
61 63
         build_table = self.connection.zuul_build_table
62 64
         buildset_table = self.connection.zuul_buildset_table
65
+        provides_table = self.connection.zuul_provides_table
63 66
 
64 67
         # contains_eager allows us to perform eager loading on the
65 68
         # buildset *and* use that table in filters (unlike
66 69
         # joinedload).
67 70
         q = self.session().query(self.connection.buildModel).\
68 71
             join(self.connection.buildSetModel).\
72
+            outerjoin(self.connection.providesModel).\
69 73
             options(orm.contains_eager(self.connection.buildModel.buildset),
74
+                    orm.selectinload(self.connection.buildModel.provides),
70 75
                     orm.selectinload(self.connection.buildModel.artifacts)).\
71 76
             with_hint(build_table, 'USE INDEX (PRIMARY)', 'mysql')
72 77
 
@@ -83,6 +88,7 @@ class DatabaseSession(object):
83 88
         q = self.listFilter(q, build_table.c.voting, voting)
84 89
         q = self.listFilter(q, build_table.c.node_name, node_name)
85 90
         q = self.listFilter(q, build_table.c.result, result)
91
+        q = self.listFilter(q, provides_table.c.name, provides)
86 92
 
87 93
         q = q.order_by(build_table.c.id.desc()).\
88 94
             limit(limit).\
@@ -224,6 +230,15 @@ class SQLConnection(BaseConnection):
224 230
                 session.flush()
225 231
                 return a
226 232
 
233
+            def createProvides(self, *args, **kw):
234
+                session = orm.session.Session.object_session(self)
235
+                p = ProvidesModel(*args, **kw)
236
+                p.build_id = self.id
237
+                self.provides.append(p)
238
+                session.add(p)
239
+                session.flush()
240
+                return p
241
+
227 242
         class ArtifactModel(Base):
228 243
             __tablename__ = self.table_prefix + ARTIFACT_TABLE
229 244
             id = sa.Column(sa.Integer, primary_key=True)
@@ -233,6 +248,17 @@ class SQLConnection(BaseConnection):
233 248
             url = sa.Column(sa.TEXT())
234 249
             build = orm.relationship(BuildModel, backref="artifacts")
235 250
 
251
+        class ProvidesModel(Base):
252
+            __tablename__ = self.table_prefix + PROVIDES_TABLE
253
+            id = sa.Column(sa.Integer, primary_key=True)
254
+            build_id = sa.Column(sa.Integer, sa.ForeignKey(
255
+                self.table_prefix + BUILD_TABLE + ".id"))
256
+            name = sa.Column(sa.String(255))
257
+            build = orm.relationship(BuildModel, backref="provides")
258
+
259
+        self.providesModel = ProvidesModel
260
+        self.zuul_provides_table = self.providesModel.__table__
261
+
236 262
         self.artifactModel = ArtifactModel
237 263
         self.zuul_artifact_table = self.artifactModel.__table__
238 264
 

+ 8
- 47
zuul/driver/sql/sqlreporter.py View File

@@ -16,9 +16,9 @@ import datetime
16 16
 import logging
17 17
 import time
18 18
 import voluptuous as v
19
-import urllib.parse
20 19
 
21 20
 from zuul.reporter import BaseReporter
21
+from zuul.lib.artifacts import get_artifacts_from_result_data
22 22
 
23 23
 
24 24
 class SQLReporter(BaseReporter):
@@ -27,26 +27,6 @@ class SQLReporter(BaseReporter):
27 27
     name = 'sql'
28 28
     log = logging.getLogger("zuul.SQLReporter")
29 29
 
30
-    artifact = {
31
-        'name': str,
32
-        'url': str,
33
-    }
34
-    zuul_data = {
35
-        'zuul': {
36
-            'log_url': str,
37
-            'artifacts': [artifact],
38
-            v.Extra: object,
39
-        }
40
-    }
41
-    artifact_schema = v.Schema(zuul_data)
42
-
43
-    def validateArtifactSchema(self, data):
44
-        try:
45
-            self.artifact_schema(data)
46
-        except Exception:
47
-            return False
48
-        return True
49
-
50 30
     def report(self, item):
51 31
         """Create an entry into a database."""
52 32
 
@@ -104,32 +84,13 @@ class SQLReporter(BaseReporter):
104 84
                     node_name=build.node_name,
105 85
                 )
106 86
 
107
-                if self.validateArtifactSchema(build.result_data):
108
-                    artifacts = build.result_data.get('zuul', {}).get(
109
-                        'artifacts', [])
110
-                    default_url = build.result_data.get('zuul', {}).get(
111
-                        'log_url')
112
-                    if default_url:
113
-                        if default_url[-1] != '/':
114
-                            default_url += '/'
115
-                    for artifact in artifacts:
116
-                        url = artifact['url']
117
-                        if default_url:
118
-                            # If the artifact url is relative, it will
119
-                            # be combined with the log_url; if it is
120
-                            # absolute, it will replace it.
121
-                            try:
122
-                                url = urllib.parse.urljoin(default_url, url)
123
-                            except Exception:
124
-                                self.log.debug("Error parsing URL:",
125
-                                               exc_info=1)
126
-                        db_build.createArtifact(
127
-                            name=artifact['name'],
128
-                            url=url,
129
-                        )
130
-                else:
131
-                    self.log.debug("Result data did not pass artifact schema "
132
-                                   "validation: %s", build.result_data)
87
+                for provides in job.provides:
88
+                    db_build.createProvides(name=provides)
89
+
90
+                for artifact in get_artifacts_from_result_data(
91
+                    build.result_data,
92
+                    logger=self.log):
93
+                    db_build.createArtifact(**artifact)
133 94
 
134 95
 
135 96
 def getSchema():

+ 2
- 0
zuul/executor/client.py View File

@@ -165,6 +165,8 @@ class ExecutorClient(object):
165 165
                            timeout=job.timeout,
166 166
                            jobtags=sorted(job.tags),
167 167
                            _inheritance_path=list(job.inheritance_path))
168
+        if job.artifact_data:
169
+            zuul_params['artifacts'] = job.artifact_data
168 170
         if job.override_checkout:
169 171
             zuul_params['override_checkout'] = job.override_checkout
170 172
         if hasattr(item.change, 'branch'):

+ 69
- 0
zuul/lib/artifacts.py View File

@@ -0,0 +1,69 @@
1
+# Copyright 2018-2019 Red Hat, Inc.
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+#      http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+
15
+import voluptuous as v
16
+import urllib.parse
17
+
18
+artifact = {
19
+    'name': str,
20
+    'url': str,
21
+}
22
+
23
+zuul_data = {
24
+    'zuul': {
25
+        'log_url': str,
26
+        'artifacts': [artifact],
27
+        v.Extra: object,
28
+    }
29
+}
30
+
31
+artifact_schema = v.Schema(zuul_data)
32
+
33
+
34
+def validate_artifact_schema(data):
35
+    try:
36
+        artifact_schema(data)
37
+    except Exception:
38
+        return False
39
+    return True
40
+
41
+
42
+def get_artifacts_from_result_data(result_data, logger=None):
43
+    ret = []
44
+    if validate_artifact_schema(result_data):
45
+        artifacts = result_data.get('zuul', {}).get(
46
+            'artifacts', [])
47
+        default_url = result_data.get('zuul', {}).get(
48
+            'log_url')
49
+        if default_url:
50
+            if default_url[-1] != '/':
51
+                default_url += '/'
52
+        for artifact in artifacts:
53
+            url = artifact['url']
54
+            if default_url:
55
+                # If the artifact url is relative, it will be combined
56
+                # with the log_url; if it is absolute, it will replace
57
+                # it.
58
+                try:
59
+                    url = urllib.parse.urljoin(default_url, url)
60
+                except Exception:
61
+                    if logger:
62
+                        logger.debug("Error parsing URL:",
63
+                                     exc_info=1)
64
+            ret.append({'name': artifact['name'],
65
+                        'url': url})
66
+    else:
67
+        logger.debug("Result data did not pass artifact schema "
68
+                     "validation: %s", result_data)
69
+    return ret

+ 130
- 6
zuul/model.py View File

@@ -28,6 +28,7 @@ import itertools
28 28
 
29 29
 from zuul import change_matcher
30 30
 from zuul.lib.config import get_default
31
+from zuul.lib.artifacts import get_artifacts_from_result_data
31 32
 
32 33
 MERGER_MERGE = 1          # "git merge"
33 34
 MERGER_MERGE_RESOLVE = 2  # "git merge -s resolve"
@@ -164,6 +165,11 @@ class TemplateNotFoundError(Exception):
164 165
     pass
165 166
 
166 167
 
168
+class RequirementsError(Exception):
169
+    """A job's requirements were not met."""
170
+    pass
171
+
172
+
167 173
 class Attributes(object):
168 174
     """A class to hold attributes for string formatting."""
169 175
 
@@ -1070,6 +1076,8 @@ class Job(ConfigObject):
1070 1076
             file_matcher=None,
1071 1077
             irrelevant_file_matcher=None,  # skip-if
1072 1078
             tags=frozenset(),
1079
+            provides=frozenset(),
1080
+            requires=frozenset(),
1073 1081
             dependencies=frozenset(),
1074 1082
         )
1075 1083
 
@@ -1111,6 +1119,7 @@ class Job(ConfigObject):
1111 1119
             start_mark=None,
1112 1120
             inheritance_path=(),
1113 1121
             parent_data=None,
1122
+            artifact_data=None,
1114 1123
             description=None,
1115 1124
             variant_description=None,
1116 1125
             protected_origin=None,
@@ -1161,6 +1170,10 @@ class Job(ConfigObject):
1161 1170
         d['protected'] = self.protected
1162 1171
         d['voting'] = self.voting
1163 1172
         d['timeout'] = self.timeout
1173
+        d['tags'] = list(self.tags)
1174
+        d['provides'] = list(self.provides)
1175
+        d['requires'] = list(self.requires)
1176
+        d['dependencies'] = list(self.dependencies)
1164 1177
         d['attempts'] = self.attempts
1165 1178
         d['roles'] = list(map(lambda x: x.toDict(), self.roles))
1166 1179
         d['post_review'] = self.post_review
@@ -1170,9 +1183,6 @@ class Job(ConfigObject):
1170 1183
             d['parent'] = self.parent
1171 1184
         else:
1172 1185
             d['parent'] = tenant.default_base_job
1173
-        d['dependencies'] = []
1174
-        for dependency in self.dependencies:
1175
-            d['dependencies'].append(dependency)
1176 1186
         if isinstance(self.nodeset, str):
1177 1187
             ns = tenant.layout.nodesets.get(self.nodeset)
1178 1188
         else:
@@ -1343,6 +1353,9 @@ class Job(ConfigObject):
1343 1353
         self.parent_data = v
1344 1354
         self.variables = Job._deepUpdate(self.parent_data, self.variables)
1345 1355
 
1356
+    def updateArtifactData(self, artifact_data):
1357
+        self.artifact_data = artifact_data
1358
+
1346 1359
     def updateProjectVariables(self, project_vars):
1347 1360
         # Merge project/template variables directly into the job
1348 1361
         # variables.  Job variables override project variables.
@@ -1499,11 +1512,12 @@ class Job(ConfigObject):
1499 1512
 
1500 1513
         for k in self.context_attributes:
1501 1514
             if (other._get(k) is not None and
1502
-                k not in set(['tags'])):
1515
+                k not in set(['tags', 'requires', 'provides'])):
1503 1516
                 setattr(self, k, other._get(k))
1504 1517
 
1505
-        if other._get('tags') is not None:
1506
-            self.tags = frozenset(self.tags.union(other.tags))
1518
+        for k in ('tags', 'requires', 'provides'):
1519
+            if other._get(k) is not None:
1520
+                setattr(self, k, getattr(self, k).union(other._get(k)))
1507 1521
 
1508 1522
         self.inheritance_path = self.inheritance_path + (repr(other),)
1509 1523
 
@@ -1924,6 +1938,7 @@ class BuildSet(object):
1924 1938
 
1925 1939
 
1926 1940
 class QueueItem(object):
1941
+
1927 1942
     """Represents the position of a Change in a ChangeQueue.
1928 1943
 
1929 1944
     All Changes are enqueued into ChangeQueue in a QueueItem. The QueueItem
@@ -1950,6 +1965,7 @@ class QueueItem(object):
1950 1965
         self.layout = None
1951 1966
         self.project_pipeline_config = None
1952 1967
         self.job_graph = None
1968
+        self._cached_sql_results = None
1953 1969
 
1954 1970
     def __repr__(self):
1955 1971
         if self.pipeline:
@@ -2146,6 +2162,110 @@ class QueueItem(object):
2146 2162
             return False
2147 2163
         return self.item_ahead.isHoldingFollowingChanges()
2148 2164
 
2165
+    def _getRequirementsResultFromSQL(self, requirements):
2166
+        # This either returns data or raises an exception
2167
+        if self._cached_sql_results is None:
2168
+            sql_driver = self.pipeline.manager.sched.connections.drivers['sql']
2169
+            conn = sql_driver.tenant_connections.get(self.pipeline.tenant.name)
2170
+            if conn:
2171
+                builds = conn.getBuilds(
2172
+                    tenant=self.pipeline.tenant.name,
2173
+                    project=self.change.project.name,
2174
+                    pipeline=self.pipeline.name,
2175
+                    change=self.change.number,
2176
+                    branch=self.change.branch,
2177
+                    patchset=self.change.patchset,
2178
+                    provides=list(requirements))
2179
+            else:
2180
+                builds = []
2181
+            # Just look at the most recent buildset.
2182
+            # TODO: query for a buildset instead of filtering.
2183
+            builds = [b for b in builds
2184
+                      if b.buildset.uuid == builds[0].buildset.uuid]
2185
+            self._cached_sql_results = builds
2186
+
2187
+        builds = self._cached_sql_results
2188
+        data = []
2189
+        if not builds:
2190
+            return data
2191
+
2192
+        for build in builds:
2193
+            if build.result != 'SUCCESS':
2194
+                provides = [x.name for x in build.provides]
2195
+                requirement = list(requirements.intersection(set(provides)))
2196
+                raise RequirementsError(
2197
+                    "Requirements %s not met by build %s" % (
2198
+                        requirement, build.uuid))
2199
+            else:
2200
+                artifacts = [{'name': a.name,
2201
+                              'url': a.url,
2202
+                              'project': build.buildset.project,
2203
+                              'change': str(build.buildset.change),
2204
+                              'patchset': build.buildset.patchset,
2205
+                              'job': build.job_name}
2206
+                             for a in build.artifacts]
2207
+                data += artifacts
2208
+        return data
2209
+
2210
+    def providesRequirements(self, requirements, data):
2211
+        # Mutates data and returns true/false if requirements
2212
+        # satisfied.
2213
+        if not requirements:
2214
+            return True
2215
+        if not self.live:
2216
+            # Look for this item in other queues in the pipeline.
2217
+            item = None
2218
+            found = False
2219
+            for item in self.pipeline.getAllItems():
2220
+                if item.live and item.change == self.change:
2221
+                    found = True
2222
+                    break
2223
+            if found:
2224
+                if not item.providesRequirements(requirements, data):
2225
+                    return False
2226
+            else:
2227
+                # Look for this item in the SQL DB.
2228
+                data += self._getRequirementsResultFromSQL(requirements)
2229
+        if self.hasJobGraph():
2230
+            for job in self.getJobs():
2231
+                if job.provides.intersection(requirements):
2232
+                    build = self.current_build_set.getBuild(job.name)
2233
+                    if not build:
2234
+                        return False
2235
+                    if build.result and build.result != 'SUCCESS':
2236
+                        return False
2237
+                    if not build.result and not build.paused:
2238
+                        return False
2239
+                    artifacts = get_artifacts_from_result_data(
2240
+                        build.result_data,
2241
+                        logger=self.log)
2242
+                    artifacts = [{'name': a['name'],
2243
+                                  'url': a['url'],
2244
+                                  'project': self.change.project.name,
2245
+                                  'change': self.change.number,
2246
+                                  'patchset': self.change.patchset,
2247
+                                  'job': build.job.name}
2248
+                                 for a in artifacts]
2249
+                    data += artifacts
2250
+        if not self.item_ahead:
2251
+            return True
2252
+        return self.item_ahead.providesRequirements(requirements, data)
2253
+
2254
+    def jobRequirementsReady(self, job):
2255
+        if not self.item_ahead:
2256
+            return True
2257
+        try:
2258
+            data = []
2259
+            ret = self.item_ahead.providesRequirements(job.requires, data)
2260
+            job.updateArtifactData(data)
2261
+        except RequirementsError as e:
2262
+            self.warning(str(e))
2263
+            fakebuild = Build(job, None)
2264
+            fakebuild.result = 'SKIPPED'
2265
+            self.addBuild(fakebuild)
2266
+            ret = True
2267
+        return ret
2268
+
2149 2269
     def findJobsToRun(self, semaphore_handler):
2150 2270
         torun = []
2151 2271
         if not self.live:
@@ -2173,6 +2293,8 @@ class QueueItem(object):
2173 2293
         for job in self.job_graph.getJobs():
2174 2294
             if job not in jobs_not_started:
2175 2295
                 continue
2296
+            if not self.jobRequirementsReady(job):
2297
+                continue
2176 2298
             all_parent_jobs_successful = True
2177 2299
             parent_builds_with_data = {}
2178 2300
             for parent_job in self.job_graph.getParentJobsRecursively(
@@ -2237,6 +2359,8 @@ class QueueItem(object):
2237 2359
         for job in self.job_graph.getJobs():
2238 2360
             if job not in jobs_not_requested:
2239 2361
                 continue
2362
+            if not self.jobRequirementsReady(job):
2363
+                continue
2240 2364
             all_parent_jobs_successful = True
2241 2365
             for parent_job in self.job_graph.getParentJobsRecursively(
2242 2366
                     job.name):

+ 5
- 0
zuul/web/__init__.py View File

@@ -440,6 +440,7 @@ class ZuulWebAPI(object):
440 440
             'newrev': buildset.newrev,
441 441
             'ref_url': buildset.ref_url,
442 442
             'artifacts': [],
443
+            'provides': [],
443 444
         }
444 445
 
445 446
         for artifact in build.artifacts:
@@ -447,6 +448,10 @@ class ZuulWebAPI(object):
447 448
                 'name': artifact.name,
448 449
                 'url': artifact.url,
449 450
             })
451
+        for provides in build.provides:
452
+            ret['provides'].append({
453
+                'name': artifact.name,
454
+            })
450 455
         return ret
451 456
 
452 457
     @cherrypy.expose

Loading…
Cancel
Save