Browse Source

Merge "Return artifacts as dicts and add metadata"

Zuul 2 months ago
parent
commit
5957d7a95e

+ 13
- 5
doc/source/user/jobs.rst View File

@@ -229,7 +229,7 @@ of item.
229 229
    under the ``zuul`` key:
230 230
 
231 231
    .. var:: artifacts
232
-      :type: list
232
+      :type: dict
233 233
 
234 234
       If the job has a :attr:`job.requires` attribute, and Zuul has
235 235
       found changes ahead of this change in the pipeline with matching
@@ -263,6 +263,10 @@ of item.
263 263
 
264 264
          The URL of the artifact (as supplied to :ref:`return_artifacts`).
265 265
 
266
+      .. var:: metadata
267
+
268
+         The metadata of the artifact (as supplied to :ref:`return_artifacts`).
269
+
266 270
    .. var:: build
267 271
 
268 272
       The UUID of the build.  A build is a single execution of a job.
@@ -779,7 +783,7 @@ Returning artifact URLs
779 783
 
780 784
 If a build produces artifacts, any number of URLs may be returned to
781 785
 Zuul and stored in the SQL database.  These will then be available via
782
-the web interface.
786
+the web interface and subsequent jobs.
783 787
 
784 788
 To provide artifact URLs for a build, use *zuul_return* to set keys
785 789
 under the **zuul.artifacts** dictionary.  For example:
@@ -791,13 +795,17 @@ under the **zuul.artifacts** dictionary.  For example:
791 795
         data:
792 796
           zuul:
793 797
             artifacts:
794
-              - name: tarball
798
+              tarball:
795 799
                 url: http://example.com/path/to/package.tar.gz
796
-              - name: docs
800
+                metadata:
801
+                  version: 3.0
802
+              docs:
797 803
                 url: build/docs/
798 804
 
799 805
 If the value of **url** is a relative URL, it will be combined with
800
-the **zuul.log_url** value if set to create an absolute URL.
806
+the **zuul.log_url** value if set to create an absolute URL.  The
807
+**metadata** key is optional; if it is provided, it must be a
808
+dictionary; its keys and values may be anything.
801 809
 
802 810
 Skipping child jobs
803 811
 ~~~~~~~~~~~~~~~~~~~

+ 12
- 0
releasenotes/notes/artifact-format-2de4b9c038e28115.yaml View File

@@ -0,0 +1,12 @@
1
+---
2
+features:
3
+  - Artifacts may now include a metadata field for storing arbitrary
4
+    metadata about the artifacts in the SQL database.
5
+deprecations:
6
+  - Artifacts should now be supplied to zuul_return in dictionary form
7
+    instead of a list.  See :ref:`return_artifacts`.
8
+
9
+    This is to aid in multiple playbooks providing information back to
10
+    Zuul without requiring coordination with each other.
11
+
12
+    Support for the list format will be removed in a future version.

+ 37
- 0
zuul/driver/sql/alembic/versions/c18b1277dfb5_artifact_metadata.py View File

@@ -0,0 +1,37 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+#      http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+"""artifact_metadata
14
+
15
+Revision ID: c18b1277dfb5
16
+Revises: 39d302d34d38
17
+Create Date: 2019-02-04 14:02:44.291890
18
+
19
+"""
20
+
21
+# revision identifiers, used by Alembic.
22
+revision = 'c18b1277dfb5'
23
+down_revision = '39d302d34d38'
24
+branch_labels = None
25
+depends_on = None
26
+
27
+from alembic import op
28
+import sqlalchemy as sa
29
+
30
+
31
+def upgrade(table_prefix=''):
32
+    op.add_column(
33
+        table_prefix + 'zuul_artifact', sa.Column('metadata', sa.TEXT()))
34
+
35
+
36
+def downgrade():
37
+    raise Exception("Downgrades not supported")

+ 4
- 0
zuul/driver/sql/sqlconnection.py View File

@@ -223,6 +223,9 @@ class SQLConnection(BaseConnection):
223 223
 
224 224
             def createArtifact(self, *args, **kw):
225 225
                 session = orm.session.Session.object_session(self)
226
+                if 'metadata' in kw:
227
+                    kw['meta'] = kw['metadata']
228
+                    del kw['metadata']
226 229
                 a = ArtifactModel(*args, **kw)
227 230
                 a.build_id = self.id
228 231
                 self.artifacts.append(a)
@@ -246,6 +249,7 @@ class SQLConnection(BaseConnection):
246 249
                 self.table_prefix + BUILD_TABLE + ".id"))
247 250
             name = sa.Column(sa.String(255))
248 251
             url = sa.Column(sa.TEXT())
252
+            meta = sa.Column('metadata', sa.TEXT())
249 253
             build = orm.relationship(BuildModel, backref="artifacts")
250 254
 
251 255
         class ProvidesModel(Base):

+ 3
- 0
zuul/driver/sql/sqlreporter.py View File

@@ -13,6 +13,7 @@
13 13
 # under the License.
14 14
 
15 15
 import datetime
16
+import json
16 17
 import logging
17 18
 import time
18 19
 import voluptuous as v
@@ -90,6 +91,8 @@ class SQLReporter(BaseReporter):
90 91
                 for artifact in get_artifacts_from_result_data(
91 92
                     build.result_data,
92 93
                     logger=self.log):
94
+                    if 'metadata' in artifact:
95
+                        artifact['metadata'] = json.dumps(artifact['metadata'])
93 96
                     db_build.createArtifact(**artifact)
94 97
 
95 98
 

+ 18
- 6
zuul/lib/artifacts.py View File

@@ -15,15 +15,20 @@
15 15
 import voluptuous as v
16 16
 import urllib.parse
17 17
 
18
-artifact = {
18
+old_artifact = {
19 19
     'name': str,
20 20
     'url': str,
21 21
 }
22 22
 
23
+new_artifact = {
24
+    'url': str,
25
+    'metadata': dict,
26
+}
27
+
23 28
 zuul_data = {
24 29
     'zuul': {
25 30
         'log_url': str,
26
-        'artifacts': [artifact],
31
+        'artifacts': v.Any([old_artifact], {str: new_artifact}),
27 32
         v.Extra: object,
28 33
     }
29 34
 }
@@ -43,13 +48,18 @@ def get_artifacts_from_result_data(result_data, logger=None):
43 48
     ret = []
44 49
     if validate_artifact_schema(result_data):
45 50
         artifacts = result_data.get('zuul', {}).get(
46
-            'artifacts', [])
51
+            'artifacts', {})
52
+        if isinstance(artifacts, list):
53
+            new_artifacts = {}
54
+            for a in artifacts:
55
+                new_artifacts[a['name']] = {'url': a['url']}
56
+            artifacts = new_artifacts
47 57
         default_url = result_data.get('zuul', {}).get(
48 58
             'log_url')
49 59
         if default_url:
50 60
             if default_url[-1] != '/':
51 61
                 default_url += '/'
52
-        for artifact in artifacts:
62
+        for artifact_name, artifact in artifacts.items():
53 63
             url = artifact['url']
54 64
             if default_url:
55 65
                 # If the artifact url is relative, it will be combined
@@ -61,8 +71,10 @@ def get_artifacts_from_result_data(result_data, logger=None):
61 71
                     if logger:
62 72
                         logger.debug("Error parsing URL:",
63 73
                                      exc_info=1)
64
-            ret.append({'name': artifact['name'],
65
-                        'url': url})
74
+            d = artifact.copy()
75
+            d['name'] = artifact_name
76
+            d['url'] = url
77
+            ret.append(d)
66 78
     else:
67 79
         logger.debug("Result data did not pass artifact schema "
68 80
                      "validation: %s", result_data)

+ 11
- 8
zuul/model.py View File

@@ -15,6 +15,7 @@
15 15
 import abc
16 16
 from collections import OrderedDict
17 17
 import copy
18
+import json
18 19
 import logging
19 20
 import os
20 21
 import re2
@@ -2197,14 +2198,16 @@ class QueueItem(object):
2197 2198
                     "Requirements %s not met by build %s" % (
2198 2199
                         requirement, build.uuid))
2199 2200
             else:
2200
-                artifacts = [{'name': a.name,
2201
-                              'url': a.url,
2202
-                              'project': build.buildset.project,
2203
-                              'change': str(build.buildset.change),
2204
-                              'patchset': build.buildset.patchset,
2205
-                              'job': build.job_name}
2206
-                             for a in build.artifacts]
2207
-                data += artifacts
2201
+                for a in build.artifacts:
2202
+                    artifact = {'name': a.name,
2203
+                                'url': a.url,
2204
+                                'project': build.buildset.project,
2205
+                                'change': str(build.buildset.change),
2206
+                                'patchset': build.buildset.patchset,
2207
+                                'job': build.job_name}
2208
+                    if a.meta:
2209
+                        artifact['metadata'] = json.loads(a.meta)
2210
+                    data.append(artifact)
2208 2211
         return data
2209 2212
 
2210 2213
     def providesRequirements(self, requirements, data):

Loading…
Cancel
Save