Browse Source

Return artifacts as dicts and add metadata

A recent attempt to use the artifact return feature of zuul_return
exposed some rough edges.  These two changes should make it much
easier to use.

First, return artifacts as a dictionary instead of a list.  This
requires that they have unique names (which is no bad thing -- what
would two artifacts named "docs" mean anyway?).  But mainly it allows
the dict merging behavior of zuul_return to be used, so that one
playbook may use zuul_return with some artifacts, and another playbook
may do the same, without either needing to load in the values of
the other first (assuming, of course, that they use different artifact
names).

Second, add a metadata field.  In the database and API, this is JSON
serialized, but in zuul_return and zuul.artifacts, it is exploded into
separate fields.  This lets jobs do things like associate versions or
tags with artifacts without having to abuse the url field.

Change-Id: I228687c1bd1c74ebc33b088ffd43f30c7309990d
James E. Blair 2 months ago
parent
commit
f12453f6cb

+ 13
- 5
doc/source/user/jobs.rst View File

@@ -229,7 +229,7 @@ of item.
229 229
    under the ``zuul`` key:
230 230
 
231 231
    .. var:: artifacts
232
-      :type: list
232
+      :type: dict
233 233
 
234 234
       If the job has a :attr:`job.requires` attribute, and Zuul has
235 235
       found changes ahead of this change in the pipeline with matching
@@ -263,6 +263,10 @@ of item.
263 263
 
264 264
          The URL of the artifact (as supplied to :ref:`return_artifacts`).
265 265
 
266
+      .. var:: metadata
267
+
268
+         The metadata of the artifact (as supplied to :ref:`return_artifacts`).
269
+
266 270
    .. var:: build
267 271
 
268 272
       The UUID of the build.  A build is a single execution of a job.
@@ -779,7 +783,7 @@ Returning artifact URLs
779 783
 
780 784
 If a build produces artifacts, any number of URLs may be returned to
781 785
 Zuul and stored in the SQL database.  These will then be available via
782
-the web interface.
786
+the web interface and subsequent jobs.
783 787
 
784 788
 To provide artifact URLs for a build, use *zuul_return* to set keys
785 789
 under the **zuul.artifacts** dictionary.  For example:
@@ -791,13 +795,17 @@ under the **zuul.artifacts** dictionary.  For example:
791 795
         data:
792 796
           zuul:
793 797
             artifacts:
794
-              - name: tarball
798
+              tarball:
795 799
                 url: http://example.com/path/to/package.tar.gz
796
-              - name: docs
800
+                metadata:
801
+                  version: 3.0
802
+              docs:
797 803
                 url: build/docs/
798 804
 
799 805
 If the value of **url** is a relative URL, it will be combined with
800
-the **zuul.log_url** value if set to create an absolute URL.
806
+the **zuul.log_url** value if set to create an absolute URL.  The
807
+**metadata** key is optional; if it is provided, it must be a
808
+dictionary; its keys and values may be anything.
801 809
 
802 810
 Skipping child jobs
803 811
 ~~~~~~~~~~~~~~~~~~~

+ 12
- 0
releasenotes/notes/artifact-format-2de4b9c038e28115.yaml View File

@@ -0,0 +1,12 @@
1
+---
2
+features:
3
+  - Artifacts may now include a metadata field for storing arbitrary
4
+    metadata about the artifacts in the SQL database.
5
+deprecations:
6
+  - Artifacts should now be supplied to zuul_return in dictionary form
7
+    instead of a list.  See :ref:`return_artifacts`.
8
+
9
+    This is to aid in multiple playbooks providing information back to
10
+    Zuul without requiring coordination with each other.
11
+
12
+    Support for the list format will be removed in a future version.

+ 37
- 0
zuul/driver/sql/alembic/versions/c18b1277dfb5_artifact_metadata.py View File

@@ -0,0 +1,37 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+#      http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+"""artifact_metadata
14
+
15
+Revision ID: c18b1277dfb5
16
+Revises: 39d302d34d38
17
+Create Date: 2019-02-04 14:02:44.291890
18
+
19
+"""
20
+
21
+# revision identifiers, used by Alembic.
22
+revision = 'c18b1277dfb5'
23
+down_revision = '39d302d34d38'
24
+branch_labels = None
25
+depends_on = None
26
+
27
+from alembic import op
28
+import sqlalchemy as sa
29
+
30
+
31
+def upgrade(table_prefix=''):
32
+    op.add_column(
33
+        table_prefix + 'zuul_artifact', sa.Column('metadata', sa.TEXT()))
34
+
35
+
36
+def downgrade():
37
+    raise Exception("Downgrades not supported")

+ 4
- 0
zuul/driver/sql/sqlconnection.py View File

@@ -223,6 +223,9 @@ class SQLConnection(BaseConnection):
223 223
 
224 224
             def createArtifact(self, *args, **kw):
225 225
                 session = orm.session.Session.object_session(self)
226
+                if 'metadata' in kw:
227
+                    kw['meta'] = kw['metadata']
228
+                    del kw['metadata']
226 229
                 a = ArtifactModel(*args, **kw)
227 230
                 a.build_id = self.id
228 231
                 self.artifacts.append(a)
@@ -246,6 +249,7 @@ class SQLConnection(BaseConnection):
246 249
                 self.table_prefix + BUILD_TABLE + ".id"))
247 250
             name = sa.Column(sa.String(255))
248 251
             url = sa.Column(sa.TEXT())
252
+            meta = sa.Column('metadata', sa.TEXT())
249 253
             build = orm.relationship(BuildModel, backref="artifacts")
250 254
 
251 255
         class ProvidesModel(Base):

+ 3
- 0
zuul/driver/sql/sqlreporter.py View File

@@ -13,6 +13,7 @@
13 13
 # under the License.
14 14
 
15 15
 import datetime
16
+import json
16 17
 import logging
17 18
 import time
18 19
 import voluptuous as v
@@ -90,6 +91,8 @@ class SQLReporter(BaseReporter):
90 91
                 for artifact in get_artifacts_from_result_data(
91 92
                     build.result_data,
92 93
                     logger=self.log):
94
+                    if 'metadata' in artifact:
95
+                        artifact['metadata'] = json.dumps(artifact['metadata'])
93 96
                     db_build.createArtifact(**artifact)
94 97
 
95 98
 

+ 18
- 6
zuul/lib/artifacts.py View File

@@ -15,15 +15,20 @@
15 15
 import voluptuous as v
16 16
 import urllib.parse
17 17
 
18
-artifact = {
18
+old_artifact = {
19 19
     'name': str,
20 20
     'url': str,
21 21
 }
22 22
 
23
+new_artifact = {
24
+    'url': str,
25
+    'metadata': dict,
26
+}
27
+
23 28
 zuul_data = {
24 29
     'zuul': {
25 30
         'log_url': str,
26
-        'artifacts': [artifact],
31
+        'artifacts': v.Any([old_artifact], {str: new_artifact}),
27 32
         v.Extra: object,
28 33
     }
29 34
 }
@@ -43,13 +48,18 @@ def get_artifacts_from_result_data(result_data, logger=None):
43 48
     ret = []
44 49
     if validate_artifact_schema(result_data):
45 50
         artifacts = result_data.get('zuul', {}).get(
46
-            'artifacts', [])
51
+            'artifacts', {})
52
+        if isinstance(artifacts, list):
53
+            new_artifacts = {}
54
+            for a in artifacts:
55
+                new_artifacts[a['name']] = {'url': a['url']}
56
+            artifacts = new_artifacts
47 57
         default_url = result_data.get('zuul', {}).get(
48 58
             'log_url')
49 59
         if default_url:
50 60
             if default_url[-1] != '/':
51 61
                 default_url += '/'
52
-        for artifact in artifacts:
62
+        for artifact_name, artifact in artifacts.items():
53 63
             url = artifact['url']
54 64
             if default_url:
55 65
                 # If the artifact url is relative, it will be combined
@@ -61,8 +71,10 @@ def get_artifacts_from_result_data(result_data, logger=None):
61 71
                     if logger:
62 72
                         logger.debug("Error parsing URL:",
63 73
                                      exc_info=1)
64
-            ret.append({'name': artifact['name'],
65
-                        'url': url})
74
+            d = artifact.copy()
75
+            d['name'] = artifact_name
76
+            d['url'] = url
77
+            ret.append(d)
66 78
     else:
67 79
         logger.debug("Result data did not pass artifact schema "
68 80
                      "validation: %s", result_data)

+ 11
- 8
zuul/model.py View File

@@ -15,6 +15,7 @@
15 15
 import abc
16 16
 from collections import OrderedDict
17 17
 import copy
18
+import json
18 19
 import logging
19 20
 import os
20 21
 import re2
@@ -2197,14 +2198,16 @@ class QueueItem(object):
2197 2198
                     "Requirements %s not met by build %s" % (
2198 2199
                         requirement, build.uuid))
2199 2200
             else:
2200
-                artifacts = [{'name': a.name,
2201
-                              'url': a.url,
2202
-                              'project': build.buildset.project,
2203
-                              'change': str(build.buildset.change),
2204
-                              'patchset': build.buildset.patchset,
2205
-                              'job': build.job_name}
2206
-                             for a in build.artifacts]
2207
-                data += artifacts
2201
+                for a in build.artifacts:
2202
+                    artifact = {'name': a.name,
2203
+                                'url': a.url,
2204
+                                'project': build.buildset.project,
2205
+                                'change': str(build.buildset.change),
2206
+                                'patchset': build.buildset.patchset,
2207
+                                'job': build.job_name}
2208
+                    if a.meta:
2209
+                        artifact['metadata'] = json.loads(a.meta)
2210
+                    data.append(artifact)
2208 2211
         return data
2209 2212
 
2210 2213
     def providesRequirements(self, requirements, data):

Loading…
Cancel
Save