Add commit_id to zuul vars

We added the commit_id to the MQTT reporter a while ago so that
external systems which identify changes by commit id (the Gerrit
current patchset hexsha, or the Github PR HEAD, etc) can track
Zuul work.  To enable the same sort of interfacing with external
systems in Zuul jobs, the variable is added to the set of zuul
variables.

To make it easier for use in multiple pipelines, the variable is
set to oldrev or newrev in the non-change case.

To make it easier to confirm driver parity, a new Gerrit driver
test class is added with the intention that it should parallel
the other driver test classes over time.

Change-Id: I2826349ee9ce696c8f9de8c23276094123292196

doc/source/job-content.rst

@@ -786,6 +786,15 @@ are available:
       workspace for debugging and inspection purposes does not inclued
       the ``!unsafe`` tag.
 
+   .. var:: commit_id
+
+      The git sha of the change.  This may be the commit sha of the
+      current patchset revision or the tip of a pull request branch
+      depending on the source.  Because of Zuul's speculative merge
+      process, this commit may not even appear in the prepared git
+      repos, so it should not be relied upon for git operations in
+      jobs.  It is included here to support interfacing with systems
+      that identify a change by the commit.
 
 Branch Items
 ~~~~~~~~~~~~
@@ -816,6 +825,11 @@ additional variables are available:
       being pushed to the branch, the git sha of the new revision will
       be included here.  Otherwise, this variable will be undefined.
 
+   .. var:: commit_id
+
+      The git sha of the branch.  Identical to ``newrev`` or
+      ``oldrev`` if defined.
+
 Tag Items
 ~~~~~~~~~
 
@@ -842,6 +856,11 @@ available:
       the new git sha of the tag will be included here.  If the tag
       was deleted, this variable will be undefined.
 
+   .. var:: commit_id
+
+      The git sha of the branch.  Identical to ``newrev`` or
+      ``oldrev`` if defined.
+
 Ref Items
 ~~~~~~~~~
 
@@ -865,6 +884,11 @@ available:
       the new git sha of the ref will be included here.  If the ref
       was deleted, this variable will be undefined.
 
+   .. var:: commit_id
+
+      The git sha of the branch.  Identical to ``newrev`` or
+      ``oldrev`` if defined.
+
 Working Directory
 ~~~~~~~~~~~~~~~~~
 

releasenotes/notes/commit_id-56cc40ac08535bdc.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    A new job variable, `zuul.commit_id` has been added.  This
+    provides a stable identifier for all types of pipeline queue items
+    (changes, refs, tags, or branches) from any driver to identify the
+    commit which triggered the queue item.  This can be used to
+    interface with external systems which identify changes by commit.

tests/fixtures/layouts/simple.yaml

@@ -40,6 +40,14 @@
         - event: ref-updated
           ref: ^(?!refs/).*$
 
+- pipeline:
+    name: tag
+    manager: independent
+    trigger:
+      gerrit:
+        - event: ref-updated
+          ref: ^refs/tags/.*$
+
 - job:
     name: base
     parent: null
@@ -57,6 +65,10 @@
     name: post-job
     run: playbooks/post.yaml
 
+- job:
+    name: tag-job
+    run: playbooks/tag.yaml
+
 - project:
     name: org/project
     check:
@@ -68,3 +80,6 @@
     post:
       jobs:
         - post-job
+    tag:
+      jobs:
+        - tag-job

tests/unit/test_gerrit.py

@@ -1,4 +1,5 @@
 # Copyright 2015 BMW Car IT GmbH
+# Copyright 2023 Acme Gating, LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -25,6 +26,7 @@ from tests.base import (
     skipIfMultiScheduler,
     ZuulTestCase,
 )
+from zuul.lib import strings
 from zuul.driver.gerrit import GerritDriver
 from zuul.driver.gerrit.gerritconnection import GerritConnection
 
@@ -1101,3 +1103,56 @@ class TestGerritUnicodeRefs(ZuulTestCase):
                           '52944ee370db5c87691e62e0f9079b6281319b4e',
                           'refs/heads/faster':
                           '52944ee370db5c87691e62e0f9079b6281319b4e'})
+
+
+class TestGerritDriver(ZuulTestCase):
+    # Most of the Zuul test suite tests the Gerrit driver, to some
+    # extent.  The other classes in this file test specific methods of
+    # Zuul interacting with Gerrit.  But the other drivers also test
+    # some basic driver functionality that, if tested for Gerrit at
+    # all, is spread out in random tests.  This class adds some
+    # (potentially duplicative) testing to validate parity with the
+    # other drivers.
+
+    @simple_layout('layouts/simple.yaml')
+    def test_change_event(self):
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertHistory([
+            dict(name='check-job', result='SUCCESS', changes='1,1'),
+        ])
+
+        job = self.getJobFromHistory('check-job')
+        zuulvars = job.parameters['zuul']
+        self.assertEqual(str(A.number), zuulvars['change'])
+        self.assertEqual('1', zuulvars['patchset'])
+        self.assertEqual(str(A.patchsets[-1]['revision']),
+                         zuulvars['commit_id'])
+        self.assertEqual('master', zuulvars['branch'])
+        self.assertEquals('https://review.example.com/1',
+                          zuulvars['items'][0]['change_url'])
+        self.assertEqual(zuulvars["message"], strings.b64encode('A'))
+        self.assertEqual(1, len(self.history))
+        self.assertEqual(1, len(A.messages))
+
+    @simple_layout('layouts/simple.yaml')
+    def test_tag_event(self):
+        event = self.fake_gerrit.addFakeTag('org/project', 'master', 'foo')
+        tagsha = event['refUpdate']['newRev']
+        self.fake_gerrit.addEvent(event)
+        self.waitUntilSettled()
+
+        self.assertHistory([
+            dict(name='tag-job', result='SUCCESS', ref='refs/tags/foo'),
+        ])
+
+        job = self.getJobFromHistory('tag-job')
+        zuulvars = job.parameters['zuul']
+        zuulvars = job.parameters['zuul']
+        self.assertEqual('refs/tags/foo', zuulvars['ref'])
+        self.assertEqual('tag', zuulvars['pipeline'])
+        self.assertEqual('tag-job', zuulvars['job'])
+        self.assertEqual(tagsha, zuulvars['newrev'])
+        self.assertEqual(tagsha, zuulvars['commit_id'])

tests/unit/test_github_driver.py

@@ -71,6 +71,7 @@ class TestGithubDriver(ZuulTestCase):
         zuulvars = job.parameters['zuul']
         self.assertEqual(str(A.number), zuulvars['change'])
         self.assertEqual(str(A.head_sha), zuulvars['patchset'])
+        self.assertEqual(str(A.head_sha), zuulvars['commit_id'])
         self.assertEqual('master', zuulvars['branch'])
         self.assertEquals('https://github.com/org/project/pull/1',
                           zuulvars['items'][0]['change_url'])
@@ -293,6 +294,7 @@ class TestGithubDriver(ZuulTestCase):
         self.assertEqual('refs/tags/newtag', build_params['zuul']['ref'])
         self.assertFalse('oldrev' in build_params['zuul'])
         self.assertEqual(sha, build_params['zuul']['newrev'])
+        self.assertEqual(sha, build_params['zuul']['commit_id'])
         self.assertEqual(
             'https://github.com/org/project/releases/tag/newtag',
             build_params['zuul']['change_url'])

tests/unit/test_gitlab_driver.py

@@ -108,6 +108,7 @@ class TestGitlabDriver(ZuulTestCase):
         zuulvars = job.parameters['zuul']
         self.assertEqual(str(A.number), zuulvars['change'])
         self.assertEqual(str(A.sha), zuulvars['patchset'])
+        self.assertEqual(str(A.sha), zuulvars['commit_id'])
         self.assertEqual('master', zuulvars['branch'])
         self.assertEquals(f'{self.fake_gitlab._test_baseurl}/'
                           'org/project/merge_requests/1',
@@ -425,6 +426,7 @@ class TestGitlabDriver(ZuulTestCase):
         self.assertEqual('tag', zuulvars['pipeline'])
         self.assertEqual('project-tag-job', zuulvars['job'])
         self.assertEqual(tagsha, zuulvars['newrev'])
+        self.assertEqual(tagsha, zuulvars['commit_id'])
 
     @simple_layout('layouts/basic-gitlab.yaml', driver='gitlab')
     def test_pull_request_with_dyn_reconf(self):

tests/unit/test_pagure_driver.py

@@ -50,6 +50,7 @@ class TestPagureDriver(ZuulTestCase):
         zuulvars = job.parameters['zuul']
         self.assertEqual(str(A.number), zuulvars['change'])
         self.assertEqual(str(A.commit_stop), zuulvars['patchset'])
+        self.assertEqual(str(A.commit_stop), zuulvars['commit_id'])
         self.assertEqual('master', zuulvars['branch'])
         self.assertEquals('https://pagure/org/project/pull-request/1',
                           zuulvars['items'][0]['change_url'])
@@ -207,6 +208,7 @@ class TestPagureDriver(ZuulTestCase):
             zuulvars['change_url'])
         self.assertEqual(expected_newrev, zuulvars['newrev'])
         self.assertEqual(expected_oldrev, zuulvars['oldrev'])
+        self.assertEqual(expected_newrev, zuulvars['commit_id'])
 
     @simple_layout('layouts/basic-pagure.yaml', driver='pagure')
     def test_ref_created(self):

zuul/driver/pagure/pagureconnection.py

@@ -779,6 +779,7 @@ class PagureConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
         change.branch = change.pr.get('branch')
         change.is_current_patchset = (change.pr.get('commit_stop') ==
                                       change.patchset)
+        change.commit_id = change.pr.get('commit_stop')
         change.files = change.pr.get('files')
         change.title = change.pr.get('title')
         change.tags = change.pr.get('tags')

zuul/executor/common.py

@@ -66,12 +66,20 @@ def construct_build_params(uuid, connections, job, item, pipeline,
     if hasattr(item.change, 'message'):
         zuul_params['message'] = strings.b64encode(item.change.message)
         zuul_params['change_message'] = item.change.message
+    commit_id = None
     if (hasattr(item.change, 'oldrev') and item.change.oldrev
         and item.change.oldrev != '0' * 40):
         zuul_params['oldrev'] = item.change.oldrev
+        commit_id = item.change.oldrev
     if (hasattr(item.change, 'newrev') and item.change.newrev
         and item.change.newrev != '0' * 40):
         zuul_params['newrev'] = item.change.newrev
+        commit_id = item.change.newrev
+    if hasattr(item.change, 'commit_id'):
+        commit_id = item.change.commit_id
+    if commit_id:
+        zuul_params['commit_id'] = commit_id
+
     zuul_params['projects'] = {}  # Set below
     zuul_params['items'] = dependent_changes
     zuul_params['child_jobs'] = list(item.current_build_set.job_graph.