Emit per-branch queue stats separately

We currently emit 4 statsd metrics for each shared queue, but in
the case that a queue is configured as per-branch, we disregard
the branch and emit the stats under the same hierarchy for any
branch of that queue.  This means that if we have a queue for
integrated-master and a queue for integrated-stable at the same
time, we would emit the stats for the master queue, then
immediately emit the same stats for the stable queue, overwriting
the master stats.

To correct this, move the metrics down a level in the case that
the queue is configured per-branch, and include the branch name
in the key.

Change-Id: I2f4b22394bc3774410a02ae76281eddf080e5c7f

doc/source/monitoring.rst

@@ -237,13 +237,17 @@ These metrics are emitted by the Zuul :ref:`scheduler`:
          This hierarchy holds more specific metrics for each
          :term:`project queue` in the pipeline.
 
-         .. stat:: <queue name>
+         .. stat:: <queue>
 
             The name of the queue.  If the queue is automatically
             generated for a single project, the name of the project is
             used by default.  Embedded ``.`` characters will be
             translated to ``_``, and ``/`` to ``.``.
 
+            If the queue is configured as per-branch, the metrics
+            below are omitted and instead found under
+            :stat:`zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.branch`.
+
             .. stat:: current_changes
                :type: gauge
 
@@ -266,6 +270,20 @@ These metrics are emitted by the Zuul :ref:`scheduler`:
 
                The number of changes processed by the queue.
 
+            .. stat:: branch
+
+               If the queue is configured as per-branch, this
+               hierarchy will be present and will hold stats for each
+               branch seen.
+
+               .. stat:: <branch>
+
+                  The name of the branch.  Embedded ``.`` characters
+                  will be translated to ``_``, and ``/`` to ``.``.
+
+                  Underneath this key are per-branch values of the
+                  metrics above.
+
       .. stat:: project
 
          This hierarchy holds more specific metrics for each project

releasenotes/notes/per-branch-stats-0c9f262a70d5cc81.yaml

@@ -0,0 +1,15 @@
+---
+fixes:
+  - |
+    Monitoring stats for per-branch queues are now distinct from
+    shared-branch queues.  Shared branch queue stats are at:
+
+      :stat:`zuul.tenant.<tenant>.pipeline.<pipeline>.queue`.
+
+    Per-branch queue stats are now at:
+
+      :stat:`zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.branch`.
+
+    Prior to this change, per-branch queue stats for one branch queue
+    may have overwritten the stats from another queue resulting in
+    incomplete or incorrect data.

tests/unit/test_scheduler.py

@@ -6804,6 +6804,21 @@ class TestChangeQueues(ZuulTestCase):
         known upfront and the queues are pre-seeded.
         """
         self._test_dependent_queues_per_branch('org/project')
+        self.assertReportedStat(
+            'zuul.tenant.tenant-one.pipeline.gate.queue.'
+            'integrated.branch.master.current_changes',
+            value='1', kind='g')
+        self.assertReportedStat(
+            'zuul.tenant.tenant-one.pipeline.gate.queue.'
+            'integrated.branch.master.window',
+            value='20', kind='g')
+        self.assertReportedStat(
+            'zuul.tenant.tenant-one.pipeline.gate.queue.'
+            'integrated.branch.master.resident_time', kind='ms')
+        self.assertReportedStat(
+            'zuul.tenant.tenant-one.pipeline.gate.queue.'
+            'integrated.branch.master.total_changes', value='1',
+            kind='c')
 
     def test_dependent_queues_per_branch_no_config(self):
         """

zuul/manager/__init__.py

@@ -2269,7 +2269,23 @@ class PipelineManager(metaclass=ABCMeta):
                 # stats.gauges.zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.total_changes
                 # stats.gauges.zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.current_changes
                 # stats.gauges.zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.window
-                queuekey = '%s.queue.%s' % (key, queuename)
+                queuekey = f'{key}.queue.{queuename}'
+
+                # Handle per-branch queues
+                layout = self.pipeline.tenant.layout
+                queue_config = layout.queues.get(item.queue.name)
+                per_branch = queue_config and queue_config.per_branch
+                if per_branch and item.queue.project_branches:
+                    # Get the first project-branch of this queue,
+                    # which is a tuple of project, branch, and get
+                    # second item of that tuple, the branch name.  In
+                    # a per-branch queue, we expect the branch name to
+                    # be the same for every project.
+                    branch = item.queue.project_branches[0][1]
+                    if branch:
+                        branch = branch.replace('.', '_').replace('/', '.')
+                        queuekey = f'{queuekey}.branch.{branch}'
+
                 queue_changes = sum(len(i.changes) for i in item.queue.queue)
                 self.sched.statsd.gauge(queuekey + '.current_changes',
                                         queue_changes)