Prevent race on multibranch scan during update

When Jenkins triggers a repository scan while updating multibranch
pipelines, child jobs may be removed (e.g. if a PR has been merged or
closed). Address such cases more gracefully.

Change-Id: I882ce5111cc451494c0312d99ea6adaca4cf848f
Signed-off-by: Tyler Yankee <tyler.yankee@kitware.com>

jenkins_jobs/builder.py

@@ -22,6 +22,7 @@ import logging
 import os
 from pprint import pformat
 import re
+import requests
 from six.moves.urllib.parse import quote
 import time
 import xml.etree.ElementTree as XML
@@ -184,7 +185,21 @@ class JenkinsManager(object):
         return self.jobs
 
     def is_managed_job(self, job_name):
-        xml = self.jenkins.get_job_config(job_name)
+        # Under concurrent Jenkins repository scans (triggered by reconfiguring
+        # a multibranch pipeline), ephemeral child jobs (PR branches, etc.)
+        # may be removed. Treat a vanished job as unmanaged rather than raising.
+        try:
+            xml = self.jenkins.get_job_config(job_name)
+        except requests.exceptions.HTTPError as e:
+            status_code = getattr(getattr(e, "response", None), "status_code", None)
+            if status_code == 404:
+                logger.debug(
+                    "Job '%s' vanished before managed-status check; "
+                    "treating as unmanaged",
+                    job_name,
+                )
+                return False
+            raise
         try:
             out = XML.fromstring(xml.encode("utf-8"))
             description = out.find(".//description").text
@@ -200,7 +215,10 @@ class JenkinsManager(object):
         return self._plugins_list
 
     def delete_old_managed_jobs(self, keep=None):
-        jobs = self.get_jobs()
+        # Refresh the job list so we don't act on the startup snapshot.
+        # Multibranch scans triggered by reconfig_job may have created or
+        # deleted child jobs (PR branches, etc.) since we last fetched.
+        jobs = self.get_jobs(cache=False)
         deleted_jobs = 0
         if keep is None:
             keep = []

tests/jenkins_manager/test_manager.py

@@ -17,6 +17,7 @@
 from unittest import mock
 
 import pytest
+import requests
 
 from jenkins_jobs.config import JJBConfig
 import jenkins_jobs.builder
@@ -101,3 +102,48 @@ def test_get_plugins_info_error(mocker, jjb_config, error_string):
     mocker.patch.object(builder.jenkins, "get_plugins", side_effect=exception)
     plugins_info = builder.get_plugins_info()
     assert [_plugins_info["plugin1"]] == plugins_info
+
+
+def test_delete_old_managed_jobs_with_concurrent_http_404(mocker, jjb_config):
+    jjb_config.builder["plugins_info"] = None
+    builder = jenkins_jobs.builder.JenkinsManager(jjb_config)
+
+    pr_job_name = "my-pipeline/PR-123"
+    patches = mocker.patch.multiple(
+        "jenkins_jobs.builder.JenkinsManager",
+        get_jobs=mock.DEFAULT,
+        is_job=mock.DEFAULT,
+        delete_job=mock.DEFAULT,
+    )
+    patches["get_jobs"].return_value = [
+        {"fullname": "my-pipeline"},
+        {"fullname": pr_job_name},
+    ]
+    patches["is_job"].return_value = True
+
+    # After is_job() but before get_job_config(), the child job (PR, etc.) is
+    # deleted by Jenkins concurrent scanning.
+    def _get_job_config_side_effect(job_name):
+        if job_name == pr_job_name:
+            response = requests.Response()
+            response.status_code = 404
+            response.url = (
+                "https://jenkins.example/job/my-pipeline/" "job/PR-123/config.xml"
+            )
+            raise requests.exceptions.HTTPError(
+                "404 Client Error: Not found for url: " + response.url,
+                response=response,
+            )
+        return (
+            '<?xml version="1.0"?><org.jenkinsci.plugins.workflow.multibranch'
+            ".WorkflowMultiBranchProject>"
+            "<description>Managed by JJB</description>"
+            "</org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject>"
+        )
+
+    mocker.patch.object(
+        builder.jenkins, "get_job_config", side_effect=_get_job_config_side_effect
+    )
+
+    result = builder.delete_old_managed_jobs(keep=["my-pipeline"])
+    assert result == 0