Test that job regexes in zuul/layout match

Enhance our zuul/layout.yaml checks to check that the names in the jobs
section actually match an existing job. A mismatch implies that the
pattern does not work as intented.

Add note to run-layout.sh that the job files is used elsewhere as well,
so that it does not get deleted by accident.

Change-Id: I5c4cd97829acf90bb8f72dde32e5fe8063933cdb

tools/layout-checks.py

@@ -16,6 +16,8 @@
 # limitations under the License.
 
 import yaml
+import os
+import re
 import sys
 
 layout = yaml.load(open('zuul/layout.yaml'))
@@ -89,10 +91,44 @@ def check_formatting():
 
     return errors
 
+def grep(source, pattern):
+    found = False
+    p = re.compile(pattern)
+    for line in open(source, "r").readlines():
+        if p.match(line):
+            found = True
+            break
+    return found
+
+
+def check_jobs():
+    """Check that jobs have matches"""
+    errors = False
+
+    print("Checking job section regex expressions")
+    print("======================================")
+
+    # The job-list.txt file is created by tools/run-layout.sh and
+    # thus should exist if this is run from tox. If this is manually invoked
+    # the file might not exist, in that case pass the test.
+    job_list = ".test/job-list.txt"
+    if not os.path.isfile(job_list):
+        print("Job list file %s does not exist, not checking jobs section"
+              % job_list)
+        return False
+    for jobs in layout['jobs']:
+        found = grep(job_list, jobs['name'])
+        if not found:
+            print ("Regex %s has no matches in job list" % jobs['name'])
+            errors = True
+
+    return errors
+
 def check_all():
     errors = check_projects_sorted()
     errors = check_merge_template() or errors
     errors = check_formatting() or errors
+    errors = check_jobs() or errors
 
     if errors:
         print("\nFound errors in layout.yaml!")

tools/run-layout.sh

@@ -44,8 +44,12 @@ cp jenkins/jobs/* .test/jenkins-job-builder/.test/new/config
 cd .test/jenkins-job-builder
 tox -e compare-xml-new
 
+# Note that the job-list.txt file is also used by
+# tools/layout-checks.py, if this file gets changed, we need to adapt
+# the test as well.
 cd ..
 find jenkins-job-builder/.test/new/out/ -printf "%f\n" > job-list.txt
 
+
 cd zuul
 tox -e venv -- zuul-server -c etc/zuul.conf-sample -l ../../zuul/layout.yaml -t ../job-list.txt