Add check job for upstream URLs on imported repos

Add a check job to require upstream URLs to use a scheme
that we are confident won't cause connection errors during
the import.

Add a tox configuration for this new job, as well as the
existing alphabetical check script, to make it easier
for developers to run those tests locally just by running
"tox".

Change-Id: I894ac0f8afb234404bb6e80e64b90aa25e052f10

modules/openstack_project/files/jenkins_job_builder/config/infra.yaml

@@ -87,7 +87,26 @@
     builders:
       - revoke-sudo
       - gerrit-git-prep
-      - shell: "tools/check_projects_yaml_alphabetized.sh"
+      - tox:
+          envlist: projects_alphabetized
+          github-org: openstack-infra
+          project: config
+
+    publishers:
+      - console-log
+
+
+- job:
+    name: check-projects-yaml-upstream
+    node: bare-precise
+
+    builders:
+      - revoke-sudo
+      - gerrit-git-prep
+      - tox:
+          envlist: upstream
+          github-org: openstack-infra
+          project: config
 
     publishers:
       - console-log

modules/openstack_project/files/zuul/layout.yaml

@@ -2660,6 +2660,7 @@ projects:
       - gate-config-irc-access
       - gate-ci-docs
       - check-projects-yaml-alphabetized
+      - check-projects-yaml-upstream
     gate:
       - gate-config-layout
       - gate-config-pep8
@@ -2667,6 +2668,7 @@ projects:
       - gate-config-puppet-syntax
       - gate-config-irc-access
       - check-projects-yaml-alphabetized
+      - check-projects-yaml-upstream
     post:
       - ci-docs
 

tools/check_upstream_url_scheme.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+"""
+Allow git:// and https:// URLs for importing upstream repositories,
+but not git@
+"""
+
+import argparse
+
+import yaml
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-v', '--verbose',
+                    dest='verbose',
+                    default=False,
+                    action='store_true',
+                    )
+parser.add_argument(
+    'infile',
+    help='path to review.projects.yaml',
+)
+args = parser.parse_args()
+
+projects = yaml.load(open(args.infile, 'r'))
+
+VALID_SCHEMES = ['https://', 'http://', 'git://']
+
+for p in projects:
+    name = p.get('project')
+    if not name:
+        # not a project
+        continue
+    upstream = p.get('upstream')
+    if args.verbose:
+        print 'Checking %s: %r' % (name, upstream)
+    if not upstream:
+        continue
+    for prefix in VALID_SCHEMES:
+        if upstream.startswith(prefix):
+            break
+    else:
+        raise ValueError(
+            'Upstream URLs should use a scheme in %s, found %s' %
+            (VALID_SCHEMES, p['project'])
+        )

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 minversion = 1.6
-envlist = pep8
+envlist = pep8,upstream,projects_alphabetized
 skipsdist = True
 
 [testenv]
@@ -11,6 +11,14 @@ deps = -r{toxinidir}/test-requirements.txt
 [testenv:pep8]
 commands = flake8
 
+[testenv:upstream]
+deps = PyYAML
+commands =
+  {toxinidir}/tools/check_upstream_url_scheme.py -v modules/openstack_project/files/review.projects.yaml
+
+[testenv:projects_alphabetized]
+commands = {toxinidir}/tools/check_projects_yaml_alphabetized.sh
+
 [testenv:venv]
 commands = {posargs}