diff --git a/tools/configs/non-defaults.conf b/tools/configs/non-defaults.conf
new file mode 100644
index 000000000..5fb7aa276
--- /dev/null
+++ b/tools/configs/non-defaults.conf
@@ -0,0 +1,16 @@
+[job_builder]
+keep_descriptions=True
+ignore_cache=True
+recursive=True
+exclude=.*:manual:./development
+allow_duplicates=True
+allow_empty_variables=True
+
+[jenkins]
+user=jenkins
+password=1234567890abcdef1234567890abcdef
+url=https://jenkins.example.com
+query_plugins_info=False
+
+[__future__]
+param_order_from_yaml=True
diff --git a/tools/run-compare-configs.sh b/tools/run-compare-configs.sh
new file mode 100755
index 000000000..42406fe81
--- /dev/null
+++ b/tools/run-compare-configs.sh
@@ -0,0 +1,76 @@
+#!/bin/bash -e
+#
+# Copyright 2016 Hewlett-Packard Development Company, L.P.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+rm -fr .test
+mkdir -p .test/run-conf/config
+
+CONFIGS_DIR=$(dirname ${0})/configs
+CONFIGS=$(ls -1 ${CONFIGS_DIR}/*.conf 2>/dev/null)
+
+cd .test
+if [ -e /usr/zuul-env/bin/zuul-cloner ];
+then
+    /usr/zuul-env/bin/zuul-cloner -m ../tools/run-compare-clonemap.yaml --cache-dir /opt/git git://git.openstack.org openstack-infra/project-config
+else
+    git clone --depth=1 git://git.openstack.org/openstack-infra/project-config
+fi
+# setup a default configuration to compare results against
+cp -r project-config/jenkins/jobs/* run-conf/config
+cd ..
+
+mkdir -p .test/run-conf/default/out
+tox -e compare-xml-config
+
+echo "############################################################"
+echo "Starting processing configs"
+for conf_file in ${CONFIGS}
+do
+    echo "============================================================"
+    echo "Processing non-default config ${conf_file}"
+    conf_name=$(basename ${conf_file%%.conf})
+    mkdir -p .test/run-conf/${conf_name}/out
+    tox -e compare-xml-config -- --conf ${conf_file} test -o .test/run-conf/${conf_name}/out/ .test/run-conf/config
+    echo "------------------------------------------------------------"
+done
+
+echo "############################################################"
+echo "Comparing differences from default to alternative configs"
+
+for conf_file in ${CONFIGS}
+do
+    echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
+    echo "Comparing config ${conf_file}"
+    conf_name=$(basename ${conf_file%%.conf})
+    CHANGED=0
+    for x in `(cd .test/run-conf/default/out/ && find -type f)`
+    do
+        differences=$(diff -u .test/run-conf/default/out/${x} .test/run-conf/${conf_name}/out/${x} 2>&1)
+        if [ $? -ne 0 ]
+        then
+            CHANGED=1
+            echo "============================================================"
+            echo ${x}
+            echo "------------------------------------------------------------"
+            echo "${differences}"
+        fi
+    done
+    if [ "${CHANGED}" -eq "0" ]
+    then
+        echo "No differences between default and ${conf_name} configs"
+    fi
+done
+# should only fail if previous command exited with a non-zero status
+exit 0
diff --git a/tox.ini b/tox.ini
index be1a887c6..3bd156660 100644
--- a/tox.ini
+++ b/tox.ini
@@ -26,6 +26,9 @@ commands = flake8
 deps = pyflakes
 commands = pyflakes jenkins_jobs tests setup.py
 
+[testenv:compare-xml-config]
+commands = jenkins-jobs {posargs:test -o .test/run-conf/default/out/ .test/run-conf/config/}
+
 [testenv:compare-xml-old]
 commands = jenkins-jobs test -o .test/old/out/ .test/old/config/