From affd95d24ec2fb85ec4f19296cca1607d27fec86 Mon Sep 17 00:00:00 2001
From: Chris Dent <chris.dent@gmail.com>
Date: Thu, 23 Jul 2015 11:48:32 +0100
Subject: [PATCH] Treat REPLACER errors as test failures

Make sure that when one of the magic replacers is used that if there
is an error attempting to access the referenced data it is treated
as a test failure and not as an error.

This effectively means trapping various exceptions and turning them
into AssertionErrorS.
---
 gabbi/case.py                        |  7 ++++++-
 gabbi/gabbits_intercept/backref.yaml |  9 +++++++++
 gabbi/gabbits_intercept/self.yaml    | 15 +++++++++++++++
 gabbi/handlers.py                    |  2 ++
 test-failskip.sh                     |  2 +-
 5 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/gabbi/case.py b/gabbi/case.py
index d252fb0..ec0c439 100644
--- a/gabbi/case.py
+++ b/gabbi/case.py
@@ -128,7 +128,12 @@ class HTTPTestCase(testcase.TestCase):
             method = '_%s_replace' % replacer.lower()
             try:
                 if template in message:
-                    message = getattr(self, method)(message)
+                    try:
+                        message = getattr(self, method)(message)
+                    except (KeyError, AttributeError, ValueError) as exc:
+                        raise AssertionError(
+                            'unable to replace %s in %s, data unavailable: %s'
+                            % (template, message, exc))
             except TypeError:
                 # Message is not a string
                 pass
diff --git a/gabbi/gabbits_intercept/backref.yaml b/gabbi/gabbits_intercept/backref.yaml
index e0cec5b..78e97d8 100644
--- a/gabbi/gabbits_intercept/backref.yaml
+++ b/gabbi/gabbits_intercept/backref.yaml
@@ -64,3 +64,12 @@ tests:
      location: $SCHEME://$NETLOC$RESPONSE['c']
      x-gabbi-url: $SCHEME://$NETLOC/v2
      content-type: $HEADERS['content-type']
+
+- name: backref json fail start
+  url:  /
+  method: POST
+  data: ''
+
+- name: backref json fail end
+  xfail: true
+  url: $RESPONSE['url']
diff --git a/gabbi/gabbits_intercept/self.yaml b/gabbi/gabbits_intercept/self.yaml
index 14ea19a..31ab83c 100644
--- a/gabbi/gabbits_intercept/self.yaml
+++ b/gabbi/gabbits_intercept/self.yaml
@@ -96,6 +96,11 @@ tests:
   response_headers:
       x-gabbi-url: $SCHEME://$NETLOC/takingnames
 
+- name: confirm environ no key fail
+  desc: this confirms that no key leads to failure rather than error
+  xfail: true
+  url: /$ENVIRON['1385F1EB-DC5C-4A95-8928-58673FB272DC']
+
 - name: test pluggable response
   url: /foo?alpha=1
   response_test:
@@ -114,3 +119,13 @@ tests:
   url: /
   xfail: true
   method: DIE
+
+- name: non json response failure
+  desc: asking for json in a non json test should be failure not error
+  url: /
+  xfail: true
+  method: GET
+  request_headers:
+      accept: text/plain
+  response_json_paths:
+      $.data: hello
diff --git a/gabbi/handlers.py b/gabbi/handlers.py
index 114e907..60782a6 100644
--- a/gabbi/handlers.py
+++ b/gabbi/handlers.py
@@ -100,6 +100,8 @@ class JSONResponseHandler(ResponseHandler):
         # to do their own processing.
         try:
             match = test.extract_json_path_value(test.json_data, path)
+        except AttributeError:
+            raise AssertionError('unable to extract JSON from test results')
         except ValueError:
             raise AssertionError('json path %s cannot match %s' %
                                  (path, test.json_data))
diff --git a/test-failskip.sh b/test-failskip.sh
index 833eff7..1c34977 100755
--- a/test-failskip.sh
+++ b/test-failskip.sh
@@ -2,7 +2,7 @@
 # Run the tests and confirm that the stuff we expect to skip or fail
 # does.
 
-GREP_FAIL_MATCH='expected failures=5'
+GREP_FAIL_MATCH='expected failures=8'
 GREP_SKIP_MATCH='skips=2'
 
 python setup.py testr && \