Treat REPLACER errors as test failures

Make sure that when one of the magic replacers is used that if there
is an error attempting to access the referenced data it is treated
as a test failure and not as an error.

This effectively means trapping various exceptions and turning them
into AssertionErrorS.

gabbi/case.py

@@ -128,7 +128,12 @@ class HTTPTestCase(testcase.TestCase):
             method = '_%s_replace' % replacer.lower()
             try:
                 if template in message:
-                    message = getattr(self, method)(message)
+                    try:
+                        message = getattr(self, method)(message)
+                    except (KeyError, AttributeError, ValueError) as exc:
+                        raise AssertionError(
+                            'unable to replace %s in %s, data unavailable: %s'
+                            % (template, message, exc))
             except TypeError:
                 # Message is not a string
                 pass

gabbi/gabbits_intercept/backref.yaml

@@ -64,3 +64,12 @@ tests:
      location: $SCHEME://$NETLOC$RESPONSE['c']
      x-gabbi-url: $SCHEME://$NETLOC/v2
      content-type: $HEADERS['content-type']
+
+- name: backref json fail start
+  url:  /
+  method: POST
+  data: ''
+
+- name: backref json fail end
+  xfail: true
+  url: $RESPONSE['url']

gabbi/gabbits_intercept/self.yaml

@@ -96,6 +96,11 @@ tests:
   response_headers:
       x-gabbi-url: $SCHEME://$NETLOC/takingnames
 
+- name: confirm environ no key fail
+  desc: this confirms that no key leads to failure rather than error
+  xfail: true
+  url: /$ENVIRON['1385F1EB-DC5C-4A95-8928-58673FB272DC']
+
 - name: test pluggable response
   url: /foo?alpha=1
   response_test:
@@ -114,3 +119,13 @@ tests:
   url: /
   xfail: true
   method: DIE
+
+- name: non json response failure
+  desc: asking for json in a non json test should be failure not error
+  url: /
+  xfail: true
+  method: GET
+  request_headers:
+      accept: text/plain
+  response_json_paths:
+      $.data: hello

gabbi/handlers.py

@@ -100,6 +100,8 @@ class JSONResponseHandler(ResponseHandler):
         # to do their own processing.
         try:
             match = test.extract_json_path_value(test.json_data, path)
+        except AttributeError:
+            raise AssertionError('unable to extract JSON from test results')
         except ValueError:
             raise AssertionError('json path %s cannot match %s' %
                                  (path, test.json_data))

test-failskip.sh

@@ -2,7 +2,7 @@
 # Run the tests and confirm that the stuff we expect to skip or fail
 # does.
 
-GREP_FAIL_MATCH='expected failures=5'
+GREP_FAIL_MATCH='expected failures=8'
 GREP_SKIP_MATCH='skips=2'
 
 python setup.py testr && \