added new tests category between "pre" and "post: "mid". added new test cases

src/saml2test/check.py

@@ -213,6 +213,28 @@ class CheckSpHttpResponseOK(Error):
         return res
 
 
+class CheckSpHttpResponse500(Error):
+    """ Checks that the SP's HTTP response status is >= 500. This is useful
+        to check if the SP correctly flags errors such as an invalid signature
+    """
+    cid = "check-sp-http-response-500"
+    msg = "SP does not return a HTTP 5xx status when it shold do so."
+
+    def _func(self, conv):
+        _response = conv.last_response
+        _content = conv.last_response.content
+
+        res = {}
+        if _response.status_code < 500:
+            self._status = self.status
+            self._message = self.msg
+            #res["content"] = _content   #too big + charset converstion needed
+            res["url"] = conv.position
+            res["http_status"] = _response.status_code
+
+        return res
+
+
 class MissingRedirect(CriticalError):
     """ At this point in the flow a redirect back to the client was expected.
     """

src/sp_test/base.py

@@ -282,13 +282,6 @@ class Conversation():
         :param resp_flow: The flow to prepare the response
         :return: The SP's HTTP response on receiving the SAML response
         """
-        # make sure I got the request I expected
-        assert isinstance(self.saml_request.message, req._class)
-
-        try:
-            self.test_sequence(req.tests["post"])
-        except KeyError:
-            pass
 
         # Pick information from the request that should be in the response
         args = self.instance.response_args(self.saml_request.message,
@@ -381,7 +374,7 @@ class Conversation():
 
         self._log_response(self.last_response)
 
-    def do_flow(self, flow):
+    def do_flow(self, flow, mid_tests):
         """
         Solicited or 'un-solicited' flows.
 
@@ -392,6 +385,12 @@ class Conversation():
             self.wb_send_GET_startpage()
             self.intermit(flow[0]._interaction)
             self.parse_saml_message()
+        # make sure I got the request I expected
+        assert isinstance(self.saml_request.message, flow[1]._class)
+        try:
+            self.test_sequence(mid_tests)
+        except KeyError:
+            pass
         self.send_idp_response(flow[1], flow[2])
         if len(flow) == 4:
             self.handle_result(flow[3])
@@ -399,6 +398,7 @@ class Conversation():
             self.handle_result()
 
     def do_sequence_and_tests(self, oper, tests=None):
+        self.current_oper = oper
         try:
             self.test_sequence(tests["pre"])
         except KeyError:
@@ -406,7 +406,7 @@ class Conversation():
 
         for flow in oper:
             try:
-                self.do_flow(flow)
+                self.do_flow(flow, tests["mid"])
             except InteractionNeeded:
                 self.test_output.append({"status": INTERACTION,
                                          "message": "see detail log for response content",

src/sp_test/check.py

@@ -3,7 +3,7 @@ import logging
 import re
 import sys
 
-from saml2 import BINDING_HTTP_REDIRECT
+from saml2 import BINDING_HTTP_POST, BINDING_HTTP_REDIRECT
 from saml2test.check import Check
 from saml2test.check import ERROR, INFORMATION, WARNING
 from saml2test import check
@@ -86,6 +86,15 @@ class VerifyDigestAlgorithm(Check):
         if request.signature:
             if not self._digest_algo(request.signature, _algs):
                 return {}
+        elif conv._binding == BINDING_HTTP_REDIRECT:
+            self._message = "no digest with redirect binding"
+            self._status = INFORMATION
+            return {}
+        elif conv._binding == BINDING_HTTP_POST:
+            self._message = "cannot verify digest algorithm: request not signed"
+            self._status = WARNING
+            return {}
+
 
         return {}
 
@@ -204,6 +213,29 @@ class VerifyEchopageContents(Check):
         return False
 
 
+class SetResponseAndAssertionSignaturesFalse(Check):
+    """ Prepare config to suppress signatures of both response and assertion"""
+    cid = "set-response-and-assertion-signature-false"
+    msg = "Prepare config to suppress signatures of both response and assertion"
+
+    def _func(self, conv):
+        conv.json_config['args']['AuthnResponse']['sign_assertion'] = 'never'
+        conv.json_config['args']['AuthnResponse']['sign_response'] = 'never'
+        self._status = INFORMATION
+        return {}
+
+
+#class SetInvalidIdpKey(Check):
+#    """ Prepare config to set IDP signing key to some useless key"""
+#    cid = "set-idp-key-invalid"
+#    msg = "Prepare config to set IDP signing key invalid"
+#
+#    def _func(self, conv):
+#        conv.instance.sec.cert_file = conv.instance.config.invalid_idp_cert_file
+#        conv.instance.sec.key_file = conv.instance.config.invalid_idp_key_file
+#        return {}
+
+
 # =============================================================================
 
 

src/sp_test/tests.py

@@ -11,9 +11,11 @@ from saml2.saml import SCM_SENDER_VOUCHES
 from saml2.saml import ConditionAbstractType_
 from saml2.samlp import STATUS_AUTHN_FAILED
 from saml2.time_util import in_a_while, a_while_ago
-from sp_test.check import VerifyAuthnRequest, VerifyDigestAlgorithm, \
-    VerifySignatureAlgorithm, VerifyIfRequestIsSigned
 from sp_test import check
+from sp_test.check import VerifyAuthnRequest, VerifyDigestAlgorithm
+from sp_test.check import VerifySignatureAlgorithm, VerifyIfRequestIsSigned
+from sp_test.check import SetResponseAndAssertionSignaturesFalse
+from saml2test.check import CheckSpHttpResponseOK, CheckSpHttpResponse500
 from saml2test import ip_addresses
 
 __author__ = 'rolandh'
@@ -83,9 +85,8 @@ class Request(object):
     response = ""
     _class = None
     tests = {"pre": [],
-             "post": [VerifyAuthnRequest,
+             "mid": [VerifyAuthnRequest],
-                      VerifyDigestAlgorithm,
+             "post": []}
-                      VerifySignatureAlgorithm,]}
 
     def __init__(self):
         pass
@@ -171,13 +172,19 @@ class AuthnResponse_without_SubjectConfirmationData_2(AuthnResponse):
 
 class AuthnResponse_rnd_Response_inresponseto(AuthnResponse):
     def pre_processing(self, message, **kwargs):
-        message.in_response_to = rndstr(16)
+        message.in_response_to = "invalid_rand_" + rndstr(6)
         return message
 
 
 class AuthnResponse_rnd_Response_assertion_inresponseto(AuthnResponse):
     def pre_processing(self, message, **kwargs):
-        message.assertion.in_response_to = rndstr(16)
+        message.assertion.in_response_to = "invalid_rand_" + rndstr(6)
+        return message
+
+
+class AuthnResponse_Response_no_inresponse(AuthnResponse):
+    def pre_processing(self, message, **kwargs):
+        message.in_response_to = None
         return message
 
 
@@ -367,25 +374,60 @@ PHASES = {
     "login_redirect": (Login, AuthnRequest, AuthnResponse_redirect),
 }
 
+# Each operation defines 4 flows and 3 sets of tests, in chronological order:
+# test "pre": executes before anything is sent to the SP
+# flow 0: Start conversation flow
+# flow 1: SAML request flow
+# test "mid": executes after receiving the SAML request
+# flow 2: SAML response flow
+# flow 3: check SP response after authentication
+# test "post": executes after finals response has been received from SP
+
 OPERATIONS = {
     'sp-00': {
-        "name": 'Basic Login test',
+        "name": 'Basic Login test expect HTTP 200 result',
-        "descr": 'GET startpage from SP, verify authentication request, verify '
+        "descr": 'WebSSO verify authentication request, verify '
                  'HTTP-Response after sending the SAML response',
-        "sequence": [(Login, AuthnRequest, AuthnResponse, None)],
+        "sequence": [(Login, AuthnRequest, AuthnResponse, CheckSpHttpResponseOK)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'sp-01': {
-        "name": 'Login & echo page verification test',
+        "name": 'Login OK & echo page verification test',
         "descr": 'Same as SP-00, then check if result page is displayed',
         "sequence": [(Login, AuthnRequest, AuthnResponse, check.VerifyEchopageContents)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'sp-02': {
         "name": 'Require AuthnRequest to be signed',
         "descr": 'Same as SP-00, and check if a request signature can be found',
         "sequence": [(Login, AuthnRequest, AuthnResponse, None)],
-        "tests": {"pre": [], "post": [VerifyIfRequestIsSigned]}
+        "tests": {"pre": [], "mid": [VerifyIfRequestIsSigned], "post": []}
+    },
+    'sp-03': {
+        "name": 'Reject unsigned reponse/assertion',
+        "descr": 'Check if SP flags missing signature with HTTP 500',
+        "sequence": [(Login, AuthnRequest, AuthnResponse, CheckSpHttpResponse500)],
+        "tests": {"pre": [SetResponseAndAssertionSignaturesFalse], "mid": [], "post": []}
+    },
+    'sp-04': {  # test-case specific code in sp_test/__init__
+        "name": 'Reject siganture with invalid IDP key',
+        "descr": 'IDP-key for otherwise valid signature not in metadata - expect HTTP 500 result',
+        "sequence": [(Login, AuthnRequest, AuthnResponse, CheckSpHttpResponse500)],
+        "tests": {"pre": [], "mid": [], "post": []}
+    },
+    'sp-05': {
+        "name": 'Verify digest algorithm',
+        "descr": 'Trigger WebSSO AuthnRequest and verify that the used '
+                 'digest algorithm was one from the approved set.',
+        "sequence": [(Login, AuthnRequest, AuthnResponse, None)],
+        "tests": {"pre": [], "mid": [VerifyDigestAlgorithm], "post": []}
+    },
+    'sp-06': {
+        "name": 'Verify signature algorithm',
+        "descr": 'Trigger WebSSO AuthnRequest and verify that the used '
+                 'signature algorithm was one from the approved set.',
+        "sequence": [(Login, AuthnRequest, AuthnResponse, None)],
+        "tests": {"pre": [], "mid": [VerifySignatureAlgorithm], "post": []}
     },
     'sp-08': {
         "name": "SP should accept a Response without a "
@@ -394,37 +436,37 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_without_SubjectConfirmationData_2,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL02': {
         "name": 'Verify various aspects of the generated AuthnRequest message',
         "descr": 'Basic Login test',
         "sequence": [],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL03': {
         "name": "SP should not accept a Response as valid, when the StatusCode"
                 " is not success",
         "sequence": [(Login, AuthnRequest, ErrorResponse, check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL04': {
         "name": "SP should accept a NameID with Format: persistent",
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_NameIDformat_persistent, None)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL05': {
         "name": "SP should accept a NameID with Format: e-mail",
         "sequence": [(Login, AuthnRequest, AuthnResponse_NameIDformat_email,
                       None)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL06': {
         "name": "Do SP work with unknown NameID Format, such as : foo",
         "sequence": [(Login, AuthnRequest, AuthnResponse_NameIDformat_foo,
                       None)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL07': {
         "name": "SP should accept a Response without a "
@@ -432,7 +474,7 @@ OPERATIONS = {
                  "is SCM_SENDER_VOUCHES",
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_without_SubjectConfirmationData_1, None)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL09': {
         "name": "SP should not accept a response InResponseTo "
@@ -440,7 +482,7 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_rnd_Response_inresponseto,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL10': {
         "name": "SP should not accept an assertion InResponseTo "
@@ -448,22 +490,31 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_rnd_Response_assertion_inresponseto,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
+    },
+    'FL11': {
+        "name": "Does the SP allow the InResponseTo attribute to be missing"
+                "from the Response element?",
+        "sequence": [(Login, AuthnRequest,
+                      AuthnResponse_Response_no_inresponse,
+                      check.ErrorResponse)],
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL12': {
-        "name": "Do the SP allow the InResponseTo attribute to be missing"
+        "name": "Does the SP allow the InResponseTo attribute to be missing"
-                "from the SubjectConfirmationData element?",
+                "from the SubjectConfirmationData element?"
+                "(Test is questionable - review)",  # TODO
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_SubjectConfirmationData_no_inresponse,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL13': {
         "name": "SP should not accept a broken DestinationURL attribute",
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_broken_destination,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     # New untested
     'FL14a': {
@@ -471,14 +522,14 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_broken_destination,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL14b': {
         "name": "SP should not accept missing Recipient attribute",
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_missing_Recipient,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL20': {
         "name": "Accept a Response with a SubjectConfirmationData elements "
@@ -486,7 +537,7 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_correct_recipient_address,
                       None)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL21': {
         "name": "Accept a Response with a SubjectConfirmationData elements "
@@ -494,7 +545,7 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_incorrect_recipient_address,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL22': {
         "name": "Accept a Response with two SubjectConfirmationData elements"
@@ -502,7 +553,7 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_2_recipients_me_last,
                       None)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL23': {
         "name": "Accept a Response with two SubjectConfirmationData elements"
@@ -510,14 +561,14 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_2_recipients_me_first,
                       None)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL26': {
         "name": "Reject an assertion containing an unknown Condition.",
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_unknown_condition,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL27': {
         "name": "Reject a Response with a Condition with a NotBefore in the "
@@ -525,7 +576,7 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_future_NotBefore,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL28': {
         "name": "Reject a Response with a Condition with a NotOnOrAfter in "
@@ -533,7 +584,7 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_future_NotBefore,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL29': {
         "name": "Reject a Response with a SubjectConfirmationData@NotOnOrAfter "
@@ -541,7 +592,7 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_past_SubjectConfirmationData_NotOnOrAfter,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL24': {
         "name": "Reject a Response with a SubjectConfirmationData@NotBefore "
@@ -549,7 +600,7 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_future_SubjectConfirmationData_NotBefore,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL30': {
         "name": "Reject a Response with an AuthnStatement where "
@@ -557,28 +608,28 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_past_AuthnStatement_SessionNotOnOrAfter,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL31': {
         "name": "Reject a Response with an AuthnStatement missing",
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_missing_AuthnStatement,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL32': {
         "name": "Reject an IssueInstant far (24 hours) into the future",
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_missing_AuthnStatement,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL33': {
         "name": "Reject an IssueInstant far (24 hours) into the past",
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_missing_AuthnStatement,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL34': {
         "name": "Accept xs:datetime with millisecond precision "
@@ -586,7 +637,7 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_datetime_millisecond,
                       None)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL36': {
         "name": "Reject a Response with a Condition with a empty set of "
@@ -594,14 +645,14 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_AudienceRestriction_no_audience,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL37': {
         "name": "Reject a Response with a Condition with a wrong Audience.",
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_AudienceRestriction_wrong_audience,
                       check.ErrorResponse)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL38': {
         "name": "Accept a Response with a Condition with an additional "
@@ -609,7 +660,7 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_AudienceRestriction_prepended_audience,
                       None)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
     'FL39': {
         "name": "Accept a Response with a Condition with an additional "
@@ -617,7 +668,7 @@ OPERATIONS = {
         "sequence": [(Login, AuthnRequest,
                       AuthnResponse_AudienceRestriction_appended_audience,
                       None)],
-        "tests": {"pre": [], "post": []}
+        "tests": {"pre": [], "mid": [], "post": []}
     },
 }