From 8ec445b13d329e6d347ff7a0a69a529f589ba649 Mon Sep 17 00:00:00 2001
From: Colleen Murphy <colleen.murphy@suse.de>
Date: Thu, 17 Oct 2019 10:02:31 -0700
Subject: [PATCH] Add option to disable testing against external idp

Currently, the federation tests are non-voting because they require
connecting to an external service that is not under our control, and is
therefore unreliable. Non-voting tests are a problem because they are
often ignored even when their results are related to new changes. This
change adds a tempest config option
``[identity-feature-enabled]/external_idp``, defaulting to true for
backwards compatibility, which when disabled causes the tests that rely
on the external IdP to be disabled leaving only the K2K federation tests
to be executed. Exercising only the K2K tests is still a good means of
regression testing and we can safely make those tests voting.

Change-Id: I534470df7ca529511ab9a7631f167ec2035ab4be
---
 keystone_tempest_plugin/config.py             |  5 +++
 .../scenario/test_federated_authentication.py | 36 +++++++++++++++----
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/keystone_tempest_plugin/config.py b/keystone_tempest_plugin/config.py
index 87872c5..d3e3c02 100644
--- a/keystone_tempest_plugin/config.py
+++ b/keystone_tempest_plugin/config.py
@@ -20,6 +20,11 @@ identity_feature_option = [
                 default=False,
                 help='Does the environment support the Federated Identity '
                      'feature?'),
+    cfg.BoolOpt('external_idp',
+                default=True,
+                help='Whether to test federated scenarios against an external '
+                     'identity provider. If disabled, only '
+                     'Keystone-to-Keystone tests will be enabled.'),
 ]
 
 fed_scenario_group = cfg.OptGroup(name='fed_scenario',
diff --git a/keystone_tempest_plugin/tests/scenario/test_federated_authentication.py b/keystone_tempest_plugin/tests/scenario/test_federated_authentication.py
index 89af4ce..e930205 100644
--- a/keystone_tempest_plugin/tests/scenario/test_federated_authentication.py
+++ b/keystone_tempest_plugin/tests/scenario/test_federated_authentication.py
@@ -183,14 +183,10 @@ class TestSaml2EcpFederatedAuthentication(base.BaseIdentityTest):
 
         return resp
 
-    @testtools.skipUnless(CONF.identity_feature_enabled.federation,
-                          "Federated Identity feature not enabled")
-    def test_request_unscoped_token(self):
+    def _test_request_unscoped_token(self):
         self._request_unscoped_token()
 
-    @testtools.skipUnless(CONF.identity_feature_enabled.federation,
-                          "Federated Identity feature not enabled")
-    def test_request_scoped_token(self):
+    def _test_request_scoped_token(self):
         resp = self._request_unscoped_token()
         token_id = resp.headers['X-Subject-Token']
 
@@ -203,6 +199,24 @@ class TestSaml2EcpFederatedAuthentication(base.BaseIdentityTest):
             project_id=projects[0]['id'], token=token_id)
 
 
+class TestSaml2FederatedExternalAuthentication(
+    TestSaml2EcpFederatedAuthentication):
+
+    @testtools.skipUnless(CONF.identity_feature_enabled.federation,
+                          "Federated Identity feature not enabled")
+    @testtools.skipUnless(CONF.identity_feature_enabled.external_idp,
+                          "External identity provider is not available")
+    def test_request_unscoped_token(self):
+        self._test_request_unscoped_token()
+
+    @testtools.skipUnless(CONF.identity_feature_enabled.federation,
+                          "Federated Identity feature not enabled")
+    @testtools.skipUnless(CONF.identity_feature_enabled.external_idp,
+                          "External identity provider is not available")
+    def test_request_scoped_token(self):
+        self._test_request_scoped_token()
+
+
 class TestK2KFederatedAuthentication(TestSaml2EcpFederatedAuthentication):
 
     def setUp(self):
@@ -253,3 +267,13 @@ class TestK2KFederatedAuthentication(TestSaml2EcpFederatedAuthentication):
         self.auth_client.expected_success(200, resp.status)
 
         return etree.XML(saml), self.sp_url
+
+    @testtools.skipUnless(CONF.identity_feature_enabled.federation,
+                          "Federated Identity feature not enabled")
+    def test_request_unscoped_token(self):
+        self._test_request_unscoped_token()
+
+    @testtools.skipUnless(CONF.identity_feature_enabled.federation,
+                          "Federated Identity feature not enabled")
+    def test_request_scoped_token(self):
+        self._test_request_scoped_token()