Fetch discovery documents with auth when needed

Some services, like Nova, default to requiring auth for their
versioned discovery documents. This means strict discovery
does not work on them, because discovery as it is now defaults
to not sending auth. Just changing the default would be a behavior
change resulting in sending unneeded data with *every* request.
Instead, respond to Unauthorized exceptions by retrying the request
with auth token. This way discovery will work for services that
are otherwise blocking unauthenticated access, and will get more
efficient over time as those services improve.

Depends-On: https://review.opendev.org/#/c/685999
Change-Id: I8a33e8a05bed0f18e4e42431f6d16b8a6a5270ef

keystoneauth1/discover.py

@@ -98,13 +98,23 @@ def get_version_data(session, url, authenticated=None):
     """
     headers = {'Accept': 'application/json'}
 
-    resp = session.get(url, headers=headers, authenticated=authenticated)
+    try:
+        resp = session.get(url, headers=headers, authenticated=authenticated)
+    except exceptions.Unauthorized:
+        resp = session.get(url, headers=headers, authenticated=True)
 
     try:
         body_resp = resp.json()
     except ValueError:
         pass
     else:
+        # Swift returns the list of containers for an account on an
+        # authenticated GET from /, not a version document. To our knowledge
+        # it's the only thing returning a [] here - and that's ok.
+        if isinstance(body_resp, list):
+            raise exceptions.DiscoveryFailure(
+                'Invalid Response - List returned instead of dict')
+
         # In the event of querying a root URL we will get back a list of
         # available versions.
         try:

keystoneauth1/tests/unit/test_discovery.py

@@ -1241,6 +1241,29 @@ class EndpointDataTests(utils.TestCase):
             [mock.call('sess', url, cache='cache', authenticated=False)
              for url in ('url1', 'url2', 'url3')])
 
+    def test_run_discovery_auth(self):
+        url = 'https://example.com'
+        headers = {'Accept': 'application/json'}
+
+        session = mock.Mock()
+        session.get.side_effect = [
+            exceptions.Unauthorized('unauthorized'),
+            # Throw a different exception the second time so that we can
+            # catch it in the test and verify the retry.
+            exceptions.BadRequest('bad request'),
+        ]
+
+        try:
+            discover.get_version_data(session, url)
+        except exceptions.BadRequest:
+            pass
+        # Only one call with 'url'
+        self.assertEqual(2, session.get.call_count)
+        session.get.assert_has_calls([
+            mock.call(url, headers=headers, authenticated=None),
+            mock.call(url, headers=headers, authenticated=True),
+        ])
+
     def test_endpoint_data_str(self):
         """Validate EndpointData.__str__."""
         # Populate a few fields to make sure they come through.

releasenotes/notes/retry-authenticated-discovery-19c4354ff983f507.yaml

@@ -0,0 +1,14 @@
+---
+fixes:
+  - |
+    Retry version discovery with auth token when the initial request
+    throws 401 Unauthorized. There are some services that are erroneously
+    defaulting to authenticated discovery, and this allows discovery
+    to work properly on them.
+upgrade:
+  - |
+    If keystoneauth and openstacksdk are both in use and keystoneauth
+    is upgraded to this release **before** upgrading openstacksdk to
+    ``0.36.1`` or later, creation of ServerGroup objects with policies
+    and use of Ansible Inventory could be adversely affected. See
+    https://review.opendev.org/#/c/685999/ for more details.