Handle special cases with msgpack and python3

We attempt to be clever about string types in the token formatters.
We do this because in some cases, not all items in a token payload
are serialized to byte strings. To add flexibility for this, we use
tuples with a boolean value that denotes if the accompanying value is
a byte string or not. This helps us safely re-inflate the value from
a byte string back to it's .hex representations, typically with UUID
strings.

With python3, we actually hit an interesting case where what we pass
into the token payload doesn't actually maintain that state due to the
usage of msgpack. The msgpack library returns byte strings even though
the initial value may not have been a byte string. This breaks the
logic we have for the associated boolean value because the string type
changes and the boolean does not.

This commit adds a couple of if/statements to detect if we running on
python3 and if the boolean mismatches the actual value type. Then, it
attempts to do the right thing by decoding the string.

We should think about how we want to do this, or if there is a better
way.

Change-Id: Iaecd45ef985cbf5ff4a6a724df96c1304a927247
Closes-Bug: 1813085

keystone/tests/unit/test_v3_federation.py

@@ -1912,6 +1912,57 @@ class FederatedTokenTests(test_v3.RestfulTestCase, FederatedSetupMixin):
         token_resp = render_token.render_token_response_from_model(r)['token']
         self.assertValidMappedUser(token_resp)
 
+    def test_default_domain_scoped_token(self):
+        # Make sure federated users can get tokens scoped to the default
+        # domain, which has a non-uuid ID by default (e.g., `default`). We want
+        # to make sure the token provider handles string types properly if the
+        # ID isn't compressed into byte format during validation. Turn off
+        # cache on issue so that we validate the token online right after we
+        # get it to make sure the token provider is called.
+        self.config_fixture.config(group='token', cache_on_issue=False)
+
+        # Grab an unscoped token to get a domain-scoped token with.
+        token = self._issue_unscoped_token()
+
+        # Give the user a direct role assignment on the default domain, so they
+        # can get a federated domain-scoped token.
+        PROVIDERS.assignment_api.create_grant(
+            self.role_admin['id'], user_id=token.user_id,
+            domain_id=CONF.identity.default_domain_id
+        )
+
+        # Get a token scoped to the default domain with an ID of `default`,
+        # which isn't a uuid type, but we should be able to handle it
+        # accordingly in the token formatters/providers.
+        auth_request = {
+            'auth': {
+                'identity': {
+                    'methods': [
+                        'token'
+                    ],
+                    'token': {
+                        'id': token.id
+                    }
+                },
+                'scope': {
+                    'domain': {
+                        'id': CONF.identity.default_domain_id
+                    }
+                }
+            }
+        }
+        r = self.v3_create_token(auth_request)
+        domain_scoped_token_id = r.headers.get('X-Subject-Token')
+
+        # Validate the token to make sure the token providers handle non-uuid
+        # domain IDs properly.
+        headers = {'X-Subject-Token': domain_scoped_token_id}
+        self.get(
+            '/auth/tokens',
+            token=domain_scoped_token_id,
+            headers=headers
+        )
+
     def test_issue_the_same_unscoped_token_with_user_deleted(self):
         r = self._issue_unscoped_token()
         token = render_token.render_token_response_from_model(r)['token']

keystone/token/token_formatters.py

@@ -611,6 +611,23 @@ class FederatedScopedPayload(FederatedUnscopedPayload):
         (is_stored_as_bytes, scope_id) = payload[2]
         if is_stored_as_bytes:
             scope_id = cls.convert_uuid_bytes_to_hex(scope_id)
+        else:
+            # NOTE(lbragstad): We assembled the token payload scope as a tuple
+            # (False, domain_id) for cases like (False, 'default'), since the
+            # default domain ID isn't converted to a byte string when it's not
+            # in UUID format. Despite the boolean indicator in the tuple that
+            # denotes if the value is stored as a byte string or not, msgpack
+            # apparently returns the serialized input as byte strings anyway.
+            # For example, this means what we though we were passing in as
+            # (False, 'default') during token creation actually comes out as
+            # (False, b'default') in token validation through msgpack, which
+            # clearly isn't correct according to our boolean indicator. This
+            # causes comparison issues due to different string types (e.g.,
+            # b'default' != 'default') with python 3. See bug 1813085 for
+            # details. We use this pattern for other strings in the payload
+            # like idp_id and protocol_id for the same reason.
+            if six.PY3 and isinstance(scope_id, six.binary_type):
+                scope_id = scope_id.decode('utf-8')
         project_id = (
             scope_id
             if cls.version == FederatedProjectScopedPayload.version else None)
@@ -621,7 +638,12 @@ class FederatedScopedPayload(FederatedUnscopedPayload):
         (is_stored_as_bytes, idp_id) = payload[4]
         if is_stored_as_bytes:
             idp_id = cls.convert_uuid_bytes_to_hex(idp_id)
+        else:
+            if six.PY3 and isinstance(idp_id, six.binary_type):
+                idp_id = idp_id.decode('utf-8')
         protocol_id = payload[5]
+        if six.PY3 and isinstance(protocol_id, six.binary_type):
+            protocol_id = protocol_id.decode('utf-8')
         expires_at_str = cls._convert_float_to_time_string(payload[6])
         audit_ids = list(map(cls.base64_encode, payload[7]))
         system = None

releasenotes/notes/bug-1813085-cf24b204e95fd7f5.yaml

@@ -0,0 +1,7 @@
+---
+fixes:
+  - |
+    [`bug 1813085 <https://bugs.launchpad.net/keystone/+bug/1813085>`]
+    Validation of federated domain-scoped tokens scoped to the ``default``
+    domain no longer results in an ``HTTP 404 Domain Not Found`` due
+    to byte string conversion issues with msgpack in python 3.