s3token: Pass service auth token to Keystone

Recent versions of Keystone require auth tokens when accessing the
/v3/s3tokens endpoint to prevent exposure of a lot of information that
a user who just has a presigned URL should not be able to see.

UpgradeImpact
=============
The s3token middleware now requires Keystone auth credentials to be
configured. If secret_cache_duration is enabled, these credentials
should already be configured. Without these credentials, Keystone users
will no longer be able to make S3 API requests.

Closes-Bug: #2119646
Change-Id: Ie80bc33d0d9de17ca6eaad3b43628724538001f6
Signed-off-by: Tim Burke <tim.burke@gmail.com>

etc/proxy-server.conf-sample

@@ -785,17 +785,6 @@ auth_uri = http://keystonehost:5000/v3
 # Connect/read timeout (in seconds) to use when communicating with Keystone
 http_timeout = 10.0
 
-# Number of seconds to cache the S3 secret. By setting this to a positive
-# number, the S3 authorization validation checks can happen locally.
-# secret_cache_duration = 0
-
-# If S3 secret caching is enabled, Keystone auth credentials to be used to
-# validate S3 authorization must be provided here. The appropriate options
-# are the same as used in the authtoken middleware above. The values are
-# likely the same as used in the authtoken middleware.
-# Note that the Keystone auth credentials used by s3token will need to be
-# able to view all project credentials too.
-
 # SSL-related options
 # insecure = False
 # certfile =
@@ -804,12 +793,10 @@ http_timeout = 10.0
 # You can override the default log routing for this filter here:
 # log_name = s3token
 
-# Secrets may be cached to reduce latency for the client and load on Keystone.
+# Recent Keystone deployments require credentials similar to the authtoken
-# Set this to some number of seconds greater than zero to enable caching.
+# middleware; these credentials require access to the s3tokens endpoint.
-# secret_cache_duration = 0
+# Additionally, if secret caching is enabled, the credentials should have
-
+# access to view all project credentials.
-# Secret caching requires Keystone credentials similar to the authtoken middleware;
-# these credentials require access to view all project credentials.
 # auth_url = http://keystonehost:5000
 # auth_type = password
 # project_domain_id = default
@@ -818,6 +805,11 @@ http_timeout = 10.0
 # username = swift
 # password = password
 
+# Secrets may be cached to reduce latency for the client and load on Keystone.
+# Set this to some number of seconds greater than zero to enable caching and
+# allow some S3 authorization validation checks to happen entirely in the proxy.
+# secret_cache_duration = 0
+
 [filter:healthcheck]
 use = egg:swift#healthcheck
 # An optional filesystem path, which if present, will cause the healthcheck

swift/common/middleware/s3api/s3token.py

@@ -180,31 +180,45 @@ class S3Token(object):
         self._secret_cache_duration = int(conf.get('secret_cache_duration', 0))
         if self._secret_cache_duration < 0:
             raise ValueError('secret_cache_duration must be non-negative')
-        if self._secret_cache_duration:
-            try:
-                auth_plugin = keystone_loading.get_plugin_loader(
-                    conf.get('auth_type', 'password'))
-                available_auth_options = auth_plugin.get_options()
-                auth_options = {}
-                for option in available_auth_options:
-                    name = option.name.replace('-', '_')
-                    value = conf.get(name)
-                    if value:
-                        auth_options[name] = value
 
+        # Service authentication for s3tokens API calls
+        self.keystoneclient = None
+        try:
+            auth_plugin = keystone_loading.get_plugin_loader(
+                conf.get('auth_type', 'password'))
+            available_auth_options = auth_plugin.get_options()
+            auth_options = {}
+            for option in available_auth_options:
+                name = option.name.replace('-', '_')
+                value = conf.get(name)
+                if value:
+                    auth_options[name] = value
+
+            if not auth_options:
+                self._logger.warning(
+                    "No service auth configuration. "
+                    "s3tokens API calls will be unauthenticated. "
+                    "New versions of keystone require service auth.")
+            else:
                 auth = auth_plugin.load_from_options(**auth_options)
                 session = keystone_session.Session(auth=auth)
                 self.keystoneclient = keystone_client.Client(
                     session=session,
                     region_name=conf.get('region_name'))
-                self._logger.info("Caching s3tokens for %s seconds",
+                self._logger.info(
-                                  self._secret_cache_duration)
+                    "Service authentication configured for s3tokens API")
-            except Exception:
+        except Exception:
-                self._logger.warning("Unable to load keystone auth_plugin. "
+            self._logger.warning(
-                                     "Secret caching will be unavailable.",
+                "Unable to load service auth configuration. "
-                                     exc_info=True)
+                "s3tokens API calls will be unauthenticated "
-                self.keystoneclient = None
+                "and secret caching will be unavailable.",
-                self._secret_cache_duration = 0
+                exc_info=True)
+
+        if self._secret_cache_duration and self.keystoneclient:
+            self._logger.info("Caching s3tokens for %s seconds",
+                              self._secret_cache_duration)
+        else:
+            self._secret_cache_duration = 0
 
     def _deny_request(self, code):
         error_cls, message = {
@@ -222,6 +236,16 @@ class S3Token(object):
 
     def _json_request(self, creds_json):
         headers = {'Content-Type': 'application/json'}
+
+        # Add service authentication headers if configured
+        if self.keystoneclient:
+            try:
+                headers.update(
+                    self.keystoneclient.session.get_auth_headers())
+            except Exception:
+                self._logger.warning("Failed to get service token",
+                                     exc_info=True)
+
         try:
             response = requests.post(self._request_uri,
                                      headers=headers, data=creds_json,

test/unit/common/middleware/s3api/test_s3token.py

@@ -589,6 +589,9 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
         cache.get.return_value = None
 
         keystone_client = MOCK_KEYSTONE.return_value
+        keystone_client.session.get_auth_headers.return_value = {
+            'X-Auth-Token': 'bearer token',
+        }
         keystone_client.ec2.get.return_value = mock.Mock(secret='secret')
 
         MOCK_REQUEST.return_value = FakeResponse({
@@ -615,6 +618,18 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
         }
 
         self.assertTrue(MOCK_REQUEST.called)
+        self.assertEqual(MOCK_REQUEST.mock_calls, [
+            mock.call('http://example.com/s3tokens', headers={
+                'Content-Type': 'application/json',
+                'X-Auth-Token': 'bearer token',
+            }, data=json.dumps({
+                "credentials": {
+                    "access": "access",
+                    "token": "dG9rZW4=",
+                    "signature": "signature",
+                }
+            }), verify=None, timeout=10.0)
+        ])
         tenant = GOOD_RESPONSE_V2['access']['token']['tenant']
         expected_cache = (expected_headers, tenant, 'secret')
         cache.set.assert_called_once_with('s3secret/access', expected_cache,