OpenStack Storage (Swift)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1565 lines
59KB

  1. # Copyright (c) 2014 OpenStack Foundation.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12. # implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. import base64
  16. from collections import defaultdict, OrderedDict
  17. from email.header import Header
  18. from hashlib import sha1, sha256, md5
  19. import hmac
  20. import re
  21. import six
  22. # pylint: disable-msg=import-error
  23. from six.moves.urllib.parse import quote, unquote, parse_qsl
  24. import string
  25. from swift.common.utils import split_path, json, get_swift_info, \
  26. close_if_possible
  27. from swift.common import swob
  28. from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \
  29. HTTP_NO_CONTENT, HTTP_UNAUTHORIZED, HTTP_FORBIDDEN, HTTP_NOT_FOUND, \
  30. HTTP_CONFLICT, HTTP_UNPROCESSABLE_ENTITY, HTTP_REQUEST_ENTITY_TOO_LARGE, \
  31. HTTP_PARTIAL_CONTENT, HTTP_NOT_MODIFIED, HTTP_PRECONDITION_FAILED, \
  32. HTTP_REQUESTED_RANGE_NOT_SATISFIABLE, HTTP_LENGTH_REQUIRED, \
  33. HTTP_BAD_REQUEST, HTTP_REQUEST_TIMEOUT, HTTP_SERVICE_UNAVAILABLE, \
  34. HTTP_TOO_MANY_REQUESTS, HTTP_RATE_LIMITED, is_success
  35. from swift.common.constraints import check_utf8
  36. from swift.proxy.controllers.base import get_container_info, \
  37. headers_to_container_info
  38. from swift.common.request_helpers import check_path_header
  39. from swift.common.middleware.s3api.controllers import ServiceController, \
  40. ObjectController, AclController, MultiObjectDeleteController, \
  41. LocationController, LoggingStatusController, PartController, \
  42. UploadController, UploadsController, VersioningController, \
  43. UnsupportedController, S3AclController, BucketController
  44. from swift.common.middleware.s3api.s3response import AccessDenied, \
  45. InvalidArgument, InvalidDigest, BucketAlreadyOwnedByYou, \
  46. RequestTimeTooSkewed, S3Response, SignatureDoesNotMatch, \
  47. BucketAlreadyExists, BucketNotEmpty, EntityTooLarge, \
  48. InternalError, NoSuchBucket, NoSuchKey, PreconditionFailed, InvalidRange, \
  49. MissingContentLength, InvalidStorageClass, S3NotImplemented, InvalidURI, \
  50. MalformedXML, InvalidRequest, RequestTimeout, InvalidBucketName, \
  51. BadDigest, AuthorizationHeaderMalformed, SlowDown, \
  52. AuthorizationQueryParametersError, ServiceUnavailable
  53. from swift.common.middleware.s3api.exception import NotS3Request, \
  54. BadSwiftRequest
  55. from swift.common.middleware.s3api.utils import utf8encode, \
  56. S3Timestamp, mktime, MULTIUPLOAD_SUFFIX
  57. from swift.common.middleware.s3api.subresource import decode_acl, encode_acl
  58. from swift.common.middleware.s3api.utils import sysmeta_header, \
  59. validate_bucket_name
  60. from swift.common.middleware.s3api.acl_utils import handle_acl_header
  61. # List of sub-resources that must be maintained as part of the HMAC
  62. # signature string.
  63. ALLOWED_SUB_RESOURCES = sorted([
  64. 'acl', 'delete', 'lifecycle', 'location', 'logging', 'notification',
  65. 'partNumber', 'policy', 'requestPayment', 'torrent', 'uploads', 'uploadId',
  66. 'versionId', 'versioning', 'versions', 'website',
  67. 'response-cache-control', 'response-content-disposition',
  68. 'response-content-encoding', 'response-content-language',
  69. 'response-content-type', 'response-expires', 'cors', 'tagging', 'restore'
  70. ])
  71. MAX_32BIT_INT = 2147483647
  72. SIGV2_TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%S'
  73. SIGV4_X_AMZ_DATE_FORMAT = '%Y%m%dT%H%M%SZ'
  74. SERVICE = 's3' # useful for mocking out in tests
  75. def _header_strip(value):
  76. # S3 seems to strip *all* control characters
  77. if value is None:
  78. return None
  79. stripped = _header_strip.re.sub('', value)
  80. if value and not stripped:
  81. # If there's nothing left after stripping,
  82. # behave as though it wasn't provided
  83. return None
  84. return stripped
  85. _header_strip.re = re.compile('^[\x00-\x20]*|[\x00-\x20]*$')
  86. def _header_acl_property(resource):
  87. """
  88. Set and retrieve the acl in self.headers
  89. """
  90. def getter(self):
  91. return getattr(self, '_%s' % resource)
  92. def setter(self, value):
  93. self.headers.update(encode_acl(resource, value))
  94. setattr(self, '_%s' % resource, value)
  95. def deleter(self):
  96. self.headers[sysmeta_header(resource, 'acl')] = ''
  97. return property(getter, setter, deleter,
  98. doc='Get and set the %s acl property' % resource)
  99. class HashingInput(object):
  100. """
  101. wsgi.input wrapper to verify the hash of the input as it's read.
  102. """
  103. def __init__(self, reader, content_length, hasher, expected_hex_hash):
  104. self._input = reader
  105. self._to_read = content_length
  106. self._hasher = hasher()
  107. self._expected = expected_hex_hash
  108. def read(self, size=None):
  109. chunk = self._input.read(size)
  110. self._hasher.update(chunk)
  111. self._to_read -= len(chunk)
  112. if self._to_read < 0 or (size > len(chunk) and self._to_read) or (
  113. self._to_read == 0 and
  114. self._hasher.hexdigest() != self._expected):
  115. self.close()
  116. # Since we don't return the last chunk, the PUT never completes
  117. raise swob.HTTPUnprocessableEntity(
  118. 'The X-Amz-Content-SHA56 you specified did not match '
  119. 'what we received.')
  120. return chunk
  121. def close(self):
  122. close_if_possible(self._input)
  123. class SigV4Mixin(object):
  124. """
  125. A request class mixin to provide S3 signature v4 functionality
  126. """
  127. def check_signature(self, secret):
  128. secret = utf8encode(secret)
  129. user_signature = self.signature
  130. derived_secret = 'AWS4' + secret
  131. for scope_piece in self.scope.values():
  132. derived_secret = hmac.new(
  133. derived_secret, scope_piece, sha256).digest()
  134. valid_signature = hmac.new(
  135. derived_secret, self.string_to_sign, sha256).hexdigest()
  136. return user_signature == valid_signature
  137. @property
  138. def _is_query_auth(self):
  139. return 'X-Amz-Credential' in self.params
  140. @property
  141. def timestamp(self):
  142. """
  143. Return timestamp string according to the auth type
  144. The difference from v2 is v4 have to see 'X-Amz-Date' even though
  145. it's query auth type.
  146. """
  147. if not self._timestamp:
  148. try:
  149. if self._is_query_auth and 'X-Amz-Date' in self.params:
  150. # NOTE(andrey-mp): Date in Signature V4 has different
  151. # format
  152. timestamp = mktime(
  153. self.params['X-Amz-Date'], SIGV4_X_AMZ_DATE_FORMAT)
  154. else:
  155. if self.headers.get('X-Amz-Date'):
  156. timestamp = mktime(
  157. self.headers.get('X-Amz-Date'),
  158. SIGV4_X_AMZ_DATE_FORMAT)
  159. else:
  160. timestamp = mktime(self.headers.get('Date'))
  161. except (ValueError, TypeError):
  162. raise AccessDenied('AWS authentication requires a valid Date '
  163. 'or x-amz-date header')
  164. if timestamp < 0:
  165. raise AccessDenied('AWS authentication requires a valid Date '
  166. 'or x-amz-date header')
  167. try:
  168. self._timestamp = S3Timestamp(timestamp)
  169. except ValueError:
  170. # Must be far-future; blame clock skew
  171. raise RequestTimeTooSkewed()
  172. return self._timestamp
  173. def _validate_expire_param(self):
  174. """
  175. Validate X-Amz-Expires in query parameter
  176. :raises: AccessDenied
  177. :raises: AuthorizationQueryParametersError
  178. :raises: AccessDenined
  179. """
  180. err = None
  181. try:
  182. expires = int(self.params['X-Amz-Expires'])
  183. except KeyError:
  184. raise AccessDenied()
  185. except ValueError:
  186. err = 'X-Amz-Expires should be a number'
  187. else:
  188. if expires < 0:
  189. err = 'X-Amz-Expires must be non-negative'
  190. elif expires >= 2 ** 63:
  191. err = 'X-Amz-Expires should be a number'
  192. elif expires > 604800:
  193. err = ('X-Amz-Expires must be less than a week (in seconds); '
  194. 'that is, the given X-Amz-Expires must be less than '
  195. '604800 seconds')
  196. if err:
  197. raise AuthorizationQueryParametersError(err)
  198. if int(self.timestamp) + expires < S3Timestamp.now():
  199. raise AccessDenied('Request has expired')
  200. def _parse_credential(self, credential_string):
  201. parts = credential_string.split("/")
  202. # credential must be in following format:
  203. # <access-key-id>/<date>/<AWS-region>/<AWS-service>/aws4_request
  204. if not parts[0] or len(parts) != 5:
  205. raise AccessDenied()
  206. return dict(zip(['access', 'date', 'region', 'service', 'terminal'],
  207. parts))
  208. def _parse_query_authentication(self):
  209. """
  210. Parse v4 query authentication
  211. - version 4:
  212. 'X-Amz-Credential' and 'X-Amz-Signature' should be in param
  213. :raises: AccessDenied
  214. :raises: AuthorizationHeaderMalformed
  215. """
  216. if self.params.get('X-Amz-Algorithm') != 'AWS4-HMAC-SHA256':
  217. raise InvalidArgument('X-Amz-Algorithm',
  218. self.params.get('X-Amz-Algorithm'))
  219. try:
  220. cred_param = self._parse_credential(
  221. self.params['X-Amz-Credential'])
  222. sig = self.params['X-Amz-Signature']
  223. if not sig:
  224. raise AccessDenied()
  225. except KeyError:
  226. raise AccessDenied()
  227. try:
  228. signed_headers = self.params['X-Amz-SignedHeaders']
  229. except KeyError:
  230. # TODO: make sure if is it malformed request?
  231. raise AuthorizationHeaderMalformed()
  232. self._signed_headers = set(signed_headers.split(';'))
  233. invalid_messages = {
  234. 'date': 'Invalid credential date "%s". This date is not the same '
  235. 'as X-Amz-Date: "%s".',
  236. 'region': "Error parsing the X-Amz-Credential parameter; "
  237. "the region '%s' is wrong; expecting '%s'",
  238. 'service': 'Error parsing the X-Amz-Credential parameter; '
  239. 'incorrect service "%s". This endpoint belongs to "%s".',
  240. 'terminal': 'Error parsing the X-Amz-Credential parameter; '
  241. 'incorrect terminal "%s". This endpoint uses "%s".',
  242. }
  243. for key in ('date', 'region', 'service', 'terminal'):
  244. if cred_param[key] != self.scope[key]:
  245. kwargs = {}
  246. if key == 'region':
  247. kwargs = {'region': self.scope['region']}
  248. raise AuthorizationQueryParametersError(
  249. invalid_messages[key] % (cred_param[key], self.scope[key]),
  250. **kwargs)
  251. return cred_param['access'], sig
  252. def _parse_header_authentication(self):
  253. """
  254. Parse v4 header authentication
  255. - version 4:
  256. 'X-Amz-Credential' and 'X-Amz-Signature' should be in param
  257. :raises: AccessDenied
  258. :raises: AuthorizationHeaderMalformed
  259. """
  260. auth_str = self.headers['Authorization']
  261. cred_param = self._parse_credential(auth_str.partition(
  262. "Credential=")[2].split(',')[0])
  263. sig = auth_str.partition("Signature=")[2].split(',')[0]
  264. if not sig:
  265. raise AccessDenied()
  266. signed_headers = auth_str.partition(
  267. "SignedHeaders=")[2].split(',', 1)[0]
  268. if not signed_headers:
  269. # TODO: make sure if is it Malformed?
  270. raise AuthorizationHeaderMalformed()
  271. invalid_messages = {
  272. 'date': 'Invalid credential date "%s". This date is not the same '
  273. 'as X-Amz-Date: "%s".',
  274. 'region': "The authorization header is malformed; the region '%s' "
  275. "is wrong; expecting '%s'",
  276. 'service': 'The authorization header is malformed; incorrect '
  277. 'service "%s". This endpoint belongs to "%s".',
  278. 'terminal': 'The authorization header is malformed; incorrect '
  279. 'terminal "%s". This endpoint uses "%s".',
  280. }
  281. for key in ('date', 'region', 'service', 'terminal'):
  282. if cred_param[key] != self.scope[key]:
  283. kwargs = {}
  284. if key == 'region':
  285. kwargs = {'region': self.scope['region']}
  286. raise AuthorizationHeaderMalformed(
  287. invalid_messages[key] % (cred_param[key], self.scope[key]),
  288. **kwargs)
  289. self._signed_headers = set(signed_headers.split(';'))
  290. return cred_param['access'], sig
  291. def _canonical_query_string(self):
  292. return '&'.join(
  293. '%s=%s' % (quote(key, safe='-_.~'),
  294. quote(value, safe='-_.~'))
  295. for key, value in sorted(self.params.items())
  296. if key not in ('Signature', 'X-Amz-Signature'))
  297. def _headers_to_sign(self):
  298. """
  299. Select the headers from the request that need to be included
  300. in the StringToSign.
  301. :return : dict of headers to sign, the keys are all lower case
  302. """
  303. if 'headers_raw' in self.environ: # eventlet >= 0.19.0
  304. # See https://github.com/eventlet/eventlet/commit/67ec999
  305. headers_lower_dict = defaultdict(list)
  306. for key, value in self.environ['headers_raw']:
  307. headers_lower_dict[key.lower().strip()].append(
  308. ' '.join(_header_strip(value or '').split()))
  309. headers_lower_dict = {k: ','.join(v)
  310. for k, v in headers_lower_dict.items()}
  311. else: # mostly-functional fallback
  312. headers_lower_dict = dict(
  313. (k.lower().strip(), ' '.join(_header_strip(v or '').split()))
  314. for (k, v) in six.iteritems(self.headers))
  315. if 'host' in headers_lower_dict and re.match(
  316. 'Boto/2.[0-9].[0-2]',
  317. headers_lower_dict.get('user-agent', '')):
  318. # Boto versions < 2.9.3 strip the port component of the host:port
  319. # header, so detect the user-agent via the header and strip the
  320. # port if we detect an old boto version.
  321. headers_lower_dict['host'] = \
  322. headers_lower_dict['host'].split(':')[0]
  323. headers_to_sign = [
  324. (key, value) for key, value in sorted(headers_lower_dict.items())
  325. if key in self._signed_headers]
  326. if len(headers_to_sign) != len(self._signed_headers):
  327. # NOTE: if we are missing the header suggested via
  328. # signed_header in actual header, it results in
  329. # SignatureDoesNotMatch in actual S3 so we can raise
  330. # the error immediately here to save redundant check
  331. # process.
  332. raise SignatureDoesNotMatch()
  333. return headers_to_sign
  334. def _canonical_uri(self):
  335. """
  336. It won't require bucket name in canonical_uri for v4.
  337. """
  338. return self.environ.get('RAW_PATH_INFO', self.path)
  339. def _canonical_request(self):
  340. # prepare 'canonical_request'
  341. # Example requests are like following:
  342. #
  343. # GET
  344. # /
  345. # Action=ListUsers&Version=2010-05-08
  346. # content-type:application/x-www-form-urlencoded; charset=utf-8
  347. # host:iam.amazonaws.com
  348. # x-amz-date:20150830T123600Z
  349. #
  350. # content-type;host;x-amz-date
  351. # e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
  352. #
  353. # 1. Add verb like: GET
  354. cr = [self.method.upper()]
  355. # 2. Add path like: /
  356. path = self._canonical_uri()
  357. cr.append(path)
  358. # 3. Add query like: Action=ListUsers&Version=2010-05-08
  359. cr.append(self._canonical_query_string())
  360. # 4. Add headers like:
  361. # content-type:application/x-www-form-urlencoded; charset=utf-8
  362. # host:iam.amazonaws.com
  363. # x-amz-date:20150830T123600Z
  364. headers_to_sign = self._headers_to_sign()
  365. cr.append(''.join('%s:%s\n' % (key, value)
  366. for key, value in headers_to_sign))
  367. # 5. Add signed headers into canonical request like
  368. # content-type;host;x-amz-date
  369. cr.append(';'.join(k for k, v in headers_to_sign))
  370. # 6. Add payload string at the tail
  371. if 'X-Amz-Credential' in self.params:
  372. # V4 with query parameters only
  373. hashed_payload = 'UNSIGNED-PAYLOAD'
  374. elif 'X-Amz-Content-SHA256' not in self.headers:
  375. msg = 'Missing required header for this request: ' \
  376. 'x-amz-content-sha256'
  377. raise InvalidRequest(msg)
  378. else:
  379. hashed_payload = self.headers['X-Amz-Content-SHA256']
  380. if hashed_payload != 'UNSIGNED-PAYLOAD':
  381. if self.content_length == 0:
  382. if hashed_payload != sha256().hexdigest():
  383. raise BadDigest(
  384. 'The X-Amz-Content-SHA56 you specified did not '
  385. 'match what we received.')
  386. elif self.content_length:
  387. self.environ['wsgi.input'] = HashingInput(
  388. self.environ['wsgi.input'],
  389. self.content_length,
  390. sha256,
  391. hashed_payload)
  392. # else, length not provided -- Swift will kick out a
  393. # 411 Length Required which will get translated back
  394. # to a S3-style response in S3Request._swift_error_codes
  395. cr.append(hashed_payload)
  396. return '\n'.join(cr).encode('utf-8')
  397. @property
  398. def scope(self):
  399. return OrderedDict([
  400. ('date', self.timestamp.amz_date_format.split('T')[0]),
  401. ('region', self.location),
  402. ('service', SERVICE),
  403. ('terminal', 'aws4_request'),
  404. ])
  405. def _string_to_sign(self):
  406. """
  407. Create 'StringToSign' value in Amazon terminology for v4.
  408. """
  409. return '\n'.join(['AWS4-HMAC-SHA256',
  410. self.timestamp.amz_date_format,
  411. '/'.join(self.scope.values()),
  412. sha256(self._canonical_request()).hexdigest()])
  413. def signature_does_not_match_kwargs(self):
  414. kwargs = super(SigV4Mixin, self).signature_does_not_match_kwargs()
  415. cr = self._canonical_request()
  416. kwargs.update({
  417. 'canonical_request': cr,
  418. 'canonical_request_bytes': ' '.join(
  419. format(ord(c), '02x') for c in cr),
  420. })
  421. return kwargs
  422. def get_request_class(env, s3_acl):
  423. """
  424. Helper function to find a request class to use from Map
  425. """
  426. if s3_acl:
  427. request_classes = (S3AclRequest, SigV4S3AclRequest)
  428. else:
  429. request_classes = (S3Request, SigV4Request)
  430. req = swob.Request(env)
  431. if 'X-Amz-Credential' in req.params or \
  432. req.headers.get('Authorization', '').startswith(
  433. 'AWS4-HMAC-SHA256 '):
  434. # This is an Amazon SigV4 request
  435. return request_classes[1]
  436. else:
  437. # The others using Amazon SigV2 class
  438. return request_classes[0]
  439. class S3Request(swob.Request):
  440. """
  441. S3 request object.
  442. """
  443. bucket_acl = _header_acl_property('container')
  444. object_acl = _header_acl_property('object')
  445. def __init__(self, env, app=None, slo_enabled=True, storage_domain='',
  446. location='us-east-1', force_request_log=False,
  447. dns_compliant_bucket_names=True, allow_multipart_uploads=True,
  448. allow_no_owner=False):
  449. # NOTE: app and allow_no_owner are not used by this class, need for
  450. # compatibility of S3acl
  451. swob.Request.__init__(self, env)
  452. self.storage_domain = storage_domain
  453. self.location = location
  454. self.force_request_log = force_request_log
  455. self.dns_compliant_bucket_names = dns_compliant_bucket_names
  456. self.allow_multipart_uploads = allow_multipart_uploads
  457. self._timestamp = None
  458. self.access_key, self.signature = self._parse_auth_info()
  459. self.bucket_in_host = self._parse_host()
  460. self.container_name, self.object_name = self._parse_uri()
  461. self._validate_headers()
  462. # Lock in string-to-sign now, before we start messing with query params
  463. self.string_to_sign = self._string_to_sign()
  464. self.environ['s3api.auth_details'] = {
  465. 'access_key': self.access_key,
  466. 'signature': self.signature,
  467. 'string_to_sign': self.string_to_sign,
  468. 'check_signature': self.check_signature,
  469. }
  470. self.token = None
  471. self.account = None
  472. self.user_id = None
  473. self.slo_enabled = slo_enabled
  474. # Avoids that swift.swob.Response replaces Location header value
  475. # by full URL when absolute path given. See swift.swob for more detail.
  476. self.environ['swift.leave_relative_location'] = True
  477. def check_signature(self, secret):
  478. secret = utf8encode(secret)
  479. user_signature = self.signature
  480. valid_signature = base64.b64encode(hmac.new(
  481. secret, self.string_to_sign, sha1).digest()).strip()
  482. return user_signature == valid_signature
  483. @property
  484. def timestamp(self):
  485. """
  486. S3Timestamp from Date header. If X-Amz-Date header specified, it
  487. will be prior to Date header.
  488. :return : S3Timestamp instance
  489. """
  490. if not self._timestamp:
  491. try:
  492. if self._is_query_auth and 'Timestamp' in self.params:
  493. # If Timestamp specified in query, it should be prior
  494. # to any Date header (is this right?)
  495. timestamp = mktime(
  496. self.params['Timestamp'], SIGV2_TIMESTAMP_FORMAT)
  497. else:
  498. timestamp = mktime(
  499. self.headers.get('X-Amz-Date',
  500. self.headers.get('Date')))
  501. except ValueError:
  502. raise AccessDenied('AWS authentication requires a valid Date '
  503. 'or x-amz-date header')
  504. if timestamp < 0:
  505. raise AccessDenied('AWS authentication requires a valid Date '
  506. 'or x-amz-date header')
  507. try:
  508. self._timestamp = S3Timestamp(timestamp)
  509. except ValueError:
  510. # Must be far-future; blame clock skew
  511. raise RequestTimeTooSkewed()
  512. return self._timestamp
  513. @property
  514. def _is_header_auth(self):
  515. return 'Authorization' in self.headers
  516. @property
  517. def _is_query_auth(self):
  518. return 'AWSAccessKeyId' in self.params
  519. def _parse_host(self):
  520. storage_domain = self.storage_domain
  521. if not storage_domain:
  522. return None
  523. if not storage_domain.startswith('.'):
  524. storage_domain = '.' + storage_domain
  525. if 'HTTP_HOST' in self.environ:
  526. given_domain = self.environ['HTTP_HOST']
  527. elif 'SERVER_NAME' in self.environ:
  528. given_domain = self.environ['SERVER_NAME']
  529. else:
  530. return None
  531. port = ''
  532. if ':' in given_domain:
  533. given_domain, port = given_domain.rsplit(':', 1)
  534. if given_domain.endswith(storage_domain):
  535. return given_domain[:-len(storage_domain)]
  536. return None
  537. def _parse_uri(self):
  538. if not check_utf8(self.environ['PATH_INFO']):
  539. raise InvalidURI(self.path)
  540. if self.bucket_in_host:
  541. obj = self.environ['PATH_INFO'][1:] or None
  542. return self.bucket_in_host, obj
  543. bucket, obj = self.split_path(0, 2, True)
  544. if bucket and not validate_bucket_name(
  545. bucket, self.dns_compliant_bucket_names):
  546. # Ignore GET service case
  547. raise InvalidBucketName(bucket)
  548. return (bucket, obj)
  549. def _parse_query_authentication(self):
  550. """
  551. Parse v2 authentication query args
  552. TODO: make sure if 0, 1, 3 is supported?
  553. - version 0, 1, 2, 3:
  554. 'AWSAccessKeyId' and 'Signature' should be in param
  555. :return: a tuple of access_key and signature
  556. :raises: AccessDenied
  557. """
  558. try:
  559. access = self.params['AWSAccessKeyId']
  560. expires = self.params['Expires']
  561. sig = self.params['Signature']
  562. except KeyError:
  563. raise AccessDenied()
  564. if not all([access, sig, expires]):
  565. raise AccessDenied()
  566. return access, sig
  567. def _parse_header_authentication(self):
  568. """
  569. Parse v2 header authentication info
  570. :returns: a tuple of access_key and signature
  571. :raises: AccessDenied
  572. """
  573. auth_str = self.headers['Authorization']
  574. if not auth_str.startswith('AWS ') or ':' not in auth_str:
  575. raise AccessDenied()
  576. # This means signature format V2
  577. access, sig = auth_str.split(' ', 1)[1].rsplit(':', 1)
  578. return access, sig
  579. def _parse_auth_info(self):
  580. """Extract the access key identifier and signature.
  581. :returns: a tuple of access_key and signature
  582. :raises: NotS3Request
  583. """
  584. if self._is_query_auth:
  585. self._validate_expire_param()
  586. return self._parse_query_authentication()
  587. elif self._is_header_auth:
  588. self._validate_dates()
  589. return self._parse_header_authentication()
  590. else:
  591. # if this request is neither query auth nor header auth
  592. # s3api regard this as not s3 request
  593. raise NotS3Request()
  594. def _validate_expire_param(self):
  595. """
  596. Validate Expires in query parameters
  597. :raises: AccessDenied
  598. """
  599. # Expires header is a float since epoch
  600. try:
  601. ex = S3Timestamp(float(self.params['Expires']))
  602. except (KeyError, ValueError):
  603. raise AccessDenied()
  604. if S3Timestamp.now() > ex:
  605. raise AccessDenied('Request has expired')
  606. if ex >= 2 ** 31:
  607. raise AccessDenied(
  608. 'Invalid date (should be seconds since epoch): %s' %
  609. self.params['Expires'])
  610. def _validate_dates(self):
  611. """
  612. Validate Date/X-Amz-Date headers for signature v2
  613. :raises: AccessDenied
  614. :raises: RequestTimeTooSkewed
  615. """
  616. date_header = self.headers.get('Date')
  617. amz_date_header = self.headers.get('X-Amz-Date')
  618. if not date_header and not amz_date_header:
  619. raise AccessDenied('AWS authentication requires a valid Date '
  620. 'or x-amz-date header')
  621. # Anyways, request timestamp should be validated
  622. epoch = S3Timestamp(0)
  623. if self.timestamp < epoch:
  624. raise AccessDenied()
  625. # If the standard date is too far ahead or behind, it is an
  626. # error
  627. delta = 60 * 5
  628. if abs(int(self.timestamp) - int(S3Timestamp.now())) > delta:
  629. raise RequestTimeTooSkewed()
  630. def _validate_headers(self):
  631. if 'CONTENT_LENGTH' in self.environ:
  632. try:
  633. if self.content_length < 0:
  634. raise InvalidArgument('Content-Length',
  635. self.content_length)
  636. except (ValueError, TypeError):
  637. raise InvalidArgument('Content-Length',
  638. self.environ['CONTENT_LENGTH'])
  639. value = _header_strip(self.headers.get('Content-MD5'))
  640. if value is not None:
  641. if not re.match('^[A-Za-z0-9+/]+={0,2}$', value):
  642. # Non-base64-alphabet characters in value.
  643. raise InvalidDigest(content_md5=value)
  644. try:
  645. self.headers['ETag'] = value.decode('base64').encode('hex')
  646. except Exception:
  647. raise InvalidDigest(content_md5=value)
  648. if len(self.headers['ETag']) != 32:
  649. raise InvalidDigest(content_md5=value)
  650. if self.method == 'PUT' and any(h in self.headers for h in (
  651. 'If-Match', 'If-None-Match',
  652. 'If-Modified-Since', 'If-Unmodified-Since')):
  653. raise S3NotImplemented(
  654. 'Conditional object PUTs are not supported.')
  655. if 'X-Amz-Copy-Source' in self.headers:
  656. try:
  657. check_path_header(self, 'X-Amz-Copy-Source', 2, '')
  658. except swob.HTTPException:
  659. msg = 'Copy Source must mention the source bucket and key: ' \
  660. 'sourcebucket/sourcekey'
  661. raise InvalidArgument('x-amz-copy-source',
  662. self.headers['X-Amz-Copy-Source'],
  663. msg)
  664. if 'x-amz-metadata-directive' in self.headers:
  665. value = self.headers['x-amz-metadata-directive']
  666. if value not in ('COPY', 'REPLACE'):
  667. err_msg = 'Unknown metadata directive.'
  668. raise InvalidArgument('x-amz-metadata-directive', value,
  669. err_msg)
  670. if 'x-amz-storage-class' in self.headers:
  671. # Only STANDARD is supported now.
  672. if self.headers['x-amz-storage-class'] != 'STANDARD':
  673. raise InvalidStorageClass()
  674. if 'x-amz-mfa' in self.headers:
  675. raise S3NotImplemented('MFA Delete is not supported.')
  676. sse_value = self.headers.get('x-amz-server-side-encryption')
  677. if sse_value is not None:
  678. if sse_value not in ('aws:kms', 'AES256'):
  679. raise InvalidArgument(
  680. 'x-amz-server-side-encryption', sse_value,
  681. 'The encryption method specified is not supported')
  682. encryption_enabled = get_swift_info(admin=True)['admin'].get(
  683. 'encryption', {}).get('enabled')
  684. if not encryption_enabled or sse_value != 'AES256':
  685. raise S3NotImplemented(
  686. 'Server-side encryption is not supported.')
  687. if 'x-amz-website-redirect-location' in self.headers:
  688. raise S3NotImplemented('Website redirection is not supported.')
  689. # https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
  690. # describes some of what would be required to support this
  691. if any(['aws-chunked' in self.headers.get('content-encoding', ''),
  692. 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' == self.headers.get(
  693. 'x-amz-content-sha256', ''),
  694. 'x-amz-decoded-content-length' in self.headers]):
  695. raise S3NotImplemented('Transfering payloads in multiple chunks '
  696. 'using aws-chunked is not supported.')
  697. if 'x-amz-tagging' in self.headers:
  698. raise S3NotImplemented('Object tagging is not supported.')
  699. @property
  700. def body(self):
  701. """
  702. swob.Request.body is not secure against malicious input. It consumes
  703. too much memory without any check when the request body is excessively
  704. large. Use xml() instead.
  705. """
  706. raise AttributeError("No attribute 'body'")
  707. def xml(self, max_length):
  708. """
  709. Similar to swob.Request.body, but it checks the content length before
  710. creating a body string.
  711. """
  712. te = self.headers.get('transfer-encoding', '')
  713. te = [x.strip() for x in te.split(',') if x.strip()]
  714. if te and (len(te) > 1 or te[-1] != 'chunked'):
  715. raise S3NotImplemented('A header you provided implies '
  716. 'functionality that is not implemented',
  717. header='Transfer-Encoding')
  718. if self.message_length() > max_length:
  719. raise MalformedXML()
  720. if te or self.message_length():
  721. # Limit the read similar to how SLO handles manifests
  722. body = self.body_file.read(max_length)
  723. else:
  724. # No (or zero) Content-Length provided, and not chunked transfer;
  725. # no body. Assume zero-length, and enforce a required body below.
  726. return None
  727. return body
  728. def check_md5(self, body):
  729. if 'HTTP_CONTENT_MD5' not in self.environ:
  730. raise InvalidRequest('Missing required header for this request: '
  731. 'Content-MD5')
  732. digest = md5(body).digest().encode('base64').strip()
  733. if self.environ['HTTP_CONTENT_MD5'] != digest:
  734. raise BadDigest(content_md5=self.environ['HTTP_CONTENT_MD5'])
  735. def _copy_source_headers(self):
  736. env = {}
  737. for key, value in self.environ.items():
  738. if key.startswith('HTTP_X_AMZ_COPY_SOURCE_'):
  739. env[key.replace('X_AMZ_COPY_SOURCE_', '')] = value
  740. return swob.HeaderEnvironProxy(env)
  741. def check_copy_source(self, app):
  742. """
  743. check_copy_source checks the copy source existence and if copying an
  744. object to itself, for illegal request parameters
  745. :returns: the source HEAD response
  746. """
  747. try:
  748. src_path = self.headers['X-Amz-Copy-Source']
  749. except KeyError:
  750. return None
  751. if '?' in src_path:
  752. src_path, qs = src_path.split('?', 1)
  753. query = parse_qsl(qs, True)
  754. if not query:
  755. pass # ignore it
  756. elif len(query) > 1 or query[0][0] != 'versionId':
  757. raise InvalidArgument('X-Amz-Copy-Source',
  758. self.headers['X-Amz-Copy-Source'],
  759. 'Unsupported copy source parameter.')
  760. elif query[0][1] != 'null':
  761. # TODO: once we support versioning, we'll need to translate
  762. # src_path to the proper location in the versions container
  763. raise S3NotImplemented('Versioning is not yet supported')
  764. self.headers['X-Amz-Copy-Source'] = src_path
  765. src_path = unquote(src_path)
  766. src_path = src_path if src_path.startswith('/') else ('/' + src_path)
  767. src_bucket, src_obj = split_path(src_path, 0, 2, True)
  768. headers = swob.HeaderKeyDict()
  769. headers.update(self._copy_source_headers())
  770. src_resp = self.get_response(app, 'HEAD', src_bucket, src_obj,
  771. headers=headers)
  772. if src_resp.status_int == 304: # pylint: disable-msg=E1101
  773. raise PreconditionFailed()
  774. self.headers['X-Amz-Copy-Source'] = \
  775. '/' + self.headers['X-Amz-Copy-Source'].lstrip('/')
  776. source_container, source_obj = \
  777. split_path(self.headers['X-Amz-Copy-Source'], 1, 2, True)
  778. if (self.container_name == source_container and
  779. self.object_name == source_obj and
  780. self.headers.get('x-amz-metadata-directive',
  781. 'COPY') == 'COPY'):
  782. raise InvalidRequest("This copy request is illegal "
  783. "because it is trying to copy an "
  784. "object to itself without "
  785. "changing the object's metadata, "
  786. "storage class, website redirect "
  787. "location or encryption "
  788. "attributes.")
  789. return src_resp
  790. def _canonical_uri(self):
  791. """
  792. Require bucket name in canonical_uri for v2 in virtual hosted-style.
  793. """
  794. raw_path_info = self.environ.get('RAW_PATH_INFO', self.path)
  795. if self.bucket_in_host:
  796. raw_path_info = '/' + self.bucket_in_host + raw_path_info
  797. return raw_path_info
  798. def _string_to_sign(self):
  799. """
  800. Create 'StringToSign' value in Amazon terminology for v2.
  801. """
  802. amz_headers = {}
  803. buf = [self.method,
  804. _header_strip(self.headers.get('Content-MD5')) or '',
  805. _header_strip(self.headers.get('Content-Type')) or '']
  806. if 'headers_raw' in self.environ: # eventlet >= 0.19.0
  807. # See https://github.com/eventlet/eventlet/commit/67ec999
  808. amz_headers = defaultdict(list)
  809. for key, value in self.environ['headers_raw']:
  810. key = key.lower()
  811. if not key.startswith('x-amz-'):
  812. continue
  813. amz_headers[key.strip()].append(value.strip())
  814. amz_headers = dict((key, ','.join(value))
  815. for key, value in amz_headers.items())
  816. else: # mostly-functional fallback
  817. amz_headers = dict((key.lower(), value)
  818. for key, value in self.headers.items()
  819. if key.lower().startswith('x-amz-'))
  820. if self._is_header_auth:
  821. if 'x-amz-date' in amz_headers:
  822. buf.append('')
  823. elif 'Date' in self.headers:
  824. buf.append(self.headers['Date'])
  825. elif self._is_query_auth:
  826. buf.append(self.params['Expires'])
  827. else:
  828. # Should have already raised NotS3Request in _parse_auth_info,
  829. # but as a sanity check...
  830. raise AccessDenied()
  831. for key, value in sorted(amz_headers.items()):
  832. buf.append("%s:%s" % (key, value))
  833. path = self._canonical_uri()
  834. if self.query_string:
  835. path += '?' + self.query_string
  836. params = []
  837. if '?' in path:
  838. path, args = path.split('?', 1)
  839. for key, value in sorted(self.params.items()):
  840. if key in ALLOWED_SUB_RESOURCES:
  841. params.append('%s=%s' % (key, value) if value else key)
  842. if params:
  843. buf.append('%s?%s' % (path, '&'.join(params)))
  844. else:
  845. buf.append(path)
  846. return '\n'.join(buf)
  847. def signature_does_not_match_kwargs(self):
  848. return {
  849. 'a_w_s_access_key_id': self.access_key,
  850. 'string_to_sign': self.string_to_sign,
  851. 'signature_provided': self.signature,
  852. 'string_to_sign_bytes': ' '.join(
  853. format(ord(c), '02x') for c in self.string_to_sign),
  854. }
  855. @property
  856. def controller_name(self):
  857. return self.controller.__name__[:-len('Controller')]
  858. @property
  859. def controller(self):
  860. if self.is_service_request:
  861. return ServiceController
  862. if not self.slo_enabled:
  863. multi_part = ['partNumber', 'uploadId', 'uploads']
  864. if len([p for p in multi_part if p in self.params]):
  865. raise S3NotImplemented("Multi-part feature isn't support")
  866. if 'acl' in self.params:
  867. return AclController
  868. if 'delete' in self.params:
  869. return MultiObjectDeleteController
  870. if 'location' in self.params:
  871. return LocationController
  872. if 'logging' in self.params:
  873. return LoggingStatusController
  874. if 'partNumber' in self.params:
  875. return PartController
  876. if 'uploadId' in self.params:
  877. return UploadController
  878. if 'uploads' in self.params:
  879. return UploadsController
  880. if 'versioning' in self.params:
  881. return VersioningController
  882. unsupported = ('notification', 'policy', 'requestPayment', 'torrent',
  883. 'website', 'cors', 'tagging', 'restore')
  884. if set(unsupported) & set(self.params):
  885. return UnsupportedController
  886. if self.is_object_request:
  887. return ObjectController
  888. return BucketController
  889. @property
  890. def is_service_request(self):
  891. return not self.container_name
  892. @property
  893. def is_bucket_request(self):
  894. return self.container_name and not self.object_name
  895. @property
  896. def is_object_request(self):
  897. return self.container_name and self.object_name
  898. @property
  899. def is_authenticated(self):
  900. return self.account is not None
  901. def to_swift_req(self, method, container, obj, query=None,
  902. body=None, headers=None):
  903. """
  904. Create a Swift request based on this request's environment.
  905. """
  906. if self.account is None:
  907. account = self.access_key
  908. else:
  909. account = self.account
  910. env = self.environ.copy()
  911. def sanitize(value):
  912. if set(value).issubset(string.printable):
  913. return value
  914. value = Header(value, 'UTF-8').encode()
  915. if value.startswith('=?utf-8?q?'):
  916. return '=?UTF-8?Q?' + value[10:]
  917. elif value.startswith('=?utf-8?b?'):
  918. return '=?UTF-8?B?' + value[10:]
  919. else:
  920. return value
  921. if 'headers_raw' in env: # eventlet >= 0.19.0
  922. # See https://github.com/eventlet/eventlet/commit/67ec999
  923. for key, value in env['headers_raw']:
  924. if not key.lower().startswith('x-amz-meta-'):
  925. continue
  926. # AWS ignores user-defined headers with these characters
  927. if any(c in key for c in ' "),/;<=>?@[\\]{}'):
  928. # NB: apparently, '(' *is* allowed
  929. continue
  930. # Note that this may have already been deleted, e.g. if the
  931. # client sent multiple headers with the same name, or both
  932. # x-amz-meta-foo-bar and x-amz-meta-foo_bar
  933. env.pop('HTTP_' + key.replace('-', '_').upper(), None)
  934. # Need to preserve underscores. Since we know '=' can't be
  935. # present, quoted-printable seems appropriate.
  936. key = key.replace('_', '=5F').replace('-', '_').upper()
  937. key = 'HTTP_X_OBJECT_META_' + key[11:]
  938. if key in env:
  939. env[key] += ',' + sanitize(value)
  940. else:
  941. env[key] = sanitize(value)
  942. else: # mostly-functional fallback
  943. for key in self.environ:
  944. if not key.startswith('HTTP_X_AMZ_META_'):
  945. continue
  946. # AWS ignores user-defined headers with these characters
  947. if any(c in key for c in ' "),/;<=>?@[\\]{}'):
  948. # NB: apparently, '(' *is* allowed
  949. continue
  950. env['HTTP_X_OBJECT_META_' + key[16:]] = sanitize(env[key])
  951. del env[key]
  952. if 'HTTP_X_AMZ_COPY_SOURCE' in env:
  953. env['HTTP_X_COPY_FROM'] = env['HTTP_X_AMZ_COPY_SOURCE']
  954. del env['HTTP_X_AMZ_COPY_SOURCE']
  955. env['CONTENT_LENGTH'] = '0'
  956. if env.pop('HTTP_X_AMZ_METADATA_DIRECTIVE', None) == 'REPLACE':
  957. env['HTTP_X_FRESH_METADATA'] = 'True'
  958. else:
  959. copy_exclude_headers = ('HTTP_CONTENT_DISPOSITION',
  960. 'HTTP_CONTENT_ENCODING',
  961. 'HTTP_CONTENT_LANGUAGE',
  962. 'CONTENT_TYPE',
  963. 'HTTP_EXPIRES',
  964. 'HTTP_CACHE_CONTROL',
  965. 'HTTP_X_ROBOTS_TAG')
  966. for key in copy_exclude_headers:
  967. env.pop(key, None)
  968. for key in list(env.keys()):
  969. if key.startswith('HTTP_X_OBJECT_META_'):
  970. del env[key]
  971. if self.force_request_log:
  972. env['swift.proxy_access_log_made'] = False
  973. env['swift.source'] = 'S3'
  974. if method is not None:
  975. env['REQUEST_METHOD'] = method
  976. env['HTTP_X_AUTH_TOKEN'] = self.token
  977. if obj:
  978. path = '/v1/%s/%s/%s' % (account, container, obj)
  979. elif container:
  980. path = '/v1/%s/%s' % (account, container)
  981. else:
  982. path = '/v1/%s' % (account)
  983. env['PATH_INFO'] = path
  984. query_string = ''
  985. if query is not None:
  986. params = []
  987. for key, value in sorted(query.items()):
  988. if value is not None:
  989. params.append('%s=%s' % (key, quote(str(value))))
  990. else:
  991. params.append(key)
  992. query_string = '&'.join(params)
  993. env['QUERY_STRING'] = query_string
  994. return swob.Request.blank(quote(path), environ=env, body=body,
  995. headers=headers)
  996. def _swift_success_codes(self, method, container, obj):
  997. """
  998. Returns a list of expected success codes from Swift.
  999. """
  1000. if not container:
  1001. # Swift account access.
  1002. code_map = {
  1003. 'GET': [
  1004. HTTP_OK,
  1005. ],
  1006. }
  1007. elif not obj:
  1008. # Swift container access.
  1009. code_map = {
  1010. 'HEAD': [
  1011. HTTP_NO_CONTENT,
  1012. ],
  1013. 'GET': [
  1014. HTTP_OK,
  1015. HTTP_NO_CONTENT,
  1016. ],
  1017. 'PUT': [
  1018. HTTP_CREATED,
  1019. ],
  1020. 'POST': [
  1021. HTTP_NO_CONTENT,
  1022. ],
  1023. 'DELETE': [
  1024. HTTP_NO_CONTENT,
  1025. ],
  1026. }
  1027. else:
  1028. # Swift object access.
  1029. code_map = {
  1030. 'HEAD': [
  1031. HTTP_OK,
  1032. HTTP_PARTIAL_CONTENT,
  1033. HTTP_NOT_MODIFIED,
  1034. ],
  1035. 'GET': [
  1036. HTTP_OK,
  1037. HTTP_PARTIAL_CONTENT,
  1038. HTTP_NOT_MODIFIED,
  1039. ],
  1040. 'PUT': [
  1041. HTTP_CREATED,
  1042. HTTP_ACCEPTED, # For SLO with heartbeating
  1043. ],
  1044. 'POST': [
  1045. HTTP_ACCEPTED,
  1046. ],
  1047. 'DELETE': [
  1048. HTTP_OK,
  1049. HTTP_NO_CONTENT,
  1050. ],
  1051. }
  1052. return code_map[method]
  1053. def _bucket_put_accepted_error(self, container, app):
  1054. sw_req = self.to_swift_req('HEAD', container, None)
  1055. info = get_container_info(sw_req.environ, app)
  1056. sysmeta = info.get('sysmeta', {})
  1057. try:
  1058. acl = json.loads(sysmeta.get('s3api-acl',
  1059. sysmeta.get('swift3-acl', '{}')))
  1060. owner = acl.get('Owner')
  1061. except (ValueError, TypeError, KeyError):
  1062. owner = None
  1063. if owner is None or owner == self.user_id:
  1064. raise BucketAlreadyOwnedByYou(container)
  1065. raise BucketAlreadyExists(container)
  1066. def _swift_error_codes(self, method, container, obj, env, app):
  1067. """
  1068. Returns a dict from expected Swift error codes to the corresponding S3
  1069. error responses.
  1070. """
  1071. if not container:
  1072. # Swift account access.
  1073. code_map = {
  1074. 'GET': {
  1075. },
  1076. }
  1077. elif not obj:
  1078. # Swift container access.
  1079. code_map = {
  1080. 'HEAD': {
  1081. HTTP_NOT_FOUND: (NoSuchBucket, container),
  1082. },
  1083. 'GET': {
  1084. HTTP_NOT_FOUND: (NoSuchBucket, container),
  1085. },
  1086. 'PUT': {
  1087. HTTP_ACCEPTED: (self._bucket_put_accepted_error, container,
  1088. app),
  1089. },
  1090. 'POST': {
  1091. HTTP_NOT_FOUND: (NoSuchBucket, container),
  1092. },
  1093. 'DELETE': {
  1094. HTTP_NOT_FOUND: (NoSuchBucket, container),
  1095. HTTP_CONFLICT: BucketNotEmpty,
  1096. },
  1097. }
  1098. else:
  1099. # Swift object access.
  1100. # 404s differ depending upon whether the bucket exists
  1101. # Note that base-container-existence checks happen elsewhere for
  1102. # multi-part uploads, and get_container_info should be pulling
  1103. # from the env cache
  1104. def not_found_handler():
  1105. if container.endswith(MULTIUPLOAD_SUFFIX) or \
  1106. is_success(get_container_info(
  1107. env, app, swift_source='S3').get('status')):
  1108. return NoSuchKey(obj)
  1109. return NoSuchBucket(container)
  1110. code_map = {
  1111. 'HEAD': {
  1112. HTTP_NOT_FOUND: not_found_handler,
  1113. HTTP_PRECONDITION_FAILED: PreconditionFailed,
  1114. },
  1115. 'GET': {
  1116. HTTP_NOT_FOUND: not_found_handler,
  1117. HTTP_PRECONDITION_FAILED: PreconditionFailed,
  1118. HTTP_REQUESTED_RANGE_NOT_SATISFIABLE: InvalidRange,
  1119. },
  1120. 'PUT': {
  1121. HTTP_NOT_FOUND: (NoSuchBucket, container),
  1122. HTTP_UNPROCESSABLE_ENTITY: BadDigest,
  1123. HTTP_REQUEST_ENTITY_TOO_LARGE: EntityTooLarge,
  1124. HTTP_LENGTH_REQUIRED: MissingContentLength,
  1125. HTTP_REQUEST_TIMEOUT: RequestTimeout,
  1126. },
  1127. 'POST': {
  1128. HTTP_NOT_FOUND: not_found_handler,
  1129. HTTP_PRECONDITION_FAILED: PreconditionFailed,
  1130. },
  1131. 'DELETE': {
  1132. HTTP_NOT_FOUND: (NoSuchKey, obj),
  1133. },
  1134. }
  1135. return code_map[method]
  1136. def _get_response(self, app, method, container, obj,
  1137. headers=None, body=None, query=None):
  1138. """
  1139. Calls the application with this request's environment. Returns a
  1140. S3Response object that wraps up the application's result.
  1141. """
  1142. method = method or self.environ['REQUEST_METHOD']
  1143. if container is None:
  1144. container = self.container_name
  1145. if obj is None:
  1146. obj = self.object_name
  1147. sw_req = self.to_swift_req(method, container, obj, headers=headers,
  1148. body=body, query=query)
  1149. try:
  1150. sw_resp = sw_req.get_response(app)
  1151. except swob.HTTPException as err:
  1152. sw_resp = err
  1153. else:
  1154. # reuse account and tokens
  1155. _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
  1156. 2, 3, True)
  1157. self.account = utf8encode(self.account)
  1158. resp = S3Response.from_swift_resp(sw_resp)
  1159. status = resp.status_int # pylint: disable-msg=E1101
  1160. if not self.user_id:
  1161. if 'HTTP_X_USER_NAME' in sw_resp.environ:
  1162. # keystone
  1163. self.user_id = \
  1164. utf8encode("%s:%s" %
  1165. (sw_resp.environ['HTTP_X_TENANT_NAME'],
  1166. sw_resp.environ['HTTP_X_USER_NAME']))
  1167. else:
  1168. # tempauth
  1169. self.user_id = self.access_key
  1170. success_codes = self._swift_success_codes(method, container, obj)
  1171. error_codes = self._swift_error_codes(method, container, obj,
  1172. sw_req.environ, app)
  1173. if status in success_codes:
  1174. return resp
  1175. err_msg = resp.body
  1176. if status in error_codes:
  1177. err_resp = \
  1178. error_codes[sw_resp.status_int] # pylint: disable-msg=E1101
  1179. if isinstance(err_resp, tuple):
  1180. raise err_resp[0](*err_resp[1:])
  1181. else:
  1182. raise err_resp()
  1183. if status == HTTP_BAD_REQUEST:
  1184. raise BadSwiftRequest(err_msg)
  1185. if status == HTTP_UNAUTHORIZED:
  1186. raise SignatureDoesNotMatch(
  1187. **self.signature_does_not_match_kwargs())
  1188. if status == HTTP_FORBIDDEN:
  1189. raise AccessDenied()
  1190. if status == HTTP_SERVICE_UNAVAILABLE:
  1191. raise ServiceUnavailable()
  1192. if status in (HTTP_RATE_LIMITED, HTTP_TOO_MANY_REQUESTS):
  1193. raise SlowDown()
  1194. raise InternalError('unexpected status code %d' % status)
  1195. def get_response(self, app, method=None, container=None, obj=None,
  1196. headers=None, body=None, query=None):
  1197. """
  1198. get_response is an entry point to be extended for child classes.
  1199. If additional tasks needed at that time of getting swift response,
  1200. we can override this method.
  1201. swift.common.middleware.s3api.s3request.S3Request need to just call
  1202. _get_response to get pure swift response.
  1203. """
  1204. if 'HTTP_X_AMZ_ACL' in self.environ:
  1205. handle_acl_header(self)
  1206. return self._get_response(app, method, container, obj,
  1207. headers, body, query)
  1208. def get_validated_param(self, param, default, limit=MAX_32BIT_INT):
  1209. value = default
  1210. if param in self.params:
  1211. try:
  1212. value = int(self.params[param])
  1213. if value < 0:
  1214. err_msg = 'Argument %s must be an integer between 0 and' \
  1215. ' %d' % (param, MAX_32BIT_INT)
  1216. raise InvalidArgument(param, self.params[param], err_msg)
  1217. if value > MAX_32BIT_INT:
  1218. # check the value because int() could build either a long
  1219. # instance or a 64bit integer.
  1220. raise ValueError()
  1221. if limit < value:
  1222. value = limit
  1223. except ValueError:
  1224. err_msg = 'Provided %s not an integer or within ' \
  1225. 'integer range' % param
  1226. raise InvalidArgument(param, self.params[param], err_msg)
  1227. return value
  1228. def get_container_info(self, app):
  1229. """
  1230. get_container_info will return a result dict of get_container_info
  1231. from the backend Swift.
  1232. :returns: a dictionary of container info from
  1233. swift.controllers.base.get_container_info
  1234. :raises: NoSuchBucket when the container doesn't exist
  1235. :raises: InternalError when the request failed without 404
  1236. """
  1237. if self.is_authenticated:
  1238. # if we have already authenticated, yes we can use the account
  1239. # name like as AUTH_xxx for performance efficiency
  1240. sw_req = self.to_swift_req(app, self.container_name, None)
  1241. info = get_container_info(sw_req.environ, app)
  1242. if is_success(info['status']):
  1243. return info
  1244. elif info['status'] == 404:
  1245. raise NoSuchBucket(self.container_name)
  1246. else:
  1247. raise InternalError(
  1248. 'unexpected status code %d' % info['status'])
  1249. else:
  1250. # otherwise we do naive HEAD request with the authentication
  1251. resp = self.get_response(app, 'HEAD', self.container_name, '')
  1252. return headers_to_container_info(
  1253. resp.sw_headers, resp.status_int) # pylint: disable-msg=E1101
  1254. def gen_multipart_manifest_delete_query(self, app, obj=None):
  1255. if not self.allow_multipart_uploads:
  1256. return None
  1257. query = {'multipart-manifest': 'delete'}
  1258. if not obj:
  1259. obj = self.object_name
  1260. resp = self.get_response(app, 'HEAD', obj=obj)
  1261. return query if resp.is_slo else None
  1262. def set_acl_handler(self, handler):
  1263. pass
  1264. class S3AclRequest(S3Request):
  1265. """
  1266. S3Acl request object.
  1267. """
  1268. def __init__(self, env, app, slo_enabled=True, storage_domain='',
  1269. location='us-east-1', force_request_log=False,
  1270. dns_compliant_bucket_names=True, allow_multipart_uploads=True,
  1271. allow_no_owner=False):
  1272. super(S3AclRequest, self).__init__(
  1273. env, app, slo_enabled, storage_domain, location, force_request_log,
  1274. dns_compliant_bucket_names, allow_multipart_uploads)
  1275. self.allow_no_owner = allow_no_owner
  1276. self.authenticate(app)
  1277. self.acl_handler = None
  1278. @property
  1279. def controller(self):
  1280. if 'acl' in self.params and not self.is_service_request:
  1281. return S3AclController
  1282. return super(S3AclRequest, self).controller
  1283. def authenticate(self, app):
  1284. """
  1285. authenticate method will run pre-authenticate request and retrieve
  1286. account information.
  1287. Note that it currently supports only keystone and tempauth.
  1288. (no support for the third party authentication middleware)
  1289. """
  1290. sw_req = self.to_swift_req('TEST', None, None, body='')
  1291. # don't show log message of this request
  1292. sw_req.environ['swift.proxy_access_log_made'] = True
  1293. sw_resp = sw_req.get_response(app)
  1294. if not sw_req.remote_user:
  1295. raise SignatureDoesNotMatch(
  1296. **self.signature_does_not_match_kwargs())
  1297. _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
  1298. 2, 3, True)
  1299. self.account = utf8encode(self.account)
  1300. if 'HTTP_X_USER_NAME' in sw_resp.environ:
  1301. # keystone
  1302. self.user_id = "%s:%s" % (sw_resp.environ['HTTP_X_TENANT_NAME'],
  1303. sw_resp.environ['HTTP_X_USER_NAME'])
  1304. self.user_id = utf8encode(self.user_id)
  1305. self.token = sw_resp.environ.get('HTTP_X_AUTH_TOKEN')
  1306. else:
  1307. # tempauth
  1308. self.user_id = self.access_key
  1309. sw_req.environ.get('swift.authorize', lambda req: None)(sw_req)
  1310. self.environ['swift_owner'] = sw_req.environ.get('swift_owner', False)
  1311. # Need to skip S3 authorization on subsequent requests to prevent
  1312. # overwriting the account in PATH_INFO
  1313. del self.environ['s3api.auth_details']
  1314. def to_swift_req(self, method, container, obj, query=None,
  1315. body=None, headers=None):
  1316. sw_req = super(S3AclRequest, self).to_swift_req(
  1317. method, container, obj, query, body, headers)
  1318. if self.account:
  1319. sw_req.environ['swift_owner'] = True # needed to set ACL
  1320. sw_req.environ['swift.authorize_override'] = True
  1321. sw_req.environ['swift.authorize'] = lambda req: None
  1322. return sw_req
  1323. def get_acl_response(self, app, method=None, container=None, obj=None,
  1324. headers=None, body=None, query=None):
  1325. """
  1326. Wrapper method of _get_response to add s3 acl information
  1327. from response sysmeta headers.
  1328. """
  1329. resp = self._get_response(
  1330. app, method, container, obj, headers, body, query)
  1331. resp.bucket_acl = decode_acl(
  1332. 'container', resp.sysmeta_headers, self.allow_no_owner)
  1333. resp.object_acl = decode_acl(
  1334. 'object', resp.sysmeta_headers, self.allow_no_owner)
  1335. return resp
  1336. def get_response(self, app, method=None, container=None, obj=None,
  1337. headers=None, body=None, query=None):
  1338. """
  1339. Wrap up get_response call to hook with acl handling method.
  1340. """
  1341. if not self.acl_handler:
  1342. # we should set acl_handler all time before calling get_response
  1343. raise Exception('get_response called before set_acl_handler')
  1344. resp = self.acl_handler.handle_acl(
  1345. app, method, container, obj, headers)
  1346. # possible to skip recalling get_response_acl if resp is not
  1347. # None (e.g. HEAD)
  1348. if resp:
  1349. return resp
  1350. return self.get_acl_response(app, method, container, obj,
  1351. headers, body, query)
  1352. def set_acl_handler(self, acl_handler):
  1353. self.acl_handler = acl_handler
  1354. class SigV4Request(SigV4Mixin, S3Request):
  1355. pass
  1356. class SigV4S3AclRequest(SigV4Mixin, S3AclRequest):
  1357. pass