OpenStack Storage (Swift)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

469 lines
18 KiB

  1. # Copyright (c) 2011 OpenStack Foundation
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12. # implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. r"""
  16. FormPost Middleware
  17. Translates a browser form post into a regular Swift object PUT.
  18. The format of the form is::
  19. <form action="<swift-url>" method="POST"
  20. enctype="multipart/form-data">
  21. <input type="hidden" name="redirect" value="<redirect-url>" />
  22. <input type="hidden" name="max_file_size" value="<bytes>" />
  23. <input type="hidden" name="max_file_count" value="<count>" />
  24. <input type="hidden" name="expires" value="<unix-timestamp>" />
  25. <input type="hidden" name="signature" value="<hmac>" />
  26. <input type="file" name="file1" /><br />
  27. <input type="submit" />
  28. </form>
  29. Optionally, if you want the uploaded files to be temporary you can set
  30. x-delete-at or x-delete-after attributes by adding one of these as a
  31. form input::
  32. <input type="hidden" name="x_delete_at" value="<unix-timestamp>" />
  33. <input type="hidden" name="x_delete_after" value="<seconds>" />
  34. If you want to specify the content type or content encoding of the files you
  35. can set content-encoding or content-type by adding them to the form input::
  36. <input type="hidden" name="content-type" value="text/html" />
  37. <input type="hidden" name="content-encoding" value="gzip" />
  38. The above example applies these parameters to all uploaded files. You can also
  39. set the content-type and content-encoding on a per-file basis by adding the
  40. parameters to each part of the upload.
  41. The <swift-url> is the URL of the Swift destination, such as::
  42. https://swift-cluster.example.com/v1/AUTH_account/container/object_prefix
  43. The name of each file uploaded will be appended to the <swift-url>
  44. given. So, you can upload directly to the root of container with a
  45. url like::
  46. https://swift-cluster.example.com/v1/AUTH_account/container/
  47. Optionally, you can include an object prefix to better separate
  48. different users' uploads, such as::
  49. https://swift-cluster.example.com/v1/AUTH_account/container/object_prefix
  50. Note the form method must be POST and the enctype must be set as
  51. "multipart/form-data".
  52. The redirect attribute is the URL to redirect the browser to after the upload
  53. completes. This is an optional parameter. If you are uploading the form via an
  54. XMLHttpRequest the redirect should not be included. The URL will have status
  55. and message query parameters added to it, indicating the HTTP status code for
  56. the upload (2xx is success) and a possible message for further information if
  57. there was an error (such as "max_file_size exceeded").
  58. The max_file_size attribute must be included and indicates the
  59. largest single file upload that can be done, in bytes.
  60. The max_file_count attribute must be included and indicates the
  61. maximum number of files that can be uploaded with the form. Include
  62. additional ``<input type="file" name="filexx" />`` attributes if
  63. desired.
  64. The expires attribute is the Unix timestamp before which the form
  65. must be submitted before it is invalidated.
  66. The signature attribute is the HMAC-SHA1 signature of the form. Here is
  67. sample code for computing the signature::
  68. import hmac
  69. from hashlib import sha1
  70. from time import time
  71. path = '/v1/account/container/object_prefix'
  72. redirect = 'https://srv.com/some-page' # set to '' if redirect not in form
  73. max_file_size = 104857600
  74. max_file_count = 10
  75. expires = int(time() + 600)
  76. key = 'mykey'
  77. hmac_body = '%s\n%s\n%s\n%s\n%s' % (path, redirect,
  78. max_file_size, max_file_count, expires)
  79. signature = hmac.new(key, hmac_body, sha1).hexdigest()
  80. The key is the value of either the account (X-Account-Meta-Temp-URL-Key,
  81. X-Account-Meta-Temp-Url-Key-2) or the container
  82. (X-Container-Meta-Temp-URL-Key, X-Container-Meta-Temp-Url-Key-2) TempURL keys.
  83. Be certain to use the full path, from the /v1/ onward.
  84. Note that x_delete_at and x_delete_after are not used in signature generation
  85. as they are both optional attributes.
  86. The command line tool ``swift-form-signature`` may be used (mostly
  87. just when testing) to compute expires and signature.
  88. Also note that the file attributes must be after the other attributes
  89. in order to be processed correctly. If attributes come after the
  90. file, they won't be sent with the subrequest (there is no way to
  91. parse all the attributes on the server-side without reading the whole
  92. thing into memory -- to service many requests, some with large files,
  93. there just isn't enough memory on the server, so attributes following
  94. the file are simply ignored).
  95. """
  96. __all__ = ['FormPost', 'filter_factory', 'READ_CHUNK_SIZE', 'MAX_VALUE_LENGTH']
  97. import hmac
  98. from hashlib import sha1
  99. from time import time
  100. import six
  101. from six.moves.urllib.parse import quote
  102. from swift.common.constraints import valid_api_version
  103. from swift.common.exceptions import MimeInvalid
  104. from swift.common.middleware.tempurl import get_tempurl_keys_from_metadata
  105. from swift.common.utils import streq_const_time, register_swift_info, \
  106. parse_content_disposition, parse_mime_headers, \
  107. iter_multipart_mime_documents, reiterate, close_if_possible
  108. from swift.common.wsgi import make_pre_authed_env
  109. from swift.common.swob import HTTPUnauthorized, wsgi_to_str, str_to_wsgi
  110. from swift.proxy.controllers.base import get_account_info, get_container_info
  111. #: The size of data to read from the form at any given time.
  112. READ_CHUNK_SIZE = 4096
  113. #: The maximum size of any attribute's value. Any additional data will be
  114. #: truncated.
  115. MAX_VALUE_LENGTH = 4096
  116. class FormInvalid(Exception):
  117. pass
  118. class FormUnauthorized(Exception):
  119. pass
  120. class _CappedFileLikeObject(object):
  121. """
  122. A file-like object wrapping another file-like object that raises
  123. an EOFError if the amount of data read exceeds a given
  124. max_file_size.
  125. :param fp: The file-like object to wrap.
  126. :param max_file_size: The maximum bytes to read before raising an
  127. EOFError.
  128. """
  129. def __init__(self, fp, max_file_size):
  130. self.fp = fp
  131. self.max_file_size = max_file_size
  132. self.amount_read = 0
  133. self.file_size_exceeded = False
  134. def read(self, size=None):
  135. ret = self.fp.read(size)
  136. self.amount_read += len(ret)
  137. if self.amount_read > self.max_file_size:
  138. self.file_size_exceeded = True
  139. raise EOFError('max_file_size exceeded')
  140. return ret
  141. def readline(self):
  142. ret = self.fp.readline()
  143. self.amount_read += len(ret)
  144. if self.amount_read > self.max_file_size:
  145. self.file_size_exceeded = True
  146. raise EOFError('max_file_size exceeded')
  147. return ret
  148. class FormPost(object):
  149. """
  150. FormPost Middleware
  151. See above for a full description.
  152. The proxy logs created for any subrequests made will have swift.source set
  153. to "FP".
  154. :param app: The next WSGI filter or app in the paste.deploy
  155. chain.
  156. :param conf: The configuration dict for the middleware.
  157. """
  158. def __init__(self, app, conf):
  159. #: The next WSGI application/filter in the paste.deploy pipeline.
  160. self.app = app
  161. #: The filter configuration dict.
  162. self.conf = conf
  163. def __call__(self, env, start_response):
  164. """
  165. Main hook into the WSGI paste.deploy filter/app pipeline.
  166. :param env: The WSGI environment dict.
  167. :param start_response: The WSGI start_response hook.
  168. :returns: Response as per WSGI.
  169. """
  170. if env['REQUEST_METHOD'] == 'POST':
  171. try:
  172. content_type, attrs = \
  173. parse_content_disposition(env.get('CONTENT_TYPE') or '')
  174. if content_type == 'multipart/form-data' and \
  175. 'boundary' in attrs:
  176. http_user_agent = "%s FormPost" % (
  177. env.get('HTTP_USER_AGENT', ''))
  178. env['HTTP_USER_AGENT'] = http_user_agent.strip()
  179. status, headers, body = self._translate_form(
  180. env, attrs['boundary'])
  181. start_response(status, headers)
  182. return [body]
  183. except MimeInvalid:
  184. body = b'FormPost: invalid starting boundary'
  185. start_response(
  186. '400 Bad Request',
  187. (('Content-Type', 'text/plain'),
  188. ('Content-Length', str(len(body)))))
  189. return [body]
  190. except (FormInvalid, EOFError) as err:
  191. body = 'FormPost: %s' % err
  192. if six.PY3:
  193. body = body.encode('utf-8')
  194. start_response(
  195. '400 Bad Request',
  196. (('Content-Type', 'text/plain'),
  197. ('Content-Length', str(len(body)))))
  198. return [body]
  199. except FormUnauthorized as err:
  200. message = 'FormPost: %s' % str(err).title()
  201. return HTTPUnauthorized(body=message)(
  202. env, start_response)
  203. return self.app(env, start_response)
  204. def _translate_form(self, env, boundary):
  205. """
  206. Translates the form data into subrequests and issues a
  207. response.
  208. :param env: The WSGI environment dict.
  209. :param boundary: The MIME type boundary to look for.
  210. :returns: status_line, headers_list, body
  211. """
  212. keys = self._get_keys(env)
  213. if six.PY3:
  214. boundary = boundary.encode('utf-8')
  215. status = message = ''
  216. attributes = {}
  217. subheaders = []
  218. file_count = 0
  219. for fp in iter_multipart_mime_documents(
  220. env['wsgi.input'], boundary, read_chunk_size=READ_CHUNK_SIZE):
  221. hdrs = parse_mime_headers(fp)
  222. disp, attrs = parse_content_disposition(
  223. hdrs.get('Content-Disposition', ''))
  224. if disp == 'form-data' and attrs.get('filename'):
  225. file_count += 1
  226. try:
  227. if file_count > int(attributes.get('max_file_count') or 0):
  228. status = '400 Bad Request'
  229. message = 'max file count exceeded'
  230. break
  231. except ValueError:
  232. raise FormInvalid('max_file_count not an integer')
  233. attributes['filename'] = attrs['filename'] or 'filename'
  234. if 'content-type' not in attributes and 'content-type' in hdrs:
  235. attributes['content-type'] = \
  236. hdrs['Content-Type'] or 'application/octet-stream'
  237. if 'content-encoding' not in attributes and \
  238. 'content-encoding' in hdrs:
  239. attributes['content-encoding'] = hdrs['Content-Encoding']
  240. status, subheaders = \
  241. self._perform_subrequest(env, attributes, fp, keys)
  242. if not status.startswith('2'):
  243. break
  244. else:
  245. data = b''
  246. mxln = MAX_VALUE_LENGTH
  247. while mxln:
  248. chunk = fp.read(mxln)
  249. if not chunk:
  250. break
  251. mxln -= len(chunk)
  252. data += chunk
  253. while fp.read(READ_CHUNK_SIZE):
  254. pass
  255. if six.PY3:
  256. data = data.decode('utf-8')
  257. if 'name' in attrs:
  258. attributes[attrs['name'].lower()] = data.rstrip('\r\n--')
  259. if not status:
  260. status = '400 Bad Request'
  261. message = 'no files to process'
  262. headers = [(k, v) for k, v in subheaders
  263. if k.lower().startswith('access-control')]
  264. redirect = attributes.get('redirect')
  265. if not redirect:
  266. body = status
  267. if message:
  268. body = status + '\r\nFormPost: ' + message.title()
  269. headers.extend([('Content-Type', 'text/plain'),
  270. ('Content-Length', len(body))])
  271. if six.PY3:
  272. body = body.encode('utf-8')
  273. return status, headers, body
  274. status = status.split(' ', 1)[0]
  275. if '?' in redirect:
  276. redirect += '&'
  277. else:
  278. redirect += '?'
  279. redirect += 'status=%s&message=%s' % (quote(status), quote(message))
  280. body = '<html><body><p><a href="%s">' \
  281. 'Click to continue...</a></p></body></html>' % redirect
  282. if six.PY3:
  283. body = body.encode('utf-8')
  284. headers.extend(
  285. [('Location', redirect), ('Content-Length', str(len(body)))])
  286. return '303 See Other', headers, body
  287. def _perform_subrequest(self, orig_env, attributes, fp, keys):
  288. """
  289. Performs the subrequest and returns the response.
  290. :param orig_env: The WSGI environment dict; will only be used
  291. to form a new env for the subrequest.
  292. :param attributes: dict of the attributes of the form so far.
  293. :param fp: The file-like object containing the request body.
  294. :param keys: The account keys to validate the signature with.
  295. :returns: (status_line, headers_list)
  296. """
  297. if not keys:
  298. raise FormUnauthorized('invalid signature')
  299. try:
  300. max_file_size = int(attributes.get('max_file_size') or 0)
  301. except ValueError:
  302. raise FormInvalid('max_file_size not an integer')
  303. subenv = make_pre_authed_env(orig_env, 'PUT', agent=None,
  304. swift_source='FP')
  305. if 'QUERY_STRING' in subenv:
  306. del subenv['QUERY_STRING']
  307. subenv['HTTP_TRANSFER_ENCODING'] = 'chunked'
  308. subenv['wsgi.input'] = _CappedFileLikeObject(fp, max_file_size)
  309. if not subenv['PATH_INFO'].endswith('/') and \
  310. subenv['PATH_INFO'].count('/') < 4:
  311. subenv['PATH_INFO'] += '/'
  312. subenv['PATH_INFO'] += str_to_wsgi(
  313. attributes['filename'] or 'filename')
  314. if 'x_delete_at' in attributes:
  315. try:
  316. subenv['HTTP_X_DELETE_AT'] = int(attributes['x_delete_at'])
  317. except ValueError:
  318. raise FormInvalid('x_delete_at not an integer: '
  319. 'Unix timestamp required.')
  320. if 'x_delete_after' in attributes:
  321. try:
  322. subenv['HTTP_X_DELETE_AFTER'] = int(
  323. attributes['x_delete_after'])
  324. except ValueError:
  325. raise FormInvalid('x_delete_after not an integer: '
  326. 'Number of seconds required.')
  327. if 'content-type' in attributes:
  328. subenv['CONTENT_TYPE'] = \
  329. attributes['content-type'] or 'application/octet-stream'
  330. if 'content-encoding' in attributes:
  331. subenv['HTTP_CONTENT_ENCODING'] = attributes['content-encoding']
  332. try:
  333. if int(attributes.get('expires') or 0) < time():
  334. raise FormUnauthorized('form expired')
  335. except ValueError:
  336. raise FormInvalid('expired not an integer')
  337. hmac_body = '%s\n%s\n%s\n%s\n%s' % (
  338. wsgi_to_str(orig_env['PATH_INFO']),
  339. attributes.get('redirect') or '',
  340. attributes.get('max_file_size') or '0',
  341. attributes.get('max_file_count') or '0',
  342. attributes.get('expires') or '0')
  343. if six.PY3:
  344. hmac_body = hmac_body.encode('utf-8')
  345. has_valid_sig = False
  346. for key in keys:
  347. # Encode key like in swift.common.utls.get_hmac.
  348. if not isinstance(key, six.binary_type):
  349. key = key.encode('utf8')
  350. sig = hmac.new(key, hmac_body, sha1).hexdigest()
  351. if streq_const_time(sig, (attributes.get('signature') or
  352. 'invalid')):
  353. has_valid_sig = True
  354. if not has_valid_sig:
  355. raise FormUnauthorized('invalid signature')
  356. substatus = [None]
  357. subheaders = [None]
  358. wsgi_input = subenv['wsgi.input']
  359. def _start_response(status, headers, exc_info=None):
  360. if wsgi_input.file_size_exceeded:
  361. raise EOFError("max_file_size exceeded")
  362. substatus[0] = status
  363. subheaders[0] = headers
  364. # reiterate to ensure the response started,
  365. # but drop any data on the floor
  366. close_if_possible(reiterate(self.app(subenv, _start_response)))
  367. return substatus[0], subheaders[0]
  368. def _get_keys(self, env):
  369. """
  370. Returns the X-[Account|Container]-Meta-Temp-URL-Key[-2] header values
  371. for the account or container, or an empty list if none are set.
  372. Returns 0-4 elements depending on how many keys are set in the
  373. account's or container's metadata.
  374. Also validate that the request
  375. path indicates a valid container; if not, no keys will be returned.
  376. :param env: The WSGI environment for the request.
  377. :returns: list of tempurl keys
  378. """
  379. parts = env['PATH_INFO'].split('/', 4)
  380. if len(parts) < 4 or parts[0] or not valid_api_version(parts[1]) \
  381. or not parts[2] or not parts[3]:
  382. return []
  383. account_info = get_account_info(env, self.app, swift_source='FP')
  384. account_keys = get_tempurl_keys_from_metadata(account_info['meta'])
  385. container_info = get_container_info(env, self.app, swift_source='FP')
  386. container_keys = get_tempurl_keys_from_metadata(
  387. container_info.get('meta', []))
  388. return account_keys + container_keys
  389. def filter_factory(global_conf, **local_conf):
  390. """Returns the WSGI filter for use with paste.deploy."""
  391. conf = global_conf.copy()
  392. conf.update(local_conf)
  393. register_swift_info('formpost')
  394. return lambda app: FormPost(app, conf)