511 lines
22 KiB
Python
511 lines
22 KiB
Python
# Copyright (c) 2012-2015, Eucalyptus Systems, Inc.
|
|
#
|
|
# Permission to use, copy, modify, and/or distribute this software for
|
|
# any purpose with or without fee is hereby granted, provided that the
|
|
# above copyright notice and this permission notice appear in all copies.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
|
|
# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
|
|
|
from __future__ import absolute_import
|
|
|
|
import base64
|
|
import calendar
|
|
import datetime
|
|
import email.utils
|
|
import hashlib
|
|
import hmac
|
|
import os
|
|
import re
|
|
import time
|
|
import warnings
|
|
|
|
import six
|
|
import six.moves.urllib_parse as urlparse
|
|
|
|
from requestbuilder import Arg
|
|
from requestbuilder.auth import BaseAuth
|
|
from requestbuilder.exceptions import AuthError
|
|
|
|
|
|
ISO8601 = '%Y-%m-%dT%H:%M:%SZ'
|
|
ISO8601_BASIC = '%Y%m%dT%H%M%SZ'
|
|
|
|
|
|
class HmacKeyAuth(BaseAuth):
|
|
'''
|
|
Basis for AWS HMAC-based authentication
|
|
'''
|
|
ARGS = [Arg('-I', '--access-key-id', dest='key_id', metavar='KEY_ID'),
|
|
Arg('-S', '--secret-key', dest='secret_key', metavar='KEY'),
|
|
Arg('--security-token', dest='security_token', metavar='TOKEN')]
|
|
|
|
@classmethod
|
|
def from_other(cls, other, **kwargs):
|
|
kwargs.setdefault('loglevel', other.log.level)
|
|
kwargs.setdefault('key_id', other.args.get('key_id'))
|
|
kwargs.setdefault('secret_key', other.args.get('secret_key'))
|
|
kwargs.setdefault('security_token', other.args.get('security_token'))
|
|
new = cls(other.config, **kwargs)
|
|
new.configure()
|
|
return new
|
|
|
|
def configure(self):
|
|
# If the current user/region was explicitly set (e.g. with --region),
|
|
# use that first
|
|
self.configure_from_configfile(only_if_explicit=True)
|
|
# Try the environment next
|
|
self.args['key_id'] = (self.args.get('key_id') or
|
|
os.getenv('AWS_ACCESS_KEY_ID') or
|
|
os.getenv('AWS_ACCESS_KEY'))
|
|
self.args['secret_key'] = (self.args.get('secret_key') or
|
|
os.getenv('AWS_SECRET_ACCESS_KEY') or
|
|
os.getenv('AWS_SECRET_KEY'))
|
|
self.args['security_token'] = (self.args.get('security_token') or
|
|
os.getenv('AWS_SECURITY_TOKEN'))
|
|
# See if an AWS credential file was given in the environment
|
|
self.configure_from_aws_credential_file()
|
|
# Try the requestbuilder config file next
|
|
self.configure_from_configfile()
|
|
|
|
if not self.args.get('key_id'):
|
|
raise AuthError('missing access key ID; please supply one with -I')
|
|
if not self.args.get('secret_key'):
|
|
raise AuthError('missing secret key; please supply one with -S')
|
|
|
|
def configure_from_aws_credential_file(self):
|
|
if 'AWS_CREDENTIAL_FILE' in os.environ:
|
|
path = os.getenv('AWS_CREDENTIAL_FILE')
|
|
path = os.path.expandvars(path)
|
|
path = os.path.expanduser(path)
|
|
with open(path) as credfile:
|
|
for line in credfile:
|
|
line = line.split('#', 1)[0]
|
|
if '=' in line:
|
|
(key, val) = line.split('=', 1)
|
|
if (key.strip() == 'AWSAccessKeyId' and
|
|
not self.args.get('key_id')):
|
|
# There's probably a better way to do this, but it
|
|
# seems to work for me. Patches are welcome. :)
|
|
self.args['key_id'] = val.strip()
|
|
elif (key.strip() == 'AWSSecretKey' and
|
|
not self.args.get('secret_key')):
|
|
self.args['secret_key'] = val.strip()
|
|
|
|
def configure_from_configfile(self, only_if_explicit=False):
|
|
if only_if_explicit and not self.args.get('region'): # Somewhat hacky
|
|
# The current user/region were not explicitly set, so do nothing.
|
|
return
|
|
if not self.args.get('key_id'):
|
|
config_key_id = self.config.get_user_option('key-id')
|
|
if config_key_id:
|
|
self.args['key_id'] = config_key_id
|
|
if not self.args.get('secret_key'):
|
|
config_secret_key = self.config.get_user_option('secret-key',
|
|
redact=True)
|
|
if config_secret_key:
|
|
self.args['secret_key'] = config_secret_key
|
|
|
|
|
|
class HmacV1Auth(HmacKeyAuth):
|
|
'''
|
|
S3 REST authentication
|
|
http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html
|
|
'''
|
|
|
|
# This list comes from the CanonicalizedResource section of the above page
|
|
HASHED_PARAMS = set((
|
|
'acl', 'lifecycle', 'location', 'logging', 'notification',
|
|
'partNumber', 'policy', 'requestPayment', 'torrent', 'uploadId',
|
|
'uploads', 'versionId', 'versioning', 'versions', 'website'))
|
|
|
|
def apply_to_request(self, req, service):
|
|
self._update_request_before_signing(req)
|
|
c_headers = self.get_canonicalized_headers(req)
|
|
c_resource = self.get_canonicalized_resource(req, service)
|
|
to_sign = self._get_string_to_sign(req, c_headers, c_resource)
|
|
self.log.debug('string to sign: %s', repr(to_sign))
|
|
signature = self.sign_string(to_sign.encode('utf-8'))
|
|
self.log.debug('b64-encoded signature: %s', signature)
|
|
self._apply_signature(req, signature)
|
|
return req
|
|
|
|
def apply_to_request_params(self, req, service, expiration_datetime):
|
|
# This does not implement security tokens.
|
|
msg = ('S3RestAuth.apply_to_request_params is deprecated; use '
|
|
'S3.get_request_url with S3QueryAuth instead')
|
|
self.log.warn(msg)
|
|
warnings.warn(msg, DeprecationWarning)
|
|
|
|
for param in ('AWSAccessKeyId', 'Expires', 'Signature'):
|
|
req.params.pop(param, None)
|
|
|
|
expiration = calendar.timegm(expiration_datetime.utctimetuple())
|
|
delta_t = expiration_datetime - datetime.datetime.utcnow()
|
|
delta_t_sec = ((delta_t.microseconds +
|
|
(delta_t.seconds + delta_t.days * 24 * 3600) * 10**6)
|
|
/ 10**6)
|
|
self.log.debug('expiration: %i (%f seconds from now)',
|
|
expiration, delta_t_sec)
|
|
c_headers = self.get_canonicalized_headers(req)
|
|
self.log.debug('canonicalized headers: %s', repr(c_headers))
|
|
c_resource = self.get_canonicalized_resource(req, service)
|
|
self.log.debug('canonicalized resource: %s', repr(c_resource))
|
|
to_sign = '\n'.join((req.method,
|
|
req.headers.get('Content-MD5', ''),
|
|
req.headers.get('Content-Type', ''),
|
|
six.text_type(expiration),
|
|
c_headers + c_resource))
|
|
self.log.debug('string to sign: %s', repr(to_sign))
|
|
signature = self.sign_string(to_sign.encode('utf-8'))
|
|
self.log.debug('b64-encoded signature: %s', signature)
|
|
req.params['AWSAccessKeyId'] = self.args['key_id']
|
|
req.params['Expires'] = six.text_type(expiration)
|
|
req.params['Signature'] = signature
|
|
if self.args.get('security_token'):
|
|
# This is a guess. I have no evidence that this actually works.
|
|
req.params['SecurityToken'] = self.args['security_token']
|
|
|
|
def _update_request_before_signing(self, req):
|
|
if not req.headers:
|
|
req.headers = {}
|
|
req.headers['Date'] = email.utils.formatdate()
|
|
req.headers['Host'] = urlparse.urlparse(req.url).netloc
|
|
if self.args.get('security_token'):
|
|
req.headers['x-amz-security-token'] = self.args['security_token']
|
|
req.headers.pop('Signature', None)
|
|
|
|
def _get_string_to_sign(self, req, c_headers, c_resource):
|
|
return '\n'.join((req.method.upper(),
|
|
req.headers.get('Content-MD5', ''),
|
|
req.headers.get('Content-Type', ''),
|
|
req.headers.get('Date'),
|
|
c_headers + c_resource))
|
|
|
|
def _apply_signature(self, req, signature):
|
|
req.headers['Authorization'] = 'AWS {0}:{1}'.format(
|
|
self.args['key_id'], signature)
|
|
|
|
def get_canonicalized_resource(self, req, service):
|
|
# /bucket/keyname
|
|
parsed_req_path = urlparse.urlparse(req.url).path
|
|
assert service.endpoint is not None
|
|
parsed_svc_path = urlparse.urlparse(service.endpoint).path
|
|
# IMPORTANT: this only supports path-style requests
|
|
assert parsed_req_path.startswith(parsed_svc_path)
|
|
resource = parsed_req_path[len(parsed_svc_path):]
|
|
if parsed_svc_path.endswith('/'):
|
|
# The leading / got stripped off
|
|
resource = '/' + resource
|
|
|
|
# Now append sub-resources, a.k.a. query string parameters
|
|
if getattr(req, 'params', None):
|
|
# A regular Request
|
|
params = req.params
|
|
else:
|
|
# A PreparedRequest
|
|
params = _get_params_from_url(req.url)
|
|
if params:
|
|
subresources = []
|
|
for key, val in sorted(params.iteritems()):
|
|
if key in self.HASHED_PARAMS:
|
|
if val is None:
|
|
subresources.append(key)
|
|
else:
|
|
print '{0}={1}'.format(key, val), key + '=' + val
|
|
subresources.append(key + '=' + val)
|
|
if subresources:
|
|
resource += '?' + '&'.join(subresources)
|
|
self.log.debug('canonicalized resource: %s', repr(resource))
|
|
return resource
|
|
|
|
def get_canonicalized_headers(self, req):
|
|
headers_dict = {}
|
|
for key, val in req.headers.iteritems():
|
|
if key.lower().startswith('x-amz-'):
|
|
headers_dict.setdefault(key.lower(), [])
|
|
headers_dict[key.lower()].append(' '.join(val.split()))
|
|
headers_strs = []
|
|
for key, vals in sorted(headers_dict.iteritems()):
|
|
headers_strs.append('{0}:{1}'.format(key, ','.join(vals)))
|
|
if headers_strs:
|
|
c_headers = '\n'.join(headers_strs) + '\n'
|
|
else:
|
|
c_headers = ''
|
|
self.log.debug('canonicalized headers: %s', repr(c_headers))
|
|
return c_headers
|
|
|
|
def sign_string(self, to_sign):
|
|
req_hmac = hmac.new(self.args['secret_key'], digestmod=hashlib.sha1)
|
|
req_hmac.update(to_sign)
|
|
return base64.b64encode(req_hmac.digest())
|
|
|
|
|
|
class QueryHmacV1Auth(HmacV1Auth):
|
|
DEFAULT_TIMEOUT = 600 # 10 minutes
|
|
|
|
def _update_request_before_signing(self, req):
|
|
timeout = int(self.args.get('timeout')) or self.DEFAULT_TIMEOUT
|
|
assert timeout > 0
|
|
params = _get_params_from_url(req.url)
|
|
params['AWSAccessKeyId'] = self.args['key_id']
|
|
params['Expires'] = int(time.time() + timeout)
|
|
params.pop('Signature', None)
|
|
req.prepare_url(_remove_params_from_url(req.url), params)
|
|
|
|
def _get_string_to_sign(self, req, c_headers, c_resource):
|
|
params = _get_params_from_url(req.url)
|
|
return '\n'.join((req.method.upper(),
|
|
req.headers.get('Content-MD5', ''),
|
|
req.headers.get('Content-Type', ''),
|
|
params['Expires'],
|
|
c_headers + c_resource))
|
|
|
|
def _apply_signature(self, req, signature):
|
|
req.prepare_url(req.url, {'Signature': signature})
|
|
|
|
|
|
class QueryHmacV2Auth(HmacKeyAuth):
|
|
'''
|
|
AWS signature version 2
|
|
http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
|
|
'''
|
|
|
|
def apply_to_request(self, req, service):
|
|
parsed = urlparse.urlparse(req.url)
|
|
if req.method == 'POST':
|
|
# This is probably going to break when given multipart data.
|
|
params = urlparse.parse_qs(req.body or '', keep_blank_values=True)
|
|
else:
|
|
params = urlparse.parse_qs(parsed.query, keep_blank_values=True)
|
|
params = dict((key, vals[0]) for key, vals in params.iteritems())
|
|
params['AWSAccessKeyId'] = self.args['key_id']
|
|
params['SignatureVersion'] = 2
|
|
params['SignatureMethod'] = 'HmacSHA256'
|
|
params['Timestamp'] = time.strftime(ISO8601, time.gmtime())
|
|
if self.args.get('security_token'):
|
|
params['SecurityToken'] = self.args['security_token']
|
|
# Needed for retries so old signatures aren't included in to_sign
|
|
params.pop('Signature', None)
|
|
to_sign = '{method}\n{host}\n{path}\n'.format(
|
|
method=req.method, host=parsed.netloc.lower(),
|
|
path=(parsed.path or '/'))
|
|
quoted_params = []
|
|
for key in sorted(params):
|
|
val = six.text_type(params[key])
|
|
quoted_params.append(urlparse.quote(key, safe='') + '=' +
|
|
urlparse.quote(val, safe='-_~'))
|
|
query_string = '&'.join(quoted_params)
|
|
to_sign += query_string
|
|
# Redact passwords
|
|
redacted_to_sign = re.sub('assword=[^&]*', 'assword=<redacted>',
|
|
to_sign)
|
|
self.log.debug('string to sign: %s', repr(redacted_to_sign))
|
|
signature = self.sign_string(to_sign)
|
|
self.log.debug('b64-encoded signature: %s', signature)
|
|
params['Signature'] = signature
|
|
if req.method == 'POST':
|
|
req.prepare_body(params, {})
|
|
else:
|
|
req.prepare_url(_remove_params_from_url(req.url), params)
|
|
|
|
return req
|
|
|
|
def sign_string(self, to_sign):
|
|
req_hmac = hmac.new(self.args['secret_key'], digestmod=hashlib.sha256)
|
|
req_hmac.update(to_sign)
|
|
return base64.b64encode(req_hmac.digest())
|
|
|
|
|
|
class HmacV4Auth(HmacKeyAuth):
|
|
"""
|
|
AWS signature version 4
|
|
http://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
|
|
"""
|
|
|
|
def apply_to_request(self, req, service):
|
|
if not service.region_name:
|
|
self.log.error('a region name is required to use sigv4')
|
|
raise AuthError('region name is required; use a config file '
|
|
'or prepend the region name and :: to the URL')
|
|
if not service.NAME:
|
|
self.log.critical('service class %s must have a NAME attribute '
|
|
'to use sigv4', service.__class__.__name__)
|
|
raise AuthError('BUG: service class {0} does not have a name'
|
|
.format(service.__class__.__name__))
|
|
payload_hash = self._hash_payload(req) # large files will be slow here
|
|
now = time.time()
|
|
date_header = time.strftime(ISO8601_BASIC, time.gmtime(now))
|
|
scope = self._build_scope(service, now)
|
|
credential = '/'.join((self.args['key_id'],) + scope)
|
|
self._update_request_before_signing(req, credential, payload_hash,
|
|
date_header)
|
|
|
|
c_uri = self._get_canonical_uri(req)
|
|
c_query = self._get_canonical_query(req)
|
|
c_headers = self._get_canonical_headers(req)
|
|
s_headers = self._get_signed_headers(req)
|
|
c_request = '\n'.join((req.method.upper(), c_uri, c_query, c_headers,
|
|
'', s_headers, payload_hash))
|
|
self.log.debug('canonical request: %s', repr(c_request))
|
|
|
|
to_sign = '\n'.join(('AWS4-HMAC-SHA256', date_header, '/'.join(scope),
|
|
hashlib.sha256(c_request).hexdigest()))
|
|
# Redact passwords
|
|
redacted_to_sign = re.sub('assword=[^&]*', 'assword=<redacted>',
|
|
to_sign)
|
|
self.log.debug('string to sign: %s', repr(redacted_to_sign))
|
|
|
|
derived_hmac = hmac.new('AWS4{0}'.format(self.args['secret_key']),
|
|
digestmod=hashlib.sha256)
|
|
for chunk in scope:
|
|
derived_hmac.update(chunk)
|
|
derived_hmac = hmac.new(derived_hmac.digest(),
|
|
digestmod=hashlib.sha256)
|
|
derived_hmac.update(to_sign)
|
|
signature = derived_hmac.hexdigest()
|
|
self.log.debug('signature: %s', signature)
|
|
self._apply_signature(req, credential, signature)
|
|
return req
|
|
|
|
def _update_request_before_signing(self, req, credential, payload_sha256,
|
|
date_header):
|
|
parsed = urlparse.urlparse(req.url)
|
|
req.headers['Host'] = parsed.netloc
|
|
req.headers.pop('Authorization', None)
|
|
req.headers['X-Amz-Content-SHA256'] = payload_sha256
|
|
req.headers['X-Amz-Date'] = date_header
|
|
|
|
def _apply_signature(self, req, credential, signature):
|
|
auth_header = ', '.join((
|
|
'AWS4-HMAC-SHA256 Credential={0}'.format(credential),
|
|
'SignedHeaders={0}'.format(self._get_signed_headers(req)),
|
|
'Signature={0}'.format(signature)))
|
|
req.headers['Authorization'] = auth_header
|
|
|
|
def _build_scope(self, service, timestamp):
|
|
scope = (time.strftime('%Y%m%d', time.gmtime(timestamp)),
|
|
service.region_name, service.NAME, 'aws4_request')
|
|
self.log.debug('scope: %s', '/'.join(scope))
|
|
return scope
|
|
|
|
def _get_canonical_uri(self, req):
|
|
path = urlparse.urlsplit(req.url).path or '/'
|
|
# TODO: Normalize stuff like ".."
|
|
c_uri = urlparse.quote(path, safe='/~')
|
|
self.log.debug('canonical URI: %s', c_uri)
|
|
return c_uri
|
|
|
|
def _get_canonical_query(self, req):
|
|
req_params = urlparse.parse_qsl(urlparse.urlparse(req.url).query,
|
|
keep_blank_values=True)
|
|
params = []
|
|
for key, val in sorted(req_params or []):
|
|
params.append('='.join((urlparse.quote(key, safe='~-_.'),
|
|
urlparse.quote(val, safe='~-_.'))))
|
|
c_params = '&'.join(params)
|
|
self.log.debug('canonical query: %s', c_params)
|
|
return c_params
|
|
|
|
def _get_normalized_headers(self, req):
|
|
# This doesn't currently support multi-value headers.
|
|
headers = {}
|
|
for key, val in req.headers.iteritems():
|
|
headers[key.lower().strip()] = val.strip()
|
|
return headers
|
|
|
|
def _get_canonical_headers(self, req):
|
|
headers = []
|
|
normalized_headers = self._get_normalized_headers(req)
|
|
for key, val in sorted(normalized_headers.items()):
|
|
headers.append(':'.join((key, val)))
|
|
self.log.debug('canonical headers: %s', str(headers))
|
|
return '\n'.join(headers)
|
|
|
|
def _get_signed_headers(self, req):
|
|
normalized_headers = self._get_normalized_headers(req)
|
|
s_headers = ';'.join(sorted(normalized_headers))
|
|
self.log.debug('signed headers: %s', s_headers)
|
|
return s_headers
|
|
|
|
def _hash_payload(self, req):
|
|
if self.args.get('payload_hash'):
|
|
return self.args['payload_hash']
|
|
digest = hashlib.sha256()
|
|
if not req.body:
|
|
pass
|
|
elif hasattr(req.body, 'seek'):
|
|
body_position = req.data.tell()
|
|
self.log.debug('payload hashing starting')
|
|
while True:
|
|
chunk = req.body.read(16384)
|
|
if not chunk:
|
|
break
|
|
digest.update(chunk)
|
|
req.body.seek(body_position)
|
|
self.log.debug('payload hashing done')
|
|
elif hasattr(req.body, 'read'):
|
|
self.log.debug('payload spooling/hashing starting')
|
|
# 10M happens to be the size of a bundle part, the thing we upload
|
|
# most frequently.
|
|
spool = tempfile.SpooledTemporaryFile(max_size=(10 * 1024 * 1024))
|
|
while True:
|
|
chunk = req.body.read(16384)
|
|
if not chunk:
|
|
break
|
|
digest.update(chunk)
|
|
spool.write(chunk)
|
|
self.log.debug('payload spooling/hashing done')
|
|
spool.seek(0)
|
|
self.log.info('re-pointing request body at spooled payload')
|
|
req.body = spool
|
|
# Should we close the original req.body here?
|
|
else:
|
|
digest.update(req.body)
|
|
self.log.debug('payload hash: %s', digest.hexdigest())
|
|
return digest.hexdigest()
|
|
|
|
|
|
class QueryHmacV4Auth(HmacV4Auth):
|
|
def _update_request_before_signing(self, req, credential, payload_sha256,
|
|
date_header):
|
|
# We don't do anything with payload_sha256. Is that bad?
|
|
parsed = urlparse.urlparse(req.url)
|
|
req.headers['Host'] = parsed.netloc
|
|
req.headers.pop('Authorization', None)
|
|
params = {
|
|
'X-Amz-Algorithm': 'AWS4-HMAC-SHA256',
|
|
'X-Amz-Credential': credential,
|
|
'X-Amz-Date': date_header,
|
|
'X-Amz-SignedHeaders': self._get_signed_headers(req)}
|
|
if self.args.get('timeout'):
|
|
params['X-Amz-Expires'] = self.args['timeout']
|
|
req.prepare_url(req.url, params)
|
|
|
|
def _apply_signature(self, req, credential, signature):
|
|
req.prepare_url(req.url, {'X-Amz-Signature': signature})
|
|
|
|
|
|
def _get_params_from_url(url):
|
|
"""
|
|
Given a URL, return a dict of parameters and their values. If a
|
|
parameter appears more than once all but the first value will be lost.
|
|
"""
|
|
parsed = urlparse.urlparse(url)
|
|
params = urlparse.parse_qs(parsed.query, keep_blank_values=True)
|
|
return dict((key, vals[0]) for key, vals in params.iteritems())
|
|
|
|
|
|
def _remove_params_from_url(url):
|
|
"""
|
|
Return a copy of a URL with its parameters, fragments, and query
|
|
string removed.
|
|
"""
|
|
parsed = urlparse.urlparse(url)
|
|
return urlparse.urlunparse((parsed[0], parsed[1], parsed[2], '', '', ''))
|