# Copyright (c) 2012-2013, Eucalyptus Systems, Inc. # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the # above copyright notice and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT # OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. from __future__ import absolute_import from functools import partial import logging import platform import sys import textwrap from . import __version__, EMPTY from .command import BaseCommand from .exceptions import ClientError, ServerError from .service import BaseService from .util import aggregate_subclass_fields from .xmlparse import parse_listdelimited_aws_xml class BaseRequest(BaseCommand): ''' The basis for a command line tool that represents a request. To invoke this as a command line tool, call the do_cli() method on an instance of the class; arguments will be parsed from the command line. To invoke this in another context, pass keyword args to __init__() with names that match those stored by the argument parser and then call main(). Important methods in this class include: - do_cli: command line entry point - main: pre/post-request processing and request sending - send: actually send a request to the server and return a response (called by the main() method) - print_result: format data from the main method and print it to stdout To be useful a tool should inherit from this class and implement the main() and print_result() methods. The do_cli() method functions as the entry point for the command line, populating self.args from the command line and then calling main() and print_result() in sequence. Other tools may instead supply arguments via __init__() and then call main() alone. Important members of this class include: - SERVICE_CLASS: a class corresponding to the web service in use - API_VERSION: the API version to send along with the request. This is only necessary to override the service class's API version for a specific request. - NAME: a string containing the Action query parameter. This defaults to the class's name. - DESCRIPTION: a string describing the tool. This becomes part of the command line help string. - ARGS: a list of Arg and/or MutuallyExclusiveArgGroup objects are used to generate command line arguments. Inheriting classes needing to add command line arguments should contain their own Args lists, which are *prepended* to those of their parent classes. - FILTERS: a list of Filter objects that are used to generate filter options at the command line. Inheriting classes needing to add filters should contain their own FILTERS lists, which are *prepended* to those of their parent classes. ''' SERVICE_CLASS = BaseService API_VERSION = None NAME = None FILTERS = [] LIST_MARKERS = [] def __init__(self, service=None, **kwargs): self.service = service # Parts of the HTTP request to be sent to the server. # Note that self.serialize_params will update self.params for each # entry in self.args that routes to PARAMS. self.headers = {} self.params = {} self.post_data = None self.method = 'GET' # HTTP response obtained from the server self.response = None self.__user_agent = None BaseCommand.__init__(self, **kwargs) def _post_init(self): if self.service is None: self.service = self.SERVICE_CLASS(self.config, self.log) BaseCommand._post_init(self) @property def default_route(self): return self.params def collect_arg_objs(self): request_args = BaseCommand.collect_arg_objs(self) service_args = self.service.collect_arg_objs() # Note that the service is likely to include auth args as well. return request_args + service_args def preprocess_arg_objs(self, arg_objs): self.service.preprocess_arg_objs(arg_objs) def populate_parser(self, parser, arg_objs): BaseCommand.populate_parser(self, parser, arg_objs) if self.FILTERS: parser.add_argument('--filter', metavar='NAME=VALUE', action='append', dest='filters', help='restrict results to those that meet criteria', type=partial(_parse_filter, filter_objs=self.FILTERS)) parser.epilog = self.__build_filter_help() self._arg_routes['filters'] = None def process_cli_args(self): BaseCommand.process_cli_args(self) if 'filters' in self.args: self.args['Filter'] = _process_filters(self.args.pop('filters')) self._arg_routes['Filter'] = self.params def configure(self): self.service.configure() @property def name(self): ''' The name of this action. Used when choosing what to supply for the Action query parameter. ''' return self.NAME or self.__class__.__name__ @property def user_agent(self): ''' Return a user-agent string for this program. ''' if not self.__user_agent: template = 'requestbuilder/{ver} ({os} {osver}; {python} {pyver})' self.__user_agent = template.format(ver=__version__, os=platform.uname()[0], osver=platform.uname()[2], python=platform.python_implementation(), pyver=platform.python_version()) return self.__user_agent @property def status(self): if self.response is not None: return self.response.status else: return None def serialize_params(self, args, prefix=None): ''' Given a possibly-nested dict of args and an arg routing destination, transform each element in the dict that matches the corresponding arg routing table into a simple dict containing key-value pairs suitable for use as query parameters. This implementation flattens dicts and lists into the format given by the EC2 query API, which uses dotted lists of dict keys and list indices to indicate nested structures. Keys with nonzero values that evaluate as false are ignored. If a collection of keys is supplied with ignore then keys that do not appear in that collection are also ignored. Examples: in: {'InstanceId': 'i-12345678', 'PublicIp': '1.2.3.4'} out: {'InstanceId': 'i-12345678', 'PublicIp': '1.2.3.4'} in: {'RegionName': ['us-east-1', 'us-west-1']} out: {'RegionName.1': 'us-east-1', 'RegionName.2': 'us-west-1'} in: {'Filter': [{'Name': 'image-id', 'Value': ['ami-12345678']}, {'Name': 'instance-type', 'Value': ['m1.small', 't1.micro']}], 'InstanceId': ['i-24680135']} out: {'Filter.1.Name': 'image-id', 'Filter.1.Value.1': 'ami-12345678', 'Filter.2.Name': 'instance-type', 'Filter.2.Value.1': 'm1.small', 'Filter.2.Value.2': 't1.micro', 'InstanceId.1': 'i-24680135'} ''' flattened = {} if args is None: return {} elif isinstance(args, dict): for (key, val) in args.iteritems(): # Prefix.Key1, Prefix.Key2, ... if prefix: prefixed_key = prefix + '.' + str(key) else: prefixed_key = str(key) if isinstance(val, dict) or isinstance(val, list): flattened.update(self.serialize_params(val, prefixed_key)) elif isinstance(val, file): flattened[prefixed_key] = val.read() elif val or val is 0: flattened[prefixed_key] = str(val) elif val is EMPTY: flattened[prefixed_key] = '' elif isinstance(args, list): for (i_item, item) in enumerate(args, 1): # Prefix.1, Prefix.2, ... if prefix: prefixed_key = prefix + '.' + str(i_item) else: prefixed_key = str(i_item) if isinstance(item, dict) or isinstance(item, list): flattened.update(self.serialize_params(item, prefixed_key)) elif isinstance(item, file): flattened[prefixed_key] = item.read() elif item or item == 0: flattened[prefixed_key] = str(item) elif val is EMPTY: flattened[prefixed_key] = '' else: raise TypeError('non-flattenable type: ' + args.__class__.__name__) return flattened def send(self): ''' Send a request to the server and return its response. More precisely: 1. Build a dict of params suitable for submission as HTTP request parameters, based first upon the content of self.params, and second upon everything in self.args that routes to PARAMS. 2. Send an HTTP request via self.service with the HTTP method given in self.method using query parameters from the aforementioned serialized dict, headers based on self.headers, and POST data based on self.post_data. 3. If the response's status code indicates success, parse the response with self.parse_response and return the result. 4. If the response's status code does not indicate success, log an error and raise a ServerError. ''' params = self.serialize_params(self.params) headers = dict(self.headers or {}) headers.setdefault('User-Agent', self.user_agent) self.log.info('parameters: %s', params) self.response = self.service.make_request(self.name, method=self.method, headers=headers, params=params, data=self.post_data, api_version=self.API_VERSION) try: if 200 <= self.response.status_code < 300: parsed = self.parse_response(self.response) self.log.info('result: success') return parsed else: self.log.debug('-- response content --\n', extra={'append': True}) self.log.debug(self.response.text, extra={'append': True}) self.log.debug('-- end of response content --') self.log.info('result: failure') raise ServerError(self.response.status_code, self.response.content) finally: # Empty the socket buffer so it can be reused try: self.response.content except RuntimeError: # The content was already consumed pass def parse_response(self, response): # Parser for list-delimited responses like EC2's # We do some extra handling here to log stuff as it comes in rather # than reading it all into memory at once. self.log.debug('-- response content --\n', extra={'append': True}) # Using Response.iter_content gives us automatic decoding, but we then # have to make the generator look like a file so etree can use it. with _IteratorFileObjAdapter(self.response.iter_content(16384)) \ as content_fileobj: logged_fileobj = _ReadLoggingFileWrapper(content_fileobj, self.log, logging.DEBUG) response_dict = parse_listdelimited_aws_xml(logged_fileobj, self.LIST_MARKERS) self.log.debug('-- end of response content --') # Strip off the root element return response_dict[list(response_dict.keys())[0]] def main(self): ''' The main processing method for this type of request. In this method, inheriting classes generally populate self.headers, self.params, and self.post_data with information gathered from self.args or elsewhere, call self.send, and return the response. BaseRequest's default behavior is to simply return the result of a request with everything that routes to PARAMS. ''' self.preprocess() response = self.send() self.postprocess(response) return response def preprocess(self): pass def postprocess(self, response): pass def handle_cli_exception(self, err): if isinstance(err, ServerError): if err.code: print >> sys.stderr, 'error ({code}) {msg}'.format( code=err.code, msg=err.message or '') else: print >> sys.stderr, 'error {msg}'.format( msg=err.message or '') if self.debug: raise sys.exit(1) else: BaseCommand.handle_cli_exception(self, err) def __build_filter_help(self, force=False): ''' Return a pre-formatted help string for all of the filters defined in self.FILTERS. The result is meant to be used as command line help output. ''' # Does not have access to self.config if '-h' not in sys.argv and '--help' not in sys.argv and not force: # Performance optimization return '' # FIXME: This code has a bug with triple-quoted strings that contain # embedded indentation. textwrap.dedent doesn't seem to help. # Reproducer: 'whether the volume will be deleted' max_len = 24 col_len = max([len(filter_obj.name) for filter_obj in self.FILTERS if len(filter_obj.name) < max_len]) - 1 helplines = ['allowed filter names:'] for filter_obj in self.FILTERS: if filter_obj.help: if len(filter_obj.name) <= col_len: # filter-name Description of the filter that # continues on the next line right_space = ' ' * (max_len - len(filter_obj.name) - 2) wrapper = textwrap.TextWrapper(fix_sentence_endings=True, initial_indent=(' ' + filter_obj.name + right_space), subsequent_indent=(' ' * max_len)) else: # really-long-filter-name # Description that begins on the next line helplines.append(' ' + filter_obj.name) wrapper = textwrap.TextWrapper(fix_sentence_endings=True, initial_indent=( ' ' * max_len), subsequent_indent=(' ' * max_len)) helplines.extend(wrapper.wrap(filter_obj.help)) else: helplines.append(' ' + filter_obj.name) return '\n'.join(helplines) def _parse_filter(filter_str, filter_objs=None): ''' Given a "key=value" string given as a command line parameter, return a pair with the matching filter's dest member and the given value after converting it to the type expected by the filter. If this is impossible, an ArgumentTypeError will result instead. ''' # Find the appropriate filter object filter_objs = [obj for obj in (filter_objs or []) if obj.matches_argval(filter_str)] if not filter_objs: msg = '"{0}" matches no available filters'.format(filter_str) raise argparse.ArgumentTypeError(msg) return filter_objs[0].convert(filter_str) def _process_filters(cli_filters): ''' Change filters from the [(key, value), ...] format given at the command line to [{'Name': key, 'Value': [value, ...]}, ...] format, which flattens to the form the server expects. ''' filter_args = {} # Compile [(key, value), ...] pairs into {key: [value, ...], ...} for (key, val) in cli_filters or {}: filter_args.setdefault(key, []) filter_args[key].append(val) # Build the flattenable [{'Name': key, 'Value': [value, ...]}, ...] filters = [{'Name': name, 'Value': values} for (name, values) in filter_args.iteritems()] return filters class _IteratorFileObjAdapter(object): def __init__(self, source): self._source = source self._buflist = [] self._closed = False self._len = 0 def __enter__(self): return self def __exit__(self, *args): self.close() @property def closed(self): return self._closed def close(self): if not self._closed: self.buflist = None self._closed = True def read(self, size=-1): if size is None or size < 0: for chunk in self._source: self._buflist.append(chunk) result = ''.join(self._buflist) self._buflist = [] self._len = 0 else: while self._len < size: try: chunk = next(self._source) self._buflist.append(chunk) self._len += len(chunk) except StopIteration: break result = ''.join(self._buflist) extra_len = len(result) - size self._buflist = [] self._len = 0 if extra_len > 0: self._buflist = [result[-extra_len:]] self._len = extra_len result = result[:-extra_len] return result class _ReadLoggingFileWrapper(object): def __init__(self, fileobj, logger, level): self.fileobj = fileobj self.logger = logger self.level = level def read(self, size=-1): chunk = self.fileobj.read(size) self.logger.log(self.level, chunk, extra={'append': True}) return chunk