heat : add heat-boto test client

Add initial version of the heat cli tool which uses boto
- revised following review comments to remove jeos/cfn paths
ref #92

Change-Id: I61b5815b250f3b01d33844ff46dd1612000d51fd
Signed-off-by: Steven Hardy <shardy@redhat.com>

bin/heat-boto

@@ -0,0 +1,711 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+This is the administration program for heat. It is simply a command-line
+interface for adding, modifying, and retrieving information about the stacks
+belonging to a user.  It is a convenience application that talks to the heat
+API server.
+"""
+
+import gettext
+import optparse
+import os
+import os.path
+import sys
+import time
+import json
+import logging
+
+from urlparse import urlparse
+
+# If ../heat/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(possible_topdir, 'heat', '__init__.py')):
+    sys.path.insert(0, possible_topdir)
+
+gettext.install('heat', unicode=1)
+
+import boto
+from heat import version
+from heat.common import config
+from heat.common import exception
+from heat import utils
+
+
+# FIXME : would be better if each of the boto response classes
+# implemented __str__ or a print method, so we could just print
+# them, avoiding all these print functions, boto patch TODO
+
+
+def print_stack_event(event):
+    '''
+    Print contents of a boto.cloudformation.stack.StackEvent object
+    '''
+    print "EventId : %s" % event.event_id
+    print "LogicalResourceId : %s" % event.logical_resource_id
+    print "PhysicalResourceId : %s" % event.physical_resource_id
+    print "ResourceProperties : %s" % event.resource_properties
+    print "ResourceStatus : %s" % event.resource_status
+    print "ResourceStatusReason : %s" % event.resource_status_reason
+    print "ResourceType : %s" % event.resource_type
+    print "StackId : %s" % event.stack_id
+    print "StackName : %s" % event.stack_name
+    print "Timestamp : %s" % event.timestamp
+    print "--"
+
+
+def print_stack(s):
+    '''
+    Print contents of a boto.cloudformation.stack.Stack object
+    '''
+    print "Capabilities : %s" % s.capabilities
+    print "CreationTime : %s" % s.creation_time
+    print "Description : %s" % s.description
+    print "DisableRollback : %s" % s.disable_rollback
+    # FIXME : boto doesn't populate this field, but AWS defines it.
+    # bit unclear because the docs say LastUpdatedTime, where all
+    # other response structures define LastUpdatedTimestamp
+    # need confirmation of real AWS format, probably a documentation bug..
+#    print "LastUpdatedTime : %s" % s.last_updated_time
+    print "NotificationARNs : %s" % s.notification_arns
+    print "Outputs : %s" % s.outputs
+    print "Parameters : %s" % s.parameters
+    print "StackId : %s" % s.stack_id
+    print "StackName : %s" % s.stack_name
+    print "StackStatus : %s" % s.stack_status
+    print "StackStatusReason : %s" % s.stack_status_reason
+    print "TimeoutInMinutes : %s" % s.timeout_in_minutes
+    print "--"
+
+
+def print_stack_resource(res):
+    '''
+    Print contents of a boto.cloudformation.stack.StackResource object
+    '''
+    print "LogicalResourceId : %s" % res.logical_resource_id
+    print "PhysicalResourceId : %s" % res.physical_resource_id
+    print "ResourceStatus : %s" % res.resource_status
+    print "ResourceStatusReason : %s" % res.resource_status_reason
+    print "ResourceType : %s" % res.resource_type
+    print "StackId : %s" % res.stack_id
+    print "StackName : %s" % res.stack_name
+    print "Timestamp : %s" % res.timestamp
+    print "--"
+
+
+def print_stack_resource_summary(res):
+    '''
+    Print contents of a boto.cloudformation.stack.StackResourceSummary object
+    '''
+    print "LastUpdatedTimestamp : %s" % res.last_updated_timestamp
+    print "LogicalResourceId : %s" % res.logical_resource_id
+    print "PhysicalResourceId : %s" % res.physical_resource_id
+    print "ResourceStatus : %s" % res.resource_status
+    print "ResourceStatusReason : %s" % res.resource_status_reason
+    print "ResourceType : %s" % res.resource_type
+    print "--"
+
+
+def print_stack_summary(s):
+    '''
+    Print contents of a boto.cloudformation.stack.StackSummary object
+    '''
+    print "StackId : %s" % s.stack_id
+    print "StackName : %s" % s.stack_name
+    print "CreationTime : %s" % s.creation_time
+    print "StackStatus : %s" % s.stack_status
+    print "TemplateDescription : %s" % s.template_description
+    print "--"
+
+
+@utils.catch_error('validate')
+def template_validate(options, arguments):
+    '''
+    Validate a template.  This command parses a template and verifies that
+    it is in the correct format.
+
+    Usage: heat validate \\
+        [--template-file=<template file>|--template-url=<template URL>] \\
+        [options]
+
+    --template-file: Specify a local template file.
+    --template-url:  Specify a URL pointing to a stack description template.
+    '''
+    parameters = {}
+    if options.template_file:
+        parameters['TemplateBody'] = open(options.template_file).read()
+    elif options.template_url:
+        parameters['TemplateUrl'] = options.template_url
+    else:
+        logging.error('Please specify a template file or url')
+        return utils.FAILURE
+
+    if options.parameters:
+        count = 1
+        for p in options.parameters.split(';'):
+            (n, v) = p.split('=')
+            parameters['Parameters.member.%d.ParameterKey' % count] = n
+            parameters['Parameters.member.%d.ParameterValue' % count] = v
+            count = count + 1
+
+    client = get_client(options)
+    result = client.validate_template(**parameters)
+    print result
+
+
+@utils.catch_error('estimatetemplatecost')
+def estimate_template_cost(options, arguments):
+    parameters = {}
+    try:
+        parameters['StackName'] = arguments.pop(0)
+    except IndexError:
+        logging.error("Please specify the stack name you wish to estimate")
+        logging.error("as the first argument")
+        return utils.FAILURE
+
+    if options.parameters:
+        count = 1
+        for p in options.parameters.split(';'):
+            (n, v) = p.split('=')
+            parameters['Parameters.member.%d.ParameterKey' % count] = n
+            parameters['Parameters.member.%d.ParameterValue' % count] = v
+            count = count + 1
+
+    if options.template_file:
+        parameters['TemplateBody'] = open(options.template_file).read()
+    elif options.template_url:
+        parameters['TemplateUrl'] = options.template_url
+    else:
+        logging.error('Please specify a template file or url')
+        return utils.FAILURE
+
+    c = get_client(options)
+    result = c.estimate_template_cost(**parameters)
+    print result
+
+
+@utils.catch_error('gettemplate')
+def get_template(options, arguments):
+    '''
+    Gets an existing stack template.
+    '''
+    if len(arguments):
+        stack_name = arguments.pop(0)
+    else:
+        logging.error("Please specify the stack you wish to get")
+        logging.error("as the first argument")
+        return utils.FAILURE
+
+    c = get_client(options)
+    result = c.get_template(stack_name)
+    print json.dumps(result)
+
+
+@utils.catch_error('create')
+def stack_create(options, arguments):
+    '''
+    Create a new stack from a template.
+
+    Usage: heat create <stack name> \\
+        [--template-file=<template file>|--template-url=<template URL>] \\
+        [options]
+
+    Stack Name: The user specified name of the stack you wish to create.
+
+    --template-file: Specify a local template file containing a valid
+                     stack description template.
+    --template-url:  Specify a URL pointing to a valid stack description
+                     template.
+    '''
+
+    if len(arguments):
+        stack_name = arguments.pop(0)
+    else:
+        logging.error("Please specify the stack name you wish to create")
+        logging.error("as the first argument")
+        return utils.FAILURE
+
+    params = []
+    if options.parameters:
+        for p in options.parameters.split(';'):
+            (n, v) = p.split('=')
+            # Boto expects parameters as a list-of-tuple
+            params.append((n, v))
+
+    result = None
+    c = get_client(options)
+    if options.template_file:
+        t = open(options.template_file).read()
+        result = c.create_stack(stack_name, template_body=t, parameters=params)
+    elif options.template_url:
+        result = c.create_stack(stack_name, template_url=options.template_url)
+    else:
+        logging.error('Please specify a template file or url')
+        return utils.FAILURE
+
+    print result
+
+
+@utils.catch_error('update')
+def stack_update(options, arguments):
+    '''
+    Update an existing stack.
+
+    Usage: heat update <stack name> \\
+        [--template-file=<template file>|--template-url=<template URL>] \\
+        [options]
+
+    Stack Name: The name of the stack you wish to modify.
+
+    --template-file: Specify a local template file containing a valid
+                     stack description template.
+    --template-url:  Specify a URL pointing to a valid stack description
+                     template.
+
+    Options:
+        --parameters: A list of key/value pairs separated by ';'s used
+                      to specify allowed values in the template file.
+    '''
+    parameters = {}
+    if len(arguments):
+        stack_name = arguments.pop(0)
+    else:
+        logging.error("Please specify the stack name you wish to update")
+        logging.error("as the first argument")
+        return utils.FAILURE
+
+    result = None
+    c = get_client(options)
+    if options.template_file:
+        t = open(options.template_file).read()
+        result = c.update_stack(stack_name, template_body=t)
+    elif options.template_url:
+        t = options.template_url
+        result = c.update_stack(stack_name, template_url=t)
+
+    if options.parameters:
+        count = 1
+        for p in options.parameters.split(';'):
+            (n, v) = p.split('=')
+            parameters['Parameters.member.%d.ParameterKey' % count] = n
+            parameters['Parameters.member.%d.ParameterValue' % count] = v
+            count = count + 1
+
+    print result
+
+
+@utils.catch_error('delete')
+def stack_delete(options, arguments):
+    '''
+    Delete an existing stack.  This shuts down all VMs associated with
+    the stack and (perhaps wrongly) also removes all events associated
+    with the given stack.
+
+    Usage: heat delete <stack name>
+    '''
+    if len(arguments):
+        stack_name = arguments.pop(0)
+    else:
+        logging.error("Please specify the stack name you wish to delete")
+        logging.error("as the first argument")
+        return utils.FAILURE
+
+    c = get_client(options)
+    result = c.delete_stack(stack_name)
+    print result
+
+
+@utils.catch_error('describe')
+def stack_describe(options, arguments):
+    '''
+    Describes an existing stack.
+
+    Usage: heat describe <stack name>
+    '''
+    if len(arguments):
+        stack_name = arguments.pop(0)
+    else:
+        logging.info("No stack name passed, getting results for ALL stacks")
+        stack_name = None
+
+    c = get_client(options)
+    result = c.describe_stacks(stack_name)
+    for s in result:
+        print_stack(s)
+
+
+@utils.catch_error('event-list')
+def stack_events_list(options, arguments):
+    '''
+    List events associated with the given stack.
+
+    Usage: heat event-list <stack name>
+    '''
+    if len(arguments):
+        stack_name = arguments.pop(0)
+    else:
+        logging.info("No stack name passed, getting events for ALL stacks")
+        stack_name = None
+
+    c = get_client(options)
+    result = c.describe_stack_events(stack_name)
+    for e in result:
+        print_stack_event(e)
+
+
+@utils.catch_error('resource')
+def stack_resource_show(options, arguments):
+    '''
+    Display details of the specified resource.
+    '''
+    c = get_client(options)
+    try:
+        stack_name, resource_name = arguments
+    except ValueError:
+        print 'Enter stack name and logical resource id'
+        return
+
+    result = c.describe_stack_resources(stack_name, resource_name)
+    for r in result:
+        print_stack_resource(r)
+
+
+@utils.catch_error('resource-list')
+def stack_resources_list(options, arguments):
+    '''
+    Display summary of all resources in the specified stack.
+    '''
+    c = get_client(options)
+    try:
+        stack_name = arguments.pop(0)
+    except IndexError:
+        print 'Enter stack name'
+        return
+
+    result = c.list_stack_resources(stack_name)
+    for r in result:
+        print_stack_resource_summary(r)
+
+
+@utils.catch_error('resource-list-details')
+def stack_resources_list_details(options, arguments):
+    '''
+    Display details of all resources in the specified stack.
+    '''
+    c = get_client(options)
+    logical_resource_id = arguments.pop(0) if arguments else None
+    if not options.stack_name and not options.physical_resource_id:
+        logging.error(
+            "Must specify either stack-name physical-resource-id")
+        return
+    result = c.describe_stack_resources(options.stack_name,
+        logical_resource_id, options.physical_resource_id)
+    for r in result:
+        print_stack_resource(r)
+
+
+@utils.catch_error('list')
+def stack_list(options, arguments):
+    '''
+    List all running stacks.
+
+    Usage: heat list
+    '''
+    c = get_client(options)
+    result = c.list_stacks()
+    for s in result:
+        print_stack_summary(s)
+
+
+def get_client(options):
+    """
+    Returns a new boto client object to a heat server
+    """
+
+    logging.debug("boto config expected in /etc/boto.cfg, reading contents")
+
+    # Retrieve the credentials from the config file
+    # FIXME : send patch to boto so it retrives these credentials from its
+    # config file, we shouldn't need to manually parse it...
+    access_key_id = boto.config.get_value('Credentials', 'aws_access_key_id')
+    secret_access_key = boto.config.get_value('Credentials',
+        'aws_secret_access_key')
+    if (access_key_id and secret_access_key):
+        logging.debug("Got credentials id=%s secret=%s" % (access_key_id,
+            secret_access_key))
+    else:
+        logging.error(
+            "Failed to get required credentials, please pass in /etc/boto.cfg")
+
+    # FIXME : reopen boto pull-request so that --host can override the
+    # host specified in the hostname by passing it via the constructor
+    # I implemented this previously, but they preferred the config-file
+    # solution..
+    cloudformation = boto.connect_cloudformation(
+        aws_access_key_id=access_key_id,
+        aws_secret_access_key=secret_access_key,
+        debug=options.debug, is_secure=False, port=options.port, path="/v1")
+    if cloudformation:
+        logging.debug("Got CF connection object OK")
+    else:
+        logging.error("Error establishing connection!")
+        sys.exit(1)
+
+    return cloudformation
+
+
+def create_options(parser):
+    """
+    Sets up the CLI and config-file options that may be
+    parsed and program commands.
+
+    :param parser: The option parser
+    """
+    parser.add_option('-v', '--verbose', default=False, action="store_true",
+                      help="Print more verbose output")
+    parser.add_option('-d', '--debug', default=False, action="store_true",
+                      help="Print more verbose output")
+    parser.add_option('-y', '--yes', default=False, action="store_true",
+                      help="Don't prompt for user input; assume the answer to "
+                           "every question is 'yes'.")
+    parser.add_option('-H', '--host', metavar="ADDRESS", default="0.0.0.0",
+                      help="Address of heat API host. "
+                           "Default: %default")
+    parser.add_option('-p', '--port', dest="port", metavar="PORT",
+                      type=int, default=config.DEFAULT_PORT,
+                      help="Port the heat API host listens on. "
+                           "Default: %default")
+    parser.add_option('-U', '--url', metavar="URL", default=None,
+                      help="URL of heat service. This option can be used "
+                           "to specify the hostname, port and protocol "
+                           "(http/https) of the heat server, for example "
+                           "-U https://localhost:" + str(config.DEFAULT_PORT) +
+                           "/v1 Default: No<F3>ne")
+    parser.add_option('-k', '--insecure', dest="insecure",
+                      default=False, action="store_true",
+                      help="Explicitly allow heat to perform \"insecure\" "
+                      "SSL (https) requests. The server's certificate will "
+                      "not be verified against any certificate authorities. "
+                      "This option should be used with caution.")
+    parser.add_option('-A', '--auth_token', dest="auth_token",
+                      metavar="TOKEN", default=None,
+                      help="Authentication token to use to identify the "
+                           "client to the heat server")
+    parser.add_option('-I', '--username', dest="username",
+                      metavar="USER", default=None,
+                      help="User name used to acquire an authentication token")
+    parser.add_option('-K', '--password', dest="password",
+                      metavar="PASSWORD", default=None,
+                      help="Password used to acquire an authentication token")
+    parser.add_option('-T', '--tenant', dest="tenant",
+                      metavar="TENANT", default=None,
+                      help="Tenant name used for Keystone authentication")
+    parser.add_option('-R', '--region', dest="region",
+                      metavar="REGION", default=None,
+                      help="Region name. When using keystone authentication "
+                      "version 2.0 or later this identifies the region "
+                      "name to use when selecting the service endpoint. A "
+                      "region name must be provided if more than one "
+                      "region endpoint is available")
+    parser.add_option('-N', '--auth_url', dest="auth_url",
+                      metavar="AUTH_URL", default=None,
+                      help="Authentication URL")
+    parser.add_option('-S', '--auth_strategy', dest="auth_strategy",
+                      metavar="STRATEGY", default=None,
+                      help="Authentication strategy (keystone or noauth)")
+
+    parser.add_option('-u', '--template-url', metavar="template_url",
+                      default=None, help="URL of template. Default: None")
+    parser.add_option('-f', '--template-file', metavar="template_file",
+                      default=None, help="Path to the template. Default: None")
+    parser.add_option('-t', '--timeout', metavar="timeout",
+                      default='60',
+                      help='Stack creation timeout in minutes. Default: 60')
+
+    parser.add_option('-P', '--parameters', metavar="parameters", default=None,
+                      help="Parameter values used to create the stack.")
+    parser.add_option('-n', '--stack-name', default=None,
+                      help="Name of the queried stack")
+    parser.add_option('-c', '--physical-resource-id', default=None,
+                      help="Physical ID of the queried resource")
+
+
+def credentials_from_env():
+    return dict(username=os.getenv('OS_USERNAME'),
+                password=os.getenv('OS_PASSWORD'),
+                tenant=os.getenv('OS_TENANT_NAME'),
+                auth_url=os.getenv('OS_AUTH_URL'),
+                auth_strategy=os.getenv('OS_AUTH_STRATEGY'))
+
+
+def parse_options(parser, cli_args):
+    """
+    Returns the parsed CLI options, command to run and its arguments, merged
+    with any same-named options found in a configuration file
+
+    :param parser: The option parser
+    """
+    if not cli_args:
+        cli_args.append('-h')  # Show options in usage output...
+
+    (options, args) = parser.parse_args(cli_args)
+    env_opts = credentials_from_env()
+    for option, env_val in env_opts.items():
+        if not getattr(options, option):
+            setattr(options, option, env_val)
+
+    if options.url is not None:
+        u = urlparse(options.url)
+        options.port = u.port
+        options.host = u.hostname
+
+    if not options.auth_strategy:
+        options.auth_strategy = 'noauth'
+
+    options.use_ssl = (options.url is not None and u.scheme == 'https')
+
+    # HACK(sirp): Make the parser available to the print_help method
+    # print_help is a command, so it only accepts (options, args); we could
+    # one-off have it take (parser, options, args), however, for now, I think
+    # this little hack will suffice
+    options.__parser = parser
+
+    if not args:
+        parser.print_usage()
+        sys.exit(0)
+
+    command_name = args.pop(0)
+    command = lookup_command(parser, command_name)
+
+    if options.debug:
+        logging.basicConfig(format='%(levelname)s:%(message)s',
+            level=logging.DEBUG)
+        logging.debug("Debug level logging enabled")
+    elif options.verbose:
+        logging.basicConfig(format='%(levelname)s:%(message)s',
+            level=logging.INFO)
+    else:
+        logging.basicConfig(format='%(levelname)s:%(message)s',
+            level=logging.WARNING)
+
+    return (options, command, args)
+
+
+def print_help(options, args):
+    """
+    Print help specific to a command
+    """
+    parser = options.__parser
+
+    if not args:
+        parser.print_usage()
+
+    subst = {'prog': os.path.basename(sys.argv[0])}
+    docs = [lookup_command(parser, cmd).__doc__ % subst for cmd in args]
+    print '\n\n'.join(docs)
+
+
+def lookup_command(parser, command_name):
+    base_commands = {'help': print_help}
+
+    stack_commands = {
+                'create': stack_create,
+                'update': stack_update,
+                'delete': stack_delete,
+                'list': stack_list,
+                'event-list': stack_events_list,
+                'resource': stack_resource_show,
+                'resource-list': stack_resources_list,
+                'resource-list-details': stack_resources_list_details,
+                'validate': template_validate,
+                'gettemplate': get_template,
+                'estimate-template-cost': estimate_template_cost,
+                'describe': stack_describe}
+
+    commands = {}
+    for command_set in (base_commands, stack_commands):
+        commands.update(command_set)
+
+    try:
+        command = commands[command_name]
+    except KeyError:
+        parser.print_usage()
+        sys.exit("Unknown command: %s" % command_name)
+
+    return command
+
+
+def main():
+    '''
+    '''
+    usage = """
+%prog <command> [options] [args]
+
+Commands:
+
+    help <command>  Output help for one of the commands below
+
+    create          Create the stack
+
+    delete          Delete the stack
+
+    describe        Describe the stack
+
+    update          Update the stack
+
+    list            List the user's stacks
+
+    gettemplate     Get the template
+
+    estimate-template-cost     Returns the estimated monthly cost of a template
+
+    validate        Validate a template
+
+    event-list      List events for a stack
+
+    resource        Describe the resource
+
+    resource-list   Show list of resources belonging to a stack
+
+    resource-list-details    Detailed view of resources belonging to a stack
+
+"""
+
+    oparser = optparse.OptionParser(version='%%prog %s'
+                                    % version.version_string(),
+                                    usage=usage.strip())
+    create_options(oparser)
+    (opts, cmd, args) = parse_options(oparser, sys.argv[1:])
+
+    try:
+        start_time = time.time()
+        result = cmd(opts, args)
+        end_time = time.time()
+        logging.debug("Completed in %-0.4f sec." % (end_time - start_time))
+        sys.exit(result)
+    except (RuntimeError,
+            NotImplementedError,
+            exception.ClientConfigurationError), ex:
+        oparser.print_usage()
+        logging.error("ERROR: " % ex)
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

etc/boto.cfg

@@ -0,0 +1,16 @@
+[Credentials]
+# AWS credentials, from keystone ec2-credentials-list
+aws_access_key_id = YOUR_KEY
+aws_secret_access_key = YOUR_SECKEY
+
+[Boto]
+# Make boto output verbose debugging information
+debug = 0
+
+# Override the default AWS endpoint to connect to heat on localhost
+cfn_region_name = heat
+cfn_region_endpoint = 127.0.0.1
+
+# Set the client retries to 1, or errors connecting to heat repeat
+# which is not useful when debugging API issues
+num_retries = 1

run_tests.sh

@@ -64,7 +64,7 @@ function run_tests {
 function run_pep8 {
   echo "Running pep8 ..."
   PEP8_OPTIONS="--exclude=$PEP8_EXCLUDE --repeat"
-  PEP8_INCLUDE="bin/heat bin/heat-api bin/heat-engine heat tools setup.py heat/testing/runner.py"
+  PEP8_INCLUDE="bin/heat bin/heat-boto bin/heat-api bin/heat-engine heat tools setup.py heat/testing/runner.py"
   ${wrapper} pep8 $PEP8_OPTIONS $PEP8_INCLUDE
 }
 

setup.py

@@ -47,6 +47,7 @@ setuptools.setup(
     ],
     scripts=['bin/heat',
              'bin/heat-api',
+             'bin/heat-boto',
              'bin/heat-metadata',
              'bin/heat-engine',
              'bin/heat-db-setup'],