openstack
/
heat
Code Issues Proposed changes
Browse Source

Remove cfntools and jeos 

These files are now available in the heat-jeos repository.

Change-Id: I392e7443348a31e8454ae14d957b0b54560c2ec3
Signed-off-by: Steven Dake <sdake@redhat.com>
changes/40/140/1
Steven Dake 10 years ago
parent
4b5c3fee7b
commit
4c359db1b0
21 changed files with 14 additions and 2089 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 14
      MANIFEST.in
  2. 25
      bin/heat
  3. 28
      docs/GettingStarted.rst
  4. 13
      heat/cfntools/README
  5. 0
      heat/cfntools/__init__.py
  6. 91
      heat/cfntools/cfn-get-metadata
  7. 114
      heat/cfntools/cfn-hup
  8. 74
      heat/cfntools/cfn-init
  9. 169
      heat/cfntools/cfn-push-stats
  10. 91
      heat/cfntools/cfn-signal
  11. 893
      heat/cfntools/cfn_helper.py
  12. 28
      heat/jeos/F16-i386-cfntools-jeos.tdl
  13. 60
      heat/jeos/F16-i386-gold-jeos.tdl
  14. 28
      heat/jeos/F16-x86_64-cfntools-jeos.tdl
  15. 60
      heat/jeos/F16-x86_64-gold-jeos.tdl
  16. 28
      heat/jeos/F17-i386-cfntools-jeos.tdl
  17. 60
      heat/jeos/F17-i386-gold-jeos.tdl
  18. 28
      heat/jeos/F17-x86_64-cfntools-jeos.tdl
  19. 60
      heat/jeos/F17-x86_64-gold-jeos.tdl
  20. 25
      heat/jeos/U10-amd64-cfntools-jeos.tdl
  21. 214
      heat/utils.py

14
MANIFEST.in
Unescape View File

@ -7,20 +7,6 @@ include MANIFEST.in pylintrc
include openstack-common.conf
include babel.cfg
graft templates
include heat/jeos/F16-i386-gold-jeos.tdl
include heat/jeos/F17-i386-gold-jeos.tdl
include heat/jeos/F16-i386-cfntools-jeos.tdl
include heat/jeos/F17-i386-cfntools-jeos.tdl
include heat/jeos/F16-x86_64-gold-jeos.tdl
include heat/jeos/F17-x86_64-gold-jeos.tdl
include heat/jeos/F16-x86_64-cfntools-jeos.tdl
include heat/jeos/F17-x86_64-cfntools-jeos.tdl
include heat/jeos/U10-amd64-cfntools-jeos.tdl
include heat/cfntools/cfn-init
include heat/cfntools/cfn-hup
include heat/cfntools/cfn-signal
include heat/cfntools/cfn-get-metadata
include heat/cfntools/cfn-push-stats
include heat/cloudinit/config
include heat/cloudinit/part-handler.py
include heat/db/sqlalchemy/migrate_repo/migrate.cfg

25
bin/heat
Unescape View File

@ -270,25 +270,6 @@ def stack_list(options, arguments):
print json.dumps(result, indent=2)
@utils.catch_error('jeos-create')
def jeos_create(options, arguments):
'''
Create a new JEOS (Just Enough Operating System) image.
Usage: heat jeos-create <distribution> <architecture> <image type>
Distribution: Distribution such as 'F16', 'F17', 'U10', 'D6'.
Architecture: Architecture such as 'i386' 'i686' or 'x86_64'.
Image Type: Image type such as 'gold' or 'cfntools'.
'gold' is a basic gold JEOS.
'cfntools' contains the cfntools helper scripts.
The command must be run as root in order for libvirt to have permissions
to create virtual machines and read the raw DVDs.
'''
utils.jeos_create(options, arguments, jeos_path, cfntools_path)
def get_client(options):
"""
Returns a new client object to a heat server
@ -466,9 +447,7 @@ def lookup_command(parser, command_name):
'event-list': stack_events_list,
'validate': template_validate,
'gettemplate': get_template,
'describe': stack_describe,
'jeos_create': jeos_create, # DEPRECATED
'jeos-create': jeos_create}
'describe': stack_describe}
commands = {}
for command_set in (base_commands, stack_commands):
@ -507,8 +486,6 @@ Commands:
validate Validate a template
jeos-create Create a JEOS image
event-list List events for a stack
"""

28
docs/GettingStarted.rst
Unescape View File

@ -121,29 +121,27 @@ This is for Heat to associate with the virtual machines.
nova keypair-add --pub_key ~/.ssh/id_rsa.pub ${USER}_key
Install Oz
----------
Verify that Oz_ is installed ::
Download and install heat_jeos via git
--------------------------------------
Download heat_jeos via git
sudo yum -y install oz
Oz is used below to create the JEOS.
.. _Oz: http://aeolusproject.org/oz.html
Create a JEOS
-------------
::
git clone git://github.com/heat-api/heat-jeos.git
cd heat-jeos
setup.py install
Create a JEOS with heat_jeos tools
----------------------------------
::
sudo -E heat -y jeos-create F16 x86_64 cfntools
sudo -E heat-jeos -y create F16 x86_64 cfntools
Note: The ``-E`` option to ``sudo`` preserves the environment, specifically the keystone credentials, when ``jeos-create`` is run as root.
Note: The ``-E`` option to ``sudo`` preserves the environment, specifically the keystone credentials, when ``heat-jeos`` is run as root.
Note: ``jeos-create`` must be run as root in order to create the cfntools disk image.
Note: ``heat-jeos`` must be run as root in order to create the cfntools disk image.
Note: If you want to enable debugging output from Oz, add '``-d``' (debugging) to the ``jeos-create`` command.
Note: If you want to enable debugging output from Oz, add '``-d``' (debugging) to the ``heat-jeos`` command.
Verify JEOS registration
~~~~~~~~~~~~~~~~~~~~~~~~

13
heat/cfntools/README
Unescape View File

@ -1,13 +0,0 @@
There are several bootstrap methods for cloudformations:
1. Create image with application ready to go
2. Use cloud-init to run a startup script passed as userdata to the nova
server create
3. Use the CloudFormation instance helper scripts
This directory contains files required for choice #3.
cfn-init - Reads the AWS::CloudFormation::Init for the instance resource,
installs packages, and starts services
cfn-signal - Waits for an application to be ready before continuing, ie:
supporting the WaitCondition feature
cfn-hup - Handle updates from the UpdateStack CloudFormation API call

0
heat/cfntools/__init__.py
Unescape View File

91
heat/cfntools/cfn-get-metadata
Unescape View File

@ -1,91 +0,0 @@
#!/usr/bin/env python
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Implements cfn-get-metadata CloudFormation functionality
"""
import argparse
import io
import logging
import os
import os.path
import sys
if os.path.exists('/opt/aws/bin'):
sys.path.insert(0, '/opt/aws/bin')
from cfn_helper import *
else:
from heat.cfntools.cfn_helper import *
description = " "
parser = argparse.ArgumentParser(description=description)
parser.add_argument('-s', '--stack',
dest="stack_name",
help="A Heat stack name",
required=True)
parser.add_argument('-r', '--resource',
dest="logical_resource_id",
help="A Heat logical resource ID",
required=True)
parser.add_argument('--access-key',
dest="access_key",
help="A Keystone access key",
required=False)
parser.add_argument('--secret-key',
dest="secret_key",
help="A Keystone secret key",
required=False)
parser.add_argument('--region',
dest="region",
help="Openstack region",
required=False)
parser.add_argument('--credential-file',
dest="credential_file",
help="credential-file",
required=False)
parser.add_argument('-u', '--url',
dest="url",
help="service url",
required=False)
parser.add_argument('-k', '--key',
dest="key",
help="key",
required=False)
args = parser.parse_args()
if not args.stack_name:
print 'The Stack name must not be empty.'
exit(1)
if not args.logical_resource_id:
print 'The Resource ID must not be empty'
exit(1)
log_format = '%(levelname)s [%(asctime)s] %(message)s'
logging.basicConfig(format=log_format, level=logging.DEBUG)
logger = logging.getLogger('cfn-get-metadata')
log_file_name = "/var/log/cfn-get-metadata.log"
file_handler = logging.FileHandler(log_file_name)
file_handler.setFormatter(logging.Formatter(log_format))
logger.addHandler(file_handler)
metadata = Metadata(args.stack_name,
args.logical_resource_id,
access_key=args.access_key,
secret_key=args.secret_key,
region=args.region)
metadata.retrieve()
print str(metadata)

114
heat/cfntools/cfn-hup
Unescape View File

@ -1,114 +0,0 @@
#!/usr/bin/env python
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Implements cfn-hup CloudFormation functionality
"""
import argparse
import io
import logging
import os
import os.path
import sys
if os.path.exists('/opt/aws/bin'):
sys.path.insert(0, '/opt/aws/bin')
from cfn_helper import *
else:
from heat.cfntools.cfn_helper import *
description = " "
parser = argparse.ArgumentParser(description=description)
parser.add_argument('-c', '--config',
dest="config_dir",
help="Hook Config Directory",
required=False,
default='/etc/cfn/hooks.d')
parser.add_argument('-f', '--no-daemon',
dest="no_deamon",
action="store_true",
help="Do not run as a deamon",
required=False)
parser.add_argument('-v', '--verbose',
action="store_true",
dest="verbose",
help="Verbose logging",
required=False)
args = parser.parse_args()
log_format = '%(levelname)s [%(asctime)s] %(message)s'
if args.verbose:
logging.basicConfig(format=log_format, level=logging.DEBUG)
else:
logging.basicConfig(format=log_format, level=logging.INFO)
logger = logging.getLogger('cfntools')
log_file_name = "/var/log/cfn-hup.log"
file_handler = logging.FileHandler(log_file_name)
file_handler.setFormatter(logging.Formatter(log_format))
logger.addHandler(file_handler)
main_conf_path = '/etc/cfn/cfn-hup.conf'
try:
main_config_file = open(main_conf_path)
except IOError as exc:
logger.error('Could not open main configuration at %s' % main_conf_path)
exit(1)
config_files = []
hooks_conf_path = '/etc/cfn/hooks.conf'
if os.path.exists(hooks_conf_path):
try:
config_files.append(open(hooks_conf_path))
except IOError as exc:
logger.exception(exc)
if args.config_dir and os.path.exists(args.config_dir):
try:
for f in os.listdir(args.config_dir):
config_files.append(open(os.path.join(args.config_dir, f)))
except OSError as exc:
logger.exception(exc)
if not config_files:
logger.error('No hook files found at %s or %s' % (hooks_conf_path,
args.config_dir))
exit(1)
try:
mainconfig = HupConfig([main_config_file] + config_files)
except Exception as ex:
logger.error('Cannot load configuration: %s' % str(ex))
exit(1)
if not mainconfig.unique_resources_get():
logger.error('No hooks were found. Add some to %s or %s' % (hooks_conf_path,
args.config_dir))
exit(1)
for r in mainconfig.unique_resources_get():
print r
metadata = Metadata(mainconfig.stack,
r,
credentials_file=mainconfig.credential_file,
region=mainconfig.region)
metadata.retrieve()
try:
metadata.cfn_hup(mainconfig.hooks)
except Exception as e:
logger.exception("Error processing metadata")
exit(1)

74
heat/cfntools/cfn-init
Unescape View File

@ -1,74 +0,0 @@
#!/usr/bin/python
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Implements cfn-init CloudFormation functionality
"""
import argparse
import logging
import os
import sys
if os.path.exists('/opt/aws/bin'):
sys.path.insert(0, '/opt/aws/bin')
from cfn_helper import *
else:
from heat.cfntools.cfn_helper import *
description = " "
parser = argparse.ArgumentParser(description=description)
parser.add_argument('-s', '--stack',
dest="stack_name",
help="A Heat stack name",
required=False)
parser.add_argument('-r', '--resource',
dest="logical_resource_id",
help="A Heat logical resource ID",
required=False)
parser.add_argument('--access-key',
dest="access_key",
help="A Keystone access key",
required=False)
parser.add_argument('--secret-key',
dest="secret_key",
help="A Keystone secret key",
required=False)
parser.add_argument('--region',
dest="region",
help="Openstack region",
required=False)
args = parser.parse_args()
log_format = '%(levelname)s [%(asctime)s] %(message)s'
logging.basicConfig(format=log_format, level=logging.DEBUG)
logger = logging.getLogger('cfn-init')
log_file_name = "/var/log/cfn-init.log"
file_handler = logging.FileHandler(log_file_name)
file_handler.setFormatter(logging.Formatter(log_format))
logger.addHandler(file_handler)
metadata = Metadata(args.stack_name,
args.logical_resource_id,
access_key=args.access_key,
secret_key=args.secret_key,
region=args.region)
metadata.retrieve()
try:
metadata.cfn_init()
except Exception as e:
logger.exception("Error processing metadata")
exit(1)

169
heat/cfntools/cfn-push-stats
Unescape View File

@ -1,169 +0,0 @@
#!/usr/bin/env python
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Implements cfn-signal CloudFormation functionality
"""
import argparse
import logging
import os
import random
import sys
log_format = '%(levelname)s [%(asctime)s] %(message)s'
logging.basicConfig(format=log_format, level=logging.DEBUG)
logger = logging.getLogger('cfn-push-stats')
try:
import psutil
except ImportError:
logger.warn("psutil not available. If you want process and memory "
"statistics, you need to install it.")
#
# --aws-credential-file=PATH Specifies the location of the file with AWS
# credentials.
# --aws-access-key-id=VALUE Specifies the AWS access key ID to use to
# identify the caller.
# --aws-secret-key=VALUE Specifies the AWS secret key to use to sign
# the request.
# --from-cron Specifies that this script is running from cron.
#
# Examples
#
# To perform a simple test run without posting data to Amazon CloudWatch
#
# ./mon-put-instance-data.pl --mem-util --verify --verbose
#
# To set a five-minute cron schedule to report memory and disk space utilization to CloudWatch
#
# */5 * * * * ~/aws-scripts-mon/mon-put-instance-data.pl --mem-util --disk-space-util --disk-path=/ --from-cron
#
if os.path.exists('/opt/aws/bin'):
sys.path.insert(0, '/opt/aws/bin')
from cfn_helper import *
else:
from heat.cfntools.cfn_helper import *
KILO = 1024
MEGA = 1048576
GIGA = 1073741824
unit_map = {'bytes': 1,
'kilobytes': KILO,
'megabytes': MEGA,
'gigabytes': GIGA}
description = " "
parser = argparse.ArgumentParser(description=description)
parser.add_argument('-v', '--verbose', action="store_true",
help="Verbose logging", required=False)
parser.add_argument('--service-failure', required=False, action="store_true",
help='Reports a service falure.')
parser.add_argument('--mem-util', required=False, action="store_true",
help='Reports memory utilization in percentages.')
parser.add_argument('--mem-used', required=False, action="store_true",
help='Reports memory used (excluding cache and buffers) in megabytes.')
parser.add_argument('--mem-avail', required=False, action="store_true",
help='Reports available memory (including cache and buffers) in megabytes.')
parser.add_argument('--swap-util', required=False, action="store_true",
help='Reports swap utilization in percentages.')
parser.add_argument('--swap-used', required=False, action="store_true",
help='Reports allocated swap space in megabytes.')
parser.add_argument('--disk-space-util', required=False, action="store_true",
help='Reports disk space utilization in percentages.')
parser.add_argument('--disk-space-used', required=False, action="store_true",
help='Reports allocated disk space in gigabytes.')
parser.add_argument('--disk-space-avail',required=False, action="store_true",
help='Reports available disk space in gigabytes.')
parser.add_argument('--memory-units', required=False, default='megabytes',
help='Specifies units for memory metrics.')
parser.add_argument('--disk-units', required=False, default='megabytes',
help='Specifies units for disk metrics.')
parser.add_argument('--disk-path', required=False, default='/',
help='Selects the disk by the path on which to report.')
parser.add_argument('--watch', required=True,
help='the name of the watch to post to.')
args = parser.parse_args()
data = {'Namespace': 'system/linux'}
# service failure
# ===============
if args.service_failure:
data['ServiceFailure'] = {
'Value': 1,
'Units': 'Counter'}
# memory space
# ==========
if args.mem_util or args.mem_used or args.mem_avail:
mem = psutil.phymem_usage()
if args.mem_util:
data['MemoryUtilization'] = {
'Value': mem.percent,
'Units': 'Percent'}
if args.mem_used:
data['MemoryUsed'] = {
'Value': mem.used / unit_map[args.memory_units],
'Units': args.memory_units}
if args.mem_avail:
data['MemoryAvailable'] = {
'Value': mem.free / unit_map[args.memory_units],
'Units': args.memory_units}
# swap space
# ==========
if args.swap_util or args.swap_used:
swap = psutil.virtmem_usage()
if args.swap_util:
data['SwapUtilization'] = {
'Value': swap.percent,
'Units': 'Percent'}
if args.swap_used:
data['SwapUsed'] = {
'Value': swap.used / unit_map[args.memory_units],
'Units': args.memory_units}
# disk space
# ==========
if args.disk_space_util or args.disk_space_used or args.disk_space_avail:
disk = psutil.disk_usage(args.disk_path)
if args.disk_space_util:
data['DiskSpaceUtilization'] = {
'Value': disk.percent,
'Units': 'Percent'}
if args.disk_space_used:
data['DiskSpaceUsed'] = {
'Value': disk.used / unit_map[args.disk_units],
'Units': args.disk_units}
if args.disk_space_avail:
data['DiskSpaceAvailable'] = {
'Value': disk.free / unit_map[args.disk_units],
'Units': args.disk_units}
logger.info(str(data))
server_url = metadata_server_url()
if not server_url:
logger.error('can not get the metadata_server_url')
exit(1)
url = '%sstats/%s/data/' % (server_url, args.watch)
cmd_str = "curl -X POST -H \'Content-Type:\' --data-binary \'%s\' %s" % \
(json.dumps(data), url)
cmd = CommandRunner(cmd_str)
cmd.run()
logger.info(cmd.stdout)

91
heat/cfntools/cfn-signal
Unescape View File

@ -1,91 +0,0 @@
#!/usr/bin/env python
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Implements cfn-signal CloudFormation functionality
"""
import argparse
import logging
import os
import sys
if os.path.exists('/opt/aws/bin'):
sys.path.insert(0, '/opt/aws/bin')
from cfn_helper import *
else:
from heat.cfntools.cfn_helper import *
description = " "
parser = argparse.ArgumentParser(description=description)
parser.add_argument('-s', '--success',
dest="success",
help="signal status to report",
default='true',
required=False)
parser.add_argument('-r', '--reason',
dest="reason",
help="The reason for the failure",
default="Configuration Complete",
required=False)
parser.add_argument('--data',
dest="data",
default="Application has completed configuration.",
help="The data to send",
required=False)
parser.add_argument('-i', '--id',
dest="unique_id",
help="the unique id to send back to the WaitCondition",
default='00000',
required=False)
parser.add_argument('-e', '--exit',
dest="exit_code",
help="The exit code from a procecc to interpret",
default=None,
required=False)
parser.add_argument('url',
help='the url to post to')
args = parser.parse_args()
log_format = '%(levelname)s [%(asctime)s] %(message)s'
logging.basicConfig(format=log_format, level=logging.DEBUG)
logger = logging.getLogger('cfn-init')
log_file_name = "/var/log/cfn-signal.log"
file_handler = logging.FileHandler(log_file_name)
file_handler.setFormatter(logging.Formatter(log_format))
logger.addHandler(file_handler)
logger.debug('cfn-signal called %s ' % (str(args)))
status = 'FAILURE'
if args.exit_code:
# "exit_code" takes presedence over "success".
if args.exit_code == '0':
status = 'SUCCESS'
else:
if args.success == 'true':
status = 'SUCCESS'
body = {
"Status" : status,
"Reason" : args.reason,
"UniqueId" : args.unique_id,
"Data" : args.data
}
cmd_str = "curl -X PUT -H \'Content-Type:\' --data-binary \'%s\' %s" % \
(json.dumps(body), args.url)
CommandRunner(cmd_str).run()

893
heat/cfntools/cfn_helper.py
Unescape View File

@ -1,893 +0,0 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Implements cfn metadata handling
Resource metadata currently implemented:
* config/packages
* config/services
Not implemented yet:
* config sets
* config/sources
* config/commands
* config/files
* config/users
* config/groups
* command line args
- placeholders are ignored
"""
import ConfigParser
import errno
import grp
import json
import logging
import os
import os.path
import pwd
try:
import rpmUtils.updates as rpmupdates
import rpmUtils.miscutils as rpmutils
rpmutils_present = True
except:
rpmutils_present = False
import subprocess
import sys
from urllib2 import urlopen, Request
from urlparse import urlparse, urlunparse
logger = logging.getLogger('cfntools')
def to_boolean(b):
val = b.lower().strip() if isinstance(b, basestring) else b
return val in [True, 'true', 'yes', '1', 1]
class HupConfig(object):
def __init__(self, fp_list):
self.config = ConfigParser.SafeConfigParser()
for fp in fp_list:
self.config.readfp(fp)
self.load_main_section()
self.hooks = {}
for s in self.config.sections():
if s != 'main':
self.hooks[s] = Hook(s,
self.config.get(s, 'triggers'),
self.config.get(s, 'path'),
self.config.get(s, 'runas'),
self.config.get(s, 'action'))
def load_main_section(self):
# required values
self.stack = self.config.get('main', 'stack')
self.credential_file = self.config.get('main', 'credential-file')
try:
with open(self.credential_file) as f:
self.credentials = f.read()
except:
raise Exception("invalid credentials file %s" %
self.credential_file)
# optional values
try:
self.region = self.config.get('main', 'region')
except ConfigParser.NoOptionError:
self.region = 'nova'
try:
self.interval = self.config.getint('main', 'interval')
except ConfigParser.NoOptionError:
self.interval = 10
def __str__(self):
return '{stack: %s, credential_file: %s, region: %s, interval:%d}' % \
(self.stack, self.credential_file, self.region, self.interval)
def unique_resources_get(self):
resources = []
for h in self.hooks:
r = self.hooks[h].resource_name_get()
if not r in resources:
resources.append(self.hooks[h].resource_name_get())
return resources
class Hook(object):
def __init__(self, name, triggers, path, runas, action):
self.name = name
self.triggers = triggers
self.path = path
self.runas = runas
self.action = action
def resource_name_get(self):
sp = self.path.split('.')
return sp[1]
def event(self, ev_name, ev_object, ev_resource):
if self.resource_name_get() == ev_resource and \
ev_name in self.triggers:
CommandRunner(self.action).run(user=self.runas)
else:
logger.debug('event: {%s, %s, %s} did not match %s' %
(ev_name, ev_object, ev_resource, self.__str__()))
def __str__(self):
return '{%s, %s, %s, %s, %s}' % \
(self.name,
self.triggers,
self.path,
self.runas,
self.action)
class CommandRunner(object):
"""
Helper class to run a command and store the output.
"""
def __init__(self, command, nextcommand=None):
self._command = command
self._next = nextcommand
self._stdout = None
self._stderr = None
self._status = None
def __str__(self):
s = "CommandRunner:"
s += "\n\tcommand: %s" % self._command
if self._status:
s += "\n\tstatus: %s" % self._status
if self._stdout:
s += "\n\tstdout: %s" % self._stdout
if self._stderr:
s += "\n\tstderr: %s" % self._stderr
return s
def run(self, user='root'):
"""
Run the Command and return the output.
Returns:
self
"""
logger.debug("Running command: %s" % self._command)
cmd = ['su', user, '-c', self._command]
subproc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = subproc.communicate()
self._status = subproc.returncode
self._stdout = output[0]
self._stderr = output[1]
if self._next:
self._next.run()
return self
@property
def stdout(self):
return self._stdout
@property
def stderr(self):
return self._stderr
@property
def status(self):
return self._status
class RpmHelper(object):
if rpmutils_present:
_rpm_util = rpmupdates.Updates([], [])
@classmethod
def prepcache(cls):
"""
Prepare the yum cache
"""
CommandRunner("yum -y makecache").run()
@classmethod
def compare_rpm_versions(cls, v1, v2):
"""
Compare two RPM version strings.
Arguments:
v1 -- a version string
v2 -- a version string
Returns:
0 -- the versions are equal
1 -- v1 is greater
-1 -- v2 is greater
"""
if v1 and v2:
return rpmutils.compareVerOnly(v1, v2)
elif v1:
return 1
elif v2:
return -1
else:
return 0
@classmethod
def newest_rpm_version(cls, versions):
"""
Returns the highest (newest) version from a list of versions.
Arguments:
versions -- A list of version strings
e.g., ['2.0', '2.2', '2.2-1.fc16', '2.2.22-1.fc16']
"""
if versions:
if isinstance(versions, basestring):
return versions
versions = sorted(versions, rpmutils.compareVerOnly,
reverse=True)
return versions[0]
else:
return None
@classmethod
def rpm_package_version(cls, pkg):
"""
Returns the version of an installed RPM.
Arguments:
pkg -- A package name
"""
cmd = "rpm -q --queryformat '%{VERSION}-%{RELEASE}' %s" % pkg
command = CommandRunner(cmd).run()
return command.stdout
@classmethod
def rpm_package_installed(cls, pkg):
"""
Indicates whether pkg is in rpm database.
Arguments:
pkg -- A package name (with optional version and release spec).
e.g., httpd
e.g., httpd-2.2.22
e.g., httpd-2.2.22-1.fc16
"""
command = CommandRunner("rpm -q %s" % pkg).run()
return command.status == 0
@classmethod
def yum_package_available(cls, pkg):
"""
Indicates whether pkg is available via yum
Arguments:
pkg -- A package name (with optional version and release spec).
e.g., httpd
e.g., httpd-2.2.22
e.g., httpd-2.2.22-1.fc16
"""
cmd_str = "yum -C -y --showduplicates list available %s" % pkg
command = CommandRunner(cmd_str).run()
return command.status == 0
@classmethod
def install(cls, packages, rpms=True):
"""
Installs (or upgrades) a set of packages via RPM or via Yum.
Arguments:
packages -- a list of packages to install
rpms -- if True:
* use RPM to install the packages
* packages must be a list of URLs to retrieve RPMs
if False:
* use Yum to install packages
* packages is a list of:
- pkg name (httpd), or
- pkg name with version spec (httpd-2.2.22), or
- pkg name with version-release spec
(httpd-2.2.22-1.fc16)
"""
if rpms:
cmd = "rpm -U --force --nosignature "
cmd += " ".join(packages)
logger.info("Installing packages: %s" % cmd)
else:
cmd = "yum -y install "
cmd += " ".join(packages)
logger.info("Installing packages: %s" % cmd)
command = CommandRunner(cmd).run()
if command.status:
logger.warn("Failed to install packages: %s" % cmd)
@classmethod
def downgrade(cls, packages, rpms=True):
"""
Downgrades a set of packages via RPM or via Yum.
Arguments:
packages -- a list of packages to downgrade
rpms -- if True:
* use RPM to downgrade (replace) the packages
* packages must be a list of URLs to retrieve the RPMs
if False:
* use Yum to downgrade packages
* packages is a list of:
- pkg name with version spec (httpd-2.2.22), or
- pkg name with version-release spec
(httpd-2.2.22-1.fc16)
"""
if rpms:
cls.install(packages)
else:
cmd = "yum -y downgrade "
cmd += " ".join(packages)
logger.info("Downgrading packages: %s" % cmd)
command = Command(cmd).run()
if command.status:
logger.warn("Failed to downgrade packages: %s" % cmd)
class PackagesHandler(object):
_packages = {}
_package_order = ["dpkg", "rpm", "apt", "yum"]
@staticmethod
def _pkgsort(pkg1, pkg2):
order = PackagesHandler._package_order
p1_name = pkg1[0]
p2_name = pkg2[0]
if p1_name in order and p2_name in order:
return cmp(order.index(p1_name), order.index(p2_name))
elif p1_name in order:
return -1
elif p2_name in order:
return 1
else:
return cmp(p1_name.lower(), p2_name.lower())
def __init__(self, packages):
self._packages = packages
def _handle_gem_packages(self, packages):
"""
very basic support for gems
"""
# TODO(asalkeld) support versions
# -b == local & remote install
# -y == install deps
opts = '-b -y'
for pkg_name, versions in packages.iteritems():
if len(versions) > 0:
cmd_str = 'gem install %s --version %s %s' % (opts,
versions[0],
pkg_name)
CommandRunner(cmd_str).run()
else:
CommandRunner('gem install %s %s' % (opts, pkg_name)).run()
def _handle_python_packages(self, packages):
"""
very basic support for easy_install
"""
# TODO(asalkeld) support versions
for pkg_name, versions in packages.iteritems():
cmd_str = 'easy_install %s' % (pkg_name)
CommandRunner(cmd_str).run()
def _handle_yum_packages(self, packages):
"""
Handle installation, upgrade, or downgrade of a set of
packages via yum.
Arguments:
packages -- a package entries map of the form:
"pkg_name" : "version",
"pkg_name" : ["v1", "v2"],
"pkg_name" : []
For each package entry:
* if no version is supplied and the package is already installed, do
nothing
* if no version is supplied and the package is _not_ already
installed, install it
* if a version string is supplied, and the package is already
installed, determine whether to downgrade or upgrade (or do nothing
if version matches installed package)
* if a version array is supplied, choose the highest version from the
array and follow same logic for version string above
"""
# collect pkgs for batch processing at end
installs = []
downgrades = []
# update yum cache
RpmHelper.prepcache()
for pkg_name, versions in packages.iteritems():
ver = RpmHelper.newest_rpm_version(versions)
pkg = "%s-%s" % (pkg_name, ver) if ver else pkg_name
if RpmHelper.rpm_package_installed(pkg):
# FIXME:print non-error, but skipping pkg
pass
elif not RpmHelper.yum_package_available(pkg):
logger.warn("Skipping package '%s'. Not available via yum" %
pkg)
elif not ver:
installs.append(pkg)
else:
current_ver = RpmHelper.rpm_package_version(pkg)
rc = RpmHelper.compare_rpm_versions(current_ver, ver)
if rc < 0:
installs.append(pkg)
elif rc > 0:
downgrades.append(pkg)
if installs:
RpmHelper.install(installs, rpms=False)
if downgrades:
RpmHelper.downgrade(downgrades)
def _handle_rpm_packages(sef, packages):
"""
Handle installation, upgrade, or downgrade of a set of
packages via rpm.
Arguments:
packages -- a package entries map of the form:
"pkg_name" : "url"
For each package entry:
* if the EXACT package is already installed, skip it
* if a different version of the package is installed, overwrite it
* if the package isn't installed, install it
"""
#FIXME: handle rpm installs
pass
def _handle_apt_packages(self, packages):
"""
very basic support for apt
"""
# TODO(asalkeld) support versions
pkg_list = ' '.join([p for p in packages])
cmd_str = 'apt-get -y install %s' % pkg_list
CommandRunner(cmd_str).run()
# map of function pionters to handle different package managers
_package_handlers = {
"yum": _handle_yum_packages,
"rpm": _handle_rpm_packages,
"apt": _handle_apt_packages,
"rubygems": _handle_gem_packages,
"python": _handle_python_packages
}
def _package_handler(self, manager_name):
handler = None
if manager_name in self._package_handlers:
handler = self._package_handlers[manager_name]
return handler
def apply_packages(self):
"""
Install, upgrade, or downgrade packages listed
Each package is a dict containing package name and a list of versions
Install order:
* dpkg
* rpm
* apt
* yum
"""
if not self._packages:
return
packages = sorted(self._packages.iteritems(), PackagesHandler._pkgsort)
for manager, package_entries in packages:
handler = self._package_handler(manager)
if not handler:
logger.warn("Skipping invalid package type: %s" % manager)
else:
handler(self, package_entries)
class FilesHandler(object):
def __init__(self, files):
self._files = files
def apply_files(self):
if not self._files:
return
for fdest, meta in self._files.iteritems():
dest = fdest.encode()
try:
os.makedirs(os.path.dirname(dest))
except OSError as e:
if e.errno == errno.EEXIST:
logger.debug(str(e))
else:
logger.exception(e)
if 'content' in meta:
if isinstance(meta['content'], basestring):
f = open(dest, 'w')
f.write(meta['content'])
f.close()
else:
f = open(dest, 'w')
f.write(json.dumps(meta['content'], indent=4))
f.close()
elif 'source' in meta:
CommandRunner('wget -O %s %s' % (dest, meta['source'])).run()
else:
logger.error('%s %s' % (dest, str(meta)))
continue
uid = -1
gid = -1
if 'owner' in meta:
try:
user_info = pwd.getpwnam(meta['owner'])
uid = user_info[2]
except KeyError as ex:
pass
if 'group' in meta:
try:
group_info = grp.getgrnam(meta['group'])
gid = group_info[2]
except KeyError as ex:
pass
os.chown(dest, uid, gid)
if 'mode' in meta:
os.chmod(dest, int(meta['mode'], 8))
class SourcesHandler(object):
'''
tar, tar+gzip,tar+bz2 and zip
'''
_sources = {}
def __init__(self, sources):
self._sources = sources
def _url_to_tmp_filename(self, url):
sp = url.split('/')
if 'https://github.com' in url:
if 'zipball' == sp[-2]:
return '/tmp/%s-%s.zip' % (sp[-3], sp[-1])
elif 'tarball' == sp[-2]:
return '/tmp/%s-%s.tar.gz' % (sp[-3], sp[-1])
else:
pass
return '/tmp/%s' % (sp[-1])
def _decompress(self, archive, dest_dir):
cmd_str = ''
logger.debug("Decompressing")
(r, ext) = os.path.splitext(archive)
if ext == '.tgz':
cmd_str = 'tar -C %s -xzf %s' % (dest_dir, archive)
elif ext == '.tbz2':
cmd_str = 'tar -C %s -xjf %s' % (dest_dir, archive)
elif ext == '.zip':
cmd_str = 'unzip -d %s %s' % (dest_dir, archive)
elif ext == '.tar':
cmd_str = 'tar -C %s -xf %s' % (dest_dir, archive)
elif ext == '.gz':
(r, ext) = os.path.splitext(r)
if ext:
cmd_str = 'tar -C %s -xzf %s' % (dest_dir, archive)
else:
cmd_str = 'gunzip -c %s > %s' % (archive, dest_dir)
elif ext == 'bz2':
(r, ext) = os.path.splitext(r)
if ext:
cmd_str = 'tar -C %s -xjf %s' % (dest_dir, archive)
else:
cmd_str = 'bunzip2 -c %s > %s' % (archive, dest_dir)
else:
pass
return CommandRunner(cmd_str)
def apply_sources(self):
if not self._sources:
return
for dest, url in self._sources.iteritems():
tmp_name = self._url_to_tmp_filename(url)
cmd_str = 'wget -O %s %s' % (tmp_name, url)
decompress_command = self._decompress(tmp_name, dest)
CommandRunner(cmd_str, decompress_command).run()
try:
os.makedirs(dest)
except OSError as e:
if e.errno == errno.EEXIST:
logger.debug(str(e))
else:
logger.exception(e)
class ServicesHandler(object):
_services = {}
def __init__(self, services, resource=None, hooks=None):
self._services = services
self.resource = resource
self.hooks = hooks
def _handle_sysv_command(self, service, command):
service_exe = "/sbin/service"
enable_exe = "/sbin/chkconfig"
cmd = ""
if "enable" == command:
cmd = "%s %s on" % (enable_exe, service)
elif "disable" == command:
cmd = "%s %s off" % (enable_exe, service)
elif "start" == command:
cmd = "%s %s start" % (service_exe, service)
elif "stop" == command:
cmd = "%s %s stop" % (service_exe, service)
elif "status" == command:
cmd = "%s %s status" % (service_exe, service)
command = CommandRunner(cmd)
command.run()
return command
def _handle_systemd_command(self, service, command):
exe = "/bin/systemctl"
cmd = ""
service = '%s.service' % service
if "enable" == command:
cmd = "%s enable %s" % (exe, service)
elif "disable" == command:
cmd = "%s disable %s" % (exe, service)
elif "start" == command:
cmd = "%s start %s" % (exe, service)
elif "stop" == command:
cmd = "%s stop %s" % (exe, service)
elif "status" == command:
cmd = "%s status %s" % (exe, service)
command = CommandRunner(cmd)
command.run()
return command
def _initialize_service(self, handler, service, properties):
if "enabled" in properties:
enable = to_boolean(properties["enabled"])
if enable:
logger.info("Enabling service %s" % service)
handler(self, service, "enable")
else:
logger.info("Disabling service %s" % service)
handler(self, service, "disable")
if "ensureRunning" in properties:
ensure_running = to_boolean(properties["ensureRunning"])
command = handler(self, service, "status")
running = command.status == 0
if ensure_running and not running:
logger.info("Starting service %s" % service)
handler(self, service, "start")
elif not ensure_running and running:
logger.info("Stopping service %s" % service)
handler(self, service, "stop")
def _monitor_service(self, handler, service, properties):
if "ensureRunning" in properties:
ensure_running = to_boolean(properties["ensureRunning"])
command = handler(self, service, "status")
running = command.status == 0
if ensure_running and not running:
logger.warn("Restarting service %s" % service)
start_cmd = handler(self, service, "start")
if start_cmd.status != 0:
logger.warning('Service %s did not start. STDERR: %s' %
(service, start_cmd.stderr))
return
for h in self.hooks:
self.hooks[h].event('service.restarted',
service, self.resource)
def _monitor_services(self, handler, services):
for service, properties in services.iteritems():
self._monitor_service(handler, service, properties)
def _initialize_services(self, handler, services):
for service, properties in services.iteritems():
self._initialize_service(handler, service, properties)
# map of function pointers to various service handlers
_service_handlers = {
"sysvinit": _handle_sysv_command,
"systemd": _handle_systemd_command
}
def _service_handler(self, manager_name):
handler = None
if manager_name in self._service_handlers:
handler = self._service_handlers[manager_name]
return handler
def apply_services(self):
"""
Starts, stops, enables, disables services
"""
if not self._services:
return
for manager, service_entries in self._services.iteritems():
handler = self._service_handler(manager)
if not handler:
logger.warn("Skipping invalid service type: %s" % manager)
else:
self._initialize_services(handler, service_entries)
def monitor_services(self):
"""
Restarts failed services, and runs hooks.
"""
if not self._services:
return
for manager, service_entries in self._services.iteritems():
handler = self._service_handler(manager)
if not handler:
logger.warn("Skipping invalid service type: %s" % manager)
else:
self._monitor_services(handler, service_entries)
def metadata_server_url():
"""
Return the url to the metadata server.
"""
try:
f = open("/var/lib/cloud/data/cfn-metadata-server")
server_url = f.read().strip()
f.close()
if not server_url[-1] == '/':
server_url += '/'
return server_url
except IOError:
return None
class MetadataServerConnectionError(Exception):
pass
class Metadata(object):
_metadata = None
_init_key = "AWS::CloudFormation::Init"
def __init__(self, stack, resource, access_key=None,
secret_key=None, credentials_file=None, region=None):
self.stack = stack
self.resource = resource
self.access_key = access_key
self.secret_key = secret_key