x
/
inception
Code Issues Proposed changes

Cleanup oslo.config 

- Remove config opts and orchestrator ctor document redundancy

- Make "atomic" an attribute of orchestrator

- A bit reordering of opts

- Cleanup and rename inception.conf.sample

- Some other minors: Authors, setup.py

Change-Id: I9c501acf4a11c2fc0056130bd163bbccd2ec9e24
This commit is contained in:
Changbin Liu 2013-07-08 21:00:40 -04:00
parent 03ed5b7e9f
commit d0f14b4b3b
5 changed files with 98 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Authors
View File

@ -1,2 +1,3 @@
Andrew Forrest <aj4est@gmail.com>
Changbin Liu <changbin.liu@gmail.com> Changbin Liu <changbin.liu@gmail.com>
Yun Mao <yunmao@gmail.com> Yun Mao <yunmao@gmail.com>

48
etc/inception/inception.conf
View File

@ -1,48 +0,0 @@
# inception.conf
#
# To use this configuration, add one of the following options to your command
# line:
#
# --config-file <d>/inception.conf
# or
# --config-dir <d>
#
# where <d> is the path to the directory containing this file
[DEFAULT]
# user - login id with sudo for image used for nodes in Inception Cloud
user = ubuntu
# image - id of image used to construct nodes in Inception Cloud
image = f3d62d5b-a76b-4997-a579-ff946a606132
# flavor - id of machine flavor used for nodes in Inception Cloud (3 = med)
flavor = 3
# gateway_flavor - id of machine flavor used to construct GW in Inception Cloud
gateway_flavor = 1
# key_name - name of PK pair for node access
key_name = <your_key_name_here>
# security_groups - firewall rules
security_groups = default, ssh
# src_dir - source location of various chef-related setup scripts on client
src_dir = ../bin/
# dst_dir - absolute destination path for chef-related setup scripts on nodes
dst_dir = /home/ubuntu/
# userdata - bash script run by cloud-init in late boot stage (rc.local-like)
userdata = userdata.sh.template
# timeout - maximum time to wait for all servers to be reachable (seconds)
timeout = 999999
# poll_interval - interval between readiness [ssh+userdata] polls (seconds)
poll_interval = 5
# pool - name of pool for floating IP addresses
pool = research

25
etc/inception/inception.conf.sample Normal file
View File

@ -0,0 +1,25 @@
[DEFAULT]
user = ubuntu
image = f3d62d5b-a76b-4997-a579-ff946a606132
flavor = 3
gateway_flavor = 1
key_name = shared
security_groups = default, ssh
src_dir = ../bin/
dst_dir = /home/ubuntu/
userdata = userdata.sh.template
timeout = 999999
poll_interval = 5
pool = research

161
inception/orchestrator.py
View File

@ -27,7 +27,6 @@ developer/user
from collections import OrderedDict from collections import OrderedDict
import functools import functools
import getopt
import os import os
import Queue import Queue
import subprocess import subprocess
@ -43,84 +42,86 @@ from oslo.config import cfg
from inception import __version__ from inception import __version__
from inception.utils import cmd from inception.utils import cmd
CONF = cfg.CONF orchestrator_opts = [
inception_opts = [
cfg.StrOpt('prefix', cfg.StrOpt('prefix',
default=None, default=None,
metavar='PREFIX',
required=True, required=True,
short='p', short='p',
help='(unique) prefix for node names (no hyphens allowed)'), help='unique prefix for node names (no hyphens allowed)'),
cfg.IntOpt('num_workers', cfg.IntOpt('num_workers',
default=2, default=2,
metavar='NUM',
short='n', short='n',
help='number of worker nodes to create'), help='number of worker nodes to create'),
cfg.BoolOpt('shell',
default=False,
help='initialize, then drop to embedded IPython shell'),
cfg.BoolOpt('atomic', cfg.BoolOpt('atomic',
default=False, default=False,
help='on error, run as if --cleanup was specified'), help='on error, whether rollback, i.e., auto delete all'
cfg.BoolOpt('cleanup', ' created virtual resources'),
default=False,
help='take down the inception cloud'),
cfg.BoolOpt('parallel', cfg.BoolOpt('parallel',
default=False, default=False,
help='execute chef-related setup tasks in parallel'), help='execute Chef-related setup tasks in parallel'),
cfg.StrOpt('chef_repo', cfg.StrOpt('chef_repo',
default='git://github.com/maoy/inception-chef-repo.git', default='git://github.com/maoy/inception-chef-repo.git',
metavar='URL', help='URL of Chef repository'),
help='URL of Chef repo'),
cfg.StrOpt('chef_repo_branch', cfg.StrOpt('chef_repo_branch',
default='master', default='master',
metavar='BRANCH',
help='name of branch of Chef repo to use'), help='name of branch of Chef repo to use'),
cfg.StrOpt('ssh_keyfile', cfg.StrOpt('ssh_keyfile',
default=None, default=None,
metavar='PATH', help='path of extra public key(s) for node access via ssh'),
help='path of additional keyfile for node access via ssh'),
cfg.StrOpt('pool', cfg.StrOpt('pool',
default='research', default='research',
help='name of pool for floating IP addresses'), help='name of pool for floating IP addresses'),
cfg.StrOpt('user', cfg.StrOpt('user',
default='ubuntu', default='ubuntu',
help=''), help='login id with sudo for all nodes'),
cfg.StrOpt('image', cfg.StrOpt('image',
default='f3d62d5b-a76b-4997-a579-ff946a606132', default='f3d62d5b-a76b-4997-a579-ff946a606132',
help=''), help='id of image used to construct nodes (=u1204-130531-gv)'),
cfg.IntOpt('flavor', cfg.IntOpt('flavor',
default=3, default=3,
help='id of machine flavor used for nodes'), help='id of machine flavor used for nodes (3=medium)'),
cfg.IntOpt('gateway_flavor', cfg.IntOpt('gateway_flavor',
default=1, default=1,
help='id of machine flavor used for gateway'), help='id of machine flavor used to construct GW (1=tiny)'),
cfg.StrOpt('key_name', cfg.StrOpt('key_name',
default='<your_key_name_here>', default='shared',
help='name of key for node access via ssh'), help='name of public key for node access via ssh'),
cfg.ListOpt('security_groups', cfg.ListOpt('security_groups',
default=['default', 'ssh'], default=['default', 'ssh'],
help='list of security groups for nodes'), help='list of security groups (firewall rules) for nodes'),
cfg.StrOpt('src_dir', cfg.StrOpt('src_dir',
default='../bin/', default='../bin/',
help='path of setup script source dir on client'), help='relative source location (to __file__) of various'
' chef-related setup scripts on client'),
cfg.StrOpt('dst_dir', cfg.StrOpt('dst_dir',
default='/home/ubuntu/', default='/home/ubuntu/',
help='path of setup script destination dir on nodes'), help='absolute destination path for chef-related setup scripts'
' on nodes'),
cfg.StrOpt('userdata', cfg.StrOpt('userdata',
default='userdata.sh.template', default='userdata.sh.template',
help='template for user data script'), help='bash script run by cloud-init in late boot stage'
' (rc.local-like)'),
cfg.IntOpt('timeout', cfg.IntOpt('timeout',
default=999999, default=999999,
help='number of seconds for creation timeout'), help='maximum time (in seconds) to wait for all nodes to be'
' ready [ssh-able + userdata]'),
cfg.IntOpt('poll_interval', cfg.IntOpt('poll_interval',
default=5, default=5,
help='interval (in seconds) between readiness polls'), help='interval (in seconds) between readiness polls'),
] ]
# Register options cmd_opts = [
CONF.register_cli_opts(inception_opts) cfg.BoolOpt('shell',
default=False,
help='initialize, then drop to embedded IPython shell'),
cfg.BoolOpt('cleanup',
default=False,
help='take down the inception cloud'),
]
CONF = cfg.CONF
CONF.register_cli_opts(orchestrator_opts)
CONF.register_cli_opts(cmd_opts)
class Orchestrator(object): class Orchestrator(object):
@ -131,46 +132,25 @@ class Orchestrator(object):
def __init__(self, def __init__(self,
prefix, prefix,
num_workers, num_workers,
atomic,
parallel,
chef_repo, chef_repo,
chef_repo_branch, chef_repo_branch,
parallel, ssh_keyfile,
ssh_keyfile=None, pool,
pool='research', user,
user='ubuntu', image,
image='f3d62d5b-a76b-4997-a579-ff946a606132', flavor,
flavor=3, gateway_flavor,
gateway_flavor=1, key_name,
key_name='<your_key_name_here>', security_groups,
security_groups=('default', 'ssh'), src_dir,
src_dir='../bin/', dst_dir,
dst_dir='/home/ubuntu/', userdata,
userdata='userdata.sh.template', timeout,
timeout=999999, poll_interval):
poll_interval=5):
""" """
@param prefix: unique name as prefix For doc on each param refer to orchestrator_opts
@param num_workers: how many worker nodes you'd like
@param chef_repo: chef repository location
@param chef_repo_branch: which branch to use in repo
@param parallel: whether run functions in parallel (via threads, for
accelerating) or sequential
@param ssh_keyfile: extra ssh public key to login user account
@param pool: floating ip pool
@param user: username (with root permission) for all servers
@param image: default u1204-130531-gv
@param flavor: default medium
@param gateway_flavor: default tiny
@param key_name: ssh public key to be injected
@param security_groups: firewall rules
@param src_dir: location from where scripts are uploaded to servers.
Relative path to __file__
@param dst_dir: target location of scripts on servers. Must be absolte
path
@param userdata: a bash script to be executed by cloud-init (in late
booting stage, rc.local-like)
@param timeout: sleep time (s) for servers to be launched
@param poll_interval: every this time poll to check whether a server
has finished launching, i.e., ssh-able + userdata done
""" """
## check args ## check args
#TODO(changbl): remove the restriction of "num_workers <= 5" #TODO(changbl): remove the restriction of "num_workers <= 5"
@ -183,9 +163,11 @@ class Orchestrator(object):
## args ## args
self.prefix = prefix self.prefix = prefix
self.num_workers = num_workers self.num_workers = num_workers
self.atomic = atomic
self.parallel = parallel
self.chef_repo = chef_repo self.chef_repo = chef_repo
self.chef_repo_branch = chef_repo_branch self.chef_repo_branch = chef_repo_branch
self.parallel = parallel self.ssh_keyfile = ssh_keyfile
self.pool = pool self.pool = pool
self.user = user self.user = user
self.image = image self.image = image
@ -198,14 +180,14 @@ class Orchestrator(object):
self.dst_dir = os.path.abspath(dst_dir) self.dst_dir = os.path.abspath(dst_dir)
with open(os.path.join(self.src_dir, userdata), 'r') as fin: with open(os.path.join(self.src_dir, userdata), 'r') as fin:
self.userdata = fin.read() self.userdata = fin.read()
# Inject the extra ssh public key if any
ssh_keycontent = ''
if ssh_keyfile:
with open(ssh_keyfile, 'r') as fin:
ssh_keycontent = fin.read()
self.userdata = self.userdata % (user, ssh_keycontent)
self.timeout = timeout self.timeout = timeout
self.poll_interval = poll_interval self.poll_interval = poll_interval
# Inject the extra ssh public key if any
ssh_keycontent = ''
if self.ssh_keyfile:
with open(self.ssh_keyfile, 'r') as fin:
ssh_keycontent = fin.read()
self.userdata = self.userdata % (user, ssh_keycontent)
# scripts to run on chefserver, execute one by one (sequence matters) # scripts to run on chefserver, execute one by one (sequence matters)
self.chefserver_commands = [] self.chefserver_commands = []
self.chefserver_files = OrderedDict() self.chefserver_files = OrderedDict()
@ -243,12 +225,9 @@ class Orchestrator(object):
self._worker_names = [] self._worker_names = []
self._gateway_floating_ip = None self._gateway_floating_ip = None
def start(self, atomic): def start(self):
""" """
run the whole process run the whole process
@param atomic: upon exception, whether rollback, i.e., auto delete all
created servers
""" """
try: try:
self._check_existence() self._check_existence()
@ -265,15 +244,16 @@ class Orchestrator(object):
print "OpenStack dashboard is https://%s" % self._controller_ip print "OpenStack dashboard is https://%s" % self._controller_ip
except Exception: except Exception:
print traceback.format_exc() print traceback.format_exc()
if atomic: if self.atomic:
self.cleanup() self.cleanup()
def _check_existence(self): def _check_existence(self):
""" """
Check whether inception cloud existence based on given self.prefix Check whether inception cloud existence based on given self.prefix
""" """
full_prefix = self.prefix + '-'
for server in self.client.servers.list(): for server in self.client.servers.list():
if '-' in server.name and server.name.split('-')[0] == self.prefix: if server.name.startswith(full_prefix):
raise ValueError('prefix=%s is already used' % self.prefix) raise ValueError('prefix=%s is already used' % self.prefix)
def _create_servers(self): def _create_servers(self):
@ -589,19 +569,19 @@ def main():
""" """
program starting point program starting point
""" """
# processes args
# Processes both config file and cmd line opts
try: try:
CONF(args=sys.argv[1:], version="Inception: version %s" % __version__) CONF(args=sys.argv[1:], version="Inception: version %s" % __version__)
except Exception, e: except Exception as e:
print e print e
sys.exit(-2) sys.exit(1)
# start orchestator
orchestrator = Orchestrator(CONF.prefix, orchestrator = Orchestrator(CONF.prefix,
CONF.num_workers, CONF.num_workers,
CONF.atomic,
CONF.parallel,
CONF.chef_repo, CONF.chef_repo,
CONF.chef_repo_branch, CONF.chef_repo_branch,
CONF.parallel,
CONF.ssh_keyfile, CONF.ssh_keyfile,
CONF.pool, CONF.pool,
CONF.user, CONF.user,
@ -615,7 +595,6 @@ def main():
CONF.userdata, CONF.userdata,
CONF.timeout, CONF.timeout,
CONF.poll_interval) CONF.poll_interval)
if CONF.shell: if CONF.shell:
# give me a ipython shell # give me a ipython shell
IPython.embed() IPython.embed()
@ -623,7 +602,7 @@ def main():
if CONF.cleanup: if CONF.cleanup:
orchestrator.cleanup() orchestrator.cleanup()
else: else:
orchestrator.start(CONF.atomic) orchestrator.start()
############################################## ##############################################
if __name__ == "__main__": if __name__ == "__main__":

6
setup.py
View File

@ -1,13 +1,11 @@
#!/usr/bin/python #!/usr/bin/env python
# Copyright (c) 2012 AT&T. All right reserved. # Copyright (c) 2012 AT&T. All right reserved.
#
try: try:
from setuptools import setup from setuptools import setup
except ImportError: except ImportError:
from distutils.core import setup from distutils.core import setup
# move version string out of setup so it is readily available to others # move version string out of setup so it is readily available to others
from inception import __version__ from inception import __version__
@ -17,7 +15,7 @@ setup(
description="Inception: Towards a Nested Cloud Architecture", description="Inception: Towards a Nested Cloud Architecture",
license="Apache 2.0", license="Apache 2.0",
classifiers=["Programming Language :: Python"], classifiers=["Programming Language :: Python"],
url='https://github.com/maoy/inception', url='https://github.com/stackforge/inception',
packages=["inception"], packages=["inception"],
install_requires=[ install_requires=[
"oslo.config>=1.1.1", "oslo.config>=1.1.1",