diff --git a/conductor/conductor/app.py b/conductor/conductor/app.py index a69e8e85..da17e9bb 100644 --- a/conductor/conductor/app.py +++ b/conductor/conductor/app.py @@ -17,7 +17,8 @@ def task_received(task, message_id): print 'Starting at', datetime.datetime.now() reporter = reporting.Reporter(rmqclient, message_id, task['id']) - command_dispatcher = CommandDispatcher(task['name'], rmqclient) + command_dispatcher = CommandDispatcher( + task['name'], rmqclient, task['token']) workflows = [] for path in glob.glob("data/workflows/*.xml"): print "loading", path @@ -26,9 +27,12 @@ def task_received(task, message_id): workflows.append(workflow) while True: - for workflow in workflows: - workflow.execute() - if not command_dispatcher.execute_pending(): + try: + for workflow in workflows: + workflow.execute() + if not command_dispatcher.execute_pending(): + break + except Exception: break command_dispatcher.close() diff --git a/conductor/conductor/cloud_formation.py b/conductor/conductor/cloud_formation.py index 16427141..e7894bf2 100644 --- a/conductor/conductor/cloud_formation.py +++ b/conductor/conductor/cloud_formation.py @@ -1,7 +1,7 @@ import base64 import xml_code_engine - +import config def update_cf_stack(engine, context, body, template, mappings, arguments, **kwargs): @@ -17,15 +17,15 @@ def update_cf_stack(engine, context, body, template, def prepare_user_data(context, template='Default', **kwargs): - config = context['/config'] + settings = config.CONF.rabbitmq + with open('data/init.ps1') as init_script_file: with open('data/templates/agent-config/%s.template' % template) as template_file: init_script = init_script_file.read() template_data = template_file.read() template_data = template_data.replace( - '%RABBITMQ_HOST%', - config.get_setting('rabbitmq', 'host') or 'localhost') + '%RABBITMQ_HOST%', settings.host) template_data = template_data.replace( '%RESULT_QUEUE%', '-execution-results-%s' % str(context['/dataSource']['name'])) diff --git a/conductor/conductor/commands/cloud_formation.py b/conductor/conductor/commands/cloud_formation.py index d0225edb..ab62ac57 100644 --- a/conductor/conductor/commands/cloud_formation.py +++ b/conductor/conductor/commands/cloud_formation.py @@ -1,16 +1,23 @@ import anyjson import os import uuid +import eventlet import conductor.helpers from command import CommandBase -from subprocess import call +import conductor.config +from heatclient.client import Client +import heatclient.exc + class HeatExecutor(CommandBase): - def __init__(self, stack): + def __init__(self, stack, token): self._pending_list = [] self._stack = stack + settings = conductor.config.CONF.heat + self._heat_client = Client('1', settings.url, + token_only=True, token=token) def execute(self, template, mappings, arguments, callback): with open('data/templates/cf/%s.template' % template) as template_file: @@ -43,15 +50,15 @@ class HeatExecutor(CommandBase): print 'Executing heat template', anyjson.dumps(template), \ 'with arguments', arguments, 'on stack', self._stack - if not os.path.exists("tmp"): - os.mkdir("tmp") - file_name = "tmp/" + str(uuid.uuid4()) - print "Saving template to", file_name - with open(file_name, "w") as f: - f.write(anyjson.dumps(template)) - - arguments_str = ';'.join(['%s=%s' % (key, value) - for (key, value) in arguments.items()]) + # if not os.path.exists("tmp"): + # os.mkdir("tmp") + # file_name = "tmp/" + str(uuid.uuid4()) + # print "Saving template to", file_name + # with open(file_name, "w") as f: + # f.write(anyjson.dumps(template)) + # + # arguments_str = ';'.join(['%s=%s' % (key, value) + # for (key, value) in arguments.items()]) # call([ # "./heat_run", "stack-create", # "-f" + file_name, @@ -59,12 +66,35 @@ class HeatExecutor(CommandBase): # self._stack # ]) + try: + self._heat_client.stacks.update( + stack_id=self._stack, + parameters=arguments, + template=template) + self._wait_state('UPDATE_COMPLETE') + except heatclient.exc.HTTPNotFound: + self._heat_client.stacks.create( + stack_name=self._stack, + parameters=arguments, + template=template) + self._wait_state('CREATE_COMPLETE') + pending_list = self._pending_list self._pending_list = [] for item in pending_list: item['callback'](True) - - return True + + def _wait_state(self, state): + while True: + status = self._heat_client.stacks.get( + stack_id=self._stack).stack_status + if 'IN_PROGRESS' in status: + eventlet.sleep(1) + continue + if status != state: + raise EnvironmentError() + return + diff --git a/conductor/conductor/commands/dispatcher.py b/conductor/conductor/commands/dispatcher.py index 606266e5..624ce302 100644 --- a/conductor/conductor/commands/dispatcher.py +++ b/conductor/conductor/commands/dispatcher.py @@ -4,9 +4,9 @@ import windows_agent class CommandDispatcher(command.CommandBase): - def __init__(self, environment_id, rmqclient): + def __init__(self, environment_id, rmqclient, token): self._command_map = { - 'cf': cloud_formation.HeatExecutor(environment_id), + 'cf': cloud_formation.HeatExecutor(environment_id, token), 'agent': windows_agent.WindowsAgentExecutor( environment_id, rmqclient, environment_id) } diff --git a/conductor/conductor/config.py b/conductor/conductor/config.py index 1e42cada..51909189 100644 --- a/conductor/conductor/config.py +++ b/conductor/conductor/config.py @@ -45,9 +45,14 @@ rabbit_opts = [ cfg.StrOpt('virtual_host', default='/'), ] +heat_opts = [ + cfg.StrOpt('url') +] + CONF = cfg.CONF CONF.register_opts(paste_deploy_opts, group='paste_deploy') CONF.register_opts(rabbit_opts, group='rabbitmq') +CONF.register_opts(heat_opts, group='heat') CONF.import_opt('verbose', 'conductor.openstack.common.log') diff --git a/conductor/etc/conductor.conf b/conductor/etc/conductor.conf index 189eeedf..24930b10 100644 --- a/conductor/etc/conductor.conf +++ b/conductor/etc/conductor.conf @@ -1,9 +1,11 @@ [DEFAULT] log_file = logs/conductor.log +[heat] +url = http://172.18.124.101:8004/v1/16eb78cbb688459c8308d89678bcef50 [rabbitmq] -host = localhost +host = 172.18.124.101 port = 5672 virtual_host = keero login = keero diff --git a/conductor/test.json b/conductor/test.json index 24270704..c2815d50 100644 --- a/conductor/test.json +++ b/conductor/test.json @@ -1,6 +1,7 @@ { - "name": "MyDataCenter", + "name": "MyDataCenterx", "id": "adc6d143f9584d10808c7ef4d07e4802", + "token": "MIINIQYJKoZIhvcNAQcCoIINEjCCDQ4CAQExCTAHBgUrDgMCGjCCC-oGCSqGSIb3DQEHAaCCC+sEggvneyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxMy0wMy0yNlQwNjo0NTozNy4zOTI0MDAiLCAiZXhwaXJlcyI6ICIyMDEzLTAzLTI3VDA2OjQ1OjM3WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMTZlYjc4Y2JiNjg4NDU5YzgzMDhkODk2NzhiY2VmNTAiLCAibmFtZSI6ICJhZG1pbiJ9fSwgInNlcnZpY2VDYXRhbG9nIjogW3siZW5kcG9pbnRzIjogW3siYWRtaW5VUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjg3NzQvdjIvMTZlYjc4Y2JiNjg4NDU5YzgzMDhkODk2NzhiY2VmNTAiLCAicmVnaW9uIjogIlJlZ2lvbk9uZSIsICJpbnRlcm5hbFVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6ODc3NC92Mi8xNmViNzhjYmI2ODg0NTljODMwOGQ4OTY3OGJjZWY1MCIsICJpZCI6ICIwNGFlNjM2ZTdhYzc0NmJjYjExM2EwYzI5NDYzMzgzMCIsICJwdWJsaWNVUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjg3NzQvdjIvMTZlYjc4Y2JiNjg4NDU5YzgzMDhkODk2NzhiY2VmNTAifV0sICJlbmRwb2ludHNfbGlua3MiOiBbXSwgInR5cGUiOiAiY29tcHV0ZSIsICJuYW1lIjogIm5vdmEifSwgeyJlbmRwb2ludHMiOiBbeyJhZG1pblVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6MzMzMyIsICJyZWdpb24iOiAiUmVnaW9uT25lIiwgImludGVybmFsVVJMIjogImh0dHA6Ly8xNzIuMTguMTI0LjEwMTozMzMzIiwgImlkIjogIjA5MmJkMjMyMGU5ZDRlYWY4ZDBlZjEzNDhjOGU3NTJjIiwgInB1YmxpY1VSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6MzMzMyJ9XSwgImVuZHBvaW50c19saW5rcyI6IFtdLCAidHlwZSI6ICJzMyIsICJuYW1lIjogInMzIn0sIHsiZW5kcG9pbnRzIjogW3siYWRtaW5VUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjkyOTIiLCAicmVnaW9uIjogIlJlZ2lvbk9uZSIsICJpbnRlcm5hbFVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6OTI5MiIsICJpZCI6ICI1ZWUzNjdjYzRhNjY0YmQzYTYyNmI2MjBkMzFhYzcwYyIsICJwdWJsaWNVUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjkyOTIifV0sICJlbmRwb2ludHNfbGlua3MiOiBbXSwgInR5cGUiOiAiaW1hZ2UiLCAibmFtZSI6ICJnbGFuY2UifSwgeyJlbmRwb2ludHMiOiBbeyJhZG1pblVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6ODAwMC92MSIsICJyZWdpb24iOiAiUmVnaW9uT25lIiwgImludGVybmFsVVJMIjogImh0dHA6Ly8xNzIuMTguMTI0LjEwMTo4MDAwL3YxIiwgImlkIjogIjM3MzMzYmQwNDkxOTQzY2FiNWEyZGM5N2I5YWQzYjE2IiwgInB1YmxpY1VSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6ODAwMC92MSJ9XSwgImVuZHBvaW50c19saW5rcyI6IFtdLCAidHlwZSI6ICJjbG91ZGZvcm1hdGlvbiIsICJuYW1lIjogImhlYXQtY2ZuIn0sIHsiZW5kcG9pbnRzIjogW3siYWRtaW5VUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjg3NzYvdjEvMTZlYjc4Y2JiNjg4NDU5YzgzMDhkODk2NzhiY2VmNTAiLCAicmVnaW9uIjogIlJlZ2lvbk9uZSIsICJpbnRlcm5hbFVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6ODc3Ni92MS8xNmViNzhjYmI2ODg0NTljODMwOGQ4OTY3OGJjZWY1MCIsICJpZCI6ICI4NTgwYjMzOTAxZWU0YTUyOWI0OGMyMzU0ZjFiMWNhZSIsICJwdWJsaWNVUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjg3NzYvdjEvMTZlYjc4Y2JiNjg4NDU5YzgzMDhkODk2NzhiY2VmNTAifV0sICJlbmRwb2ludHNfbGlua3MiOiBbXSwgInR5cGUiOiAidm9sdW1lIiwgIm5hbWUiOiAiY2luZGVyIn0sIHsiZW5kcG9pbnRzIjogW3siYWRtaW5VUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjg3NzMvc2VydmljZXMvQWRtaW4iLCAicmVnaW9uIjogIlJlZ2lvbk9uZSIsICJpbnRlcm5hbFVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6ODc3My9zZXJ2aWNlcy9DbG91ZCIsICJpZCI6ICIwYTViOTIyNTNiZjg0NTAwYTA4OWY1N2VkMmYzZDY3NSIsICJwdWJsaWNVUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjg3NzMvc2VydmljZXMvQ2xvdWQifV0sICJlbmRwb2ludHNfbGlua3MiOiBbXSwgInR5cGUiOiAiZWMyIiwgIm5hbWUiOiAiZWMyIn0sIHsiZW5kcG9pbnRzIjogW3siYWRtaW5VUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjgwMDQvdjEvMTZlYjc4Y2JiNjg4NDU5YzgzMDhkODk2NzhiY2VmNTAiLCAicmVnaW9uIjogIlJlZ2lvbk9uZSIsICJpbnRlcm5hbFVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6ODAwNC92MS8xNmViNzhjYmI2ODg0NTljODMwOGQ4OTY3OGJjZWY1MCIsICJpZCI6ICJhMjRjMGY1ZmUzMmQ0ZDU5YWEwMTk1Mzg3OGFlMDQwNyIsICJwdWJsaWNVUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjgwMDQvdjEvMTZlYjc4Y2JiNjg4NDU5YzgzMDhkODk2NzhiY2VmNTAifV0sICJlbmRwb2ludHNfbGlua3MiOiBbXSwgInR5cGUiOiAib3JjaGVzdHJhdGlvbiIsICJuYW1lIjogImhlYXQifSwgeyJlbmRwb2ludHMiOiBbeyJhZG1pblVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6MzUzNTcvdjIuMCIsICJyZWdpb24iOiAiUmVnaW9uT25lIiwgImludGVybmFsVVJMIjogImh0dHA6Ly8xNzIuMTguMTI0LjEwMTo1MDAwL3YyLjAiLCAiaWQiOiAiNGM4M2VlYjk3MDA5NDg3M2FiNjg3NjUzNWJlZjgxZWEiLCAicHVibGljVVJMIjogImh0dHA6Ly8xNzIuMTguMTI0LjEwMTo1MDAwL3YyLjAifV0sICJlbmRwb2ludHNfbGlua3MiOiBbXSwgInR5cGUiOiAiaWRlbnRpdHkiLCAibmFtZSI6ICJrZXlzdG9uZSJ9XSwgInVzZXIiOiB7InVzZXJuYW1lIjogImFkbWluIiwgInJvbGVzX2xpbmtzIjogW10sICJpZCI6ICJmMmNkZWM4NTQ2MmQ0N2UzODQ5ZTZmMzE3NGRhMTk4NSIsICJyb2xlcyI6IFt7Im5hbWUiOiAiYWRtaW4ifV0sICJuYW1lIjogImFkbWluIn0sICJtZXRhZGF0YSI6IHsiaXNfYWRtaW4iOiAwLCAicm9sZXMiOiBbIjc4N2JlODdjMGFkMjQ3ODJiNTQ4NWU5NjNhZjllNzllIl19fX0xgf8wgfwCAQEwXDBXMQswCQYDVQQGEwJVUzEOMAwGA1UECBMFVW5zZXQxDjAMBgNVBAcTBVVuc2V0MQ4wDAYDVQQKEwVVbnNldDEYMBYGA1UEAxMPd3d3LmV4YW1wbGUuY29tAgEBMAcGBSsOAwIaMA0GCSqGSIb3DQEBAQUABIGAURfgqd8iZ-UWZTta2pyKzXBXm9nmdzlOY-TN8526LWH4jrU1uuimAZKSjZUCwmnaSvoXHLlP6CSGvNUJWDDu6YFNmDfmatVqFrTij4EFGruExmtUxmhbQOnAyhKqIxHFg2t3VKEB2tVhLGSzoSH1dM2+j0-I0JgOLWIStVFEF5A=", "services": { "activeDirectories": [ { diff --git a/conductor/tools/pip-requires b/conductor/tools/pip-requires index 816ab986..ac910b04 100644 --- a/conductor/tools/pip-requires +++ b/conductor/tools/pip-requires @@ -5,5 +5,6 @@ puka Paste PasteDeploy iso8601>=0.1.4 +python-heatclient http://tarballs.openstack.org/oslo-config/oslo-config-2013.1b4.tar.gz#egg=oslo-config