From 38e3ef873fabe93acf961cb8ebb68b9b301d9d62 Mon Sep 17 00:00:00 2001 From: Yao Lu Date: Mon, 26 Sep 2016 10:31:22 +0800 Subject: [PATCH] Support auto scaling for kolla backend This patch mainly does thress things: 1) Refactor auto scaling code to support enforcing different satisfy check for different backends. 2) Introduce satisfy check for kolla backend, keep the old check in tecs directory. 3) Update role state after finish deploy. Change-Id: Ice323dc7a96a1e922ba3545262b31dd634517863 Signed-off-by: Yao Lu --- .../daisy/daisy/api/backends/kolla/install.py | 2 + code/daisy/daisy/cmd/orchestration.py | 16 ++- .../daisy/orchestration/kolla/__init__.py | 0 .../daisy/orchestration/kolla/install.py | 100 ++++++++++++++++++ code/daisy/daisy/orchestration/manager.py | 74 +------------ .../daisy/orchestration/tecs/__init__.py | 0 .../daisy/daisy/orchestration/tecs/install.py | 99 +++++++++++++++++ 7 files changed, 216 insertions(+), 75 deletions(-) create mode 100755 code/daisy/daisy/orchestration/kolla/__init__.py create mode 100755 code/daisy/daisy/orchestration/kolla/install.py create mode 100755 code/daisy/daisy/orchestration/tecs/__init__.py create mode 100755 code/daisy/daisy/orchestration/tecs/install.py diff --git a/code/daisy/daisy/api/backends/kolla/install.py b/code/daisy/daisy/api/backends/kolla/install.py index 52f70dea..6ec8d5a5 100644 --- a/code/daisy/daisy/api/backends/kolla/install.py +++ b/code/daisy/daisy/api/backends/kolla/install.py @@ -482,3 +482,5 @@ class KOLLAInstallTask(Thread): host_id_list, kolla_state['ACTIVE'], self.message, 100) + update_progress_to_db(self.req, role_id_list, + kolla_state['ACTIVE'], 100) diff --git a/code/daisy/daisy/cmd/orchestration.py b/code/daisy/daisy/cmd/orchestration.py index 05e45f12..225c76de 100755 --- a/code/daisy/daisy/cmd/orchestration.py +++ b/code/daisy/daisy/cmd/orchestration.py @@ -29,8 +29,10 @@ from oslo_log import log as logging from daisy.common import exception from daisy.common import config from oslo_service import loopingcall -from daisy.orchestration import manager import six +import ConfigParser +from oslo_utils import importutils +import daisy.api.backends.common as daisy_cmn # Monkey patch socket and time eventlet.patcher.monkey_patch(all=False, socket=True, time=True, thread=True) @@ -58,12 +60,22 @@ def fail(returncode, e): sys.stderr.write("ERROR: %s\n" % six.text_type(e)) +def get_backend(): + config = ConfigParser.ConfigParser() + config.read(daisy_cmn.daisy_conf_file) + backend = config.get("BACKEND", "default_backend_types") + return backend + + def main(): try: config.parse_args() logging.setup(CONF, 'daisy') + backend = get_backend() + manager = "daisy.orchestration.%s.install" % backend + api = importutils.import_module(manager) timer = loopingcall.FixedIntervalLoopingCall( - manager.OrchestrationManager.find_auto_scale_cluster) + api.find_auto_scale_cluster) timer.start(float(CONF.orchestration.auto_scale_interval)).wait() except exception.WorkerCreationFailure as e: fail(2, e) diff --git a/code/daisy/daisy/orchestration/kolla/__init__.py b/code/daisy/daisy/orchestration/kolla/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/code/daisy/daisy/orchestration/kolla/install.py b/code/daisy/daisy/orchestration/kolla/install.py new file mode 100755 index 00000000..ef3b65b7 --- /dev/null +++ b/code/daisy/daisy/orchestration/kolla/install.py @@ -0,0 +1,100 @@ +# Copyright 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +/orchestration for kolla API +""" + +from oslo_config import cfg +from oslo_log import log as logging +from webob import exc + +from daisy.common import exception +from daisyclient.v1 import client +import ConfigParser +from daisy.orchestration import manager + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +def find_auto_scale_cluster(): + try: + daisy_version = 1.0 + config_discoverd = ConfigParser.ConfigParser() + config_discoverd.read("/etc/daisy/daisy-api.conf") + bind_port = config_discoverd.get("DEFAULT", "bind_port") + daisy_endpoint = "http://127.0.0.1:" + bind_port + # daisy_endpoint="http://127.0.0.1:19292" + daisy_client = client.Client( + version=daisy_version, endpoint=daisy_endpoint) + orchestrationManager = manager.OrchestrationManager() + cluster_meta = {'auto_scale': '1'} + params = {'filters': cluster_meta} + clusters_gen = daisy_client.clusters.list(**params) + clusters = [cluster.to_dict() + for cluster in clusters_gen if cluster.auto_scale == 1] + if clusters: + cluster_id = clusters[0]['id'] + params = {'filters': ''} + hosts_gen = daisy_client.hosts.list(**params) + init_hosts = [host.to_dict( + ) for host in hosts_gen if host.os_status == "init" or + host.os_status == "install-failed"] + if not init_hosts: + LOG.info("no init or install-failed host") + return {"status": "no init host"} + + params = {'filters': {'cluster_id': cluster_id}} + roles_gen = daisy_client.roles.list(**params) + roles_in_cluster = [role.to_dict() for role in roles_gen] + roles = [role for role in roles_in_cluster if role[ + 'name'] == "CONTROLLER_LB" and role['status'] == "active"] + if not roles: + LOG.info("no active CONTROLLER_LB role") + return {"status": "no active CONTROLLER_LB role"} + for host in init_hosts: + if host['status'] == "init": + host_info = daisy_client.hosts.get(host['id']) + if hasattr(host_info, "interfaces"): + scale_host = \ + orchestrationManager.set_scale_host_interface( + cluster_id, host_info, daisy_client) + if scale_host: + host_meta = { + 'hugepagesize': scale_host.hugepagesize, + 'hugepages': scale_host.hugepages, + 'isolcpus': scale_host.isolcpus, + 'name': scale_host.name, + 'os_version': scale_host.os_version_file, + 'root_lv_size': scale_host.root_lv_size, + 'swap_lv_size': scale_host.swap_lv_size, + 'role': ['COMPUTER'], + 'cluster': cluster_id, + 'interfaces': scale_host.interfaces} + daisy_client.hosts.update( + host['id'], **host_meta) + else: + LOG.error("can not set scale host") + return {"status": "no scale host"} + + else: + LOG.info("not interfaces in host %s" % host['id']) + raise exc.HTTPNotFound( + "not interfaces in host %s" % host['id']) + orchestrationManager._os_install( + cluster_id, daisy_client) + except exception.Invalid as e: + LOG.exception(e.message) diff --git a/code/daisy/daisy/orchestration/manager.py b/code/daisy/daisy/orchestration/manager.py index afd3c2f3..e88f93d1 100755 --- a/code/daisy/daisy/orchestration/manager.py +++ b/code/daisy/daisy/orchestration/manager.py @@ -19,11 +19,8 @@ from oslo_config import cfg from oslo_log import log as logging -from webob import exc from daisy.common import exception -from daisyclient.v1 import client -import ConfigParser LOG = logging.getLogger(__name__) CONF = cfg.CONF @@ -35,76 +32,7 @@ class OrchestrationManager(): """Load orchestration options and initialization.""" pass - @staticmethod - def find_auto_scale_cluster(): - try: - daisy_version = 1.0 - config_discoverd = ConfigParser.ConfigParser() - config_discoverd.read("/etc/daisy/daisy-api.conf") - bind_port = config_discoverd.get("DEFAULT", "bind_port") - daisy_endpoint = "http://127.0.0.1:" + bind_port - # daisy_endpoint="http://127.0.0.1:19292" - daisy_client = client.Client( - version=daisy_version, endpoint=daisy_endpoint) - orchestrationManager = OrchestrationManager() - cluster_meta = {'auto_scale': '1'} - params = {'filters': cluster_meta} - clusters_gen = daisy_client.clusters.list(**params) - clusters = [cluster.to_dict() - for cluster in clusters_gen if cluster.auto_scale == 1] - if clusters: - cluster_id = clusters[0]['id'] - params = {'filters': ''} - hosts_gen = daisy_client.hosts.list(**params) - init_hosts = [host.to_dict( - ) for host in hosts_gen if host.os_status == "init" or - host.os_status == "install-failed"] - if not init_hosts: - LOG.info("no init or install-failed host") - return {"status": "no init host"} - - params = {'filters': {'cluster_id': cluster_id}} - roles_gen = daisy_client.roles.list(**params) - roles_in_cluster = [role.to_dict() for role in roles_gen] - roles = [role for role in roles_in_cluster if role[ - 'name'] == "CONTROLLER_HA" and role['status'] == "active"] - if not roles: - LOG.info("no active CONTROLLER_HA role") - return {"status": "no active CONTROLLER_HA role"} - for host in init_hosts: - if host['status'] == "init": - host_info = daisy_client.hosts.get(host['id']) - if hasattr(host_info, "interfaces"): - scale_host = \ - orchestrationManager.set_scale_host_interface( - cluster_id, host_info, daisy_client) - if scale_host: - host_meta = { - 'hugepagesize': scale_host.hugepagesize, - 'hugepages': scale_host.hugepages, - 'isolcpus': scale_host.isolcpus, - 'name': scale_host.name, - 'os_version': scale_host.os_version_file, - 'root_lv_size': scale_host.root_lv_size, - 'swap_lv_size': scale_host.swap_lv_size, - 'role': ['COMPUTER'], - 'cluster': cluster_id, - 'interfaces': scale_host.interfaces} - daisy_client.hosts.update( - host['id'], **host_meta) - else: - LOG.error("can not set scale host") - return {"status": "no scale host"} - - else: - LOG.info("not interfaces in host %s" % host['id']) - raise exc.HTTPNotFound( - "not interfaces in host %s" % host['id']) - orchestrationManager._os_tecs_install(cluster_id, daisy_client) - except exception.Invalid as e: - LOG.exception(e.message) - - def _os_tecs_install(self, cluster_id, daisy_client): + def _os_install(self, cluster_id, daisy_client): try: install_meta = {'cluster_id': cluster_id} daisy_client.install.install(**install_meta) diff --git a/code/daisy/daisy/orchestration/tecs/__init__.py b/code/daisy/daisy/orchestration/tecs/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/code/daisy/daisy/orchestration/tecs/install.py b/code/daisy/daisy/orchestration/tecs/install.py new file mode 100755 index 00000000..631528d7 --- /dev/null +++ b/code/daisy/daisy/orchestration/tecs/install.py @@ -0,0 +1,99 @@ +# Copyright 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +/orchestration for tecs API +""" + +from oslo_config import cfg +from oslo_log import log as logging +from webob import exc + +from daisy.common import exception +from daisyclient.v1 import client +import ConfigParser +from daisy.orchestration import manager + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +def find_auto_scale_cluster(): + try: + daisy_version = 1.0 + config_discoverd = ConfigParser.ConfigParser() + config_discoverd.read("/etc/daisy/daisy-api.conf") + bind_port = config_discoverd.get("DEFAULT", "bind_port") + daisy_endpoint = "http://127.0.0.1:" + bind_port + # daisy_endpoint="http://127.0.0.1:19292" + daisy_client = client.Client( + version=daisy_version, endpoint=daisy_endpoint) + orchestrationManager = manager.OrchestrationManager() + cluster_meta = {'auto_scale': '1'} + params = {'filters': cluster_meta} + clusters_gen = daisy_client.clusters.list(**params) + clusters = [cluster.to_dict() + for cluster in clusters_gen if cluster.auto_scale == 1] + if clusters: + cluster_id = clusters[0]['id'] + params = {'filters': ''} + hosts_gen = daisy_client.hosts.list(**params) + init_hosts = [host.to_dict( + ) for host in hosts_gen if host.os_status == "init" or + host.os_status == "install-failed"] + if not init_hosts: + LOG.info("no init or install-failed host") + return {"status": "no init host"} + + params = {'filters': {'cluster_id': cluster_id}} + roles_gen = daisy_client.roles.list(**params) + roles_in_cluster = [role.to_dict() for role in roles_gen] + roles = [role for role in roles_in_cluster if role[ + 'name'] == "CONTROLLER_HA" and role['status'] == "active"] + if not roles: + LOG.info("no active CONTROLLER_HA role") + return {"status": "no active CONTROLLER_HA role"} + for host in init_hosts: + if host['status'] == "init": + host_info = daisy_client.hosts.get(host['id']) + if hasattr(host_info, "interfaces"): + scale_host = \ + orchestrationManager.set_scale_host_interface( + cluster_id, host_info, daisy_client) + if scale_host: + host_meta = { + 'hugepagesize': scale_host.hugepagesize, + 'hugepages': scale_host.hugepages, + 'isolcpus': scale_host.isolcpus, + 'name': scale_host.name, + 'os_version': scale_host.os_version_file, + 'root_lv_size': scale_host.root_lv_size, + 'swap_lv_size': scale_host.swap_lv_size, + 'role': ['COMPUTER'], + 'cluster': cluster_id, + 'interfaces': scale_host.interfaces} + daisy_client.hosts.update( + host['id'], **host_meta) + else: + LOG.error("can not set scale host") + return {"status": "no scale host"} + + else: + LOG.info("not interfaces in host %s" % host['id']) + raise exc.HTTPNotFound( + "not interfaces in host %s" % host['id']) + orchestrationManager._os_install(cluster_id, daisy_client) + except exception.Invalid as e: + LOG.exception(e.message)