diff --git a/backend/os/.gitignore b/backend/os/.gitignore new file mode 100644 index 00000000..5e7d2734 --- /dev/null +++ b/backend/os/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/backend/versionfile/kolla/.gitignore b/backend/versionfile/kolla/.gitignore new file mode 100644 index 00000000..5e7d2734 --- /dev/null +++ b/backend/versionfile/kolla/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/code/daisy/daisy/api/backends/kolla/api.py b/code/daisy/daisy/api/backends/kolla/api.py index 453bb4e2..1d234669 100644 --- a/code/daisy/daisy/api/backends/kolla/api.py +++ b/code/daisy/daisy/api/backends/kolla/api.py @@ -21,6 +21,7 @@ import subprocess from daisy import i18n from daisy.api.backends import driver import daisy.api.backends.kolla.install as instl +import daisy.api.backends.kolla.upgrade as upgrd import daisy.api.backends.common as daisy_cmn @@ -50,6 +51,20 @@ class API(driver.DeploymentDriver): kolla_install_task = instl.KOLLAInstallTask(req, cluster_id) kolla_install_task.start() + def upgrade(self, req, cluster_id, version_id, vpatch_id, update_file, hosts): + """ + update openstack to a cluster. + + :param req: The WSGI/Webob Request object + + :raises HTTPBadRequest if x-install-cluster is missing + """ + LOG.info( + _("Begin to update OpenStack nodes, please waiting....")) + + kolla_install_task = upgrd.KOLLAUpgradeTask(req, cluster_id, version_id, update_file) + kolla_install_task.start() + def update_progress_to_db(self, req, update_info, discover_host_meta): discover = {} discover['status'] = update_info['status'] diff --git a/code/daisy/daisy/api/backends/kolla/common.py b/code/daisy/daisy/api/backends/kolla/common.py index 06d62823..a05f936c 100644 --- a/code/daisy/daisy/api/backends/kolla/common.py +++ b/code/daisy/daisy/api/backends/kolla/common.py @@ -34,6 +34,7 @@ _LI = i18n._LI _LW = i18n._LW daisy_kolla_path = '/var/lib/daisy/kolla/' +daisy_kolla_ver_path = '/var/lib/daisy/versionfile/kolla/' KOLLA_STATE = { 'INIT': 'init', 'INSTALLING': 'installing', @@ -280,3 +281,42 @@ def mask_string(unmasked, mask_list=None, replace_list=None): word = word.replace(before, after) masked = masked.replace(word, STR_MASK) return masked + + +def check_and_get_kolla_version(daisy_kolla_pkg_path, file_name=None): + kolla_version_pkg_file = "" + if file_name: + get_kolla_version_pkg = "ls %s| grep %s$" % (daisy_kolla_pkg_path, + file_name) + else: + get_kolla_version_pkg = "ls %s| grep ^kolla.*\.tgz$"\ + % daisy_kolla_pkg_path + obj = subprocess.Popen(get_kolla_version_pkg, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (stdoutput, erroutput) = obj.communicate() + if stdoutput: + kolla_version_pkg_name = stdoutput.split('\n')[0] + kolla_version_pkg_file = daisy_kolla_pkg_path + kolla_version_pkg_name + chmod_for_kolla_version = 'chmod +x %s' % kolla_version_pkg_file + daisy_cmn.subprocess_call(chmod_for_kolla_version) + return kolla_version_pkg_file + + +def version_load(kolla_version_pkg_file, fp): + tar_for_kolla_version = 'tar mzxvf %s' % kolla_version_pkg_file + subprocess.call(tar_for_kolla_version, shell=True) + get_container_id = "docker ps -a |grep registry |awk -F ' ' '{printf $1}' " + container_id = subprocess.check_output(get_container_id, shell=True) + if container_id: + stop_container = 'docker stop %s' % container_id + daisy_cmn.subprocess_call(stop_container, fp) + remove_container = 'docker rm %s' % container_id + daisy_cmn.subprocess_call(remove_container, fp) + registry_file = daisy_kolla_ver_path + "/tmp/registry" + daisy_cmn.subprocess_call( + 'docker run -d -p 4000:5000 --restart=always \ + -e REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY=/tmp/registry \ + -v %s:/tmp/registry --name registry registry:2' % registry_file, fp) + diff --git a/code/daisy/daisy/api/backends/kolla/install.py b/code/daisy/daisy/api/backends/kolla/install.py index 568a568e..88258ead 100644 --- a/code/daisy/daisy/api/backends/kolla/install.py +++ b/code/daisy/daisy/api/backends/kolla/install.py @@ -56,6 +56,7 @@ install_kolla_progress = 0.0 install_mutex = threading.Lock() kolla_file = "/home/kolla_install" kolla_config_file = "/etc/kolla/globals.yml" +daisy_kolla_ver_path = kolla_cmn.daisy_kolla_ver_path def update_progress_to_db(req, role_id_list, @@ -446,6 +447,21 @@ class KOLLAInstallTask(Thread): % self.cluster_id)) def _run(self): + # check and get version + cluster_data = registry.get_cluster_metadata(self.req.context, self.cluster_id) + if cluster_data.get('tecs_version_id', None): + vid = cluster_data['tecs_version_id'] + version_info = registry.get_version_metadata(self.req.context, vid) + kolla_version_pkg_file = \ + kolla_cmn.check_and_get_kolla_version(daisy_kolla_ver_path, version_info['name']) + else: + kolla_version_pkg_file =\ + kolla_cmn.check_and_get_tecs_version(daisy_kolla_ver_path) + if not kolla_version_pkg_file: + self.state = kolla_state['INSTALL_FAILED'] + self.message =\ + "kolla version file not found in %s" % daisy_kolla_ver_path + raise exception.NotFound(message=self.message) (kolla_config, self.mgt_ip_list, host_name_ip_list) = \ get_cluster_kolla_config(self.req, self.cluster_id) if not self.mgt_ip_list: @@ -480,6 +496,7 @@ class KOLLAInstallTask(Thread): self.message, 0) docker_registry_ip = _get_local_ip() with open(self.log_file, "w+") as fp: + kolla_cmn.version_load(kolla_version_pkg_file, fp) threads = [] for host in hosts_list: t = threading.Thread(target=thread_bin, @@ -588,3 +605,7 @@ class KOLLAInstallTask(Thread): self.message, 100) update_progress_to_db(self.req, role_id_list, kolla_state['ACTIVE'], 100) + for host_id in host_id_list: + daisy_cmn.update_db_host_status( + self.req, host_id, {'tecs_version_id': cluster_data['tecs_version_id'], + 'tecs_patch_id': ''}) diff --git a/code/daisy/daisy/api/backends/kolla/upgrade.py b/code/daisy/daisy/api/backends/kolla/upgrade.py new file mode 100644 index 00000000..1dedbfae --- /dev/null +++ b/code/daisy/daisy/api/backends/kolla/upgrade.py @@ -0,0 +1,160 @@ +# Copyright 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +/update endpoint for Daisy v1 API +""" + +import subprocess +import time +from oslo_log import log as logging +from daisy import i18n +import daisy.api.backends.common as daisy_cmn +import daisy.api.backends.kolla.common as kolla_cmn +import daisy.registry.client.v1.api as registry +from threading import Thread +from daisy.common import exception + + +LOG = logging.getLogger(__name__) +_ = i18n._ +_LE = i18n._LE +_LI = i18n._LI +_LW = i18n._LW + +kolla_state = kolla_cmn.KOLLA_STATE + + +def update_all_host_progress_to_db(req, hosts_id_list, role_host_meta={}): + for host_id in hosts_id_list: + host_roles = registry.get_host_roles_by_host_id(req.context, host_id) + for host_role_id in host_roles: + if role_host_meta: + daisy_cmn.update_role_host(req, host_role_id['id'], + role_host_meta) + + +class KOLLAUpgradeTask(Thread): + """ + Class for kolla upgrade openstack. + """ + + def __init__(self, req, cluster_id, version_id, update_file): + super(KOLLAUpgradeTask, self).__init__() + self.req = req + self.cluster_id = cluster_id + self.progress = 0 + self.version_id = version_id + self.update_file = update_file + self.message = "" + self.kolla_file = "/home/kolla_install" + self.log_file = "/var/log/daisy/kolla_%s_upgrade.log" % self.cluster_id + + + def run(self): + hosts = registry.get_cluster_hosts(self.req.context, self.cluster_id) + hosts_id_list = [host['host_id'] for host in hosts] + cluster_meta = registry.get_cluster_metadata(self.req.context, self.cluster_id) + self.message = "prechecking envirnoment" + update_all_host_progress_to_db(self.req, hosts_id_list, + {'progress': 0, + 'status': kolla_state['UPDATING'], + 'messages': self.message}) + kolla_version_pkg_file = kolla_cmn.check_and_get_kolla_version( + kolla_cmn.daisy_kolla_ver_path, self.update_file) + if not kolla_version_pkg_file: + self.message = "kolla version file not found in %s"\ + % kolla_cmn.daisy_kolla_path + update_all_host_progress_to_db(self.req, hosts_id_list, + {'progress': 0, + 'status': kolla_state['UPDATE_FAILED'], + 'messages': self.message}) + raise exception.NotFound(message=self.message) + if cluster_meta['tecs_version_id']: + version_data = registry.get_version_metadata( + self.req.context, cluster_meta['tecs_version_id']) + if version_data['name'] == self.update_file: + LOG.error(_("kolla version %s is not need to upgrade!" + % version_data['name'])) + self.message = "kolla version %s is not need to upgrade!" % version_data['name'] + update_all_host_progress_to_db(self.req, hosts_id_list, + {'progress': 0, + 'status': kolla_state['UPDATE_FAILED'], + 'messages': self.message}) + return + for host in hosts: + host_meta = daisy_cmn.get_host_detail(self.req, host["host_id"]) + host_ip = daisy_cmn.get_management_ip(host_meta) + host_ip_set = set() + host_ip_set.add(host_ip) + unreached_hosts = daisy_cmn.check_ping_hosts( + host_ip_set, 3) + if unreached_hosts: + self.message = "hosts %s ping failed" % unreached_hosts + update_all_host_progress_to_db(self.req, hosts_id_list, + {'progress': 0, + 'status': kolla_state['UPDATE_FAILED'], + 'messages': self.message}) + raise exception.NotFound(message=self.message) + + LOG.info(_("precheck envirnoment successfully ...")) + self.message = "openstack upgrading" + update_all_host_progress_to_db(self.req, hosts_id_list, + {'progress': 10, + 'status': kolla_state['UPDATING'], + 'messages': self.message}) + with open(self.log_file, "w+") as fp: + kolla_cmn.version_load(kolla_version_pkg_file, fp) + update_all_host_progress_to_db(self.req, hosts_id_list, + {'progress': 20, + 'status': kolla_state['UPDATING'], + 'messages': self.message}) + try: + LOG.info(_("begin to kolla-ansible " + "upgrade for all nodes...")) + exc_result = subprocess.check_output( + 'cd %s/kolla && ./tools/kolla-ansible upgrade -i ' + '%s/kolla/ansible/inventory/multinode' % + (self.kolla_file, self.kolla_file), + shell=True, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + LOG.error("kolla-ansible upgrade failed!") + self.message = "kolla-ansible upgrade failed!" + update_all_host_progress_to_db(self.req, hosts_id_list, + {'progress': 20, + 'status': kolla_state['UPDATE_FAILED'], + 'messages': self.message}) + LOG.info(_("kolla-ansible upgrade failed!")) + fp.write(e.output.strip()) + exit() + else: + LOG.info(_("openstack upgraded successfully")) + fp.write(exc_result) + self.message = "openstack upgraded successfully" + update_all_host_progress_to_db(self.req, hosts_id_list, + {'progress': 100, + 'status': kolla_state['ACTIVE'], + 'messages': self.message}) + for host_id in hosts_id_list: + daisy_cmn.update_db_host_status( + self.req, host_id, {'tecs_version_id': self.version_id, + 'tecs_patch_id': ''}) + cluster_meta = {} + cluster_meta['tecs_version_id'] = self.version_id + cluster_meta = registry.update_cluster_metadata( + self.req.context, self.cluster_id, cluster_meta) + LOG.info(_("openstack upgraded for cluster %s successfully." + % self.cluster_id)) + diff --git a/code/daisy/daisy/api/v1/install.py b/code/daisy/daisy/api/v1/install.py index 61a44a19..4a2fa336 100755 --- a/code/daisy/daisy/api/v1/install.py +++ b/code/daisy/daisy/api/v1/install.py @@ -436,10 +436,13 @@ class Controller(controller.BaseController): request=req, content_type="text/plain") if not install_meta.get('hosts', None): - msg = "upgrade hosts is null!" - raise HTTPBadRequest(explanation=msg, - request=req, - content_type="text/plain") + if install_meta['update_object'] == "kolla": + hosts = [] + else: + msg = "upgrade hosts is null!" + raise HTTPBadRequest(explanation=msg, + request=req, + content_type="text/plain") else: hosts = eval(install_meta['hosts']) update_file = "" diff --git a/rpm/SPECS/daisy.spec b/rpm/SPECS/daisy.spec index 40686fda..06141498 100755 --- a/rpm/SPECS/daisy.spec +++ b/rpm/SPECS/daisy.spec @@ -184,6 +184,7 @@ install -d -m 755 %{buildroot}%{_localstatedir}/log/daisy mkdir -p %{buildroot}/var/lib/daisy cp -Rf ../kolla %{buildroot}/var/lib/daisy cp -Rf ../trustme.sh %{buildroot}/var/lib/daisy +cp -Rf ../versionfile %{buildroot}/var/lib/daisy %pre id daisy @@ -264,6 +265,8 @@ fi %dir %attr(0777, all, all) /var/lib/daisy/kolla/ %attr(0755, daisy, daisy) /var/lib/daisy/kolla/* %attr(0755, daisy, daisy) /var/lib/daisy/trustme.sh +%dir %attr(0777, daisy, daisy) /var/lib/daisy/versionfile/ +%attr(0777, daisy, daisy) /var/lib/daisy/versionfile/* %files -n python-daisy %doc README.rst diff --git a/tools/setup/install/install_func.sh b/tools/setup/install/install_func.sh index 7e21b475..9a9683de 100755 --- a/tools/setup/install/install_func.sh +++ b/tools/setup/install/install_func.sh @@ -62,7 +62,6 @@ function ip_to_cidr() function kolla_install { write_install_log "Begin install kolla depends..." - catalog_url="http://127.0.0.1:4000/v2/_catalog" check_installed "docker-engine" if [[ "$has_installed" == "yes" ]];then echo "docker-engine has been already installed" @@ -91,28 +90,24 @@ function kolla_install if [[ "$has_installed" == "yes" ]];then echo "jinja2 has been already installed" else - yum install -y https://kojipkgs.fedoraproject.org//packages/python-jinja2/2.8/2.fc23/noarch/python-jinja2-2.8-2.fc23.noarch.rpm + yum install -y https://kojipkgs.fedoraproject.org//packages/python-jinja2/2.8/2.fc23/noarch/python-jinja2-2.8-2.fc23.noarch.rpm fi imagebranch="newton" imageversion="latest" imageserver="http://120.24.17.215" - imagedir="/home/kolla_install/docker/" + imagedir="/var/lib/daisy/versionfile/kolla" imagename="kolla-image-$imagebranch-$imageversion.tgz" + sourcedir="/home/kolla_install/" write_install_log "Begin copy images..." + cd $imagedir if [ -f "$imagedir/$imagename" ];then echo "$imagename already exist!" - cd $imagedir - tar mzxvf $imagename else - mkdir -p $imagedir - cd $imagedir wget "$imageserver/$imagename" - tar mzxvf $imagename fi - - sourcedir="/home/kolla_install/" + tar mzxvf $imagename sourceversion=$(cat $imagedir/registry-*.version | head -1) write_install_log "Begin clone kolla... $sourceversion" @@ -129,18 +124,15 @@ function kolla_install cp -r /home/kolla_install/kolla/etc/kolla /etc # TODO: (huzhj)Use latest registry server from upstream - catalog=`curl $catalog_url |grep repositories` - if [ -z $catalog ];then - if [ -f "/home/kolla_install/docker/registry-server.tar" ];then - echo "registry-server.tar already exist!" - else - cd /home/kolla_install/docker - wget "http://daisycloud.org/static/files/registry-server.tar" - fi - cd /home/kolla_install/docker - docker load < ./registry-server.tar - docker run -d -p 4000:5000 --restart=always -e REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY=/tmp/registry -v /home/kolla_install/docker/tmp/registry:/tmp/registry --name registry registry:2 + cd $imagedir + if [ -f "$imagedir/registry-server.tar" ];then + echo "registry-server.tar already exist!" + else + wget "http://daisycloud.org/static/files/registry-server.tar" fi + docker load < ./registry-server.tar + rm -rf $imagedir/tmp + rm -rf $imagedir/registry-*.version } #rm daisy yum config file function delete_unused_repo_file