From 2b5e5704bf82f0cfa1d415481891939b8e9298fe Mon Sep 17 00:00:00 2001 From: Alexander Kislitsky Date: Tue, 24 Jun 2014 11:53:50 +0400 Subject: [PATCH] Development environment on masternode Help scripts for deploy develpment environment on masternode: - command line arguments handled - deploy/revert for nailgun project - requirements.txt installation into virtual env handled - common functons for execute commands in containers Change-Id: I3c60708e01867cd8455f79b3c02cfd505118cad5 Closes-Bug: #1279303 --- fuel_development/configurator/__init__.py | 15 ++ fuel_development/configurator/common.py | 57 +++++ fuel_development/configurator/nailgun.py | 248 ++++++++++++++++++++++ fuel_development/manage.py | 98 +++++++++ fuel_development/requirements.txt | 1 + fuel_development/tox.ini | 38 ++++ run_tests.sh | 1 + 7 files changed, 458 insertions(+) create mode 100644 fuel_development/configurator/__init__.py create mode 100644 fuel_development/configurator/common.py create mode 100644 fuel_development/configurator/nailgun.py create mode 100644 fuel_development/manage.py create mode 100644 fuel_development/requirements.txt create mode 100644 fuel_development/tox.ini diff --git a/fuel_development/configurator/__init__.py b/fuel_development/configurator/__init__.py new file mode 100644 index 0000000000..8d06a39b94 --- /dev/null +++ b/fuel_development/configurator/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/fuel_development/configurator/common.py b/fuel_development/configurator/common.py new file mode 100644 index 0000000000..762fc749a5 --- /dev/null +++ b/fuel_development/configurator/common.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from fabric.api import run +from fabric.colors import green +from fabric.colors import yellow +from fabric.context_managers import hide +from fabric.context_managers import settings + + +def is_package_installed(package): + print(green("Checking package {0} is installed".format(package))) + with settings(hide('warnings'), warn_only=True): + result = run('rpm -q {0}'.format(package)) + if result.succeeded: + print(green("Package {0} is installed".format(package))) + else: + print(yellow("Package {0} is not installed".format(package))) + return result.succeeded + + +def install_package(package): + if not is_package_installed(package): + print(green("Installing package {0}".format(package))) + run('sudo yum -y install {0}'.format(package)) + print(green("Package {0} installed".format(package))) + + +def run_in_container(container_name, command): + return run('dockerctl shell {0} {1}'.format(container_name, command)) + + +def backup_file(container_name, origin, backup): + command = 'cp --no-clobber {0} {1}'.format(origin, backup) + run_in_container(container_name, command) + + +def revert_file(container_name, backup, origin): + command = 'cp {0} {1}'.format(backup, origin) + run_in_container(container_name, command) + + +def escape_path(s): + return s.replace('/', '\/') diff --git a/fuel_development/configurator/nailgun.py b/fuel_development/configurator/nailgun.py new file mode 100644 index 0000000000..62000fede9 --- /dev/null +++ b/fuel_development/configurator/nailgun.py @@ -0,0 +1,248 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from fabric.api import env +from fabric.colors import green +from fabric.context_managers import hide +from fabric.context_managers import settings +from fabric.contrib.project import rsync_project +from fabric.operations import run + +from common import backup_file +from common import escape_path +from common import install_package +from common import revert_file +from common import run_in_container + + +REMOTE_DEST_DIR = '/etc/fuel/development' +NGINX_CONFIG_FILE = '/etc/nginx/conf.d/nailgun.conf' +NGINX_CONFIG_FILE_BACKUP = '{0}_backup'.format(NGINX_CONFIG_FILE) +NGINX_CONTAINER_NAME = 'nginx' + +NAILGUN_CONTAINER_NAME = 'nailgun' +NAILGUN_APPS = ('assassind', 'nailgun', 'receiverd') +NAILGUN_REMOTE_DIR = os.path.join(REMOTE_DEST_DIR, 'nailgun') +NAILGUN_REMOTE_VENV_DIR = os.path.join(REMOTE_DEST_DIR, '.nailgun_venv') + +SUPERVISORD_CONFIG_FILE = '/etc/supervisord.conf' +SUPERVISORD_CONFIG_FILE_BACKUP = '{0}_backup'.format( + SUPERVISORD_CONFIG_FILE +) + + +def install_required_packages(): + install_package('rsync') + install_package('postgresql-devel') + install_package('python-virtualenv') + install_package('gcc') + + +def sync(repo_dir): + print(green("Synchronizing Nailgun code")) + run('mkdir -p {0}'.format(NAILGUN_REMOTE_DIR)) + nailgun_dir = os.path.join(repo_dir, 'nailgun/') + rsync_project(NAILGUN_REMOTE_DIR, local_dir=nailgun_dir, delete=True, + extra_opts='-a', exclude=('*pyc', '.tox*')) + print(green("Nailgun code synchronized")) + + +def configure_nginx(): + print(green("Configuring nginx")) + backup_file( + NGINX_CONTAINER_NAME, + NGINX_CONFIG_FILE, + NGINX_CONFIG_FILE_BACKUP + ) + + replaces = ( + ('/usr/share/nailgun/static', + os.path.join(REMOTE_DEST_DIR, 'nailgun', 'static')), + ('access_nailgun.log', 'access_nailgun_dev.log'), + ('error_nailgun.log', 'error_nailgun_dev.log'), + ) + for f, t in replaces: + command = 'sed -i.bak -r -e \'s/{0}/{1}/g\' $(echo "{2}")'.format( + escape_path(f), + escape_path(t), + NGINX_CONFIG_FILE + ) + run_in_container(NGINX_CONTAINER_NAME, command) + print(green("Nginx configured")) + + +def reload_nginx(): + print(green("Reloading nginx configs")) + run_in_container( + NGINX_CONTAINER_NAME, + 'service nginx reload' + ) + print(green("Nginx configs reloaded")) + + +def is_supervisord_configured(dev_env_string): + # grep return code is not 0 in this case + with settings(hide('warnings'), warn_only=True): + result = run_in_container( + NAILGUN_CONTAINER_NAME, + 'grep -Fxc "{0}" {1}'.format( + dev_env_string, + SUPERVISORD_CONFIG_FILE + ) + ) + return bool(result and int(result)) + + +def configure_supervisord(): + print(green("Configuring supervisord")) + backup_file( + NAILGUN_CONTAINER_NAME, + SUPERVISORD_CONFIG_FILE, + SUPERVISORD_CONFIG_FILE_BACKUP + ) + + dev_lib_path = os.path.join( + NAILGUN_REMOTE_VENV_DIR, + 'lib', + 'python2.6', + 'site-packages' + ) + dev_env_string = 'environment=PYTHONPATH={0}:{1}'.format( + NAILGUN_REMOTE_DIR, + dev_lib_path + ) + if is_supervisord_configured(dev_env_string): + print(green("Supervisord already configured")) + else: + for app in NAILGUN_APPS: + header = '\[program:{0}\]'.format(app) + header_dev = '{0}\\n{1}'.format(header, dev_env_string) + command = 'sed -i.bak -r -e \'s/{0}/{1}/g\' $(echo "{2}")'.format( + header, + escape_path(header_dev), + SUPERVISORD_CONFIG_FILE + ) + run_in_container(NAILGUN_CONTAINER_NAME, command) + print(green("Supervisor configured")) + + +def reload_supervisord(): + """When supervisord is reloaded, services are restarted + automatically + """ + print(green("Reloading supervisord configs")) + run_in_container( + NAILGUN_CONTAINER_NAME, + 'service supervisord reload' + ) + print(green("Supervisord configs reloaded")) + + +def restart_nailgun(): + print(green("Restarting nailgun applications")) + for app in NAILGUN_APPS: + print(green("Restarting application {0}".format(app))) + command = 'supervisorctl restart {0}'.format(app) + run_in_container(NAILGUN_CONTAINER_NAME, command) + print(green("Application {0} restarted".format(app))) + print(green("Nailgun applications restarted")) + + +def revert_nginx(): + print(green("Restoring origin nginx configuration")) + with settings(warn_only=True): + revert_file( + NGINX_CONTAINER_NAME, + NGINX_CONFIG_FILE_BACKUP, + NGINX_CONFIG_FILE + ) + reload_nginx() + print(green("Origin nginx configuration restored")) + + +def revert_supervisord(): + print(green("Restoring origin supervisord configuration")) + with settings(warn_only=True): + revert_file( + NAILGUN_CONTAINER_NAME, + SUPERVISORD_CONFIG_FILE_BACKUP, + SUPERVISORD_CONFIG_FILE + ) + reload_supervisord() + print(green("Origin supervisord configuration restored")) + + +def configure_nailgun_venv(): + print(green("Creating nailgun virtualenv")) + command = 'virtualenv {0}'.format(NAILGUN_REMOTE_VENV_DIR) + run(command) + print(green("Nailgun virtualenv created")) + + print(green("Installing nailgun requirements")) + pip_path = os.path.join( + NAILGUN_REMOTE_VENV_DIR, + 'bin', + 'pip' + ) + dev_requirments = os.path.join( + NAILGUN_REMOTE_DIR, + 'requirements.txt' + ) + command = '{0} install -r {1}'.format(pip_path, dev_requirments) + run(command) + print(green("Nailgun requirements installed")) + + +def deploy(params): + print(green("Deploying development environment to {0}".format( + env.host_string + ))) + repo_dir = params.fuelweb_dir + if params.synconly: + sync(repo_dir) + restart_nailgun() + else: + install_required_packages() + sync(repo_dir) + configure_nailgun_venv() + configure_supervisord() + reload_supervisord() + configure_nginx() + reload_nginx() + print(green("Development environment deployed to {0}".format( + env.host_string + ))) + + +def revert(): + print(green("Restoring production environment on {0}".format( + env.host_string + ))) + revert_nginx() + revert_supervisord() + restart_nailgun() + print(green("Production environment restored on {0}".format( + env.host_string + ))) + + +def action(params): + if params.command == 'deploy': + deploy(params) + elif params.command == 'revert': + revert() diff --git a/fuel_development/manage.py b/fuel_development/manage.py new file mode 100644 index 0000000000..02405bc017 --- /dev/null +++ b/fuel_development/manage.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import os +import sys + +from fabric.api import env +from fabric.colors import red +from fabric.context_managers import hide +from fabric.context_managers import settings + + +def load_nailgun_deploy_parser(operation_parser): + deploy_parser = operation_parser.add_parser('deploy') + + cur_dir = os.path.dirname(__file__) + default_dir = os.path.realpath(os.path.join(cur_dir, '..')) + deploy_parser.add_argument( + '-d', '--fuelweb-dir', + type=str, + help="Path to fuel-web repository " + "(if not set '{0}' will be used)".format(default_dir), + default=default_dir + ) + deploy_parser.add_argument( + '--synconly', + action='store_true', + help="Synchronize source and restart service " + "without masternode configuration" + ) + + +def load_nailgun_revert_parser(operation_parser): + operation_parser.add_parser('revert') + + +def load_nailgun_parser(subparsers): + nailgun_parser = subparsers.add_parser( + 'nailgun', help="Processing nailgun operations" + ) + operation_parser = nailgun_parser.add_subparsers( + dest='command', + help="Deploy or revert nailgun development environment on masternode" + ) + load_nailgun_deploy_parser(operation_parser) + load_nailgun_revert_parser(operation_parser) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + + default_masternode_addr = '10.20.0.2' + parser.add_argument( + '-m', '--masternode-addr', + help="Master node address ('{0}' by default)".format( + default_masternode_addr + ), + default=default_masternode_addr + ) + + subparsers = parser.add_subparsers( + dest='action', + help="Targets to be developed on master node" + ) + + load_nailgun_parser(subparsers) + params = parser.parse_args() + + # Configuring fabric global params + env.host_string = params.masternode_addr + env.user = 'root' + + # Loading configurator by action value + action_module = __import__('configurator.{0}'.format(params.action)) + processor = getattr(action_module, params.action) + + # Executing action + try: + with settings(hide('running', 'stdout')): + processor.action(params) + except Exception as e: + print(red("Configuration failed: {0}".format(e))) + # Exiting with general error code + sys.exit(1) diff --git a/fuel_development/requirements.txt b/fuel_development/requirements.txt new file mode 100644 index 0000000000..35efc6d844 --- /dev/null +++ b/fuel_development/requirements.txt @@ -0,0 +1 @@ +Fabric==1.7.0 diff --git a/fuel_development/tox.ini b/fuel_development/tox.ini new file mode 100644 index 0000000000..c2108e6758 --- /dev/null +++ b/fuel_development/tox.ini @@ -0,0 +1,38 @@ +[tox] +minversion = 1.6 +skipsdist = True +envlist = py26,py27,pep8 + +[testenv] +usedevelop = True +install_command = pip install {packages} +setenv = VIRTUAL_ENV={envdir} +deps = -r{toxinidir}/requirements.txt +commands = + nosetests {posargs:fuel_upgrade} + +[tox:jenkins] +downloadcache = ~/cache/pip + +[testenv:pep8] +deps = hacking==0.7 +usedevelop = False +commands = + flake8 {posargs:.} + +[testenv:venv] +commands = {posargs:} + +[testenv:devenv] +envdir = devenv +usedevelop = True + +[flake8] +ignore = H234,H302,H802 +exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools,__init__.py,docs +show-pep8 = True +show-source = True +count = True + +[hacking] +import_exceptions = testtools.matchers diff --git a/run_tests.sh b/run_tests.sh index feaef285d7..c8f4f37bcf 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -437,6 +437,7 @@ function run_flake8 { run_flake8_subproject network_checker && \ run_flake8_subproject fuel_upgrade_system/fuel_update_downloader && \ run_flake8_subproject fuel_upgrade_system/fuel_upgrade && \ + run_flake8_subproject fuel_development && \ run_flake8_subproject shotgun || result=1 return $result }