diff --git a/.gitignore b/.gitignore index cd68f093..2d37d5c7 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ pip-log.txt .tox nosetests.xml .testrepository +cover # Translations *.mo diff --git a/bin/__init__.py b/bin/__init__.py index e69de29b..4ee55a4c 100644 --- a/bin/__init__.py +++ b/bin/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/bin/chef/addcookbooks.py b/bin/chef/addcookbooks.py index badcf921..e2bc8dac 100755 --- a/bin/chef/addcookbooks.py +++ b/bin/chef/addcookbooks.py @@ -1,9 +1,24 @@ #!/usr/bin/env python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """import cookbooks to chef server.""" import logging import os import os.path - +import sys from compass.utils import flags from compass.utils import logsetting @@ -15,13 +30,16 @@ flags.add('cookbooks_dir', def main(): - """main entry""" + """main entry.""" flags.init() logsetting.init() cookbooks_dir = flags.OPTIONS.cookbooks_dir logging.info('add cookbooks %s', cookbooks_dir) cmd = "knife cookbook upload --all --cookbook-path %s" % cookbooks_dir - os.system(cmd) + status = os.system(cmd) + logging.info('run cmd %s returns %s', cmd, status) + if status: + sys.exit(1) if __name__ == '__main__': diff --git a/bin/chef/adddatabags.py b/bin/chef/adddatabags.py index 1fffde21..210251af 100755 --- a/bin/chef/adddatabags.py +++ b/bin/chef/adddatabags.py @@ -1,8 +1,24 @@ #!/usr/bin/env python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """import databags to chef server.""" import logging import os import os.path +import sys from compass.utils import flags from compass.utils import logsetting @@ -14,7 +30,7 @@ flags.add('databags_dir', def main(): - """main entry""" + """main entry.""" flags.init() logsetting.init() databags = [] @@ -35,7 +51,10 @@ def main(): logging.info('add databag item %s to databag %s', databag_item, databag) cmd = 'knife data bag from file %s %s' % (databag, databag_item) - os.system(cmd) + status = os.system(cmd) + logging.info('run cmd %s returns %s', cmd, status) + if status: + sys.exit(1) if __name__ == '__main__': diff --git a/bin/chef/addroles.py b/bin/chef/addroles.py index 94237fd3..9a2b6c25 100755 --- a/bin/chef/addroles.py +++ b/bin/chef/addroles.py @@ -1,8 +1,24 @@ #!/usr/bin/env python -"""script to import roles to chef server""" +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""script to import roles to chef server.""" import logging import os import os.path +import sys from compass.utils import flags from compass.utils import logsetting @@ -14,7 +30,7 @@ flags.add('roles_dir', def main(): - """main entry""" + """main entry.""" flags.init() logsetting.init() rolelist = [] @@ -27,7 +43,10 @@ def main(): for role in rolelist: logging.info('add role %s', role) cmd = "knife role from file %s" % role - os.system(cmd) + status = os.system(cmd) + logging.info('run cmd %s returns %s', cmd, status) + if status: + sys.exit(1) if __name__ == '__main__': diff --git a/bin/cobbler/migrate_ks.py b/bin/cobbler/migrate_ks.py deleted file mode 100755 index 9f14f2c3..00000000 --- a/bin/cobbler/migrate_ks.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/python -"""script to migrate rendered kickstart files from cobbler to outside.""" -import xmlrpclib -import logging - -from compass.utils import setting_wrapper as setting - - -def main(): - """main entry""" - remote = xmlrpclib.Server(setting.COBBLER_INSTALLER_URL, allow_none=True) - token = remote.login(*setting.COBBLER_INSTALLER_TOKEN) - systems = remote.get_systems(token) - for system in systems: - data = remote.generate_kickstart('', system['name']) - try: - with open( - '/var/www/cblr_ks/%s' % system['name'], 'w' - ) as kickstart_file: - logging.info("Migrating kickstart for %s", system['name']) - kickstart_file.write(data) - except Exception as error: - logging.error("Directory /var/www/cblr_ks/ does not exist.") - logging.exception(error) - - -if __name__ == '__main__': - logging.info("Running kickstart migration") - main() diff --git a/bin/csvdeploy.py b/bin/csvdeploy.py index eb8673b8..c95e4133 100755 --- a/bin/csvdeploy.py +++ b/bin/csvdeploy.py @@ -1,11 +1,29 @@ #!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""deploy cluster from csv file.""" +import ast +import copy +import csv import os import re -import csv -import ast import sys -from copy import deepcopy -from multiprocessing import Process, Queue + +from multiprocessing import Process +from multiprocessing import Queue from optparse import OptionParser try: @@ -35,8 +53,7 @@ TABLES = { def start(csv_dir, compass_url): - """ Start deploy both failed clusters and new clusters. - """ + """Start deploy both failed clusters and new clusters.""" # Get clusters and hosts data from CSV clusters_data = get_csv('cluster.csv', csv_dir) hosts_data = get_csv('cluster_host.csv', csv_dir) @@ -95,11 +112,15 @@ def start(csv_dir, compass_url): def get_csv(fname, csv_dir): - """ Parse csv files into python variables. all nested fields in db will be - assembled. + """Parse csv files into python variables. + + .. note:: + all nested fields in db will be assembled. + :param fname: CSV file name :param csv_dir: CSV files directory - Return a list of dict which key is column name and value is its data. + + :returns: list of dict which key is column name and value is its data. """ headers = [] rows = [] @@ -168,7 +189,7 @@ def merge_dict(lhs, rhs, override=True): merge_dict(lhs[key], value, override) else: if override or key not in lhs: - lhs[key] = deepcopy(value) + lhs[key] = copy.deepcopy(value) class _APIClient(Client): @@ -180,10 +201,11 @@ class _APIClient(Client): return self._put(url, data=data) def execute(self, cluster_data, hosts_data, resp_results): - """ The process including create or update a cluster and the cluster - configuration, add or update a host in the cluster, and deploy - the updated hosts. - :param cluster_data: the dictionary of cluster data + """The process including create or update a cluster and the cluster + configuration, add or update a host in the cluster, and deploy + the updated hosts. + + :param cluster_data: the dictionary of cluster data """ cluster_id = cluster_data['id'] code, resp = self.get_cluster(cluster_id) diff --git a/bin/manage_db.py b/bin/manage_db.py index caa1b523..bb23037f 100755 --- a/bin/manage_db.py +++ b/bin/manage_db.py @@ -1,4 +1,19 @@ #!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """utility binary to manage database.""" import os import os.path @@ -7,16 +22,23 @@ import sys from flask.ext.script import Manager from compass.actions import clean_deployment -from compass.actions import reinstall -from compass.actions import deploy from compass.actions import clean_installing_progress +from compass.actions import deploy +from compass.actions import reinstall from compass.actions import search from compass.api import app from compass.config_management.utils import config_manager from compass.db import database -from compass.db.model import Adapter, Role, Switch, SwitchConfig -from compass.db.model import Machine, HostState, ClusterState -from compass.db.model import Cluster, ClusterHost, LogProgressingHistory +from compass.db.model import Adapter +from compass.db.model import Cluster +from compass.db.model import ClusterHost +from compass.db.model import ClusterState +from compass.db.model import HostState +from compass.db.model import LogProgressingHistory +from compass.db.model import Machine +from compass.db.model import Role +from compass.db.model import Switch +from compass.db.model import SwitchConfig from compass.tasks.client import celery from compass.utils import flags from compass.utils import logsetting @@ -76,14 +98,14 @@ TABLE_MAPPING = { @app_manager.command def list_config(): - "List the configuration" + "List the commands." for key, value in app.config.items(): print key, value @app_manager.command def checkdb(): - """check if db exists""" + """check if db exists.""" if setting.DATABASE_TYPE == 'file': if os.path.exists(setting.DATABASE_FILE): sys.exit(0) @@ -95,24 +117,24 @@ def checkdb(): @app_manager.command def createdb(): - """Creates database from sqlalchemy models""" + """Creates database from sqlalchemy models.""" if setting.DATABASE_TYPE == 'file': if os.path.exists(setting.DATABASE_FILE): os.remove(setting.DATABASE_FILE) database.create_db() if setting.DATABASE_TYPE == 'file': - os.chmod(setting.DATABASE_FILE, 0777) + os.chmod(setting.DATABASE_FILE, 0o777) @app_manager.command def dropdb(): - """Drops database from sqlalchemy models""" + """Drops database from sqlalchemy models.""" database.drop_db() @app_manager.command def createtable(): - """Create database table by --table_name""" + """Create database table.""" if not flags.OPTIONS.table_name: print 'flag --table_name is missing' return @@ -127,7 +149,7 @@ def createtable(): @app_manager.command def droptable(): - """Drop database table by --talbe_name""" + """Drop database table.""" if not flags.OPTIONS.table_name: print 'flag --table_name is missing' return diff --git a/bin/poll_switch.py b/bin/poll_switch.py index cb4a2585..4dd77240 100755 --- a/bin/poll_switch.py +++ b/bin/poll_switch.py @@ -1,4 +1,19 @@ #!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """main script to poll machines which is connected to the switches.""" import functools import lockfile @@ -31,7 +46,7 @@ flags.add('run_interval', def pollswitches(switch_ips): - """poll switch""" + """poll switch.""" poll_switch_ips = [] with database.session(): poll_switch_ips = util.update_switch_ips(switch_ips) diff --git a/bin/progress_update.py b/bin/progress_update.py index 9a45dde1..e5862e9b 100755 --- a/bin/progress_update.py +++ b/bin/progress_update.py @@ -1,4 +1,19 @@ #!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """main script to run as service to update hosts installing progress.""" import functools import lockfile diff --git a/bin/query_switch.py b/bin/query_switch.py old mode 100644 new mode 100755 index 6fbb0a20..835bfe8c --- a/bin/query_switch.py +++ b/bin/query_switch.py @@ -1,24 +1,40 @@ #!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""query switch.""" +import optparse +import Queue import threading import time -from Queue import Queue, Empty -from optparse import OptionParser - from compass.apiclient.restful import Client class AddSwitch(object): - """ A utility class that handles adding a switch and retrieving - corresponding machines associated with the switch. """ + """A utility class that handles adding a switch and retrieving + corresponding machines associated with the switch. + """ def __init__(self, server_url): print server_url, " ...." self._client = Client(server_url) def add_switch(self, queue, ip, snmp_community): - """ Add a switch with SNMP credentials and retrieve attached - server machines. + """Add a switch with SNMP credentials and retrieve attached + server machines. :param queue: The result holder for the machine details. :type queue: A Queue object(thread-safe). @@ -88,10 +104,10 @@ class AddSwitch(object): if __name__ == "__main__": usage = "usage: %prog [options] switch_ips" - parser = OptionParser(usage) + parser = optparse.OptionParser(usage) parser.add_option("-u", "--server-url", dest="server_url", - default="http://localhost/", + default="http://localhost/api", help="The Compass Server URL") parser.add_option("-c", "--community", dest="community", @@ -104,7 +120,7 @@ if __name__ == "__main__": parser.error("Wrong number of arguments") threads = [] - queue = Queue() + queue = Queue.Queue() add_switch = AddSwitch(options.server_url) print "Add switch to the server. This may take a while ..." @@ -122,5 +138,5 @@ if __name__ == "__main__": try: ip, result = queue.get(block=False) print ip, " : ", result - except Empty: + except Queue.Empty: break diff --git a/bin/refresh.sh b/bin/refresh.sh index c5dc5d13..890fdd75 100755 --- a/bin/refresh.sh +++ b/bin/refresh.sh @@ -6,6 +6,9 @@ fi /opt/compass/bin/manage_db.py createdb /opt/compass/bin/manage_db.py sync_switch_configs /opt/compass/bin/manage_db.py sync_from_installers -service compassd restart service httpd restart service rsyslog restart +service redis restart +redis-cli flushall +service compassd restart + diff --git a/bin/runserver.py b/bin/runserver.py index c27e0321..73a1e5f7 100755 --- a/bin/runserver.py +++ b/bin/runserver.py @@ -1,4 +1,19 @@ #!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """main script to start an instance of compass server .""" import logging diff --git a/compass/__init__.py b/compass/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/__init__.py +++ b/compass/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/actions/__init__.py b/compass/actions/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/actions/__init__.py +++ b/compass/actions/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/actions/clean_deployment.py b/compass/actions/clean_deployment.py index b610d253..f74fad0c 100644 --- a/compass/actions/clean_deployment.py +++ b/compass/actions/clean_deployment.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to clean deployment of a given cluster .. moduleauthor:: Xiaodong Wang @@ -13,16 +27,17 @@ def clean_deployment(cluster_hosts): """Clean deployment of clusters. :param cluster_hosts: clusters and hosts in each cluster to clean. - :type cluster_hosts: dict of int to list of int + :type cluster_hosts: dict of int or str to list of int or str .. note:: The function should be called out of database session. """ - logging.debug('clean cluster_hosts: %s', cluster_hosts) - with database.session(): - cluster_hosts, os_versions, target_systems = ( - util.update_cluster_hosts(cluster_hosts)) - manager = ConfigManager() - manager.clean_cluster_and_hosts( - cluster_hosts, os_versions, target_systems) - manager.sync() + with util.lock('serialized_action'): + logging.debug('clean cluster_hosts: %s', cluster_hosts) + with database.session(): + cluster_hosts, os_versions, target_systems = ( + util.update_cluster_hosts(cluster_hosts)) + manager = ConfigManager() + manager.clean_cluster_and_hosts( + cluster_hosts, os_versions, target_systems) + manager.sync() diff --git a/compass/actions/clean_installing_progress.py b/compass/actions/clean_installing_progress.py index 7b43feb3..0c9e51ea 100644 --- a/compass/actions/clean_installing_progress.py +++ b/compass/actions/clean_installing_progress.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to clean installing progress of a given cluster .. moduleauthor:: Xiaodong Wang @@ -13,17 +27,19 @@ def clean_installing_progress(cluster_hosts): """Clean installing progress of clusters. :param cluster_hosts: clusters and hosts in each cluster to clean. - :type cluster_hosts: dict of int to list of int + :type cluster_hosts: dict of int or str to list of int or str .. note:: The function should be called out of database session. """ - logging.debug('clean installing progress of cluster_hosts: %s', - cluster_hosts) - with database.session(): - cluster_hosts, os_versions, target_systems = ( - util.update_cluster_hosts(cluster_hosts)) - manager = ConfigManager() - manager.clean_cluster_and_hosts_installing_progress( - cluster_hosts, os_versions, target_systems) - manager.sync() + with util.lock('serialized_action'): + logging.debug( + 'clean installing progress of cluster_hosts: %s', + cluster_hosts) + with database.session(): + cluster_hosts, os_versions, target_systems = ( + util.update_cluster_hosts(cluster_hosts)) + manager = ConfigManager() + manager.clean_cluster_and_hosts_installing_progress( + cluster_hosts, os_versions, target_systems) + manager.sync() diff --git a/compass/actions/cli.py b/compass/actions/cli.py index db7dc1a4..4eb4c8b2 100644 --- a/compass/actions/cli.py +++ b/compass/actions/cli.py @@ -1,11 +1,25 @@ -"""Compass Command Line Interface""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Compass Command Line Interface""" +import subprocess import sys -from subprocess import Popen from compass.actions.health_check import check from compass.utils.util import pretty_print + ACTION_MAP = { "check": "apache celery dhcp dns hds misc os_installer " "package_installer squid tftp".split(" "), @@ -20,9 +34,7 @@ class BootCLI(object): return def run(self, args): - """ - cli takes the commands and calls respective modules - """ + """cli takes the commands and calls respective modules.""" action = self.get_action(args) if action is None: self.print_help() @@ -36,9 +48,10 @@ class BootCLI(object): @classmethod def get_action(cls, args): - """ - This method returns an action type. - For 'compass check dhcp' command, it will return 'check'. + """This method returns an action type. + + .. note:: + For 'compass check dhcp' command, it will return 'check'. """ if len(args) == 1: return None @@ -48,9 +61,10 @@ class BootCLI(object): @classmethod def get_module(cls, action, args): - """ - This method returns a module. - For 'compass check dhcp' command, it will return 'dhcp'. + """This method returns a module. + + .. note:: + For 'compass check dhcp' command, it will return 'dhcp'. """ if len(args) <= 2: return None @@ -59,12 +73,12 @@ class BootCLI(object): return "invalid" def run_check(self, module=None): - """ - param module default set to None. - This provides a flexible sanity check, - if parameter module is none. Compass checks all modules. - If module specified, Compass will only check such module. + """This provides a flexible sanity check. + .. note:: + param module default set to None. + if parameter module is none. Compass checks all modules. + If module specified, Compass will only check such module. """ if module is None: pretty_print("Starting: Compass Health Check", @@ -101,27 +115,32 @@ class BootCLI(object): print "Compass Check completes. No problems found, all systems go" sys.exit(0) else: - print "Compass has ERRORS shown above. Please fix them before " \ - "deploying!" + print ( + "Compass has ERRORS shown above. Please fix them before " + "deploying!") sys.exit(1) @classmethod def run_refresh(cls, action=None): """Run refresh.""" - ## TODO: replace refresh.sh with refresh.py + # TODO(xicheng): replace refresh.sh with refresh.py if action is None: pretty_print("Refreshing Compass...", "=================") - Popen(['/opt/compass/bin/refresh.sh'], shell=True) + subprocess.Popen( + ['/opt/compass/bin/refresh.sh'], shell=True) elif action == "db": pretty_print("Refreshing Compass Database...", "===================") - Popen(['/opt/compass/bin/manage_db.py createdb'], shell=True) + subprocess.Popen( + ['/opt/compass/bin/manage_db.py createdb'], shell=True) else: pretty_print("Syncing with Installers...", "================") - Popen(['/opt/compass/bin/manage_db.py sync_from_installers'], - shell=True) + subprocess.Popen( + ['/opt/compass/bin/manage_db.py sync_from_installers'], + shell=True + ) @classmethod def print_help(cls, module_help=""): @@ -145,12 +164,11 @@ class BootCLI(object): def main(): - """ - Compass cli entry point - """ + """Compass cli entry point.""" cli = BootCLI() output = cli.run(sys.argv) return sys.exit(output) + if __name__ == "__main__": main() diff --git a/compass/actions/deploy.py b/compass/actions/deploy.py index a07601d2..838cd10d 100644 --- a/compass/actions/deploy.py +++ b/compass/actions/deploy.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to deploy a given cluster .. moduleauthor:: Xiaodong Wang @@ -13,16 +27,17 @@ def deploy(cluster_hosts): """Deploy clusters. :param cluster_hosts: clusters and hosts in each cluster to deploy. - :type cluster_hosts: dict of int to list of int + :type cluster_hosts: dict of int or str to list of int or str .. note:: The function should be called out of database session. """ - logging.debug('deploy cluster_hosts: %s', cluster_hosts) - with database.session(): - cluster_hosts, os_versions, target_systems = ( - util.update_cluster_hosts(cluster_hosts)) - manager = ConfigManager() - manager.install_cluster_and_hosts( - cluster_hosts, os_versions, target_systems) - manager.sync() + with util.lock('serialized_action'): + logging.debug('deploy cluster_hosts: %s', cluster_hosts) + with database.session(): + cluster_hosts, os_versions, target_systems = ( + util.update_cluster_hosts(cluster_hosts)) + manager = ConfigManager() + manager.install_cluster_and_hosts( + cluster_hosts, os_versions, target_systems) + manager.sync() diff --git a/compass/actions/health_check/__init__.py b/compass/actions/health_check/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/actions/health_check/__init__.py +++ b/compass/actions/health_check/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/actions/health_check/base.py b/compass/actions/health_check/base.py index d0013afb..a64fcd84 100644 --- a/compass/actions/health_check/base.py +++ b/compass/actions/health_check/base.py @@ -1,4 +1,18 @@ -"""Base class for Compass Health Check""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Base class for Compass Health Check.""" from compass.actions.health_check import utils as health_check_utils from compass.utils import setting_wrapper as setting @@ -13,10 +27,10 @@ class BaseCheck(object): self.dist, self.version, self.release = health_check_utils.get_dist() def _set_status(self, code, message): - """set status""" + """set status.""" self.code = code self.messages.append(message) def get_status(self): - """get status""" + """get status.""" return (self.code, self.messages) diff --git a/compass/actions/health_check/check.py b/compass/actions/health_check/check.py index 461c2834..c1adbc6b 100644 --- a/compass/actions/health_check/check.py +++ b/compass/actions/health_check/check.py @@ -1,85 +1,96 @@ -"""Main Entry Point of Compass Health Check""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -from compass.actions.health_check import check_apache as apache -from compass.actions.health_check import check_celery as celery -from compass.actions.health_check import check_dhcp as dhcp -from compass.actions.health_check import check_dns as dns -from compass.actions.health_check import check_hds as hds -from compass.actions.health_check import ( - check_os_installer as os_installer) -from compass.actions.health_check import ( - check_package_installer as package_installer) -from compass.actions.health_check import check_squid as squid -from compass.actions.health_check import check_tftp as tftp -from compass.actions.health_check import check_misc as misc +"""Main Entry Point of Compass Health Check.""" from compass.actions.health_check import base +from compass.actions.health_check import check_apache +from compass.actions.health_check import check_celery +from compass.actions.health_check import check_dhcp +from compass.actions.health_check import check_dns +from compass.actions.health_check import check_hds +from compass.actions.health_check import check_misc +from compass.actions.health_check import check_os_installer +from compass.actions.health_check import check_package_installer +from compass.actions.health_check import check_squid +from compass.actions.health_check import check_tftp class BootCheck(base.BaseCheck): - """health check for all components""" + """health check for all components.""" def run(self): - """do health check""" + """do health check.""" status = {} - status['apache'] = self.check_apache() - status['celery'] = self.check_celery() - status['dhcp'] = self.check_dhcp() - status['dns'] = self.check_dns() - status['hds'] = self.check_hds() - status['os_installer'] = self.check_os_installer() - status['package_installer'] = self.check_package_installer() - status['squid'] = self.check_squid() - status['tftp'] = self.check_tftp() - status['other'] = self.check_misc() + status['apache'] = self._check_apache() + status['celery'] = self._check_celery() + status['dhcp'] = self._check_dhcp() + status['dns'] = self._check_dns() + status['hds'] = self._check_hds() + status['os_installer'] = self._check_os_installer() + status['package_installer'] = self._check_package_installer() + status['squid'] = self._check_squid() + status['tftp'] = self._check_tftp() + status['other'] = self._check_misc() return status - def check_apache(self): - """do apache health check""" - checker = apache.ApacheCheck() + def _check_apache(self): + """do apache health check.""" + checker = check_apache.ApacheCheck() return checker.run() - def check_celery(self): - """do celery health check""" - checker = celery.CeleryCheck() + def _check_celery(self): + """do celery health check.""" + checker = check_celery.CeleryCheck() return checker.run() - def check_dhcp(self): - """do dhcp health check""" - checker = dhcp.DhcpCheck() + def _check_dhcp(self): + """do dhcp health check.""" + checker = check_dhcp.DhcpCheck() return checker.run() - def check_dns(self): - """do dns health check""" - checker = dns.DnsCheck() + def _check_dns(self): + """do dns health check.""" + checker = check_dns.DnsCheck() return checker.run() - def check_hds(self): - """do hds health check""" - checker = hds.HdsCheck() + def _check_hds(self): + """do hds health check.""" + checker = check_hds.HdsCheck() return checker.run() - def check_os_installer(self): - """do os installer health check""" - checker = os_installer.OsInstallerCheck() + def _check_os_installer(self): + """do os installer health check.""" + checker = check_os_installer.OsInstallerCheck() return checker.run() - def check_package_installer(self): - """do package installer health check""" - checker = package_installer.PackageInstallerCheck() + def _check_package_installer(self): + """do package installer health check.""" + checker = check_package_installer.PackageInstallerCheck() return checker.run() - def check_squid(self): - """do squid health check""" - checker = squid.SquidCheck() + def _check_squid(self): + """do squid health check.""" + checker = check_squid.SquidCheck() return checker.run() - def check_tftp(self): - """do tftp health check""" - checker = tftp.TftpCheck() + def _check_tftp(self): + """do tftp health check.""" + checker = check_tftp.TftpCheck() return checker.run() - def check_misc(self): - """do misc health check""" - checker = misc.MiscCheck() + def _check_misc(self): + """do misc health check.""" + checker = check_misc.MiscCheck() return checker.run() diff --git a/compass/actions/health_check/check_apache.py b/compass/actions/health_check/check_apache.py index 7cf2c774..84f20712 100644 --- a/compass/actions/health_check/check_apache.py +++ b/compass/actions/health_check/check_apache.py @@ -1,4 +1,18 @@ -"""Health Check module for Apache service""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Health Check module for Apache service.""" import socket import urllib2 @@ -12,7 +26,7 @@ class ApacheCheck(base.BaseCheck): NAME = "Apache Check" def run(self): - """do the healthcheck""" + """do the healthcheck.""" if self.dist in ("centos", "redhat", "fedora", "scientific linux"): apache_service = 'httpd' else: @@ -51,7 +65,7 @@ class ApacheCheck(base.BaseCheck): return True def check_apache_running(self, apache_service): - """Checks if Apache service is running on port 80""" + """Checks if Apache service is running on port 80.""" print "Checking Apache service......", serv_err_msg = health_check_utils.check_service_running(self.NAME, @@ -70,7 +84,7 @@ class ApacheCheck(base.BaseCheck): self._set_status( 0, "[%s]Error: Compass web is not redirected by Apache.") - except: + except Exception: self._set_status( 0, "[%s]Error: Apache is not running on Port 80." diff --git a/compass/actions/health_check/check_celery.py b/compass/actions/health_check/check_celery.py index 6513ad71..ded3ce61 100644 --- a/compass/actions/health_check/check_celery.py +++ b/compass/actions/health_check/check_celery.py @@ -1,7 +1,20 @@ -"""Health Check module for Celery""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -import os +"""Health Check module for Celery.""" import commands +import os from celery.task.control import inspect @@ -11,10 +24,10 @@ from compass.actions.health_check import utils as health_check_utils class CeleryCheck(base.BaseCheck): """celery health check class.""" - NAME = "Celery Check" + NAME = "Celery Check." def run(self): - """do health check""" + """do health check.""" self.check_compass_celery_setting() print "[Done]" self.check_celery_backend() @@ -26,7 +39,7 @@ class CeleryCheck(base.BaseCheck): return (self.code, self.messages) def check_compass_celery_setting(self): - """Validates Celery settings""" + """Validates Celery settings.""" print "Checking Celery setting......", setting_map = { @@ -75,10 +88,10 @@ class CeleryCheck(base.BaseCheck): return True def check_celery_backend(self): - """Checks if Celery backend is running and configured properly""" + """Checks if Celery backend is running and configured properly.""" print "Checking Celery Backend......", - if not 'celeryd' in commands.getoutput('ps -ef'): + if 'celeryd' not in commands.getoutput('ps -ef'): self._set_status(0, "[%s]Error: celery is not running" % self.NAME) return True diff --git a/compass/actions/health_check/check_dhcp.py b/compass/actions/health_check/check_dhcp.py index c7a40363..8989cdf3 100644 --- a/compass/actions/health_check/check_dhcp.py +++ b/compass/actions/health_check/check_dhcp.py @@ -1,10 +1,23 @@ -"""Health Check module for DHCP service""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Health Check module for DHCP service.""" +import commands import os import re -import commands -import xmlrpclib import socket +import xmlrpclib from compass.actions.health_check import base @@ -15,13 +28,13 @@ class DhcpCheck(base.BaseCheck): NAME = "DHCP Check" def run(self): - """do health check""" + """do health check.""" installer = self.config.OS_INSTALLER method_name = "self.check_" + installer + "_dhcp()" return eval(method_name) def check_cobbler_dhcp(self): - """Checks if Cobbler has taken over DHCP service""" + """Checks if Cobbler has taken over DHCP service.""" try: remote = xmlrpclib.Server( @@ -29,7 +42,7 @@ class DhcpCheck(base.BaseCheck): allow_none=True) remote.login( *self.config.COBBLER_INSTALLER_TOKEN) - except: + except Exception: self._set_status( 0, "[%s]Error: Cannot login to Cobbler with " @@ -42,6 +55,7 @@ class DhcpCheck(base.BaseCheck): "[%s]Info: DHCP service is " "not managed by Compass" % self.NAME) return (self.code, self.messages) + self.check_cobbler_dhcp_template() print "[Done]" self.check_dhcp_service() @@ -50,11 +64,11 @@ class DhcpCheck(base.BaseCheck): self.messages.append( "[%s]Info: DHCP health check has completed. " "No problems found, all systems go." % self.NAME) + return (self.code, self.messages) def check_cobbler_dhcp_template(self): - """Validates Cobbler's DHCP template file""" - + """Validates Cobbler's DHCP template file.""" print "Checking DHCP template......", if os.path.exists("/etc/cobbler/dhcp.template"): var_map = { @@ -127,13 +141,13 @@ class DhcpCheck(base.BaseCheck): 0, "[%s]Error: DHCP template file doesn't exist, " "health check failed." % self.NAME) + return True def check_dhcp_service(self): - """Checks if DHCP is running on port 67""" - + """Checks if DHCP is running on port 67.""" print "Checking DHCP service......", - if not 'dhcp' in commands.getoutput('ps -ef'): + if 'dhcp' not in commands.getoutput('ps -ef'): self._set_status( 0, "[%s]Error: dhcp service does not " @@ -144,4 +158,5 @@ class DhcpCheck(base.BaseCheck): 0, "[%s]Error: bootps is not listening " "on port 67" % self.NAME) + return True diff --git a/compass/actions/health_check/check_dns.py b/compass/actions/health_check/check_dns.py index 6a425950..0c4648e9 100644 --- a/compass/actions/health_check/check_dns.py +++ b/compass/actions/health_check/check_dns.py @@ -1,4 +1,18 @@ -"""Health Check module for DNS service""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Health Check module for DNS service.""" import commands import os @@ -13,21 +27,20 @@ class DnsCheck(base.BaseCheck): NAME = "DNS Check" def run(self): - """do health check""" + """do health check.""" installer = self.config.OS_INSTALLER method_name = "self.check_" + installer + "_dns()" return eval(method_name) def check_cobbler_dns(self): - """Checks if Cobbler has taken over DNS service""" - + """Checks if Cobbler has taken over DNS service.""" try: remote = xmlrpclib.Server( self.config.COBBLER_INSTALLER_URL, allow_none=True) remote.login( *self.config.COBBLER_INSTALLER_TOKEN) - except: + except Exception: self._set_status(0, "[%s]Error: Cannot login to Cobbler " "with the tokens provided in the config file" @@ -49,7 +62,7 @@ class DnsCheck(base.BaseCheck): return (self.code, self.messages) def check_cobbler_dns_template(self): - """Validates Cobbler's DNS template file""" + """Validates Cobbler's DNS template file.""" print "Checking DNS template......", if os.path.exists("/etc/cobbler/named.template"): @@ -66,7 +79,7 @@ class DnsCheck(base.BaseCheck): if "allow-query" in line: for subnet in ["127.0.0.0/8"]: - if not subnet in line: + if subnet not in line: missing_query.append(subnet) named_template.close() @@ -108,10 +121,10 @@ class DnsCheck(base.BaseCheck): return True def check_dns_service(self): - """Checks if DNS is running on port 53""" + """Checks if DNS is running on port 53.""" print "Checking DNS service......", - if not 'named' in commands.getoutput('ps -ef'): + if 'named' not in commands.getoutput('ps -ef'): self._set_status( 0, "[%s]Error: named service does not seem to be " diff --git a/compass/actions/health_check/check_hds.py b/compass/actions/health_check/check_hds.py index ded5c6ef..48ee8aa6 100644 --- a/compass/actions/health_check/check_hds.py +++ b/compass/actions/health_check/check_hds.py @@ -1,24 +1,40 @@ -"""Health Check module for Hardware Discovery""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Health Check module for Hardware Discovery.""" from compass.actions.health_check import base from compass.actions.health_check import utils as health_check_utils class HdsCheck(base.BaseCheck): - """hds health check class""" + """hds health check class.""" NAME = "HDS Check" def run(self): - """do health check""" + """do health check.""" if self.dist in ("centos", "redhat", "fedora", "scientific linux"): pkg_type = "yum" else: pkg_type = "apt" + try: pkg_module = __import__(pkg_type) - except: + except Exception: self.messages.append("[%s]Error: No module named %s, " "please install it first." % (self.NAME, pkg_module)) + method_name = 'self.check_' + pkg_type + '_snmp(pkg_module)' eval(method_name) print "[Done]" @@ -28,11 +44,11 @@ class HdsCheck(base.BaseCheck): self.messages.append("[%s]Info: hds health check has complated. " "No problems found, all systems go." % self.NAME) + return (self.code, self.messages) def check_yum_snmp(self, pkg_module): - """ - Check if SNMP yum dependencies are installed + """Check if SNMP yum dependencies are installed :param pkg_module : python yum library :type pkg_module : python module @@ -46,19 +62,21 @@ class HdsCheck(base.BaseCheck): self.messages.append("[%s]Error: %s package is required " "for HDS" % (self.NAME, package)) uninstalled.append(package) + if len(uninstalled) != 0: self._set_status(0, "[%s]Info: Uninstalled packages: %s" % (self.NAME, ', '.join(item for item in uninstalled))) + return True def check_apt_snmp(self, pkg_module): - """do apt health check""" - ## TODO: add ubuntu package check here + """do apt health check.""" + ## TODO(xicheng): add ubuntu package check here return None def check_snmp_mibs(self): - """Checks if SNMP MIB files are properly placed""" + """Checks if SNMP MIB files are properly placed.""" print "Checking SNMP MIBs......", conf_err_msg = health_check_utils.check_path(self.NAME, @@ -71,4 +89,5 @@ class HdsCheck(base.BaseCheck): '/usr/local/share/snmp/mibs') if not mibs_err_msg == "": self._set_status(0, mibs_err_msg) + return True diff --git a/compass/actions/health_check/check_misc.py b/compass/actions/health_check/check_misc.py index 5484b65c..0f1d7b3f 100644 --- a/compass/actions/health_check/check_misc.py +++ b/compass/actions/health_check/check_misc.py @@ -1,11 +1,24 @@ -"""Miscellaneous Health Check for Compass""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Miscellaneous Health Check for Compass.""" from compass.actions.health_check import base from compass.actions.health_check import utils as health_check_utils class MiscCheck(base.BaseCheck): - """health check for misc""" + """health check for misc.""" NAME = "Miscellaneous Check" MISC_MAPPING = { @@ -23,7 +36,7 @@ class MiscCheck(base.BaseCheck): } def run(self): - """do health check""" + """do health check.""" self.check_linux_dependencies() print "[Done]" self.check_pip_dependencies() @@ -44,26 +57,26 @@ class MiscCheck(base.BaseCheck): return (self.code, self.messages) def check_linux_dependencies(self): - """Checks if dependencies are installed""" - + """Checks if dependencies are installed.""" print "Checking Linux dependencies....", if self.dist in ("centos", "redhat", "fedora", "scientific linux"): pkg_type = "yum" else: pkg_type = "apt" + try: pkg_module = __import__(pkg_type) - except: + except Exception: self._set_status( 0, "[%s]Error: No module named %s, " "please install it first." % (self.NAME, pkg_module)) + method_name = 'self.check_' + pkg_type + '_dependencies(pkg_module)' eval(method_name) def check_yum_dependencies(self, pkg_module): - """ - Checks if yum dependencies are installed. + """Checks if yum dependencies are installed. :param pkg_module : python yum library :type pkg_module : python module @@ -89,14 +102,13 @@ class MiscCheck(base.BaseCheck): return True def check_pip_dependencies(self): - """Checks if required pip packages are installed""" - + """Checks if required pip packages are installed.""" print "Checking pip dependencies......", uninstalled = [] for module in self.MISC_MAPPING['pip']: try: __import__(module) - except: + except Exception: self._set_status( 0, "[%s]Error: pip package %s is requred" @@ -112,7 +124,7 @@ class MiscCheck(base.BaseCheck): return True def check_ntp(self): - """Validates ntp configuration and service""" + """Validates ntp configuration and service.""" print "Checking NTP......", conf_err_msg = health_check_utils.check_path(self.NAME, @@ -128,7 +140,7 @@ class MiscCheck(base.BaseCheck): return True def check_rsyslogd(self): - """Validates rsyslogd configuration and service""" + """Validates rsyslogd configuration and service.""" print "Checking rsyslog......", conf_err_msg = health_check_utils.check_path(self.NAME, @@ -149,7 +161,7 @@ class MiscCheck(base.BaseCheck): return True def check_chkconfig(self): - """Check if required services are enabled on the start up""" + """Check if required services are enabled on the start up.""" print "Checking chkconfig......", serv_to_disable = [] @@ -160,6 +172,7 @@ class MiscCheck(base.BaseCheck): "[%s]Error: %s is not disabled" % (self.NAME, serv)) serv_to_disable.append(serv) + if len(serv_to_disable) != 0: self._set_status( 0, @@ -174,6 +187,7 @@ class MiscCheck(base.BaseCheck): self._set_status( 0, "[%s]Error: %s is disabled" % (self.NAME, serv)) serv_to_enable.append(serv) + if len(serv_to_enable) != 0: self._set_status(0, "[%s]Info: You need to enable these " "services on system start-up: %s" @@ -183,20 +197,19 @@ class MiscCheck(base.BaseCheck): return True def check_selinux(self): - """Check if SELinux is disabled""" - + """Check if SELinux is disabled.""" print "Checking Selinux......", - selinux = open("/etc/selinux/config") disabled = False - for line in selinux.readlines(): - if "SELINUX=disabled" in line: - disabled = True - break + with open("/etc/selinux/config") as selinux: + for line in selinux: + if "SELINUX=disabled" in line: + disabled = True + break + if disabled is False: self._set_status( 0, "[%s]Selinux is not disabled, " "please disable it in /etc/selinux/config." % self.NAME) - selinux.close() return True diff --git a/compass/actions/health_check/check_os_installer.py b/compass/actions/health_check/check_os_installer.py index ee31ce9a..d23903dc 100644 --- a/compass/actions/health_check/check_os_installer.py +++ b/compass/actions/health_check/check_os_installer.py @@ -1,4 +1,18 @@ -"""Compass Health Check module for OS Installer""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compass Health Check module for OS Installer.""" import os import xmlrpclib @@ -7,25 +21,24 @@ from compass.actions.health_check import base class OsInstallerCheck(base.BaseCheck): - """os installer health check""" + """os installer health check.""" NAME = "OS Installer Check" def run(self): - """do health check""" + """do health check.""" installer = self.config.OS_INSTALLER method_name = 'self.' + installer + '_check()' return eval(method_name) def cobbler_check(self): - """Runs cobbler check from xmlrpc client""" - + """Runs cobbler check from xmlrpc client.""" try: remote = xmlrpclib.Server( self.config.COBBLER_INSTALLER_URL, allow_none=True) token = remote.login( *self.config.COBBLER_INSTALLER_TOKEN) - except: + except Exception: self.code = 0 self.messages.append( "[%s]Error: Cannot login to Cobbler with " @@ -42,6 +55,7 @@ class OsInstallerCheck(base.BaseCheck): for index, message in enumerate(check_result): if "SELinux" in message: check_result.pop(index) + if len(check_result) != 0: self.code = 0 for error_msg in check_result: @@ -62,6 +76,7 @@ class OsInstallerCheck(base.BaseCheck): if 'ppa_repo' in repo['mirror']: found_ppa = True break + if found_ppa is False: self._set_status(0, "[%s]Error: No repository ppa_repo found" diff --git a/compass/actions/health_check/check_package_installer.py b/compass/actions/health_check/check_package_installer.py index 34fe1354..dd47e1e6 100644 --- a/compass/actions/health_check/check_package_installer.py +++ b/compass/actions/health_check/check_package_installer.py @@ -1,11 +1,25 @@ -"""Health Check module for Package Installer""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Health Check module for Package Installer.""" import os import requests from compass.actions.health_check import base -from compass.actions.health_check import utils as health_check_utils from compass.actions.health_check import setting as health_check_setting +from compass.actions.health_check import utils as health_check_utils class PackageInstallerCheck(base.BaseCheck): @@ -13,14 +27,13 @@ class PackageInstallerCheck(base.BaseCheck): NAME = "Package Installer Check" def run(self): - """do health check""" + """do health check.""" installer = self.config.PACKAGE_INSTALLER method_name = "self." + installer + "_check()" return eval(method_name) def chef_check(self): - """Checks chef setting, cookbooks, databags and roles""" - + """Checks chef setting, cookbooks, databags and roles.""" chef_data_map = { 'CookBook': health_check_setting.COOKBOOKS, 'DataBag': health_check_setting.DATABAGS, @@ -63,8 +76,7 @@ class PackageInstallerCheck(base.BaseCheck): return (self.code, self.messages) def check_chef_data(self, data_type, github_url): - """ - Checks if chef cookbooks/roles/databags are correct. + """Checks if chef cookbooks/roles/databags are correct. :param data_type : chef data type should be one of ['CookBook','DataBag','Role'] @@ -76,7 +88,7 @@ class PackageInstallerCheck(base.BaseCheck): print "Checking Chef %s......" % (data_type.lower().strip() + 's'), try: import chef - except: + except Exception: self._set_status( 0, "[%s]Error: pychef is not installed." % self.NAME) @@ -113,7 +125,7 @@ class PackageInstallerCheck(base.BaseCheck): return (data_type, list(diff)) def check_chef_config_dir(self): - """Validates chef configuration directories""" + """Validates chef configuration directories.""" print "Checking Chef configurations......", message = health_check_utils.check_path(self.NAME, '/etc/chef-server/') @@ -123,4 +135,5 @@ class PackageInstallerCheck(base.BaseCheck): message = health_check_utils.check_path(self.NAME, '/opt/chef-server/') if not message == "": self._set_status(0, message) + return None diff --git a/compass/actions/health_check/check_squid.py b/compass/actions/health_check/check_squid.py index 682d2d6a..97930a16 100644 --- a/compass/actions/health_check/check_squid.py +++ b/compass/actions/health_check/check_squid.py @@ -1,7 +1,20 @@ -"""Health Check module for Squid service""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -import os +"""Health Check module for Squid service.""" import commands +import os import pwd import socket @@ -14,7 +27,7 @@ class SquidCheck(base.BaseCheck): NAME = "Squid Check" def run(self): - """do health check""" + """do health check.""" self.check_squid_files() print "[Done]" self.check_squid_service() @@ -26,8 +39,7 @@ class SquidCheck(base.BaseCheck): return (self.code, self.messages) def check_squid_files(self): - """Validates squid config, cache directory and ownership""" - + """Validates squid config, cache directory and ownership.""" print "Checking Squid Files......", var_map = { 'match_squid_conf': False, @@ -91,10 +103,10 @@ class SquidCheck(base.BaseCheck): return True def check_squid_service(self): - """Checks if squid is running on port 3128""" + """Checks if squid is running on port 3128.""" print "Checking Squid service......", - if not 'squid' in commands.getoutput('ps -ef'): + if 'squid' not in commands.getoutput('ps -ef'): self._set_status( 0, "[%s]Error: squid service does not seem " @@ -107,7 +119,7 @@ class SquidCheck(base.BaseCheck): "[%s]Error: squid is not listening on " "3128" % self.NAME) - except: + except Exception: self._set_status( 0, "[%s]Error: No service is listening on 3128, " diff --git a/compass/actions/health_check/check_tftp.py b/compass/actions/health_check/check_tftp.py index e732fbeb..4a86e32a 100644 --- a/compass/actions/health_check/check_tftp.py +++ b/compass/actions/health_check/check_tftp.py @@ -1,26 +1,38 @@ -"""Health Check module for TFTP service""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Health Check module for TFTP service.""" import os -import xmlrpclib import socket +import xmlrpclib from compass.actions.health_check import base from compass.actions.health_check import utils as health_check_utils class TftpCheck(base.BaseCheck): - """tftp health check class""" + """tftp health check class.""" NAME = "TFTP Check" def run(self): - """do health check""" + """do health check.""" installer = self.config.OS_INSTALLER method_name = "self.check_" + installer + "_tftp()" return eval(method_name) def check_cobbler_tftp(self): - """ - Checks if Cobbler manages TFTP service + """Checks if Cobbler manages TFTP service. :note: we assume TFTP service is running at the same machine where this health check runs at @@ -32,7 +44,7 @@ class TftpCheck(base.BaseCheck): allow_none=True) remote.login( *self.config.COBBLER_INSTALLER_TOKEN) - except: + except Exception: self._set_status( 0, "[%s]Error: Cannot login to Cobbler with the tokens " @@ -56,8 +68,7 @@ class TftpCheck(base.BaseCheck): return (self.code, self.messages) def check_tftp_dir(self): - """Validates TFTP directories and configurations""" - + """Validates TFTP directories and configurations.""" print "Checking TFTP directories......", if not os.path.exists('/var/lib/tftpboot/'): self._set_status( @@ -69,8 +80,7 @@ class TftpCheck(base.BaseCheck): return True def check_tftp_service(self): - """Checks if TFTP is running on port 69""" - + """Checks if TFTP is running on port 69.""" print "Checking TFTP services......", serv_err_msg = health_check_utils.check_service_running(self.NAME, 'xinetd') diff --git a/compass/actions/health_check/setting.py b/compass/actions/health_check/setting.py index a7a7d8c6..16be3ce2 100644 --- a/compass/actions/health_check/setting.py +++ b/compass/actions/health_check/setting.py @@ -1,4 +1,18 @@ -"""Health Check Settings""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Health Check Settings.""" # Chef data on github COOKBOOKS = ( diff --git a/compass/actions/health_check/utils.py b/compass/actions/health_check/utils.py index 6d35abc5..9de1ab73 100644 --- a/compass/actions/health_check/utils.py +++ b/compass/actions/health_check/utils.py @@ -1,13 +1,25 @@ -"""Compass Health Check heavy-lifting utilities""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Compass Health Check heavy-lifting utilities""" +import commands import os import platform -import commands def validate_setting(module, setting, param): - """ - Checks if a Compass setting exists in the config file. + """Checks if a Compass setting exists in the config file. :param module : module name to be checked :type module : string @@ -25,15 +37,14 @@ def validate_setting(module, setting, param): def get_dist(): - """Returns the operating system related information""" + """Returns the operating system related information.""" os_version, version, release = platform.linux_distribution() return (os_version.lower().strip(), version, release.lower().strip()) def check_path(module_name, path): - """ - Checks if a directory or file exisits. + """Checks if a directory or file exisits. :param module_name : module name to be checked :type module_name : string @@ -50,8 +61,7 @@ def check_path(module_name, path): def check_service_running(module_name, service_name): - """ - Checks if a certain service is running. + """Checks if a certain service is running. :param module_name : module name to be checked :type module_name : string @@ -60,7 +70,7 @@ def check_service_running(module_name, service_name): """ err_msg = "" - if not service_name in commands.getoutput('ps -ef'): + if service_name not in commands.getoutput('ps -ef'): err_msg = "[%s]Error: %s is not running." % ( module_name, service_name) @@ -68,8 +78,7 @@ def check_service_running(module_name, service_name): def check_chkconfig(service_name): - """ - Checks if a service is enabled at the start up. + """Checks if a service is enabled at the start up. :param service_name : service name to be checked :type service_name : string diff --git a/compass/actions/poll_switch.py b/compass/actions/poll_switch.py index 4f649934..ccc69d86 100644 --- a/compass/actions/poll_switch.py +++ b/compass/actions/poll_switch.py @@ -1,8 +1,24 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to provider function to poll switch.""" import logging from compass.db import database -from compass.db.model import Switch, Machine, SwitchConfig +from compass.db.model import Machine +from compass.db.model import Switch +from compass.db.model import SwitchConfig from compass.hdsdiscovery.hdmanager import HDManager @@ -77,9 +93,12 @@ def poll_switch(ip_addr, req_obj='mac', oper="SCAN"): switch_id = switch.id filter_ports = session.query( - SwitchConfig.filter_port).filter( - SwitchConfig.ip == Switch.ip).filter( - Switch.id == switch_id).all() + SwitchConfig.filter_port + ).filter( + SwitchConfig.ip == Switch.ip + ).filter( + Switch.id == switch_id + ).all() logging.info("***********filter posts are %s********", filter_ports) if filter_ports: #Get all ports from tuples into list diff --git a/compass/actions/reinstall.py b/compass/actions/reinstall.py index 281b73c2..6c93bb1f 100644 --- a/compass/actions/reinstall.py +++ b/compass/actions/reinstall.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to reinstall a given cluster .. moduleauthor:: Xiaodong Wang @@ -13,16 +27,17 @@ def reinstall(cluster_hosts): """Reinstall clusters. :param cluster_hosts: clusters and hosts in each cluster to reinstall. - :type cluster_hosts: dict of int to list of int + :type cluster_hosts: dict of int or str to list of int or str .. note:: The function should be called out of database session. """ - logging.debug('reinstall cluster_hosts: %s', cluster_hosts) - with database.session(): - cluster_hosts, os_versions, target_systems = ( - util.update_cluster_hosts(cluster_hosts)) - manager = ConfigManager() - manager.reinstall_cluster_and_hosts( - cluster_hosts, os_versions, target_systems) - manager.sync() + with util.lock('serialized_action'): + logging.debug('reinstall cluster_hosts: %s', cluster_hosts) + with database.session(): + cluster_hosts, os_versions, target_systems = ( + util.update_cluster_hosts(cluster_hosts)) + manager = ConfigManager() + manager.reinstall_cluster_and_hosts( + cluster_hosts, os_versions, target_systems) + manager.sync() diff --git a/compass/actions/search.py b/compass/actions/search.py index cd8f45ea..f130d98e 100644 --- a/compass/actions/search.py +++ b/compass/actions/search.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to search configs of given clusters .. moduleauthor:: Xiaodong Wang @@ -15,7 +29,7 @@ def search(cluster_hosts, cluster_propreties_match, """search clusters. :param cluster_hosts: clusters and hosts in each cluster to search. - :type cluster_hosts: dict of int to list of int + :type cluster_hosts: dict of int or str to list of int or str .. note:: The function should be called out of database session. @@ -26,7 +40,7 @@ def search(cluster_hosts, cluster_propreties_match, util.update_cluster_hosts(cluster_hosts)) manager = ConfigManager() return manager.filter_cluster_and_hosts( - cluster_hosts, cluster_propreties_match, + cluster_hosts, os_versions, + target_systems, cluster_propreties_match, cluster_properties_name, host_properties_match, - host_properties_name, os_versions, - target_systems) + host_properties_name) diff --git a/compass/actions/update_progress.py b/compass/actions/update_progress.py index 27442f55..4febf6d3 100644 --- a/compass/actions/update_progress.py +++ b/compass/actions/update_progress.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to update status and installing progress of the given cluster. .. moduleauthor:: Xiaodong Wang @@ -5,8 +19,8 @@ import logging from compass.actions import util -from compass.log_analyzor import progress_calculator from compass.db import database +from compass.log_analyzor import progress_calculator from compass.utils import setting_wrapper as setting @@ -44,7 +58,7 @@ def update_progress(cluster_hosts): """Update status and installing progress of the given cluster. :param cluster_hosts: clusters and hosts in each cluster to update. - :type cluster_hosts: dict of int to list of int + :type cluster_hosts: dict of int or str to list of int or str .. note:: The function should be called out of the database session scope. @@ -58,17 +72,19 @@ def update_progress(cluster_hosts): After the progress got updated, these information will be stored back to the log_progressing_history for next time run. """ - logging.debug('update installing progress of cluster_hosts: %s', - cluster_hosts) - os_versions = {} - target_systems = {} - with database.session(): - cluster_hosts, os_versions, target_systems = ( - util.update_cluster_hosts( - cluster_hosts, _cluster_filter, _host_filter)) + with util.lock('log_progressing', blocking=False): + logging.debug('update installing progress of cluster_hosts: %s', + cluster_hosts) + os_versions = {} + target_systems = {} + with database.session(): + cluster_hosts, os_versions, target_systems = ( + util.update_cluster_hosts( + cluster_hosts, _cluster_filter, _host_filter)) - progress_calculator.update_progress(setting.OS_INSTALLER, - os_versions, - setting.PACKAGE_INSTALLER, - target_systems, - cluster_hosts) + progress_calculator.update_progress( + setting.OS_INSTALLER, + os_versions, + setting.PACKAGE_INSTALLER, + target_systems, + cluster_hosts) diff --git a/compass/actions/util.py b/compass/actions/util.py index abee5328..3bf20aae 100644 --- a/compass/actions/util.py +++ b/compass/actions/util.py @@ -1,12 +1,52 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to provide util for actions .. moduleauthor:: Xiaodong Wang ,xiaodongwang@huawei.com> """ import logging +import redis + +from contextlib import contextmanager from compass.db import database -from compass.db.model import Switch from compass.db.model import Cluster +from compass.db.model import Switch + + +@contextmanager +def lock(lock_name, blocking=True, timeout=10): + redis_instance = redis.Redis() + instance_lock = redis_instance.lock(lock_name, timeout=timeout) + try: + locked = instance_lock.acquire(blocking=blocking) + if locked: + logging.debug('acquired lock %s', lock_name) + yield instance_lock + else: + logging.info('lock %s is already hold', lock_name) + + except Exception as error: + logging.info( + 'redis fails to acquire the lock %s', lock_name) + logging.exception(error) + + finally: + instance_lock.acquired_until = 0 + instance_lock.release() + logging.debug('released lock %s', lock_name) def update_switch_ips(switch_ips): @@ -31,7 +71,11 @@ def update_cluster_hosts(cluster_hosts, updated_cluster_hosts = {} clusters = session.query(Cluster).all() for cluster in clusters: - if cluster_hosts and cluster.id not in cluster_hosts: + if cluster_hosts and ( + cluster.id not in cluster_hosts and + str(cluster.id) not in cluster_hosts and + cluster.name not in cluster_hosts + ): logging.debug('ignore cluster %s sinc it is not in %s', cluster.id, cluster_hosts) continue @@ -50,18 +94,26 @@ def update_cluster_hosts(cluster_hosts, os_versions[cluster.id] = adapter.os target_systems[cluster.id] = adapter.target_system - if ( - cluster.id not in cluster_hosts or - not cluster_hosts[cluster.id] - ): - hostids = [host.id for host in cluster.hosts] + if cluster.id in cluster_hosts: + hosts = cluster_hosts[cluster.id] + elif str(cluster.id) in cluster_hosts: + hosts = cluster_hosts[str(cluster.id)] + elif cluster.name in cluster_hosts: + hosts = cluster_hosts[cluster.name] else: - hostids = cluster_hosts[cluster.id] + hosts = [] + + if not hosts: + hosts = [host.id for host in cluster.hosts] for host in cluster.hosts: - if host.id not in hostids: + if ( + host.id not in hosts and + str(host.id) not in hosts and + host.hostname not in hosts + ): logging.debug('ignore host %s which is not in %s', - host.id, hostids) + host.id, hosts) continue if host_filter and not host_filter(host): diff --git a/compass/api/__init__.py b/compass/api/__init__.py index 08e582f9..2d0b4721 100644 --- a/compass/api/__init__.py +++ b/compass/api/__init__.py @@ -1,9 +1,26 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + __all__ = ['Flask', 'SQLAlchemy', 'compass_api'] -from flask import Flask + from flask.ext.sqlalchemy import SQLAlchemy +from flask import Flask + app = Flask(__name__) app.debug = True + from compass.api import api as compass_api diff --git a/compass/api/api.py b/compass/api/api.py index 7feb63b4..eb254f49 100644 --- a/compass/api/api.py +++ b/compass/api/api.py @@ -1,27 +1,46 @@ -"""Define all the RestfulAPI entry points""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Define all the RestfulAPI entry points.""" import logging +import netaddr import re import simplejson as json -from flask import request -from flask.ext.restful import Resource -from sqlalchemy.sql import and_, or_ -from compass.api import app, util, errors -from compass.tasks.client import celery +from flask.ext.restful import Resource +from flask import request +from sqlalchemy.sql import and_ +from sqlalchemy.sql import or_ + +from compass.api import app +from compass.api import errors +from compass.api import util from compass.db import database -from compass.db.model import Switch as ModelSwitch -from compass.db.model import SwitchConfig -from compass.db.model import Machine as ModelMachine +from compass.db.model import Adapter from compass.db.model import Cluster as ModelCluster from compass.db.model import ClusterHost as ModelClusterHost from compass.db.model import ClusterState from compass.db.model import HostState -from compass.db.model import Adapter +from compass.db.model import Machine as ModelMachine from compass.db.model import Role +from compass.db.model import Switch as ModelSwitch +from compass.db.model import SwitchConfig +from compass.tasks.client import celery class SwitchList(Resource): - """Query detals of switches and poll swithes""" + """Query details of switches and poll swithes.""" ENDPOINT = "/switches" @@ -30,9 +49,10 @@ class SwitchList(Resource): LIMIT = 'limit' def get(self): - """ - List details of all switches, optionally filtered by some conditions. - Note: switchIp and swtichIpNetwork cannot be combined to use. + """List details of all switches filtered by some conditions. + + .. note:: + switchIp and swtichIpNetwork cannot be combined to use. :param switchIp: switch IP address :param switchIpNetwork: switch IP network @@ -85,17 +105,17 @@ class SwitchList(Resource): errors.UserInvalidUsage(error_msg)) def get_queried_ip_prefix(network, prefix): - """ Get Ip prefex as pattern used to query switches. - Switches' Ip addresses need to match this pattern. + """Get Ip prefex as pattern used to query switches. + + .. note:: + Switches' Ip addresses need to match this pattern. """ count = int(prefix / 8) if count == 0: count = 1 return network.rsplit('.', count)[0] + '.' - from netaddr import IPNetwork, IPAddress - - ip_network = IPNetwork(switch_ip_network) + ip_network = netaddr.IPNetwork(switch_ip_network) ip_filter = get_queried_ip_prefix(str(ip_network.network), ip_network.prefixlen) @@ -103,15 +123,16 @@ class SwitchList(Resource): result_set = [] if limit: result_set = session.query(ModelSwitch).filter( - ModelSwitch.ip.startswith(ip_filter)).limit( - limit).all() + ModelSwitch.ip.startswith(ip_filter) + ).limit(limit).all() else: result_set = session.query(ModelSwitch).filter( - ModelSwitch.ip.startswith(ip_filter)).all() + ModelSwitch.ip.startswith(ip_filter) + ).all() for switch in result_set: ip_addr = str(switch.ip) - if IPAddress(ip_addr) in ip_network: + if netaddr.IPAddress(ip_addr) in ip_network: switches.append(switch) logging.info('[SwitchList][get] ip %s', ip_addr) @@ -139,9 +160,10 @@ class SwitchList(Resource): "switches": switch_list}) def post(self): - """ - Insert switch IP and the credential to db. Invoke a task to poll - switch at the same time. + """Insert switch IP and the credential to db. + + .. note:: + Invoke a task to poll switch at the same time. :param ip: switch IP address :param credential: a dict for accessing the switch @@ -197,7 +219,7 @@ class SwitchList(Resource): class Switch(Resource): - """Get and update a single switch information""" + """Get and update a single switch information.""" ENDPOINT = "/switches" def get(self, switch_id): @@ -288,7 +310,7 @@ class Switch(Resource): class MachineList(Resource): - """Query machines by filters""" + """Query machines by filters.""" ENDPOINT = "/machines" SWITCHID = 'switchId' @@ -298,10 +320,12 @@ class MachineList(Resource): LIMIT = 'limit' def get(self): - """ - Lists details of machines, optionally filtered by some conditions as - the following. According to SwitchConfig, machines with some ports will - be filtered. + """Lists details of machines. + + .. note:: + The machines are filtered by some conditions as + the following. According to SwitchConfig, machines + with some ports will be filtered. :param switchId: the unique identifier of the switch :param mac: the MAC address @@ -335,7 +359,7 @@ class MachineList(Resource): return errors.UserInvalidUsage( errors.UserInvalidUsage(error_msg) ) - #TODO: support query filtered port + # TODO(grace): support query filtered port if filter_clause: machines = session.query(ModelMachine)\ .filter(and_(*filter_clause)).all() @@ -362,8 +386,8 @@ class MachineList(Resource): continue machine_res = {} - machine_res['switch_ip'] = None if not machine.switch else \ - machine.switch.ip + machine_res['switch_ip'] = ( + None if not machine.switch else machine.switch.ip) machine_res['id'] = machine.id machine_res['mac'] = machine.mac machine_res['port'] = machine.port @@ -380,12 +404,11 @@ class MachineList(Resource): class Machine(Resource): - """List details of the machine with specific machine id""" + """List details of the machine with specific machine id.""" ENDPOINT = '/machines' def get(self, machine_id): - """ - Lists details of the specified machine. + """Lists details of the specified machine. :param machine_id: the unique identifier of the machine """ @@ -422,8 +445,10 @@ class Machine(Resource): class Cluster(Resource): - """Creates cluster and lists cluster details; Update and list the cluster's - configuration information. + """Creates cluster and lists cluster details. + + .. note:: + Update and list the cluster's configuration information. """ ENDPOINT = '/clusters' SECURITY = 'security' @@ -431,9 +456,7 @@ class Cluster(Resource): PARTITION = 'partition' def get(self, cluster_id, resource=None): - """ - Lists details of the specified cluster if resource is not specified. - Otherwise, lists details of the resource of this cluster + """Lists details of the resource specified cluster. :param cluster_id: the unique identifier of the cluster :param resource: the resource name(security, networking, partition) @@ -527,8 +550,7 @@ class Cluster(Resource): ) def put(self, cluster_id, resource): - """ - Update the resource information of the specified cluster in database + """Update the resource information of the specified cluster. :param cluster_id: the unique identifier of the cluster :param resource: resource name(security, networking, partition) @@ -573,8 +595,9 @@ class Cluster(Resource): if is_valid: column = resources[resource]['column'] session.query( - ModelCluster).filter_by(id=cluster_id).update( - {column: json.dumps(value)} + ModelCluster + ).filter_by(id=cluster_id).update( + {column: json.dumps(value)} ) else: return errors.handle_mssing_input( @@ -588,7 +611,7 @@ class Cluster(Resource): @app.route("/clusters", methods=['GET']) def list_clusters(): - """Lists the details of all clusters""" + """Lists the details of all clusters.""" endpoint = '/clusters' state = request.args.get('state', None, type=str) results = [] @@ -608,23 +631,29 @@ def list_clusters(): elif state == 'installing': # The deployment of this cluster is in progress. clusters = session.query( - ModelCluster).filter( + ModelCluster + ).filter( ModelCluster.id == ClusterState.id, or_( ClusterState.state == 'INSTALLING', ClusterState.state == 'UNINITIALIZED' - )).all() + ) + ).all() elif state == 'failed': # The deployment of this cluster is failed. - clusters = session.query(ModelCluster)\ - .filter(ModelCluster.id == ClusterState.id, - ClusterState.state == 'ERROR')\ - .all() + clusters = session.query( + ModelCluster + ).filter( + ModelCluster.id == ClusterState.id, + ClusterState.state == 'ERROR' + ).all() elif state == 'successful': - clusters = session.query(ModelCluster)\ - .filter(ModelCluster.id == ClusterState.id, - ClusterState.state == 'READY')\ - .all() + clusters = session.query( + ModelCluster + ).filter( + ModelCluster.id == ClusterState.id, + ClusterState.state == 'READY' + ).all() if clusters: for cluster in clusters: @@ -637,8 +666,11 @@ def list_clusters(): results.append(cluster_res) return util.make_json_response( - 200, {"status": "OK", - "clusters": results}) + 200, { + "status": "OK", + "clusters": results + } + ) @app.route("/clusters//action", methods=['POST']) @@ -653,7 +685,7 @@ def execute_cluster_action(cluster_id): :param deploy: the action of starting to deploy """ def _add_hosts(cluster_id, hosts): - """Add cluster host(s) to the cluster by cluster_id""" + """Add cluster host(s) to the cluster by cluster_id.""" cluseter_hosts = [] available_machines = [] @@ -708,15 +740,17 @@ def execute_cluster_action(cluster_id): ) def _remove_hosts(cluster_id, hosts): - """Remove existing cluster host from the cluster""" + """Remove existing cluster host from the cluster.""" removed_hosts = [] with database.session() as session: failed_hosts = [] for host_id in hosts: host = session.query( - ModelClusterHost).filter_by( - id=host_id, cluster_id=cluster_id).first() + ModelClusterHost + ).filter_by( + id=host_id, cluster_id=cluster_id + ).first() if not host: failed_hosts.append(host_id) @@ -742,8 +776,9 @@ def execute_cluster_action(cluster_id): filter_clause.append('id=%s' % host_id) # Delete the requested hosts from database - session.query(ModelClusterHost).filter(or_(*filter_clause))\ - .delete(synchronize_session='fetch') + session.query(ModelClusterHost).filter( + or_(*filter_clause) + ).delete(synchronize_session='fetch') return util.make_json_response( 200, { @@ -753,7 +788,7 @@ def execute_cluster_action(cluster_id): ) def _replace_all_hosts(cluster_id, hosts): - """Remove all existing hosts from the cluster and add new ones""" + """Remove all existing hosts from the cluster and add new ones.""" with database.session() as session: # Delete all existing hosts of the cluster @@ -763,7 +798,7 @@ def execute_cluster_action(cluster_id): return _add_hosts(cluster_id, hosts) def _deploy(cluster_id, hosts): - """Deploy the cluster""" + """Deploy the cluster.""" deploy_hosts_info = [] deploy_cluster_info = {} @@ -862,10 +897,10 @@ def execute_cluster_action(cluster_id): class ClusterHostConfig(Resource): - """Lists and update/delete cluster host configurations""" + """Lists and update/delete cluster host configurations.""" def get(self, host_id): - """Lists configuration details of the specified cluster host + """Lists configuration details of the specified cluster host. :param host_id: the unique identifier of the host """ @@ -887,8 +922,7 @@ class ClusterHostConfig(Resource): "config": config_res}) def put(self, host_id): - """ - Update configuration of the specified cluster host + """Update configuration of the specified cluster host. :param host_id: the unique identifier of the host """ @@ -943,8 +977,7 @@ class ClusterHostConfig(Resource): 200, {"status": "OK"}) def delete(self, host_id, subkey): - """ - Delete one attribute in configuration of the specified cluster host + """Delete one attribute of the specified cluster host. :param host_id: the unique identifier of the host :param subkey: the attribute name in configuration @@ -978,11 +1011,11 @@ class ClusterHostConfig(Resource): class ClusterHost(Resource): - """List details of the cluster host by host id""" + """List details of the cluster host by host id.""" ENDPOINT = '/clusterhosts' def get(self, host_id): - """Lists details of the specified cluster host + """Lists details of the specified cluster host. :param host_id: the unique identifier of the host """ @@ -1010,8 +1043,10 @@ class ClusterHost(Resource): @app.route("/clusterhosts", methods=['GET']) def list_clusterhosts(): - """ - Lists details of all cluster hosts, optionally filtered by some conditions. + """Lists details of all cluster hosts. + + .. note:: + the cluster hosts are optionally filtered by some conditions. :param hostname: the name of the host :param clstername: the name of the cluster @@ -1027,9 +1062,11 @@ def list_clusterhosts(): hosts = None if hostname and clustername: hosts = session.query( - ModelClusterHost).join(ModelCluster).filter( + ModelClusterHost + ).join(ModelCluster).filter( ModelClusterHost.hostname == hostname, - ModelCluster.name == clustername).all() + ModelCluster.name == clustername + ).all() elif hostname: hosts = session.query( @@ -1062,8 +1099,7 @@ def list_clusterhosts(): @app.route("/adapters/", methods=['GET']) def list_adapter(adapter_id): - """ - Lists details of the specified adapter. + """Lists details of the specified adapter. :param adapter_id: the unique identifier of the adapter """ @@ -1103,12 +1139,15 @@ def list_adapter_roles(adapter_id): if not adapter_q: error_msg = "Adapter id=%s does not exist!" % adapter_id return errors.handle_not_exist( - errors.ObjectDoesNotExist(error_msg)) + errors.ObjectDoesNotExist(error_msg) + ) roles = session.query( - Role, Adapter).filter( + Role, Adapter + ).filter( Adapter.id == adapter_id, - Adapter.target_system == Role.target_system).all() + Adapter.target_system == Role.target_system + ).all() for role, _ in roles: role_res = {} @@ -1117,8 +1156,11 @@ def list_adapter_roles(adapter_id): roles_list.append(role_res) return util.make_json_response( - 200, {"status": "OK", - "roles": roles_list}) + 200, { + "status": "OK", + "roles": roles_list + } + ) @app.route("/adapters", methods=['GET']) @@ -1155,10 +1197,10 @@ def list_adapters(): class HostInstallingProgress(Resource): - """Get host installing progress information""" + """Get host installing progress information.""" def get(self, host_id): - """Lists progress details of a specific cluster host + """Lists progress details of a specific cluster host. :param host_id: the unique identifier of the host """ @@ -1193,10 +1235,10 @@ class HostInstallingProgress(Resource): class ClusterInstallingProgress(Resource): - """Get cluster installing progress information""" + """Get cluster installing progress information.""" def get(self, cluster_id): - """Lists progress details of a specific cluster + """Lists progress details of a specific cluster. :param cluster_id: the unique identifier of the cluster """ @@ -1232,66 +1274,94 @@ class ClusterInstallingProgress(Resource): class DashboardLinks(Resource): - """Lists dashboard links""" + """Lists dashboard links.""" ENDPOINT = "/dashboardlinks/" def get(self): - """ - Return a list of dashboard links + """Return a list of dashboard links. """ cluster_id = request.args.get('cluster_id', None) logging.info('get cluster links with cluster_id=%s', cluster_id) links = {} with database.session() as session: - hosts = session.query(ModelClusterHost)\ - .filter_by(cluster_id=cluster_id).all() + hosts = session.query( + ModelClusterHost + ).filter_by(cluster_id=cluster_id).all() if not hosts: error_msg = "Cannot find hosts in cluster id=%s" % cluster_id return errors.handle_not_exist( - errors.ObjectDoesNotExist(error_msg)) + errors.ObjectDoesNotExist(error_msg) + ) for host in hosts: config = host.config - if ('has_dashboard_roles' in config and - config['has_dashboard_roles']): + if ( + 'has_dashboard_roles' in config and + config['has_dashboard_roles'] + ): ip_addr = config.get( - 'networking', {}).get( - 'interfaces', {}).get( - 'management', {}).get( - 'ip', '') + 'networking', {} + ).get( + 'interfaces', {} + ).get( + 'management', {} + ).get( + 'ip', '' + ) roles = config.get('roles', []) for role in roles: links[role] = 'http://%s' % ip_addr return util.make_json_response( - 200, {"status": "OK", - "dashboardlinks": links} + 200, { + "status": "OK", + "dashboardlinks": links + } ) TABLES = { - 'switch_config': {'name': SwitchConfig, - 'columns': ['id', 'ip', 'filter_port']}, - 'switch': {'name': ModelSwitch, - 'columns': ['id', 'ip', 'credential_data']}, - 'machine': {'name': ModelMachine, - 'columns': ['id', 'mac', 'port', 'vlan', 'switch_id']}, - 'cluster': {'name': ModelCluster, - 'columns': ['id', 'name', 'security_config', - 'networking_config', 'partition_config', - 'adapter_id', 'state']}, - 'cluster_host': {'name': ModelClusterHost, - 'columns': ['id', 'cluster_id', 'hostname', 'machine_id', - 'config_data', 'state']}, - 'adapter': {'name': Adapter, - 'columns': ['id', 'name', 'os', 'target_system']}, - 'role': {'name': Role, - 'columns': ['id', 'name', 'target_system', 'description']} + 'switch_config': { + 'name': SwitchConfig, + 'columns': ['id', 'ip', 'filter_port'] + }, + 'switch': { + 'name': ModelSwitch, + 'columns': ['id', 'ip', 'credential_data'] + }, + 'machine': { + 'name': ModelMachine, + 'columns': ['id', 'mac', 'port', 'vlan', 'switch_id'] + }, + 'cluster': { + 'name': ModelCluster, + 'columns': [ + 'id', 'name', 'security_config', + 'networking_config', 'partition_config', + 'adapter_id', 'state' + ] + }, + 'cluster_host': { + 'name': ModelClusterHost, + 'columns': [ + 'id', 'cluster_id', 'hostname', 'machine_id', + 'config_data', 'state' + ] + }, + 'adapter': { + 'name': Adapter, + 'columns': ['id', 'name', 'os', 'target_system'] + }, + 'role': { + 'name': Role, + 'columns': ['id', 'name', 'target_system', 'description'] + } } @app.route("/export/", methods=['GET']) def export_csv(tname): + """export to csv file.""" if tname not in TABLES: error_msg = "Table '%s' is not supported to export or wrong table name" return util.handle_invalid_usage( @@ -1369,5 +1439,6 @@ util.add_resource(ClusterInstallingProgress, '/clusters//progress') util.add_resource(DashboardLinks, '/dashboardlinks') + if __name__ == '__main__': app.run(debug=True) diff --git a/compass/api/errors.py b/compass/api/errors.py index 95d7bb9f..a6caf166 100644 --- a/compass/api/errors.py +++ b/compass/api/errors.py @@ -1,10 +1,24 @@ -"""Exception and its handler""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exception and its handler.""" from compass.api import app from compass.api import util class ObjectDoesNotExist(Exception): - """Define the exception for referring non-existing object""" + """Define the exception for referring non-existing object.""" def __init__(self, message): super(ObjectDoesNotExist, self).__init__(message) self.message = message @@ -14,7 +28,7 @@ class ObjectDoesNotExist(Exception): class UserInvalidUsage(Exception): - """Define the exception for fault usage of users""" + """Define the exception for fault usage of users.""" def __init__(self, message): super(UserInvalidUsage, self).__init__(message) self.message = message @@ -24,7 +38,7 @@ class UserInvalidUsage(Exception): class ObjectDuplicateError(Exception): - """Define the duplicated object exception""" + """Define the duplicated object exception.""" def __init__(self, message): super(ObjectDuplicateError, self).__init__(message) self.message = message @@ -34,7 +48,7 @@ class ObjectDuplicateError(Exception): class InputMissingError(Exception): - """Define the insufficient input exception""" + """Define the insufficient input exception.""" def __init__(self, message): super(InputMissingError, self).__init__(message) self.message = message @@ -44,7 +58,7 @@ class InputMissingError(Exception): class MethodNotAllowed(Exception): - """Define the exception which invalid method is called""" + """Define the exception which invalid method is called.""" def __init__(self, message): super(MethodNotAllowed, self).__init__(message) self.message = message @@ -55,7 +69,7 @@ class MethodNotAllowed(Exception): @app.errorhandler(ObjectDoesNotExist) def handle_not_exist(error, failed_objs=None): - """Handler of ObjectDoesNotExist Exception""" + """Handler of ObjectDoesNotExist Exception.""" message = {'status': 'Not Found', 'message': error.message} @@ -68,30 +82,36 @@ def handle_not_exist(error, failed_objs=None): @app.errorhandler(UserInvalidUsage) def handle_invalid_usage(error): - """Handler of UserInvalidUsage Exception""" + """Handler of UserInvalidUsage Exception.""" - message = {'status': 'Invalid parameters', - 'message': error.message} + message = { + 'status': 'Invalid parameters', + 'message': error.message + } return util.make_json_response(400, message) @app.errorhandler(InputMissingError) def handle_mssing_input(error): - """Handler of InputMissingError Exception""" + """Handler of InputMissingError Exception.""" - message = {'status': 'Insufficient data', - 'message': error.message} + message = { + 'status': 'Insufficient data', + 'message': error.message + } return util.make_json_response(400, message) @app.errorhandler(ObjectDuplicateError) def handle_duplicate_object(error, failed_objs=None): - """Handler of ObjectDuplicateError Exception""" + """Handler of ObjectDuplicateError Exception.""" - message = {'status': 'Conflict Error', - 'message': error.message} + message = { + 'status': 'Conflict Error', + 'message': error.message + } if failed_objs and isinstance(failed_objs, dict): message.update(failed_objs) @@ -101,8 +121,10 @@ def handle_duplicate_object(error, failed_objs=None): @app.errorhandler(MethodNotAllowed) def handle_not_allowed_method(error): - """Handler of MethodNotAllowed Exception""" + """Handler of MethodNotAllowed Exception.""" - message = {"status": "Method Not Allowed", - "message": "The method is not allowed to use"} + message = { + "status": "Method Not Allowed", + "message": "The method is not allowed to use" + } return util.make_json_response(405, message) diff --git a/compass/api/util.py b/compass/api/util.py index d6044009..3872bece 100644 --- a/compass/api/util.py +++ b/compass/api/util.py @@ -1,18 +1,33 @@ -"""Utils for API usage""" -from flask import make_response -from flask.ext.restful import Api +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utils for API usage.""" +import netaddr import re -from netaddr import IPAddress + +from flask.ext.restful import Api +from flask import make_response import simplejson as json from compass.api import app + API = Api(app) def make_json_response(status_code, data): - """Wrap json format to the reponse object""" + """Wrap json format to the reponse object.""" result = json.dumps(data, indent=4) resp = make_response(result, status_code) @@ -21,7 +36,7 @@ def make_json_response(status_code, data): def make_csv_response(status_code, csv_data, fname): - """Wrap CSV format to the reponse object""" + """Wrap CSV format to the reponse object.""" fname = '.'.join((fname, 'csv')) resp = make_response(csv_data, status_code) resp.mimetype = 'text/csv' @@ -30,12 +45,12 @@ def make_csv_response(status_code, csv_data, fname): def add_resource(*args, **kwargs): - """Add resource""" + """Add resource.""" API.add_resource(*args, **kwargs) def is_valid_ip(ip_address): - """Valid the format of an Ip address""" + """Valid the format of an Ip address.""" if not ip_address: return False @@ -50,7 +65,7 @@ def is_valid_ip(ip_address): def is_valid_ipnetowrk(ip_network): - """Valid the format of an Ip network""" + """Valid the format of an Ip network.""" if not ip_network: return False @@ -66,9 +81,9 @@ def is_valid_ipnetowrk(ip_network): def is_valid_netmask(ip_addr): - """Valid the format of a netmask""" + """Valid the format of a netmask.""" try: - ip_address = IPAddress(ip_addr) + ip_address = netaddr.IPAddress(ip_addr) return ip_address.is_netmask() except Exception: @@ -76,14 +91,14 @@ def is_valid_netmask(ip_addr): def is_valid_gateway(ip_addr): - """Valid the format of gateway""" + """Valid the format of gateway.""" invalid_ip_prefix = ['0', '224', '169', '127'] try: # Check if ip_addr is an IP address and not start with 0 ip_addr_prefix = ip_addr.split('.')[0] if is_valid_ip(ip_addr) and ip_addr_prefix not in invalid_ip_prefix: - ip_address = IPAddress(ip_addr) + ip_address = netaddr.IPAddress(ip_addr) if not ip_address.is_multicast(): # Check if ip_addr is not multicast and reserved IP return True @@ -106,7 +121,7 @@ def _is_valid_nameservers(value): def is_valid_security_config(config): - """Valid the format of security section in config""" + """Valid the format of security section in config.""" outer_format = { "server_credentials": {}, "service_credentials": {}, "console_credentials": {} @@ -132,10 +147,10 @@ def is_valid_security_config(config): def is_valid_networking_config(config): - """Valid the format of networking config""" + """Valid the format of networking config.""" def _is_valid_interfaces_config(interfaces_config): - """Valid the format of interfaces section in config""" + """Valid the format of interfaces section in config.""" interfaces_section = { "management": {}, "tenant": {}, "public": {}, "storage": {} } @@ -206,7 +221,7 @@ def is_valid_networking_config(config): return (True, '') def _is_valid_global_config(global_config): - """Valid the format of 'global' section in config""" + """Valid the format of 'global' section in config.""" global_section = { "nameservers": {"req": 1, "validator": _is_valid_nameservers}, "search_path": {"req": 1, "validator": ""}, @@ -253,7 +268,7 @@ def is_valid_networking_config(config): def is_valid_partition_config(config): - """Valid the configuration format""" + """Valid the configuration format.""" if not config: return (False, '%s in partition cannot be null!' % config) @@ -262,10 +277,13 @@ def is_valid_partition_config(config): def valid_host_config(config): - """ valid_format is used to check if the input config is qualified - the required fields and format. - The key is the required field and format of the input config - The value is the validator function name of the config value + """Valid the host configuration format. + + .. note:: + Valid_format is used to check if the input config is qualified + the required fields and format. + The key is the required field and format of the input config + The value is the validator function name of the config value """ from api import errors @@ -295,10 +313,12 @@ def valid_host_config(config): def flatten_dict(dictionary, output, flat_key=""): - """This function will convert the dictionary into a list - For example: - dict = {'a':{'b': 'c'}, 'd': 'e'} ==> - list = ['a/b/c', 'd/e'] + """This function will convert the dictionary into a flatten dict. + + .. note:: + For example: + dict = {'a':{'b': 'c'}, 'd': 'e'} ==> + flatten dict = {'a/b': 'c', 'd': 'e'} """ keywords = dictionary.keys() @@ -311,13 +331,13 @@ def flatten_dict(dictionary, output, flat_key=""): def update_dict_value(searchkey, dictionary): - """Update dictionary value""" + """Update dictionary value.""" keywords = dictionary.keys() for key in keywords: if key == searchkey: if isinstance(dictionary[key], str): - dictionary[key] = "" + dictionary[key] = '' elif isinstance(dictionary[key], list): dictionary[key] = [] @@ -328,7 +348,7 @@ def update_dict_value(searchkey, dictionary): def is_valid_keys(expected, input_dict, section=""): - """Valid keys""" + """Validate keys.""" excepted_keys = set(expected.keys()) input_keys = set(input_dict.keys()) if excepted_keys != input_keys: @@ -344,15 +364,15 @@ def is_valid_keys(expected, input_dict, section=""): def get_col_val_from_dict(result, data): - """ - Convert a dict's values to a list. + """Convert a dict's values to a list. + :param result: a list of values for each column :param data: input data - for example: - data = {"a": {"b": {"c": 1}, - "d": 2}} - the result will be [1, 2] + .. note:: + for example: + data = {"a": {"b": {"c": 1}, "d": 2}} + the result will be [1, 2] """ if not isinstance(data, dict): data = str(data) if str(data) else 'None' @@ -365,16 +385,18 @@ def get_col_val_from_dict(result, data): def get_headers_from_dict(headers, colname, data): """Convert a column which value is dict to a list of column name and keys. - nested keys in dict will be joined by '.' as a column name in CSV. - :param headers: the result list to hold dict keys - :param colname: the column name - :param data: input data - for example: - the column name is 'config_data', and - the value is {"a": {"b": {"c": 1}, - "d": 2}} - then headers will be ['config_data.a.b.c', 'config_data.a.d'] + .. note:: + nested keys in dict will be joined by '.' as a column name in CSV. + for example: + the column name is 'config_data', and + the value is {"a": {"b": {"c": 1}, "d": 2}} + then headers will be ['config_data.a.b.c', 'config_data.a.d'] + + :param headers: the result list to hold dict keys + :param colname: the column name + :param data: input data + """ if not colname: raise "colname cannot be None!" diff --git a/compass/apiclient/example.py b/compass/apiclient/example.py index 6c655740..5f0d939f 100755 --- a/compass/apiclient/example.py +++ b/compass/apiclient/example.py @@ -1,9 +1,26 @@ #!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Example code to deploy a cluster by compass client api.""" +import os +import re +import requests import sys import time -import os -import requests + from compass.apiclient.restful import Client @@ -233,7 +250,7 @@ print 'deploy cluster %s status: %s, resp: %s' % (cluster_id, status, resp) # get intalling progress. -timeout = time.time() + 60 * 30 +timeout = time.time() + 60 * 60 while True: status, resp = client.get_cluster_installing_progress(cluster_id) print 'get cluster %s installing progress status: %s, resp: %s' % ( @@ -260,8 +277,11 @@ print 'get cluster %s dashboardlinks status: %s, resp: %s' % ( dashboardlinks = resp['dashboardlinks'] r = requests.get(dashboardlinks['os-dashboard'], verify=False) r.raise_for_status() -if r.text.find('username') == 1054: - print 'dashboard login page can be downloaded with keyword username' +match = re.search(r'(?m)(http://\d+\.\d+\.\d+\.\d+:5000/v2\.0)', r.text) +if match: + print 'dashboard login page can be downloaded' else: - print 'dashboard login page failed to be downloaded' + print ( + 'dashboard login page failed to be downloaded\n' + 'the context is:\n%s\n') % r.text raise Exception("os-dashboard is not properly installed!") diff --git a/compass/apiclient/restful.py b/compass/apiclient/restful.py index 998f623b..449b3f93 100644 --- a/compass/apiclient/restful.py +++ b/compass/apiclient/restful.py @@ -1,9 +1,23 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Compass api client library. .. moduleauthor:: Xiaodong Wang """ -import logging import json +import logging import requests @@ -390,10 +404,13 @@ class Client(object): else: if '_' not in key: continue + key_name, key_value = key.split('_', 1) data.setdefault( - 'interfaces', {}).setdefault( - key_name, {})[key_value] = value + 'interfaces', {} + ).setdefault( + key_name, {} + )[key_value] = value return data diff --git a/compass/config_management/__init__.py b/compass/config_management/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/config_management/__init__.py +++ b/compass/config_management/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/config_management/installers/__init__.py b/compass/config_management/installers/__init__.py index f0cba3df..3f15a786 100644 --- a/compass/config_management/installers/__init__.py +++ b/compass/config_management/installers/__init__.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """modules to read/write cluster/host config from installers. .. moduleauthor:: Xiaodong Wang @@ -14,12 +28,16 @@ __all__ = [ from compass.config_management.installers.os_installer import ( - get_installer_by_name as get_os_installer_by_name, - get_installer as get_os_installer, + get_installer as get_os_installer) +from compass.config_management.installers.os_installer import ( + get_installer_by_name as get_os_installer_by_name) +from compass.config_management.installers.os_installer import ( register as register_os_installer) from compass.config_management.installers.package_installer import ( - get_installer_by_name as get_package_installer_by_name, - get_installer as get_package_installer, + get_installer as get_package_installer) +from compass.config_management.installers.package_installer import ( + get_installer_by_name as get_package_installer_by_name) +from compass.config_management.installers.package_installer import ( register as register_package_installer) from compass.config_management.installers.plugins import chefhandler from compass.config_management.installers.plugins import cobbler diff --git a/compass/config_management/installers/installer.py b/compass/config_management/installers/installer.py index f9e1712c..bc4a9e57 100644 --- a/compass/config_management/installers/installer.py +++ b/compass/config_management/installers/installer.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to provider installer interface. .. moduleauthor:: Xiaodong Wang diff --git a/compass/config_management/installers/os_installer.py b/compass/config_management/installers/os_installer.py index e80d2f65..2c4e8af2 100644 --- a/compass/config_management/installers/os_installer.py +++ b/compass/config_management/installers/os_installer.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module for interface of os installer. .. moduleauthor::: Xiaodong Wang diff --git a/compass/config_management/installers/package_installer.py b/compass/config_management/installers/package_installer.py index c3203ac6..d6f7770c 100644 --- a/compass/config_management/installers/package_installer.py +++ b/compass/config_management/installers/package_installer.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to provider interface for package installer. .. moduleauthor:: Xiaodong Wang diff --git a/compass/config_management/installers/plugins/__init__.py b/compass/config_management/installers/plugins/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/config_management/installers/plugins/__init__.py +++ b/compass/config_management/installers/plugins/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/config_management/installers/plugins/chefhandler.py b/compass/config_management/installers/plugins/chefhandler.py index 021ccdd7..4e7c2406 100644 --- a/compass/config_management/installers/plugins/chefhandler.py +++ b/compass/config_management/installers/plugins/chefhandler.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """package instaler chef plugin. .. moduleauthor:: Xiaodong Wang @@ -5,12 +19,12 @@ import fnmatch import logging -from compass.utils import util from compass.config_management.installers import package_installer from compass.config_management.utils.config_translator import ConfigTranslator from compass.config_management.utils.config_translator import KeyTranslator from compass.config_management.utils import config_translator_callbacks from compass.utils import setting_wrapper as setting +from compass.utils import util TO_CLUSTER_TRANSLATORS = { @@ -73,6 +87,9 @@ FROM_CLUSTER_TRANSLATORS = { '/dashboard_roles': [KeyTranslator( translated_keys=['/dashboard_roles'] )], + '/role_mapping': [KeyTranslator( + translated_keys=['/role_mapping'] + )], } ), } @@ -81,26 +98,13 @@ FROM_CLUSTER_TRANSLATORS = { TO_HOST_TRANSLATORS = { 'openstack': ConfigTranslator( mapping={ - '/networking/interfaces/management/ip': [KeyTranslator( - translated_keys=[ - '/db/mysql/bind_address', - '/mq/rabbitmq/bind_address', - '/endpoints/compute/metadata/host', - '/endpoints/compute/novnc/host', - '/endpoints/compute/service/host', - '/endpoints/compute/xvpvnc/host', - '/endpoints/ec2/admin/host', - '/endpoints/ec2/service/host', - '/endpoints/identity/admin/host', - '/endpoints/identity/service/host', - '/endpoints/image/registry/host', - '/endpoints/image/service/host', - '/endpoints/metering/service/host', - '/endpoints/network/service/host', - '/endpoints/volume/service/host', - ], - translated_value=config_translator_callbacks.get_value_if, - from_values={'condition': '/has_dashboard_roles'} + '/roles': [KeyTranslator( + translated_keys=( + config_translator_callbacks.get_keys_from_role_mapping), + from_keys={'mapping': '/role_mapping'}, + translated_value=( + config_translator_callbacks.get_value_from_role_mapping), + from_values={'mapping': '/role_mapping'} )], } ), @@ -121,40 +125,36 @@ class Installer(package_installer.Installer): def __repr__(self): return '%s[name=%s,installer_url=%s,global_databag_name=%s]' % ( - self.__class__.__name__, self.installer_url_, - self.NAME, self.global_databag_name_) + self.__class__.__name__, self.NAME, self.installer_url_, + self.global_databag_name_) @classmethod def _cluster_databag_name(cls, clusterid, target_system): - """get cluster databag name""" - return '%s_%s' % (target_system, str(clusterid)) + """get cluster databag name.""" + return '%s_%s' % (target_system, clusterid) @classmethod - def _get_client_name(cls, hostname, clusterid, target_system): - """get client name""" - return cls._get_node_name(hostname, clusterid, target_system) + def _get_client_name(cls, fullname, target_system): + """get client name.""" + return cls._get_node_name(fullname, target_system) @classmethod - def _get_node_name(cls, hostname, clusterid, target_system): - """get node name""" - return '%s_%s_%s' % (hostname, target_system, clusterid) + def _get_node_name(cls, fullname, target_system): + """get node name.""" + return '%s.%s' % (target_system, fullname) def os_installer_config(self, config, target_system, **kwargs): """get os installer config.""" - clusterid = config['clusterid'] - roles = config['roles'] return { '%s_url' % self.NAME: self.installer_url_, 'run_list': ','.join( - ['"role[%s]"' % role for role in roles if role]), + ['"role[%s]"' % role for role in config['roles'] if role]), 'cluster_databag': self._cluster_databag_name( - clusterid, target_system), + config['clusterid'], target_system), 'chef_client_name': self._get_client_name( - config['hostname'], config['clusterid'], - target_system), + config['fullname'], target_system), 'chef_node_name': self._get_node_name( - config['hostname'], config['clusterid'], - target_system) + config['fullname'], target_system) } def get_target_systems(self, oses): @@ -251,12 +251,12 @@ class Installer(package_installer.Installer): bag_item.save() def _clean_client(self, hostid, config, target_system, **kwargs): - """clean client""" + """clean client.""" from chef import Client try: client = Client( self._get_client_name( - config['hostname'], config['clusterid'], target_system), + config['fullname'], target_system), api=self.api_) client.delete() logging.debug('client is removed for host %s ' @@ -268,12 +268,12 @@ class Installer(package_installer.Installer): hostid, config, target_system) def _clean_node(self, hostid, config, target_system, **kwargs): - """clean node""" + """clean node.""" from chef import Node try: node = Node( self._get_node_name( - config['hostname'], config['clusterid'], target_system), + config['fullname'], target_system), api=self.api_ ) node.delete() diff --git a/compass/config_management/installers/plugins/cobbler.py b/compass/config_management/installers/plugins/cobbler.py index 7fe5f9ce..05744d0e 100644 --- a/compass/config_management/installers/plugins/cobbler.py +++ b/compass/config_management/installers/plugins/cobbler.py @@ -1,4 +1,21 @@ -"""os installer cobbler plugin""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""os installer cobbler plugin. + + .. moduleauthor:: Xiaodong Wang +""" import functools import logging import os.path @@ -156,7 +173,7 @@ class Installer(os_installer.Installer): def _get_modify_system(self, profile, config, **kwargs): """get modified system config.""" system_config = { - 'name': self._get_system_name(config), + 'name': config['fullname'], 'hostname': config['hostname'], 'profile': profile, } @@ -180,15 +197,9 @@ class Installer(os_installer.Installer): {'name': os_version}) return profile_found[0] - @classmethod - def _get_system_name(cls, config): - """get system name""" - return '%s.%s' % ( - config['hostname'], config['clusterid']) - def _get_system(self, config, create_if_not_exists=True): """get system reference id.""" - sys_name = self._get_system_name(config) + sys_name = config['fullname'] try: sys_id = self.remote_.get_system_handle( sys_name, self.token_) @@ -206,7 +217,7 @@ class Installer(os_installer.Installer): def _clean_system(self, config): """clean system.""" - sys_name = self._get_system_name(config) + sys_name = config['fullname'] try: self.remote_.remove_system(sys_name, self.token_) logging.debug('system %s is removed', sys_name) @@ -218,13 +229,13 @@ class Installer(os_installer.Installer): self.remote_.save_system(sys_id, self.token_) def _update_modify_system(self, sys_id, system_config): - """update modify system""" + """update modify system.""" for key, value in system_config.items(): self.remote_.modify_system( sys_id, key, value, self.token_) def _netboot_enabled(self, sys_id): - """enable netboot""" + """enable netboot.""" self.remote_.modify_system( sys_id, 'netboot_enabled', True, self.token_) @@ -236,7 +247,7 @@ class Installer(os_installer.Installer): @classmethod def _clean_log(cls, system_name): - """clean log""" + """clean log.""" log_dir = os.path.join( setting.INSTALLATION_LOGDIR, system_name) @@ -246,7 +257,7 @@ class Installer(os_installer.Installer): self, hostid, config, **kwargs ): """clean host installing progress.""" - self._clean_log(self._get_system_name(config)) + self._clean_log(config['fullname']) def reinstall_host(self, hostid, config, **kwargs): """reinstall host.""" @@ -255,6 +266,7 @@ class Installer(os_installer.Installer): self.clean_host_installing_progress( hostid, config, **kwargs) self._netboot_enabled(sys_id) + self._save_system(sys_id) def update_host_config(self, hostid, config, **kwargs): """update host config.""" diff --git a/compass/config_management/providers/__init__.py b/compass/config_management/providers/__init__.py index 3c254d3e..f35c65f8 100644 --- a/compass/config_management/providers/__init__.py +++ b/compass/config_management/providers/__init__.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """modules to provider providers to read/write cluster/host config .. moduleauthor:: Xiaodong Wang @@ -9,7 +23,11 @@ __all__ = [ from compass.config_management.providers.config_provider import ( - get_provider, get_provider_by_name, register_provider) + get_provider) +from compass.config_management.providers.config_provider import ( + get_provider_by_name) +from compass.config_management.providers.config_provider import ( + register_provider) from compass.config_management.providers.plugins import db_config_provider from compass.config_management.providers.plugins import file_config_provider from compass.config_management.providers.plugins import mix_config_provider diff --git a/compass/config_management/providers/config_provider.py b/compass/config_management/providers/config_provider.py index ff4c4ce7..5d1ef89b 100644 --- a/compass/config_management/providers/config_provider.py +++ b/compass/config_management/providers/config_provider.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to provide interface to read/update global/cluster/host config. .. moduleauthor:: Xiaodong Wang diff --git a/compass/config_management/providers/plugins/__init__.py b/compass/config_management/providers/plugins/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/config_management/providers/plugins/__init__.py +++ b/compass/config_management/providers/plugins/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/config_management/providers/plugins/db_config_provider.py b/compass/config_management/providers/plugins/db_config_provider.py index 65fd5397..dd3f864c 100644 --- a/compass/config_management/providers/plugins/db_config_provider.py +++ b/compass/config_management/providers/plugins/db_config_provider.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to provide ConfigProvider that reads config from db. .. moduleauthor:: Xiaodong Wang @@ -8,15 +22,26 @@ import os.path from compass.config_management.providers import config_provider from compass.config_management.utils import config_filter from compass.db import database -from compass.db.model import Adapter, Role, SwitchConfig, Switch, Machine -from compass.db.model import Cluster, ClusterHost -from compass.db.model import ClusterState, HostState, LogProgressingHistory +from compass.db.model import Adapter +from compass.db.model import Cluster +from compass.db.model import ClusterHost +from compass.db.model import ClusterState +from compass.db.model import HostState +from compass.db.model import LogProgressingHistory +from compass.db.model import Machine +from compass.db.model import Role +from compass.db.model import Switch +from compass.db.model import SwitchConfig from compass.utils import setting_wrapper as setting -CLUSTER_ALLOWS = ['*'] +CLUSTER_ALLOWS = ['/security', '/networking', '/partition'] CLUSTER_DENIES = [] -HOST_ALLOWS = ['*'] +HOST_ALLOWS = [ + '/roles', + '/has_dashboard_roles', + '/networking/interfaces/*/ip' +] HOST_DENIES = [] @@ -86,7 +111,6 @@ class DBProvider(config_provider.ConfigProvider): if switch_filter_tuple in switch_filter_tuples: logging.debug('ignore adding switch filter: %s', switch_filter) - continue else: logging.debug('add switch filter: %s', switch_filter) @@ -112,7 +136,7 @@ class DBProvider(config_provider.ConfigProvider): log_dir = os.path.join( setting.INSTALLATION_LOGDIR, - '%s.%s' % (host.hostname, host.cluster_id), + host.fullname, '') session.query(LogProgressingHistory).filter( LogProgressingHistory.pathname.startswith( @@ -165,7 +189,7 @@ class DBProvider(config_provider.ConfigProvider): log_dir = os.path.join( setting.INSTALLATION_LOGDIR, - '%s.%s' % (host.hostname, host.cluster_id), + host.fullname, '') session.query(LogProgressingHistory).filter( LogProgressingHistory.pathname.startswith( @@ -186,20 +210,20 @@ class DBProvider(config_provider.ConfigProvider): id=clusterid).delete(synchronize_session='fetch') def get_cluster_hosts(self, clusterid): - """get cluster hosts""" + """get cluster hosts.""" session = database.current_session() hosts = session.query(ClusterHost).filter_by( cluster_id=clusterid).all() return [host.id for host in hosts] def get_clusters(self): - """get clusters""" + """get clusters.""" session = database.current_session() clusters = session.query(Cluster).all() return [cluster.id for cluster in clusters] def get_switch_and_machines(self): - """get switches and machines""" + """get switches and machines.""" session = database.current_session() switches = session.query(Switch).all() switches_data = [] @@ -224,7 +248,7 @@ class DBProvider(config_provider.ConfigProvider): def update_switch_and_machines( self, switches, switch_machines ): - """update switches and machines""" + """update switches and machines.""" session = database.current_session() session.query(Switch).delete(synchronize_session='fetch') session.query(Machine).delete(synchronize_session='fetch') diff --git a/compass/config_management/providers/plugins/file_config_provider.py b/compass/config_management/providers/plugins/file_config_provider.py index e36dc16f..c13e340a 100644 --- a/compass/config_management/providers/plugins/file_config_provider.py +++ b/compass/config_management/providers/plugins/file_config_provider.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """config provider read config from file. .. moduleauthor:: Xiaodong Wang diff --git a/compass/config_management/providers/plugins/mix_config_provider.py b/compass/config_management/providers/plugins/mix_config_provider.py index 5dd61291..e86f37b8 100644 --- a/compass/config_management/providers/plugins/mix_config_provider.py +++ b/compass/config_management/providers/plugins/mix_config_provider.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Mix provider which read config from different other providers. .. moduleauthor:: Xiaodong Wang @@ -49,7 +63,7 @@ class MixProvider(config_provider.ConfigProvider): adapters, roles_per_target_system) def update_switch_filters(self, switch_filters): - """update switch filters""" + """update switch filters.""" self.host_provider_.update_switch_filters(switch_filters) def clean_host_config(self, hostid): @@ -57,23 +71,23 @@ class MixProvider(config_provider.ConfigProvider): self.host_provider_.clean_host_config(hostid) def reinstall_host(self, hostid): - """reinstall host config""" + """reinstall host config.""" self.host_provider_.reinstall_host(hostid) def reinstall_cluster(self, clusterid): - """reinstall cluster""" + """reinstall cluster.""" self.host_provider_.reinstall_cluster(clusterid) def clean_host_installing_progress(self, hostid): - """clean host installing progress""" + """clean host installing progress.""" self.host_provider_.clean_host_installing_progress(hostid) def clean_cluster_installing_progress(self, clusterid): - """clean cluster installing progress""" + """clean cluster installing progress.""" self.host_provider_.clean_cluster_installing_progress(clusterid) def clean_cluster_config(self, clusterid): - """clean cluster config""" + """clean cluster config.""" self.host_provider_.clean_cluster_config(clusterid) def get_cluster_hosts(self, clusterid): @@ -81,7 +95,7 @@ class MixProvider(config_provider.ConfigProvider): return self.host_provider_.get_cluster_hosts(clusterid) def get_clusters(self): - """get clusters""" + """get clusters.""" return self.host_provider_.get_clusters() def get_switch_and_machines(self): diff --git a/compass/config_management/utils/__init__.py b/compass/config_management/utils/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/config_management/utils/__init__.py +++ b/compass/config_management/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/config_management/utils/config_filter.py b/compass/config_management/utils/config_filter.py index 45847919..15cae49c 100644 --- a/compass/config_management/utils/config_filter.py +++ b/compass/config_management/utils/config_filter.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to filter configuration when upddating. .. moduleauthor:: Xiaodong Wang @@ -8,7 +22,7 @@ from compass.config_management.utils import config_reference class ConfigFilter(object): - """config filter based on allows and denies rules""" + """config filter based on allows and denies rules.""" def __init__(self, allows=['*'], denies=[]): """Constructor @@ -27,7 +41,7 @@ class ConfigFilter(object): self.__class__.__name__, self.allows_, self.denies_) def _is_allows_valid(self): - """Check if allows are valid""" + """Check if allows are valid.""" if not isinstance(self.allows_, list): raise TypeError( 'allows type is %s but expected type is list: %s' % ( @@ -80,7 +94,9 @@ class ConfigFilter(object): if not allow: continue + logging.debug('filter by allow rule %s', allow) for sub_key, sub_ref in ref.ref_items(allow): + logging.debug('%s is added to filtered config', sub_key) filtered_ref.setdefault(sub_key).update(sub_ref.config) def _filter_denies(self, filtered_ref): @@ -89,5 +105,7 @@ class ConfigFilter(object): if not deny: continue + logging.debug('filter by deny rule %s', deny) for ref_key in filtered_ref.ref_keys(deny): + logging.debug('%s is removed from filtered config', ref_key) del filtered_ref[ref_key] diff --git a/compass/config_management/utils/config_manager.py b/compass/config_management/utils/config_manager.py index 027896ce..a2a10007 100644 --- a/compass/config_management/utils/config_manager.py +++ b/compass/config_management/utils/config_manager.py @@ -1,6 +1,18 @@ -""" -Module to get configs from provider and isntallers and update -them to provider and installers. +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module to get configs from provider and isntallers and update + them to provider and installers. .. moduleauthor:: Xiaodong wang ,xiaodongwang@huawei.com> """ @@ -9,9 +21,9 @@ import logging from compass.config_management import installers from compass.config_management import providers -from compass.config_management.utils import config_merger_callbacks from compass.config_management.utils.config_merger import ConfigMapping from compass.config_management.utils.config_merger import ConfigMerger +from compass.config_management.utils import config_merger_callbacks from compass.config_management.utils.config_reference import ConfigReference from compass.utils import setting_wrapper as setting from compass.utils import util @@ -40,6 +52,9 @@ CLUSTER_HOST_MERGER = ConfigMerger( to_key='/has_dashboard_roles', value=config_merger_callbacks.has_intersection ), + ConfigMapping( + path_list=['/role_mapping'], + ), ConfigMapping( path_list=[ '/networking/global/nameservers', @@ -88,10 +103,12 @@ CLUSTER_HOST_MERGER = ConfigMerger( class ConfigManager(object): - """ - Class is to get global/clsuter/host configs from provider, - os installer, package installer, process them, and - update them to provider, os installer, package installer. + """Class to get global/clsuter/host configs. + + .. note:: + The class is used to get global/clsuter/host configs + from provider, os installer, package installer, process them, + and update them to provider, os installer, package installer. """ def __init__(self): @@ -172,7 +189,7 @@ class ConfigManager(object): adapters, roles_per_target_system) def update_switch_filters(self): - """Update switch filter from setting.SWITCHES""" + """Update switch filter from setting.SWITCHES.""" if not hasattr(setting, 'SWITCHES'): logging.info('no switch configs to set') return @@ -183,7 +200,7 @@ class ConfigManager(object): self.config_provider_.update_switch_filters(switch_filters) def get_switch_and_machines(self): - """Get switches and machines""" + """Get switches and machines.""" switches, machines_per_switch = ( self.config_provider_.get_switch_and_machines()) logging.debug('got switches %s from %s', @@ -475,7 +492,7 @@ class ConfigManager(object): return hostids def get_clusters(self): - """get clusters""" + """get clusters.""" clusters = self.config_provider_.get_clusters() logging.debug('got clusters from %s: %s', self.config_provider_, clusters) diff --git a/compass/config_management/utils/config_merger.py b/compass/config_management/utils/config_merger.py index 6deda1f1..8a55e717 100644 --- a/compass/config_management/utils/config_merger.py +++ b/compass/config_management/utils/config_merger.py @@ -1,9 +1,23 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to set the hosts configs from cluster config. .. moduleauthor:: Xiaodong Wang """ +import copy import logging -from copy import deepcopy from compass.config_management.utils import config_reference from compass.utils import util @@ -173,14 +187,14 @@ class ConfigMapping(object): if self.value_ is None: lower_values = {} for lower_key in lower_sub_refs.keys(): - lower_values[lower_key] = deepcopy(sub_ref.config) + lower_values[lower_key] = copy.deepcopy(sub_ref.config) return lower_values if not callable(self.value_): lower_values = {} for lower_key in lower_sub_refs.keys(): - lower_values[lower_key] = deepcopy(self.value_) + lower_values[lower_key] = copy.deepcopy(self.value_) return lower_values diff --git a/compass/config_management/utils/config_merger_callbacks.py b/compass/config_management/utils/config_merger_callbacks.py index 789a3cb7..245b724f 100644 --- a/compass/config_management/utils/config_merger_callbacks.py +++ b/compass/config_management/utils/config_merger_callbacks.py @@ -1,11 +1,25 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ConfigMerger Callbacks module. .. moduleauthor:: Xiaodong Wang """ +import copy import itertools import logging -from copy import deepcopy -from netaddr import IPSet, IPRange +import netaddr from compass.utils import util @@ -58,7 +72,7 @@ def _get_bundled_exclusives(exclusives, bundle_mapping): def _get_max(lhs, rhs): - """Get max value""" + """Get max value.""" if lhs < 0: return lhs @@ -69,7 +83,7 @@ def _get_max(lhs, rhs): def _get_min(lhs, rhs): - """Get min value""" + """Get min value.""" if lhs < 0: return rhs @@ -79,6 +93,14 @@ def _get_min(lhs, rhs): return min(lhs, rhs) +def _dec_max_min(value): + """dec max and min value.""" + if value > 0: + return value - 1 + else: + return value + + def _get_bundled_max_mins(maxs, mins, default_max, default_min, role_bundles): """Get max and mins for each bundled role.""" bundled_maxs = {} @@ -119,10 +141,7 @@ def _get_bundled_max_mins(maxs, mins, default_max, default_min, role_bundles): def _update_assigned_roles(lower_refs, to_key, bundle_mapping, role_bundles, bundled_maxs, bundled_mins): - """ - Update bundled maxs/mins and get assign roles to each host, - unassigned host. - """ + """Update bundled maxs/mins and assign roles to each unassigned host.""" lower_roles = {} unassigned_hosts = [] for lower_key, lower_ref in lower_refs.items(): @@ -135,8 +154,11 @@ def _update_assigned_roles(lower_refs, to_key, bundle_mapping, bundled_roles.add(bundled_role) roles |= set(role_bundles[bundled_role]) for bundled_role in bundled_roles: - bundled_maxs[bundled_role] -= 1 - bundled_mins[bundled_role] -= 1 + bundled_maxs[bundled_role] = _dec_max_min( + bundled_maxs[bundled_role]) + bundled_mins[bundled_role] = _dec_max_min( + bundled_mins[bundled_role]) + lower_roles[lower_key] = list(roles) if not roles: unassigned_hosts.append(lower_key) @@ -158,8 +180,10 @@ def _update_exclusive_roles(bundled_exclusives, lower_roles, raise ValueError('no enough unassigned hosts for exlusive %s', bundled_exclusive) host = unassigned_hosts.pop(0) - bundled_mins[bundled_exclusive] -= 1 - bundled_maxs[bundled_exclusive] -= 1 + bundled_mins[bundled_exclusive] = _dec_max_min( + bundled_mins[bundled_exclusive]) + bundled_maxs[bundled_exclusive] = _dec_max_min( + bundled_maxs[bundled_exclusive]) lower_roles[host] = list(role_bundles[bundled_exclusive]) del role_bundles[bundled_exclusive] @@ -174,7 +198,7 @@ def _update_exclusive_roles(bundled_exclusives, lower_roles, def _assign_roles_by_mins(role_bundles, lower_roles, unassigned_hosts, bundled_maxs, bundled_mins): """Assign roles to hosts by min restriction.""" - available_hosts = deepcopy(unassigned_hosts) + available_hosts = copy.deepcopy(unassigned_hosts) for bundled_role, roles in role_bundles.items(): while bundled_mins[bundled_role] > 0: if not available_hosts: @@ -186,8 +210,10 @@ def _assign_roles_by_mins(role_bundles, lower_roles, unassigned_hosts, if host in unassigned_hosts: unassigned_hosts.remove(host) - bundled_mins[bundled_role] -= 1 - bundled_maxs[bundled_role] -= 1 + bundled_mins[bundled_role] = _dec_max_min( + bundled_mins[bundled_role]) + bundled_maxs[bundled_role] = _dec_max_min( + bundled_maxs[bundled_role]) lower_roles[host] = list(roles) logging.debug('assigned roles after assigning mins: %s', lower_roles) @@ -200,13 +226,14 @@ def _assign_roles_by_maxs(role_bundles, lower_roles, unassigned_hosts, bundled_maxs): """Assign roles to host by max restriction.""" available_lists = [] - default_roles = [] + default_roles_lists = [] for bundled_role in role_bundles.keys(): if bundled_maxs[bundled_role] > 0: available_lists.append( [bundled_role] * bundled_maxs[bundled_role]) - else: - default_roles.append(bundled_role) + elif bundled_maxs[bundled_role] < 0: + default_roles_lists.append( + [bundled_role] * (-bundled_maxs[bundled_role])) available_list = util.flat_lists_with_possibility(available_lists) @@ -221,6 +248,9 @@ def _assign_roles_by_maxs(role_bundles, lower_roles, unassigned_hosts, logging.debug('unassigned_hosts after assigning maxs: %s', unassigned_hosts) + default_roles = util.flat_lists_with_possibility( + default_roles_lists) + if default_roles: default_iter = itertools.cycle(default_roles) while unassigned_hosts: @@ -232,6 +262,22 @@ def _assign_roles_by_maxs(role_bundles, lower_roles, unassigned_hosts, logging.debug('unassigned hosts: %s', unassigned_hosts) +def _sort_roles(lower_roles, roles): + """Sort roles with the same order as in all roles.""" + for lower_key, lower_value in lower_roles.items(): + updated_roles = [] + for role in roles: + if role in lower_value: + updated_roles.append(role) + + for role in lower_value: + if role not in updated_roles: + logging.debug('found role %s not in roles %s', role, roles) + updated_roles.append(role) + + lower_roles[lower_key] = updated_roles + + def assign_roles(_upper_ref, _from_key, lower_refs, to_key, roles=[], maxs={}, mins={}, default_max=-1, default_min=0, exclusives=[], bundles=[], **_kwargs): @@ -258,6 +304,7 @@ def assign_roles(_upper_ref, _from_key, lower_refs, to_key, _assign_roles_by_maxs( role_bundles, lower_roles, unassigned_hosts, bundled_maxs) + _sort_roles(lower_roles, roles) return lower_roles @@ -267,7 +314,7 @@ def assign_roles_by_host_numbers(upper_ref, from_key, lower_refs, to_key, **_kwargs): """Assign roles by role assign policy.""" host_numbers = str(len(lower_refs)) - policy_kwargs = deepcopy(default) + policy_kwargs = copy.deepcopy(default) if host_numbers in policy_by_host_numbers: util.merge_dict(policy_kwargs, policy_by_host_numbers[host_numbers]) else: @@ -305,7 +352,7 @@ def assign_ips(_upper_ref, _from_key, lower_refs, to_key, return {} host_ips = {} unassigned_hosts = [] - ips = IPSet(IPRange(ip_start, ip_end)) + ips = netaddr.IPSet(netaddr.IPRange(ip_start, ip_end)) for lower_key, lower_ref in lower_refs.items(): ip_addr = lower_ref.get(to_key, '') if ip_addr: @@ -334,7 +381,7 @@ def assign_from_pattern(_upper_ref, _from_key, lower_refs, to_key, upper_configs[key] = kwargs[key] for lower_key, _ in lower_refs.items(): - group = deepcopy(upper_configs) + group = copy.deepcopy(upper_configs) for key in lower_keys: group[key] = kwargs[key][lower_key] @@ -353,7 +400,7 @@ def assign_noproxy(_upper_ref, _from_key, lower_refs, noproxy_pattern='', hostnames={}, ips={}, **_kwargs): """Assign no proxy to hosts.""" - no_proxy_list = deepcopy(default) + no_proxy_list = copy.deepcopy(default) for lower_key, _ in lower_refs.items(): mapping = { @@ -368,7 +415,7 @@ def assign_noproxy(_upper_ref, _from_key, lower_refs, lower_key, to_key, noproxy_pattern, mapping) raise error - no_proxy = ','.join(no_proxy_list) + no_proxy = ','.join([no_proxy for no_proxy in no_proxy_list if no_proxy]) host_no_proxy = {} for lower_key, _ in lower_refs.items(): host_no_proxy[lower_key] = no_proxy diff --git a/compass/config_management/utils/config_reference.py b/compass/config_management/utils/config_reference.py index c1f05f48..80ef2a1e 100644 --- a/compass/config_management/utils/config_reference.py +++ b/compass/config_management/utils/config_reference.py @@ -1,13 +1,26 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to provide util class to access item in nested dict easily. .. moduleauthor:: Xiaodong Wang """ +import copy import fnmatch import os.path import re -from copy import deepcopy - from compass.utils import util @@ -261,7 +274,7 @@ class ConfigReference(object): util.merge_dict(self.config, config, override) elif self.config is None or override: - self.config = deepcopy(config) + self.config = copy.deepcopy(config) else: return diff --git a/compass/config_management/utils/config_translator.py b/compass/config_management/utils/config_translator.py index 8f225bc1..3492c693 100644 --- a/compass/config_management/utils/config_translator.py +++ b/compass/config_management/utils/config_translator.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Config Translator module to translate orign config to dest config. .. moduleauthor:: Xiaodong Wang @@ -16,7 +30,7 @@ class KeyTranslator(object): """Constructor :param translated_keys: keys in dest ref to be translated to. - :type translated_keys: list of str + :type translated_keys: callable or list of (str or callable) :param from_keys: extra kwargs parsed to translated key callback. :type: from_keys: dict mapping name of kwargs to path in origin ref :param translated_value: value or callback to get translated value. @@ -48,6 +62,9 @@ class KeyTranslator(object): def _is_valid_translated_keys(self): """Check translated keys are valid.""" + if callable(self.translated_keys_): + return + for i, translated_key in enumerate(self.translated_keys_): if util.is_instance(translated_key, [str, unicode]): if '*' in translated_key: @@ -121,6 +138,11 @@ class KeyTranslator(object): logging.error('%s from_key %s missing in %s', self, from_key, sub_ref) + if callable(self.translated_keys_): + translated_keys = self.translated_keys_( + sub_ref, ref_key, **key_configs) + return translated_keys + translated_keys = [] for translated_key in self.translated_keys_: if callable(translated_key): diff --git a/compass/config_management/utils/config_translator_callbacks.py b/compass/config_management/utils/config_translator_callbacks.py index 140b64b4..0e8fca7b 100644 --- a/compass/config_management/utils/config_translator_callbacks.py +++ b/compass/config_management/utils/config_translator_callbacks.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """callback lib for config translator callbacks.""" import crypt import logging @@ -10,7 +24,7 @@ def get_key_from_pattern( _ref, path, from_pattern='.*', to_pattern='', **kwargs ): - """Get translated key from pattern""" + """Get translated key from pattern.""" match = re.match(from_pattern, path) if not match: return None @@ -25,11 +39,64 @@ def get_key_from_pattern( return translated_key +def get_keys_from_role_mapping(ref, _path, mapping={}, **_kwargs): + """get translated keys from roles.""" + roles = ref.config + translated_keys = [] + for role in roles: + if role not in mapping: + continue + + translated_keys.extend(mapping[role].keys()) + + logging.debug('got translated_keys %s from roles %s and mapping %s', + translated_keys, roles, mapping) + return translated_keys + + +def get_value_from_role_mapping( + ref, _path, _translated_ref, translated_path, + mapping={}, **_kwargs +): + """get translated value from roles and translated_path.""" + roles = ref.config + for role in roles: + if role not in mapping: + continue + + if translated_path not in mapping[role]: + continue + + value = mapping[role][translated_path] + translated_value = ref.get(value) + logging.debug('got translated_value %s from %s and roles %s', + translated_value, roles, translated_path) + return translated_value + + return None + + def get_encrypted_value(ref, _path, _translated_ref, _translated_path, crypt_method=None, **_kwargs): """Get encrypted value.""" if not crypt_method: - crypt_method = crypt.METHOD_MD5 + if hasattr(crypt, 'METHOD_MD5'): + crypt_method = crypt.METHOD_MD5 + else: + # for python2.7, copy python2.6 METHOD_MD5 logic here. + from random import choice + import string + + _saltchars = string.ascii_letters + string.digits + './' + + def _mksalt(): + """generate salt.""" + salt = '$1$' + salt += ''.join(choice(_saltchars) for _ in range(8)) + return salt + + crypt_method = _mksalt() + return crypt.crypt(ref.config, crypt_method) @@ -58,12 +125,12 @@ def add_value(ref, _path, translated_ref, def override_if_any(_ref, _path, _translated_ref, _translated_path, **kwargs): - """override if any kwargs is True""" + """override if any kwargs is True.""" return any(kwargs.values()) def override_if_all(_ref, _path, _translated_ref, _translated_path, **kwargs): - """override if all kwargs are True""" + """override if all kwargs are True.""" return all(kwargs.values()) diff --git a/compass/db/__init__.py b/compass/db/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/db/__init__.py +++ b/compass/db/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/db/database.py b/compass/db/database.py index 7447a4c3..377ffd40 100644 --- a/compass/db/database.py +++ b/compass/db/database.py @@ -1,13 +1,27 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Provider interface to manipulate database.""" import logging -from threading import local from contextlib import contextmanager from sqlalchemy import create_engine from sqlalchemy.orm import scoped_session, sessionmaker +from threading import local -from compass.utils import setting_wrapper as setting from compass.db import model +from compass.utils import setting_wrapper as setting ENGINE = create_engine(setting.SQLALCHEMY_DATABASE_URI, convert_unicode=True) @@ -39,9 +53,10 @@ def in_session(): @contextmanager def session(): - """ - database session scope. To operate database, it should be called in - database session. + """database session scope. + + .. note:: + To operate database, it should be called in database session. """ if hasattr(SESSION_HOLDER, 'session'): logging.error('we are already in session') @@ -78,7 +93,7 @@ def current_session(): def create_db(): - """Create database""" + """Create database.""" model.BASE.metadata.create_all(bind=ENGINE) diff --git a/compass/db/model.py b/compass/db/model.py index 4079068c..16ae492c 100644 --- a/compass/db/model.py +++ b/compass/db/model.py @@ -1,13 +1,30 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """database model.""" -from datetime import datetime -import simplejson as json import logging +import simplejson as json import uuid + +from datetime import datetime from sqlalchemy import Column, ColumnDefault, Integer, String from sqlalchemy import Float, Enum, DateTime, ForeignKey, Text, Boolean from sqlalchemy import UniqueConstraint from sqlalchemy.orm import relationship, backref from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.hybrid import hybrid_property + from compass.utils import util @@ -73,7 +90,7 @@ class Switch(BASE): return ''\ % (self.ip, self.credential, self.vendor, self.state) - @property + @hybrid_property def vendor(self): """vendor property getter""" return self.vendor_info @@ -129,9 +146,11 @@ class Switch(BASE): class Machine(BASE): - """ - Machine table.Note: currently, we are taking care of management plane. - Therefore, we assume one machine is connected to one switch. + """Machine table. + + .. note:: + currently, we are taking care of management plane. + Therefore, we assume one machine is connected to one switch. :param id: int, identity as primary key :param mac: string, the MAC address of the machine. @@ -199,16 +218,24 @@ class HostState(BASE): def __init__(self, **kwargs): super(HostState, self).__init__(**kwargs) - @property + @hybrid_property def hostname(self): """hostname getter""" return self.host.hostname + @hybrid_property + def fullname(self): + """fullname getter""" + return self.host.fullname + def __repr__(self): - return ('') % ( + return ( + '' + ) % ( self.hostname, self.state, self.progress, - self.message, self.severity) + self.message, self.severity + ) class ClusterState(BASE): @@ -244,16 +271,19 @@ class ClusterState(BASE): def __init__(self, **kwargs): super(ClusterState, self).__init__(**kwargs) - @property + @hybrid_property def clustername(self): """clustername getter""" return self.cluster.name def __repr__(self): - return ('') % ( + return ( + '' + ) % ( self.clustername, self.state, self.progress, - self.message, self.severity) + self.message, self.severity + ) class Cluster(BASE): @@ -285,9 +315,8 @@ class Cluster(BASE): def __init__(self, **kwargs): if 'name' not in kwargs or not kwargs['name']: - self.name = str(uuid.uuid4()) - if 'name' in kwargs: - del kwargs['name'] + kwargs['name'] = str(uuid.uuid4()) + super(Cluster, self).__init__(**kwargs) def __repr__(self): @@ -367,7 +396,7 @@ class Cluster(BASE): @networking.setter def networking(self, value): - """networking setter""" + """networking setter.""" logging.debug('cluster %s set networking %s', self.id, value) if value: try: @@ -380,9 +409,9 @@ class Cluster(BASE): else: self.networking_config = None - @property + @hybrid_property def config(self): - """get config from security, networking, partition""" + """get config from security, networking, partition.""" config = {} if self.raw_config: try: @@ -462,14 +491,17 @@ class ClusterHost(BASE): def __init__(self, **kwargs): if 'hostname' not in kwargs or not kwargs['hostname']: - self.hostname = str(uuid.uuid4()) - if 'hostname' in kwargs: - del kwargs['hostname'] + kwargs['hostname'] = str(uuid.uuid4()) + super(ClusterHost, self).__init__(**kwargs) def __repr__(self): - return ''\ - % (self.hostname, self.cluster, self.machine) + return '' % ( + self.hostname, self.cluster, self.machine) + + @hybrid_property + def fullname(self): + return '%s.%s' % (self.hostname, self.cluster.id) @property def config(self): @@ -479,10 +511,16 @@ class ClusterHost(BASE): if self.config_data: config.update(json.loads(self.config_data)) - config.update({'hostid': self.id, 'hostname': self.hostname}) + config.update({ + 'hostid': self.id, + 'hostname': self.hostname, + }) if self.cluster: - config.update({'clusterid': self.cluster.id, - 'clustername': self.cluster.name}) + config.update({ + 'clusterid': self.cluster.id, + 'clustername': self.cluster.name, + 'fullname': self.fullname, + }) if self.machine: util.merge_dict( @@ -560,14 +598,17 @@ class LogProgressingHistory(BASE): super(LogProgressingHistory, self).__init__(**kwargs) def __repr__(self): - return ('LogProgressingHistory[%r: position %r,' - 'partial_line %r,progress %r,message %r,' - 'severity %r]') % ( + return ( + 'LogProgressingHistory[%r: position %r,' + 'partial_line %r,progress %r,message %r,' + 'severity %r]' + ) % ( self.pathname, self.position, self.partial_line, self.progress, self.message, - self.severity) + self.severity + ) class Adapter(BASE): @@ -592,13 +633,15 @@ class Adapter(BASE): def __repr__(self): return '' % ( - self.name, self.os, self.target_system) + self.name, self.os, self.target_system + ) class Role(BASE): - """ - The Role table stores avaiable roles of one target system - where the host can be deployed to one or several roles in the cluster. + """The Role table stores avaiable roles of one target system. + + .. note:: + the host can be deployed to one or several roles in the cluster. :param id: int, identity as primary key. :param name: role name. diff --git a/compass/hdsdiscovery/__init__.py b/compass/hdsdiscovery/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/hdsdiscovery/__init__.py +++ b/compass/hdsdiscovery/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/hdsdiscovery/base.py b/compass/hdsdiscovery/base.py index 8729e9ed..9dd5a9ed 100644 --- a/compass/hdsdiscovery/base.py +++ b/compass/hdsdiscovery/base.py @@ -1,18 +1,32 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ Base class extended by specific vendor in vendors directory. A vendor needs to implement abstract methods of base class. """ -import re import logging +import re from abc import ABCMeta -from compass.hdsdiscovery import utils from compass.hdsdiscovery.error import TimeoutError +from compass.hdsdiscovery import utils class BaseVendor(object): - """Basic Vendor object""" + """Basic Vendor object.""" __metaclass__ = ABCMeta def is_this_vendor(self, host, credential, sys_info, **kwargs): @@ -23,8 +37,11 @@ class BaseVendor(object): class BaseSnmpVendor(BaseVendor): - """Base SNMP-based vendor plugin. It uses MIB-II sysDescr value - to determine the vendor of the switch. """ + """Base SNMP-based vendor plugin. + + .. note:: + It uses MIB-II sysDescr value to determine the vendor of the switch. + """ def __init__(self, matched_names): super(BaseSnmpVendor, self).__init__() @@ -57,15 +74,15 @@ class BasePlugin(object): # At least one of these three functions below must be implemented. def scan(self, **kwargs): - """Get multiple records at once""" + """Get multiple records at once.""" pass def set(self, **kwargs): - """Set value to desired variable""" + """Set value to desired variable.""" pass def get(self, **kwargs): - """Get one record from a host""" + """Get one record from a host.""" pass @@ -82,10 +99,12 @@ class BaseSnmpMacPlugin(BasePlugin): self.vlan_oid = vlan_oid def process_data(self, oper='SCAN', **kwargs): + """progress data.""" func_name = oper.lower() return getattr(self, func_name)(**kwargs) def scan(self, **kwargs): + """scan.""" results = None try: results = utils.snmpwalk_by_cl(self.host, self.credential, @@ -109,7 +128,7 @@ class BaseSnmpMacPlugin(BasePlugin): return mac_list def get_vlan_id(self, port): - """Get vlan Id""" + """Get vlan Id.""" if not port: return None @@ -127,7 +146,7 @@ class BaseSnmpMacPlugin(BasePlugin): return vlan_id def get_port(self, if_index): - """Get port number""" + """Get port number.""" if_name = '.'.join((self.port_oid, if_index)) result = None @@ -143,12 +162,12 @@ class BaseSnmpMacPlugin(BasePlugin): return port def convert_to_hex(self, value): - """Convert the integer from decimal to hex""" + """Convert the integer from decimal to hex.""" return "%0.2x" % int(value) def get_mac_address(self, mac_numbers): - """Assemble mac address from the list""" + """Assemble mac address from the list.""" if len(mac_numbers) != 6: logging.error("[PluginMac:get_mac_address] MAC address must be " "6 digitals") diff --git a/compass/hdsdiscovery/error.py b/compass/hdsdiscovery/error.py index 2fb66867..5bcf1a2c 100644 --- a/compass/hdsdiscovery/error.py +++ b/compass/hdsdiscovery/error.py @@ -1,4 +1,18 @@ -"""hdsdiscovery module errors""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""hdsdiscovery module errors.""" class TimeoutError(Exception): diff --git a/compass/hdsdiscovery/hdmanager.py b/compass/hdsdiscovery/hdmanager.py index f5f579e6..94a030df 100644 --- a/compass/hdsdiscovery/hdmanager.py +++ b/compass/hdsdiscovery/hdmanager.py @@ -1,10 +1,25 @@ -"""Manage hdsdiscovery functionalities""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Manage hdsdiscovery functionalities.""" +import logging import os import re -import logging -from compass.hdsdiscovery import utils from compass.hdsdiscovery.error import TimeoutError +from compass.hdsdiscovery import utils + UNREACHABLE = 'unreachable' NOTSUPPORTED = 'notsupported' @@ -38,14 +53,14 @@ class HDManager(object): plugin = utils.load_module(req_obj, plugin_dir, host, credential) if not plugin: # No plugin found! - #TODO add more code to catch excpetion or unexpected state + # TODO(Grace): add more code to catch excpetion or unexpected state logging.error('no plugin %s to load from %s', req_obj, plugin_dir) return None return plugin.process_data(oper, **kwargs) def is_valid_vendor(self, host, credential, vendor): - """ Check if vendor is associated with this host and credential + """Check if vendor is associated with this host and credential :param host: switch ip :param credential: credential to access switch @@ -76,7 +91,7 @@ class HDManager(object): return False def get_vendor(self, host, credential): - """ Check and get vendor of the switch. + """Check and get vendor of the switch. :param host: switch ip: :param credential: credential to access switch @@ -131,7 +146,7 @@ class HDManager(object): return (vendor, "Found", "") def get_sys_info(self, host, credential): - """get sys info""" + """get sys info.""" sys_info = None try: sys_info = utils.snmpget_by_cl(host, diff --git a/compass/hdsdiscovery/utils.py b/compass/hdsdiscovery/utils.py index 56d8ebc2..eff87eee 100644 --- a/compass/hdsdiscovery/utils.py +++ b/compass/hdsdiscovery/utils.py @@ -1,16 +1,30 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Utility functions Including functions of get/getbulk/walk/set of snmp for three versions """ import imp -import re import logging +import re import subprocess from compass.hdsdiscovery.error import TimeoutError def load_module(mod_name, path, host=None, credential=None): - """ Load a module instance. + """Load a module instance. :param str mod_name: module name :param str path: directory of the module @@ -75,7 +89,7 @@ def ssh_remote_execute(host, username, password, cmd): def valid_ip_format(ip_address): - """Valid the format of an Ip address""" + """Valid the format of an Ip address.""" if not re.match(r'^((([0-2]?\d{0,2}\.){3}([0-2]?\d{0,2}))' r'|(([\da-fA-F]{1,4}:){7}([\da-fA-F]{1,4})))$', @@ -89,9 +103,11 @@ def valid_ip_format(ip_address): # Implement snmpwalk and snmpget funtionality # The structure of returned dictionary will by tag/iid/value/type ################################################################# -AUTH_VERSIONS = {'1': 1, - '2c': 2, - '3': 3} +AUTH_VERSIONS = { + '1': 1, + '2c': 2, + '3': 3 +} def snmp_walk(host, credential, *args, **kwargs): diff --git a/compass/hdsdiscovery/vendors/__init__.py b/compass/hdsdiscovery/vendors/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/hdsdiscovery/vendors/__init__.py +++ b/compass/hdsdiscovery/vendors/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/hdsdiscovery/vendors/hp/__init__.py b/compass/hdsdiscovery/vendors/hp/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/hdsdiscovery/vendors/hp/__init__.py +++ b/compass/hdsdiscovery/vendors/hp/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/hdsdiscovery/vendors/hp/hp.py b/compass/hdsdiscovery/vendors/hp/hp.py index eaa43976..0fc53534 100644 --- a/compass/hdsdiscovery/vendors/hp/hp.py +++ b/compass/hdsdiscovery/vendors/hp/hp.py @@ -1,4 +1,18 @@ -"""Vendor: HP""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Vendor: HP.""" from compass.hdsdiscovery import base @@ -7,7 +21,7 @@ CLASS_NAME = 'Hp' class Hp(base.BaseSnmpVendor): - """Hp switch object""" + """Hp switch object.""" def __init__(self): base.BaseSnmpVendor.__init__(self, ['hp', 'procurve']) @@ -15,5 +29,5 @@ class Hp(base.BaseSnmpVendor): @property def name(self): - """Get 'name' proptery""" + """Get 'name' proptery.""" return self.names[0] diff --git a/compass/hdsdiscovery/vendors/hp/plugins/__init__.py b/compass/hdsdiscovery/vendors/hp/plugins/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/hdsdiscovery/vendors/hp/plugins/__init__.py +++ b/compass/hdsdiscovery/vendors/hp/plugins/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/hdsdiscovery/vendors/hp/plugins/mac.py b/compass/hdsdiscovery/vendors/hp/plugins/mac.py index fd4c699a..3bc81f46 100644 --- a/compass/hdsdiscovery/vendors/hp/plugins/mac.py +++ b/compass/hdsdiscovery/vendors/hp/plugins/mac.py @@ -1,9 +1,23 @@ -"""HP Switch Mac module""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""HP Switch Mac module.""" from compass.hdsdiscovery.base import BaseSnmpMacPlugin CLASS_NAME = 'Mac' class Mac(BaseSnmpMacPlugin): - """Process MAC address by HP switch""" + """Process MAC address by HP switch.""" pass diff --git a/compass/hdsdiscovery/vendors/huawei/__init__.py b/compass/hdsdiscovery/vendors/huawei/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/hdsdiscovery/vendors/huawei/__init__.py +++ b/compass/hdsdiscovery/vendors/huawei/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/hdsdiscovery/vendors/huawei/huawei.py b/compass/hdsdiscovery/vendors/huawei/huawei.py index d840fb18..1be789b8 100644 --- a/compass/hdsdiscovery/vendors/huawei/huawei.py +++ b/compass/hdsdiscovery/vendors/huawei/huawei.py @@ -1,4 +1,18 @@ -"""Huawei Switch""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Huawei Switch.""" from compass.hdsdiscovery import base @@ -7,7 +21,7 @@ CLASS_NAME = "Huawei" class Huawei(base.BaseSnmpVendor): - """Huawei switch""" + """Huawei switch.""" def __init__(self): base.BaseSnmpVendor.__init__(self, ["huawei"]) @@ -15,5 +29,5 @@ class Huawei(base.BaseSnmpVendor): @property def name(self): - """Return switch name""" + """Return switch name.""" return self.__name diff --git a/compass/hdsdiscovery/vendors/huawei/plugins/__init__.py b/compass/hdsdiscovery/vendors/huawei/plugins/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/hdsdiscovery/vendors/huawei/plugins/__init__.py +++ b/compass/hdsdiscovery/vendors/huawei/plugins/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/hdsdiscovery/vendors/huawei/plugins/mac.py b/compass/hdsdiscovery/vendors/huawei/plugins/mac.py index 68c82a4a..49d3863f 100644 --- a/compass/hdsdiscovery/vendors/huawei/plugins/mac.py +++ b/compass/hdsdiscovery/vendors/huawei/plugins/mac.py @@ -1,22 +1,41 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Huawei Switch Mac module.""" import logging -from compass.hdsdiscovery import utils + from compass.hdsdiscovery.base import BaseSnmpMacPlugin +from compass.hdsdiscovery import utils CLASS_NAME = "Mac" class Mac(BaseSnmpMacPlugin): - """Processes MAC address""" + """Processes MAC address.""" def __init__(self, host, credential): - super(Mac, self).__init__(host, credential, - 'HUAWEI-L2MAM-MIB::hwDynFdbPort') + super(Mac, self).__init__( + host, credential, + 'HUAWEI-L2MAM-MIB::hwDynFdbPort') def scan(self): - """ - Implemnets the scan method in BasePlugin class. In this mac module, - mac addesses were retrieved by snmpwalk commandline. + """Implemnets the scan method in BasePlugin class. + + .. note:: + In this mac module, mac addesses were retrieved by + snmpwalk commandline. """ results = utils.snmpwalk_by_cl(self.host, self.credential, self.oid) diff --git a/compass/hdsdiscovery/vendors/ovswitch/__init__.py b/compass/hdsdiscovery/vendors/ovswitch/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/hdsdiscovery/vendors/ovswitch/__init__.py +++ b/compass/hdsdiscovery/vendors/ovswitch/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/hdsdiscovery/vendors/ovswitch/ovswitch.py b/compass/hdsdiscovery/vendors/ovswitch/ovswitch.py index 91ebc105..d3ef3c26 100644 --- a/compass/hdsdiscovery/vendors/ovswitch/ovswitch.py +++ b/compass/hdsdiscovery/vendors/ovswitch/ovswitch.py @@ -1,6 +1,20 @@ -"""Open Vswitch module""" -import re +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Open Vswitch module.""" import logging +import re from compass.hdsdiscovery import base from compass.hdsdiscovery import utils @@ -11,7 +25,7 @@ CLASS_NAME = "OVSwitch" class OVSwitch(base.BaseVendor): - """Open Vswitch""" + """Open Vswitch.""" def __init__(self): self.__name = "Open vSwitch" @@ -57,5 +71,5 @@ class OVSwitch(base.BaseVendor): @property def name(self): - """Open Vswitch name""" + """Open Vswitch name.""" return self.__name diff --git a/compass/hdsdiscovery/vendors/ovswitch/plugins/__init__.py b/compass/hdsdiscovery/vendors/ovswitch/plugins/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/hdsdiscovery/vendors/ovswitch/plugins/__init__.py +++ b/compass/hdsdiscovery/vendors/ovswitch/plugins/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/hdsdiscovery/vendors/ovswitch/plugins/mac.py b/compass/hdsdiscovery/vendors/ovswitch/plugins/mac.py index 3c92518e..5f497a0d 100644 --- a/compass/hdsdiscovery/vendors/ovswitch/plugins/mac.py +++ b/compass/hdsdiscovery/vendors/ovswitch/plugins/mac.py @@ -1,15 +1,29 @@ -"""Open Vswitch Mac address module""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Open Vswitch Mac address module.""" import logging -from compass.hdsdiscovery import utils from compass.hdsdiscovery import base +from compass.hdsdiscovery import utils CLASS_NAME = "Mac" class Mac(base.BasePlugin): - """Open Vswitch MAC address module""" + """Open Vswitch MAC address module.""" def __init__(self, host, credential): self.host = host self.credential = credential @@ -23,9 +37,10 @@ class Mac(base.BasePlugin): return getattr(self, func_name)(**kwargs) def scan(self, **kwargs): - """ - Implemnets the scan method in BasePlugin class. In this module, - mac addesses were retrieved by ssh + """Implemnets the scan method in BasePlugin class. + + .. note:: + In this module, mac addesses were retrieved by ssh. """ try: user = self.credential['username'] @@ -47,7 +62,8 @@ class Mac(base.BasePlugin): output = None try: output = utils.ssh_remote_execute(self.host, user, pwd, cmd) - except: + except Exception as error: + logging.exception(error) return None logging.debug("[scan][output] output is %s", output) @@ -60,9 +76,12 @@ class Mac(base.BasePlugin): for line in output: if not line or line == '\n': continue + values_arr = line.split() temp = {} for field, value in zip(fields_arr, values_arr): temp[field] = value + result.append(temp.copy()) + return result diff --git a/compass/hdsdiscovery/vendors/pica8/__init__.py b/compass/hdsdiscovery/vendors/pica8/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/hdsdiscovery/vendors/pica8/__init__.py +++ b/compass/hdsdiscovery/vendors/pica8/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/hdsdiscovery/vendors/pica8/pica8.py b/compass/hdsdiscovery/vendors/pica8/pica8.py index 1cde9964..d0c9459a 100644 --- a/compass/hdsdiscovery/vendors/pica8/pica8.py +++ b/compass/hdsdiscovery/vendors/pica8/pica8.py @@ -1,4 +1,18 @@ -"""Vendor: Pica8""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Vendor: Pica8.""" from compass.hdsdiscovery import base @@ -7,7 +21,7 @@ CLASS_NAME = 'Pica8' class Pica8(base.BaseSnmpVendor): - """Pica8 switch object""" + """Pica8 switch object.""" def __init__(self): base.BaseSnmpVendor.__init__(self, ['pica8']) @@ -15,5 +29,5 @@ class Pica8(base.BaseSnmpVendor): @property def name(self): - """Get 'name' proptery""" + """Get 'name' proptery.""" return self._name diff --git a/compass/hdsdiscovery/vendors/pica8/plugins/__init__.py b/compass/hdsdiscovery/vendors/pica8/plugins/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/hdsdiscovery/vendors/pica8/plugins/__init__.py +++ b/compass/hdsdiscovery/vendors/pica8/plugins/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/hdsdiscovery/vendors/pica8/plugins/mac.py b/compass/hdsdiscovery/vendors/pica8/plugins/mac.py index 1ff5f88f..d5ccfc09 100644 --- a/compass/hdsdiscovery/vendors/pica8/plugins/mac.py +++ b/compass/hdsdiscovery/vendors/pica8/plugins/mac.py @@ -1,4 +1,18 @@ -"""Pica8 Switch Mac module""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pica8 Switch Mac module.""" from compass.hdsdiscovery.base import BaseSnmpMacPlugin @@ -6,5 +20,5 @@ CLASS_NAME = 'Mac' class Mac(BaseSnmpMacPlugin): - """Process MAC address by Pica8 switch""" + """Process MAC address by Pica8 switch.""" pass diff --git a/compass/log_analyzor/__init__.py b/compass/log_analyzor/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/log_analyzor/__init__.py +++ b/compass/log_analyzor/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/log_analyzor/adapter_matcher.py b/compass/log_analyzor/adapter_matcher.py index d868a6c2..3d6bb522 100644 --- a/compass/log_analyzor/adapter_matcher.py +++ b/compass/log_analyzor/adapter_matcher.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to provider installing progress calculation for the adapter. .. moduleauthor:: Xiaodong Wang @@ -6,7 +20,8 @@ import logging import re from compass.db import database -from compass.db.model import Cluster, ClusterHost +from compass.db.model import Cluster +from compass.db.model import ClusterHost from compass.log_analyzor.line_matcher import Progress @@ -31,17 +46,15 @@ class AdapterItemMatcher(object): self.__class__.__name__, self.file_matchers_, self.min_progress_, self.max_progress_) - def update_progress(self, hostname, clusterid, progress): + def update_progress(self, fullname, progress): """Update progress. - :param hostname: the hostname of the installing host. - :type hostname: str - :param clusterid: the cluster id of the installing host. - :type clusterid: int + :param fullname: the fullname of the installing host. + :type fullname: str :param progress: Progress instance to update. """ for file_matcher in self.file_matchers_: - file_matcher.update_progress(hostname, clusterid, progress) + file_matcher.update_progress(fullname, progress) class OSMatcher(object): @@ -72,9 +85,9 @@ class OSMatcher(object): self.name_ == os_installer_name, self.os_regex_.match(os_name)]) - def update_progress(self, hostname, clusterid, progress): + def update_progress(self, fullname, progress): """Update progress.""" - self.matcher_.update_progress(hostname, clusterid, progress) + self.matcher_.update_progress(fullname, progress) class PackageMatcher(object): @@ -105,9 +118,9 @@ class PackageMatcher(object): self.name_ == package_installer_name, self.target_system_ == target_system]) - def update_progress(self, hostname, clusterid, progress): + def update_progress(self, fullname, progress): """Update progress.""" - self.matcher_.update_progress(hostname, clusterid, progress) + self.matcher_.update_progress(fullname, progress) class AdapterMatcher(object): @@ -155,8 +168,8 @@ class AdapterMatcher(object): """ session = database.current_session() host = session.query( - ClusterHost).filter_by( - id=hostid).first() + ClusterHost + ).filter_by(id=hostid).first() if not host: logging.error( 'there is no host for %s in ClusterHost', hostid) @@ -165,10 +178,10 @@ class AdapterMatcher(object): if not host.state: logging.error('there is no related HostState for %s', hostid) - return host.hostname, None, None + return host.fullname, None, None return ( - host.hostname, + host.fullname, host.state.state, Progress(host.state.progress, host.state.message, @@ -310,32 +323,32 @@ class AdapterMatcher(object): host_progresses = {} with database.session(): for hostid in hostids: - hostname, host_state, host_progress = ( + fullname, host_state, host_progress = ( self._get_host_progress(hostid)) - if not hostname or not host_progress: + if not fullname or not host_progress: logging.error( - 'nothing to update host %s => hostname %s ' + 'nothing to update host %s => ' 'state %s progress %s', - hostid, hostname, host_state, host_progress) + fullname, host_state, host_progress) continue - logging.debug('got host %s hostname %s state %s progress %s', - hostid, hostname, host_state, host_progress) + logging.debug('got host %s state %s progress %s', + fullname, host_state, host_progress) host_progresses[hostid] = ( - hostname, host_state, host_progress) + fullname, host_state, host_progress) for hostid, host_value in host_progresses.items(): - hostname, host_state, host_progress = host_value + fullname, host_state, host_progress = host_value if host_state == 'INSTALLING' and host_progress.progress < 1.0: self.os_matcher_.update_progress( - hostname, clusterid, host_progress) + fullname, host_progress) self.package_matcher_.update_progress( - hostname, clusterid, host_progress) + fullname, host_progress) else: logging.error( 'there is no need to update host %s ' - 'progress: hostname %s state %s progress %s', - hostid, hostname, host_state, host_progress) + 'progress: state %s progress %s', + fullname, host_state, host_progress) with database.session(): for hostid in hostids: diff --git a/compass/log_analyzor/file_matcher.py b/compass/log_analyzor/file_matcher.py index a79b9876..82514e4d 100644 --- a/compass/log_analyzor/file_matcher.py +++ b/compass/log_analyzor/file_matcher.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to update intalling progress by processing log file. .. moduleauthor:: Xiaodong Wang @@ -25,7 +39,7 @@ class FileFilter(object): class CompositeFileFilter(FileFilter): - """filter log file based on the list of filters""" + """filter log file based on the list of filters.""" def __init__(self, filters): self.filters_ = filters @@ -97,8 +111,10 @@ class FileReader(object): """ with database.session() as session: history = session.query( - LogProgressingHistory).filter_by( - pathname=self.pathname_).first() + LogProgressingHistory + ).filter_by( + pathname=self.pathname_ + ).first() if history: self.position_ = history.position self.partial_line_ = history.partial_line @@ -198,18 +214,15 @@ class FileReaderFactory(object): return '%s[logdir: %s filefilter: %s]' % ( self.__class__.__name__, self.logdir_, self.filefilter_) - def get_file_reader(self, hostname, clusterid, filename): + def get_file_reader(self, fullname, filename): """Get FileReader instance. - :param hostname: hostname of installing host. - :param clusterid: cluster id of the installing host. + :param fullname: fullname of installing host. :param filename: the filename of the log file. :returns: :class:`FileReader` instance if it is not filtered. """ - pathname = os.path.join( - self.logdir_, '%s.%s' % (hostname, clusterid), - filename) + pathname = os.path.join(self.logdir_, fullname, filename) logging.debug('get FileReader from %s', pathname) if not self.filefilter_.filter(pathname): logging.error('%s is filtered', pathname) @@ -223,10 +236,7 @@ FILE_READER_FACTORY = FileReaderFactory( class FileMatcher(object): - """ - File matcher the get the lastest installing progress - from the log file. - """ + """File matcher to get the installing progress from the log file.""" def __init__(self, line_matchers, min_progress, max_progress, filename): if not 0.0 <= min_progress <= max_progress <= 1.0: raise IndexError( @@ -272,10 +282,13 @@ class FileMatcher(object): return total_progress_data = min( - self.absolute_min_progress_ - + - file_progress.progress * self.absolute_progress_diff_, - self.absolute_max_progress_) + ( + self.absolute_min_progress_ + ( + file_progress.progress * self.absolute_progress_diff_ + ) + ), + self.absolute_max_progress_ + ) # total progress should only be updated when the new calculated # progress is greater than the recored total progress or the @@ -296,13 +309,11 @@ class FileMatcher(object): 'ignore update file %s progress %s to total progress %s', self.filename_, file_progress, total_progress) - def update_progress(self, hostname, clusterid, total_progress): + def update_progress(self, fullname, total_progress): """update progress from file. - :param hostname: the hostname of the installing host. - :type hostname: str - :param clusterid: the cluster id of the installing host. - :type clusterid: int + :param fullname: the fullname of the installing host. + :type fullname: str :param total_progress: Progress instance to update. the function update installing progress by reading the log file. @@ -315,7 +326,7 @@ class FileMatcher(object): no line end indicator for the last line of the file. """ file_reader = FILE_READER_FACTORY.get_file_reader( - hostname, clusterid, self.filename_) + fullname, self.filename_) if not file_reader: return diff --git a/compass/log_analyzor/line_matcher.py b/compass/log_analyzor/line_matcher.py index d72b0a30..81305792 100644 --- a/compass/log_analyzor/line_matcher.py +++ b/compass/log_analyzor/line_matcher.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to get the progress when found match with a line of the log.""" import logging import re @@ -39,9 +53,7 @@ class ProgressCalculator(object): cls, progress_data, message, severity, progress ): - """ - Update progress with the given progress_data, - message and severity. + """Update progress with the given progress_data, message and severity. :param progress_data: installing progress. :type progress_data: float between 0 to 1. diff --git a/compass/log_analyzor/progress_calculator.py b/compass/log_analyzor/progress_calculator.py index bdbe3e0f..55746b9c 100644 --- a/compass/log_analyzor/progress_calculator.py +++ b/compass/log_analyzor/progress_calculator.py @@ -1,15 +1,30 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """module to provide updating installing process function. .. moduleauthor:: Xiaodong Wang """ import logging -from compass.log_analyzor.line_matcher import LineMatcher, IncrementalProgress -from compass.log_analyzor.file_matcher import FileMatcher -from compass.log_analyzor.adapter_matcher import AdapterMatcher from compass.log_analyzor.adapter_matcher import AdapterItemMatcher +from compass.log_analyzor.adapter_matcher import AdapterMatcher from compass.log_analyzor.adapter_matcher import OSMatcher from compass.log_analyzor.adapter_matcher import PackageMatcher +from compass.log_analyzor.file_matcher import FileMatcher +from compass.log_analyzor.line_matcher import IncrementalProgress +from compass.log_analyzor.line_matcher import LineMatcher # TODO(weidong): reconsider intialization method for the following. diff --git a/compass/tasks/__init__.py b/compass/tasks/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/tasks/__init__.py +++ b/compass/tasks/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tasks/client.py b/compass/tasks/client.py index f8b9d179..ca7ad142 100644 --- a/compass/tasks/client.py +++ b/compass/tasks/client.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to setup celery client. .. moduleauthor:: Xiaodong Wang diff --git a/compass/tasks/tasks.py b/compass/tasks/tasks.py index c9e971ff..1448b919 100644 --- a/compass/tasks/tasks.py +++ b/compass/tasks/tasks.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to define celery tasks. .. moduleauthor:: Xiaodong Wang @@ -8,8 +22,8 @@ from compass.actions import clean_deployment from compass.actions import clean_installing_progress from compass.actions import deploy from compass.actions import poll_switch -from compass.actions import update_progress from compass.actions import reinstall +from compass.actions import update_progress from compass.tasks.client import celery from compass.utils import flags from compass.utils import logsetting diff --git a/compass/tests/__init__.py b/compass/tests/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/tests/__init__.py +++ b/compass/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/actions/__init__.py b/compass/tests/actions/__init__.py new file mode 100644 index 00000000..4ee55a4c --- /dev/null +++ b/compass/tests/actions/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/actions/data/test1 b/compass/tests/actions/data/test1 index 607ca607..47cefd60 100644 --- a/compass/tests/actions/data/test1 +++ b/compass/tests/actions/data/test1 @@ -1,3 +1,5 @@ +import simplejson as json + ADAPTERS = [ {'name': 'CentOS_openstack', 'os': 'CentOS', 'target_system': 'openstack'}, ] @@ -7,7 +9,7 @@ ROLES = [ {'name': 'os-compute', 'target_system': 'openstack'}, ] SWITCHES = [ - {'ip': '1.2.3.4', 'vendor': 'huawei', 'credential': {'version': 'v2c', 'community': 'public'}}, + {'ip': '1.2.3.4', 'vendor_info': 'huawei', 'credential_data': json.dumps({'version': 'v2c', 'community': 'public'})}, ] MACHINES_BY_SWITCH = { '1.2.3.4': [ @@ -19,7 +21,7 @@ CLUSTERS = [ 'name': 'cluster1', 'adapter': 'CentOS_openstack', 'mutable': False, - 'security': { + 'security_config': json.dumps({ 'server_credentials': { 'username': 'root', 'password': 'huawei' }, @@ -29,8 +31,8 @@ CLUSTERS = [ 'console_credentials': { 'username': 'admin', 'password': 'huawei' } - }, - 'networking': { + }), + 'networking_config': json.dumps({ 'interfaces': { 'management': { 'nic': 'eth0', @@ -72,8 +74,8 @@ CLUSTERS = [ 'search_path': 'ods.com', 'gateway': '10.145.88.1' }, - }, - 'partition': '/home 20%%;/tmp 10%%;/var 30%%;', + }), + 'partition_config': json.dumps('/home 20%%;/tmp 10%%;/var 30%%;'), }, ] HOSTS_BY_CLUSTER = { @@ -82,8 +84,8 @@ HOSTS_BY_CLUSTER = { 'hostname': 'host1', 'mac': '00:00:01:02:03:04', 'mutable': False, - 'config': { - 'networking': { + 'config_data': json.dumps({ + 'netwoon.dumps(king': { 'interfaces': { 'management': { 'ip': '192.168.20.100', @@ -91,7 +93,7 @@ HOSTS_BY_CLUSTER = { }, }, 'roles': ["os-single-controller", "os-network", "os-compute"], - }, + }), }, ], } @@ -109,6 +111,29 @@ chef_MOCK = { 'os-network': 'openstack network node', 'os-compute-worker': 'openstack nova node' }, + 'role_mapping': { + 'os-single-controller': { + '/db/mysql/bind_address': '/networking/interfaces/management/ip', + '/mq/rabbitmq/bind_address': '/networking/interfaces/management/ip', + '/endpoints/compute/metadata/host': '/networking/interfaces/management/ip', + '/endpoints/compute/novnc/host': '/networking/interfaces/management/ip', + '/endpoints/compute/service/host': '/networking/interfaces/management/ip', + '/endpoints/compute/xvpvnc/host': '/networking/interfaces/management/ip', + '/endpoints/ec2/admin/host': '/networking/interfaces/management/ip', + '/endpoints/ec2/service/host': '/networking/interfaces/management/ip', + '/endpoints/identity/admin/host': '/networking/interfaces/management/ip', + '/endpoints/identity/service/host': '/networking/interfaces/management/ip', + '/endpoints/image/registry/host': '/networking/interfaces/management/ip', + '/endpoints/image/service/host': '/networking/interfaces/management/ip', + '/endpoints/metering/service/host': '/networking/interfaces/management/ip', + '/endpoints/network/service/host': '/networking/interfaces/management/ip', + '/endpoints/volume/service/host': '/networking/interfaces/management/ip' + }, + 'os-network': { + }, + 'os-compute-worker': { + } + }, 'dashboard_roles': ['os-single-controller'], 'role_assign_policy': { 'default':{ @@ -165,9 +190,9 @@ cobbler_EXPECTED = { 'run_list': '"role[os-single-controller]","role[os-network]","role[os-compute]"', 'ignore_proxy': '127.0.0.1,localhost,host1.1,192.168.20.100', 'ntp_server': '192.168.20.254', - 'chef_client_name': 'host1_openstack_1', + 'chef_client_name': 'openstack.host1.1', 'cluster_databag': 'openstack_1', - 'chef_node_name': u'host1_openstack_1' + 'chef_node_name': u'openstack.host1.1' }, }], } diff --git a/compass/tests/actions/data/test2 b/compass/tests/actions/data/test2 index 4047226c..537c3d9b 100644 --- a/compass/tests/actions/data/test2 +++ b/compass/tests/actions/data/test2 @@ -1,3 +1,5 @@ +import simplejson as json + ADAPTERS = [ {'name': 'CentOS_openstack', 'os': 'CentOS', 'target_system': 'openstack'}, ] @@ -7,7 +9,7 @@ ROLES = [ {'name': 'os-compute', 'target_system': 'openstack'}, ] SWITCHES = [ - {'ip': '1.2.3.4', 'vendor': 'huawei', 'credential': {'version': 'v2c', 'community': 'public'}}, + {'ip': '1.2.3.4', 'vendor_info': 'huawei', 'credential_data': json.dumps({'version': 'v2c', 'community': 'public'})}, ] MACHINES_BY_SWITCH = { '1.2.3.4': [ @@ -20,7 +22,7 @@ CLUSTERS = [ 'name': 'cluster1', 'adapter': 'CentOS_openstack', 'mutable': False, - 'security': { + 'security_config': json.dumps({ 'server_credentials': { 'username': 'root', 'password': 'huawei' }, @@ -30,8 +32,8 @@ CLUSTERS = [ 'console_credentials': { 'username': 'admin', 'password': 'huawei' } - }, - 'networking': { + }), + 'networking_config': json.dumps({ 'interfaces': { 'management': { 'nic': 'eth0', @@ -73,8 +75,8 @@ CLUSTERS = [ 'search_path': 'ods.com', 'gateway': '10.145.88.1' }, - }, - 'partition': '/home 20%%;/tmp 10%%;/var 30%%;', + }), + 'partition_config': json.dumps('/home 20%%;/tmp 10%%;/var 30%%;'), }, ] HOSTS_BY_CLUSTER = { @@ -83,7 +85,7 @@ HOSTS_BY_CLUSTER = { 'hostname': 'host1', 'mac': '00:00:01:02:03:04', 'mutable': False, - 'config': { + 'config_data': json.dumps({ 'networking': { 'interfaces': { 'management': { @@ -92,12 +94,12 @@ HOSTS_BY_CLUSTER = { }, }, 'roles': ["os-single-controller", "os-network"], - }, + }), },{ 'hostname': 'host2', 'mac': '00:00:01:02:03:05', 'mutable': False, - 'config': { + 'config_data': json.dumps({ 'networking': { 'interfaces': { 'management': { @@ -106,7 +108,7 @@ HOSTS_BY_CLUSTER = { }, }, 'roles': ["os-compute"], - }, + }), }, ], @@ -126,6 +128,29 @@ chef_MOCK = { 'os-compute-worker': 'openstack nova node' }, 'dashboard_roles': ['os-single-controller'], + 'role_mapping': { + 'os-single-controller': { + '/db/mysql/bind_address': '/networking/interfaces/management/ip', + '/mq/rabbitmq/bind_address': '/networking/interfaces/management/ip', + '/endpoints/compute/metadata/host': '/networking/interfaces/management/ip', + '/endpoints/compute/novnc/host': '/networking/interfaces/management/ip', + '/endpoints/compute/service/host': '/networking/interfaces/management/ip', + '/endpoints/compute/xvpvnc/host': '/networking/interfaces/management/ip', + '/endpoints/ec2/admin/host': '/networking/interfaces/management/ip', + '/endpoints/ec2/service/host': '/networking/interfaces/management/ip', + '/endpoints/identity/admin/host': '/networking/interfaces/management/ip', + '/endpoints/identity/service/host': '/networking/interfaces/management/ip', + '/endpoints/image/registry/host': '/networking/interfaces/management/ip', + '/endpoints/image/service/host': '/networking/interfaces/management/ip', + '/endpoints/metering/service/host': '/networking/interfaces/management/ip', + '/endpoints/network/service/host': '/networking/interfaces/management/ip', + '/endpoints/volume/service/host': '/networking/interfaces/management/ip' + }, + 'os-network': { + }, + 'os-compute-worker': { + } + }, 'role_assign_policy': { 'default':{ 'bundles': [], @@ -181,9 +206,9 @@ cobbler_EXPECTED = { 'run_list': '"role[os-single-controller]","role[os-network]"', 'ignore_proxy': '127.0.0.1,localhost,host1.1,192.168.20.100,host2.1,192.168.20.101', 'ntp_server': '192.168.20.254', - 'chef_client_name': 'host1_openstack_1', + 'chef_client_name': 'openstack.host1.1', 'cluster_databag': 'openstack_1', - 'chef_node_name': 'host1_openstack_1' + 'chef_node_name': 'openstack.host1.1' }, },{ 'profile': 'CentOS', @@ -215,9 +240,9 @@ cobbler_EXPECTED = { 'run_list': '"role[os-compute]"', 'ignore_proxy': '127.0.0.1,localhost,host1.1,192.168.20.100,host2.1,192.168.20.101', 'ntp_server': '192.168.20.254', - 'chef_client_name': 'host2_openstack_1', + 'chef_client_name': 'openstack.host2.1', 'cluster_databag': 'openstack_1', - 'chef_node_name': 'host2_openstack_1' + 'chef_node_name': 'openstack.host2.1' }, }], } diff --git a/compass/tests/actions/data/test3 b/compass/tests/actions/data/test3 index f363c517..71378e46 100644 --- a/compass/tests/actions/data/test3 +++ b/compass/tests/actions/data/test3 @@ -1,3 +1,5 @@ +import simplejson as json + ADAPTERS = [ {'name': 'CentOS_openstack', 'os': 'CentOS', 'target_system': 'openstack'}, ] @@ -7,7 +9,7 @@ ROLES = [ {'name': 'os-compute', 'target_system': 'openstack'}, ] SWITCHES = [ - {'ip': '1.2.3.4', 'vendor': 'huawei', 'credential': {'version': 'v2c', 'community': 'public'}}, + {'ip': '1.2.3.4', 'vendor_info': 'huawei', 'credential_data': json.dumps({'version': 'v2c', 'community': 'public'})}, ] MACHINES_BY_SWITCH = { '1.2.3.4': [ @@ -22,7 +24,7 @@ CLUSTERS = [ 'name': 'cluster1', 'adapter': 'CentOS_openstack', 'mutable': False, - 'security': { + 'security_config': json.dumps({ 'server_credentials': { 'username': 'root', 'password': 'huawei' }, @@ -32,8 +34,8 @@ CLUSTERS = [ 'console_credentials': { 'username': 'admin', 'password': 'huawei' } - }, - 'networking': { + }), + 'networking_config': json.dumps({ 'interfaces': { 'management': { 'nic': 'eth0', @@ -75,13 +77,13 @@ CLUSTERS = [ 'search_path': 'ods.com', 'gateway': '10.145.88.1' }, - }, - 'partition': '/home 20%%;/tmp 10%%;/var 30%%;', + }), + 'partition_config': json.dumps('/home 20%%;/tmp 10%%;/var 30%%;'), }, { 'name': 'cluster2', 'adapter': 'CentOS_openstack', 'mutable': False, - 'security': { + 'security_config': json.dumps({ 'server_credentials': { 'username': 'root', 'password': 'huawei' }, @@ -91,8 +93,8 @@ CLUSTERS = [ 'console_credentials': { 'username': 'admin', 'password': 'huawei' } - }, - 'networking': { + }), + 'networking_config': json.dumps({ 'interfaces': { 'management': { 'nic': 'eth0', @@ -134,8 +136,8 @@ CLUSTERS = [ 'search_path': 'ods.com', 'gateway': '10.145.88.1' }, - }, - 'partition': '/home 20%%;/tmp 10%%;/var 30%%;', + }), + 'partition_config': json.dumps('/home 20%%;/tmp 10%%;/var 30%%;'), }, ] @@ -145,7 +147,7 @@ HOSTS_BY_CLUSTER = { 'hostname': 'host1', 'mac': '00:00:01:02:03:04', 'mutable': False, - 'config': { + 'config_data': json.dumps({ 'networking': { 'interfaces': { 'management': { @@ -154,12 +156,12 @@ HOSTS_BY_CLUSTER = { }, }, 'roles': ["os-single-controller"], - }, + }), }, { 'hostname': 'host2', 'mac': '00:00:01:02:03:05', 'mutable': False, - 'config': { + 'config_data': json.dumps({ 'networking': { 'interfaces': { 'management': { @@ -168,7 +170,7 @@ HOSTS_BY_CLUSTER = { }, }, 'roles': ["os-network", "os-compute"], - }, + }), }, ], 'cluster2': [ @@ -176,7 +178,7 @@ HOSTS_BY_CLUSTER = { 'hostname': 'host1', 'mac': '00:00:01:02:03:06', 'mutable': False, - 'config': { + 'config_data': json.dumps({ 'networking': { 'interfaces': { 'management': { @@ -185,12 +187,12 @@ HOSTS_BY_CLUSTER = { }, }, 'roles': ["os-single-controller"], - }, + }), }, { 'hostname': 'host2', 'mac': '00:00:01:02:03:07', 'mutable': False, - 'config': { + 'config_data': json.dumps({ 'networking': { 'interfaces': { 'management': { @@ -199,7 +201,7 @@ HOSTS_BY_CLUSTER = { }, }, 'roles': ["os-network", "os-compute"], - }, + }), }, ], } @@ -218,6 +220,29 @@ chef_MOCK = { 'os-compute-worker': 'openstack nova node' }, 'dashboard_roles': ['os-single-controller'], + 'role_mapping': { + 'os-single-controller': { + '/db/mysql/bind_address': '/networking/interfaces/management/ip', + '/mq/rabbitmq/bind_address': '/networking/interfaces/management/ip', + '/endpoints/compute/metadata/host': '/networking/interfaces/management/ip', + '/endpoints/compute/novnc/host': '/networking/interfaces/management/ip', + '/endpoints/compute/service/host': '/networking/interfaces/management/ip', + '/endpoints/compute/xvpvnc/host': '/networking/interfaces/management/ip', + '/endpoints/ec2/admin/host': '/networking/interfaces/management/ip', + '/endpoints/ec2/service/host': '/networking/interfaces/management/ip', + '/endpoints/identity/admin/host': '/networking/interfaces/management/ip', + '/endpoints/identity/service/host': '/networking/interfaces/management/ip', + '/endpoints/image/registry/host': '/networking/interfaces/management/ip', + '/endpoints/image/service/host': '/networking/interfaces/management/ip', + '/endpoints/metering/service/host': '/networking/interfaces/management/ip', + '/endpoints/network/service/host': '/networking/interfaces/management/ip', + '/endpoints/volume/service/host': '/networking/interfaces/management/ip' + }, + 'os-network': { + }, + 'os-compute-worker': { + } + }, 'role_assign_policy': { 'default':{ 'bundles': [], @@ -273,9 +298,9 @@ cobbler_EXPECTED = { 'run_list': '"role[os-single-controller]"', 'ignore_proxy': '127.0.0.1,localhost,host1.1,192.168.20.100,host2.1,192.168.20.101', 'ntp_server': '192.168.20.254', - 'chef_client_name': 'host1_openstack_1', + 'chef_client_name': 'openstack.host1.1', 'cluster_databag': 'openstack_1', - 'chef_node_name': u'host1_openstack_1' + 'chef_node_name': u'openstack.host1.1' }, },{ 'profile': 'CentOS', @@ -307,9 +332,9 @@ cobbler_EXPECTED = { 'run_list': '"role[os-network]","role[os-compute]"', 'ignore_proxy': '127.0.0.1,localhost,host1.1,192.168.20.100,host2.1,192.168.20.101', 'ntp_server': '192.168.20.254', - 'chef_client_name': 'host2_openstack_1', + 'chef_client_name': 'openstack.host2.1', 'cluster_databag': 'openstack_1', - 'chef_node_name': u'host2_openstack_1' + 'chef_node_name': u'openstack.host2.1' }, },{ 'profile': 'CentOS', @@ -341,9 +366,9 @@ cobbler_EXPECTED = { 'run_list': '"role[os-single-controller]"', 'ignore_proxy': '127.0.0.1,localhost,host1.2,192.168.20.110,host2.2,192.168.20.111', 'ntp_server': '192.168.20.254', - 'chef_client_name': 'host1_openstack_2', + 'chef_client_name': 'openstack.host1.2', 'cluster_databag': 'openstack_2', - 'chef_node_name': u'host1_openstack_2' + 'chef_node_name': u'openstack.host1.2' }, },{ 'profile': 'CentOS', @@ -375,9 +400,9 @@ cobbler_EXPECTED = { 'run_list': '"role[os-network]","role[os-compute]"', 'ignore_proxy': '127.0.0.1,localhost,host1.2,192.168.20.110,host2.2,192.168.20.111', 'ntp_server': '192.168.20.254', - 'chef_client_name': 'host2_openstack_2', + 'chef_client_name': 'openstack.host2.2', 'cluster_databag': 'openstack_2', - 'chef_node_name': u'host2_openstack_2' + 'chef_node_name': u'openstack.host2.2' }, }], } diff --git a/compass/tests/actions/test_deploy.py b/compass/tests/actions/test_deploy.py old mode 100644 new mode 100755 index e1a89a3d..86d6b277 --- a/compass/tests/actions/test_deploy.py +++ b/compass/tests/actions/test_deploy.py @@ -1,13 +1,33 @@ -"""integration test for action deploy""" +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""integration test for action deploy. + + .. moduleauthor:: Xiaodong Wang +""" import chef import logging +import mock import os import os.path import shutil import unittest2 import xmlrpclib -from mock import Mock +from contextlib import contextmanager os.environ['COMPASS_IGNORE_SETTING'] = 'true' @@ -17,17 +37,15 @@ from compass.utils import setting_wrapper as setting reload(setting) -setting.OS_INSTALLER = 'cobbler' -setting.COBBLER_INSTALLER_URL = 'http://localhost/cobbler_api' -setting.PACKAGE_INSTALLER = 'chef' -setting.CHEF_INSTALLER_URL = 'https://localhost/' -setting.CONFIG_DIR = '%s/data' % os.path.dirname(os.path.abspath(__file__)) - - from compass.actions import deploy +from compass.actions import util from compass.db import database -from compass.db.model import Switch, Machine -from compass.db.model import Cluster, ClusterHost, Adapter, Role +from compass.db.model import Adapter +from compass.db.model import Cluster +from compass.db.model import ClusterHost +from compass.db.model import Machine +from compass.db.model import Role +from compass.db.model import Switch from compass.utils import flags from compass.utils import logsetting @@ -60,36 +78,55 @@ class TestEndToEnd(unittest2.TestCase): else: return expected_config == origin_config + def _mock_lock(self): + @contextmanager + def _lock(lock_name, blocking=True, timeout=10): + """mock lock.""" + try: + yield lock_name + finally: + pass + + self.lock_backup_ = util.lock + util.lock = mock.Mock(side_effect=_lock) + + def _unmock_lock(self): + util.lock = self.lock_backup_ + + def _unmock_cobbler(self): + xmlrpclib.Server = self.cobbler_server_backup_ + def _mock_cobbler(self, host_configs): - """mock cobbler""" - mock_server = Mock() + """mock cobbler.""" + self.cobbler_server_backup_ = xmlrpclib.Server + mock_server = mock.Mock() xmlrpclib.Server = mock_server mock_server.return_value.login.return_value = '' - mock_server.return_value.sync = Mock() - mock_server.return_value.find_profile = Mock( + mock_server.return_value.sync = mock.Mock() + mock_server.return_value.find_profile = mock.Mock( side_effect=lambda x: [x['name']]) def _get_system_handle(sys_name, token): - """mock get_system_handle""" + """mock get_system_handle.""" for i, config in enumerate(host_configs): if config['name'] == sys_name: return i raise Exception('Not Found %s' % sys_name) - mock_server.return_value.get_system_handle = Mock( + mock_server.return_value.get_system_handle = mock.Mock( side_effect=_get_system_handle) def _new_system(token): - """mock new_system""" + """mock new_system.""" host_configs.append({'name': ''}) return len(host_configs) - 1 - mock_server.return_value.new_system = Mock( + mock_server.return_value.new_system = mock.Mock( side_effect=_new_system) def _remove_system(sys_name, token): - """mock remove system""" + """mock remove system.""" for i, config in host_configs: if config['name'] == sys_name: del host_configs[i] @@ -97,29 +134,42 @@ class TestEndToEnd(unittest2.TestCase): raise Exception('Not Found %s' % sys_name) - mock_server.return_value.remove_system = Mock( + mock_server.return_value.remove_system = mock.Mock( side_effect=_remove_system) - mock_server.return_value.save_system = Mock() + mock_server.return_value.save_system = mock.Mock() def _modify_system(sys_id, key, value, token): - """mock modify_system""" + """mock modify_system.""" host_configs[sys_id][key] = value - mock_server.return_value.modify_system = Mock( + mock_server.return_value.modify_system = mock.Mock( side_effect=_modify_system) def _check_cobbler(self, host_configs, expected_host_configs): - """check cobbler config generated correctly""" + """check cobbler config generated correctly.""" self.assertEqual(len(host_configs), len(expected_host_configs)) for i in range(len(host_configs)): self.assertTrue( - self._contains(host_configs[i], expected_host_configs[i])) + self._contains(host_configs[i], expected_host_configs[i]), + 'host %s config is\n%s\nwhile expected config is\n%s' % ( + i, host_configs[i], expected_host_configs[i] + ) + ) + + def _unmock_chef(self): + chef.autoconfigure = self.chef_autoconfigure_backup_ + chef.DataBag = chef.chef_databag_backup_ + chef.DataBagItem = self.chef_databagitem_backup_ + chef.Client = self.chef_client_backup_ + chef.Node = self.chef_node_backup_ def _mock_chef(self, configs): - """mock chef""" - chef.autoconfigure = Mock() - chef.DataBag = Mock() + """mock chef.""" + self.chef_autoconfigure_backup_ = chef.autoconfigure + chef.chef_databag_backup_ = chef.DataBag + chef.autoconfigure = mock.Mock() + chef.DataBag = mock.Mock() import collections @@ -146,35 +196,49 @@ class TestEndToEnd(unittest2.TestCase): del in_self.config_[name] def delete(in_self): - """mock delete""" + """mock delete.""" del configs[in_self.bag_item_name_] def save(in_self): - """mock save""" + """mock save.""" configs[in_self.bag_item_name_] = in_self.config_ - chef.DataBagItem = Mock(side_effect=_mockDict) - chef.Client = Mock() - chef.Client.return_value.delete = Mock() - chef.Node = Mock() - chef.Node.return_value.delete = Mock() + self.chef_databagitem_backup_ = chef.DataBagItem + chef.DataBagItem = mock.Mock(side_effect=_mockDict) + self.chef_client_backup_ = chef.Client + chef.Client = mock.Mock() + chef.Client.return_value.delete = mock.Mock() + self.chef_node_backup_ = chef.Node + chef.Node = mock.Mock() + chef.Node.return_value.delete = mock.Mock() def _check_chef(self, configs, expected_configs): """check chef config is generated correctly.""" - self.assertTrue(self._contains(configs, expected_configs)) + self.assertTrue( + self._contains(configs, expected_configs), + 'configs\n%s\nwhile expected configs\n%s' % ( + configs, expected_configs + ) + ) def _mock_os_installer(self, config_locals): - """mock os installer""" + """mock os installer.""" self.os_installer_mock_[setting.OS_INSTALLER]( **config_locals['%s_MOCK' % setting.OS_INSTALLER]) + def _unmock_os_installer(self): + self.os_installer_unmock_[setting.OS_INSTALLER]() + def _mock_package_installer(self, config_locals): - """mock package installer""" + """mock package installer.""" self.package_installer_mock_[setting.PACKAGE_INSTALLER]( **config_locals['%s_MOCK' % setting.PACKAGE_INSTALLER]) + def _unmock_package_installer(self): + self.package_installer_unmock_[setting.PACKAGE_INSTALLER]() + def _check_os_installer(self, config_locals): - """check os installer generate correct configs""" + """check os installer generate correct configs.""" mock_kwargs = config_locals['%s_MOCK' % setting.OS_INSTALLER] expected_kwargs = config_locals['%s_EXPECTED' % setting.OS_INSTALLER] kwargs = {} @@ -183,7 +247,7 @@ class TestEndToEnd(unittest2.TestCase): self.os_installer_checker_[setting.OS_INSTALLER](**kwargs) def _check_package_installer(self, config_locals): - """check package installer generate correct configs""" + """check package installer generate correct configs.""" mock_kwargs = config_locals['%s_MOCK' % setting.PACKAGE_INSTALLER] expected_kwargs = config_locals[ '%s_EXPECTED' % setting.PACKAGE_INSTALLER] @@ -193,7 +257,7 @@ class TestEndToEnd(unittest2.TestCase): self.package_installer_checker_[setting.PACKAGE_INSTALLER](**kwargs) def _test(self, config_filename): - """run the test""" + """run the test.""" full_path = '%s/data/%s' % ( os.path.dirname(os.path.abspath(__file__)), config_filename) @@ -214,9 +278,11 @@ class TestEndToEnd(unittest2.TestCase): self._check_os_installer(config_locals) self._check_package_installer(config_locals) + self._unmock_os_installer() + self._unmock_package_installer() def _prepare_database(self, config_locals): - """prepare database""" + """prepare database.""" with database.session() as session: adapters = {} for adapter_config in config_locals['ADAPTERS']: @@ -269,25 +335,64 @@ class TestEndToEnd(unittest2.TestCase): host.cluster = clusters[cluster_name] session.add(host) + def _mock_setting(self): + self.backup_os_installer_ = setting.OS_INSTALLER + self.backup_package_installer_ = setting.PACKAGE_INSTALLER + self.backup_cobbler_url_ = setting.COBBLER_INSTALLER_URL + self.backup_chef_url_ = setting.CHEF_INSTALLER_URL + self.backup_config_dir_ = setting.CONFIG_DIR + self.backup_global_config_filename_ = setting.GLOBAL_CONFIG_FILENAME + self.backup_config_file_format_ = setting.CONFIG_FILE_FORMAT + setting.OS_INSTALLER = 'cobbler' + setting.PACKAGE_INSTALLER = 'chef' + setting.COBBLER_INSTALLER_URL = 'http://localhost/cobbler_api' + setting.CHEF_INSTALLER_URL = 'https://localhost/' + setting.CONFIG_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'data') + setting.GLOBAL_CONFIG_FILENAME = 'global_config' + setting.CONFIG_FILE_FORMAT = 'python' + + def _unmock_setting(self): + setting.OS_INSTALLER = self.backup_os_installer_ + setting.PACKAGE_INSTALLER = self.backup_package_installer_ + setting.COBBLER_INSTALLER_URL = self.backup_cobbler_url_ + setting.CHEF_INSTALLER_URL = self.backup_chef_url_ + setting.CONFIG_DIR = self.backup_config_dir_ + setting.GLOBAL_CONFIG_FILENAME = self.backup_global_config_filename_ + self.backup_config_file_format_ = setting.CONFIG_FILE_FORMAT + def setUp(self): - """test setup""" + """test setup.""" super(TestEndToEnd, self).setUp() + self._mock_setting() logsetting.init() database.create_db() - shutil.rmtree = Mock() - os.system = Mock() + self._mock_lock() + self.rmtree_backup_ = shutil.rmtree + shutil.rmtree = mock.Mock() + self.system_backup_ = os.system + os.system = mock.Mock() self.os_installer_mock_ = {} self.os_installer_mock_['cobbler'] = self._mock_cobbler + self.os_installer_unmock_ = {} + self.os_installer_unmock_['cobbler'] = self._unmock_cobbler self.package_installer_mock_ = {} self.package_installer_mock_['chef'] = self._mock_chef + self.package_installer_unmock_ = {} + self.package_installer_unmock_['chef'] = self._unmock_chef self.os_installer_checker_ = {} self.os_installer_checker_['cobbler'] = self._check_cobbler self.package_installer_checker_ = {} self.package_installer_checker_['chef'] = self._check_chef def tearDown(self): - """test teardown""" + """test teardown.""" database.drop_db() + self._unmock_lock() + shutil.rmtree = self.rmtree_backup_ + os.system = self.system_backup_ + self._unmock_setting() super(TestEndToEnd, self).tearDown() def test_1(self): @@ -299,7 +404,7 @@ class TestEndToEnd(unittest2.TestCase): self._test('test2') def test_3(self): - """test multi clusters multi hosts""" + """test multi clusters multi hosts.""" self._test('test3') diff --git a/compass/tests/api/__init__.py b/compass/tests/api/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/tests/api/__init__.py +++ b/compass/tests/api/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/api/test_api.py b/compass/tests/api/test_api.py old mode 100644 new mode 100755 index 84910f0c..8eabaa10 --- a/compass/tests/api/test_api.py +++ b/compass/tests/api/test_api.py @@ -1,10 +1,26 @@ -"""test api module""" -from copy import deepcopy -from celery import current_app -from mock import Mock -import simplejson as json -import os +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test api module.""" +import celery +import copy import csv +import mock +import os +import simplejson as json import unittest2 @@ -17,14 +33,14 @@ reload(setting) from compass.api import app from compass.db import database -from compass.db.model import Switch -from compass.db.model import Machine -from compass.db.model import Cluster -from compass.db.model import ClusterState -from compass.db.model import ClusterHost -from compass.db.model import HostState from compass.db.model import Adapter +from compass.db.model import Cluster +from compass.db.model import ClusterHost +from compass.db.model import ClusterState +from compass.db.model import HostState +from compass.db.model import Machine from compass.db.model import Role +from compass.db.model import Switch from compass.db.model import SwitchConfig from compass.utils import flags from compass.utils import logsetting @@ -49,7 +65,7 @@ class ApiTestCase(unittest2.TestCase): # We do not want to send a real task as our test environment # does not have a AMQP system set up. TODO(): any better way? - current_app.send_task = Mock() + celery.current_app.send_task = mock.Mock() def tearDown(self): database.drop_db() @@ -57,7 +73,7 @@ class ApiTestCase(unittest2.TestCase): class TestSwtichMachineAPI(ApiTestCase): - """test switch machine api""" + """test switch machine api.""" SWITCH_RESP_TPL = {"state": "under_monitoring", "ip": "", @@ -79,21 +95,29 @@ class TestSwtichMachineAPI(ApiTestCase): super(TestSwtichMachineAPI, self).tearDown() def test_get_switch_list(self): - """tst get switch list api""" + """tst get switch list api.""" # Prepare testing data with database.session() as session: - switches = [Switch(ip='192.168.1.1', - credential=self.SWITCH_CREDENTIAL), - Switch(ip='192.168.1.2', - credential=self.SWITCH_CREDENTIAL), - Switch(ip='192.1.192.1', - credential=self.SWITCH_CREDENTIAL), - Switch(ip='192.1.192.2', - credential=self.SWITCH_CREDENTIAL), - Switch(ip='192.1.195.3', - credential=self.SWITCH_CREDENTIAL), - Switch(ip='192.2.192.4', - credential=self.SWITCH_CREDENTIAL)] + switches = [ + Switch( + ip='192.168.1.1', + credential_data=json.dumps(self.SWITCH_CREDENTIAL)), + Switch( + ip='192.168.1.2', + credential_data=json.dumps(self.SWITCH_CREDENTIAL)), + Switch( + ip='192.1.192.1', + credential_data=json.dumps(self.SWITCH_CREDENTIAL)), + Switch( + ip='192.1.192.2', + credential_data=json.dumps(self.SWITCH_CREDENTIAL)), + Switch( + ip='192.1.195.3', + credential_data=json.dumps(self.SWITCH_CREDENTIAL)), + Switch( + ip='192.2.192.4', + credential_data=json.dumps(self.SWITCH_CREDENTIAL)) + ] session.add_all(switches) # Start to query switches @@ -137,7 +161,7 @@ class TestSwtichMachineAPI(ApiTestCase): self.assertEqual(switch_count, expected_count) def test_post_switch_list(self): - """test post switch list""" + """test post switch list.""" # Test SwitchList POST method url = '/switches' @@ -169,7 +193,7 @@ class TestSwtichMachineAPI(ApiTestCase): self.assertEqual(return_value.status_code, 400) def test_get_switch_by_id(self): - """test get switch y id""" + """test get switch by id.""" # Test Get /switches/{id} # Non-exist switch id url = '/switches/1000' @@ -190,7 +214,7 @@ class TestSwtichMachineAPI(ApiTestCase): self.assertDictEqual(data["switch"], expected_switch_resp) def test_put_switch_by_id(self): - """test put switch by id""" + """test put switch by id.""" # Test put a switch by id url = '/switches/1000' # Put a non-existing switch @@ -200,7 +224,7 @@ class TestSwtichMachineAPI(ApiTestCase): # Put sucessfully url = '/switches/1' - credential = deepcopy(self.SWITCH_CREDENTIAL) + credential = copy.deepcopy(self.SWITCH_CREDENTIAL) credential['version'] = '1v' data = {'switch': {'credential': credential}} return_value = self.test_client.put(url, data=json.dumps(data)) @@ -210,13 +234,13 @@ class TestSwtichMachineAPI(ApiTestCase): 'repolling') def test_delete_switch(self): - """test delete switch""" + """test delete switch.""" url = '/switches/1' return_value = self.test_client.delete(url) self.assertEqual(return_value.status_code, 405) def test_get_machine_by_id(self): - """test get machine by id""" + """test get machine by id.""" # Prepare testing data with database.session() as session: machine = Machine(mac='00:27:88:0c:a6', port='1', vlan='1', @@ -234,7 +258,7 @@ class TestSwtichMachineAPI(ApiTestCase): self.assertEqual(return_value.status_code, 404) def test_get_machine_list(self): - """test get machine list""" + """test get machine list.""" #Prepare testing data with database.session() as session: switch_config = [ @@ -295,7 +319,7 @@ class TestSwtichMachineAPI(ApiTestCase): class TestClusterAPI(ApiTestCase): - """test cluster api""" + """test cluster api.""" SECURITY_CONFIG = { 'server_credentials': { @@ -377,7 +401,7 @@ class TestClusterAPI(ApiTestCase): super(TestClusterAPI, self).tearDown() def test_get_cluster_by_id(self): - """test get cluster by id""" + """test get cluster by id.""" # a. Get an existing cluster # b. Get a non-existing cluster, return 404 test_list = [{'url': '/clusters/1', 'expected_code': 200, @@ -399,7 +423,7 @@ class TestClusterAPI(ApiTestCase): # Create a cluster def test_post_cluster(self): - """test post cluster""" + """test post cluster.""" # a. Post a new cluster but no adapter exists cluster_req = {'cluster': {'name': 'cluster_09', 'adapter_id': 1}} @@ -430,7 +454,7 @@ class TestClusterAPI(ApiTestCase): self.assertEqual(data['cluster']['id'], 10) def test_get_clusters(self): - """test get clusters""" + """test get clusters.""" # a. get all clusters url = "/clusters" return_value = self.test_client.get(url) @@ -462,7 +486,7 @@ class TestClusterAPI(ApiTestCase): self.assertEqual(len(data['clusters']), 2) def test_put_cluster_security_resource(self): - """test put cluster security resource""" + """test put cluster security resource.""" # Prepare testing data security = {'security': self.SECURITY_CONFIG} @@ -488,82 +512,82 @@ class TestClusterAPI(ApiTestCase): # d. Security config is invalid -- some required field is null url = "/clusters/1/security" - invalid_security = deepcopy(security) + invalid_security = copy.deepcopy(security) invalid_security['security']['server_credentials']['username'] = None return_value = self.test_client.put( url, data=json.dumps(invalid_security)) self.assertEqual(return_value.status_code, 400) # e. Security config is invalid -- keyword is incorrect - invalid_security = deepcopy(security) + invalid_security = copy.deepcopy(security) invalid_security['security']['xxxx'] = {'xxx': 'xxx'} return_value = self.test_client.put( url, data=json.dumps(invalid_security)) self.assertEqual(return_value.status_code, 400) # f. Security config is invalid -- missing keyword - invalid_security = deepcopy(security) + invalid_security = copy.deepcopy(security) del invalid_security["security"]["server_credentials"] return_value = self.test_client.put( url, data=json.dumps(invalid_security)) self.assertEqual(return_value.status_code, 400) # g. Security config is invalid -- missing subkey keyword - invalid_security = deepcopy(security) + invalid_security = copy.deepcopy(security) del invalid_security["security"]["server_credentials"]["username"] return_value = self.test_client.put( url, data=json.dumps(invalid_security)) self.assertEqual(return_value.status_code, 400) def test_put_cluster_networking_resource(self): - """test put cluster networking resource""" + """test put cluster networking resource.""" networking = {"networking": self.NETWORKING_CONFIG} url = "/clusters/1/networking" return_value = self.test_client.put(url, data=json.dumps(networking)) self.assertEqual(return_value.status_code, 200) # Missing some required keyword in interfaces section - invalid_config = deepcopy(networking) + invalid_config = copy.deepcopy(networking) del invalid_config["networking"]["interfaces"]["management"]["nic"] return_value = self.test_client.put( url, data=json.dumps(invalid_config)) self.assertEqual(return_value.status_code, 400) - invalid_config = deepcopy(networking) + invalid_config = copy.deepcopy(networking) del invalid_config["networking"]["interfaces"]["management"] return_value = self.test_client.put( url, data=json.dumps(invalid_config)) self.assertEqual(return_value.status_code, 400) - invalid_config = deepcopy(networking) + invalid_config = copy.deepcopy(networking) invalid_config["networking"]["interfaces"]["xxx"] = {} return_value = self.test_client.put( url, data=json.dumps(invalid_config)) self.assertEqual(return_value.status_code, 400) # Missing some required keyword in global section - invalid_config = deepcopy(networking) + invalid_config = copy.deepcopy(networking) del invalid_config["networking"]["global"]["gateway"] return_value = self.test_client.put( url, data=json.dumps(invalid_config)) self.assertEqual(return_value.status_code, 400) # Invalid value in interfaces section - invalid_config = deepcopy(networking) + invalid_config = copy.deepcopy(networking) invalid_config["networking"]["interfaces"]["tenant"]["nic"] = "eth0" return_value = self.test_client.put( url, data=json.dumps(invalid_config)) self.assertEqual(return_value.status_code, 400) # Invalid value in global section - invalid_config = deepcopy(networking) + invalid_config = copy.deepcopy(networking) invalid_config["networking"]["global"]["nameservers"] = "*.*.*.*," return_value = self.test_client.put( url, data=json.dumps(invalid_config)) self.assertEqual(return_value.status_code, 400) def test_get_cluster_resource(self): - """test get cluster resource""" + """test get cluster resource.""" # Test resource with database.session() as session: cluster = session.query(Cluster).filter_by(id=1).first() @@ -592,7 +616,7 @@ class TestClusterAPI(ApiTestCase): self.assertEqual(data['message'], excepted_err_msg) def test_cluster_action(self): - """test cluster action""" + """test cluster action.""" from sqlalchemy import func #Prepare testing data: create machines, clusters in database #The first three machines will belong to cluster_01, the last one @@ -629,8 +653,8 @@ class TestClusterAPI(ApiTestCase): # ClusterHost table should not have any records. with database.session() as session: hosts_num = session.query( - func.count(ClusterHost.id)).filter_by( - cluster_id=1).scalar() + func.count(ClusterHost.id) + ).filter_by(cluster_id=1).scalar() self.assertEqual(hosts_num, 0) # 2. add a host with a installed machine @@ -647,8 +671,8 @@ class TestClusterAPI(ApiTestCase): total_hosts = 0 with database.session() as session: total_hosts = session.query( - func.count(ClusterHost.id)).filter_by( - cluster_id=1).scalar() + func.count(ClusterHost.id) + ).filter_by(cluster_id=1).scalar() data = json.loads(return_value.get_data()) self.assertEqual(len(data['cluster_hosts']), total_hosts) self.assertEqual(total_hosts, 3) @@ -661,7 +685,8 @@ class TestClusterAPI(ApiTestCase): self.assertEqual(len(data['failedHosts']), 3) with database.session() as session: count = session.query( - func.count(ClusterHost.id)).filter_by(cluster_id=1).scalar() + func.count(ClusterHost.id) + ).filter_by(cluster_id=1).scalar() self.assertEqual(count, 3) # 5. sucessfully remove requested hosts @@ -672,7 +697,8 @@ class TestClusterAPI(ApiTestCase): self.assertEqual(len(data['cluster_hosts']), 2) with database.session() as session: count = session.query( - func.count(ClusterHost.id)).filter_by(cluster_id=1).scalar() + func.count(ClusterHost.id) + ).filter_by(cluster_id=1).scalar() self.assertEqual(count, 1) # 6. Test 'replaceAllHosts' action on cluster_01 @@ -683,7 +709,8 @@ class TestClusterAPI(ApiTestCase): self.assertEqual(len(data['cluster_hosts']), 3) with database.session() as session: count = session.query( - func.count(ClusterHost.id)).filter_by(cluster_id=1).scalar() + func.count(ClusterHost.id) + ).filter_by(cluster_id=1).scalar() self.assertEqual(count, 3) # 7. Test 'deploy' action on cluster_01 @@ -699,10 +726,13 @@ class TestClusterAPI(ApiTestCase): url = '/clusters/2/action' with database.session() as session: session.query( - ClusterHost).filter_by(cluster_id=2).delete( - synchronize_session=False) + ClusterHost + ).filter_by(cluster_id=2).delete( + synchronize_session=False + ) host = session.query( - ClusterHost).filter_by(cluster_id=2).first() + ClusterHost + ).filter_by(cluster_id=2).first() return_value = self.test_client.post(url, data=json.dumps(request)) self.assertEqual(return_value.status_code, 404) @@ -743,7 +773,7 @@ class TestClusterAPI(ApiTestCase): class ClusterHostAPITest(ApiTestCase): - """test cluster host api""" + """test cluster host api.""" def setUp(self): super(ClusterHostAPITest, self).setUp() @@ -787,43 +817,62 @@ class ClusterHostAPITest(ApiTestCase): super(ClusterHostAPITest, self).tearDown() def test_clusterhost_get_config(self): - """test get cluster host config""" + """test get cluster host config.""" # 1. Try to get a config of the cluster host which does not exist url = '/clusterhosts/1000/config' return_value = self.test_client.get(url) self.assertEqual(404, return_value.status_code) # 2. Get a config of a cluster host sucessfully - test_config_data = deepcopy(self.test_config_data) + test_config_data = copy.deepcopy(self.test_config_data) test_config_data['hostname'] = 'host_01' url = '/clusterhosts/1/config' return_value = self.test_client.get(url) self.assertEqual(200, return_value.status_code) config = json.loads(return_value.get_data())['config'] - expected_config = deepcopy(test_config_data) + expected_config = copy.deepcopy(test_config_data) expected_config['hostid'] = 1 expected_config['hostname'] = 'host_01' expected_config['clusterid'] = 1 expected_config['clustername'] = 'cluster_01' + expected_config['fullname'] = 'host_01.1' expected_config[ - 'networking']['interfaces'][ - 'management']['mac'] = "00:27:88:0c:01" + 'networking' + ][ + 'interfaces' + ][ + 'management' + ][ + 'mac' + ] = "00:27:88:0c:01" expected_config['switch_port'] = '' expected_config['switch_ip'] = '192.168.1.1' expected_config['vlan'] = 0 self.assertDictEqual(config, expected_config) def test_clusterhost_put_config(self): - """test put clusterhost config""" - config = deepcopy(self.test_config_data) + """test put clusterhost config.""" + config = copy.deepcopy(self.test_config_data) config['roles'] = ['base'] config[ - 'networking']['interfaces'][ - 'management']['ip'] = '192.168.1.2' + 'networking' + ][ + 'interfaces' + ][ + 'management' + ][ + 'ip' + ] = '192.168.1.2' config[ - 'networking']['interfaces'][ - 'tenant']['ip'] = '10.12.1.2' + 'networking' + ][ + 'interfaces' + ][ + 'tenant' + ][ + 'ip' + ] = '10.12.1.2' # 1. Try to put a config of the cluster host which does not exist url = '/clusterhosts/1000/config' @@ -832,7 +881,7 @@ class ClusterHostAPITest(ApiTestCase): # 2. Config with incorrect ip format url = '/clusterhosts/2/config' - incorrect_conf = deepcopy(config) + incorrect_conf = copy.deepcopy(config) incorrect_conf['hostname'] = 'host_02' incorrect_conf[ 'networking']['interfaces']['management']['ip'] = 'xxx' @@ -852,7 +901,7 @@ class ClusterHostAPITest(ApiTestCase): self.assertDictEqual(config, config_db) def test_clusterhost_delete_subkey(self): - """test delete cluster host subkey""" + """test delete cluster host subkey.""" # 1. Try to delete an unqalified subkey of config url = '/clusterhosts/1/config/gateway' return_value = self.test_client.delete(url) @@ -862,7 +911,7 @@ class ClusterHostAPITest(ApiTestCase): url = 'clusterhosts/1/config/roles' return_value = self.test_client.delete(url) self.assertEqual(200, return_value.status_code) - expected_config = deepcopy(self.test_config_data) + expected_config = copy.deepcopy(self.test_config_data) with database.session() as session: config_db = session.query( ClusterHost.config_data).filter_by(id=1).first()[0] @@ -878,7 +927,7 @@ class ClusterHostAPITest(ApiTestCase): self.assertEqual(400, return_value.status_code) def test_clusterhost_get_by_id(self): - """test get cluster host by id""" + """test get cluster host by id.""" # 1. Get host sucessfully url = '/clusterhosts/1' return_value = self.test_client.get(url) @@ -893,7 +942,7 @@ class ClusterHostAPITest(ApiTestCase): self.assertEqual(404, return_value.status_code) def test_list_clusterhosts(self): - """test list cluster hosts""" + """test list cluster hosts.""" # 1. list the cluster host whose hostname is host_01 url = '/clusterhosts?hostname=host_02' return_value = self.test_client.get(url) @@ -932,7 +981,7 @@ class ClusterHostAPITest(ApiTestCase): self.assertListEqual([], hosts_result) def test_host_installing_progress(self): - """test get host installing progress""" + """test get host installing progress.""" # 1. Get progress of a non-existing host url = '/clusterhosts/1000/progress' return_value = self.test_client.get(url) @@ -959,10 +1008,14 @@ class ClusterHostAPITest(ApiTestCase): host = session.query(ClusterHost).filter_by(id=1).first() host.state.state = 'INSTALLING' session.query( - HostState).filter_by(id=1).update( - {'progress': 0.3, - 'message': 'Configuring...', - 'severity': 'INFO'}) + HostState + ).filter_by( + id=1 + ).update({ + 'progress': 0.3, + 'message': 'Configuring...', + 'severity': 'INFO' + }) return_value = self.test_client.get(url) self.assertEqual(200, return_value.status_code) @@ -972,7 +1025,7 @@ class ClusterHostAPITest(ApiTestCase): class TestAdapterAPI(ApiTestCase): - """test adapter api""" + """test adapter api.""" def setUp(self): super(TestAdapterAPI, self).setUp() @@ -992,7 +1045,7 @@ class TestAdapterAPI(ApiTestCase): super(TestAdapterAPI, self).tearDown() def test_list_adapter_by_id(self): - """test list adapter by id""" + """test list adapter by id.""" url = '/adapters/1' return_value = self.test_client.get(url) self.assertEqual(200, return_value.status_code) @@ -1000,7 +1053,7 @@ class TestAdapterAPI(ApiTestCase): self.assertEqual('Centos_openstack', data['adapter']['name']) def test_list_adapter_roles(self): - """test list adapter roles""" + """test list adapter roles.""" url = '/adapters/1/roles' return_value = self.test_client.get(url) self.assertEqual(200, return_value.status_code) @@ -1008,7 +1061,7 @@ class TestAdapterAPI(ApiTestCase): self.assertEqual(2, len(data['roles'])) def test_list_adapters(self): - """test list adapters""" + """test list adapters.""" url = '/adapters?name=Centos_openstack' return_value = self.test_client.get(url) data = json.loads(return_value.get_data()) @@ -1031,7 +1084,7 @@ class TestAdapterAPI(ApiTestCase): class TestAPIWorkFlow(ApiTestCase): - """test api workflow""" + """test api workflow.""" CLUSTER_SECURITY_CONFIG = { "security": { @@ -1118,11 +1171,12 @@ class TestAPIWorkFlow(ApiTestCase): #Prepare test data with database.session() as session: # Populate switch info to DB - switch = Switch(ip="192.168.2.1", - credential={"version": "2c", - "community": "public"}, - vendor="huawei", - state="under_monitoring") + switch = Switch( + ip="192.168.2.1", + credential_data=json.dumps( + {"version": "2c", "community": "public"}), + vendor_info="huawei", + state="under_monitoring") session.add(switch) # Populate machines info to DB @@ -1142,7 +1196,7 @@ class TestAPIWorkFlow(ApiTestCase): super(TestAPIWorkFlow, self).tearDown() def test_work_flow(self): - """test api workflow""" + """test api workflow.""" # Polling switch: mock post switch # url = '/switches' # data = {"ip": "192.168.2.1", @@ -1207,9 +1261,9 @@ class TestAPIWorkFlow(ApiTestCase): # Put cluster host config individually hosts_configs = [ - deepcopy(self.CLUSTERHOST_CONFIG), - deepcopy(self.CLUSTERHOST_CONFIG), - deepcopy(self.CLUSTERHOST_CONFIG) + copy.deepcopy(self.CLUSTERHOST_CONFIG), + copy.deepcopy(self.CLUSTERHOST_CONFIG), + copy.deepcopy(self.CLUSTERHOST_CONFIG) ] names = ["host_01", "host_02", "host_03"] mgmt_ips = ["10.120.8.100", "10.120.8.101", "10.120.8.102"] @@ -1254,6 +1308,8 @@ class TestAPIWorkFlow(ApiTestCase): excepted["clusterid"] = cluster_id excepted["clustername"] = "cluster_01" excepted["hostid"] = host_info["id"] + excepted["fullname"] = "%s.%s" % ( + excepted["hostname"], excepted["clusterid"]) excepted["networking"]["interfaces"]["management"]["mac"] = mac excepted['switch_port'] = machine.port excepted['vlan'] = machine.vlan @@ -1266,6 +1322,7 @@ class TestAPIWorkFlow(ApiTestCase): class TestExport(ApiTestCase): + """test export functions.""" CLUSTER_SECURITY_CONFIG = { "security": { @@ -1364,26 +1421,32 @@ class TestExport(ApiTestCase): session.add(adapter) #Populate switches info to DB - switches = [Switch(ip="192.168.2.1", - credential={"version": "2c", - "community": "public"}, - vendor="huawei", - state="under_monitoring"), - Switch(ip="192.168.2.2", - credential={"version": "2c", - "community": "public"}, - vendor="huawei", - state="under_monitoring"), - Switch(ip="192.168.2.3", - credential={"version": "2c", - "community": "public"}, - vendor="huawei", - state="under_monitoring"), - Switch(ip="192.168.2.4", - credential={"version": "2c", - "community": "public"}, - vendor="huawei", - state="under_monitoring")] + switches = [ + Switch( + ip="192.168.2.1", + credential_data=json.dumps( + {"version": "2c", "community": "public"}), + vendor_info="huawei", + state="under_monitoring"), + Switch( + ip="192.168.2.2", + credential_data=json.dumps( + {"version": "2c", "community": "public"}), + vendor_info="huawei", + state="under_monitoring"), + Switch( + ip="192.168.2.3", + credential_data=json.dumps( + {"version": "2c", "community": "public"}), + vendor_info="huawei", + state="under_monitoring"), + Switch( + ip="192.168.2.4", + credential_data=json.dumps( + {"version": "2c", "community": "public"}), + vendor_info="huawei", + state="under_monitoring") + ] session.add_all(switches) # Populate machines info to DB @@ -1443,7 +1506,7 @@ class TestExport(ApiTestCase): cluster_names = ['cluster_01', 'cluster_02'] for name, networking_config in zip(cluster_names, clusters_networking_config): - nconfig = deepcopy(self.CLUSTER_NETWORKING_CONFIG) + nconfig = copy.deepcopy(self.CLUSTER_NETWORKING_CONFIG) util.merge_dict(nconfig, networking_config) c = Cluster(name=name, adapter_id=1, security_config=json.dumps( @@ -1461,7 +1524,7 @@ class TestExport(ApiTestCase): '192.168.2.100', '192.168.2.101', '192.168.2.102'] hosts_config = [] for mip, tip in zip(host_mips, host_tips): - config = deepcopy(self.CLUSTERHOST_CONFIG) + config = copy.deepcopy(self.CLUSTERHOST_CONFIG) config['networking']['interfaces']['management']['ip'] = mip config['networking']['interfaces']['tenant']['ip'] = tip hosts_config.append(json.dumps(config)) @@ -1512,6 +1575,7 @@ class TestExport(ApiTestCase): super(TestExport, self).tearDown() def test_export(self): + """test export.""" talbes = ['switch', 'machine', 'cluster', 'cluster_host', 'adapter', 'role', 'switch_config'] for tname in talbes: @@ -1530,6 +1594,5 @@ class TestExport(ApiTestCase): if __name__ == '__main__': flags.init() - flags.OPTIONS.logfile = '/var/log/compass/test.log' logsetting.init() unittest2.main() diff --git a/compass/tests/apicommand/__init__.py b/compass/tests/apicommand/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/tests/apicommand/__init__.py +++ b/compass/tests/apicommand/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/apicommand/run_server.py b/compass/tests/apicommand/run_server.py old mode 100644 new mode 100755 index cb17f3a1..ba4e23ea --- a/compass/tests/apicommand/run_server.py +++ b/compass/tests/apicommand/run_server.py @@ -1,8 +1,24 @@ #!/usr/bin/env python -from copy import deepcopy +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""run fake flask server for test.""" +import copy +import os import simplejson as json import sys -import os + curr_dir = os.path.dirname(os.path.realpath(__file__)) compass_dir = os.path.dirname(os.path.dirname(os.path.dirname(curr_dir))) @@ -11,20 +27,20 @@ sys.path.append(compass_dir) from compass.api import app from compass.db import database -from compass.db.model import Switch -from compass.db.model import Machine -from compass.db.model import Cluster -from compass.db.model import ClusterState -from compass.db.model import ClusterHost -from compass.db.model import HostState from compass.db.model import Adapter +from compass.db.model import Cluster +from compass.db.model import ClusterHost +from compass.db.model import ClusterState +from compass.db.model import HostState +from compass.db.model import Machine from compass.db.model import Role +from compass.db.model import Switch from compass.db.model import SwitchConfig from compass.utils import util def setupDb(): - + """setup database.""" SECURITY_CONFIG = { "security": { "server_credentials": { @@ -197,7 +213,7 @@ def setupDb(): cluster_names = ['cluster_01', 'cluster_02'] for name, networking_config in zip(cluster_names, clusters_networking_config): - nconfig = deepcopy(NET_CONFIG) + nconfig = copy.deepcopy(NET_CONFIG) util.merge_dict(nconfig, networking_config) c = Cluster( name=name, adapter_id=1, @@ -213,7 +229,7 @@ def setupDb(): hosts_config = [] for mip, tip in zip(host_mips, host_tips): - config = deepcopy(HOST_CONFIG) + config = copy.deepcopy(HOST_CONFIG) config['networking']['interfaces']['management']['ip'] = mip config['networking']['interfaces']['tenant']['ip'] = tip hosts_config.append(json.dumps(config)) @@ -259,20 +275,23 @@ def setupDb(): ] session.add_all(host_states) + if __name__ == '__main__': db_url, port = sys.argv[1:] print db_url try: database.init(db_url) database.create_db() - except Exception as e: + except Exception as error: print "=====> Failed to create database" - print e + print error try: setupDb() - except: - pass + except Exception as error: + print "setupDb=====>Failed to setup database" + print error + print "Starting server ....." print "port is ", port app.run(use_reloader=False, host="0.0.0.0", port=port) diff --git a/compass/tests/apicommand/test_csvdeploy.py b/compass/tests/apicommand/test_csvdeploy.py old mode 100644 new mode 100755 index b2117c67..d938747f --- a/compass/tests/apicommand/test_csvdeploy.py +++ b/compass/tests/apicommand/test_csvdeploy.py @@ -1,19 +1,39 @@ -import simplejson as json +#!/usr/bin/env python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test deploy from csv file.""" +import mock import os -import sys -import unittest2 -import tempfile -import subprocess import shutil import signal +import simplejson as json import socket +import subprocess +import sys +import tempfile import time -from mock import Mock +import unittest2 + curr_dir = os.path.dirname(os.path.realpath(__file__)) api_cmd_path = '/'.join(( os.path.dirname(os.path.dirname(os.path.dirname(curr_dir))), 'bin')) sys.path.append(api_cmd_path) + + import csvdeploy @@ -51,12 +71,12 @@ class ApiTestCase(unittest2.TestCase): tmp_socket.bind(('', 0)) self.port = tmp_socket.getsockname()[-1] tmp_socket.close() - time.sleep(5) + time.sleep(10) except socket.error: sys.exit(1) - cmd = '%s run_server.py %s %d' % (sys.executable, database_url, - self.port) + cmd = '%s run_server.py %s %d' % ( + sys.executable, database_url, self.port) self.proc = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, preexec_fn=os.setsid, @@ -73,6 +93,7 @@ class ApiTestCase(unittest2.TestCase): super(ApiTestCase, self).tearDown() database.ENGINE.dispose() + database.init('sqlite://') os.killpg(self.proc.pid, signal.SIGTERM) try: if os.path.exists(self.db_dir): @@ -102,7 +123,9 @@ class TestAPICommand(ApiTestCase): super(TestAPICommand, self).tearDown() def test_start(self): - Client.deploy_hosts = Mock(return_value=(202, self.deploy_return_val)) + """test start deploy from csv.""" + Client.deploy_hosts = mock.Mock( + return_value=(202, self.deploy_return_val)) url = "http://127.0.0.1:%d" % self.port csvdeploy.start(self.CSV_IMPORT_DIR, url) clusters = csvdeploy.get_csv('cluster.csv', @@ -110,8 +133,9 @@ class TestAPICommand(ApiTestCase): with database.session() as session: for csv_cluster in clusters: cluster_id = csv_cluster['id'] - cluster = session.query(Cluster)\ - .filter_by(id=cluster_id).first() + cluster = session.query( + Cluster + ).filter_by(id=cluster_id).first() self.assertIsNotNone(cluster) self.assertEqual(csv_cluster['name'], cluster.name) self.assertDictEqual(csv_cluster['security_config'], diff --git a/compass/tests/config_management/__init__.py b/compass/tests/config_management/__init__.py new file mode 100644 index 00000000..4ee55a4c --- /dev/null +++ b/compass/tests/config_management/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/config_management/installers/test_os_installer.py b/compass/tests/config_management/installers/test_os_installer.py old mode 100644 new mode 100755 index afc7a6dc..e02eb28f --- a/compass/tests/config_management/installers/test_os_installer.py +++ b/compass/tests/config_management/installers/test_os_installer.py @@ -1,4 +1,23 @@ -"""test os installer module""" +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test os installer module. + + .. moduleauthor:: Xiaodong Wang +""" import os import unittest2 @@ -16,7 +35,8 @@ from compass.utils import logsetting class DummyInstaller(os_installer.Installer): - """dummy installer""" + """dummy installer.""" + NAME = 'dummy' def __init__(self): @@ -24,7 +44,8 @@ class DummyInstaller(os_installer.Installer): class Dummy2Installer(os_installer.Installer): - """another dummy installer""" + """another dummy installer.""" + NAME = 'dummy' def __init__(self): @@ -32,31 +53,31 @@ class Dummy2Installer(os_installer.Installer): class TestInstallerFunctions(unittest2.TestCase): - """test installer functions""" + """test installer functions.""" def setUp(self): super(TestInstallerFunctions, self).setUp() logsetting.init() - self.installers_backup = os_installer.INSTALLERS + self.installers_backup_ = os_installer.INSTALLERS os_installer.INSTALLERS = {} def tearDown(self): - os_installer.INSTALLERS = self.installers_backup + os_installer.INSTALLERS = self.installers_backup_ super(TestInstallerFunctions, self).tearDown() def test_found_installer(self): - """test found installer""" + """test found installer.""" os_installer.register(DummyInstaller) intaller = os_installer.get_installer_by_name(DummyInstaller.NAME) self.assertIsInstance(intaller, DummyInstaller) def test_notfound_unregistered_installer(self): - """test not found unregistered installer""" + """test not found unregistered installer.""" self.assertRaises(KeyError, os_installer.get_installer_by_name, DummyInstaller.NAME) def test_multi_registered_installer(self): - """test register multi installers with the same name""" + """test register multi installers with the same name.""" os_installer.register(DummyInstaller) self.assertRaises(KeyError, os_installer.register, Dummy2Installer) diff --git a/compass/tests/config_management/installers/test_package_installer.py b/compass/tests/config_management/installers/test_package_installer.py old mode 100644 new mode 100755 index e15b1d7d..eeab3c88 --- a/compass/tests/config_management/installers/test_package_installer.py +++ b/compass/tests/config_management/installers/test_package_installer.py @@ -1,4 +1,23 @@ -"""test package_installer module""" +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test package_installer module + + .. moduleauthor:: Xiaodong Wang +""" import os import unittest2 @@ -16,7 +35,8 @@ from compass.utils import logsetting class DummyInstaller(package_installer.Installer): - """dummy installer""" + """dummy installer.""" + NAME = 'dummy' def __init__(self): @@ -24,7 +44,7 @@ class DummyInstaller(package_installer.Installer): class Dummy2Installer(package_installer.Installer): - """another dummy installer""" + """another dummy installer.""" NAME = 'dummy' def __init__(self): @@ -32,16 +52,16 @@ class Dummy2Installer(package_installer.Installer): class TestInstallerFunctions(unittest2.TestCase): - """test installer functions""" + """test installer functions.""" def setUp(self): super(TestInstallerFunctions, self).setUp() logsetting.init() - self.installers_backup = package_installer.INSTALLERS + self.installers_backup_ = package_installer.INSTALLERS package_installer.INSTALLERS = {} def tearDown(self): - package_installer.INSTALLERS = self.installers_backup + package_installer.INSTALLERS = self.installers_backup_ super(TestInstallerFunctions, self).tearDown() def test_found_installer(self): @@ -52,12 +72,12 @@ class TestInstallerFunctions(unittest2.TestCase): self.assertIsInstance(intaller, DummyInstaller) def test_notfound_unregistered_installer(self): - """test not found unregistered installer""" + """test not found unregistered installer.""" self.assertRaises(KeyError, package_installer.get_installer_by_name, DummyInstaller.NAME) def test_multi_registered_installer(self): - """test register multi installers with the same name""" + """test register multi installers with the same name.""" package_installer.register(DummyInstaller) self.assertRaises(KeyError, package_installer.register, Dummy2Installer) diff --git a/compass/tests/config_management/providers/__init__.py b/compass/tests/config_management/providers/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/tests/config_management/providers/__init__.py +++ b/compass/tests/config_management/providers/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/config_management/providers/test_config_provider.py b/compass/tests/config_management/providers/test_config_provider.py old mode 100644 new mode 100755 index 96183167..de5f3c52 --- a/compass/tests/config_management/providers/test_config_provider.py +++ b/compass/tests/config_management/providers/test_config_provider.py @@ -1,4 +1,22 @@ -"""test config provider module""" +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""test config provider module. + + .. moduleauthor:: Xiaodong Wang +""" import os import unittest2 @@ -16,7 +34,8 @@ from compass.utils import logsetting class DummyProvider(config_provider.ConfigProvider): - """Dummy provider""" + """Dummy provider.""" + NAME = 'dummy' def __init__(self): @@ -24,7 +43,8 @@ class DummyProvider(config_provider.ConfigProvider): class Dummy2Provider(config_provider.ConfigProvider): - """another dummy provider""" + """another dummy provider.""" + NAME = 'dummy' def __init__(self): @@ -32,26 +52,27 @@ class Dummy2Provider(config_provider.ConfigProvider): class TestProviderRegisterFunctions(unittest2.TestCase): - """test provider register""" + """test provider register.""" def setUp(self): super(TestProviderRegisterFunctions, self).setUp() logsetting.init() + self.config_provider_backup_ = config_provider.PROVIDERS config_provider.PROVIDERS = {} def tearDown(self): - config_provider.PROVIDERS = {} + config_provider.PROVIDERS = self.config_provider_backup_ super(TestProviderRegisterFunctions, self).tearDown() def test_found_provider(self): - """test found provider""" + """test found provider.""" config_provider.register_provider(DummyProvider) provider = config_provider.get_provider_by_name( DummyProvider.NAME) self.assertIsInstance(provider, DummyProvider) def test_notfound_unregistered_provider(self): - """test notfound unregistered provider""" + """test notfound unregistered provider.""" self.assertRaises(KeyError, config_provider.get_provider_by_name, DummyProvider.NAME) diff --git a/compass/tests/config_management/utils/__init__.py b/compass/tests/config_management/utils/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/tests/config_management/utils/__init__.py +++ b/compass/tests/config_management/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/config_management/utils/test_config_filter.py b/compass/tests/config_management/utils/test_config_filter.py old mode 100644 new mode 100755 index ca04e09b..33af56ae --- a/compass/tests/config_management/utils/test_config_filter.py +++ b/compass/tests/config_management/utils/test_config_filter.py @@ -1,13 +1,41 @@ -"""test config_filter module""" +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test config_filter module. + + .. moduleauthor:: Xiaodong Wang +""" +import os import unittest2 + +os.environ['COMPASS_IGNORE_SETTING'] = 'true' + + +from compass.utils import setting_wrapper as setting +reload(setting) + + from compass.config_management.utils import config_filter from compass.utils import flags from compass.utils import logsetting class TestConfigFilter(unittest2.TestCase): - """test config filter class""" + """test config filter class.""" def setUp(self): super(TestConfigFilter, self).setUp() @@ -17,7 +45,7 @@ class TestConfigFilter(unittest2.TestCase): super(TestConfigFilter, self).tearDown() def test_allows(self): - """test allows rules""" + """test allows rules.""" config = {'1': '1', '2': {'22': '22', '33': {'333': '333', @@ -41,7 +69,7 @@ class TestConfigFilter(unittest2.TestCase): self.assertEqual(filtered_config, expected_config) def test_denies(self): - """test denies rules""" + """test denies rules.""" config = {'1': '1', '2': {'22': '22', '33': {'333': '333', '44': '444'}}, diff --git a/compass/tests/config_management/utils/test_config_merger.py b/compass/tests/config_management/utils/test_config_merger.py old mode 100644 new mode 100755 index 9b7396f6..fea54fb9 --- a/compass/tests/config_management/utils/test_config_merger.py +++ b/compass/tests/config_management/utils/test_config_merger.py @@ -1,7 +1,34 @@ -"""test config merger module""" +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test config merger module. + + .. moduleauthor:: Xiaodong Wang +""" import functools +import os import unittest2 +os.environ['COMPASS_IGNORE_SETTING'] = 'true' + + +from compass.utils import setting_wrapper as setting +reload(setting) + + from compass.config_management.utils import config_merger from compass.config_management.utils import config_merger_callbacks from compass.utils import flags @@ -9,7 +36,7 @@ from compass.utils import logsetting class TestConfigMerger(unittest2.TestCase): - """test config merger class""" + """test config merger class.""" def setUp(self): super(TestConfigMerger, self).setUp() @@ -19,7 +46,7 @@ class TestConfigMerger(unittest2.TestCase): super(TestConfigMerger, self).tearDown() def test_merge(self): - """test merge""" + """test merge.""" upper_config = { 'networking': { 'interfaces': { diff --git a/compass/tests/config_management/utils/test_config_merger_callbacks.py b/compass/tests/config_management/utils/test_config_merger_callbacks.py old mode 100644 new mode 100755 index b44cd8c7..fb433a6e --- a/compass/tests/config_management/utils/test_config_merger_callbacks.py +++ b/compass/tests/config_management/utils/test_config_merger_callbacks.py @@ -1,14 +1,42 @@ -"""test config merger callbacks module""" +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test config merger callbacks module. + + .. moduleauthor:: Xiaodong Wang +""" +import os import unittest2 + +os.environ['COMPASS_IGNORE_SETTING'] = 'true' + + +from compass.utils import setting_wrapper as setting +reload(setting) + + from compass.config_management.utils import config_merger_callbacks from compass.config_management.utils import config_reference from compass.utils import flags -from comapss.utils import logsetting +from compass.utils import logsetting class TestAssignRoles(unittest2.TestCase): - """test assign roles""" + """test assign roles.""" def setUp(self): super(TestAssignRoles, self).setUp() @@ -18,7 +46,7 @@ class TestAssignRoles(unittest2.TestCase): super(TestAssignRoles, self).tearDown() def test_assign_roles(self): - """test assign roles""" + """test assign roles.""" lower_configs = { 1: {'roles': ['control']}, 2: {'roles': ['api', 'compute']}, diff --git a/compass/tests/config_management/utils/test_config_reference.py b/compass/tests/config_management/utils/test_config_reference.py old mode 100644 new mode 100755 index a6ec19ef..aa979622 --- a/compass/tests/config_management/utils/test_config_reference.py +++ b/compass/tests/config_management/utils/test_config_reference.py @@ -1,21 +1,40 @@ -"""test config reference module""" -import unittest2 -from copy import deepcopy +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test config reference module. + + .. moduleauthor:: Xiaodong Wang +""" +import copy +import unittest2 -from compass.utils import util from compass.config_management.utils import config_reference +from compass.utils import util class TestConfigReference(unittest2.TestCase): - """test config reference class""" + """test config reference class.""" def test_init(self): - """test init function""" + """test init function.""" config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8} ref = config_reference.ConfigReference(config) config2 = {'5': {'6': 6}} ref2 = config_reference.ConfigReference(config2['5'], ref, '5') - expected_config = deepcopy(config) + expected_config = copy.deepcopy(config) util.merge_dict(expected_config, config2) self.assertEqual(ref.config, expected_config) self.assertEqual(id(ref.config['5']), id(ref2.config)) @@ -24,7 +43,7 @@ class TestConfigReference(unittest2.TestCase): self.assertEqual(id(ref.config['5']), id(ref3.config)) def test_ref(self): - """test ref function""" + """test ref function.""" config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8} ref = config_reference.ConfigReference(config) self.assertRaises(KeyError, ref.ref, '') @@ -52,7 +71,7 @@ class TestConfigReference(unittest2.TestCase): self.assertEqual(ref.ref('9'), subref2) def test_refs(self): - """test refs function""" + """test refs function.""" config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8, '88': 88} ref = config_reference.ConfigReference(config) refkeys = ref.ref_keys('1') @@ -66,7 +85,7 @@ class TestConfigReference(unittest2.TestCase): self.assertRaises(KeyError, ref.ref_keys, '') def test_contains(self): - """test contains function""" + """test contains function.""" config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8} ref = config_reference.ConfigReference(config) self.assertIn('/1/2', ref) @@ -77,7 +96,7 @@ class TestConfigReference(unittest2.TestCase): self.assertNotIn('/1/2/3/..', ref) def test_setitem(self): - """test setitem function""" + """test setitem function.""" config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8} ref = config_reference.ConfigReference(config) ref['/1/2'] = '6' @@ -91,7 +110,7 @@ class TestConfigReference(unittest2.TestCase): self.assertEqual(ref['3/6/8'], [1, 3, 5]) def test_del(self): - """test del function""" + """test del function.""" config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8} ref = config_reference.ConfigReference(config) del ref['/8'] @@ -103,7 +122,7 @@ class TestConfigReference(unittest2.TestCase): self.assertRaises(KeyError, ref.__delitem__, '9') def test_get(self): - """test get function""" + """test get function.""" config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8} ref = config_reference.ConfigReference(config) self.assertEqual(ref.get('1/2'), config['1']['2']) @@ -112,7 +131,7 @@ class TestConfigReference(unittest2.TestCase): self.assertNotIn('3', config['1']) def test_setdefault(self): - """test setdefault function""" + """test setdefault function.""" config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8} ref = config_reference.ConfigReference(config) self.assertEqual(ref.setdefault('1/2').config, config['1']['2']) @@ -121,9 +140,9 @@ class TestConfigReference(unittest2.TestCase): self.assertEqual(4, config['1']['4']) def test_update(self): - """test update function""" + """test update function.""" config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8} - expected_config = deepcopy(config) + expected_config = copy.deepcopy(config) ref = config_reference.ConfigReference(config) config2 = {'9': 9, '10': {'10': 10}} @@ -136,7 +155,7 @@ class TestConfigReference(unittest2.TestCase): self.assertEqual(ref.config, 10) def test_iter(self): - """test iter function""" + """test iter function.""" config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8} ref = config_reference.ConfigReference(config) keys = ref.keys() diff --git a/compass/tests/config_management/utils/test_config_translator.py b/compass/tests/config_management/utils/test_config_translator.py old mode 100644 new mode 100755 index f11b142f..a64aee62 --- a/compass/tests/config_management/utils/test_config_translator.py +++ b/compass/tests/config_management/utils/test_config_translator.py @@ -1,7 +1,35 @@ -"""test config translator module""" +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test config translator module. + + .. moduleauthor:: Xiaodong Wang +""" import functools +import os import unittest2 + +os.environ['COMPASS_IGNORE_SETTING'] = 'true' + + +from compass.utils import setting_wrapper as setting +reload(setting) + + from compass.config_management.utils import config_translator from compass.config_management.utils import config_translator_callbacks from compass.utils import flags @@ -9,7 +37,7 @@ from compass.utils import logsetting class TestConfigTranslatorFunctions(unittest2.TestCase): - """test config translator class""" + """test config translator class.""" def setUp(self): super(TestConfigTranslatorFunctions, self).setUp() @@ -19,7 +47,7 @@ class TestConfigTranslatorFunctions(unittest2.TestCase): super(TestConfigTranslatorFunctions, self).tearDown() def test_translate_1(self): - """config translate test""" + """config translate test.""" config = { 'networking': { 'interfaces': { @@ -209,7 +237,7 @@ class TestConfigTranslatorFunctions(unittest2.TestCase): self.assertEqual(translated_config, expected_config) def test_translate_2(self): - """config translate test""" + """config translate test.""" translator = config_translator.ConfigTranslator( mapping={ '/networking/interfaces/management/ip': [ @@ -338,7 +366,6 @@ class TestConfigTranslatorFunctions(unittest2.TestCase): expected_config2 = None translated_config2 = translator.translate(config2) - print translated_config2 self.assertEqual(translated_config2, expected_config2) diff --git a/compass/tests/hdsdiscovery/__init__.py b/compass/tests/hdsdiscovery/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/tests/hdsdiscovery/__init__.py +++ b/compass/tests/hdsdiscovery/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/hdsdiscovery/test_base.py b/compass/tests/hdsdiscovery/test_base.py old mode 100644 new mode 100755 index 545e2eeb..131903a3 --- a/compass/tests/hdsdiscovery/test_base.py +++ b/compass/tests/hdsdiscovery/test_base.py @@ -1,6 +1,23 @@ -"""test hdsdiscovery base module""" +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test hdsdiscovery base module.""" import os import unittest2 + from mock import patch @@ -11,21 +28,21 @@ from compass.utils import setting_wrapper as setting reload(setting) -from compass.hdsdiscovery.base import BaseSnmpVendor from compass.hdsdiscovery.base import BaseSnmpMacPlugin +from compass.hdsdiscovery.base import BaseSnmpVendor from compass.utils import flags from compass.utils import logsetting class MockSnmpVendor(BaseSnmpVendor): - """snmp vendor mock class""" + """snmp vendor mock class.""" def __init__(self): BaseSnmpVendor.__init__(self, ["MockVendor", "FakeVendor"]) class TestBaseSnmpMacPlugin(unittest2.TestCase): - """teset base snmp plugin class""" + """teset base snmp plugin class.""" def setUp(self): super(TestBaseSnmpMacPlugin, self).setUp() @@ -40,14 +57,14 @@ class TestBaseSnmpMacPlugin(unittest2.TestCase): @patch('compass.hdsdiscovery.utils.snmpget_by_cl') def test_get_port(self, mock_snmpget): - """test snmp get port""" + """test snmp get port.""" mock_snmpget.return_value = 'IF-MIB::ifName.4 = STRING: ge-1/1/4' result = self.test_plugin.get_port('4') self.assertEqual('4', result) @patch('compass.hdsdiscovery.utils.snmpget_by_cl') def test_get_vlan_id(self, mock_snmpget): - """test snmp get vlan""" + """test snmp get vlan.""" # Port is None self.assertIsNone(self.test_plugin.get_vlan_id(None)) @@ -57,7 +74,7 @@ class TestBaseSnmpMacPlugin(unittest2.TestCase): self.assertEqual('100', result) def test_get_mac_address(self): - """tet snmp get mac address""" + """tet snmp get mac address.""" # Correct input for mac numbers mac_numbers = '0.224.129.230.57.173'.split('.') mac = self.test_plugin.get_mac_address(mac_numbers) @@ -70,7 +87,7 @@ class TestBaseSnmpMacPlugin(unittest2.TestCase): class BaseTest(unittest2.TestCase): - """base test class""" + """base test class.""" def setUp(self): super(BaseTest, self).setUp() @@ -80,7 +97,7 @@ class BaseTest(unittest2.TestCase): super(BaseTest, self).tearDown() def test_base_snmp_vendor(self): - """test base snmp vendor""" + """test base snmp vendor.""" fake = MockSnmpVendor() credential = {"version": "2c", diff --git a/compass/tests/hdsdiscovery/test_hdsdiscovery.py b/compass/tests/hdsdiscovery/test_hdsdiscovery.py old mode 100644 new mode 100755 index 4ad6c134..63d17357 --- a/compass/tests/hdsdiscovery/test_hdsdiscovery.py +++ b/compass/tests/hdsdiscovery/test_hdsdiscovery.py @@ -1,6 +1,23 @@ -"""test hdsdiscovery module""" +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test hdsdiscovery module.""" import os import unittest2 + from mock import patch @@ -19,7 +36,7 @@ from compass.utils import logsetting class HuaweiTest(unittest2.TestCase): - """test huawei switch snmp get""" + """test huawei switch snmp get.""" def setUp(self): super(HuaweiTest, self).setUp() @@ -34,7 +51,7 @@ class HuaweiTest(unittest2.TestCase): super(HuaweiTest, self).tearDown() def test_is_this_vendor(self): - """test device vendor is haiwei""" + """test device vendor is haiwei.""" #Credential's keyword is incorrect self.assertFalse( self.huawei.is_this_vendor(self.correct_host, @@ -57,7 +74,7 @@ class HuaweiTest(unittest2.TestCase): class HuaweiMacTest(unittest2.TestCase): - """test get mac from huawei device""" + """test get mac from huawei device.""" def setUp(self): super(HuaweiMacTest, self).setUp() @@ -71,7 +88,7 @@ class HuaweiMacTest(unittest2.TestCase): super(HuaweiMacTest, self).tearDown() def test_process_data(self): - """get progress data function""" + """get progress data function.""" # GET operation haven't been implemeneted. self.assertIsNone(self.mac_plugin.process_data('GET')) @@ -80,7 +97,7 @@ from compass.hdsdiscovery.vendors.ovswitch.plugins.mac import Mac as OVSMac class OVSMacTest(unittest2.TestCase): - """ovs switch test""" + """ovs switch test.""" def setUp(self): super(OVSMacTest, self).setUp() @@ -90,7 +107,7 @@ class OVSMacTest(unittest2.TestCase): @patch('compass.hdsdiscovery.utils.ssh_remote_execute') def test_scan(self, ovs_mock): - """test scan ovs switch""" + """test scan ovs switch.""" ovs_mock.return_value = [] mac_instance = OVSMac(self.host, self.credential) self.assertIsNone(mac_instance.scan()) @@ -103,7 +120,7 @@ class OVSMacTest(unittest2.TestCase): class HDManagerTest(unittest2.TestCase): - """test HDManager""" + """test HDManager.""" def setUp(self): super(HDManagerTest, self).setUp() @@ -118,7 +135,7 @@ class HDManagerTest(unittest2.TestCase): @patch('compass.hdsdiscovery.hdmanager.HDManager.get_sys_info') def test_get_vendor(self, sys_info_mock): - """test get_vendor""" + """test get_vendor.""" # Incorrect ip self.assertIsNone(self.manager.get_vendor('1234.1.1.1', self.correct_credential)[0]) @@ -161,7 +178,7 @@ class HDManagerTest(unittest2.TestCase): @patch('compass.hdsdiscovery.hdmanager.HDManager.get_sys_info') def test_is_valid_vendor(self, sys_info_mock): - """test is_valid_vendor""" + """test is_valid_vendor.""" #non-exsiting vendor self.assertFalse(self.manager.is_valid_vendor(self.correct_host, self.correct_credential, @@ -184,7 +201,7 @@ class HDManagerTest(unittest2.TestCase): 'pica8')) def test_learn(self): - """test learn""" + """test learn.""" #non-exsiting plugin self.assertIsNone(self.manager.learn(self.correct_host, self.correct_credential, @@ -200,14 +217,14 @@ from compass.hdsdiscovery import utils class UtilsTest(unittest2.TestCase): - """hdsdiscovery util test class""" + """hdsdiscovery util test class.""" def setUp(self): super(UtilsTest, self).setUp() logsetting.init() def test_load_module(self): - """test load_module""" + """test load_module.""" self.assertIsNone(utils.load_module('xxx', 'fake/path/to/module')) diff --git a/compass/tests/utils/__init__.py b/compass/tests/utils/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/tests/utils/__init__.py +++ b/compass/tests/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/utils/test_util.py b/compass/tests/utils/test_util.py old mode 100644 new mode 100755 index 1026b275..be9aa05f --- a/compass/tests/utils/test_util.py +++ b/compass/tests/utils/test_util.py @@ -1,4 +1,21 @@ -"""test util module""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test util module. + + .. moduleauthor:: Xiaodong Wang +""" import os import unittest2 @@ -16,7 +33,7 @@ from compass.utils import util class TestDictMerge(unittest2.TestCase): - """Test dict merge""" + """Test dict merge.""" def setUp(self): super(TestDictMerge, self).setUp() @@ -26,21 +43,21 @@ class TestDictMerge(unittest2.TestCase): super(TestDictMerge, self).tearDown() def test_simple_merge(self): - """simple test of merge""" + """simple test of merge.""" lhs = {1: 1} rhs = {2: 2} util.merge_dict(lhs, rhs) self.assertEqual(lhs, {1: 1, 2: 2}) def test_recursive_merge(self): - """test merge recursively""" + """test merge recursively.""" lhs = {1: {2: 3}} rhs = {1: {3: 4}} util.merge_dict(lhs, rhs) self.assertEqual(lhs, {1: {2: 3, 3: 4}}) def test_merge_override(self): - """test merge override""" + """test merge override.""" lhs = {1: 1} rhs = {1: 2} util.merge_dict(lhs, rhs) @@ -52,7 +69,7 @@ class TestDictMerge(unittest2.TestCase): self.assertEqual(lhs, {1: {2: 4, 3: 5, 4: 6}}) def test_merge_not_override(self): - """test merge not override""" + """test merge not override.""" lhs = {1: 1} rhs = {1: 2} util.merge_dict(lhs, rhs, False) @@ -64,7 +81,7 @@ class TestDictMerge(unittest2.TestCase): self.assertEqual(lhs, {1: {2: 3, 3: 5, 4: 6}}) def test_change_after_merge(self): - """test change after merge""" + """test change after merge.""" lhs = {1: {2: 3}} rhs = {1: {3: [4, 5, 6]}} util.merge_dict(lhs, rhs) @@ -75,7 +92,7 @@ class TestDictMerge(unittest2.TestCase): self.assertEqual(rhs, {1: {3: [4, 5, 6, 7]}}) def test_lhs_rhs_notdict(self): - """test merge not dict""" + """test merge not dict.""" lhs = [1, 2, 3] rhs = {1: 2} self.assertRaises(TypeError, util.merge_dict, (lhs, rhs)) @@ -85,10 +102,10 @@ class TestDictMerge(unittest2.TestCase): class TestOrderKeys(unittest2.TestCase): - """test order keys""" + """test order keys.""" def test_simple_order_keys(self): - """test simple order keys""" + """test simple order keys.""" keys = [1, 2, 3, 4, 5] orders = [3, 4, 5] ordered_keys = util.order_keys(keys, orders) @@ -102,21 +119,21 @@ class TestOrderKeys(unittest2.TestCase): self.assertEqual(ordered_keys, [3, 4, 1, 2, 5]) def test_order_keys_with_multidot(self): - """test order keys with multi dots in it""" + """test order keys with multi dots in it.""" keys = [1, 2, 3, 4, 5] orders = [3, '.', 4, '.', 5] ordered_keys = util.order_keys(keys, orders) self.assertEqual(ordered_keys, [3, 1, 2, 4, 5]) def test_others_in_orders(self): - """test other key in order""" + """test other key in order.""" keys = [1, 2, 3, 4, 5] orders = [3, '.', 5, 6] ordered_keys = util.order_keys(keys, orders) self.assertEqual(ordered_keys, [3, 1, 2, 4, 5]) def test_keys_orders_notlist(self): - """test keys not in order""" + """test keys not in order.""" keys = {1: 1} orders = [3, 4, 5] self.assertRaises(TypeError, util.order_keys, keys, orders) @@ -127,19 +144,19 @@ class TestOrderKeys(unittest2.TestCase): class TestIsInstanceOf(unittest2.TestCase): - """test isinstanceof""" + """test isinstanceof function.""" def test_isinstance(self): - """test isinstance""" + """test isinstance.""" self.assertTrue(util.is_instance({}, [dict, list])) self.assertFalse(util.is_instance({}, [str, list])) self.assertFalse(util.is_instance({}, [])) class TestGetListWithPossibility(unittest2.TestCase): - """test get list with possibility""" + """test get list with possibility.""" def test_simple_case(self): - """test simple case""" + """test simple case.""" lists = [['role1'], ['role2'], ['role3']] self.assertEqual(util.flat_lists_with_possibility(lists), ['role1', 'role2', 'role3']) diff --git a/compass/utils/__init__.py b/compass/utils/__init__.py index e69de29b..4ee55a4c 100644 --- a/compass/utils/__init__.py +++ b/compass/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/utils/celeryconfig_wrapper.py b/compass/utils/celeryconfig_wrapper.py index fe4e000f..4fa51ee7 100644 --- a/compass/utils/celeryconfig_wrapper.py +++ b/compass/utils/celeryconfig_wrapper.py @@ -1,4 +1,21 @@ -"""celeryconfig wrapper.""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""celeryconfig wrapper. + + .. moduleauthor:: Xiaodong Wang +""" import logging import os.path diff --git a/compass/utils/daemonize.py b/compass/utils/daemonize.py index 45db7e7f..f02bfb99 100644 --- a/compass/utils/daemonize.py +++ b/compass/utils/daemonize.py @@ -1,11 +1,25 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to provider util functions in all compass code .. moduleauthor:: Xiaodong Wang """ import daemon import logging -import sys import signal +import sys import time from compass.utils import flags @@ -30,7 +44,7 @@ def handle_term(signum, frame): def _daemon(callback, run_interval): - """help function to run callback in daemon""" + """help function to run callback in daemon.""" global BUSY signal.signal(signal.SIGTERM, handle_term) signal.signal(signal.SIGHUP, handle_term) diff --git a/compass/utils/flags.py b/compass/utils/flags.py index 2db8f9b0..21a74a7e 100644 --- a/compass/utils/flags.py +++ b/compass/utils/flags.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to load flags. .. moduleauthor:: Xiaodong Wang diff --git a/compass/utils/logsetting.py b/compass/utils/logsetting.py index 2ca3e018..b82540c9 100644 --- a/compass/utils/logsetting.py +++ b/compass/utils/logsetting.py @@ -1,3 +1,17 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to setup logging configuration. .. moduleauthor:: Xiaodong Wang @@ -6,8 +20,8 @@ import logging import logging.handlers import os -import sys import os.path +import sys from compass.utils import flags from compass.utils import setting_wrapper as setting diff --git a/compass/utils/setting_wrapper.py b/compass/utils/setting_wrapper.py index 9aeabafc..afcc6acf 100644 --- a/compass/utils/setting_wrapper.py +++ b/compass/utils/setting_wrapper.py @@ -1,4 +1,21 @@ -"""comapss setting wrapper.""" +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""comapss setting wrapper. + + .. moduleauthor:: Xiaodong Wang ,xiaodongwang@huawei.com> +""" import logging import os diff --git a/compass/utils/util.py b/compass/utils/util.py index 66ea975c..d8951c5b 100644 --- a/compass/utils/util.py +++ b/compass/utils/util.py @@ -1,11 +1,24 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Module to provider util functions in all compass code .. moduleauthor:: Xiaodong Wang """ +import copy import re -from copy import deepcopy - def merge_dict(lhs, rhs, override=True): """Merge nested right dict into left nested dict recursively. @@ -38,7 +51,7 @@ def merge_dict(lhs, rhs, override=True): merge_dict(lhs[key], value, override) else: if override or key not in lhs: - lhs[key] = deepcopy(value) + lhs[key] = copy.deepcopy(value) def order_keys(keys, orders): @@ -106,7 +119,7 @@ def flat_lists_with_possibility(lists): ['a', 'a', 'a', 'a'], ['b', 'b'], ['c'], the expected output is ['a', 'b', 'c', 'a', 'a', 'b', 'a']. """ - lists = deepcopy(lists) + lists = copy.deepcopy(lists) lists = sorted(lists, key=len, reverse=True) list_possibility = [] max_index = 0 @@ -146,23 +159,22 @@ def pretty_print(*contents): def get_clusters_from_str(clusters_str): """get clusters from string.""" clusters = {} - for clusterid_and_hostnames in clusters_str.split(';'): - if not clusterid_and_hostnames: + for cluster_and_hosts in clusters_str.split(';'): + if not cluster_and_hosts: continue - if ':' in clusterid_and_hostnames: - clusterid_str, hostnames_str = clusterid_and_hostnames.split( + if ':' in cluster_and_hosts: + cluster_str, hosts_str = cluster_and_hosts.split( ':', 1) else: - clusterid_str = clusterid_and_hostnames - hostnames_str = '' + cluster_str = cluster_and_hosts + hosts_str = '' - clusterid = int(clusterid_str) - hostnames = [ - hostname for hostname in hostnames_str.split(',') - if hostname + hosts = [ + host for host in hosts_str.split(',') + if host ] - clusters[clusterid] = hostnames + clusters[cluster_str] = hosts return clusters @@ -335,7 +347,11 @@ def get_properties_name_from_str(properties_name_str): def print_properties(properties): """print properties.""" print '-----------------------------------------------' - for property_name, property_value in properties.items(): - print '%s=%s' % (property_name, property_value) + for property_item in properties: + property_pairs = [] + for property_name, property_value in property_item.items(): + property_pairs.append('%s=%s' % (property_name, property_value)) + + print ','.join(property_pairs) print '-----------------------------------------------' diff --git a/conf/global_config b/conf/global_config index eb1e7b5f..3bff8f3e 100644 --- a/conf/global_config +++ b/conf/global_config @@ -1,8 +1,8 @@ networking = { 'global': { - 'default_no_proxy': ['127.0.0.1', 'localhost'], + 'default_no_proxy': ['127.0.0.1', 'localhost', '$compass_ip', '$compass_hostname'], 'search_path_pattern': '%(clusterid)s.%(search_path)s %(search_path)s', - 'noproxy_pattern': '%(hostname)s.%(clusterid)s,%(ip)s' + 'noproxy_pattern': '%(hostname)s,%(ip)s' }, 'interfaces': { 'management': { diff --git a/conf/setting b/conf/setting index 1977114c..bcbcaedb 100644 --- a/conf/setting +++ b/conf/setting @@ -15,7 +15,7 @@ PACKAGE_INSTALLER = 'chef' CHEF_INSTALLER_URL = 'https://localhost' CHEF_GLOBAL_DATABAG_NAME = 'env_default' INSTALLATION_LOGDIR = '/var/log/cobbler/anamon' -DEFAULT_LOGLEVEL = 'info' +DEFAULT_LOGLEVEL = 'debug' DEFAULT_LOGDIR = '/var/log/compass' DEFAULT_LOGINTERVAL = 1 DEFAULT_LOGINTERVAL_UNIT = 'h' diff --git a/ez_setup.py b/ez_setup.py index 6830344f..41d35d52 100644 --- a/ez_setup.py +++ b/ez_setup.py @@ -1,4 +1,19 @@ #!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Bootstrap setuptools installation If you want to use setuptools in your package's setup.py, just include this @@ -13,13 +28,13 @@ the appropriate options to ``use_setuptools()``. This file can also be run as a script to install or upgrade setuptools. """ +import optparse import os import shutil -import sys -import tempfile -import tarfile -import optparse import subprocess +import sys +import tarfile +import tempfile from distutils import log @@ -35,13 +50,13 @@ DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" def _python_cmd(*args): - """run cmd in python""" + """run cmd in python.""" args = (sys.executable,) + args return subprocess.call(args) == 0 def _install(tarball, install_args=()): - """install tarball""" + """install tarball.""" # extracting the tarball tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) @@ -71,7 +86,7 @@ def _install(tarball, install_args=()): def _build_egg(egg, tarball, to_dir): - """build egg""" + """build egg.""" # extracting the tarball tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) @@ -102,7 +117,7 @@ def _build_egg(egg, tarball, to_dir): def _do_download(version, download_base, to_dir, download_delay): - """download package""" + """download package.""" egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' % (version, sys.version_info[0], sys.version_info[1])) if not os.path.exists(egg): @@ -117,7 +132,7 @@ def _do_download(version, download_base, to_dir, download_delay): def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, download_delay=15): - """use setuptools to do the setup""" + """use setuptools to do the setup.""" # making sure we use the absolute path to_dir = os.path.abspath(to_dir) was_imported = 'pkg_resources' in sys.modules or \ @@ -243,7 +258,8 @@ def _extractall(self, path=".", members=None): def _build_install_args(options): - """ + """Build install args + Build the arguments to 'python setup.py install' on the setuptools package """ install_args = [] @@ -258,7 +274,7 @@ def _build_install_args(options): def _parse_args(): - """Parse the command line for options""" + """Parse the command line for options.""" parser = optparse.OptionParser() parser.add_option( '--user', dest='user_install', action='store_true', default=False, @@ -273,7 +289,7 @@ def _parse_args(): def main(version=DEFAULT_VERSION): - """Install or upgrade setuptools and EasyInstall""" + """Install or upgrade setuptools and EasyInstall.""" options = _parse_args() tarball = download_setuptools(download_base=options.download_base) return _install(tarball, _build_install_args(options)) diff --git a/install/cobbler.sh b/install/cobbler.sh index 6085a979..cf5c315a 100755 --- a/install/cobbler.sh +++ b/install/cobbler.sh @@ -92,14 +92,21 @@ CBLR_PASSWD=${CBLR_PASSWD:-"cobbler"} # update cobbler config sudo cp -rn /var/lib/cobbler/snippets /root/backup/cobbler/ sudo cp -rn /var/lib/cobbler/kickstarts/ /root/backup/cobbler/ +sudo cp -rn /var/lib/cobbler/triggers /root/backup/cobbler/ sudo rm -rf /var/lib/cobbler/snippets/* sudo cp -rf $ADAPTER_HOME/cobbler/snippets/* /var/lib/cobbler/snippets/ +sudo cp -rf $ADAPTER_HOME/cobbler/triggers/* /var/lib/cobbler/triggers/ sudo chmod 777 /var/lib/cobbler/snippets -sudo chmod 666 /var/lib/cobbler/snippets/* +sudo chmod -R 666 /var/lib/cobbler/snippets/* +sudo chmod -R 755 /var/lib/cobbler/triggers sudo sed -i "s/# \$compass_ip \$compass_hostname/$ipaddr $HOSTNAME/g" /var/lib/cobbler/snippets/hosts sudo rm -f /var/lib/cobbler/kickstarts/default.ks sudo cp -rf $ADAPTER_HOME/cobbler/kickstarts/default.ks /var/lib/cobbler/kickstarts/ sudo chmod 666 /var/lib/cobbler/kickstarts/default.ks +sudo mkdir /var/www/cblr_ks +sudo chmod 755 /var/www/cblr_ks +sudo cp -rf $ADAPTER_HOME/cobbler/conf/cobbler.conf /etc/httpd/conf.d/ +chmod 644 /etc/httpd/conf.d/cobbler.conf sudo cp -rn /etc/xinetd.d /root/backup/ sudo sed -i 's/disable\([ \t]\+\)=\([ \t]\+\)yes/disable\1=\2no/g' /etc/xinetd.d/rsync diff --git a/install/compass.sh b/install/compass.sh index db45bace..196b9ca1 100755 --- a/install/compass.sh +++ b/install/compass.sh @@ -42,7 +42,8 @@ fi sudo sed -i "/COBBLER_INSTALLER_URL/c\COBBLER_INSTALLER_URL = 'http:\/\/$ipaddr/cobbler_api'" /etc/compass/setting sudo sed -i "/CHEF_INSTALLER_URL/c\CHEF_INSTALLER_URL = 'https:\/\/$ipaddr/'" /etc/compass/setting - +sudo sed -i "s/\$compass_ip/$ipaddr/g" /etc/compass/global_config +sudo sed -i "s/\$compass_hostname/$HOSTNAME/g" /etc/compass/global_config # add cookbooks, databags and roles sudo mkdir -p /var/chef/cookbooks/ @@ -52,13 +53,25 @@ sudo cp -r $ADAPTER_HOME/chef/cookbooks/* /var/chef/cookbooks/ sudo cp -r $ADAPTER_HOME/chef/databags/* /var/chef/databags/ sudo cp -r $ADAPTER_HOME/chef/roles/* /var/chef/roles/ -sudo chmod +x /opt/compass/bin/addcookbooks.py -sudo chmod +x /opt/compass/bin/adddatabags.py +sudo chmod +x /opt/compass/bin/addcookbooks.py +sudo chmod +x /opt/compass/bin/adddatabags.py sudo chmod +x /opt/compass/bin/addroles.py sudo /opt/compass/bin/addcookbooks.py --cookbooks_dir=/var/chef/cookbooks +if [[ "$?" != "0" ]]; then + echo "failed to add cookbooks" + exit 1 +fi sudo /opt/compass/bin/adddatabags.py --databags_dir=/var/chef/databags +if [[ "$?" != "0" ]]; then + echo "failed to add databags" + exit 1 +fi sudo /opt/compass/bin/addroles.py --roles_dir=/var/chef/roles +if [[ "$?" != "0" ]]; then + echo "failed to add roles" + exit 1 +fi # copy the chef validatation keys to cobbler snippets sudo cp -rf /etc/chef-server/chef-validator.pem /var/lib/cobbler/snippets/chef-validator.pem @@ -73,6 +86,14 @@ else echo "httpd has already started" fi +sudo service redis status +if [[ "$?" != "0" ]]; then + echo "redis is not started" + exit 1 +else + echo "redis has already started" +fi + sudo service compassd status if [[ "$?" != "0" ]]; then echo "compassd is not started" diff --git a/install/dependency.sh b/install/dependency.sh index c44a74aa..3b32d3d5 100755 --- a/install/dependency.sh +++ b/install/dependency.sh @@ -2,7 +2,7 @@ echo 'Installing Required packages for Compass...' -sudo yum install -y rsyslog logrotate ntp iproute openssh-clients python git wget python-setuptools python-netaddr python-flask python-flask-sqlalchemy python-amqplib amqp python-paramiko python-mock mod_wsgi httpd squid dhcp bind rsync yum-utils xinetd tftp-server gcc net-snmp-utils net-snmp net-snmp-python python-daemon unzip openssl openssl098e ca-certificates +sudo yum install -y rsyslog logrotate ntp iproute openssh-clients python git wget python-setuptools python-netaddr python-flask python-flask-sqlalchemy python-amqplib amqp python-paramiko python-mock mod_wsgi httpd squid dhcp bind rsync yum-utils xinetd tftp-server gcc net-snmp-utils net-snmp net-snmp-python python-daemon unzip openssl openssl098e ca-certificates redis python-redis if [[ "$?" != "0" ]]; then echo "failed to install yum dependency" exit 1 @@ -28,5 +28,6 @@ sudo chkconfig named on sudo chkconfig sshd on sudo chkconfig rsyslog on sudo chkconfig ntpd on +sudo chkconfig redis on sudo chkconfig iptables off sudo chkconfig ip6tables off diff --git a/misc/ci/pxe-deploy.sh b/misc/ci/pxe-deploy.sh index 9647b6b5..bba64ec2 100755 --- a/misc/ci/pxe-deploy.sh +++ b/misc/ci/pxe-deploy.sh @@ -1,4 +1,6 @@ #!/bin/bash -xe +ln -s /var/log/cobbler/anamon cobbler_logs +ln -s /var/log/compass compass_logs cp compass-core/compass/apiclient/example.py /tmp/test.py chmod +x /tmp/test.py virsh destroy pxe01 diff --git a/misc/squid/squid.conf b/misc/squid/squid.conf index a6312e7f..04e42d56 100644 --- a/misc/squid/squid.conf +++ b/misc/squid/squid.conf @@ -9,9 +9,6 @@ acl to_localhost dst 127.0.0.0/8 0.0.0.0/32 ::1 # Adapt to list your (internal) IP networks from where browsing # should be allowed acl localnet src $subnet # the subnet of local network -acl localnet src 10.0.0.0/8 # RFC1918 possible internal network -acl localnet src 172.16.0.0/12 # RFC1918 possible internal network -acl localnet src 192.168.0.0/16 # RFC1918 possible internal network acl localnet src fc00::/7 # RFC 4193 local private network range acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines diff --git a/requirements.txt b/requirements.txt index 623e7de6..08c1c1b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,5 @@ netaddr paramiko simplejson requests +PyChef +redis diff --git a/setup.py b/setup.py index d561cb70..7211b8e0 100644 --- a/setup.py +++ b/setup.py @@ -1,20 +1,52 @@ +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bootstrap setuptools installation + +If you want to use setuptools in your package's setup.py, just include this +file in the same directory with it, and add this to the top of your setup.py:: + + from ez_setup import use_setuptools + use_setuptools() + +If you want to require a specific version of setuptools, set a download +mirror, or use an alternate download directory, you can do so by supplying +the appropriate options to ``use_setuptools()``. + +This file can also be run as a script to install or upgrade setuptools. +""" + """setup script.""" try: - from setuptools import setup, find_packages - from setuptools.command.test import test as TestCommand + from setuptools import find_packages except ImportError: from ez_setup import use_setuptools use_setuptools() - from setuptools import setup - from setuptools.command.test import test as TestCommand -import sys +from setuptools.command.test import test as TestCommand +from setuptools import setup + + import os +import sys class Tox(TestCommand): - """Tox to do the setup""" + """Tox to do the setup.""" def finalize_options(self): TestCommand.finalize_options(self) diff --git a/test-requirements.txt b/test-requirements.txt index 3bedbfdf..406995e0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,8 @@ discover mock unittest2 +testtools>=0.9.32 testrepository>=0.0.17 mimeparse +coverage>=3.6 +hacking>=0.8.0,<0.9 diff --git a/tox.ini b/tox.ini index 5e1211f6..1b58b2a2 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,8 @@ setenv = VIRTUAL_ENV={envdir} LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=C + COMPASS_IGNORE_SETTING=1 + deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt install_command = pip install -U {opts} {packages} @@ -29,7 +31,7 @@ commands = python setup.py testr --coverage --testr-args='{posargs}' downloadcache = ~/cache/pip [flake8] -ignore = E126,H703 +ignore = H302,H233,H803 show-source = true builtins = _ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools