x
/
compass-core
Code Issues Proposed changes

add code to match new pep8 style checking and fix bugs. 

Change-Id: Id5fd48b8149d3934e4854366e59475f37624ec24
This commit is contained in:
xiaodongwang 2014-03-07 11:15:14 -08:00 committed by zhaoxinyu
parent 479f9923eb
commit ece0a3e936
146 changed files with 3667 additions and 1214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -29,6 +29,7 @@ pip-log.txt
.tox
nosetests.xml
.testrepository
cover
# Translations
*.mo

13
bin/__init__.py
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

24
bin/chef/addcookbooks.py
View File

@ -1,9 +1,24 @@
#!/usr/bin/env python
#
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""import cookbooks to chef server."""
import logging
import os
import os.path
import sys
from compass.utils import flags
from compass.utils import logsetting
@ -15,13 +30,16 @@ flags.add('cookbooks_dir',
def main():
"""main entry"""
"""main entry."""
flags.init()
logsetting.init()
cookbooks_dir = flags.OPTIONS.cookbooks_dir
logging.info('add cookbooks %s', cookbooks_dir)
cmd = "knife cookbook upload --all --cookbook-path %s" % cookbooks_dir
os.system(cmd)
status = os.system(cmd)
logging.info('run cmd %s returns %s', cmd, status)
if status:
sys.exit(1)
if __name__ == '__main__':

23
bin/chef/adddatabags.py
View File

@ -1,8 +1,24 @@
#!/usr/bin/env python
#
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""import databags to chef server."""
import logging
import os
import os.path
import sys
from compass.utils import flags
from compass.utils import logsetting
@ -14,7 +30,7 @@ flags.add('databags_dir',
def main():
"""main entry"""
"""main entry."""
flags.init()
logsetting.init()
databags = []
@ -35,7 +51,10 @@ def main():
logging.info('add databag item %s to databag %s',
databag_item, databag)
cmd = 'knife data bag from file %s %s' % (databag, databag_item)
os.system(cmd)
status = os.system(cmd)
logging.info('run cmd %s returns %s', cmd, status)
if status:
sys.exit(1)
if __name__ == '__main__':

25
bin/chef/addroles.py
View File

@ -1,8 +1,24 @@
#!/usr/bin/env python
"""script to import roles to chef server"""
#
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""script to import roles to chef server."""
import logging
import os
import os.path
import sys
from compass.utils import flags
from compass.utils import logsetting
@ -14,7 +30,7 @@ flags.add('roles_dir',
def main():
"""main entry"""
"""main entry."""
flags.init()
logsetting.init()
rolelist = []
@ -27,7 +43,10 @@ def main():
for role in rolelist:
logging.info('add role %s', role)
cmd = "knife role from file %s" % role
os.system(cmd)
status = os.system(cmd)
logging.info('run cmd %s returns %s', cmd, status)
if status:
sys.exit(1)
if __name__ == '__main__':

29
bin/cobbler/migrate_ks.py
View File

@ -1,29 +0,0 @@
#!/usr/bin/python
"""script to migrate rendered kickstart files from cobbler to outside."""
import xmlrpclib
import logging
from compass.utils import setting_wrapper as setting
def main():
"""main entry"""
remote = xmlrpclib.Server(setting.COBBLER_INSTALLER_URL, allow_none=True)
token = remote.login(*setting.COBBLER_INSTALLER_TOKEN)
systems = remote.get_systems(token)
for system in systems:
data = remote.generate_kickstart('', system['name'])
try:
with open(
'/var/www/cblr_ks/%s' % system['name'], 'w'
) as kickstart_file:
logging.info("Migrating kickstart for %s", system['name'])
kickstart_file.write(data)
except Exception as error:
logging.error("Directory /var/www/cblr_ks/ does not exist.")
logging.exception(error)
if __name__ == '__main__':
logging.info("Running kickstart migration")
main()

50
bin/csvdeploy.py
View File

@ -1,11 +1,29 @@
#!/usr/bin/python
#
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""deploy cluster from csv file."""
import ast
import copy
import csv
import os
import re
import csv
import ast
import sys
from copy import deepcopy
from multiprocessing import Process, Queue
from multiprocessing import Process
from multiprocessing import Queue
from optparse import OptionParser
try:
@ -35,8 +53,7 @@ TABLES = {
def start(csv_dir, compass_url):
""" Start deploy both failed clusters and new clusters.
"""
"""Start deploy both failed clusters and new clusters."""
# Get clusters and hosts data from CSV
clusters_data = get_csv('cluster.csv', csv_dir)
hosts_data = get_csv('cluster_host.csv', csv_dir)
@ -95,11 +112,15 @@ def start(csv_dir, compass_url):
def get_csv(fname, csv_dir):
""" Parse csv files into python variables. all nested fields in db will be
assembled.
"""Parse csv files into python variables.
.. note::
all nested fields in db will be assembled.
:param fname: CSV file name
:param csv_dir: CSV files directory
Return a list of dict which key is column name and value is its data.
:returns: list of dict which key is column name and value is its data.
"""
headers = []
rows = []
@ -168,7 +189,7 @@ def merge_dict(lhs, rhs, override=True):
merge_dict(lhs[key], value, override)
else:
if override or key not in lhs:
lhs[key] = deepcopy(value)
lhs[key] = copy.deepcopy(value)
class _APIClient(Client):
@ -180,10 +201,11 @@ class _APIClient(Client):
return self._put(url, data=data)
def execute(self, cluster_data, hosts_data, resp_results):
""" The process including create or update a cluster and the cluster
configuration, add or update a host in the cluster, and deploy
the updated hosts.
:param cluster_data: the dictionary of cluster data
"""The process including create or update a cluster and the cluster
configuration, add or update a host in the cluster, and deploy
the updated hosts.
:param cluster_data: the dictionary of cluster data
"""
cluster_id = cluster_data['id']
code, resp = self.get_cluster(cluster_id)

46
bin/manage_db.py
View File

@ -1,4 +1,19 @@
#!/usr/bin/python
#
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""utility binary to manage database."""
import os
import os.path
@ -7,16 +22,23 @@ import sys
from flask.ext.script import Manager
from compass.actions import clean_deployment
from compass.actions import reinstall
from compass.actions import deploy
from compass.actions import clean_installing_progress
from compass.actions import deploy
from compass.actions import reinstall
from compass.actions import search
from compass.api import app
from compass.config_management.utils import config_manager
from compass.db import database
from compass.db.model import Adapter, Role, Switch, SwitchConfig
from compass.db.model import Machine, HostState, ClusterState
from compass.db.model import Cluster, ClusterHost, LogProgressingHistory
from compass.db.model import Adapter
from compass.db.model import Cluster
from compass.db.model import ClusterHost
from compass.db.model import ClusterState
from compass.db.model import HostState
from compass.db.model import LogProgressingHistory
from compass.db.model import Machine
from compass.db.model import Role
from compass.db.model import Switch
from compass.db.model import SwitchConfig
from compass.tasks.client import celery
from compass.utils import flags
from compass.utils import logsetting
@ -76,14 +98,14 @@ TABLE_MAPPING = {
@app_manager.command
def list_config():
"List the configuration"
"List the commands."
for key, value in app.config.items():
print key, value
@app_manager.command
def checkdb():
"""check if db exists"""
"""check if db exists."""
if setting.DATABASE_TYPE == 'file':
if os.path.exists(setting.DATABASE_FILE):
sys.exit(0)
@ -95,24 +117,24 @@ def checkdb():
@app_manager.command
def createdb():
"""Creates database from sqlalchemy models"""
"""Creates database from sqlalchemy models."""
if setting.DATABASE_TYPE == 'file':
if os.path.exists(setting.DATABASE_FILE):
os.remove(setting.DATABASE_FILE)
database.create_db()
if setting.DATABASE_TYPE == 'file':
os.chmod(setting.DATABASE_FILE, 0777)
os.chmod(setting.DATABASE_FILE, 0o777)
@app_manager.command
def dropdb():
"""Drops database from sqlalchemy models"""
"""Drops database from sqlalchemy models."""
database.drop_db()
@app_manager.command
def createtable():
"""Create database table by --table_name"""
"""Create database table."""
if not flags.OPTIONS.table_name:
print 'flag --table_name is missing'
return
@ -127,7 +149,7 @@ def createtable():
@app_manager.command
def droptable():
"""Drop database table by --talbe_name"""
"""Drop database table."""
if not flags.OPTIONS.table_name:
print 'flag --table_name is missing'
return

17
bin/poll_switch.py
View File

@ -1,4 +1,19 @@
#!/usr/bin/python
#
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""main script to poll machines which is connected to the switches."""
import functools
import lockfile
@ -31,7 +46,7 @@ flags.add('run_interval',
def pollswitches(switch_ips):
"""poll switch"""
"""poll switch."""
poll_switch_ips = []
with database.session():
poll_switch_ips = util.update_switch_ips(switch_ips)

15
bin/progress_update.py
View File

@ -1,4 +1,19 @@
#!/usr/bin/python
#
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""main script to run as service to update hosts installing progress."""
import functools
import lockfile

38
bin/query_switch.py Normal file → Executable file
View File

@ -1,24 +1,40 @@
#!/usr/bin/python
#
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""query switch."""
import optparse
import Queue
import threading
import time
from Queue import Queue, Empty
from optparse import OptionParser
from compass.apiclient.restful import Client
class AddSwitch(object):
""" A utility class that handles adding a switch and retrieving
corresponding machines associated with the switch. """
"""A utility class that handles adding a switch and retrieving
corresponding machines associated with the switch.
"""
def __init__(self, server_url):
print server_url, " ...."
self._client = Client(server_url)
def add_switch(self, queue, ip, snmp_community):
""" Add a switch with SNMP credentials and retrieve attached
server machines.
"""Add a switch with SNMP credentials and retrieve attached
server machines.
:param queue: The result holder for the machine details.
:type queue: A Queue object(thread-safe).
@ -88,10 +104,10 @@ class AddSwitch(object):
if __name__ == "__main__":
usage = "usage: %prog [options] switch_ips"
parser = OptionParser(usage)
parser = optparse.OptionParser(usage)
parser.add_option("-u", "--server-url", dest="server_url",
default="http://localhost/",
default="http://localhost/api",
help="The Compass Server URL")
parser.add_option("-c", "--community", dest="community",
@ -104,7 +120,7 @@ if __name__ == "__main__":
parser.error("Wrong number of arguments")
threads = []
queue = Queue()
queue = Queue.Queue()
add_switch = AddSwitch(options.server_url)
print "Add switch to the server. This may take a while ..."
@ -122,5 +138,5 @@ if __name__ == "__main__":
try:
ip, result = queue.get(block=False)
print ip, " : ", result
except Empty:
except Queue.Empty:
break

5
bin/refresh.sh
View File

@ -6,6 +6,9 @@ fi
/opt/compass/bin/manage_db.py createdb
/opt/compass/bin/manage_db.py sync_switch_configs
/opt/compass/bin/manage_db.py sync_from_installers
service compassd restart
service httpd restart
service rsyslog restart
service redis restart
redis-cli flushall
service compassd restart

15
bin/runserver.py
View File

@ -1,4 +1,19 @@
#!/usr/bin/python
#
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""main script to start an instance of compass server ."""
import logging

13
compass/__init__.py
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

13
compass/actions/__init__.py
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

33
compass/actions/clean_deployment.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to clean deployment of a given cluster
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
@ -13,16 +27,17 @@ def clean_deployment(cluster_hosts):
"""Clean deployment of clusters.
:param cluster_hosts: clusters and hosts in each cluster to clean.
:type cluster_hosts: dict of int to list of int
:type cluster_hosts: dict of int or str to list of int or str
.. note::
The function should be called out of database session.
"""
logging.debug('clean cluster_hosts: %s', cluster_hosts)
with database.session():
cluster_hosts, os_versions, target_systems = (
util.update_cluster_hosts(cluster_hosts))
manager = ConfigManager()
manager.clean_cluster_and_hosts(
cluster_hosts, os_versions, target_systems)
manager.sync()
with util.lock('serialized_action'):
logging.debug('clean cluster_hosts: %s', cluster_hosts)
with database.session():
cluster_hosts, os_versions, target_systems = (
util.update_cluster_hosts(cluster_hosts))
manager = ConfigManager()
manager.clean_cluster_and_hosts(
cluster_hosts, os_versions, target_systems)
manager.sync()

36
compass/actions/clean_installing_progress.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to clean installing progress of a given cluster
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
@ -13,17 +27,19 @@ def clean_installing_progress(cluster_hosts):
"""Clean installing progress of clusters.
:param cluster_hosts: clusters and hosts in each cluster to clean.
:type cluster_hosts: dict of int to list of int
:type cluster_hosts: dict of int or str to list of int or str
.. note::
The function should be called out of database session.
"""
logging.debug('clean installing progress of cluster_hosts: %s',
cluster_hosts)
with database.session():
cluster_hosts, os_versions, target_systems = (
util.update_cluster_hosts(cluster_hosts))
manager = ConfigManager()
manager.clean_cluster_and_hosts_installing_progress(
cluster_hosts, os_versions, target_systems)
manager.sync()
with util.lock('serialized_action'):
logging.debug(
'clean installing progress of cluster_hosts: %s',
cluster_hosts)
with database.session():
cluster_hosts, os_versions, target_systems = (
util.update_cluster_hosts(cluster_hosts))
manager = ConfigManager()
manager.clean_cluster_and_hosts_installing_progress(
cluster_hosts, os_versions, target_systems)
manager.sync()

70
compass/actions/cli.py
View File

@ -1,11 +1,25 @@
"""Compass Command Line Interface"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Compass Command Line Interface"""
import subprocess
import sys
from subprocess import Popen
from compass.actions.health_check import check
from compass.utils.util import pretty_print
ACTION_MAP = {
"check": "apache celery dhcp dns hds misc os_installer "
"package_installer squid tftp".split(" "),
@ -20,9 +34,7 @@ class BootCLI(object):
return
def run(self, args):
"""
cli takes the commands and calls respective modules
"""
"""cli takes the commands and calls respective modules."""
action = self.get_action(args)
if action is None:
self.print_help()
@ -36,9 +48,10 @@ class BootCLI(object):
@classmethod
def get_action(cls, args):
"""
This method returns an action type.
For 'compass check dhcp' command, it will return 'check'.
"""This method returns an action type.
.. note::
For 'compass check dhcp' command, it will return 'check'.
"""
if len(args) == 1:
return None
@ -48,9 +61,10 @@ class BootCLI(object):
@classmethod
def get_module(cls, action, args):
"""
This method returns a module.
For 'compass check dhcp' command, it will return 'dhcp'.
"""This method returns a module.
.. note::
For 'compass check dhcp' command, it will return 'dhcp'.
"""
if len(args) <= 2:
return None
@ -59,12 +73,12 @@ class BootCLI(object):
return "invalid"
def run_check(self, module=None):
"""
param module default set to None.
This provides a flexible sanity check,
if parameter module is none. Compass checks all modules.
If module specified, Compass will only check such module.
"""This provides a flexible sanity check.
.. note::
param module default set to None.
if parameter module is none. Compass checks all modules.
If module specified, Compass will only check such module.
"""
if module is None:
pretty_print("Starting: Compass Health Check",
@ -101,27 +115,32 @@ class BootCLI(object):
print "Compass Check completes. No problems found, all systems go"
sys.exit(0)
else:
print "Compass has ERRORS shown above. Please fix them before " \
"deploying!"
print (
"Compass has ERRORS shown above. Please fix them before "
"deploying!")
sys.exit(1)
@classmethod
def run_refresh(cls, action=None):
"""Run refresh."""
## TODO: replace refresh.sh with refresh.py
# TODO(xicheng): replace refresh.sh with refresh.py
if action is None:
pretty_print("Refreshing Compass...",
"=================")
Popen(['/opt/compass/bin/refresh.sh'], shell=True)
subprocess.Popen(
['/opt/compass/bin/refresh.sh'], shell=True)
elif action == "db":
pretty_print("Refreshing Compass Database...",
"===================")
Popen(['/opt/compass/bin/manage_db.py createdb'], shell=True)
subprocess.Popen(
['/opt/compass/bin/manage_db.py createdb'], shell=True)
else:
pretty_print("Syncing with Installers...",
"================")
Popen(['/opt/compass/bin/manage_db.py sync_from_installers'],
shell=True)
subprocess.Popen(
['/opt/compass/bin/manage_db.py sync_from_installers'],
shell=True
)
@classmethod
def print_help(cls, module_help=""):
@ -145,12 +164,11 @@ class BootCLI(object):
def main():
"""
Compass cli entry point
"""
"""Compass cli entry point."""
cli = BootCLI()
output = cli.run(sys.argv)
return sys.exit(output)
if __name__ == "__main__":
main()

33
compass/actions/deploy.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to deploy a given cluster
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
@ -13,16 +27,17 @@ def deploy(cluster_hosts):
"""Deploy clusters.
:param cluster_hosts: clusters and hosts in each cluster to deploy.
:type cluster_hosts: dict of int to list of int
:type cluster_hosts: dict of int or str to list of int or str
.. note::
The function should be called out of database session.
"""
logging.debug('deploy cluster_hosts: %s', cluster_hosts)
with database.session():
cluster_hosts, os_versions, target_systems = (
util.update_cluster_hosts(cluster_hosts))
manager = ConfigManager()
manager.install_cluster_and_hosts(
cluster_hosts, os_versions, target_systems)
manager.sync()
with util.lock('serialized_action'):
logging.debug('deploy cluster_hosts: %s', cluster_hosts)
with database.session():
cluster_hosts, os_versions, target_systems = (
util.update_cluster_hosts(cluster_hosts))
manager = ConfigManager()
manager.install_cluster_and_hosts(
cluster_hosts, os_versions, target_systems)
manager.sync()

13
compass/actions/health_check/__init__.py
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

20
compass/actions/health_check/base.py
View File

@ -1,4 +1,18 @@
"""Base class for Compass Health Check"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Base class for Compass Health Check."""
from compass.actions.health_check import utils as health_check_utils
from compass.utils import setting_wrapper as setting
@ -13,10 +27,10 @@ class BaseCheck(object):
self.dist, self.version, self.release = health_check_utils.get_dist()
def _set_status(self, code, message):
"""set status"""
"""set status."""
self.code = code
self.messages.append(message)
def get_status(self):
"""get status"""
"""get status."""
return (self.code, self.messages)

121
compass/actions/health_check/check.py
View File

@ -1,85 +1,96 @@
"""Main Entry Point of Compass Health Check"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from compass.actions.health_check import check_apache as apache
from compass.actions.health_check import check_celery as celery
from compass.actions.health_check import check_dhcp as dhcp
from compass.actions.health_check import check_dns as dns
from compass.actions.health_check import check_hds as hds
from compass.actions.health_check import (
check_os_installer as os_installer)
from compass.actions.health_check import (
check_package_installer as package_installer)
from compass.actions.health_check import check_squid as squid
from compass.actions.health_check import check_tftp as tftp
from compass.actions.health_check import check_misc as misc
"""Main Entry Point of Compass Health Check."""
from compass.actions.health_check import base
from compass.actions.health_check import check_apache
from compass.actions.health_check import check_celery
from compass.actions.health_check import check_dhcp
from compass.actions.health_check import check_dns
from compass.actions.health_check import check_hds
from compass.actions.health_check import check_misc
from compass.actions.health_check import check_os_installer
from compass.actions.health_check import check_package_installer
from compass.actions.health_check import check_squid
from compass.actions.health_check import check_tftp
class BootCheck(base.BaseCheck):
"""health check for all components"""
"""health check for all components."""
def run(self):
"""do health check"""
"""do health check."""
status = {}
status['apache'] = self.check_apache()
status['celery'] = self.check_celery()
status['dhcp'] = self.check_dhcp()
status['dns'] = self.check_dns()
status['hds'] = self.check_hds()
status['os_installer'] = self.check_os_installer()
status['package_installer'] = self.check_package_installer()
status['squid'] = self.check_squid()
status['tftp'] = self.check_tftp()
status['other'] = self.check_misc()
status['apache'] = self._check_apache()
status['celery'] = self._check_celery()
status['dhcp'] = self._check_dhcp()
status['dns'] = self._check_dns()
status['hds'] = self._check_hds()
status['os_installer'] = self._check_os_installer()
status['package_installer'] = self._check_package_installer()
status['squid'] = self._check_squid()
status['tftp'] = self._check_tftp()
status['other'] = self._check_misc()
return status
def check_apache(self):
"""do apache health check"""
checker = apache.ApacheCheck()
def _check_apache(self):
"""do apache health check."""
checker = check_apache.ApacheCheck()
return checker.run()
def check_celery(self):
"""do celery health check"""
checker = celery.CeleryCheck()
def _check_celery(self):
"""do celery health check."""
checker = check_celery.CeleryCheck()
return checker.run()
def check_dhcp(self):
"""do dhcp health check"""
checker = dhcp.DhcpCheck()
def _check_dhcp(self):
"""do dhcp health check."""
checker = check_dhcp.DhcpCheck()
return checker.run()
def check_dns(self):
"""do dns health check"""
checker = dns.DnsCheck()
def _check_dns(self):
"""do dns health check."""
checker = check_dns.DnsCheck()
return checker.run()
def check_hds(self):
"""do hds health check"""
checker = hds.HdsCheck()
def _check_hds(self):
"""do hds health check."""
checker = check_hds.HdsCheck()
return checker.run()
def check_os_installer(self):
"""do os installer health check"""
checker = os_installer.OsInstallerCheck()
def _check_os_installer(self):
"""do os installer health check."""
checker = check_os_installer.OsInstallerCheck()
return checker.run()
def check_package_installer(self):
"""do package installer health check"""
checker = package_installer.PackageInstallerCheck()
def _check_package_installer(self):
"""do package installer health check."""
checker = check_package_installer.PackageInstallerCheck()
return checker.run()
def check_squid(self):
"""do squid health check"""
checker = squid.SquidCheck()
def _check_squid(self):
"""do squid health check."""
checker = check_squid.SquidCheck()
return checker.run()
def check_tftp(self):
"""do tftp health check"""
checker = tftp.TftpCheck()
def _check_tftp(self):
"""do tftp health check."""
checker = check_tftp.TftpCheck()
return checker.run()
def check_misc(self):
"""do misc health check"""
checker = misc.MiscCheck()
def _check_misc(self):
"""do misc health check."""
checker = check_misc.MiscCheck()
return checker.run()

22
compass/actions/health_check/check_apache.py
View File

@ -1,4 +1,18 @@
"""Health Check module for Apache service"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Health Check module for Apache service."""
import socket
import urllib2
@ -12,7 +26,7 @@ class ApacheCheck(base.BaseCheck):
NAME = "Apache Check"
def run(self):
"""do the healthcheck"""
"""do the healthcheck."""
if self.dist in ("centos", "redhat", "fedora", "scientific linux"):
apache_service = 'httpd'
else:
@ -51,7 +65,7 @@ class ApacheCheck(base.BaseCheck):
return True
def check_apache_running(self, apache_service):
"""Checks if Apache service is running on port 80"""
"""Checks if Apache service is running on port 80."""
print "Checking Apache service......",
serv_err_msg = health_check_utils.check_service_running(self.NAME,
@ -70,7 +84,7 @@ class ApacheCheck(base.BaseCheck):
self._set_status(
0,
"[%s]Error: Compass web is not redirected by Apache.")
except:
except Exception:
self._set_status(
0,
"[%s]Error: Apache is not running on Port 80."

27
compass/actions/health_check/check_celery.py
View File

@ -1,7 +1,20 @@
"""Health Check module for Celery"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
"""Health Check module for Celery."""
import commands
import os
from celery.task.control import inspect
@ -11,10 +24,10 @@ from compass.actions.health_check import utils as health_check_utils
class CeleryCheck(base.BaseCheck):
"""celery health check class."""
NAME = "Celery Check"
NAME = "Celery Check."
def run(self):
"""do health check"""
"""do health check."""
self.check_compass_celery_setting()
print "[Done]"
self.check_celery_backend()
@ -26,7 +39,7 @@ class CeleryCheck(base.BaseCheck):
return (self.code, self.messages)
def check_compass_celery_setting(self):
"""Validates Celery settings"""
"""Validates Celery settings."""
print "Checking Celery setting......",
setting_map = {
@ -75,10 +88,10 @@ class CeleryCheck(base.BaseCheck):
return True
def check_celery_backend(self):
"""Checks if Celery backend is running and configured properly"""
"""Checks if Celery backend is running and configured properly."""
print "Checking Celery Backend......",
if not 'celeryd' in commands.getoutput('ps -ef'):
if 'celeryd' not in commands.getoutput('ps -ef'):
self._set_status(0, "[%s]Error: celery is not running" % self.NAME)
return True

37
compass/actions/health_check/check_dhcp.py
View File

@ -1,10 +1,23 @@
"""Health Check module for DHCP service"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Health Check module for DHCP service."""
import commands
import os
import re
import commands
import xmlrpclib
import socket
import xmlrpclib
from compass.actions.health_check import base
@ -15,13 +28,13 @@ class DhcpCheck(base.BaseCheck):
NAME = "DHCP Check"
def run(self):
"""do health check"""
"""do health check."""
installer = self.config.OS_INSTALLER
method_name = "self.check_" + installer + "_dhcp()"
return eval(method_name)
def check_cobbler_dhcp(self):
"""Checks if Cobbler has taken over DHCP service"""
"""Checks if Cobbler has taken over DHCP service."""
try:
remote = xmlrpclib.Server(
@ -29,7 +42,7 @@ class DhcpCheck(base.BaseCheck):
allow_none=True)
remote.login(
*self.config.COBBLER_INSTALLER_TOKEN)
except:
except Exception:
self._set_status(
0,
"[%s]Error: Cannot login to Cobbler with "
@ -42,6 +55,7 @@ class DhcpCheck(base.BaseCheck):
"[%s]Info: DHCP service is "
"not managed by Compass" % self.NAME)
return (self.code, self.messages)
self.check_cobbler_dhcp_template()
print "[Done]"
self.check_dhcp_service()
@ -50,11 +64,11 @@ class DhcpCheck(base.BaseCheck):
self.messages.append(
"[%s]Info: DHCP health check has completed. "
"No problems found, all systems go." % self.NAME)
return (self.code, self.messages)
def check_cobbler_dhcp_template(self):
"""Validates Cobbler's DHCP template file"""
"""Validates Cobbler's DHCP template file."""
print "Checking DHCP template......",
if os.path.exists("/etc/cobbler/dhcp.template"):
var_map = {
@ -127,13 +141,13 @@ class DhcpCheck(base.BaseCheck):
0,
"[%s]Error: DHCP template file doesn't exist, "
"health check failed." % self.NAME)
return True
def check_dhcp_service(self):
"""Checks if DHCP is running on port 67"""
"""Checks if DHCP is running on port 67."""
print "Checking DHCP service......",
if not 'dhcp' in commands.getoutput('ps -ef'):
if 'dhcp' not in commands.getoutput('ps -ef'):
self._set_status(
0,
"[%s]Error: dhcp service does not "
@ -144,4 +158,5 @@ class DhcpCheck(base.BaseCheck):
0,
"[%s]Error: bootps is not listening "
"on port 67" % self.NAME)
return True

31
compass/actions/health_check/check_dns.py
View File

@ -1,4 +1,18 @@
"""Health Check module for DNS service"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Health Check module for DNS service."""
import commands
import os
@ -13,21 +27,20 @@ class DnsCheck(base.BaseCheck):
NAME = "DNS Check"
def run(self):
"""do health check"""
"""do health check."""
installer = self.config.OS_INSTALLER
method_name = "self.check_" + installer + "_dns()"
return eval(method_name)
def check_cobbler_dns(self):
"""Checks if Cobbler has taken over DNS service"""
"""Checks if Cobbler has taken over DNS service."""
try:
remote = xmlrpclib.Server(
self.config.COBBLER_INSTALLER_URL,
allow_none=True)
remote.login(
*self.config.COBBLER_INSTALLER_TOKEN)
except:
except Exception:
self._set_status(0,
"[%s]Error: Cannot login to Cobbler "
"with the tokens provided in the config file"
@ -49,7 +62,7 @@ class DnsCheck(base.BaseCheck):
return (self.code, self.messages)
def check_cobbler_dns_template(self):
"""Validates Cobbler's DNS template file"""
"""Validates Cobbler's DNS template file."""
print "Checking DNS template......",
if os.path.exists("/etc/cobbler/named.template"):
@ -66,7 +79,7 @@ class DnsCheck(base.BaseCheck):
if "allow-query" in line:
for subnet in ["127.0.0.0/8"]:
if not subnet in line:
if subnet not in line:
missing_query.append(subnet)
named_template.close()
@ -108,10 +121,10 @@ class DnsCheck(base.BaseCheck):
return True
def check_dns_service(self):
"""Checks if DNS is running on port 53"""
"""Checks if DNS is running on port 53."""
print "Checking DNS service......",
if not 'named' in commands.getoutput('ps -ef'):
if 'named' not in commands.getoutput('ps -ef'):
self._set_status(
0,
"[%s]Error: named service does not seem to be "

37
compass/actions/health_check/check_hds.py
View File

@ -1,24 +1,40 @@
"""Health Check module for Hardware Discovery"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Health Check module for Hardware Discovery."""
from compass.actions.health_check import base
from compass.actions.health_check import utils as health_check_utils
class HdsCheck(base.BaseCheck):
"""hds health check class"""
"""hds health check class."""
NAME = "HDS Check"
def run(self):
"""do health check"""
"""do health check."""
if self.dist in ("centos", "redhat", "fedora", "scientific linux"):
pkg_type = "yum"
else:
pkg_type = "apt"
try:
pkg_module = __import__(pkg_type)
except:
except Exception:
self.messages.append("[%s]Error: No module named %s, "
"please install it first."
% (self.NAME, pkg_module))
method_name = 'self.check_' + pkg_type + '_snmp(pkg_module)'
eval(method_name)
print "[Done]"
@ -28,11 +44,11 @@ class HdsCheck(base.BaseCheck):
self.messages.append("[%s]Info: hds health check has complated. "
"No problems found, all systems go."
% self.NAME)
return (self.code, self.messages)
def check_yum_snmp(self, pkg_module):
"""
Check if SNMP yum dependencies are installed
"""Check if SNMP yum dependencies are installed
:param pkg_module : python yum library
:type pkg_module : python module
@ -46,19 +62,21 @@ class HdsCheck(base.BaseCheck):
self.messages.append("[%s]Error: %s package is required "
"for HDS" % (self.NAME, package))
uninstalled.append(package)
if len(uninstalled) != 0:
self._set_status(0, "[%s]Info: Uninstalled packages: %s"
% (self.NAME,
', '.join(item for item in uninstalled)))
return True
def check_apt_snmp(self, pkg_module):
"""do apt health check"""
## TODO: add ubuntu package check here
"""do apt health check."""
## TODO(xicheng): add ubuntu package check here
return None
def check_snmp_mibs(self):
"""Checks if SNMP MIB files are properly placed"""
"""Checks if SNMP MIB files are properly placed."""
print "Checking SNMP MIBs......",
conf_err_msg = health_check_utils.check_path(self.NAME,
@ -71,4 +89,5 @@ class HdsCheck(base.BaseCheck):
'/usr/local/share/snmp/mibs')
if not mibs_err_msg == "":
self._set_status(0, mibs_err_msg)
return True

57
compass/actions/health_check/check_misc.py
View File

@ -1,11 +1,24 @@
"""Miscellaneous Health Check for Compass"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Miscellaneous Health Check for Compass."""
from compass.actions.health_check import base
from compass.actions.health_check import utils as health_check_utils
class MiscCheck(base.BaseCheck):
"""health check for misc"""
"""health check for misc."""
NAME = "Miscellaneous Check"
MISC_MAPPING = {
@ -23,7 +36,7 @@ class MiscCheck(base.BaseCheck):
}
def run(self):
"""do health check"""
"""do health check."""
self.check_linux_dependencies()
print "[Done]"
self.check_pip_dependencies()
@ -44,26 +57,26 @@ class MiscCheck(base.BaseCheck):
return (self.code, self.messages)
def check_linux_dependencies(self):
"""Checks if dependencies are installed"""
"""Checks if dependencies are installed."""
print "Checking Linux dependencies....",
if self.dist in ("centos", "redhat", "fedora", "scientific linux"):
pkg_type = "yum"
else:
pkg_type = "apt"
try:
pkg_module = __import__(pkg_type)
except:
except Exception:
self._set_status(
0,
"[%s]Error: No module named %s, "
"please install it first." % (self.NAME, pkg_module))
method_name = 'self.check_' + pkg_type + '_dependencies(pkg_module)'
eval(method_name)
def check_yum_dependencies(self, pkg_module):
"""
Checks if yum dependencies are installed.
"""Checks if yum dependencies are installed.
:param pkg_module : python yum library
:type pkg_module : python module
@ -89,14 +102,13 @@ class MiscCheck(base.BaseCheck):
return True
def check_pip_dependencies(self):
"""Checks if required pip packages are installed"""
"""Checks if required pip packages are installed."""
print "Checking pip dependencies......",
uninstalled = []
for module in self.MISC_MAPPING['pip']:
try:
__import__(module)
except:
except Exception:
self._set_status(
0,
"[%s]Error: pip package %s is requred"
@ -112,7 +124,7 @@ class MiscCheck(base.BaseCheck):
return True
def check_ntp(self):
"""Validates ntp configuration and service"""
"""Validates ntp configuration and service."""
print "Checking NTP......",
conf_err_msg = health_check_utils.check_path(self.NAME,
@ -128,7 +140,7 @@ class MiscCheck(base.BaseCheck):
return True
def check_rsyslogd(self):
"""Validates rsyslogd configuration and service"""
"""Validates rsyslogd configuration and service."""
print "Checking rsyslog......",
conf_err_msg = health_check_utils.check_path(self.NAME,
@ -149,7 +161,7 @@ class MiscCheck(base.BaseCheck):
return True
def check_chkconfig(self):
"""Check if required services are enabled on the start up"""
"""Check if required services are enabled on the start up."""
print "Checking chkconfig......",
serv_to_disable = []
@ -160,6 +172,7 @@ class MiscCheck(base.BaseCheck):
"[%s]Error: %s is not disabled"
% (self.NAME, serv))
serv_to_disable.append(serv)
if len(serv_to_disable) != 0:
self._set_status(
0,
@ -174,6 +187,7 @@ class MiscCheck(base.BaseCheck):
self._set_status(
0, "[%s]Error: %s is disabled" % (self.NAME, serv))
serv_to_enable.append(serv)
if len(serv_to_enable) != 0:
self._set_status(0, "[%s]Info: You need to enable these "
"services on system start-up: %s"
@ -183,20 +197,19 @@ class MiscCheck(base.BaseCheck):
return True
def check_selinux(self):
"""Check if SELinux is disabled"""
"""Check if SELinux is disabled."""
print "Checking Selinux......",
selinux = open("/etc/selinux/config")
disabled = False
for line in selinux.readlines():
if "SELINUX=disabled" in line:
disabled = True
break
with open("/etc/selinux/config") as selinux:
for line in selinux:
if "SELINUX=disabled" in line:
disabled = True
break
if disabled is False:
self._set_status(
0,
"[%s]Selinux is not disabled, "
"please disable it in /etc/selinux/config." % self.NAME)
selinux.close()
return True

27
compass/actions/health_check/check_os_installer.py
View File

@ -1,4 +1,18 @@
"""Compass Health Check module for OS Installer"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Compass Health Check module for OS Installer."""
import os
import xmlrpclib
@ -7,25 +21,24 @@ from compass.actions.health_check import base
class OsInstallerCheck(base.BaseCheck):
"""os installer health check"""
"""os installer health check."""
NAME = "OS Installer Check"
def run(self):
"""do health check"""
"""do health check."""
installer = self.config.OS_INSTALLER
method_name = 'self.' + installer + '_check()'
return eval(method_name)
def cobbler_check(self):
"""Runs cobbler check from xmlrpc client"""
"""Runs cobbler check from xmlrpc client."""
try:
remote = xmlrpclib.Server(
self.config.COBBLER_INSTALLER_URL,
allow_none=True)
token = remote.login(
*self.config.COBBLER_INSTALLER_TOKEN)
except:
except Exception:
self.code = 0
self.messages.append(
"[%s]Error: Cannot login to Cobbler with "
@ -42,6 +55,7 @@ class OsInstallerCheck(base.BaseCheck):
for index, message in enumerate(check_result):
if "SELinux" in message:
check_result.pop(index)
if len(check_result) != 0:
self.code = 0
for error_msg in check_result:
@ -62,6 +76,7 @@ class OsInstallerCheck(base.BaseCheck):
if 'ppa_repo' in repo['mirror']:
found_ppa = True
break
if found_ppa is False:
self._set_status(0,
"[%s]Error: No repository ppa_repo found"

31
compass/actions/health_check/check_package_installer.py
View File

@ -1,11 +1,25 @@
"""Health Check module for Package Installer"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Health Check module for Package Installer."""
import os
import requests
from compass.actions.health_check import base
from compass.actions.health_check import utils as health_check_utils
from compass.actions.health_check import setting as health_check_setting
from compass.actions.health_check import utils as health_check_utils
class PackageInstallerCheck(base.BaseCheck):
@ -13,14 +27,13 @@ class PackageInstallerCheck(base.BaseCheck):
NAME = "Package Installer Check"
def run(self):
"""do health check"""
"""do health check."""
installer = self.config.PACKAGE_INSTALLER
method_name = "self." + installer + "_check()"
return eval(method_name)
def chef_check(self):
"""Checks chef setting, cookbooks, databags and roles"""
"""Checks chef setting, cookbooks, databags and roles."""
chef_data_map = {
'CookBook': health_check_setting.COOKBOOKS,
'DataBag': health_check_setting.DATABAGS,
@ -63,8 +76,7 @@ class PackageInstallerCheck(base.BaseCheck):
return (self.code, self.messages)
def check_chef_data(self, data_type, github_url):
"""
Checks if chef cookbooks/roles/databags are correct.
"""Checks if chef cookbooks/roles/databags are correct.
:param data_type : chef data type
should be one of ['CookBook','DataBag','Role']
@ -76,7 +88,7 @@ class PackageInstallerCheck(base.BaseCheck):
print "Checking Chef %s......" % (data_type.lower().strip() + 's'),
try:
import chef
except:
except Exception:
self._set_status(
0,
"[%s]Error: pychef is not installed." % self.NAME)
@ -113,7 +125,7 @@ class PackageInstallerCheck(base.BaseCheck):
return (data_type, list(diff))
def check_chef_config_dir(self):
"""Validates chef configuration directories"""
"""Validates chef configuration directories."""
print "Checking Chef configurations......",
message = health_check_utils.check_path(self.NAME, '/etc/chef-server/')
@ -123,4 +135,5 @@ class PackageInstallerCheck(base.BaseCheck):
message = health_check_utils.check_path(self.NAME, '/opt/chef-server/')
if not message == "":
self._set_status(0, message)
return None

28
compass/actions/health_check/check_squid.py
View File

@ -1,7 +1,20 @@
"""Health Check module for Squid service"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
"""Health Check module for Squid service."""
import commands
import os
import pwd
import socket
@ -14,7 +27,7 @@ class SquidCheck(base.BaseCheck):
NAME = "Squid Check"
def run(self):
"""do health check"""
"""do health check."""
self.check_squid_files()
print "[Done]"
self.check_squid_service()
@ -26,8 +39,7 @@ class SquidCheck(base.BaseCheck):
return (self.code, self.messages)
def check_squid_files(self):
"""Validates squid config, cache directory and ownership"""
"""Validates squid config, cache directory and ownership."""
print "Checking Squid Files......",
var_map = {
'match_squid_conf': False,
@ -91,10 +103,10 @@ class SquidCheck(base.BaseCheck):
return True
def check_squid_service(self):
"""Checks if squid is running on port 3128"""
"""Checks if squid is running on port 3128."""
print "Checking Squid service......",
if not 'squid' in commands.getoutput('ps -ef'):
if 'squid' not in commands.getoutput('ps -ef'):
self._set_status(
0,
"[%s]Error: squid service does not seem "
@ -107,7 +119,7 @@ class SquidCheck(base.BaseCheck):
"[%s]Error: squid is not listening on "
"3128" % self.NAME)
except:
except Exception:
self._set_status(
0,
"[%s]Error: No service is listening on 3128, "

32
compass/actions/health_check/check_tftp.py
View File

@ -1,26 +1,38 @@
"""Health Check module for TFTP service"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Health Check module for TFTP service."""
import os
import xmlrpclib
import socket
import xmlrpclib
from compass.actions.health_check import base
from compass.actions.health_check import utils as health_check_utils
class TftpCheck(base.BaseCheck):
"""tftp health check class"""
"""tftp health check class."""
NAME = "TFTP Check"
def run(self):
"""do health check"""
"""do health check."""
installer = self.config.OS_INSTALLER
method_name = "self.check_" + installer + "_tftp()"
return eval(method_name)
def check_cobbler_tftp(self):
"""
Checks if Cobbler manages TFTP service
"""Checks if Cobbler manages TFTP service.
:note: we assume TFTP service is running at the
same machine where this health check runs at
@ -32,7 +44,7 @@ class TftpCheck(base.BaseCheck):
allow_none=True)
remote.login(
*self.config.COBBLER_INSTALLER_TOKEN)
except:
except Exception:
self._set_status(
0,
"[%s]Error: Cannot login to Cobbler with the tokens "
@ -56,8 +68,7 @@ class TftpCheck(base.BaseCheck):
return (self.code, self.messages)
def check_tftp_dir(self):
"""Validates TFTP directories and configurations"""
"""Validates TFTP directories and configurations."""
print "Checking TFTP directories......",
if not os.path.exists('/var/lib/tftpboot/'):
self._set_status(
@ -69,8 +80,7 @@ class TftpCheck(base.BaseCheck):
return True
def check_tftp_service(self):
"""Checks if TFTP is running on port 69"""
"""Checks if TFTP is running on port 69."""
print "Checking TFTP services......",
serv_err_msg = health_check_utils.check_service_running(self.NAME,
'xinetd')

16
compass/actions/health_check/setting.py
View File

@ -1,4 +1,18 @@
"""Health Check Settings"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Health Check Settings."""
# Chef data on github
COOKBOOKS = (

33
compass/actions/health_check/utils.py
View File

@ -1,13 +1,25 @@
"""Compass Health Check heavy-lifting utilities"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Compass Health Check heavy-lifting utilities"""
import commands
import os
import platform
import commands
def validate_setting(module, setting, param):
"""
Checks if a Compass setting exists in the config file.
"""Checks if a Compass setting exists in the config file.
:param module : module name to be checked
:type module : string
@ -25,15 +37,14 @@ def validate_setting(module, setting, param):
def get_dist():
"""Returns the operating system related information"""
"""Returns the operating system related information."""
os_version, version, release = platform.linux_distribution()
return (os_version.lower().strip(), version, release.lower().strip())
def check_path(module_name, path):
"""
Checks if a directory or file exisits.
"""Checks if a directory or file exisits.
:param module_name : module name to be checked
:type module_name : string
@ -50,8 +61,7 @@ def check_path(module_name, path):
def check_service_running(module_name, service_name):
"""
Checks if a certain service is running.
"""Checks if a certain service is running.
:param module_name : module name to be checked
:type module_name : string
@ -60,7 +70,7 @@ def check_service_running(module_name, service_name):
"""
err_msg = ""
if not service_name in commands.getoutput('ps -ef'):
if service_name not in commands.getoutput('ps -ef'):
err_msg = "[%s]Error: %s is not running." % (
module_name, service_name)
@ -68,8 +78,7 @@ def check_service_running(module_name, service_name):
def check_chkconfig(service_name):
"""
Checks if a service is enabled at the start up.
"""Checks if a service is enabled at the start up.
:param service_name : service name to be checked
:type service_name : string

27
compass/actions/poll_switch.py
View File

@ -1,8 +1,24 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to provider function to poll switch."""
import logging
from compass.db import database
from compass.db.model import Switch, Machine, SwitchConfig
from compass.db.model import Machine
from compass.db.model import Switch
from compass.db.model import SwitchConfig
from compass.hdsdiscovery.hdmanager import HDManager
@ -77,9 +93,12 @@ def poll_switch(ip_addr, req_obj='mac', oper="SCAN"):
switch_id = switch.id
filter_ports = session.query(
SwitchConfig.filter_port).filter(
SwitchConfig.ip == Switch.ip).filter(
Switch.id == switch_id).all()
SwitchConfig.filter_port
).filter(
SwitchConfig.ip == Switch.ip
).filter(
Switch.id == switch_id
).all()
logging.info("***********filter posts are %s********", filter_ports)
if filter_ports:
#Get all ports from tuples into list

33
compass/actions/reinstall.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to reinstall a given cluster
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
@ -13,16 +27,17 @@ def reinstall(cluster_hosts):
"""Reinstall clusters.
:param cluster_hosts: clusters and hosts in each cluster to reinstall.
:type cluster_hosts: dict of int to list of int
:type cluster_hosts: dict of int or str to list of int or str
.. note::
The function should be called out of database session.
"""
logging.debug('reinstall cluster_hosts: %s', cluster_hosts)
with database.session():
cluster_hosts, os_versions, target_systems = (
util.update_cluster_hosts(cluster_hosts))
manager = ConfigManager()
manager.reinstall_cluster_and_hosts(
cluster_hosts, os_versions, target_systems)
manager.sync()
with util.lock('serialized_action'):
logging.debug('reinstall cluster_hosts: %s', cluster_hosts)
with database.session():
cluster_hosts, os_versions, target_systems = (
util.update_cluster_hosts(cluster_hosts))
manager = ConfigManager()
manager.reinstall_cluster_and_hosts(
cluster_hosts, os_versions, target_systems)
manager.sync()

22
compass/actions/search.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to search configs of given clusters
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
@ -15,7 +29,7 @@ def search(cluster_hosts, cluster_propreties_match,
"""search clusters.
:param cluster_hosts: clusters and hosts in each cluster to search.
:type cluster_hosts: dict of int to list of int
:type cluster_hosts: dict of int or str to list of int or str
.. note::
The function should be called out of database session.
@ -26,7 +40,7 @@ def search(cluster_hosts, cluster_propreties_match,
util.update_cluster_hosts(cluster_hosts))
manager = ConfigManager()
return manager.filter_cluster_and_hosts(
cluster_hosts, cluster_propreties_match,
cluster_hosts, os_versions,
target_systems, cluster_propreties_match,
cluster_properties_name, host_properties_match,
host_properties_name, os_versions,
target_systems)
host_properties_name)

46
compass/actions/update_progress.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to update status and installing progress of the given cluster.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
@ -5,8 +19,8 @@
import logging
from compass.actions import util
from compass.log_analyzor import progress_calculator
from compass.db import database
from compass.log_analyzor import progress_calculator
from compass.utils import setting_wrapper as setting
@ -44,7 +58,7 @@ def update_progress(cluster_hosts):
"""Update status and installing progress of the given cluster.
:param cluster_hosts: clusters and hosts in each cluster to update.
:type cluster_hosts: dict of int to list of int
:type cluster_hosts: dict of int or str to list of int or str
.. note::
The function should be called out of the database session scope.
@ -58,17 +72,19 @@ def update_progress(cluster_hosts):
After the progress got updated, these information will be stored back
to the log_progressing_history for next time run.
"""
logging.debug('update installing progress of cluster_hosts: %s',
cluster_hosts)
os_versions = {}
target_systems = {}
with database.session():
cluster_hosts, os_versions, target_systems = (
util.update_cluster_hosts(
cluster_hosts, _cluster_filter, _host_filter))
with util.lock('log_progressing', blocking=False):
logging.debug('update installing progress of cluster_hosts: %s',
cluster_hosts)
os_versions = {}
target_systems = {}
with database.session():
cluster_hosts, os_versions, target_systems = (
util.update_cluster_hosts(
cluster_hosts, _cluster_filter, _host_filter))
progress_calculator.update_progress(setting.OS_INSTALLER,
os_versions,
setting.PACKAGE_INSTALLER,
target_systems,
cluster_hosts)
progress_calculator.update_progress(
setting.OS_INSTALLER,
os_versions,
setting.PACKAGE_INSTALLER,
target_systems,
cluster_hosts)

72
compass/actions/util.py
View File

@ -1,12 +1,52 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to provide util for actions
.. moduleauthor:: Xiaodong Wang ,xiaodongwang@huawei.com>
"""
import logging
import redis
from contextlib import contextmanager
from compass.db import database
from compass.db.model import Switch
from compass.db.model import Cluster
from compass.db.model import Switch
@contextmanager
def lock(lock_name, blocking=True, timeout=10):
redis_instance = redis.Redis()
instance_lock = redis_instance.lock(lock_name, timeout=timeout)
try:
locked = instance_lock.acquire(blocking=blocking)
if locked:
logging.debug('acquired lock %s', lock_name)
yield instance_lock
else:
logging.info('lock %s is already hold', lock_name)
except Exception as error:
logging.info(
'redis fails to acquire the lock %s', lock_name)
logging.exception(error)
finally:
instance_lock.acquired_until = 0
instance_lock.release()
logging.debug('released lock %s', lock_name)
def update_switch_ips(switch_ips):
@ -31,7 +71,11 @@ def update_cluster_hosts(cluster_hosts,
updated_cluster_hosts = {}
clusters = session.query(Cluster).all()
for cluster in clusters:
if cluster_hosts and cluster.id not in cluster_hosts:
if cluster_hosts and (
cluster.id not in cluster_hosts and
str(cluster.id) not in cluster_hosts and
cluster.name not in cluster_hosts
):
logging.debug('ignore cluster %s sinc it is not in %s',
cluster.id, cluster_hosts)
continue
@ -50,18 +94,26 @@ def update_cluster_hosts(cluster_hosts,
os_versions[cluster.id] = adapter.os
target_systems[cluster.id] = adapter.target_system
if (
cluster.id not in cluster_hosts or
not cluster_hosts[cluster.id]
):
hostids = [host.id for host in cluster.hosts]
if cluster.id in cluster_hosts:
hosts = cluster_hosts[cluster.id]
elif str(cluster.id) in cluster_hosts:
hosts = cluster_hosts[str(cluster.id)]
elif cluster.name in cluster_hosts:
hosts = cluster_hosts[cluster.name]
else:
hostids = cluster_hosts[cluster.id]
hosts = []
if not hosts:
hosts = [host.id for host in cluster.hosts]
for host in cluster.hosts:
if host.id not in hostids:
if (
host.id not in hosts and
str(host.id) not in hosts and
host.hostname not in hosts
):
logging.debug('ignore host %s which is not in %s',
host.id, hostids)
host.id, hosts)
continue
if host_filter and not host_filter(host):

19
compass/api/__init__.py
View File

@ -1,9 +1,26 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
__all__ = ['Flask', 'SQLAlchemy', 'compass_api']
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy
from flask import Flask
app = Flask(__name__)
app.debug = True
from compass.api import api as compass_api

315
compass/api/api.py
View File

@ -1,27 +1,46 @@
"""Define all the RestfulAPI entry points"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Define all the RestfulAPI entry points."""
import logging
import netaddr
import re
import simplejson as json
from flask import request
from flask.ext.restful import Resource
from sqlalchemy.sql import and_, or_
from compass.api import app, util, errors
from compass.tasks.client import celery
from flask.ext.restful import Resource
from flask import request
from sqlalchemy.sql import and_
from sqlalchemy.sql import or_
from compass.api import app
from compass.api import errors
from compass.api import util
from compass.db import database
from compass.db.model import Switch as ModelSwitch
from compass.db.model import SwitchConfig
from compass.db.model import Machine as ModelMachine
from compass.db.model import Adapter
from compass.db.model import Cluster as ModelCluster
from compass.db.model import ClusterHost as ModelClusterHost
from compass.db.model import ClusterState
from compass.db.model import HostState
from compass.db.model import Adapter
from compass.db.model import Machine as ModelMachine
from compass.db.model import Role
from compass.db.model import Switch as ModelSwitch
from compass.db.model import SwitchConfig
from compass.tasks.client import celery
class SwitchList(Resource):
"""Query detals of switches and poll swithes"""
"""Query details of switches and poll swithes."""
ENDPOINT = "/switches"
@ -30,9 +49,10 @@ class SwitchList(Resource):
LIMIT = 'limit'
def get(self):
"""
List details of all switches, optionally filtered by some conditions.
Note: switchIp and swtichIpNetwork cannot be combined to use.
"""List details of all switches filtered by some conditions.
.. note::
switchIp and swtichIpNetwork cannot be combined to use.
:param switchIp: switch IP address
:param switchIpNetwork: switch IP network
@ -85,17 +105,17 @@ class SwitchList(Resource):
errors.UserInvalidUsage(error_msg))
def get_queried_ip_prefix(network, prefix):
""" Get Ip prefex as pattern used to query switches.
Switches' Ip addresses need to match this pattern.
"""Get Ip prefex as pattern used to query switches.
.. note::
Switches' Ip addresses need to match this pattern.
"""
count = int(prefix / 8)
if count == 0:
count = 1
return network.rsplit('.', count)[0] + '.'
from netaddr import IPNetwork, IPAddress
ip_network = IPNetwork(switch_ip_network)
ip_network = netaddr.IPNetwork(switch_ip_network)
ip_filter = get_queried_ip_prefix(str(ip_network.network),
ip_network.prefixlen)
@ -103,15 +123,16 @@ class SwitchList(Resource):
result_set = []
if limit:
result_set = session.query(ModelSwitch).filter(
ModelSwitch.ip.startswith(ip_filter)).limit(
limit).all()
ModelSwitch.ip.startswith(ip_filter)
).limit(limit).all()
else:
result_set = session.query(ModelSwitch).filter(
ModelSwitch.ip.startswith(ip_filter)).all()
ModelSwitch.ip.startswith(ip_filter)
).all()
for switch in result_set:
ip_addr = str(switch.ip)
if IPAddress(ip_addr) in ip_network:
if netaddr.IPAddress(ip_addr) in ip_network:
switches.append(switch)
logging.info('[SwitchList][get] ip %s', ip_addr)
@ -139,9 +160,10 @@ class SwitchList(Resource):
"switches": switch_list})
def post(self):
"""
Insert switch IP and the credential to db. Invoke a task to poll
switch at the same time.
"""Insert switch IP and the credential to db.
.. note::
Invoke a task to poll switch at the same time.
:param ip: switch IP address
:param credential: a dict for accessing the switch
@ -197,7 +219,7 @@ class SwitchList(Resource):
class Switch(Resource):
"""Get and update a single switch information"""
"""Get and update a single switch information."""
ENDPOINT = "/switches"
def get(self, switch_id):
@ -288,7 +310,7 @@ class Switch(Resource):
class MachineList(Resource):
"""Query machines by filters"""
"""Query machines by filters."""
ENDPOINT = "/machines"
SWITCHID = 'switchId'
@ -298,10 +320,12 @@ class MachineList(Resource):
LIMIT = 'limit'
def get(self):
"""
Lists details of machines, optionally filtered by some conditions as
the following. According to SwitchConfig, machines with some ports will
be filtered.
"""Lists details of machines.
.. note::
The machines are filtered by some conditions as
the following. According to SwitchConfig, machines
with some ports will be filtered.
:param switchId: the unique identifier of the switch
:param mac: the MAC address
@ -335,7 +359,7 @@ class MachineList(Resource):
return errors.UserInvalidUsage(
errors.UserInvalidUsage(error_msg)
)
#TODO: support query filtered port
# TODO(grace): support query filtered port
if filter_clause:
machines = session.query(ModelMachine)\
.filter(and_(*filter_clause)).all()
@ -362,8 +386,8 @@ class MachineList(Resource):
continue
machine_res = {}
machine_res['switch_ip'] = None if not machine.switch else \
machine.switch.ip
machine_res['switch_ip'] = (
None if not machine.switch else machine.switch.ip)
machine_res['id'] = machine.id
machine_res['mac'] = machine.mac
machine_res['port'] = machine.port
@ -380,12 +404,11 @@ class MachineList(Resource):
class Machine(Resource):
"""List details of the machine with specific machine id"""
"""List details of the machine with specific machine id."""
ENDPOINT = '/machines'
def get(self, machine_id):
"""
Lists details of the specified machine.
"""Lists details of the specified machine.
:param machine_id: the unique identifier of the machine
"""
@ -422,8 +445,10 @@ class Machine(Resource):
class Cluster(Resource):
"""Creates cluster and lists cluster details; Update and list the cluster's
configuration information.
"""Creates cluster and lists cluster details.
.. note::
Update and list the cluster's configuration information.
"""
ENDPOINT = '/clusters'
SECURITY = 'security'
@ -431,9 +456,7 @@ class Cluster(Resource):
PARTITION = 'partition'
def get(self, cluster_id, resource=None):
"""
Lists details of the specified cluster if resource is not specified.
Otherwise, lists details of the resource of this cluster
"""Lists details of the resource specified cluster.
:param cluster_id: the unique identifier of the cluster
:param resource: the resource name(security, networking, partition)
@ -527,8 +550,7 @@ class Cluster(Resource):
)
def put(self, cluster_id, resource):
"""
Update the resource information of the specified cluster in database
"""Update the resource information of the specified cluster.
:param cluster_id: the unique identifier of the cluster
:param resource: resource name(security, networking, partition)
@ -573,8 +595,9 @@ class Cluster(Resource):
if is_valid:
column = resources[resource]['column']
session.query(
ModelCluster).filter_by(id=cluster_id).update(
{column: json.dumps(value)}
ModelCluster
).filter_by(id=cluster_id).update(
{column: json.dumps(value)}
)
else:
return errors.handle_mssing_input(
@ -588,7 +611,7 @@ class Cluster(Resource):
@app.route("/clusters", methods=['GET'])
def list_clusters():
"""Lists the details of all clusters"""
"""Lists the details of all clusters."""
endpoint = '/clusters'
state = request.args.get('state', None, type=str)
results = []
@ -608,23 +631,29 @@ def list_clusters():
elif state == 'installing':
# The deployment of this cluster is in progress.
clusters = session.query(
ModelCluster).filter(
ModelCluster
).filter(
ModelCluster.id == ClusterState.id,
or_(
ClusterState.state == 'INSTALLING',
ClusterState.state == 'UNINITIALIZED'
)).all()
)
).all()
elif state == 'failed':
# The deployment of this cluster is failed.
clusters = session.query(ModelCluster)\
.filter(ModelCluster.id == ClusterState.id,
ClusterState.state == 'ERROR')\
.all()
clusters = session.query(
ModelCluster
).filter(
ModelCluster.id == ClusterState.id,
ClusterState.state == 'ERROR'
).all()
elif state == 'successful':
clusters = session.query(ModelCluster)\
.filter(ModelCluster.id == ClusterState.id,
ClusterState.state == 'READY')\
.all()
clusters = session.query(
ModelCluster
).filter(
ModelCluster.id == ClusterState.id,
ClusterState.state == 'READY'
).all()
if clusters:
for cluster in clusters:
@ -637,8 +666,11 @@ def list_clusters():
results.append(cluster_res)
return util.make_json_response(
200, {"status": "OK",
"clusters": results})
200, {
"status": "OK",
"clusters": results
}
)
@app.route("/clusters/<int:cluster_id>/action", methods=['POST'])
@ -653,7 +685,7 @@ def execute_cluster_action(cluster_id):
:param deploy: the action of starting to deploy
"""
def _add_hosts(cluster_id, hosts):
"""Add cluster host(s) to the cluster by cluster_id"""
"""Add cluster host(s) to the cluster by cluster_id."""
cluseter_hosts = []
available_machines = []
@ -708,15 +740,17 @@ def execute_cluster_action(cluster_id):
)
def _remove_hosts(cluster_id, hosts):
"""Remove existing cluster host from the cluster"""
"""Remove existing cluster host from the cluster."""
removed_hosts = []
with database.session() as session:
failed_hosts = []
for host_id in hosts:
host = session.query(
ModelClusterHost).filter_by(
id=host_id, cluster_id=cluster_id).first()
ModelClusterHost
).filter_by(
id=host_id, cluster_id=cluster_id
).first()
if not host:
failed_hosts.append(host_id)
@ -742,8 +776,9 @@ def execute_cluster_action(cluster_id):
filter_clause.append('id=%s' % host_id)
# Delete the requested hosts from database
session.query(ModelClusterHost).filter(or_(*filter_clause))\
.delete(synchronize_session='fetch')
session.query(ModelClusterHost).filter(
or_(*filter_clause)
).delete(synchronize_session='fetch')
return util.make_json_response(
200, {
@ -753,7 +788,7 @@ def execute_cluster_action(cluster_id):
)
def _replace_all_hosts(cluster_id, hosts):
"""Remove all existing hosts from the cluster and add new ones"""
"""Remove all existing hosts from the cluster and add new ones."""
with database.session() as session:
# Delete all existing hosts of the cluster
@ -763,7 +798,7 @@ def execute_cluster_action(cluster_id):
return _add_hosts(cluster_id, hosts)
def _deploy(cluster_id, hosts):
"""Deploy the cluster"""
"""Deploy the cluster."""
deploy_hosts_info = []
deploy_cluster_info = {}
@ -862,10 +897,10 @@ def execute_cluster_action(cluster_id):
class ClusterHostConfig(Resource):
"""Lists and update/delete cluster host configurations"""
"""Lists and update/delete cluster host configurations."""
def get(self, host_id):
"""Lists configuration details of the specified cluster host
"""Lists configuration details of the specified cluster host.
:param host_id: the unique identifier of the host
"""
@ -887,8 +922,7 @@ class ClusterHostConfig(Resource):
"config": config_res})
def put(self, host_id):
"""
Update configuration of the specified cluster host
"""Update configuration of the specified cluster host.
:param host_id: the unique identifier of the host
"""
@ -943,8 +977,7 @@ class ClusterHostConfig(Resource):
200, {"status": "OK"})
def delete(self, host_id, subkey):
"""
Delete one attribute in configuration of the specified cluster host
"""Delete one attribute of the specified cluster host.
:param host_id: the unique identifier of the host
:param subkey: the attribute name in configuration
@ -978,11 +1011,11 @@ class ClusterHostConfig(Resource):
class ClusterHost(Resource):
"""List details of the cluster host by host id"""
"""List details of the cluster host by host id."""
ENDPOINT = '/clusterhosts'
def get(self, host_id):
"""Lists details of the specified cluster host
"""Lists details of the specified cluster host.
:param host_id: the unique identifier of the host
"""
@ -1010,8 +1043,10 @@ class ClusterHost(Resource):
@app.route("/clusterhosts", methods=['GET'])
def list_clusterhosts():
"""
Lists details of all cluster hosts, optionally filtered by some conditions.
"""Lists details of all cluster hosts.
.. note::
the cluster hosts are optionally filtered by some conditions.
:param hostname: the name of the host
:param clstername: the name of the cluster
@ -1027,9 +1062,11 @@ def list_clusterhosts():
hosts = None
if hostname and clustername:
hosts = session.query(
ModelClusterHost).join(ModelCluster).filter(
ModelClusterHost
).join(ModelCluster).filter(
ModelClusterHost.hostname == hostname,
ModelCluster.name == clustername).all()
ModelCluster.name == clustername
).all()
elif hostname:
hosts = session.query(
@ -1062,8 +1099,7 @@ def list_clusterhosts():
@app.route("/adapters/<int:adapter_id>", methods=['GET'])
def list_adapter(adapter_id):
"""
Lists details of the specified adapter.
"""Lists details of the specified adapter.
:param adapter_id: the unique identifier of the adapter
"""
@ -1103,12 +1139,15 @@ def list_adapter_roles(adapter_id):
if not adapter_q:
error_msg = "Adapter id=%s does not exist!" % adapter_id
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg))
errors.ObjectDoesNotExist(error_msg)
)
roles = session.query(
Role, Adapter).filter(
Role, Adapter
).filter(
Adapter.id == adapter_id,
Adapter.target_system == Role.target_system).all()
Adapter.target_system == Role.target_system
).all()
for role, _ in roles:
role_res = {}
@ -1117,8 +1156,11 @@ def list_adapter_roles(adapter_id):
roles_list.append(role_res)
return util.make_json_response(
200, {"status": "OK",
"roles": roles_list})
200, {
"status": "OK",
"roles": roles_list
}
)
@app.route("/adapters", methods=['GET'])
@ -1155,10 +1197,10 @@ def list_adapters():
class HostInstallingProgress(Resource):
"""Get host installing progress information"""
"""Get host installing progress information."""
def get(self, host_id):
"""Lists progress details of a specific cluster host
"""Lists progress details of a specific cluster host.
:param host_id: the unique identifier of the host
"""
@ -1193,10 +1235,10 @@ class HostInstallingProgress(Resource):
class ClusterInstallingProgress(Resource):
"""Get cluster installing progress information"""
"""Get cluster installing progress information."""
def get(self, cluster_id):
"""Lists progress details of a specific cluster
"""Lists progress details of a specific cluster.
:param cluster_id: the unique identifier of the cluster
"""
@ -1232,66 +1274,94 @@ class ClusterInstallingProgress(Resource):
class DashboardLinks(Resource):
"""Lists dashboard links"""
"""Lists dashboard links."""
ENDPOINT = "/dashboardlinks/"
def get(self):
"""
Return a list of dashboard links
"""Return a list of dashboard links.
"""
cluster_id = request.args.get('cluster_id', None)
logging.info('get cluster links with cluster_id=%s', cluster_id)
links = {}
with database.session() as session:
hosts = session.query(ModelClusterHost)\
.filter_by(cluster_id=cluster_id).all()
hosts = session.query(
ModelClusterHost
).filter_by(cluster_id=cluster_id).all()
if not hosts:
error_msg = "Cannot find hosts in cluster id=%s" % cluster_id
return errors.handle_not_exist(
errors.ObjectDoesNotExist(error_msg))
errors.ObjectDoesNotExist(error_msg)
)
for host in hosts:
config = host.config
if ('has_dashboard_roles' in config and
config['has_dashboard_roles']):
if (
'has_dashboard_roles' in config and
config['has_dashboard_roles']
):
ip_addr = config.get(
'networking', {}).get(
'interfaces', {}).get(
'management', {}).get(
'ip', '')
'networking', {}
).get(
'interfaces', {}
).get(
'management', {}
).get(
'ip', ''
)
roles = config.get('roles', [])
for role in roles:
links[role] = 'http://%s' % ip_addr
return util.make_json_response(
200, {"status": "OK",
"dashboardlinks": links}
200, {
"status": "OK",
"dashboardlinks": links
}
)
TABLES = {
'switch_config': {'name': SwitchConfig,
'columns': ['id', 'ip', 'filter_port']},
'switch': {'name': ModelSwitch,
'columns': ['id', 'ip', 'credential_data']},
'machine': {'name': ModelMachine,
'columns': ['id', 'mac', 'port', 'vlan', 'switch_id']},
'cluster': {'name': ModelCluster,
'columns': ['id', 'name', 'security_config',
'networking_config', 'partition_config',
'adapter_id', 'state']},
'cluster_host': {'name': ModelClusterHost,
'columns': ['id', 'cluster_id', 'hostname', 'machine_id',
'config_data', 'state']},
'adapter': {'name': Adapter,
'columns': ['id', 'name', 'os', 'target_system']},
'role': {'name': Role,
'columns': ['id', 'name', 'target_system', 'description']}
'switch_config': {
'name': SwitchConfig,
'columns': ['id', 'ip', 'filter_port']
},
'switch': {
'name': ModelSwitch,
'columns': ['id', 'ip', 'credential_data']
},
'machine': {
'name': ModelMachine,
'columns': ['id', 'mac', 'port', 'vlan', 'switch_id']
},
'cluster': {
'name': ModelCluster,
'columns': [
'id', 'name', 'security_config',
'networking_config', 'partition_config',
'adapter_id', 'state'
]
},
'cluster_host': {
'name': ModelClusterHost,
'columns': [
'id', 'cluster_id', 'hostname', 'machine_id',
'config_data', 'state'
]
},
'adapter': {
'name': Adapter,
'columns': ['id', 'name', 'os', 'target_system']
},
'role': {
'name': Role,
'columns': ['id', 'name', 'target_system', 'description']
}
}
@app.route("/export/<string:tname>", methods=['GET'])
def export_csv(tname):
"""export to csv file."""
if tname not in TABLES:
error_msg = "Table '%s' is not supported to export or wrong table name"
return util.handle_invalid_usage(
@ -1369,5 +1439,6 @@ util.add_resource(ClusterInstallingProgress,
'/clusters/<int:cluster_id>/progress')
util.add_resource(DashboardLinks, '/dashboardlinks')
if __name__ == '__main__':
app.run(debug=True)

60
compass/api/errors.py
View File

@ -1,10 +1,24 @@
"""Exception and its handler"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Exception and its handler."""
from compass.api import app
from compass.api import util
class ObjectDoesNotExist(Exception):
"""Define the exception for referring non-existing object"""
"""Define the exception for referring non-existing object."""
def __init__(self, message):
super(ObjectDoesNotExist, self).__init__(message)
self.message = message
@ -14,7 +28,7 @@ class ObjectDoesNotExist(Exception):
class UserInvalidUsage(Exception):
"""Define the exception for fault usage of users"""
"""Define the exception for fault usage of users."""
def __init__(self, message):
super(UserInvalidUsage, self).__init__(message)
self.message = message
@ -24,7 +38,7 @@ class UserInvalidUsage(Exception):
class ObjectDuplicateError(Exception):
"""Define the duplicated object exception"""
"""Define the duplicated object exception."""
def __init__(self, message):
super(ObjectDuplicateError, self).__init__(message)
self.message = message
@ -34,7 +48,7 @@ class ObjectDuplicateError(Exception):
class InputMissingError(Exception):
"""Define the insufficient input exception"""
"""Define the insufficient input exception."""
def __init__(self, message):
super(InputMissingError, self).__init__(message)
self.message = message
@ -44,7 +58,7 @@ class InputMissingError(Exception):
class MethodNotAllowed(Exception):
"""Define the exception which invalid method is called"""
"""Define the exception which invalid method is called."""
def __init__(self, message):
super(MethodNotAllowed, self).__init__(message)
self.message = message
@ -55,7 +69,7 @@ class MethodNotAllowed(Exception):
@app.errorhandler(ObjectDoesNotExist)
def handle_not_exist(error, failed_objs=None):
"""Handler of ObjectDoesNotExist Exception"""
"""Handler of ObjectDoesNotExist Exception."""
message = {'status': 'Not Found',
'message': error.message}
@ -68,30 +82,36 @@ def handle_not_exist(error, failed_objs=None):
@app.errorhandler(UserInvalidUsage)
def handle_invalid_usage(error):
"""Handler of UserInvalidUsage Exception"""
"""Handler of UserInvalidUsage Exception."""
message = {'status': 'Invalid parameters',
'message': error.message}
message = {
'status': 'Invalid parameters',
'message': error.message
}
return util.make_json_response(400, message)
@app.errorhandler(InputMissingError)
def handle_mssing_input(error):
"""Handler of InputMissingError Exception"""
"""Handler of InputMissingError Exception."""
message = {'status': 'Insufficient data',
'message': error.message}
message = {
'status': 'Insufficient data',
'message': error.message
}
return util.make_json_response(400, message)
@app.errorhandler(ObjectDuplicateError)
def handle_duplicate_object(error, failed_objs=None):
"""Handler of ObjectDuplicateError Exception"""
"""Handler of ObjectDuplicateError Exception."""
message = {'status': 'Conflict Error',
'message': error.message}
message = {
'status': 'Conflict Error',
'message': error.message
}
if failed_objs and isinstance(failed_objs, dict):
message.update(failed_objs)
@ -101,8 +121,10 @@ def handle_duplicate_object(error, failed_objs=None):
@app.errorhandler(MethodNotAllowed)
def handle_not_allowed_method(error):
"""Handler of MethodNotAllowed Exception"""
"""Handler of MethodNotAllowed Exception."""
message = {"status": "Method Not Allowed",
"message": "The method is not allowed to use"}
message = {
"status": "Method Not Allowed",
"message": "The method is not allowed to use"
}
return util.make_json_response(405, message)

110
compass/api/util.py
View File

@ -1,18 +1,33 @@
"""Utils for API usage"""
from flask import make_response
from flask.ext.restful import Api
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utils for API usage."""
import netaddr
import re
from netaddr import IPAddress
from flask.ext.restful import Api
from flask import make_response
import simplejson as json
from compass.api import app
API = Api(app)
def make_json_response(status_code, data):
"""Wrap json format to the reponse object"""
"""Wrap json format to the reponse object."""
result = json.dumps(data, indent=4)
resp = make_response(result, status_code)
@ -21,7 +36,7 @@ def make_json_response(status_code, data):
def make_csv_response(status_code, csv_data, fname):
"""Wrap CSV format to the reponse object"""
"""Wrap CSV format to the reponse object."""
fname = '.'.join((fname, 'csv'))
resp = make_response(csv_data, status_code)
resp.mimetype = 'text/csv'
@ -30,12 +45,12 @@ def make_csv_response(status_code, csv_data, fname):
def add_resource(*args, **kwargs):
"""Add resource"""
"""Add resource."""
API.add_resource(*args, **kwargs)
def is_valid_ip(ip_address):
"""Valid the format of an Ip address"""
"""Valid the format of an Ip address."""
if not ip_address:
return False
@ -50,7 +65,7 @@ def is_valid_ip(ip_address):
def is_valid_ipnetowrk(ip_network):
"""Valid the format of an Ip network"""
"""Valid the format of an Ip network."""
if not ip_network:
return False
@ -66,9 +81,9 @@ def is_valid_ipnetowrk(ip_network):
def is_valid_netmask(ip_addr):
"""Valid the format of a netmask"""
"""Valid the format of a netmask."""
try:
ip_address = IPAddress(ip_addr)
ip_address = netaddr.IPAddress(ip_addr)
return ip_address.is_netmask()
except Exception:
@ -76,14 +91,14 @@ def is_valid_netmask(ip_addr):
def is_valid_gateway(ip_addr):
"""Valid the format of gateway"""
"""Valid the format of gateway."""
invalid_ip_prefix = ['0', '224', '169', '127']
try:
# Check if ip_addr is an IP address and not start with 0
ip_addr_prefix = ip_addr.split('.')[0]
if is_valid_ip(ip_addr) and ip_addr_prefix not in invalid_ip_prefix:
ip_address = IPAddress(ip_addr)
ip_address = netaddr.IPAddress(ip_addr)
if not ip_address.is_multicast():
# Check if ip_addr is not multicast and reserved IP
return True
@ -106,7 +121,7 @@ def _is_valid_nameservers(value):
def is_valid_security_config(config):
"""Valid the format of security section in config"""
"""Valid the format of security section in config."""
outer_format = {
"server_credentials": {}, "service_credentials": {},
"console_credentials": {}
@ -132,10 +147,10 @@ def is_valid_security_config(config):
def is_valid_networking_config(config):
"""Valid the format of networking config"""
"""Valid the format of networking config."""
def _is_valid_interfaces_config(interfaces_config):
"""Valid the format of interfaces section in config"""
"""Valid the format of interfaces section in config."""
interfaces_section = {
"management": {}, "tenant": {}, "public": {}, "storage": {}
}
@ -206,7 +221,7 @@ def is_valid_networking_config(config):
return (True, '')
def _is_valid_global_config(global_config):
"""Valid the format of 'global' section in config"""
"""Valid the format of 'global' section in config."""
global_section = {
"nameservers": {"req": 1, "validator": _is_valid_nameservers},
"search_path": {"req": 1, "validator": ""},
@ -253,7 +268,7 @@ def is_valid_networking_config(config):
def is_valid_partition_config(config):
"""Valid the configuration format"""
"""Valid the configuration format."""
if not config:
return (False, '%s in partition cannot be null!' % config)
@ -262,10 +277,13 @@ def is_valid_partition_config(config):
def valid_host_config(config):
""" valid_format is used to check if the input config is qualified
the required fields and format.
The key is the required field and format of the input config
The value is the validator function name of the config value
"""Valid the host configuration format.
.. note::
Valid_format is used to check if the input config is qualified
the required fields and format.
The key is the required field and format of the input config
The value is the validator function name of the config value
"""
from api import errors
@ -295,10 +313,12 @@ def valid_host_config(config):
def flatten_dict(dictionary, output, flat_key=""):
"""This function will convert the dictionary into a list
For example:
dict = {'a':{'b': 'c'}, 'd': 'e'} ==>
list = ['a/b/c', 'd/e']
"""This function will convert the dictionary into a flatten dict.
.. note::
For example:
dict = {'a':{'b': 'c'}, 'd': 'e'} ==>
flatten dict = {'a/b': 'c', 'd': 'e'}
"""
keywords = dictionary.keys()
@ -311,13 +331,13 @@ def flatten_dict(dictionary, output, flat_key=""):
def update_dict_value(searchkey, dictionary):
"""Update dictionary value"""
"""Update dictionary value."""
keywords = dictionary.keys()
for key in keywords:
if key == searchkey:
if isinstance(dictionary[key], str):
dictionary[key] = ""
dictionary[key] = ''
elif isinstance(dictionary[key], list):
dictionary[key] = []
@ -328,7 +348,7 @@ def update_dict_value(searchkey, dictionary):
def is_valid_keys(expected, input_dict, section=""):
"""Valid keys"""
"""Validate keys."""
excepted_keys = set(expected.keys())
input_keys = set(input_dict.keys())
if excepted_keys != input_keys:
@ -344,15 +364,15 @@ def is_valid_keys(expected, input_dict, section=""):
def get_col_val_from_dict(result, data):
"""
Convert a dict's values to a list.
"""Convert a dict's values to a list.
:param result: a list of values for each column
:param data: input data
for example:
data = {"a": {"b": {"c": 1},
"d": 2}}
the result will be [1, 2]
.. note::
for example:
data = {"a": {"b": {"c": 1}, "d": 2}}
the result will be [1, 2]
"""
if not isinstance(data, dict):
data = str(data) if str(data) else 'None'
@ -365,16 +385,18 @@ def get_col_val_from_dict(result, data):
def get_headers_from_dict(headers, colname, data):
"""Convert a column which value is dict to a list of column name and keys.
nested keys in dict will be joined by '.' as a column name in CSV.
:param headers: the result list to hold dict keys
:param colname: the column name
:param data: input data
for example:
the column name is 'config_data', and
the value is {"a": {"b": {"c": 1},
"d": 2}}
then headers will be ['config_data.a.b.c', 'config_data.a.d']
.. note::
nested keys in dict will be joined by '.' as a column name in CSV.
for example:
the column name is 'config_data', and
the value is {"a": {"b": {"c": 1}, "d": 2}}
then headers will be ['config_data.a.b.c', 'config_data.a.d']
:param headers: the result list to hold dict keys
:param colname: the column name
:param data: input data
"""
if not colname:
raise "colname cannot be None!"

32
compass/apiclient/example.py
View File

@ -1,9 +1,26 @@
#!/usr/bin/python
#
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Example code to deploy a cluster by compass client api."""
import os
import re
import requests
import sys
import time
import os
import requests
from compass.apiclient.restful import Client
@ -233,7 +250,7 @@ print 'deploy cluster %s status: %s, resp: %s' % (cluster_id, status, resp)
# get intalling progress.
timeout = time.time() + 60 * 30
timeout = time.time() + 60 * 60
while True:
status, resp = client.get_cluster_installing_progress(cluster_id)
print 'get cluster %s installing progress status: %s, resp: %s' % (
@ -260,8 +277,11 @@ print 'get cluster %s dashboardlinks status: %s, resp: %s' % (
dashboardlinks = resp['dashboardlinks']
r = requests.get(dashboardlinks['os-dashboard'], verify=False)
r.raise_for_status()
if r.text.find('username') == 1054:
print 'dashboard login page can be downloaded with keyword username'
match = re.search(r'(?m)(http://\d+\.\d+\.\d+\.\d+:5000/v2\.0)', r.text)
if match:
print 'dashboard login page can be downloaded'
else:
print 'dashboard login page failed to be downloaded'
print (
'dashboard login page failed to be downloaded\n'
'the context is:\n%s\n') % r.text
raise Exception("os-dashboard is not properly installed!")

23
compass/apiclient/restful.py
View File

@ -1,9 +1,23 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Compass api client library.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
"""
import logging
import json
import logging
import requests
@ -390,10 +404,13 @@ class Client(object):
else:
if '_' not in key:
continue
key_name, key_value = key.split('_', 1)
data.setdefault(
'interfaces', {}).setdefault(
key_name, {})[key_value] = value
'interfaces', {}
).setdefault(
key_name, {}
)[key_value] = value
return data

13
compass/config_management/__init__.py
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

26
compass/config_management/installers/__init__.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""modules to read/write cluster/host config from installers.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
@ -14,12 +28,16 @@ __all__ = [
from compass.config_management.installers.os_installer import (
get_installer_by_name as get_os_installer_by_name,
get_installer as get_os_installer,
get_installer as get_os_installer)
from compass.config_management.installers.os_installer import (
get_installer_by_name as get_os_installer_by_name)
from compass.config_management.installers.os_installer import (
register as register_os_installer)
from compass.config_management.installers.package_installer import (
get_installer_by_name as get_package_installer_by_name,
get_installer as get_package_installer,
get_installer as get_package_installer)
from compass.config_management.installers.package_installer import (
get_installer_by_name as get_package_installer_by_name)
from compass.config_management.installers.package_installer import (
register as register_package_installer)
from compass.config_management.installers.plugins import chefhandler
from compass.config_management.installers.plugins import cobbler

14
compass/config_management/installers/installer.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to provider installer interface.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>

14
compass/config_management/installers/os_installer.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module for interface of os installer.
.. moduleauthor::: Xiaodong Wang <xiaodongwang@huawei.com>

14
compass/config_management/installers/package_installer.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to provider interface for package installer.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>

13
compass/config_management/installers/plugins/__init__.py
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

86
compass/config_management/installers/plugins/chefhandler.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""package instaler chef plugin.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@gmail.com>
@ -5,12 +19,12 @@
import fnmatch
import logging
from compass.utils import util
from compass.config_management.installers import package_installer
from compass.config_management.utils.config_translator import ConfigTranslator
from compass.config_management.utils.config_translator import KeyTranslator
from compass.config_management.utils import config_translator_callbacks
from compass.utils import setting_wrapper as setting
from compass.utils import util
TO_CLUSTER_TRANSLATORS = {
@ -73,6 +87,9 @@ FROM_CLUSTER_TRANSLATORS = {
'/dashboard_roles': [KeyTranslator(
translated_keys=['/dashboard_roles']
)],
'/role_mapping': [KeyTranslator(
translated_keys=['/role_mapping']
)],
}
),
}
@ -81,26 +98,13 @@ FROM_CLUSTER_TRANSLATORS = {
TO_HOST_TRANSLATORS = {
'openstack': ConfigTranslator(
mapping={
'/networking/interfaces/management/ip': [KeyTranslator(
translated_keys=[
'/db/mysql/bind_address',
'/mq/rabbitmq/bind_address',
'/endpoints/compute/metadata/host',
'/endpoints/compute/novnc/host',
'/endpoints/compute/service/host',
'/endpoints/compute/xvpvnc/host',
'/endpoints/ec2/admin/host',
'/endpoints/ec2/service/host',
'/endpoints/identity/admin/host',
'/endpoints/identity/service/host',
'/endpoints/image/registry/host',
'/endpoints/image/service/host',
'/endpoints/metering/service/host',
'/endpoints/network/service/host',
'/endpoints/volume/service/host',
],
translated_value=config_translator_callbacks.get_value_if,
from_values={'condition': '/has_dashboard_roles'}
'/roles': [KeyTranslator(
translated_keys=(
config_translator_callbacks.get_keys_from_role_mapping),
from_keys={'mapping': '/role_mapping'},
translated_value=(
config_translator_callbacks.get_value_from_role_mapping),
from_values={'mapping': '/role_mapping'}
)],
}
),
@ -121,40 +125,36 @@ class Installer(package_installer.Installer):
def __repr__(self):
return '%s[name=%s,installer_url=%s,global_databag_name=%s]' % (
self.__class__.__name__, self.installer_url_,
self.NAME, self.global_databag_name_)
self.__class__.__name__, self.NAME, self.installer_url_,
self.global_databag_name_)
@classmethod
def _cluster_databag_name(cls, clusterid, target_system):
"""get cluster databag name"""
return '%s_%s' % (target_system, str(clusterid))
"""get cluster databag name."""
return '%s_%s' % (target_system, clusterid)
@classmethod
def _get_client_name(cls, hostname, clusterid, target_system):
"""get client name"""
return cls._get_node_name(hostname, clusterid, target_system)
def _get_client_name(cls, fullname, target_system):
"""get client name."""
return cls._get_node_name(fullname, target_system)
@classmethod
def _get_node_name(cls, hostname, clusterid, target_system):
"""get node name"""
return '%s_%s_%s' % (hostname, target_system, clusterid)
def _get_node_name(cls, fullname, target_system):
"""get node name."""
return '%s.%s' % (target_system, fullname)
def os_installer_config(self, config, target_system, **kwargs):
"""get os installer config."""
clusterid = config['clusterid']
roles = config['roles']
return {
'%s_url' % self.NAME: self.installer_url_,
'run_list': ','.join(
['"role[%s]"' % role for role in roles if role]),
['"role[%s]"' % role for role in config['roles'] if role]),
'cluster_databag': self._cluster_databag_name(
clusterid, target_system),
config['clusterid'], target_system),
'chef_client_name': self._get_client_name(
config['hostname'], config['clusterid'],
target_system),
config['fullname'], target_system),
'chef_node_name': self._get_node_name(
config['hostname'], config['clusterid'],
target_system)
config['fullname'], target_system)
}
def get_target_systems(self, oses):
@ -251,12 +251,12 @@ class Installer(package_installer.Installer):
bag_item.save()
def _clean_client(self, hostid, config, target_system, **kwargs):
"""clean client"""
"""clean client."""
from chef import Client
try:
client = Client(
self._get_client_name(
config['hostname'], config['clusterid'], target_system),
config['fullname'], target_system),
api=self.api_)
client.delete()
logging.debug('client is removed for host %s '
@ -268,12 +268,12 @@ class Installer(package_installer.Installer):
hostid, config, target_system)
def _clean_node(self, hostid, config, target_system, **kwargs):
"""clean node"""
"""clean node."""
from chef import Node
try:
node = Node(
self._get_node_name(
config['hostname'], config['clusterid'], target_system),
config['fullname'], target_system),
api=self.api_
)
node.delete()

40
compass/config_management/installers/plugins/cobbler.py
View File

@ -1,4 +1,21 @@
"""os installer cobbler plugin"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""os installer cobbler plugin.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
"""
import functools
import logging
import os.path
@ -156,7 +173,7 @@ class Installer(os_installer.Installer):
def _get_modify_system(self, profile, config, **kwargs):
"""get modified system config."""
system_config = {
'name': self._get_system_name(config),
'name': config['fullname'],
'hostname': config['hostname'],
'profile': profile,
}
@ -180,15 +197,9 @@ class Installer(os_installer.Installer):
{'name': os_version})
return profile_found[0]
@classmethod
def _get_system_name(cls, config):
"""get system name"""
return '%s.%s' % (
config['hostname'], config['clusterid'])
def _get_system(self, config, create_if_not_exists=True):
"""get system reference id."""
sys_name = self._get_system_name(config)
sys_name = config['fullname']
try:
sys_id = self.remote_.get_system_handle(
sys_name, self.token_)
@ -206,7 +217,7 @@ class Installer(os_installer.Installer):
def _clean_system(self, config):
"""clean system."""
sys_name = self._get_system_name(config)
sys_name = config['fullname']
try:
self.remote_.remove_system(sys_name, self.token_)
logging.debug('system %s is removed', sys_name)
@ -218,13 +229,13 @@ class Installer(os_installer.Installer):
self.remote_.save_system(sys_id, self.token_)
def _update_modify_system(self, sys_id, system_config):
"""update modify system"""
"""update modify system."""
for key, value in system_config.items():
self.remote_.modify_system(
sys_id, key, value, self.token_)
def _netboot_enabled(self, sys_id):
"""enable netboot"""
"""enable netboot."""
self.remote_.modify_system(
sys_id, 'netboot_enabled', True, self.token_)
@ -236,7 +247,7 @@ class Installer(os_installer.Installer):
@classmethod
def _clean_log(cls, system_name):
"""clean log"""
"""clean log."""
log_dir = os.path.join(
setting.INSTALLATION_LOGDIR,
system_name)
@ -246,7 +257,7 @@ class Installer(os_installer.Installer):
self, hostid, config, **kwargs
):
"""clean host installing progress."""
self._clean_log(self._get_system_name(config))
self._clean_log(config['fullname'])
def reinstall_host(self, hostid, config, **kwargs):
"""reinstall host."""
@ -255,6 +266,7 @@ class Installer(os_installer.Installer):
self.clean_host_installing_progress(
hostid, config, **kwargs)
self._netboot_enabled(sys_id)
self._save_system(sys_id)
def update_host_config(self, hostid, config, **kwargs):
"""update host config."""

20
compass/config_management/providers/__init__.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""modules to provider providers to read/write cluster/host config
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
@ -9,7 +23,11 @@ __all__ = [
from compass.config_management.providers.config_provider import (
get_provider, get_provider_by_name, register_provider)
get_provider)
from compass.config_management.providers.config_provider import (
get_provider_by_name)
from compass.config_management.providers.config_provider import (
register_provider)
from compass.config_management.providers.plugins import db_config_provider
from compass.config_management.providers.plugins import file_config_provider
from compass.config_management.providers.plugins import mix_config_provider

14
compass/config_management/providers/config_provider.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to provide interface to read/update global/cluster/host config.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>

13
compass/config_management/providers/plugins/__init__.py
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

48
compass/config_management/providers/plugins/db_config_provider.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to provide ConfigProvider that reads config from db.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
@ -8,15 +22,26 @@ import os.path
from compass.config_management.providers import config_provider
from compass.config_management.utils import config_filter
from compass.db import database
from compass.db.model import Adapter, Role, SwitchConfig, Switch, Machine
from compass.db.model import Cluster, ClusterHost
from compass.db.model import ClusterState, HostState, LogProgressingHistory
from compass.db.model import Adapter
from compass.db.model import Cluster
from compass.db.model import ClusterHost
from compass.db.model import ClusterState
from compass.db.model import HostState
from compass.db.model import LogProgressingHistory
from compass.db.model import Machine
from compass.db.model import Role
from compass.db.model import Switch
from compass.db.model import SwitchConfig
from compass.utils import setting_wrapper as setting
CLUSTER_ALLOWS = ['*']
CLUSTER_ALLOWS = ['/security', '/networking', '/partition']
CLUSTER_DENIES = []
HOST_ALLOWS = ['*']
HOST_ALLOWS = [
'/roles',
'/has_dashboard_roles',
'/networking/interfaces/*/ip'
]
HOST_DENIES = []
@ -86,7 +111,6 @@ class DBProvider(config_provider.ConfigProvider):
if switch_filter_tuple in switch_filter_tuples:
logging.debug('ignore adding switch filter: %s',
switch_filter)
continue
else:
logging.debug('add switch filter: %s', switch_filter)
@ -112,7 +136,7 @@ class DBProvider(config_provider.ConfigProvider):
log_dir = os.path.join(
setting.INSTALLATION_LOGDIR,
'%s.%s' % (host.hostname, host.cluster_id),
host.fullname,
'')
session.query(LogProgressingHistory).filter(
LogProgressingHistory.pathname.startswith(
@ -165,7 +189,7 @@ class DBProvider(config_provider.ConfigProvider):
log_dir = os.path.join(
setting.INSTALLATION_LOGDIR,
'%s.%s' % (host.hostname, host.cluster_id),
host.fullname,
'')
session.query(LogProgressingHistory).filter(
LogProgressingHistory.pathname.startswith(
@ -186,20 +210,20 @@ class DBProvider(config_provider.ConfigProvider):
id=clusterid).delete(synchronize_session='fetch')
def get_cluster_hosts(self, clusterid):
"""get cluster hosts"""
"""get cluster hosts."""
session = database.current_session()
hosts = session.query(ClusterHost).filter_by(
cluster_id=clusterid).all()
return [host.id for host in hosts]
def get_clusters(self):
"""get clusters"""
"""get clusters."""
session = database.current_session()
clusters = session.query(Cluster).all()
return [cluster.id for cluster in clusters]
def get_switch_and_machines(self):
"""get switches and machines"""
"""get switches and machines."""
session = database.current_session()
switches = session.query(Switch).all()
switches_data = []
@ -224,7 +248,7 @@ class DBProvider(config_provider.ConfigProvider):
def update_switch_and_machines(
self, switches, switch_machines
):
"""update switches and machines"""
"""update switches and machines."""
session = database.current_session()
session.query(Switch).delete(synchronize_session='fetch')
session.query(Machine).delete(synchronize_session='fetch')

14
compass/config_management/providers/plugins/file_config_provider.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""config provider read config from file.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>

28
compass/config_management/providers/plugins/mix_config_provider.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Mix provider which read config from different other providers.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
@ -49,7 +63,7 @@ class MixProvider(config_provider.ConfigProvider):
adapters, roles_per_target_system)
def update_switch_filters(self, switch_filters):
"""update switch filters"""
"""update switch filters."""
self.host_provider_.update_switch_filters(switch_filters)
def clean_host_config(self, hostid):
@ -57,23 +71,23 @@ class MixProvider(config_provider.ConfigProvider):
self.host_provider_.clean_host_config(hostid)
def reinstall_host(self, hostid):
"""reinstall host config"""
"""reinstall host config."""
self.host_provider_.reinstall_host(hostid)
def reinstall_cluster(self, clusterid):
"""reinstall cluster"""
"""reinstall cluster."""
self.host_provider_.reinstall_cluster(clusterid)
def clean_host_installing_progress(self, hostid):
"""clean host installing progress"""
"""clean host installing progress."""
self.host_provider_.clean_host_installing_progress(hostid)
def clean_cluster_installing_progress(self, clusterid):
"""clean cluster installing progress"""
"""clean cluster installing progress."""
self.host_provider_.clean_cluster_installing_progress(clusterid)
def clean_cluster_config(self, clusterid):
"""clean cluster config"""
"""clean cluster config."""
self.host_provider_.clean_cluster_config(clusterid)
def get_cluster_hosts(self, clusterid):
@ -81,7 +95,7 @@ class MixProvider(config_provider.ConfigProvider):
return self.host_provider_.get_cluster_hosts(clusterid)
def get_clusters(self):
"""get clusters"""
"""get clusters."""
return self.host_provider_.get_clusters()
def get_switch_and_machines(self):

13
compass/config_management/utils/__init__.py
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

22
compass/config_management/utils/config_filter.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to filter configuration when upddating.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
@ -8,7 +22,7 @@ from compass.config_management.utils import config_reference
class ConfigFilter(object):
"""config filter based on allows and denies rules"""
"""config filter based on allows and denies rules."""
def __init__(self, allows=['*'], denies=[]):
"""Constructor
@ -27,7 +41,7 @@ class ConfigFilter(object):
self.__class__.__name__, self.allows_, self.denies_)
def _is_allows_valid(self):
"""Check if allows are valid"""
"""Check if allows are valid."""
if not isinstance(self.allows_, list):
raise TypeError(
'allows type is %s but expected type is list: %s' % (
@ -80,7 +94,9 @@ class ConfigFilter(object):
if not allow:
continue
logging.debug('filter by allow rule %s', allow)
for sub_key, sub_ref in ref.ref_items(allow):
logging.debug('%s is added to filtered config', sub_key)
filtered_ref.setdefault(sub_key).update(sub_ref.config)
def _filter_denies(self, filtered_ref):
@ -89,5 +105,7 @@ class ConfigFilter(object):
if not deny:
continue
logging.debug('filter by deny rule %s', deny)
for ref_key in filtered_ref.ref_keys(deny):
logging.debug('%s is removed from filtered config', ref_key)
del filtered_ref[ref_key]

39
compass/config_management/utils/config_manager.py
View File

@ -1,6 +1,18 @@
"""
Module to get configs from provider and isntallers and update
them to provider and installers.
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to get configs from provider and isntallers and update
them to provider and installers.
.. moduleauthor:: Xiaodong wang ,xiaodongwang@huawei.com>
"""
@ -9,9 +21,9 @@ import logging
from compass.config_management import installers
from compass.config_management import providers
from compass.config_management.utils import config_merger_callbacks
from compass.config_management.utils.config_merger import ConfigMapping
from compass.config_management.utils.config_merger import ConfigMerger
from compass.config_management.utils import config_merger_callbacks
from compass.config_management.utils.config_reference import ConfigReference
from compass.utils import setting_wrapper as setting
from compass.utils import util
@ -40,6 +52,9 @@ CLUSTER_HOST_MERGER = ConfigMerger(
to_key='/has_dashboard_roles',
value=config_merger_callbacks.has_intersection
),
ConfigMapping(
path_list=['/role_mapping'],
),
ConfigMapping(
path_list=[
'/networking/global/nameservers',
@ -88,10 +103,12 @@ CLUSTER_HOST_MERGER = ConfigMerger(
class ConfigManager(object):
"""
Class is to get global/clsuter/host configs from provider,
os installer, package installer, process them, and
update them to provider, os installer, package installer.
"""Class to get global/clsuter/host configs.
.. note::
The class is used to get global/clsuter/host configs
from provider, os installer, package installer, process them,
and update them to provider, os installer, package installer.
"""
def __init__(self):
@ -172,7 +189,7 @@ class ConfigManager(object):
adapters, roles_per_target_system)
def update_switch_filters(self):
"""Update switch filter from setting.SWITCHES"""
"""Update switch filter from setting.SWITCHES."""
if not hasattr(setting, 'SWITCHES'):
logging.info('no switch configs to set')
return
@ -183,7 +200,7 @@ class ConfigManager(object):
self.config_provider_.update_switch_filters(switch_filters)
def get_switch_and_machines(self):
"""Get switches and machines"""
"""Get switches and machines."""
switches, machines_per_switch = (
self.config_provider_.get_switch_and_machines())
logging.debug('got switches %s from %s',
@ -475,7 +492,7 @@ class ConfigManager(object):
return hostids
def get_clusters(self):
"""get clusters"""
"""get clusters."""
clusters = self.config_provider_.get_clusters()
logging.debug('got clusters from %s: %s',
self.config_provider_, clusters)

20
compass/config_management/utils/config_merger.py
View File

@ -1,9 +1,23 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to set the hosts configs from cluster config.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
"""
import copy
import logging
from copy import deepcopy
from compass.config_management.utils import config_reference
from compass.utils import util
@ -173,14 +187,14 @@ class ConfigMapping(object):
if self.value_ is None:
lower_values = {}
for lower_key in lower_sub_refs.keys():
lower_values[lower_key] = deepcopy(sub_ref.config)
lower_values[lower_key] = copy.deepcopy(sub_ref.config)
return lower_values
if not callable(self.value_):
lower_values = {}
for lower_key in lower_sub_refs.keys():
lower_values[lower_key] = deepcopy(self.value_)
lower_values[lower_key] = copy.deepcopy(self.value_)
return lower_values

93
compass/config_management/utils/config_merger_callbacks.py
View File

@ -1,11 +1,25 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""ConfigMerger Callbacks module.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
"""
import copy
import itertools
import logging
from copy import deepcopy
from netaddr import IPSet, IPRange
import netaddr
from compass.utils import util
@ -58,7 +72,7 @@ def _get_bundled_exclusives(exclusives, bundle_mapping):
def _get_max(lhs, rhs):
"""Get max value"""
"""Get max value."""
if lhs < 0:
return lhs
@ -69,7 +83,7 @@ def _get_max(lhs, rhs):
def _get_min(lhs, rhs):
"""Get min value"""
"""Get min value."""
if lhs < 0:
return rhs
@ -79,6 +93,14 @@ def _get_min(lhs, rhs):
return min(lhs, rhs)
def _dec_max_min(value):
"""dec max and min value."""
if value > 0:
return value - 1
else:
return value
def _get_bundled_max_mins(maxs, mins, default_max, default_min, role_bundles):
"""Get max and mins for each bundled role."""
bundled_maxs = {}
@ -119,10 +141,7 @@ def _get_bundled_max_mins(maxs, mins, default_max, default_min, role_bundles):
def _update_assigned_roles(lower_refs, to_key, bundle_mapping,
role_bundles, bundled_maxs, bundled_mins):
"""
Update bundled maxs/mins and get assign roles to each host,
unassigned host.
"""
"""Update bundled maxs/mins and assign roles to each unassigned host."""
lower_roles = {}
unassigned_hosts = []
for lower_key, lower_ref in lower_refs.items():
@ -135,8 +154,11 @@ def _update_assigned_roles(lower_refs, to_key, bundle_mapping,
bundled_roles.add(bundled_role)
roles |= set(role_bundles[bundled_role])
for bundled_role in bundled_roles:
bundled_maxs[bundled_role] -= 1
bundled_mins[bundled_role] -= 1
bundled_maxs[bundled_role] = _dec_max_min(
bundled_maxs[bundled_role])
bundled_mins[bundled_role] = _dec_max_min(
bundled_mins[bundled_role])
lower_roles[lower_key] = list(roles)
if not roles:
unassigned_hosts.append(lower_key)
@ -158,8 +180,10 @@ def _update_exclusive_roles(bundled_exclusives, lower_roles,
raise ValueError('no enough unassigned hosts for exlusive %s',
bundled_exclusive)
host = unassigned_hosts.pop(0)
bundled_mins[bundled_exclusive] -= 1
bundled_maxs[bundled_exclusive] -= 1
bundled_mins[bundled_exclusive] = _dec_max_min(
bundled_mins[bundled_exclusive])
bundled_maxs[bundled_exclusive] = _dec_max_min(
bundled_maxs[bundled_exclusive])
lower_roles[host] = list(role_bundles[bundled_exclusive])
del role_bundles[bundled_exclusive]
@ -174,7 +198,7 @@ def _update_exclusive_roles(bundled_exclusives, lower_roles,
def _assign_roles_by_mins(role_bundles, lower_roles, unassigned_hosts,
bundled_maxs, bundled_mins):
"""Assign roles to hosts by min restriction."""
available_hosts = deepcopy(unassigned_hosts)
available_hosts = copy.deepcopy(unassigned_hosts)
for bundled_role, roles in role_bundles.items():
while bundled_mins[bundled_role] > 0:
if not available_hosts:
@ -186,8 +210,10 @@ def _assign_roles_by_mins(role_bundles, lower_roles, unassigned_hosts,
if host in unassigned_hosts:
unassigned_hosts.remove(host)
bundled_mins[bundled_role] -= 1
bundled_maxs[bundled_role] -= 1
bundled_mins[bundled_role] = _dec_max_min(
bundled_mins[bundled_role])
bundled_maxs[bundled_role] = _dec_max_min(
bundled_maxs[bundled_role])
lower_roles[host] = list(roles)
logging.debug('assigned roles after assigning mins: %s', lower_roles)
@ -200,13 +226,14 @@ def _assign_roles_by_maxs(role_bundles, lower_roles, unassigned_hosts,
bundled_maxs):
"""Assign roles to host by max restriction."""
available_lists = []
default_roles = []
default_roles_lists = []
for bundled_role in role_bundles.keys():
if bundled_maxs[bundled_role] > 0:
available_lists.append(
[bundled_role] * bundled_maxs[bundled_role])
else:
default_roles.append(bundled_role)
elif bundled_maxs[bundled_role] < 0:
default_roles_lists.append(
[bundled_role] * (-bundled_maxs[bundled_role]))
available_list = util.flat_lists_with_possibility(available_lists)
@ -221,6 +248,9 @@ def _assign_roles_by_maxs(role_bundles, lower_roles, unassigned_hosts,
logging.debug('unassigned_hosts after assigning maxs: %s',
unassigned_hosts)
default_roles = util.flat_lists_with_possibility(
default_roles_lists)
if default_roles:
default_iter = itertools.cycle(default_roles)
while unassigned_hosts:
@ -232,6 +262,22 @@ def _assign_roles_by_maxs(role_bundles, lower_roles, unassigned_hosts,
logging.debug('unassigned hosts: %s', unassigned_hosts)
def _sort_roles(lower_roles, roles):
"""Sort roles with the same order as in all roles."""
for lower_key, lower_value in lower_roles.items():
updated_roles = []
for role in roles:
if role in lower_value:
updated_roles.append(role)
for role in lower_value:
if role not in updated_roles:
logging.debug('found role %s not in roles %s', role, roles)
updated_roles.append(role)
lower_roles[lower_key] = updated_roles
def assign_roles(_upper_ref, _from_key, lower_refs, to_key,
roles=[], maxs={}, mins={}, default_max=-1,
default_min=0, exclusives=[], bundles=[], **_kwargs):
@ -258,6 +304,7 @@ def assign_roles(_upper_ref, _from_key, lower_refs, to_key,
_assign_roles_by_maxs(
role_bundles, lower_roles, unassigned_hosts,
bundled_maxs)
_sort_roles(lower_roles, roles)
return lower_roles
@ -267,7 +314,7 @@ def assign_roles_by_host_numbers(upper_ref, from_key, lower_refs, to_key,
**_kwargs):
"""Assign roles by role assign policy."""
host_numbers = str(len(lower_refs))
policy_kwargs = deepcopy(default)
policy_kwargs = copy.deepcopy(default)
if host_numbers in policy_by_host_numbers:
util.merge_dict(policy_kwargs, policy_by_host_numbers[host_numbers])
else:
@ -305,7 +352,7 @@ def assign_ips(_upper_ref, _from_key, lower_refs, to_key,
return {}
host_ips = {}
unassigned_hosts = []
ips = IPSet(IPRange(ip_start, ip_end))
ips = netaddr.IPSet(netaddr.IPRange(ip_start, ip_end))
for lower_key, lower_ref in lower_refs.items():
ip_addr = lower_ref.get(to_key, '')
if ip_addr:
@ -334,7 +381,7 @@ def assign_from_pattern(_upper_ref, _from_key, lower_refs, to_key,
upper_configs[key] = kwargs[key]
for lower_key, _ in lower_refs.items():
group = deepcopy(upper_configs)
group = copy.deepcopy(upper_configs)
for key in lower_keys:
group[key] = kwargs[key][lower_key]
@ -353,7 +400,7 @@ def assign_noproxy(_upper_ref, _from_key, lower_refs,
noproxy_pattern='',
hostnames={}, ips={}, **_kwargs):
"""Assign no proxy to hosts."""
no_proxy_list = deepcopy(default)
no_proxy_list = copy.deepcopy(default)
for lower_key, _ in lower_refs.items():
mapping = {
@ -368,7 +415,7 @@ def assign_noproxy(_upper_ref, _from_key, lower_refs,
lower_key, to_key, noproxy_pattern, mapping)
raise error
no_proxy = ','.join(no_proxy_list)
no_proxy = ','.join([no_proxy for no_proxy in no_proxy_list if no_proxy])
host_no_proxy = {}
for lower_key, _ in lower_refs.items():
host_no_proxy[lower_key] = no_proxy

19
compass/config_management/utils/config_reference.py
View File

@ -1,13 +1,26 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to provide util class to access item in nested dict easily.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
"""
import copy
import fnmatch
import os.path
import re
from copy import deepcopy
from compass.utils import util
@ -261,7 +274,7 @@ class ConfigReference(object):
util.merge_dict(self.config, config, override)
elif self.config is None or override:
self.config = deepcopy(config)
self.config = copy.deepcopy(config)
else:
return

24
compass/config_management/utils/config_translator.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Config Translator module to translate orign config to dest config.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
@ -16,7 +30,7 @@ class KeyTranslator(object):
"""Constructor
:param translated_keys: keys in dest ref to be translated to.
:type translated_keys: list of str
:type translated_keys: callable or list of (str or callable)
:param from_keys: extra kwargs parsed to translated key callback.
:type: from_keys: dict mapping name of kwargs to path in origin ref
:param translated_value: value or callback to get translated value.
@ -48,6 +62,9 @@ class KeyTranslator(object):
def _is_valid_translated_keys(self):
"""Check translated keys are valid."""
if callable(self.translated_keys_):
return
for i, translated_key in enumerate(self.translated_keys_):
if util.is_instance(translated_key, [str, unicode]):
if '*' in translated_key:
@ -121,6 +138,11 @@ class KeyTranslator(object):
logging.error('%s from_key %s missing in %s',
self, from_key, sub_ref)
if callable(self.translated_keys_):
translated_keys = self.translated_keys_(
sub_ref, ref_key, **key_configs)
return translated_keys
translated_keys = []
for translated_key in self.translated_keys_:
if callable(translated_key):

75
compass/config_management/utils/config_translator_callbacks.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""callback lib for config translator callbacks."""
import crypt
import logging
@ -10,7 +24,7 @@ def get_key_from_pattern(
_ref, path, from_pattern='.*',
to_pattern='', **kwargs
):
"""Get translated key from pattern"""
"""Get translated key from pattern."""
match = re.match(from_pattern, path)
if not match:
return None
@ -25,11 +39,64 @@ def get_key_from_pattern(
return translated_key
def get_keys_from_role_mapping(ref, _path, mapping={}, **_kwargs):
"""get translated keys from roles."""
roles = ref.config
translated_keys = []
for role in roles:
if role not in mapping:
continue
translated_keys.extend(mapping[role].keys())
logging.debug('got translated_keys %s from roles %s and mapping %s',
translated_keys, roles, mapping)
return translated_keys
def get_value_from_role_mapping(
ref, _path, _translated_ref, translated_path,
mapping={}, **_kwargs
):
"""get translated value from roles and translated_path."""
roles = ref.config
for role in roles:
if role not in mapping:
continue
if translated_path not in mapping[role]:
continue
value = mapping[role][translated_path]
translated_value = ref.get(value)
logging.debug('got translated_value %s from %s and roles %s',
translated_value, roles, translated_path)
return translated_value
return None
def get_encrypted_value(ref, _path, _translated_ref, _translated_path,
crypt_method=None, **_kwargs):
"""Get encrypted value."""
if not crypt_method:
crypt_method = crypt.METHOD_MD5
if hasattr(crypt, 'METHOD_MD5'):
crypt_method = crypt.METHOD_MD5
else:
# for python2.7, copy python2.6 METHOD_MD5 logic here.
from random import choice
import string
_saltchars = string.ascii_letters + string.digits + './'
def _mksalt():
"""generate salt."""
salt = '$1$'
salt += ''.join(choice(_saltchars) for _ in range(8))
return salt
crypt_method = _mksalt()
return crypt.crypt(ref.config, crypt_method)
@ -58,12 +125,12 @@ def add_value(ref, _path, translated_ref,
def override_if_any(_ref, _path, _translated_ref, _translated_path, **kwargs):
"""override if any kwargs is True"""
"""override if any kwargs is True."""
return any(kwargs.values())
def override_if_all(_ref, _path, _translated_ref, _translated_path, **kwargs):
"""override if all kwargs are True"""
"""override if all kwargs are True."""
return all(kwargs.values())

13
compass/db/__init__.py
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

27
compass/db/database.py
View File

@ -1,13 +1,27 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Provider interface to manipulate database."""
import logging
from threading import local
from contextlib import contextmanager
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
from threading import local
from compass.utils import setting_wrapper as setting
from compass.db import model
from compass.utils import setting_wrapper as setting
ENGINE = create_engine(setting.SQLALCHEMY_DATABASE_URI, convert_unicode=True)
@ -39,9 +53,10 @@ def in_session():
@contextmanager
def session():
"""
database session scope. To operate database, it should be called in
database session.
"""database session scope.
.. note::
To operate database, it should be called in database session.
"""
if hasattr(SESSION_HOLDER, 'session'):
logging.error('we are already in session')
@ -78,7 +93,7 @@ def current_session():
def create_db():
"""Create database"""
"""Create database."""
model.BASE.metadata.create_all(bind=ENGINE)

115
compass/db/model.py
View File

@ -1,13 +1,30 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""database model."""
from datetime import datetime
import simplejson as json
import logging
import simplejson as json
import uuid
from datetime import datetime
from sqlalchemy import Column, ColumnDefault, Integer, String
from sqlalchemy import Float, Enum, DateTime, ForeignKey, Text, Boolean
from sqlalchemy import UniqueConstraint
from sqlalchemy.orm import relationship, backref
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.hybrid import hybrid_property
from compass.utils import util
@ -73,7 +90,7 @@ class Switch(BASE):
return '<Switch ip: %r, credential: %r, vendor: %r, state: %s>'\
% (self.ip, self.credential, self.vendor, self.state)
@property
@hybrid_property
def vendor(self):
"""vendor property getter"""
return self.vendor_info
@ -129,9 +146,11 @@ class Switch(BASE):
class Machine(BASE):
"""
Machine table.Note: currently, we are taking care of management plane.
Therefore, we assume one machine is connected to one switch.
"""Machine table.
.. note::
currently, we are taking care of management plane.
Therefore, we assume one machine is connected to one switch.
:param id: int, identity as primary key
:param mac: string, the MAC address of the machine.
@ -199,16 +218,24 @@ class HostState(BASE):
def __init__(self, **kwargs):
super(HostState, self).__init__(**kwargs)
@property
@hybrid_property
def hostname(self):
"""hostname getter"""
return self.host.hostname
@hybrid_property
def fullname(self):
"""fullname getter"""
return self.host.fullname
def __repr__(self):
return ('<HostState %r: state=%r, progress=%s, '
'message=%s, severity=%s>') % (
return (
'<HostState %r: state=%r, progress=%s, '
'message=%s, severity=%s>'
) % (
self.hostname, self.state, self.progress,
self.message, self.severity)
self.message, self.severity
)
class ClusterState(BASE):
@ -244,16 +271,19 @@ class ClusterState(BASE):
def __init__(self, **kwargs):
super(ClusterState, self).__init__(**kwargs)
@property
@hybrid_property
def clustername(self):
"""clustername getter"""
return self.cluster.name
def __repr__(self):
return ('<ClusterState %r: state=%r, progress=%s, '
'message=%s, severity=%s>') % (
return (
'<ClusterState %r: state=%r, progress=%s, '
'message=%s, severity=%s>'
) % (
self.clustername, self.state, self.progress,
self.message, self.severity)
self.message, self.severity
)
class Cluster(BASE):
@ -285,9 +315,8 @@ class Cluster(BASE):
def __init__(self, **kwargs):
if 'name' not in kwargs or not kwargs['name']:
self.name = str(uuid.uuid4())
if 'name' in kwargs:
del kwargs['name']
kwargs['name'] = str(uuid.uuid4())
super(Cluster, self).__init__(**kwargs)
def __repr__(self):
@ -367,7 +396,7 @@ class Cluster(BASE):
@networking.setter
def networking(self, value):
"""networking setter"""
"""networking setter."""
logging.debug('cluster %s set networking %s', self.id, value)
if value:
try:
@ -380,9 +409,9 @@ class Cluster(BASE):
else:
self.networking_config = None
@property
@hybrid_property
def config(self):
"""get config from security, networking, partition"""
"""get config from security, networking, partition."""
config = {}
if self.raw_config:
try:
@ -462,14 +491,17 @@ class ClusterHost(BASE):
def __init__(self, **kwargs):
if 'hostname' not in kwargs or not kwargs['hostname']:
self.hostname = str(uuid.uuid4())
if 'hostname' in kwargs:
del kwargs['hostname']
kwargs['hostname'] = str(uuid.uuid4())
super(ClusterHost, self).__init__(**kwargs)
def __repr__(self):
return '<ClusterHost %r: cluster=%r machine=%r>'\
% (self.hostname, self.cluster, self.machine)
return '<ClusterHost %r: cluster=%r machine=%r>' % (
self.hostname, self.cluster, self.machine)
@hybrid_property
def fullname(self):
return '%s.%s' % (self.hostname, self.cluster.id)
@property
def config(self):
@ -479,10 +511,16 @@ class ClusterHost(BASE):
if self.config_data:
config.update(json.loads(self.config_data))
config.update({'hostid': self.id, 'hostname': self.hostname})
config.update({
'hostid': self.id,
'hostname': self.hostname,
})
if self.cluster:
config.update({'clusterid': self.cluster.id,
'clustername': self.cluster.name})
config.update({
'clusterid': self.cluster.id,
'clustername': self.cluster.name,
'fullname': self.fullname,
})
if self.machine:
util.merge_dict(
@ -560,14 +598,17 @@ class LogProgressingHistory(BASE):
super(LogProgressingHistory, self).__init__(**kwargs)
def __repr__(self):
return ('LogProgressingHistory[%r: position %r,'
'partial_line %r,progress %r,message %r,'
'severity %r]') % (
return (
'LogProgressingHistory[%r: position %r,'
'partial_line %r,progress %r,message %r,'
'severity %r]'
) % (
self.pathname, self.position,
self.partial_line,
self.progress,
self.message,
self.severity)
self.severity
)
class Adapter(BASE):
@ -592,13 +633,15 @@ class Adapter(BASE):
def __repr__(self):
return '<Adapter %r: os %r, target_system %r>' % (
self.name, self.os, self.target_system)
self.name, self.os, self.target_system
)
class Role(BASE):
"""
The Role table stores avaiable roles of one target system
where the host can be deployed to one or several roles in the cluster.
"""The Role table stores avaiable roles of one target system.
.. note::
the host can be deployed to one or several roles in the cluster.
:param id: int, identity as primary key.
:param name: role name.

13
compass/hdsdiscovery/__init__.py
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

43
compass/hdsdiscovery/base.py
View File

@ -1,18 +1,32 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Base class extended by specific vendor in vendors directory.
A vendor needs to implement abstract methods of base class.
"""
import re
import logging
import re
from abc import ABCMeta
from compass.hdsdiscovery import utils
from compass.hdsdiscovery.error import TimeoutError
from compass.hdsdiscovery import utils
class BaseVendor(object):
"""Basic Vendor object"""
"""Basic Vendor object."""
__metaclass__ = ABCMeta
def is_this_vendor(self, host, credential, sys_info, **kwargs):
@ -23,8 +37,11 @@ class BaseVendor(object):
class BaseSnmpVendor(BaseVendor):
"""Base SNMP-based vendor plugin. It uses MIB-II sysDescr value
to determine the vendor of the switch. """
"""Base SNMP-based vendor plugin.
.. note::
It uses MIB-II sysDescr value to determine the vendor of the switch.
"""
def __init__(self, matched_names):
super(BaseSnmpVendor, self).__init__()
@ -57,15 +74,15 @@ class BasePlugin(object):
# At least one of these three functions below must be implemented.
def scan(self, **kwargs):
"""Get multiple records at once"""
"""Get multiple records at once."""
pass
def set(self, **kwargs):
"""Set value to desired variable"""
"""Set value to desired variable."""
pass
def get(self, **kwargs):
"""Get one record from a host"""
"""Get one record from a host."""
pass
@ -82,10 +99,12 @@ class BaseSnmpMacPlugin(BasePlugin):
self.vlan_oid = vlan_oid
def process_data(self, oper='SCAN', **kwargs):
"""progress data."""
func_name = oper.lower()
return getattr(self, func_name)(**kwargs)
def scan(self, **kwargs):
"""scan."""
results = None
try:
results = utils.snmpwalk_by_cl(self.host, self.credential,
@ -109,7 +128,7 @@ class BaseSnmpMacPlugin(BasePlugin):
return mac_list
def get_vlan_id(self, port):
"""Get vlan Id"""
"""Get vlan Id."""
if not port:
return None
@ -127,7 +146,7 @@ class BaseSnmpMacPlugin(BasePlugin):
return vlan_id
def get_port(self, if_index):
"""Get port number"""
"""Get port number."""
if_name = '.'.join((self.port_oid, if_index))
result = None
@ -143,12 +162,12 @@ class BaseSnmpMacPlugin(BasePlugin):
return port
def convert_to_hex(self, value):
"""Convert the integer from decimal to hex"""
"""Convert the integer from decimal to hex."""
return "%0.2x" % int(value)
def get_mac_address(self, mac_numbers):
"""Assemble mac address from the list"""
"""Assemble mac address from the list."""
if len(mac_numbers) != 6:
logging.error("[PluginMac:get_mac_address] MAC address must be "
"6 digitals")

16
compass/hdsdiscovery/error.py
View File

@ -1,4 +1,18 @@
"""hdsdiscovery module errors"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""hdsdiscovery module errors."""
class TimeoutError(Exception):

29
compass/hdsdiscovery/hdmanager.py
View File

@ -1,10 +1,25 @@
"""Manage hdsdiscovery functionalities"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Manage hdsdiscovery functionalities."""
import logging
import os
import re
import logging
from compass.hdsdiscovery import utils
from compass.hdsdiscovery.error import TimeoutError
from compass.hdsdiscovery import utils
UNREACHABLE = 'unreachable'
NOTSUPPORTED = 'notsupported'
@ -38,14 +53,14 @@ class HDManager(object):
plugin = utils.load_module(req_obj, plugin_dir, host, credential)
if not plugin:
# No plugin found!
#TODO add more code to catch excpetion or unexpected state
# TODO(Grace): add more code to catch excpetion or unexpected state
logging.error('no plugin %s to load from %s', req_obj, plugin_dir)
return None
return plugin.process_data(oper, **kwargs)
def is_valid_vendor(self, host, credential, vendor):
""" Check if vendor is associated with this host and credential
"""Check if vendor is associated with this host and credential
:param host: switch ip
:param credential: credential to access switch
@ -76,7 +91,7 @@ class HDManager(object):
return False
def get_vendor(self, host, credential):
""" Check and get vendor of the switch.
"""Check and get vendor of the switch.
:param host: switch ip:
:param credential: credential to access switch
@ -131,7 +146,7 @@ class HDManager(object):
return (vendor, "Found", "")
def get_sys_info(self, host, credential):
"""get sys info"""
"""get sys info."""
sys_info = None
try:
sys_info = utils.snmpget_by_cl(host,

28
compass/hdsdiscovery/utils.py
View File

@ -1,16 +1,30 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utility functions
Including functions of get/getbulk/walk/set of snmp for three versions
"""
import imp
import re
import logging
import re
import subprocess
from compass.hdsdiscovery.error import TimeoutError
def load_module(mod_name, path, host=None, credential=None):
""" Load a module instance.
"""Load a module instance.
:param str mod_name: module name
:param str path: directory of the module
@ -75,7 +89,7 @@ def ssh_remote_execute(host, username, password, cmd):
def valid_ip_format(ip_address):
"""Valid the format of an Ip address"""
"""Valid the format of an Ip address."""
if not re.match(r'^((([0-2]?\d{0,2}\.){3}([0-2]?\d{0,2}))'
r'|(([\da-fA-F]{1,4}:){7}([\da-fA-F]{1,4})))$',
@ -89,9 +103,11 @@ def valid_ip_format(ip_address):
# Implement snmpwalk and snmpget funtionality
# The structure of returned dictionary will by tag/iid/value/type
#################################################################
AUTH_VERSIONS = {'1': 1,
'2c': 2,
'3': 3}
AUTH_VERSIONS = {
'1': 1,
'2c': 2,
'3': 3
}
def snmp_walk(host, credential, *args, **kwargs):

13
compass/hdsdiscovery/vendors/__init__.py vendored
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

13
compass/hdsdiscovery/vendors/hp/__init__.py vendored
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

20
compass/hdsdiscovery/vendors/hp/hp.py vendored
View File

@ -1,4 +1,18 @@
"""Vendor: HP"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Vendor: HP."""
from compass.hdsdiscovery import base
@ -7,7 +21,7 @@ CLASS_NAME = 'Hp'
class Hp(base.BaseSnmpVendor):
"""Hp switch object"""
"""Hp switch object."""
def __init__(self):
base.BaseSnmpVendor.__init__(self, ['hp', 'procurve'])
@ -15,5 +29,5 @@ class Hp(base.BaseSnmpVendor):
@property
def name(self):
"""Get 'name' proptery"""
"""Get 'name' proptery."""
return self.names[0]

13
compass/hdsdiscovery/vendors/hp/plugins/__init__.py vendored
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

18
compass/hdsdiscovery/vendors/hp/plugins/mac.py vendored
View File

@ -1,9 +1,23 @@
"""HP Switch Mac module"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""HP Switch Mac module."""
from compass.hdsdiscovery.base import BaseSnmpMacPlugin
CLASS_NAME = 'Mac'
class Mac(BaseSnmpMacPlugin):
"""Process MAC address by HP switch"""
"""Process MAC address by HP switch."""
pass

13
compass/hdsdiscovery/vendors/huawei/__init__.py vendored
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

20
compass/hdsdiscovery/vendors/huawei/huawei.py vendored
View File

@ -1,4 +1,18 @@
"""Huawei Switch"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Huawei Switch."""
from compass.hdsdiscovery import base
@ -7,7 +21,7 @@ CLASS_NAME = "Huawei"
class Huawei(base.BaseSnmpVendor):
"""Huawei switch"""
"""Huawei switch."""
def __init__(self):
base.BaseSnmpVendor.__init__(self, ["huawei"])
@ -15,5 +29,5 @@ class Huawei(base.BaseSnmpVendor):
@property
def name(self):
"""Return switch name"""
"""Return switch name."""
return self.__name

13
compass/hdsdiscovery/vendors/huawei/plugins/__init__.py vendored
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

33
compass/hdsdiscovery/vendors/huawei/plugins/mac.py vendored
View File

@ -1,22 +1,41 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Huawei Switch Mac module."""
import logging
from compass.hdsdiscovery import utils
from compass.hdsdiscovery.base import BaseSnmpMacPlugin
from compass.hdsdiscovery import utils
CLASS_NAME = "Mac"
class Mac(BaseSnmpMacPlugin):
"""Processes MAC address"""
"""Processes MAC address."""
def __init__(self, host, credential):
super(Mac, self).__init__(host, credential,
'HUAWEI-L2MAM-MIB::hwDynFdbPort')
super(Mac, self).__init__(
host, credential,
'HUAWEI-L2MAM-MIB::hwDynFdbPort')
def scan(self):
"""
Implemnets the scan method in BasePlugin class. In this mac module,
mac addesses were retrieved by snmpwalk commandline.
"""Implemnets the scan method in BasePlugin class.
.. note::
In this mac module, mac addesses were retrieved by
snmpwalk commandline.
"""
results = utils.snmpwalk_by_cl(self.host, self.credential, self.oid)

13
compass/hdsdiscovery/vendors/ovswitch/__init__.py vendored
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

22
compass/hdsdiscovery/vendors/ovswitch/ovswitch.py vendored
View File

@ -1,6 +1,20 @@
"""Open Vswitch module"""
import re
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Open Vswitch module."""
import logging
import re
from compass.hdsdiscovery import base
from compass.hdsdiscovery import utils
@ -11,7 +25,7 @@ CLASS_NAME = "OVSwitch"
class OVSwitch(base.BaseVendor):
"""Open Vswitch"""
"""Open Vswitch."""
def __init__(self):
self.__name = "Open vSwitch"
@ -57,5 +71,5 @@ class OVSwitch(base.BaseVendor):
@property
def name(self):
"""Open Vswitch name"""
"""Open Vswitch name."""
return self.__name

13
compass/hdsdiscovery/vendors/ovswitch/plugins/__init__.py vendored
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

33
compass/hdsdiscovery/vendors/ovswitch/plugins/mac.py vendored
View File

@ -1,15 +1,29 @@
"""Open Vswitch Mac address module"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Open Vswitch Mac address module."""
import logging
from compass.hdsdiscovery import utils
from compass.hdsdiscovery import base
from compass.hdsdiscovery import utils
CLASS_NAME = "Mac"
class Mac(base.BasePlugin):
"""Open Vswitch MAC address module"""
"""Open Vswitch MAC address module."""
def __init__(self, host, credential):
self.host = host
self.credential = credential
@ -23,9 +37,10 @@ class Mac(base.BasePlugin):
return getattr(self, func_name)(**kwargs)
def scan(self, **kwargs):
"""
Implemnets the scan method in BasePlugin class. In this module,
mac addesses were retrieved by ssh
"""Implemnets the scan method in BasePlugin class.
.. note::
In this module, mac addesses were retrieved by ssh.
"""
try:
user = self.credential['username']
@ -47,7 +62,8 @@ class Mac(base.BasePlugin):
output = None
try:
output = utils.ssh_remote_execute(self.host, user, pwd, cmd)
except:
except Exception as error:
logging.exception(error)
return None
logging.debug("[scan][output] output is %s", output)
@ -60,9 +76,12 @@ class Mac(base.BasePlugin):
for line in output:
if not line or line == '\n':
continue
values_arr = line.split()
temp = {}
for field, value in zip(fields_arr, values_arr):
temp[field] = value
result.append(temp.copy())
return result

13
compass/hdsdiscovery/vendors/pica8/__init__.py vendored
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

20
compass/hdsdiscovery/vendors/pica8/pica8.py vendored
View File

@ -1,4 +1,18 @@
"""Vendor: Pica8"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Vendor: Pica8."""
from compass.hdsdiscovery import base
@ -7,7 +21,7 @@ CLASS_NAME = 'Pica8'
class Pica8(base.BaseSnmpVendor):
"""Pica8 switch object"""
"""Pica8 switch object."""
def __init__(self):
base.BaseSnmpVendor.__init__(self, ['pica8'])
@ -15,5 +29,5 @@ class Pica8(base.BaseSnmpVendor):
@property
def name(self):
"""Get 'name' proptery"""
"""Get 'name' proptery."""
return self._name

13
compass/hdsdiscovery/vendors/pica8/plugins/__init__.py vendored
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

18
compass/hdsdiscovery/vendors/pica8/plugins/mac.py vendored
View File

@ -1,4 +1,18 @@
"""Pica8 Switch Mac module"""
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Pica8 Switch Mac module."""
from compass.hdsdiscovery.base import BaseSnmpMacPlugin
@ -6,5 +20,5 @@ CLASS_NAME = 'Mac'
class Mac(BaseSnmpMacPlugin):
"""Process MAC address by Pica8 switch"""
"""Process MAC address by Pica8 switch."""
pass

13
compass/log_analyzor/__init__.py
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

67
compass/log_analyzor/adapter_matcher.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to provider installing progress calculation for the adapter.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
@ -6,7 +20,8 @@ import logging
import re
from compass.db import database
from compass.db.model import Cluster, ClusterHost
from compass.db.model import Cluster
from compass.db.model import ClusterHost
from compass.log_analyzor.line_matcher import Progress
@ -31,17 +46,15 @@ class AdapterItemMatcher(object):
self.__class__.__name__, self.file_matchers_,
self.min_progress_, self.max_progress_)
def update_progress(self, hostname, clusterid, progress):
def update_progress(self, fullname, progress):
"""Update progress.
:param hostname: the hostname of the installing host.
:type hostname: str
:param clusterid: the cluster id of the installing host.
:type clusterid: int
:param fullname: the fullname of the installing host.
:type fullname: str
:param progress: Progress instance to update.
"""
for file_matcher in self.file_matchers_:
file_matcher.update_progress(hostname, clusterid, progress)
file_matcher.update_progress(fullname, progress)
class OSMatcher(object):
@ -72,9 +85,9 @@ class OSMatcher(object):
self.name_ == os_installer_name,
self.os_regex_.match(os_name)])
def update_progress(self, hostname, clusterid, progress):
def update_progress(self, fullname, progress):
"""Update progress."""
self.matcher_.update_progress(hostname, clusterid, progress)
self.matcher_.update_progress(fullname, progress)
class PackageMatcher(object):
@ -105,9 +118,9 @@ class PackageMatcher(object):
self.name_ == package_installer_name,
self.target_system_ == target_system])
def update_progress(self, hostname, clusterid, progress):
def update_progress(self, fullname, progress):
"""Update progress."""
self.matcher_.update_progress(hostname, clusterid, progress)
self.matcher_.update_progress(fullname, progress)
class AdapterMatcher(object):
@ -155,8 +168,8 @@ class AdapterMatcher(object):
"""
session = database.current_session()
host = session.query(
ClusterHost).filter_by(
id=hostid).first()
ClusterHost
).filter_by(id=hostid).first()
if not host:
logging.error(
'there is no host for %s in ClusterHost', hostid)
@ -165,10 +178,10 @@ class AdapterMatcher(object):
if not host.state:
logging.error('there is no related HostState for %s',
hostid)
return host.hostname, None, None
return host.fullname, None, None
return (
host.hostname,
host.fullname,
host.state.state,
Progress(host.state.progress,
host.state.message,
@ -310,32 +323,32 @@ class AdapterMatcher(object):
host_progresses = {}
with database.session():
for hostid in hostids:
hostname, host_state, host_progress = (
fullname, host_state, host_progress = (
self._get_host_progress(hostid))
if not hostname or not host_progress:
if not fullname or not host_progress:
logging.error(
'nothing to update host %s => hostname %s '
'nothing to update host %s => '
'state %s progress %s',
hostid, hostname, host_state, host_progress)
fullname, host_state, host_progress)
continue
logging.debug('got host %s hostname %s state %s progress %s',
hostid, hostname, host_state, host_progress)
logging.debug('got host %s state %s progress %s',
fullname, host_state, host_progress)
host_progresses[hostid] = (
hostname, host_state, host_progress)
fullname, host_state, host_progress)
for hostid, host_value in host_progresses.items():
hostname, host_state, host_progress = host_value
fullname, host_state, host_progress = host_value
if host_state == 'INSTALLING' and host_progress.progress < 1.0:
self.os_matcher_.update_progress(
hostname, clusterid, host_progress)
fullname, host_progress)
self.package_matcher_.update_progress(
hostname, clusterid, host_progress)
fullname, host_progress)
else:
logging.error(
'there is no need to update host %s '
'progress: hostname %s state %s progress %s',
hostid, hostname, host_state, host_progress)
'progress: state %s progress %s',
fullname, host_state, host_progress)
with database.session():
for hostid in hostids:

57
compass/log_analyzor/file_matcher.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to update intalling progress by processing log file.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
@ -25,7 +39,7 @@ class FileFilter(object):
class CompositeFileFilter(FileFilter):
"""filter log file based on the list of filters"""
"""filter log file based on the list of filters."""
def __init__(self, filters):
self.filters_ = filters
@ -97,8 +111,10 @@ class FileReader(object):
"""
with database.session() as session:
history = session.query(
LogProgressingHistory).filter_by(
pathname=self.pathname_).first()
LogProgressingHistory
).filter_by(
pathname=self.pathname_
).first()
if history:
self.position_ = history.position
self.partial_line_ = history.partial_line
@ -198,18 +214,15 @@ class FileReaderFactory(object):
return '%s[logdir: %s filefilter: %s]' % (
self.__class__.__name__, self.logdir_, self.filefilter_)
def get_file_reader(self, hostname, clusterid, filename):
def get_file_reader(self, fullname, filename):
"""Get FileReader instance.
:param hostname: hostname of installing host.
:param clusterid: cluster id of the installing host.
:param fullname: fullname of installing host.
:param filename: the filename of the log file.
:returns: :class:`FileReader` instance if it is not filtered.
"""
pathname = os.path.join(
self.logdir_, '%s.%s' % (hostname, clusterid),
filename)
pathname = os.path.join(self.logdir_, fullname, filename)
logging.debug('get FileReader from %s', pathname)
if not self.filefilter_.filter(pathname):
logging.error('%s is filtered', pathname)
@ -223,10 +236,7 @@ FILE_READER_FACTORY = FileReaderFactory(
class FileMatcher(object):
"""
File matcher the get the lastest installing progress
from the log file.
"""
"""File matcher to get the installing progress from the log file."""
def __init__(self, line_matchers, min_progress, max_progress, filename):
if not 0.0 <= min_progress <= max_progress <= 1.0:
raise IndexError(
@ -272,10 +282,13 @@ class FileMatcher(object):
return
total_progress_data = min(
self.absolute_min_progress_
+
file_progress.progress * self.absolute_progress_diff_,
self.absolute_max_progress_)
(
self.absolute_min_progress_ + (
file_progress.progress * self.absolute_progress_diff_
)
),
self.absolute_max_progress_
)
# total progress should only be updated when the new calculated
# progress is greater than the recored total progress or the
@ -296,13 +309,11 @@ class FileMatcher(object):
'ignore update file %s progress %s to total progress %s',
self.filename_, file_progress, total_progress)
def update_progress(self, hostname, clusterid, total_progress):
def update_progress(self, fullname, total_progress):
"""update progress from file.
:param hostname: the hostname of the installing host.
:type hostname: str
:param clusterid: the cluster id of the installing host.
:type clusterid: int
:param fullname: the fullname of the installing host.
:type fullname: str
:param total_progress: Progress instance to update.
the function update installing progress by reading the log file.
@ -315,7 +326,7 @@ class FileMatcher(object):
no line end indicator for the last line of the file.
"""
file_reader = FILE_READER_FACTORY.get_file_reader(
hostname, clusterid, self.filename_)
fullname, self.filename_)
if not file_reader:
return

18
compass/log_analyzor/line_matcher.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to get the progress when found match with a line of the log."""
import logging
import re
@ -39,9 +53,7 @@ class ProgressCalculator(object):
cls, progress_data, message,
severity, progress
):
"""
Update progress with the given progress_data,
message and severity.
"""Update progress with the given progress_data, message and severity.
:param progress_data: installing progress.
:type progress_data: float between 0 to 1.

21
compass/log_analyzor/progress_calculator.py
View File

@ -1,15 +1,30 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""module to provide updating installing process function.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
"""
import logging
from compass.log_analyzor.line_matcher import LineMatcher, IncrementalProgress
from compass.log_analyzor.file_matcher import FileMatcher
from compass.log_analyzor.adapter_matcher import AdapterMatcher
from compass.log_analyzor.adapter_matcher import AdapterItemMatcher
from compass.log_analyzor.adapter_matcher import AdapterMatcher
from compass.log_analyzor.adapter_matcher import OSMatcher
from compass.log_analyzor.adapter_matcher import PackageMatcher
from compass.log_analyzor.file_matcher import FileMatcher
from compass.log_analyzor.line_matcher import IncrementalProgress
from compass.log_analyzor.line_matcher import LineMatcher
# TODO(weidong): reconsider intialization method for the following.

13
compass/tasks/__init__.py
View File

@ -0,0 +1,13 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

14
compass/tasks/client.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to setup celery client.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>

16
compass/tasks/tasks.py
View File

@ -1,3 +1,17 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to define celery tasks.
.. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
@ -8,8 +22,8 @@ from compass.actions import clean_deployment
from compass.actions import clean_installing_progress
from compass.actions import deploy
from compass.actions import poll_switch
from compass.actions import update_progress
from compass.actions import reinstall
from compass.actions import update_progress
from compass.tasks.client import celery
from compass.utils import flags
from compass.utils import logsetting

Some files were not shown because too many files have changed in this diff Show More