x/timmy
Code Issues Proposed changes

Timmy modular rewrite

Browse Source 
Change-Id: I3923784db7d7e7f6fb7d4e16b6db1a66c1475980
This commit is contained in:
Aleksandr Dobdin 2016-09-14 13:01:11 +00:00
parent 6ed9f2ba64
commit a544641657
8 changed files with 582 additions and 464 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
timmy/cli.py
View File

@ -17,7 +17,7 @@
from timmy.conf import load_conf
from timmy.env import project_name, version
from timmy.nodes import Node, NodeManager
from timmy.nodes import Node
from timmy.tools import signal_wrapper
import argparse
import logging
@ -36,22 +36,18 @@ def pretty_run(quiet, msg, f, args=[], kwargs={}):
return result
def parse_args():
parser = argparse.ArgumentParser(description=('Parallel remote command'
' execution and file'
' manipulation tool'))
def add_args(parser, module):
parser = module.add_args(parser)
return parser
def parser_init(add_help=False):
desc = 'Parallel remote command execution and file manipulation tool'
parser = argparse.ArgumentParser(description=desc, add_help=add_help)
parser.add_argument('-V', '--version', action='store_true',
help='Print Timmy version and exit.')
parser.add_argument('-c', '--config',
help='Path to a YAML configuration file.')
parser.add_argument('-j', '--nodes-json',
help=('Path to a json file retrieved via'
' "fuel node --json". Useful to speed up'
' initialization, skips "fuel node" call.'))
parser.add_argument('--fuel-ip', help='fuel ip address')
parser.add_argument('--fuel-user', help='fuel username')
parser.add_argument('--fuel-pass', help='fuel password')
parser.add_argument('--fuel-token', help='fuel auth token')
parser.add_argument('-o', '--dest-file',
help=('Output filename for the archive in tar.gz'
' format for command outputs and collected'
@ -116,8 +112,6 @@ def parse_args():
help=('Do not use default log collection parameters,'
' only use what has been provided either via -L'
' or in rqfile(s). Implies "-l".'))
parser.add_argument('--logs-no-fuel-remote', action='store_true',
help='Do not collect remote logs from Fuel.')
parser.add_argument('--logs-speed', type=int, metavar='MBIT/S',
help=('Limit log collection bandwidth to 90%% of the'
' specified speed in Mbit/s.'))
@ -133,9 +127,6 @@ def parse_args():
' of a total size larger than locally available'
'. Values lower than 0.3 are not recommended'
' and may result in filling up local disk.'))
parser.add_argument('--fuel-proxy',
help='use os system proxy variables for fuelclient',
action='store_true')
parser.add_argument('--only-logs',
action='store_true',
help=('Only collect logs, do not run commands or'
@ -143,8 +134,6 @@ def parse_args():
parser.add_argument('--fake-logs',
help='Do not collect logs, only calculate size.',
action='store_true')
parser.add_argument('-x', '--extended', action='store_true',
help='Execute extended commands.')
parser.add_argument('--no-archive',
help=('Do not create results archive. By default,'
' an archive with all outputs and files'
@ -159,7 +148,7 @@ def parse_args():
' messages. Good for quick runs / "watch" wrap.'
' This option disables any -v parameters.'),
action='store_true')
parser.add_argument('-m', '--maxthreads', type=int, default=100,
parser.add_argument('--maxthreads', type=int, default=100,
metavar='NUMBER',
help=('Maximum simultaneous nodes for command'
'execution.'))
@ -180,6 +169,9 @@ def parse_args():
' results. Do not forget to clean up the results'
' manually when using this option.'),
action='store_true')
parser.add_argument('-m', '--module', metavar='INVENTORY MODULE',
default='fuel',
help='Use module to get node data')
parser.add_argument('-v', '--verbose', action='count', default=0,
help=('This works for -vvvv, -vvv, -vv, -v, -v -v,'
'etc, If no -v then logging.WARNING is '
@ -193,7 +185,13 @@ def parse_args():
def main(argv=None):
if argv is None:
argv = sys.argv
parser = parse_args()
parser = parser_init()
args, unknown = parser.parse_known_args(argv[1:])
parser = parser_init(add_help=True)
if args.module:
inventory = __import__('timmy.modules.%s' % args.module,
fromlist=['timmy.modules'])
parser = add_args(parser, inventory)
args = parser.parse_args(argv[1:])
if args.version:
print(version)
@ -215,21 +213,9 @@ def main(argv=None):
logger.addHandler(log_handler)
logger.setLevel(loglevel)
conf = load_conf(args.config)
if args.fuel_ip:
conf['fuel_ip'] = args.fuel_ip
if args.fuel_user:
conf['fuel_user'] = args.fuel_user
if args.fuel_pass:
conf['fuel_pass'] = args.fuel_pass
if any([args.fuel_user and not args.fuel_pass,
args.fuel_pass and not args.fuel_user]):
logger.critical('You must specify both --fuel-user and --fuel-pass')
exit(112)
if args.fuel_token:
conf['fuel_api_token'] = args.fuel_token
conf['fuelclient'] = False
if args.fuel_proxy:
conf['fuel_skip_proxy'] = False
if inventory:
inventory.add_conf(conf)
inventory.check_args(args, conf)
if args.put or args.command or args.script or args.get:
conf['shell_mode'] = True
conf['do_print_results'] = True
@ -244,8 +230,6 @@ def main(argv=None):
if args.logs_no_default:
conf['logs_no_default'] = True
args.logs = True
if args.logs_no_fuel_remote:
conf['logs_no_fuel_remote'] = True
if args.logs_speed or args.logs_speed_auto:
conf['logs_speed_limit'] = True
if args.logs_speed:
@ -305,8 +289,8 @@ def main(argv=None):
logger.info('Using rqdir: %s, rqfile: %s' %
(conf['rqdir'], conf['rqfile']))
nm = pretty_run(args.quiet, 'Initializing node data',
NodeManager,
kwargs={'conf': conf, 'extended': args.extended,
inventory.NodeManager,
kwargs={'conf': conf,
'nodes_json': args.nodes_json})
if args.only_logs or args.logs:
size = pretty_run(args.quiet, 'Calculating logs size',

20
timmy/conf.py
View File

@ -31,20 +31,6 @@ def load_conf(filename):
'-oUserKnownHostsFile=/dev/null', '-oLogLevel=error',
'-oBatchMode=yes', '-oUser=root']
conf['env_vars'] = ['OPENRC=/root/openrc', 'LC_ALL="C"', 'LANG="C"']
conf['fuel_ip'] = '127.0.0.1'
conf['fuel_api_user'] = 'admin'
conf['fuel_api_pass'] = 'admin'
conf['fuel_api_token'] = None
conf['fuel_api_tenant'] = 'admin'
conf['fuel_api_port'] = '8000'
conf['fuel_api_keystone_port'] = '5000'
# The three parameters below are used to override FuelClient, API, CLI auth
conf['fuel_user'] = None
conf['fuel_pass'] = None
conf['fuel_tenant'] = None
conf['fuelclient'] = True # use fuelclient library by default
conf['fuel_skip_proxy'] = True
conf['timeout'] = 30
conf['prefix'] = 'nice -n 19 ionice -c 3'
rqdir = 'rq'
@ -66,12 +52,6 @@ def load_conf(filename):
conf['filelists'] = []
conf['logs'] = []
conf['logs_no_default'] = False # skip logs defined in default.yaml
conf['logs_fuel_remote_dir'] = ['/var/log/docker-logs/remote',
'/var/log/remote']
conf['logs_no_fuel_remote'] = False # do not collect /var/log/remote
'''Do not collect from /var/log/remote/<node>
if node is in the array of nodes filtered out by soft filter'''
conf['logs_exclude_filtered'] = True
conf['logs_days'] = 30
conf['logs_speed_limit'] = False # enable speed limiting of log transfers
conf['logs_speed_default'] = 100 # Mbit/s, used when autodetect fails

0
timmy/modules/__init__.py Normal file
View File

491
timmy/modules/fuel.py Normal file
View File

@ -0,0 +1,491 @@
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import os
import sys
import urllib2
from timmy import tools
from timmy.nodes import NodeManager as BaseNodeManager
from timmy.nodes import Node as BaseNode
try:
import fuelclient
if hasattr(fuelclient, 'connect'):
# fuel > 9.0.1 - drop support, use API and CLI instead
FuelClient = None
else:
import fuelclient.client
if type(fuelclient.client.APIClient) is fuelclient.client.Client:
# fuel 9.0.1 and below
from fuelclient.client import Client as FuelClient
else:
FuelClient = None
except:
FuelClient = None
try:
from fuelclient.client import logger
logger.handlers = []
except:
pass
def add_args(parser):
parser.add_argument('--fuel-ip', help='fuel ip address')
parser.add_argument('--fuel-user', help='fuel username')
parser.add_argument('--fuel-pass', help='fuel password')
parser.add_argument('--fuel-token', help='fuel auth token')
parser.add_argument('--fuel-logs-no-remote', action='store_true',
help='Do not collect remote logs from Fuel.')
parser.add_argument('--fuel-proxy',
help='use os system proxy variables for fuelclient',
action='store_true')
parser.add_argument('-j', '--nodes-json',
help=('Path to a json file retrieved via'
' "fuel node --json". Useful to speed up'
' initialization, skips "fuel node" call.'))
return parser
def check_args(args, conf):
if args.fuel_ip:
conf['fuel_ip'] = args.fuel_ip
if args.fuel_user:
conf['fuel_user'] = args.fuel_user
if args.fuel_pass:
conf['fuel_pass'] = args.fuel_pass
if args.fuel_proxy:
conf['fuel_skip_proxy'] = False
if args.fuel_token:
conf['fuel_api_token'] = args.fuel_token
conf['fuelclient'] = False
if args.fuel_logs_no_remote:
conf['fuel_logs_no_remote'] = True
def add_conf(conf):
conf['fuel_ip'] = '127.0.0.1'
conf['fuel_api_user'] = 'admin'
conf['fuel_api_pass'] = 'admin'
conf['fuel_api_token'] = None
conf['fuel_api_tenant'] = 'admin'
conf['fuel_api_port'] = '8000'
conf['fuel_api_keystone_port'] = '5000'
# The three parameters below are used to override FuelClient, API, CLI auth
conf['fuel_user'] = None
conf['fuel_pass'] = None
conf['fuel_tenant'] = None
conf['fuelclient'] = True # use fuelclient library by default
conf['fuel_skip_proxy'] = True
conf['fuel_logs_remote_dir'] = ['/var/log/docker-logs/remote',
'/var/log/remote']
conf['fuel_logs_no_remote'] = False # do not collect /var/log/remote
'''Do not collect from /var/log/remote/<node>
if node is in the array of nodes filtered out by soft filter'''
conf['fuel_logs_exclude_filtered'] = True
class Node(BaseNode):
def get_release(self):
if self.id == 0:
cmd = ("awk -F ':' '/release/ {print $2}' "
"/etc/nailgun/version.yaml")
else:
cmd = ("awk -F ':' '/fuel_version/ {print $2}' "
"/etc/astute.yaml")
release, err, code = tools.ssh_node(ip=self.ip,
command=cmd,
ssh_opts=self.ssh_opts,
timeout=self.timeout,
prefix=self.prefix)
if code != 0:
self.logger.warning('%s: could not determine'
' MOS release' % self.repr)
release = 'n/a'
else:
release = release.strip('\n "\'')
self.logger.info('%s, MOS release: %s' %
(self.repr, release))
return release
def get_roles_hiera(self):
def trim_primary(roles):
trim_roles = [r for r in roles if not r.startswith('primary-')]
trim_roles += [r[8:] for r in roles if r.startswith('primary-')]
return trim_roles
self.logger.debug('%s: roles not defined, trying hiera' % self.repr)
cmd = 'hiera roles'
outs, errs, code = tools.ssh_node(ip=self.ip,
command=cmd,
ssh_opts=self.ssh_opts,
env_vars=self.env_vars,
timeout=self.timeout,
prefix=self.prefix)
self.check_code(code, 'get_roles_hiera', cmd, errs, [0])
if code == 0:
try:
roles = trim_primary(json.loads(outs))
except:
self.logger.warning("%s: failed to parse '%s' output as JSON" %
(self.repr, cmd))
return self.roles
self.logger.debug('%s: got roles: %s' % (self.repr, roles))
if roles is not None:
return roles
else:
return self.roles
else:
self.logger.warning("%s: failed to load roles via hiera" %
self.repr)
self.roles
def get_cluster_id(self):
self.logger.debug('%s: cluster id not defined, trying to determine' %
self.repr)
astute_file = '/etc/astute.yaml'
cmd = ("python -c 'import yaml; a = yaml.load(open(\"%s\")"
".read()); print a[\"cluster\"][\"id\"]'" % astute_file)
outs, errs, code = tools.ssh_node(ip=self.ip,
command=cmd,
ssh_opts=self.ssh_opts,
env_vars=self.env_vars,
timeout=self.timeout,
prefix=self.prefix)
return int(outs.rstrip('\n')) if code == 0 else None
def log_item_manipulate(self, item):
if self.fuel_logs_no_remote and 'fuel' in self.roles:
self.logger.debug('adding Fuel remote logs to exclude list')
if 'exclude' not in item:
item['exclude'] = []
for remote_dir in self.fuel_logs_remote_dir:
item['exclude'].append(remote_dir)
if 'fuel' in self.roles:
for n in self.logs_excluded_nodes:
self.logger.debug('removing remote logs for node:%s' % n)
if 'exclude' not in item:
item['exclude'] = []
for remote_dir in self.fuel_logs_remote_dir:
ipd = os.path.join(remote_dir, n)
item['exclude'].append(ipd)
class NodeManager(BaseNodeManager):
def __init__(self, conf, nodes_json=None, logger=None):
self.base_init(conf, logger)
self.token = self.conf['fuel_api_token']
fuelnode = self.fuel_init()
self.logs_excluded_nodes = []
if FuelClient and conf['fuelclient']:
# save os environment variables
environ = os.environ
try:
if self.conf['fuel_skip_proxy']:
os.environ['HTTPS_PROXY'] = ''
os.environ['HTTP_PROXY'] = ''
os.environ['https_proxy'] = ''
os.environ['http_proxy'] = ''
self.logger.info('Setup fuelclient instance')
self.fuelclient = FuelClient()
if self.conf['fuel_user']:
self.fuelclient.username = self.conf['fuel_user']
if self.conf['fuel_pass']:
self.fuelclient.password = self.conf['fuel_pass']
if self.conf['fuel_tenant']:
self.fuelclient.tenant_name = self.conf['fuel_tenant']
# self.fuelclient.debug_mode(True)
except Exception as e:
self.logger.info('Failed to setup fuelclient instance:%s' % e,
exc_info=True)
self.fuelclient = None
os.environ = environ
else:
self.logger.info('Skipping setup fuelclient instance')
self.fuelclient = None
if nodes_json:
self.nodes_json = tools.load_json_file(nodes_json)
else:
if (not self.get_nodes_fuelclient() and
not self.get_nodes_api() and
not self.get_nodes_cli()):
sys.exit(105)
self.nodes_init(Node)
# get release information for all nodes
self.get_release()
self.post_init()
fuelnode.logs_excluded_nodes = self.logs_excluded_nodes
def fuel_init(self):
if not self.conf['fuel_ip']:
self.logger.critical('NodeManager: fuel_ip not set')
sys.exit(106)
fuelnode = Node(id=0,
cluster=0,
name='fuel',
fqdn='n/a',
mac='n/a',
os_platform='centos',
roles=['fuel'],
status='ready',
online=True,
ip=self.conf['fuel_ip'],
conf=self.conf)
fuelnode.cluster_repr = ""
fuelnode.repr = "fuel"
# soft-skip Fuel if it is hard-filtered
if not self.filter(fuelnode, self.conf['hard_filter']):
fuelnode.filtered_out = True
self.nodes[self.conf['fuel_ip']] = fuelnode
return fuelnode
def apply_soft_filter(self):
# apply soft-filter on all nodes
for node in self.nodes.values():
if not self.filter(node, self.conf['soft_filter']):
node.filtered_out = True
if self.conf['fuel_logs_exclude_filtered']:
self.logs_excluded_nodes.append(node.fqdn)
self.logs_excluded_nodes.append(node.ip)
def get_release(self):
if (not self.get_release_fuel_client() and
not self.get_release_api() and
not self.get_release_cli()):
self.logger.warning('could not get Fuel and MOS versions')
def get_nodes_fuelclient(self):
if not self.fuelclient:
return False
try:
self.logger.info('using fuelclient to get nodes json')
self.nodes_json = self.fuelclient.get_request('nodes')
return True
except Exception as e:
self.logger.warning(("NodeManager: can't "
"get node list from fuel client:\n%s" % (e)),
exc_info=True)
return False
def get_release_api(self):
self.logger.info('getting release via API')
version_json = self.get_api_request('version')
if version_json:
version = json.loads(version_json)
fuel = self.nodes[self.conf['fuel_ip']]
fuel.release = version['release']
else:
return False
clusters_json = self.get_api_request('clusters')
if clusters_json:
clusters = json.loads(clusters_json)
self.set_nodes_release(clusters)
return True
else:
return False
def get_release_fuel_client(self):
if not self.fuelclient:
return False
self.logger.info('getting release via fuelclient')
try:
v = self.fuelclient.get_request('version')
fuel_version = v['release']
self.logger.debug('version response:%s' % v)
clusters = self.fuelclient.get_request('clusters')
self.logger.debug('clusters response:%s' % clusters)
except:
self.logger.warning(("Can't get fuel version or "
"clusters information"))
return False
self.nodes[self.conf['fuel_ip']].release = fuel_version
self.set_nodes_release(clusters)
return True
def auth_token(self):
'''Get keystone token to access Nailgun API. Requires Fuel 5+'''
if self.token:
return True
self.logger.info('getting token for Nailgun')
v2_body = ('{"auth": {"tenantName": "%s", "passwordCredentials": {'
'"username": "%s", "password": "%s"}}}')
# v3 not fully implemented yet
# v3_body = ('{ "auth": {'
# ' "scope": {'
# ' "project": {'
# ' "name": "%s",'
# ' "domain": { "id": "default" }'
# ' }'
# ' },'
# ' "identity": {'
# ' "methods": ["password"],'
# ' "password": {'
# ' "user": {'
# ' "name": "%s",'
# ' "domain": { "id": "default" },'
# ' "password": "%s"'
# ' }'
# ' }'
# ' }'
# '}}')
# Sticking to v2 API for now because Fuel 9.1 has a custom
# domain_id defined in keystone.conf which we do not know.
args = {'user': None, 'pass': None, 'tenant': None}
for a in args:
if self.conf['fuel_%s' % a]:
args[a] = self.conf['fuel_%s' % a]
else:
args[a] = self.conf['fuel_api_%s' % a]
req_data = v2_body % (args['tenant'], args['user'], args['pass'])
req = urllib2.Request("http://%s:%s/v2.0/tokens" %
(self.conf['fuel_ip'],
self.conf['fuel_api_keystone_port']), req_data,
{'Content-Type': 'application/json'})
try:
# Disabling v3 token retrieval for now
# token = urllib2.urlopen(req).info().getheader('X-Subject-Token')
result = urllib2.urlopen(req)
resp_body = result.read()
resp_json = json.loads(resp_body)
token = resp_json['access']['token']['id']
self.token = token
return True
except:
return False
def get_api_request(self, request):
if self.auth_token():
url = "http://%s:%s/api/%s" % (self.conf['fuel_ip'],
self.conf['fuel_api_port'],
request)
req = urllib2.Request(url, None, {'X-Auth-Token': self.token})
try:
result = urllib2.urlopen(req)
code = result.getcode()
if code == 200:
return result.read()
else:
self.logger.error('NodeManager: cannot get API response'
' from %s, code %s' % (url, code))
except:
pass
def get_nodes_api(self):
self.logger.info('using API to get nodes json')
nodes_json = self.get_api_request('nodes')
if nodes_json:
self.nodes_json = json.loads(nodes_json)
return True
else:
return False
def get_nodes_cli(self):
self.logger.info('using CLI to get nodes json')
fuelnode = self.nodes[self.conf['fuel_ip']]
o_auth = n_auth = ''
entropy = bool(self.conf['fuel_user']) + bool(self.conf['fuel_pass'])
if entropy == 2:
# auth for Fuel up to 8.0
o_auth = '--user %s --password %s' % (self.conf['fuel_user'],
self.conf['fuel_pass'])
# Fuel 9.0+
n_auth = 'OS_USERNAME=%s OS_PASSWORD=%s' % (self.conf['fuel_user'],
self.conf['fuel_pass'])
elif entropy == 1:
self.logger.warning('Must specify both fuel_user and fuel_pass')
cmd = 'bash -c "%s fuel node --json"' % n_auth
nodes_json, err, code = tools.ssh_node(ip=fuelnode.ip,
command=cmd,
ssh_opts=fuelnode.ssh_opts,
timeout=fuelnode.timeout,
prefix=fuelnode.prefix)
if code != 0:
self.logger.warning(('NodeManager: cannot get fuel node list from'
' CLI, will fallback. Error: %s') % err)
cmd = 'bash -c "fuel %s node --json"' % o_auth
nodes_json, err, code = tools.ssh_node(ip=fuelnode.ip,
command=cmd,
ssh_opts=fuelnode.ssh_opts,
timeout=fuelnode.timeout,
prefix=fuelnode.prefix)
if code != 0:
self.logger.warning(('NodeManager: cannot get '
'fuel node list from CLI: %s') % err)
self.nodes_json = None
return False
self.nodes_json = json.loads(nodes_json)
return True
def get_release_cli(self):
run_items = []
for key, node in self.nodes.items():
if not node.filtered_out:
run_items.append(tools.RunItem(target=node.get_release,
key=key))
result = tools.run_batch(run_items, 100, dict_result=True)
if result:
for key in result:
self.nodes[key].release = result[key]
return True
else:
return False
def nodes_init_fallbacks(self):
self.nodes_get_roles_hiera()
self.nodes_get_os()
self.nodes_get_cluster_ids()
def nodes_get_roles_hiera(self, maxthreads=100):
run_items = []
for key, node in self.nodes.items():
if all([not node.filtered_out, not node.roles,
node.status != 'discover']):
run_items.append(tools.RunItem(target=node.get_roles_hiera,
key=key))
result = tools.run_batch(run_items, maxthreads, dict_result=True)
for key in result:
if result[key]:
self.nodes[key].roles = result[key]
def nodes_get_cluster_ids(self, maxthreads=100):
self.logger.debug('getting cluster ids from nodes')
run_items = []
for key, node in self.nodes.items():
if not node.filtered_out and not node.cluster:
run_items.append(tools.RunItem(target=node.get_cluster_id,
key=key))
result = tools.run_batch(run_items, maxthreads, dict_result=True)
for key in result:
if result[key] is not None:
self.nodes[key].cluster = result[key]
def set_nodes_release(self, clusters):
cldict = {}
for cluster in clusters:
cldict[cluster['id']] = cluster
if cldict:
for node in self.nodes.values():
if node.cluster:
node.release = cldict[node.cluster]['fuel_version']
else:
# set to n/a or may be fuel_version
if node.id != 0:
node.release = 'n/a'
self.logger.info('%s: release: %s' % (node.repr, node.release))

38
timmy/modules/local.py Normal file
View File

@ -0,0 +1,38 @@
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from timmy.nodes import NodeManager as BaseNodeManager
def add_args(parser):
parser.add_argument('-j', '--nodes-json', required=True,
help=('Path to a json file containing host info:'
' ip, roles, etc.'))
return parser
def check_args(args, conf):
pass
def add_conf(conf):
pass
class NodeManager(BaseNodeManager):
pass

428
timmy/nodes.py
View File

@ -23,35 +23,12 @@ from copy import deepcopy
from datetime import datetime, date, timedelta
from timmy.env import project_name
from tools import w_list, run_with_lock
import json
import logging
import os
import re
import shutil
import sys
import tools
import urllib2
try:
import fuelclient
if hasattr(fuelclient, 'connect'):
# fuel > 9.0.1 - drop support, use API and CLI instead
FuelClient = None
else:
import fuelclient.client
if type(fuelclient.client.APIClient) is fuelclient.client.Client:
# fuel 9.0.1 and below
from fuelclient.client import Client as FuelClient
else:
FuelClient = None
except:
FuelClient = None
try:
from fuelclient.client import logger
logger.handlers = []
except:
pass
class Node(object):
@ -186,60 +163,6 @@ class Node(object):
setattr(self, f, [])
r_apply(conf, p, c_a, k_d, overridden, d, clean=clean)
def get_release(self):
if self.id == 0:
cmd = ("awk -F ':' '/release/ {print $2}' "
"/etc/nailgun/version.yaml")
else:
cmd = ("awk -F ':' '/fuel_version/ {print $2}' "
"/etc/astute.yaml")
release, err, code = tools.ssh_node(ip=self.ip,
command=cmd,
ssh_opts=self.ssh_opts,
timeout=self.timeout,
prefix=self.prefix)
if code != 0:
self.logger.warning('%s: could not determine'
' MOS release' % self.repr)
release = 'n/a'
else:
release = release.strip('\n "\'')
self.logger.info('%s, MOS release: %s' %
(self.repr, release))
return release
def get_roles_hiera(self):
def trim_primary(roles):
trim_roles = [r for r in roles if not r.startswith('primary-')]
trim_roles += [r[8:] for r in roles if r.startswith('primary-')]
return trim_roles
self.logger.debug('%s: roles not defined, trying hiera' % self.repr)
cmd = 'hiera roles'
outs, errs, code = tools.ssh_node(ip=self.ip,
command=cmd,
ssh_opts=self.ssh_opts,
env_vars=self.env_vars,
timeout=self.timeout,
prefix=self.prefix)
self.check_code(code, 'get_roles_hiera', cmd, errs, [0])
if code == 0:
try:
roles = trim_primary(json.loads(outs))
except:
self.logger.warning("%s: failed to parse '%s' output as JSON" %
(self.repr, cmd))
return self.roles
self.logger.debug('%s: got roles: %s' % (self.repr, roles))
if roles is not None:
return roles
else:
return self.roles
else:
self.logger.warning("%s: failed to load roles via hiera" %
self.repr)
self.roles
def get_os(self):
self.logger.debug('%s: os_platform not defined, trying to determine' %
self.repr)
@ -252,22 +175,6 @@ class Node(object):
prefix=self.prefix)
return 'centos' if code else 'ubuntu'
def get_cluster_id(self):
self.logger.debug('%s: cluster id not defined, trying to determine' %
self.repr)
astute_file = '/etc/astute.yaml'
cmd = ("python -c 'import os,yaml; "
"exit(1) if not os.path.exists(\"%s\") else 0; "
"a = yaml.load(open(\"%s\").read()); "
"print a[\"cluster\"][\"id\"]'" % (astute_file, astute_file))
outs, errs, code = tools.ssh_node(ip=self.ip,
command=cmd,
ssh_opts=self.ssh_opts,
env_vars=self.env_vars,
timeout=self.timeout,
prefix=self.prefix)
return int(outs.rstrip('\n')) if code == 0 else None
def check_access(self):
self.logger.debug('%s: verifyng node access' %
self.repr)
@ -487,7 +394,10 @@ class Node(object):
recursive=True)
self.check_code(code, 'put_files', 'tools.put_file_scp', errs)
def logs_populate(self, timeout=5, logs_excluded_nodes=[]):
def log_item_manipulate(self, item):
pass
def logs_populate(self, timeout=5):
def filter_by_re(item, string):
return (('include' not in item or not item['include'] or
@ -496,20 +406,7 @@ class Node(object):
any([re.search(e, string) for e in item['exclude']])))
for item in self.logs:
if self.logs_no_fuel_remote and 'fuel' in self.roles:
self.logger.debug('adding Fuel remote logs to exclude list')
if 'exclude' not in item:
item['exclude'] = []
for remote_dir in self.logs_fuel_remote_dir:
item['exclude'].append(remote_dir)
if 'fuel' in self.roles:
for n in logs_excluded_nodes:
self.logger.debug('removing remote logs for node:%s' % n)
if 'exclude' not in item:
item['exclude'] = []
for remote_dir in self.logs_fuel_remote_dir:
ipd = os.path.join(remote_dir, n)
item['exclude'].append(ipd)
self.log_item_manipulate(item)
start_str = None
if 'start' in item or hasattr(self, 'logs_days'):
if hasattr(self, 'logs_days') and 'start' not in item:
@ -602,7 +499,16 @@ class Node(object):
class NodeManager(object):
"""Class nodes """
def __init__(self, conf, extended=False, nodes_json=None, logger=None):
def __init__(self, conf, nodes_json, logger=None):
self.base_init(conf, logger)
self.nodes_json = tools.load_json_file(nodes_json)
self.nodes_init(Node)
self.post_init()
def nodes_init_fallbacks(self):
self.nodes_get_os()
def base_init(self, conf, logger=None):
self.conf = conf
self.logger = logger or logging.getLogger(project_name)
if conf['outputs_timestamp'] or conf['dir_timestamp']:
@ -623,63 +529,17 @@ class NodeManager(object):
if self.conf['rqfile']:
self.import_rq()
self.nodes = {}
self.token = self.conf['fuel_api_token']
self.fuel_init()
# save os environment variables
environ = os.environ
self.logs_excluded_nodes = []
if FuelClient and conf['fuelclient']:
try:
if self.conf['fuel_skip_proxy']:
os.environ['HTTPS_PROXY'] = ''
os.environ['HTTP_PROXY'] = ''
os.environ['https_proxy'] = ''
os.environ['http_proxy'] = ''
self.logger.info('Setup fuelclient instance')
self.fuelclient = FuelClient()
if self.conf['fuel_user']:
self.fuelclient.username = self.conf['fuel_user']
if self.conf['fuel_pass']:
self.fuelclient.password = self.conf['fuel_pass']
if self.conf['fuel_tenant']:
self.fuelclient.tenant_name = self.conf['fuel_tenant']
# self.fuelclient.debug_mode(True)
except Exception as e:
self.logger.info('Failed to setup fuelclient instance:%s' % e,
exc_info=True)
self.fuelclient = None
else:
self.logger.info('Skipping setup fuelclient instance')
self.fuelclient = None
if nodes_json:
self.nodes_json = tools.load_json_file(nodes_json)
else:
if (not self.get_nodes_fuelclient() and
not self.get_nodes_api() and
not self.get_nodes_cli()):
self.logger.critical('Failed to retrieve node information.')
sys.exit(105)
self.nodes_init()
self.nodes_check_access()
# get release information for all nodes
if (not self.get_release_fuel_client() and
not self.get_release_api() and
not self.get_release_cli()):
self.logger.warning('could not get Fuel and MOS versions')
# fallbacks
self.nodes_get_roles_hiera()
self.nodes_get_os()
self.nodes_get_cluster_ids()
def apply_soft_filter(self):
# apply soft-filter on all nodes
for node in self.nodes.values():
# apply soft-filter on all nodes
if not self.filter(node, self.conf['soft_filter']):
node.filtered_out = True
if self.conf['logs_exclude_filtered']:
self.logs_excluded_nodes.append(node.fqdn)
self.logs_excluded_nodes.append(node.ip)
def post_init(self):
self.nodes_reapply_conf()
self.apply_soft_filter()
self.conf_assign_once()
os.environ = environ
def __str__(self):
def ml_column(matrix, i):
@ -773,219 +633,7 @@ class NodeManager(object):
for rqfile in self.conf['rqfile']:
merge_rq(rqfile, dst)
def fuel_init(self):
if not self.conf['fuel_ip']:
self.logger.critical('NodeManager: fuel_ip not set')
sys.exit(106)
fuelnode = Node(id=0,
cluster=0,
name='fuel',
fqdn='n/a',
mac='n/a',
os_platform='centos',
roles=['fuel'],
status='ready',
online=True,
ip=self.conf['fuel_ip'],
conf=self.conf)
fuelnode.cluster_repr = ""
fuelnode.repr = "fuel"
# soft-skip Fuel if it is hard-filtered
if not self.filter(fuelnode, self.conf['hard_filter']):
fuelnode.filtered_out = True
self.nodes[self.conf['fuel_ip']] = fuelnode
def get_nodes_fuelclient(self):
if not self.fuelclient:
return False
try:
self.logger.info('using fuelclient to get nodes json')
self.nodes_json = self.fuelclient.get_request('nodes')
return True
except Exception as e:
self.logger.warning(("NodeManager: can't "
"get node list from fuel client:\n%s" % (e)),
exc_info=True)
return False
def get_release_api(self):
self.logger.info('getting release via API')
version_json = self.get_api_request('version')
if version_json:
version = json.loads(version_json)
fuel = self.nodes[self.conf['fuel_ip']]
fuel.release = version['release']
else:
return False
clusters_json = self.get_api_request('clusters')
if clusters_json:
clusters = json.loads(clusters_json)
self.set_nodes_release(clusters)
return True
else:
return False
def get_release_fuel_client(self):
if not self.fuelclient:
return False
self.logger.info('getting release via fuelclient')
try:
v = self.fuelclient.get_request('version')
fuel_version = v['release']
self.logger.debug('version response:%s' % v)
clusters = self.fuelclient.get_request('clusters')
self.logger.debug('clusters response:%s' % clusters)
except:
self.logger.warning(("Can't get fuel version or "
"clusters information"))
return False
self.nodes[self.conf['fuel_ip']].release = fuel_version
self.set_nodes_release(clusters)
return True
def set_nodes_release(self, clusters):
cldict = {}
for cluster in clusters:
cldict[cluster['id']] = cluster
if cldict:
for node in self.nodes.values():
if node.cluster:
node.release = cldict[node.cluster]['fuel_version']
else:
# set to n/a or may be fuel_version
if node.id != 0:
node.release = 'n/a'
self.logger.info('%s: release: %s' % (node.repr, node.release))
def auth_token(self):
'''Get keystone token to access Nailgun API. Requires Fuel 5+'''
if self.token:
return True
self.logger.info('getting token for Nailgun')
v2_body = ('{"auth": {"tenantName": "%s", "passwordCredentials": {'
'"username": "%s", "password": "%s"}}}')
# v3 not fully implemented yet
# v3_body = ('{ "auth": {'
# ' "scope": {'
# ' "project": {'
# ' "name": "%s",'
# ' "domain": { "id": "default" }'
# ' }'
# ' },'
# ' "identity": {'
# ' "methods": ["password"],'
# ' "password": {'
# ' "user": {'
# ' "name": "%s",'
# ' "domain": { "id": "default" },'
# ' "password": "%s"'
# ' }'
# ' }'
# ' }'
# '}}')
# Sticking to v2 API for now because Fuel 9.1 has a custom
# domain_id defined in keystone.conf which we do not know.
args = {'user': None, 'pass': None, 'tenant': None}
for a in args:
if self.conf['fuel_%s' % a]:
args[a] = self.conf['fuel_%s' % a]
else:
args[a] = self.conf['fuel_api_%s' % a]
req_data = v2_body % (args['tenant'], args['user'], args['pass'])
req = urllib2.Request("http://%s:%s/v2.0/tokens" %
(self.conf['fuel_ip'],
self.conf['fuel_api_keystone_port']), req_data,
{'Content-Type': 'application/json'})
try:
# Disabling v3 token retrieval for now
# token = urllib2.urlopen(req).info().getheader('X-Subject-Token')
result = urllib2.urlopen(req)
resp_body = result.read()
resp_json = json.loads(resp_body)
token = resp_json['access']['token']['id']
self.token = token
return True
except:
return False
def get_api_request(self, request):
if self.auth_token():
url = "http://%s:%s/api/%s" % (self.conf['fuel_ip'],
self.conf['fuel_api_port'],
request)
req = urllib2.Request(url, None, {'X-Auth-Token': self.token})
try:
result = urllib2.urlopen(req)
code = result.getcode()
if code == 200:
return result.read()
else:
self.logger.error('NodeManager: cannot get API response'
' from %s, code %s' % (url, code))
except:
pass
def get_nodes_api(self):
self.logger.info('using API to get nodes json')
nodes_json = self.get_api_request('nodes')
if nodes_json:
self.nodes_json = json.loads(nodes_json)
return True
else:
return False
def get_nodes_cli(self):
self.logger.info('using CLI to get nodes json')
fuelnode = self.nodes[self.conf['fuel_ip']]
o_auth = n_auth = ''
entropy = bool(self.conf['fuel_user']) + bool(self.conf['fuel_pass'])
if entropy == 2:
# auth for Fuel up to 8.0
o_auth = '--user %s --password %s' % (self.conf['fuel_user'],
self.conf['fuel_pass'])
# Fuel 9.0+
n_auth = 'OS_USERNAME=%s OS_PASSWORD=%s' % (self.conf['fuel_user'],
self.conf['fuel_pass'])
elif entropy == 1:
self.logger.warning('Must specify both fuel_user and fuel_pass')
cmd = 'bash -c "%s fuel node --json"' % n_auth
nodes_json, err, code = tools.ssh_node(ip=fuelnode.ip,
command=cmd,
ssh_opts=fuelnode.ssh_opts,
timeout=fuelnode.timeout,
prefix=fuelnode.prefix)
if code != 0:
self.logger.warning(('NodeManager: cannot get fuel node list from'
' CLI, will fallback. Error: %s') % err)
cmd = 'bash -c "fuel %s node --json"' % o_auth
nodes_json, err, code = tools.ssh_node(ip=fuelnode.ip,
command=cmd,
ssh_opts=fuelnode.ssh_opts,
timeout=fuelnode.timeout,
prefix=fuelnode.prefix)
if code != 0:
self.logger.warning(('NodeManager: cannot get '
'fuel node list from CLI: %s') % err)
self.nodes_json = None
return False
self.nodes_json = json.loads(nodes_json)
return True
def get_release_cli(self):
run_items = []
for key, node in self.nodes.items():
if not node.filtered_out:
run_items.append(tools.RunItem(target=node.get_release,
key=key))
result = tools.run_batch(run_items, 100, dict_result=True)
if result:
for key in result:
self.nodes[key].release = result[key]
return True
else:
return False
def nodes_init(self):
def nodes_init(self, NodeClass):
for node_data in self.nodes_json:
params = {'conf': self.conf}
keys = ['id', 'cluster', 'roles', 'fqdn', 'name', 'mac',
@ -993,9 +641,11 @@ class NodeManager(object):
for key in keys:
if key in node_data:
params[key] = node_data[key]
node = Node(**params)
node = NodeClass(**params)
if self.filter(node, self.conf['hard_filter']):
self.nodes[node.ip] = node
self.nodes_check_access()
self.nodes_init_fallbacks()
def conf_assign_once(self):
once = Node.conf_once_prefix
@ -1021,18 +671,6 @@ class NodeManager(object):
for node in self.nodes.values():
node.apply_conf(self.conf)
def nodes_get_roles_hiera(self, maxthreads=100):
run_items = []
for key, node in self.nodes.items():
if all([not node.filtered_out, not node.roles,
node.status != 'discover']):
run_items.append(tools.RunItem(target=node.get_roles_hiera,
key=key))
result = tools.run_batch(run_items, maxthreads, dict_result=True)
for key in result:
if result[key]:
self.nodes[key].roles = result[key]
def nodes_get_os(self, maxthreads=100):
run_items = []
for key, node in self.nodes.items():
@ -1043,18 +681,6 @@ class NodeManager(object):
if result[key]:
self.nodes[key].os_platform = result[key]
def nodes_get_cluster_ids(self, maxthreads=100):
self.logger.debug('getting cluster ids from nodes')
run_items = []
for key, node in self.nodes.items():
if not node.filtered_out and not node.cluster:
run_items.append(tools.RunItem(target=node.get_cluster_id,
key=key))
result = tools.run_batch(run_items, maxthreads, dict_result=True)
for key in result:
if result[key] is not None:
self.nodes[key].cluster = result[key]
def nodes_check_access(self, maxthreads=100):
self.logger.debug('checking if nodes are accessible')
run_items = []
@ -1111,10 +737,8 @@ class NodeManager(object):
run_items = []
for key, node in self.nodes.items():
if not node.filtered_out:
args = {'timeout': timeout,
'logs_excluded_nodes': self.logs_excluded_nodes}
run_items.append(tools.RunItem(target=node.logs_populate,
args=args,
args={'timeout': timeout},
key=key))
result = tools.run_batch(run_items, maxthreads, dict_result=True)
for key in result:

1
timmy/tests/test_conf.py
View File

@ -45,7 +45,6 @@ class ConfTest(unittest.TestCase):
'filelists': list,
'logs': list,
'logs_no_default': bool,
'logs_exclude_filtered': bool,
'logs_days': int,
'logs_speed_limit': bool,
'logs_speed_default': int,

2
timmy/tools.py
View File

@ -55,6 +55,8 @@ def signal_wrapper(f):
try:
f(*args, **kwargs)
except Exception as e:
if not logger.handlers:
logging.basicConfig()
logger.error('Error: %s' % e, exc_info=True)
for k in dir(e):
'''debug: print all exception attrs except internal