Remove openrc/admin-token usage in q-agent-cleanup.py
Removes the openrc/admin-token deps in q-agent-cleanup.py. Instead it will read the neutron service credentials out of neutron.conf. This has the advantage that it must be up to date for neutron to function so if the credential is modified it will have to be updated by the administrator. Related-bug: #1347542 Closes-bug: #1396594 Change-Id: Iee0968a079deacef24b56f34e9314b0ebecad0ae
This commit is contained in:
parent
f9d7cea160
commit
fa44e4ac5e
@ -1,43 +1,102 @@
|
||||
#!/usr/bin/env python
|
||||
# flake8: noqa
|
||||
import re
|
||||
import time
|
||||
import os
|
||||
import sys
|
||||
import random
|
||||
import string
|
||||
import json
|
||||
# Copyright 2013 - 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
from ConfigParser import SafeConfigParser
|
||||
import functools
|
||||
import json
|
||||
import logging
|
||||
import logging.config
|
||||
import logging.handlers
|
||||
import shlex
|
||||
import subprocess
|
||||
import StringIO
|
||||
import re
|
||||
import socket
|
||||
from neutronclient.neutron import client as q_client
|
||||
from keystoneclient.v2_0 import client as ks_client
|
||||
from keystoneclient.apiclient.exceptions import NotFound as ks_NotFound
|
||||
import StringIO
|
||||
import subprocess
|
||||
import sys
|
||||
from time import sleep
|
||||
|
||||
from neutronclient.neutron import client as n_client
|
||||
|
||||
LOG_NAME = 'q-agent-cleanup'
|
||||
|
||||
API_VER = '2.0'
|
||||
PORT_ID_PART_LEN = 11
|
||||
TMP_USER_NAME = 'tmp_neutron_admin'
|
||||
|
||||
|
||||
def get_authconfig(cfg_file):
|
||||
# Read OS auth config file
|
||||
rv = {}
|
||||
stripchars=" \'\""
|
||||
def make_logger(handler=logging.StreamHandler(sys.stdout), level=logging.INFO):
|
||||
format = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
|
||||
handler.setFormatter(format)
|
||||
logger = logging.getLogger(LOG_NAME)
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(level)
|
||||
return logger
|
||||
|
||||
LOG = make_logger()
|
||||
|
||||
AUTH_KEYS = {
|
||||
'tenant_name': 'admin_tenant_name',
|
||||
'username': 'admin_user',
|
||||
'password': 'admin_password',
|
||||
'auth_url': 'auth_uri',
|
||||
}
|
||||
|
||||
|
||||
def get_auth_data(cfg_file, section='keystone_authtoken', keys=AUTH_KEYS):
|
||||
cfg = SafeConfigParser()
|
||||
with open(cfg_file) as f:
|
||||
for line in f:
|
||||
rg = re.match(r'\s*export\s+(\w+)\s*=\s*(.*)',line)
|
||||
if rg :
|
||||
#Use shlex to unescape bash shell escape characters
|
||||
value = "".join(x for x in
|
||||
shlex.split(rg.group(2).strip(stripchars)))
|
||||
rv[rg.group(1).strip(stripchars)] = value
|
||||
return rv
|
||||
cfg.readfp(f)
|
||||
auth_data = {}
|
||||
for key, value in keys.iteritems():
|
||||
auth_data[key] = cfg.get(section, value)
|
||||
return auth_data
|
||||
|
||||
# Note(xarses): be careful not to inject \n's into the regex pattern
|
||||
# or it will case the maching to fail
|
||||
RECOVERABLE = re.compile((
|
||||
'(HTTP\s+400\))|'
|
||||
'(400-\{\'message\'\:\s+\'\'\})|'
|
||||
'(\[Errno 111\]\s+Connection\s+refused)|'
|
||||
'(503\s+Service\s+Unavailable)|'
|
||||
'(504\s+Gateway\s+Time-out)|'
|
||||
'(\:\s+Maximum\s+attempts\s+reached)|'
|
||||
'(Unauthorized\:\s+bad\s+credentials)|'
|
||||
'(Max\s+retries\s+exceeded)|'
|
||||
"""('*NoneType'*\s+object\s+ha'\s+no\s+attribute\s+'*__getitem__'*$)|"""
|
||||
'(No\s+route\s+to\s+host$)|'
|
||||
'(Lost\s+connection\s+to\s+MySQL\s+server)'), flags=re.M)
|
||||
|
||||
RETRY_COUNT = 50
|
||||
RETRY_DELAY = 2
|
||||
|
||||
|
||||
def retry(func, pattern=RECOVERABLE):
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
i = 0
|
||||
while True:
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except Exception as e:
|
||||
if pattern and not pattern.match(e.message):
|
||||
raise e
|
||||
i += 1
|
||||
if i >= RETRY_COUNT:
|
||||
raise e
|
||||
print("retry request {0}: {1}".format(i, e))
|
||||
sleep(RETRY_DELAY)
|
||||
return wrapper
|
||||
|
||||
|
||||
class NeutronCleaner(object):
|
||||
@ -54,7 +113,7 @@ class NeutronCleaner(object):
|
||||
PORT_NAME_PREFIXES_BY_DEV_OWNER['network:router_interface']
|
||||
)
|
||||
}
|
||||
BRIDGES_FOR_PORTS_BY_AGENT ={
|
||||
BRIDGES_FOR_PORTS_BY_AGENT = {
|
||||
'dhcp': ('br-int',),
|
||||
'l3': ('br-int', 'br-ex'),
|
||||
}
|
||||
@ -78,11 +137,12 @@ class NeutronCleaner(object):
|
||||
CMD__ip_netns_list = ['ip', 'netns', 'list']
|
||||
CMD__ip_netns_exec = ['ip', 'netns', 'exec']
|
||||
|
||||
RE__port_in_portlist = re.compile(r"^\s*\d+\:\s+([\w-]+)\:") # 14: tap-xxxyyyzzz:
|
||||
# 14: tap-xxxyyyzzz:
|
||||
RE__port_in_portlist = re.compile(r"^\s*\d+\:\s+([\w-]+)\:")
|
||||
|
||||
def __init__(self, openrc, options, log=None):
|
||||
def __init__(self, options, log=None):
|
||||
self.log = log
|
||||
self.auth_config = openrc
|
||||
self.auth_data = get_auth_data(cfg_file=options.get('authconf'))
|
||||
self.options = options
|
||||
self.agents = {}
|
||||
self.debug = options.get('debug')
|
||||
@ -91,180 +151,93 @@ class NeutronCleaner(object):
|
||||
'l3': self._reschedule_agent_l3,
|
||||
}
|
||||
|
||||
self._token = None
|
||||
self._keystone = None
|
||||
self._client = None
|
||||
self._need_cleanup_tmp_admin = False
|
||||
|
||||
def __del__(self):
|
||||
if self._need_cleanup_tmp_admin and self._keystone and self._keystone.username:
|
||||
try:
|
||||
self._keystone.users.delete(self._keystone.users.find(username=self._keystone.username))
|
||||
except:
|
||||
# if we get exception while cleaning temporary account -- nothing harm
|
||||
pass
|
||||
|
||||
def generate_random_passwd(self, length=13):
|
||||
chars = string.ascii_letters + string.digits + '!@#$%^&*()'
|
||||
random.seed = (os.urandom(1024))
|
||||
return ''.join(random.choice(chars) for i in range(length))
|
||||
|
||||
@property
|
||||
def keystone(self):
|
||||
if self._keystone is None:
|
||||
ret_count = self.options.get('retries', 1)
|
||||
tmp_passwd = self.generate_random_passwd()
|
||||
while True:
|
||||
if ret_count <= 0:
|
||||
self.log.error(">>> Keystone error: no more retries for connect to keystone server.")
|
||||
sys.exit(1)
|
||||
try:
|
||||
a_token = self.options.get('auth-token')
|
||||
a_url = self.options.get('admin-auth-url')
|
||||
if a_token and a_url:
|
||||
self.log.debug("Authentication by predefined token.")
|
||||
# create keystone instance, authorized by service token
|
||||
ks = ks_client.Client(
|
||||
token=a_token,
|
||||
endpoint=a_url,
|
||||
)
|
||||
service_tenant = ks.tenants.find(name='services')
|
||||
auth_url = ks.endpoints.find(
|
||||
service_id=ks.services.find(type='identity').id
|
||||
).internalurl
|
||||
# find and re-create temporary rescheduling-admin user with random password
|
||||
try:
|
||||
user = ks.users.find(username=TMP_USER_NAME)
|
||||
ks.users.delete(user)
|
||||
except ks_NotFound:
|
||||
# user not found, it's OK
|
||||
pass
|
||||
user = ks.users.create(TMP_USER_NAME, tmp_passwd, tenant_id=service_tenant.id)
|
||||
ks.roles.add_user_role(user, ks.roles.find(name='admin'), service_tenant)
|
||||
# authenticate newly-created tmp neutron admin
|
||||
self._keystone = ks_client.Client(
|
||||
username=user.username,
|
||||
password=tmp_passwd,
|
||||
tenant_id=user.tenantId,
|
||||
auth_url=auth_url,
|
||||
)
|
||||
self._need_cleanup_tmp_admin = True
|
||||
else:
|
||||
self.log.debug("Authentication by given credentionals.")
|
||||
self._keystone = ks_client.Client(
|
||||
username=self.auth_config['OS_USERNAME'],
|
||||
password=self.auth_config['OS_PASSWORD'],
|
||||
tenant_name=self.auth_config['OS_TENANT_NAME'],
|
||||
auth_url=self.auth_config['OS_AUTH_URL'],
|
||||
)
|
||||
break
|
||||
except Exception as e:
|
||||
errmsg = str(e.message).strip() # str() need, because keystone may use int as message in exception
|
||||
if re.search(r"Connection\s+refused$", errmsg, re.I) or \
|
||||
re.search(r"Connection\s+timed\s+out$", errmsg, re.I) or\
|
||||
re.search(r"Lost\s+connection\s+to\s+MySQL\s+server", errmsg, re.I) or\
|
||||
re.search(r"Service\s+Unavailable$", errmsg, re.I) or\
|
||||
re.search(r"'*NoneType'*\s+object\s+has\s+no\s+attribute\s+'*__getitem__'*$", errmsg, re.I) or \
|
||||
re.search(r"No\s+route\s+to\s+host$", errmsg, re.I):
|
||||
self.log.info(">>> Can't connect to {0}, wait for server ready...".format(self.auth_config['OS_AUTH_URL']))
|
||||
time.sleep(self.options.sleep)
|
||||
else:
|
||||
self.log.error(">>> Keystone error:\n{0}".format(e.message))
|
||||
raise e
|
||||
ret_count -= 1
|
||||
return self._keystone
|
||||
@property
|
||||
def token(self):
|
||||
if self._token is None:
|
||||
self._token = self._keystone.auth_token
|
||||
#self.log.debug("Auth_token: '{0}'".format(self._token))
|
||||
#todo: Validate existing token
|
||||
return self._token
|
||||
|
||||
@property
|
||||
@retry
|
||||
def client(self):
|
||||
if self._client is None:
|
||||
self._client = q_client.Client(
|
||||
API_VER,
|
||||
endpoint_url=self.keystone.endpoints.find(
|
||||
service_id=self.keystone.services.find(type='network').id
|
||||
).adminurl,
|
||||
token=self.token,
|
||||
)
|
||||
self._client = n_client.Client(API_VER, **self.auth_data)
|
||||
return self._client
|
||||
|
||||
def _neutron_API_call(self, method, *args):
|
||||
ret_count = self.options.get('retries')
|
||||
while True:
|
||||
if ret_count <= 0:
|
||||
self.log.error("Q-server error: no more retries for connect to server.")
|
||||
return []
|
||||
try:
|
||||
rv = method (*args)
|
||||
break
|
||||
except Exception as e:
|
||||
errmsg = str(e.message).strip()
|
||||
if re.search(r"Connection\s+refused", errmsg, re.I) or\
|
||||
re.search(r"Connection\s+timed\s+out", errmsg, re.I) or\
|
||||
re.search(r"Lost\s+connection\s+to\s+MySQL\s+server", errmsg, re.I) or\
|
||||
re.search(r"503\s+Service\s+Unavailable", errmsg, re.I) or\
|
||||
re.search(r"No\s+route\s+to\s+host", errmsg, re.I):
|
||||
self.log.info("Can't connect to {0}, wait for server ready...".format(self.keystone.service_catalog.url_for(service_type='network')))
|
||||
time.sleep(self.options.sleep)
|
||||
else:
|
||||
self.log.error("Neutron error:\n{0}".format(e.message))
|
||||
raise e
|
||||
ret_count -= 1
|
||||
return rv
|
||||
|
||||
def _get_agents(self,use_cache=True):
|
||||
return self._neutron_API_call(self.client.list_agents)['agents']
|
||||
@retry
|
||||
def _get_agents(self, use_cache=True):
|
||||
return self.client.list_agents()['agents']
|
||||
|
||||
@retry
|
||||
def _get_routers(self, use_cache=True):
|
||||
return self._neutron_API_call(self.client.list_routers)['routers']
|
||||
return self.client.list_routers()['routers']
|
||||
|
||||
@retry
|
||||
def _get_networks(self, use_cache=True):
|
||||
return self._neutron_API_call(self.client.list_networks)['networks']
|
||||
return self.client.list_networks()['networks']
|
||||
|
||||
@retry
|
||||
def _list_networks_on_dhcp_agent(self, agent_id):
|
||||
return self._neutron_API_call(self.client.list_networks_on_dhcp_agent, agent_id)['networks']
|
||||
return self.client.list_networks_on_dhcp_agent(
|
||||
agent_id)['networks']
|
||||
|
||||
@retry
|
||||
def _list_routers_on_l3_agent(self, agent_id):
|
||||
return self._neutron_API_call(self.client.list_routers_on_l3_agent, agent_id)['routers']
|
||||
return self.client.list_routers_on_l3_agent(
|
||||
agent_id)['routers']
|
||||
|
||||
@retry
|
||||
def _list_l3_agents_on_router(self, router_id):
|
||||
return self._neutron_API_call(self.client.list_l3_agent_hosting_routers, router_id)['agents']
|
||||
return self.client.list_l3_agent_hosting_routers(
|
||||
router_id)['agents']
|
||||
|
||||
@retry
|
||||
def _list_dhcp_agents_on_network(self, network_id):
|
||||
return self._neutron_API_call(self.client.list_dhcp_agent_hosting_networks, network_id)['agents']
|
||||
return self.client.list_dhcp_agent_hosting_networks(
|
||||
network_id)['agents']
|
||||
|
||||
def _list_orphaned_networks(self):
|
||||
networks = self._get_networks()
|
||||
self.log.debug("_list_orphaned_networks:, got list of networks {0}".format(json.dumps(networks,indent=4)))
|
||||
self.log.debug(
|
||||
"_list_orphaned_networks:, got list of networks {0}".format(
|
||||
json.dumps(networks, indent=4)))
|
||||
orphaned_networks = []
|
||||
for network in networks:
|
||||
if len(self._list_dhcp_agents_on_network(network['id'])) == 0:
|
||||
orphaned_networks.append(network['id'])
|
||||
self.log.debug("_list_orphaned_networks:, got list of orphaned networks {0}".format(orphaned_networks))
|
||||
self.log.debug(
|
||||
"_list_orphaned_networks:, got list of orphaned networks {0}".
|
||||
format(orphaned_networks))
|
||||
return orphaned_networks
|
||||
|
||||
def _list_orphaned_routers(self):
|
||||
routers = self._get_routers()
|
||||
self.log.debug("_list_orphaned_routers:, got list of routers {0}".format(json.dumps(routers,indent=4)))
|
||||
self.log.debug(
|
||||
"_list_orphaned_routers:, got list of routers {0}".format(
|
||||
json.dumps(routers, indent=4)))
|
||||
orphaned_routers = []
|
||||
for router in routers:
|
||||
if len(self._list_l3_agents_on_router(router['id'])) == 0:
|
||||
orphaned_routers.append(router['id'])
|
||||
self.log.debug("_list_orphaned_routers:, got list of orphaned routers {0}".format(orphaned_routers))
|
||||
self.log.debug(
|
||||
"_list_orphaned_routers:, got list of orphaned routers {0}".format(
|
||||
orphaned_routers))
|
||||
return orphaned_routers
|
||||
|
||||
@retry
|
||||
def _add_network_to_dhcp_agent(self, agent_id, net_id):
|
||||
return self._neutron_API_call(self.client.add_network_to_dhcp_agent, agent_id, {"network_id": net_id})
|
||||
def _add_router_to_l3_agent(self, agent_id, router_id):
|
||||
return self._neutron_API_call(self.client.add_router_to_l3_agent, agent_id, {"router_id": router_id})
|
||||
return self.client.add_network_to_dhcp_agent(
|
||||
agent_id, {"network_id": net_id})
|
||||
|
||||
@retry
|
||||
def _add_router_to_l3_agent(self, agent_id, router_id):
|
||||
return self.client.add_router_to_l3_agent(
|
||||
agent_id, {"router_id": router_id})
|
||||
|
||||
@retry
|
||||
def _remove_router_from_l3_agent(self, agent_id, router_id):
|
||||
return self._neutron_API_call(self.client.remove_router_from_l3_agent, agent_id, router_id)
|
||||
return self.client.remove_router_from_l3_agent(
|
||||
agent_id, router_id)
|
||||
|
||||
@retry
|
||||
def _delete_agent(self, agent_id):
|
||||
return self.client.delete_agent(agent_id)
|
||||
|
||||
def _get_agents_by_type(self, agent, use_cache=True):
|
||||
self.log.debug("_get_agents_by_type: start.")
|
||||
@ -277,7 +250,9 @@ class NeutronCleaner(object):
|
||||
from_cache = ''
|
||||
else:
|
||||
from_cache = ' from local cache'
|
||||
self.log.debug("_get_agents_by_type: end, {0} rv: {1}".format(from_cache, json.dumps(rv, indent=4)))
|
||||
self.log.debug(
|
||||
"_get_agents_by_type: end, {0} rv: {1}".format(
|
||||
from_cache, json.dumps(rv, indent=4)))
|
||||
return rv
|
||||
|
||||
def __collect_namespaces_for_agent(self, agent):
|
||||
@ -291,7 +266,9 @@ class NeutronCleaner(object):
|
||||
)
|
||||
rc = process.wait()
|
||||
if rc != 0:
|
||||
self.log.error("ERROR (rc={0}) while execution {1}".format(rc, ' '.join(cmd)))
|
||||
self.log.error(
|
||||
"ERROR (rc={0}) while execution {1}".format(
|
||||
rc, ' '.join(cmd)))
|
||||
return []
|
||||
# filter namespaces by given agent type
|
||||
netns = []
|
||||
@ -299,7 +276,7 @@ class NeutronCleaner(object):
|
||||
for ns in StringIO.StringIO(stdout):
|
||||
ns = ns.strip()
|
||||
self.log.debug("Found network namespace '{0}'".format(ns))
|
||||
if ns.startswith("{0}-".format(self.NS_NAME_PREFIXES[agent])):
|
||||
if ns.startswith(self.NS_NAME_PREFIXES[agent]):
|
||||
netns.append(ns)
|
||||
return netns
|
||||
|
||||
@ -315,7 +292,9 @@ class NeutronCleaner(object):
|
||||
)
|
||||
rc = process.wait()
|
||||
if rc != 0:
|
||||
self.log.error("ERROR (rc={0}) while execution {1}".format(rc, ' '.join(cmd)))
|
||||
self.log.error(
|
||||
"ERROR (rc={0}) while execution {1}".format(
|
||||
rc, ' '.join(cmd)))
|
||||
return []
|
||||
ports = []
|
||||
stdout = process.communicate()[0]
|
||||
@ -356,7 +335,9 @@ class NeutronCleaner(object):
|
||||
)
|
||||
rc = process.wait()
|
||||
if rc != 0:
|
||||
self.log.error("ERROR (rc={0}) while execution {1}".format(rc, ' '.join(cmd)))
|
||||
self.log.error(
|
||||
"ERROR (rc={0}) while execution {1}".format(
|
||||
rc, ' '.join(cmd)))
|
||||
self.log.debug("_cleanup_ports: end.")
|
||||
|
||||
return True
|
||||
@ -371,11 +352,13 @@ class NeutronCleaner(object):
|
||||
dead_networks = []
|
||||
for agent in self._get_agents_by_type(agent_type):
|
||||
if agent['alive']:
|
||||
self.log.info("found alive DHCP agent: {0}".format(agent['id']))
|
||||
self.log.info(
|
||||
"found alive DHCP agent: {0}".format(agent['id']))
|
||||
agents['alive'].append(agent)
|
||||
else:
|
||||
# dead agent
|
||||
self.log.info("found dead DHCP agent: {0}".format(agent['id']))
|
||||
self.log.info(
|
||||
"found dead DHCP agent: {0}".format(agent['id']))
|
||||
agents['dead'].append(agent)
|
||||
for net in self._list_networks_on_dhcp_agent(agent['id']):
|
||||
dead_networks.append(net)
|
||||
@ -391,28 +374,33 @@ class NeutronCleaner(object):
|
||||
for net in dead_networks:
|
||||
if net['id'] not in lucky_ids:
|
||||
# attach network to agent
|
||||
self.log.info("attach network {net} to DHCP agent {agent}".format(
|
||||
net=net['id'],
|
||||
agent=agents['alive'][0]['id']
|
||||
))
|
||||
self.log.info(
|
||||
"attach network {net} to DHCP agent {agent}".format(
|
||||
net=net['id'],
|
||||
agent=agents['alive'][0]['id']))
|
||||
if not self.options.get('noop'):
|
||||
self._add_network_to_dhcp_agent(agents['alive'][0]['id'], net['id'])
|
||||
#if error:
|
||||
# return
|
||||
self._add_network_to_dhcp_agent(
|
||||
agents['alive'][0]['id'], net['id'])
|
||||
|
||||
# remove dead agents if need (and if found alive agent)
|
||||
if self.options.get('remove-dead'):
|
||||
for agent in agents['dead']:
|
||||
self.log.info("remove dead DHCP agent: {0}".format(agent['id']))
|
||||
self.log.info(
|
||||
"remove dead DHCP agent: {0}".format(agent['id']))
|
||||
if not self.options.get('noop'):
|
||||
self._neutron_API_call(self.client.delete_agent, agent['id'])
|
||||
orphaned_networks=self._list_orphaned_networks()
|
||||
self._delete_agent(agent['id'])
|
||||
orphaned_networks = self._list_orphaned_networks()
|
||||
self.log.info("_reschedule_agent_dhcp: rescheduling orphaned networks")
|
||||
if orphaned_networks and agents['alive']:
|
||||
for network in orphaned_networks:
|
||||
self.log.info("_reschedule_agent_dhcp: rescheduling {0} to {1}".format(network,agents['alive'][0]['id']))
|
||||
self.log.info(
|
||||
"_reschedule_agent_dhcp: rescheduling {0} to {1}".format(
|
||||
network, agents['alive'][0]['id']))
|
||||
if not self.options.get('noop'):
|
||||
self._add_network_to_dhcp_agent(agents['alive'][0]['id'], network)
|
||||
self.log.info("_reschedule_agent_dhcp: ended rescheduling of orphaned networks")
|
||||
self._add_network_to_dhcp_agent(
|
||||
agents['alive'][0]['id'], network)
|
||||
self.log.info(
|
||||
"_reschedule_agent_dhcp: ended rescheduling of orphaned networks")
|
||||
self.log.debug("_reschedule_agent_dhcp: end.")
|
||||
|
||||
def _reschedule_agent_l3(self, agent_type):
|
||||
@ -435,9 +423,11 @@ class NeutronCleaner(object):
|
||||
lambda rou: dead_routers.append((rou, agent['id'])),
|
||||
self._list_routers_on_l3_agent(agent['id'])
|
||||
)
|
||||
self.log.debug("L3 agents in cluster: {ags}".format(ags=json.dumps(agents, indent=4)))
|
||||
self.log.debug("Routers, attached to dead L3 agents: {rr}".format(rr=json.dumps(dead_routers, indent=4)))
|
||||
|
||||
self.log.debug(
|
||||
"L3 agents in cluster: {0}".format(
|
||||
json.dumps(agents, indent=4)))
|
||||
self.log.debug("Routers, attached to dead L3 agents: {0}".format(
|
||||
json.dumps(dead_routers, indent=4)))
|
||||
|
||||
if dead_routers and agents['alive']:
|
||||
# get router-ID list of already attached to alive agent routerss
|
||||
@ -450,49 +440,48 @@ class NeutronCleaner(object):
|
||||
for agent in agents['dead']:
|
||||
self.log.info("remove dead L3 agent: {0}".format(agent['id']))
|
||||
if not self.options.get('noop'):
|
||||
self._neutron_API_call(self.client.delete_agent, agent['id'])
|
||||
self._delete_agent(agent['id'])
|
||||
# move routers from dead to alive agent
|
||||
for rou in filter(lambda rr: not(rr[0]['id'] in lucky_ids), dead_routers):
|
||||
# self.log.info("unschedule router {rou} from L3 agent {agent}".format(
|
||||
# rou=rou[0]['id'],
|
||||
# agent=rou[1]
|
||||
# ))
|
||||
# if not self.options.get('noop'):
|
||||
# self._remove_router_from_l3_agent(rou[1], rou[0]['id'])
|
||||
# #todo: if error:
|
||||
# #
|
||||
self.log.info("schedule router {rou} to L3 agent {agent}".format(
|
||||
rou=rou[0]['id'],
|
||||
agent=agents['alive'][0]['id']
|
||||
))
|
||||
for rou in filter(
|
||||
lambda rr: not(rr[0]['id'] in lucky_ids), dead_routers):
|
||||
self.log.info(
|
||||
"schedule router {0} to L3 agent {1}".format(
|
||||
rou[0]['id'],
|
||||
agents['alive'][0]['id']))
|
||||
if not self.options.get('noop'):
|
||||
self._add_router_to_l3_agent(agents['alive'][0]['id'], rou[0]['id'])
|
||||
self._add_router_to_l3_agent(
|
||||
agents['alive'][0]['id'], rou[0]['id'])
|
||||
|
||||
orphaned_routers=self._list_orphaned_routers()
|
||||
orphaned_routers = self._list_orphaned_routers()
|
||||
self.log.info("_reschedule_agent_l3: rescheduling orphaned routers")
|
||||
if orphaned_routers and agents['alive']:
|
||||
for router in orphaned_routers:
|
||||
self.log.info("_reschedule_agent_l3: rescheduling {0} to {1}".format(router,agents['alive'][0]['id']))
|
||||
self.log.info(
|
||||
"_reschedule_agent_l3: rescheduling {0} to {1}".format(
|
||||
router, agents['alive'][0]['id']))
|
||||
if not self.options.get('noop'):
|
||||
self._add_router_to_l3_agent(agents['alive'][0]['id'], router)
|
||||
self.log.info("_reschedule_agent_l3: ended rescheduling of orphaned routers")
|
||||
self._add_router_to_l3_agent(
|
||||
agents['alive'][0]['id'], router)
|
||||
self.log.info(
|
||||
"_reschedule_agent_l3: ended rescheduling of orphaned routers")
|
||||
self.log.debug("_reschedule_agent_l3: end.")
|
||||
|
||||
def _remove_self(self,agent_type):
|
||||
def _remove_self(self, agent_type):
|
||||
self.log.debug("_remove_self: start.")
|
||||
for agent in self._get_agents_by_type(agent_type):
|
||||
if agent['host'] == socket.gethostname():
|
||||
self.log.info("_remove_self: deleting our own agent {0} of type {1}".format(agent['id'],agent_type))
|
||||
if not self.options.get('noop'):
|
||||
self._neutron_API_call(self.client.delete_agent, agent['id'])
|
||||
self.log.info(
|
||||
"_remove_self: deleting our own agent {0} of type {1}".
|
||||
format(agent['id'], agent_type))
|
||||
if not self.options.get('noop'):
|
||||
self._delete_agent(agent['id'])
|
||||
self.log.debug("_remove_self: end.")
|
||||
|
||||
|
||||
def _reschedule_agent(self, agent):
|
||||
self.log.debug("_reschedule_agents: start.")
|
||||
task = self.RESCHEDULING_CALLS.get(agent, None)
|
||||
if task:
|
||||
task (agent)
|
||||
task(agent)
|
||||
self.log.debug("_reschedule_agents: end.")
|
||||
|
||||
def do(self, agent):
|
||||
@ -502,8 +491,6 @@ class NeutronCleaner(object):
|
||||
self._reschedule_agent(agent)
|
||||
if self.options.get('remove-self'):
|
||||
self._remove_self(agent)
|
||||
# if self.options.get('remove-agent'):
|
||||
# self._cleanup_agents(agent)
|
||||
|
||||
def _test_healthy(self, agent_list, hostname):
|
||||
rv = False
|
||||
@ -513,7 +500,10 @@ class NeutronCleaner(object):
|
||||
return rv
|
||||
|
||||
def test_healthy(self, agent_type):
|
||||
rc = 9 # OCF_FAILED_MASTER, http://www.linux-ha.org/doc/dev-guides/_literal_ocf_failed_master_literal_9.html
|
||||
# OCF_FAILED_MASTER,
|
||||
# http://www.linux-ha.org/doc/dev-guides/_literal_ocf_failed_master_literal_9.html
|
||||
|
||||
rc = 9
|
||||
agentlist = self._get_agents_by_type(agent_type)
|
||||
for hostname in self.options.get('test-hostnames'):
|
||||
if self._test_healthy(agentlist, hostname):
|
||||
@ -521,74 +511,130 @@ class NeutronCleaner(object):
|
||||
return rc
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Neutron network node cleaning tool.')
|
||||
parser.add_argument("-c", "--auth-config", dest="authconf", default="/root/openrc",
|
||||
help="Authenticating config FILE", metavar="FILE")
|
||||
parser.add_argument("-t", "--auth-token", dest="auth-token", default=None,
|
||||
help="Authenticating token (instead username/passwd)", metavar="TOKEN")
|
||||
parser.add_argument("-u", "--admin-auth-url", dest="admin-auth-url", default=None,
|
||||
help="Authenticating URL (admin)", metavar="URL")
|
||||
parser.add_argument("--retries", dest="retries", type=int, default=50,
|
||||
help="try NN retries for API call", metavar="NN")
|
||||
parser.add_argument("--sleep", dest="sleep", type=int, default=2,
|
||||
help="sleep seconds between retries", metavar="SEC")
|
||||
parser.add_argument("-a", "--agent", dest="agent", action="append",
|
||||
help="specyfy agents for cleaning", required=True)
|
||||
parser.add_argument("--cleanup-ports", dest="cleanup-ports", action="store_true", default=False,
|
||||
help="cleanup ports for given agents on this node")
|
||||
parser.add_argument("--remove-self", dest="remove-self", action="store_true", default=False,
|
||||
help="remove ourselves from agent list")
|
||||
parser.add_argument("--activeonly", dest="activeonly", action="store_true", default=False,
|
||||
help="cleanup only active ports")
|
||||
parser.add_argument("--reschedule", dest="reschedule", action="store_true", default=False,
|
||||
help="reschedule given agents")
|
||||
parser.add_argument("--remove-dead", dest="remove-dead", action="store_true", default=False,
|
||||
help="remove dead agents while rescheduling")
|
||||
parser.add_argument("--test-alive-for-hostname", dest="test-hostnames", action="append",
|
||||
help="testing agent's healthy for given hostname")
|
||||
parser.add_argument("--external-bridge", dest="external-bridge", default="br-ex",
|
||||
help="external bridge name", metavar="IFACE")
|
||||
parser.add_argument("--integration-bridge", dest="integration-bridge", default="br-int",
|
||||
help="integration bridge name", metavar="IFACE")
|
||||
parser.add_argument("-l", "--log", dest="log", action="store",
|
||||
help="log file or logging.conf location")
|
||||
parser.add_argument("--noop", dest="noop", action="store_true", default=False,
|
||||
help="do not execute, print to log instead")
|
||||
parser.add_argument("--debug", dest="debug", action="store_true", default=False,
|
||||
help="debug")
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Neutron network node cleaning tool.')
|
||||
parser.add_argument(
|
||||
"-c",
|
||||
"--auth-config",
|
||||
dest="authconf",
|
||||
default="/etc/neutron/neutron.conf",
|
||||
help="Read authconfig from service file",
|
||||
metavar="FILE")
|
||||
parser.add_argument(
|
||||
"-t",
|
||||
"--auth-token",
|
||||
dest="auth-token",
|
||||
default=None,
|
||||
help="Authenticating token (instead username/passwd)",
|
||||
metavar="TOKEN")
|
||||
parser.add_argument(
|
||||
"-u",
|
||||
"--admin-auth-url",
|
||||
dest="admin-auth-url",
|
||||
default=None,
|
||||
help="Authenticating URL (admin)",
|
||||
metavar="URL")
|
||||
parser.add_argument(
|
||||
"--retries",
|
||||
dest="retries",
|
||||
type=int,
|
||||
default=50,
|
||||
help="try NN retries for API call",
|
||||
metavar="NN")
|
||||
parser.add_argument(
|
||||
"--sleep",
|
||||
dest="sleep",
|
||||
type=int,
|
||||
default=2,
|
||||
help="sleep seconds between retries",
|
||||
metavar="SEC")
|
||||
parser.add_argument(
|
||||
"-a",
|
||||
"--agent",
|
||||
dest="agent",
|
||||
action="append",
|
||||
help="specyfy agents for cleaning",
|
||||
required=True)
|
||||
parser.add_argument(
|
||||
"--cleanup-ports",
|
||||
dest="cleanup-ports",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="cleanup ports for given agents on this node")
|
||||
parser.add_argument(
|
||||
"--remove-self",
|
||||
dest="remove-self",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="remove ourselves from agent list")
|
||||
parser.add_argument(
|
||||
"--activeonly",
|
||||
dest="activeonly",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="cleanup only active ports")
|
||||
parser.add_argument(
|
||||
"--reschedule",
|
||||
dest="reschedule",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="reschedule given agents")
|
||||
parser.add_argument(
|
||||
"--remove-dead",
|
||||
dest="remove-dead",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="remove dead agents while rescheduling")
|
||||
parser.add_argument(
|
||||
"--test-alive-for-hostname",
|
||||
dest="test-hostnames",
|
||||
action="append",
|
||||
help="testing agent's healthy for given hostname")
|
||||
parser.add_argument(
|
||||
"--external-bridge",
|
||||
dest="external-bridge",
|
||||
default="br-ex",
|
||||
help="external bridge name",
|
||||
metavar="IFACE")
|
||||
parser.add_argument(
|
||||
"--integration-bridge",
|
||||
dest="integration-bridge",
|
||||
default="br-int",
|
||||
help="integration bridge name",
|
||||
metavar="IFACE")
|
||||
parser.add_argument(
|
||||
"-l",
|
||||
"--log",
|
||||
dest="log",
|
||||
action="store",
|
||||
help="log to file instead of STDOUT")
|
||||
parser.add_argument(
|
||||
"--noop",
|
||||
dest="noop",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="do not execute, print to log instead")
|
||||
parser.add_argument(
|
||||
"--debug",
|
||||
dest="debug",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="debug")
|
||||
args = parser.parse_args()
|
||||
# if len(args) != 1:
|
||||
# parser.error("incorrect number of arguments")
|
||||
# parser.print_help() args = parser.parse_args()
|
||||
RETRY_COUNT = args.retries
|
||||
RETRY_DELAY = args.sleep
|
||||
|
||||
# setup logging
|
||||
if args.log:
|
||||
LOG = make_logger(
|
||||
handler=logging.handlers.WatchedFileHandler(args.log))
|
||||
|
||||
#setup logging
|
||||
if args.debug:
|
||||
_log_level = logging.DEBUG
|
||||
else:
|
||||
_log_level = logging.INFO
|
||||
if not args.log:
|
||||
# log config or file not given -- log to console
|
||||
LOG = logging.getLogger(LOG_NAME) # do not move to UP of file
|
||||
_log_handler = logging.StreamHandler(sys.stdout)
|
||||
_log_handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s"))
|
||||
LOG.addHandler(_log_handler)
|
||||
LOG.setLevel(_log_level)
|
||||
elif args.log.split(os.sep)[-1] == 'logging.conf':
|
||||
# setup logging by external file
|
||||
import logging.config
|
||||
logging.config.fileConfig(args.log)
|
||||
LOG = logging.getLogger(LOG_NAME) # do not move to UP of file
|
||||
else:
|
||||
# log to given file
|
||||
LOG = logging.getLogger(LOG_NAME) # do not move to UP of file
|
||||
LOG.addHandler(logging.handlers.WatchedFileHandler(args.log))
|
||||
LOG.setLevel(_log_level)
|
||||
LOG.setLevel(logging.DEBUG)
|
||||
|
||||
LOG.info("Started: {0}".format(' '.join(sys.argv)))
|
||||
cleaner = NeutronCleaner(get_authconfig(args.authconf), options=vars(args), log=LOG)
|
||||
cleaner = NeutronCleaner(options=vars(args), log=LOG)
|
||||
rc = 0
|
||||
if vars(args).get('test-hostnames'):
|
||||
rc = cleaner.test_healthy(args.agent[0])
|
||||
@ -597,5 +643,3 @@ if __name__ == '__main__':
|
||||
cleaner.do(i)
|
||||
LOG.debug("End.")
|
||||
sys.exit(rc)
|
||||
#
|
||||
###
|
||||
|
83
deployment/puppet/cluster/files/test_q_agent_cleanup.py
Normal file
83
deployment/puppet/cluster/files/test_q_agent_cleanup.py
Normal file
@ -0,0 +1,83 @@
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from mock import Mock
|
||||
from mock import mock_open
|
||||
from mock import patch
|
||||
import pytest
|
||||
|
||||
# Yes, q-agent-cleanup has dashes, this is necessary to bypass the import
|
||||
# restriction. It's normally not a module, we only need this for testing.
|
||||
qagent = __import__('q-agent-cleanup')
|
||||
|
||||
|
||||
qagent.RETRY_DELAY = RETRY_DELAY = 0
|
||||
qagent.RETRY_COUNT = RETRY_COUNT = 10
|
||||
|
||||
|
||||
def build_mock(effect):
|
||||
mock = Mock()
|
||||
mock.side_effect = effect
|
||||
# http://stackoverflow.com/questions/22204660/python-mock-wrapsf-problems
|
||||
mock.__name__ = 'foo'
|
||||
return mock
|
||||
|
||||
|
||||
def test_retry_NotEpectedError():
|
||||
mock = build_mock(TypeError)
|
||||
pytest.raises(TypeError, qagent.retry(mock))
|
||||
mock.assert_called_once_with()
|
||||
|
||||
|
||||
def test_retry_RecoverableError():
|
||||
mock = build_mock([Exception('503 Service Unavailable')] * 5 + [True])
|
||||
ret = qagent.retry(mock)()
|
||||
assert ret
|
||||
assert mock.call_count == 6
|
||||
|
||||
|
||||
def test_retry_RecoveryTimedOut():
|
||||
mock = build_mock(Exception('503 Service Unavailable'))
|
||||
pytest.raises(Exception, qagent.retry(mock))
|
||||
assert mock.call_count == RETRY_COUNT
|
||||
|
||||
|
||||
def test_retry_CallSuccess():
|
||||
mock = build_mock([True])
|
||||
qagent.retry(mock)()
|
||||
mock.assert_called_once_with()
|
||||
|
||||
|
||||
def test_get_authdata():
|
||||
cfg = """[keystone_authtoken]
|
||||
admin_tenant_name = tenant
|
||||
admin_user = admin
|
||||
admin_password = password
|
||||
auth_uri = http://127.0.0.1:5000/v2.0/
|
||||
|
||||
"""
|
||||
actual = {
|
||||
'tenant_name': 'tenant',
|
||||
'username': 'admin',
|
||||
'password': 'password',
|
||||
'auth_url': 'http://127.0.0.1:5000/v2.0/',
|
||||
}
|
||||
|
||||
mock = mock_open()
|
||||
mock.return_value.readline = Mock()
|
||||
mock.return_value.readline.side_effect = cfg.split('\n')
|
||||
|
||||
with patch("__builtin__.open", mock):
|
||||
read_data = qagent.get_auth_data('file')
|
||||
assert read_data == actual
|
@ -1,2 +1,3 @@
|
||||
mock>=1.0.1
|
||||
pytest>=2.6.4
|
||||
python-neutronclient
|
||||
|
Loading…
Reference in New Issue
Block a user