openstack/fuel-qa
Code Issues Proposed changes

Merge "Add tests for zabbix subplugins"

Browse Source
This commit is contained in:
Jenkins
2015-08-04 13:55:34 +00:00
committed by Gerrit Code Review
parent 6e069e355b fde58d51c3
commit ff2260f486
2 changed files with 444 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fuelweb_test/settings.py
View File

@@ -341,6 +341,10 @@ EXAMPLE_PLUGIN_PATH = os.environ.get('EXAMPLE_PLUGIN_PATH')
EXAMPLE_PLUGIN_V3_PATH = os.environ.get('EXAMPLE_PLUGIN_V3_PATH')
LBAAS_PLUGIN_PATH = os.environ.get('LBAAS_PLUGIN_PATH')
ZABBIX_PLUGIN_PATH = os.environ.get('ZABBIX_PLUGIN_PATH')
ZABBIX_SNMP_PLUGIN_PATH = os.environ.get('ZABBIX_SNMP_PLUGIN_PATH')
ZABBIX_SNMP_EMC_PLUGIN_PATH = os.environ.get('ZABBIX_SNMP_EMC_PLUGIN_PATH')
ZABBIX_SNMP_EXTREME_PLUGIN_PATH = os.environ.get(
'ZABBIX_SNMP_EXTREME_PLUGIN_PATH')
LMA_COLLECTOR_PLUGIN_PATH = os.environ.get('LMA_COLLECTOR_PLUGIN_PATH')
ELASTICSEARCH_KIBANA_PLUGIN_PATH = os.environ.get(
'ELASTICSEARCH_KIBANA_PLUGIN_PATH')

489
fuelweb_test/tests/plugins/plugin_zabbix/test_plugin_zabbix.py
View File

@@ -12,25 +12,134 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
import re
import urllib
import urlparse
import bs4
from devops.helpers.helpers import wait
from proboscis.asserts import assert_equal
from proboscis.asserts import assert_not_equal
from proboscis.asserts import assert_true
from proboscis import test
import requests
from fuelweb_test import logger
from fuelweb_test import settings as conf
from fuelweb_test.helpers import checkers
from fuelweb_test.helpers.decorators import log_snapshot_after_test
from fuelweb_test import settings as conf
from fuelweb_test.tests.base_test_case import SetupEnvironment
from fuelweb_test.tests.base_test_case import TestBasic
@test(groups=["plugins"])
class ZabbixWeb(object):
def __init__(self, public_vip, username, password):
self.session = requests.Session()
self.base_url = "http://{0}/zabbix/".format(public_vip)
self.username = username
self.password = password
def login(self):
login_params = urllib.urlencode({'request': '',
'name': self.username,
'password': self.password,
'autologin': 1,
'enter': 'Sign in'})
url = urlparse.urljoin(self.base_url, '?{0}'.format(login_params))
response = self.session.post(url)
assert_equal(response.status_code, 200,
"Login to Zabbix failed: {0}".format(response.content))
def get_trigger_statuses(self):
url = urlparse.urljoin(self.base_url, 'tr_status.php')
response = self.session.get(url)
assert_equal(response.status_code, 200,
"Getting Zabbix trigger statuses failed: {0}"
.format(response.content))
return response.content
def get_screens(self):
url = urlparse.urljoin(self.base_url, 'screens.php')
response = self.session.get(url)
assert_equal(response.status_code, 200,
"Getting Zabbix screens failed: {0}"
.format(response.content))
return response.content
@test(groups=["plugins", "zabbix_plugins"])
class ZabbixPlugin(TestBasic):
"""ZabbixPlugin.""" # TODO documentation
"""ZabbixPlugin."""
def setup_zabbix_plugin(self,
cluster_id,
zabbix_username='admin',
zabbix_password='zabbix'):
plugin_name = 'zabbix_monitoring'
assert_true(
self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
"Plugin couldn't be enabled. Check plugin version. Test aborted")
plugin_options = {'metadata/enabled': True,
'username/value': zabbix_username,
'password/value': zabbix_password}
self.fuel_web.update_plugin_data(
cluster_id, plugin_name, plugin_options)
def setup_snmp_plugin(self,
cluster_id,
snmp_community='public'):
plugin_name = 'zabbix_snmptrapd'
assert_true(
self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
"Plugin couldn't be enabled. Check plugin version. Test aborted")
plugin_options = {'metadata/enabled': True,
'community/value': snmp_community}
self.fuel_web.update_plugin_data(
cluster_id, plugin_name, plugin_options)
def setup_snmp_emc_plugin(self, cluster_id):
plugin_name = 'zabbix_monitoring_emc'
assert_true(
self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
"Plugin couldn't be enabled. Check plugin version. Test aborted")
plugin_options = {'metadata/enabled': True,
'hosts/value': 'emc:10.109.2.2'}
self.fuel_web.update_plugin_data(
cluster_id, plugin_name, plugin_options)
def setup_snmp_extreme_plugin(self, cluster_id):
plugin_name = 'zabbix_monitoring_extreme_networks'
assert_true(
self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
"Plugin couldn't be enabled. Check plugin version. Test aborted")
plugin_options = {'metadata/enabled': True,
'hosts/value': 'extreme:10.109.2.2'}
self.fuel_web.update_plugin_data(
cluster_id, plugin_name, plugin_options)
def check_event_message(self, zabbix_web, zabbix_hostgroup, message):
statuses_html = bs4.BeautifulSoup(zabbix_web.get_trigger_statuses())
status_lines = statuses_html.find_all('tr', {'class': 'even_row'})
for status_line in status_lines:
host_span = status_line.find('span', {'class': 'link_menu'})
if not host_span or host_span.get_text() != zabbix_hostgroup:
continue
host_span = (status_line.find('span', {'class': 'pointer'}).
find('span', {'class': 'link_menu'}))
if host_span and message in host_span.get_text():
return True
return False
@test(depends_on=[SetupEnvironment.prepare_slaves_5],
groups=["deploy_zabbix_ha"])
@@ -47,26 +156,109 @@ class ZabbixPlugin(TestBasic):
6. Add 1 node with cinder role
7. Deploy the cluster
8. Run network verification
9. check plugin health
10. Run OSTF
9. Run OSTF
10. Check zabbix service in pacemaker
11. Check login to zabbix dashboard
Duration 70m
Snapshot deploy_zabbix_ha
"""
self.env.revert_snapshot("ready_with_5_slaves")
# copy plugin to the master node
checkers.upload_tarball(
self.env.d_env.get_admin_remote(), conf.ZABBIX_PLUGIN_PATH, "/var")
# install plugin
checkers.install_plugin_check_code(
self.env.d_env.get_admin_remote(),
plugin=os.path.basename(conf.ZABBIX_PLUGIN_PATH))
settings = None
if conf.NEUTRON_ENABLE:
settings = {
"net_provider": "neutron",
"net_segment_type": conf.NEUTRON_SEGMENT_TYPE
}
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=conf.DEPLOYMENT_MODE,
settings=settings
)
zabbix_username = 'admin'
zabbix_password = 'zabbix'
self.setup_zabbix_plugin(cluster_id, zabbix_username, zabbix_password)
self.fuel_web.update_nodes(
cluster_id,
{
"slave-01": ["controller"],
"slave-02": ["controller"],
"slave-03": ["controller"],
"slave-04": ["compute"],
"slave-05": ["cinder"]
}
)
self.fuel_web.deploy_cluster_wait(cluster_id)
self.fuel_web.verify_network(cluster_id)
self.fuel_web.run_ostf(cluster_id=cluster_id)
cmd = "crm resource status p_zabbix-server"
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
response = remote.execute(cmd)["stdout"][0]
assert_true("p_zabbix-server is running" in response,
"p_zabbix-server resource wasn't found in pacemaker:\n{0}"
.format(response))
public_vip = self.fuel_web.get_public_vip(cluster_id)
zabbix_web = ZabbixWeb(public_vip, zabbix_username, zabbix_password)
zabbix_web.login()
screens_html = bs4.BeautifulSoup(zabbix_web.get_screens())
screens_links = screens_html.find_all('a')
assert_true(any('charts.php?graphid=' in link.get('href')
for link in screens_links),
"Zabbix screen page does not contain graphs:\n{0}".
format(screens_links))
self.env.make_snapshot("deploy_zabbix_ha")
@test(depends_on=[SetupEnvironment.prepare_slaves_5],
groups=["deploy_zabbix_snmptrap_ha"])
@log_snapshot_after_test
def deploy_zabbix_snmptrap_ha(self):
"""Deploy cluster in ha mode with zabbix snmptrap plugin
Scenario:
1. Upload plugin to the master node
2. Install plugins
3. Create cluster
4. Add 3 nodes with controller role
5. Add 1 node with compute role
6. Add 1 node with cinder role
7. Deploy the cluster
8. Run network verification
9. Run OSTF
10. Check zabbix service in pacemaker
11. Check login to zabbix dashboard
12. Check SNMP services on controllers
13. Check test SNMP trap
Duration 70m
Snapshot deploy_zabbix_snmptrap_ha
"""
self.env.revert_snapshot("ready_with_5_slaves")
for plugin in [conf.ZABBIX_PLUGIN_PATH,
conf.ZABBIX_SNMP_PLUGIN_PATH]:
checkers.upload_tarball(
self.env.d_env.get_admin_remote(), plugin, "/var")
checkers.install_plugin_check_code(
self.env.d_env.get_admin_remote(),
plugin=os.path.basename(plugin))
settings = None
if conf.NEUTRON_ENABLE:
@@ -81,14 +273,12 @@ class ZabbixPlugin(TestBasic):
settings=settings
)
plugin_name = 'zabbix_monitoring'
msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
assert_true(
self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
msg)
logger.debug('we have zabbix_monitoring element')
options = {'metadata/enabled': True}
self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)
zabbix_username = 'admin'
zabbix_password = 'zabbix'
snmp_community = 'public'
self.setup_zabbix_plugin(cluster_id)
self.setup_snmp_plugin(cluster_id, snmp_community)
self.fuel_web.update_nodes(
cluster_id,
@@ -101,37 +291,238 @@ class ZabbixPlugin(TestBasic):
}
)
self.fuel_web.deploy_cluster_wait(cluster_id)
logger.debug("Start checking service on node slave-01")
cmd = "crm resource status p_zabbix-server"
remote = self.fuel_web.get_ssh_for_node("slave-01")
response = remote.execute(cmd)["stdout"][0]
assert_true("p_zabbix-server is running" in response,
"Response: {0}".format(response))
self.fuel_web.verify_network(cluster_id)
self.fuel_web.run_ostf(cluster_id=cluster_id)
logger.debug("Start checking login to zabbix dashboard")
zabbix_ip = self.fuel_web.get_public_vip(cluster_id)
zabbix_url = "http://{0}/zabbix/".format(zabbix_ip)
req = "request=&name=admin&password=zabbix&autologin=1&enter=Sign+in"
session = requests.Session()
res = session.post("{0}?{1}".format(zabbix_url, req))
assert_equal(res.status_code, 200, "HTTP {0}".format(res.status_code))
cmd = "crm resource status p_zabbix-server"
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
response = remote.execute(cmd)["stdout"][0]
assert_true("p_zabbix-server is running" in response,
"p_zabbix-server resource wasn't found in pacemaker:\n{0}"
.format(response))
logger.debug("Zabbix dashboard {0}".format(zabbix_url))
res = session.get("{0}/screens.php".format(zabbix_url))
assert_equal(res.status_code, 200, "HTTP {0}".format(res.status_code))
soup = bs4.BeautifulSoup(res.content)
output = soup.find_all('a')
public_vip = self.fuel_web.get_public_vip(cluster_id)
screen_graphids = []
for s in output:
m = re.match(r".*graphid=(?P<id>\d+).*", "{0}".format(s))
if m:
screen_graphids.append(m.group("id"))
assert_true(screen_graphids, "No Graphs at the screen")
zabbix_web = ZabbixWeb(public_vip, zabbix_username, zabbix_password)
zabbix_web.login()
self.env.make_snapshot("deploy_zabbix_ha")
screens_html = bs4.BeautifulSoup(zabbix_web.get_screens())
screens_links = screens_html.find_all('a')
assert_true(any('charts.php?graphid=' in link.get('href')
for link in screens_links),
"Zabbix screen page does not contain graphs:\n{0}".
format(screens_links))
for node_name in ['slave-01', 'slave-02', 'slave-03']:
with self.fuel_web.get_ssh_for_node(node_name) as remote:
cmd = 'pgrep {0}'
response = \
''.join(remote.execute(cmd.format('snmptrapd'))["stdout"])
assert_not_equal(response.strip(), "OK",
"Service {0} not started".format('snmptrapd'))
response = \
''.join(remote.execute(cmd.format('snmptt'))["stdout"])
assert_not_equal(response.strip(), "OK",
"Service {0} not started".format('snmptt'))
management_vip = self.fuel_web.get_mgmt_vip(cluster_id)
snmp_heartbeat_command = \
("snmptrap -v 2c -c {0} {1} '' .1.3.6.1.4.1.8072.2.3.0.1"
.format(snmp_community, management_vip))
self.env.d_env.get_admin_remote().execute(snmp_heartbeat_command)
mgmt_vip_devops_node = self.fuel_web.get_pacemaker_resource_location(
'slave-01', 'vip__management')[0]
mgmt_vip_nailgun_node = self.fuel_web.get_nailgun_node_by_devops_node(
mgmt_vip_devops_node)
with self.env.d_env.get_ssh_to_remote(
mgmt_vip_nailgun_node['ip']) as remote:
cmd = ('grep netSnmpExampleHeartbeatNotification '
'/var/log/zabbix/zabbix_server.log | '
'grep "Status Events"')
check_snmp_event = lambda: \
"".join(remote.execute(cmd).get('output', [])).strip() != ""
wait(check_snmp_event)
self.env.make_snapshot("deploy_zabbix_snmptrap_ha")
@test(depends_on=[SetupEnvironment.prepare_slaves_5],
groups=["deploy_zabbix_snmp_emc_ha"])
@log_snapshot_after_test
def deploy_zabbix_snmp_emc_ha(self):
"""Deploy cluster in ha mode with zabbix emc plugin
Scenario:
1. Upload plugin to the master node
2. Install plugins: zabbix, zabbix snmp and zabbix emc
3. Create cluster
4. Add 3 nodes with controller role
5. Add 1 node with compute role
6. Add 1 node with cinder role
7. Deploy the cluster
8. Run network verification
9. Run OSTF
10. Check EMC trigger with test SNMP message
Duration 70m
Snapshot deploy_zabbix_snmp_emc_ha
"""
self.env.revert_snapshot("ready_with_5_slaves")
for plugin in [conf.ZABBIX_PLUGIN_PATH,
conf.ZABBIX_SNMP_PLUGIN_PATH,
conf.ZABBIX_SNMP_EMC_PLUGIN_PATH]:
checkers.upload_tarball(
self.env.d_env.get_admin_remote(), plugin, "/var")
checkers.install_plugin_check_code(
self.env.d_env.get_admin_remote(),
plugin=os.path.basename(plugin))
settings = None
if conf.NEUTRON_ENABLE:
settings = {
"net_provider": "neutron",
"net_segment_type": conf.NEUTRON_SEGMENT_TYPE
}
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=conf.DEPLOYMENT_MODE,
settings=settings
)
zabbix_username = 'admin'
zabbix_password = 'zabbix'
snmp_community = 'public'
self.setup_zabbix_plugin(cluster_id, zabbix_username, zabbix_password)
self.setup_snmp_plugin(cluster_id, snmp_community)
self.setup_snmp_emc_plugin(cluster_id)
self.fuel_web.update_nodes(
cluster_id,
{
"slave-01": ["controller"],
"slave-02": ["controller"],
"slave-03": ["controller"],
"slave-04": ["compute"],
"slave-05": ["cinder"]
}
)
self.fuel_web.deploy_cluster_wait(cluster_id)
self.fuel_web.verify_network(cluster_id)
self.fuel_web.run_ostf(cluster_id=cluster_id)
management_vip = self.fuel_web.get_mgmt_vip(cluster_id)
snmp_emc_critical_command = \
("snmptrap -v 1 -c {snmp_community} {management_vip} "
"'.1.3.6.1.4.1.1981' {management_vip} 6 6 '10' .1.3.6.1.4.1.1981 "
"s 'null' .1.3.6.1.4.1.1981 s 'null' .1.3.6.1.4.1.1981 s 'a37'"
.format(snmp_community=snmp_community,
management_vip=management_vip))
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
remote.execute("apt-get install snmp -y")
remote.execute(snmp_emc_critical_command)
public_vip = self.fuel_web.get_public_vip(cluster_id)
zabbix_web = ZabbixWeb(public_vip, zabbix_username, zabbix_password)
zabbix_web.login()
wait(lambda: self.check_event_message(
zabbix_web, 'emc', 'SNMPtrigger Critical'))
self.env.make_snapshot("deploy_zabbix_snmp_emc_ha")
@test(depends_on=[SetupEnvironment.prepare_slaves_5],
groups=["deploy_zabbix_snmp_extreme_ha"])
@log_snapshot_after_test
def deploy_zabbix_snmp_extreme_ha(self):
"""Deploy cluster in ha mode with zabbix snmptrap plugin
Scenario:
1. Upload plugin to the master node
2. Install plugins
3. Create cluster
4. Add 3 nodes with controller role
5. Add 1 node with compute role
6. Add 1 node with cinder role
7. Deploy the cluster
8. Run network verification
9. Run OSTF
10. Check Extreme Swich trigger with test SNMP message
Duration 70m
Snapshot deploy_zabbix_snmp_extreme_ha
"""
self.env.revert_snapshot("ready_with_5_slaves")
for plugin in [conf.ZABBIX_PLUGIN_PATH,
conf.ZABBIX_SNMP_PLUGIN_PATH,
conf.ZABBIX_SNMP_EXTREME_PLUGIN_PATH]:
checkers.upload_tarball(
self.env.d_env.get_admin_remote(), plugin, "/var")
checkers.install_plugin_check_code(
self.env.d_env.get_admin_remote(),
plugin=os.path.basename(plugin))
settings = None
if conf.NEUTRON_ENABLE:
settings = {
"net_provider": "neutron",
"net_segment_type": conf.NEUTRON_SEGMENT_TYPE
}
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=conf.DEPLOYMENT_MODE,
settings=settings
)
zabbix_username = 'admin'
zabbix_password = 'zabbix'
snmp_community = 'public'
self.setup_zabbix_plugin(cluster_id, zabbix_username, zabbix_password)
self.setup_snmp_plugin(cluster_id, snmp_community)
self.setup_snmp_extreme_plugin(cluster_id)
self.fuel_web.update_nodes(
cluster_id,
{
"slave-01": ["controller"],
"slave-02": ["controller"],
"slave-03": ["controller"],
"slave-04": ["compute"],
"slave-05": ["cinder"]
}
)
self.fuel_web.deploy_cluster_wait(cluster_id)
self.fuel_web.verify_network(cluster_id)
self.fuel_web.run_ostf(cluster_id=cluster_id)
management_vip = self.fuel_web.get_mgmt_vip(cluster_id)
snmp_extreme_critical_command = \
("snmptrap -v 1 -c {snmp_community} {management_vip} "
"'.1.3.6.1.4.1.1916' {management_vip} 6 10 '10' .1.3.6.1.4.1.1916"
" s 'null' .1.3.6.1.4.1.1916 s 'null' .1.3.6.1.4.1.1916 s '2'"
.format(snmp_community=snmp_community,
management_vip=management_vip))
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
remote.execute("apt-get install snmp -y")
remote.execute(snmp_extreme_critical_command)
public_vip = self.fuel_web.get_public_vip(cluster_id)
zabbix_web = ZabbixWeb(public_vip, zabbix_username, zabbix_password)
zabbix_web.login()
wait(lambda: self.check_event_message(
zabbix_web, 'extreme', 'Power Supply Failed'))
self.env.make_snapshot("deploy_zabbix_snmp_extreme_ha")