diff --git a/openrc.default b/openrc.default index 10be44e..b195cff 100644 --- a/openrc.default +++ b/openrc.default @@ -10,8 +10,8 @@ export ADMIN_NODE_CPU=2 export SLAVE_NODE_MEMORY=5120 # Locations for fuel-qa, MOS and plugins artefacts -export FUELQA_GITREF=stable/8.0 -export ISO_PATH=$HOME/iso/MirantisOpenStack-8.0.iso +export FUELQA_GITREF=stable/mitaka +export ISO_PATH=$HOME/iso/MirantisOpenStack-9.0.iso export LMA_COLLECTOR_PLUGIN_PATH=$HOME/plugins/lma_collector-0.9-0.9.0-1.noarch.rpm export LMA_INFRA_ALERTING_PLUGIN_PATH=$HOME/plugins/lma_infrastructure_alerting-0.9-0.9.0-1.noarch.rpm export ELASTICSEARCH_KIBANA_PLUGIN_PATH=$HOME/plugins/elasticsearch_kibana-0.9-0.9.0-1.noarch.rpm diff --git a/requirements.txt b/requirements.txt index 770c207..28fcaf7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,9 @@ elasticsearch git+git://github.com/openstack/fuel-devops.git@2.9.22 +netaddr oslo.i18n>=3.1.0 # the more recent python-*client (dependencies of fuel-qa) require at least this version of oslo.i18n python-ceilometerclient +py-zabbix PyYAML requests selenium diff --git a/stacklight_tests/run_tests.py b/stacklight_tests/run_tests.py index 4ff179d..ba26ff3 100644 --- a/stacklight_tests/run_tests.py +++ b/stacklight_tests/run_tests.py @@ -71,6 +71,7 @@ def import_tests(): from stacklight_tests.toolchain import test_smoke_bvt # noqa from stacklight_tests.toolchain import test_system # noqa from stacklight_tests.zabbix import test_smoke_bvt # noqa + from stacklight_tests.zabbix import test_system # noqa def run_tests(): diff --git a/stacklight_tests/zabbix/api.py b/stacklight_tests/zabbix/api.py index 66c5c63..ffae421 100644 --- a/stacklight_tests/zabbix/api.py +++ b/stacklight_tests/zabbix/api.py @@ -16,7 +16,10 @@ import requests import urllib import urlparse +from devops.helpers import helpers as devops_helpers +from fuelweb_test import logger from proboscis import asserts +from pyzabbix import ZabbixAPI from stacklight_tests.helpers import checkers from stacklight_tests.helpers import helpers @@ -106,6 +109,10 @@ class ZabbixApi(base_test.PluginApi): def get_zabbix_vip(self): return self.helpers.fuel_web.get_public_vip(self.helpers.cluster_id) + def get_zabbix_mgmt_vip(self): + return self.helpers.fuel_web.client.get_networks( + self.helpers.cluster_id)['vips']['zbx_vip_mgmt']['ipaddr'] + def check_plugin_online(self): controller = self.fuel_web.get_nailgun_cluster_nodes_by_roles( self.helpers.cluster_id, ['controller'])[0] @@ -140,3 +147,105 @@ class ZabbixApi(base_test.PluginApi): return ZabbixWeb( self.get_zabbix_url(), username, password, self.protocol) + + def get_zabbix_api(self): + zabbix_api = ZabbixAPI( + url=self.get_zabbix_url(), + user=self.settings.zabbix_username, + password=self.settings.zabbix_password) + zabbix_api.session.verify = False + return zabbix_api + + def get_node_with_zabbix_vip_fqdn(self): + controller = self.fuel_web.get_nailgun_cluster_nodes_by_roles( + self.helpers.cluster_id, ['controller'])[0] + + with self.fuel_web.get_ssh_for_nailgun_node(controller) as remote: + result = remote.check_call( + "crm status | grep {} | awk '{{print $4}}'".format( + self.helpers.get_vip_resource_name( + self.settings.zabbix_vip))) + return result['stdout'][0].rstrip() + + def get_triggers(self, params=None): + params = params or { + "output": ["triggerid", "description", "priority"], + "filter": {"value": 1}, "sortfield": "priority" + } + return self.get_zabbix_api().do_request('trigger.get', params) + + def wait_for_trigger(self, triggers, params=None, timeout=3 * 60): + def check_triggers(): + for trigger in triggers: + found = False + for line in self.get_triggers(params)['result']: + if line["description"] in trigger["description"]: + found = True + if line["priority"] != trigger["priority"]: + logger.error( + "Trigger '{0}' has wrong priority! Expecteed" + " '{1}' but found '{2}'".format( + line["description"], trigger["priority"], + line["priority"])) + return False + if not found: + logger.error("Failed to find trigger: {0}".format( + trigger["description"])) + return False + return True + devops_helpers.wait( + check_triggers, timeout=timeout, + timeout_msg="Failed to get all expected triggers!") + + def send_extreme_snmptraps(self, remote, extreme_host_ip): + snmp_traps = { + 'ps': "snmptrap -v 1 -c {snmp_community} {zabbix_vip} " + "'.1.3.6.1.4.1.1916' '{extreme_host_ip}' 6 {parameter} '10'" + " .1.3.6.1.4.1.1916 s \"null\" .1.3.6.1.4.1.1916 s \"null\"" + " .1.3.6.1.4.1.1916 s \"2\"", + 'port': "snmptrap -v 1 -c {snmp_community} {zabbix_vip}" + " '.1.3.6.1.6.3.1.1' '{extreme_host_ip}' {parameter} 10" + " '10' .1.3.6.1.6.3.1.1 s \"eth1\"", + 'fan': "snmptrap -v 1 -c {snmp_community} {zabbix_vip}" + " '.1.3.6.1.4.1.1916' '{extreme_host_ip}' 6 {parameter}" + " '10' .1.3.6.1.4.1.1916 s \"null\" .1.3.6.1.4.1.1916 s" + " \"null\" .1.3.6.1.4.1.1916 s \"5\"", + } + remote.check_call(snmp_traps['ps'].format( + snmp_community='public', zabbix_vip=self.get_zabbix_mgmt_vip(), + extreme_host_ip=extreme_host_ip, parameter=10)) + remote.check_call(snmp_traps['ps'].format( + snmp_community='public', zabbix_vip=self.get_zabbix_mgmt_vip(), + extreme_host_ip=extreme_host_ip, parameter=11)) + remote.check_call(snmp_traps['port'].format( + snmp_community='public', zabbix_vip=self.get_zabbix_mgmt_vip(), + extreme_host_ip=extreme_host_ip, parameter=2)) + remote.check_call(snmp_traps['port'].format( + snmp_community='public', zabbix_vip=self.get_zabbix_mgmt_vip(), + extreme_host_ip=extreme_host_ip, parameter=3)) + remote.check_call(snmp_traps['fan'].format( + snmp_community='public', zabbix_vip=self.get_zabbix_mgmt_vip(), + extreme_host_ip=extreme_host_ip, parameter=7)) + remote.check_call(snmp_traps['fan'].format( + snmp_community='public', zabbix_vip=self.get_zabbix_mgmt_vip(), + extreme_host_ip=extreme_host_ip, parameter=8)) + + def send_emc_snmptraps(self, remote, emc_host_ip): + emc_trap = ( + "snmptrap -v 1 -c {snmp_community} {zabbix_vip}" + " '.1.3.6.1.4.1.1981' '{emc_host_ip}' 6 {parameter1} '10'" + " .1.3.6.1.4.1.1981 s \"null\" .1.3.6.1.4.1.1981 s \"null\"" + " .1.3.6.1.4.1.1981 s \"{parameter2}\"" + ) + remote.check_call(emc_trap.format( + snmp_community='public', zabbix_vip=self.get_zabbix_mgmt_vip(), + emc_host_ip=emc_host_ip, parameter1=6, parameter2="a37")) + remote.check_call(emc_trap.format( + snmp_community='public', zabbix_vip=self.get_zabbix_mgmt_vip(), + emc_host_ip=emc_host_ip, parameter1=5, parameter2=966)) + remote.check_call(emc_trap.format( + snmp_community='public', zabbix_vip=self.get_zabbix_mgmt_vip(), + emc_host_ip=emc_host_ip, parameter1=4, parameter2=7220)) + remote.check_call(emc_trap.format( + snmp_community='public', zabbix_vip=self.get_zabbix_mgmt_vip(), + emc_host_ip=emc_host_ip, parameter1=3, parameter2=2004)) diff --git a/stacklight_tests/zabbix/test_system.py b/stacklight_tests/zabbix/test_system.py new file mode 100644 index 0000000..6fa70e7 --- /dev/null +++ b/stacklight_tests/zabbix/test_system.py @@ -0,0 +1,168 @@ +# coding=utf-8 +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from fuelweb_test.helpers.decorators import log_snapshot_after_test +from netaddr import IPNetwork +from proboscis import test + +from stacklight_tests.zabbix import api + + +@test(groups=["plugins"]) +class TestZabbixPluginSystem(api.ZabbixApi): + """Class for system testing the Zabbix plugin.""" + + @test(depends_on_groups=["prepare_slaves_5"], + groups=["deploy_zabbix_ha_offline", "system", "zabbix", + "createmirror"]) + @log_snapshot_after_test + def deploy_zabbix_ha_offline(self): + """Run fuel-createmirror and deploy environment + + Scenario: + 1. Copy Zabbix plugin to the Fuel Master + node and install the plugin. + 2. Run the following command on the master node: + fuel-createmirror + 3. Create an environment with enabled plugin in the + Fuel Web UI and deploy it. + 4. Run OSTF. + + Duration 60m + """ + self.env.revert_snapshot("ready_with_5_slaves") + + self.prepare_plugin() + + self.helpers.fuel_createmirror() + + self.helpers.create_cluster(name=self.__class__.__name__) + + self.activate_plugin() + + self.helpers.deploy_cluster( + { + 'slave-01': ['controller'], + 'slave-02': ['controller'], + 'slave-03': ['controller'], + 'slave-04': ['compute'], + 'slave-05': ['cinder'] + } + ) + + self.check_plugin_online() + + self.helpers.run_ostf() + + @test(depends_on_groups=["prepare_slaves_5"], + groups=["test_dependant_plugins", "system", "zabbix"]) + @log_snapshot_after_test + def test_dependant_plugins(self): + """Check Zabbix dependant plugins + + Scenario: + 1. Upload and install Zabbix plugins. + 2. Configure EMC plugin with a fake Name/IP pair: + MyEMCHost:10.109.0.100 + 3. Configure Extreme Networks plugin with a fake Name/IP pair: + MyXNHost:10.109.0.101 + 4. Add 3 nodes with controller role. + 5. Add 1 node with compute role. + 6. Add 1 node with cinder role. + 7. Deploy cluster. + 8. Check plugin health. + 9. Run OSTF. + 10. Send and verify that traps have been received by Zabbix. + + Duration 60m + """ + self.env.revert_snapshot("ready_with_5_slaves") + + self.prepare_plugin(dependat_plugins=True) + + self.helpers.create_cluster(name=self.__class__.__name__) + + networks = self.helpers.fuel_web.client.get_networks( + self.helpers.cluster_id)['networks'] + public_cidr = filter( + lambda x: x['name'] == "public", networks)[0]['cidr'] + extreme_host_ip = str(IPNetwork(public_cidr)[100]) + emc_host_ip = str(IPNetwork(public_cidr)[101]) + + self.activate_plugin() + self.activate_dependant_plugin( + self.settings.dependant_plugins["ZABBIX_SNMPTRAPD"]) + self.activate_dependant_plugin( + self.settings.dependant_plugins[ + "ZABBIX_MONITORING_EXTREME_NETWORKS"], + options={'metadata/enabled': True, + 'hosts/value': 'MyXNHost:{}'.format(extreme_host_ip)}) + self.activate_dependant_plugin( + self.settings.dependant_plugins["ZABBIX_MONITORING_EMC"], + options={'metadata/enabled': True, + 'hosts/value': 'MyEMCHost:{}'.format(emc_host_ip)}) + + self.helpers.deploy_cluster( + { + 'slave-01': ['controller'], + 'slave-02': ['controller'], + 'slave-03': ['controller'], + 'slave-04': ['compute'], + 'slave-05': ['cinder'] + }, timeout=10800 + ) + + self.check_plugin_online() + + self.helpers.run_ostf() + + zabbix_api = self.get_zabbix_api() + myxnhost_id = zabbix_api.do_request( + 'host.get', {"output": ["hostid"], "filter": {"host": [ + "MyXNHost"]}})['result'][0]['hostid'] + myemchost_id = zabbix_api.do_request( + 'host.get', {"output": ["hostid"], "filter": {"host": [ + "MyEMCHost"]}})['result'][0]['hostid'] + + controller = self.fuel_web.get_nailgun_cluster_nodes_by_roles( + self.helpers.cluster_id, ['controller'])[0] + + with self.fuel_web.get_ssh_for_nailgun_node(controller) as remote: + remote.check_call("apt-get install snmp -y") + self.send_extreme_snmptraps(remote, extreme_host_ip) + self.send_emc_snmptraps(remote, emc_host_ip) + + triggers = [ + {'priority': '4', 'description': 'Power Supply Failed:' + ' {ITEM.VALUE1}'}, + {'priority': '1', 'description': 'Power Supply OK: {ITEM.VALUE1}'}, + {'priority': '4', 'description': 'Fan Failed: {ITEM.VALUE1}'}, + {'priority': '1', 'description': 'Fan OK: {ITEM.VALUE1}'}, + {'priority': '4', 'description': 'Link Down: {ITEM.VALUE1}'}, + {'priority': '1', 'description': 'Link Up: {ITEM.LASTVALUE1}'}, + {'priority': '1', 'description': 'SNMPtrigger Information:' + ' {ITEM.VALUE1}'}, + {'priority': '2', 'description': 'SNMPtrigger Warning:' + ' {ITEM.VALUE1}'}, + {'priority': '3', 'description': 'SNMPtrigger Error:' + ' {ITEM.VALUE1}'}, + {'priority': '4', 'description': 'SNMPtrigger Critical:' + ' {ITEM.VALUE1}'} + ] + + self.wait_for_trigger(triggers, {"output": [ + "triggerid", "description", "priority"], "filter": {"value": 1}, + "hostids": [str(myxnhost_id), str(myemchost_id)] + })