fuel-qa/fuelweb_test/tests/plugins/plugin_zabbix/test_plugin_zabbix.py

#    Copyright 2015 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
import os

import bs4
from devops.helpers.helpers import wait
from proboscis.asserts import assert_equal
from proboscis.asserts import assert_not_equal
from proboscis.asserts import assert_true
from proboscis import test
import requests
# pylint: disable=import-error
from six.moves import urllib
# pylint: enable=import-error

from fuelweb_test.helpers.checkers import check_plugin_path_env
from fuelweb_test.helpers import utils
from fuelweb_test.helpers.decorators import log_snapshot_after_test
from fuelweb_test import settings
from fuelweb_test.tests.base_test_case import SetupEnvironment
from fuelweb_test.tests.base_test_case import TestBasic


class ZabbixWeb(object):
    def __init__(self, public_vip, username, password, verify=False):
        self.session = requests.Session()
        self.base_url = "https://{0}/zabbix/".format(public_vip)
        self.username = username
        self.password = password
        self.verify = verify

    def login(self):
        login_params = urllib.parse.urlencode(
            {'request': '',
             'name': self.username,
             'password': self.password,
             'autologin': 1,
             'enter': 'Sign in'})
        url = urllib.parse.urljoin(self.base_url, '?{0}'.format(login_params))
        response = self.session.post(url, verify=self.verify)

        assert_equal(response.status_code, 200,
                     "Login to Zabbix failed: {0}".format(response.content))

    def get_trigger_statuses(self):
        url = urllib.parse.urljoin(self.base_url, 'tr_status.php')
        response = self.session.get(url, verify=self.verify)

        assert_equal(response.status_code, 200,
                     "Getting Zabbix trigger statuses failed: {0}"
                     .format(response.content))

        return response.content

    def get_screens(self):
        url = urllib.parse.urljoin(self.base_url, 'screens.php')
        response = self.session.get(url, verify=self.verify)

        assert_equal(response.status_code, 200,
                     "Getting Zabbix screens failed: {0}"
                     .format(response.content))

        return response.content


@test(groups=["plugins", "zabbix_plugins"])
class ZabbixPlugin(TestBasic):
    """ZabbixPlugin."""
    def __init__(self):
        super(ZabbixPlugin, self).__init__()
        check_plugin_path_env(
            var_name='ZABBIX_PLUGIN_PATH',
            plugin_path=settings.ZABBIX_PLUGIN_PATH
        )

    def setup_zabbix_plugin(self,
                            cluster_id,
                            zabbix_username='admin',
                            zabbix_password='zabbix'):
        plugin_name = 'zabbix_monitoring'

        assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
            "Plugin couldn't be enabled. Check plugin version. Test aborted")
        plugin_options = {'metadata/enabled': True,
                          'username/value': zabbix_username,
                          'password/value': zabbix_password}
        self.fuel_web.update_plugin_data(
            cluster_id, plugin_name, plugin_options)

    def setup_snmp_plugin(self,
                          cluster_id,
                          snmp_community='public'):
        plugin_name = 'zabbix_snmptrapd'

        assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
            "Plugin couldn't be enabled. Check plugin version. Test aborted")
        plugin_options = {'metadata/enabled': True,
                          'community/value': snmp_community}
        self.fuel_web.update_plugin_data(
            cluster_id, plugin_name, plugin_options)

    def setup_snmp_emc_plugin(self, cluster_id):
        plugin_name = 'zabbix_monitoring_emc'

        assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
            "Plugin couldn't be enabled. Check plugin version. Test aborted")

        plugin_options = {'metadata/enabled': True,
                          'hosts/value': 'emc:10.109.2.2'}
        self.fuel_web.update_plugin_data(
            cluster_id, plugin_name, plugin_options)

    def setup_snmp_extreme_plugin(self, cluster_id):
        plugin_name = 'zabbix_monitoring_extreme_networks'

        assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
            "Plugin couldn't be enabled. Check plugin version. Test aborted")

        plugin_options = {'metadata/enabled': True,
                          'hosts/value': 'extreme:10.109.2.2'}
        self.fuel_web.update_plugin_data(
            cluster_id, plugin_name, plugin_options)

    @staticmethod
    def check_event_message(zabbix_web, zabbix_hostgroup, message):
        statuses_html = bs4.BeautifulSoup(zabbix_web.get_trigger_statuses())
        status_lines = statuses_html.find_all('tr', {'class': 'even_row'})

        for status_line in status_lines:
            host_span = status_line.find('span', {'class': 'link_menu'})
            if not host_span or host_span.get_text() != zabbix_hostgroup:
                continue

            host_span = (status_line.find('span', {'class': 'pointer'}).
                         find('span', {'class': 'link_menu'}))
            if host_span and message in host_span.get_text():
                return True
        return False

    @test(depends_on=[SetupEnvironment.prepare_slaves_5],
          groups=["deploy_zabbix_ha"])
    @log_snapshot_after_test
    def deploy_zabbix_ha(self):
        """Deploy cluster in ha mode with zabbix plugin

        Scenario:
            1. Upload plugin to the master node
            2. Install plugin
            3. Create cluster
            4. Add 3 nodes with controller role
            5. Add 1 node with compute role
            6. Add 1 node with cinder role
            7. Deploy the cluster
            8. Run network verification
            9. Run OSTF
            10. Check zabbix service in pacemaker
            11. Check login to zabbix dashboard

        Duration 70m
        Snapshot deploy_zabbix_ha

        """
        self.env.revert_snapshot("ready_with_5_slaves")

        utils.upload_tarball(
            ip=self.ssh_manager.admin_ip,
            tar_path=settings.ZABBIX_PLUGIN_PATH,
            tar_target="/var")
        utils.install_plugin_check_code(
            ip=self.ssh_manager.admin_ip,
            plugin=os.path.basename(settings.ZABBIX_PLUGIN_PATH))

        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=settings.DEPLOYMENT_MODE,
        )

        zabbix_username = 'admin'
        zabbix_password = 'zabbix'
        self.setup_zabbix_plugin(cluster_id, zabbix_username, zabbix_password)

        self.fuel_web.update_nodes(
            cluster_id,
            {
                "slave-01": ["controller"],
                "slave-02": ["controller"],
                "slave-03": ["controller"],
                "slave-04": ["compute"],
                "slave-05": ["cinder"]
            }
        )

        self.fuel_web.deploy_cluster_wait(cluster_id)
        self.fuel_web.verify_network(cluster_id)
        self.fuel_web.run_ostf(cluster_id=cluster_id)
        self.check_zabbix_configuration(cluster_id, zabbix_username,
                                        zabbix_password)

        self.env.make_snapshot("deploy_zabbix_ha")

    def check_zabbix_configuration(self, cluster_id, user, password):
        cmd = "crm resource status p_zabbix-server"
        with self.fuel_web.get_ssh_for_node("slave-01") as remote:
            response = remote.execute(cmd)["stdout"][0]
        assert_true("p_zabbix-server is running" in response,
                    "p_zabbix-server resource wasn't found in pacemaker:\n{0}"
                    .format(response))

        public_vip = self.fuel_web.get_public_vip(cluster_id)

        zabbix_web = ZabbixWeb(public_vip, user, password)
        zabbix_web.login()

        screens_html = bs4.BeautifulSoup(zabbix_web.get_screens())
        screens_links = screens_html.find_all('a')
        assert_true(any('charts.php?graphid=' in link.get('href')
                        for link in screens_links),
                    "Zabbix screen page does not contain graphs:\n{0}".
                    format(screens_links))

    @test(depends_on=[SetupEnvironment.prepare_slaves_5],
          groups=["deploy_zabbix_snmptrap_ha"])
    @log_snapshot_after_test
    def deploy_zabbix_snmptrap_ha(self):
        """Deploy cluster in ha mode with zabbix snmptrap plugin

        Scenario:
            1. Upload plugin to the master node
            2. Install plugins
            3. Create cluster
            4. Add 3 nodes with controller role
            5. Add 1 node with compute role
            6. Add 1 node with cinder role
            7. Deploy the cluster
            8. Run network verification
            9. Run OSTF
            10. Check zabbix service in pacemaker
            11. Check login to zabbix dashboard
            12. Check SNMP services on controllers
            13. Check test SNMP trap

        Duration 70m
        Snapshot deploy_zabbix_snmptrap_ha

        """
        check_plugin_path_env(
            var_name='ZABBIX_SNMP_PLUGIN_PATH',
            plugin_path=settings.ZABBIX_SNMP_PLUGIN_PATH
        )

        self.env.revert_snapshot("ready_with_5_slaves")

        for plugin in [settings.ZABBIX_PLUGIN_PATH,
                       settings.ZABBIX_SNMP_PLUGIN_PATH]:
            utils.upload_tarball(
                ip=self.ssh_manager.admin_ip,
                tar_path=plugin,
                tar_target="/var")
            utils.install_plugin_check_code(
                ip=self.ssh_manager.admin_ip,
                plugin=os.path.basename(plugin))

        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=settings.DEPLOYMENT_MODE,
        )

        zabbix_username = 'admin'
        zabbix_password = 'zabbix'
        snmp_community = 'public'

        self.setup_zabbix_plugin(cluster_id)
        self.setup_snmp_plugin(cluster_id, snmp_community)

        self.fuel_web.update_nodes(
            cluster_id,
            {
                "slave-01": ["controller"],
                "slave-02": ["controller"],
                "slave-03": ["controller"],
                "slave-04": ["compute"],
                "slave-05": ["cinder"]
            }
        )
        self.fuel_web.deploy_cluster_wait(cluster_id)
        self.fuel_web.verify_network(cluster_id)
        self.fuel_web.run_ostf(cluster_id=cluster_id)
        self.check_zabbix_configuration(cluster_id, zabbix_username,
                                        zabbix_password)

        for node_name in ['slave-01', 'slave-02', 'slave-03']:
            with self.fuel_web.get_ssh_for_node(node_name) as remote:
                cmd = 'pgrep {0}'
                response = \
                    ''.join(remote.execute(cmd.format('snmptrapd'))["stdout"])
                assert_not_equal(response.strip(), "OK",
                                 "Service {0} not started".format('snmptrapd'))
                response = \
                    ''.join(remote.execute(cmd.format('snmptt'))["stdout"])
                assert_not_equal(response.strip(), "OK",
                                 "Service {0} not started".format('snmptt'))

        management_vip = self.fuel_web.get_mgmt_vip(cluster_id)
        snmp_heartbeat_command = \
            ("snmptrap -v 2c -c {0} {1} '' .1.3.6.1.4.1.8072.2.3.0.1"
             .format(snmp_community, management_vip))

        with self.fuel_web.get_ssh_for_node("slave-01") as remote:
            remote.execute("apt-get install snmp -y")
            remote.execute(snmp_heartbeat_command)

        mgmt_vip_devops_node = self.fuel_web.get_pacemaker_resource_location(
            'slave-01', 'vip__management')[0]
        mgmt_vip_nailgun_node = self.fuel_web.get_nailgun_node_by_devops_node(
            mgmt_vip_devops_node)

        with self.env.d_env.get_ssh_to_remote(
                mgmt_vip_nailgun_node['ip']) as remote:
            cmd = ('grep netSnmpExampleHeartbeatNotification '
                   '/var/log/zabbix/zabbix_server.log | '
                   'grep "Status Events"')

            wait(lambda: remote.execute(cmd)['exit_code'] == 0)

        self.env.make_snapshot("deploy_zabbix_snmptrap_ha")

    @test(depends_on=[SetupEnvironment.prepare_slaves_5],
          groups=["deploy_zabbix_snmp_emc_ha"])
    @log_snapshot_after_test
    def deploy_zabbix_snmp_emc_ha(self):
        """Deploy cluster in ha mode with zabbix emc plugin

        Scenario:
            1. Upload plugin to the master node
            2. Install plugins: zabbix, zabbix snmp and zabbix emc
            3. Create cluster
            4. Add 3 nodes with controller role
            5. Add 1 node with compute role
            6. Add 1 node with cinder role
            7. Deploy the cluster
            8. Run network verification
            9. Run OSTF
            10. Check EMC trigger with test SNMP message

        Duration 70m
        Snapshot deploy_zabbix_snmp_emc_ha

        """
        check_plugin_path_env(
            var_name='ZABBIX_SNMP_PLUGIN_PATH',
            plugin_path=settings.ZABBIX_SNMP_PLUGIN_PATH
        )
        check_plugin_path_env(
            var_name='ZABBIX_SNMP_EMC_PLUGIN_PATH',
            plugin_path=settings.ZABBIX_SNMP_EMC_PLUGIN_PATH
        )

        self.env.revert_snapshot("ready_with_5_slaves")

        for plugin in [settings.ZABBIX_PLUGIN_PATH,
                       settings.ZABBIX_SNMP_PLUGIN_PATH,
                       settings.ZABBIX_SNMP_EMC_PLUGIN_PATH]:
            utils.upload_tarball(
                ip=self.ssh_manager.admin_ip,
                tar_path=plugin,
                tar_target="/var")
            utils.install_plugin_check_code(
                ip=self.ssh_manager.admin_ip,
                plugin=os.path.basename(plugin))

        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=settings.DEPLOYMENT_MODE,
        )

        zabbix_username = 'admin'
        zabbix_password = 'zabbix'
        snmp_community = 'public'

        self.setup_zabbix_plugin(cluster_id, zabbix_username, zabbix_password)
        self.setup_snmp_plugin(cluster_id, snmp_community)
        self.setup_snmp_emc_plugin(cluster_id)

        self.fuel_web.update_nodes(
            cluster_id,
            {
                "slave-01": ["controller"],
                "slave-02": ["controller"],
                "slave-03": ["controller"],
                "slave-04": ["compute"],
                "slave-05": ["cinder"]
            }
        )
        self.fuel_web.deploy_cluster_wait(cluster_id)
        self.fuel_web.verify_network(cluster_id)
        self.fuel_web.run_ostf(cluster_id=cluster_id)

        management_vip = self.fuel_web.get_mgmt_vip(cluster_id)
        snmp_emc_critical_command = \
            ("snmptrap -v 1 -c {snmp_community} {management_vip} "
             "'.1.3.6.1.4.1.1981' {management_vip} 6 6 '10' .1.3.6.1.4.1.1981 "
             "s 'null' .1.3.6.1.4.1.1981 s 'null' .1.3.6.1.4.1.1981 s 'a37'"
             .format(snmp_community=snmp_community,
                     management_vip=management_vip))

        with self.fuel_web.get_ssh_for_node("slave-01") as remote:
            remote.execute("apt-get install snmp -y")
            remote.execute(snmp_emc_critical_command)

        public_vip = self.fuel_web.get_public_vip(cluster_id)
        zabbix_web = ZabbixWeb(public_vip, zabbix_username, zabbix_password)
        zabbix_web.login()

        wait(lambda: self.check_event_message(
            zabbix_web, 'emc', 'SNMPtrigger Critical'))

        self.env.make_snapshot("deploy_zabbix_snmp_emc_ha")

    @test(depends_on=[SetupEnvironment.prepare_slaves_5],
          groups=["deploy_zabbix_snmp_extreme_ha"])
    @log_snapshot_after_test
    def deploy_zabbix_snmp_extreme_ha(self):
        """Deploy cluster in ha mode with zabbix snmptrap plugin

        Scenario:
            1. Upload plugin to the master node
            2. Install plugins
            3. Create cluster
            4. Add 3 nodes with controller role
            5. Add 1 node with compute role
            6. Add 1 node with cinder role
            7. Deploy the cluster
            8. Run network verification
            9. Run OSTF
            10. Check Extreme Switch trigger with test SNMP message

        Duration 70m
        Snapshot deploy_zabbix_snmp_extreme_ha

        """
        check_plugin_path_env(
            var_name='ZABBIX_SNMP_PLUGIN_PATH',
            plugin_path=settings.ZABBIX_SNMP_PLUGIN_PATH
        )
        check_plugin_path_env(
            var_name='ZABBIX_SNMP_EXTREME_PLUGIN_PATH',
            plugin_path=settings.ZABBIX_SNMP_EXTREME_PLUGIN_PATH
        )
        self.env.revert_snapshot("ready_with_5_slaves")

        for plugin in [settings.ZABBIX_PLUGIN_PATH,
                       settings.ZABBIX_SNMP_PLUGIN_PATH,
                       settings.ZABBIX_SNMP_EXTREME_PLUGIN_PATH]:
            utils.upload_tarball(
                ip=self.ssh_manager.admin_ip,
                tar_path=plugin,
                tar_target="/var")
            utils.install_plugin_check_code(
                ip=self.ssh_manager.admin_ip,
                plugin=os.path.basename(plugin))

        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=settings.DEPLOYMENT_MODE,
        )

        zabbix_username = 'admin'
        zabbix_password = 'zabbix'
        snmp_community = 'public'

        self.setup_zabbix_plugin(cluster_id, zabbix_username, zabbix_password)
        self.setup_snmp_plugin(cluster_id, snmp_community)
        self.setup_snmp_extreme_plugin(cluster_id)

        self.fuel_web.update_nodes(
            cluster_id,
            {
                "slave-01": ["controller"],
                "slave-02": ["controller"],
                "slave-03": ["controller"],
                "slave-04": ["compute"],
                "slave-05": ["cinder"]
            }
        )
        self.fuel_web.deploy_cluster_wait(cluster_id)
        self.fuel_web.verify_network(cluster_id)
        self.fuel_web.run_ostf(cluster_id=cluster_id)

        management_vip = self.fuel_web.get_mgmt_vip(cluster_id)
        snmp_extreme_critical_command = \
            ("snmptrap -v 1 -c {snmp_community} {management_vip} "
             "'.1.3.6.1.4.1.1916' {management_vip} 6 10 '10' .1.3.6.1.4.1.1916"
             " s 'null' .1.3.6.1.4.1.1916 s 'null' .1.3.6.1.4.1.1916 s '2'"
             .format(snmp_community=snmp_community,
                     management_vip=management_vip))

        with self.fuel_web.get_ssh_for_node("slave-01") as remote:
            remote.execute("apt-get install snmp -y")
            remote.execute(snmp_extreme_critical_command)

        public_vip = self.fuel_web.get_public_vip(cluster_id)
        zabbix_web = ZabbixWeb(public_vip, zabbix_username, zabbix_password)
        zabbix_web.login()

        wait(lambda: self.check_event_message(
            zabbix_web, 'extreme', 'Power Supply Failed'))

        self.env.make_snapshot("deploy_zabbix_snmp_extreme_ha")

    @test(depends_on=[SetupEnvironment.prepare_slaves_5],
          groups=["deploy_zabbix_ceph_ha"])
    @log_snapshot_after_test
    def deploy_zabbix_ceph_ha(self):
        """Deploy cluster in ha mode with zabbix plugin

        Scenario:
            1. Upload plugin to the master node
            2. Install plugin
            3. Create cluster
            4. Add 3 nodes with controller,ceph-osd roles
            5. Add 2 node with compute,ceph-osd roles
            6. Deploy the cluster
            7. Run network verification
            8. Run OSTF
            9. Check zabbix service in pacemaker
            10. Check login to zabbix dashboard

        Duration 180m
        Snapshot deploy_zabbix_ceph_ha

        """
        self.env.revert_snapshot("ready_with_5_slaves")

        utils.upload_tarball(
            ip=self.ssh_manager.admin_ip,
            tar_path=settings.ZABBIX_PLUGIN_PATH,
            tar_target="/var")
        utils.install_plugin_check_code(
            ip=self.ssh_manager.admin_ip,
            plugin=os.path.basename(settings.ZABBIX_PLUGIN_PATH))

        cluster_settings = {
            "net_provider": "neutron",
            "net_segment_type": settings.NEUTRON_SEGMENT_TYPE,
            'volumes_ceph': True,
            'images_ceph': True,
            'volumes_lvm': False,
            'tenant': 'cephHA',
            'user': 'cephHA',
            'password': 'cephHA',
            'osd_pool_size': "3"
        }

        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=settings.DEPLOYMENT_MODE,
            settings=cluster_settings
        )

        zabbix_username = 'admin'
        zabbix_password = 'zabbix'
        self.setup_zabbix_plugin(cluster_id, zabbix_username, zabbix_password)

        self.fuel_web.update_nodes(
            cluster_id,
            {
                'slave-01': ['controller', 'ceph-osd'],
                'slave-02': ['controller', 'ceph-osd'],
                'slave-03': ['controller', 'ceph-osd'],
                'slave-04': ['compute', 'ceph-osd'],
                'slave-05': ['compute', 'ceph-osd'],
            }
        )

        self.fuel_web.deploy_cluster_wait(cluster_id, timeout=190 * 60)
        self.fuel_web.verify_network(cluster_id)
        self.fuel_web.run_ostf(cluster_id=cluster_id)
        self.check_zabbix_configuration(cluster_id, zabbix_username,
                                        zabbix_password)

        self.env.make_snapshot("deploy_zabbix_ceph_ha")