diff --git a/conf.d/libvirt.yaml.example b/conf.d/libvirt.yaml.example index 96a8ed5a..e1f15e20 100644 --- a/conf.d/libvirt.yaml.example +++ b/conf.d/libvirt.yaml.example @@ -2,10 +2,13 @@ init_config: # These are Nova credentials, [keystone_authtoken] in /etc/nova/nova.conf - admin_password: pass - admin_tenant_name: service - admin_user: nova - identity_uri: 'http://192.168.10.5:35357/v2.0' + password: pass + project_name: service + username: nova + auth_url: 'http://192.168.10.5/identity' + # Options to specify endpoint type, default to 'publicURL', other choices: + # 'internalURL' and 'adminURL' + endpoint_type: 'publicURL' # Location of temporary files maintained by the plugin. Ramdisk preferred. cache_dir: /dev/shm # How long to wait before querying Nova for instance updates? (seconds) diff --git a/conf.d/ovs.yaml.example b/conf.d/ovs.yaml.example index 60da1fca..e98c1c9a 100644 --- a/conf.d/ovs.yaml.example +++ b/conf.d/ovs.yaml.example @@ -1,12 +1,15 @@ # (C) Copyright 2016 Hewlett Packard Enterprise Development LP init_config: # These are Neutron credentials, [keystone_authtoken] in /etc/neutron/neutron.conf - admin_password: password - admin_tenant_name: services - admin_user: neutron - identity_uri: 'http://192.168.10.5:35357/v2.0' + password: password + project_name: services + username: neutron + auth_url: 'http://192.168.10.5/identity' # Number of seconds to wait before updating the neutron router cache file. neutron_refresh: 14400 + # Options to specify endpoint type, default to 'publicURL', other choices: + # 'internalURL' and 'adminURL' + endpoint_type: 'publicURL' # The region name in /etc/neutron/neutron.conf region_name: 'region1' # Location of temporary files maintained by the plugin. Ramdisk preferred. diff --git a/docs/Libvirt.md b/docs/Libvirt.md index 50507865..828e17a9 100644 --- a/docs/Libvirt.md +++ b/docs/Libvirt.md @@ -33,19 +33,21 @@ The Libvirt plugin provides metrics for virtual machines when run on the hypervi ## Configuration The `monasca-setup` program will configure the Libvirt plugin if `nova-compute` is running, its `nova.conf` config file is readable by the Monasca Agent user (default: 'mon-agent'), and `python-novaclient` is installed. -In order to fetch data on hosted compute instances, the Libvirt plugin needs to be able to talk to the Nova API. It does this using credentials found in `nova.conf` under `[keystone_authtoken]`, obtained when `monasca-setup` is run, and stored in `/etc/monasca/agent/conf.d/libvirt.yaml` as `admin_user`, `admin_password`, `admin_tenant_name`, and `admin_password`. These credentials are only used to build and update the [Instance Cache](#instance-cache). +In order to fetch data on hosted compute instances, the Libvirt plugin needs to be able to talk to the Nova API. It does this using credentials found in `nova.conf` under `[keystone_authtoken]`, obtained when `monasca-setup` is run, and stored in `/etc/monasca/agent/conf.d/libvirt.yaml` as `username`, `project_name`, and `password`. These credentials are only used to build and update the [Instance Cache](#instance-cache). The Libvirt plugin uses a cache directory to persist data, which is `/dev/shm` by default. On non-Linux systems (BSD, Mac OSX), `/dev/shm` may not exist, so `cache_dir` would need to be changed accordingly, either in `monasca_setup/detection/plugins/libvirt.py` prior to running `monasca-setup`, or `/etc/monasca/agent/conf.d/libvirt.yaml` afterwards. If the owner of the VM is in a different tenant the Agent Cross-Tenant Metric Submission can be setup. See this [documentation](https://github.com/openstack/monasca-agent/blob/master/docs/MonascaMetrics.md#cross-tenant-metric-submission) for details. -`admin_user` is the username capable of making administrative nova calls. +`username` is the username capable of making administrative nova calls. -`admin_password` password for the nova user. +`password` password for the nova user. -`admin_tenant_name` is the project/tenant to POST metrics with the `vm.` prefix. +`project_name` is the project/tenant to POST metrics with the `vm.` prefix. -`identity_url` is the keystone endpoint for auth. +`auth_url` is the keystone endpoint for auth. + +`endpoint_type` is the endpoint type for making nova/neutron calls. `region_name` is used to add the region dimension to metrics. @@ -84,10 +86,11 @@ If the owner of the VM is in a different tenant the Agent Cross-Tenant Metric Su Example config: ``` init_config: - admin_password: pass - admin_tenant_name: service - admin_user: nova - identity_uri: 'http://192.168.10.5:35357/v2.0' + password: pass + project_name: service + username: nova + auth_url: 'http://192.168.10.5/identity' + endpoint_type: 'publicURL' region_name: 'region1' cache_dir: /dev/shm nova_refresh: 14400 diff --git a/docs/Ovs.md b/docs/Ovs.md index 7c0f8f07..9da798b3 100644 --- a/docs/Ovs.md +++ b/docs/Ovs.md @@ -31,15 +31,17 @@ NOTE: The `/usr/bin/ovs-vsctl` command requires sudo to run. See notes in `ovs_ This plugin is not currently automatically configured using the monasca-setup program -- it must be explicitly configured using the configuration file example below. -`admin_password` password for the neutron user. +`password` password for the neutron user. -`admin_tenant_name` is the project/tenant to POST metrics with the `ovs.` prefix. +`project_name` is the project/tenant to POST metrics with the `ovs.` prefix. -`admin_user` is the username capable of making administrative neutron calls. +`username` is the username capable of making administrative neutron calls. `neutron_refresh` is the number of seconds to wait before updating the neutron router cache file. This requires two neutron calls to get port and router info, so we intentionally overload neutron by making these calls each time the agent wakes up. -`identity_url` is the keystone endpoint for auth. +`auth_url` is the keystone endpoint for auth. + +`endpoint_type` is the endpoint type for making neutron calls. `region_name` is used to add the region dimension to metrics. @@ -67,11 +69,12 @@ Example config (`ovs.yaml`): ``` --- init_config: - admin_password: password - admin_tenant_name: services - admin_user: neutron + password: password + project_name: services + username: neutron neutron_refresh: 14400 - identity_uri: 'http://192.168.10.5:35357/v2.0' + auth_url: 'http://192.168.10.5/identity' + endpoint_type: 'publicURL' region_name: 'region1' cache_dir: /dev/shm network_use_bits: true diff --git a/monasca_agent/collector/checks/utils.py b/monasca_agent/collector/checks/utils.py index 41dac3d3..9b4cd525 100644 --- a/monasca_agent/collector/checks/utils.py +++ b/monasca_agent/collector/checks/utils.py @@ -1,4 +1,5 @@ # (C) Copyright 2015,2017 Hewlett Packard Enterprise Development LP +# (C) Copyright 2017 KylinCloud import base64 import logging @@ -14,6 +15,9 @@ log = logging.getLogger(__name__) DEFAULT_TIMEOUT = 20 +from keystoneclient.v2_0 import client as kc +from monasca_agent.common import keystone + def add_basic_auth(request, username, password): """A helper to add basic authentication to a urllib2 request. @@ -26,17 +30,11 @@ def add_basic_auth(request, username, password): def get_keystone_client(config): - import keystoneclient.v2_0.client as kc - kwargs = { - 'username': config.get('admin_user'), - 'project_name': config.get('admin_tenant_name'), - 'password': config.get('admin_password'), - 'auth_url': config.get('identity_uri'), - 'endpoint_type': 'internalURL', - 'region_name': config.get('region_name'), - } + session = keystone.get_session(config) - return kc.Client(**kwargs) + return kc.Client(session=session, + endpoint_type=config.get('endpoint_type', 'publicURL'), + region_name=config.get('region_name')) def get_tenant_name(tenants, tenant_id): diff --git a/monasca_agent/collector/checks_d/libvirt.py b/monasca_agent/collector/checks_d/libvirt.py index 6c94c0d8..8a212837 100644 --- a/monasca_agent/collector/checks_d/libvirt.py +++ b/monasca_agent/collector/checks_d/libvirt.py @@ -31,8 +31,13 @@ from datetime import datetime from datetime import timedelta from monasca_agent.collector.checks import AgentCheck from monasca_agent.collector.virt import inspector +from monasca_agent.common import keystone from multiprocessing.dummy import Pool from netaddr import all_matching_cidrs +from neutronclient.v2_0 import client as neutron_client +from novaclient import client as n_client +from novaclient.exceptions import NotFound + DOM_STATES = {libvirt.VIR_DOMAIN_BLOCKED: 'VM is blocked', libvirt.VIR_DOMAIN_CRASHED: 'VM has crashed', @@ -122,35 +127,27 @@ class LibvirtCheck(AgentCheck): def _update_instance_cache(self): """Collect instance_id, project_id, and AZ for all instance UUIDs """ - from novaclient import client - from novaclient.exceptions import NotFound id_cache = {} flavor_cache = {} port_cache = None netns = None # Get a list of all instances from the Nova API - nova_client = client.Client(2, - username=self.init_config.get('admin_user'), - password=self.init_config.get('admin_password'), - project_name=self.init_config.get('admin_tenant_name'), - auth_url=self.init_config.get('identity_uri'), - endpoint_type='internalURL', - service_type="compute", - region_name=self.init_config.get('region_name')) - instances = nova_client.servers.list(search_opts={'all_tenants': 1, - 'host': self.hostname}) + session = keystone.get_session(self.init_config) + nova_client = n_client.Client( + "2.1", session=session, + endpoint_type=self.init_config.get("endpoint_type", "publicURL"), + service_type="compute", + region_name=self.init_config.get('region_name')) self._get_this_host_aggregate(nova_client) - + instances = nova_client.servers.list( + search_opts={'all_tenants': 1, 'host': self.hostname}) # Lay the groundwork for fetching VM IPs and network namespaces if self.init_config.get('ping_check'): - from neutronclient.v2_0 import client - nu = client.Client(username=self.init_config.get('admin_user'), - password=self.init_config.get('admin_password'), - tenant_name=self.init_config.get('admin_tenant_name'), - auth_url=self.init_config.get('identity_uri'), - endpoint_type='internalURL', - region_name=self.init_config.get('region_name')) + nu = neutron_client.Client( + session=session, + endpoint_type=self.init_config.get("endpoint_type", "publicURL"), + region_name=self.init_config.get('region_name')) port_cache = nu.list_ports()['ports'] # Finding existing network namespaces is an indication that either # DVR agent_mode is enabled, or this is all-in-one (like devstack) diff --git a/monasca_agent/collector/checks_d/ovs.py b/monasca_agent/collector/checks_d/ovs.py index d4358189..51e96450 100644 --- a/monasca_agent/collector/checks_d/ovs.py +++ b/monasca_agent/collector/checks_d/ovs.py @@ -16,6 +16,7 @@ import time from copy import deepcopy from monasca_agent.collector.checks import AgentCheck +from monasca_agent.common import keystone from neutronclient.v2_0 import client as neutron_client from novaclient import client as nova_client @@ -55,6 +56,7 @@ class OvsCheck(AgentCheck): else: include_re = include_re + '|' + 'qg.*' self.include_iface_re = re.compile(include_re) + self.session = keystone.get_session(self.init_config) def check(self, instance): time_start = time.time() @@ -292,38 +294,21 @@ class OvsCheck(AgentCheck): return None def _get_nova_client(self): - - username = self.init_config.get('admin_user') - password = self.init_config.get('admin_password') - tenant_name = self.init_config.get('admin_tenant_name') - auth_url = self.init_config.get('identity_uri') region_name = self.init_config.get('region_name') - - nc = nova_client.Client(2, - username=username, - password=password, - project_name=tenant_name, - auth_url=auth_url, - endpoint_type='internalURL', + endpoint_type = self.init_config.get("endpoint_type", "publicURL") + nc = nova_client.Client(2, session=self.session, + endpoint_type=endpoint_type, service_type="compute", region_name=region_name) return nc def _get_neutron_client(self): - - username = self.init_config.get('admin_user') - password = self.init_config.get('admin_password') - tenant_name = self.init_config.get('admin_tenant_name') - auth_url = self.init_config.get('identity_uri') region_name = self.init_config.get('region_name') - - return neutron_client.Client(username=username, - password=password, - tenant_name=tenant_name, - auth_url=auth_url, + endpoint_type = self.init_config.get("endpoint_type", "publicURL") + return neutron_client.Client(session=self.session, region_name=region_name, - endpoint_type='internalURL') + endpoint_type=endpoint_type) def _run_command(self, command, input=None): self.log.debug("Executing command - {0}".format(command)) diff --git a/monasca_agent/common/keystone.py b/monasca_agent/common/keystone.py index 8cfc4470..34fbaa11 100644 --- a/monasca_agent/common/keystone.py +++ b/monasca_agent/common/keystone.py @@ -1,8 +1,11 @@ # (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP +# (C) Copyright 2017 KylinCloud import logging import six +from keystoneauth1 import identity +from keystoneauth1 import session from monascaclient import ksclient import monasca_agent.common.singleton as singleton @@ -10,6 +13,19 @@ import monasca_agent.common.singleton as singleton log = logging.getLogger(__name__) +def get_session(config): + auth = identity.Password(auth_url=config.get('auth_url'), + username=config.get('username'), + password=config.get('password'), + project_name=config.get('project_name'), + user_domain_name=config.get( + 'user_domain_name', 'default'), + project_domain_name=config.get( + 'project_domain_name', 'default')) + sess = session.Session(auth=auth) + return sess + + # Make this a singleton class so we don't get the token every time # the class is created @six.add_metaclass(singleton.Singleton) diff --git a/monasca_setup/detection/plugins/libvirt.py b/monasca_setup/detection/plugins/libvirt.py index fe2175d1..f2576de3 100644 --- a/monasca_setup/detection/plugins/libvirt.py +++ b/monasca_setup/detection/plugins/libvirt.py @@ -95,19 +95,11 @@ class Libvirt(Plugin): nova_cfg.read(self.nova_conf) # Which configuration options are needed for the plugin YAML? # Use a dict so that they can be renamed later if necessary - cfg_needed = {'admin_user': 'admin_user', - 'admin_password': 'admin_password', - 'admin_tenant_name': 'admin_tenant_name'} + cfg_needed = {'username': 'username', + 'password': 'password', + 'project_name': 'project_name'} cfg_section = 'keystone_authtoken' - # Handle Devstack's slightly different nova.conf names - if (nova_cfg.has_option(cfg_section, 'username') - and nova_cfg.has_option(cfg_section, 'password') - and nova_cfg.has_option(cfg_section, 'project_name')): - cfg_needed = {'admin_user': 'username', - 'admin_password': 'password', - 'admin_tenant_name': 'project_name'} - # Start with plugin-specific configuration parameters init_config = {'cache_dir': cache_dir, 'nova_refresh': nova_refresh, @@ -130,9 +122,9 @@ class Libvirt(Plugin): # Create an identity URI (again, slightly different for Devstack) if nova_cfg.has_option(cfg_section, 'auth_url'): - init_config['identity_uri'] = "{0}/v2.0".format(nova_cfg.get(cfg_section, 'auth_url')) + init_config['auth_url'] = nova_cfg.get(cfg_section, 'auth_url') else: - init_config['identity_uri'] = "{0}/v2.0".format(nova_cfg.get(cfg_section, 'identity_uri')) + init_config['auth_url'] = nova_cfg.get(cfg_section, 'identity_uri') # Verify requirements to enable ping checks init_config['ping_check'] = self.literal_eval('False') diff --git a/monasca_setup/detection/plugins/ovs.py b/monasca_setup/detection/plugins/ovs.py index ca4b709d..730346c5 100644 --- a/monasca_setup/detection/plugins/ovs.py +++ b/monasca_setup/detection/plugins/ovs.py @@ -32,16 +32,14 @@ use_health_metrics = True # If set, router max capacity metrics will be published publish_router_capacity = False # Acceptable arguments -acceptable_args = ['admin_user', 'admin_password', 'admin_tenant_name', - 'identity_uri', 'cache_dir', 'neutron_refresh', 'ovs_cmd', +acceptable_args = ['username', 'password', 'project_name', + 'auth_url', 'cache_dir', 'neutron_refresh', 'ovs_cmd', 'network_use_bits', 'check_router_ha', 'region_name', 'included_interface_re', 'conf_file_path', 'use_absolute_metrics', 'use_rate_metrics', 'use_health_metrics', 'publish_router_capacity'] # Arguments which must be ignored if provided -ignorable_args = ['admin_user', 'admin_password', 'admin_tenant_name', - 'identity_uri', 'region_name', 'conf_file_path'] -# Regular expression to match the URI version -uri_version_re = re.compile('.*v2.0|.*v3.0|.*v1|.*v2') +ignorable_args = ['username', 'password', 'project_name', + 'auth_url', 'region_name', 'conf_file_path'] class Ovs(detection.Plugin): @@ -137,19 +135,9 @@ class Ovs(detection.Plugin): log.info("\tUsing neutron configuration file %s", self.neutron_conf) neutron_cfg.read(self.neutron_conf) cfg_section = 'keystone_authtoken' - - # Handle Devstack's slightly different neutron.conf names - if ( - neutron_cfg.has_option(cfg_section, 'username') and - neutron_cfg.has_option(cfg_section, 'password') and - neutron_cfg.has_option(cfg_section, 'project_name')): - cfg_needed = {'admin_user': 'username', - 'admin_password': 'password', - 'admin_tenant_name': 'project_name'} - else: - cfg_needed = {'admin_user': 'admin_user', - 'admin_password': 'admin_password', - 'admin_tenant_name': 'admin_tenant_name'} + cfg_needed = {'username': 'username', + 'password': 'password', + 'project_name': 'project_name'} # Start with plugin-specific configuration parameters init_config = {'cache_dir': cache_dir, @@ -165,17 +153,11 @@ class Ovs(detection.Plugin): for option in cfg_needed: init_config[option] = neutron_cfg.get(cfg_section, cfg_needed[option]) - uri_version = 'v2.0' - if neutron_cfg.has_option(cfg_section, 'auth_version'): - uri_version = str(neutron_cfg.get(cfg_section, 'auth_version')) - # Create an identity URI (again, slightly different for Devstack) if neutron_cfg.has_option(cfg_section, 'auth_url'): - if re.match(uri_version_re, str(neutron_cfg.get(cfg_section, 'auth_url'))): - uri_version = '' - init_config['identity_uri'] = "{0}/{1}".format(neutron_cfg.get(cfg_section, 'auth_url'), uri_version) + init_config['auth_url'] = neutron_cfg.get(cfg_section, 'auth_url') else: - init_config['identity_uri'] = "{0}/{1}".format(neutron_cfg.get(cfg_section, 'identity_uri'), uri_version) + init_config['auth_url'] = neutron_cfg.get(cfg_section, 'identity_uri') # Create an region_name (again, slightly different for Devstack) if neutron_cfg.has_option('service_auth', 'region'): diff --git a/tests/detection/test_ovs.py b/tests/detection/test_ovs.py index fa3d1d60..a47bb8ca 100644 --- a/tests/detection/test_ovs.py +++ b/tests/detection/test_ovs.py @@ -38,7 +38,7 @@ class TestOvs(unittest.TestCase): unittest.TestCase.setUp(self) with patch.object(Ovs, '_detect') as mock_detect: self.ovs_obj = Ovs('temp_dir') - self.has_option = [True, True, True, False, False, True] + self.has_option = [False, True] self.get_value = [MagicMock(), MagicMock(), MagicMock(), 'http://10.10.10.10', 'region1'] self.assertTrue(mock_detect.called) @@ -88,14 +88,14 @@ class TestOvs(unittest.TestCase): self.assertEqual(result['ovs']['init_config']['neutron_refresh'], 13000) self.assertFalse(result['ovs']['init_config']['network_use_bits']) - self.assertIsInstance(result['ovs']['init_config']['admin_user'], + self.assertIsInstance(result['ovs']['init_config']['username'], MagicMock) - self.assertIsInstance(result['ovs']['init_config']['admin_password'], + self.assertIsInstance(result['ovs']['init_config']['password'], MagicMock) - self.assertIsInstance(result['ovs']['init_config']['admin_tenant_name'], + self.assertIsInstance(result['ovs']['init_config']['project_name'], MagicMock) - self.assertEqual(result['ovs']['init_config']['identity_uri'], - 'http://10.10.10.10/v2.0') + self.assertEqual(result['ovs']['init_config']['auth_url'], + 'http://10.10.10.10') self.assertEqual(result['ovs']['init_config']['region_name'], 'region1') self.assertEqual(result['ovs']['init_config']['cache_dir'], @@ -118,11 +118,11 @@ class TestOvs(unittest.TestCase): "sudo /usr/bin/ovs-vsctl") self.assertEqual(result['ovs']['init_config']['included_interface_re'], 'qg.*|vhu.*|sg.*') - self.assertIsInstance(result['ovs']['init_config']['admin_user'], + self.assertIsInstance(result['ovs']['init_config']['username'], MagicMock) - self.assertIsInstance(result['ovs']['init_config']['admin_password'], + self.assertIsInstance(result['ovs']['init_config']['password'], MagicMock) - self.assertIsInstance(result['ovs']['init_config']['admin_tenant_name'], + self.assertIsInstance(result['ovs']['init_config']['project_name'], MagicMock) self.assertTrue(result['ovs']['init_config']['use_absolute_metrics']) self.assertTrue(result['ovs']['init_config']['use_rate_metrics']) @@ -179,10 +179,10 @@ class TestOvs(unittest.TestCase): def test_build_config_with_args(self): with patch.object(LOG, 'warn') as mock_log_warn: self.ovs_obj.neutron_conf = 'neutron-conf' - self.ovs_obj.args = {'admin_user': 'admin', - 'admin_password': 'password', - 'admin_tenant_name': 'tenant', - 'identity_uri': '10.10.10.20', + self.ovs_obj.args = {'username': 'admin', + 'password': 'password', + 'project_name': 'tenant', + 'auth_url': '10.10.10.20', 'region_name': 'region0', 'neutron_refresh': 13000, 'use_absolute_metrics': False} @@ -199,10 +199,10 @@ class TestOvs(unittest.TestCase): def test_build_config_invalid_arg_warning(self): with patch.object(LOG, 'warn') as mock_log_warn: self.ovs_obj.neutron_conf = 'neutron-conf' - self.ovs_obj.args = {'admin_user': 'admin', - 'admin_password': 'password', - 'admin_tenant_name': 'tenant', - 'identity_uri': '10.10.10.20', + self.ovs_obj.args = {'username': 'admin', + 'password': 'password', + 'project_name': 'tenant', + 'auth_url': '10.10.10.20', 'region_name': 'region0', 'neutron_refresh': 13000, 'use_absolute_metrics': False, @@ -215,32 +215,32 @@ class TestOvs(unittest.TestCase): def test_build_config_if_auth_version(self): self.ovs_obj.neutron_conf = 'neutron-conf' - self.has_option = [True, True, True, True, True, True] + self.has_option = [True, True] self.get_value = [MagicMock(), MagicMock(), MagicMock(), - 'v3.0', 'http://10.10.10.10', + 'http://10.10.10.10', 'http://10.10.10.10', 'region1'] result = self._build_config_without_args(self.ovs_obj) - self.assertEqual(result['ovs']['init_config']['identity_uri'], - 'http://10.10.10.10/v3.0') + self.assertEqual(result['ovs']['init_config']['auth_url'], + 'http://10.10.10.10') def test_build_config_if_auth_url_has_version(self): self.ovs_obj.neutron_conf = 'neutron-conf' - self.has_option = [True, True, True, True, True, True] + self.has_option = [True, True] self.get_value = [MagicMock(), MagicMock(), MagicMock(), - 'v3.0', 'http://10.10.10.10/v1', + 'http://10.10.10.10/v1', 'http://10.10.10.10/v1', 'region1'] result = self._build_config_without_args(self.ovs_obj) - self.assertEqual(result['ovs']['init_config']['identity_uri'], - 'http://10.10.10.10/v1/') + self.assertEqual(result['ovs']['init_config']['auth_url'], + 'http://10.10.10.10/v1') def test_build_config_region_name_from_nova(self): self.ovs_obj.neutron_conf = 'neutron-conf' - self.has_option = [True, True, True, False, False, False] + self.has_option = [False, False] self.get_value = [MagicMock(), MagicMock(), MagicMock(), 'http://10.10.10.10', 'region2'] result = self._build_config_without_args(self.ovs_obj) - self.assertEqual(result['ovs']['init_config']['identity_uri'], - 'http://10.10.10.10/v2.0') + self.assertEqual(result['ovs']['init_config']['auth_url'], + 'http://10.10.10.10') self.assertEqual(result['ovs']['init_config']['region_name'], 'region2')