NoUniqueMatch: ClientException on Gnocchi publisher
Ceilometer can ignore/discard measurements that come from possible
VMs that can be used to host Gnocchi. The assumption is that one
can use OpenStack itself to host the VMs that are used to run Gnocchi.
The configuration is called `filter_project`. This configuration can
be used in `event_pipeline.yaml` and `pipeline.yaml` configuration files.
However, when this config is not used, it has a default project name
for Gnocchi as `service`, as we can see in the following code snippet:
```
def __init__(self, conf, parsed_url):
super(GnocchiPublisher, self).__init__(conf, parsed_url)
# TODO(jd) allow to override Gnocchi endpoint via the host in the URL
options = urlparse.parse_qs(parsed_url.query)
self.filter_project = options.get('filter_project', ['service'])[-1]
```
Which means that if somebody creates a project called `service`, this project would not push measurements to Gnocchi.
This configuration is then used by the following code:
```
def gnocchi_project_id(self):
if self._gnocchi_project_id is not None:
return self._gnocchi_project_id
with self._gnocchi_project_id_lock:
if self._gnocchi_project_id is None:
try:
project = self._ks_client.projects.find(
name=self.filter_project,
domain=self.filter_domain)
except ka_exceptions.NotFound:
LOG.warning('filtered project not found in keystone,'
' ignoring the filter_project '
'option')
self.filter_project = None
return None
except Exception:
LOG.exception('fail to retrieve filtered project ')
raise
self._gnocchi_project_id = project.id
LOG.debug("filtered project found: %s",
self._gnocchi_project_id)
return self._gnocchi_project_id
```
Basically, this method will look for the project ID of the project
name that is configured in `filter_project` option. If it does not
find any project, it returns None, and it changes the value of
`filter_project` to None as well. Before this `gnocchi_project_id`
method/property is called, there is a verification if
`filter_project` is None. Therefore, it is assumed that when we
set the value of `filter_project` to None, this method
(`gnocchi_project_id`) would not be called anymore.
However, that is not taking into account concurrency parallel
executions.
In the code, we can see `with self._gnocchi_project_id_lock:`
statement, which seems to execute locking in the execution flow.
However, that will not always be the case because multiple
concurrent calls can be queued in that part of the code, and
when the first one finishes setting the `filter_project` to None,
the others will execute with this variable as None, which will cause
Keystone command `self._ks_client.projects.find` to find/list all
projects. That command was designed to list/find only one project;
therefore, when it finds more than one project, it throws an error.
That is the cause for the exception we were seeing from time to time
in the log files.
Change-Id: I3b4ac918015b2fd3fbe24047c3eb13419f580b27
This commit is contained in:
Rafael Weingärtner
parent
341ec07827
commit
463594b229
|
|
@ -280,22 +280,31 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
|
|||
return self._gnocchi_project_id
|
||||
with self._gnocchi_project_id_lock:
|
||||
if self._gnocchi_project_id is None:
|
||||
if not self.filter_project:
|
||||
LOG.debug(
|
||||
"Multiple executions were locked on "
|
||||
"self._gnocchi_project_id_lock`. This execution "
|
||||
"should no call `_internal_gnocchi_project_discovery` "
|
||||
"as `self.filter_project` is None.")
|
||||
return None
|
||||
try:
|
||||
project = self._ks_client.projects.find(
|
||||
name=self.filter_project,
|
||||
domain=self.filter_domain)
|
||||
except ka_exceptions.NotFound:
|
||||
LOG.warning('project %s not found in keystone,'
|
||||
' ignoring the filter_project '
|
||||
'option', self.filter_project)
|
||||
LOG.warning('Filtered project [%s] not found in keystone, '
|
||||
'ignoring the filter_project option' %
|
||||
self.filter_project)
|
||||
|
||||
self.filter_project = None
|
||||
return None
|
||||
except Exception:
|
||||
LOG.exception('fail to retrieve filtered project ')
|
||||
LOG.exception('Failed to retrieve filtered project [%s].'
|
||||
% self.filter_project)
|
||||
raise
|
||||
self._gnocchi_project_id = project.id
|
||||
LOG.debug("filtered project found: %s",
|
||||
self._gnocchi_project_id)
|
||||
LOG.debug("Filtered project [%s] found with ID [%s].",
|
||||
self.filter_project, self._gnocchi_project_id)
|
||||
return self._gnocchi_project_id
|
||||
|
||||
def _is_swift_account_sample(self, sample):
|
||||
|
|
@ -320,11 +329,29 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
|
|||
if operation:
|
||||
return rd, operation
|
||||
|
||||
def filter_gnocchi_activity_openstack(self, samples):
|
||||
"""Skip sample generated by gnocchi itself
|
||||
|
||||
This method will filter out the samples that are generated by
|
||||
Gnocchi itself.
|
||||
"""
|
||||
filtered_samples = []
|
||||
for sample in samples:
|
||||
if not self._is_gnocchi_activity(sample):
|
||||
filtered_samples.append(sample)
|
||||
LOG.debug("Sample [%s] is not a Gnocchi activity; therefore, "
|
||||
"we do not filter it out and push it to Gnocchi.",
|
||||
sample)
|
||||
else:
|
||||
LOG.debug("Sample [%s] is a Gnocchi activity; therefore, "
|
||||
"we filter it out and do not push it to Gnocchi.",
|
||||
sample)
|
||||
return filtered_samples
|
||||
|
||||
def publish_samples(self, data):
|
||||
self.ensures_archives_policies()
|
||||
|
||||
# NOTE(sileht): skip sample generated by gnocchi itself
|
||||
data = [s for s in data if not self._is_gnocchi_activity(s)]
|
||||
data = self.filter_gnocchi_activity_openstack(data)
|
||||
|
||||
def value_to_sort(object_to_sort):
|
||||
value = object_to_sort.resource_id
|
||||
|
|
|
|||
|
|
@ -339,9 +339,9 @@ class PublisherTest(base.BaseTestCase):
|
|||
def test_activity_gnocchi_project_not_found(self, logger):
|
||||
self.ks_client.projects.find.side_effect = ka_exceptions.NotFound
|
||||
self._do_test_activity_filter(2)
|
||||
logger.warning.assert_called_with('project %s not found in '
|
||||
'keystone, ignoring the '
|
||||
'filter_project option', 'service')
|
||||
logger.warning.assert_called_with(
|
||||
'Filtered project [service] not found in keystone, ignoring the '
|
||||
'filter_project option')
|
||||
|
||||
def test_activity_filter_match_swift_event(self):
|
||||
self.samples[0].name = 'storage.objects.outgoing.bytes'
|
||||
|
|
@ -749,8 +749,11 @@ class PublisherWorkflowTest(base.BaseTestCase,
|
|||
resource_type = resource_definition.cfg['resource_type']
|
||||
|
||||
expected_debug = [
|
||||
mock.call('filtered project found: %s',
|
||||
mock.call('Filtered project [%s] found with ID [%s].', 'service',
|
||||
'a2d42c23-d518-46b6-96ab-3fba2e146859'),
|
||||
mock.call('Sample [%s] is not a Gnocchi activity; therefore, we '
|
||||
'do not filter it out and push it to Gnocchi.',
|
||||
self.sample),
|
||||
mock.call('Processing sample [%s] for resource ID [%s].',
|
||||
self.sample, resource_id),
|
||||
mock.call('Executing batch resource metrics measures for resource '
|
||||
|
|
|
|||
Loading…
Reference in New Issue