Browse Source

Merge "Support pagination for OpenStack services"

Jenkins 2 years ago
parent
commit
15c2ff534b

+ 57
- 26
deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py View File

@@ -115,7 +115,8 @@ class OSClient(object):
115 115
         self.logger.debug("Got token '%s'" % self.token)
116 116
         return self.token
117 117
 
118
-    def make_request(self, verb, url, data=None, token_required=True):
118
+    def make_request(self, verb, url, data=None, token_required=True,
119
+                     params=None):
119 120
         kwargs = {
120 121
             'url': url,
121 122
             'timeout': self.timeout,
@@ -131,6 +132,9 @@ class OSClient(object):
131 132
         if data is not None:
132 133
             kwargs['data'] = data
133 134
 
135
+        if params is not None:
136
+            kwargs['params'] = params
137
+
134 138
         func = getattr(self.session, verb.lower())
135 139
 
136 140
         try:
@@ -159,6 +163,7 @@ class CollectdPlugin(base.Base):
159 163
         self.max_retries = 2
160 164
         self.os_client = None
161 165
         self.extra_config = {}
166
+        self.pagination_limit = None
162 167
 
163 168
     def _build_url(self, service, resource):
164 169
         s = (self.get_service(service) or {})
@@ -241,12 +246,12 @@ class CollectdPlugin(base.Base):
241 246
 
242 247
                     yield data
243 248
 
244
-    def get(self, service, resource):
249
+    def get(self, service, resource, params=None):
245 250
         url = self._build_url(service, resource)
246 251
         if not url:
247 252
             return
248 253
         self.logger.info("GET '%s'" % url)
249
-        return self.os_client.make_request('get', url)
254
+        return self.os_client.make_request('get', url, params=params)
250 255
 
251 256
     @property
252 257
     def service_catalog(self):
@@ -271,46 +276,72 @@ class CollectdPlugin(base.Base):
271 276
                 tenant_name = node.values[0]
272 277
             elif node.key == 'KeystoneUrl':
273 278
                 keystone_url = node.values[0]
279
+            elif node.key == 'PaginationLimit':
280
+                self.pagination_limit = int(node.values[0])
274 281
         self.os_client = OSClient(username, password, tenant_name,
275 282
                                   keystone_url, self.timeout, self.logger,
276 283
                                   self.max_retries)
277 284
 
278 285
     def get_objects(self, project, object_name, api_version='',
279
-                    params='all_tenants=1'):
286
+                    params=None, detail=False):
280 287
         """ Return a list of OpenStack objects
281 288
 
282
-            See get_objects_details()
283
-        """
284
-        return self._get_objects(project, object_name, api_version, params,
285
-                                 False)
286
-
287
-    def get_objects_details(self, project, object_name, api_version='',
288
-                            params='all_tenants=1'):
289
-        """ Return a list of details about OpenStack objects
290
-
291 289
             The API version is not always included in the URL endpoint
292 290
             registered in Keystone (eg Glance). In this case, use the
293 291
             api_version parameter to specify which version should be used.
292
+
294 293
         """
295
-        return self._get_objects(project, object_name, api_version, params,
296
-                                 True)
294
+        if params is None:
295
+            params = {}
297 296
 
298
-    def _get_objects(self, project, object_name, api_version, params, detail):
299 297
         if api_version:
300 298
             resource = '%s/%s' % (api_version, object_name)
301 299
         else:
302 300
             resource = '%s' % (object_name)
301
+
303 302
         if detail:
304
-            resource = '%s/detail' % (resource)
305
-        if params:
306
-            resource = '%s?%s' % (resource, params)
307
-        # TODO(scroiset): use pagination to handle large collection
308
-        r = self.get(project, resource)
309
-        if not r or object_name not in r.json():
310
-            self.logger.warning('Could not find %s %s' % (project,
311
-                                                          object_name))
312
-            return []
313
-        return r.json().get(object_name)
303
+            resource = '{}/detail'.format(resource)
304
+
305
+        url = self._build_url(project, resource)
306
+        if not url:
307
+            return
308
+
309
+        opts = {}
310
+        if self.pagination_limit:
311
+            opts['limit'] = self.pagination_limit
312
+
313
+        opts.update(params)
314
+        objs = []
315
+
316
+        while True:
317
+            r = self.os_client.make_request('get', url, params=opts)
318
+            if not r or object_name not in r.json():
319
+                self.logger.warning('Could not find %s %s' % (project,
320
+                                                              object_name))
321
+                return objs
322
+
323
+            resp = r.json()
324
+            bulk_objs = resp.get(object_name)
325
+
326
+            if not bulk_objs:
327
+                break
328
+
329
+            objs.extend(bulk_objs)
330
+
331
+            links = resp.get('{}_links'.format(object_name))
332
+            if links is None or self.pagination_limit is None:
333
+                # Either the pagination is not supported or there is no more
334
+                # data
335
+                break
336
+
337
+            # if there is no 'next' link in the response, all data has been
338
+            # read.
339
+            if len([i for i in links if i.get('rel') == 'next']) == 0:
340
+                break
341
+
342
+            opts['marker'] = bulk_objs[-1]['id']
343
+
344
+        return objs
314 345
 
315 346
     def count_objects_group_by(self,
316 347
                                list_object,

+ 6
- 2
deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py View File

@@ -33,10 +33,13 @@ class CinderStatsPlugin(openstack.CollectdPlugin):
33 33
         super(CinderStatsPlugin, self).__init__(*args, **kwargs)
34 34
         self.plugin = PLUGIN_NAME
35 35
         self.interval = INTERVAL
36
+        self.pagination_limit = 500
36 37
 
37 38
     def itermetrics(self):
38 39
 
39
-        volumes_details = self.get_objects_details('cinder', 'volumes')
40
+        volumes_details = self.get_objects('cinder', 'volumes',
41
+                                           params={'all_tenants': 1},
42
+                                           detail=True)
40 43
 
41 44
         def groupby(d):
42 45
             return d.get('status', 'unknown').lower()
@@ -63,7 +66,8 @@ class CinderStatsPlugin(openstack.CollectdPlugin):
63 66
                 'values': size
64 67
             }
65 68
 
66
-        snaps_details = self.get_objects_details('cinder', 'snapshots')
69
+        snaps_details = self.get_objects('cinder', 'snapshots',
70
+                                         params={'all_tenants': 1})
67 71
         status_snaps = self.count_objects_group_by(snaps_details,
68 72
                                                    group_by_func=groupby)
69 73
         for s, nb in status_snaps.iteritems():

+ 4
- 1
deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py View File

@@ -33,6 +33,7 @@ class GlanceStatsPlugin(openstack.CollectdPlugin):
33 33
         super(GlanceStatsPlugin, self).__init__(*args, **kwargs)
34 34
         self.plugin = PLUGIN_NAME
35 35
         self.interval = INTERVAL
36
+        self.pagination_limit = 25
36 37
 
37 38
     def itermetrics(self):
38 39
 
@@ -48,7 +49,9 @@ class GlanceStatsPlugin(openstack.CollectdPlugin):
48 49
 
49 50
         images_details = self.get_objects_details('glance', 'images',
50 51
                                                   api_version='v1',
51
-                                                  params='is_public=None')
52
+                                                  params={},
53
+                                                  detail=True)
54
+
52 55
         status = self.count_objects_group_by(images_details,
53 56
                                              group_by_func=groupby)
54 57
         for s, nb in status.iteritems():

+ 1
- 0
deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py View File

@@ -36,6 +36,7 @@ class NeutronStatsPlugin(openstack.CollectdPlugin):
36 36
         super(NeutronStatsPlugin, self).__init__(*args, **kwargs)
37 37
         self.plugin = PLUGIN_NAME
38 38
         self.interval = INTERVAL
39
+        self.pagination_limit = 100
39 40
 
40 41
     def itermetrics(self):
41 42
 

+ 4
- 1
deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py View File

@@ -31,9 +31,12 @@ class NovaInstanceStatsPlugin(openstack.CollectdPlugin):
31 31
         super(NovaInstanceStatsPlugin, self).__init__(*args, **kwargs)
32 32
         self.plugin = PLUGIN_NAME
33 33
         self.interval = INTERVAL
34
+        self.pagination_limit = 500
34 35
 
35 36
     def itermetrics(self):
36
-        servers_details = self.get_objects_details('nova', 'servers')
37
+        servers_details = self.get_objects('nova', 'servers',
38
+                                           params={'all_tenants': 1},
39
+                                           detail=True)
37 40
 
38 41
         def groupby(d):
39 42
             return d.get('status', 'unknown').lower()

Loading…
Cancel
Save