Browse Source

Remove TaskManager and just use keystoneauth

Support for concurrency and rate limiting has been added to keystoneauth,
which is the library openstacksdk uses to talk to OpenStack. Instead
of managing concurrency in nodepool using the TaskManager and pool of
worker threads, let keystoneauth take over. This also means we no longer
have a hook into the request process, so we defer statsd reporting to
the openstacksdk layer as well.

Change-Id: If21a10c56f43a121d30aa802f2c89d31df97f121
tags/3.6.0
Monty Taylor 2 months ago
parent
commit
34aae137fa

+ 4
- 5
doc/source/operation.rst View File

@@ -517,12 +517,11 @@ OpenStack API stats
517 517
 ~~~~~~~~~~~~~~~~~~~
518 518
 
519 519
 Low level details on the timing of OpenStack API calls will be logged
520
-by the API task manager.  These calls are logged under
520
+by ``openstacksdk``. These calls are logged under
521 521
 ``nodepool.task.<provider>.<api-call>``.  The API call name is of the
522
-generic format ``<endpoint><method><operation>`` transformed into a
523
-CamelCase value with no deliminators; for example the
524
-``compute.GET.servers`` call becomes ``ComputeGetServers`` and
525
-``compute.POST.os-volumes_boot`` becomes ``ComputePostOsVolumesBoot``.
522
+generic format ``<service-type>.<method>.<operation>``. For example, the
523
+``GET /servers`` call to the ``compute`` service becomes
524
+``compute.GET.servers``.
526 525
 
527 526
 Since these calls reflect the internal operations of the
528 527
 ``openstacksdk``, the exact keys logged may vary across providers and

+ 13
- 16
nodepool/driver/openstack/provider.py View File

@@ -17,6 +17,7 @@
17 17
 import copy
18 18
 import logging
19 19
 import operator
20
+import os
20 21
 import time
21 22
 
22 23
 import openstack
@@ -25,7 +26,6 @@ from nodepool import exceptions
25 26
 from nodepool.driver import Provider
26 27
 from nodepool.driver.utils import QuotaInformation
27 28
 from nodepool.nodeutils import iterate_timeout
28
-from nodepool.task_manager import TaskManager
29 29
 from nodepool import stats
30 30
 from nodepool import version
31 31
 from nodepool import zk
@@ -47,8 +47,6 @@ class OpenStackProvider(Provider):
47 47
         self._networks = {}
48 48
         self.__flavors = {}  # TODO(gtema): caching
49 49
         self.__azs = None
50
-        self._use_taskmanager = use_taskmanager
51
-        self._taskmanager = None
52 50
         self._current_nodepool_quota = None
53 51
         self._zk = None
54 52
         self._down_ports = set()
@@ -57,20 +55,14 @@ class OpenStackProvider(Provider):
57 55
         self._statsd = stats.get_client()
58 56
 
59 57
     def start(self, zk_conn):
60
-        if self._use_taskmanager:
61
-            self._taskmanager = TaskManager(self.provider.name,
62
-                                            self.provider.rate)
63
-            self._taskmanager.start()
64 58
         self.resetClient()
65 59
         self._zk = zk_conn
66 60
 
67 61
     def stop(self):
68
-        if self._taskmanager:
69
-            self._taskmanager.stop()
62
+        pass
70 63
 
71 64
     def join(self):
72
-        if self._taskmanager:
73
-            self._taskmanager.join()
65
+        pass
74 66
 
75 67
     def getRequestHandler(self, poolworker, request):
76 68
         return handler.OpenStackNodeRequestHandler(poolworker, request)
@@ -83,13 +75,18 @@ class OpenStackProvider(Provider):
83 75
         return self.__flavors
84 76
 
85 77
     def _getClient(self):
86
-        if self._use_taskmanager:
87
-            manager = self._taskmanager
88
-        else:
89
-            manager = None
78
+        rate_limit = None
79
+        # nodepool tracks rate limit in time between requests.
80
+        # openstacksdk tracks rate limit in requests per second.
81
+        # 1/time = requests-per-second.
82
+        if self.provider.rate:
83
+            rate_limit = 1 / self.provider.rate
90 84
         return openstack.connection.Connection(
91 85
             config=self.provider.cloud_config,
92
-            task_manager=manager,
86
+            rate_limit=rate_limit,
87
+            statsd_host=os.getenv('STATSD_HOST', None),
88
+            statsd_port=os.getenv('STATSD_PORT ', None),
89
+            statsd_prefix='nodepool.task.{0}'.format(self.provider.name),
93 90
             app_name='nodepool',
94 91
             app_version=version.version_info.version_string()
95 92
         )

+ 0
- 99
nodepool/task_manager.py View File

@@ -1,99 +0,0 @@
1
-#!/usr/bin/env python
2
-
3
-# Copyright (C) 2011-2013 OpenStack Foundation
4
-#
5
-# Licensed under the Apache License, Version 2.0 (the "License");
6
-# you may not use this file except in compliance with the License.
7
-# You may obtain a copy of the License at
8
-#
9
-#    http://www.apache.org/licenses/LICENSE-2.0
10
-#
11
-# Unless required by applicable law or agreed to in writing, software
12
-# distributed under the License is distributed on an "AS IS" BASIS,
13
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
14
-# implied.
15
-#
16
-# See the License for the specific language governing permissions and
17
-# limitations under the License.
18
-
19
-import threading
20
-import logging
21
-import re
22
-import queue
23
-import time
24
-
25
-from openstack import task_manager as openstack_task_manager
26
-
27
-from nodepool import stats
28
-
29
-
30
-def _transform_task_name(task_name):
31
-    # Transform openstacksdk internal task name to something more
32
-    # suitable for sending to statsd for tracking; e.g.
33
-    #
34
-    #  compute.DELETE.servers -> ComputeDeleteServers
35
-    #  compute.POST.os-volumes_boot -> ComputePostOsVolumesBoot
36
-    parts = re.split('[.\-_]', task_name)
37
-    return "".join(
38
-        [part.lower().capitalize() for part in parts]
39
-    )
40
-
41
-
42
-class TaskManager(openstack_task_manager.TaskManager):
43
-    log = logging.getLogger("nodepool.TaskManager")
44
-
45
-    def __init__(self, name, rate, workers=5):
46
-        super(TaskManager, self).__init__(name=name, workers=workers)
47
-        self.daemon = True
48
-        self.queue = queue.Queue()
49
-        self._running = True
50
-        self.rate = float(rate)
51
-        self.statsd = stats.get_client()
52
-        self._thread = threading.Thread(name=name, target=self.run)
53
-        self._thread.daemon = True
54
-
55
-    def start(self):
56
-        self._thread.start()
57
-
58
-    def stop(self):
59
-        self._running = False
60
-        self.queue.put(None)
61
-
62
-    def join(self):
63
-        self._thread.join()
64
-
65
-    def run(self):
66
-        last_ts = 0
67
-        try:
68
-            while True:
69
-                task = self.queue.get()
70
-                if not task:
71
-                    if not self._running:
72
-                        break
73
-                    continue
74
-                while True:
75
-                    delta = time.time() - last_ts
76
-                    if delta >= self.rate:
77
-                        break
78
-                    time.sleep(self.rate - delta)
79
-                self.log.debug("Manager %s running task %s (queue %s)" %
80
-                               (self.name,
81
-                                _transform_task_name(task.name),
82
-                                self.queue.qsize()))
83
-                self.run_task(task)
84
-                self.queue.task_done()
85
-        except Exception:
86
-            self.log.exception("Task manager died.")
87
-            raise
88
-
89
-    def post_run_task(self, elapsed_time, task):
90
-        task_name = _transform_task_name(task.name)
91
-        self.log.debug(
92
-            "Manager %s ran task %s in %ss" %
93
-            (self.name, task_name, elapsed_time))
94
-
95
-        if self.statsd:
96
-            # nodepool.task.PROVIDER.TASK_NAME
97
-            key = 'nodepool.task.%s.%s' % (self.name, task_name)
98
-            self.statsd.timing(key, int(elapsed_time * 1000))
99
-            self.statsd.incr(key)

+ 0
- 13
nodepool/tests/unit/test_sdk_integration.py View File

@@ -20,23 +20,10 @@ import yaml
20 20
 
21 21
 from nodepool import config as nodepool_config
22 22
 from nodepool import provider_manager
23
-from nodepool import task_manager
24 23
 from nodepool import tests
25 24
 
26 25
 
27 26
 class TestShadeIntegration(tests.IntegrationTestCase):
28
-    def test_task_name_transformation(self):
29
-        t = task_manager._transform_task_name
30
-        self.assertEqual(
31
-            t('compute.DELETE.servers'),
32
-            'ComputeDeleteServers')
33
-        self.assertEqual(
34
-            t('compute.POST.os-volumes_boot'),
35
-            'ComputePostOsVolumesBoot')
36
-        self.assertEqual(
37
-            t('compute.GET.os-availability-zone'),
38
-            'ComputeGetOsAvailabilityZone')
39
-
40 27
     def _cleanup_cloud_config(self):
41 28
         os.remove(self.clouds_path)
42 29
 

+ 15
- 0
releasenotes/notes/task-manager-replaced-12e4b3a0108f9358.yaml View File

@@ -0,0 +1,15 @@
1
+---
2
+upgrade:
3
+  - |
4
+    The ``TaskManager`` used by the OpenStack provider has been removed.
5
+    The ``keystoneauth1`` library underneath ``openstacksdk`` has grown
6
+    support for rate limiting using a ``FairSemaphore`` instead of a pool
7
+    of worker threads. This should reduce the overall thread count.
8
+  - |
9
+    statsd key names have changed. Because of the removal of ``TaskManager``
10
+    statsd calls are being deferred to openstacksdk. Instead of keys of the
11
+    form ``ComputeGetServers``, the openstacksdk keys are of the form
12
+    ``compute.GET.servers``. They will always start with the normalized
13
+    ``service-type``, followed by the HTTP verb, followed by a ``.`` separated
14
+    list of url segments. Any service version, project-id entries in the url
15
+    or ``.json`` suffixes will be removed.

+ 2
- 3
requirements.txt View File

@@ -6,9 +6,8 @@ python-daemon>=2.0.4,<2.1.0
6 6
 extras
7 7
 statsd>=3.0
8 8
 PrettyTable>=0.6,<0.8
9
-# openstacksdk before 0.21.0 has issues with dogpile.cache
10
-# openstacksdk removes taskmanager in 0.27.0
11
-openstacksdk>=0.21.0,<0.27.0
9
+# openstacksdk before 0.27.0 is TaskManager based
10
+openstacksdk>=0.27.0
12 11
 diskimage-builder>=2.0.0
13 12
 voluptuous
14 13
 kazoo

Loading…
Cancel
Save