Browse Source

Merge "web: add /{tenant}/labels route"

tags/3.4.0
Zuul 5 months ago
parent
commit
0fc3485454
5 changed files with 112 additions and 4 deletions
  1. 12
    2
      tests/base.py
  2. 8
    1
      tests/unit/test_web.py
  3. 3
    0
      zuul/cmd/web.py
  4. 23
    1
      zuul/web/__init__.py
  5. 66
    0
      zuul/zk.py

+ 12
- 2
tests/base.py View File

@@ -1736,6 +1736,7 @@ class FakeSMTP(object):
1736 1736
 class FakeNodepool(object):
1737 1737
     REQUEST_ROOT = '/nodepool/requests'
1738 1738
     NODE_ROOT = '/nodepool/nodes'
1739
+    LAUNCHER_ROOT = '/nodepool/launchers'
1739 1740
 
1740 1741
     log = logging.getLogger("zuul.test.FakeNodepool")
1741 1742
 
@@ -1745,6 +1746,7 @@ class FakeNodepool(object):
1745 1746
         self.client = kazoo.client.KazooClient(
1746 1747
             hosts='%s:%s%s' % (host, port, chroot))
1747 1748
         self.client.start()
1749
+        self.registerLauncher()
1748 1750
         self._running = True
1749 1751
         self.paused = False
1750 1752
         self.thread = threading.Thread(target=self.run)
@@ -1783,6 +1785,12 @@ class FakeNodepool(object):
1783 1785
         for req in self.getNodeRequests():
1784 1786
             self.fulfillRequest(req)
1785 1787
 
1788
+    def registerLauncher(self, labels=["label1"], id="FakeLauncher"):
1789
+        path = os.path.join(self.LAUNCHER_ROOT, id)
1790
+        data = {'id': id, 'supported_labels': labels}
1791
+        self.client.create(
1792
+            path, json.dumps(data).encode('utf8'), makepath=True)
1793
+
1786 1794
     def getNodeRequests(self):
1787 1795
         try:
1788 1796
             reqids = self.client.get_children(self.REQUEST_ROOT)
@@ -2054,7 +2062,7 @@ class WebProxyFixture(fixtures.Fixture):
2054 2062
 
2055 2063
 
2056 2064
 class ZuulWebFixture(fixtures.Fixture):
2057
-    def __init__(self, gearman_server_port, config, info=None):
2065
+    def __init__(self, gearman_server_port, config, info=None, zk_hosts=None):
2058 2066
         super(ZuulWebFixture, self).__init__()
2059 2067
         self.gearman_server_port = gearman_server_port
2060 2068
         self.connections = zuul.lib.connections.ConnectionRegistry()
@@ -2066,6 +2074,7 @@ class ZuulWebFixture(fixtures.Fixture):
2066 2074
             self.info = zuul.model.WebInfo()
2067 2075
         else:
2068 2076
             self.info = info
2077
+        self.zk_hosts = zk_hosts
2069 2078
 
2070 2079
     def _setUp(self):
2071 2080
         # Start the web server
@@ -2073,7 +2082,8 @@ class ZuulWebFixture(fixtures.Fixture):
2073 2082
             listen_address='::', listen_port=0,
2074 2083
             gear_server='127.0.0.1', gear_port=self.gearman_server_port,
2075 2084
             info=self.info,
2076
-            connections=self.connections)
2085
+            connections=self.connections,
2086
+            zk_hosts=self.zk_hosts)
2077 2087
         self.web.start()
2078 2088
         self.addCleanup(self.stop)
2079 2089
 

+ 8
- 1
tests/unit/test_web.py View File

@@ -52,7 +52,8 @@ class BaseTestWeb(ZuulTestCase):
52 52
             ZuulWebFixture(
53 53
                 self.gearman_server.port,
54 54
                 self.config,
55
-                info=zuul.model.WebInfo.fromConfig(self.zuul_ini_config)))
55
+                info=zuul.model.WebInfo.fromConfig(self.zuul_ini_config),
56
+                zk_hosts=self.zk_config))
56 57
 
57 58
         self.executor_server.hold_jobs_in_build = True
58 59
 
@@ -372,6 +373,12 @@ class TestWeb(BaseTestWeb):
372 373
                 'voting': True
373 374
             }], data)
374 375
 
376
+    def test_web_labels_list(self):
377
+        # can we fetch the labels list
378
+        data = self.get_url('api/tenant/tenant-one/labels').json()
379
+        expected_list = [{'name': 'label1'}]
380
+        self.assertEqual(expected_list, data)
381
+
375 382
     def test_web_pipeline_list(self):
376 383
         # can we fetch the list of pipelines
377 384
         data = self.get_url('api/tenant/tenant-one/pipelines').json()

+ 3
- 0
zuul/cmd/web.py View File

@@ -64,6 +64,9 @@ class WebServer(zuul.cmd.ZuulDaemonApp):
64 64
                 self.log.exception("Error validating config")
65 65
                 sys.exit(1)
66 66
 
67
+        params["zk_hosts"] = get_default(
68
+            self.config, 'zookeeper', 'hosts', '127.0.0.1:2181')
69
+
67 70
         try:
68 71
             self.web = zuul.web.ZuulWeb(**params)
69 72
         except Exception:

+ 23
- 1
zuul/web/__init__.py View File

@@ -31,6 +31,7 @@ import threading
31 31
 
32 32
 import zuul.model
33 33
 import zuul.rpcclient
34
+import zuul.zk
34 35
 
35 36
 STATIC_DIR = os.path.join(os.path.dirname(__file__), 'static')
36 37
 cherrypy.tools.websocket = WebSocketTool()
@@ -195,6 +196,7 @@ class ZuulWebAPI(object):
195 196
 
196 197
     def __init__(self, zuulweb):
197 198
         self.rpc = zuulweb.rpc
199
+        self.zk = zuulweb.zk
198 200
         self.zuulweb = zuulweb
199 201
         self.cache = {}
200 202
         self.cache_time = {}
@@ -346,6 +348,19 @@ class ZuulWebAPI(object):
346 348
         resp.headers['Access-Control-Allow-Origin'] = '*'
347 349
         return ret
348 350
 
351
+    @cherrypy.expose
352
+    @cherrypy.tools.save_params()
353
+    @cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
354
+    def labels(self, tenant):
355
+        labels = set()
356
+        for launcher in self.zk.getRegisteredLaunchers():
357
+            for label in launcher.supported_labels:
358
+                labels.add(label)
359
+        ret = [{'name': label} for label in sorted(labels)]
360
+        resp = cherrypy.response
361
+        resp.headers['Access-Control-Allow-Origin'] = '*'
362
+        return ret
363
+
349 364
     @cherrypy.expose
350 365
     @cherrypy.tools.save_params()
351 366
     def key(self, tenant, project):
@@ -560,7 +575,8 @@ class ZuulWeb(object):
560 575
                  static_cache_expiry=3600,
561 576
                  connections=None,
562 577
                  info=None,
563
-                 static_path=None):
578
+                 static_path=None,
579
+                 zk_hosts=None):
564 580
         self.start_time = time.time()
565 581
         self.listen_address = listen_address
566 582
         self.listen_port = listen_port
@@ -573,6 +589,9 @@ class ZuulWeb(object):
573 589
         # instanciate handlers
574 590
         self.rpc = zuul.rpcclient.RPCClient(gear_server, gear_port,
575 591
                                             ssl_key, ssl_cert, ssl_ca)
592
+        self.zk = zuul.zk.ZooKeeper()
593
+        if zk_hosts:
594
+            self.zk.connect(hosts=zk_hosts, read_only=True)
576 595
         self.connections = connections
577 596
         self.stream_manager = StreamManager()
578 597
 
@@ -598,6 +617,8 @@ class ZuulWeb(object):
598 617
                           controller=api, action='project')
599 618
         route_map.connect('api', '/api/tenant/{tenant}/pipelines',
600 619
                           controller=api, action='pipelines')
620
+        route_map.connect('api', '/api/tenant/{tenant}/labels',
621
+                          controller=api, action='labels')
601 622
         route_map.connect('api', '/api/tenant/{tenant}/key/{project:.*}.pub',
602 623
                           controller=api, action='key')
603 624
         route_map.connect('api', '/api/tenant/{tenant}/'
@@ -661,6 +682,7 @@ class ZuulWeb(object):
661 682
         cherrypy.server.httpserver = None
662 683
         self.wsplugin.unsubscribe()
663 684
         self.stream_manager.stop()
685
+        self.zk.disconnect()
664 686
 
665 687
 
666 688
 if __name__ == "__main__":

+ 66
- 0
zuul/zk.py View File

@@ -382,3 +382,69 @@ class ZooKeeper(object):
382 382
                     node_data.get('hold_job') == identifier):
383 383
                 count += 1
384 384
         return count
385
+
386
+    # Copy of nodepool/zk.py begins here
387
+    LAUNCHER_ROOT = "/nodepool/launchers"
388
+
389
+    def _bytesToDict(self, data):
390
+        return json.loads(data.decode('utf8'))
391
+
392
+    def _launcherPath(self, launcher):
393
+        return "%s/%s" % (self.LAUNCHER_ROOT, launcher)
394
+
395
+    def getRegisteredLaunchers(self):
396
+        '''
397
+        Get a list of all launchers that have registered with ZooKeeper.
398
+
399
+        :returns: A list of Launcher objects, or empty list if none are found.
400
+        '''
401
+        try:
402
+            launcher_ids = self.client.get_children(self.LAUNCHER_ROOT)
403
+        except kze.NoNodeError:
404
+            return []
405
+
406
+        objs = []
407
+        for launcher in launcher_ids:
408
+            path = self._launcherPath(launcher)
409
+            try:
410
+                data, _ = self.client.get(path)
411
+            except kze.NoNodeError:
412
+                # launcher disappeared
413
+                continue
414
+
415
+            objs.append(Launcher.fromDict(self._bytesToDict(data)))
416
+        return objs
417
+
418
+
419
+class Launcher():
420
+    '''
421
+    Class to describe a nodepool launcher.
422
+    '''
423
+
424
+    def __init__(self):
425
+        self.id = None
426
+        self._supported_labels = set()
427
+
428
+    def __eq__(self, other):
429
+        if isinstance(other, Launcher):
430
+            return (self.id == other.id and
431
+                    self.supported_labels == other.supported_labels)
432
+        else:
433
+            return False
434
+
435
+    @property
436
+    def supported_labels(self):
437
+        return self._supported_labels
438
+
439
+    @supported_labels.setter
440
+    def supported_labels(self, value):
441
+        if not isinstance(value, set):
442
+            raise TypeError("'supported_labels' attribute must be a set")
443
+        self._supported_labels = value
444
+
445
+    @staticmethod
446
+    def fromDict(d):
447
+        obj = Launcher()
448
+        obj.id = d.get('id')
449
+        obj.supported_labels = set(d.get('supported_labels', []))
450
+        return obj

Loading…
Cancel
Save