diff --git a/tests/base.py b/tests/base.py index cac2102fd7..97af80d57e 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1736,6 +1736,7 @@ class FakeSMTP(object): class FakeNodepool(object): REQUEST_ROOT = '/nodepool/requests' NODE_ROOT = '/nodepool/nodes' + LAUNCHER_ROOT = '/nodepool/launchers' log = logging.getLogger("zuul.test.FakeNodepool") @@ -1745,6 +1746,7 @@ class FakeNodepool(object): self.client = kazoo.client.KazooClient( hosts='%s:%s%s' % (host, port, chroot)) self.client.start() + self.registerLauncher() self._running = True self.paused = False self.thread = threading.Thread(target=self.run) @@ -1783,6 +1785,12 @@ class FakeNodepool(object): for req in self.getNodeRequests(): self.fulfillRequest(req) + def registerLauncher(self, labels=["label1"], id="FakeLauncher"): + path = os.path.join(self.LAUNCHER_ROOT, id) + data = {'id': id, 'supported_labels': labels} + self.client.create( + path, json.dumps(data).encode('utf8'), makepath=True) + def getNodeRequests(self): try: reqids = self.client.get_children(self.REQUEST_ROOT) @@ -2054,7 +2062,7 @@ class WebProxyFixture(fixtures.Fixture): class ZuulWebFixture(fixtures.Fixture): - def __init__(self, gearman_server_port, config, info=None): + def __init__(self, gearman_server_port, config, info=None, zk_hosts=None): super(ZuulWebFixture, self).__init__() self.gearman_server_port = gearman_server_port self.connections = zuul.lib.connections.ConnectionRegistry() @@ -2066,6 +2074,7 @@ class ZuulWebFixture(fixtures.Fixture): self.info = zuul.model.WebInfo() else: self.info = info + self.zk_hosts = zk_hosts def _setUp(self): # Start the web server @@ -2073,7 +2082,8 @@ class ZuulWebFixture(fixtures.Fixture): listen_address='::', listen_port=0, gear_server='127.0.0.1', gear_port=self.gearman_server_port, info=self.info, - connections=self.connections) + connections=self.connections, + zk_hosts=self.zk_hosts) self.web.start() self.addCleanup(self.stop) diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py index 31b2d9b4cf..063cf8d156 100755 --- a/tests/unit/test_web.py +++ b/tests/unit/test_web.py @@ -52,7 +52,8 @@ class BaseTestWeb(ZuulTestCase): ZuulWebFixture( self.gearman_server.port, self.config, - info=zuul.model.WebInfo.fromConfig(self.zuul_ini_config))) + info=zuul.model.WebInfo.fromConfig(self.zuul_ini_config), + zk_hosts=self.zk_config)) self.executor_server.hold_jobs_in_build = True @@ -372,6 +373,12 @@ class TestWeb(BaseTestWeb): 'voting': True }], data) + def test_web_labels_list(self): + # can we fetch the labels list + data = self.get_url('api/tenant/tenant-one/labels').json() + expected_list = [{'name': 'label1'}] + self.assertEqual(expected_list, data) + def test_web_pipeline_list(self): # can we fetch the list of pipelines data = self.get_url('api/tenant/tenant-one/pipelines').json() diff --git a/zuul/cmd/web.py b/zuul/cmd/web.py index 89c910f90a..c77a5db430 100755 --- a/zuul/cmd/web.py +++ b/zuul/cmd/web.py @@ -64,6 +64,9 @@ class WebServer(zuul.cmd.ZuulDaemonApp): self.log.exception("Error validating config") sys.exit(1) + params["zk_hosts"] = get_default( + self.config, 'zookeeper', 'hosts', '127.0.0.1:2181') + try: self.web = zuul.web.ZuulWeb(**params) except Exception: diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py index 1b49e8fdbe..ff3154330d 100755 --- a/zuul/web/__init__.py +++ b/zuul/web/__init__.py @@ -31,6 +31,7 @@ import threading import zuul.model import zuul.rpcclient +import zuul.zk STATIC_DIR = os.path.join(os.path.dirname(__file__), 'static') cherrypy.tools.websocket = WebSocketTool() @@ -195,6 +196,7 @@ class ZuulWebAPI(object): def __init__(self, zuulweb): self.rpc = zuulweb.rpc + self.zk = zuulweb.zk self.zuulweb = zuulweb self.cache = {} self.cache_time = {} @@ -346,6 +348,19 @@ class ZuulWebAPI(object): resp.headers['Access-Control-Allow-Origin'] = '*' return ret + @cherrypy.expose + @cherrypy.tools.save_params() + @cherrypy.tools.json_out(content_type='application/json; charset=utf-8') + def labels(self, tenant): + labels = set() + for launcher in self.zk.getRegisteredLaunchers(): + for label in launcher.supported_labels: + labels.add(label) + ret = [{'name': label} for label in sorted(labels)] + resp = cherrypy.response + resp.headers['Access-Control-Allow-Origin'] = '*' + return ret + @cherrypy.expose @cherrypy.tools.save_params() def key(self, tenant, project): @@ -560,7 +575,8 @@ class ZuulWeb(object): static_cache_expiry=3600, connections=None, info=None, - static_path=None): + static_path=None, + zk_hosts=None): self.start_time = time.time() self.listen_address = listen_address self.listen_port = listen_port @@ -573,6 +589,9 @@ class ZuulWeb(object): # instanciate handlers self.rpc = zuul.rpcclient.RPCClient(gear_server, gear_port, ssl_key, ssl_cert, ssl_ca) + self.zk = zuul.zk.ZooKeeper() + if zk_hosts: + self.zk.connect(hosts=zk_hosts, read_only=True) self.connections = connections self.stream_manager = StreamManager() @@ -598,6 +617,8 @@ class ZuulWeb(object): controller=api, action='project') route_map.connect('api', '/api/tenant/{tenant}/pipelines', controller=api, action='pipelines') + route_map.connect('api', '/api/tenant/{tenant}/labels', + controller=api, action='labels') route_map.connect('api', '/api/tenant/{tenant}/key/{project:.*}.pub', controller=api, action='key') route_map.connect('api', '/api/tenant/{tenant}/' @@ -661,6 +682,7 @@ class ZuulWeb(object): cherrypy.server.httpserver = None self.wsplugin.unsubscribe() self.stream_manager.stop() + self.zk.disconnect() if __name__ == "__main__": diff --git a/zuul/zk.py b/zuul/zk.py index bd476c095a..0ccc1f6210 100644 --- a/zuul/zk.py +++ b/zuul/zk.py @@ -382,3 +382,69 @@ class ZooKeeper(object): node_data.get('hold_job') == identifier): count += 1 return count + + # Copy of nodepool/zk.py begins here + LAUNCHER_ROOT = "/nodepool/launchers" + + def _bytesToDict(self, data): + return json.loads(data.decode('utf8')) + + def _launcherPath(self, launcher): + return "%s/%s" % (self.LAUNCHER_ROOT, launcher) + + def getRegisteredLaunchers(self): + ''' + Get a list of all launchers that have registered with ZooKeeper. + + :returns: A list of Launcher objects, or empty list if none are found. + ''' + try: + launcher_ids = self.client.get_children(self.LAUNCHER_ROOT) + except kze.NoNodeError: + return [] + + objs = [] + for launcher in launcher_ids: + path = self._launcherPath(launcher) + try: + data, _ = self.client.get(path) + except kze.NoNodeError: + # launcher disappeared + continue + + objs.append(Launcher.fromDict(self._bytesToDict(data))) + return objs + + +class Launcher(): + ''' + Class to describe a nodepool launcher. + ''' + + def __init__(self): + self.id = None + self._supported_labels = set() + + def __eq__(self, other): + if isinstance(other, Launcher): + return (self.id == other.id and + self.supported_labels == other.supported_labels) + else: + return False + + @property + def supported_labels(self): + return self._supported_labels + + @supported_labels.setter + def supported_labels(self, value): + if not isinstance(value, set): + raise TypeError("'supported_labels' attribute must be a set") + self._supported_labels = value + + @staticmethod + def fromDict(d): + obj = Launcher() + obj.id = d.get('id') + obj.supported_labels = set(d.get('supported_labels', [])) + return obj