From 89790013f3b8a5b39e48fd5c9c3f3f3b754590f4 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 27 Feb 2018 13:16:44 +1100 Subject: [PATCH] Consolidate node_list, add generic filter node_list takes an argument "detail" which adds a rather arbitrary list of results to the output. This comes from the command-line, where we're trying to keep width under a certain length; but doesn't make as much sense here (especially for json). For dashboard type applications, replace this with a simple "fields" parameter which, if set, will only return those fields it sees in the common text output function. Note, this purposely doesn't apply to the JSON output, as it expected client-side filtering is more appropriate there. We could also add generic field support to the command-line tools, if considered worthwhile. Add some documentation on all the end-points, and add info about these parameters. Change-Id: Ifbf1019b77368124961e7aa28dae403cabe50de1 --- doc/source/conf.py | 3 +- doc/source/operation.rst | 62 +++++++++++++++++++++++++++++++++++ nodepool/cmd/nodepoolcmd.py | 12 +++++-- nodepool/status.py | 56 +++++++++++++++++++------------ nodepool/tests/test_webapp.py | 21 ++++++++++++ nodepool/webapp.py | 5 ++- test-requirements.txt | 1 + 7 files changed, 135 insertions(+), 25 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 4540b43a8..0eea16a16 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -25,7 +25,8 @@ sys.path.insert(0, os.path.abspath('../..')) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [ 'sphinxcontrib.programoutput' ] +extensions = [ 'sphinxcontrib.programoutput', + 'sphinxcontrib.httpdomain'] #extensions = ['sphinx.ext.intersphinx'] #intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None)} diff --git a/doc/source/operation.rst b/doc/source/operation.rst index bd57c876d..9dcb6a3d3 100644 --- a/doc/source/operation.rst +++ b/doc/source/operation.rst @@ -218,3 +218,65 @@ same and makes it easy to turn the provider back on). If urgency is required you can delete the nodes directly instead of waiting for them to go through their normal lifecycle but the effect is the same. + +Web interface +------------- + +If configured (see :ref:`webapp-conf`), a ``nodepool-launcher`` +instance can provide a range of end-points that can provide +information in text and ``json`` format. Note if there are multiple +launchers, all will provide the same information. + +.. http:get:: /image-list + + The status of uploaded images + + :query fields: comma-separated list of fields to display + :resheader Content-Type: text/plain + +.. http:get:: /image-list.json + + The status of uploaded images + + :resheader Content-Type: application/json + +.. http:get:: /dib-image-list + + The status of images built by ``diskimage-builder`` + + :query fields: comma-separated list of fields to display + :resheader Content-Type: text/plain + +.. http:get:: /dib-image-list.json + + The status of images built by ``diskimage-builder`` + + :resheader Content-Type: application/json + +.. http:get:: /node-list + + The status of currently active nodes + + :query node_id: restrict to a specific node + :query fields: comma-separated list of fields to display + :resheader Content-Type: text/plain + +.. http:get:: /node-list.json + + The status of currently active nodes + + :query node_id: restrict to a specific node + :resheader Content-Type: application/json + +.. http:get:: /request-list + + Outstanding requests + + :query fields: comma-separated list of fields to display + :resheader Content-Type: text/plain + +.. http:get:: /request-list.json + + Outstanding requests + + :resheader Content-Type: application/json diff --git a/nodepool/cmd/nodepoolcmd.py b/nodepool/cmd/nodepoolcmd.py index 5c39e7c6b..76ad00772 100755 --- a/nodepool/cmd/nodepoolcmd.py +++ b/nodepool/cmd/nodepoolcmd.py @@ -156,8 +156,16 @@ class NodePoolCmd(NodepoolApp): def list(self, node_id=None, detail=False): if hasattr(self.args, 'detail'): detail = self.args.detail - results = status.node_list(self.zk, node_id, detail) - print(status.output(results, 'pretty')) + + fields = ['id', 'provider', 'label', 'server_id', + 'public_ipv4', 'ipv6', 'state', 'age', 'locked'] + if detail: + fields.extend(['hostname', 'private_ipv4', 'AZ', + 'connection_port', 'launcher', + 'allocated_to', 'hold_job', + 'comment']) + results = status.node_list(self.zk, node_id) + print(status.output(results, 'pretty', fields)) def dib_image_list(self): results = status.dib_image_list(self.zk) diff --git a/nodepool/status.py b/nodepool/status.py index 949e40605..a074ebbe3 100755 --- a/nodepool/status.py +++ b/nodepool/status.py @@ -53,13 +53,26 @@ def age(timestamp): return '%02d:%02d:%02d:%02d' % (d, h, m, s) -def _to_pretty_table(objs, headers_table): +def _to_pretty_table(objs, headers_table, fields): + '''Construct a pretty table output + + :param objs: list of output objects + :param headers_table: list of (key, desr) header tuples + :param fields: list of fields to show; None means all + + :return str: text output + ''' + if fields: + headers_table = OrderedDict( + [h for h in headers_table.items() if h[0] in fields]) headers = headers_table.values() t = PrettyTable(headers) t.align = 'l' for obj in objs: values = [] for k in headers_table: + if fields and k not in fields: + continue if k == 'age': try: obj_age = age(int(obj[k])) @@ -76,10 +89,18 @@ def _to_pretty_table(objs, headers_table): return t -def output(results, fmt): +def output(results, fmt, fields=None): + '''Generate output for webapp results + + :param results: tuple (objs, headers) as returned by various _list + functions + :param fmt: select from ascii pretty-table or json + :param fields: list of fields to show in pretty-table output + ''' objs, headers_table = results + if fmt == 'pretty': - t = _to_pretty_table(objs, headers_table) + t = _to_pretty_table(objs, headers_table, fields) return str(t) elif fmt == 'json': return json.dumps(objs) @@ -87,7 +108,7 @@ def output(results, fmt): raise ValueError('Unknown format "%s"' % fmt) -def node_list(zk, node_id=None, detail=False): +def node_list(zk, node_id=None): headers_table = [ ("id", "ID"), ("provider", "Provider"), @@ -97,9 +118,7 @@ def node_list(zk, node_id=None, detail=False): ("ipv6", "IPv6"), ("state", "State"), ("age", "Age"), - ("locked", "Locked") - ] - detail_headers_table = [ + ("locked", "Locked"), ("hostname", "Hostname"), ("private_ipv4", "Private IPv4"), ("AZ", "AZ"), @@ -109,8 +128,6 @@ def node_list(zk, node_id=None, detail=False): ("hold_job", "Hold Job"), ("comment", "Comment") ] - if detail: - headers_table += detail_headers_table headers_table = OrderedDict(headers_table) def _get_node_values(node): @@ -131,19 +148,16 @@ def node_list(zk, node_id=None, detail=False): node.public_ipv6, node.state, age(node.state_time), - locked + locked, + node.hostname, + node.private_ipv4, + node.az, + node.connection_port, + node.launcher, + node.allocated_to, + node.hold_job, + node.comment ] - if detail: - values += [ - node.hostname, - node.private_ipv4, - node.az, - node.connection_port, - node.launcher, - node.allocated_to, - node.hold_job, - node.comment - ] return values objs = [] diff --git a/nodepool/tests/test_webapp.py b/nodepool/tests/test_webapp.py index 1c79784d6..cdbe06350 100644 --- a/nodepool/tests/test_webapp.py +++ b/nodepool/tests/test_webapp.py @@ -45,6 +45,27 @@ class TestWebApp(tests.DBTestCase): data = f.read() self.assertTrue('fake-image' in data.decode('utf8')) + def test_image_list_filtered(self): + configfile = self.setup_config('node.yaml') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.useBuilder(configfile) + pool.start() + webapp = self.useWebApp(pool, port=0) + webapp.start() + port = webapp.server.socket.getsockname()[1] + + self.waitForImage('fake-provider', 'fake-image') + self.waitForNodes('fake-label') + + req = request.Request( + "http://localhost:%s/image-list?fields=id,image,state" % port) + f = request.urlopen(req) + self.assertEqual(f.info().get('Content-Type'), + 'text/plain; charset=UTF-8') + data = f.read() + self.assertIn("| 0000000001 | fake-image | ready |", + data.decode('utf8')) + def test_image_list_json(self): configfile = self.setup_config('node.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) diff --git a/nodepool/webapp.py b/nodepool/webapp.py index acf2179c2..8050dc671 100644 --- a/nodepool/webapp.py +++ b/nodepool/webapp.py @@ -105,8 +105,11 @@ class WebApp(threading.Thread): else: return None - output = status.output(results, out_fmt) + fields = None + if params.get('fields'): + fields = params.get('fields').split(',') + output = status.output(results, out_fmt, fields) return self.cache.put(index, output) def app(self, request): diff --git a/test-requirements.txt b/test-requirements.txt index c1a641031..233be08c6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,6 +3,7 @@ hacking>=0.10.2,<0.11 coverage sphinx>=1.5.1,<1.6 sphinxcontrib-programoutput +sphinxcontrib-httpdomain fixtures>=0.3.12 mock>=1.0 python-subunit