Reorganize REST API and dashboard urls

The existing structure with the API and HTML interleaved makes it hard
to have apache do html offloading for whitelabeled deploys (like
openstack's) where the api and web are served from the same host.
Similarly, the tenant selector in the url for the html content being bare
and not prefixed by anything makes it harder to pull routing information
javascript-side.

Introduce an 'api' prefix to the REST API calls so that we can apply
rewrite rules differently for things starting with /api than things that
don't. Add the word 'tenant' before each tenant subpath.

Also add a '/t/' to the url for the html, so that we have anchors for
routing regexes but the urls don't get too long and unweildy.

Finally, also add /key as a prefix to the key route for similar reasons.

Change-Id: I0cbbf6f1958e91b5910738da9b4fb4c563d48dd4

doc/source/admin/installation.rst

@@ -86,8 +86,11 @@ White Label
 
 Static Offload
   Shift the duties of serving static files, such as HTML, Javascript, CSS or
-  images either to the Reverse Proxy server or to a completely separate
+  images to the Reverse Proxy server.
-  location such as a Swift Object Store or a CDN-enabled static web server.
+
+Static External
+  Serve the static files from a completely separate location that does not
+  support programmatic rewrite rules such as a Swift Object Store.
 
 Sub-URL
   Serve a Zuul dashboard from a location below the root URL as part of
@@ -107,4 +110,64 @@ Using Apache as the Reverse Proxy requires the ``mod_proxy``,
 ``mod_proxy_http`` and ``mod_proxy_wstunnel`` modules to be installed and
 enabled. Static Offload and White Label additionally require ``mod_rewrite``.
 
-.. TODO(mordred): Fill in specifics for all three methods
+All of the cases require a rewrite rule for the websocket streaming, so the
+simplest reverse-proxy case is::
+
+  RewriteEngine on
+  RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P]
+  RewriteRule ^/(.*)$ http://localhost:9000/$1 [P]
+
+
+Static Offload
+--------------
+
+To have the Reverse Proxy serve the static html/javscript assets instead of
+proxying them to the REST layer, register the location where you unpacked
+the web application as the document root and add a simple rewrite rule::
+
+  DocumentRoot /var/lib/html
+  <Directory /var/lib/html>
+    Require all granted
+  </Directory>
+  RewriteEngine on
+  RewriteRule ^/t/.*/(.*)$ /$1 [L]
+  RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P]
+  RewriteRule ^/api/(.*)$ http://localhost:9000/api/$1 [P]
+
+White Labeled Tenant
+--------------------
+
+Running a white-labeled tenant is similar to the offload case, but adds a
+rule to ensure connection webhooks don't try to get put into the tenant scope.
+
+.. note::
+
+  It's possible to do white-labelling without static offload, but it is more
+  complex with no benefit.
+
+Assuming the zuul tenant name is "example", the rewrite rules are::
+
+  DocumentRoot /var/lib/html
+  <Directory /var/lib/html>
+    Require all granted
+  </Directory>
+  RewriteEngine on
+  RewriteRule ^/api/connection/(.*)$ http://localhost:9000/api/connection/$1 [P]
+  RewriteRule ^/api/console-stream ws://localhost:9000/api/tenant/example/console-stream [P]
+  RewriteRule ^/api/(.*)$ http://localhost:9000/api/tenant/example/$1 [P]
+
+Static External
+---------------
+
+.. note::
+
+  Hosting zuul dashboard on an external static location that does not support
+  dynamic url rewrite rules only works for white-labeled deployments.
+
+In order to serve the zuul dashboard code from an external static location,
+``ZUUL_API_URL`` must be set at javascript build time by passing the
+``--define`` flag to the ``npm build:dist`` command.
+
+.. code-block:: bash
+
+  npm build:dist -- --define "ZUUL_API_URL='http://zuul-web.example.com'"

tests/base.py

@@ -1031,7 +1031,7 @@ class FakeGithubConnection(githubconnection.GithubConnection):
 
         if use_zuulweb:
             return requests.post(
-                'http://127.0.0.1:%s/connection/%s/payload'
+                'http://127.0.0.1:%s/api/connection/%s/payload'
                 % (self.zuul_web_port, self.connection_name),
                 json=data, headers=headers)
         else:
@@ -1863,7 +1863,8 @@ class ZuulWebFixture(fixtures.Fixture):
         # Start the web server
         self.web = zuul.web.ZuulWeb(
             listen_address='127.0.0.1', listen_port=0,
-            gear_server='127.0.0.1', gear_port=self.gearman_server_port)
+            gear_server='127.0.0.1', gear_port=self.gearman_server_port,
+            info=zuul.model.WebInfo())
         loop = asyncio.new_event_loop()
         loop.set_debug(True)
         ws_thread = threading.Thread(target=self.web.run, args=(loop,))

tests/unit/test_streaming.py

@@ -170,7 +170,7 @@ class TestStreaming(tests.base.AnsibleZuulTestCase):
 
     def runWSClient(self, build_uuid, event):
         async def client(loop, build_uuid, event):
-            uri = 'http://[::1]:9000/tenant-one/console-stream'
+            uri = 'http://[::1]:9000/api/tenant/tenant-one/console-stream'
             try:
                 session = aiohttp.ClientSession(loop=loop)
                 async with session.ws_connect(uri) as ws:

tests/unit/test_web.py

@@ -114,7 +114,7 @@ class TestWeb(BaseTestWeb):
         self.executor_server.release('project-merge')
         self.waitUntilSettled()
 
-        resp = self.get_url("tenant-one/status")
+        resp = self.get_url("api/tenant/tenant-one/status")
         self.assertIn('Content-Length', resp.headers)
         self.assertIn('Content-Type', resp.headers)
         self.assertEqual(
@@ -211,7 +211,7 @@ class TestWeb(BaseTestWeb):
         self.executor_server.release('project-merge')
         self.waitUntilSettled()
 
-        resp = self.get_url("tenants")
+        resp = self.get_url("api/tenants")
         self.assertIn('Content-Length', resp.headers)
         self.assertIn('Content-Type', resp.headers)
         self.assertEqual(
@@ -230,7 +230,7 @@ class TestWeb(BaseTestWeb):
         self.executor_server.release()
         self.waitUntilSettled()
 
-        data = self.get_url("tenants").json()
+        data = self.get_url("api/tenants").json()
         self.assertEqual('tenant-one', data[0]['name'])
         self.assertEqual(3, data[0]['projects'])
         self.assertEqual(0, data[0]['queue'])
@@ -242,12 +242,12 @@ class TestWeb(BaseTestWeb):
 
     def test_web_find_change(self):
         # can we filter by change id
-        data = self.get_url("tenant-one/status/change/1,1").json()
+        data = self.get_url("api/tenant/tenant-one/status/change/1,1").json()
 
         self.assertEqual(1, len(data), data)
         self.assertEqual("org/project", data[0]['project'])
 
-        data = self.get_url("tenant-one/status/change/2,1").json()
+        data = self.get_url("api/tenant/tenant-one/status/change/2,1").json()
 
         self.assertEqual(1, len(data), data)
         self.assertEqual("org/project1", data[0]['project'], data)
@@ -256,11 +256,11 @@ class TestWeb(BaseTestWeb):
         with open(os.path.join(FIXTURE_DIR, 'public.pem'), 'rb') as f:
             public_pem = f.read()
 
-        resp = self.get_url("tenant-one/org/project.pub")
+        resp = self.get_url("api/tenant/tenant-one/key/org/project.pub")
         self.assertEqual(resp.content, public_pem)
 
     def test_web_404_on_unknown_tenant(self):
-        resp = self.get_url("non-tenant/status")
+        resp = self.get_url("api/tenant/non-tenant/status")
         self.assertEqual(404, resp.status_code)
 
 
@@ -275,7 +275,7 @@ class TestInfo(BaseTestWeb):
         self.stats_prefix = statsd_config.get('prefix')
 
     def test_info(self):
-        info = self.get_url("info").json()
+        info = self.get_url("api/info").json()
         self.assertEqual(
             info, {
                 "info": {
@@ -293,7 +293,7 @@ class TestInfo(BaseTestWeb):
             })
 
     def test_tenant_info(self):
-        info = self.get_url("tenant-one/info").json()
+        info = self.get_url("api/tenant/tenant-one/info").json()
         self.assertEqual(
             info, {
                 "info": {

tests/unit/test_web_urls.py

@@ -14,6 +14,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import json
 import urllib
 
 from bs4 import BeautifulSoup
@@ -66,7 +67,7 @@ class TestDirect(TestWebURLs, ZuulTestCase):
         self.port = self.web.port
 
     def test_status_page(self):
-        self._crawl('/tenant-one/status.html')
+        self._crawl('/t/tenant-one/status.html')
 
 
 class TestWhiteLabel(TestWebURLs, ZuulTestCase):
@@ -75,8 +76,7 @@ class TestWhiteLabel(TestWebURLs, ZuulTestCase):
     def setUp(self):
         super(TestWhiteLabel, self).setUp()
         rules = [
-            ('^/(.*)$', 'http://localhost:{}/tenant-one/\\1'.format(
+            ('^/(.*)$', 'http://localhost:{}/\\1'.format(self.web.port)),
-                self.web.port)),
         ]
         self.proxy = self.useFixture(WebProxyFixture(rules))
         self.port = self.proxy.port
@@ -85,6 +85,24 @@ class TestWhiteLabel(TestWebURLs, ZuulTestCase):
         self._crawl('/status.html')
 
 
+class TestWhiteLabelAPI(TestWebURLs, ZuulTestCase):
+    # Test a zuul-web behind a whitelabel proxy (i.e., what
+    # zuul.openstack.org does).
+    def setUp(self):
+        super(TestWhiteLabelAPI, self).setUp()
+        rules = [
+            ('^/api/(.*)$',
+             'http://localhost:{}/api/tenant/tenant-one/\\1'.format(
+                 self.web.port)),
+        ]
+        self.proxy = self.useFixture(WebProxyFixture(rules))
+        self.port = self.proxy.port
+
+    def test_info(self):
+        info = json.loads(self._get(self.port, '/api/info').decode('utf-8'))
+        self.assertEqual('tenant-one', info['info']['tenant'])
+
+
 class TestSuburl(TestWebURLs, ZuulTestCase):
     # Test a zuul-web mounted on a suburl (i.e., what software factory
     # does).
@@ -98,4 +116,4 @@ class TestSuburl(TestWebURLs, ZuulTestCase):
         self.port = self.proxy.port
 
     def test_status_page(self):
-        self._crawl('/zuul3/tenant-one/status.html')
+        self._crawl('/zuul3/t/tenant-one/status.html')

tools/encrypt_secret.py

@@ -70,7 +70,7 @@ def main():
                          "unencrypted connection. Your secret may get "
                          "compromised.\n")
 
-    req = Request("%s/%s.pub" % (args.url.rstrip('/'), args.project))
+    req = Request("%s/key/%s.pub" % (args.url.rstrip('/'), args.project))
     pubkey = urlopen(req)
 
     if args.infile:

web/dashboard.js

@@ -29,7 +29,7 @@ angular.module('zuulTenants', []).controller(
     'mainController', function ($scope, $http, $location) {
       $scope.tenants = undefined
       $scope.tenants_fetch = function () {
-        $http.get(getSourceUrl('tenants', $location))
+        $http.get(getSourceUrl('api/tenants', $location))
             .then(function success (result) {
               $scope.tenants = result.data
             })

web/status.js

@@ -36,7 +36,7 @@ import { getSourceUrl } from './util'
 /**
  * @return The $.zuul instance
  */
-function zuulStart ($) {
+function zuulStart ($, $location) {
   // Start the zuul app (expects default dom)
 
   let $container, $indicator
@@ -60,7 +60,7 @@ function zuulStart ($) {
       params['source_data'] = DemoStatusTree
     }
   } else {
-    params['source'] = getSourceUrl('status')
+    params['source'] = getSourceUrl('status', $location)
   }
 
   let zuul = $.zuul(params)
@@ -120,7 +120,7 @@ if (module.hot) {
 }
 
 angular.module('zuulStatus', []).controller(
-  'mainController', function ($scope, $http) {
+  'mainController', function ($scope, $http, $location) {
-    zuulStart(jQuery)
+    zuulStart(jQuery, $location)
   }
 )

web/stream.js

@@ -24,6 +24,8 @@
 import angular from 'angular'
 import './styles/stream.css'
 
+import { getSourceUrl } from './util'
+
 function escapeLog (text) {
   const pattern = /[<>&"']/g
 
@@ -32,7 +34,7 @@ function escapeLog (text) {
   })
 }
 
-function zuulStartStream () {
+function zuulStartStream ($location) {
   let pageUpdateInMS = 250
   let receiveBuffer = ''
 
@@ -71,7 +73,7 @@ function zuulStartStream () {
     } else {
       protocol = 'ws://'
     }
-    let path = url['pathname'].replace(/stream.html.*$/g, '') + 'console-stream'
+    let path = getSourceUrl('console-stream', $location)
     params['websocket_url'] = protocol + url['host'] + path
   }
   let ws = new WebSocket(params['websocket_url'])
@@ -93,7 +95,7 @@ function zuulStartStream () {
 }
 
 angular.module('zuulStream', []).controller(
-  'mainController', function ($scope, $http) {
+  'mainController', function ($scope, $http, $location) {
-    window.onload = zuulStartStream()
+    window.onload = zuulStartStream($location)
   }
 )

web/templates/builds.ejs

@@ -23,7 +23,7 @@ under the License.
   <nav class="navbar navbar-default">
   <div class="container-fluid">
     <div class="navbar-header">
-      <a class="navbar-brand" href="../" target="_self">Zuul Dashboard</a>
+      <a class="navbar-brand" href="../../" target="_self">Zuul Dashboard</a>
     </div>
     <ul class="nav navbar-nav">
       <li><a href="status.html" target="_self">Status</a></li>

web/templates/jobs.ejs

@@ -23,7 +23,7 @@ under the License.
   <nav class="navbar navbar-default">
   <div class="container-fluid">
     <div class="navbar-header">
-      <a class="navbar-brand" href="../" target="_self">Zuul Dashboard</a>
+      <a class="navbar-brand" href="../../" target="_self">Zuul Dashboard</a>
     </div>
     <ul class="nav navbar-nav">
       <li><a href="status.html" target="_self">Status</a></li>

web/templates/status.ejs

@@ -23,7 +23,7 @@ under the License.
   <nav class="navbar navbar-default">
       <div class="container-fluid">
           <div class="navbar-header">
-              <a class="navbar-brand" href="../" target="_self">Zuul Dashboard</a>
+              <a class="navbar-brand" href="../../" target="_self">Zuul Dashboard</a>
           </div>
           <ul class="nav navbar-nav">
               <li class="active"><a href="status.html" target="_self">Status</a></li>

web/templates/tenants.ejs

@@ -44,9 +44,9 @@ under the License.
     <tbody>
       <tr ng-repeat="tenant in tenants">
         <td>{{ tenant.name }}</td>
-        <td><a href="{{ tenant.name }}/status.html">status</a></td>
+        <td><a href="t/{{ tenant.name }}/status.html">status</a></td>
-        <td><a href="{{ tenant.name }}/jobs.html">jobs</a></td>
+        <td><a href="t/{{ tenant.name }}/jobs.html">jobs</a></td>
-        <td><a href="{{ tenant.name }}/builds.html">builds</a></td>
+        <td><a href="t/{{ tenant.name }}/builds.html">builds</a></td>
         <td>{{ tenant.projects }}</td>
         <td>{{ tenant.queue }}</td>
       </tr>

web/util.js

@@ -19,12 +19,33 @@
 // @licend  The above is the entire license notice
 // for the JavaScript code in this page.
 
+// TODO(mordred) This is a temporary hack until we're on @angular/router
+function extractTenant (url) {
+  if (url.includes('/t/')) {
+    // This is a multi-tenant deploy, find the tenant
+    const tenantStart = url.lastIndexOf('/t/') + 3
+    const tenantEnd = url.indexOf('/', tenantStart)
+    return url.slice(tenantStart, tenantEnd)
+  } else {
+    return null
+  }
+}
+
 // TODO(mordred) This should be encapsulated in an Angular Service singleton
 // that fetches the other things from the info endpoint.
 export function getSourceUrl (filename, $location) {
   if (typeof ZUUL_API_URL !== 'undefined') {
-    return ZUUL_API_URL + '/' + filename
+    return `${ZUUL_API_URL}/api/${filename}`
   } else {
-    return filename
+    let tenant = extractTenant($location.url())
+    if (tenant) {
+      // Multi-tenant deploy. This is at t/a-tenant/x.html. api path is at
+      // api/tenant/a-tenant/x, so should be at ../../api/tenant/a-tenant/x
+      return `../../api/tenant/${tenant}/${filename}`
+    } else {
+      // Whilelabel deploy. This is at x.html. api path is at
+      // api/x, so should be at api/x
+      return `api/${filename}`
+    }
   }
 }

zuul/driver/sql/sqlconnection.py

@@ -26,7 +26,7 @@ import voluptuous
 
 from zuul.connection import BaseConnection
 from zuul.lib.config import get_default
-from zuul.web.handler import BaseWebHandler, StaticHandler
+from zuul.web.handler import BaseTenantWebHandler
 
 BUILDSET_TABLE = 'zuul_buildset'
 BUILD_TABLE = 'zuul_build'
@@ -129,8 +129,7 @@ class SQLConnection(BaseConnection):
     def getWebHandlers(self, zuul_web, info):
         info.capabilities.job_history = True
         return [
-            SqlWebHandler(self, zuul_web, 'GET', '/{tenant}/builds'),
+            SqlWebHandler(self, zuul_web, 'GET', 'builds'),
-            StaticHandler(zuul_web, '/{tenant}/builds.html'),
         ]
 
     def validateWebConfig(self, config, connections):
@@ -155,7 +154,7 @@ class SQLConnection(BaseConnection):
             return True
 
 
-class SqlWebHandler(BaseWebHandler):
+class SqlWebHandler(BaseTenantWebHandler):
     log = logging.getLogger("zuul.web.SqlHandler")
     filters = ("project", "pipeline", "change", "branch", "patchset", "ref",
                "result", "uuid", "job_name", "voting", "node_name", "newrev")

zuul/web/__init__.py

@@ -320,19 +320,21 @@ class ZuulWeb(object):
             is run within a separate (non-main) thread.
         """
         routes = [
-            ('GET', '/info', self._handleRootInfo),
+            ('GET', '/api/info', self._handleRootInfo),
-            ('GET', '/{tenant}/info', self._handleTenantInfo),
+            ('GET', '/api/tenants', self._handleTenantsRequest),
-            ('GET', '/tenants', self._handleTenantsRequest),
+            ('GET', '/api/tenant/{tenant}/info', self._handleTenantInfo),
-            ('GET', '/{tenant}/status', self._handleStatusRequest),
+            ('GET', '/api/tenant/{tenant}/status', self._handleStatusRequest),
-            ('GET', '/{tenant}/jobs', self._handleJobsRequest),
+            ('GET', '/api/tenant/{tenant}/jobs', self._handleJobsRequest),
-            ('GET', '/{tenant}/status/change/{change}',
+            ('GET', '/api/tenant/{tenant}/status/change/{change}',
              self._handleStatusChangeRequest),
-            ('GET', '/{tenant}/console-stream', self._handleWebsocket),
+            ('GET', '/api/tenant/{tenant}/console-stream',
-            ('GET', '/{tenant}/{project:.*}.pub', self._handleKeyRequest),
+             self._handleWebsocket),
+            ('GET', '/api/tenant/{tenant}/key/{project:.*}.pub',
+             self._handleKeyRequest),
         ]
 
         static_routes = [
-            StaticHandler(self, '/{tenant}/', 'status.html'),
+            StaticHandler(self, '/t/{tenant}/', 'status.html'),
             StaticHandler(self, '/', 'tenants.html'),
         ]
 
@@ -340,7 +342,7 @@ class ZuulWeb(object):
             routes.append((route.method, route.path, route.handleRequest))
 
         # Add fallthrough routes at the end for the static html/js files
-        routes.append(('GET', '/{tenant}/{path:.*}', self._handleStatic))
+        routes.append(('GET', '/t/{tenant}/{path:.*}', self._handleStatic))
         routes.append(('GET', '/{path:.*}', self._handleStatic))
 
         self.log.debug("ZuulWeb starting")

zuul/web/handler.py

@@ -31,6 +31,13 @@ class BaseWebHandler(object, metaclass=abc.ABCMeta):
         """Process a web request."""
 
 
+class BaseTenantWebHandler(BaseWebHandler):
+
+    def __init__(self, connection, zuul_web, method, path):
+        super(BaseTenantWebHandler, self).__init__(
+            connection, zuul_web, method, '/api/tenant/{tenant}/' + path)
+
+
 class BaseDriverWebHandler(BaseWebHandler):
 
     def __init__(self, connection, zuul_web, method, path):
@@ -38,7 +45,7 @@ class BaseDriverWebHandler(BaseWebHandler):
             connection=connection, zuul_web=zuul_web, method=method, path=path)
         if path.startswith('/'):
             path = path[1:]
-        self.path = '/connection/{connection}/{path}'.format(
+        self.path = '/api/connection/{connection}/{path}'.format(
             connection=self.connection.connection_name,
             path=path)