Scripts for UI plugins development using mitmproxy

Starts a local proxy that rewrites prod server responses, adding new
plugins, serving local files, enabling or disabling assets bundles.

Prerequisite: Docker
Target platform: OS X

Change-Id: I9abd94c816f987bf43e4335aff5be7ad17dd0fde

contrib/mitm-ui/README.md

@@ -0,0 +1,47 @@
+# Scripts for PolyGerrit local development against prod using MitmProxy.
+
+## Installation (OSX)
+
+1. Install Docker from http://docker.com
+2. Start the proxy and create a new proxied browser instance
+   ```
+   cd ~/gerrit
+   ~/mitm-gerrit/mitm-serve-app-dev.sh
+   ```
+3. Install MITM certificates
+   - Open http://mitm.it in the proxied browser window
+   - Follow the instructions to install MITM certs
+
+## Usage
+
+### Add or replace a single plugin containing static content
+
+To develop unminified plugin that loads multiple files, use this.
+
+1. Create a new proxied browser window and start mitmproxy via Docker:
+   ```
+   ~/mitm-gerrit/mitm-single-plugin.sh ./path/to/static/plugin.html
+   ```
+2. Open any *.googlesource.com domain in proxied window
+3. plugin.html and ./path/to/static/* will be served
+
+### Add or replace a minified plugin for *.googlesource.com
+
+This flow assumes no additional .html/.js are needed, i.e. the plugin is a single file.
+
+1. Create a new proxied browser window and start mitmproxy via Docker:
+   ```
+   ~/mitm-gerrit/mitm-plugins.sh ./path/to/plugin.html,./maybe/one/more.js
+   ```
+2. Open any *.googlesource.com domain in proxied window
+3. plugin.html and more.js are served
+
+### Serve uncompiled PolyGerrit
+
+1. Create a new proxied browser window and start mitmproxy via Docker:
+   ```
+   cd ~/gerrit
+   ~/mitm-gerrit/mitm-serve-app-dev.sh
+   ```
+2. Open any *.googlesource.com domain in proxied window
+3. Instead of prod UI (gr-app.html, gr-app.js), local source files will be served

contrib/mitm-ui/add-header.py

@@ -0,0 +1,5 @@
+# mitmdump -s add-header.py
+def response(flow):
+    if flow.request.host == 'gerrit-review.googlesource.com' and flow.request.path == "/c/92000?1":
+        #flow.response.headers['any'] = '<meta.rdf>; rel=meta'
+        flow.response.headers['Link'] = '</changes/98000/detail?O=11640c>;rel="preload";crossorigin;'

contrib/mitm-ui/dev-chrome.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+if [[ "$OSTYPE" != "darwin"* ]]; then
+    echo Only works on OSX.
+    exit 1
+fi
+
+/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --remote-debugging-port=9222 --user-data-dir=${HOME}/devchrome --proxy-server="127.0.0.1:8888"

contrib/mitm-ui/force-version.py

@@ -0,0 +1,22 @@
+# mitmdump -q -p 8888 -s "force-version.py --version $1"
+# Request URL is not changed, only the response context
+from mitmproxy import http
+import argparse
+import re
+
+class Server:
+    def __init__(self, version):
+        self.version = version
+
+    def request(self, flow: http.HTTPFlow) -> None:
+        if "gr-app." in flow.request.pretty_url:
+            flow.request.url = re.sub(
+                r"polygerrit_ui/([\d.]+)/elements",
+                "polygerrit_ui/" + self.version + "/elements",
+                flow.request.url)
+
+def start():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--version", type=str, help="Rapid release version, e.g. 432.0")
+    args = parser.parse_args()
+    return Server(args.version)

contrib/mitm-ui/mitm-docker.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+
+extra_volume='/tmp:/tmp'
+
+POSITIONAL=()
+while [[ $# -gt 0 ]]
+do
+key="$1"
+
+case $key in
+    -v|--volume)
+    extra_volume="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    *)    # unknown option
+    POSITIONAL+=("$1") # save it in an array for later
+    shift # past argument
+    ;;
+esac
+done
+set -- "${POSITIONAL[@]}" # restore positional parameters
+
+if [[ -z "$1" ]]; then
+    echo This is a runner for higher-level scripts, e.g. mitm-serve-app-dev.sh
+    echo Alternatively, pass mitmproxy script from the same dir as a parameter, e.g. serve-app-dev.py
+    exit 1
+fi
+
+gerrit_dir=$(pwd)
+mitm_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+CMD="${mitm_dir}/$1"
+
+docker run --rm -it \
+       -v ~/.mitmproxy:/home/mitmproxy/.mitmproxy \
+       -v ${mitm_dir}:${mitm_dir} \
+       -v ${gerrit_dir}:${gerrit_dir} \
+       -v ${extra_volume} \
+       -p 8888:8888 \
+       mitmproxy/mitmproxy:2.0.2 \
+       mitmdump -q -p 8888 -s "${CMD}"

contrib/mitm-ui/mitm-plugins.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+if [[ -z "$1" ]]; then
+    echo This script injects plugins for *.googlesource.com.
+    echo Provide plugin paths, comma-separated, as a parameter.
+    echo This script assumes files do not have dependencies, i.e. minified.
+    exit 1
+fi
+
+realpath() {
+    [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"
+}
+
+join () {
+  local IFS="$1"
+  shift
+  echo "$*"
+}
+
+plugins=$1
+plugin_paths=()
+for plugin in $(echo ${plugins} | sed "s/,/ /g")
+do
+    plugin_paths+=($(realpath ${plugin}))
+done
+
+absolute_plugin_paths=$(join , "${plugin_paths[@]}")
+
+mitm_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+${mitm_dir}/dev-chrome.sh &
+
+${mitm_dir}/mitm-docker.sh "serve-app-dev.py --plugins ${absolute_plugin_paths} --strip_assets"

contrib/mitm-ui/mitm-serve-app-dev.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+workspace="./WORKSPACE"
+if [[ ! -f ${workspace} ]] || [[ ! $(head -n 1 ${workspace}) == *"gerrit"* ]]; then
+    echo Please change to cloned Gerrit repo from https://gerrit.googlesource.com/gerrit/
+    exit 1
+fi
+
+mitm_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+${mitm_dir}/dev-chrome.sh &
+
+${mitm_dir}/mitm-docker.sh "serve-app-dev.py --app $(pwd)/polygerrit-ui/app/"

contrib/mitm-ui/mitm-single-plugin.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+if [[ -z "$1" ]]; then
+    echo This script serves one plugin with the rest of static content.
+    echo Provide path to index plugin file, e.g. buildbucket.html for buildbucket plugin
+    exit 1
+fi
+
+realpath() {
+  OURPWD=$PWD
+  cd "$(dirname "$1")"
+  LINK=$(basename "$1")
+  while [ -L "$LINK" ]; do
+      LINK=$(readlink "$LINK")
+      cd "$(dirname "$LINK")"
+      LINK="$(basename "$1")"
+  done
+  REAL_DIR=`pwd -P`
+  RESULT=$REAL_DIR/$LINK
+  cd "$OURPWD"
+  echo "$RESULT"
+}
+
+plugin=$(realpath $1)
+plugin_root=$(dirname ${plugin})
+
+mitm_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+${mitm_dir}/dev-chrome.sh &
+
+${mitm_dir}/mitm-docker.sh -v ${plugin_root}:${plugin_root} "serve-app-dev.py --plugins ${plugin} --strip_assets --plugin_root ${plugin_root}"

contrib/mitm-ui/serve-app-dev.py

@@ -0,0 +1,139 @@
+# 1. install and setup mitmproxy v2.0.2: https://mitmproxy.readthedocs.io/en/v2.0.2/install.html
+#   (In case of python versions trouble, use https://www.anaconda.com/)
+# 2. mitmdump -q -s -p 8888 \
+#   "serve-app-dev.py --app /path/to/polygerrit-ui/app/"
+# 3. start Chrome with --proxy-server="127.0.0.1:8888" --user-data-dir=/tmp/devchrome
+# 4. open, say, gerrit-review.googlesource.com. Or chromium-review.googlesource.com. Any.
+# 5. uncompiled source files are served and you can log in, too.
+# 6. enjoy!
+#
+# P.S. For replacing plugins, use --plugins or --plugin_root
+#
+# --plugin takes comma-separated list of plugins to add or replace.
+#
+# Example: Adding a new plugin to the server response:
+# --plugins ~/gerrit-testsite/plugins/myplugin.html
+#
+# Example: Replace all matching plugins with local versions:
+# --plugins ~/gerrit-testsite/plugins/
+# Following files will be served if they exist for /plugins/tricium/static/tricium.html:
+#  ~/gerrit-testsite/plugins/tricium.html
+#  ~/gerrit-testsite/plugins/tricium/static/tricium.html
+#
+# --assets takes assets bundle.html, expecting rest of the assets files to be in the same folder
+#
+# Example:
+#  --assets ~/gerrit-testsite/assets/a3be19f.html
+#
+
+from mitmproxy import http
+from mitmproxy.script import concurrent
+import re
+import argparse
+import os.path
+import json
+
+class Server:
+    def __init__(self, devpath, plugins, pluginroot, assets, strip_assets):
+        if devpath:
+            print("Serving app from " + devpath)
+        if pluginroot:
+            print("Serving plugins from " + pluginroot)
+        if assets:
+            self.assets_root, self.assets_file = os.path.split(assets)
+            print("Assets: using " + self.assets_file + " from " + self.assets_root)
+        else:
+            self.assets_root = None
+        if plugins:
+            self.plugins = {path.split("/")[-1:][0]: path for path in map(expandpath, plugins.split(","))}
+            for filename, path in self.plugins.items():
+                print("Serving " + filename + " from " + path)
+        else:
+            self.plugins = {}
+        self.devpath = devpath
+        self.pluginroot = pluginroot
+        self.strip_assets = strip_assets
+
+    def readfile(self, path):
+        with open(path, 'rb') as contentfile:
+            return contentfile.read()
+
+@concurrent
+def response(flow: http.HTTPFlow) -> None:
+    if server.strip_assets:
+        assets_bundle = 'googlesource.com/polygerrit_assets'
+        assets_pos = flow.response.text.find(assets_bundle)
+        if assets_pos != -1:
+            t = flow.response.text
+            flow.response.text = t[:t.rfind('<', 0, assets_pos)] + t[t.find('>', assets_pos) + 1:]
+            return
+
+    if server.assets_root:
+        marker = 'webcomponents-lite.js"></script>'
+        pos = flow.response.text.find(marker)
+        if pos != -1:
+            pos += len(marker)
+            flow.response.text = ''.join([
+                flow.response.text[:pos],
+                '<link rel="import" href="/gerrit_assets/123.0/' + server.assets_file + '">',
+                flow.response.text[pos:]
+            ])
+
+        assets_prefix = "/gerrit_assets/123.0/"
+        if flow.request.path.startswith(assets_prefix):
+            assets_file = flow.request.path[len(assets_prefix):]
+            flow.response.content = server.readfile(server.assets_root + '/' + assets_file)
+            flow.response.status_code = 200
+            if assets_file.endswith('.js'):
+                flow.response.headers['Content-type'] = 'text/javascript'
+            return
+    m = re.match(".+polygerrit_ui/\d+\.\d+/(.+)", flow.request.path)
+    pluginmatch = re.match("^/plugins/(.+)", flow.request.path)
+    localfile = ""
+    if flow.request.path == "/config/server/info":
+        config = json.loads(flow.response.content[5:].decode('utf8'))
+        for filename, path in server.plugins.items():
+            pluginname = filename.split(".")[0]
+            payload = config["plugin"]["js_resource_paths" if filename.endswith(".js") else "html_resource_paths"]
+            if list(filter(lambda url: filename in url, payload)):
+                continue
+            payload.append("plugins/" + pluginname + "/static/" + filename)
+        flow.response.content = str.encode(")]}'\n" + json.dumps(config))
+    if m is not None:
+        filepath = m.groups()[0]
+        localfile = server.devpath + filepath
+    elif pluginmatch is not None:
+        pluginfile = flow.request.path_components[-1]
+        if server.plugins and pluginfile in server.plugins:
+            if os.path.isfile(server.plugins[pluginfile]):
+                localfile = server.plugins[pluginfile]
+            else:
+                print("Can't find file " + server.plugins[pluginfile] + " for " + flow.request.path)
+        elif server.pluginroot:
+            pluginurl = pluginmatch.groups()[0]
+            if os.path.isfile(server.pluginroot + pluginfile):
+                localfile = server.pluginroot + pluginfile
+            elif os.path.isfile(server.pluginroot + pluginurl):
+                localfile = server.pluginroot + pluginurl
+    if localfile and os.path.isfile(localfile):
+        if pluginmatch is not None:
+            print("Serving " + flow.request.path + " from " + localfile)
+        flow.response.content = server.readfile(localfile)
+        flow.response.status_code = 200
+        if localfile.endswith('.js'):
+            flow.response.headers['Content-type'] = 'text/javascript'
+
+def expandpath(path):
+    return os.path.realpath(os.path.expanduser(path))
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--app", type=str, default="", help="Path to /polygerrit-ui/app/")
+parser.add_argument("--plugins", type=str, default="", help="Comma-separated list of plugin files to add/replace")
+parser.add_argument("--plugin_root", type=str, default="", help="Path containing individual plugin files to replace")
+parser.add_argument("--assets", type=str, default="", help="Path containing assets file to import.")
+parser.add_argument("--strip_assets", action="store_true", help="Strip plugin bundles from the response.")
+args = parser.parse_args()
+server = Server(expandpath(args.app) + '/',
+                args.plugins, expandpath(args.plugin_root) + '/',
+                args.assets and expandpath(args.assets),
+                args.strip_assets)

contrib/mitm-ui/serve-app-locally.py

@@ -0,0 +1,46 @@
+# bazel build polygerrit-ui/app:gr-app
+# mitmdump -s "serve-app-locally.py ~/gerrit/bazel-bin/polygerrit-ui/app"
+from mitmproxy import http
+import argparse
+import os
+import zipfile
+
+class Server:
+    def __init__(self, bundle):
+        self.bundle = bundle
+        self.bundlemtime = 0
+        self.files = {
+            'polygerrit_ui/elements/gr-app.js': '',
+            'polygerrit_ui/elements/gr-app.html': '',
+            'polygerrit_ui/styles/main.css': '',
+        }
+        self.read_files()
+
+    def read_files(self):
+        if not os.path.isfile(self.bundle):
+            print("bundle not found!")
+            return
+        mtime = os.stat(self.bundle).st_mtime
+        if mtime <= self.bundlemtime:
+            return
+        self.bundlemtime = mtime
+        with zipfile.ZipFile(self.bundle) as z:
+            for fname in self.files:
+                print('Reading new content for ' + fname)
+                with z.open(fname, 'r') as content_file:
+                    self.files[fname] = content_file.read()
+
+    def response(self, flow: http.HTTPFlow) -> None:
+        self.read_files()
+        for name in self.files:
+            if name.rsplit('/', 1)[1] in flow.request.pretty_url:
+                flow.response.content = self.files[name]
+
+def expandpath(path):
+    return os.path.expanduser(path)
+
+def start():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("bundle", type=str)
+    args = parser.parse_args()
+    return Server(expandpath(args.bundle))