/projects/: Support JSON output format

Implement a machine readable JSON output for both ls-projects and
GET /projects/ interfaces. This will eventually permit replacing
the RPC used by ProjectListScreen to be this new common JSON API.

The JSON output is a single JSON object containing an object entry
for each matched project:

$ curl 'http://127.0.0.1:8080/projects/?format=JSON&d'
)]}'
{
  "external/bison": {
    "description": "A general-purpose parser generator"
  },
  "external/flex": {
    "description": "A fast lexical analyser generator"
  },
  "external/gcc": {},
  "tools/repo": {}
}

Change-Id: I2014a2209bab8be4fb9f61f15e3546c17a2debd5

Documentation/cmd-ls-projects.txt

@@ -64,6 +64,15 @@ Line-feeds are escaped to allow ls-project to keep the
 `all`:: Any type of project.
 --
 
+--format::
+	What output format to display the results in.
++
+--
+`text`:: Simple text based format.
+`json`:: Map of JSON objects describing each project.
+`json_compact`:: Minimized JSON output.
+--
+
 --all::
 	Display all projects that are accessible by the calling user
 	account. Besides the projects that the calling user account has
@@ -78,6 +87,15 @@ This command is also available over HTTP, as `/projects/` for
 anonymous access and `/a/projects/` for authenticated access.
 Named options are available as query parameters.
 
+Over HTTP the `json_compact` output format is assumed if the client
+explicitly asks for JSON using HTTP header `Accept: application/json`.
+When any JSON output format is used on HTTP, readers must skip the
+first line produced. The first line is a garbage JSON string crafted
+to prevent a browser from executing the response in a script tag.
+
+Output will be gzip compressed if `Accept-Encoding: gzip` was used
+by the client in the request headers.
+
 EXAMPLES
 --------
 

Documentation/index.txt

@@ -18,6 +18,7 @@ User Guide
 * link:user-signedoffby.html[Signed-off-by Lines]
 * link:access-control.html[Access Controls]
 * link:error-messages.html[Error Messages]
+* link:rest-api.html[REST API]
 * link:user-submodules.html[Subscribing to Git Submodules]
 
 Installation

Documentation/rest-api.txt

@@ -0,0 +1,89 @@
+Gerrit Code Review - REST API
+=============================
+
+Gerrit Code Review comes with a REST like API available over HTTP.
+The API is suitable for automated tools to build upon, as well as
+supporting some ad-hoc scripting use cases.
+
+Protocol Details
+----------------
+
+Authentication
+~~~~~~~~~~~~~~
+By default all REST endpoints assume anonymous access and filter
+results to correspond to what anonymous users can read (which may
+be nothing at all).
+
+Users (and programs) may authenticate using HTTP authentication by
+supplying the HTTP password from the user's account settings page.
+Gerrit by default uses HTTP digest authentication. To authenticate,
+prefix the endpoint URL with `/a/`. For example to authenticate to
+`/projects/` request URL `/a/projects/`.
+
+Output Format
+~~~~~~~~~~~~~
+Most APIs return text format by default. JSON can be requested
+by setting the `Accept` HTTP request header to include
+`application/json`, for example:
+
+----
+  GET /projects/ HTTP/1.0
+  Accept: application/json
+----
+
+JSON responses are encoded using UTF-8 and use content type
+`application/json`. The JSON response body starts with magic prefix
+line that must be stripped before feeding the rest of the response
+body to a JSON parser:
+
+----
+  )]}'
+  [ ... valid JSON ... ]
+----
+
+The default JSON format is `JSON_COMPACT`, which skips unnecessary
+whitespace. This is not the easiest format for a human to read. Many
+examples in this documentation use `format=JSON` as a query parameter
+to obtain pretty formatting in the response. Producing (and parsing)
+the compact format is more efficient, so most tools should prefer the
+default compact format.
+
+Responses will be gzip compressed by the server if the HTTP
+`Accept-Encoding` request header is set to `gzip`. This may
+save on network transfer time for larger responses.
+
+Endpoints
+---------
+
+List Projects: /projects/
+~~~~~~~~~~~~~~~~~~~~~~~~~
+Lists the projects accessible by the caller. This is the same as
+using the link:cmd-ls-projects.html[ls-projects] command over SSH,
+and accepts the same options as query parameters.
+
+----
+  GET /projects/?format=JSON&d HTTP/1.0
+
+  HTTP/1.1 200 OK
+  Content-Disposition: attachment
+  Content-Type: application/json;charset=UTF-8
+   
+  )]}'
+  {
+    "external/bison": {
+      "description": "GNU parser generator"
+    },
+    "external/gcc": {},
+    "external/openssl": {
+      "description": "encryption\ncrypto routines"
+    },
+    "test": {
+      "description": "\u003chtml\u003e is escaped"
+    }
+  }
+----
+
+
+GERRIT
+------
+Part of link:index.html[Gerrit Code Review]

gerrit-httpd/src/main/java/com/google/gerrit/httpd/RestApiServlet.java

@@ -17,6 +17,7 @@ package com.google.gerrit.httpd;
 import com.google.common.base.Strings;
 import com.google.gerrit.util.cli.CmdLineParser;
 import com.google.gwtjsonrpc.server.RPCServletUtils;
+import com.google.gwtjsonrpc.common.JsonConstants;
 import com.google.inject.Inject;
 
 import org.kohsuke.args4j.CmdLineException;
@@ -26,6 +27,7 @@ import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
 import java.util.Map;
 
 import javax.servlet.ServletException;
@@ -38,6 +40,28 @@ public abstract class RestApiServlet extends HttpServlet {
   private static final Logger log =
       LoggerFactory.getLogger(RestApiServlet.class);
 
+  /** MIME type used for a JSON response body. */
+  protected static final String JSON_TYPE = JsonConstants.JSON_TYPE;
+
+  /**
+   * Garbage prefix inserted before JSON output to prevent XSSI.
+   * <p>
+   * This prefix is ")]}'\n" and is designed to prevent a web browser from
+   * executing the response body if the resource URI were to be referenced using
+   * a &lt;script src="...&gt; HTML tag from another web site. Clients using the
+   * HTTP interface will need to always strip the first line of response data to
+   * remove this magic header.
+   */
+  protected static final byte[] JSON_MAGIC;
+
+  static {
+    try {
+      JSON_MAGIC = ")]}'\n".getBytes("UTF-8");
+    } catch (UnsupportedEncodingException e) {
+      throw new RuntimeException("UTF-8 not supported", e);
+    }
+  }
+
   @Override
   protected void service(HttpServletRequest req, HttpServletResponse res)
       throws ServletException, IOException {
@@ -75,6 +99,23 @@ public abstract class RestApiServlet extends HttpServlet {
     }
   }
 
+  protected static boolean acceptsJson(HttpServletRequest req) {
+    String accept = req.getHeader("Accept");
+    if (accept == null) {
+      return false;
+    } else if (JSON_TYPE.equals(accept)) {
+      return true;
+    } else if (accept.startsWith(JSON_TYPE + ",")) {
+      return true;
+    }
+    for (String p : accept.split("[ ,;][ ,;]*")) {
+      if (JSON_TYPE.equals(p)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   protected static void sendText(HttpServletRequest req,
       HttpServletResponse res, String data) throws IOException {
     res.setContentType("text/plain");

gerrit-httpd/src/main/java/com/google/gerrit/httpd/rpc/project/ListProjectsServlet.java

@@ -15,6 +15,7 @@
 package com.google.gerrit.httpd.rpc.project;
 
 import com.google.gerrit.httpd.RestApiServlet;
+import com.google.gerrit.server.OutputFormat;
 import com.google.gerrit.server.project.ListProjects;
 import com.google.inject.Inject;
 import com.google.inject.Provider;
@@ -42,11 +43,19 @@ public class ListProjectsServlet extends RestApiServlet {
   protected void doGet(HttpServletRequest req, HttpServletResponse res)
       throws IOException {
     ListProjects impl = factory.get();
+    if (acceptsJson(req)) {
+      impl.setFormat(OutputFormat.JSON_COMPACT);
+    }
     if (paramParser.parse(impl, req, res)) {
       ByteArrayOutputStream buf = new ByteArrayOutputStream();
-      res.setContentType("text/plain");
-      res.setCharacterEncoding("UTF-8");
+      if (impl.getFormat().isJson()) {
+        res.setContentType(JSON_TYPE);
+        buf.write(JSON_MAGIC);
+      } else {
+        res.setContentType("text/plain");
+      }
       impl.display(buf);
+      res.setCharacterEncoding("UTF-8");
       send(req, res, buf.toByteArray());
     }
   }

gerrit-server/src/main/java/com/google/gerrit/server/OutputFormat.java

@@ -0,0 +1,71 @@
+// Copyright (C) 2012 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server;
+
+import com.google.gson.FieldNamingPolicy;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gwtjsonrpc.server.SqlTimestampDeserializer;
+
+import java.sql.Timestamp;
+
+/** Standard output format used by an API call. */
+public enum OutputFormat {
+  /**
+   * The output is a human readable text format. It may also be regular enough
+   * to be machine readable. Whether or not the text format is machine readable
+   * and will be committed to as a long term format that tools can build upon is
+   * specific to each API call.
+   */
+  TEXT,
+
+  /**
+   * Pretty-printed JSON format. This format uses whitespace to make the output
+   * readable by a human, but is also machine readable with a JSON library. The
+   * structure of the output is a long term format that tools can rely upon.
+   */
+  JSON,
+
+  /**
+   * Same as {@link #JSON}, but with unnecessary whitespace removed to save
+   * generation time and copy costs. Typically JSON_COMPACT format is used by a
+   * browser based HTML client running over the network.
+   */
+  JSON_COMPACT;
+
+  /** @return true when the format is either JSON or JSON_COMPACT. */
+  public boolean isJson() {
+    return this == JSON_COMPACT || this == JSON;
+  }
+
+  /** @return a new Gson instance configured according to the format. */
+  public GsonBuilder newGsonBuilder() {
+    if (!isJson()) {
+      throw new IllegalStateException(String.format("%s is not JSON", this));
+    }
+    GsonBuilder gb = new GsonBuilder()
+      .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
+      .registerTypeAdapter(Timestamp.class, new SqlTimestampDeserializer());
+    if (this == OutputFormat.JSON) {
+      gb.setPrettyPrinting();
+    }
+    return gb;
+  }
+
+  /** @return a new Gson instance configured according to the format. */
+  public Gson newGson() {
+    return newGsonBuilder().create();
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/project/ListProjects.java

@@ -14,10 +14,13 @@
 
 package com.google.gerrit.server.project;
 
+import com.google.common.collect.Maps;
 import com.google.gerrit.reviewdb.client.Project;
 import com.google.gerrit.server.CurrentUser;
+import com.google.gerrit.server.OutputFormat;
 import com.google.gerrit.server.git.GitRepositoryManager;
 import com.google.gerrit.server.util.TreeFormatter;
+import com.google.gson.reflect.TypeToken;
 import com.google.inject.Inject;
 
 import org.eclipse.jgit.errors.RepositoryNotFoundException;
@@ -36,6 +39,7 @@ import java.io.PrintWriter;
 import java.io.UnsupportedEncodingException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
@@ -75,6 +79,9 @@ public class ListProjects {
   private final GitRepositoryManager repoManager;
   private final ProjectNode.Factory projectNodeFactory;
 
+  @Option(name = "--format", metaVar = "FMT", usage = "Output display format")
+  private OutputFormat format = OutputFormat.TEXT;
+
   @Option(name = "--show-branch", aliases = {"-b"}, multiValued = true,
       usage = "displays the sha of each project in the specified branch")
   private List<String> showBranch;
@@ -115,6 +122,15 @@ public class ListProjects {
     return showDescription;
   }
 
+  public OutputFormat getFormat() {
+    return format;
+  }
+
+  public ListProjects setFormat(OutputFormat fmt) {
+    this.format = fmt;
+    return this;
+  }
+
   public void display(OutputStream out) {
     final PrintWriter stdout;
     try {
@@ -124,6 +140,9 @@ public class ListProjects {
       throw new RuntimeException("JVM lacks UTF-8 encoding", e);
     }
 
+    Map<String, ProjectInfo> output = Maps.newTreeMap();
+    Map<String, String> hiddenNames = Maps.newHashMap();
+
     final TreeMap<Project.NameKey, ProjectNode> treeMap =
         new TreeMap<Project.NameKey, ProjectNode>();
     try {
@@ -137,18 +156,39 @@ public class ListProjects {
 
         final ProjectControl pctl = e.controlFor(currentUser);
         final boolean isVisible = pctl.isVisible() || (all && pctl.isOwner());
-        if (showTree) {
+        if (showTree && !format.isJson()) {
           treeMap.put(projectName,
               projectNodeFactory.create(pctl.getProject(), isVisible));
           continue;
         }
 
-        if (!isVisible) {
+        if (!isVisible && !(showTree && pctl.isOwner())) {
           // Require the project itself to be visible to the user.
           //
           continue;
         }
 
+        ProjectInfo info = new ProjectInfo();
+        info.name = projectName.get();
+        if (showTree && format.isJson()) {
+          ProjectState parent = e.getParentState();
+          if (parent != null) {
+            ProjectControl parentCtrl = parent.controlFor(currentUser);
+            if (parentCtrl.isVisible() || parentCtrl.isOwner()) {
+              info.parent = parent.getProject().getName();
+            } else {
+              info.parent = hiddenNames.get(parent.getProject().getName());
+              if (info.parent == null) {
+                info.parent = "?-" + (hiddenNames.size() + 1);
+                hiddenNames.put(parent.getProject().getName(), info.parent);
+              }
+            }
+          }
+        }
+        if (showDescription && !e.getProject().getDescription().isEmpty()) {
+          info.description = e.getProject().getDescription();
+        }
+
         try {
           if (showBranch != null) {
             Repository git = repoManager.openRepository(projectName);
@@ -162,20 +202,19 @@ public class ListProjects {
                continue;
               }
 
-              for (Ref ref : refs) {
-                if (ref == null) {
-                  // Print stub (forty '-' symbols)
-                  stdout.print("----------------------------------------");
-                } else {
-                  stdout.print(ref.getObjectId().name());
+              for (int i = 0; i < showBranch.size(); i++) {
+                Ref ref = refs.get(i);
+                if (ref != null && ref.getObjectId() != null) {
+                  if (info.branches == null) {
+                    info.branches = Maps.newLinkedHashMap();
+                  }
+                  info.branches.put(showBranch.get(i), ref.getObjectId().name());
                 }
-                stdout.print(' ');
               }
             } finally {
               git.close();
             }
-
-          } else if (type != FilterType.ALL) {
+          } else if (!showTree && type != FilterType.ALL) {
             Repository git = repoManager.openRepository(projectName);
             try {
               if (!type.matches(git)) {
@@ -194,18 +233,36 @@ public class ListProjects {
           continue;
         }
 
-        stdout.print(projectName.get());
-
-        String desc;
-        if (showDescription && !(desc = e.getProject().getDescription()).isEmpty()) {
-          // We still want to list every project as one-liners, hence escaping \n.
-          stdout.print(" - " + desc.replace("\n", "\\n"));
+        if (format.isJson()) {
+          output.put(info.name, info);
+          continue;
         }
 
-        stdout.print("\n");
+        if (showBranch != null) {
+          for (String name : showBranch) {
+            String ref = info.branches != null ? info.branches.get(name) : null;
+            if (ref == null) {
+              // Print stub (forty '-' symbols)
+              ref = "----------------------------------------";
+            }
+            stdout.print(ref);
+            stdout.print(' ');
+          }
+        }
+        stdout.print(info.name);
+
+        if (info.description != null) {
+          // We still want to list every project as one-liners, hence escaping \n.
+          stdout.print(" - " + info.description.replace("\n", "\\n"));
+        }
+        stdout.print('\n');
       }
 
-      if (showTree && treeMap.size() > 0) {
+      if (format.isJson()) {
+        format.newGson().toJson(
+            output, new TypeToken<Map<String, ProjectInfo>>() {}.getType(), stdout);
+        stdout.print('\n');
+      } else if (showTree && treeMap.size() > 0) {
         printProjectTree(stdout, treeMap);
       }
     } finally {
@@ -270,4 +327,11 @@ public class ListProjects {
     }
     return false;
   }
+
+  private static class ProjectInfo {
+    transient String name;
+    String parent;
+    String description;
+    Map<String, String> branches;
+  }
 }

gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/ListProjectsCommand.java

@@ -30,11 +30,13 @@ final class ListProjectsCommand extends BaseCommand {
       @Override
       public void run() throws Exception {
         parseCommandLine(impl);
-        if (impl.isShowTree() && (impl.getShowBranch() != null)) {
-          throw new UnloggedFailure(1, "fatal: --tree and --show-branch options are not compatible.");
-        }
-        if (impl.isShowTree() && impl.isShowDescription()) {
-          throw new UnloggedFailure(1, "fatal: --tree and --description options are not compatible.");
+        if (!impl.getFormat().isJson()) {
+          if (impl.isShowTree() && (impl.getShowBranch() != null)) {
+            throw new UnloggedFailure(1, "fatal: --tree and --show-branch options are not compatible.");
+          }
+          if (impl.isShowTree() && impl.isShowDescription()) {
+            throw new UnloggedFailure(1, "fatal: --tree and --description options are not compatible.");
+          }
         }
         impl.display(out);
       }