Implement RawMailParser

This change adds a parser to parse raw emails received by either POP3 or
IMAP into a MailMessage. It adds a dependency to Apache Mime4j to handle
the mime message parsing and tests.

Change-Id: I97ead9615ffcd0a7839ae1aa1581be4005cf67f1

WORKSPACE

@@ -397,6 +397,20 @@ maven_jar(
   sha1 = 'ab5daef2f881c42c8e280cbe918ec4d7fdfd7efe',
 )
 
+MIME4J_VERS = '0.8.0'
+
+maven_jar(
+  name = 'mime4j_core',
+  artifact = 'org.apache.james:apache-mime4j-core:' + MIME4J_VERS,
+  sha1 = 'd54f45fca44a2f210569656b4ca3574b42911c95',
+)
+
+maven_jar(
+  name = 'mime4j_dom',
+  artifact = 'org.apache.james:apache-mime4j-dom:' + MIME4J_VERS,
+  sha1 = '6720c93d14225c3e12c4a69768a0370c80e376a3',
+)
+
 OW2_VERS = '5.1'
 
 maven_jar(

gerrit-server/BUCK

@@ -70,6 +70,8 @@ java_library(
     '//lib/lucene:lucene-analyzers-common',
     '//lib/lucene:lucene-core-and-backward-codecs',
     '//lib/lucene:lucene-queryparser',
+    '//lib/mime4j:core',
+    '//lib/mime4j:dom',
     '//lib/ow2:ow2-asm',
     '//lib/ow2:ow2-asm-tree',
     '//lib/ow2:ow2-asm-util',

gerrit-server/BUILD

@@ -72,6 +72,8 @@ java_library(
     '//lib/lucene:lucene-analyzers-common',
     '//lib/lucene:lucene-core-and-backward-codecs',
     '//lib/lucene:lucene-queryparser',
+    '//lib/mime4j:core',
+    '//lib/mime4j:dom',
     '//lib/ow2:ow2-asm',
     '//lib/ow2:ow2-asm-tree',
     '//lib/ow2:ow2-asm-util',

gerrit-server/src/main/java/com/google/gerrit/server/mail/Address.java

@@ -24,7 +24,16 @@ public class Address {
     if (0 <= lt && lt < gt && lt + 1 < at && at + 1 < gt) {
       final String email = in.substring(lt + 1, gt).trim();
       final String name = in.substring(0, lt).trim();
-      return new Address(name.length() > 0 ? name : null, email);
+      int nameStart = 0;
+      int nameEnd = name.length();
+      if (name.startsWith("\"")) {
+        nameStart++;
+      }
+      if (name.endsWith("\"")) {
+        nameEnd--;
+      }
+      return new Address(name.length() > 0 ?
+          name.substring(nameStart, nameEnd): null, email);
     }
 
     if (lt < 0 && gt < 0 && 0 < at && at < in.length() - 1) {

gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/MailMessage.java

@@ -16,6 +16,8 @@ package com.google.gerrit.server.mail.receive;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
+import com.google.gerrit.common.Nullable;
+import com.google.gerrit.server.mail.Address;
 
 import org.joda.time.DateTime;
 
@@ -25,57 +27,63 @@ import org.joda.time.DateTime;
  * by the MailParser after MailReceiver has received a message. Transformations
  * done by the parser include stitching mime parts together, transforming all
  * content to UTF-16 and removing attachments.
+ *
+ * A valid MailMessage contains at least the following fields: id, from, to,
+ * subject and dateReceived.
  */
 @AutoValue
 public abstract class MailMessage {
   // Unique Identifier
   public abstract String id();
-  // Envelope Information
-  public abstract String from();
-  public abstract ImmutableList<String> to();
-  public abstract ImmutableList<String> cc();
+  // Envelop Information
+  public abstract Address from();
+  public abstract ImmutableList<Address> to();
+  @Nullable
+  public abstract ImmutableList<Address> cc();
   // Metadata
   public abstract DateTime dateReceived();
   public abstract ImmutableList<String> additionalHeaders();
   // Content
   public abstract String subject();
+  @Nullable
   public abstract String textContent();
+  @Nullable
   public abstract String htmlContent();
 
-  static Builder builder() {
+  public static Builder builder() {
     return new AutoValue_MailMessage.Builder();
   }
 
   @AutoValue.Builder
-  abstract static class Builder {
-    abstract Builder id(String val);
-    abstract Builder from(String val);
-    abstract ImmutableList.Builder<String> toBuilder();
+  public abstract static class Builder {
+    public abstract Builder id(String val);
+    public abstract Builder from(Address val);
+    public abstract ImmutableList.Builder<Address> toBuilder();
 
-    public Builder addTo(String val) {
+    public Builder addTo(Address val) {
       toBuilder().add(val);
       return this;
     }
 
-    abstract ImmutableList.Builder<String> ccBuilder();
+    public abstract ImmutableList.Builder<Address> ccBuilder();
 
-    public Builder addCc(String val) {
+    public Builder addCc(Address val) {
       ccBuilder().add(val);
       return this;
     }
 
-    abstract Builder dateReceived(DateTime val);
-    abstract ImmutableList.Builder<String> additionalHeadersBuilder();
+    public abstract Builder dateReceived(DateTime val);
+    public abstract ImmutableList.Builder<String> additionalHeadersBuilder();
 
     public Builder addAdditionalHeader(String val) {
       additionalHeadersBuilder().add(val);
       return this;
     }
 
-    abstract Builder subject(String val);
-    abstract Builder textContent(String val);
-    abstract Builder htmlContent(String val);
+    public abstract Builder subject(String val);
+    public abstract Builder textContent(String val);
+    public abstract Builder htmlContent(String val);
 
-    abstract MailMessage build();
+    public abstract MailMessage build();
   }
 }

gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/MailParsingException.java

@@ -21,4 +21,8 @@ public class MailParsingException extends Exception {
   public MailParsingException(String msg) {
     super(msg);
   }
+
+  public MailParsingException(String msg, Throwable cause) {
+    super(msg, cause);
+  }
 }

gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/RawMailParser.java

@@ -14,13 +14,157 @@
 
 package com.google.gerrit.server.mail.receive;
 
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.io.CharStreams;
+import com.google.gerrit.server.mail.Address;
+
+import org.apache.james.mime4j.MimeException;
+import org.apache.james.mime4j.dom.Entity;
+import org.apache.james.mime4j.dom.Message;
+import org.apache.james.mime4j.dom.MessageBuilder;
+import org.apache.james.mime4j.dom.Multipart;
+import org.apache.james.mime4j.dom.TextBody;
+import org.apache.james.mime4j.dom.address.Mailbox;
+import org.apache.james.mime4j.message.DefaultMessageBuilder;
+import org.joda.time.DateTime;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
 /**
  * RawMailParser parses raw email content received through POP3 or IMAP into
  * an internal {@link MailMessage}.
  */
 public class RawMailParser {
+  private static final ImmutableSet<String> MAIN_HEADERS =
+      ImmutableSet.of("to", "from", "cc", "date", "message-id",
+          "subject", "content-type");
+
+  /**
+   * Parses a MailMessage from a string.
+   * @param raw String as received over the wire
+   * @return Parsed MailMessage
+   * @throws MailParsingException
+   */
   public static MailMessage parse(String raw) throws MailParsingException {
-    // TODO(hiesel) Implement.
-    return null;
+    MailMessage.Builder messageBuilder = MailMessage.builder();
+    Message mimeMessage;
+    try {
+      MessageBuilder builder = new DefaultMessageBuilder();
+      mimeMessage =
+          builder.parseMessage(new ByteArrayInputStream(raw.getBytes()));
+    } catch (IOException | MimeException e) {
+      throw new MailParsingException("Can't parse email", e);
+    }
+    // Add general headers
+    messageBuilder.id(mimeMessage.getMessageId());
+    messageBuilder.subject(mimeMessage.getSubject());
+    messageBuilder.dateReceived(new DateTime(mimeMessage.getDate()));
+
+    // Add From, To and Cc
+    if (mimeMessage.getFrom() != null && mimeMessage.getFrom().size() > 0) {
+      Mailbox from = mimeMessage.getFrom().get(0);
+      messageBuilder.from(new Address(from.getName(), from.getAddress()));
+    }
+    if (mimeMessage.getTo() != null) {
+      for (Mailbox m : mimeMessage.getTo().flatten()) {
+        messageBuilder.addTo(new Address(m.getName(), m.getAddress()));
+      }
+    }
+    if (mimeMessage.getCc() != null) {
+      for (Mailbox m : mimeMessage.getCc().flatten()) {
+        messageBuilder.addCc(new Address(m.getName(), m.getAddress()));
+      }
+    }
+
+    // Add additional headers
+    mimeMessage.getHeader().getFields().stream()
+        .filter(f -> !MAIN_HEADERS.contains(f.getName().toLowerCase()))
+        .forEach(f -> messageBuilder.addAdditionalHeader(
+            f.getName() + ": " + f.getBody()));
+
+    // Add text and html body parts
+    StringBuilder textBuilder = new StringBuilder();
+    StringBuilder htmlBuilder = new StringBuilder();
+    try {
+      handleMimePart(mimeMessage, textBuilder, htmlBuilder);
+    } catch (IOException e) {
+      throw new MailParsingException("Can't parse email", e);
+    }
+    messageBuilder.textContent(Strings.emptyToNull(textBuilder.toString()));
+    messageBuilder.htmlContent(Strings.emptyToNull(htmlBuilder.toString()));
+
+    try {
+      // build() will only succeed if all required attributes were set. We wrap
+      // the IllegalStateException in a MailParsingException indicating that
+      // required attributes are missing, so that the caller doesn't fall over.
+      return messageBuilder.build();
+    } catch (IllegalStateException e) {
+      throw new MailParsingException(
+          "Missing required attributes after email was parsed", e);
+    }
+  }
+
+  /**
+   * Parses a MailMessage from an array of characters. Note that the character
+   * array is int-typed. This method is only used by POP3, which specifies that
+   * all transferred characters are US-ASCII (RFC 6856). When reading the input
+   * in Java, io.Reader yields ints. These can be safely converted to chars
+   * as all US-ASCII characters fit in a char. If emails contain non-ASCII
+   * characters, such as UTF runes, these will be encoded in ASCII using either
+   * Base64 or quoted-printable encoding.
+   * @param chars Array as received over the wire
+   * @return Parsed MailMessage
+   * @throws MailParsingException
+   */
+  public static MailMessage parse(int[] chars) throws MailParsingException {
+    StringBuilder b = new StringBuilder(chars.length);
+    for (int c : chars) {
+      b.append((char) c);
+    }
+    return parse(b.toString());
+  }
+
+  /**
+   * Traverses a mime tree and parses out text and html parts. All other parts
+   * will be dropped.
+   * @param part MimePart to parse
+   * @param textBuilder StringBuilder to append all plaintext parts
+   * @param htmlBuilder StringBuilder to append all html parts
+   * @throws IOException
+   */
+  private static void handleMimePart(Entity part, StringBuilder textBuilder,
+      StringBuilder htmlBuilder) throws IOException {
+    if (isPlainOrHtml(part.getMimeType()) &&
+        !isAttachment(part.getDispositionType())) {
+      TextBody tb = (TextBody) part.getBody();
+      String result = CharStreams.toString(new InputStreamReader(
+          tb.getInputStream(), tb.getMimeCharset()));
+      if (part.getMimeType().equals("text/plain")) {
+        textBuilder.append(result);
+      } else if (part.getMimeType().equals("text/html")) {
+        htmlBuilder.append(result);
+      }
+    } else if (isMixedOrAlternative(part.getMimeType())) {
+      Multipart multipart = (Multipart) part.getBody();
+      for (Entity e : multipart.getBodyParts()) {
+        handleMimePart(e, textBuilder, htmlBuilder);
+      }
+    }
+  }
+
+  private static boolean isPlainOrHtml(String mimeType) {
+    return (mimeType.equals("text/plain") || mimeType.equals("text/html"));
+  }
+
+  private static boolean isMixedOrAlternative(String mimeType) {
+    return mimeType.equals("multipart/alternative") ||
+        mimeType.equals("multipart/mixed");
+  }
+
+  private static boolean isAttachment(String dispositionType) {
+    return dispositionType != null && dispositionType.equals("attachment");
   }
 }

gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/RawMailParserTest.java

@@ -0,0 +1,76 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.mail.receive;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.gerrit.server.mail.receive.data.AttachmentMessage;
+import com.google.gerrit.server.mail.receive.data.Base64HeaderMessage;
+import com.google.gerrit.server.mail.receive.data.HtmlMimeMessage;
+import com.google.gerrit.server.mail.receive.data.NonUTF8Message;
+import com.google.gerrit.server.mail.receive.data.QuotedPrintableHeaderMessage;
+import com.google.gerrit.server.mail.receive.data.RawMailMessage;
+import com.google.gerrit.server.mail.receive.data.SimpleTextMessage;
+import com.google.gerrit.testutil.GerritBaseTests;
+
+import org.junit.Test;
+
+public class RawMailParserTest extends GerritBaseTests {
+  @Test
+  public void testParseEmail() throws Exception {
+    RawMailMessage[] messages = new RawMailMessage[] {
+        new SimpleTextMessage(),
+        new Base64HeaderMessage(),
+        new QuotedPrintableHeaderMessage(),
+        new HtmlMimeMessage(),
+        new AttachmentMessage(),
+        new NonUTF8Message(),
+    };
+    for (RawMailMessage rawMailMessage : messages) {
+      if (rawMailMessage.rawChars() != null) {
+        // Assert Character to Mail Parser
+        MailMessage parsedMailMessage =
+            RawMailParser.parse(rawMailMessage.rawChars());
+        assertMail(parsedMailMessage, rawMailMessage.expectedMailMessage());
+      }
+      if (rawMailMessage.raw() != null) {
+        // Assert String to Mail Parser
+        MailMessage parsedMailMessage = RawMailParser
+            .parse(rawMailMessage.raw());
+        assertMail(parsedMailMessage, rawMailMessage.expectedMailMessage());
+      }
+    }
+  }
+
+  /**
+   *  This method makes it easier to debug failing tests by checking each
+   *  property individual instead of calling equals as it will immediately
+   *  reveal the property that diverges between the two objects.
+   * @param have MailMessage retrieved from the parser
+   * @param want MailMessage that would be expected
+   */
+  private void assertMail(MailMessage have, MailMessage want) {
+    assertThat(have.id()).isEqualTo(want.id());
+    assertThat(have.to()).isEqualTo(want.to());
+    assertThat(have.from()).isEqualTo(want.from());
+    assertThat(have.cc()).isEqualTo(want.cc());
+    assertThat(have.dateReceived().getMillis())
+        .isEqualTo(want.dateReceived().getMillis());
+    assertThat(have.additionalHeaders()).isEqualTo(want.additionalHeaders());
+    assertThat(have.subject()).isEqualTo(want.subject());
+    assertThat(have.textContent()).isEqualTo(want.textContent());
+    assertThat(have.htmlContent()).isEqualTo(want.htmlContent());
+  }
+}

gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/data/AttachmentMessage.java

@@ -0,0 +1,89 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.mail.receive.data;
+
+
+import com.google.gerrit.server.mail.Address;
+import com.google.gerrit.server.mail.receive.MailMessage;
+
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+/**
+ * Tests that all mime parts that are neither text/plain, nor text/html are
+ * dropped.
+ */
+public class AttachmentMessage extends RawMailMessage {
+  private static String raw = "MIME-Version: 1.0\n" +
+      "Date: Tue, 25 Oct 2016 02:11:35 -0700\n" +
+      "Message-ID: <CAM7sg=3meaAVUxW3KXeJEVs8sv_ADw1BnvpcHHiYVR2TQQi__w" +
+      "@mail.gmail.com>\n" +
+      "Subject: Test Subject\n" +
+      "From: Patrick Hiesel <hiesel@google.com>\n" +
+      "To: Patrick Hiesel <hiesel@google.com>\n" +
+      "Content-Type: multipart/mixed; boundary=001a114e019a56962d054062708f\n" +
+      "\n" +
+      "--001a114e019a56962d054062708f\n" +
+      "Content-Type: multipart/alternative; boundary=001a114e019a5696250540" +
+      "62708d\n" +
+      "\n" +
+      "--001a114e019a569625054062708d\n" +
+      "Content-Type: text/plain; charset=UTF-8\n" +
+      "\n" +
+      "Contains unwanted attachment" +
+      "\n" +
+      "--001a114e019a569625054062708d\n" +
+      "Content-Type: text/html; charset=UTF-8\n" +
+      "\n" +
+      "<div dir=\"ltr\">Contains unwanted attachment</div>" +
+      "\n" +
+      "--001a114e019a569625054062708d--\n" +
+      "--001a114e019a56962d054062708f\n" +
+      "Content-Type: text/plain; charset=US-ASCII; name=\"test.txt\"\n" +
+      "Content-Disposition: attachment; filename=\"test.txt\"\n" +
+      "Content-Transfer-Encoding: base64\n" +
+      "X-Attachment-Id: f_iv264bt50\n" +
+      "\n" +
+      "VEVTVAo=\n" +
+      "--001a114e019a56962d054062708f--";
+
+  @Override
+  public String raw() {
+    return raw;
+  }
+
+  @Override
+  public int[] rawChars() {
+    return null;
+  }
+
+  @Override
+  public MailMessage expectedMailMessage() {
+    System.out.println("\uD83D\uDE1B test");
+    MailMessage.Builder expect = MailMessage.builder();
+    expect
+        .id("<CAM7sg=3meaAVUxW3KXeJEVs8sv_ADw1BnvpcHHiYVR2TQQi__w" +
+            "@mail.gmail.com>")
+        .from(new Address("Patrick Hiesel", "hiesel@google.com"))
+        .addTo(new Address("Patrick Hiesel", "hiesel@google.com"))
+        .textContent("Contains unwanted attachment")
+        .htmlContent("<div dir=\"ltr\">Contains unwanted attachment</div>")
+        .subject("Test Subject")
+        .addAdditionalHeader("MIME-Version: 1.0")
+        .dateReceived(
+            new DateTime(2016, 10, 25, 9, 11, 35, 0, DateTimeZone.UTC));
+    return expect.build();
+  }
+}

gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/data/Base64HeaderMessage.java

@@ -0,0 +1,62 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.mail.receive.data;
+
+import com.google.gerrit.server.mail.Address;
+import com.google.gerrit.server.mail.receive.MailMessage;
+
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+/**
+ * Tests parsing a Base64 encoded subject.
+ */
+public class Base64HeaderMessage extends RawMailMessage {
+  private static String textContent = "Some Text";
+  private static String raw = "" +
+      "Date: Tue, 25 Oct 2016 02:11:35 -0700\n" +
+      "Message-ID: <001a114da7ae26e2eb053fe0c29c@google.com>\n" +
+      "Subject: =?UTF-8?B?8J+YmyB0ZXN0?=\n" +
+      "From: \"Jonathan Nieder (Gerrit)\" <noreply-gerritcodereview-" +
+      "CtTy0igsBrnvL7dKoWEIEg@google.com>\n" +
+      "To: ekempin <ekempin@google.com>\n" +
+      "Content-Type: text/plain; charset=UTF-8; format=flowed; delsp=yes\n" +
+      "\n" + textContent;
+
+  @Override
+  public String raw() {
+    return raw;
+  }
+
+  @Override
+  public int[] rawChars() {
+    return null;
+  }
+
+  @Override
+  public MailMessage expectedMailMessage() {
+    MailMessage.Builder expect = MailMessage.builder();
+    expect
+        .id("<001a114da7ae26e2eb053fe0c29c@google.com>")
+        .from(new Address("Jonathan Nieder (Gerrit)",
+            "noreply-gerritcodereview-CtTy0igsBrnvL7dKoWEIEg@google.com"))
+        .addTo(new Address("ekempin","ekempin@google.com"))
+        .textContent(textContent)
+        .subject("\uD83D\uDE1B test")
+        .dateReceived(
+            new DateTime(2016, 10, 25, 9, 11, 35, 0, DateTimeZone.UTC));
+    return expect.build();
+  }
+}

gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/data/HtmlMimeMessage.java

@@ -0,0 +1,103 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.mail.receive.data;
+
+import com.google.gerrit.server.mail.Address;
+import com.google.gerrit.server.mail.receive.MailMessage;
+
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+/**
+ * Tests a message containing mime/alternative (text + html) content.
+ */
+public class HtmlMimeMessage extends RawMailMessage {
+  private static String textContent = "Simple test";
+
+  // htmlContent is encoded in quoted-printable
+  private static String htmlContent = "<div dir=3D\"ltr\">Test <span style" +
+      "=3D\"background-color:rgb(255,255,0)\">Messa=\n" +
+      "ge</span> in <u>HTML=C2=A0</u><a href=3D\"https://en.wikipedia.org/" +
+      "wiki/%C3%=\n9Cmlaut_(band)\" class=3D\"gmail-mw-redirect\" title=3D\"" +
+      "=C3=9Cmlaut (band)\" st=\nyle=3D\"text-decoration:none;color:rgb(11," +
+      "0,128);background-image:none;backg=\nround-position:initial;background" +
+      "-size:initial;background-repeat:initial;ba=\nckground-origin:initial;" +
+      "background-clip:initial;font-family:sans-serif;font=\n" +
+      "-size:14px\">=C3=9C</a></div>";
+
+  private static String unencodedHtmlContent = "" +
+      "<div dir=\"ltr\">Test <span style=\"background-color:rgb(255,255,0)\">" +
+      "Message</span> in <u>HTML </u><a href=\"https://en.wikipedia.org/wiki/" +
+      "%C3%9Cmlaut_(band)\" class=\"gmail-mw-redirect\" title=\"Ümlaut " +
+      "(band)\" style=\"text-decoration:none;color:rgb(11,0,128);" +
+      "background-image:none;background-position:initial;background-size:" +
+      "initial;background-repeat:initial;background-origin:initial;background" +
+      "-clip:initial;font-family:sans-serif;font-size:14px\">Ü</a></div>";
+
+  private static String raw = "" +
+      "MIME-Version: 1.0\n" +
+      "Date: Tue, 25 Oct 2016 02:11:35 -0700\n" +
+      "Message-ID: <001a114cd8be55b4ab053face5cd@google.com>\n" +
+      "Subject: Change in gerrit[master]: Implement receiver class structure " +
+      "and bindings\n" +
+      "From: \"ekempin (Gerrit)\" <noreply-gerritcodereview-qUgXfQecoDLHwp0Ml" +
+      "dAzig@google.com>\n" +
+      "To: Patrick Hiesel <hiesel@google.com>\n" +
+      "Cc: ekempin <ekempin@google.com>\n" +
+      "Content-Type: multipart/alternative; boundary=001a114cd8b" +
+      "e55b486053face5ca\n" +
+      "\n" +
+      "--001a114cd8be55b486053face5ca\n" +
+      "Content-Type: text/plain; charset=UTF-8; format=flowed; delsp=yes\n" +
+      "\n" +
+      textContent +
+      "\n" +
+      "--001a114cd8be55b486053face5ca\n" +
+      "Content-Type: text/html; charset=UTF-8\n" +
+      "Content-Transfer-Encoding: quoted-printable\n" +
+      "\n" +
+      htmlContent +
+      "\n" +
+      "--001a114cd8be55b486053face5ca--";
+
+  @Override
+  public String raw() {
+    return raw;
+  }
+
+  @Override
+  public int[] rawChars() {
+    return null;
+  }
+
+  @Override
+  public MailMessage expectedMailMessage() {
+    MailMessage.Builder expect = MailMessage.builder();
+    expect
+        .id("<001a114cd8be55b4ab053face5cd@google.com>")
+        .from(new Address("ekempin (Gerrit)",
+            "noreply-gerritcodereview-qUgXfQecoDLHwp0MldAzig@google.com"))
+        .addCc(new Address("ekempin","ekempin@google.com"))
+        .addTo(new Address("Patrick Hiesel","hiesel@google.com"))
+        .textContent(textContent)
+        .htmlContent(unencodedHtmlContent)
+        .subject("Change in gerrit[master]: Implement " +
+            "receiver class structure and bindings")
+        .addAdditionalHeader("MIME-Version: 1.0")
+        .dateReceived(
+            new DateTime(2016, 10, 25, 9, 11, 35, 0, DateTimeZone.UTC));
+    return expect.build();
+  }
+}

gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/data/NonUTF8Message.java

@@ -0,0 +1,66 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.gerrit.server.mail.receive.data;
+
+import com.google.gerrit.server.mail.Address;
+import com.google.gerrit.server.mail.receive.MailMessage;
+
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+/**
+ * Tests that non-UTF8 encodings are handled correctly.
+ */
+public class NonUTF8Message extends RawMailMessage {
+  private static String textContent = "Some Text";
+  private static String raw = "" +
+      "Date: Tue, 25 Oct 2016 02:11:35 -0700\n" +
+      "Message-ID: <001a114da7ae26e2eb053fe0c29c@google.com>\n" +
+      "Subject: =?UTF-8?B?8J+YmyB0ZXN0?=\n" +
+      "From: \"Jonathan Nieder (Gerrit)\" <noreply-gerritcodereview-" +
+      "CtTy0igsBrnvL7dKoWEIEg@google.com>\n" +
+      "To: ekempin <ekempin@google.com>\n" +
+      "Content-Type: text/plain; charset=UTF-8; format=flowed; delsp=yes\n" +
+      "\n" + textContent;
+
+  @Override
+  public String raw() {
+    return null;
+  }
+
+  @Override
+  public int[] rawChars() {
+    int[] arr = new int[raw.length()];
+    int i = 0;
+    for (char c : raw.toCharArray()) {
+      arr[i++] = (int) c;
+    }
+    return arr;
+  }
+
+  @Override
+  public MailMessage expectedMailMessage() {
+    MailMessage.Builder expect = MailMessage.builder();
+    expect
+        .id("<001a114da7ae26e2eb053fe0c29c@google.com>")
+        .from(new Address("Jonathan Nieder (Gerrit)",
+            "noreply-gerritcodereview-CtTy0igsBrnvL7dKoWEIEg@google.com"))
+        .addTo(new Address("ekempin","ekempin@google.com"))
+        .textContent(textContent)
+        .subject("\uD83D\uDE1B test")
+        .dateReceived(
+            new DateTime(2016, 10, 25, 9, 11, 35, 0, DateTimeZone.UTC));
+    return expect.build();
+  }
+}

gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/data/QuotedPrintableHeaderMessage.java

@@ -0,0 +1,63 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.mail.receive.data;
+
+import com.google.gerrit.server.mail.Address;
+import com.google.gerrit.server.mail.receive.MailMessage;
+
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+/**
+ * Tests parsing a quoted printable encoded subject
+ */
+public class QuotedPrintableHeaderMessage extends RawMailMessage {
+  private static String textContent = "Some Text";
+  private static String raw = "" +
+      "Date: Tue, 25 Oct 2016 02:11:35 -0700\n" +
+      "Message-ID: <001a114da7ae26e2eb053fe0c29c@google.com>\n" +
+      "Subject: =?UTF-8?Q?=C3=A2me vulgaire?=\n" +
+      "From: \"Jonathan Nieder (Gerrit)\" <noreply-gerritcodereview-" +
+      "CtTy0igsBrnvL7dKoWEIEg@google.com>\n" +
+      "To: ekempin <ekempin@google.com>\n" +
+      "Content-Type: text/plain; charset=UTF-8; format=flowed; delsp=yes\n" +
+      "\n" + textContent;
+
+  @Override
+  public String raw() {
+    return raw;
+  }
+
+  @Override
+  public int[] rawChars() {
+    return null;
+  }
+
+  @Override
+  public MailMessage expectedMailMessage() {
+    System.out.println("\uD83D\uDE1B test");
+    MailMessage.Builder expect = MailMessage.builder();
+    expect
+        .id("<001a114da7ae26e2eb053fe0c29c@google.com>")
+        .from(new Address("Jonathan Nieder (Gerrit)",
+            "noreply-gerritcodereview-CtTy0igsBrnvL7dKoWEIEg@google.com"))
+        .addTo(new Address("ekempin","ekempin@google.com"))
+        .textContent(textContent)
+        .subject("âme vulgaire")
+        .dateReceived(
+            new DateTime(2016, 10, 25, 9, 11, 35, 0, DateTimeZone.UTC));
+    return expect.build();
+  }
+}

gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/data/RawMailMessage.java

@@ -0,0 +1,28 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.mail.receive.data;
+
+import com.google.gerrit.server.mail.receive.MailMessage;
+
+/**
+ * Base class for all email parsing tests.
+ */
+public abstract class RawMailMessage {
+  // Raw content to feed the parser
+  public abstract String raw();
+  public abstract int[] rawChars();
+  // Parsed representation for asserting the expected parser output
+  public abstract MailMessage expectedMailMessage();
+}

gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/data/SimpleTextMessage.java

@@ -0,0 +1,134 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.mail.receive.data;
+
+import com.google.gerrit.server.mail.Address;
+import com.google.gerrit.server.mail.receive.MailMessage;
+
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+/**
+ * Tests parsing a simple text message with different headers.
+ */
+public class SimpleTextMessage extends RawMailMessage {
+  private static String textContent = "" +
+      "Jonathan Nieder has posted comments on this change. (  \n" +
+      "https://gerrit-review.googlesource.com/90018 )\n" +
+      "\n" +
+      "Change subject: (Re)enable voting buttons for merged changes\n" +
+      "...........................................................\n" +
+      "\n" +
+      "\n" +
+      "Patch Set 2:\n" +
+      "\n" +
+      "This is producing NPEs server-side and 500s for the client.   \n" +
+      "when I try to load this change:\n" +
+      "\n" +
+      "  Error in GET /changes/90018/detail?O=10004\n" +
+      "  com.google.gwtorm.OrmException: java.lang.NullPointerException\n" +
+      "\tat com.google.gerrit.change.ChangeJson.format(ChangeJson.java:303)\n" +
+      "\tat com.google.gerrit.change.ChangeJson.format(ChangeJson.java:285)\n" +
+      "\tat com.google.gerrit.change.ChangeJson.format(ChangeJson.java:263)\n" +
+      "\tat com.google.gerrit.change.GetChange.apply(GetChange.java:50)\n" +
+      "\tat com.google.gerrit.change.GetDetail.apply(GetDetail.java:51)\n" +
+      "\tat com.google.gerrit.change.GetDetail.apply(GetDetail.java:26)\n" +
+      "\tat  \n" +
+      "com.google.gerrit.RestApiServlet.service(RestApiServlet.java:367)\n" +
+      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:717)\n" +
+      "[...]\n" +
+      "  Caused by: java.lang.NullPointerException\n" +
+      "\tat  \n" +
+      "com.google.gerrit.ChangeJson.setLabelScores(ChangeJson.java:670)\n" +
+      "\tat  \n" +
+      "com.google.gerrit.ChangeJson.labelsFor(ChangeJson.java:845)\n" +
+      "\tat  \n" +
+      "com.google.gerrit.change.ChangeJson.labelsFor(ChangeJson.java:598)\n" +
+      "\tat  \n" +
+      "com.google.gerrit.change.ChangeJson.toChange(ChangeJson.java:499)\n" +
+      "\tat com.google.gerrit.change.ChangeJson.format(ChangeJson.java:294)\n" +
+      "\t... 105 more\n" +
+      "-- \n" +
+      "To view, visit https://gerrit-review.googlesource.com/90018\n" +
+      "To unsubscribe, visit https://gerrit-review.googlesource.com\n" +
+      "\n" +
+      "Gerrit-MessageType: comment\n" +
+      "Gerrit-Change-Id: Iba501e00bee77be3bd0ced72f88fd04ba0accaed\n" +
+      "Gerrit-PatchSet: 2\n" +
+      "Gerrit-Project: gerrit\n" +
+      "Gerrit-Branch: master\n" +
+      "Gerrit-Owner: ekempin <ekempin@google.com>\n" +
+      "Gerrit-Reviewer: Dave Borowitz <dborowitz@google.com>\n" +
+      "Gerrit-Reviewer: Edwin Kempin <ekempin@google.com>\n" +
+      "Gerrit-Reviewer: GerritForge CI <gerritforge@gmail.com>\n" +
+      "Gerrit-Reviewer: Jonathan Nieder <jrn@google.com>\n" +
+      "Gerrit-Reviewer: Patrick Hiesel <hiesel@google.com>\n" +
+      "Gerrit-Reviewer: ekempin <ekempin@google.com>\n" +
+      "Gerrit-HasComments: No";
+
+  private static String raw = "" +
+      "Authentication-Results: mx.google.com; dkim=pass header.i=" +
+      "@google.com;\n" +
+      "Date: Tue, 25 Oct 2016 02:11:35 -0700\n" +
+      "In-Reply-To: <gerrit.1477487889000.Iba501e00bee77be3bd0ced" +
+      "72f88fd04ba0accaed@gerrit-review.googlesource.com>\n" +
+      "References: <gerrit.1477487889000.Iba501e00bee77be3bd0ced72f8" +
+      "8fd04ba0accaed@gerrit-review.googlesource.com>\n" +
+      "Message-ID: <001a114da7ae26e2eb053fe0c29c@google.com>\n" +
+      "Subject: Change in gerrit[master]: (Re)enable voting buttons for " +
+      "merged changes\n" +
+      "From: \"Jonathan Nieder (Gerrit)\" <noreply-gerritcodereview-CtTy0" +
+      "igsBrnvL7dKoWEIEg@google.com>\n" +
+      "To: ekempin <ekempin@google.com>\n" +
+      "Cc: Dave Borowitz <dborowitz@google.com>, Jonathan Nieder " +
+      "<jrn@google.com>, Patrick Hiesel <hiesel@google.com>\n" +
+      "Content-Type: text/plain; charset=UTF-8; format=flowed; delsp=yes\n" +
+      "\n" + textContent;
+
+  @Override
+  public String raw() {
+    return raw;
+  }
+
+  @Override
+  public int[] rawChars() {
+    return null;
+  }
+
+  @Override
+  public MailMessage expectedMailMessage() {
+    MailMessage.Builder expect = MailMessage.builder();
+    expect
+        .id("<001a114da7ae26e2eb053fe0c29c@google.com>")
+        .from(new Address("Jonathan Nieder (Gerrit)",
+            "noreply-gerritcodereview-CtTy0igsBrnvL7dKoWEIEg@google.com"))
+        .addTo(new Address("ekempin","ekempin@google.com"))
+        .addCc(new Address("Dave Borowitz", "dborowitz@google.com"))
+        .addCc(new Address("Jonathan Nieder", "jrn@google.com"))
+        .addCc(new Address("Patrick Hiesel", "hiesel@google.com"))
+        .textContent(textContent)
+        .subject("Change in gerrit[master]: (Re)enable voting"
+            + " buttons for merged changes")
+        .dateReceived(
+            new DateTime(2016, 10, 25, 9, 11, 35, 0, DateTimeZone.UTC))
+        .addAdditionalHeader("Authentication-Results: mx.google.com; " +
+            "dkim=pass header.i=@google.com;")
+        .addAdditionalHeader("In-Reply-To: <gerrit.1477487889000.Iba501e00bee" +
+            "77be3bd0ced72f88fd04ba0accaed@gerrit-review.googlesource.com>")
+        .addAdditionalHeader("References: <gerrit.1477487889000.Iba501e00bee" +
+            "77be3bd0ced72f88fd04ba0accaed@gerrit-review.googlesource.com>");
+    return expect.build();
+  }
+}

lib/mime4j/BUCK

@@ -0,0 +1,35 @@
+include_defs('//lib/maven.defs')
+
+VERSION = '0.8.0'
+
+java_library(
+  name = 'core',
+  exported_deps = [
+    ':core_library',
+  ],
+  visibility = ['PUBLIC'],
+)
+
+maven_jar(
+  name = 'core_library',
+  id = 'org.apache.james:apache-mime4j-core:' + VERSION,
+  sha1 = 'd54f45fca44a2f210569656b4ca3574b42911c95',
+  license = 'Apache2.0',
+  visibility = ['PUBLIC'],
+)
+
+java_library(
+  name = 'dom',
+  exported_deps = [
+    ':dom_library',
+  ],
+  visibility = ['PUBLIC'],
+)
+
+maven_jar(
+  name = 'dom_library',
+  id = 'org.apache.james:apache-mime4j-dom:' + VERSION,
+  sha1 = '6720c93d14225c3e12c4a69768a0370c80e376a3',
+  license = 'Apache2.0',
+  visibility = ['PUBLIC'],
+)

lib/mime4j/BUILD

@@ -0,0 +1,13 @@
+java_library(
+    name = "core",
+    data = ["//lib:LICENSE-Apache2.0"],
+    visibility = ["//visibility:public"],
+    exports = ["@mime4j_core//jar"],
+)
+
+java_library(
+    name = "dom",
+    data = ["//lib:LICENSE-Apache2.0"],
+    visibility = ["//visibility:public"],
+    exports = ["@mime4j_dom//jar"],
+)