Implement Plaintext Email Parser
This implements a plaintext email parser and tests to parse comments from inbound email. Change-Id: I4e56d4b74115d7aae8c40c70f679573d3b1c4b7d
This commit is contained in:
Patrick Hiesel
@@ -0,0 +1,31 @@
|
||||
// Copyright (C) 2016 The Android Open Source Project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.gerrit.server.mail.receive;
|
||||
|
||||
import com.google.gerrit.reviewdb.client.Comment;
|
||||
|
||||
/** A comment parsed from inbound email */
|
||||
public class MailComment {
|
||||
enum CommentType {
|
||||
CHANGE_MESSAGE,
|
||||
FILE_COMMENT,
|
||||
INLINE_COMMENT
|
||||
}
|
||||
|
||||
CommentType type;
|
||||
Comment inReplyTo;
|
||||
String fileName;
|
||||
String message;
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
// Copyright (C) 2016 The Android Open Source Project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.gerrit.server.mail.receive;
|
||||
|
||||
import com.google.common.collect.Iterators;
|
||||
import com.google.common.collect.PeekingIterator;
|
||||
import com.google.gerrit.reviewdb.client.Comment;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
/** TextParser provides parsing functionality for plaintext email. */
|
||||
public class TextParser {
|
||||
/**
|
||||
* Parses comments from plaintext email.
|
||||
*
|
||||
* @param email MailMessage as received from the email service.
|
||||
* @param comments Comments previously persisted on the change that caused the
|
||||
* original notification email to be sent out. Ordering must
|
||||
* be the same as in the outbound email
|
||||
* @param changeUrl Canonical change url that points to the change on this
|
||||
* Gerrit instance.
|
||||
* Example: https://go-review.googlesource.com/#/c/91570
|
||||
* @return List of MailComments parsed from the plaintext part of the email.
|
||||
*/
|
||||
public static List<MailComment> parse(
|
||||
MailMessage email, Collection<Comment> comments, String changeUrl) {
|
||||
String body = email.textContent();
|
||||
// Replace CR-LF by \n
|
||||
body = body.replace("\r\n", "\n");
|
||||
|
||||
List<MailComment> parsedComments = new ArrayList<>();
|
||||
|
||||
// Some email clients (like GMail) use >> for enquoting text when there are
|
||||
// inline comments that the users typed. These will then be enquoted by a
|
||||
// single >. We sanitize this by unifying it into >. Inline comments typed
|
||||
// by the user will not be enquoted.
|
||||
//
|
||||
// Example:
|
||||
// Some comment
|
||||
// >> Quoted Text
|
||||
// >> Quoted Text
|
||||
// > A comment typed in the email directly
|
||||
String singleQuotePattern = "\n> ";
|
||||
String doubleQuotePattern = "\n>> ";
|
||||
if (countOccurrences(body, doubleQuotePattern) >
|
||||
countOccurrences(body, singleQuotePattern)) {
|
||||
body = body.replace(doubleQuotePattern, singleQuotePattern);
|
||||
}
|
||||
|
||||
PeekingIterator<Comment> iter =
|
||||
Iterators.peekingIterator(comments.iterator());
|
||||
|
||||
String[] lines = body.split("\n");
|
||||
MailComment currentComment = null;
|
||||
String lastEncounteredFileName = null;
|
||||
Comment lastEncounteredComment = null;
|
||||
for (String line : lines) {
|
||||
if (line.startsWith("> ")) {
|
||||
line = line.substring("> ".length()).trim();
|
||||
// This is not a comment, try to advance the file/comment pointers and
|
||||
// add previous comment to list if applicable
|
||||
if (currentComment != null) {
|
||||
parsedComments.add(currentComment);
|
||||
currentComment = null;
|
||||
}
|
||||
|
||||
if (!iter.hasNext()) {
|
||||
continue;
|
||||
}
|
||||
Comment perspectiveComment = iter.peek();
|
||||
if (line.equals(filePath(changeUrl, perspectiveComment))) {
|
||||
if (lastEncounteredFileName == null ||
|
||||
!lastEncounteredFileName
|
||||
.equals(perspectiveComment.key.filename)) {
|
||||
// This is the annotation of a file
|
||||
lastEncounteredFileName = perspectiveComment.key.filename;
|
||||
lastEncounteredComment = null;
|
||||
} else if (perspectiveComment.lineNbr == 0) {
|
||||
// This was originally a file-level comment
|
||||
lastEncounteredComment = perspectiveComment;
|
||||
iter.next();
|
||||
}
|
||||
} else if (isCommentUrl(line, changeUrl, perspectiveComment)) {
|
||||
lastEncounteredComment = perspectiveComment;
|
||||
iter.next();
|
||||
}
|
||||
} else {
|
||||
// This is a comment. Try to append to previous comment if applicable or
|
||||
// create a new comment.
|
||||
if (currentComment == null) {
|
||||
// Start new comment
|
||||
currentComment = new MailComment();
|
||||
currentComment.message = line;
|
||||
if (lastEncounteredComment == null) {
|
||||
if (lastEncounteredFileName == null) {
|
||||
// Change message
|
||||
currentComment.type = MailComment.CommentType.CHANGE_MESSAGE;
|
||||
} else {
|
||||
// File comment not sent in reply to another comment
|
||||
currentComment.type = MailComment.CommentType.FILE_COMMENT;
|
||||
currentComment.fileName = lastEncounteredFileName;
|
||||
}
|
||||
} else {
|
||||
// Comment sent in reply to another comment
|
||||
currentComment.inReplyTo = lastEncounteredComment;
|
||||
currentComment.type = MailComment.CommentType.INLINE_COMMENT;
|
||||
}
|
||||
} else {
|
||||
// Attach to previous comment
|
||||
currentComment.message += "\n" + line;
|
||||
}
|
||||
}
|
||||
}
|
||||
// There is no need to attach the currentComment after this loop as all
|
||||
// emails have footers and other enquoted text after the last comment
|
||||
// appeared and the last comment will have already been added to the list
|
||||
// at this point.
|
||||
|
||||
return parsedComments;
|
||||
}
|
||||
|
||||
/** Counts the occurrences of pattern in s */
|
||||
private static int countOccurrences(String s, String pattern) {
|
||||
return (s.length() - s.replace(pattern, "").length()) / pattern.length();
|
||||
}
|
||||
|
||||
/** Check if string is an inline comment url on a patch set or the base */
|
||||
private static boolean isCommentUrl(String str, String changeUrl,
|
||||
Comment comment) {
|
||||
return str.equals(filePath(changeUrl, comment) + "@" + comment.lineNbr) ||
|
||||
str.equals(filePath(changeUrl, comment) + "@a" + comment.lineNbr);
|
||||
}
|
||||
|
||||
/** Generate the fully qualified filepath */
|
||||
private static String filePath(String changeUrl, Comment comment) {
|
||||
return changeUrl + "/" + comment.key.patchSetId + "/" +
|
||||
comment.key.filename;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
// Copyright (C) 2016 The Android Open Source Project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.gerrit.server.mail.receive;
|
||||
|
||||
import com.google.gerrit.reviewdb.client.Account;
|
||||
import com.google.gerrit.reviewdb.client.Comment;
|
||||
import com.google.gerrit.server.mail.Address;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
import java.sql.Timestamp;
|
||||
import org.junit.Ignore;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
|
||||
@Ignore
|
||||
public class AbstractParserTest {
|
||||
protected static void assertChangeMessage(String message,
|
||||
MailComment comment) {
|
||||
assertThat(comment.fileName).isNull();
|
||||
assertThat(comment.message).isEqualTo(message);
|
||||
assertThat(comment.inReplyTo).isNull();
|
||||
assertThat(comment.type).isEqualTo(MailComment.CommentType.CHANGE_MESSAGE);
|
||||
}
|
||||
|
||||
protected static void assertInlineComment(String message, MailComment comment,
|
||||
Comment inReplyTo) {
|
||||
assertThat(comment.fileName).isNull();
|
||||
assertThat(comment.message).isEqualTo(message);
|
||||
assertThat(comment.inReplyTo).isEqualTo(inReplyTo);
|
||||
assertThat(comment.type).isEqualTo(MailComment.CommentType.INLINE_COMMENT);
|
||||
}
|
||||
|
||||
protected static void assertFileComment(String message, MailComment comment,
|
||||
String file) {
|
||||
assertThat(comment.fileName).isEqualTo(file);
|
||||
assertThat(comment.message).isEqualTo(message);
|
||||
assertThat(comment.inReplyTo).isNull();
|
||||
assertThat(comment.type).isEqualTo(MailComment.CommentType.FILE_COMMENT);
|
||||
}
|
||||
|
||||
protected static Comment newComment(String uuid, String file,
|
||||
String message, int line) {
|
||||
Comment c = new Comment(new Comment.Key(uuid, file, 1),
|
||||
new Account.Id(0), new Timestamp(0l), (short) 0, message, "");
|
||||
c.lineNbr = line;
|
||||
return c;
|
||||
}
|
||||
|
||||
/** Returns a MailMessage.Builder with all required fields populated. */
|
||||
protected static MailMessage.Builder newMailMessageBuilder() {
|
||||
MailMessage.Builder b = MailMessage.builder();
|
||||
b.id("id");
|
||||
b.from(new Address("Foo Bar", "foo@bar.com"));
|
||||
b.dateReceived(new DateTime());
|
||||
b.subject("");
|
||||
return b;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
// Copyright (C) 2016 The Android Open Source Project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.gerrit.server.mail.receive;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
|
||||
import com.google.gerrit.reviewdb.client.Comment;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class TextParserTest extends AbstractParserTest {
|
||||
private static final String quotedFooter = "" +
|
||||
"> To view, visit https://gerrit-review.googlesource.com/123\n" +
|
||||
"> To unsubscribe, visit https://gerrit-review.googlesource.com\n" +
|
||||
"> \n" +
|
||||
"> Gerrit-MessageType: comment\n" +
|
||||
"> Gerrit-Change-Id: Ie1234021bf1e8d1425641af58fd648fc011db153\n" +
|
||||
"> Gerrit-PatchSet: 1\n" +
|
||||
"> Gerrit-Project: gerrit\n" +
|
||||
"> Gerrit-Branch: master\n" +
|
||||
"> Gerrit-Owner: Foo Bar <foo@bar.com>\n" +
|
||||
"> Gerrit-HasComments: Yes";
|
||||
private static final String changeURL =
|
||||
"https://gerrit-review.googlesource.com/#/changes/123";
|
||||
|
||||
@Test
|
||||
public void simpleChangeMessage() {
|
||||
MailMessage.Builder b = newMailMessageBuilder();
|
||||
b.textContent("Looks good to me\n" + quotedFooter);
|
||||
|
||||
List<Comment> comments = defaultComments();
|
||||
List<MailComment> parsedComments =
|
||||
TextParser.parse(b.build(), comments, changeURL);
|
||||
|
||||
assertThat(parsedComments).hasSize(1);
|
||||
assertChangeMessage("Looks good to me", parsedComments.get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void simpleInlineComments() {
|
||||
MailMessage.Builder b = newMailMessageBuilder();
|
||||
b.textContent(newPlaintextBody("Looks good to me",
|
||||
"I have a comment on this.", null, "Also have a comment here.",
|
||||
null, null, null) + quotedFooter);
|
||||
|
||||
List<Comment> comments = defaultComments();
|
||||
List<MailComment> parsedComments =
|
||||
TextParser.parse(b.build(), comments, changeURL);
|
||||
|
||||
assertThat(parsedComments).hasSize(3);
|
||||
assertChangeMessage("Looks good to me", parsedComments.get(0));
|
||||
assertInlineComment("I have a comment on this.", parsedComments.get(1),
|
||||
comments.get(1));
|
||||
assertInlineComment("Also have a comment here.", parsedComments.get(2),
|
||||
comments.get(3));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void simpleFileComment() {
|
||||
MailMessage.Builder b = newMailMessageBuilder();
|
||||
b.textContent(newPlaintextBody("Looks good to me",
|
||||
null, null, "Also have a comment here.",
|
||||
"This is a nice file", null, null) + quotedFooter);
|
||||
|
||||
List<Comment> comments = defaultComments();
|
||||
List<MailComment> parsedComments =
|
||||
TextParser.parse(b.build(), comments, changeURL);
|
||||
|
||||
assertThat(parsedComments).hasSize(3);
|
||||
assertChangeMessage("Looks good to me", parsedComments.get(0));
|
||||
assertFileComment("This is a nice file", parsedComments.get(1),
|
||||
comments.get(1).key.filename);
|
||||
assertInlineComment("Also have a comment here.", parsedComments.get(2),
|
||||
comments.get(3));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void noComments() {
|
||||
MailMessage.Builder b = newMailMessageBuilder();
|
||||
b.textContent(newPlaintextBody(null, null, null, null, null, null, null) +
|
||||
quotedFooter);
|
||||
|
||||
List<Comment> comments = defaultComments();
|
||||
List<MailComment> parsedComments =
|
||||
TextParser.parse(b.build(), comments, changeURL);
|
||||
|
||||
assertThat(parsedComments).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void noChangeMessage() {
|
||||
MailMessage.Builder b = newMailMessageBuilder();
|
||||
b.textContent(newPlaintextBody(null, null, null,
|
||||
"Also have a comment here.", "This is a nice file", null, null) +
|
||||
quotedFooter);
|
||||
|
||||
List<Comment> comments = defaultComments();
|
||||
List<MailComment> parsedComments =
|
||||
TextParser.parse(b.build(), comments, changeURL);
|
||||
|
||||
assertThat(parsedComments).hasSize(2);
|
||||
assertFileComment("This is a nice file", parsedComments.get(0),
|
||||
comments.get(1).key.filename);
|
||||
assertInlineComment("Also have a comment here.", parsedComments.get(1),
|
||||
comments.get(3));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void allCommentsGmail() {
|
||||
MailMessage.Builder b = newMailMessageBuilder();
|
||||
b.textContent((newPlaintextBody("Looks good to me",
|
||||
null, null, "Also have a comment here.",
|
||||
"This is a nice file", null, null) + quotedFooter)
|
||||
.replace("> ", ">> "));
|
||||
|
||||
List<Comment> comments = defaultComments();
|
||||
List<MailComment> parsedComments =
|
||||
TextParser.parse(b.build(), comments, changeURL);
|
||||
|
||||
assertThat(parsedComments).hasSize(3);
|
||||
assertChangeMessage("Looks good to me", parsedComments.get(0));
|
||||
assertFileComment("This is a nice file", parsedComments.get(1),
|
||||
comments.get(1).key.filename);
|
||||
assertInlineComment("Also have a comment here.", parsedComments.get(2),
|
||||
comments.get(3));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void replyToFileComment() {
|
||||
MailMessage.Builder b = newMailMessageBuilder();
|
||||
b.textContent(newPlaintextBody("Looks good to me", null, null, null, null,
|
||||
null, "Comment in reply to file comment") + quotedFooter);
|
||||
|
||||
List<Comment> comments = defaultComments();
|
||||
List<MailComment> parsedComments =
|
||||
TextParser.parse(b.build(), comments, changeURL);
|
||||
|
||||
assertThat(parsedComments).hasSize(2);
|
||||
assertChangeMessage("Looks good to me", parsedComments.get(0));
|
||||
assertInlineComment("Comment in reply to file comment",
|
||||
parsedComments.get(1), comments.get(0));
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a plaintext message body with the specified comments.
|
||||
*
|
||||
* @param changeMessage
|
||||
* @param c1 Comment in reply to first inline comment.
|
||||
* @param c2 Comment in reply to second inline comment.
|
||||
* @param c3 Comment in reply to third inline comment.
|
||||
* @param f1 Comment on file one.
|
||||
* @param f2 Comment on file two.
|
||||
* @param fc1 Comment in reply to a comment of file 1.
|
||||
* @return A string with all inline comments and the original quoted email.
|
||||
*/
|
||||
private static String newPlaintextBody(String changeMessage, String c1,
|
||||
String c2, String c3, String f1, String f2, String fc1) {
|
||||
return (changeMessage == null ? "" : changeMessage + "\n") +
|
||||
"> Foo Bar has posted comments on this change. ( \n" +
|
||||
"> " + changeURL +"/1 )\n" +
|
||||
"> \n" +
|
||||
"> Change subject: Test change\n" +
|
||||
"> ...............................................................\n" +
|
||||
"> \n" +
|
||||
"> \n" +
|
||||
"> Patch Set 1: Code-Review+1\n" +
|
||||
"> \n" +
|
||||
"> (3 comments)\n" +
|
||||
"> \n" +
|
||||
"> " + changeURL + "/1/gerrit-server/test.txt\n" +
|
||||
"> File \n" +
|
||||
"> gerrit-server/test.txt:\n" +
|
||||
(f1 == null ? "" : f1 + "\n") +
|
||||
"> \n" +
|
||||
"> Patch Set #4:\n" +
|
||||
"> " + changeURL + "/1/gerrit-server/test.txt\n" +
|
||||
"> \n" +
|
||||
"> Some comment" +
|
||||
"> \n" +
|
||||
(fc1 == null ? "" : fc1 + "\n") +
|
||||
"> " + changeURL + "/1/gerrit-server/test.txt@2\n" +
|
||||
"> PS1, Line 2: throw new Exception(\"Object has unsupported: \" +\n" +
|
||||
"> : entry.getValue() +\n" +
|
||||
"> : \" must be java.util.Date\");\n" +
|
||||
"> Should entry.getKey() be included in this message?\n" +
|
||||
"> \n" +
|
||||
(c1 == null ? "" : c1 + "\n") +
|
||||
"> \n" +
|
||||
"> " + changeURL + "/1/gerrit-server/test.txt@3\n" +
|
||||
"> PS1, Line 3: throw new Exception(\"Object has: \" +\n" +
|
||||
"> : entry.getValue().getClass() +\n" +
|
||||
"> : \" must be java.util.Date\");\n" +
|
||||
"> same here\n" +
|
||||
"> \n" +
|
||||
(c2 == null ? "" : c2 + "\n") +
|
||||
"> \n" +
|
||||
"> " + changeURL + "/1/gerrit-server/readme.txt\n" +
|
||||
"> File \n" +
|
||||
"> gerrit-server/readme.txt:\n" +
|
||||
(f2 == null ? "" : f2 + "\n") +
|
||||
"> \n" +
|
||||
"> " + changeURL + "/1/gerrit-server/readme.txt@3\n" +
|
||||
"> PS1, Line 3: E\n" +
|
||||
"> Should this be EEE like in other places?\n" +
|
||||
(c3 == null ? "" : c3 + "\n");
|
||||
}
|
||||
|
||||
private List<Comment> defaultComments() {
|
||||
List<Comment> comments = new ArrayList<>();
|
||||
comments.add(newComment("c1", "gerrit-server/test.txt", "comment", 0));
|
||||
comments.add(newComment("c2", "gerrit-server/test.txt", "comment", 2));
|
||||
comments.add(newComment("c3", "gerrit-server/test.txt", "comment", 3));
|
||||
comments.add(newComment("c4", "gerrit-server/readme.txt", "comment", 3));
|
||||
return comments;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user