opendev/gerrit
Code Issues Proposed changes

Merge changes from topic 'email-ingestion'

Browse Source 
* changes:
  Add IMAP implementation for receiving emails
  Add POP3 implementation for receiving emails
  Implement RawMailParser
  Implement receiver class structure and bindings
This commit is contained in:
ekempin 2016-11-07 12:22:01 +00:00 committed by Gerrit Code Review
commit dff208be39
38 changed files with 1716 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
Documentation/config-gerrit.txt
View File

@ -3419,6 +3419,67 @@ which miscellaneous tasks are handled.
+
Default is 1.
[[receiveemail]]
=== Section receiveemail
[[receiveemail.protocol]]receiveemail.protocol::
+
Specifies the protocol used for receiving emails. Valid options are
'POP3', 'IMAP' and 'NONE'. Note that Gerrit will automatically switch between
POP3 and POP3s as well as IMAP and IMAPS depending on the specified
link:#receiveemail.encryption[encryption].
+
Defaults to 'NONE' which means that receiving emails is disabled.
[[receiveemail.host]]receiveemail.host::
+
The hostname of the mailserver. Example: 'imap.gmail.com'.
+
Defaults to an empty string which means that receiving emails is disabled.
[[receiveemail.port]]receiveemail.port::
+
The port the email server exposes for receving emails.
+
Defaults to the industry standard for a given protocol and encryption:
POP3: 110; POP3S: 995; IMAP: 143; IMAPS: 995.
[[receiveemail.username]]receiveemail.username::
+
Username used for authenticating with the email server.
+
Defaults to an empty string.
[[receiveemail.password]]receiveemail.password::
+
Password used for authenticating with the email server.
+
Defaults to an empty string.
[[receiveemail.encryption]]receiveemail.encryption::
+
Encryption standard used for transport layer security between Gerrit and the
email server. Possible values include 'NONE', 'SSL' and 'TLS'.
+
Defaults to 'NONE'.
[[receiveemail.fetchInterval]]receiveemail.fetchInterval::
+
Time between two consecutive fetches from the email server. Communication with
the email server is not kept alive. Examples: 60s, 10m, 1h.
+
Defaults to 60 seconds.
[[receiveemail.enableImapIdle]]receiveemail.enableImapIdle::
+
If the IMAP protocol is used for retrieving emails, IMAPv4 IDLE can be used to
keep the connection with the email server alive and receive a push when a new
email is delivered to the inbox. In this case, Gerrit will process the email
immediately and will not have a fetch delay.
+
Defaults to false.
[[sendemail]]
=== Section sendemail

30
WORKSPACE
View File

@ -381,6 +381,36 @@ maven_jar(
sha1 = '2e35862b0435c1b027a21f3d6eecbe50e6e08d54',
)
GREENMAIL_VERS = '1.5.2'
maven_jar(
name = 'greenmail',
artifact = 'com.icegreen:greenmail:' + GREENMAIL_VERS,
sha1 = '6b4862a09f8642da58c109117b24ccc19a4a6d39',
)
MAIL_VERS = '1.5.6'
maven_jar(
name = 'mail',
artifact = 'com.sun.mail:javax.mail:' + MAIL_VERS,
sha1 = 'ab5daef2f881c42c8e280cbe918ec4d7fdfd7efe',
)
MIME4J_VERS = '0.8.0'
maven_jar(
name = 'mime4j_core',
artifact = 'org.apache.james:apache-mime4j-core:' + MIME4J_VERS,
sha1 = 'd54f45fca44a2f210569656b4ca3574b42911c95',
)
maven_jar(
name = 'mime4j_dom',
artifact = 'org.apache.james:apache-mime4j-dom:' + MIME4J_VERS,
sha1 = '6720c93d14225c3e12c4a69768a0370c80e376a3',
)
OW2_VERS = '5.1'
maven_jar(

2
gerrit-acceptance-framework/BUCK
View File

@ -56,10 +56,12 @@ java_library(
'//lib:truth',
],
provided_deps = PROVIDED + [
'//lib/greenmail:greenmail',
'//lib:gwtorm',
'//lib/guice:guice',
'//lib/guice:guice-assistedinject',
'//lib/guice:guice-servlet',
'//lib/mail:mail',
],
visibility = ['PUBLIC'],
)

2
gerrit-acceptance-framework/BUILD
View File

@ -49,10 +49,12 @@ java_library2(
'//lib:truth',
],
deps = PROVIDED + [ # We want these deps to be exported_deps
'//lib/greenmail:greenmail',
'//lib:gwtorm',
'//lib/guice:guice',
'//lib/guice:guice-assistedinject',
'//lib/guice:guice-servlet',
'//lib/mail:mail',
],
visibility = ['//visibility:public'],
)

2
gerrit-acceptance-tests/BUCK
View File

@ -29,11 +29,13 @@ java_library(
'//lib/bouncycastle:bcpg',
'//lib/bouncycastle:bcprov',
'//lib/greenmail:greenmail',
'//lib/guice:guice',
'//lib/guice:guice-assistedinject',
'//lib/guice:guice-servlet',
'//lib/log:api',
'//lib/jgit/org.eclipse.jgit:jgit',
'//lib/mail:mail',
'//lib/mina:sshd',
],
visibility = [

7
gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/server/mail/BUCK Normal file
View File

@ -0,0 +1,7 @@
include_defs('//gerrit-acceptance-tests/tests.defs')
acceptance_tests(
group = 'server_mail',
srcs = glob(['*IT.java']),
labels = ['server'],
)

11
gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/server/mail/BUILD Normal file
View File

@ -0,0 +1,11 @@
load("//gerrit-acceptance-tests:tests.bzl", "acceptance_tests")
acceptance_tests(
srcs = glob(["*IT.java"]),
group = "server_mail",
labels = ["server"],
deps = [
"//lib/greenmail",
"//lib/mail",
],
)

106
gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/server/mail/MailIT.java Normal file
View File

@ -0,0 +1,106 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.acceptance.server.mail;
import static com.google.common.truth.Truth.assertThat;
import com.google.gerrit.acceptance.AbstractDaemonTest;
import com.google.gerrit.acceptance.GerritConfig;
import com.google.gerrit.acceptance.GerritConfigs;
import com.google.gerrit.acceptance.NoHttpd;
import com.google.gerrit.server.mail.receive.MailReceiver;
import com.google.gerrit.testutil.ConfigSuite;
import com.google.inject.Inject;
import com.icegreen.greenmail.junit.GreenMailRule;
import com.icegreen.greenmail.user.GreenMailUser;
import com.icegreen.greenmail.util.GreenMail;
import com.icegreen.greenmail.util.GreenMailUtil;
import com.icegreen.greenmail.util.ServerSetupTest;
import org.eclipse.jgit.lib.Config;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import javax.mail.internet.MimeMessage;
@NoHttpd
@RunWith(ConfigSuite.class)
public class MailIT extends AbstractDaemonTest {
private final static String RECEIVEEMAIL = "receiveemail";
private final static String HOST = "localhost";
private final static String USERNAME = "user@domain.com";
private final static String PASSWORD = "password";
@Inject
private MailReceiver mailReceiver;
@Inject
private GreenMail greenMail;
@Rule
public final GreenMailRule mockPop3Server = new GreenMailRule(
ServerSetupTest.SMTP_POP3_IMAP);
@ConfigSuite.Default
public static Config pop3Config() {
Config cfg = new Config();
cfg.setString(RECEIVEEMAIL, null, "host", HOST);
cfg.setString(RECEIVEEMAIL, null, "port", "3110");
cfg.setString(RECEIVEEMAIL, null, "username", USERNAME);
cfg.setString(RECEIVEEMAIL, null, "password", PASSWORD);
cfg.setString(RECEIVEEMAIL, null, "protocol", "POP3");
cfg.setString(RECEIVEEMAIL, null, "fetchInterval", "99");
return cfg;
}
@ConfigSuite.Config
public static Config imapConfig() {
Config cfg = new Config();
cfg.setString(RECEIVEEMAIL, null, "host", HOST);
cfg.setString(RECEIVEEMAIL, null, "port", "3143");
cfg.setString(RECEIVEEMAIL, null, "username", USERNAME);
cfg.setString(RECEIVEEMAIL, null, "password", PASSWORD);
cfg.setString(RECEIVEEMAIL, null, "protocol", "IMAP");
cfg.setString(RECEIVEEMAIL, null, "fetchInterval", "99");
return cfg;
}
@Test
public void testDelete() throws Exception {
GreenMailUser user = mockPop3Server.setUser(USERNAME, USERNAME, PASSWORD);
user.deliver(createSimpleMessage());
assertThat(mockPop3Server.getReceivedMessages().length).isEqualTo(1);
// Let Gerrit handle emails
mailReceiver.handleEmails();
// Check that the message is still present
assertThat(mockPop3Server.getReceivedMessages().length).isEqualTo(1);
// Mark the message for deletion
mailReceiver.requestDeletion(
mockPop3Server.getReceivedMessages()[0].getMessageID());
// Let Gerrit handle emails
mailReceiver.handleEmails();
// Check that the message was deleted
assertThat(mockPop3Server.getReceivedMessages().length).isEqualTo(0);
}
private MimeMessage createSimpleMessage() {
return GreenMailUtil
.createTextEmail(USERNAME, "from@localhost.com", "subject",
"body",
greenMail.getImap().getServerSetup());
}
}

2
gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java
View File

@ -68,6 +68,7 @@ import com.google.gerrit.server.index.DummyIndexModule;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gerrit.server.mail.SignedTokenEmailTokenVerifier;
import com.google.gerrit.server.mail.receive.MailReceiver;
import com.google.gerrit.server.mail.send.SmtpEmailSender;
import com.google.gerrit.server.mime.MimeUtil2Module;
import com.google.gerrit.server.patch.DiffExecutorModule;
@ -362,6 +363,7 @@ public class Daemon extends SiteProgram {
modules.add(new SearchingChangeCacheImpl.Module(slave));
modules.add(new InternalAccountDirectory.Module());
modules.add(new DefaultCacheFactory.Module());
modules.add(cfgInjector.getInstance(MailReceiver.Module.class));
if (emailModule != null) {
modules.add(emailModule);
} else {

2
gerrit-pgm/src/main/java/com/google/gerrit/pgm/init/InitSendEmail.java
View File

@ -21,7 +21,7 @@ import com.google.gerrit.pgm.init.api.ConsoleUI;
import com.google.gerrit.pgm.init.api.InitStep;
import com.google.gerrit.pgm.init.api.Section;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.mail.send.SmtpEmailSender.Encryption;
import com.google.gerrit.server.mail.Encryption;
import com.google.inject.Inject;
import com.google.inject.Singleton;

2
gerrit-server/BUCK
View File

@ -70,6 +70,8 @@ java_library(
'//lib/lucene:lucene-analyzers-common',
'//lib/lucene:lucene-core-and-backward-codecs',
'//lib/lucene:lucene-queryparser',
'//lib/mime4j:core',
'//lib/mime4j:dom',
'//lib/ow2:ow2-asm',
'//lib/ow2:ow2-asm-tree',
'//lib/ow2:ow2-asm-util',

2
gerrit-server/BUILD
View File

@ -72,6 +72,8 @@ java_library(
'//lib/lucene:lucene-analyzers-common',
'//lib/lucene:lucene-core-and-backward-codecs',
'//lib/lucene:lucene-queryparser',
'//lib/mime4j:core',
'//lib/mime4j:dom',
'//lib/ow2:ow2-asm',
'//lib/ow2:ow2-asm-tree',
'//lib/ow2:ow2-asm-util',

11
gerrit-server/src/main/java/com/google/gerrit/server/mail/Address.java
View File

@ -24,7 +24,16 @@ public class Address {
if (0 <= lt && lt < gt && lt + 1 < at && at + 1 < gt) {
final String email = in.substring(lt + 1, gt).trim();
final String name = in.substring(0, lt).trim();
return new Address(name.length() > 0 ? name : null, email);
int nameStart = 0;
int nameEnd = name.length();
if (name.startsWith("\"")) {
nameStart++;
}
if (name.endsWith("\"")) {
nameEnd--;
}
return new Address(name.length() > 0 ?
name.substring(nameStart, nameEnd): null, email);
}
if (lt < 0 && gt < 0 && 0 < at && at < in.length() - 1) {

32
gerrit-server/src/main/java/com/google/gerrit/server/mail/EmailSettings.java
View File

@ -15,21 +15,47 @@
package com.google.gerrit.server.mail;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.mail.receive.Protocol;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.eclipse.jgit.lib.Config;
import java.util.concurrent.TimeUnit;
@Singleton
public class EmailSettings {
private static final String SEND_EMAL = "sendemail";
private static final String RECEIVE_EMAL = "receiveemail";
// Send
public final boolean html;
public final boolean includeDiff;
public final int maximumDiffSize;
// Receive
public final Protocol protocol;
public final String host;
public final int port;
public final String username;
public final String password;
public final Encryption encryption;
public final long fetchInterval; // in milliseconds
@Inject
EmailSettings(@GerritServerConfig Config cfg) {
html = cfg.getBoolean("sendemail", "html", true);
includeDiff = cfg.getBoolean("sendemail", "includeDiff", false);
maximumDiffSize = cfg.getInt("sendemail", "maximumDiffSize", 256 << 10);
// Send
html = cfg.getBoolean(SEND_EMAL, "html", true);
includeDiff = cfg.getBoolean(SEND_EMAL, "includeDiff", false);
maximumDiffSize = cfg.getInt(SEND_EMAL, "maximumDiffSize", 256 << 10);
// Receive
protocol = cfg.getEnum(RECEIVE_EMAL, null, "protocol", Protocol.NONE);
host = cfg.getString(RECEIVE_EMAL, null, "host");
port = cfg.getInt(RECEIVE_EMAL, "port", 0);
username = cfg.getString(RECEIVE_EMAL, null, "username");
password = cfg.getString(RECEIVE_EMAL, null, "password");
encryption =
cfg.getEnum(RECEIVE_EMAL, null, "encryption", Encryption.NONE);
fetchInterval = cfg.getTimeUnit(RECEIVE_EMAL, null, "fetchInterval",
TimeUnit.MILLISECONDS.convert(60, TimeUnit.SECONDS),
TimeUnit.MILLISECONDS);
}
}

19
gerrit-server/src/main/java/com/google/gerrit/server/mail/Encryption.java Normal file
View File

@ -0,0 +1,19 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail;
public enum Encryption {
NONE, SSL, TLS
}

143
gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/ImapMailReceiver.java Normal file
View File

@ -0,0 +1,143 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail.receive;
import com.google.gerrit.server.mail.EmailSettings;
import com.google.gerrit.server.mail.Encryption;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.commons.net.imap.IMAPClient;
import org.apache.commons.net.imap.IMAPSClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@Singleton
public class ImapMailReceiver extends MailReceiver {
private static final Logger log =
LoggerFactory.getLogger(ImapMailReceiver.class);
private static final String inboxFolder = "INBOX";
@Inject
public ImapMailReceiver(EmailSettings mailSettings) {
super(mailSettings);
}
/**
* handleEmails will open a connection to the mail server, remove emails
* where deletion is pending, read new email and close the connection.
*/
@Override
public synchronized void handleEmails() {
IMAPClient imap;
if (mailSettings.encryption != Encryption.NONE) {
imap = new IMAPSClient(mailSettings.encryption.name(), false);
} else {
imap = new IMAPClient();
}
if (mailSettings.port > 0) {
imap.setDefaultPort(mailSettings.port);
}
// Set a 30s timeout for each operation
imap.setDefaultTimeout(30 * 1000);
try {
imap.connect(mailSettings.host);
try {
if (!imap.login(mailSettings.username, mailSettings.password)) {
log.error("Could not login to IMAP server");
return;
}
try {
if (!imap.select(inboxFolder)){
log.error("Could not select IMAP folder " + inboxFolder);
return;
}
// Fetch just the internal dates first to know how many messages we
// should fetch.
if (!imap.fetch("1:*", "(INTERNALDATE)")) {
log.error("IMAP fetch failed. Will retry in next fetch cycle.");
return;
}
// Format of reply is one line per email and one line to indicate
// that the fetch was successful.
// Example:
// * 1 FETCH (INTERNALDATE "Mon, 24 Oct 2016 16:53:22 +0200 (CEST)")
// * 2 FETCH (INTERNALDATE "Mon, 24 Oct 2016 16:53:22 +0200 (CEST)")
// AAAC OK FETCH completed.
int numMessages = imap.getReplyStrings().length - 1;
log.info("Fetched " + numMessages + " messages via IMAP");
if (numMessages == 0) {
return;
}
// Fetch the full version of all emails
List<MailMessage> mailMessages = new ArrayList<>(numMessages);
for (int i = 1; i <= numMessages; i++) {
if (imap.fetch(i + ":" + i, "(BODY.PEEK[])")) {
// Obtain full reply
String[] rawMessage = imap.getReplyStrings();
if (rawMessage.length < 2) {
continue;
}
// First and last line are IMAP status codes. We have already
// checked, that the fetch returned true (OK), so we safely ignore
// those two lines.
StringBuilder b = new StringBuilder(2 * (rawMessage.length - 2));
for(int j = 1; j < rawMessage.length - 1; j++) {
if (j > 1) {
b.append("\n");
}
b.append(rawMessage[j]);
}
try {
MailMessage mailMessage = RawMailParser.parse(b.toString());
if (pendingDeletion.contains(mailMessage.id())) {
// Mark message as deleted
if (imap.store(i + ":" + i, "+FLAGS", "(\\Deleted)")) {
pendingDeletion.remove(mailMessage.id());
} else {
log.error("Could not mark mail message as deleted: " +
mailMessage.id());
}
} else {
mailMessages.add(mailMessage);
}
} catch (MailParsingException e) {
log.error("Exception while parsing email after IMAP fetch", e);
}
} else {
log.error("IMAP fetch failed. Will retry in next fetch cycle.");
}
}
// Permanently delete emails marked for deletion
if (!imap.expunge()) {
log.error("Could not expunge IMAP emails");
}
// TODO(hiesel) Call email handling logic with mailMessages
} finally {
imap.logout();
}
} finally {
imap.disconnect();
}
} catch (IOException e) {
log.error("Error while talking to IMAP server", e);
return;
}
}
}

89
gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/MailMessage.java Normal file
View File

@ -0,0 +1,89 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail.receive;
import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableList;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.server.mail.Address;
import org.joda.time.DateTime;
/**
* MailMessage is a simplified representation of an RFC 2045-2047 mime email
* message used for representing received emails inside Gerrit. It is populated
* by the MailParser after MailReceiver has received a message. Transformations
* done by the parser include stitching mime parts together, transforming all
* content to UTF-16 and removing attachments.
*
* A valid MailMessage contains at least the following fields: id, from, to,
* subject and dateReceived.
*/
@AutoValue
public abstract class MailMessage {
// Unique Identifier
public abstract String id();
// Envelop Information
public abstract Address from();
public abstract ImmutableList<Address> to();
@Nullable
public abstract ImmutableList<Address> cc();
// Metadata
public abstract DateTime dateReceived();
public abstract ImmutableList<String> additionalHeaders();
// Content
public abstract String subject();
@Nullable
public abstract String textContent();
@Nullable
public abstract String htmlContent();
public static Builder builder() {
return new AutoValue_MailMessage.Builder();
}
@AutoValue.Builder
public abstract static class Builder {
public abstract Builder id(String val);
public abstract Builder from(Address val);
public abstract ImmutableList.Builder<Address> toBuilder();
public Builder addTo(Address val) {
toBuilder().add(val);
return this;
}
public abstract ImmutableList.Builder<Address> ccBuilder();
public Builder addCc(Address val) {
ccBuilder().add(val);
return this;
}
public abstract Builder dateReceived(DateTime val);
public abstract ImmutableList.Builder<String> additionalHeadersBuilder();
public Builder addAdditionalHeader(String val) {
additionalHeadersBuilder().add(val);
return this;
}
public abstract Builder subject(String val);
public abstract Builder textContent(String val);
public abstract Builder htmlContent(String val);
public abstract MailMessage build();
}
}

28
gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/MailParsingException.java Normal file
View File

@ -0,0 +1,28 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail.receive;
/** MailParsingException indicates that an email could not be parsed. */
public class MailParsingException extends Exception {
private static final long serialVersionUID = 1L;
public MailParsingException(String msg) {
super(msg);
}
public MailParsingException(String msg, Throwable cause) {
super(msg, cause);
}
}

106
gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/MailReceiver.java Normal file
View File

@ -0,0 +1,106 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail.receive;
import com.google.common.annotations.VisibleForTesting;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.mail.EmailSettings;
import com.google.inject.Inject;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
/** MailReceiver implements base functionality for receiving emails. */
public abstract class MailReceiver implements LifecycleListener {
protected EmailSettings mailSettings;
protected Set<String> pendingDeletion;
private Timer timer;
public static class Module extends LifecycleModule {
private final EmailSettings mailSettings;
@Inject
Module(EmailSettings mailSettings) {
this.mailSettings = mailSettings;
}
@Override
protected void configure() {
if (mailSettings.protocol == Protocol.NONE) {
return;
}
listener().to(MailReceiver.class);
switch (mailSettings.protocol) {
case IMAP:
bind(MailReceiver.class).to(ImapMailReceiver.class);
break;
case POP3:
bind(MailReceiver.class).to(Pop3MailReceiver.class);
break;
case NONE:
default:
}
}
}
@Inject
public MailReceiver(EmailSettings mailSettings) {
this.mailSettings = mailSettings;
pendingDeletion = Collections.synchronizedSet(new HashSet<>());
}
@Override
public void start() {
if (timer == null) {
timer = new Timer();
} else {
timer.cancel();
}
timer.scheduleAtFixedRate(new TimerTask() {
@Override
public void run() {
MailReceiver.this.handleEmails();
}
}, 0l, mailSettings.fetchInterval);
}
@Override
public void stop() {
if (timer != null) {
timer.cancel();
}
}
/**
* requestDeletion will enqueue an email for deletion and delete it the
* next time we connect to the email server. This does not guarantee deletion
* as the Gerrit instance might fail before we connect to the email server.
* @param messageId
*/
public void requestDeletion(String messageId) {
pendingDeletion.add(messageId);
}
/**
* handleEmails will open a connection to the mail server, remove emails
* where deletion is pending, read new email and close the connection.
*/
@VisibleForTesting
public abstract void handleEmails();
}

148
gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/Pop3MailReceiver.java Normal file
View File

@ -0,0 +1,148 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail.receive;
import com.google.common.primitives.Ints;
import com.google.gerrit.server.mail.EmailSettings;
import com.google.gerrit.server.mail.Encryption;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.net.pop3.POP3Client;
import org.apache.commons.net.pop3.POP3MessageInfo;
import org.apache.commons.net.pop3.POP3SClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Singleton
public class Pop3MailReceiver extends MailReceiver {
private static final Logger log =
LoggerFactory.getLogger(Pop3MailReceiver.class);
@Inject
public Pop3MailReceiver(EmailSettings mailSettings) {
super(mailSettings);
}
/**
* handleEmails will open a connection to the mail server, remove emails
* where deletion is pending, read new email and close the connection.
*/
@Override
public synchronized void handleEmails() {
POP3Client pop3;
if (mailSettings.encryption != Encryption.NONE) {
pop3 = new POP3SClient(mailSettings.encryption.name());
} else {
pop3 = new POP3Client();
}
if (mailSettings.port > 0) {
pop3.setDefaultPort(mailSettings.port);
}
try {
pop3.connect(mailSettings.host);
} catch (IOException e) {
log.error("Could not connect to POP3 email server", e);
return;
}
try {
try {
if (!pop3.login(mailSettings.username, mailSettings.password)) {
log.error("Could not login to POP3 email server."
+ " Check username and password");
return;
}
try {
POP3MessageInfo[] messages = pop3.listMessages();
if (messages == null) {
log.error("Could not retrieve message list via POP3");
return;
}
log.info("Received " + messages.length + " messages via POP3");
// Fetch messages
List<MailMessage> mailMessages = new ArrayList<>();
for (POP3MessageInfo msginfo : messages) {
if (msginfo == null) {
// Message was deleted
continue;
}
Reader reader = pop3.retrieveMessage(msginfo.number);
if (reader == null) {
log.error("Could not retrieve POP3 message header for message " +
msginfo.identifier);
return;
}
int[] message = fetchMessage(reader);
try {
MailMessage mailMessage = RawMailParser.parse(message);
// Delete messages where deletion is pending. This requires
// knowing the integer message ID of the email. We therefore parse
// the message first and extract the Message-ID specified in RFC
// 822 and delete the message if deletion is pending.
if (pendingDeletion.contains(mailMessage.id())) {
if (pop3.deleteMessage(msginfo.number)) {
pendingDeletion.remove(mailMessage.id());
} else {
log.error("Could not delete message " + msginfo.number);
}
} else {
// Process message further
mailMessages.add(mailMessage);
}
} catch (MailParsingException e) {
log.error("Could not parse message " + msginfo.number);
}
}
// TODO(hiesel) Call processing logic with mailMessages
} finally {
pop3.logout();
}
} finally {
pop3.disconnect();
}
} catch (IOException e) {
log.error("Error while issuing POP3 command", e);
}
}
public final int[] fetchMessage(Reader reader) throws IOException {
BufferedReader bufferedReader;
if (reader instanceof BufferedReader) {
bufferedReader = (BufferedReader) reader;
} else {
bufferedReader = new BufferedReader(reader);
}
try {
List<Integer> character = new ArrayList<>();
int ch;
while ((ch = bufferedReader.read()) != -1) {
character.add(ch);
}
return Ints.toArray(character);
} finally {
bufferedReader.close();
if (bufferedReader != reader) {
reader.close();
}
}
}
}

19
gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/Protocol.java Normal file
View File

@ -0,0 +1,19 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail.receive;
public enum Protocol {
NONE, POP3, IMAP
}

170
gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/RawMailParser.java Normal file
View File

@ -0,0 +1,170 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail.receive;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import com.google.common.io.CharStreams;
import com.google.gerrit.server.mail.Address;
import org.apache.james.mime4j.MimeException;
import org.apache.james.mime4j.dom.Entity;
import org.apache.james.mime4j.dom.Message;
import org.apache.james.mime4j.dom.MessageBuilder;
import org.apache.james.mime4j.dom.Multipart;
import org.apache.james.mime4j.dom.TextBody;
import org.apache.james.mime4j.dom.address.Mailbox;
import org.apache.james.mime4j.message.DefaultMessageBuilder;
import org.joda.time.DateTime;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
/**
* RawMailParser parses raw email content received through POP3 or IMAP into
* an internal {@link MailMessage}.
*/
public class RawMailParser {
private static final ImmutableSet<String> MAIN_HEADERS =
ImmutableSet.of("to", "from", "cc", "date", "message-id",
"subject", "content-type");
/**
* Parses a MailMessage from a string.
* @param raw String as received over the wire
* @return Parsed MailMessage
* @throws MailParsingException
*/
public static MailMessage parse(String raw) throws MailParsingException {
MailMessage.Builder messageBuilder = MailMessage.builder();
Message mimeMessage;
try {
MessageBuilder builder = new DefaultMessageBuilder();
mimeMessage =
builder.parseMessage(new ByteArrayInputStream(raw.getBytes()));
} catch (IOException | MimeException e) {
throw new MailParsingException("Can't parse email", e);
}
// Add general headers
messageBuilder.id(mimeMessage.getMessageId());
messageBuilder.subject(mimeMessage.getSubject());
messageBuilder.dateReceived(new DateTime(mimeMessage.getDate()));
// Add From, To and Cc
if (mimeMessage.getFrom() != null && mimeMessage.getFrom().size() > 0) {
Mailbox from = mimeMessage.getFrom().get(0);
messageBuilder.from(new Address(from.getName(), from.getAddress()));
}
if (mimeMessage.getTo() != null) {
for (Mailbox m : mimeMessage.getTo().flatten()) {
messageBuilder.addTo(new Address(m.getName(), m.getAddress()));
}
}
if (mimeMessage.getCc() != null) {
for (Mailbox m : mimeMessage.getCc().flatten()) {
messageBuilder.addCc(new Address(m.getName(), m.getAddress()));
}
}
// Add additional headers
mimeMessage.getHeader().getFields().stream()
.filter(f -> !MAIN_HEADERS.contains(f.getName().toLowerCase()))
.forEach(f -> messageBuilder.addAdditionalHeader(
f.getName() + ": " + f.getBody()));
// Add text and html body parts
StringBuilder textBuilder = new StringBuilder();
StringBuilder htmlBuilder = new StringBuilder();
try {
handleMimePart(mimeMessage, textBuilder, htmlBuilder);
} catch (IOException e) {
throw new MailParsingException("Can't parse email", e);
}
messageBuilder.textContent(Strings.emptyToNull(textBuilder.toString()));
messageBuilder.htmlContent(Strings.emptyToNull(htmlBuilder.toString()));
try {
// build() will only succeed if all required attributes were set. We wrap
// the IllegalStateException in a MailParsingException indicating that
// required attributes are missing, so that the caller doesn't fall over.
return messageBuilder.build();
} catch (IllegalStateException e) {
throw new MailParsingException(
"Missing required attributes after email was parsed", e);
}
}
/**
* Parses a MailMessage from an array of characters. Note that the character
* array is int-typed. This method is only used by POP3, which specifies that
* all transferred characters are US-ASCII (RFC 6856). When reading the input
* in Java, io.Reader yields ints. These can be safely converted to chars
* as all US-ASCII characters fit in a char. If emails contain non-ASCII
* characters, such as UTF runes, these will be encoded in ASCII using either
* Base64 or quoted-printable encoding.
* @param chars Array as received over the wire
* @return Parsed MailMessage
* @throws MailParsingException
*/
public static MailMessage parse(int[] chars) throws MailParsingException {
StringBuilder b = new StringBuilder(chars.length);
for (int c : chars) {
b.append((char) c);
}
return parse(b.toString());
}
/**
* Traverses a mime tree and parses out text and html parts. All other parts
* will be dropped.
* @param part MimePart to parse
* @param textBuilder StringBuilder to append all plaintext parts
* @param htmlBuilder StringBuilder to append all html parts
* @throws IOException
*/
private static void handleMimePart(Entity part, StringBuilder textBuilder,
StringBuilder htmlBuilder) throws IOException {
if (isPlainOrHtml(part.getMimeType()) &&
!isAttachment(part.getDispositionType())) {
TextBody tb = (TextBody) part.getBody();
String result = CharStreams.toString(new InputStreamReader(
tb.getInputStream(), tb.getMimeCharset()));
if (part.getMimeType().equals("text/plain")) {
textBuilder.append(result);
} else if (part.getMimeType().equals("text/html")) {
htmlBuilder.append(result);
}
} else if (isMixedOrAlternative(part.getMimeType())) {
Multipart multipart = (Multipart) part.getBody();
for (Entity e : multipart.getBodyParts()) {
handleMimePart(e, textBuilder, htmlBuilder);
}
}
}
private static boolean isPlainOrHtml(String mimeType) {
return (mimeType.equals("text/plain") || mimeType.equals("text/html"));
}
private static boolean isMixedOrAlternative(String mimeType) {
return mimeType.equals("multipart/alternative") ||
mimeType.equals("multipart/mixed");
}
private static boolean isAttachment(String dispositionType) {
return dispositionType != null && dispositionType.equals("attachment");
}
}

5
gerrit-server/src/main/java/com/google/gerrit/server/mail/send/SmtpEmailSender.java
View File

@ -25,6 +25,7 @@ import com.google.gerrit.common.errors.EmailException;
import com.google.gerrit.server.config.ConfigUtil;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.mail.Address;
import com.google.gerrit.server.mail.Encryption;
import com.google.inject.AbstractModule;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@ -61,10 +62,6 @@ public class SmtpEmailSender implements EmailSender {
}
}
public enum Encryption {
NONE, SSL, TLS
}
private final boolean enabled;
private final int connectTimeout;

76
gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/RawMailParserTest.java Normal file
View File

@ -0,0 +1,76 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail.receive;
import static com.google.common.truth.Truth.assertThat;
import com.google.gerrit.server.mail.receive.data.AttachmentMessage;
import com.google.gerrit.server.mail.receive.data.Base64HeaderMessage;
import com.google.gerrit.server.mail.receive.data.HtmlMimeMessage;
import com.google.gerrit.server.mail.receive.data.NonUTF8Message;
import com.google.gerrit.server.mail.receive.data.QuotedPrintableHeaderMessage;
import com.google.gerrit.server.mail.receive.data.RawMailMessage;
import com.google.gerrit.server.mail.receive.data.SimpleTextMessage;
import com.google.gerrit.testutil.GerritBaseTests;
import org.junit.Test;
public class RawMailParserTest extends GerritBaseTests {
@Test
public void testParseEmail() throws Exception {
RawMailMessage[] messages = new RawMailMessage[] {
new SimpleTextMessage(),
new Base64HeaderMessage(),
new QuotedPrintableHeaderMessage(),
new HtmlMimeMessage(),
new AttachmentMessage(),
new NonUTF8Message(),
};
for (RawMailMessage rawMailMessage : messages) {
if (rawMailMessage.rawChars() != null) {
// Assert Character to Mail Parser
MailMessage parsedMailMessage =
RawMailParser.parse(rawMailMessage.rawChars());
assertMail(parsedMailMessage, rawMailMessage.expectedMailMessage());
}
if (rawMailMessage.raw() != null) {
// Assert String to Mail Parser
MailMessage parsedMailMessage = RawMailParser
.parse(rawMailMessage.raw());
assertMail(parsedMailMessage, rawMailMessage.expectedMailMessage());
}
}
}
/**
* This method makes it easier to debug failing tests by checking each
* property individual instead of calling equals as it will immediately
* reveal the property that diverges between the two objects.
* @param have MailMessage retrieved from the parser
* @param want MailMessage that would be expected
*/
private void assertMail(MailMessage have, MailMessage want) {
assertThat(have.id()).isEqualTo(want.id());
assertThat(have.to()).isEqualTo(want.to());
assertThat(have.from()).isEqualTo(want.from());
assertThat(have.cc()).isEqualTo(want.cc());
assertThat(have.dateReceived().getMillis())
.isEqualTo(want.dateReceived().getMillis());
assertThat(have.additionalHeaders()).isEqualTo(want.additionalHeaders());
assertThat(have.subject()).isEqualTo(want.subject());
assertThat(have.textContent()).isEqualTo(want.textContent());
assertThat(have.htmlContent()).isEqualTo(want.htmlContent());
}
}

89
gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/data/AttachmentMessage.java Normal file
View File

@ -0,0 +1,89 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail.receive.data;
import com.google.gerrit.server.mail.Address;
import com.google.gerrit.server.mail.receive.MailMessage;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
/**
* Tests that all mime parts that are neither text/plain, nor text/html are
* dropped.
*/
public class AttachmentMessage extends RawMailMessage {
private static String raw = "MIME-Version: 1.0\n" +
"Date: Tue, 25 Oct 2016 02:11:35 -0700\n" +
"Message-ID: <CAM7sg=3meaAVUxW3KXeJEVs8sv_ADw1BnvpcHHiYVR2TQQi__w" +
"@mail.gmail.com>\n" +
"Subject: Test Subject\n" +
"From: Patrick Hiesel <hiesel@google.com>\n" +
"To: Patrick Hiesel <hiesel@google.com>\n" +
"Content-Type: multipart/mixed; boundary=001a114e019a56962d054062708f\n" +
"\n" +
"--001a114e019a56962d054062708f\n" +
"Content-Type: multipart/alternative; boundary=001a114e019a5696250540" +
"62708d\n" +
"\n" +
"--001a114e019a569625054062708d\n" +
"Content-Type: text/plain; charset=UTF-8\n" +
"\n" +
"Contains unwanted attachment" +
"\n" +
"--001a114e019a569625054062708d\n" +
"Content-Type: text/html; charset=UTF-8\n" +
"\n" +
"<div dir=\"ltr\">Contains unwanted attachment</div>" +
"\n" +
"--001a114e019a569625054062708d--\n" +
"--001a114e019a56962d054062708f\n" +
"Content-Type: text/plain; charset=US-ASCII; name=\"test.txt\"\n" +
"Content-Disposition: attachment; filename=\"test.txt\"\n" +
"Content-Transfer-Encoding: base64\n" +
"X-Attachment-Id: f_iv264bt50\n" +
"\n" +
"VEVTVAo=\n" +
"--001a114e019a56962d054062708f--";
@Override
public String raw() {
return raw;
}
@Override
public int[] rawChars() {
return null;
}
@Override
public MailMessage expectedMailMessage() {
System.out.println("\uD83D\uDE1B test");
MailMessage.Builder expect = MailMessage.builder();
expect
.id("<CAM7sg=3meaAVUxW3KXeJEVs8sv_ADw1BnvpcHHiYVR2TQQi__w" +
"@mail.gmail.com>")
.from(new Address("Patrick Hiesel", "hiesel@google.com"))
.addTo(new Address("Patrick Hiesel", "hiesel@google.com"))
.textContent("Contains unwanted attachment")
.htmlContent("<div dir=\"ltr\">Contains unwanted attachment</div>")
.subject("Test Subject")
.addAdditionalHeader("MIME-Version: 1.0")
.dateReceived(
new DateTime(2016, 10, 25, 9, 11, 35, 0, DateTimeZone.UTC));
return expect.build();
}
}

62
gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/data/Base64HeaderMessage.java Normal file
View File

@ -0,0 +1,62 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail.receive.data;
import com.google.gerrit.server.mail.Address;
import com.google.gerrit.server.mail.receive.MailMessage;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
/**
* Tests parsing a Base64 encoded subject.
*/
public class Base64HeaderMessage extends RawMailMessage {
private static String textContent = "Some Text";
private static String raw = "" +
"Date: Tue, 25 Oct 2016 02:11:35 -0700\n" +
"Message-ID: <001a114da7ae26e2eb053fe0c29c@google.com>\n" +
"Subject: =?UTF-8?B?8J+YmyB0ZXN0?=\n" +
"From: \"Jonathan Nieder (Gerrit)\" <noreply-gerritcodereview-" +
"CtTy0igsBrnvL7dKoWEIEg@google.com>\n" +
"To: ekempin <ekempin@google.com>\n" +
"Content-Type: text/plain; charset=UTF-8; format=flowed; delsp=yes\n" +
"\n" + textContent;
@Override
public String raw() {
return raw;
}
@Override
public int[] rawChars() {
return null;
}
@Override
public MailMessage expectedMailMessage() {
MailMessage.Builder expect = MailMessage.builder();
expect
.id("<001a114da7ae26e2eb053fe0c29c@google.com>")
.from(new Address("Jonathan Nieder (Gerrit)",
"noreply-gerritcodereview-CtTy0igsBrnvL7dKoWEIEg@google.com"))
.addTo(new Address("ekempin","ekempin@google.com"))
.textContent(textContent)
.subject("\uD83D\uDE1B test")
.dateReceived(
new DateTime(2016, 10, 25, 9, 11, 35, 0, DateTimeZone.UTC));
return expect.build();
}
}

103
gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/data/HtmlMimeMessage.java Normal file
View File

@ -0,0 +1,103 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail.receive.data;
import com.google.gerrit.server.mail.Address;
import com.google.gerrit.server.mail.receive.MailMessage;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
/**
* Tests a message containing mime/alternative (text + html) content.
*/
public class HtmlMimeMessage extends RawMailMessage {
private static String textContent = "Simple test";
// htmlContent is encoded in quoted-printable
private static String htmlContent = "<div dir=3D\"ltr\">Test <span style" +
"=3D\"background-color:rgb(255,255,0)\">Messa=\n" +
"ge</span> in <u>HTML=C2=A0</u><a href=3D\"https://en.wikipedia.org/" +
"wiki/%C3%=\n9Cmlaut_(band)\" class=3D\"gmail-mw-redirect\" title=3D\"" +
"=C3=9Cmlaut (band)\" st=\nyle=3D\"text-decoration:none;color:rgb(11," +
"0,128);background-image:none;backg=\nround-position:initial;background" +
"-size:initial;background-repeat:initial;ba=\nckground-origin:initial;" +
"background-clip:initial;font-family:sans-serif;font=\n" +
"-size:14px\">=C3=9C</a></div>";
private static String unencodedHtmlContent = "" +
"<div dir=\"ltr\">Test <span style=\"background-color:rgb(255,255,0)\">" +
"Message</span> in <u>HTML </u><a href=\"https://en.wikipedia.org/wiki/" +
"%C3%9Cmlaut_(band)\" class=\"gmail-mw-redirect\" title=\"Ümlaut " +
"(band)\" style=\"text-decoration:none;color:rgb(11,0,128);" +
"background-image:none;background-position:initial;background-size:" +
"initial;background-repeat:initial;background-origin:initial;background" +
"-clip:initial;font-family:sans-serif;font-size:14px\">Ü</a></div>";
private static String raw = "" +
"MIME-Version: 1.0\n" +
"Date: Tue, 25 Oct 2016 02:11:35 -0700\n" +
"Message-ID: <001a114cd8be55b4ab053face5cd@google.com>\n" +
"Subject: Change in gerrit[master]: Implement receiver class structure " +
"and bindings\n" +
"From: \"ekempin (Gerrit)\" <noreply-gerritcodereview-qUgXfQecoDLHwp0Ml" +
"dAzig@google.com>\n" +
"To: Patrick Hiesel <hiesel@google.com>\n" +
"Cc: ekempin <ekempin@google.com>\n" +
"Content-Type: multipart/alternative; boundary=001a114cd8b" +
"e55b486053face5ca\n" +
"\n" +
"--001a114cd8be55b486053face5ca\n" +
"Content-Type: text/plain; charset=UTF-8; format=flowed; delsp=yes\n" +
"\n" +
textContent +
"\n" +
"--001a114cd8be55b486053face5ca\n" +
"Content-Type: text/html; charset=UTF-8\n" +
"Content-Transfer-Encoding: quoted-printable\n" +
"\n" +
htmlContent +
"\n" +
"--001a114cd8be55b486053face5ca--";
@Override
public String raw() {
return raw;
}
@Override
public int[] rawChars() {
return null;
}
@Override
public MailMessage expectedMailMessage() {
MailMessage.Builder expect = MailMessage.builder();
expect
.id("<001a114cd8be55b4ab053face5cd@google.com>")
.from(new Address("ekempin (Gerrit)",
"noreply-gerritcodereview-qUgXfQecoDLHwp0MldAzig@google.com"))
.addCc(new Address("ekempin","ekempin@google.com"))
.addTo(new Address("Patrick Hiesel","hiesel@google.com"))
.textContent(textContent)
.htmlContent(unencodedHtmlContent)
.subject("Change in gerrit[master]: Implement " +
"receiver class structure and bindings")
.addAdditionalHeader("MIME-Version: 1.0")
.dateReceived(
new DateTime(2016, 10, 25, 9, 11, 35, 0, DateTimeZone.UTC));
return expect.build();
}
}

66
gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/data/NonUTF8Message.java Normal file
View File

@ -0,0 +1,66 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail.receive.data;
import com.google.gerrit.server.mail.Address;
import com.google.gerrit.server.mail.receive.MailMessage;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
/**
* Tests that non-UTF8 encodings are handled correctly.
*/
public class NonUTF8Message extends RawMailMessage {
private static String textContent = "Some Text";
private static String raw = "" +
"Date: Tue, 25 Oct 2016 02:11:35 -0700\n" +
"Message-ID: <001a114da7ae26e2eb053fe0c29c@google.com>\n" +
"Subject: =?UTF-8?B?8J+YmyB0ZXN0?=\n" +
"From: \"Jonathan Nieder (Gerrit)\" <noreply-gerritcodereview-" +
"CtTy0igsBrnvL7dKoWEIEg@google.com>\n" +
"To: ekempin <ekempin@google.com>\n" +
"Content-Type: text/plain; charset=UTF-8; format=flowed; delsp=yes\n" +
"\n" + textContent;
@Override
public String raw() {
return null;
}
@Override
public int[] rawChars() {
int[] arr = new int[raw.length()];
int i = 0;
for (char c : raw.toCharArray()) {
arr[i++] = (int) c;
}
return arr;
}
@Override
public MailMessage expectedMailMessage() {
MailMessage.Builder expect = MailMessage.builder();
expect
.id("<001a114da7ae26e2eb053fe0c29c@google.com>")
.from(new Address("Jonathan Nieder (Gerrit)",
"noreply-gerritcodereview-CtTy0igsBrnvL7dKoWEIEg@google.com"))
.addTo(new Address("ekempin","ekempin@google.com"))
.textContent(textContent)
.subject("\uD83D\uDE1B test")
.dateReceived(
new DateTime(2016, 10, 25, 9, 11, 35, 0, DateTimeZone.UTC));
return expect.build();
}
}

63
gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/data/QuotedPrintableHeaderMessage.java Normal file
View File

@ -0,0 +1,63 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail.receive.data;
import com.google.gerrit.server.mail.Address;
import com.google.gerrit.server.mail.receive.MailMessage;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
/**
* Tests parsing a quoted printable encoded subject
*/
public class QuotedPrintableHeaderMessage extends RawMailMessage {
private static String textContent = "Some Text";
private static String raw = "" +
"Date: Tue, 25 Oct 2016 02:11:35 -0700\n" +
"Message-ID: <001a114da7ae26e2eb053fe0c29c@google.com>\n" +
"Subject: =?UTF-8?Q?=C3=A2me vulgaire?=\n" +
"From: \"Jonathan Nieder (Gerrit)\" <noreply-gerritcodereview-" +
"CtTy0igsBrnvL7dKoWEIEg@google.com>\n" +
"To: ekempin <ekempin@google.com>\n" +
"Content-Type: text/plain; charset=UTF-8; format=flowed; delsp=yes\n" +
"\n" + textContent;
@Override
public String raw() {
return raw;
}
@Override
public int[] rawChars() {
return null;
}
@Override
public MailMessage expectedMailMessage() {
System.out.println("\uD83D\uDE1B test");
MailMessage.Builder expect = MailMessage.builder();
expect
.id("<001a114da7ae26e2eb053fe0c29c@google.com>")
.from(new Address("Jonathan Nieder (Gerrit)",
"noreply-gerritcodereview-CtTy0igsBrnvL7dKoWEIEg@google.com"))
.addTo(new Address("ekempin","ekempin@google.com"))
.textContent(textContent)
.subject("âme vulgaire")
.dateReceived(
new DateTime(2016, 10, 25, 9, 11, 35, 0, DateTimeZone.UTC));
return expect.build();
}
}

28
gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/data/RawMailMessage.java Normal file
View File

@ -0,0 +1,28 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail.receive.data;
import com.google.gerrit.server.mail.receive.MailMessage;
/**
* Base class for all email parsing tests.
*/
public abstract class RawMailMessage {
// Raw content to feed the parser
public abstract String raw();
public abstract int[] rawChars();
// Parsed representation for asserting the expected parser output
public abstract MailMessage expectedMailMessage();
}

134
gerrit-server/src/test/java/com/google/gerrit/server/mail/receive/data/SimpleTextMessage.java Normal file
View File

@ -0,0 +1,134 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.mail.receive.data;
import com.google.gerrit.server.mail.Address;
import com.google.gerrit.server.mail.receive.MailMessage;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
/**
* Tests parsing a simple text message with different headers.
*/
public class SimpleTextMessage extends RawMailMessage {
private static String textContent = "" +
"Jonathan Nieder has posted comments on this change. ( \n" +
"https://gerrit-review.googlesource.com/90018 )\n" +
"\n" +
"Change subject: (Re)enable voting buttons for merged changes\n" +
"...........................................................\n" +
"\n" +
"\n" +
"Patch Set 2:\n" +
"\n" +
"This is producing NPEs server-side and 500s for the client. \n" +
"when I try to load this change:\n" +
"\n" +
" Error in GET /changes/90018/detail?O=10004\n" +
" com.google.gwtorm.OrmException: java.lang.NullPointerException\n" +
"\tat com.google.gerrit.change.ChangeJson.format(ChangeJson.java:303)\n" +
"\tat com.google.gerrit.change.ChangeJson.format(ChangeJson.java:285)\n" +
"\tat com.google.gerrit.change.ChangeJson.format(ChangeJson.java:263)\n" +
"\tat com.google.gerrit.change.GetChange.apply(GetChange.java:50)\n" +
"\tat com.google.gerrit.change.GetDetail.apply(GetDetail.java:51)\n" +
"\tat com.google.gerrit.change.GetDetail.apply(GetDetail.java:26)\n" +
"\tat \n" +
"com.google.gerrit.RestApiServlet.service(RestApiServlet.java:367)\n" +
"\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:717)\n" +
"[...]\n" +
" Caused by: java.lang.NullPointerException\n" +
"\tat \n" +
"com.google.gerrit.ChangeJson.setLabelScores(ChangeJson.java:670)\n" +
"\tat \n" +
"com.google.gerrit.ChangeJson.labelsFor(ChangeJson.java:845)\n" +
"\tat \n" +
"com.google.gerrit.change.ChangeJson.labelsFor(ChangeJson.java:598)\n" +
"\tat \n" +
"com.google.gerrit.change.ChangeJson.toChange(ChangeJson.java:499)\n" +
"\tat com.google.gerrit.change.ChangeJson.format(ChangeJson.java:294)\n" +
"\t... 105 more\n" +
"-- \n" +
"To view, visit https://gerrit-review.googlesource.com/90018\n" +
"To unsubscribe, visit https://gerrit-review.googlesource.com\n" +
"\n" +
"Gerrit-MessageType: comment\n" +
"Gerrit-Change-Id: Iba501e00bee77be3bd0ced72f88fd04ba0accaed\n" +
"Gerrit-PatchSet: 2\n" +
"Gerrit-Project: gerrit\n" +
"Gerrit-Branch: master\n" +
"Gerrit-Owner: ekempin <ekempin@google.com>\n" +
"Gerrit-Reviewer: Dave Borowitz <dborowitz@google.com>\n" +
"Gerrit-Reviewer: Edwin Kempin <ekempin@google.com>\n" +
"Gerrit-Reviewer: GerritForge CI <gerritforge@gmail.com>\n" +
"Gerrit-Reviewer: Jonathan Nieder <jrn@google.com>\n" +
"Gerrit-Reviewer: Patrick Hiesel <hiesel@google.com>\n" +
"Gerrit-Reviewer: ekempin <ekempin@google.com>\n" +
"Gerrit-HasComments: No";
private static String raw = "" +
"Authentication-Results: mx.google.com; dkim=pass header.i=" +
"@google.com;\n" +
"Date: Tue, 25 Oct 2016 02:11:35 -0700\n" +
"In-Reply-To: <gerrit.1477487889000.Iba501e00bee77be3bd0ced" +
"72f88fd04ba0accaed@gerrit-review.googlesource.com>\n" +
"References: <gerrit.1477487889000.Iba501e00bee77be3bd0ced72f8" +
"8fd04ba0accaed@gerrit-review.googlesource.com>\n" +
"Message-ID: <001a114da7ae26e2eb053fe0c29c@google.com>\n" +
"Subject: Change in gerrit[master]: (Re)enable voting buttons for " +
"merged changes\n" +
"From: \"Jonathan Nieder (Gerrit)\" <noreply-gerritcodereview-CtTy0" +
"igsBrnvL7dKoWEIEg@google.com>\n" +
"To: ekempin <ekempin@google.com>\n" +
"Cc: Dave Borowitz <dborowitz@google.com>, Jonathan Nieder " +
"<jrn@google.com>, Patrick Hiesel <hiesel@google.com>\n" +
"Content-Type: text/plain; charset=UTF-8; format=flowed; delsp=yes\n" +
"\n" + textContent;
@Override
public String raw() {
return raw;
}
@Override
public int[] rawChars() {
return null;
}
@Override
public MailMessage expectedMailMessage() {
MailMessage.Builder expect = MailMessage.builder();
expect
.id("<001a114da7ae26e2eb053fe0c29c@google.com>")
.from(new Address("Jonathan Nieder (Gerrit)",
"noreply-gerritcodereview-CtTy0igsBrnvL7dKoWEIEg@google.com"))
.addTo(new Address("ekempin","ekempin@google.com"))
.addCc(new Address("Dave Borowitz", "dborowitz@google.com"))
.addCc(new Address("Jonathan Nieder", "jrn@google.com"))
.addCc(new Address("Patrick Hiesel", "hiesel@google.com"))
.textContent(textContent)
.subject("Change in gerrit[master]: (Re)enable voting"
+ " buttons for merged changes")
.dateReceived(
new DateTime(2016, 10, 25, 9, 11, 35, 0, DateTimeZone.UTC))
.addAdditionalHeader("Authentication-Results: mx.google.com; " +
"dkim=pass header.i=@google.com;")
.addAdditionalHeader("In-Reply-To: <gerrit.1477487889000.Iba501e00bee" +
"77be3bd0ced72f88fd04ba0accaed@gerrit-review.googlesource.com>")
.addAdditionalHeader("References: <gerrit.1477487889000.Iba501e00bee" +
"77be3bd0ced72f88fd04ba0accaed@gerrit-review.googlesource.com>");
return expect.build();
}
}

2
gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java
View File

@ -53,6 +53,7 @@ import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gerrit.server.mail.SignedTokenEmailTokenVerifier;
import com.google.gerrit.server.mail.receive.MailReceiver;
import com.google.gerrit.server.mail.send.SmtpEmailSender;
import com.google.gerrit.server.mime.MimeUtil2Module;
import com.google.gerrit.server.notedb.ConfigNotesMigration;
@ -310,6 +311,7 @@ public class WebAppInitializer extends GuiceServletContextListener
modules.add(new SearchingChangeCacheImpl.Module());
modules.add(new InternalAccountDirectory.Module());
modules.add(new DefaultCacheFactory.Module());
modules.add(cfgInjector.getInstance(MailReceiver.Module.class));
modules.add(new SmtpEmailSender.Module());
modules.add(new SignedTokenEmailTokenVerifier.Module());
modules.add(new PluginRestApiModule());

21
lib/greenmail/BUCK Normal file
View File

@ -0,0 +1,21 @@
include_defs('//lib/maven.defs')
VERSION = '1.5.2'
java_library(
name = 'greenmail',
exported_deps = [
':greenmail_library',
],
visibility = ['PUBLIC'],
)
maven_jar(
name = 'greenmail_library',
id = 'com.icegreen:greenmail:' + VERSION,
sha1 = '6b4862a09f8642da58c109117b24ccc19a4a6d39',
license = 'Apache2.0',
exclude_java_sources = True,
visibility = ['PUBLIC'],
)

7
lib/greenmail/BUILD Normal file
View File

@ -0,0 +1,7 @@
package(default_visibility = ['//visibility:public'])
java_library(
name = 'greenmail',
exports = ['@greenmail//jar'],
visibility = ['//visibility:public'],
data = ['//lib:LICENSE-Apache2.0'],
)

21
lib/mail/BUCK Normal file
View File

@ -0,0 +1,21 @@
include_defs('//lib/maven.defs')
VERSION = '1.5.6'
java_library(
name = 'mail',
exported_deps = [
':mail_library',
],
visibility = ['PUBLIC'],
)
maven_jar(
name = 'mail_library',
id = 'com.sun.mail:javax.mail:' + VERSION,
sha1 = 'ab5daef2f881c42c8e280cbe918ec4d7fdfd7efe',
license = 'DO_NOT_DISTRIBUTE',
exclude_java_sources = True,
visibility = ['PUBLIC'],
)

6
lib/mail/BUILD Normal file
View File

@ -0,0 +1,6 @@
java_library(
name = 'mail',
exports = ['@mail//jar'],
visibility = ['//visibility:public'],
data = ['//lib:LICENSE-DO_NOT_DISTRIBUTE'],
)

35
lib/mime4j/BUCK Normal file
View File

@ -0,0 +1,35 @@
include_defs('//lib/maven.defs')
VERSION = '0.8.0'
java_library(
name = 'core',
exported_deps = [
':core_library',
],
visibility = ['PUBLIC'],
)
maven_jar(
name = 'core_library',
id = 'org.apache.james:apache-mime4j-core:' + VERSION,
sha1 = 'd54f45fca44a2f210569656b4ca3574b42911c95',
license = 'Apache2.0',
visibility = ['PUBLIC'],
)
java_library(
name = 'dom',
exported_deps = [
':dom_library',
],
visibility = ['PUBLIC'],
)
maven_jar(
name = 'dom_library',
id = 'org.apache.james:apache-mime4j-dom:' + VERSION,
sha1 = '6720c93d14225c3e12c4a69768a0370c80e376a3',
license = 'Apache2.0',
visibility = ['PUBLIC'],
)

13
lib/mime4j/BUILD Normal file
View File

@ -0,0 +1,13 @@
java_library(
name = "core",
data = ["//lib:LICENSE-Apache2.0"],
visibility = ["//visibility:public"],
exports = ["@mime4j_core//jar"],
)
java_library(
name = "dom",
data = ["//lib:LICENSE-Apache2.0"],
visibility = ["//visibility:public"],
exports = ["@mime4j_dom//jar"],
)