Implement receiver class structure and bindings

This commit implements the basic class structure for receiving emails,
all required config parameters, all necessary bindings depending on
which protocol the administrator has configured and updates the
documentation accordingly.

It also adds test-only dependencies to Greenmail and javax.mail. These
will be used to create integration tests.

It's the first change in a topic of changes to implement email ingestion
for Gerrit.

Change-Id: I0edec7ca2655fcd70284bb75ca8eb94ce2491d7a

Documentation/config-gerrit.txt

@@ -3419,6 +3419,67 @@ which miscellaneous tasks are handled.
 +
 Default is 1.
 
+[[receiveemail]]
+=== Section receiveemail
+
+[[receiveemail.protocol]]receiveemail.protocol::
++
+Specifies the protocol used for receiving emails. Valid options are
+'POP3', 'IMAP' and 'NONE'. Note that Gerrit will automatically switch between
+POP3 and POP3s as well as IMAP and IMAPS depending on the specified
+link:#receiveemail.encryption[encryption].
++
+Defaults to 'NONE' which means that receiving emails is disabled.
+
+[[receiveemail.host]]receiveemail.host::
++
+The hostname of the mailserver. Example: 'imap.gmail.com'.
++
+Defaults to an empty string which means that receiving emails is disabled.
+
+[[receiveemail.port]]receiveemail.port::
++
+The port the email server exposes for receving emails.
++
+Defaults to the industry standard for a given protocol and encryption:
+POP3: 110; POP3S: 995; IMAP: 143; IMAPS: 995.
+
+[[receiveemail.username]]receiveemail.username::
++
+Username used for authenticating with the email server.
++
+Defaults to an empty string.
+
+[[receiveemail.password]]receiveemail.password::
++
+Password used for authenticating with the email server.
++
+Defaults to an empty string.
+
+[[receiveemail.encryption]]receiveemail.encryption::
++
+Encryption standard used for transport layer security between Gerrit and the
+email server. Possible values include 'NONE', 'SSL' and 'TLS'.
++
+Defaults to 'NONE'.
+
+[[receiveemail.fetchInterval]]receiveemail.fetchInterval::
++
+Time between two consecutive fetches from the email server. Communication with
+the email server is not kept alive. Examples: 60s, 10m, 1h.
++
+Defaults to 60 seconds.
+
+[[receiveemail.enableImapIdle]]receiveemail.enableImapIdle::
++
+If the IMAP protocol is used for retrieving emails, IMAPv4 IDLE can be used to
+keep the connection with the email server alive and receive a push when a new
+email is delivered to the inbox. In this case, Gerrit will process the email
+immediately and will not have a fetch delay.
+
++
+Defaults to false.
+
 [[sendemail]]
 === Section sendemail
 

WORKSPACE

@@ -381,6 +381,22 @@ maven_jar(
   sha1 = '2e35862b0435c1b027a21f3d6eecbe50e6e08d54',
 )
 
+GREENMAIL_VERS = '1.5.2'
+
+maven_jar(
+  name = 'greenmail',
+  artifact = 'com.icegreen:greenmail:' + GREENMAIL_VERS,
+  sha1 = '6b4862a09f8642da58c109117b24ccc19a4a6d39',
+)
+
+MAIL_VERS = '1.5.6'
+
+maven_jar(
+  name = 'mail',
+  artifact = 'com.sun.mail:javax.mail:' + MAIL_VERS,
+  sha1 = 'ab5daef2f881c42c8e280cbe918ec4d7fdfd7efe',
+)
+
 OW2_VERS = '5.1'
 
 maven_jar(

gerrit-acceptance-framework/BUCK

@@ -56,10 +56,12 @@ java_library(
     '//lib:truth',
   ],
   provided_deps = PROVIDED + [
+    '//lib/greenmail:greenmail',
     '//lib:gwtorm',
     '//lib/guice:guice',
     '//lib/guice:guice-assistedinject',
     '//lib/guice:guice-servlet',
+    '//lib/mail:mail',
   ],
   visibility = ['PUBLIC'],
 )

gerrit-acceptance-tests/BUCK

@@ -29,11 +29,13 @@ java_library(
 
     '//lib/bouncycastle:bcpg',
     '//lib/bouncycastle:bcprov',
+    '//lib/greenmail:greenmail',
     '//lib/guice:guice',
     '//lib/guice:guice-assistedinject',
     '//lib/guice:guice-servlet',
     '//lib/log:api',
     '//lib/jgit/org.eclipse.jgit:jgit',
+    '//lib/mail:mail',
     '//lib/mina:sshd',
   ],
   visibility = [

gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java

@@ -68,6 +68,7 @@ import com.google.gerrit.server.index.DummyIndexModule;
 import com.google.gerrit.server.index.IndexModule;
 import com.google.gerrit.server.index.IndexModule.IndexType;
 import com.google.gerrit.server.mail.SignedTokenEmailTokenVerifier;
+import com.google.gerrit.server.mail.receive.MailReceiver;
 import com.google.gerrit.server.mail.send.SmtpEmailSender;
 import com.google.gerrit.server.mime.MimeUtil2Module;
 import com.google.gerrit.server.patch.DiffExecutorModule;
@@ -362,6 +363,7 @@ public class Daemon extends SiteProgram {
     modules.add(new SearchingChangeCacheImpl.Module(slave));
     modules.add(new InternalAccountDirectory.Module());
     modules.add(new DefaultCacheFactory.Module());
+    modules.add(cfgInjector.getInstance(MailReceiver.Module.class));
     if (emailModule != null) {
       modules.add(emailModule);
     } else {

gerrit-pgm/src/main/java/com/google/gerrit/pgm/init/InitSendEmail.java

@@ -21,7 +21,7 @@ import com.google.gerrit.pgm.init.api.ConsoleUI;
 import com.google.gerrit.pgm.init.api.InitStep;
 import com.google.gerrit.pgm.init.api.Section;
 import com.google.gerrit.server.config.SitePaths;
-import com.google.gerrit.server.mail.send.SmtpEmailSender.Encryption;
+import com.google.gerrit.server.mail.Encryption;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 

gerrit-server/src/main/java/com/google/gerrit/server/mail/EmailSettings.java

@@ -15,21 +15,47 @@
 package com.google.gerrit.server.mail;
 
 import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.mail.receive.Protocol;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 
 import org.eclipse.jgit.lib.Config;
 
+import java.util.concurrent.TimeUnit;
+
 @Singleton
 public class EmailSettings {
+  private static final String SEND_EMAL = "sendemail";
+  private static final String RECEIVE_EMAL = "receiveemail";
+  // Send
   public final boolean html;
   public final boolean includeDiff;
   public final int maximumDiffSize;
+  // Receive
+  public final Protocol protocol;
+  public final String host;
+  public final int port;
+  public final String username;
+  public final String password;
+  public final Encryption encryption;
+  public final long fetchInterval; // in milliseconds
 
   @Inject
   EmailSettings(@GerritServerConfig Config cfg) {
-    html = cfg.getBoolean("sendemail", "html", true);
-    includeDiff = cfg.getBoolean("sendemail", "includeDiff", false);
-    maximumDiffSize = cfg.getInt("sendemail", "maximumDiffSize", 256 << 10);
+    // Send
+    html = cfg.getBoolean(SEND_EMAL, "html", true);
+    includeDiff = cfg.getBoolean(SEND_EMAL, "includeDiff", false);
+    maximumDiffSize = cfg.getInt(SEND_EMAL, "maximumDiffSize", 256 << 10);
+    // Receive
+    protocol = cfg.getEnum(RECEIVE_EMAL, null, "protocol", Protocol.NONE);
+    host = cfg.getString(RECEIVE_EMAL, null, "host");
+    port = cfg.getInt(RECEIVE_EMAL, "port", 0);
+    username = cfg.getString(RECEIVE_EMAL, null, "username");
+    password = cfg.getString(RECEIVE_EMAL, null, "password");
+    encryption =
+        cfg.getEnum(RECEIVE_EMAL, null, "encryption", Encryption.NONE);
+    fetchInterval = cfg.getTimeUnit(RECEIVE_EMAL, null, "fetchInterval",
+        TimeUnit.MILLISECONDS.convert(60, TimeUnit.SECONDS),
+        TimeUnit.MILLISECONDS);
   }
 }

gerrit-server/src/main/java/com/google/gerrit/server/mail/Encryption.java

@@ -0,0 +1,19 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.mail;
+
+public enum Encryption {
+  NONE, SSL, TLS
+}

gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/ImapMailReceiver.java

@@ -0,0 +1,37 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.mail.receive;
+
+import com.google.gerrit.server.mail.EmailSettings;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+@Singleton
+public class ImapMailReceiver extends MailReceiver {
+
+  @Inject
+  public ImapMailReceiver(EmailSettings mailSettings) {
+    super(mailSettings);
+  }
+
+  /**
+   * handleEmails will open a connection to the mail server, remove emails
+   * where deletion is pending, read new email and close the connection.
+   */
+  @Override
+  protected synchronized void handleEmails() {
+    // TODO(hiesel) Implement.
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/MailMessage.java

@@ -0,0 +1,81 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.mail.receive;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
+
+import org.joda.time.DateTime;
+
+/**
+ * MailMessage is a simplified representation of an RFC 2045-2047 mime email
+ * message used for representing received emails inside Gerrit. It is populated
+ * by the MailParser after MailReceiver has received a message. Transformations
+ * done by the parser include stitching mime parts together, transforming all
+ * content to UTF-16 and removing attachments.
+ */
+@AutoValue
+public abstract class MailMessage {
+  // Unique Identifier
+  public abstract String id();
+  // Envelope Information
+  public abstract String from();
+  public abstract ImmutableList<String> to();
+  public abstract ImmutableList<String> cc();
+  // Metadata
+  public abstract DateTime dateReceived();
+  public abstract ImmutableList<String> additionalHeaders();
+  // Content
+  public abstract String subject();
+  public abstract String textContent();
+  public abstract String htmlContent();
+
+  static Builder builder() {
+    return new AutoValue_MailMessage.Builder();
+  }
+
+  @AutoValue.Builder
+  abstract static class Builder {
+    abstract Builder id(String val);
+    abstract Builder from(String val);
+    abstract ImmutableList.Builder<String> toBuilder();
+
+    public Builder addTo(String val) {
+      toBuilder().add(val);
+      return this;
+    }
+
+    abstract ImmutableList.Builder<String> ccBuilder();
+
+    public Builder addCc(String val) {
+      ccBuilder().add(val);
+      return this;
+    }
+
+    abstract Builder dateReceived(DateTime val);
+    abstract ImmutableList.Builder<String> additionalHeadersBuilder();
+
+    public Builder addAdditionalHeader(String val) {
+      additionalHeadersBuilder().add(val);
+      return this;
+    }
+
+    abstract Builder subject(String val);
+    abstract Builder textContent(String val);
+    abstract Builder htmlContent(String val);
+
+    abstract MailMessage build();
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/MailParsingException.java

@@ -0,0 +1,24 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.mail.receive;
+
+/** MailParsingException indicates that an email could not be parsed. */
+public class MailParsingException extends Exception {
+  private static final long serialVersionUID = 1L;
+
+  public MailParsingException(String msg) {
+    super(msg);
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/MailReceiver.java

@@ -0,0 +1,104 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.mail.receive;
+
+import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.lifecycle.LifecycleModule;
+import com.google.gerrit.server.mail.EmailSettings;
+import com.google.inject.Inject;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+
+/** MailReceiver implements base functionality for receiving emails. */
+public abstract class MailReceiver implements LifecycleListener {
+  protected EmailSettings mailSettings;
+  protected Set<String> pendingDeletion;
+  private Timer timer;
+
+  public static class Module extends LifecycleModule {
+    private final EmailSettings mailSettings;
+
+    @Inject
+    Module(EmailSettings mailSettings) {
+      this.mailSettings = mailSettings;
+    }
+
+    @Override
+    protected void configure() {
+      if (mailSettings.protocol == Protocol.NONE) {
+        return;
+      }
+      listener().to(MailReceiver.class);
+      switch (mailSettings.protocol) {
+        case IMAP:
+          bind(MailReceiver.class).to(ImapMailReceiver.class);
+          break;
+        case POP3:
+          bind(MailReceiver.class).to(Pop3MailReceiver.class);
+          break;
+        case NONE:
+        default:
+      }
+    }
+  }
+
+  @Inject
+  public MailReceiver(EmailSettings mailSettings) {
+    this.mailSettings = mailSettings;
+    pendingDeletion = Collections.synchronizedSet(new HashSet<>());
+  }
+
+  @Override
+  public void start() {
+    if (timer == null) {
+      timer = new Timer();
+    } else {
+      timer.cancel();
+    }
+    timer.scheduleAtFixedRate(new TimerTask() {
+      @Override
+      public void run() {
+        MailReceiver.this.handleEmails();
+      }
+    }, 0l, mailSettings.fetchInterval);
+  }
+
+  @Override
+  public void stop() {
+    if (timer != null) {
+      timer.cancel();
+    }
+  }
+
+  /**
+   * requestDeletion will enqueue an email for deletion and delete it the
+   * next time we connect to the email server. This does not guarantee deletion
+   * as the Gerrit instance might fail before we connect to the email server.
+   * @param messageId
+   */
+  public void requestDeletion(String messageId) {
+    pendingDeletion.add(messageId);
+  }
+
+  /**
+   * handleEmails will open a connection to the mail server, remove emails
+   * where deletion is pending, read new email and close the connection.
+   */
+  protected abstract void handleEmails();
+}

gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/Pop3MailReceiver.java

@@ -0,0 +1,36 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.mail.receive;
+
+import com.google.gerrit.server.mail.EmailSettings;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+@Singleton
+public class Pop3MailReceiver extends MailReceiver {
+  @Inject
+  public Pop3MailReceiver(EmailSettings mailSettings) {
+    super(mailSettings);
+  }
+
+  /**
+   * handleEmails will open a connection to the mail server, remove emails
+   * where deletion is pending, read new email and close the connection.
+   */
+  @Override
+  protected synchronized void handleEmails() {
+    // TODO(hiesel) Implement.
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/Protocol.java

@@ -0,0 +1,19 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.mail.receive;
+
+public enum Protocol {
+  NONE, POP3, IMAP
+}

gerrit-server/src/main/java/com/google/gerrit/server/mail/receive/RawMailParser.java

@@ -0,0 +1,26 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.mail.receive;
+
+/**
+ * RawMailParser parses raw email content received through POP3 or IMAP into
+ * an internal {@link MailMessage}.
+ */
+public class RawMailParser {
+  public static MailMessage parse(String raw) throws MailParsingException {
+    // TODO(hiesel) Implement.
+    return null;
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/mail/send/SmtpEmailSender.java

@@ -25,6 +25,7 @@ import com.google.gerrit.common.errors.EmailException;
 import com.google.gerrit.server.config.ConfigUtil;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.mail.Address;
+import com.google.gerrit.server.mail.Encryption;
 import com.google.inject.AbstractModule;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
@@ -61,10 +62,6 @@ public class SmtpEmailSender implements EmailSender {
     }
   }
 
-  public enum Encryption {
-    NONE, SSL, TLS
-  }
-
   private final boolean enabled;
   private final int connectTimeout;
 

gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java

@@ -53,6 +53,7 @@ import com.google.gerrit.server.git.WorkQueue;
 import com.google.gerrit.server.index.IndexModule;
 import com.google.gerrit.server.index.IndexModule.IndexType;
 import com.google.gerrit.server.mail.SignedTokenEmailTokenVerifier;
+import com.google.gerrit.server.mail.receive.MailReceiver;
 import com.google.gerrit.server.mail.send.SmtpEmailSender;
 import com.google.gerrit.server.mime.MimeUtil2Module;
 import com.google.gerrit.server.notedb.ConfigNotesMigration;
@@ -310,6 +311,7 @@ public class WebAppInitializer extends GuiceServletContextListener
     modules.add(new SearchingChangeCacheImpl.Module());
     modules.add(new InternalAccountDirectory.Module());
     modules.add(new DefaultCacheFactory.Module());
+    modules.add(cfgInjector.getInstance(MailReceiver.Module.class));
     modules.add(new SmtpEmailSender.Module());
     modules.add(new SignedTokenEmailTokenVerifier.Module());
     modules.add(new PluginRestApiModule());

lib/greenmail/BUCK

@@ -0,0 +1,21 @@
+include_defs('//lib/maven.defs')
+
+VERSION = '1.5.2'
+
+java_library(
+  name = 'greenmail',
+  exported_deps = [
+    ':greenmail_library',
+  ],
+  visibility = ['PUBLIC'],
+)
+
+maven_jar(
+  name = 'greenmail_library',
+  id = 'com.icegreen:greenmail:' + VERSION,
+  sha1 = '6b4862a09f8642da58c109117b24ccc19a4a6d39',
+  license = 'Apache2.0',
+  exclude_java_sources = True,
+  visibility = ['PUBLIC'],
+)
+

lib/greenmail/BUILD

@@ -0,0 +1,7 @@
+package(default_visibility = ['//visibility:public'])
+java_library(
+  name = 'greenmail',
+  exports = ['@greenmail//jar'],
+  visibility = ['//visibility:public'],
+  data = ['//lib:LICENSE-Apache2.0'],
+)

lib/mail/BUCK

@@ -0,0 +1,21 @@
+include_defs('//lib/maven.defs')
+
+VERSION = '1.5.6'
+
+java_library(
+  name = 'mail',
+  exported_deps = [
+    ':mail_library',
+  ],
+  visibility = ['PUBLIC'],
+)
+
+maven_jar(
+  name = 'mail_library',
+  id = 'com.sun.mail:javax.mail:' + VERSION,
+  sha1 = 'ab5daef2f881c42c8e280cbe918ec4d7fdfd7efe',
+  license = 'DO_NOT_DISTRIBUTE',
+  exclude_java_sources = True,
+  visibility = ['PUBLIC'],
+)
+

lib/mail/BUILD

@@ -0,0 +1,6 @@
+java_library(
+  name = 'mail',
+  exports = ['@mail//jar'],
+  visibility = ['//visibility:public'],
+  data = ['//lib:LICENSE-DO_NOT_DISTRIBUTE'],
+)