Migrate accounts to NoteDb (part 2)

This is the second part of migrating accounts from ReviewDb to NoteDb.

This change:
* migrates the accounts from ReviewDb to NoteDb (for single instance
  Gerrit servers)
* adds a configuration parameter (user.readAccountsFromGit) that
  controls whether external IDs are read from ReviewDb or NoteDb

AccountIT is now loading external IDs of an account directly from NoteDb
instead of retrieving them via the account cache. This is because for
the test deleteUserBranchWithAccessDatabaseCapability() the admin
account gets deleted by deleting its user branch and then the @After
restoreExternalIds() method couldn't delete the external IDs of that
account anymore (because the account was deleted it couldn't be
retrieved via the account cache anymore).

Change-Id: I41fa3a6bdb76f497c79a05bdc76e97a7e73624a6
Signed-off-by: Edwin Kempin <ekempin@google.com>

gerrit-server/src/main/java/com/google/gerrit/server/account/AccountManager.java

@@ -193,7 +193,8 @@ public class AccountManager {
     }
   }
 
-  private Account load(Account toUpdate, Account.Id accountId, ReviewDb db) throws OrmException {
+  private Account load(Account toUpdate, Account.Id accountId, ReviewDb db)
+      throws OrmException, IOException, ConfigInvalidException {
     if (toUpdate == null) {
       toUpdate = accounts.get(db, accountId);
       if (toUpdate == null) {

gerrit-server/src/main/java/com/google/gerrit/server/account/AccountResolver.java

@@ -23,12 +23,14 @@ import com.google.gwtorm.server.OrmException;
 import com.google.inject.Inject;
 import com.google.inject.Provider;
 import com.google.inject.Singleton;
+import java.io.IOException;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import org.eclipse.jgit.errors.ConfigInvalidException;
 
 @Singleton
 public class AccountResolver {
@@ -61,7 +63,8 @@ public class AccountResolver {
    * @return the single account that matches; null if no account matches or there are multiple
    *     candidates.
    */
-  public Account find(ReviewDb db, String nameOrEmail) throws OrmException {
+  public Account find(ReviewDb db, String nameOrEmail)
+      throws OrmException, IOException, ConfigInvalidException {
     Set<Account.Id> r = findAll(db, nameOrEmail);
     if (r.size() == 1) {
       return byId.get(r.iterator().next()).getAccount();
@@ -90,7 +93,8 @@ public class AccountResolver {
    *     name ("username").
    * @return the accounts that match, empty collection if none. Never null.
    */
-  public Set<Account.Id> findAll(ReviewDb db, String nameOrEmail) throws OrmException {
+  public Set<Account.Id> findAll(ReviewDb db, String nameOrEmail)
+      throws OrmException, IOException, ConfigInvalidException {
     Matcher m = Pattern.compile("^.* \\(([1-9][0-9]*)\\)$").matcher(nameOrEmail);
     if (m.matches()) {
       Account.Id id = Account.Id.parse(m.group(1));
@@ -118,7 +122,8 @@ public class AccountResolver {
     return findAllByNameOrEmail(db, nameOrEmail);
   }
 
-  private boolean exists(ReviewDb db, Account.Id id) throws OrmException {
+  private boolean exists(ReviewDb db, Account.Id id)
+      throws OrmException, IOException, ConfigInvalidException {
     return accounts.get(db, id) != null;
   }
 

gerrit-server/src/main/java/com/google/gerrit/server/account/Accounts.java

@@ -22,35 +22,70 @@ import com.google.gerrit.reviewdb.client.Account;
 import com.google.gerrit.reviewdb.client.RefNames;
 import com.google.gerrit.reviewdb.server.ReviewDb;
 import com.google.gerrit.server.config.AllUsersName;
+import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.git.GitRepositoryManager;
+import com.google.gerrit.server.mail.send.OutgoingEmailValidator;
 import com.google.gwtorm.server.OrmException;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Stream;
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.lib.Config;
 import org.eclipse.jgit.lib.Repository;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /** Class to access accounts. */
 @Singleton
 public class Accounts {
+  private static final Logger log = LoggerFactory.getLogger(Accounts.class);
+
+  private final boolean readFromGit;
   private final GitRepositoryManager repoManager;
   private final AllUsersName allUsersName;
+  private final OutgoingEmailValidator emailValidator;
 
   @Inject
-  Accounts(GitRepositoryManager repoManager, AllUsersName allUsersName) {
+  Accounts(
+      @GerritServerConfig Config cfg,
+      GitRepositoryManager repoManager,
+      AllUsersName allUsersName,
+      OutgoingEmailValidator emailValidator) {
+    this.readFromGit = cfg.getBoolean("user", null, "readAccountsFromGit", false);
     this.repoManager = repoManager;
     this.allUsersName = allUsersName;
+    this.emailValidator = emailValidator;
   }
 
-  public Account get(ReviewDb db, Account.Id accountId) throws OrmException {
+  public Account get(ReviewDb db, Account.Id accountId)
+      throws OrmException, IOException, ConfigInvalidException {
+    if (readFromGit) {
+      try (Repository repo = repoManager.openRepository(allUsersName)) {
+        return read(repo, accountId);
+      }
+    }
+
     return db.accounts().get(accountId);
   }
 
-  public List<Account> get(ReviewDb db, Collection<Account.Id> accountIds) throws OrmException {
+  public List<Account> get(ReviewDb db, Collection<Account.Id> accountIds)
+      throws OrmException, IOException, ConfigInvalidException {
+    if (readFromGit) {
+      List<Account> accounts = new ArrayList<>(accountIds.size());
+      try (Repository repo = repoManager.openRepository(allUsersName)) {
+        for (Account.Id accountId : accountIds) {
+          accounts.add(read(repo, accountId));
+        }
+      }
+      return accounts;
+    }
+
     return db.accounts().get(accountIds).toList();
   }
 
@@ -59,7 +94,22 @@ public class Accounts {
    *
    * @return all accounts
    */
-  public List<Account> all(ReviewDb db) throws OrmException {
+  public List<Account> all(ReviewDb db) throws OrmException, IOException {
+    if (readFromGit) {
+      Set<Account.Id> accountIds = allIds();
+      List<Account> accounts = new ArrayList<>(accountIds.size());
+      try (Repository repo = repoManager.openRepository(allUsersName)) {
+        for (Account.Id accountId : accountIds) {
+          try {
+            accounts.add(read(repo, accountId));
+          } catch (Exception e) {
+            log.error(String.format("Ignoring invalid account %s", accountId.get()), e);
+          }
+        }
+      }
+      return accounts;
+    }
+
     return db.accounts().all().toList();
   }
 
@@ -103,6 +153,13 @@ public class Accounts {
     }
   }
 
+  private Account read(Repository allUsersRepository, Account.Id accountId)
+      throws IOException, ConfigInvalidException {
+    AccountConfig accountConfig = new AccountConfig(emailValidator, accountId);
+    accountConfig.load(allUsersRepository);
+    return accountConfig.getAccount();
+  }
+
   private static Stream<Account.Id> readUserRefs(Repository repo) throws IOException {
     return repo.getRefDatabase()
         .getRefs(RefNames.REFS_USERS)

gerrit-server/src/main/java/com/google/gerrit/server/account/AccountsCollection.java

@@ -33,6 +33,8 @@ import com.google.gwtorm.server.OrmException;
 import com.google.inject.Inject;
 import com.google.inject.Provider;
 import com.google.inject.Singleton;
+import java.io.IOException;
+import org.eclipse.jgit.errors.ConfigInvalidException;
 
 @Singleton
 public class AccountsCollection
@@ -68,7 +70,8 @@ public class AccountsCollection
 
   @Override
   public AccountResource parse(TopLevelResource root, IdString id)
-      throws ResourceNotFoundException, AuthException, OrmException {
+      throws ResourceNotFoundException, AuthException, OrmException, IOException,
+          ConfigInvalidException {
     IdentifiedUser user = parseId(id.get());
     if (user == null) {
       throw new ResourceNotFoundException(id);
@@ -89,7 +92,8 @@ public class AccountsCollection
    *     account is not visible to the calling user
    */
   public IdentifiedUser parse(String id)
-      throws AuthException, UnprocessableEntityException, OrmException {
+      throws AuthException, UnprocessableEntityException, OrmException, IOException,
+          ConfigInvalidException {
     return parseOnBehalfOf(null, id);
   }
 
@@ -104,8 +108,11 @@ public class AccountsCollection
    * @throws AuthException thrown if 'self' is used as account ID and the current user is not
    *     authenticated
    * @throws OrmException
+   * @throws ConfigInvalidException
+   * @throws IOException
    */
-  public IdentifiedUser parseId(String id) throws AuthException, OrmException {
+  public IdentifiedUser parseId(String id)
+      throws AuthException, OrmException, IOException, ConfigInvalidException {
     return parseIdOnBehalfOf(null, id);
   }
 
@@ -113,7 +120,8 @@ public class AccountsCollection
    * Like {@link #parse(String)}, but also sets the {@link CurrentUser#getRealUser()} on the result.
    */
   public IdentifiedUser parseOnBehalfOf(@Nullable CurrentUser caller, String id)
-      throws AuthException, UnprocessableEntityException, OrmException {
+      throws AuthException, UnprocessableEntityException, OrmException, IOException,
+          ConfigInvalidException {
     IdentifiedUser user = parseIdOnBehalfOf(caller, id);
     if (user == null) {
       throw new UnprocessableEntityException(String.format("Account Not Found: %s", id));
@@ -124,7 +132,7 @@ public class AccountsCollection
   }
 
   private IdentifiedUser parseIdOnBehalfOf(@Nullable CurrentUser caller, String id)
-      throws AuthException, OrmException {
+      throws AuthException, OrmException, IOException, ConfigInvalidException {
     if (id.equals("self")) {
       CurrentUser user = self.get();
       if (user.isIdentifiedUser()) {