Retry account updates on LockFailureException

If an account update fails with LockFailureException (because the same
account was concurrently updated by another thread) we now retry the
account update.

In future AccountsUpdate should allow to update an account and its
external IDs atomically. For external ID updates there is a higher
chance of lock failures since the external IDs of all users are stored
in a single notes branch. Hence when AccountsUpdates takes to also
update external IDs it gets important to retry account updates that
failed due to lock failure. So this change is a preparation to support
updating accounts and external IDs atomically in future.

Change-Id: I2bc6b4cceb111855f48d02a68690003992b3dc53
Signed-off-by: Edwin Kempin <ekempin@google.com>

javatests/com/google/gerrit/acceptance/api/accounts/AccountIT.java

@@ -35,6 +35,7 @@ import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toSet;
 import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
 
+import com.github.rholder.retry.StopStrategies;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableList;
@@ -92,16 +93,21 @@ import com.google.gerrit.server.account.WatchConfig.NotifyType;
 import com.google.gerrit.server.account.externalids.ExternalId;
 import com.google.gerrit.server.account.externalids.ExternalIds;
 import com.google.gerrit.server.account.externalids.ExternalIdsUpdate;
+import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
+import com.google.gerrit.server.git.LockFailureException;
 import com.google.gerrit.server.git.ProjectConfig;
 import com.google.gerrit.server.index.account.AccountIndexer;
 import com.google.gerrit.server.index.account.StalenessChecker;
 import com.google.gerrit.server.mail.Address;
+import com.google.gerrit.server.mail.send.OutgoingEmailValidator;
 import com.google.gerrit.server.notedb.rebuild.ChangeRebuilderImpl;
 import com.google.gerrit.server.project.RefPattern;
 import com.google.gerrit.server.query.account.InternalAccountQuery;
+import com.google.gerrit.server.update.RetryHelper;
 import com.google.gerrit.server.util.MagicBranch;
 import com.google.gerrit.testing.ConfigSuite;
 import com.google.gerrit.testing.FakeEmailSender.Message;
+import com.google.gwtorm.server.OrmException;
 import com.google.inject.Inject;
 import com.google.inject.Provider;
 import com.google.inject.name.Named;
@@ -119,10 +125,12 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.bouncycastle.bcpg.ArmoredOutputStream;
 import org.bouncycastle.openpgp.PGPPublicKey;
 import org.bouncycastle.openpgp.PGPPublicKeyRing;
 import org.eclipse.jgit.api.errors.TransportException;
+import org.eclipse.jgit.errors.ConfigInvalidException;
 import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
 import org.eclipse.jgit.junit.TestRepository;
 import org.eclipse.jgit.lib.CommitBuilder;
@@ -181,6 +189,12 @@ public class AccountIT extends AbstractDaemonTest {
 
   @Inject private AccountIndexer accountIndexer;
 
+  @Inject private OutgoingEmailValidator emailValidator;
+
+  @Inject private GitReferenceUpdated gitReferenceUpdated;
+
+  @Inject private RetryHelper.Metrics retryMetrics;
+
   @Inject
   @Named("accounts")
   private LoadingCache<Account.Id, Optional<AccountState>> accountsCache;
@@ -1880,6 +1894,106 @@ public class AccountIT extends AbstractDaemonTest {
         Permission.CREATE);
   }
 
+  @Test
+  public void retryOnLockFailure() throws Exception {
+    String status = "happy";
+    String fullName = "Foo";
+    AtomicBoolean doneBgUpdate = new AtomicBoolean(false);
+    AccountsUpdate update =
+        new AccountsUpdate(
+            repoManager,
+            gitReferenceUpdated,
+            null,
+            allUsers,
+            emailValidator,
+            serverIdent.get(),
+            () -> metaDataUpdateFactory.create(allUsers),
+            new RetryHelper(
+                cfg,
+                retryMetrics,
+                null,
+                null,
+                null,
+                r -> r.withBlockStrategy(noSleepBlockStrategy)),
+            () -> {
+              if (!doneBgUpdate.getAndSet(true)) {
+                try {
+                  accountsUpdate.create().update(admin.id, u -> u.setStatus(status));
+                } catch (IOException | ConfigInvalidException | OrmException e) {
+                  // Ignore, the successful update of the account is asserted later
+                }
+              }
+            });
+    assertThat(doneBgUpdate.get()).isFalse();
+    AccountInfo accountInfo = gApi.accounts().id(admin.id.get()).get();
+    assertThat(accountInfo.status).isNull();
+    assertThat(accountInfo.name).isNotEqualTo(fullName);
+
+    Account updatedAccount = update.update(admin.id, u -> u.setFullName(fullName));
+    assertThat(doneBgUpdate.get()).isTrue();
+
+    assertThat(updatedAccount.getStatus()).isEqualTo(status);
+    assertThat(updatedAccount.getFullName()).isEqualTo(fullName);
+
+    accountInfo = gApi.accounts().id(admin.id.get()).get();
+    assertThat(accountInfo.status).isEqualTo(status);
+    assertThat(accountInfo.name).isEqualTo(fullName);
+  }
+
+  @Test
+  public void failAfterRetryerGivesUp() throws Exception {
+    List<String> status = ImmutableList.of("foo", "bar", "baz");
+    String fullName = "Foo";
+    AtomicInteger bgCounter = new AtomicInteger(0);
+    AccountsUpdate update =
+        new AccountsUpdate(
+            repoManager,
+            gitReferenceUpdated,
+            null,
+            allUsers,
+            emailValidator,
+            serverIdent.get(),
+            () -> metaDataUpdateFactory.create(allUsers),
+            new RetryHelper(
+                cfg,
+                retryMetrics,
+                null,
+                null,
+                null,
+                r ->
+                    r.withStopStrategy(StopStrategies.stopAfterAttempt(status.size()))
+                        .withBlockStrategy(noSleepBlockStrategy)),
+            () -> {
+              try {
+                accountsUpdate
+                    .create()
+                    .update(admin.id, u -> u.setStatus(status.get(bgCounter.getAndAdd(1))));
+              } catch (IOException | ConfigInvalidException | OrmException e) {
+                // Ignore, the expected exception is asserted later
+              }
+            });
+    assertThat(bgCounter.get()).isEqualTo(0);
+    AccountInfo accountInfo = gApi.accounts().id(admin.id.get()).get();
+    assertThat(accountInfo.status).isNull();
+    assertThat(accountInfo.name).isNotEqualTo(fullName);
+
+    try {
+      update.update(admin.id, u -> u.setFullName(fullName));
+      fail("expected LockFailureException");
+    } catch (LockFailureException e) {
+      // Ignore, expected
+    }
+    assertThat(bgCounter.get()).isEqualTo(status.size());
+
+    Account updatedAccount = accounts.get(admin.id);
+    assertThat(updatedAccount.getStatus()).isEqualTo(Iterables.getLast(status));
+    assertThat(updatedAccount.getFullName()).isEqualTo(admin.fullName);
+
+    accountInfo = gApi.accounts().id(admin.id.get()).get();
+    assertThat(accountInfo.status).isEqualTo(Iterables.getLast(status));
+    assertThat(accountInfo.name).isEqualTo(admin.fullName);
+  }
+
   @Test
   public void stalenessChecker() throws Exception {
     // Newly created account is not stale.