Merge branch 'stable-2.11'

* stable-2.11:
  Add documentation for SecureStore
  Add SwitchSecureStore site program
  Make SecureStore an abstract class
  Add API to make SecureStore switching possible

Change-Id: I60f6e58f590a94004329bf663506923a95c3a500

Documentation/dev-plugins.txt

@@ -1731,6 +1731,34 @@ MyType(@PluginData java.io.File myDir) {
 }
 ----
 
+[[secure-store]]
+== SecureStore
+
+SecureStore allows to change the way Gerrit stores sensitive data like
+passwords.
+
+In order to replace the default SecureStore (no-op) implementation,
+a class that extends `com.google.gerrit.server.securestore.SecureStore`
+needs to be provided (with dependencies) in a separate jar file. Then
+link:pgm-SwitchSecureStore.html[SwitchSecureStore] must be run to
+switch implementations.
+
+The SecureStore implementation is instantiated using a Guice injector
+which binds the `File` annotated with the `@SitePath` annotation.
+This means that a SecureStore implementation class can get access to
+the `site_path` like in the following example:
+
+[source,java]
+----
+@Inject
+MySecureStore(@SitePath java.io.File sitePath) {
+  // your code
+}
+----
+
+No Guice bindings or modules are required. Gerrit will automatically
+discover and bind the implementation.
+
 [[download-commands]]
 == Download Commands
 

Documentation/pgm-SwitchSecureStore.txt

@@ -0,0 +1,39 @@
+= SwitchSecureStore
+
+== NAME
+SwitchSecureStore - Changes the currently used SecureStore implementation
+
+== SYNOPSIS
+--
+'java' -jar gerrit.war 'SwitchSecureStore' [<OPTIONS>]
+--
+
+== DESCRIPTION
+Changes the SecureStore implementation used by Gerrit. It migrates all data
+stored in the old implementation, removes the old implementation jar file
+from `$site_path/lib` and puts the new one there. As a final step
+the link:config-gerrit.html#gerrit.secureStoreClass[gerrit.secureStoreClass]
+property of `gerrit.config` will be updated.
+
+All dependencies not provided by Gerrit should be put the in `$site_path/lib`
+directory manually, before running the `SwitchSecureStore` program.
+
+After this operation there is no automatic way back the to standard Gerrit no-op
+secure store implementation, however there is a manual procedure:
+* stop Gerrit,
+* remove SecureStore jar file from `$site_path/lib`,
+* put plain text passwords into `$site_path/etc/secure.conf` file,
+* start Gerrit.
+
+== OPTIONS
+
+--new-secure-store-lib::
+	Path to jar file with new SecureStore implementation. Jar dependencies must be
+	put in `$site_path/lib` directory.
+
+GERRIT
+------
+Part of link:index.html[Gerrit Code Review]
+
+SEARCHBOX
+---------

Documentation/pgm-index.txt

@@ -24,6 +24,9 @@ link:pgm-prolog-shell.html[prolog-shell]::
 link:pgm-reindex.html[reindex]::
 	Rebuild the secondary index.
 
+link:pgm-SwitchSecureStore.html[SwitchSecureStore]::
+	Change used SecureStore implementation.
+
 link:pgm-rulec.html[rulec]::
 	Compile project-specific Prolog rules to JARs.
 

gerrit-common/src/main/java/com/google/gerrit/common/SiteLibraryLoaderUtil.java

@@ -22,14 +22,7 @@ import java.util.Comparator;
 public final class SiteLibraryLoaderUtil {
 
   public static void loadSiteLib(File libdir) {
-    File[] jars = libdir.listFiles(new FileFilter() {
-      @Override
-      public boolean accept(File path) {
-        String name = path.getName();
-        return (name.endsWith(".jar") || name.endsWith(".zip"))
-            && path.isFile();
-      }
-    });
+    File[] jars = listJars(libdir);
     if (jars != null && 0 < jars.length) {
       Arrays.sort(jars, new Comparator<File>() {
         @Override
@@ -46,6 +39,18 @@ public final class SiteLibraryLoaderUtil {
     }
   }
 
+  public static File[] listJars(File libdir) {
+    File[] jars = libdir.listFiles(new FileFilter() {
+      @Override
+      public boolean accept(File path) {
+        String name = path.getName();
+        return (name.endsWith(".jar") || name.endsWith(".zip"))
+            && path.isFile();
+      }
+    });
+    return jars;
+  }
+
   private SiteLibraryLoaderUtil() {
   }
 }

gerrit-pgm/src/main/java/com/google/gerrit/pgm/SwitchSecureStore.java

@@ -0,0 +1,217 @@
+// Copyright (C) 2014 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.pgm;
+
+import static com.google.gerrit.server.schema.DataSourceProvider.Context.SINGLE_USER;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import com.google.common.collect.Iterables;
+import com.google.common.io.Files;
+import com.google.gerrit.common.IoUtil;
+import com.google.gerrit.common.SiteLibraryLoaderUtil;
+import com.google.gerrit.pgm.util.SiteProgram;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.plugins.JarScanner;
+import com.google.gerrit.server.securestore.DefaultSecureStore;
+import com.google.gerrit.server.securestore.SecureStore;
+import com.google.gerrit.server.securestore.SecureStore.EntryKey;
+import com.google.inject.Injector;
+
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+import org.kohsuke.args4j.Option;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.jar.JarFile;
+import java.util.zip.ZipEntry;
+
+public class SwitchSecureStore extends SiteProgram {
+  private static String getSecureStoreClassFromGerritConfig(SitePaths sitePaths) {
+    FileBasedConfig cfg =
+        new FileBasedConfig(sitePaths.gerrit_config, FS.DETECTED);
+    try {
+      cfg.load();
+    } catch (IOException | ConfigInvalidException e) {
+      throw new RuntimeException("Cannot read gerrit.config file", e);
+    }
+    return cfg.getString("gerrit", null, "secureStoreClass");
+  }
+
+  private static final Logger log = LoggerFactory
+      .getLogger(SwitchSecureStore.class);
+
+  @Option(name = "--new-secure-store-lib",
+      usage = "Path to new SecureStore implementation",
+      required = true)
+  private String newSecureStoreLib;
+
+  @Override
+  public int run() throws Exception {
+    SitePaths sitePaths = new SitePaths(getSitePath());
+    File newSecureStoreFile = new File(newSecureStoreLib);
+    if (!newSecureStoreFile.exists()) {
+      log.error(String.format("File %s doesn't exists",
+          newSecureStoreFile.getAbsolutePath()));
+      return -1;
+    }
+
+    String newSecureStore = getNewSecureStoreClassName(newSecureStoreFile);
+    String currentSecureStoreName = getCurrentSecureStoreClassName(sitePaths);
+
+    if (currentSecureStoreName.equals(newSecureStore)) {
+      log.error("Old and new SecureStore implementation names "
+          + "are the same. Migration will not work");
+      return -1;
+    }
+
+    IoUtil.loadJARs(newSecureStoreFile);
+    SiteLibraryLoaderUtil.loadSiteLib(sitePaths.lib_dir);
+
+    log.info("Current secureStoreClass property ({}) will be replaced with {}",
+        currentSecureStoreName, newSecureStore);
+    Injector dbInjector = createDbInjector(SINGLE_USER);
+    SecureStore currentStore =
+        getSecureStore(currentSecureStoreName, dbInjector);
+    SecureStore newStore = getSecureStore(newSecureStore, dbInjector);
+
+    migrateProperties(currentStore, newStore);
+
+    removeOldLib(sitePaths, currentSecureStoreName);
+    copyNewLib(sitePaths, newSecureStoreFile);
+
+    updateGerritConfig(sitePaths, newSecureStore);
+
+    return 0;
+  }
+
+  private void migrateProperties(SecureStore currentStore, SecureStore newStore) {
+    log.info("Migrate entries");
+    for (EntryKey key : currentStore.list()) {
+      String[] value =
+          currentStore.getList(key.section, key.subsection, key.name);
+      if (value != null) {
+        newStore.setList(key.section, key.subsection, key.name,
+            Arrays.asList(value));
+      } else {
+        String msg =
+            String.format("Cannot migrate entry for %s", key.section);
+        if (key.subsection != null) {
+          msg = msg + String.format(".%s", key.subsection);
+        }
+        msg = msg + String.format(".%s", key.name);
+        throw new RuntimeException(msg);
+      }
+    }
+  }
+
+  private void removeOldLib(SitePaths sitePaths, String currentSecureStoreName) {
+    File oldSecureStore =
+        findJarWithSecureStore(sitePaths, currentSecureStoreName);
+    if (oldSecureStore != null) {
+      log.info("Removing old SecureStore ({}) from lib/ directory",
+          oldSecureStore.getName());
+      if (!oldSecureStore.delete()) {
+        log.error("Cannot remove {}", oldSecureStore.getAbsolutePath());
+      }
+    } else {
+      log.info("Cannot find jar with old SecureStore ({}) in lib/ directory",
+          currentSecureStoreName);
+    }
+  }
+
+  private void copyNewLib(SitePaths sitePaths, File newSecureStoreFile)
+      throws IOException {
+    log.info("Copy new SecureStore ({}) into lib/ directory",
+        newSecureStoreFile.getName());
+    Files.copy(newSecureStoreFile, new File(sitePaths.lib_dir,
+        newSecureStoreFile.getName()));
+  }
+
+  private void updateGerritConfig(SitePaths sitePaths, String newSecureStore)
+      throws IOException, ConfigInvalidException {
+    log.info("Set gerrit.secureStoreClass property of gerrit.config to {}",
+        newSecureStore);
+    FileBasedConfig config =
+        new FileBasedConfig(sitePaths.gerrit_config, FS.DETECTED);
+    config.load();
+    config.setString("gerrit", null, "secureStoreClass", newSecureStore);
+    config.save();
+  }
+
+  private String getNewSecureStoreClassName(File secureStore)
+      throws IOException {
+    JarScanner scanner = new JarScanner(secureStore);
+    List<String> newSecureStores =
+        scanner.findSubClassesOf(SecureStore.class);
+    if (newSecureStores.isEmpty()) {
+      throw new RuntimeException(String.format(
+          "Cannot find implementation of SecureStore interface in %s",
+          secureStore.getAbsolutePath()));
+    }
+    if (newSecureStores.size() > 1) {
+      throw new RuntimeException(String.format(
+          "Found too many implementations of SecureStore:\n%s\nin %s", Joiner
+              .on("\n").join(newSecureStores), secureStore.getAbsolutePath()));
+    }
+    return Iterables.getOnlyElement(newSecureStores);
+  }
+
+  private String getCurrentSecureStoreClassName(SitePaths sitePaths) {
+    String current = getSecureStoreClassFromGerritConfig(sitePaths);
+    if (!Strings.isNullOrEmpty(current)) {
+      return current;
+    }
+    return DefaultSecureStore.class.getName();
+  }
+
+  private SecureStore getSecureStore(String className, Injector injector) {
+    try {
+      @SuppressWarnings("unchecked")
+      Class<? extends SecureStore> clazz =
+          (Class<? extends SecureStore>) Class.forName(className);
+      return injector.getInstance(clazz);
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException(
+          String.format("Cannot load SecureStore implementation: %s", className), e);
+    }
+  }
+
+  private File findJarWithSecureStore(SitePaths sitePaths,
+      String secureStoreClass) {
+    File[] jars = SiteLibraryLoaderUtil.listJars(sitePaths.lib_dir);
+    if (jars == null || jars.length == 0) {
+      return null;
+    }
+    String secureStoreClassPath = secureStoreClass.replace('.', '/') + ".class";
+    for (File jar : jars) {
+      try (JarFile jarFile = new JarFile(jar)) {
+        ZipEntry entry = jarFile.getEntry(secureStoreClassPath);
+        if (entry != null) {
+          return jar;
+        }
+      } catch (IOException e) {
+        log.error(e.getMessage(), e);
+      }
+    }
+    return null;
+  }
+}

gerrit-pgm/src/main/java/com/google/gerrit/pgm/init/BaseInit.java

@@ -217,7 +217,8 @@ public class BaseInit extends SiteProgram {
         && !currentSecureStoreClassName.equals(secureStoreInitData.className)) {
       String err =
           String.format(
-              "Different secure store was previously configured: %s.",
+              "Different secure store was previously configured: %s. "
+              + "Use SwitchSecureStore program to switch between implementations.",
               currentSecureStoreClassName);
       die(err, new RuntimeException("secure store mismatch"));
     }
@@ -293,7 +294,7 @@ public class BaseInit extends SiteProgram {
       }
       JarScanner scanner = new JarScanner(secureStoreLib);
       List<String> secureStores =
-          scanner.findImplementationsOf(SecureStore.class);
+          scanner.findSubClassesOf(SecureStore.class);
       if (secureStores.isEmpty()) {
         throw new InvalidSecureStoreException(String.format(
             "Cannot find class implementing %s interface in %s",

gerrit-pgm/src/test/java/com/google/gerrit/pgm/init/UpgradeFrom2_0_xTest.java

@@ -116,24 +116,14 @@ public class UpgradeFrom2_0_xTest extends InitTestCase {
     u.run();
   }
 
-  private static class InMemorySecureStore implements SecureStore {
+  private static class InMemorySecureStore extends SecureStore {
     private final Config cfg = new Config();
 
-    @Override
-    public String get(String section, String subsection, String name) {
-      return cfg.getString(section, subsection, name);
-    }
-
     @Override
     public String[] getList(String section, String subsection, String name) {
       return cfg.getStringList(section, subsection, name);
     }
 
-    @Override
-    public void set(String section, String subsection, String name, String value) {
-      cfg.setString(section, subsection, name, value);
-    }
-
     @Override
     public void setList(String section, String subsection, String name,
         List<String> values) {
@@ -144,5 +134,10 @@ public class UpgradeFrom2_0_xTest extends InitTestCase {
     public void unset(String section, String subsection, String name) {
       cfg.unset(section, subsection, name);
     }
+
+    @Override
+    public Iterable<EntryKey> list() {
+      throw new UnsupportedOperationException("not used by tests");
+    }
   }
 }

gerrit-server/src/main/java/com/google/gerrit/server/plugins/JarScanner.java

@@ -43,7 +43,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.lang.annotation.Annotation;
-import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Enumeration;
@@ -134,11 +134,14 @@ public class JarScanner implements PluginContentScanner {
     return result.build();
   }
 
-  public List<String> findImplementationsOf(Class<?> requestedInterface)
-      throws IOException {
-    List<String> result = Lists.newArrayList();
-    String name = requestedInterface.getName().replace('.', '/');
+  public List<String> findSubClassesOf(Class<?> superClass) throws IOException {
+    return findSubClassesOf(superClass.getName());
+  }
 
+  private List<String> findSubClassesOf(String superClass) throws IOException {
+    String name = superClass.replace('.', '/');
+
+    List<String> classes = new ArrayList<>();
     Enumeration<JarEntry> e = jarFile.entries();
     while (e.hasMoreElements()) {
       JarEntry entry = e.nextElement();
@@ -155,13 +158,15 @@ public class JarScanner implements PluginContentScanner {
         continue;
       }
 
-      if (def.isConcrete() && def.interfaces != null
-          && Iterables.contains(Arrays.asList(def.interfaces), name)) {
-        result.add(def.className);
+      if (name.equals(def.superName)) {
+        classes.addAll(findSubClassesOf(def.className));
+        if (def.isConcrete()) {
+          classes.add(def.className);
+        }
       }
     }
 
-    return result;
+    return classes;
   }
 
   private static boolean skip(JarEntry entry) {
@@ -192,6 +197,7 @@ public class JarScanner implements PluginContentScanner {
   public static class ClassData extends ClassVisitor {
     int access;
     String className;
+    String superName;
     String annotationName;
     String annotationValue;
     String[] interfaces;
@@ -212,7 +218,7 @@ public class JarScanner implements PluginContentScanner {
         String superName, String[] interfaces) {
       this.className = Type.getObjectType(name).getClassName();
       this.access = access;
-      this.interfaces = interfaces;
+      this.superName = superName;
     }
 
     @Override

gerrit-server/src/main/java/com/google/gerrit/server/securestore/DefaultSecureStore.java

@@ -26,10 +26,11 @@ import org.eclipse.jgit.util.FS;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 
 @Singleton
-public class DefaultSecureStore implements SecureStore {
+public class DefaultSecureStore extends SecureStore {
   private final FileBasedConfig sec;
 
   @Inject
@@ -43,26 +44,11 @@ public class DefaultSecureStore implements SecureStore {
     }
   }
 
-  @Override
-  public String get(String section, String subsection, String name) {
-    return sec.getString(section, subsection, name);
-  }
-
   @Override
   public String[] getList(String section, String subsection, String name) {
     return sec.getStringList(section, subsection, name);
   }
 
-  @Override
-  public void set(String section, String subsection, String name, String value) {
-    if (value != null) {
-      sec.setString(section, subsection, name, value);
-    } else {
-      sec.unset(section, subsection, name);
-    }
-    save();
-  }
-
   @Override
   public void setList(String section, String subsection, String name,
       List<String> values) {
@@ -80,6 +66,22 @@ public class DefaultSecureStore implements SecureStore {
     save();
   }
 
+  @Override
+  public Iterable<EntryKey> list() {
+    List<EntryKey> result = new ArrayList<>();
+    for (String section : sec.getSections()) {
+      for (String subsection : sec.getSubsections(section)) {
+        for (String name : sec.getNames(section, subsection)) {
+          result.add(new EntryKey(section, subsection, name));
+        }
+      }
+      for (String name : sec.getNames(section)) {
+        result.add(new EntryKey(section, null, name));
+      }
+    }
+    return result;
+  }
+
   private void save() {
     try {
       saveSecure(sec);

gerrit-server/src/main/java/com/google/gerrit/server/securestore/SecureStore.java

@@ -14,17 +14,115 @@
 
 package com.google.gerrit.server.securestore;
 
+import com.google.common.collect.Lists;
+
 import java.util.List;
 
-public interface SecureStore {
+/**
+ * Abstract class for providing new SecureStore implementation for Gerrit.
+ *
+ * SecureStore is responsible for storing sensitive data like passwords in a
+ * secure manner.
+ *
+ * It is implementator's responsibility to encrypt and store values.
+ *
+ * To deploy new SecureStore one needs to provide a jar file with explicitly one
+ * class that extends {@code SecureStore} and put it in Gerrit server. Then run:
+ *
+ * `java -jar gerrit.war SwitchSecureStore -d $gerrit_site --new-secure-store-lib
+ *  $path_to_new_secure_store.jar`
+ *
+ * on stopped Gerrit instance.
+ */
+public abstract class SecureStore {
+  /**
+   * Describes {@link SecureStore} entry
+   */
+  public static class EntryKey {
+    public final String name;
+    public final String section;
+    public final String subsection;
 
-  String get(String section, String subsection, String name);
+    /**
+     * Creates EntryKey.
+     *
+     * @param section
+     * @param subsection
+     * @param name
+     */
+    public EntryKey(String section, String subsection, String name) {
+      this.name = name;
+      this.section = section;
+      this.subsection = subsection;
+    }
+  }
 
-  String[] getList(String section, String subsection, String name);
+  /**
+   * Extract decrypted value of stored property from SecureStore or {@code null}
+   * when property was not found.
+   *
+   * @param section
+   * @param subsection
+   * @param name
+   * @return decrypted String value or {@code null} if not found
+   */
+  public final String get(String section, String subsection, String name) {
+    String[] values = getList(section, subsection, name);
+    if (values != null && values.length > 0) {
+      return values[0];
+    }
+    return null;
+  }
 
-  void set(String section, String subsection, String name, String value);
+  /**
+   * Extract list of values from SecureStore and decrypt every value in that
+   * list or {@code null} when property was not found.
+   *
+   * @param section
+   * @param subsection
+   * @param name
+   * @return decrypted list of string values or {@code null}
+   */
+  public abstract String[] getList(String section, String subsection, String name);
 
-  void setList(String section, String subsection, String name, List<String> values);
+  /**
+   * Store single value in SecureStore.
+   *
+   * This method is responsible for encrypting value and storing it.
+   *
+   * @param section
+   * @param subsection
+   * @param name
+   * @param value plain text value
+   */
+  public final void set(String section, String subsection, String name, String value) {
+    setList(section, subsection, name, Lists.newArrayList(value));
+  }
 
-  void unset(String section, String subsection, String name);
+  /**
+   * Store list of values in SecureStore.
+   *
+   * This method is responsible for encrypting all values in the list and storing them.
+   *
+   * @param section
+   * @param subsection
+   * @param name
+   * @param values list of plain text values
+   */
+  public abstract void setList(String section, String subsection, String name, List<String> values);
+
+  /**
+   * Remove value for given {@code section}, {@code subsection} and {@code name}
+   * from SecureStore.
+   *
+   * @param section
+   * @param subsection
+   * @param name
+   */
+  public abstract void unset(String section, String subsection, String name);
+
+  /**
+   * @return list of stored entries.
+   */
+  public abstract Iterable<EntryKey> list();
 }