From 74517e1b8b4f3a2b59800559d494ad50987ccfb6 Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Wed, 22 May 2013 17:37:36 -0700 Subject: [PATCH] Keep track of Lucene and index schema versions Like SchemaVersionCheck, use a LifecycleManager to check the schema versions on startup. We need to check both: - The on-disk format of the Lucene index. We specify the current version manually in LuceneChangeIndex, since Lucene itself doesn't store a version in the directory. (CheckIndex claims to be able to detect an out-of-date index file format, but it seems buggy.) - The version of the ChangeField schema, which indicates the semantics of the various fields. Persist both of these versions in a simple config file, index/gerrit_index.config. If the check fails, inform the user they need to reindex. Rewrite Reindex to delete all the files in the Lucene directory, not just the documents. Change-Id: Ib82b8aab21c819db3625cd9984262a40135e0894 --- .../gerrit/lucene/IndexVersionCheck.java | 92 +++++++++++++++++++ .../gerrit/lucene/LuceneChangeIndex.java | 8 +- .../gerrit/lucene/LuceneIndexModule.java | 13 +++ .../java/com/google/gerrit/pgm/Reindex.java | 22 ++++- .../gerrit/server/index/ChangeField.java | 3 + 5 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 gerrit-lucene/src/main/java/com/google/gerrit/lucene/IndexVersionCheck.java diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/IndexVersionCheck.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/IndexVersionCheck.java new file mode 100644 index 0000000000..fd8bc09610 --- /dev/null +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/IndexVersionCheck.java @@ -0,0 +1,92 @@ +// Copyright (C) 2013 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.package com.google.gerrit.server.git; + +package com.google.gerrit.lucene; + +import static com.google.gerrit.lucene.LuceneChangeIndex.LUCENE_VERSION; + +import static org.apache.lucene.util.Version.LUCENE_CURRENT; + +import com.google.common.collect.ImmutableMap; +import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.ChangeField; +import com.google.inject.Inject; +import com.google.inject.ProvisionException; + +import org.apache.lucene.util.Version; +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.util.FS; + +import java.io.File; +import java.io.IOException; +import java.util.Map; + +public class IndexVersionCheck implements LifecycleListener { + public static final Map SCHEMA_VERSIONS = ImmutableMap.of( + "changes", ChangeField.SCHEMA_VERSION); + + public static File gerritIndexConfig(SitePaths sitePaths) { + return new File(sitePaths.index_dir, "gerrit_index.config"); + } + + private final SitePaths sitePaths; + + @Inject + IndexVersionCheck(SitePaths sitePaths) { + this.sitePaths = sitePaths; + } + + @Override + public void start() { + File file = gerritIndexConfig(sitePaths); + try { + FileBasedConfig cfg = new FileBasedConfig(file, FS.detect()); + cfg.load(); + for (Map.Entry e : SCHEMA_VERSIONS.entrySet()) { + int schemaVersion = cfg.getInt("index", e.getKey(), "schemaVersion", 0); + if (schemaVersion != e.getValue()) { + throw new ProvisionException(String.format( + "wrong index schema version for \"%s\": expected %d, found %d%s", + e.getKey(), e.getValue(), schemaVersion, upgrade())); + } + } + @SuppressWarnings("deprecation") + Version luceneVersion = + cfg.getEnum("lucene", null, "version", LUCENE_CURRENT); + if (luceneVersion != LUCENE_VERSION) { + throw new ProvisionException(String.format( + "wrong Lucene version: expected %d, found %d%s", + luceneVersion, LUCENE_VERSION, upgrade())); + + } + } catch (IOException e) { + throw new ProvisionException("unable to read " + file); + } catch (ConfigInvalidException e) { + throw new ProvisionException("invalid config file " + file); + } + } + + @Override + public void stop() { + // Do nothing. + } + + private final String upgrade() { + return "\nRun reindex to rebuild the index:\n" + + "$ java -jar gerrit.war reindex -d " + + sitePaths.site_path.getAbsolutePath(); + } +} diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java index a05f1eb138..aa76355daf 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java @@ -75,7 +75,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { private static final Logger log = LoggerFactory.getLogger(LuceneChangeIndex.class); - private static final Version VERSION = Version.LUCENE_43; + public static final Version LUCENE_VERSION = Version.LUCENE_43; private final FillArgs fillArgs; private final Directory dir; @@ -88,7 +88,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { this.fillArgs = fillArgs; dir = FSDirectory.open(new File(sitePaths.index_dir, "changes")); IndexWriterConfig writerConfig = - new IndexWriterConfig(VERSION, new StandardAnalyzer(VERSION)); + new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)); writerConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); writer = new IndexWriter(dir, writerConfig); searcherManager = new SearcherManager(writer, true, null); @@ -143,6 +143,10 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener { } } + public Directory getDirectory() { + return dir; + } + public IndexWriter getWriter() { return writer; } diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java index ee88342a69..9c344bb0e6 100644 --- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java +++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java @@ -30,10 +30,23 @@ public class LuceneIndexModule extends LifecycleModule { .getBoolean("index", null, "enabled", false); } + private final boolean checkVersion; + + public LuceneIndexModule() { + this(true); + } + + public LuceneIndexModule(boolean checkVersion) { + this.checkVersion = checkVersion; + } + @Override protected void configure() { bind(ChangeIndex.class).to(LuceneChangeIndex.class); bind(ChangeIndexer.class).to(ChangeIndexerImpl.class); listener().to(LuceneChangeIndex.class); + if (checkVersion) { + listener().to(IndexVersionCheck.class); + } } } diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java index c5bb33f535..482ecc6a1a 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java @@ -14,6 +14,9 @@ package com.google.gerrit.pgm; +import static com.google.gerrit.lucene.IndexVersionCheck.SCHEMA_VERSIONS; +import static com.google.gerrit.lucene.IndexVersionCheck.gerritIndexConfig; +import static com.google.gerrit.lucene.LuceneChangeIndex.LUCENE_VERSION; import static com.google.gerrit.server.schema.DataSourceProvider.Context.SINGLE_USER; import com.google.common.base.Stopwatch; @@ -40,10 +43,14 @@ import com.google.inject.TypeLiteral; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.util.FS; import java.io.File; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -86,6 +93,7 @@ public class Reindex extends SiteProgram { index.getWriter().commit(); double elapsed = sw.elapsed(TimeUnit.MILLISECONDS) / 1000d; System.out.format("Reindexed %d changes in %.02fms", i, elapsed); + writeVersion(); manager.stop(); return 0; @@ -94,7 +102,7 @@ public class Reindex extends SiteProgram { private Injector createSysInjector() { List modules = Lists.newArrayList(); modules.add(PatchListCacheImpl.module()); - modules.add(new LuceneIndexModule()); + modules.add(new LuceneIndexModule(false)); modules.add(new AbstractModule() { @SuppressWarnings("rawtypes") @Override @@ -128,4 +136,16 @@ public class Reindex extends SiteProgram { } } } + + private void writeVersion() throws IOException, ConfigInvalidException { + FileBasedConfig cfg = + new FileBasedConfig(gerritIndexConfig(sitePaths), FS.detect()); + cfg.load(); + + for (Map.Entry e : SCHEMA_VERSIONS.entrySet()) { + cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue()); + } + cfg.setEnum("lucene", null, "version", LUCENE_VERSION); + cfg.save(); + } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java index 3b3fac7249..602c59b9c1 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java @@ -36,6 +36,9 @@ import java.util.Map; * Used to generate a schema for index implementations that require one. */ public class ChangeField { + /** Increment whenever making schema changes. */ + public static final int SCHEMA_VERSION = 1; + /** Legacy change ID. */ public static final FieldDef CHANGE_ID = new FieldDef.Single(ChangeQueryBuilder.FIELD_CHANGE,