Draft integration of Lucene search mechanism and web ui

James Moger

2012-02-28 43e32ec71a508e1bb68b247fdca74f64bcf629b3

Draft integration of Lucene search mechanism and web ui

 distrib/gitblit.properties

@@ -91,6 +91,26 @@
# SINCE 0.8.0

groovy.postReceiveScripts =



# If true, a Lucene index will be generated and maintained for each repository.

# Lucene search replaces brute-force Git repository traversal.

#

# SINCE 0.9.0

# RESTART REQUIRED

lucene.enable  = false



# If *lucene.pollingMode* = true, Gitblit will periodically check all repositories

# for branch updates.

# If *lucene.pollingMode* = false, repositories will only be indexed on pushes

# to Gitblit.

#

# Regardless of this setting, Gitblit will check all repositories for branch

# updates 1 minute after startup. Indexes will automatically be built for any

# repository that is missing its index  or if an index version change is detected.

#

# SINCE 0.9.0

# RESTART REQUIRED

lucene.pollingMode = false



#

# Authentication Settings

#


 resources/gitblit.css

@@ -266,6 +266,18 @@
    padding: 2px;

}



div.searchResult {

    padding:5px;

}



div.searchResult .summary {

    font-weight: bold;

}



div.searchResult .branch {

    color: #008000;

}



div.header, div.commitHeader, table.repositories th {

    background-color:#e0e0e0;

    background-repeat:repeat-x;


 src/com/gitblit/GitBlit.java

@@ -1824,8 +1824,8 @@
        }

        luceneExecutor = new LuceneExecutor(settings);

        if (luceneExecutor.isReady()) {

            logger.info("Lucene executor is scheduled to process the repository queue every 10 minutes.");

            scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, 10, TimeUnit.MINUTES);

            logger.info("Lucene executor is scheduled to process the repository queue every 2 minutes.");

            scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, 2, TimeUnit.MINUTES);

        } else {

            logger.warn("Lucene executor is disabled.");

        }


 src/com/gitblit/LuceneExecutor.java

@@ -29,6 +29,7 @@
import com.gitblit.models.RepositoryModel;

import com.gitblit.utils.JGitUtils;

import com.gitblit.utils.LuceneUtils;

import com.gitblit.utils.LuceneUtils.IndexResult;



/**

 * The Lucene executor handles indexing repositories synchronously and

@@ -53,8 +54,8 @@


    public LuceneExecutor(IStoredSettings settings) {

        this.settings = settings;

        this.isLuceneEnabled = settings.getBoolean("lucene.enableLucene", false);

        this.isPollingMode = settings.getBoolean("lucene.pollingMode", false);

        this.isLuceneEnabled = settings.getBoolean(Keys.lucene.enable, false);

        this.isPollingMode = settings.getBoolean(Keys.lucene.pollingMode, false);

    }



    /**

@@ -144,11 +145,14 @@
                if (LuceneUtils.shouldReindex(repository)) {

                    // (re)build the entire index

                    long start = System.currentTimeMillis();

                    boolean success = LuceneUtils.reindex(repository);

                    IndexResult result = LuceneUtils.reindex(repository);

                    long duration = System.currentTimeMillis() - start;

                    if (success) {

                        String msg = "Built {0} Lucene index in {1} msecs";

                        logger.info(MessageFormat.format(msg, repositoryName, duration));

                    if (result.success) {

                        if (result.commitCount > 0) {

                            String msg = "Built {0} Lucene index from {1} commits in {2} msecs";

                            logger.info(MessageFormat.format(msg, repositoryName,

                                    result.commitCount, duration));

                        }

                    } else {

                        String msg = "Could not build {0} Lucene index!";

                        logger.error(MessageFormat.format(msg, repositoryName));

@@ -156,11 +160,14 @@
                } else {

                    // update the index with latest commits

                    long start = System.currentTimeMillis();

                    boolean success = LuceneUtils.updateIndex(repository);

                    IndexResult result = LuceneUtils.updateIndex(repository);

                    long duration = System.currentTimeMillis() - start;

                    if (success) {

                        String msg = "Updated {0} Lucene index in {1} msecs";

                        logger.info(MessageFormat.format(msg, repositoryName, duration));

                    if (result.success) {

                        if (result.commitCount > 0) {

                            String msg = "Updated {0} Lucene index with {1} commits in {2} msecs";

                            logger.info(MessageFormat.format(msg, repositoryName,

                                    result.commitCount, duration));

                        }

                    } else {

                        String msg = "Could not update {0} Lucene index!";

                        logger.error(MessageFormat.format(msg, repositoryName));


 src/com/gitblit/utils/LuceneUtils.java

@@ -52,6 +52,7 @@
import org.eclipse.jgit.treewalk.TreeWalk;

import org.eclipse.jgit.util.FS;



import com.gitblit.GitBlit;

import com.gitblit.models.IssueModel;

import com.gitblit.models.IssueModel.Attachment;

import com.gitblit.models.PathModel.PathChangeModel;

@@ -121,10 +122,13 @@
     * @return the repository name

     */

    private static String getName(Repository repository) {

        String rootPath = GitBlit.getRepositoriesFolder().getAbsolutePath();

        if (repository.isBare()) {

            return repository.getDirectory().getName();

            return StringUtils.getRelativePath(rootPath, repository.getDirectory()

                    .getAbsolutePath());

        } else {

            return repository.getDirectory().getParentFile().getName();

            return StringUtils.getRelativePath(rootPath, repository.getDirectory().getParentFile()

                    .getAbsolutePath());

        }

    }



@@ -198,11 +202,12 @@
     * index.

     * 

     * @param repository

     * @return true if the indexing has succeeded

     * @return IndexResult

     */

    public static boolean reindex(Repository repository) {

    public static IndexResult reindex(Repository repository) {

        IndexResult result = new IndexResult();

        if (!LuceneUtils.deleteIndex(repository)) {

            return false;

            return result;

        }

        try {

            String repositoryName = getName(repository);

@@ -300,6 +305,7 @@
                            Index.NOT_ANALYZED));

                    doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));

                    writer.addDocument(doc);

                    result.commitCount += 1;

                }



                // traverse the log and index the previous commit objects

@@ -312,6 +318,7 @@
                                Index.NOT_ANALYZED));

                        doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));

                        writer.addDocument(doc);

                        result.commitCount += 1;

                    }

                }



@@ -335,11 +342,11 @@
            config.save();

            resetIndexSearcher(repository);

            writer.commit();

            return true;

            result.success = true;

        } catch (Exception e) {

            e.printStackTrace();

        }

        return false;

        return result;

    }



    /**

@@ -453,9 +460,10 @@
     * Updates a repository index incrementally from the last indexed commits.

     * 

     * @param repository

     * @return IndexResult

     */

    public static boolean updateIndex(Repository repository) {

        boolean success = false;

    public static IndexResult updateIndex(Repository repository) {

        IndexResult result = new IndexResult();

        try {

            FileBasedConfig config = getConfig(repository);

            config.load();

@@ -475,13 +483,13 @@


            // detect branch deletion

            // first assume all branches are deleted and then remove each

            // existing branch from deletedBranches during indexing			
            // existing branch from deletedBranches during indexing

            Set<String> deletedBranches = new TreeSet<String>();

            for (String alias : config.getNames(CONF_ALIAS)) {

                String branch = config.getString(CONF_ALIAS, null, alias);

                deletedBranches.add(branch);

            }

			


            // walk through each branches

            List<RefModel> branches = JGitUtils.getLocalBranches(repository, true, -1);

            for (RefModel branch : branches) {

@@ -490,7 +498,7 @@
                // remove this branch from the deletedBranches set

                deletedBranches.remove(branchName);



                // determine last commit				
                // determine last commit

                String keyName = getBranchKey(branchName);

                String lastCommit = config.getString(CONF_BRANCH, null, keyName);



@@ -507,6 +515,7 @@
                Collections.reverse(revs);

                for (RevCommit commit : revs) {

                    index(repository, branchName, commit);

                    result.commitCount += 1;

                }



                // update the config

@@ -515,7 +524,7 @@
                config.setString(CONF_BRANCH, null, keyName, branch.getObjectId().getName());

                config.save();

            }

			


            // the deletedBranches set will normally be empty by this point

            // unless a branch really was deleted and no longer exists

            if (deletedBranches.size() > 0) {

@@ -525,11 +534,11 @@
                    writer.commit();

                }

            }

            success = true;

            result.success = true;

        } catch (Throwable t) {

            t.printStackTrace();

        }

        return success;

        return result;

    }



    /**

@@ -782,4 +791,9 @@
        }

        SEARCHERS.clear();

    }



    public static class IndexResult {

        public boolean success;

        public int commitCount;

    }

}


 src/com/gitblit/wicket/pages/RootPage.java

@@ -101,6 +101,9 @@
        pages.add(new PageRegistration("gb.repositories", RepositoriesPage.class,

                getRootPageParameters()));

        pages.add(new PageRegistration("gb.activity", ActivityPage.class, getRootPageParameters()));

        if (GitBlit.getBoolean(Keys.lucene.enable, false)) {

            pages.add(new PageRegistration("gb.search", LucenePage.class));

        }

        if (showAdmin) {

            pages.add(new PageRegistration("gb.users", UsersPage.class));

        }