More Lucene progress

James Moger

2012-03-09 3d0494ad6db80843add9bf9d6eac783a691ac5a1

More Lucene progress

 src/com/gitblit/GitBlit.java

@@ -184,6 +184,7 @@
        }

        return self().timezone;

    }

	


    /**

     * Returns the boolean value for the specified key. If the key does not


 src/com/gitblit/LuceneExecutor.java

@@ -105,26 +105,26 @@
        Set<String> processed = new HashSet<String>();

        if (!queue.isEmpty()) {

            // update the repository Lucene index

            String repositoryName = null;

            while ((repositoryName = queue.poll()) != null) {

                if (processed.contains(repositoryName)) {

            String name = null;

            while ((name = queue.poll()) != null) {

                if (processed.contains(name)) {

                    // skipping multi-queued repository

                    continue;

                }

                try {

                    Repository repository = GitBlit.self().getRepository(repositoryName);

                    Repository repository = GitBlit.self().getRepository(name);

                    if (repository == null) {

                        logger.warn(MessageFormat.format(

                                "Lucene executor could not find repository {0}. Skipping.",

                                repositoryName));

                                name));

                        continue;

                    }

                    index(repositoryName, repository);

                    index(name, repository);

                    repository.close();

                    processed.add(repositoryName);

                    processed.add(name);

                } catch (Throwable e) {

                    logger.error(MessageFormat.format("Failed to update {0} Lucene index",

                            repositoryName), e);

                            name), e);

                }

            }

        }

@@ -134,51 +134,53 @@
     * Synchronously indexes a repository. This may build a complete index of a

     * repository or it may update an existing index.

     * 

     * @param repositoryName

     * @param name

     *            the name of the repository

     * @param repository

     *            the repository object

     */

    public void index(String repositoryName, Repository repository) {

    public void index(String name, Repository repository) {

        try {

            if (JGitUtils.hasCommits(repository)) {

                if (LuceneUtils.shouldReindex(repository)) {

                    // (re)build the entire index

                    long start = System.currentTimeMillis();

                    IndexResult result = LuceneUtils.reindex(repository);

                    long duration = System.currentTimeMillis() - start;

                    String msg = "Building {0} Lucene index...";

                    logger.info(MessageFormat.format(msg, name));

                    IndexResult result = LuceneUtils.reindex(name, repository, true);

                    float duration = (System.currentTimeMillis() - start)/1000f;

                    if (result.success) {

                        if (result.commitCount > 0) {

                            String msg = "Built {0} Lucene index from {1} commits in {2} msecs";

                            logger.info(MessageFormat.format(msg, repositoryName,

                                    result.commitCount, duration));

                            msg = "Built {0} Lucene index from {1} commits and {2} files across {3} branches in {4} secs";

                            logger.info(MessageFormat.format(msg, name,

                                    result.commitCount, result.blobCount, result.branchCount, duration));

                        }

                    } else {

                        String msg = "Could not build {0} Lucene index!";

                        logger.error(MessageFormat.format(msg, repositoryName));

                        msg = "Could not build {0} Lucene index!";

                        logger.error(MessageFormat.format(msg, name));

                    }

                } else {

                    // update the index with latest commits

                    long start = System.currentTimeMillis();

                    IndexResult result = LuceneUtils.updateIndex(repository);

                    long duration = System.currentTimeMillis() - start;

                    IndexResult result = LuceneUtils.updateIndex(name, repository);

                    float duration = (System.currentTimeMillis() - start)/1000f;

                    if (result.success) {

                        if (result.commitCount > 0) {

                            String msg = "Updated {0} Lucene index with {1} commits in {2} msecs";

                            logger.info(MessageFormat.format(msg, repositoryName,

                                    result.commitCount, duration));

                            String msg = "Updated {0} Lucene index with {1} commits and {2} files across {3} branches in {4} secs";

                            logger.info(MessageFormat.format(msg, name,

                                    result.commitCount, result.blobCount, result.branchCount, duration));

                        }

                    } else {

                        String msg = "Could not update {0} Lucene index!";

                        logger.error(MessageFormat.format(msg, repositoryName));

                        logger.error(MessageFormat.format(msg, name));

                    }

                }

            } else {

                logger.info(MessageFormat.format("Skipped Lucene index of empty repository {0}",

                        repositoryName));

                        name));

            }

        } catch (Throwable t) {

            logger.error(MessageFormat.format("Lucene indexing failure for {0}", repositoryName), t);

            logger.error(MessageFormat.format("Lucene indexing failure for {0}", name), t);

        }

    }




 src/com/gitblit/models/SearchResult.java

@@ -32,6 +32,8 @@


    public String id;



    public List<String> tags;

	
    public List<String> labels;



    public ObjectType type;


 src/com/gitblit/utils/LuceneUtils.java

@@ -1,3 +1,18 @@
/*

 * Copyright 2012 gitblit.com.

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *     http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package com.gitblit.utils;



import java.io.ByteArrayOutputStream;

@@ -50,9 +65,11 @@
import org.eclipse.jgit.revwalk.RevWalk;

import org.eclipse.jgit.storage.file.FileBasedConfig;

import org.eclipse.jgit.treewalk.TreeWalk;

import org.eclipse.jgit.treewalk.filter.AndTreeFilter;

import org.eclipse.jgit.treewalk.filter.PathFilterGroup;

import org.eclipse.jgit.treewalk.filter.TreeFilter;

import org.eclipse.jgit.util.FS;



import com.gitblit.GitBlit;

import com.gitblit.models.IssueModel;

import com.gitblit.models.IssueModel.Attachment;

import com.gitblit.models.PathModel.PathChangeModel;

@@ -96,6 +113,7 @@
    private static final String FIELD_AUTHOR = "author";

    private static final String FIELD_COMMITTER = "committer";

    private static final String FIELD_DATE = "date";

    private static final String FIELD_TAG = "tag";

    private static final String FIELD_LABEL = "label";

    private static final String FIELD_ATTACHMENT = "attachment";



@@ -109,27 +127,47 @@
    private static final Map<File, IndexSearcher> SEARCHERS = new ConcurrentHashMap<File, IndexSearcher>();

    private static final Map<File, IndexWriter> WRITERS = new ConcurrentHashMap<File, IndexWriter>();



    private static final String LUCENE_DIR = "lucene";

    private static final String CONF_FILE = "lucene.conf";

    private static final String CONF_INDEX = "index";

    private static final String CONF_VERSION = "version";

    private static final String CONF_ALIAS = "aliases";

    private static final String CONF_BRANCH = "branches";



	
    /**

     * Returns the name of the repository.

     * Returns the author for the commit, if this information is available.

     * 

     * @param repository

     * @return the repository name

     * @param commit

     * @return an author or unknown

     */

    private static String getName(Repository repository) {

        String rootPath = GitBlit.getRepositoriesFolder().getAbsolutePath();

        if (repository.isBare()) {

            return StringUtils.getRelativePath(rootPath, repository.getDirectory()

                    .getAbsolutePath());

        } else {

            return StringUtils.getRelativePath(rootPath, repository.getDirectory().getParentFile()

                    .getAbsolutePath());

    private static String getAuthor(RevCommit commit) {

        String name = "unknown";

        try {

            name = commit.getAuthorIdent().getName();

            if (StringUtils.isEmpty(name)) {

                name = commit.getAuthorIdent().getEmailAddress();

            }

        } catch (NullPointerException n) {						
        }

        return name;

    }

	
    /**

     * Returns the committer for the commit, if this information is available.

     * 
     * @param commit

     * @return an committer or unknown

     */

    private static String getCommitter(RevCommit commit) {

        String name = "unknown";

        try {

            name = commit.getCommitterIdent().getName();

            if (StringUtils.isEmpty(name)) {

                name = commit.getCommitterIdent().getEmailAddress();

            }

        } catch (NullPointerException n) {						
        }

        return name;

    }



    /**

@@ -182,7 +220,7 @@
     */

    public static boolean deleteIndex(Repository repository) {

        try {

            File luceneIndex = new File(repository.getDirectory(), "lucene");

            File luceneIndex = new File(repository.getDirectory(), LUCENE_DIR);

            if (luceneIndex.exists()) {

                org.eclipse.jgit.util.FileUtils.delete(luceneIndex,

                        org.eclipse.jgit.util.FileUtils.RECURSIVE);

@@ -201,16 +239,22 @@
     * This completely indexes the repository and will destroy any existing

     * index.

     * 

     * @param repositoryName

     * @param repository

     * @param fullIndex

     *            If false blob metadata is set to the HEAD revision of each

     *            branch.  If true, each the last commit of each blob is

     *            determined to properly index the author, committer, and date.

     *            Full indexing can be time-consuming.

     * @return IndexResult

     */

    public static IndexResult reindex(Repository repository) {

    public static IndexResult reindex(String repositoryName, Repository repository,

            boolean fullIndex) {

        IndexResult result = new IndexResult();

        if (!LuceneUtils.deleteIndex(repository)) {

            return result;

        }

        try {

            String repositoryName = getName(repository);

        try {			
            FileBasedConfig config = getConfig(repository);

            Set<String> indexedCommits = new TreeSet<String>();

            IndexWriter writer = getIndexWriter(repository, true);

@@ -235,40 +279,64 @@
                }

                String branchName = branch.getName();

                RevWalk revWalk = new RevWalk(repository);

                RevCommit rev = revWalk.parseCommit(branch.getObjectId());

                RevCommit branchHead = revWalk.parseCommit(branch.getObjectId());

                String head = branchHead.getId().getName();



                String keyName = getBranchKey(branchName);

                config.setString(CONF_ALIAS, null, keyName, branchName);

                config.setString(CONF_BRANCH, null, keyName, rev.getName());

                config.setString(CONF_BRANCH, null, keyName, head);



                // index the blob contents of the tree

                ByteArrayOutputStream os = new ByteArrayOutputStream();

                byte[] tmp = new byte[32767];

                TreeWalk treeWalk = new TreeWalk(repository);

                treeWalk.addTree(rev.getTree());

                treeWalk.addTree(branchHead.getTree());

                treeWalk.setRecursive(true);

                String revDate = DateTools.timeToString(rev.getCommitTime() * 1000L,

                        Resolution.MINUTE);

				
                while (treeWalk.next()) {

                    result.blobCount++;

                    String blobPath = treeWalk.getPathString();

                    RevCommit blobRev = branchHead;

					
                    RevWalk blobWalk = null;

                    if (fullIndex) {

                        // XXX this is _really_ slow, there must be a better way

                        // determine the most recent commit for this blob

                        blobWalk = new RevWalk(repository);

                        blobWalk.markStart(blobWalk.parseCommit(branch.getObjectId()));

                        TreeFilter filter = AndTreeFilter.create(

                                PathFilterGroup.createFromStrings(Collections.singleton(blobPath)),

                                TreeFilter.ANY_DIFF);

                        blobWalk.setTreeFilter(filter);



                        for (RevCommit commit : blobWalk) {

                            blobRev = commit;

                            break;

                        }

                    }

					
                    String blobAuthor = getAuthor(blobRev);

                    String blobCommitter = getCommitter(blobRev);

                    String blobDate = DateTools.timeToString(blobRev.getCommitTime() * 1000L,

                            Resolution.MINUTE);

					
                    if (blobWalk != null) {

                        blobWalk.dispose();

                    }

					
                    Document doc = new Document();

                    doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES,

                            Index.NOT_ANALYZED_NO_NORMS));

                    doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,

                            Index.NOT_ANALYZED));

                    doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));

                    doc.add(new Field(FIELD_OBJECT_ID, treeWalk.getPathString(), Store.YES,

                            Index.NOT_ANALYZED));

                    doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO));

                    doc.add(new Field(FIELD_AUTHOR, rev.getAuthorIdent().getName(), Store.YES,

                            Index.NOT_ANALYZED_NO_NORMS));

                    doc.add(new Field(FIELD_COMMITTER, rev.getCommitterIdent().getName(),

                            Store.YES, Index.NOT_ANALYZED_NO_NORMS));

                    doc.add(new Field(FIELD_LABEL, branch.getName(), Store.YES, Index.ANALYZED));

                    doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES, Index.NOT_ANALYZED_NO_NORMS));

                    doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED));

                    doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED));

                    doc.add(new Field(FIELD_OBJECT_ID, blobPath, Store.YES, Index.ANALYZED));

                    doc.add(new Field(FIELD_DATE, blobDate, Store.YES, Index.NO));

                    doc.add(new Field(FIELD_AUTHOR, blobAuthor, Store.YES, Index.ANALYZED));

                    doc.add(new Field(FIELD_COMMITTER, blobCommitter, Store.YES, Index.ANALYZED));					


                    // determine extension to compare to the extension

                    // blacklist

                    String ext = null;

                    String name = treeWalk.getPathString().toLowerCase();

                    String name = blobPath.toLowerCase();

                    if (name.indexOf('.') > -1) {

                        ext = name.substring(name.lastIndexOf('.') + 1);

                    }

@@ -298,25 +366,25 @@
                treeWalk.release();



                // index the head commit object

                String head = rev.getId().getName();

                if (indexedCommits.add(head)) {

                    Document doc = createDocument(rev, tags.get(head));

                    doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,

                            Index.NOT_ANALYZED));

                    doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));

                    Document doc = createDocument(branchHead, tags.get(head));

                    doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED));

                    doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED));

                    writer.addDocument(doc);

                    result.commitCount += 1;

                    result.branchCount += 1;

                }



                // traverse the log and index the previous commit objects

                revWalk.markStart(rev);

                revWalk.reset();

                revWalk.markStart(branchHead);

                RevCommit rev;

                while ((rev = revWalk.next()) != null) {

                    String hash = rev.getId().getName();

                    if (indexedCommits.add(hash)) {

                        Document doc = createDocument(rev, tags.get(hash));

                        doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,

                                Index.NOT_ANALYZED));

                        doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));

                        doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED));

                        doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED));

                        writer.addDocument(doc);

                        result.commitCount += 1;

                    }

@@ -329,10 +397,13 @@
            // this repository has a gb-issues branch, index all issues

            if (IssueUtils.getIssuesBranch(repository) != null) {

                List<IssueModel> issues = IssueUtils.getIssues(repository, null);

                if (issues.size() > 0) {

                    result.branchCount += 1;

                }

                for (IssueModel issue : issues) {

                    result.issueCount++;

                    Document doc = createDocument(issue);

                    doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,

                            Index.NOT_ANALYZED));

                    doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED));

                    writer.addDocument(doc);

                }

            }

@@ -353,13 +424,16 @@
     * Incrementally update the index with the specified commit for the

     * repository.

     * 

     * @param repositoryName

     * @param repository

     * @param branch

     *            the fully qualified branch name (e.g. refs/heads/master)

     * @param commit

     * @return true, if successful

     */

    public static boolean index(Repository repository, String branch, RevCommit commit) {

    private static IndexResult index(String repositoryName, Repository repository, 
            String branch, RevCommit commit) {

        IndexResult result = new IndexResult();

        try {

            if (excludedBranches.contains(branch)) {

                if (IssueUtils.GB_ISSUES.equals(branch)) {

@@ -367,20 +441,23 @@
                    String issueId = commit.getShortMessage().substring(2).trim();

                    IssueModel issue = IssueUtils.getIssue(repository, issueId);

                    if (issue == null) {

                        // delete the old issue from the index, if exists

                        // issue was deleted, remove from index

                        IndexWriter writer = getIndexWriter(repository, false);

                        writer.deleteDocuments(

                                new Term(FIELD_OBJECT_TYPE, ObjectType.issue.name()), new Term(

                                        FIELD_OBJECT_ID, issueId));

                        writer.commit();

                        return true;

                        result.success = true;

                        return result;

                    }

                    return index(repository, issue);

                    result.success = index(repositoryName, repository, issue);

                    result.issueCount++;

                    return result;

					
                }

                return false;

                return result;

            }

            List<PathChangeModel> changedPaths = JGitUtils.getFilesInCommit(repository, commit);

            String repositoryName = getName(repository);

            String revDate = DateTools.timeToString(commit.getCommitTime() * 1000L,

                    Resolution.MINUTE);

            IndexWriter writer = getIndexWriter(repository, false);

@@ -391,19 +468,16 @@


                // re-index the blob

                if (!ChangeType.DELETE.equals(path.changeType)) {

                    result.blobCount++;

                    Document doc = new Document();

                    doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES,

                            Index.NOT_ANALYZED_NO_NORMS));

                    doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,

                            Index.NOT_ANALYZED));

                    doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.NOT_ANALYZED));

                    doc.add(new Field(FIELD_OBJECT_ID, path.path, Store.YES, Index.NOT_ANALYZED));

                    doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED));

                    doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.ANALYZED));

                    doc.add(new Field(FIELD_OBJECT_ID, path.path, Store.YES, Index.ANALYZED));

                    doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO));

                    doc.add(new Field(FIELD_AUTHOR, commit.getAuthorIdent().getName(), Store.YES,

                            Index.NOT_ANALYZED_NO_NORMS));

                    doc.add(new Field(FIELD_COMMITTER, commit.getCommitterIdent().getName(),

                            Store.YES, Index.NOT_ANALYZED_NO_NORMS));

                    doc.add(new Field(FIELD_LABEL, branch, Store.YES, Index.ANALYZED));

                    doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), Store.YES, Index.ANALYZED));

                    doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), Store.YES, Index.ANALYZED));



                    // determine extension to compare to the extension

                    // blacklist

@@ -425,11 +499,12 @@
            writer.commit();



            Document doc = createDocument(commit, null);

            return index(repository, doc);

            result.commitCount++;

            result.success = index(repositoryName, repository, doc);

        } catch (Exception e) {

            e.printStackTrace();

        }

        return false;

        return result;

    }



    /**

@@ -440,7 +515,7 @@
     * @param issue

     * @return true, if successful

     */

    public static boolean index(Repository repository, IssueModel issue) {

    public static boolean index(String repositoryName, Repository repository, IssueModel issue) {

        try {

            // delete the old issue from the index, if exists

            IndexWriter writer = getIndexWriter(repository, false);

@@ -449,7 +524,7 @@
            writer.commit();



            Document doc = createDocument(issue);

            return index(repository, doc);

            return index(repositoryName, repository, doc);

        } catch (Exception e) {

            e.printStackTrace();

        }

@@ -459,10 +534,11 @@
    /**

     * Updates a repository index incrementally from the last indexed commits.

     * 

     * @param repositoryName

     * @param repository

     * @return IndexResult

     */

    public static IndexResult updateIndex(Repository repository) {

    public static IndexResult updateIndex(String repositoryName, Repository repository) {

        IndexResult result = new IndexResult();

        try {

            FileBasedConfig config = getConfig(repository);

@@ -511,11 +587,14 @@
                    revs = JGitUtils.getRevLog(repository, lastCommit, branchName);

                }



                // reverse the list of commits so we start with the first commit

                if (revs.size() > 0) {

                    result.branchCount += 1;

                }

				
                // reverse the list of commits so we start with the first commit				
                Collections.reverse(revs);

                for (RevCommit commit : revs) {

                    index(repository, branchName, commit);

                    result.commitCount += 1;

                    result.add(index(repositoryName, repository, branchName, commit));					
                }



                // update the config

@@ -550,12 +629,12 @@
    private static Document createDocument(IssueModel issue) {

        Document doc = new Document();

        doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.issue.name(), Store.YES,

                Field.Index.NOT_ANALYZED_NO_NORMS));

        doc.add(new Field(FIELD_OBJECT_ID, issue.id, Store.YES, Index.NOT_ANALYZED));

        doc.add(new Field(FIELD_BRANCH, IssueUtils.GB_ISSUES, Store.YES, Index.NOT_ANALYZED));

                Field.Index.NOT_ANALYZED));

        doc.add(new Field(FIELD_OBJECT_ID, issue.id, Store.YES, Index.ANALYZED));

        doc.add(new Field(FIELD_BRANCH, IssueUtils.GB_ISSUES, Store.YES, Index.ANALYZED));

        doc.add(new Field(FIELD_DATE, DateTools.dateToString(issue.created, Resolution.MINUTE),

                Store.YES, Field.Index.NO));

        doc.add(new Field(FIELD_AUTHOR, issue.reporter, Store.YES, Index.NOT_ANALYZED_NO_NORMS));

        doc.add(new Field(FIELD_AUTHOR, issue.reporter, Store.YES, Index.ANALYZED));

        List<String> attachments = new ArrayList<String>();

        for (Attachment attachment : issue.getAttachments()) {

            attachments.add(attachment.name.toLowerCase());

@@ -579,19 +658,16 @@
    private static Document createDocument(RevCommit commit, List<String> tags) {

        Document doc = new Document();

        doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.commit.name(), Store.YES,

                Index.NOT_ANALYZED_NO_NORMS));

        doc.add(new Field(FIELD_OBJECT_ID, commit.getName(), Store.YES, Index.NOT_ANALYZED));

                Index.NOT_ANALYZED));

        doc.add(new Field(FIELD_OBJECT_ID, commit.getName(), Store.YES, Index.ANALYZED));

        doc.add(new Field(FIELD_DATE, DateTools.timeToString(commit.getCommitTime() * 1000L,

                Resolution.MINUTE), Store.YES, Index.NO));

        doc.add(new Field(FIELD_AUTHOR, commit.getCommitterIdent().getName(), Store.YES,

                Index.NOT_ANALYZED_NO_NORMS));

        doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), Store.YES, Index.ANALYZED));

        doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), Store.YES, Index.ANALYZED));

        doc.add(new Field(FIELD_SUMMARY, commit.getShortMessage(), Store.YES, Index.ANALYZED));

        doc.add(new Field(FIELD_CONTENT, commit.getFullMessage(), Store.NO, Index.ANALYZED));

        if (!ArrayUtils.isEmpty(tags)) {

            if (!ArrayUtils.isEmpty(tags)) {

                doc.add(new Field(FIELD_LABEL, StringUtils.flattenStrings(tags), Store.YES,

                        Index.ANALYZED));

            }

            doc.add(new Field(FIELD_TAG, StringUtils.flattenStrings(tags), Store.YES, Index.ANALYZED));

        }

        return doc;

    }

@@ -599,13 +675,13 @@
    /**

     * Incrementally index an object for the repository.

     * 

     * @param repositoryName

     * @param repository

     * @param doc

     * @return true, if successful

     */

    private static boolean index(Repository repository, Document doc) {

        try {

            String repositoryName = getName(repository);

    private static boolean index(String repositoryName, Repository repository, Document doc) {

        try {			
            doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.NOT_ANALYZED));

            IndexWriter writer = getIndexWriter(repository, false);

            writer.addDocument(doc);

@@ -629,6 +705,9 @@
        result.repository = doc.get(FIELD_REPOSITORY);

        result.branch = doc.get(FIELD_BRANCH);

        result.id = doc.get(FIELD_OBJECT_ID);

        if (doc.get(FIELD_TAG) != null) {

            result.tags = StringUtils.getStringsFromValue(doc.get(FIELD_TAG));

        }

        if (doc.get(FIELD_LABEL) != null) {

            result.labels = StringUtils.getStringsFromValue(doc.get(FIELD_LABEL));

        }

@@ -672,7 +751,7 @@
    private static IndexWriter getIndexWriter(Repository repository, boolean forceCreate)

            throws IOException {

        IndexWriter indexWriter = WRITERS.get(repository.getDirectory());

        File indexFolder = new File(repository.getDirectory(), "lucene");

        File indexFolder = new File(repository.getDirectory(), LUCENE_DIR);

        Directory directory = FSDirectory.open(indexFolder);

        if (forceCreate || !indexFolder.exists()) {

            // if the writer is going to blow away the existing index and create

@@ -794,6 +873,16 @@


    public static class IndexResult {

        public boolean success;

        public int branchCount;

        public int commitCount;

        public int blobCount;

        public int issueCount;

		
        public void add(IndexResult result) {

            this.branchCount += result.branchCount;

            this.commitCount += result.commitCount;

            this.blobCount += result.blobCount;

            this.issueCount += result.issueCount;			
        }

    }

}


 src/com/gitblit/wicket/GitBlitWebApp.java

@@ -26,6 +26,7 @@
import com.gitblit.GitBlit;

import com.gitblit.Keys;

import com.gitblit.wicket.pages.ActivityPage;

import com.gitblit.wicket.pages.BasePage;

import com.gitblit.wicket.pages.BlamePage;

import com.gitblit.wicket.pages.BlobDiffPage;

import com.gitblit.wicket.pages.BlobPage;

@@ -90,7 +91,12 @@
        mount("/commitdiff", CommitDiffPage.class, "r", "h");

        mount("/patch", PatchPage.class, "r", "h", "f");

        mount("/history", HistoryPage.class, "r", "h", "f");

        mount("/search", SearchPage.class);

        if (GitBlit.getBoolean(Keys.lucene.enable, false)) {

            // TODO switch this to LucenePage when it is ready

            mount("/search", SearchPage.class);

        } else {

            mount("/search", SearchPage.class);

        }

        mount("/metrics", MetricsPage.class, "r");

        mount("/blame", BlamePage.class, "r", "h", "f");



@@ -124,6 +130,20 @@
    public Class<? extends Page> getHomePage() {

        return RepositoriesPage.class;

    }

	
    /**

     * Returns the preferred search page class.

     * 
     * @return a Wicket class representing a search page

     */

    public Class<? extends BasePage> getSearchPageClass() {

        if (GitBlit.getBoolean(Keys.lucene.enable, false)) {

            // TODO switch this to LucenePage when it is ready

            return SearchPage.class;//LucenePage.class;

        }

        return SearchPage.class;

    }





    @Override

    public final Session newSession(Request request, Response response) {


 src/com/gitblit/wicket/GitBlitWebApp.properties

@@ -212,4 +212,6 @@
gb.reset = reset

gb.pages = pages

gb.workingCopy = working copy

gb.workingCopyWarning = this repository has a working copy and can not receive pushes
gb.workingCopyWarning = this repository has a working copy and can not receive pushes

gb.query = query

gb.queryHelp = write brief explanation of Lucene syntax here

 src/com/gitblit/wicket/pages/LucenePage.html

@@ -6,17 +6,46 @@
<body onload="document.getElementById('fragment').focus();">

<wicket:extend>

    <div class="pageTitle">

        <h2><wicket:message key="gb.search"></wicket:message><small> lucene</small></h2>

        <h2><wicket:message key="gb.search"></wicket:message></h2>

    </div>

    <form class="form-inline" wicket:id="searchForm">

        <input class="span6" wicket:id="fragment" placeholder="enter search text"></input>

        <button class="btn btn-primary" type="submit" value="Search"><wicket:message key="gb.search"></wicket:message></button>

    <form class="form-inline" style="margin:0px;" wicket:id="searchForm">

        <div class="row-fluid">

            <div class="span3">

                <h3><wicket:message key="gb.repositories"></wicket:message></h3>

                <select wicket:id="repositories" ></select>

            </div>

            <div class="span9">

                <div>

                    <h3><wicket:message key="gb.query"></wicket:message></h3>

                    <input class="span8" wicket:id="fragment" placeholder="enter search text"></input>

                    <button class="btn btn-primary" type="submit" value="Search"><wicket:message key="gb.search"></wicket:message></button>

                </div>

                <div style="margin-top:10px;">

                    <div style="margin-left:0px;" class="span4">

                        <div class="alert alert">

                            <b>type:</b> commit or blob<br/>

                            <b>id:</b> commit id or file path<br/>

                            <b>branch:</b><br/>

                            <b>author:</b><br/>

                            <b>committer:</b><br/>

                            <b>tag:</b> tag<br/>

                        </div>

                    </div>

                    <div class="span5">

                        <wicket:message key="gb.queryHelp"></wicket:message>

                    </div>

                </div>

            </div>

        </div>

    </form>

    <hr/>

    <div class="row-fluid">

    <div class="searchResult" wicket:id="searchResults">

        <div><i wicket:id="type"></i><span class="summary" wicket:id="summary"></span></div>

        <span class="author" wicket:id="author"></span> committed to <span class="repository" wicket:id="repository"></span>:<span class="branch" wicket:id="branch"></span><br/>

        <span class="date" wicket:id="date"></span>

    </div>

    </div>

</wicket:extend>

</body>

</html>

 src/com/gitblit/wicket/pages/LucenePage.java

@@ -20,7 +20,8 @@


import org.apache.wicket.PageParameters;

import org.apache.wicket.markup.html.basic.Label;

import org.apache.wicket.markup.html.form.Form;

import org.apache.wicket.markup.html.form.ListMultipleChoice;

import org.apache.wicket.markup.html.form.StatelessForm;

import org.apache.wicket.markup.html.form.TextField;

import org.apache.wicket.markup.repeater.Item;

import org.apache.wicket.markup.repeater.data.DataView;

@@ -29,6 +30,7 @@
import org.eclipse.jgit.lib.Constants;

import org.eclipse.jgit.lib.Repository;



import com.gitblit.Constants.SearchType;

import com.gitblit.GitBlit;

import com.gitblit.models.SearchResult;

import com.gitblit.utils.LuceneUtils;

@@ -50,6 +52,18 @@


    private void setup(PageParameters params) {

        setupPage("", "");

		
        String repository = null;		
        String value = null;

        com.gitblit.Constants.SearchType searchType = null;



        if (params != null) {

            repository = WicketUtils.getRepositoryName(params);

            value = WicketUtils.getSearchString(params);

            String type = WicketUtils.getSearchType(params);

            searchType = com.gitblit.Constants.SearchType.forName(type);

        }

		
        final List<SearchResult> results = new ArrayList<SearchResult>();

        ListDataProvider<SearchResult> resultsDp = new ListDataProvider<SearchResult>(results);

        final DataView<SearchResult> resultsView = new DataView<SearchResult>("searchResults", resultsDp) {

@@ -86,26 +100,70 @@
            }

        };        

        

        // initial query

        final Model<String> fragment = new Model<String>();

        Form<Void> form = new Form<Void>("searchForm") {

        if (!StringUtils.isEmpty(value)) {

            if (searchType == SearchType.COMMIT) {

                fragment.setObject("type:" + searchType.name().toLowerCase() + " AND \"" + value + "\"");	
            } else {

                fragment.setObject(searchType.name().toLowerCase() + ":\"" + value + "\"");

            }

        }

		
        // selected repositories

        final Model<ArrayList<String>> repositories = new Model<ArrayList<String>>();

        if (!StringUtils.isEmpty(repository)) {			
            ArrayList<String> list = new ArrayList<String>();

            list.add(repository);

            repositories.setObject(list);

        }

		
        // search form

        StatelessForm<Void> form = new StatelessForm<Void>("searchForm") {

			
            private static final long serialVersionUID = 1L;



            @Override

            public void onSubmit() {

                String f = fragment.getObject();

                List<Repository> repositories = new ArrayList<Repository>();

                for (String r : GitBlit.self().getRepositoryList()) {

                    repositories.add(GitBlit.self().getRepository(r));

                if (StringUtils.isEmpty(f)) {

                    error("Query is empty!");

                    return;

                }				
                if (repositories.getObject().size() == 0) {

                    error("Please select one or more repositories!");

                    return;

                }

                List<SearchResult> srs = LuceneUtils.search(f, 100, repositories.toArray(new Repository[repositories.size()]));

                results.clear();

                results.addAll(srs);

                for (Repository r : repositories) {

                    r.close();

                }

                results.addAll(search(repositories, fragment));

                resultsView.setVisible(true);

            }

        };

        ListMultipleChoice<String> selections = new ListMultipleChoice<String>("repositories", repositories, GitBlit.self().getRepositoryList());

        selections.setMaxRows(11);

        form.add(selections);

        form.add(new TextField<String>("fragment", fragment));

        add(form);

        add(resultsView.setVisible(false));

        if (!StringUtils.isEmpty(repository) && !StringUtils.isEmpty(fragment.getObject())) {

            // search is defined by url parameters

            results.clear();

            results.addAll(search(repositories, fragment));

            add(resultsView.setVisible(true));

        } else {

            // no pre-defined search

            add(resultsView.setVisible(false));

        }

    }

	
    private List<SearchResult> search(Model<ArrayList<String>> repositories, Model<String> fragment) {

        List<Repository> repos = new ArrayList<Repository>();

        for (String r : repositories.getObject()) {

            repos.add(GitBlit.self().getRepository(r));

        }

        List<SearchResult> srs = LuceneUtils.search(fragment.getObject(), 100, repos.toArray(new Repository[repos.size()]));

        for (Repository r : repos) {

            r.close();

        }

        return srs;

    }

}


 src/com/gitblit/wicket/pages/RepositoryPage.java

@@ -47,6 +47,7 @@
import com.gitblit.utils.JGitUtils;

import com.gitblit.utils.StringUtils;

import com.gitblit.utils.TicgitUtils;

import com.gitblit.wicket.GitBlitWebApp;

import com.gitblit.wicket.GitBlitWebSession;

import com.gitblit.wicket.PageRegistration;

import com.gitblit.wicket.PageRegistration.OtherPageLink;

@@ -228,6 +229,8 @@


    protected Component createPersonPanel(String wicketId, PersonIdent identity,

            Constants.SearchType searchType) {

        GitBlitWebApp app = (GitBlitWebApp) GitBlitWebSession.get().getApplication();

        final Class<? extends BasePage> searchPageClass = app.getSearchPageClass();

        String name = identity == null ? "" : identity.getName();

        String address = identity == null ? "" : identity.getEmailAddress();

        boolean showEmail = GitBlit.getBoolean(Keys.web.showEmailAddresses, false);

@@ -241,20 +244,20 @@
                }

            }

            Fragment partial = new Fragment(wicketId, "partialPersonIdent", this);

            LinkPanel link = new LinkPanel("personName", "list", value, SearchPage.class,

            LinkPanel link = new LinkPanel("personName", "list", value, searchPageClass,

                    WicketUtils.newSearchParameter(repositoryName, objectId, value, searchType));

            setPersonSearchTooltip(link, value, searchType);

            partial.add(link);

            return partial;

        } else {

            Fragment fullPerson = new Fragment(wicketId, "fullPersonIdent", this);

            LinkPanel nameLink = new LinkPanel("personName", "list", name, SearchPage.class,

            LinkPanel nameLink = new LinkPanel("personName", "list", name, searchPageClass,

                    WicketUtils.newSearchParameter(repositoryName, objectId, name, searchType));

            setPersonSearchTooltip(nameLink, name, searchType);

            fullPerson.add(nameLink);



            LinkPanel addressLink = new LinkPanel("personAddress", "list", "<" + address + ">",

                    SearchPage.class, WicketUtils.newSearchParameter(repositoryName, objectId,

                    searchPageClass, WicketUtils.newSearchParameter(repositoryName, objectId,

                            address, searchType));

            setPersonSearchTooltip(addressLink, address, searchType);

            fullPerson.add(addressLink);

@@ -357,7 +360,8 @@
                    break;

                }

            }

            setResponsePage(SearchPage.class,

            GitBlitWebApp app = (GitBlitWebApp) GitBlitWebSession.get().getApplication();

            setResponsePage(app.getSearchPageClass(),

                    WicketUtils.newSearchParameter(repositoryName, null, searchString, searchType));

        }

    }


 src/com/gitblit/wicket/panels/ActivityPanel.java

@@ -28,7 +28,10 @@
import com.gitblit.models.Activity;

import com.gitblit.models.Activity.RepositoryCommit;

import com.gitblit.utils.StringUtils;

import com.gitblit.wicket.GitBlitWebApp;

import com.gitblit.wicket.GitBlitWebSession;

import com.gitblit.wicket.WicketUtils;

import com.gitblit.wicket.pages.BasePage;

import com.gitblit.wicket.pages.CommitDiffPage;

import com.gitblit.wicket.pages.CommitPage;

import com.gitblit.wicket.pages.LogPage;

@@ -50,6 +53,9 @@
        super(wicketId);



        Collections.sort(recentActivity);

		
        GitBlitWebApp app = (GitBlitWebApp) GitBlitWebSession.get().getApplication();

        final Class<? extends BasePage> searchPageClass = app.getSearchPageClass();



        DataView<Activity> activityView = new DataView<Activity>("activity",

                new ListDataProvider<Activity>(recentActivity)) {

@@ -86,7 +92,7 @@
                        // author search link

                        String author = commit.getAuthorIdent().getName();

                        LinkPanel authorLink = new LinkPanel("author", "list", author,

                                SearchPage.class, WicketUtils.newSearchParameter(commit.repository,

                                searchPageClass, WicketUtils.newSearchParameter(commit.repository,

                                        commit.getName(), author, Constants.SearchType.AUTHOR), true);

                        setPersonSearchTooltip(authorLink, author, Constants.SearchType.AUTHOR);

                        fragment.add(authorLink);


 src/com/gitblit/wicket/panels/BranchesPanel.java

@@ -35,7 +35,10 @@
import com.gitblit.models.RepositoryModel;

import com.gitblit.utils.JGitUtils;

import com.gitblit.utils.StringUtils;

import com.gitblit.wicket.GitBlitWebApp;

import com.gitblit.wicket.GitBlitWebSession;

import com.gitblit.wicket.WicketUtils;

import com.gitblit.wicket.pages.BasePage;

import com.gitblit.wicket.pages.BranchesPage;

import com.gitblit.wicket.pages.CommitPage;

import com.gitblit.wicket.pages.LogPage;

@@ -75,6 +78,9 @@
            add(new Label("branches", new StringResourceModel("gb.branches", this, null)));

        }



        GitBlitWebApp app = (GitBlitWebApp) GitBlitWebSession.get().getApplication();

        final Class<? extends BasePage> searchPageClass = app.getSearchPageClass();



        ListDataProvider<RefModel> branchesDp = new ListDataProvider<RefModel>(branches);

        DataView<RefModel> branchesView = new DataView<RefModel>("branch", branchesDp) {

            private static final long serialVersionUID = 1L;

@@ -91,7 +97,7 @@


                String author = entry.getAuthorIdent().getName();

                LinkPanel authorLink = new LinkPanel("branchAuthor", "list", author,

                        SearchPage.class, WicketUtils.newSearchParameter(model.name,

                        searchPageClass, WicketUtils.newSearchParameter(model.name,

                                entry.getName(), author, Constants.SearchType.AUTHOR));

                setPersonSearchTooltip(authorLink, author, Constants.SearchType.AUTHOR);

                item.add(authorLink);


 src/com/gitblit/wicket/panels/HistoryPanel.java

@@ -38,7 +38,10 @@
import com.gitblit.models.RefModel;

import com.gitblit.utils.JGitUtils;

import com.gitblit.utils.StringUtils;

import com.gitblit.wicket.GitBlitWebApp;

import com.gitblit.wicket.GitBlitWebSession;

import com.gitblit.wicket.WicketUtils;

import com.gitblit.wicket.pages.BasePage;

import com.gitblit.wicket.pages.BlobDiffPage;

import com.gitblit.wicket.pages.BlobPage;

import com.gitblit.wicket.pages.CommitDiffPage;

@@ -94,6 +97,9 @@
        // breadcrumbs

        add(new PathBreadcrumbsPanel("breadcrumbs", repositoryName, path, objectId));



        GitBlitWebApp app = (GitBlitWebApp) GitBlitWebSession.get().getApplication();

        final Class<? extends BasePage> searchPageClass = app.getSearchPageClass();



        ListDataProvider<RevCommit> dp = new ListDataProvider<RevCommit>(commits);

        DataView<RevCommit> logView = new DataView<RevCommit>("commit", dp) {

            private static final long serialVersionUID = 1L;

@@ -108,7 +114,7 @@
                // author search link

                String author = entry.getAuthorIdent().getName();

                LinkPanel authorLink = new LinkPanel("commitAuthor", "list", author,

                        SearchPage.class, WicketUtils.newSearchParameter(repositoryName, objectId,

                        searchPageClass, WicketUtils.newSearchParameter(repositoryName, objectId,

                                author, Constants.SearchType.AUTHOR));

                setPersonSearchTooltip(authorLink, author, Constants.SearchType.AUTHOR);

                item.add(authorLink);


 src/com/gitblit/wicket/panels/LogPanel.java

@@ -35,7 +35,10 @@
import com.gitblit.models.RefModel;

import com.gitblit.utils.JGitUtils;

import com.gitblit.utils.StringUtils;

import com.gitblit.wicket.GitBlitWebApp;

import com.gitblit.wicket.GitBlitWebSession;

import com.gitblit.wicket.WicketUtils;

import com.gitblit.wicket.pages.BasePage;

import com.gitblit.wicket.pages.CommitDiffPage;

import com.gitblit.wicket.pages.CommitPage;

import com.gitblit.wicket.pages.LogPage;

@@ -82,6 +85,9 @@
                    WicketUtils.newRepositoryParameter(repositoryName)));

        }



        GitBlitWebApp app = (GitBlitWebApp) GitBlitWebSession.get().getApplication();

        final Class<? extends BasePage> searchPageClass = app.getSearchPageClass();



        ListDataProvider<RevCommit> dp = new ListDataProvider<RevCommit>(commits);

        DataView<RevCommit> logView = new DataView<RevCommit>("commit", dp) {

            private static final long serialVersionUID = 1L;

@@ -96,7 +102,7 @@
                // author search link

                String author = entry.getAuthorIdent().getName();

                LinkPanel authorLink = new LinkPanel("commitAuthor", "list", author,

                        SearchPage.class, WicketUtils.newSearchParameter(repositoryName, objectId,

                        searchPageClass, WicketUtils.newSearchParameter(repositoryName, objectId,

                                author, Constants.SearchType.AUTHOR));

                setPersonSearchTooltip(authorLink, author, Constants.SearchType.AUTHOR);

                item.add(authorLink);


 tests/com/gitblit/tests/IssuesTest.java

@@ -36,6 +36,7 @@
import com.gitblit.utils.IssueUtils;

import com.gitblit.utils.IssueUtils.IssueFilter;

import com.gitblit.utils.LuceneUtils;

import com.gitblit.utils.StringUtils;



/**

 * Tests the mechanics of distributed issue management on the gb-issues branch.

@@ -48,6 +49,8 @@
    @Test

    public void testLifecycle() throws Exception {

        Repository repository = GitBlitSuite.getIssuesTestRepository();

        String name = StringUtils.getRelativePath(GitBlitSuite.REPOSITORIES.getAbsolutePath(),

                repository.getDirectory().getAbsolutePath());

        

        // create and insert an issue

        Change c1 = newChange("testCreation() " + Long.toHexString(System.currentTimeMillis()));

@@ -128,7 +131,7 @@
        // build a new Lucene index

        LuceneUtils.deleteIndex(repository);

        for (IssueModel anIssue : allIssues) {

            LuceneUtils.index(repository, anIssue);

            LuceneUtils.index(name, repository, anIssue);

        }

        List<SearchResult> hits = LuceneUtils.search("working", 10, repository);

        assertTrue(hits.size() > 0);

@@ -139,7 +142,7 @@
        change.comment("this is a test of reindexing an issue");

        IssueUtils.updateIssue(repository, issue.id, change);

        issue = IssueUtils.getIssue(repository, issue.id);

        LuceneUtils.index(repository, issue);

        LuceneUtils.index(name, repository, issue);



        // delete all issues

        for (IssueModel anIssue : allIssues) {


 tests/com/gitblit/tests/LuceneUtilsTest.java

@@ -24,6 +24,7 @@


import com.gitblit.models.SearchResult;

import com.gitblit.utils.LuceneUtils;

import com.gitblit.utils.StringUtils;



/**

 * Tests Lucene indexing and querying.

@@ -37,19 +38,25 @@
    public void testFullIndex() throws Exception {

        // reindex helloworld

        Repository repository = GitBlitSuite.getHelloworldRepository();

        LuceneUtils.reindex(repository);

        String name = StringUtils.getRelativePath(GitBlitSuite.REPOSITORIES.getAbsolutePath(),

                repository.getDirectory().getAbsolutePath());

        LuceneUtils.reindex(name, repository, false);

        repository.close();



        // reindex theoretical physics

        repository = GitBlitSuite.getTheoreticalPhysicsRepository();

        LuceneUtils.reindex(repository);

        name = StringUtils.getRelativePath(GitBlitSuite.REPOSITORIES.getAbsolutePath(),

                repository.getDirectory().getAbsolutePath());

        LuceneUtils.reindex(name, repository, false);

        repository.close();



		
        // reindex JGit

        repository = GitBlitSuite.getJGitRepository();

        LuceneUtils.reindex(repository);

        name = StringUtils.getRelativePath(GitBlitSuite.REPOSITORIES.getAbsolutePath(),

                repository.getDirectory().getAbsolutePath());

        LuceneUtils.reindex(name, repository, false);

        repository.close();



		
        LuceneUtils.close();

    }