Properly index blobs on a branch

James Moger

2012-02-19 fb15c2c3a50aa4fe33b6bdfc9f16d759336caa6c

Properly index blobs on a branch

 src/com/gitblit/models/SearchResult.java

@@ -27,6 +27,8 @@
    public String summary;

    

    public String repository;

	
    public String branch;



    public String id;



@@ -39,6 +41,6 @@


    @Override

    public String toString() {

        return  score + " : " + type.name() + " : " + repository + " : " + id;

        return  score + " : " + type.name() + " : " + repository + " : " + id + " (" + branch + ")";

    }

}

 src/com/gitblit/utils/LuceneUtils.java

@@ -38,6 +38,7 @@
import org.apache.lucene.store.Directory;

import org.apache.lucene.store.FSDirectory;

import org.apache.lucene.util.Version;

import org.eclipse.jgit.diff.DiffEntry.ChangeType;

import org.eclipse.jgit.lib.Constants;

import org.eclipse.jgit.lib.FileMode;

import org.eclipse.jgit.lib.ObjectId;

@@ -50,6 +51,7 @@


import com.gitblit.models.IssueModel;

import com.gitblit.models.IssueModel.Attachment;

import com.gitblit.models.PathModel.PathChangeModel;

import com.gitblit.models.RefModel;

import com.gitblit.models.SearchResult;



@@ -82,6 +84,7 @@


    private static final String FIELD_OBJECT_TYPE = "type";

    private static final String FIELD_OBJECT_ID = "id";

    private static final String FIELD_BRANCH = "branch";

    private static final String FIELD_REPOSITORY = "repository";

    private static final String FIELD_SUMMARY = "summary";

    private static final String FIELD_CONTENT = "content";

@@ -163,6 +166,7 @@
                if (excludedBranches.contains(branch.getName())) {

                    continue;

                }

                String branchName = branch.getName();

                RevWalk revWalk = new RevWalk(repository);

                RevCommit rev = revWalk.parseCommit(branch.getObjectId());



@@ -179,6 +183,8 @@
                    doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES,

                            Index.NOT_ANALYZED_NO_NORMS));

                    doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,

                            Index.NOT_ANALYZED));

                    doc.add(new Field(FIELD_BRANCH, branchName, Store.YES,

                            Index.NOT_ANALYZED));

                    doc.add(new Field(FIELD_OBJECT_ID, treeWalk.getPathString(), Store.YES,

                            Index.NOT_ANALYZED));

@@ -227,6 +233,8 @@
                    Document doc = createDocument(rev, tags.get(head));

                    doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,

                            Index.NOT_ANALYZED));

                    doc.add(new Field(FIELD_BRANCH, branchName, Store.YES,

                            Index.NOT_ANALYZED));

                    writer.addDocument(doc);

                }



@@ -237,6 +245,8 @@
                    if (indexedCommits.add(hash)) {

                        Document doc = createDocument(rev, tags.get(hash));

                        doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,

                                Index.NOT_ANALYZED));

                        doc.add(new Field(FIELD_BRANCH, branchName, Store.YES,

                                Index.NOT_ANALYZED));

                        writer.addDocument(doc);

                    }

@@ -272,11 +282,69 @@
     * repository.

     * 

     * @param repository

     * @param branch

     *            the fully qualified branch name (e.g. refs/heads/master)

     * @param commit

     * @return true, if successful

     */

    public static boolean index(Repository repository, RevCommit commit) {

        try {

    public static boolean index(Repository repository, String branch, RevCommit commit) {

        try {			
            if (excludedBranches.contains(branch)) {

                if (IssueUtils.GB_ISSUES.equals(branch)) {

                    // index an issue

                    String issueId = commit.getShortMessage().substring(2).trim();

                    IssueModel issue = IssueUtils.getIssue(repository, issueId);

                    return index(repository, issue, true);

                }

                return false;

            }

            List<PathChangeModel> changedPaths = JGitUtils.getFilesInCommit(repository, commit);

            String repositoryName = getName(repository);

            String revDate = DateTools.timeToString(commit.getCommitTime() * 1000L,

                    Resolution.MINUTE);

            IndexWriter writer = getIndexWriter(repository, false);

            for (PathChangeModel path : changedPaths) {

                // delete the indexed blob

                writer.deleteDocuments(new Term(FIELD_OBJECT_TYPE, ObjectType.blob.name()),

                        new Term(FIELD_BRANCH, branch),

                        new Term(FIELD_OBJECT_ID, path.path));

				
                // re-index the blob

                if (!ChangeType.DELETE.equals(path.changeType)) {

                    Document doc = new Document();

                    doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES,

                            Index.NOT_ANALYZED_NO_NORMS));

                    doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,

                            Index.NOT_ANALYZED));

                    doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.NOT_ANALYZED));

                    doc.add(new Field(FIELD_OBJECT_ID, path.path, Store.YES,

                            Index.NOT_ANALYZED));

                    doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO));

                    doc.add(new Field(FIELD_AUTHOR, commit.getAuthorIdent().getName(), Store.YES,

                            Index.NOT_ANALYZED_NO_NORMS));

                    doc.add(new Field(FIELD_COMMITTER, commit.getCommitterIdent().getName(),

                            Store.YES, Index.NOT_ANALYZED_NO_NORMS));

                    doc.add(new Field(FIELD_LABEL, branch, Store.YES, Index.ANALYZED));



                    // determine extension to compare to the extension

                    // blacklist

                    String ext = null;

                    String name = path.name.toLowerCase();

                    if (name.indexOf('.') > -1) {

                        ext = name.substring(name.lastIndexOf('.') + 1);

                    }



                    if (StringUtils.isEmpty(ext) || !excludedExtensions.contains(ext)) {

                        // read the blob content

                        String str = JGitUtils.getStringContent(repository, 
                                commit.getTree(), path.path);

                        doc.add(new Field(FIELD_CONTENT, str, Store.NO, Index.ANALYZED));

                        writer.addDocument(doc);

                    }

                }

            }

            writer.commit();

			
            Document doc = createDocument(commit, null);

            return index(repository, doc);

        } catch (Exception e) {

@@ -324,6 +392,7 @@
        doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.issue.name(), Store.YES,

                Field.Index.NOT_ANALYZED_NO_NORMS));

        doc.add(new Field(FIELD_OBJECT_ID, issue.id, Store.YES, Index.NOT_ANALYZED));

        doc.add(new Field(FIELD_BRANCH, IssueUtils.GB_ISSUES, Store.YES, Index.NOT_ANALYZED));

        doc.add(new Field(FIELD_DATE, DateTools.dateToString(issue.created, Resolution.MINUTE),

                Store.YES, Field.Index.NO));

        doc.add(new Field(FIELD_AUTHOR, issue.reporter, Store.YES, Index.NOT_ANALYZED_NO_NORMS));

@@ -399,6 +468,7 @@
        result.committer = doc.get(FIELD_COMMITTER);

        result.type = ObjectType.fromName(doc.get(FIELD_OBJECT_TYPE));

        result.repository = doc.get(FIELD_REPOSITORY);

        result.branch = doc.get(FIELD_BRANCH);

        result.id = doc.get(FIELD_OBJECT_ID);

        if (doc.get(FIELD_LABEL) != null) {

            result.labels = StringUtils.getStringsFromValue(doc.get(FIELD_LABEL));