Merge pull request #1073 from gitblit/1062-DocEditorUpdates

Paul Martin

2016-04-30 a502d96a860456ec5e8c96761db70f7cabb74751

 src/main/java/com/gitblit/service/LuceneService.java

@@ -21,7 +21,6 @@
import java.io.File;

import java.io.IOException;

import java.io.InputStream;

import java.lang.reflect.Method;

import java.text.MessageFormat;

import java.text.ParseException;

import java.util.ArrayList;

@@ -42,15 +41,16 @@
import org.apache.lucene.document.DateTools.Resolution;

import org.apache.lucene.document.Document;

import org.apache.lucene.document.Field;

import org.apache.lucene.document.Field.Index;

import org.apache.lucene.document.Field.Store;

import org.apache.lucene.document.StringField;

import org.apache.lucene.document.TextField;

import org.apache.lucene.index.DirectoryReader;

import org.apache.lucene.index.IndexReader;

import org.apache.lucene.index.IndexWriter;

import org.apache.lucene.index.IndexWriterConfig;

import org.apache.lucene.index.IndexWriterConfig.OpenMode;

import org.apache.lucene.index.MultiReader;

import org.apache.lucene.index.Term;

import org.apache.lucene.queryParser.QueryParser;

import org.apache.lucene.queryparser.classic.QueryParser;

import org.apache.lucene.search.BooleanClause.Occur;

import org.apache.lucene.search.BooleanQuery;

import org.apache.lucene.search.IndexSearcher;

@@ -105,7 +105,7 @@
public class LuceneService implements Runnable {





   private static final int INDEX_VERSION = 5;

   private static final int INDEX_VERSION = 6;



   private static final String FIELD_OBJECT_TYPE = "type";

   private static final String FIELD_PATH = "path";

@@ -125,7 +125,7 @@
   private static final String CONF_ALIAS = "aliases";

   private static final String CONF_BRANCH = "branches";



   private static final Version LUCENE_VERSION = Version.LUCENE_35;

   private static final Version LUCENE_VERSION = Version.LUCENE_4_10_0;



   private final Logger logger = LoggerFactory.getLogger(LuceneService.class);



@@ -194,7 +194,7 @@
    * Synchronously indexes a repository. This may build a complete index of a

    * repository or it may update an existing index.

    *

    * @param name

    * @param displayName

    *            the name of the repository

    * @param repository

    *            the repository object

@@ -437,7 +437,7 @@
               // skip non-annotated tags

               continue;

            }

            if (!tags.containsKey(tag.getObjectId())) {

            if (!tags.containsKey(tag.getReferencedObjectId().getName())) {

               tags.put(tag.getReferencedObjectId().getName(), new ArrayList<String>());

            }

            tags.get(tag.getReferencedObjectId().getName()).add(tag.displayName);

@@ -476,8 +476,8 @@
                  && branch.equals(defaultBranch)) {

               // indexing "default" branch

               indexBranch = true;

            } else if (branch.getName().startsWith(com.gitblit.Constants.R_GITBLIT)) {

               // skip Gitblit internal branches

            } else if (branch.getName().startsWith(com.gitblit.Constants.R_META)) {

               // skip internal meta branches

               indexBranch = false;

            } else {

               // normal explicit branch check

@@ -552,13 +552,13 @@
                        Resolution.MINUTE);



                  Document doc = new Document();

                  doc.add(new Field(FIELD_OBJECT_TYPE, SearchObjectType.blob.name(), Store.YES, Index.NOT_ANALYZED_NO_NORMS));

                  doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED));

                  doc.add(new Field(FIELD_COMMIT, commit.getName(), Store.YES, Index.ANALYZED));

                  doc.add(new Field(FIELD_PATH, path, Store.YES, Index.ANALYZED));

                  doc.add(new Field(FIELD_DATE, blobDate, Store.YES, Index.NO));

                  doc.add(new Field(FIELD_AUTHOR, blobAuthor, Store.YES, Index.ANALYZED));

                  doc.add(new Field(FIELD_COMMITTER, blobCommitter, Store.YES, Index.ANALYZED));

                  doc.add(new Field(FIELD_OBJECT_TYPE, SearchObjectType.blob.name(), StringField.TYPE_STORED));

                  doc.add(new Field(FIELD_BRANCH, branchName, TextField.TYPE_STORED));

                  doc.add(new Field(FIELD_COMMIT, commit.getName(), TextField.TYPE_STORED));

                  doc.add(new Field(FIELD_PATH, path, TextField.TYPE_STORED));

                  doc.add(new Field(FIELD_DATE, blobDate, StringField.TYPE_STORED));

                  doc.add(new Field(FIELD_AUTHOR, blobAuthor, TextField.TYPE_STORED));

                  doc.add(new Field(FIELD_COMMITTER, blobCommitter, TextField.TYPE_STORED));



                  // determine extension to compare to the extension

                  // blacklist

@@ -579,7 +579,7 @@
                     in.close();

                     byte[] content = os.toByteArray();

                     String str = StringUtils.decodeString(content, encodings);

                     doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED));

                     doc.add(new Field(FIELD_CONTENT, str, TextField.TYPE_STORED));

                     os.reset();

                  }



@@ -593,7 +593,7 @@
            // index the tip commit object

            if (indexedCommits.add(tipId)) {

               Document doc = createDocument(tip, tags.get(tipId));

               doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED));

               doc.add(new Field(FIELD_BRANCH, branchName, TextField.TYPE_STORED));

               writer.addDocument(doc);

               result.commitCount += 1;

               result.branchCount += 1;

@@ -607,7 +607,7 @@
               String hash = rev.getId().getName();

               if (indexedCommits.add(hash)) {

                  Document doc = createDocument(rev, tags.get(hash));

                  doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED));

                  doc.add(new Field(FIELD_BRANCH, branchName, TextField.TYPE_STORED));

                  writer.addDocument(doc);

                  result.commitCount += 1;

               }

@@ -615,7 +615,7 @@
         }



         // finished

         reader.release();

         reader.close();



         // commit all changes and reset the searcher

         config.setInt(CONF_INDEX, null, CONF_VERSION, INDEX_VERSION);

@@ -660,14 +660,13 @@
            if (!ChangeType.DELETE.equals(path.changeType)) {

               result.blobCount++;

               Document doc = new Document();

               doc.add(new Field(FIELD_OBJECT_TYPE, SearchObjectType.blob.name(), Store.YES,

                     Index.NOT_ANALYZED));

               doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.ANALYZED));

               doc.add(new Field(FIELD_COMMIT, commit.getName(), Store.YES, Index.ANALYZED));

               doc.add(new Field(FIELD_PATH, path.path, Store.YES, Index.ANALYZED));

               doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO));

               doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), Store.YES, Index.ANALYZED));

               doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), Store.YES, Index.ANALYZED));

               doc.add(new Field(FIELD_OBJECT_TYPE, SearchObjectType.blob.name(), StringField.TYPE_STORED));

               doc.add(new Field(FIELD_BRANCH, branch, TextField.TYPE_STORED));

               doc.add(new Field(FIELD_COMMIT, commit.getName(), TextField.TYPE_STORED));

               doc.add(new Field(FIELD_PATH, path.path, TextField.TYPE_STORED));

               doc.add(new Field(FIELD_DATE, revDate, StringField.TYPE_STORED));

               doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), TextField.TYPE_STORED));

               doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), TextField.TYPE_STORED));



               // determine extension to compare to the extension

               // blacklist

@@ -682,7 +681,7 @@
                  String str = JGitUtils.getStringContent(repository, commit.getTree(),

                        path.path, encodings);

                  if (str != null) {

                     doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED));

                     doc.add(new Field(FIELD_CONTENT, str, TextField.TYPE_STORED));

                     writer.addDocument(doc);

                  }

               }

@@ -700,7 +699,7 @@


         // create and write the Lucene document

         Document doc = createDocument(commit, commitTags);

         doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.ANALYZED));

         doc.add(new Field(FIELD_BRANCH, branch, TextField.TYPE_STORED));

         result.commitCount++;

         result.success = index(repositoryName, doc);

      } catch (Exception e) {

@@ -761,7 +760,7 @@
               // skip non-annotated tags

               continue;

            }

            if (!tags.containsKey(tag.getObjectId())) {

            if (!tags.containsKey(tag.getObjectId().getName())) {

               tags.put(tag.getReferencedObjectId().getName(), new ArrayList<String>());

            }

            tags.get(tag.getReferencedObjectId().getName()).add(tag.displayName);

@@ -808,8 +807,8 @@
                  && branch.equals(defaultBranch)) {

               // indexing "default" branch

               indexBranch = true;

            } else if (branch.getName().startsWith(com.gitblit.Constants.R_GITBLIT)) {

               // ignore internal Gitblit branches

            } else if (branch.getName().startsWith(com.gitblit.Constants.R_META)) {

               // ignore internal meta branches

               indexBranch = false;

            } else {

               // normal explicit branch check

@@ -880,17 +879,16 @@
    */

   private Document createDocument(RevCommit commit, List<String> tags) {

      Document doc = new Document();

      doc.add(new Field(FIELD_OBJECT_TYPE, SearchObjectType.commit.name(), Store.YES,

            Index.NOT_ANALYZED));

      doc.add(new Field(FIELD_COMMIT, commit.getName(), Store.YES, Index.ANALYZED));

      doc.add(new Field(FIELD_OBJECT_TYPE, SearchObjectType.commit.name(), StringField.TYPE_STORED));

      doc.add(new Field(FIELD_COMMIT, commit.getName(), TextField.TYPE_STORED));

      doc.add(new Field(FIELD_DATE, DateTools.timeToString(commit.getCommitTime() * 1000L,

            Resolution.MINUTE), Store.YES, Index.NO));

      doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), Store.YES, Index.ANALYZED));

      doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), Store.YES, Index.ANALYZED));

      doc.add(new Field(FIELD_SUMMARY, commit.getShortMessage(), Store.YES, Index.ANALYZED));

      doc.add(new Field(FIELD_CONTENT, commit.getFullMessage(), Store.YES, Index.ANALYZED));

            Resolution.MINUTE), StringField.TYPE_STORED));

      doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), TextField.TYPE_STORED));

      doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), TextField.TYPE_STORED));

      doc.add(new Field(FIELD_SUMMARY, commit.getShortMessage(), TextField.TYPE_STORED));

      doc.add(new Field(FIELD_CONTENT, commit.getFullMessage(), TextField.TYPE_STORED));

      if (!ArrayUtils.isEmpty(tags)) {

         doc.add(new Field(FIELD_TAG, StringUtils.flattenStrings(tags), Store.YES, Index.ANALYZED));

         doc.add(new Field(FIELD_TAG, StringUtils.flattenStrings(tags), TextField.TYPE_STORED));

      }

      return doc;

   }

@@ -952,7 +950,7 @@
      IndexSearcher searcher = searchers.get(repository);

      if (searcher == null) {

         IndexWriter writer = getIndexWriter(repository);

         searcher = new IndexSearcher(IndexReader.open(writer, true));

         searcher = new IndexSearcher(DirectoryReader.open(writer, true));

         searchers.put(repository, searcher);

      }

      return searcher;

@@ -1106,6 +1104,7 @@
         content = "";

      }



      int tabLength = storedSettings.getInteger(Keys.web.tabLength, 4);

      int fragmentLength = SearchObjectType.commit == result.type ? 512 : 150;



      QueryScorer scorer = new QueryScorer(query, "content");

@@ -1128,7 +1127,7 @@
         if (fragment.length() > fragmentLength) {

            fragment = fragment.substring(0, fragmentLength) + "...";

         }

         return "<pre class=\"text\">" + StringUtils.escapeForHtml(fragment, true) + "</pre>";

         return "<pre class=\"text\">" + StringUtils.escapeForHtml(fragment, true, tabLength) + "</pre>";

      }



      // make sure we have unique fragments

@@ -1226,25 +1225,14 @@
    */

   private class MultiSourceReader extends MultiReader {



      final Method method;



      MultiSourceReader(IndexReader[] subReaders) {

         super(subReaders);

         Method m = null;

         try {

            m = MultiReader.class.getDeclaredMethod("readerIndex", int.class);

            m.setAccessible(true);

         } catch (Exception e) {

            logger.error("Error getting readerIndex method", e);

         }

         method = m;

      MultiSourceReader(IndexReader [] readers) {

         super(readers, false);

      }



      int getSourceIndex(int docId) {

         int index = -1;

         try {

            Object o = method.invoke(this, docId);

            index = (Integer) o;

            index = super.readerIndex(docId);

         } catch (Exception e) {

            logger.error("Error getting source index", e);

         }