From f1d2ada42c5e3640656d805155e1bcadb95fd126 Mon Sep 17 00:00:00 2001
From: James Moger <james.moger@gitblit.com>
Date: Wed, 21 Mar 2012 21:02:46 -0400
Subject: [PATCH] Externalized the Lucene ignore extensions
---
src/com/gitblit/LuceneExecutor.java | 122 ++++++++++++++++++++++++----------------
1 files changed, 74 insertions(+), 48 deletions(-)
diff --git a/src/com/gitblit/LuceneExecutor.java b/src/com/gitblit/LuceneExecutor.java
index 29a8569..b6df254 100644
--- a/src/com/gitblit/LuceneExecutor.java
+++ b/src/com/gitblit/LuceneExecutor.java
@@ -25,7 +25,6 @@
import java.text.MessageFormat;
import java.text.ParseException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -138,10 +137,9 @@
private final Map<String, IndexSearcher> searchers = new ConcurrentHashMap<String, IndexSearcher>();
private final Map<String, IndexWriter> writers = new ConcurrentHashMap<String, IndexWriter>();
- private final Set<String> excludedExtensions = new TreeSet<String>(Arrays.asList("7z", "arc",
- "arj", "bin", "bmp", "dll", "doc", "docx", "exe", "gif", "gz", "jar", "jpg", "lib",
- "lzh", "odg", "pdf", "ppt", "png", "so", "swf", "xcf", "xls", "xlsx", "zip"));
-
+ private final String luceneIgnoreExtensions = "7z arc arj bin bmp dll doc docx exe gif gz jar jpg lib lzh odg odf odt pdf ppt png so swf xcf xls xlsx zip";
+ private Set<String> excludedExtensions;
+
public LuceneExecutor(IStoredSettings settings, File repositoriesFolder) {
this.storedSettings = settings;
this.repositoriesFolder = repositoriesFolder;
@@ -154,6 +152,10 @@
*/
@Override
public void run() {
+ // reload the excluded extensions
+ String exts = storedSettings.getString(Keys.web.luceneIgnoreExtensions, luceneIgnoreExtensions);
+ excludedExtensions = new TreeSet<String>(StringUtils.getStringsFromValue(exts));
+
for (String repositoryName: GitBlit.self().getRepositoryList()) {
RepositoryModel model = GitBlit.self().getRepositoryModel(repositoryName);
if (model.hasCommits && !ArrayUtils.isEmpty(model.indexedBranches)) {
@@ -622,7 +624,7 @@
IndexWriter writer = getIndexWriter(repositoryName);
for (PathChangeModel path : changedPaths) {
// delete the indexed blob
- deleteBlob(repositoryName, branch, path.path);
+ deleteBlob(repositoryName, branch, path.name);
// re-index the blob
if (!ChangeType.DELETE.equals(path.changeType)) {
@@ -655,8 +657,17 @@
}
}
writer.commit();
-
- Document doc = createDocument(commit, null);
+
+ // get any annotated commit tags
+ List<String> commitTags = new ArrayList<String>();
+ for (RefModel ref : JGitUtils.getTags(repository, true, -1)) {
+ if (ref.isAnnotatedTag() && ref.getReferencedObjectId().equals(commit.getId())) {
+ commitTags.add(ref.displayName);
+ }
+ }
+
+ // create and write the Lucene document
+ Document doc = createDocument(commit, commitTags);
doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.ANALYZED));
result.commitCount++;
result.success = index(repositoryName, doc);
@@ -1105,17 +1116,14 @@
*/
private String getHighlightedFragment(Analyzer analyzer, Query query,
String content, SearchResult result) throws IOException, InvalidTokenOffsetsException {
- content = content == null ? "":StringUtils.escapeForHtml(content, false);
-
+ if (content == null) {
+ content = "";
+ }
+
+ int fragmentLength = SearchObjectType.commit == result.type ? 512 : 150;
+
QueryScorer scorer = new QueryScorer(query, "content");
- Fragmenter fragmenter;
-
- // TODO improve the fragmenter - hopefully on line breaks
- if (SearchObjectType.commit == result.type) {
- fragmenter = new SimpleSpanFragmenter(scorer, 1024);
- } else {
- fragmenter = new SimpleSpanFragmenter(scorer, 150);
- }
+ Fragmenter fragmenter = new SimpleSpanFragmenter(scorer, fragmentLength);
// use an artificial delimiter for the token
String termTag = "<!--[";
@@ -1123,22 +1131,64 @@
SimpleHTMLFormatter formatter = new SimpleHTMLFormatter(termTag, termTagEnd);
Highlighter highlighter = new Highlighter(formatter, scorer);
highlighter.setTextFragmenter(fragmenter);
-
+
String [] fragments = highlighter.getBestFragments(analyzer, "content", content, 3);
if (ArrayUtils.isEmpty(fragments)) {
if (SearchObjectType.blob == result.type) {
return "";
}
- return "<pre class=\"text\">" + content + "</pre>";
+ // clip commit message
+ String fragment = content;
+ if (fragment.length() > fragmentLength) {
+ fragment = fragment.substring(0, fragmentLength) + "...";
+ }
+ return "<pre class=\"text\">" + StringUtils.escapeForHtml(fragment, true) + "</pre>";
}
+
+ int contentPos = 0;
StringBuilder sb = new StringBuilder();
for (int i = 0, len = fragments.length; i < len; i++) {
String fragment = fragments[i];
-
+ String tag = "<pre class=\"text\">";
+
// resurrect the raw fragment from removing the artificial delimiters
- String raw = fragment.replace(termTag, "").replace(termTagEnd, "");
- sb.append(getPreTag(result, raw, content));
+ String raw = fragment.replace(termTag, "").replace(termTagEnd, "");
+
+ // determine position of the raw fragment in the content
+ int pos = content.indexOf(raw, contentPos);
+
+ // restore complete first line of fragment
+ int c = pos;
+ while (c > 0) {
+ c--;
+ if (content.charAt(c) == '\n') {
+ break;
+ }
+ }
+ if (c > 0) {
+ // inject leading chunk of first fragment line
+ fragment = content.substring(c + 1, pos) + fragment;
+ }
+
+ if (SearchObjectType.blob == result.type) {
+ // count lines as offset into the content for this fragment
+ int line = StringUtils.countLines(content.substring(0, pos));
+
+ // create fragment tag with line number and language
+ String lang = "";
+ String ext = StringUtils.getFileExtension(result.path).toLowerCase();
+ if (!StringUtils.isEmpty(ext)) {
+ // maintain leading space!
+ lang = " lang-" + ext;
+ }
+ tag = MessageFormat.format("<pre class=\"prettyprint linenums:{0,number,0}{1}\">", line, lang);
+
+ // update offset into content
+ contentPos = pos + raw.length() + 1;
+ }
+ sb.append(tag);
+
// replace the artificial delimiter with html tags
String html = fragment.replace(termTag, "<span class=\"highlight\">").replace(termTagEnd, "</span>");
sb.append(html);
@@ -1148,31 +1198,7 @@
}
}
return sb.toString();
- }
-
- /**
- * Returns the appropriate tag for a fragment. Commit messages are visually
- * differentiated from blob fragments.
- *
- * @param result
- * @param fragment
- * @param content
- * @return an html tag appropriate for the fragment
- */
- private String getPreTag(SearchResult result, String fragment, String content) {
- String pre = "<pre class=\"text\">";
- if (SearchObjectType.blob == result.type) {
- int line = StringUtils.countLines(content.substring(0, content.indexOf(fragment)));
- int lastDot = result.path.lastIndexOf('.');
- if (lastDot > -1) {
- String ext = result.path.substring(lastDot + 1).toLowerCase();
- pre = MessageFormat.format("<pre class=\"prettyprint linenums:{0,number,0} lang-{1}\">", line, ext);
- } else {
- pre = MessageFormat.format("<pre class=\"prettyprint linenums:{0,number,0}\">", line);
- }
- }
- return pre;
- }
+ }
/**
* Simple class to track the results of an index update.
--
Gitblit v1.9.1