From 40538c57dd574d831d044cda50a8999941dc0a24 Mon Sep 17 00:00:00 2001
From: James Moger <james.moger@gitblit.com>
Date: Sat, 25 Feb 2012 08:18:22 -0500
Subject: [PATCH] Use proper timezone to generate all metrics
---
src/com/gitblit/utils/LuceneUtils.java | 161 ++++++++++++++++++++++++++++++++++++++++++++++++-----
1 files changed, 145 insertions(+), 16 deletions(-)
diff --git a/src/com/gitblit/utils/LuceneUtils.java b/src/com/gitblit/utils/LuceneUtils.java
index 4ca72f0..738382a 100644
--- a/src/com/gitblit/utils/LuceneUtils.java
+++ b/src/com/gitblit/utils/LuceneUtils.java
@@ -8,7 +8,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -26,6 +26,7 @@
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanClause.Occur;
@@ -37,6 +38,7 @@
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
+import org.eclipse.jgit.diff.DiffEntry.ChangeType;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.ObjectId;
@@ -49,6 +51,7 @@
import com.gitblit.models.IssueModel;
import com.gitblit.models.IssueModel.Attachment;
+import com.gitblit.models.PathModel.PathChangeModel;
import com.gitblit.models.RefModel;
import com.gitblit.models.SearchResult;
@@ -81,6 +84,7 @@
private static final String FIELD_OBJECT_TYPE = "type";
private static final String FIELD_OBJECT_ID = "id";
+ private static final String FIELD_BRANCH = "branch";
private static final String FIELD_REPOSITORY = "repository";
private static final String FIELD_SUMMARY = "summary";
private static final String FIELD_CONTENT = "content";
@@ -90,13 +94,32 @@
private static final String FIELD_LABEL = "label";
private static final String FIELD_ATTACHMENT = "attachment";
- private static Set<String> excludes = new TreeSet<String>(Arrays.asList("7z", "arc", "arj",
- "bin", "bmp", "dll", "doc", "docx", "exe", "gif", "gz", "jar", "jpg", "lib", "lzh",
- "odg", "pdf", "ppt", "png", "so", "swf", "xcf", "xls", "xlsx", "zip"));
+ private static Set<String> excludedExtensions = new TreeSet<String>(
+ Arrays.asList("7z", "arc", "arj", "bin", "bmp", "dll", "doc",
+ "docx", "exe", "gif", "gz", "jar", "jpg", "lib", "lzh",
+ "odg", "pdf", "ppt", "png", "so", "swf", "xcf", "xls",
+ "xlsx", "zip"));
+
+ private static Set<String> excludedBranches = new TreeSet<String>(
+ Arrays.asList("/refs/heads/gb-issues"));
private static final Map<File, IndexSearcher> SEARCHERS = new ConcurrentHashMap<File, IndexSearcher>();
private static final Map<File, IndexWriter> WRITERS = new ConcurrentHashMap<File, IndexWriter>();
+ /**
+ * Returns the name of the repository.
+ *
+ * @param repository
+ * @return the repository name
+ */
+ private static String getName(Repository repository) {
+ if (repository.isBare()) {
+ return repository.getDirectory().getName();
+ } else {
+ return repository.getDirectory().getParentFile().getName();
+ }
+ }
+
/**
* Deletes the Lucene index for the specified repository.
*
@@ -125,6 +148,7 @@
*/
public static boolean index(Repository repository) {
try {
+ String repositoryName = getName(repository);
Set<String> indexedCommits = new TreeSet<String>();
IndexWriter writer = getIndexWriter(repository, true);
// build a quick lookup of tags
@@ -139,6 +163,10 @@
// walk through each branch
List<RefModel> branches = JGitUtils.getLocalBranches(repository, true, -1);
for (RefModel branch : branches) {
+ if (excludedBranches.contains(branch.getName())) {
+ continue;
+ }
+ String branchName = branch.getName();
RevWalk revWalk = new RevWalk(repository);
RevCommit rev = revWalk.parseCommit(branch.getObjectId());
@@ -154,6 +182,10 @@
Document doc = new Document();
doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES,
Index.NOT_ANALYZED_NO_NORMS));
+ doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,
+ Index.NOT_ANALYZED));
+ doc.add(new Field(FIELD_BRANCH, branchName, Store.YES,
+ Index.NOT_ANALYZED));
doc.add(new Field(FIELD_OBJECT_ID, treeWalk.getPathString(), Store.YES,
Index.NOT_ANALYZED));
doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO));
@@ -171,7 +203,7 @@
ext = name.substring(name.lastIndexOf('.') + 1);
}
- if (StringUtils.isEmpty(ext) || !excludes.contains(ext)) {
+ if (StringUtils.isEmpty(ext) || !excludedExtensions.contains(ext)) {
// read the blob content
ObjectId entid = treeWalk.getObjectId(0);
FileMode entmode = treeWalk.getFileMode(0);
@@ -199,6 +231,10 @@
String head = rev.getId().getName();
if (indexedCommits.add(head)) {
Document doc = createDocument(rev, tags.get(head));
+ doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,
+ Index.NOT_ANALYZED));
+ doc.add(new Field(FIELD_BRANCH, branchName, Store.YES,
+ Index.NOT_ANALYZED));
writer.addDocument(doc);
}
@@ -208,6 +244,10 @@
String hash = rev.getId().getName();
if (indexedCommits.add(hash)) {
Document doc = createDocument(rev, tags.get(hash));
+ doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,
+ Index.NOT_ANALYZED));
+ doc.add(new Field(FIELD_BRANCH, branchName, Store.YES,
+ Index.NOT_ANALYZED));
writer.addDocument(doc);
}
}
@@ -221,6 +261,8 @@
List<IssueModel> issues = IssueUtils.getIssues(repository, null);
for (IssueModel issue : issues) {
Document doc = createDocument(issue);
+ doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,
+ Index.NOT_ANALYZED));
writer.addDocument(doc);
}
}
@@ -240,11 +282,69 @@
* repository.
*
* @param repository
+ * @param branch
+ * the fully qualified branch name (e.g. refs/heads/master)
* @param commit
* @return true, if successful
*/
- public static boolean index(Repository repository, RevCommit commit) {
- try {
+ public static boolean index(Repository repository, String branch, RevCommit commit) {
+ try {
+ if (excludedBranches.contains(branch)) {
+ if (IssueUtils.GB_ISSUES.equals(branch)) {
+ // index an issue
+ String issueId = commit.getShortMessage().substring(2).trim();
+ IssueModel issue = IssueUtils.getIssue(repository, issueId);
+ return index(repository, issue, true);
+ }
+ return false;
+ }
+ List<PathChangeModel> changedPaths = JGitUtils.getFilesInCommit(repository, commit);
+ String repositoryName = getName(repository);
+ String revDate = DateTools.timeToString(commit.getCommitTime() * 1000L,
+ Resolution.MINUTE);
+ IndexWriter writer = getIndexWriter(repository, false);
+ for (PathChangeModel path : changedPaths) {
+ // delete the indexed blob
+ writer.deleteDocuments(new Term(FIELD_OBJECT_TYPE, ObjectType.blob.name()),
+ new Term(FIELD_BRANCH, branch),
+ new Term(FIELD_OBJECT_ID, path.path));
+
+ // re-index the blob
+ if (!ChangeType.DELETE.equals(path.changeType)) {
+ Document doc = new Document();
+ doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES,
+ Index.NOT_ANALYZED_NO_NORMS));
+ doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,
+ Index.NOT_ANALYZED));
+ doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.NOT_ANALYZED));
+ doc.add(new Field(FIELD_OBJECT_ID, path.path, Store.YES,
+ Index.NOT_ANALYZED));
+ doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO));
+ doc.add(new Field(FIELD_AUTHOR, commit.getAuthorIdent().getName(), Store.YES,
+ Index.NOT_ANALYZED_NO_NORMS));
+ doc.add(new Field(FIELD_COMMITTER, commit.getCommitterIdent().getName(),
+ Store.YES, Index.NOT_ANALYZED_NO_NORMS));
+ doc.add(new Field(FIELD_LABEL, branch, Store.YES, Index.ANALYZED));
+
+ // determine extension to compare to the extension
+ // blacklist
+ String ext = null;
+ String name = path.name.toLowerCase();
+ if (name.indexOf('.') > -1) {
+ ext = name.substring(name.lastIndexOf('.') + 1);
+ }
+
+ if (StringUtils.isEmpty(ext) || !excludedExtensions.contains(ext)) {
+ // read the blob content
+ String str = JGitUtils.getStringContent(repository,
+ commit.getTree(), path.path);
+ doc.add(new Field(FIELD_CONTENT, str, Store.NO, Index.ANALYZED));
+ writer.addDocument(doc);
+ }
+ }
+ }
+ writer.commit();
+
Document doc = createDocument(commit, null);
return index(repository, doc);
} catch (Exception e) {
@@ -292,6 +392,7 @@
doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.issue.name(), Store.YES,
Field.Index.NOT_ANALYZED_NO_NORMS));
doc.add(new Field(FIELD_OBJECT_ID, issue.id, Store.YES, Index.NOT_ANALYZED));
+ doc.add(new Field(FIELD_BRANCH, IssueUtils.GB_ISSUES, Store.YES, Index.NOT_ANALYZED));
doc.add(new Field(FIELD_DATE, DateTools.dateToString(issue.created, Resolution.MINUTE),
Store.YES, Field.Index.NO));
doc.add(new Field(FIELD_AUTHOR, issue.reporter, Store.YES, Index.NOT_ANALYZED_NO_NORMS));
@@ -344,6 +445,9 @@
*/
private static boolean index(Repository repository, Document doc) {
try {
+ String repositoryName = getName(repository);
+ doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,
+ Index.NOT_ANALYZED));
IndexWriter writer = getIndexWriter(repository, false);
writer.addDocument(doc);
resetIndexSearcher(repository);
@@ -363,6 +467,8 @@
result.author = doc.get(FIELD_AUTHOR);
result.committer = doc.get(FIELD_COMMITTER);
result.type = ObjectType.fromName(doc.get(FIELD_OBJECT_TYPE));
+ result.repository = doc.get(FIELD_REPOSITORY);
+ result.branch = doc.get(FIELD_BRANCH);
result.id = doc.get(FIELD_OBJECT_ID);
if (doc.get(FIELD_LABEL) != null) {
result.labels = StringUtils.getStringsFromValue(doc.get(FIELD_LABEL));
@@ -437,17 +543,27 @@
}
/**
- * Search the repository for the given text or query
+ * Searches the specified repositories for the given text or query
*
- * @param repository
* @param text
- * @return a list of SearchResults
+ * if the text is null or empty, null is returned
+ * @param maximumHits
+ * the maximum number of hits to collect
+ * @param repositories
+ * a list of repositories to search. if no repositories are
+ * specified null is returned.
+ * @return a list of SearchResults in order from highest to the lowest score
+ *
*/
- public static List<SearchResult> search(Repository repository, String text) {
+ public static List<SearchResult> search(String text, int maximumHits,
+ Repository... repositories) {
if (StringUtils.isEmpty(text)) {
return null;
}
- Set<SearchResult> results = new HashSet<SearchResult>();
+ if (repositories.length == 0) {
+ return null;
+ }
+ Set<SearchResult> results = new LinkedHashSet<SearchResult>();
StandardAnalyzer analyzer = new StandardAnalyzer(LUCENE_VERSION);
try {
// default search checks summary and content
@@ -461,10 +577,23 @@
qp.setAllowLeadingWildcard(true);
query.add(qp.parse(text), Occur.SHOULD);
- IndexSearcher searcher = getIndexSearcher(repository);
+ IndexSearcher searcher;
+ if (repositories.length == 1) {
+ // single repository search
+ searcher = getIndexSearcher(repositories[0]);
+ } else {
+ // multiple repository search
+ List<IndexReader> readers = new ArrayList<IndexReader>();
+ for (Repository repository : repositories) {
+ IndexSearcher repositoryIndex = getIndexSearcher(repository);
+ readers.add(repositoryIndex.getIndexReader());
+ }
+ IndexReader [] rdrs = readers.toArray(new IndexReader[readers.size()]);
+ MultiReader reader = new MultiReader(rdrs);
+ searcher = new IndexSearcher(reader);
+ }
Query rewrittenQuery = searcher.rewrite(query);
-
- TopScoreDocCollector collector = TopScoreDocCollector.create(200, true);
+ TopScoreDocCollector collector = TopScoreDocCollector.create(maximumHits, true);
searcher.search(rewrittenQuery, collector);
ScoreDoc[] hits = collector.topDocs().scoreDocs;
for (int i = 0; i < hits.length; i++) {
@@ -477,7 +606,7 @@
e.printStackTrace();
}
return new ArrayList<SearchResult>(results);
- }
+ }
/**
* Close all the index writers and searchers
--
Gitblit v1.9.1