| src/com/gitblit/GitBlit.java | ●●●●● patch | view | raw | blame | history | |
| src/com/gitblit/LuceneExecutor.java | ●●●●● patch | view | raw | blame | history | |
| src/com/gitblit/models/RepositoryModel.java | ●●●●● patch | view | raw | blame | history | |
| src/com/gitblit/wicket/GitBlitWebApp.properties | ●●●●● patch | view | raw | blame | history | |
| src/com/gitblit/wicket/pages/EditRepositoryPage.html | ●●●●● patch | view | raw | blame | history | |
| src/com/gitblit/wicket/pages/EditRepositoryPage.java | ●●●●● patch | view | raw | blame | history | |
| src/com/gitblit/wicket/pages/LucenePage.java | ●●●●● patch | view | raw | blame | history | |
| tests/com/gitblit/tests/LuceneExecutorTest.java | ●●●●● patch | view | raw | blame | history |
src/com/gitblit/GitBlit.java
@@ -812,6 +812,8 @@ "gitblit", null, "postReceiveScript"))); model.mailingLists = new ArrayList<String>(Arrays.asList(config.getStringList( "gitblit", null, "mailingList"))); model.indexedBranches = new ArrayList<String>(Arrays.asList(config.getStringList( "gitblit", null, "indexBranch"))); } model.HEAD = JGitUtils.getHEADRef(r); model.availableRefs = JGitUtils.getAvailableHeadTargets(r); @@ -955,6 +957,12 @@ // create repository logger.info("create repository " + repository.name); r = JGitUtils.createRepository(repositoriesFolder, repository.name); // automatically index master branch if Lucene integration is enabled if (luceneExecutor.isReady()) { repository.indexedBranches = new ArrayList<String>(); repository.indexedBranches.add("refs/heads/master"); } } else { // rename repository if (!repositoryName.equalsIgnoreCase(repository.name)) { @@ -1063,6 +1071,9 @@ if (!ArrayUtils.isEmpty(repository.mailingLists)) { config.setStringList("gitblit", null, "mailingList", repository.mailingLists); } if (!ArrayUtils.isEmpty(repository.indexedBranches)) { config.setStringList("gitblit", null, "indexBranch", repository.indexedBranches); } try { config.save(); } catch (IOException e) { src/com/gitblit/LuceneExecutor.java
@@ -89,6 +89,7 @@ import com.gitblit.models.IssueModel.Attachment; import com.gitblit.models.PathModel.PathChangeModel; import com.gitblit.models.RefModel; import com.gitblit.models.RepositoryModel; import com.gitblit.models.SearchResult; import com.gitblit.utils.ArrayUtils; import com.gitblit.utils.IssueUtils; @@ -141,9 +142,6 @@ "arj", "bin", "bmp", "dll", "doc", "docx", "exe", "gif", "gz", "jar", "jpg", "lib", "lzh", "odg", "pdf", "ppt", "png", "so", "swf", "xcf", "xls", "xlsx", "zip")); private final Set<String> excludedBranches = new TreeSet<String>( Arrays.asList("/refs/heads/gb-issues")); public LuceneExecutor(IStoredSettings settings, File repositoriesFolder) { this.storedSettings = settings; this.repositoriesFolder = repositoriesFolder; @@ -170,18 +168,14 @@ return; } for (String repositoryName : GitBlit.self().getRepositoryList()) { Repository repository = GitBlit.self().getRepository(repositoryName); if (repository == null) { logger.warn(MessageFormat.format( "Lucene executor could not find repository {0}. Skipping.", repositoryName)); continue; for (String repositoryName: GitBlit.self().getRepositoryList()) { RepositoryModel model = GitBlit.self().getRepositoryModel(repositoryName); if (model.hasCommits && !ArrayUtils.isEmpty(model.indexedBranches)) { Repository repository = GitBlit.self().getRepository(model.name); index(model, repository); repository.close(); System.gc(); } // TODO allow repository to bypass Lucene indexing index(repositoryName, repository); repository.close(); System.gc(); } } @@ -194,43 +188,38 @@ * @param repository * the repository object */ protected void index(String name, Repository repository) { protected void index(RepositoryModel model, Repository repository) { try { if (JGitUtils.hasCommits(repository)) { if (shouldReindex(repository)) { // (re)build the entire index IndexResult result = reindex(name, repository); if (result.success) { if (result.commitCount > 0) { String msg = "Built {0} Lucene index from {1} commits and {2} files across {3} branches in {4} secs"; logger.info(MessageFormat.format(msg, name, result.commitCount, result.blobCount, result.branchCount, result.duration())); } } else { String msg = "Could not build {0} Lucene index!"; logger.error(MessageFormat.format(msg, name)); if (shouldReindex(repository)) { // (re)build the entire index IndexResult result = reindex(model, repository); if (result.success) { if (result.commitCount > 0) { String msg = "Built {0} Lucene index from {1} commits and {2} files across {3} branches in {4} secs"; logger.info(MessageFormat.format(msg, model.name, result.commitCount, result.blobCount, result.branchCount, result.duration())); } } else { // update the index with latest commits IndexResult result = updateIndex(name, repository); if (result.success) { if (result.commitCount > 0) { String msg = "Updated {0} Lucene index with {1} commits and {2} files across {3} branches in {4} secs"; logger.info(MessageFormat.format(msg, name, result.commitCount, result.blobCount, result.branchCount, result.duration())); } } else { String msg = "Could not update {0} Lucene index!"; logger.error(MessageFormat.format(msg, name)); } String msg = "Could not build {0} Lucene index!"; logger.error(MessageFormat.format(msg, model.name)); } } else { logger.info(MessageFormat.format("Skipped Lucene index of empty repository {0}", name)); // update the index with latest commits IndexResult result = updateIndex(model, repository); if (result.success) { if (result.commitCount > 0) { String msg = "Updated {0} Lucene index with {1} commits and {2} files across {3} branches in {4} secs"; logger.info(MessageFormat.format(msg, model.name, result.commitCount, result.blobCount, result.branchCount, result.duration())); } } else { String msg = "Could not update {0} Lucene index!"; logger.error(MessageFormat.format(msg, model.name)); } } } catch (Throwable t) { logger.error(MessageFormat.format("Lucene indexing failure for {0}", name), t); logger.error(MessageFormat.format("Lucene indexing failure for {0}", model.name), t); } } @@ -430,15 +419,15 @@ * @param repository * @return IndexResult */ public IndexResult reindex(String repositoryName, Repository repository) { public IndexResult reindex(RepositoryModel model, Repository repository) { IndexResult result = new IndexResult(); if (!deleteIndex(repositoryName)) { if (!deleteIndex(model.name)) { return result; } try { FileBasedConfig config = getConfig(repository); Set<String> indexedCommits = new TreeSet<String>(); IndexWriter writer = getIndexWriter(repositoryName); IndexWriter writer = getIndexWriter(model.name); // build a quick lookup of tags Map<String, List<String>> tags = new HashMap<String, List<String>>(); for (RefModel tag : JGitUtils.getTags(repository, false, -1)) { @@ -479,7 +468,9 @@ // walk through each branch for (RefModel branch : branches) { if (excludedBranches.contains(branch.getName())) { // if this branch is not specifically indexed then skip if (!model.indexedBranches.contains(branch.getName())) { continue; } @@ -624,11 +615,11 @@ // commit all changes and reset the searcher config.setInt(CONF_INDEX, null, CONF_VERSION, INDEX_VERSION); config.save(); resetIndexSearcher(repositoryName); resetIndexSearcher(model.name); writer.commit(); result.success(); } catch (Exception e) { logger.error("Exception while reindexing " + repositoryName, e); logger.error("Exception while reindexing " + model.name, e); } return result; } @@ -648,24 +639,6 @@ String branch, RevCommit commit) { IndexResult result = new IndexResult(); try { if (excludedBranches.contains(branch)) { if (IssueUtils.GB_ISSUES.equals(branch)) { // index an issue String issueId = commit.getShortMessage().substring(2).trim(); IssueModel issue = IssueUtils.getIssue(repository, issueId); if (issue == null) { // issue was deleted, remove from index deleteIssue(repositoryName, issueId); result.success = true; return result; } result.success = index(repositoryName, issue); result.issueCount++; return result; } return result; } List<PathChangeModel> changedPaths = JGitUtils.getFilesInCommit(repository, commit); String revDate = DateTools.timeToString(commit.getCommitTime() * 1000L, Resolution.MINUTE); @@ -779,11 +752,11 @@ /** * Updates a repository index incrementally from the last indexed commits. * * @param repositoryName * @param model * @param repository * @return IndexResult */ protected IndexResult updateIndex(String repositoryName, Repository repository) { protected IndexResult updateIndex(RepositoryModel model, Repository repository) { IndexResult result = new IndexResult(); try { FileBasedConfig config = getConfig(repository); @@ -816,6 +789,12 @@ for (RefModel branch : branches) { String branchName = branch.getName(); // determine if we should skip this branch if (!IssueUtils.GB_ISSUES.equals(branch) && !model.indexedBranches.contains(branch.getName())) { continue; } // remove this branch from the deletedBranches set deletedBranches.remove(branchName); @@ -836,10 +815,33 @@ result.branchCount += 1; } // track the issue ids that we have already indexed Set<String> indexedIssues = new TreeSet<String>(); // reverse the list of commits so we start with the first commit Collections.reverse(revs); for (RevCommit commit : revs) { result.add(index(repositoryName, repository, branchName, commit)); for (RevCommit commit : revs) { if (IssueUtils.GB_ISSUES.equals(branch)) { // only index an issue once during updateIndex String issueId = commit.getShortMessage().substring(2).trim(); if (indexedIssues.contains(issueId)) { continue; } indexedIssues.add(issueId); IssueModel issue = IssueUtils.getIssue(repository, issueId); if (issue == null) { // issue was deleted, remove from index deleteIssue(model.name, issueId); } else { // issue was updated index(model.name, issue); result.issueCount++; } } else { // index a commit result.add(index(model.name, repository, branchName, commit)); } } // update the config @@ -853,14 +855,14 @@ // unless a branch really was deleted and no longer exists if (deletedBranches.size() > 0) { for (String branch : deletedBranches) { IndexWriter writer = getIndexWriter(repositoryName); IndexWriter writer = getIndexWriter(model.name); writer.deleteDocuments(new Term(FIELD_BRANCH, branch)); writer.commit(); } } result.success = true; } catch (Throwable t) { logger.error(MessageFormat.format("Exception while updating {0} Lucene index", repositoryName), t); logger.error(MessageFormat.format("Exception while updating {0} Lucene index", model.name), t); } return result; } src/com/gitblit/models/RepositoryModel.java
@@ -57,6 +57,7 @@ public String origin; public String HEAD; public List<String> availableRefs; public List<String> indexedBranches; public String size; public List<String> preReceiveScripts; public List<String> postReceiveScripts; @@ -74,7 +75,7 @@ this.lastChange = lastchange; this.accessRestriction = AccessRestrictionType.NONE; this.federationSets = new ArrayList<String>(); this.federationStrategy = FederationStrategy.FEDERATE_THIS; this.federationStrategy = FederationStrategy.FEDERATE_THIS; } @Override src/com/gitblit/wicket/GitBlitWebApp.properties
@@ -216,4 +216,9 @@ gb.query = query gb.queryHelp = write brief explanation of Lucene syntax here gb.authored = authored gb.committed = committed gb.committed = committed gb.indexedBranches = indexed branches gb.indexedBranchesDescription = select the branches to include in your Lucene index gb.noIndexedRepositoriesWarning = none of your repositories are configured for Lucene indexing gb.undefinedQueryWarning = query is undefined! gb.gb.noSelectedRepositoriesWarning = please select one or more repositories! src/com/gitblit/wicket/pages/EditRepositoryPage.html
@@ -31,6 +31,8 @@ <tr><td colspan="2"><h3><wicket:message key="gb.federation"></wicket:message> <small><wicket:message key="gb.federationRepositoryDescription"></wicket:message></small></h3></td></tr> <tr><th><wicket:message key="gb.federationStrategy"></wicket:message></th><td class="edit"><select class="span4" wicket:id="federationStrategy" tabindex="15" /></td></tr> <tr><th style="vertical-align: top;"><wicket:message key="gb.federationSets"></wicket:message></th><td style="padding:2px;"><span wicket:id="federationSets"></span></td></tr> <tr><td colspan="2"><h3><wicket:message key="gb.search"></wicket:message> <small><wicket:message key="gb.indexedBranchesDescription"></wicket:message></small></h3></td></tr> <tr><th style="vertical-align: top;"><wicket:message key="gb.indexedBranches"></wicket:message></th><td style="padding:2px;"><span wicket:id="indexedBranches"></span></td></tr> <tr><td colspan="2"><h3><wicket:message key="gb.hookScripts"></wicket:message> <small><wicket:message key="gb.hookScriptsDescription"></wicket:message></small></h3></td></tr> <tr><th style="vertical-align: top;"><wicket:message key="gb.preReceiveScripts"></wicket:message><p></p><span wicket:id="inheritedPreReceive"></span></th><td style="padding:2px;"><span wicket:id="preReceiveScripts"></span></td></tr> <tr><th style="vertical-align: top;"><wicket:message key="gb.postReceiveScripts"></wicket:message><p></p><span wicket:id="inheritedPostReceive"></span></th><td style="padding:2px;"><span wicket:id="postReceiveScripts"></span></td></tr> src/com/gitblit/wicket/pages/EditRepositoryPage.java
@@ -39,6 +39,7 @@ import org.apache.wicket.model.Model; import org.apache.wicket.model.util.CollectionModel; import org.apache.wicket.model.util.ListModel; import org.eclipse.jgit.lib.Constants; import com.gitblit.Constants.AccessRestrictionType; import com.gitblit.Constants.FederationStrategy; @@ -82,6 +83,7 @@ // ensure this user can create or edit this repository checkPermissions(repositoryModel); List<String> indexedBranches = new ArrayList<String>(); List<String> federationSets = new ArrayList<String>(); List<String> repositoryUsers = new ArrayList<String>(); List<String> repositoryTeams = new ArrayList<String>(); @@ -98,6 +100,9 @@ Collections.sort(repositoryUsers); } federationSets.addAll(repositoryModel.federationSets); if (!ArrayUtils.isEmpty(repositoryModel.indexedBranches)) { indexedBranches.addAll(repositoryModel.indexedBranches); } } final String oldName = repositoryModel.name; @@ -109,13 +114,28 @@ // teams palette final Palette<String> teamsPalette = new Palette<String>("teams", new ListModel<String>( repositoryTeams), new CollectionModel<String>(GitBlit.self().getAllTeamnames()), new StringChoiceRenderer(), 5, false); new StringChoiceRenderer(), 8, false); // indexed local branches palette List<String> allLocalBranches = new ArrayList<String>(); if (!ArrayUtils.isEmpty(repositoryModel.availableRefs)) { for (String ref : repositoryModel.availableRefs) { if (ref.startsWith(Constants.R_HEADS)) { allLocalBranches.add(ref); } } } final Palette<String> indexedBranchesPalette = new Palette<String>("indexedBranches", new ListModel<String>( indexedBranches), new CollectionModel<String>(allLocalBranches), new StringChoiceRenderer(), 8, false); indexedBranchesPalette.setEnabled(allLocalBranches.size() > 0); // federation sets palette List<String> sets = GitBlit.getStrings(Keys.federation.sets); final Palette<String> federationSetsPalette = new Palette<String>("federationSets", new ListModel<String>(federationSets), new CollectionModel<String>(sets), new StringChoiceRenderer(), 5, false); new StringChoiceRenderer(), 8, false); // pre-receive palette if (!ArrayUtils.isEmpty(repositoryModel.preReceiveScripts)) { @@ -211,6 +231,14 @@ repositoryModel.mailingLists = new ArrayList<String>(list); } // indexed branches List<String> indexedBranches = new ArrayList<String>(); Iterator<String> branches = indexedBranchesPalette.getSelectedChoices(); while (branches.hasNext()) { indexedBranches.add(branches.next()); } repositoryModel.indexedBranches = indexedBranches; // pre-receive scripts List<String> preReceiveScripts = new ArrayList<String>(); Iterator<String> pres = preReceivePalette.getSelectedChoices(); @@ -293,7 +321,7 @@ form.add(new DropDownChoice<FederationStrategy>("federationStrategy", federationStrategies, new FederationTypeRenderer())); form.add(new CheckBox("useTickets")); form.add(new CheckBox("useDocs")); form.add(new CheckBox("useDocs")); form.add(new CheckBox("showRemoteBranches")); form.add(new CheckBox("showReadme")); form.add(new CheckBox("skipSizeCalculation")); @@ -301,6 +329,7 @@ mailingLists = new Model<String>(ArrayUtils.isEmpty(repositoryModel.mailingLists) ? "" : StringUtils.flattenStrings(repositoryModel.mailingLists, " ")); form.add(new TextField<String>("mailingLists", mailingLists)); form.add(indexedBranchesPalette); form.add(usersPalette); form.add(teamsPalette); form.add(federationSetsPalette); src/com/gitblit/wicket/pages/LucenePage.java
@@ -88,9 +88,30 @@ } } // display user-accessible selections UserModel user = GitBlitWebSession.get().getUser(); List<String> availableRepositories = new ArrayList<String>(); for (RepositoryModel model : GitBlit.self().getRepositoryModels(user)) { if (model.hasCommits && !ArrayUtils.isEmpty(model.indexedBranches)) { availableRepositories.add(model.name); } } if (availableRepositories.size() == 0) { info(getString("gb.noIndexedRepositoriesWarning")); } // enforce user-accessible repository selections ArrayList<String> searchRepositories = new ArrayList<String>(); for (String selectedRepository : repositories) { if (availableRepositories.contains(selectedRepository)) { searchRepositories.add(selectedRepository); } } // search form final Model<String> queryModel = new Model<String>(query); final Model<ArrayList<String>> repositoriesModel = new Model<ArrayList<String>>(repositories); final Model<ArrayList<String>> repositoriesModel = new Model<ArrayList<String>>(searchRepositories); StatelessForm<Void> form = new StatelessForm<Void>("searchForm") { private static final long serialVersionUID = 1L; @@ -99,11 +120,11 @@ public void onSubmit() { String q = queryModel.getObject(); if (StringUtils.isEmpty(q)) { error("Query is empty!"); error(getString("gb.undefinedQueryWarning")); return; } if (repositoriesModel.getObject().size() == 0) { error("Please select one or more repositories!"); error(getString("gb.noSelectedRepositoriesWarning")); return; } PageParameters params = new PageParameters(); @@ -112,14 +133,7 @@ setResponsePage(LucenePage.class, params); } }; UserModel user = GitBlitWebSession.get().getUser(); List<String> availableRepositories = new ArrayList<String>(); for (RepositoryModel model : GitBlit.self().getRepositoryModels(user)) { if (model.hasCommits) { availableRepositories.add(model.name); } } ListMultipleChoice<String> selections = new ListMultipleChoice<String>("repositories", repositoriesModel, availableRepositories, new StringChoiceRenderer()); selections.setMaxRows(10); @@ -129,8 +143,8 @@ // execute search final List<SearchResult> results = new ArrayList<SearchResult>(); if (!ArrayUtils.isEmpty(repositories) && !StringUtils.isEmpty(query)) { results.addAll(GitBlit.self().search(query, 100, repositories)); if (!ArrayUtils.isEmpty(searchRepositories) && !StringUtils.isEmpty(query)) { results.addAll(GitBlit.self().search(query, 100, searchRepositories)); } // search results view tests/com/gitblit/tests/LuceneExecutorTest.java
@@ -24,7 +24,10 @@ import org.junit.Test; import com.gitblit.LuceneExecutor; import com.gitblit.models.RefModel; import com.gitblit.models.RepositoryModel; import com.gitblit.models.SearchResult; import com.gitblit.utils.JGitUtils; import com.gitblit.utils.StringUtils; /** @@ -39,9 +42,18 @@ return new LuceneExecutor(null, GitBlitSuite.REPOSITORIES); } private String getName(Repository repository) { return StringUtils.getRelativePath(GitBlitSuite.REPOSITORIES.getAbsolutePath(), private RepositoryModel newRepositoryModel(Repository repository) { RepositoryModel model = new RepositoryModel(); model.name = StringUtils.getRelativePath(GitBlitSuite.REPOSITORIES.getAbsolutePath(), repository.getDirectory().getAbsolutePath()); model.hasCommits = JGitUtils.hasCommits(repository); // index all local branches model.indexedBranches = new ArrayList<String>(); for (RefModel ref : JGitUtils.getLocalBranches(repository, true, -1)) { model.indexedBranches.add(ref.getName()); } return model; } @Test @@ -50,25 +62,25 @@ // reindex helloworld Repository repository = GitBlitSuite.getHelloworldRepository(); String name = getName(repository); lucene.reindex(name, repository); RepositoryModel model = newRepositoryModel(repository); lucene.reindex(model, repository); repository.close(); SearchResult result = lucene.search("type:blob AND path:bit.bit", 1, name).get(0); SearchResult result = lucene.search("type:blob AND path:bit.bit", 1, model.name).get(0); assertEquals("Mike Donaghy", result.author); result = lucene.search("type:blob AND path:clipper.prg", 1, name).get(0); result = lucene.search("type:blob AND path:clipper.prg", 1, model.name).get(0); assertEquals("tinogomes", result.author); // reindex theoretical physics repository = GitBlitSuite.getTheoreticalPhysicsRepository(); name = getName(repository); lucene.reindex(name, repository); model = newRepositoryModel(repository); lucene.reindex(model, repository); repository.close(); // reindex JGit repository = GitBlitSuite.getJGitRepository(); name = getName(repository); lucene.reindex(name, repository); model = newRepositoryModel(repository); lucene.reindex(model, repository); repository.close(); lucene.close(); @@ -80,51 +92,51 @@ // 2 occurrences on the master branch Repository repository = GitBlitSuite.getHelloworldRepository(); String name = getName(repository); RepositoryModel model = newRepositoryModel(repository); repository.close(); List<SearchResult> results = lucene.search("ada", 10, name); List<SearchResult> results = lucene.search("ada", 10, model.name); assertEquals(2, results.size()); for (SearchResult res : results) { assertEquals("refs/heads/master", res.branch); } // author test results = lucene.search("author: tinogomes AND type:commit", 10, name); results = lucene.search("author: tinogomes AND type:commit", 10, model.name); assertEquals(2, results.size()); // blob test results = lucene.search("type: blob AND \"import std.stdio\"", 10, name); results = lucene.search("type: blob AND \"import std.stdio\"", 10, model.name); assertEquals(1, results.size()); assertEquals("d.D", results.get(0).path); // 1 occurrence on the gh-pages branch repository = GitBlitSuite.getTheoreticalPhysicsRepository(); name = getName(repository); model = newRepositoryModel(repository); repository.close(); results = lucene.search("\"add the .nojekyll file\"", 10, name); results = lucene.search("\"add the .nojekyll file\"", 10, model.name); assertEquals(1, results.size()); assertEquals("Ondrej Certik", results.get(0).author); assertEquals("2648c0c98f2101180715b4d432fc58d0e21a51d7", results.get(0).commitId); assertEquals("refs/heads/gh-pages", results.get(0).branch); results = lucene.search("type:blob AND \"src/intro.rst\"", 10, name); results = lucene.search("type:blob AND \"src/intro.rst\"", 10, model.name); assertEquals(4, results.size()); // hash id tests results = lucene.search("commit:57c4f26f157ece24b02f4f10f5f68db1d2ce7ff5", 10, name); results = lucene.search("commit:57c4f26f157ece24b02f4f10f5f68db1d2ce7ff5", 10, model.name); assertEquals(1, results.size()); results = lucene.search("commit:57c4f26f157*", 10, name); results = lucene.search("commit:57c4f26f157*", 10, model.name); assertEquals(1, results.size()); // annotated tag test repository = GitBlitSuite.getJGitRepository(); name = getName(repository); model = newRepositoryModel(repository); repository.close(); results = lucene.search("I663208919f297836a9c16bf458e4a43ffaca4c12", 10, name); results = lucene.search("I663208919f297836a9c16bf458e4a43ffaca4c12", 10, model.name); assertEquals(1, results.size()); assertEquals("[v1.3.0.201202151440-r]", results.get(0).tags.toString()); @@ -136,11 +148,11 @@ LuceneExecutor lucene = newLuceneExecutor(); List<String> list = new ArrayList<String>(); Repository repository = GitBlitSuite.getHelloworldRepository(); list.add(getName(repository)); list.add(newRepositoryModel(repository).name); repository.close(); repository = GitBlitSuite.getJGitRepository(); list.add(getName(repository)); list.add(newRepositoryModel(repository).name); repository.close(); List<SearchResult> results = lucene.search("test", 10, list);
