distrib/gitblit.properties
@@ -12,6 +12,17 @@ # RESTART REQUIRED git.repositoriesFolder = git # Build the available repository list at startup and cache this list for reuse. # This reduces disk io when presenting the repositories page, responding to rpcs, # etc, but it means that Gitblit will not automatically identify repositories # added or deleted by external tools. # # For this case you can use curl, wget, etc to issue an rpc request to clear the # cache (e.g. https://localhost/rpc?req=CLEAR_REPOSITORY_CACHE) # # SINCE 1.1.0 git.cacheRepositoryList = true # Search the repositories folder subfolders for other repositories. # Repositories MAY NOT be nested (i.e. one repository within another) # but they may be grouped together in subfolders. @@ -24,7 +35,7 @@ # Maximum number of folders to recurse into when searching for repositories. # The default value, -1, disables depth limits. # # SINCE 1.0.1 # SINCE 1.1.0 git.searchRecursionDepth = -1 # List of regex exclusion patterns to match against folders found in @@ -34,7 +45,7 @@ # # SPACE-DELIMITED # CASE-SENSITIVE # SINCE 1.0.1 # SINCE 1.1.0 git.searchExclusions = # List of regex url patterns for extracting a repository name when locating @@ -46,7 +57,7 @@ # # SPACE-DELIMITED # CASE-SENSITIVE # SINCE 1.0.1 # SINCE 1.1.0 git.submoduleUrlPatterns = .*?://github.com/(.*) # Allow push/pull over http/https with JGit servlet. @@ -80,7 +91,7 @@ # AUTHENTICATED = any authenticated user is granted restricted access # NAMED = only named users/teams are granted restricted access # # SINCE 1.0.1 # SINCE 1.1.0 git.defaultAuthorizationControl = NAMED # Number of bytes of a pack file to load into memory in a single read operation. docs/02_rpc.mkd
@@ -63,7 +63,8 @@ <tr><th>Release</th><th>Protocol Version</th></tr> <tr><td>Gitblit v0.7.0</td><td>1 (inferred version)</td></tr> <tr><td>Gitblit v0.8.0</td><td>2</td></tr> <tr><td>Gitblit v0.9.0+</td><td>3</td></tr> <tr><td>Gitblit v0.9.0 - v1.0.0</td><td>3</td></tr> <tr><td>Gitblit v1.1.0+</td><td>4</td></tr> </tbody> </table> @@ -101,6 +102,7 @@ <tr><td>LIST_SETTINGS</td><td>-</td><td><em>admin</em></td><td>1</td><td>-</td><td>ServerSettings (all keys)</td></tr> <tr><td>EDIT_SETTINGS</td><td>-</td><td><em>admin</em></td><td>1</td><td>Map<String, String></td><td>-</td></tr> <tr><td>LIST_STATUS</td><td>-</td><td><em>admin</em></td><td>1</td><td>-</td><td>ServerStatus (see example below)</td></tr> <tr><td>CLEAR_REPOSITORY_CACHE</td><td>-</td><td><em>admin</em></td><td>4</td><td>-</td><td>-</td></tr> </table> ### RPC/HTTP Response Codes docs/04_releases.mkd
@@ -25,6 +25,8 @@ #### additions - Identified repository list is now cached by default to reduce disk io and to improve performance (issue 103) **New:** *git.cacheRepositoryList=true* - Preliminary bare repository submodule support **New:** *git.submoduleUrlPatterns=* - *git.submoduleUrlPatterns* is a space-delimited list of regular expressions for extracting a repository name from a submodule url. src/com/gitblit/Constants.java
@@ -30,7 +30,7 @@ // The build script extracts this exact line so be careful editing it // and only use A-Z a-z 0-9 .-_ in the string. public static final String VERSION = "1.0.1-SNAPSHOT"; public static final String VERSION = "1.1.0-SNAPSHOT"; // The build script extracts this exact line so be careful editing it // and only use A-Z a-z 0-9 .-_ in the string. @@ -248,7 +248,7 @@ LIST_TEAMS, CREATE_TEAM, EDIT_TEAM, DELETE_TEAM, LIST_REPOSITORY_MEMBERS, SET_REPOSITORY_MEMBERS, LIST_REPOSITORY_TEAMS, SET_REPOSITORY_TEAMS, LIST_FEDERATION_REGISTRATIONS, LIST_FEDERATION_RESULTS, LIST_FEDERATION_PROPOSALS, LIST_FEDERATION_SETS, EDIT_SETTINGS, LIST_STATUS; EDIT_SETTINGS, LIST_STATUS, CLEAR_REPOSITORY_CACHE; public static RpcRequest fromName(String name) { for (RpcRequest type : values()) { src/com/gitblit/GitBlit.java
@@ -39,10 +39,12 @@ import java.util.TimeZone; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import javax.mail.Message; import javax.mail.MessagingException; @@ -125,6 +127,10 @@ private final ObjectCache<Long> repositorySizeCache = new ObjectCache<Long>(); private final ObjectCache<List<Metric>> repositoryMetricsCache = new ObjectCache<List<Metric>>(); private final List<String> repositoryListCache = new CopyOnWriteArrayList<String>(); private final AtomicReference<String> repositoryListSettingsChecksum = new AtomicReference<String>(""); private RepositoryResolver<Void> repositoryResolver; @@ -725,13 +731,71 @@ } /** * Adds the repository to the list of cached repositories if Gitblit is * configured to cache the repository list. * * @param name */ private void addToCachedRepositoryList(String name) { if (settings.getBoolean(Keys.git.cacheRepositoryList, true)) { repositoryListCache.add(name); } } /** * Clears all the cached data for the specified repository. * * @param repositoryName * @param isDeleted */ public void clearRepositoryCache(String repositoryName) { private void clearRepositoryCache(String repositoryName, boolean isDeleted) { repositorySizeCache.remove(repositoryName); repositoryMetricsCache.remove(repositoryName); if (isDeleted) { repositoryListCache.remove(repositoryName); } } /** * Resets the repository list cache. * */ public void resetRepositoryListCache() { logger.info("Repository cache manually reset"); repositoryListCache.clear(); } /** * Calculate the checksum of settings that affect the repository list cache. * @return a checksum */ private String getRepositoryListSettingsChecksum() { StringBuilder ns = new StringBuilder(); ns.append(settings.getString(Keys.git.cacheRepositoryList, "")).append('\n'); ns.append(settings.getString(Keys.git.onlyAccessBareRepositories, "")).append('\n'); ns.append(settings.getString(Keys.git.searchRepositoriesSubfolders, "")).append('\n'); ns.append(settings.getString(Keys.git.searchRecursionDepth, "")).append('\n'); ns.append(settings.getString(Keys.git.searchExclusions, "")).append('\n'); String checksum = StringUtils.getSHA1(ns.toString()); return checksum; } /** * Compare the last repository list setting checksum to the current checksum. * If different then clear the cache so that it may be rebuilt. * * @return true if the cached repository list is valid since the last check */ private boolean isValidRepositoryList() { String newChecksum = getRepositoryListSettingsChecksum(); boolean valid = newChecksum.equals(repositoryListSettingsChecksum.get()); repositoryListSettingsChecksum.set(newChecksum); if (!valid && settings.getBoolean(Keys.git.cacheRepositoryList, true)) { logger.info("Repository list settings have changed. Clearing repository list cache."); repositoryListCache.clear(); } return valid; } /** @@ -741,11 +805,46 @@ * @return list of all repositories */ public List<String> getRepositoryList() { return JGitUtils.getRepositoryList(repositoriesFolder, if (repositoryListCache.size() == 0 || !isValidRepositoryList()) { // we are not caching OR we have not yet cached OR the cached list is invalid long startTime = System.currentTimeMillis(); List<String> repositories = JGitUtils.getRepositoryList(repositoriesFolder, settings.getBoolean(Keys.git.onlyAccessBareRepositories, false), settings.getBoolean(Keys.git.searchRepositoriesSubfolders, true), settings.getInteger(Keys.git.searchRecursionDepth, -1), settings.getStrings(Keys.git.searchExclusions)); if (!settings.getBoolean(Keys.git.cacheRepositoryList, true)) { // we are not caching StringUtils.sortRepositorynames(repositories); return repositories; } else { // we are caching this list String msg = "{0} repositories identified in {1} msecs"; // optionally (re)calculate repository sizes if (getBoolean(Keys.web.showRepositorySizes, true)) { msg = "{0} repositories identified with calculated folder sizes in {1} msecs"; for (String repository : repositories) { RepositoryModel model = getRepositoryModel(repository); if (!model.skipSizeCalculation) { calculateSize(model); } } } // update cache repositoryListCache.addAll(repositories); long duration = System.currentTimeMillis() - startTime; logger.info(MessageFormat.format(msg, repositoryListCache.size(), duration)); } } // return sorted copy of cached list List<String> list = new ArrayList<String>(repositoryListCache); StringUtils.sortRepositorynames(list); return list; } /** @@ -804,6 +903,7 @@ * @return list of repository models accessible to user */ public List<RepositoryModel> getRepositoryModels(UserModel user) { long methodStart = System.currentTimeMillis(); List<String> list = getRepositoryList(); List<RepositoryModel> repositories = new ArrayList<RepositoryModel>(); for (String repo : list) { @@ -823,9 +923,15 @@ } } long duration = System.currentTimeMillis() - startTime; if (duration > 250) { // only log calcualtion time if > 250 msecs logger.info(MessageFormat.format("{0} repository sizes calculated in {1} msecs", repoCount, duration)); } } long duration = System.currentTimeMillis() - methodStart; logger.info(MessageFormat.format("{0} repository models loaded for {1} in {2} msecs", repositories.size(), user.username, duration)); return repositories; } @@ -869,7 +975,8 @@ model.hasCommits = JGitUtils.hasCommits(r); model.lastChange = JGitUtils.getLastChange(r); model.isBare = r.isBare(); StoredConfig config = JGitUtils.readConfig(r); StoredConfig config = r.getConfig(); if (config != null) { model.description = getConfig(config, "description", ""); model.owner = getConfig(config, "owner", ""); @@ -1062,6 +1169,9 @@ // create repository logger.info("create repository " + repository.name); r = JGitUtils.createRepository(repositoriesFolder, repository.name); // add name to cache addToCachedRepositoryList(repository.name); } else { // rename repository if (!repositoryName.equalsIgnoreCase(repository.name)) { @@ -1101,7 +1211,10 @@ } // clear the cache clearRepositoryCache(repositoryName); clearRepositoryCache(repositoryName, true); // add new name to repository list cache addToCachedRepositoryList(repository.name); } // load repository @@ -1129,7 +1242,7 @@ repository.name, currentRef, repository.HEAD)); if (JGitUtils.setHEADtoRef(r, repository.HEAD)) { // clear the cache clearRepositoryCache(repository.name); clearRepositoryCache(repository.name, false); } } @@ -1147,7 +1260,7 @@ * the Gitblit repository model */ public void updateConfiguration(Repository r, RepositoryModel repository) { StoredConfig config = JGitUtils.readConfig(r); StoredConfig config = r.getConfig(); config.setString(Constants.CONFIG_GITBLIT, null, "description", repository.description); config.setString(Constants.CONFIG_GITBLIT, null, "owner", repository.owner); config.setBoolean(Constants.CONFIG_GITBLIT, null, "useTickets", repository.useTickets); @@ -1225,6 +1338,9 @@ public boolean deleteRepository(String repositoryName) { try { closeRepository(repositoryName); // clear the repository cache clearRepositoryCache(repositoryName, true); File folder = new File(repositoriesFolder, repositoryName); if (folder.exists() && folder.isDirectory()) { FileUtils.delete(folder, FileUtils.RECURSIVE | FileUtils.RETRY); @@ -1232,9 +1348,6 @@ return true; } } // clear the repository cache clearRepositoryCache(repositoryName); } catch (Throwable t) { logger.error(MessageFormat.format("Failed to delete repository {0}", repositoryName), t); } @@ -1954,6 +2067,15 @@ logger.info("Git repositories folder " + repositoriesFolder.getAbsolutePath()); repositoryResolver = new FileResolver<Void>(repositoriesFolder, true); // calculate repository list settings checksum for future config changes repositoryListSettingsChecksum.set(getRepositoryListSettingsChecksum()); // build initial repository list if (settings.getBoolean(Keys.git.cacheRepositoryList, true)) { logger.info("Identifying available repositories..."); getRepositoryList(); } logTimezone("JVM", TimeZone.getDefault()); logTimezone(Constants.NAME, getTimezone());
@@ -49,7 +49,7 @@ private static final long serialVersionUID = 1L; public static final int PROTOCOL_VERSION = 3; public static final int PROTOCOL_VERSION = 4; public RpcServlet() { super(); @@ -319,6 +319,13 @@ } else { response.sendError(notAllowedCode); } } else if (RpcRequest.CLEAR_REPOSITORY_CACHE.equals(reqType)) { // clear the repository list cache if (allowAdmin) { GitBlit.self().resetRepositoryListCache(); } else { response.sendError(notAllowedCode); } } // send the result of the request src/com/gitblit/client/GitblitClient.java
@@ -578,6 +578,10 @@ public boolean deleteRepository(RepositoryModel repository) throws IOException { return RpcUtils.deleteRepository(repository, url, account, password); } public boolean clearRepositoryCache() throws IOException { return RpcUtils.clearRepositoryCache(url, account, password); } public boolean createUser(UserModel user) throws IOException { return RpcUtils.createUser(user, url, account, password); src/com/gitblit/client/RepositoriesPanel.java
@@ -82,6 +82,8 @@ private JTextField filterTextfield; private JButton clearCache; public RepositoriesPanel(GitblitClient gitblit) { super(); this.gitblit = gitblit; @@ -103,6 +105,13 @@ refreshRepositories.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent e) { refreshRepositories(); } }); clearCache = new JButton(Translation.get("gb.clearCache")); clearCache.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent e) { clearCache(); } }); @@ -242,6 +251,7 @@ repositoryTablePanel.add(new JScrollPane(table), BorderLayout.CENTER); JPanel repositoryControls = new JPanel(new FlowLayout(FlowLayout.CENTER, Utils.MARGIN, 0)); repositoryControls.add(clearCache); repositoryControls.add(refreshRepositories); repositoryControls.add(browseRepository); repositoryControls.add(createRepository); @@ -285,6 +295,7 @@ protected abstract void updateTeamsTable(); protected void disableManagement() { clearCache.setVisible(false); createRepository.setVisible(false); editRepository.setVisible(false); delRepository.setVisible(false); @@ -349,6 +360,26 @@ worker.execute(); } protected void clearCache() { GitblitWorker worker = new GitblitWorker(RepositoriesPanel.this, RpcRequest.CLEAR_REPOSITORY_CACHE) { @Override protected Boolean doRequest() throws IOException { if (gitblit.clearRepositoryCache()) { gitblit.refreshRepositories(); return true; } return false; } @Override protected void onSuccess() { updateTable(false); } }; worker.execute(); } /** * Displays the create repository dialog and fires a SwingWorker to update * the server, if appropriate. src/com/gitblit/utils/JGitUtils.java
@@ -47,7 +47,6 @@ import org.eclipse.jgit.errors.StopWalkException; import org.eclipse.jgit.lib.BlobBasedConfig; import org.eclipse.jgit.lib.CommitBuilder; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; @@ -59,7 +58,6 @@ import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.RepositoryCache.FileKey; import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.lib.TreeFormatter; import org.eclipse.jgit.revwalk.RevBlob; import org.eclipse.jgit.revwalk.RevCommit; @@ -1706,24 +1704,6 @@ error(t, repository, "Failed to create orphan branch {1} in repository {0}", branchName); } return success; } /** * Returns a StoredConfig object for the repository. * * @param repository * @return the StoredConfig of the repository */ public static StoredConfig readConfig(Repository repository) { StoredConfig c = repository.getConfig(); try { c.load(); } catch (ConfigInvalidException cex) { error(cex, repository, "{0} configuration is invalid!"); } catch (IOException cex) { error(cex, repository, "Could not open configuration for {0}!"); } return c; } /** src/com/gitblit/utils/ObjectCache.java
@@ -91,4 +91,8 @@ } return null; } public int size() { return cache.size(); } } src/com/gitblit/utils/RpcUtils.java
@@ -231,6 +231,21 @@ password); } /** * Clears the repository cache on the Gitblit server. * * @param serverUrl * @param account * @param password * @return true if the action succeeded * @throws IOException */ public static boolean clearRepositoryCache(String serverUrl, String account, char[] password) throws IOException { return doAction(RpcRequest.CLEAR_REPOSITORY_CACHE, null, null, serverUrl, account, password); } /** * Create a user on the Gitblit server. src/com/gitblit/wicket/GitBlitWebApp.properties
@@ -314,3 +314,4 @@ gb.allowAuthenticatedDescription = grant restricted access to all authenticated users gb.allowNamedDescription = grant restricted access to named users or teams gb.markdownFailure = Failed to parse Markdown content! gb.clearCache = clear cache src/com/gitblit/wicket/panels/RepositoriesPanel.html
@@ -21,8 +21,12 @@ <wicket:fragment wicket:id="adminLinks"> <!-- page nav links --> <div class="admin_nav"> <img style="vertical-align: middle;" src="add_16x16.png"/> <a wicket:id="newRepository"> <a class="btn-small" wicket:id="clearCache"> <i class="icon icon-remove"></i> <wicket:message key="gb.clearCache"></wicket:message> </a> <a class="btn-small" wicket:id="newRepository" style="padding-right:0px;"> <i class="icon icon-plus-sign"></i> <wicket:message key="gb.newRepository"></wicket:message> </a> </div> src/com/gitblit/wicket/panels/RepositoriesPanel.java
@@ -54,6 +54,7 @@ import com.gitblit.wicket.pages.BasePage; import com.gitblit.wicket.pages.EditRepositoryPage; import com.gitblit.wicket.pages.EmptyRepositoryPage; import com.gitblit.wicket.pages.RepositoriesPage; import com.gitblit.wicket.pages.SummaryPage; public class RepositoriesPanel extends BasePanel { @@ -73,6 +74,16 @@ final IDataProvider<RepositoryModel> dp; Fragment adminLinks = new Fragment("adminPanel", "adminLinks", this); adminLinks.add(new Link<Void>("clearCache") { private static final long serialVersionUID = 1L; @Override public void onClick() { GitBlit.self().resetRepositoryListCache(); setResponsePage(RepositoriesPage.class); } }.setVisible(GitBlit.getBoolean(Keys.git.cacheRepositoryList, true))); adminLinks.add(new BookmarkablePageLink<Void>("newRepository", EditRepositoryPage.class)); add(adminLinks.setVisible(showAdmin));
@@ -32,8 +32,8 @@ <wicket:fragment wicket:id="adminLinks"> <!-- page nav links --> <div class="admin_nav"> <img style="vertical-align: middle;" src="add_16x16.png"/> <a wicket:id="newTeam"> <a class="btn-small" wicket:id="newTeam" style="padding-right:0px;"> <i class="icon icon-plus-sign"></i> <wicket:message key="gb.newTeam"></wicket:message> </a> </div> src/com/gitblit/wicket/panels/UsersPanel.html
@@ -38,8 +38,8 @@ <wicket:fragment wicket:id="adminLinks"> <!-- page nav links --> <div class="admin_nav"> <img style="vertical-align: middle;" src="add_16x16.png"/> <a wicket:id="newUser"> <a class="btn-small" wicket:id="newUser" style="padding-right:0px;"> <i class="icon icon-plus-sign"></i> <wicket:message key="gb.newUser"></wicket:message> </a> </div> tests/com/gitblit/tests/RepositoryModelTest.java
@@ -29,7 +29,6 @@ import com.gitblit.Constants; import com.gitblit.GitBlit; import com.gitblit.models.RepositoryModel; import com.gitblit.utils.JGitUtils; public class RepositoryModelTest { @@ -49,7 +48,7 @@ @Before public void initializeConfiguration() throws Exception{ Repository r = GitBlitSuite.getHelloworldRepository(); StoredConfig config = JGitUtils.readConfig(r); StoredConfig config = r.getConfig(); config.unsetSection(Constants.CONFIG_GITBLIT, Constants.CONFIG_CUSTOM_FIELDS); config.setString(Constants.CONFIG_GITBLIT, Constants.CONFIG_CUSTOM_FIELDS, "commitMessageRegEx", "\\d"); @@ -61,7 +60,7 @@ @After public void teardownConfiguration() throws Exception { Repository r = GitBlitSuite.getHelloworldRepository(); StoredConfig config = JGitUtils.readConfig(r); StoredConfig config = r.getConfig(); config.unsetSection(Constants.CONFIG_GITBLIT, Constants.CONFIG_CUSTOM_FIELDS); config.save();
