From cdf77badd798861953ab11cad3ad4641f1b406d2 Mon Sep 17 00:00:00 2001
From: James Moger <james.moger@gitblit.com>
Date: Sat, 18 Aug 2012 15:36:11 -0400
Subject: [PATCH] Cache repository models to improve performance (issue 103)
---
src/com/gitblit/GitBlit.java | 137 +++++++++++++++++++++++++++++++++++++--------
1 files changed, 113 insertions(+), 24 deletions(-)
diff --git a/src/com/gitblit/GitBlit.java b/src/com/gitblit/GitBlit.java
index 0940c76..f8fbfef 100644
--- a/src/com/gitblit/GitBlit.java
+++ b/src/com/gitblit/GitBlit.java
@@ -39,7 +39,6 @@
import java.util.TimeZone;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -58,6 +57,7 @@
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.RepositoryCache.FileKey;
import org.eclipse.jgit.lib.StoredConfig;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.storage.file.WindowCache;
import org.eclipse.jgit.storage.file.WindowCacheConfig;
import org.eclipse.jgit.transport.ServiceMayNotContinueException;
@@ -88,6 +88,7 @@
import com.gitblit.models.UserModel;
import com.gitblit.utils.ArrayUtils;
import com.gitblit.utils.ByteFormat;
+import com.gitblit.utils.DeepCopier;
import com.gitblit.utils.FederationUtils;
import com.gitblit.utils.JGitUtils;
import com.gitblit.utils.JsonUtils;
@@ -128,7 +129,7 @@
private final ObjectCache<List<Metric>> repositoryMetricsCache = new ObjectCache<List<Metric>>();
- private final List<String> repositoryListCache = new CopyOnWriteArrayList<String>();
+ private final Map<String, RepositoryModel> repositoryListCache = new ConcurrentHashMap<String, RepositoryModel>();
private final AtomicReference<String> repositoryListSettingsChecksum = new AtomicReference<String>("");
@@ -736,25 +737,32 @@
*
* @param name
*/
- private void addToCachedRepositoryList(String name) {
+ private void addToCachedRepositoryList(String name, RepositoryModel model) {
if (settings.getBoolean(Keys.git.cacheRepositoryList, true)) {
- repositoryListCache.add(name);
+ repositoryListCache.put(name, model);
}
+ }
+
+ /**
+ * Removes the repository from the list of cached repositories.
+ *
+ * @param name
+ */
+ private void removeFromCachedRepositoryList(String name) {
+ if (StringUtils.isEmpty(name)) {
+ return;
+ }
+ repositoryListCache.remove(name);
}
/**
- * Clears all the cached data for the specified repository.
+ * Clears all the cached metadata for the specified repository.
*
* @param repositoryName
- * @param isDeleted
*/
- private void clearRepositoryCache(String repositoryName, boolean isDeleted) {
+ private void clearRepositoryMetadataCache(String repositoryName) {
repositorySizeCache.remove(repositoryName);
repositoryMetricsCache.remove(repositoryName);
-
- if (isDeleted) {
- repositoryListCache.remove(repositoryName);
- }
}
/**
@@ -831,10 +839,12 @@
calculateSize(model);
}
}
+ } else {
+ // update cache
+ for (String repository : repositories) {
+ getRepositoryModel(repository);
+ }
}
-
- // update cache
- repositoryListCache.addAll(repositories);
long duration = System.currentTimeMillis() - startTime;
logger.info(MessageFormat.format(msg, repositoryListCache.size(), duration));
@@ -842,7 +852,7 @@
}
// return sorted copy of cached list
- List<String> list = new ArrayList<String>(repositoryListCache);
+ List<String> list = new ArrayList<String>(repositoryListCache.keySet());
StringUtils.sortRepositorynames(list);
return list;
}
@@ -966,6 +976,79 @@
* @return repository model or null
*/
public RepositoryModel getRepositoryModel(String repositoryName) {
+ if (!repositoryListCache.containsKey(repositoryName)) {
+ RepositoryModel model = loadRepositoryModel(repositoryName);
+ if (model == null) {
+ return null;
+ }
+ addToCachedRepositoryList(repositoryName, model);
+ return model;
+ }
+
+ // cached model
+ RepositoryModel model = repositoryListCache.get(repositoryName);
+
+ // check for updates
+ Repository r = getRepository(repositoryName);
+ if (r == null) {
+ // repository is missing
+ removeFromCachedRepositoryList(repositoryName);
+ logger.error(MessageFormat.format("Repository \"{0}\" is missing! Removing from cache.", repositoryName));
+ return null;
+ }
+
+ FileBasedConfig config = (FileBasedConfig) getRepositoryConfig(r);
+ if (config.isOutdated()) {
+ // reload model
+ logger.info(MessageFormat.format("Config for \"{0}\" has changed. Reloading model and updating cache.", repositoryName));
+ model = loadRepositoryModel(repositoryName);
+ removeFromCachedRepositoryList(repositoryName);
+ addToCachedRepositoryList(repositoryName, model);
+ } else {
+ // update a few repository parameters
+ if (!model.hasCommits) {
+ // update hasCommits, assume a repository only gains commits :)
+ model.hasCommits = JGitUtils.hasCommits(r);
+ }
+
+ model.lastChange = JGitUtils.getLastChange(r);
+ }
+ r.close();
+
+ // return a copy of the cached model
+ return DeepCopier.copy(model);
+ }
+
+ /**
+ * Workaround JGit. I need to access the raw config object directly in order
+ * to see if the config is dirty so that I can reload a repository model.
+ * If I use the stock JGit method to get the config it already reloads the
+ * config. If the config changes are made within Gitblit this is fine as
+ * the returned config will still be flagged as dirty. BUT... if the config
+ * is manipulated outside Gitblit then it fails to recognize this as dirty.
+ *
+ * @param r
+ * @return a config
+ */
+ private StoredConfig getRepositoryConfig(Repository r) {
+ try {
+ Field f = r.getClass().getDeclaredField("repoConfig");
+ f.setAccessible(true);
+ StoredConfig config = (StoredConfig) f.get(r);
+ return config;
+ } catch (Exception e) {
+ logger.error("Failed to retrieve \"repoConfig\" via reflection", e);
+ }
+ return r.getConfig();
+ }
+
+ /**
+ * Create a repository model from the configuration and repository data.
+ *
+ * @param repositoryName
+ * @return a repositoryModel or null if the repository does not exist
+ */
+ private RepositoryModel loadRepositoryModel(String repositoryName) {
Repository r = getRepository(repositoryName);
if (r == null) {
return null;
@@ -1025,6 +1108,11 @@
* @return true if the repository exists
*/
public boolean hasRepository(String repositoryName) {
+ if (settings.getBoolean(Keys.git.cacheRepositoryList, true)) {
+ // if we are caching use the cache to determine availability
+ // otherwise we end up adding a phantom repository to the cache
+ return repositoryListCache.containsKey(repositoryName);
+ }
Repository r = getRepository(repositoryName, false);
if (r == null) {
return false;
@@ -1169,9 +1257,6 @@
// create repository
logger.info("create repository " + repository.name);
r = JGitUtils.createRepository(repositoriesFolder, repository.name);
-
- // add name to cache
- addToCachedRepositoryList(repository.name);
} else {
// rename repository
if (!repositoryName.equalsIgnoreCase(repository.name)) {
@@ -1211,10 +1296,7 @@
}
// clear the cache
- clearRepositoryCache(repositoryName, true);
-
- // add new name to repository list cache
- addToCachedRepositoryList(repository.name);
+ clearRepositoryMetadataCache(repositoryName);
}
// load repository
@@ -1242,13 +1324,18 @@
repository.name, currentRef, repository.HEAD));
if (JGitUtils.setHEADtoRef(r, repository.HEAD)) {
// clear the cache
- clearRepositoryCache(repository.name, false);
+ clearRepositoryMetadataCache(repository.name);
}
}
// close the repository object
r.close();
}
+
+ // update repository cache
+ removeFromCachedRepositoryList(repositoryName);
+ // model will actually be replaced on next load because config is stale
+ addToCachedRepositoryList(repository.name, repository);
}
/**
@@ -1339,12 +1426,14 @@
try {
closeRepository(repositoryName);
// clear the repository cache
- clearRepositoryCache(repositoryName, true);
+ clearRepositoryMetadataCache(repositoryName);
+ removeFromCachedRepositoryList(repositoryName);
File folder = new File(repositoriesFolder, repositoryName);
if (folder.exists() && folder.isDirectory()) {
FileUtils.delete(folder, FileUtils.RECURSIVE | FileUtils.RETRY);
if (userService.deleteRepositoryRole(repositoryName)) {
+ logger.info(MessageFormat.format("Repository \"{0}\" deleted", repositoryName));
return true;
}
}
--
Gitblit v1.9.1