Implemented commit cache for the dashboards, activity, and project pages

James Moger

2013-06-24 79dd0bddbd939c85f002f3a33b95ae84d0bf38cb

Implemented commit cache for the dashboards, activity, and project pages

 releases.moxie

@@ -56,6 +56,7 @@
     - Setting to automatically create an user account based on an authenticated user principal from the servlet container (issue-246)

     - Added WindowsUserService to authenticate users against Windows accounts (issue-250)

     - Global and per-repository setting to exclude authors from metrics (issue-251)

     - Added commit cache to improve Activity, Dashboard, and Project page generation times

     - Added SalesForce.com user service

     - Added simple star/unstar function to flag or bookmark interesting repositories

     - Added Dashboard page which shows a news feed for starred repositories and offers a filterable list of repositories you care about

@@ -126,7 +127,8 @@
    - { name: 'realm.salesforce.orgId', defaultValue: 0 }

    - { name: 'realm.windows.defaultDomain', defaultValue: ' ' }

    - { name: 'realm.windows.backingUserService', defaultValue: 'users.conf' }

    - { name: 'web.activityDurationChoices', defaultValue: '7 14 28 60 90 180' }

    - { name: 'web.activityDurationChoices', defaultValue: '7 14 21 28' }

    - { name: 'web.activityCacheDays', defaultValue: 14 }

    - { name: 'web.allowAppCloneLinks', defaultValue: 'true' }

    - { name: 'web.forceDefaultLocale', defaultValue: ' ' }

    - { name: 'web.metricAuthorExclusions', defaultValue: ' ' }


 src/main/distrib/data/gitblit.properties

@@ -825,7 +825,21 @@
#

# SPACE-DELIMITED

# SINCE 1.3.0

web.activityDurationChoices = 7 14 28 60 90 180

web.activityDurationChoices = 7 14 21 28



# The number of days of commits to cache in memory for the dashboard, activity,

# and project pages.  A value of 0 will disable all caching and will parse commits

# in each repository per-request.  If the value > 0 these pages will try to fulfill

# requests using the commit cache.  If the request specifies a period which falls

# outside the commit cache window, then the cache will be ignored and the request

# will be fulfilled by brute-force parsing all relevant commits per-repository.

#

# Consider the values specified for *web.activityDurationChoices* when setting

# the cache size AND consider adjusting the JVM -Xmx heap parameter appropriately.

#

# SINCE 1.3.0

# RESTART REQUIRED

web.activityCacheDays = 14



# Case-insensitive list of authors to exclude from metrics.  Useful for

# eliminating bots.


 src/main/java/com/gitblit/GitBlit.java

@@ -114,6 +114,7 @@
import com.gitblit.utils.ArrayUtils;
import com.gitblit.utils.Base64;
import com.gitblit.utils.ByteFormat;
import com.gitblit.utils.CommitCache;
import com.gitblit.utils.ContainerUtils;
import com.gitblit.utils.DeepCopier;
import com.gitblit.utils.FederationUtils;
@@ -3402,6 +3403,8 @@
        configureFanout();
        configureGitDaemon();
        
        CommitCache.instance().setCacheDays(settings.getInteger(Keys.web.activityCacheDays, 14));
		
        ContainerUtils.CVE_2007_0450.test();
    }
    

 src/main/java/com/gitblit/models/Activity.java

@@ -103,14 +103,28 @@
     */

    public RepositoryCommit addCommit(String repository, String branch, RevCommit commit) {

        RepositoryCommit commitModel = new RepositoryCommit(repository, branch, commit);

        return addCommit(commitModel);

    }



    /**

     * Adds a commit to the activity object as long as the commit is not a

     * duplicate.

     * 
     * @param repository

     * @param branch

     * @param commit

     * @return a RepositoryCommit, if one was added. Null if this is duplicate

     *         commit

     */

    public RepositoryCommit addCommit(RepositoryCommit commitModel) {

        if (commits.add(commitModel)) {

            String author = StringUtils.removeNewlines(commit.getAuthorIdent().getName());

            String author = StringUtils.removeNewlines(commitModel.getAuthorIdent().getName());

            String authorName = author.toLowerCase();

            String authorEmail = StringUtils.removeNewlines(commit.getAuthorIdent().getEmailAddress()).toLowerCase();

            if (!repositoryMetrics.containsKey(repository)) {

                repositoryMetrics.put(repository, new Metric(repository));

            String authorEmail = StringUtils.removeNewlines(commitModel.getAuthorIdent().getEmailAddress()).toLowerCase();

            if (!repositoryMetrics.containsKey(commitModel.repository)) {

                repositoryMetrics.put(commitModel.repository, new Metric(commitModel.repository));

            }

            repositoryMetrics.get(repository).count++;

            repositoryMetrics.get(commitModel.repository).count++;



            if (!authorExclusions.contains(authorName) && !authorExclusions.contains(authorEmail)) {

                if (!authorMetrics.containsKey(author)) {

@@ -122,7 +136,7 @@
        }

        return null;

    }

	


    public int getCommitCount() {

        return commits.size();

    }


 src/main/java/com/gitblit/models/RefLogEntry.java

@@ -160,7 +160,24 @@
        }

        return null;

    }

	


    /**

     * Adds a commit to the push entry object as long as the commit is not a

     * duplicate.

     * 
     * @param branch

     * @param commit

     * @return a RepositoryCommit, if one was added. Null if this is duplicate

     *         commit

     */

    public RepositoryCommit addCommit(RepositoryCommit commit) {

        if (commits.add(commit)) {

            authorCount = -1;

            return commit;

        }

        return null;

    }



    /**

     * Adds a a list of repository commits.  This is used to construct discrete

     * ref push log entries


 src/main/java/com/gitblit/models/RepositoryCommit.java

@@ -17,8 +17,10 @@

import java.io.Serializable;
import java.text.MessageFormat;
import java.util.Date;
import java.util.List;

import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.revwalk.RevCommit;

@@ -54,6 +56,10 @@
        return refs;
    }

    public ObjectId getId() {
        return commit.getId();
    }

    public String getName() {
        return commit.getName();
    }
@@ -65,9 +71,17 @@
    public String getShortMessage() {
        return commit.getShortMessage();
    }
	
    public Date getCommitDate() {
        return new Date(commit.getCommitTime() * 1000L);
    }

    public int getParentCount() {
        return commit.getParentCount();
    }
	
    public RevCommit [] getParents() {
        return commit.getParents();
    }

    public PersonIdent getAuthorIdent() {
@@ -77,7 +91,7 @@
    public PersonIdent getCommitterIdent() {
        return commit.getCommitterIdent();
    }

	
    @Override
    public boolean equals(Object o) {
        if (o instanceof RepositoryCommit) {
@@ -103,6 +117,10 @@
        return 0;
    }
    
    public RepositoryCommit clone(String withRef) {
        return new RepositoryCommit(repository, withRef, commit);
    }
	
    @Override
    public String toString() {
        return MessageFormat.format("{0} {1} {2,date,yyyy-MM-dd HH:mm} {3} {4}", 

 src/main/java/com/gitblit/utils/ActivityUtils.java

@@ -32,9 +32,7 @@
import java.util.TreeSet;



import org.eclipse.jgit.lib.Constants;

import org.eclipse.jgit.lib.ObjectId;

import org.eclipse.jgit.lib.Repository;

import org.eclipse.jgit.revwalk.RevCommit;



import com.gitblit.GitBlit;

import com.gitblit.Keys;

@@ -112,22 +110,19 @@
                } else {

                    branches.add(objectId);

                }

                Map<ObjectId, List<RefModel>> allRefs = JGitUtils

                        .getAllRefs(repository, model.showRemoteBranches);



                for (String branch : branches) {

                    String shortName = branch;

                    if (shortName.startsWith(Constants.R_HEADS)) {

                        shortName = shortName.substring(Constants.R_HEADS.length());

                    }

                    List<RevCommit> commits = JGitUtils.getRevLog(repository,

                            branch, thresholdDate);

                    List<RepositoryCommit> commits = CommitCache.instance().getCommits(model.name, repository, branch, thresholdDate);

                    if (model.maxActivityCommits > 0 && commits.size() > model.maxActivityCommits) {

                        // trim commits to maximum count

                        commits = commits.subList(0,  model.maxActivityCommits);

                    }

                    for (RevCommit commit : commits) {						
                        Date date = JGitUtils.getCommitDate(commit);

                    for (RepositoryCommit commit : commits) {						
                        Date date = commit.getCommitDate();

                        String dateStr = df.format(date);

                        if (!activity.containsKey(dateStr)) {

                            // Normalize the date to midnight

@@ -140,11 +135,7 @@
                            a.excludeAuthors(authorExclusions);

                            activity.put(dateStr, a);

                        }

                        RepositoryCommit commitModel = activity.get(dateStr)

                                .addCommit(model.name, shortName, commit);

                        if (commitModel != null) {

                            commitModel.setRefs(allRefs.get(commit.getId()));

                        }

                        activity.get(dateStr).addCommit(commit);

                    }

                }

                


 src/main/java/com/gitblit/utils/CommitCache.java

New file
@@ -0,0 +1,253 @@
/*
 * Copyright 2013 gitblit.com.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.gitblit.utils;

import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;

import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.gitblit.models.RefModel;
import com.gitblit.models.RepositoryCommit;

/**
 * Caches repository commits for re-use in the dashboard and activity pages.
 * 
 * @author James Moger
 *
 */
public class CommitCache {
	
    private static final CommitCache instance;
	
    protected final Logger logger = LoggerFactory.getLogger(getClass());
	
    protected final Map<String, ObjectCache<List<RepositoryCommit>>> cache;
	
    protected int cacheDays = -1;
	
    public static CommitCache instance() {
        return instance;
    }
	
    static {
        instance = new CommitCache();
    }
	
    protected CommitCache() {
        cache = new ConcurrentHashMap<String, ObjectCache<List<RepositoryCommit>>>();
    }
	
    /**
     * Returns the cutoff date for the cache.  Commits after this date are cached.
     * Commits before this date are not cached.
     * 
     * @return
     */
    protected Date getCacheCutoffDate() {
        final Calendar cal = Calendar.getInstance();
        cal.setTimeInMillis(System.currentTimeMillis());
        cal.set(Calendar.HOUR_OF_DAY, 0);
        cal.set(Calendar.MINUTE, 0);
        cal.set(Calendar.SECOND, 0);
        cal.set(Calendar.MILLISECOND, 0);
        cal.add(Calendar.DATE, -1*cacheDays);
        return cal.getTime();
    }
	
    /**
     * Sets the number of days to cache.
     * 
     * @param days
     */
    public synchronized void setCacheDays(int days) {
        this.cacheDays = days;
        clear();
    }
	
    /**
     * Clears the entire commit cache.
     * 
     */
    public void clear() {
        cache.clear();
    }
	
    /**
     * Clears the commit cache for a specific repository.
     * 
     * @param repositoryName
     */
    public void clear(String repositoryName) {
        String repoKey = repositoryName.toLowerCase();
        ObjectCache<List<RepositoryCommit>> repoCache = cache.remove(repoKey);
        if (repoCache != null) {
            logger.info(MessageFormat.format("{0} commit cache cleared", repositoryName));
        }
    }
	
    /**
     * Get all commits for the specified repository:branch that are in the cache.
     * 
     * @param repositoryName
     * @param repository
     * @param branch
     * @return a list of commits
     */
    public List<RepositoryCommit> getCommits(String repositoryName, Repository repository, String branch) {
        return getCommits(repositoryName, repository, branch, getCacheCutoffDate());
    }
	
    /**
     * Get all commits for the specified repository:branch since a specific date.
     * These commits may be retrieved from the cache if the sinceDate is after
     * the cacheCutoffDate.
     * 
     * @param repositoryName
     * @param repository
     * @param branch
     * @param sinceDate
     * @return a list of commits
     */
    public List<RepositoryCommit> getCommits(String repositoryName, Repository repository, String branch, Date sinceDate) {
        long start = System.nanoTime();
        Date cacheCutoffDate = getCacheCutoffDate();
        List<RepositoryCommit> list;
        if (cacheDays > 0 && (sinceDate.getTime() >= cacheCutoffDate.getTime())) {
            // request fits within the cache window
            String repoKey = repositoryName.toLowerCase();
            if (!cache.containsKey(repoKey)) {
                cache.put(repoKey, new ObjectCache<List<RepositoryCommit>>());
            }
			
            ObjectCache<List<RepositoryCommit>> repoCache = cache.get(repoKey);
            String branchKey = branch.toLowerCase();
			
            RevCommit tip = JGitUtils.getCommit(repository, branch);
            Date tipDate = JGitUtils.getCommitDate(tip);
			
            List<RepositoryCommit> commits;
            if (!repoCache.hasCurrent(branchKey, tipDate)) {
                commits = repoCache.getObject(branchKey);
                if (ArrayUtils.isEmpty(commits)) {
                    // we don't have any cached commits for this branch, reload
                    commits = get(repositoryName, repository, branch, cacheCutoffDate);
                    repoCache.updateObject(branchKey, tipDate, commits);
                    logger.debug(MessageFormat.format("parsed {0} commits from {1}:{2} since {3,date,yyyy-MM-dd} in {4} msecs",
                            commits.size(), repositoryName, branch, cacheCutoffDate, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
                } else {
                    // incrementally update cache since the last cached commit
                    ObjectId sinceCommit = commits.get(0).getId();
                    List<RepositoryCommit> incremental = get(repositoryName, repository, branch, sinceCommit);
                    logger.info(MessageFormat.format("incrementally added {0} commits to cache for {1}:{2} in {3} msecs",
                            incremental.size(), repositoryName, branch, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
                    incremental.addAll(commits);
                    repoCache.updateObject(branchKey, tipDate, incremental);
                    commits = incremental;
                }
            } else {
                // cache is current
                commits = repoCache.getObject(branchKey);
                // evict older commits outside the cache window
                commits = reduce(commits, cacheCutoffDate);
                // update cache
                repoCache.updateObject(branchKey, tipDate, commits);
            }
			
            if (sinceDate.equals(cacheCutoffDate)) {
                list = commits;
            } else {
                // reduce the commits to those since the specified date
                list = reduce(commits, sinceDate);
            }
            logger.debug(MessageFormat.format("retrieved {0} commits from cache of {1}:{2} since {3,date,yyyy-MM-dd} in {4} msecs",
                    list.size(), repositoryName, branch, sinceDate, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
        } else {
            // not caching or request outside cache window
            list = get(repositoryName, repository, branch, sinceDate);
            logger.debug(MessageFormat.format("parsed {0} commits from {1}:{2} since {3,date,yyyy-MM-dd} in {4} msecs",
                    list.size(), repositoryName, branch, sinceDate, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
        }
        return list;
    }
	
    /**
     * Returns a list of commits for the specified repository branch. 
     * 
     * @param repositoryName
     * @param repository
     * @param branch
     * @param sinceDate
     * @return a list of commits
     */
    protected List<RepositoryCommit> get(String repositoryName, Repository repository, String branch, Date sinceDate) {
        Map<ObjectId, List<RefModel>> allRefs = JGitUtils.getAllRefs(repository, false);
        List<RepositoryCommit> commits = new ArrayList<RepositoryCommit>();
        for (RevCommit commit : JGitUtils.getRevLog(repository, branch, sinceDate)) {
            RepositoryCommit commitModel = new RepositoryCommit(repositoryName, branch, commit);
            commitModel.setRefs(allRefs.get(commitModel.getName()));
            commits.add(commitModel);
        }
        return commits;
    }
	
    /**
     * Returns a list of commits for the specified repository branch since the specified commit. 
     * 
     * @param repositoryName
     * @param repository
     * @param branch
     * @param sinceCommit
     * @return a list of commits
     */
    protected List<RepositoryCommit> get(String repositoryName, Repository repository, String branch, ObjectId sinceCommit) {
        Map<ObjectId, List<RefModel>> allRefs = JGitUtils.getAllRefs(repository, false);
        List<RepositoryCommit> commits = new ArrayList<RepositoryCommit>();
        for (RevCommit commit : JGitUtils.getRevLog(repository, sinceCommit.getName(), branch)) {
            RepositoryCommit commitModel = new RepositoryCommit(repositoryName, branch, commit);
            commitModel.setRefs(allRefs.get(commitModel.getName()));
            commits.add(commitModel);
        }
        return commits;
    }
	
    /**
     * Reduces the list of commits to those since the specified date.
     * 
     * @param commits
     * @param sinceDate
     * @return  a list of commits
     */
    protected List<RepositoryCommit> reduce(List<RepositoryCommit> commits, Date sinceDate) {
        List<RepositoryCommit> filtered = new ArrayList<RepositoryCommit>();
        for (RepositoryCommit commit : commits) {
            if (commit.getCommitDate().compareTo(sinceDate) >= 0) {
                filtered.add(commit);
            }
        }
        return filtered;
    }
}

 src/main/java/com/gitblit/utils/ObjectCache.java

@@ -85,7 +85,7 @@
        obj.object = object;

    }



    public Object remove(String name) {

    public X remove(String name) {

        if (cache.containsKey(name)) {

            return cache.remove(name).object;

        }


 src/main/java/com/gitblit/utils/RefLogUtils.java

@@ -549,15 +549,15 @@
        String linearParent = null;
        for (RefModel local : JGitUtils.getLocalBranches(repository, true, -1)) {
            String branch = local.getName();
            List<RevCommit> commits = JGitUtils.getRevLog(repository, branch, minimumDate);
            for (RevCommit commit : commits) {
            List<RepositoryCommit> commits = CommitCache.instance().getCommits(repositoryName, repository,  branch, minimumDate);
            for (RepositoryCommit commit : commits) {
                if (linearParent != null) {
                    if (!commit.getName().equals(linearParent)) {
                        // only follow linear branch commits
                        continue;
                    }
                }
                Date date = JGitUtils.getCommitDate(commit);
                Date date = commit.getCommitDate();
                String dateStr = df.format(date);
                if (!dailydigests.containsKey(dateStr)) {
                    dailydigests.put(dateStr, new DailyLogEntry(repositoryName, date));
@@ -571,7 +571,7 @@
                    digest.updateRef(branch, ReceiveCommand.Type.UPDATE, linearParent, commit.getName());
                }
                
                RepositoryCommit repoCommit = digest.addCommit(branch, commit);
                RepositoryCommit repoCommit = digest.addCommit(commit);
                if (repoCommit != null) {
                    List<RefModel> matchedRefs = allRefs.get(commit.getId());
                    repoCommit.setRefs(matchedRefs);
@@ -587,7 +587,8 @@
                                }
                                RefLogEntry tagEntry = tags.get(dateStr);
                                tagEntry.updateRef(ref.getName(), ReceiveCommand.Type.CREATE);
                                tagEntry.addCommit(ref.getName(), commit);
                                RepositoryCommit rc = repoCommit.clone(ref.getName());
                                tagEntry.addCommit(rc);
                            } else if (ref.getName().startsWith(Constants.R_PULL)) {
                                // treat pull requests as special events in the log
                                if (!pulls.containsKey(dateStr)) {
@@ -597,7 +598,8 @@
                                }
                                RefLogEntry pullEntry = pulls.get(dateStr);
                                pullEntry.updateRef(ref.getName(), ReceiveCommand.Type.CREATE);
                                pullEntry.addCommit(ref.getName(), commit);
                                RepositoryCommit rc = repoCommit.clone(ref.getName());
                                pullEntry.addCommit(rc);
                            }
                        }
                    }

 src/main/java/com/gitblit/wicket/panels/ActivityPanel.java

@@ -22,6 +22,7 @@
import org.apache.wicket.markup.repeater.Item;

import org.apache.wicket.markup.repeater.data.DataView;

import org.apache.wicket.markup.repeater.data.ListDataProvider;

import org.eclipse.jgit.lib.Repository;



import com.gitblit.Constants;

import com.gitblit.GitBlit;

@@ -101,7 +102,7 @@
                        commitItem.add(repositoryLink);



                        // repository branch

                        LinkPanel branchLink = new LinkPanel("branch", "list", commit.branch,

                        LinkPanel branchLink = new LinkPanel("branch", "list", Repository.shortenRefName(commit.branch),

                                LogPage.class, WicketUtils.newObjectParameter(commit.repository,

                                        commit.branch), true);

                        WicketUtils.setCssStyle(branchLink, "color: #008000;");