Improve index page strategy in PagesServlet.

Yuhi Ishikura

2013-06-02 7eb9828ec6a31ad372c6a1dac7fa042bcdbd6818

 src/main/java/com/gitblit/utils/PushLogUtils.java

@@ -16,13 +16,18 @@
package com.gitblit.utils;

import java.io.IOException;
import java.text.DateFormat;
import java.text.MessageFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
import java.util.TreeSet;

import org.eclipse.jgit.api.errors.ConcurrentRefUpdateException;
@@ -32,7 +37,6 @@
import org.eclipse.jgit.dircache.DirCacheEntry;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.lib.CommitBuilder;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectInserter;
@@ -48,9 +52,12 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.gitblit.Constants;
import com.gitblit.models.DailyLogEntry;
import com.gitblit.models.PathModel.PathChangeModel;
import com.gitblit.models.PushLogEntry;
import com.gitblit.models.RefModel;
import com.gitblit.models.RepositoryCommit;
import com.gitblit.models.UserModel;

/**
@@ -106,6 +113,25 @@
      return null;
   }
   
   private static UserModel newUserModelFrom(PersonIdent ident) {
      String name = ident.getName();
      String username;
      String displayname;
      if (name.indexOf('/') > -1) {
         int slash = name.indexOf('/');
         displayname = name.substring(0, slash);
         username = name.substring(slash + 1);
      } else {
         displayname = name;
         username = ident.getEmailAddress();
      }
		
      UserModel user = new UserModel(username);
      user.displayName = displayname;
      user.emailAddress = ident.getEmailAddress();
      return user;
   }
	
   /**
    * Updates a push log.
    * 
@@ -132,15 +158,21 @@
            DirCache index = createIndex(repository, headId, commands);
            ObjectId indexTreeId = index.writeTree(odi);

            PersonIdent ident = 
                  new PersonIdent(MessageFormat.format("{0}/{1}", user.getDisplayName(), user.username),
            PersonIdent ident;
            if (UserModel.ANONYMOUS.equals(user)) {
               // anonymous push
               ident = new PersonIdent("anonymous", "anonymous");
            } else {
               // construct real pushing account
               ident =   new PersonIdent(MessageFormat.format("{0}/{1}", user.getDisplayName(), user.username),
                  user.emailAddress == null ? user.username : user.emailAddress);
            }

            // Create a commit object
            CommitBuilder commit = new CommitBuilder();
            commit.setAuthor(ident);
            commit.setCommitter(ident);
            commit.setEncoding(Constants.CHARACTER_ENCODING);
            commit.setEncoding(Constants.ENCODING);
            commit.setMessage(message);
            commit.setParentId(headId);
            commit.setTreeId(indexTreeId);
@@ -240,7 +272,7 @@
            dcEntry.setFileMode(FileMode.REGULAR_FILE);

            // insert object
            dcEntry.setObjectId(inserter.insert(Constants.OBJ_BLOB, content.getBytes("UTF-8")));
            dcEntry.setObjectId(inserter.insert(org.eclipse.jgit.lib.Constants.OBJ_BLOB, content.getBytes("UTF-8")));

            // add to temporary in-core index
            dcBuilder.add(dcEntry);
@@ -283,28 +315,50 @@
      }
      return inCoreIndex;
   }

	
   public static List<PushLogEntry> getPushLog(String repositoryName, Repository repository) {
      return getPushLog(repositoryName, repository, null, -1);
      return getPushLog(repositoryName, repository, null, 0, -1);
   }

   public static List<PushLogEntry> getPushLog(String repositoryName, Repository repository, int maxCount) {
      return getPushLog(repositoryName, repository, null, maxCount);
      return getPushLog(repositoryName, repository, null, 0, maxCount);
   }

   public static List<PushLogEntry> getPushLog(String repositoryName, Repository repository, int offset, int maxCount) {
      return getPushLog(repositoryName, repository, null, offset, maxCount);
   }

   public static List<PushLogEntry> getPushLog(String repositoryName, Repository repository, Date minimumDate) {
      return getPushLog(repositoryName, repository, minimumDate, -1);
      return getPushLog(repositoryName, repository, minimumDate, 0, -1);
   }
   
   public static List<PushLogEntry> getPushLog(String repositoryName, Repository repository, Date minimumDate, int maxCount) {
   /**
    * Returns the list of push log entries as they were recorded by Gitblit.
    * Each PushLogEntry may represent multiple ref updates.
    * 
    * @param repositoryName
    * @param repository
    * @param minimumDate
    * @param offset
    * @param maxCount
    *          if < 0, all pushes are returned.
    * @return a list of push log entries
    */
   public static List<PushLogEntry> getPushLog(String repositoryName, Repository repository,
         Date minimumDate, int offset, int maxCount) {
      List<PushLogEntry> list = new ArrayList<PushLogEntry>();
      RefModel ref = getPushLogBranch(repository);
      if (ref == null) {
         return list;
      }
      if (maxCount == 0) {
         return list;
      }
		
      Map<ObjectId, List<RefModel>> allRefs = JGitUtils.getAllRefs(repository);
      List<RevCommit> pushes;
      if (minimumDate == null) {
         pushes = JGitUtils.getRevLog(repository, GB_PUSHES, 0, maxCount);
         pushes = JGitUtils.getRevLog(repository, GB_PUSHES, offset, maxCount);
      } else {
         pushes = JGitUtils.getRevLog(repository, GB_PUSHES, minimumDate);
      }
@@ -314,23 +368,9 @@
            continue;
         }

         String name = push.getAuthorIdent().getName();
         String username;
         String displayname;
         if (name.indexOf('/') > -1) {
            int slash = name.indexOf('/');
            displayname = name.substring(0, slash);
            username = name.substring(slash + 1);
         } else {
            displayname = name;
            username = push.getAuthorIdent().getEmailAddress();
         }
			
         UserModel user = new UserModel(username);
         user.displayName = displayname;
         user.emailAddress = push.getAuthorIdent().getEmailAddress();
			
         UserModel user = newUserModelFrom(push.getAuthorIdent());
         Date date = push.getAuthorIdent().getWhen();
			
         PushLogEntry log = new PushLogEntry(repositoryName, date, user);
         list.add(log);
         List<PathChangeModel> changedRefs = JGitUtils.getFilesInCommit(repository, push);
@@ -344,12 +384,15 @@
            default:
               String content = JGitUtils.getStringContent(repository, push.getTree(), change.path);
               String [] fields = content.split(" ");
               log.updateRef(change.path, ReceiveCommand.Type.valueOf(fields[0]));
               String oldId = fields[1];
               String newId = fields[2];
               log.updateRef(change.path, ReceiveCommand.Type.valueOf(fields[0]), oldId, newId);
               List<RevCommit> pushedCommits = JGitUtils.getRevLog(repository, oldId, newId);
               for (RevCommit pushedCommit : pushedCommits) {
                  log.addCommit(change.path, pushedCommit);
                  RepositoryCommit repoCommit = log.addCommit(change.path, pushedCommit);
                  if (repoCommit != null) {
                     repoCommit.setRefs(allRefs.get(pushedCommit.getId()));
                  }
               }
            }
         }
@@ -357,4 +400,237 @@
      Collections.sort(list);
      return list;
   }

   /**
    * Returns the list of pushes separated by ref (e.g. each ref has it's own
    * PushLogEntry object).
    *  
    * @param repositoryName
    * @param repository
    * @param maxCount
    * @return a list of push log entries separated by ref
    */
   public static List<PushLogEntry> getPushLogByRef(String repositoryName, Repository repository, int maxCount) {
      return getPushLogByRef(repositoryName, repository, 0, maxCount);
   }
	
   /**
    * Returns the list of pushes separated by ref (e.g. each ref has it's own
    * PushLogEntry object).
    *  
    * @param repositoryName
    * @param repository
    * @param offset
    * @param maxCount
    * @return a list of push log entries separated by ref
    */
   public static List<PushLogEntry> getPushLogByRef(String repositoryName, Repository repository,  int offset,
         int maxCount) {
      // break the push log into ref push logs and then merge them back into a list
      Map<String, List<PushLogEntry>> refMap = new HashMap<String, List<PushLogEntry>>();
        List<PushLogEntry> pushes = getPushLog(repositoryName, repository, offset, maxCount);
      for (PushLogEntry push : pushes) {
         for (String ref : push.getChangedRefs()) {
            if (!refMap.containsKey(ref)) {
               refMap.put(ref, new ArrayList<PushLogEntry>());
            }
				
            // construct new ref-specific push log entry
            PushLogEntry refPush;
            if (push instanceof DailyLogEntry) {
               // simulated push log from commits grouped by date
               refPush = new DailyLogEntry(push.repository, push.date);
            } else {
               // real push log entry
               refPush = new PushLogEntry(push.repository, push.date, push.user);
            }
            refPush.updateRef(ref, push.getChangeType(ref), push.getOldId(ref), push.getNewId(ref));
            refPush.addCommits(push.getCommits(ref));
            refMap.get(ref).add(refPush);
         }
      }
		
      // merge individual ref pushes into master list
      List<PushLogEntry> refPushLog = new ArrayList<PushLogEntry>();
      for (List<PushLogEntry> refPush : refMap.values()) {
         refPushLog.addAll(refPush);
      }
		
      // sort ref push log
      Collections.sort(refPushLog);
		
      return refPushLog;
   }
	
   /**
    * Returns the list of pushes separated by ref (e.g. each ref has it's own
    * PushLogEntry object).
    *  
    * @param repositoryName
    * @param repository
    * @param minimumDate
    * @return a list of push log entries separated by ref
    */
   public static List<PushLogEntry> getPushLogByRef(String repositoryName, Repository repository,  Date minimumDate) {
      // break the push log into ref push logs and then merge them back into a list
      Map<String, List<PushLogEntry>> refMap = new HashMap<String, List<PushLogEntry>>();
      List<PushLogEntry> pushes = getPushLog(repositoryName, repository, minimumDate);
      for (PushLogEntry push : pushes) {
         for (String ref : push.getChangedRefs()) {
            if (!refMap.containsKey(ref)) {
               refMap.put(ref, new ArrayList<PushLogEntry>());
            }

                // construct new ref-specific push log entry
                PushLogEntry refPush = new PushLogEntry(push.repository, push.date, push.user);
                refPush.updateRef(ref, push.getChangeType(ref), push.getOldId(ref), push.getNewId(ref));
            refPush.addCommits(push.getCommits(ref));
            refMap.get(ref).add(refPush);
         }
      }
		
      // merge individual ref pushes into master list
      List<PushLogEntry> refPushLog = new ArrayList<PushLogEntry>();
      for (List<PushLogEntry> refPush : refMap.values()) {
         refPushLog.addAll(refPush);
      }
		
      // sort ref push log
      Collections.sort(refPushLog);
		
      return refPushLog;
   }

    /**
     * Returns a commit log grouped by day.
     *
     * @param repositoryName
     * @param repository
     * @param minimumDate
     * @param offset
     * @param maxCount
     *          if < 0, all pushes are returned.
     * @param the timezone to use when aggregating commits by date
     * @return a list of grouped commit log entries
     */
    public static List<DailyLogEntry> getDailyLog(String repositoryName, Repository repository,
                                                 Date minimumDate, int offset, int maxCount,
                                                 TimeZone timezone) {
        DateFormat df = new SimpleDateFormat("yyyy-MM-dd");
      df.setTimeZone(timezone);

        Map<ObjectId, List<RefModel>> allRefs = JGitUtils.getAllRefs(repository);
        Map<String, DailyLogEntry> tags = new HashMap<String, DailyLogEntry>();
        Map<String, DailyLogEntry> pulls = new HashMap<String, DailyLogEntry>();
        Map<String, DailyLogEntry> dailydigests = new HashMap<String, DailyLogEntry>();
        String linearParent = null;
        for (RefModel local : JGitUtils.getLocalBranches(repository, true, -1)) {
            String branch = local.getName();
            List<RevCommit> commits = JGitUtils.getRevLog(repository, branch, minimumDate);
            for (RevCommit commit : commits) {
               if (linearParent != null) {
                  if (!commit.getName().equals(linearParent)) {
                     // only follow linear branch commits
                     continue;
                  }
               }
                Date date = JGitUtils.getCommitDate(commit);
                String dateStr = df.format(date);
                if (!dailydigests.containsKey(dateStr)) {
                    dailydigests.put(dateStr, new DailyLogEntry(repositoryName, date));
                }
                DailyLogEntry digest = dailydigests.get(dateStr);
                if (commit.getParentCount() == 0) {
                   linearParent = null;
                   digest.updateRef(branch, ReceiveCommand.Type.CREATE);
                } else {
                   linearParent = commit.getParents()[0].getId().getName();
                   digest.updateRef(branch, ReceiveCommand.Type.UPDATE, linearParent, commit.getName());
                }
                
                RepositoryCommit repoCommit = digest.addCommit(branch, commit);
                if (repoCommit != null) {
                   List<RefModel> matchedRefs = allRefs.get(commit.getId());
                    repoCommit.setRefs(matchedRefs);
                    
                    if (!ArrayUtils.isEmpty(matchedRefs)) {
                        for (RefModel ref : matchedRefs) {
                            if (ref.getName().startsWith(Constants.R_TAGS)) {
                                // treat tags as special events in the log
                                if (!tags.containsKey(dateStr)) {
                                 UserModel tagUser = newUserModelFrom(commit.getAuthorIdent());
                                 Date tagDate = commit.getAuthorIdent().getWhen();
                                 tags.put(dateStr, new DailyLogEntry(repositoryName, tagDate, tagUser));
                                }
                                PushLogEntry tagEntry = tags.get(dateStr);
                                tagEntry.updateRef(ref.getName(), ReceiveCommand.Type.CREATE);
                                tagEntry.addCommit(ref.getName(), commit);
                            } else if (ref.getName().startsWith(Constants.R_PULL)) {
                                // treat pull requests as special events in the log
                                if (!pulls.containsKey(dateStr)) {
                                 UserModel commitUser = newUserModelFrom(commit.getAuthorIdent());
                                 Date commitDate = commit.getAuthorIdent().getWhen();
                                 pulls.put(dateStr, new DailyLogEntry(repositoryName, commitDate, commitUser));
                                }
                                PushLogEntry pullEntry = pulls.get(dateStr);
                                pullEntry.updateRef(ref.getName(), ReceiveCommand.Type.CREATE);
                                pullEntry.addCommit(ref.getName(), commit);
                            }
                        }
                    }
                }
            }
        }

        List<DailyLogEntry> list = new ArrayList<DailyLogEntry>(dailydigests.values());
        list.addAll(tags.values());
        //list.addAll(pulls.values());
        Collections.sort(list);
        return list;
    }

    /**
     * Returns the list of commits separated by ref (e.g. each ref has it's own
     * PushLogEntry object for each day).
     *
     * @param repositoryName
     * @param repository
     * @param minimumDate
     * @param the timezone to use when aggregating commits by date
     * @return a list of push log entries separated by ref and date
     */
    public static List<DailyLogEntry> getDailyLogByRef(String repositoryName, Repository repository,
          Date minimumDate, TimeZone timezone) {
        // break the push log into ref push logs and then merge them back into a list
        Map<String, List<DailyLogEntry>> refMap = new HashMap<String, List<DailyLogEntry>>();
        List<DailyLogEntry> pushes = getDailyLog(repositoryName, repository, minimumDate, 0, -1, timezone);
        for (DailyLogEntry push : pushes) {
            for (String ref : push.getChangedRefs()) {
                if (!refMap.containsKey(ref)) {
                    refMap.put(ref, new ArrayList<DailyLogEntry>());
                }

                // construct new ref-specific push log entry
                DailyLogEntry refPush = new DailyLogEntry(push.repository, push.date, push.user);
                refPush.updateRef(ref, push.getChangeType(ref), push.getOldId(ref), push.getNewId(ref));
                refPush.addCommits(push.getCommits(ref));
                refMap.get(ref).add(refPush);
            }
        }

        // merge individual ref pushes into master list
        List<DailyLogEntry> refPushLog = new ArrayList<DailyLogEntry>();
        for (List<DailyLogEntry> refPush : refMap.values()) {
           for (DailyLogEntry entry : refPush) {
              if (entry.getCommitCount() > 0) {
                 refPushLog.add(entry);
              }
           }
        }

        // sort ref push log
        Collections.sort(refPushLog);

        return refPushLog;
    }
}