From a502d96a860456ec5e8c96761db70f7cabb74751 Mon Sep 17 00:00:00 2001
From: Paul Martin <paul@paulsputer.com>
Date: Sat, 30 Apr 2016 04:19:14 -0400
Subject: [PATCH] Merge pull request #1073 from gitblit/1062-DocEditorUpdates
---
src/main/java/com/gitblit/manager/FilestoreManager.java | 466 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 466 insertions(+), 0 deletions(-)
diff --git a/src/main/java/com/gitblit/manager/FilestoreManager.java b/src/main/java/com/gitblit/manager/FilestoreManager.java
new file mode 100644
index 0000000..1110855
--- /dev/null
+++ b/src/main/java/com/gitblit/manager/FilestoreManager.java
@@ -0,0 +1,466 @@
+/*
+ * Copyright 2015 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gitblit.manager;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.lang.reflect.Type;
+import java.nio.file.Files;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.gitblit.IStoredSettings;
+import com.gitblit.Keys;
+import com.gitblit.models.FilestoreModel;
+import com.gitblit.models.FilestoreModel.Status;
+import com.gitblit.models.RepositoryModel;
+import com.gitblit.models.UserModel;
+import com.gitblit.utils.ArrayUtils;
+import com.gitblit.utils.JsonUtils.GmtDateTypeAdapter;
+import com.google.gson.ExclusionStrategy;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+/**
+ * FilestoreManager handles files uploaded via:
+ * + git-lfs
+ * + ticket attachment (TBD)
+ *
+ * Files are stored using their SHA256 hash (as per git-lfs)
+ * If the same file is uploaded through different repositories no additional space is used
+ * Access is controlled through the current repository permissions.
+ *
+ * TODO: Identify what and how the actual BLOBs should work with federation
+ *
+ * @author Paul Martin
+ *
+ */
+@Singleton
+public class FilestoreManager implements IFilestoreManager {
+
+ private final Logger logger = LoggerFactory.getLogger(getClass());
+
+ private final IRuntimeManager runtimeManager;
+
+ private final IRepositoryManager repositoryManager;
+
+ private final IStoredSettings settings;
+
+ public static final int UNDEFINED_SIZE = -1;
+
+ private static final String METAFILE = "filestore.json";
+
+ private static final String METAFILE_TMP = "filestore.json.tmp";
+
+ protected static final Type METAFILE_TYPE = new TypeToken<Collection<FilestoreModel>>() {}.getType();
+
+ private Map<String, FilestoreModel > fileCache = new ConcurrentHashMap<String, FilestoreModel>();
+
+
+ @Inject
+ FilestoreManager(
+ IRuntimeManager runtimeManager,
+ IRepositoryManager repositoryManager) {
+ this.runtimeManager = runtimeManager;
+ this.repositoryManager = repositoryManager;
+ this.settings = runtimeManager.getSettings();
+ }
+
+ @Override
+ public IManager start() {
+
+ // Try to load any existing metadata
+ File dir = getStorageFolder();
+ dir.mkdirs();
+ File metadata = new File(dir, METAFILE);
+
+ if (metadata.exists()) {
+ Collection<FilestoreModel> items = null;
+
+ Gson gson = gson();
+ try (FileReader file = new FileReader(metadata)) {
+ items = gson.fromJson(file, METAFILE_TYPE);
+ file.close();
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ for(Iterator<FilestoreModel> itr = items.iterator(); itr.hasNext(); ) {
+ FilestoreModel model = itr.next();
+ fileCache.put(model.oid, model);
+ }
+
+ logger.info("Loaded {} items from filestore metadata file", fileCache.size());
+ }
+ else
+ {
+ logger.info("No filestore metadata file found");
+ }
+
+ return this;
+ }
+
+ @Override
+ public IManager stop() {
+ return this;
+ }
+
+
+ @Override
+ public boolean isValidOid(String oid) {
+ //NOTE: Assuming SHA256 support only as per git-lfs
+ return Pattern.matches("[a-fA-F0-9]{64}", oid);
+ }
+
+ @Override
+ public FilestoreModel.Status addObject(String oid, long size, UserModel user, RepositoryModel repo) {
+
+ //Handle access control
+ if (!user.canPush(repo)) {
+ if (user == UserModel.ANONYMOUS) {
+ return Status.AuthenticationRequired;
+ } else {
+ return Status.Error_Unauthorized;
+ }
+ }
+
+ //Handle object details
+ if (!isValidOid(oid)) { return Status.Error_Invalid_Oid; }
+
+ if (fileCache.containsKey(oid)) {
+ FilestoreModel item = fileCache.get(oid);
+
+ if (!item.isInErrorState() && (size != UNDEFINED_SIZE) && (item.getSize() != size)) {
+ return Status.Error_Size_Mismatch;
+ }
+
+ item.addRepository(repo.name);
+
+ if (item.isInErrorState()) {
+ item.reset(user, size);
+ }
+ } else {
+
+ if (size < 0) {return Status.Error_Invalid_Size; }
+ if ((getMaxUploadSize() != UNDEFINED_SIZE) && (size > getMaxUploadSize())) { return Status.Error_Exceeds_Size_Limit; }
+
+ FilestoreModel model = new FilestoreModel(oid, size, user, repo.name);
+ fileCache.put(oid, model);
+ saveFilestoreModel(model);
+ }
+
+ return fileCache.get(oid).getStatus();
+ }
+
+ @Override
+ public FilestoreModel.Status uploadBlob(String oid, long size, UserModel user, RepositoryModel repo, InputStream streamIn) {
+
+ //Access control and object logic
+ Status state = addObject(oid, size, user, repo);
+
+ if (state != Status.Upload_Pending) {
+ return state;
+ }
+
+ FilestoreModel model = fileCache.get(oid);
+
+ if (!model.actionUpload(user)) {
+ return Status.Upload_In_Progress;
+ } else {
+ long actualSize = 0;
+ File file = getStoragePath(oid);
+
+ try {
+ file.getParentFile().mkdirs();
+ file.createNewFile();
+
+ try (FileOutputStream streamOut = new FileOutputStream(file)) {
+
+ actualSize = IOUtils.copyLarge(streamIn, streamOut);
+
+ streamOut.flush();
+ streamOut.close();
+
+ if (model.getSize() != actualSize) {
+ model.setStatus(Status.Error_Size_Mismatch, user);
+
+ logger.warn(MessageFormat.format("Failed to upload blob {0} due to size mismatch, expected {1} got {2}",
+ oid, model.getSize(), actualSize));
+ } else {
+ String actualOid = "";
+
+ try (FileInputStream fileForHash = new FileInputStream(file)) {
+ actualOid = DigestUtils.sha256Hex(fileForHash);
+ fileForHash.close();
+ }
+
+ if (oid.equalsIgnoreCase(actualOid)) {
+ model.setStatus(Status.Available, user);
+ } else {
+ model.setStatus(Status.Error_Hash_Mismatch, user);
+
+ logger.warn(MessageFormat.format("Failed to upload blob {0} due to hash mismatch, got {1}", oid, actualOid));
+ }
+ }
+ }
+ } catch (Exception e) {
+
+ model.setStatus(Status.Error_Unknown, user);
+ logger.warn(MessageFormat.format("Failed to upload blob {0}", oid), e);
+ } finally {
+ saveFilestoreModel(model);
+ }
+
+ if (model.isInErrorState()) {
+ file.delete();
+ model.removeRepository(repo.name);
+ }
+ }
+
+ return model.getStatus();
+ }
+
+ private FilestoreModel.Status canGetObject(String oid, UserModel user, RepositoryModel repo) {
+
+ //Access Control
+ if (!user.canView(repo)) {
+ if (user == UserModel.ANONYMOUS) {
+ return Status.AuthenticationRequired;
+ } else {
+ return Status.Error_Unauthorized;
+ }
+ }
+
+ //Object Logic
+ if (!isValidOid(oid)) {
+ return Status.Error_Invalid_Oid;
+ }
+
+ if (!fileCache.containsKey(oid)) {
+ return Status.Unavailable;
+ }
+
+ FilestoreModel item = fileCache.get(oid);
+
+ if (item.getStatus() == Status.Available) {
+ return Status.Available;
+ }
+
+ return Status.Unavailable;
+ }
+
+ @Override
+ public FilestoreModel getObject(String oid, UserModel user, RepositoryModel repo) {
+
+ if (canGetObject(oid, user, repo) == Status.Available) {
+ return fileCache.get(oid);
+ }
+
+ return null;
+ }
+
+ @Override
+ public FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut) {
+
+ //Access control and object logic
+ Status status = canGetObject(oid, user, repo);
+
+ if (status != Status.Available) {
+ return status;
+ }
+
+ FilestoreModel item = fileCache.get(oid);
+
+ if (streamOut != null) {
+ try (FileInputStream streamIn = new FileInputStream(getStoragePath(oid))) {
+
+ IOUtils.copyLarge(streamIn, streamOut);
+
+ streamOut.flush();
+ streamIn.close();
+ } catch (EOFException e) {
+ logger.error(MessageFormat.format("Client aborted connection for {0}", oid), e);
+ return Status.Error_Unexpected_Stream_End;
+ } catch (Exception e) {
+ logger.error(MessageFormat.format("Failed to download blob {0}", oid), e);
+ return Status.Error_Unknown;
+ }
+ }
+
+ return item.getStatus();
+ }
+
+ @Override
+ public List<FilestoreModel> getAllObjects(UserModel user) {
+
+ final List<RepositoryModel> viewableRepositories = repositoryManager.getRepositoryModels(user);
+ List<String> viewableRepositoryNames = new ArrayList<String>(viewableRepositories.size());
+
+ for (RepositoryModel repository : viewableRepositories) {
+ viewableRepositoryNames.add(repository.name);
+ }
+
+ if (viewableRepositoryNames.size() == 0) {
+ return null;
+ }
+
+ final Collection<FilestoreModel> allFiles = fileCache.values();
+ List<FilestoreModel> userViewableFiles = new ArrayList<FilestoreModel>(allFiles.size());
+
+ for (FilestoreModel file : allFiles) {
+ if (file.isInRepositoryList(viewableRepositoryNames)) {
+ userViewableFiles.add(file);
+ }
+ }
+
+ return userViewableFiles;
+ }
+
+ @Override
+ public File getStorageFolder() {
+ return runtimeManager.getFileOrFolder(Keys.filestore.storageFolder, "${baseFolder}/lfs");
+ }
+
+ @Override
+ public File getStoragePath(String oid) {
+ return new File(getStorageFolder(), oid.substring(0, 2).concat("/").concat(oid.substring(2)));
+ }
+
+ @Override
+ public long getMaxUploadSize() {
+ return settings.getLong(Keys.filestore.maxUploadSize, -1);
+ }
+
+ @Override
+ public long getFilestoreUsedByteCount() {
+ Iterator<FilestoreModel> iterator = fileCache.values().iterator();
+ long total = 0;
+
+ while (iterator.hasNext()) {
+
+ FilestoreModel item = iterator.next();
+ if (item.getStatus() == Status.Available) {
+ total += item.getSize();
+ }
+ }
+
+ return total;
+ }
+
+ @Override
+ public long getFilestoreAvailableByteCount() {
+
+ try {
+ return Files.getFileStore(getStorageFolder().toPath()).getUsableSpace();
+ } catch (IOException e) {
+ logger.error(MessageFormat.format("Failed to retrive available space in Filestore {0}", e));
+ }
+
+ return UNDEFINED_SIZE;
+ };
+
+ private synchronized void saveFilestoreModel(FilestoreModel model) {
+
+ File metaFile = new File(getStorageFolder(), METAFILE);
+ File metaFileTmp = new File(getStorageFolder(), METAFILE_TMP);
+ boolean isNewFile = false;
+
+ try {
+ if (!metaFile.exists()) {
+ metaFile.getParentFile().mkdirs();
+ metaFile.createNewFile();
+ isNewFile = true;
+ }
+ FileUtils.copyFile(metaFile, metaFileTmp);
+
+ } catch (IOException e) {
+ logger.error("Writing filestore model to file {0}, {1}", METAFILE, e);
+ }
+
+ try (RandomAccessFile fs = new RandomAccessFile(metaFileTmp, "rw")) {
+
+ if (isNewFile) {
+ fs.writeBytes("[");
+ } else {
+ fs.seek(fs.length() - 1);
+ fs.writeBytes(",");
+ }
+
+ fs.writeBytes(gson().toJson(model));
+ fs.writeBytes("]");
+
+ fs.close();
+
+ } catch (IOException e) {
+ logger.error("Writing filestore model to file {0}, {1}", METAFILE_TMP, e);
+ }
+
+ try {
+ if (metaFileTmp.exists()) {
+ FileUtils.copyFile(metaFileTmp, metaFile);
+
+ metaFileTmp.delete();
+ } else {
+ logger.error("Writing filestore model to file {0}", METAFILE);
+ }
+ }
+ catch (IOException e) {
+ logger.error("Writing filestore model to file {0}, {1}", METAFILE, e);
+ }
+ }
+
+ /*
+ * Intended for testing purposes only
+ */
+ @Override
+ public void clearFilestoreCache() {
+ fileCache.clear();
+ }
+
+ private static Gson gson(ExclusionStrategy... strategies) {
+ GsonBuilder builder = new GsonBuilder();
+ builder.registerTypeAdapter(Date.class, new GmtDateTypeAdapter());
+ if (!ArrayUtils.isEmpty(strategies)) {
+ builder.setExclusionStrategies(strategies);
+ }
+ return builder.create();
+ }
+
+}
--
Gitblit v1.9.1