From 13417cf9c6eec555b51da49742e47939d2f5715b Mon Sep 17 00:00:00 2001
From: James Moger <james.moger@gitblit.com>
Date: Fri, 19 Oct 2012 22:47:33 -0400
Subject: [PATCH] Exclude submodules from zip downloads (issue 151)
---
src/com/gitblit/utils/FederationUtils.java | 406 ++++++++++++++++++++++++++++++++-------------------------
1 files changed, 225 insertions(+), 181 deletions(-)
diff --git a/src/com/gitblit/utils/FederationUtils.java b/src/com/gitblit/utils/FederationUtils.java
index 129fe42..4d6060d 100644
--- a/src/com/gitblit/utils/FederationUtils.java
+++ b/src/com/gitblit/utils/FederationUtils.java
@@ -15,39 +15,31 @@
*/
package com.gitblit.utils;
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
import java.lang.reflect.Type;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.net.URLConnection;
-import java.security.SecureRandom;
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
+import java.text.MessageFormat;
+import java.util.ArrayList;
import java.util.Collection;
-import java.util.Date;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSession;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
import javax.servlet.http.HttpServletResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.gitblit.Constants;
+import com.gitblit.Constants.FederationProposalResult;
import com.gitblit.Constants.FederationRequest;
import com.gitblit.Constants.FederationToken;
-import com.gitblit.FederationServlet;
+import com.gitblit.IStoredSettings;
+import com.gitblit.Keys;
import com.gitblit.models.FederationModel;
+import com.gitblit.models.FederationProposal;
import com.gitblit.models.RepositoryModel;
+import com.gitblit.models.TeamModel;
import com.gitblit.models.UserModel;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
/**
@@ -58,35 +50,162 @@
*/
public class FederationUtils {
- public static final String CHARSET;
-
- public static final Type REPOSITORIES_TYPE = new TypeToken<Map<String, RepositoryModel>>() {
+ private static final Type REPOSITORIES_TYPE = new TypeToken<Map<String, RepositoryModel>>() {
}.getType();
- public static final Type SETTINGS_TYPE = new TypeToken<Map<String, String>>() {
+ private static final Type SETTINGS_TYPE = new TypeToken<Map<String, String>>() {
}.getType();
- public static final Type USERS_TYPE = new TypeToken<Collection<UserModel>>() {
+ private static final Type USERS_TYPE = new TypeToken<Collection<UserModel>>() {
}.getType();
- public static final Type RESULTS_TYPE = new TypeToken<List<FederationModel>>() {
+ private static final Type TEAMS_TYPE = new TypeToken<Collection<TeamModel>>() {
}.getType();
- private static final SSLContext SSL_CONTEXT;
+ private static final Logger LOGGER = LoggerFactory.getLogger(FederationUtils.class);
- private static final DummyHostnameVerifier HOSTNAME_VERIFIER;
+ /**
+ * Returns an url to this servlet for the specified parameters.
+ *
+ * @param sourceURL
+ * the url of the source gitblit instance
+ * @param token
+ * the federation token of the source gitblit instance
+ * @param req
+ * the pull type request
+ */
+ public static String asLink(String sourceURL, String token, FederationRequest req) {
+ return asLink(sourceURL, null, token, req, null);
+ }
- static {
- SSLContext context = null;
- try {
- context = SSLContext.getInstance("SSL");
- context.init(null, new TrustManager[] { new DummyTrustManager() }, new SecureRandom());
- } catch (Throwable t) {
- t.printStackTrace();
+ /**
+ *
+ * @param remoteURL
+ * the url of the remote gitblit instance
+ * @param tokenType
+ * the type of federation token of a gitblit instance
+ * @param token
+ * the federation token of a gitblit instance
+ * @param req
+ * the pull type request
+ * @param myURL
+ * the url of this gitblit instance
+ * @return
+ */
+ public static String asLink(String remoteURL, FederationToken tokenType, String token,
+ FederationRequest req, String myURL) {
+ if (remoteURL.length() > 0 && remoteURL.charAt(remoteURL.length() - 1) == '/') {
+ remoteURL = remoteURL.substring(0, remoteURL.length() - 1);
}
- SSL_CONTEXT = context;
- HOSTNAME_VERIFIER = new DummyHostnameVerifier();
- CHARSET = "UTF-8";
+ if (req == null) {
+ req = FederationRequest.PULL_REPOSITORIES;
+ }
+ return remoteURL + Constants.FEDERATION_PATH + "?req=" + req.name().toLowerCase()
+ + (token == null ? "" : ("&token=" + token))
+ + (tokenType == null ? "" : ("&tokenType=" + tokenType.name().toLowerCase()))
+ + (myURL == null ? "" : ("&url=" + StringUtils.encodeURL(myURL)));
+ }
+
+ /**
+ * Returns the list of federated gitblit instances that this instance will
+ * try to pull.
+ *
+ * @return list of registered gitblit instances
+ */
+ public static List<FederationModel> getFederationRegistrations(IStoredSettings settings) {
+ List<FederationModel> federationRegistrations = new ArrayList<FederationModel>();
+ List<String> keys = settings.getAllKeys(Keys.federation._ROOT);
+ keys.remove(Keys.federation.name);
+ keys.remove(Keys.federation.passphrase);
+ keys.remove(Keys.federation.allowProposals);
+ keys.remove(Keys.federation.proposalsFolder);
+ keys.remove(Keys.federation.defaultFrequency);
+ keys.remove(Keys.federation.sets);
+ Collections.sort(keys);
+ Map<String, FederationModel> federatedModels = new HashMap<String, FederationModel>();
+ for (String key : keys) {
+ String value = key.substring(Keys.federation._ROOT.length() + 1);
+ List<String> values = StringUtils.getStringsFromValue(value, "\\.");
+ String server = values.get(0);
+ if (!federatedModels.containsKey(server)) {
+ federatedModels.put(server, new FederationModel(server));
+ }
+ String setting = values.get(1);
+ if (setting.equals("url")) {
+ // url of the origin Gitblit instance
+ federatedModels.get(server).url = settings.getString(key, "");
+ } else if (setting.equals("token")) {
+ // token for the origin Gitblit instance
+ federatedModels.get(server).token = settings.getString(key, "");
+ } else if (setting.equals("frequency")) {
+ // frequency of the pull operation
+ federatedModels.get(server).frequency = settings.getString(key, "");
+ } else if (setting.equals("folder")) {
+ // destination folder of the pull operation
+ federatedModels.get(server).folder = settings.getString(key, "");
+ } else if (setting.equals("bare")) {
+ // whether pulled repositories should be bare
+ federatedModels.get(server).bare = settings.getBoolean(key, true);
+ } else if (setting.equals("mirror")) {
+ // are the repositories to be true mirrors of the origin
+ federatedModels.get(server).mirror = settings.getBoolean(key, true);
+ } else if (setting.equals("mergeAccounts")) {
+ // merge remote accounts into local accounts
+ federatedModels.get(server).mergeAccounts = settings.getBoolean(key, false);
+ } else if (setting.equals("sendStatus")) {
+ // send a status acknowledgment to source Gitblit instance
+ // at end of git pull
+ federatedModels.get(server).sendStatus = settings.getBoolean(key, false);
+ } else if (setting.equals("notifyOnError")) {
+ // notify administrators on federation pull failures
+ federatedModels.get(server).notifyOnError = settings.getBoolean(key, false);
+ } else if (setting.equals("exclude")) {
+ // excluded repositories
+ federatedModels.get(server).exclusions = settings.getStrings(key);
+ } else if (setting.equals("include")) {
+ // included repositories
+ federatedModels.get(server).inclusions = settings.getStrings(key);
+ }
+ }
+
+ // verify that registrations have a url and a token
+ for (FederationModel model : federatedModels.values()) {
+ if (StringUtils.isEmpty(model.url)) {
+ LOGGER.warn(MessageFormat.format(
+ "Dropping federation registration {0}. Missing url.", model.name));
+ continue;
+ }
+ if (StringUtils.isEmpty(model.token)) {
+ LOGGER.warn(MessageFormat.format(
+ "Dropping federation registration {0}. Missing token.", model.name));
+ continue;
+ }
+ // set default frequency if unspecified
+ if (StringUtils.isEmpty(model.frequency)) {
+ model.frequency = settings.getString(Keys.federation.defaultFrequency, "60 mins");
+ }
+ federationRegistrations.add(model);
+ }
+ return federationRegistrations;
+ }
+
+ /**
+ * Sends a federation poke to the Gitblit instance at remoteUrl. Pokes are
+ * sent by an pulling Gitblit instance to an origin Gitblit instance as part
+ * of the proposal process. This is to ensure that the pulling Gitblit
+ * instance has an IP route to the origin instance.
+ *
+ * @param remoteUrl
+ * the remote Gitblit instance to send a federation proposal to
+ * @param proposal
+ * a complete federation proposal
+ * @return true if there is a route to the remoteUrl
+ */
+ public static boolean poke(String remoteUrl) throws Exception {
+ String url = asLink(remoteUrl, null, FederationRequest.POKE);
+ String json = JsonUtils.toJsonString("POKE");
+ int status = JsonUtils.sendJsonString(url, json);
+ return status == HttpServletResponse.SC_OK;
}
/**
@@ -94,24 +213,34 @@
*
* @param remoteUrl
* the remote Gitblit instance to send a federation proposal to
- * @param tokenType
- * type of the provided federation token
- * @param myToken
- * my federation token
- * @param myUrl
- * my Gitblit url
- * @param myRepositories
- * the repositories I want to share keyed by their clone url
- * @return true if the proposal was received
+ * @param proposal
+ * a complete federation proposal
+ * @return the federation proposal result code
*/
- public static boolean propose(String remoteUrl, FederationToken tokenType, String myToken,
- String myUrl, Map<String, RepositoryModel> myRepositories) throws Exception {
- String url = FederationServlet.asFederationLink(remoteUrl, tokenType, myToken,
- FederationRequest.PROPOSAL, myUrl);
- Gson gson = new GsonBuilder().setPrettyPrinting().create();
- String json = gson.toJson(myRepositories);
- int status = writeJson(url, json);
- return status == HttpServletResponse.SC_OK;
+ public static FederationProposalResult propose(String remoteUrl, FederationProposal proposal)
+ throws Exception {
+ String url = asLink(remoteUrl, null, FederationRequest.PROPOSAL);
+ String json = JsonUtils.toJsonString(proposal);
+ int status = JsonUtils.sendJsonString(url, json);
+ switch (status) {
+ case HttpServletResponse.SC_FORBIDDEN:
+ // remote Gitblit Federation disabled
+ return FederationProposalResult.FEDERATION_DISABLED;
+ case HttpServletResponse.SC_BAD_REQUEST:
+ // remote Gitblit did not receive any JSON data
+ return FederationProposalResult.MISSING_DATA;
+ case HttpServletResponse.SC_METHOD_NOT_ALLOWED:
+ // remote Gitblit not accepting proposals
+ return FederationProposalResult.NO_PROPOSALS;
+ case HttpServletResponse.SC_NOT_ACCEPTABLE:
+ // remote Gitblit failed to poke this Gitblit instance
+ return FederationProposalResult.NO_POKE;
+ case HttpServletResponse.SC_OK:
+ // received
+ return FederationProposalResult.ACCEPTED;
+ default:
+ return FederationProposalResult.ERROR;
+ }
}
/**
@@ -126,9 +255,9 @@
*/
public static Map<String, RepositoryModel> getRepositories(FederationModel registration,
boolean checkExclusions) throws Exception {
- String url = FederationServlet.asPullLink(registration.url, registration.token,
+ String url = asLink(registration.url, registration.token,
FederationRequest.PULL_REPOSITORIES);
- Map<String, RepositoryModel> models = readGson(url, REPOSITORIES_TYPE);
+ Map<String, RepositoryModel> models = JsonUtils.retrieveJson(url, REPOSITORIES_TYPE);
if (checkExclusions) {
Map<String, RepositoryModel> includedModels = new HashMap<String, RepositoryModel>();
for (Map.Entry<String, RepositoryModel> entry : models.entrySet()) {
@@ -148,11 +277,26 @@
* @return a collection of UserModel objects
* @throws Exception
*/
- public static Collection<UserModel> getUsers(FederationModel registration) throws Exception {
- String url = FederationServlet.asPullLink(registration.url, registration.token,
- FederationRequest.PULL_USERS);
- Collection<UserModel> models = readGson(url, USERS_TYPE);
- return models;
+ public static List<UserModel> getUsers(FederationModel registration) throws Exception {
+ String url = asLink(registration.url, registration.token, FederationRequest.PULL_USERS);
+ Collection<UserModel> models = JsonUtils.retrieveJson(url, USERS_TYPE);
+ List<UserModel> list = new ArrayList<UserModel>(models);
+ return list;
+ }
+
+ /**
+ * Tries to pull the gitblit team definitions from the remote gitblit
+ * instance.
+ *
+ * @param registration
+ * @return a collection of TeamModel objects
+ * @throws Exception
+ */
+ public static List<TeamModel> getTeams(FederationModel registration) throws Exception {
+ String url = asLink(registration.url, registration.token, FederationRequest.PULL_TEAMS);
+ Collection<TeamModel> models = JsonUtils.retrieveJson(url, TEAMS_TYPE);
+ List<TeamModel> list = new ArrayList<TeamModel>(models);
+ return list;
}
/**
@@ -164,10 +308,22 @@
* @throws Exception
*/
public static Map<String, String> getSettings(FederationModel registration) throws Exception {
- String url = FederationServlet.asPullLink(registration.url, registration.token,
- FederationRequest.PULL_SETTINGS);
- Map<String, String> settings = readGson(url, SETTINGS_TYPE);
+ String url = asLink(registration.url, registration.token, FederationRequest.PULL_SETTINGS);
+ Map<String, String> settings = JsonUtils.retrieveJson(url, SETTINGS_TYPE);
return settings;
+ }
+
+ /**
+ * Tries to pull the referenced scripts from the remote gitblit instance.
+ *
+ * @param registration
+ * @return a map of the remote gitblit scripts by script name
+ * @throws Exception
+ */
+ public static Map<String, String> getScripts(FederationModel registration) throws Exception {
+ String url = asLink(registration.url, registration.token, FederationRequest.PULL_SCRIPTS);
+ Map<String, String> scripts = JsonUtils.retrieveJson(url, SETTINGS_TYPE);
+ return scripts;
}
/**
@@ -184,122 +340,10 @@
*/
public static boolean acknowledgeStatus(String identification, FederationModel registration)
throws Exception {
- String url = FederationServlet.asFederationLink(registration.url, null, registration.token,
- FederationRequest.STATUS, identification);
- Gson gson = new GsonBuilder().setPrettyPrinting().create();
- String json = gson.toJson(registration);
- int status = writeJson(url, json);
+ String url = asLink(registration.url, null, registration.token, FederationRequest.STATUS,
+ identification);
+ String json = JsonUtils.toJsonString(registration);
+ int status = JsonUtils.sendJsonString(url, json);
return status == HttpServletResponse.SC_OK;
- }
-
- /**
- * Reads a gson object from the specified url.
- *
- * @param url
- * @param type
- * @return
- * @throws Exception
- */
- public static <X> X readGson(String url, Type type) throws Exception {
- String json = readJson(url);
- if (StringUtils.isEmpty(json)) {
- return null;
- }
- Gson gson = new Gson();
- return gson.fromJson(json, type);
- }
-
- /**
- * Reads a JSON response.
- *
- * @param url
- * @return the JSON response as a string
- * @throws Exception
- */
- public static String readJson(String url) throws Exception {
- URL urlObject = new URL(url);
- URLConnection conn = urlObject.openConnection();
- conn.setRequestProperty("Accept-Charset", CHARSET);
- conn.setUseCaches(false);
- conn.setDoInput(true);
- if (conn instanceof HttpsURLConnection) {
- HttpsURLConnection secureConn = (HttpsURLConnection) conn;
- secureConn.setSSLSocketFactory(SSL_CONTEXT.getSocketFactory());
- secureConn.setHostnameVerifier(HOSTNAME_VERIFIER);
- }
- InputStream is = conn.getInputStream();
- BufferedReader reader = new BufferedReader(new InputStreamReader(is, CHARSET));
- StringBuilder json = new StringBuilder();
- char[] buffer = new char[4096];
- int len = 0;
- while ((len = reader.read(buffer)) > -1) {
- json.append(buffer, 0, len);
- }
- is.close();
- return json.toString();
- }
-
- /**
- * Writes a JSON message to the specified url.
- *
- * @param url
- * the url to write to
- * @param json
- * the json message to send
- * @return the http request result code
- * @throws Exception
- */
- public static int writeJson(String url, String json) throws Exception {
- byte[] jsonBytes = json.getBytes(CHARSET);
- URL urlObject = new URL(url);
- URLConnection conn = urlObject.openConnection();
- conn.setRequestProperty("Content-Type", "text/plain;charset=" + CHARSET);
- conn.setRequestProperty("Content-Length", "" + jsonBytes.length);
- conn.setUseCaches(false);
- conn.setDoOutput(true);
- if (conn instanceof HttpsURLConnection) {
- HttpsURLConnection secureConn = (HttpsURLConnection) conn;
- secureConn.setSSLSocketFactory(SSL_CONTEXT.getSocketFactory());
- secureConn.setHostnameVerifier(HOSTNAME_VERIFIER);
- }
-
- // write json body
- OutputStream os = conn.getOutputStream();
- os.write(jsonBytes);
- os.close();
-
- int status = ((HttpURLConnection) conn).getResponseCode();
- return status;
- }
-
- /**
- * DummyTrustManager trusts all certificates.
- */
- private static class DummyTrustManager implements X509TrustManager {
-
- @Override
- public void checkClientTrusted(X509Certificate[] certs, String authType)
- throws CertificateException {
- }
-
- @Override
- public void checkServerTrusted(X509Certificate[] certs, String authType)
- throws CertificateException {
- }
-
- @Override
- public X509Certificate[] getAcceptedIssuers() {
- return null;
- }
- }
-
- /**
- * Trusts all hostnames from a certificate, including self-signed certs.
- */
- private static class DummyHostnameVerifier implements HostnameVerifier {
- @Override
- public boolean verify(String hostname, SSLSession session) {
- return true;
- }
}
}
--
Gitblit v1.9.1