/**
* This file is part of git-as-svn. It is subject to the license terms
* in the LICENSE file found in the top-level directory of this distribution
* and at http://www.gnu.org/licenses/gpl-2.0.html. No part of git-as-svn,
* including this file, may be copied, modified, propagated, or distributed
* except according to the terms contained in the LICENSE file.
*/
package svnserver.repository.git;
import com.sun.nio.sctp.InvalidStreamException;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.RenameDetector;
import org.eclipse.jgit.lib.*;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevTree;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.util.IntList;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.mapdb.HTreeMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.tmatesoft.svn.core.SVNErrorCode;
import org.tmatesoft.svn.core.SVNErrorMessage;
import org.tmatesoft.svn.core.SVNException;
import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
import svnserver.StringHelper;
import svnserver.auth.User;
import svnserver.context.LocalContext;
import svnserver.context.SharedContext;
import svnserver.repository.*;
import svnserver.repository.git.cache.CacheChange;
import svnserver.repository.git.cache.CacheRevision;
import svnserver.repository.git.filter.GitFilter;
import svnserver.repository.git.filter.GitFilterHelper;
import svnserver.repository.git.filter.GitFilterLink;
import svnserver.repository.git.filter.GitFilterRaw;
import svnserver.repository.git.prop.GitProperty;
import svnserver.repository.git.prop.GitPropertyFactory;
import svnserver.repository.git.prop.PropertyMapping;
import svnserver.repository.git.push.GitPusher;
import svnserver.repository.locks.LockManagerFactory;
import svnserver.repository.locks.LockManagerRead;
import svnserver.repository.locks.LockManagerWrite;
import svnserver.repository.locks.LockWorker;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.BiFunction;
/**
* Implementation for Git repository.
*
* @author Artem V. Navrotskiy <bozaro@users.noreply.github.com>
*/
public class GitRepository implements VcsRepository {
private static final int REPORT_DELAY = 2500;
private static final int MARK_NO_FILE = -1;
@NotNull
private static final Logger log = LoggerFactory.getLogger(GitRepository.class);
@NotNull
private final LockManagerFactory lockManagerFactory;
@NotNull
public static final byte[] emptyBytes = new byte[0];
@NotNull
private final Repository repository;
@NotNull
private final GitPusher pusher;
@NotNull
private final List<GitRevision> revisions = new ArrayList<>();
@NotNull
private final TreeMap<Long, GitRevision> revisionByDate = new TreeMap<>();
@NotNull
private final TreeMap<ObjectId, GitRevision> revisionByHash = new TreeMap<>();
@NotNull
private final Map<String, IntList> lastUpdates = new ConcurrentHashMap<>();
@NotNull
private final ReadWriteLock lock = new ReentrantReadWriteLock();
// Lock for prevent concurrent pushes.
@NotNull
private final Object pushLock = new Object();
@NotNull
private final String uuid;
@NotNull
private final String gitBranch;
@NotNull
private final String svnBranch;
@NotNull
private final LocalContext context;
@NotNull
private final Map<String, Boolean> binaryCache;
@NotNull
private final Map<String, GitFilter> gitFilters;
@NotNull
private final Map<ObjectId, GitProperty[]> directoryPropertyCache = new ConcurrentHashMap<>();
@NotNull
private final Map<ObjectId, GitProperty[]> filePropertyCache = new ConcurrentHashMap<>();
private final boolean renameDetection;
public GitRepository(@NotNull LocalContext context,
@NotNull Repository repository,
@NotNull GitPusher pusher,
@NotNull String branch,
boolean renameDetection,
@NotNull LockManagerFactory lockManagerFactory) throws IOException, SVNException {
this.context = context;
final SharedContext shared = context.getShared();
shared.getOrCreate(GitSubmodules.class, GitSubmodules::new).register(repository);
this.repository = repository;
this.binaryCache = shared.getCacheDB().getHashMap("cache.binary");
this.pusher = pusher;
this.renameDetection = renameDetection;
this.lockManagerFactory = lockManagerFactory;
this.gitFilters = GitFilterHelper.createFilters(context);
this.svnBranch = LayoutHelper.initRepository(repository, branch).getName();
this.gitBranch = Constants.R_HEADS + branch;
final String repositoryId = loadRepositoryId(repository, svnBranch);
this.uuid = UUID.nameUUIDFromBytes((repositoryId + "\0" + gitBranch).getBytes(StandardCharsets.UTF_8)).toString();
log.info("Repository registered (branch: {})", gitBranch);
}
@NotNull
@Override
public LocalContext getContext() {
return context;
}
@Override
public void close() throws IOException {
context.getShared().sure(GitSubmodules.class).unregister(repository);
}
@NotNull
private static String loadRepositoryId(@NotNull Repository repository, @NotNull String refName) throws IOException {
final Ref ref = repository.getRef(refName);
if (ref == null) {
throw new IllegalStateException();
}
ObjectId oid = ref.getObjectId();
final RevWalk revWalk = new RevWalk(repository);
while (true) {
final RevCommit revCommit = revWalk.parseCommit(oid);
if (revCommit.getParentCount() == 0) {
return LayoutHelper.loadRepositoryId(repository.newObjectReader(), oid);
}
oid = revCommit.getParent(0);
}
}
/**
* Load all cached revisions.
*
* @throws IOException
* @throws SVNException
*/
public boolean loadRevisions() throws IOException, SVNException {
// Fast check.
lock.readLock().lock();
try {
final int lastRevision = revisions.size() - 1;
final ObjectId lastCommitId;
if (lastRevision >= 0) {
lastCommitId = revisions.get(lastRevision).getCacheCommit();
final Ref head = repository.getRef(svnBranch);
if (head.getObjectId().equals(lastCommitId)) {
return false;
}
}
} finally {
lock.readLock().unlock();
}
// Real loading.
lock.writeLock().lock();
try {
final int lastRevision = revisions.size() - 1;
final ObjectId lastCommitId = lastRevision < 0 ? null : revisions.get(lastRevision).getCacheCommit();
final Ref head = repository.getRef(svnBranch);
final List<RevCommit> newRevs = new ArrayList<>();
final RevWalk revWalk = new RevWalk(repository);
ObjectId objectId = head.getObjectId();
while (true) {
if (objectId.equals(lastCommitId)) {
break;
}
final RevCommit commit = revWalk.parseCommit(objectId);
newRevs.add(commit);
if (commit.getParentCount() == 0) break;
objectId = commit.getParent(0);
}
if (newRevs.isEmpty()) {
return false;
}
final long beginTime = System.currentTimeMillis();
int processed = 0;
long reportTime = beginTime;
log.info("Loading cached revision changes: {} revision", newRevs.size());
for (int i = newRevs.size() - 1; i >= 0; i--) {
loadRevisionInfo(newRevs.get(i));
processed++;
long currentTime = System.currentTimeMillis();
if (currentTime - reportTime > REPORT_DELAY) {
log.info(" processed cached revision: {} ({} rev/sec)", newRevs.size() - i, 1000.0f * processed / (currentTime - reportTime));
reportTime = currentTime;
processed = 0;
}
}
final long endTime = System.currentTimeMillis();
log.info("Cached revision loaded: {} ms", endTime - beginTime);
return true;
} finally {
lock.writeLock().unlock();
}
}
private static class CacheInfo {
private final int id;
@NotNull
private RevCommit commit;
@NotNull
private List<CacheInfo> childs = new ArrayList<>();
@NotNull
private List<CacheInfo> parents = new ArrayList<>();
@Nullable
private String svnBranch;
private CacheInfo(int id, @NotNull RevCommit commit) {
this.id = id;
this.commit = commit;
}
}
/**
* Create cache for new revisions.
*
* @throws IOException
* @throws SVNException
*/
public boolean cacheRevisions() throws IOException, SVNException {
// Fast check.
lock.readLock().lock();
try {
final int lastRevision = revisions.size() - 1;
if (lastRevision >= 0) {
final ObjectId lastCommitId = revisions.get(lastRevision).getGitNewCommit();
final Ref master = repository.getRef(gitBranch);
if ((master == null) || (master.getObjectId().equals(lastCommitId))) {
return false;
}
}
} finally {
lock.readLock().unlock();
}
// Real update.
final ObjectInserter inserter = repository.newObjectInserter();
lock.writeLock().lock();
try {
final Ref master = repository.getRef(gitBranch);
final List<RevCommit> newRevs = new ArrayList<>();
final RevWalk revWalk = new RevWalk(repository);
ObjectId objectId = master.getObjectId();
while (true) {
if (revisionByHash.containsKey(objectId)) {
break;
}
final RevCommit commit = revWalk.parseCommit(objectId);
newRevs.add(commit);
if (commit.getParentCount() == 0) break;
objectId = commit.getParent(0);
}
if (!newRevs.isEmpty()) {
final long beginTime = System.currentTimeMillis();
int processed = 0;
long reportTime = beginTime;
log.info("Loading revision changes: {} revision", newRevs.size());
int revisionId = revisions.size();
ObjectId cacheId = revisions.get(revisions.size() - 1).getCacheCommit();
for (int i = newRevs.size() - 1; i >= 0; i--) {
final RevCommit revCommit = newRevs.get(i);
cacheId = LayoutHelper.createCacheCommit(inserter, cacheId, revCommit, revisionId, Collections.emptyMap());
inserter.flush();
processed++;
long currentTime = System.currentTimeMillis();
if (currentTime - reportTime > REPORT_DELAY) {
log.info(" processed revision: {} ({} rev/sec)", newRevs.size() - i, 1000.0f * processed / (currentTime - reportTime));
reportTime = currentTime;
processed = 0;
final RefUpdate refUpdate = repository.updateRef(svnBranch);
refUpdate.setNewObjectId(cacheId);
refUpdate.update();
}
revisionId++;
}
final long endTime = System.currentTimeMillis();
log.info("Revision changes loaded: {} ms", endTime - beginTime);
final RefUpdate refUpdate = repository.updateRef(svnBranch);
refUpdate.setNewObjectId(cacheId);
refUpdate.update();
}
return !newRevs.isEmpty();
} finally {
lock.writeLock().unlock();
}
}
private CacheRevision loadCacheRevision(@NotNull ObjectReader reader, @NotNull RevCommit newCommit, int revisionId) throws IOException, SVNException {
final HTreeMap<String, byte[]> cache = context.getShared().getCacheDB().getHashMap("cache-revision");
CacheRevision result = CacheRevision.deserialize(cache.get(newCommit.name()));
if (result == null) {
final RevCommit baseCommit = LayoutHelper.loadOriginalCommit(reader, newCommit);
final GitFile oldTree = getSubversionTree(reader, newCommit.getParentCount() > 0 ? newCommit.getParent(0) : null, revisionId - 1);
final GitFile newTree = getSubversionTree(reader, newCommit, revisionId);
final Map<String, CacheChange> fileChange = new TreeMap<>();
for (Map.Entry<String, GitLogPair> entry : ChangeHelper.collectChanges(oldTree, newTree, true).entrySet()) {
fileChange.put(entry.getKey(), new CacheChange(entry.getValue()));
}
result = new CacheRevision(
baseCommit,
collectRename(oldTree, newTree),
fileChange
);
cache.put(newCommit.name(), CacheRevision.serialize(result));
}
return result;
}
@NotNull
private GitFile getSubversionTree(@NotNull ObjectReader reader, @Nullable RevCommit commit, int revisionId) throws IOException, SVNException {
final RevCommit revCommit = LayoutHelper.loadOriginalCommit(reader, commit);
if (revCommit == null) {
return new GitFileEmptyTree(this, "", revisionId - 1);
}
return GitFileTreeEntry.create(this, revCommit.getTree(), revisionId);
}
@Override
public void updateRevisions() throws IOException, SVNException {
while (true) {
loadRevisions();
if (!cacheRevisions()) {
break;
}
}
wrapLockWrite((lockManager) -> {
lockManager.validateLocks();
return Boolean.TRUE;
});
context.getShared().getCacheDB().commit();
}
private boolean isTreeEmpty(RevTree tree) throws IOException {
return new CanonicalTreeParser(GitRepository.emptyBytes, repository.newObjectReader(), tree).eof();
}
private void loadRevisionInfo(@NotNull RevCommit commit) throws IOException, SVNException {
final ObjectReader reader = repository.newObjectReader();
final CacheRevision cacheRevision = loadCacheRevision(reader, commit, revisions.size());
final int revisionId = revisions.size();
final Map<String, VcsCopyFrom> copyFroms = new HashMap<>();
for (Map.Entry<String, String> entry : cacheRevision.getRenames().entrySet()) {
copyFroms.put(entry.getKey(), new VcsCopyFrom(revisionId - 1, entry.getValue()));
}
final RevCommit oldCommit = revisions.isEmpty() ? null : revisions.get(revisions.size() - 1).getGitNewCommit();
final RevCommit svnCommit = cacheRevision.getGitCommitId() != null ? new RevWalk(reader).parseCommit(cacheRevision.getGitCommitId()) : null;
for (Map.Entry<String, CacheChange> entry : cacheRevision.getFileChange().entrySet()) {
lastUpdates.compute(entry.getKey(), (key, list) -> {
final IntList result = list == null ? new IntList() : list;
result.add(revisionId);
if (entry.getValue().getNewFile() == null) {
result.add(MARK_NO_FILE);
}
return result;
});
}
final GitRevision revision = new GitRevision(this, commit.getId(), revisionId, copyFroms, oldCommit, svnCommit, commit.getCommitTime());
if (revision.getId() > 0) {
if (revisionByDate.isEmpty() || revisionByDate.lastKey() <= revision.getDate()) {
revisionByDate.put(revision.getDate(), revision);
}
}
if (svnCommit != null) {
revisionByHash.put(svnCommit.getId(), revision);
}
revisions.add(revision);
}
@NotNull
private Map<String, String> collectRename(@NotNull GitFile oldTree, @NotNull GitFile newTree) throws IOException {
if (!renameDetection) {
return Collections.emptyMap();
}
final GitObject<ObjectId> oldTreeId = oldTree.getObjectId();
final GitObject<ObjectId> newTreeId = newTree.getObjectId();
if (oldTreeId == null || newTreeId == null || !Objects.equals(oldTreeId.getRepo(), newTreeId.getRepo())) {
return Collections.emptyMap();
}
final TreeWalk tw = new TreeWalk(repository);
tw.setRecursive(true);
tw.addTree(oldTree.getObjectId().getObject());
tw.addTree(newTree.getObjectId().getObject());
final RenameDetector rd = new RenameDetector(repository);
rd.addAll(DiffEntry.scan(tw));
final Map<String, String> result = new HashMap<>();
for (DiffEntry diff : rd.compute(tw.getObjectReader(), null)) {
if (diff.getScore() >= rd.getRenameScore()) {
result.put(StringHelper.normalize(diff.getNewPath()), StringHelper.normalize(diff.getOldPath()));
}
}
return result;
}
@NotNull
public GitProperty[] collectProperties(@NotNull GitTreeEntry treeEntry, @NotNull VcsSupplier<Iterable<GitTreeEntry>> entryProvider) throws IOException, SVNException {
if (treeEntry.getFileMode().getObjectType() == Constants.OBJ_BLOB)
return GitProperty.emptyArray;
GitProperty[] props = directoryPropertyCache.get(treeEntry.getObjectId().getObject());
if (props == null) {
final List<GitProperty> propList = new ArrayList<>();
try {
for (GitTreeEntry entry : entryProvider.get()) {
final GitProperty[] parseProps = parseGitProperty(entry.getFileName(), entry.getObjectId());
if (parseProps.length > 0) {
propList.addAll(Arrays.asList(parseProps));
}
}
} catch (SvnForbiddenException ignored) {
}
if (!propList.isEmpty()) {
props = propList.toArray(new GitProperty[propList.size()]);
} else {
props = GitProperty.emptyArray;
}
directoryPropertyCache.put(treeEntry.getObjectId().getObject(), props);
}
return props;
}
@NotNull
public GitFilter getFilter(@NotNull FileMode fileMode, @NotNull GitProperty[] props) throws IOException, SVNException {
if (fileMode.getObjectType() != Constants.OBJ_BLOB) {
return gitFilters.get(GitFilterRaw.NAME);
}
if (fileMode == FileMode.SYMLINK) {
return gitFilters.get(GitFilterLink.NAME);
}
for (int i = props.length - 1; i >= 0; --i) {
final String filterName = props[i].getFilterName();
if (filterName != null) {
final GitFilter filter = gitFilters.get(filterName);
if (filter == null) {
throw new InvalidStreamException("Unknown filter requested: " + filterName);
}
return filter;
}
}
return gitFilters.get(GitFilterRaw.NAME);
}
@NotNull
private GitProperty[] parseGitProperty(@NotNull String fileName, @NotNull GitObject<ObjectId> objectId) throws IOException, SVNException {
final GitPropertyFactory factory = PropertyMapping.getFactory(fileName);
if (factory == null)
return GitProperty.emptyArray;
return cachedParseGitProperty(objectId, factory);
}
@NotNull
private GitProperty[] cachedParseGitProperty(GitObject<ObjectId> objectId, GitPropertyFactory factory) throws IOException, SVNException {
GitProperty[] property = filePropertyCache.get(objectId.getObject());
if (property == null) {
property = factory.create(loadContent(objectId));
if (property.length == 0) {
property = GitProperty.emptyArray;
}
filePropertyCache.put(objectId.getObject(), property);
}
return property;
}
@NotNull
@Override
public GitRevision getLatestRevision() throws IOException {
lock.readLock().lock();
try {
return revisions.get(revisions.size() - 1);
} finally {
lock.readLock().unlock();
}
}
@NotNull
@Override
public VcsRevision getRevisionByDate(long dateTime) throws IOException {
lock.readLock().lock();
try {
final Map.Entry<Long, GitRevision> entry = revisionByDate.floorEntry(dateTime);
if (entry != null) {
return entry.getValue();
}
return revisions.get(0);
} finally {
lock.readLock().unlock();
}
}
@NotNull
@Override
public String getUuid() {
return uuid;
}
@NotNull
public Repository getRepository() {
return repository;
}
public boolean isObjectBinary(@Nullable GitFilter filter, @Nullable GitObject<? extends ObjectId> objectId) throws IOException, SVNException {
if (objectId == null || filter == null) return false;
final String key = filter.getName() + " " + objectId.getObject().name();
Boolean result = binaryCache.get(key);
if (result == null) {
try (InputStream stream = filter.inputStream(objectId)) {
result = SVNFileUtil.detectMimeType(stream) != null;
}
binaryCache.putIfAbsent(key, result);
}
return result;
}
@NotNull
@Override
public GitRevision getRevisionInfo(int revision) throws IOException, SVNException {
final GitRevision revisionInfo = getRevisionInfoUnsafe(revision);
if (revisionInfo == null) {
throw new SVNException(SVNErrorMessage.create(SVNErrorCode.FS_NO_SUCH_REVISION, "No such revision " + revision));
}
return revisionInfo;
}
@NotNull
public GitRevision sureRevisionInfo(int revision) throws IOException {
final GitRevision revisionInfo = getRevisionInfoUnsafe(revision);
if (revisionInfo == null) {
throw new IllegalStateException("No such revision " + revision);
}
return revisionInfo;
}
@Nullable
private GitRevision getRevisionInfoUnsafe(int revision) throws IOException {
lock.readLock().lock();
try {
if (revision >= revisions.size())
return null;
return revisions.get(revision);
} finally {
lock.readLock().unlock();
}
}
@NotNull
public GitRevision getRevision(@NotNull ObjectId revisionId) throws SVNException {
lock.readLock().lock();
try {
final GitRevision revision = revisionByHash.get(revisionId);
if (revision == null) {
throw new SVNException(SVNErrorMessage.create(SVNErrorCode.FS_NO_SUCH_REVISION, "No such revision " + revisionId.name()));
}
return revision;
} finally {
lock.readLock().unlock();
}
}
@NotNull
@Override
public VcsWriter createWriter(@NotNull User user) throws SVNException, IOException {
if (user.getEmail() == null || user.getEmail().isEmpty()) {
throw new SVNException(SVNErrorMessage.create(SVNErrorCode.RA_NOT_AUTHORIZED, "Users with undefined email can't create commits"));
}
return new GitWriter(this, pusher, pushLock, gitBranch, user);
}
@Override
public int getLastChange(@NotNull String nodePath, int beforeRevision) {
if (nodePath.isEmpty()) return beforeRevision;
final IntList revs = this.lastUpdates.get(nodePath);
if (revs != null) {
int prev = 0;
for (int i = revs.size() - 1; i >= 0; --i) {
final int rev = revs.get(i);
if ((rev >= 0) && (rev <= beforeRevision)) {
if (prev == MARK_NO_FILE) {
return MARK_NO_FILE;
}
return rev;
}
prev = rev;
}
}
return MARK_NO_FILE;
}
@NotNull
public static String loadContent(@NotNull GitObject<? extends ObjectId> objectId) throws IOException {
final byte[] bytes = objectId.getRepo().newObjectReader().open(objectId.getObject()).getBytes();
return new String(bytes, StandardCharsets.UTF_8);
}
@NotNull
public Iterable<GitTreeEntry> loadTree(@Nullable GitTreeEntry tree) throws IOException {
final GitObject<ObjectId> treeId = getTreeObject(tree);
// Loading tree.
if (treeId == null) {
return Collections.emptyList();
}
final List<GitTreeEntry> result = new ArrayList<>();
final Repository repo = treeId.getRepo();
final CanonicalTreeParser treeParser = new CanonicalTreeParser(GitRepository.emptyBytes, repo.newObjectReader(), treeId.getObject());
while (!treeParser.eof()) {
result.add(new GitTreeEntry(
treeParser.getEntryFileMode(),
new GitObject<>(repo, treeParser.getEntryObjectId()),
treeParser.getEntryPathString()
));
treeParser.next();
}
return result;
}
@Nullable
private GitObject<ObjectId> getTreeObject(@Nullable GitTreeEntry tree) throws IOException {
if (tree == null) {
return null;
}
// Get tree object
if (tree.getFileMode().equals(FileMode.TREE)) {
return tree.getObjectId();
}
if (tree.getFileMode().equals(FileMode.GITLINK)) {
GitObject<RevCommit> linkedCommit = loadLinkedCommit(tree.getObjectId().getObject());
if (linkedCommit == null) {
throw new SvnForbiddenException();
}
return new GitObject<>(linkedCommit.getRepo(), linkedCommit.getObject().getTree());
} else {
return null;
}
}
@Nullable
public GitObject<RevCommit> loadLinkedCommit(@NotNull ObjectId objectId) throws IOException {
return context.getShared().sure(GitSubmodules.class).findCommit(objectId);
}
@NotNull
@Override
public <T> T wrapLockRead(@NotNull LockWorker<T, LockManagerRead> work) throws SVNException, IOException {
return lockManagerFactory.wrapLockRead(this, work);
}
@NotNull
@Override
public <T> T wrapLockWrite(@NotNull LockWorker<T, LockManagerWrite> work) throws SVNException, IOException {
return lockManagerFactory.wrapLockWrite(this, work);
}
private static class ComputeBranchName implements BiFunction<RevCommit, CacheInfo, CacheInfo> {
@NotNull
private final String svnBranch;
public ComputeBranchName(@NotNull String svnBranch) {
this.svnBranch = svnBranch;
}
@NotNull
@Override
public CacheInfo apply(@NotNull RevCommit revCommit, @NotNull CacheInfo old) {
if (old.svnBranch == null || LayoutHelper.compareBranches(old.svnBranch, svnBranch) > 0) {
old.svnBranch = svnBranch;
}
return old;
}
}
}