/*
* This file is part of ReqTracker.
*
* Copyright (C) 2015 Taleh Didover, Florian Gerdes, Dmitry Gorelenkov,
* Rajab Hassan Kaoneka, Katsiaryna Krauchanka, Tobias Polzer,
* Gayathery Sathya, Lukas Tajak
*
* ReqTracker is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* ReqTracker is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with ReqTracker. If not, see <http://www.gnu.org/licenses/>.
*/
package de.fau.osr.bl;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.*;
import de.fau.osr.core.Requirement;
import de.fau.osr.core.db.CSVFileDataSource;
import de.fau.osr.core.db.CompositeDataSource;
import de.fau.osr.core.db.DataSource;
import de.fau.osr.core.db.VCSDataSource;
import de.fau.osr.core.db.dao.RequirementDao;
import de.fau.osr.core.db.dao.impl.CommitDaoImplementation;
import de.fau.osr.core.db.dao.impl.RequirementDaoImplementation;
import de.fau.osr.core.vcs.AnnotatedLine;
import de.fau.osr.core.vcs.base.Commit;
import de.fau.osr.core.vcs.base.CommitFile;
import de.fau.osr.core.vcs.base.CommitState;
import de.fau.osr.core.vcs.impl.GitBlameOperation;
import de.fau.osr.core.vcs.interfaces.VcsClient;
import de.fau.osr.util.AppProperties;
import de.fau.osr.util.NaturalOrderComparator;
import de.fau.osr.util.VisibleFilesTraverser;
import de.fau.osr.util.parser.CommitMessageParser;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.hibernate.SessionFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.naming.OperationNotSupportedException;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* This class is an interpreter for data from Vcs and Database
*/
public class Tracker {
private File repoFile;
private VcsClient vcsClient;
private DataSource dataSource;
private VisibleFilesTraverser projectDirTraverser;
private Logger logger = LoggerFactory.getLogger(Tracker.class);
private RequirementDao reqDao;
private CommitDaoImplementation commitDao;
private LoadingCache<String, List<AnnotatedLine>> blameCache;
// *cachedFiles* serves as a simple cache.
private List<Path> cachedFiles;
public Tracker(VcsClient vcsClient) throws IOException {
this(vcsClient, null, null);
}
public Tracker(VcsClient vcsClient, DataSource ds, File repoFile) throws IOException {
init(vcsClient, ds, repoFile);
this.reqDao = new RequirementDaoImplementation();
this.commitDao = new CommitDaoImplementation();
}
public Tracker(VcsClient vcsClient, DataSource ds, File repoFile, SessionFactory sessionFactory) throws IOException {
init(vcsClient, ds, repoFile);
this.reqDao = new RequirementDaoImplementation(sessionFactory);
this.commitDao = new CommitDaoImplementation(sessionFactory);
}
/**
* private method to initialize the tracker, uses default values from properties, if parameters are null
* @throws IOException
*/
private void init(VcsClient vcsClient, DataSource ds, File repoFile) throws IOException {
this.repoFile = repoFile;
this.vcsClient = vcsClient;
//assign default value, temp solution, todo
if (repoFile == null) repoFile = new File(AppProperties.GetValue("DefaultRepoPath"));
if (ds == null) {
CSVFileDataSource csvDs = new CSVFileDataSource(new File(repoFile.getParentFile(), AppProperties.GetValue("DefaultPathToCSVFile")));
CommitMessageParser parser = new CommitMessageParser(Pattern.compile(AppProperties.GetValue("RequirementPattern")));
VCSDataSource vcsDs = new VCSDataSource(vcsClient, parser);
ds = new CompositeDataSource(csvDs, vcsDs);
}
this.dataSource = ds;
blameCache = CacheBuilder.newBuilder()
.maximumSize(10000)
.build(
new CacheLoader<String, List<AnnotatedLine>>() {
public List<AnnotatedLine> load(String path) throws GitAPIException, IOException {
GitBlameOperation op = new GitBlameOperation(vcsClient, path, (s, i) -> null);
return AnnotatedLine.wordsToLine(op.wordBlame(), dataSource);
}
}
);
projectDirTraverser = VisibleFilesTraverser.Get(
repoFile.getParentFile().toPath(),
"git",
".class"
);
}
/**
* purges all caches for repository.
* useful to call if repository is updated by new commit
*/
public void purgeRepoCache() {
dataSource.clearCache(); //this will purge probably more than just VCS datasource
blameCache.cleanUp();
cachedFiles = new ArrayList<>();
}
/**
* @return collection of all commit objects
*/
public Collection<Commit> getCommits() throws IOException {
HashSet<String> ids = Sets.newHashSet(vcsClient.getCommitList());
return getCommitsByIds(ids);
}
/**
* @return all requirement objects, if possible filled with data from database
* @throws IOException
*/
public Collection<Requirement> getRequirements() throws IOException {
ArrayList<Requirement> reqs = new ArrayList<>(dataSource.getAllRequirements());
NaturalOrderComparator comparator = new NaturalOrderComparator();
Collections.sort(reqs, (o1, o2) -> comparator.compare(o1.getId(), o2.getId()));
return reqs;
}
/**
* @return requiremets by given file paths.
* @throws IOException
*/
public Collection<Requirement> getRequirementsByFile(String file) throws IOException {
String filename = file.replaceAll(Matcher.quoteReplacement("\\"), "/");
Collection<Requirement> reqs = new ArrayList<>();
try {
reqs = getBlame(filename).stream()
// fetching every requirement ids of each line
.map( annotatedLine -> annotatedLine.getRequirements())
.flatMap(reqIds -> reqIds.stream())
.distinct()
.sorted((reqId1, reqId2) -> new NaturalOrderComparator().compare(reqId1, reqId2))
.map(reqId -> {
try {
return getRequirementById(reqId);
} catch (IOException e) {
e.printStackTrace();
}
// this should never happen.
return new Requirement("", "", "", null, -1);
})
.collect(Collectors.toList());
} catch (GitAPIException e) {
e.printStackTrace();
}
return reqs;
}
/**
* This method returns a list of <tt>CommitFile</tt>'s for the given requirement ID.
*/
@Deprecated
public List<CommitFile> getCommitFilesForRequirementID(String requirementID) throws IOException {
long startTime = System.currentTimeMillis();
logger.info("Start call :: getCommitFilesForRequirementID():requirementID = " + requirementID + " Time:" + startTime);
List<CommitFile> commitFilesList = new ArrayList<>();
Set<String> commits = getAllReqCommitRelations().get(requirementID);
for (String commit : commits){
vcsClient.getCommitFiles(commit).get().forEachOrdered(commitFilesList::add);
}
for(CommitFile file: commitFilesList){
file.impact = getImpactPercentageForFileAndRequirement(file.newPath.getPath(), requirementID);
}
logger.info("End call :: getCommitFilesForRequirementID() Time: " + (System.currentTimeMillis() - startTime));
return commitFilesList;
}
public float getImpactPercentageForFileAndRequirement(String file, String requirementID){
String filename = file.replaceAll(Matcher.quoteReplacement("\\"), "/");
List<AnnotatedLine> currentBlame;
try {
currentBlame = this.getBlame(filename);
} catch (GitAPIException | IOException e) {
return -1;
}
double influenced = 0;
double currentBlameSize = currentBlame.size();
for(int i = 0; i<currentBlame.size(); i++){
if(currentBlame.get(i).getLine().matches("\\s*")) {
// ignore whitespace only lines
--currentBlameSize;
} else if(currentBlame.get(i).getRequirements().contains(requirementID)){
influenced += 1./currentBlame.get(i).getRequirements().size();
}
}
float impact = 0;
if (currentBlameSize != 0 ) {
impact = (float) ((influenced/currentBlameSize)*100);
}
return impact;
}
/**
* This method returns all the requirements for the given File.
*/
@Deprecated
public Set<String> getRequirementIdsForFile(String filePath) throws IOException {
String filename = filePath.replaceAll(Matcher.quoteReplacement("\\"), "/");
Set<String> requirementList = new HashSet<>();
Iterator<String> commitIdListIterator = vcsClient.getCommitListForFileodification(filename);
SetMultimap<String, String> relations = getAllCommitReqRelations();
while(commitIdListIterator.hasNext()){
requirementList.addAll(relations.get(commitIdListIterator.next()));
}
return requirementList;
}
/**
* relation ReqId - CommitId of VCS + Database
* @return set of the relations
* @throws IOException
*/
public SetMultimap<String, String> getAllReqCommitRelations() throws IOException {
return dataSource.getAllReqCommitRelations();
}
/**
* relation CommitId - ReqId of VCS + Database
* @return set of the relations
* @throws IOException
*/
public SetMultimap<String, String> getAllCommitReqRelations() throws IOException {
SetMultimap<String, String> result = HashMultimap.create();
Multimaps.invertFrom(getAllReqCommitRelations(), result);
return result;
}
/**
* @return all commit objects, are related to this requirement id
* @throws IOException
*/
@Deprecated
public Set<Commit> getCommitsForRequirementID(String requirementID) throws IOException {
Set<Commit> commits = new HashSet<>();
for(String commitID: getAllReqCommitRelations().get(requirementID)){
commits.add(new Commit(commitID,
vcsClient.getCommitMessage(commitID),
dataSource.getCommitRelationByReq(commitID),
vcsClient.getCommitFiles(commitID)));
}
return commits;
}
/**
* @return list of all requirement ids
*/
public Collection<String> getRequirementIds() throws IOException {
Set<Requirement> allReqs = dataSource.getAllRequirements();
HashSet<String> result = new HashSet<>();
for (Requirement req : allReqs) {
result.add(req.getId());
}
return result;
}
/**
* @return all ever committed files, as <tt>CommitFiles</tt>
*/
public Collection<CommitFile> getAllCommitFiles(){
Set<CommitFile> files = new HashSet<>();
Iterator<String> allCommits = vcsClient.getCommitList();
while (allCommits.hasNext()) {
String currentCommit = allCommits.next();
vcsClient.getCommitFiles(currentCommit).get().forEachOrdered(files::add);
}
return files;
}
/**
* @return Existing project file paths, which are in VCS.
* @throws IOException
*/
public Collection<Path> getFiles() throws IOException {
if (cachedFiles == null) {
// Store all project-related and version-controlled files in *cachedFiles*
cachedFiles = projectDirTraverser.traverse().stream()
// if file is versioned *vcsClient::getCommitListForFileodification*
// returns a iterator with size>0
.filter(
file -> Iterators.size(vcsClient.getCommitListForFileodification(file.toString())) > 0)
.collect(Collectors.toList());
}
return new ArrayList<>(this.cachedFiles);
}
/**
* @return Existing project file paths for given requirement
*/
public Collection<Path> getFilesByRequirement(String requirementId) throws IOException {
Collection<Path> rvalue = getFiles().stream()
.filter(file -> {
try {
// Returns true, if first occurring "impact" of *requirementId* is found at any line.
return getBlame(file.toString()).stream()
.filter(annotatedLine -> annotatedLine.getRequirements().contains(requirementId))
.findFirst()
.isPresent();
} catch (GitAPIException e) {
e.printStackTrace();
} catch (IOException e) {
// IOExceptions: *getBlame* could not retrieve any requirement ids for versioned file
}
return false;
})
.collect(Collectors.toList());
for(Path path: rvalue){
System.out.println(path.toString());
}
return rvalue;
}
/**
* @return Existing project file paths for given commit id
*/
public Collection<Path> getFilesByCommit(String commitId) throws IOException {
Collection<Path> rvalue = getFiles().stream()
.filter(path -> {
try {
return getCommitsByFile(path.toString()).contains(commitId);
} catch (IOException e) {
e.printStackTrace();
}
return false;
})
.collect(Collectors.toList());
return rvalue;
}
/**
* @return all existing ever committed file paths
*/
public Collection<String> getAllFilesAsString(){
Set<String> files = new HashSet<String>();
Set<String> deletedFiles = new HashSet<String>();
Iterator<String> allCommits = vcsClient.getCommitList();
while (allCommits.hasNext()) {
String currentCommit = allCommits.next();
for(CommitFile commitfile: (Iterable<CommitFile>)(vcsClient.getCommitFiles(currentCommit).get()::iterator)){
if(!(commitfile.commitState == CommitState.DELETED))
files.add(commitfile.newPath.getPath());
else
deletedFiles.add(commitfile.oldPath.getPath());
}
}
files.removeAll(deletedFiles);
return files;
}
/**
* get commits that did something with the {@code filePath} file
* @param filePath file to search for
* @return collection of commit ids
*/
public Collection<String> getCommitsByFile(String filePath) throws IOException {
String filename = filePath.replaceAll("\\\\", "/");
return Sets.newHashSet(vcsClient.getCommitListForFileodification(filename));
}
/**
* get commits that did something with the {@code filePath} file
* @param filePath file to search for
* @return collection of commit objects
*/
public Collection<Commit> getCommitsForFile(String filePath) throws IOException {
HashSet<String> ids = Sets.newHashSet(vcsClient.getCommitListForFileodification(filePath));
return getCommitsByIds(ids);
}
/**
* add Linkage between Requirement and Commit
* @param commitID and requirementId to be linked
*/
public void addRequirementCommitRelation(String requirementID, String commitID) throws IOException, OperationNotSupportedException {
dataSource.addReqCommitRelation(requirementID, commitID);
}
public String getCurrentRepositoryPath(){
return repoFile.toString();
}
public List<AnnotatedLine> getBlame(String path) throws IOException, GitAPIException {
String filename = path.replaceAll(Matcher.quoteReplacement("\\"), "/");
try {
return blameCache.get(filename);
} catch (ExecutionException e) {
throw new IOException(e.getMessage());
}
}
/**
* Method which performs the complete processing of Requirement Traceability
*/
public RequirementsTraceabilityMatrix generateRequirementsTraceability() throws IOException{
try{
Collection<String> files = getAllFilesAsString() ;
ExecutorService threadPoolExecutor = Executors.newFixedThreadPool(AppProperties.GetValueAsInt("TraceabilityMatrixProcessingThreadPoolCount"));
TraceabilityMatrixThread.setRequirementTraceabilityMatrix(this);
for(String file: files){
TraceabilityMatrixThread traceabilityMatrixWorkerThread = new TraceabilityMatrixThread(file);
threadPoolExecutor.execute(traceabilityMatrixWorkerThread);
}
threadPoolExecutor.shutdown();
while (!threadPoolExecutor.isTerminated()) {
}
return TraceabilityMatrixThread.getRequirementTraceabilityMatrix();
}catch(IndexOutOfBoundsException e){
return null;
}
catch(Exception e){
e.printStackTrace();
}
return null;
}
/**
* Method which performs the complete processing of Requirement Traceability by Impact
*/
public RequirementsTraceabilityMatrixByImpact generateRequirementsTraceabilityByImpact(){
RequirementsTraceabilityMatrixByImpact requirementsTraceabilityMatrixByImpact = new RequirementsTraceabilityMatrixByImpact(this);
return requirementsTraceabilityMatrixByImpact;
}
/**
* @return current repository name from VCS
*/
public String getRepositoryName(){
return vcsClient.getRepositoryName();
}
/**
* @param filePath
* @return one Collection per line in the file, of formatted requirements
* that are impacted by that line
* @throws Exception
*/
public List<Collection<String>> getRequirementsLineLinkageForFile(String filePath) throws IOException, GitAPIException {
String filename = filePath.replaceAll(Matcher.quoteReplacement("\\"), "/");
Collection<AnnotatedLine> lines = this.getBlame(filename);
List<Collection<String>> reqIdsByLines = new ArrayList<>();
//preload all reqs to ID -> Req IdentityHashMap
Collection<Requirement> allReqs = getRequirements();
IdentityHashMap<String, Requirement> reqsById = new IdentityHashMap<>();
for (Requirement req : allReqs) {
reqsById.put(req.getId(), req);
}
for(AnnotatedLine line: lines){
final Collection<String> requirements = line.getRequirements();
Collection<String> annotation = new ArrayList<>();
reqIdsByLines.add(annotation);
for(String reqId : requirements) {
//fetch the req data from data source
Requirement req = reqsById.get(reqId);
if(req != null)
annotation.add("Req " + reqId + ": \"" + req.getTitle() + "\" " + req.getDescription());
else // req is not in the database
annotation.add("Req " + reqId);
}
}
return reqIdsByLines;
}
/**
* @param id id of requirement
* @return data object requirement by id
*/
public Requirement getRequirementById(String id) throws IOException {
Collection<Requirement> allReqs = getRequirements();
for (Requirement req : allReqs) {
if (req.getId().equals(id)){
return req;
}
}
throw new NoSuchElementException("Requirement with id " + id + " not found");
}
/**
* create collection of known Requirement objects, by given set of requirement IDs
* @param reqIds set of req ids
* @return Collection of Requirement domain objects
* @throws IOException
*/
public Collection<Requirement> getRequirementsByIds(Set<String> reqIds) throws IOException {
final Set<String> finalReqs = reqIds;
Collection<Requirement> reqObjects = getRequirements();
return reqObjects.stream()
.filter(req -> finalReqs.contains(req.getId()))
.collect(Collectors.toList());
}
/**
* creates/gets commit objects, by given ids.
* @return set of commit objects
* @throws IOException
*/
public Set<Commit> getCommitsByIds(Collection<String> commitIds) throws IOException {
Set<Commit> commits = new HashSet<>();
SetMultimap<String, String> relations = getAllCommitReqRelations();
for (String id : commitIds) {
commits.add(new Commit(id, vcsClient.getCommitMessage(id), relations.get(id), vcsClient.getCommitFiles(id)));
}
return commits;
}
/**
* creates or updates requirement in database
* @param id id of req to update or create
* @param title new title
* @param description new description
*/
public void saveOrUpdateRequirement(String id, String title, String description) throws IOException, OperationNotSupportedException {
dataSource.saveOrUpdateRequirement(id, title, description);
}
}