package br.uff.ic.oceano.ostra.service; import br.uff.ic.oceano.core.control.ApplicationConstants; import br.uff.ic.oceano.core.exception.ObjetoNaoEncontradoException; import br.uff.ic.oceano.core.exception.ServiceException; import br.uff.ic.oceano.core.exception.VCSException; import br.uff.ic.oceano.core.factory.ObjectFactory; import br.uff.ic.oceano.core.model.*; import br.uff.ic.oceano.core.service.MetricValueService; import br.uff.ic.oceano.core.service.ProjectUserService; import br.uff.ic.oceano.core.service.RevisionService; import br.uff.ic.oceano.core.service.controletransacao.Transacional; import br.uff.ic.oceano.core.service.vcs.VCSService; import br.uff.ic.oceano.core.tools.compiler.CompilerService; import br.uff.ic.oceano.core.tools.metrics.MetricEnumeration; import br.uff.ic.oceano.core.tools.metrics.MetricManager; import br.uff.ic.oceano.core.tools.metrics.service.MeasurementService; import br.uff.ic.oceano.core.tools.metrics.service.MetricService; import br.uff.ic.oceano.core.tools.revision.RevisionUtil; import br.uff.ic.oceano.util.DateUtil; import br.uff.ic.oceano.util.file.FileUtils; import br.uff.ic.oceano.util.file.PathUtil; import br.uff.ic.oceano.ostra.exception.CompilerException; import br.uff.ic.oceano.ostra.model.Item; import br.uff.ic.oceano.ostra.model.VersionedItem; import br.uff.ic.oceano.ostra.model.VersionedItemMetricValue; import br.uff.ic.oceano.util.Output; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; /** * * //TODO Should be moved to core, since web is a front end. */ public class OstraMetricService extends MetricService { //Control private boolean retryingCompilation = false; private boolean retryingCheckout = true; private boolean usingUpdateRevision = false; private boolean CONTINUE_TO_NEXT_REVISION_IF_THROWS_EXCEPTION = false; private boolean STOP_REVISIONS_MEASUREMENT_WITH_METRICS_EXCEPTION = true; private int revisionThreshold = 100; // protected VCSService vCSService; protected ProjectUserService projectUserService; protected RevisionService revisionService; protected ItemService itemService; protected LogService logService; protected MetricValueService metricValueService; protected VersionedItemService versionedItemService; protected VersionedItemMetricValueService versionedItemMetricValueService; protected MeasurementService measurementService; protected Set<MetricManager> fontMetrics; protected Set<MetricManager> compiledMetrics; public OstraMetricService() { super(); measurementService = ObjectFactory.getObjectWithDataBaseDependencies(MeasurementService.class); vCSService = ObjectFactory.getObjectWithDataBaseDependencies(VCSService.class); projectUserService = ObjectFactory.getObjectWithDataBaseDependencies(ProjectUserService.class); revisionService = ObjectFactory.getObjectWithDataBaseDependencies(RevisionService.class); itemService = ObjectFactory.getObjectWithDataBaseDependencies(ItemService.class); metricValueService = ObjectFactory.getObjectWithDataBaseDependencies(MetricValueService.class); versionedItemService = ObjectFactory.getObjectWithDataBaseDependencies(VersionedItemService.class); versionedItemMetricValueService = ObjectFactory.getObjectWithDataBaseDependencies(VersionedItemMetricValueService.class); logService = ObjectFactory.getObjectWithDataBaseDependencies(LogService.class); } /** * //TODO break down in methods //TODO find progress pattern to use, for * future web dispatch feature * * @param project * @param oceanoUser * @param metricsToExtract * @throws ServiceException */ public void extractAndSaveMetricsFromAllFilesInProjectRevisions(SoftwareProject project, OceanoUser oceanoUser, List<MetricManager> metricsToExtract) throws ServiceException { loadMetrics(metricsToExtract); Output.println("Pegando usuário... "); ProjectUser projectUser = projectUserService.getByProjectAndOceanoUser(project, oceanoUser); Output.println("Login: <" + projectUser.getLogin() + ">"); String revisionsPath = null; try { Output.println("Pegando revisões de : " + project.getRepositoryUrl()); Set<Revision> revisions = vCSService.getRevisions(project, projectUser); Output.println("Processando " + revisions.size() + " revisões"); if (revisions.size() < getRevisionThreshold()) { Output.println("<<< This Project has less than " + getRevisionThreshold() + " revisions. Will be Skipped. >>>"); return; } int count = 0; String lastRevisionPath = ""; Boolean firstRevision = true; for (Revision revision : revisions) { count++; Output.clearLog(); Output.println("Revision " + revision.getNumber()); Output.println("Comitted files: " + (revision.getChangedFiles() != null ? revision.getChangedFiles().size() : 0)); Output.println("WARNING: stopping at 20 revisions"); if(count==20)break; //Dont save revisions without alterations at source files if (!RevisionUtil.get().hasSourceFileInChangedFiles(revision)) { Output.println(" Revision has no changed source files"); continue; } processRevision(revision); if (hasMeasuredMetrics(revision)) { Output.println(" Metrics extracted. Skipping to next revision."); continue; } try { Output.println(DateUtil.format(Calendar.getInstance()) + " [" + count + "/" + revisions.size() + "]>>Checking revision " + revision + "... "); Output.println("Identifying metrics to extract..."); final Set<MetricManager> fontMetricsToExtract = new HashSet<MetricManager>(); final Set<MetricManager> compiledMetricsToExtract = new HashSet<MetricManager>(); prepareMetricsToExtract(revision, fontMetricsToExtract, compiledMetricsToExtract); if (compiledMetricsToExtract.isEmpty() && fontMetricsToExtract.isEmpty()) { Output.println(" All metrics have been extracted from this revision. Skipping to next revision."); continue; } if (!revisionShouldBeMeasured(revision, fontMetricsToExtract, compiledMetricsToExtract)) { Output.println("Skipping to next revision. "); continue; } Output.println(""); //////////////////////////////////////////////////////////////// //TODO preparing the workspace should be a method, not this. boolean tryAgain = false; boolean failToTryAgain = false; int timesTrying = 0; do { timesTrying++; try { if (firstRevision) { String path = revision.getProject().getConfigurationItem().getName().replace(" ", "_"); path = PathUtil.getWellFormedPath(ApplicationConstants.DIR_BASE_CHECKOUTS, path); revision.setLocalPath(path); PathUtil.mkDirs(path); vCSService.doCheckout(revision, projectUser, false); revisionsPath = revision.getLocalPath(); Output.println("revisionsPath = " + revisionsPath); firstRevision = false; } else { if (isUsingUpdateRevision()) { revision.setLocalPath(revisionsPath); vCSService.doUpdate(revision, projectUser, false); } else { FileUtils.deleteDirectory(lastRevisionPath); //TODO path shoulb the same always //TODO not saving the path for new loop to delete String path = revision.getProject().getConfigurationItem().getName().replace(" ", "_") + "-r" + revision.getNumber(); path = PathUtil.getWellFormedPath(ApplicationConstants.DIR_BASE_CHECKOUTS, path); revision.setLocalPath(path); vCSService.doCheckout(revision, projectUser, false); } } } catch (VCSException ex) { Output.println(""); if (timesTrying < 2) { Output.println(" (!) There was a problem getting the code. Lets try again..."); //clean and try again if (FileUtils.deleteDirectory(ApplicationConstants.DIR_BASE_CHECKOUTS)) { //its the first revision again firstRevision = true; } Output.println(" trying one more time..."); tryAgain = true; } else { failToTryAgain = true; Output.println(" (!) There was a problem getting the code. And the problem happened again in the second time. Skipping to next revision..."); } } } while (isRetryingCheckout() && tryAgain && timesTrying == 1); if (failToTryAgain) { continue; } //finished preparing workspace //////////////////////////////////////////////////////////////// extractAndSaveMetricsForRevision(revision, fontMetricsToExtract, compiledMetricsToExtract); Output.println("OK. Metrics extracted sucessfully."); } catch (ServiceException ex) { Output.println(ex.getMessage()); if (CONTINUE_TO_NEXT_REVISION_IF_THROWS_EXCEPTION) { Output.println("===========================> Service Exception: " + ex.getClass() + " : " + ex.getMessage()); boolean cleaned = FileUtils.deleteDirectory(ApplicationConstants.DIR_BASE_CHECKOUTS); if (cleaned) { //its the first revision again firstRevision = true; } continue; } else { throw new ServiceException(ex); } } finally { logService.log(revision, Output.getLog().toString()); } lastRevisionPath = revision.getLocalPath(); } } catch (Exception ex) { throw new ServiceException(ex); } if (revisionsPath != null) { FileUtils.deleteDirectory(revisionsPath); } } private void loadMetrics(List<MetricManager> metricsToExtract) { fontMetrics = new HashSet<MetricManager>(); compiledMetrics = new HashSet<MetricManager>(); for (MetricManager mm : metricsToExtract) { if (mm.getMetric().isExtractsFromFont()) { fontMetrics.add(mm); } else { compiledMetrics.add(mm); } } Output.println("----------------------------------------------------"); Output.println("Métricas compiladas:"); for (MetricManager metricManager : compiledMetrics) { int type = metricManager.getMetric().getExtratcsFrom(); Output.println(metricManager.getName() + " - " + (type == Metric.EXTRACTS_FROM_PROJECT ? "Project" : (type == Metric.EXTRACTS_FROM_PACKAGE ? "Package" : "File"))); } Output.println(""); Output.println("Métricas não compiladas:"); for (MetricManager metricManager : fontMetrics) { int type = metricManager.getMetric().getExtratcsFrom(); Output.println(metricManager.getName() + " - " + (type == Metric.EXTRACTS_FROM_PROJECT ? "Project" : (type == Metric.EXTRACTS_FROM_PACKAGE ? "Package" : "File"))); } Output.println("----------------------------------------------------"); } private boolean revisionShouldBeMeasured(Revision revision, Set<MetricManager> fontMetricManagers, Set<MetricManager> compiledMetricManagers) throws ServiceException { boolean mustExtractFileMetrics = false; boolean mustExtractPackageMetrics = false; boolean mustExtractProjectMetrics = false; boolean mustExtracCompiledMetrics = false; boolean mustExtracFontMetrics = false; for (MetricManager metricManager : compiledMetricManagers) { if (metricManager.getMetric().getExtratcsFrom() == Metric.EXTRACTS_FROM_FILE) { mustExtractFileMetrics = true; } if (metricManager.getMetric().getExtratcsFrom() == Metric.EXTRACTS_FROM_PACKAGE) { mustExtractPackageMetrics = true; } if (metricManager.getMetric().getExtratcsFrom() == Metric.EXTRACTS_FROM_PROJECT) { mustExtractProjectMetrics = true; } mustExtracFontMetrics |= metricManager.getMetric().isExtractsFromFont(); mustExtracCompiledMetrics |= !metricManager.getMetric().isExtractsFromFont(); } for (MetricManager metricManager : fontMetricManagers) { if (metricManager.getMetric().getExtratcsFrom() == Metric.EXTRACTS_FROM_FILE) { mustExtractFileMetrics = true; } if (metricManager.getMetric().getExtratcsFrom() == Metric.EXTRACTS_FROM_PACKAGE) { mustExtractPackageMetrics = true; } if (metricManager.getMetric().getExtratcsFrom() == Metric.EXTRACTS_FROM_PROJECT) { mustExtractProjectMetrics = true; } mustExtracFontMetrics |= metricManager.getMetric().isExtractsFromFont(); mustExtracCompiledMetrics |= !metricManager.getMetric().isExtractsFromFont(); } boolean hasSourceFilesFromChangesFiles; boolean hasPackagesFromChangesFiles; try { hasSourceFilesFromChangesFiles = !RevisionUtil.get().getSourceFilesFromChangedFiles(revision).isEmpty(); hasPackagesFromChangesFiles = !RevisionUtil.get().getPackagesFromChangedFiles(revision).isEmpty(); } catch (Exception ex) { throw new ServiceException("Fail to check for alterations on revision.", ex); } if (!hasSourceFilesFromChangesFiles && !hasPackagesFromChangesFiles) { Output.print(" There's no source or package alteration to measure. "); return false; } mustExtractFileMetrics &= !hasSourceFilesFromChangesFiles; mustExtractPackageMetrics &= !hasPackagesFromChangesFiles; if (!mustExtractFileMetrics && !mustExtractPackageMetrics && !mustExtractProjectMetrics) { Output.print(" There's no metrics to extract. "); return false; } try { Revision persistentRevision = revisionService.getByNumberAndProject(revision.getNumber(), revision.getProject()); revision.setCannotCompile(persistentRevision.getCannotCompile()); revision.setId(persistentRevision.getId()); final boolean failForCompilation = !isRetryingCompilation() && revision.getCannotCompile() != null && revision.getCannotCompile() && mustExtracCompiledMetrics; if (failForCompilation && !mustExtracFontMetrics) { Output.print(" The compilation of this revision failed before. "); return false; } } catch (ObjetoNaoEncontradoException ex) { //This revision was never measured, so lets work on it! } Output.print(" Measuring revision... "); return true; } private boolean hasMeasuredMetrics(Revision revision) throws ServiceException { for (MetricManager metricManager : fontMetrics) { if (measurementService.isMeasured(metricManager, revision)) { return true; } } for (MetricManager metricManager : compiledMetrics) { if (measurementService.isMeasured(metricManager, revision)) { return true; } } return false; } private void prepareMetricsToExtract(Revision revision, Set<MetricManager> fontMetricsToExtract, Set<MetricManager> compiledMetricsToExtract) throws ServiceException { //TODO clean up after tests Output.println("WARNING: ignoring several metrics"); for (MetricManager metricManager : fontMetrics) { if (!measurementService.isMeasured(metricManager, revision)) { //TODO remove this if after tests if(metricManager.getMetric().getName().equals(MetricEnumeration.TLOC.getName())){ fontMetricsToExtract.add(metricManager); } } } Output.println("WARNING: ignoring compiled metrics"); for (MetricManager metricManager : compiledMetrics) { if (!measurementService.isMeasured(metricManager, revision)) { //TODO uncomment this if after tests //compiledMetricsToExtract.add(metricManager); } } } private void extractAndSaveMetricsForRevision(Revision revision, Set<MetricManager> fontMetricsToExtract, Set<MetricManager> compiledMetricsToExtract) throws ServiceException { Output.println("Extraindo " + (fontMetricsToExtract.size() + compiledMetricsToExtract.size()) + " metricas da revisão " + revision.getNumber() + ": " + fontMetricsToExtract + " (font), " + compiledMetricsToExtract + "(compiled)."); Collection<VersionedItemMetricValue> versionedItemMetricValues = new LinkedList<VersionedItemMetricValue>(); Collection<MetricValue> revisionMetricValues = new LinkedList<MetricValue>(); //font Output.println(" Extracting font metrics..."); final List<VersionedItemMetricValue> extractedMetricsFromVersionedItems = measurementService.extractMetricsFromVersionedItems(revision, fontMetricsToExtract, STOP_REVISIONS_MEASUREMENT_WITH_METRICS_EXCEPTION); versionedItemMetricValues.addAll(extractedMetricsFromVersionedItems); revisionMetricValues.addAll(measurementService.extractProjectMetricsOnly(revision, fontMetricsToExtract)); //compiled Output.println(" Extracting compiled metrics..."); versionedItemMetricValues.addAll(measurementService.extractMetricsFromVersionedItems(revision, compiledMetricsToExtract, STOP_REVISIONS_MEASUREMENT_WITH_METRICS_EXCEPTION)); revisionMetricValues.addAll(measurementService.extractProjectMetricsOnly(revision, compiledMetricsToExtract)); saveMetricsForRevisionVersionedItems(revision, versionedItemMetricValues, revisionMetricValues); } @Transacional private void saveMetricsForRevisionVersionedItems(Revision revision, Collection<VersionedItemMetricValue> versionedItemMetricValues, Collection<MetricValue> revisionMetricValues) throws ServiceException { Output.println(" Saving metrics... "); if (versionedItemMetricValues.isEmpty()) { Output.println(" There's no VersionedItem Metricvalues to save."); } if (revisionMetricValues.isEmpty()) { Output.println(" There's no Project MetricValues to save."); } if (versionedItemMetricValues.isEmpty() && revisionMetricValues.isEmpty()) { return; } final SoftwareProject revisionProject = revision.getProject(); for (VersionedItemMetricValue versionedItemMetricValue : versionedItemMetricValues) { if (versionedItemMetricValue == null) { Output.println("Found null metric result, ignoring it"); continue; } //recover the item id or create it final Item item = versionedItemMetricValue.getVersionedItem().getItem(); itemService.save(item, revisionProject); //save the versioned item versionedItemService.save(versionedItemMetricValue.getVersionedItem()); //save the versioned item metric value versionedItemMetricValueService.save(versionedItemMetricValue); Output.println(" " + versionedItemMetricValue.getMetric() + " (" + versionedItemMetricValue.getVersionedItem().getItem().getPath() + ") = " + versionedItemMetricValue.getDoubleValue()); } //TODO: verify if this commented line changes the expected behavior. // revisionMetricValues.addAll(getRevisionMetricValuesFromVersionedItemMetricValues(versionedItemMetricValues)); for (MetricValue value : revisionMetricValues) { if (value == null) { Output.println("Found null metric result, ignoring it"); continue; } //updates the revision value.setRevision(revision); metricValueService.save(value); Output.println(" " + value.getMetric() + " (" + revision.getProject().getConfigurationItem().getName() + revision.getNumber() + ") = " + value.getDoubleValue()); } } /** * This method calculates the metricvalues of non project metrics for a * given project. * * NOTE THAT THIS METHOD SHOULD NOT RECEIVE ANY PROJECT METRICS. Although it * has a protection to it. As project metrics already have a metricvalue, if * this method calculates one it will replace the measured value with zero, * ever. * * @param project * @param metrics */ public void calculateRevisionMetricValuesFromVersionedItemMetricValues(SoftwareProject project, List<Metric> metrics) { //it is not possible to calculate metricvalues from project metrics, because they are already in that granularity List<Metric> nonProjectMetrics = new ArrayList<Metric>(metrics.size()); for (Metric metric : metrics) { if (metric.getExtratcsFrom() == Metric.EXTRACTS_FROM_PROJECT) { continue; } nonProjectMetrics.add(metric); } if (nonProjectMetrics.isEmpty()) { System.out.println("There are no metrics to calculate."); return; } //A map to store the last value of each item Map<Metric, Map<Item, Double>> mapNewestItemMeasurement = new HashMap<Metric, Map<Item, Double>>(); for (Metric metric : nonProjectMetrics) { mapNewestItemMeasurement.put(metric, new HashMap<Item, Double>()); } System.out.println("Get revisions..."); Set<Revision> revisions = revisionService.getByProject(project); int count = 1; int total = revisions.size(); for (Revision revision : revisions) { System.out.println("\n" + count++ + "/" + total + " Revision = " + revision); for (Metric metric : nonProjectMetrics) { // Output.println(" " + metric); //get the list of items modified in this revision List<VersionedItemMetricValue> vimvs = versionedItemMetricValueService.getByRevisionAndMetric(revision, metric); // Output.println(" Updating the measured items:" + vimvs.size()); //for each of them, update the map with the last measured value for (VersionedItemMetricValue vimv : vimvs) { // Output.println(" >>> vimv = " + vimv); if (vimv.getVersionedItem().getType() == VersionedItem.TYPE_DELETED) { mapNewestItemMeasurement.get(vimv.getMetric()).remove(vimv.getVersionedItem().getItem()); } else { mapNewestItemMeasurement.get(vimv.getMetric()).put(vimv.getVersionedItem().getItem(), vimv.getDoubleValue()); } } // Output.println(" Calculating the value of revision"); //after update the map, it contains the last value of each item. // ================ início do novo método para o metric manager que deve contabilizar para MetricValue os VIMVs Double sum = 0d; int numberOfConsederedArtifacts = 0; StringBuilder strSoma = new StringBuilder(); for (Double oneValue : mapNewestItemMeasurement.get(metric).values()) { // soma.append(" + ").append(oneValue); if (!oneValue.isNaN()) { sum += oneValue; numberOfConsederedArtifacts++; } } if (metric.getAcronym().equals(MetricEnumeration.ACC.getAcronym()) || metric.getAcronym().equals(MetricEnumeration.CAM.getAcronym()) || metric.getAcronym().equals(MetricEnumeration.DAM.getAcronym()) || metric.getAcronym().equals(MetricEnumeration.RMA.getAcronym())) { sum /= numberOfConsederedArtifacts; } MetricValue mv = new MetricValue(revision, metric, sum); // ================ fim do novo método strSoma.append(" |").append(mapNewestItemMeasurement.get(metric).values().size()).append("|").append(" = ").append(mv); Output.println(" " + strSoma.toString()); metricValueService.save(mv); } } } /** * @return the usingUpdateRevision */ public boolean isUsingUpdateRevision() { return usingUpdateRevision; } /** * @param usingUpdateRevision the usingUpdateRevision to set */ public void setUsingUpdateRevision(boolean usingUpdateRevision) { this.usingUpdateRevision = usingUpdateRevision; } /** * @return the revisionThreshold */ public int getRevisionThreshold() { return revisionThreshold; } /** * @param revisionThreshold the revisionThreshold to set */ public void setRevisionThreshold(int revisionThreshold) { this.revisionThreshold = revisionThreshold; } /** * @return the retryingCheckout */ public boolean isRetryingCheckout() { return retryingCheckout; } /** * @param retryingCheckout the retryingCheckout to set */ public void setRetryingCheckout(boolean retryingCheckout) { this.retryingCheckout = retryingCheckout; } /** * @return the retryingCompilation */ public boolean isRetryingCompilation() { return retryingCompilation; } /** * @param retryingCompilation the retryingCompilation to set */ public void setRetryingCompilation(boolean retryingCompilation) { this.retryingCompilation = retryingCompilation; } private void processRevision(Revision revision) throws CompilerException { try { CompilerService.compile(revision); revision.setCannotCompile(false); } catch (CompilerException ex) { revision.setCannotCompile(true); } try { Revision dbRevision = revisionService.getByNumberAndProject(revision.getNumber(), revision.getProject()); revision.setId(dbRevision.getId()); } catch (ObjetoNaoEncontradoException ex) { // its ok, the revision is just new. So, lets save it! revisionService.save(revision); } } }