diff --git a/dependencies/s3m.jar b/dependencies/s3m.jar new file mode 100644 index 000000000..b17080374 Binary files /dev/null and b/dependencies/s3m.jar differ diff --git a/src/main/injectors/S3MMiningModule.groovy b/src/main/injectors/S3MMiningModule.groovy new file mode 100644 index 000000000..a5ff8a969 --- /dev/null +++ b/src/main/injectors/S3MMiningModule.groovy @@ -0,0 +1,28 @@ +package injectors + +@Grab('com.google.inject:guice:4.2.2') +import com.google.inject.AbstractModule +import com.google.inject.multibindings.Multibinder +import interfaces.* +import services.commitFilters.S3MCommitFilter +import services.dataCollectors.S3MMergesCollector.MergesCollector +import services.outputProcessors.S3MOutputProcessor +import services.projectProcessors.ForkAndEnableTravisProcessor + +class S3MMiningModule extends AbstractModule { + + @Override + protected void configure() { + Multibinder dataCollectorBinder = Multibinder.newSetBinder(binder(), DataCollector.class) + dataCollectorBinder.addBinding().to(MergesCollector.class) + + Multibinder projectProcessorBinder = Multibinder.newSetBinder(binder(), ProjectProcessor.class) + projectProcessorBinder.addBinding().to(ForkAndEnableTravisProcessor.class) + + Multibinder outputProcessorBinder = Multibinder.newSetBinder(binder(), OutputProcessor.class) + outputProcessorBinder.addBinding().to(S3MOutputProcessor.class) + + bind(CommitFilter.class).to(S3MCommitFilter.class) + } + +} diff --git a/src/main/services/commitFilters/S3MCommitFilter.groovy b/src/main/services/commitFilters/S3MCommitFilter.groovy new file mode 100644 index 000000000..b5a941e7d --- /dev/null +++ b/src/main/services/commitFilters/S3MCommitFilter.groovy @@ -0,0 +1,25 @@ +package services.commitFilters + +import interfaces.CommitFilter +import project.MergeCommit +import project.Project +import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector +import util.ProcessRunner + +class S3MCommitFilter implements CommitFilter { + + @Override + boolean applyFilter(Project project, MergeCommit mergeCommit) { + return thereIsAtLeastOneMergeScenario(project, mergeCommit) + } + + private static boolean thereIsAtLeastOneMergeScenario(Project project, MergeCommit mergeCommit) { + Process gitDiffTree = ProcessRunner.runProcess(project.getPath(), "git", "diff-tree", "--no-commit-id", "--name-status", "-r", mergeCommit.getSHA(), mergeCommit.getAncestorSHA()) + List modifiedFiles = gitDiffTree.getInputStream().readLines() + + return modifiedFiles.stream() + .filter(MergeScenarioCollector::isModifiedFile) + .filter(MergeScenarioCollector::isJavaFile) + .count() > 0 + } +} diff --git a/src/main/services/dataCollectors/S3MMergesCollector/DataAnalyser.groovy b/src/main/services/dataCollectors/S3MMergesCollector/DataAnalyser.groovy new file mode 100644 index 000000000..ce50aaf3d --- /dev/null +++ b/src/main/services/dataCollectors/S3MMergesCollector/DataAnalyser.groovy @@ -0,0 +1,47 @@ +package services.dataCollectors.S3MMergesCollector + +import project.MergeCommit +import project.Project +import services.util.BuildRequester +import services.util.MergeCommitSummary +import services.util.MergeScenarioSummary +import util.Handlers + +import java.nio.file.Path + +class DataAnalyser { + + /** + * Analyses each merge scenario's directories after S3M has run. It constructs a {@link MergeScenarioSummary} for each + * merge scenario and a global {@link MergeCommitSummary} for each merge commit. + * @param project + * @param mergeCommit + * @param mergeScenarios + * @return a summary of results of the merge commit + */ + static MergeCommitSummary analyseScenarios(Project project, MergeCommit mergeCommit, List mergeScenarios) { + MergeCommitSummary summary = new MergeCommitSummary() + buildCommitSummary(summary, mergeScenarios) + + checkForFalseNegatives(project, mergeCommit, mergeScenarios, summary) + return summary + } + + private static void buildCommitSummary(MergeCommitSummary summary, List mergeScenarios) { + mergeScenarios.stream() + .map(MergeScenarioSummary::new) + .forEach(summary::addMergeSummary) + } + + private static void checkForFalseNegatives(Project project, MergeCommit mergeCommit, List mergeScenarios, MergeCommitSummary summary) { + summary.numberOfConflicts.eachWithIndex { int numConflicts, int i -> + if (numConflicts == 0 && !summary.handlersHaveSameConflicts) { + // there's a merge result with at least one conflict + String buildLink = BuildRequester.requestBuildWithRevision(project, mergeCommit, mergeScenarios, i) + summary.markAsChecking(buildLink, Handlers.mergeAlgorithms[i]) + println 'Requested Travis build' + } + } + } + +} diff --git a/src/main/services/dataCollectors/S3MMergesCollector/MergeScenarioCollector.groovy b/src/main/services/dataCollectors/S3MMergesCollector/MergeScenarioCollector.groovy new file mode 100644 index 000000000..839b32c86 --- /dev/null +++ b/src/main/services/dataCollectors/S3MMergesCollector/MergeScenarioCollector.groovy @@ -0,0 +1,85 @@ +package services.dataCollectors.S3MMergesCollector + +import project.MergeCommit +import project.Project +import util.ProcessRunner +import services.util.Utils + +import java.nio.file.Files +import java.nio.file.Path +import java.util.stream.Collectors + +/** + * Class responsible for collecting and storing eligible merge scenarios (modified from base java files). + */ +class MergeScenarioCollector { + + /** + * Stores merge scenarios (left, base, right and merge files) encountered in the merge commit. + * @param project + * @param mergeCommit + * @return a list of directory paths where each merge scenario is located + */ + static List collectMergeScenarios(Project project, MergeCommit mergeCommit) { + return getModifiedJavaFiles(project, mergeCommit).stream() + .map(modifiedFile -> storeAndRetrieveMergeQuadruple(project, mergeCommit, modifiedFile)) + .map(quadruple -> quadruple.getV4().getParent()) + .collect(Collectors.toList()) + } + + private static Tuple4 storeAndRetrieveMergeQuadruple(Project project, MergeCommit mergeCommit, String modifiedFile) { + Path leftFile = storeFile(project, mergeCommit, modifiedFile, mergeCommit.getLeftSHA(), 'left') + Path baseFile = storeFile(project, mergeCommit, modifiedFile, mergeCommit.getAncestorSHA(), 'base') + Path rightFile = storeFile(project, mergeCommit, modifiedFile, mergeCommit.getRightSHA(), 'right') + Path mergeFile = storeFile(project, mergeCommit, modifiedFile, mergeCommit.getSHA(), 'merge') + return new Tuple4(leftFile, baseFile, rightFile, mergeFile) + } + + private static Path storeFile(Project project, MergeCommit mergeCommit, String modifiedFile, String commitSHA, String fileName) { + Path mergeScenarioDirectory = Utils.commitFilesPath(project, mergeCommit).resolve(modifiedFile) + createDirectories(mergeScenarioDirectory) + + Path filePath = mergeScenarioDirectory.resolve("${fileName}.java") + Files.deleteIfExists(filePath) + filePath.toFile() << getFileContent(project, modifiedFile, commitSHA) + return filePath + } + + private static String getFileContent(Project project, String modifiedFile, String commitSHA) { + StringBuilder fileContent = new StringBuilder() + + Process gitShow = ProcessRunner.runProcess(project.getPath(), "git", "show", "${commitSHA}:${modifiedFile}") + gitShow.getInputStream().eachLine { + fileContent.append(it).append('\n') + } + return fileContent.toString() + } + + private static List getModifiedJavaFiles(Project project, MergeCommit mergeCommit) { + Process gitDiffTree = ProcessRunner.runProcess(project.getPath(), "git", "diff-tree", "--no-commit-id", "--name-status", "-r", mergeCommit.getSHA(), mergeCommit.getAncestorSHA()) + List modifiedFiles = gitDiffTree.getInputStream().readLines() + + return modifiedFiles.stream() + .filter(MergeScenarioCollector::isModifiedFile) + .filter(MergeScenarioCollector::isJavaFile) + .map(MergeScenarioCollector::getPath) + .collect(Collectors.toList()) + } + + private static boolean isModifiedFile(String line) { + return line.charAt(0) == 'M' as char + } + + private static boolean isJavaFile(String line) { + return line.endsWith('.java') + } + + private static String getPath(String line) { + return line.substring(1).trim() + } + + private static void createDirectories(Path path) { + path.toFile().mkdirs() + } + +} diff --git a/src/main/services/dataCollectors/S3MMergesCollector/MergesCollector.groovy b/src/main/services/dataCollectors/S3MMergesCollector/MergesCollector.groovy new file mode 100644 index 000000000..2cbb35e58 --- /dev/null +++ b/src/main/services/dataCollectors/S3MMergesCollector/MergesCollector.groovy @@ -0,0 +1,29 @@ +package services.dataCollectors.S3MMergesCollector + +import interfaces.DataCollector +import project.MergeCommit +import project.Project +import services.util.MergeCommitSummary +import util.Handlers + +import java.nio.file.Path + +class MergesCollector implements DataCollector { + // groovy -cp src src/main/app/MiningFramework.groovy -a b22d2fc334ece38945974c789654e8f56d812b02 -i services.S3MHandlersAnalysis.MiningModule projects.csv + + @Override + void collectData(Project project, MergeCommit mergeCommit) { + List mergeScenarios = MergeScenarioCollector.collectMergeScenarios(project, mergeCommit) + println 'Collected merge scenarios' + + S3MRunner.collectS3MResults(mergeScenarios, [Handlers.Renaming]) + println 'Collected S3M results' + + MergeCommitSummary summary = DataAnalyser.analyseScenarios(project, mergeCommit, mergeScenarios) + println 'Summarized collected data' + + SpreadsheetBuilder.buildSpreadsheets(project, mergeCommit, summary) + println 'Built spreadsheets' + } + +} diff --git a/src/main/services/dataCollectors/S3MMergesCollector/S3MRunner.groovy b/src/main/services/dataCollectors/S3MMergesCollector/S3MRunner.groovy new file mode 100644 index 000000000..9ae89ebe8 --- /dev/null +++ b/src/main/services/dataCollectors/S3MMergesCollector/S3MRunner.groovy @@ -0,0 +1,85 @@ +package services.dataCollectors.S3MMergesCollector + +import util.Handlers +import util.ProcessRunner +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.StandardCopyOption + +class S3MRunner { + + static final Path S3M_PATH = Paths.get("dependencies/s3m.jar") + + /** + * Runs S3M for each merge scenario and for each handler. Store the results at the same directory + * the merge scenario is located, in a directory for each handler. + * + * To extend the analysis for more handlers, check {@link #runHandlerVariants(Path, List < Handlers >)} + * @param mergeScenarios + * @param handlers + */ + static void collectS3MResults(List mergeScenarios, List handlers) { + mergeScenarios.parallelStream() + .forEach(mergeScenario -> runHandlerVariants(mergeScenario, handlers)) + } + + private static void runHandlerVariants(Path mergeScenario, List handlers) { + Path leftFile = getInvolvedFile(mergeScenario, 'left') + Path baseFile = getInvolvedFile(mergeScenario, 'base') + Path rightFile = getInvolvedFile(mergeScenario, 'right') + + // To extend the analysis for other handlers, clone and modify the following conditional. + if (handlers.contains(Handlers.Renaming)) { + runS3M(leftFile, baseFile, rightFile, 'CT.java', Handlers.Renaming, '-hmcrdov') + runS3M(leftFile, baseFile, rightFile, 'SF.java', Handlers.Renaming, '-r', 'SAFE') + runS3M(leftFile, baseFile, rightFile, 'MM.java', Handlers.Renaming, '-r', 'MERGE') + runS3M(leftFile, baseFile, rightFile, 'KB.java', Handlers.Renaming, '-r', 'BOTH') + } + } + + private static void runS3M(Path leftFile, Path baseFile, Path rightFile, String outputFileName, Handlers handler, String... additionalParameters) { + Process S3M = ProcessRunner.startProcess(buildS3MProcess(leftFile, baseFile, rightFile, outputFileName, handler, additionalParameters)) + S3M.getInputStream().eachLine { + //println it + } + S3M.waitFor() + + renameUnstructuredMergeFile(baseFile.getParent(), handler.name(), outputFileName) + } + + private static void renameUnstructuredMergeFile(Path mergeScenario, String handlerName, String outputFileName) { + Path currentUnstructuredMergeFile = mergeScenario.resolve(handlerName).resolve("${outputFileName}.merge") + Path renamedUnstructuredMergeFile = mergeScenario.resolve("textual.java") + Files.move(currentUnstructuredMergeFile, renamedUnstructuredMergeFile, StandardCopyOption.REPLACE_EXISTING) + } + + private static ProcessBuilder buildS3MProcess(Path leftFile, Path baseFile, Path rightFile, String outputFileName, Handlers handler, String... additionalParameters) { + ProcessBuilder S3M = ProcessRunner.buildProcess(getParentAsString(S3M_PATH)) + List parameters = buildS3MParameters(leftFile, baseFile, rightFile, outputFileName, handler.name(), additionalParameters) + S3M.command().addAll(parameters) + return S3M + } + + private static List buildS3MParameters(Path leftFile, Path baseFile, Path rightFile, String outputFileName, String handlerName, String... additionalParameters) { + List parameters = ['java', '-jar', getNameAsString(S3M_PATH), leftFile.toString(), baseFile.toString(), rightFile.toString(), '-o', getOutputPath(baseFile.getParent(), handlerName, outputFileName).toString(), '-c', 'false', '-l', 'false'] + parameters.addAll(additionalParameters.toList()) + return parameters + } + + private static Path getOutputPath(Path mergeScenario, String handlerName, String fileName) { + return mergeScenario.resolve(handlerName).resolve(fileName) + } + + private static String getParentAsString(Path path) { + return path.getParent().toString() + } + + private static String getNameAsString(Path path) { + return path.getFileName().toString() + } + + private static Path getInvolvedFile(Path mergeScenario, String fileName) { + return mergeScenario.resolve("${fileName}.java").toAbsolutePath() + } +} diff --git a/src/main/services/dataCollectors/S3MMergesCollector/SpreadsheetBuilder.groovy b/src/main/services/dataCollectors/S3MMergesCollector/SpreadsheetBuilder.groovy new file mode 100644 index 000000000..d4d8e8be2 --- /dev/null +++ b/src/main/services/dataCollectors/S3MMergesCollector/SpreadsheetBuilder.groovy @@ -0,0 +1,63 @@ +package services.dataCollectors.S3MMergesCollector + +import project.MergeCommit +import project.Project +import services.outputProcessors.S3MOutputProcessor +import services.util.MergeCommitSummary +import services.util.MergeScenarioSummary +import services.util.Utils + +import java.nio.file.Path + +class SpreadsheetBuilder { + private static final String GLOBAL_SPREADSHEET_HEADER = 'project,merge commit,number of modified files,number of TM conflicts,number of CT conflicts,number of SF conflicts,number of MM conflicts,number of KB conflicts,handlers have the same outputs,handlers have the same conflicts,notes,false positives,false negatives,travis builds,,,' + private static final String COMMIT_SPREADSHEET_HEADER = 'project,merge commit,file,number of TM conflicts,number of CT conflicts,number of SF conflicts,number of MM conflicts,number of KB conflicts,CT text = SF text,CT text = MM text,CT text = KB text,SF text = MM text, SF text = KB text,MM text = KB text,CT conflicts = SF conflicts,CT conflicts = MM conflicts,CT conflicts = KB conflicts,SF conflicts = MM conflicts,SF conflicts = KB conflicts,MM conflicts = KB conflicts' + private static final String SPREADSHEET_NAME = 'results.csv' + + /** + * Builds a global spreadsheet, based on the merge commit's summary, and a local spreadsheet, for each merge commit, based on + * the merge scenario's summary. + * @param project + * @param mergeCommit + * @param summary + */ + static synchronized void buildSpreadsheets(Project project, MergeCommit mergeCommit, MergeCommitSummary summary) { + buildGlobalSpreadsheet(project, mergeCommit, summary) + buildCommitSpreadsheet(project, mergeCommit, summary.mergeScenarioSummaries) + } + + private static void buildCommitSpreadsheet(Project project, MergeCommit mergeCommit, List summaries) { + Path spreadsheetPath = Utils.commitFilesPath(project, mergeCommit).resolve(SPREADSHEET_NAME) + File spreadsheet = spreadsheetPath.toFile() + appendHeader(spreadsheet, COMMIT_SPREADSHEET_HEADER) + + summaries.each { summary -> + appendLineToSpreadsheet(spreadsheet, appendAfterProjectAndMergeCommitLinks(project, mergeCommit, summary.toString())) + } + } + + private static void buildGlobalSpreadsheet(Project project, MergeCommit mergeCommit, MergeCommitSummary summary) { + Path spreadsheetPath = Utils.getOutputPath().resolve(SPREADSHEET_NAME) + File spreadsheet = spreadsheetPath.toFile() + appendHeader(spreadsheet, GLOBAL_SPREADSHEET_HEADER) + + appendLineToSpreadsheet(spreadsheet, appendAfterProjectAndMergeCommitLinks(project, mergeCommit, summary.toString())) + } + + private static void appendHeader(File spreadsheet, String header) { + if (!spreadsheet.exists()) { + appendLineToSpreadsheet(spreadsheet, header) + } + } + + private static String appendAfterProjectAndMergeCommitLinks(Project project, MergeCommit mergeCommit, String string) { + String projectName = Utils.getHyperLink(S3MOutputProcessor.ANALYSIS_REMOTE_URL + "/${project.getName()}", project.getName()) + String commitSHA = Utils.getHyperLink(S3MOutputProcessor.ANALYSIS_REMOTE_URL + "/${project.getName()}/${mergeCommit.getSHA()}", mergeCommit.getSHA()) + return "${projectName},${commitSHA},${string}" + } + + private static void appendLineToSpreadsheet(File spreadsheet, String line) { + spreadsheet << "${line.replaceAll('\\\\', '/')}\n" + } + +} diff --git a/src/main/services/outputProcessors/FetchBuildsOutputProcessor.groovy b/src/main/services/outputProcessors/FetchBuildsOutputProcessor.groovy index 05a2c91cc..3560b0ef7 100644 --- a/src/main/services/outputProcessors/FetchBuildsOutputProcessor.groovy +++ b/src/main/services/outputProcessors/FetchBuildsOutputProcessor.groovy @@ -37,4 +37,4 @@ class FetchBuildsOutputProcessor implements OutputProcessor { throw new ExternalScriptException(FETCH_JARS_PATH, exitStatus); } } -} +} \ No newline at end of file diff --git a/src/main/services/outputProcessors/S3MOutputProcessor.groovy b/src/main/services/outputProcessors/S3MOutputProcessor.groovy new file mode 100644 index 000000000..f2e9b1250 --- /dev/null +++ b/src/main/services/outputProcessors/S3MOutputProcessor.groovy @@ -0,0 +1,33 @@ + +package services.outputProcessors + +import interfaces.OutputProcessor +import services.util.Utils + +import java.nio.file.Path +import java.nio.file.Paths + +class S3MOutputProcessor implements OutputProcessor { + + private static final Path ANALYSIS_REPOSITORY_PATH = Paths.get('../merge-tools') + static final String ANALYSIS_REMOTE_URL = "https://github.com/jvcoutinho/merge-tools/tree/master/s3m-handlers-analysis" + + @Override + void processOutput() { + stageAndPushData() + println 'Pushed data to remote analysis repository' + } + + private static void stageAndPushData() { + // Stage changes. + Utils.runGitCommand(ANALYSIS_REPOSITORY_PATH, 'add', '.') + + // Commit changes. + Utils.runGitCommand(ANALYSIS_REPOSITORY_PATH, 'commit', '-m', 'Collected data') + + // Push changes. + Utils.runGitCommand(ANALYSIS_REPOSITORY_PATH, 'push', '--force-with-lease') + } + + +} diff --git a/src/main/services/util/BuildRequester.groovy b/src/main/services/util/BuildRequester.groovy new file mode 100644 index 000000000..f9760814d --- /dev/null +++ b/src/main/services/util/BuildRequester.groovy @@ -0,0 +1,166 @@ +package services.util + +import app.MiningFramework +import project.MergeCommit +import project.Project +import util.GithubHelper +import util.Handlers +import util.HttpHelper +import util.TravisHelper + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.StandardCopyOption + +final class BuildRequester { + + private static enum BuildSystem { + Maven, + Gradle, + None + } + + private static final String projectOwnerName = getOwnerName() + private static final String travisAPIToken = getTravisToken() + + private static Map buildScripts = ['Maven': 'mvn package', 'Gradle': './gradlew build'] + + /** + * Replaces the files in a project by its correspondent merge results in a new branch and triggers a Travis build from a push + * @param project + * @param mergeCommit + * @param mergeScenarios + * @param mergeAlgorithmIndex + * @return the link for the Travis build triggered by this method + */ + static String requestBuildWithRevision(Project project, MergeCommit mergeCommit, List mergeScenarios, int mergeAlgorithmIndex) { + String toReplaceFile = Handlers.mergeResultPaths[mergeAlgorithmIndex] + String mergeAlgorithm = Handlers.mergeAlgorithms[mergeAlgorithmIndex] + + String branchName = "${mergeCommit.getSHA().take(7)}-${mergeAlgorithm}" + + createBranchFromCommit(project, mergeCommit, branchName) + replaceFilesInProject(project, mergeCommit, mergeScenarios, toReplaceFile) + replaceTravisFile(project) + stageAndPushChanges(project, branchName) + + Thread.sleep(2000) // sleep to give time to Travis to compute + return getBuildLink(project, branchName) + } + + private static String getBuildLink(Project project, String branchName) { + String buildID = getBuildAttribute(project, 'id', branchName) + return "https://travis-ci.com/${projectOwnerName}/${project.getName()}/builds/${buildID}" + } + + /** + * ATTENTION!! + * When executing this method (getBuildAttribute()) through "S3MMiningModule", + * the call always returns status 403 and the system is in an infinite loop until interrupted by a Stackoverflow. * + **/ + static String getBuildAttribute(Project project, String attribute, String branchName) { + String url = "https://api.travis-ci.com/repo/${projectOwnerName}%2F${project.getName()}/branch/${branchName}" + HttpURLConnection connection = new URL(url).openConnection() as HttpURLConnection + connection.setRequestProperty("Travis-API-Version", "3") + connection.setRequestProperty("Authorization", "token ${travisAPIToken}") + + if (connection.getResponseCode() != 200) { + Thread.sleep(3000) // sleep for some seconds and try again + return getBuildAttribute(project, attribute, branchName) + } + + def last_build = HttpHelper.responseToJSON(connection.getInputStream())['last_build'] + if (last_build == null) { + Thread.sleep(3000) // sleep for some seconds and try again + return getBuildAttribute(project, attribute, branchName) + } + return last_build[attribute] + } + + private static String getOwnerName() { + String githubToken = MiningFramework.arguments.getAccessKey() + GithubHelper githubHelper = new GithubHelper(githubToken) + + return githubHelper.getUser()['login'] + } + + private static String getTravisToken() { + String githubToken = MiningFramework.arguments.getAccessKey() + TravisHelper travisHelper = new TravisHelper(githubToken) + + return travisHelper.getToken() + } + + private static void createBranchFromCommit(Project project, MergeCommit mergeCommit, String branchName) { + Path projectPath = Paths.get(project.getPath()) + + // Checkout to new branch + Utils.runGitCommand(projectPath, 'checkout', '-b', branchName, mergeCommit.getSHA()) + } + + private static void replaceFilesInProject(Project project, MergeCommit mergeCommit, List mergeScenarios, String toReplaceFile) { + mergeScenarios.stream() + .forEach(mergeScenario -> Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario), StandardCopyOption.REPLACE_EXISTING)) + } + + private static Path getSource(Path mergeScenario, String toReplaceFile) { + return mergeScenario.resolve(toReplaceFile) + } + + private static Path getTarget(Project project, MergeCommit mergeCommit, Path mergeScenario) { + Path projectPath = Paths.get(project.getPath()) + Path filePath = Utils.commitFilesPath(project, mergeCommit).relativize(mergeScenario) + return projectPath.resolve(filePath) + } + + private static void replaceTravisFile(Project project) { + Path projectPath = Paths.get(project.getPath()) + + Path travisFile = projectPath.resolve('.travis.yml') + BuildSystem buildSystem = getBuildSystem(projectPath) + + if (buildSystem != BuildSystem.None && !Files.exists(travisFile)) { + travisFile.toFile() << buildNewTravisFile(buildSystem) + } + } + + private static BuildSystem getBuildSystem(Path projectPath) { + if (Files.exists(projectPath.resolve('pom.xml'))) { + return BuildSystem.Maven + } + + if (Files.exists(projectPath.resolve('build.gradle'))) { + return BuildSystem.Gradle + } + + return BuildSystem.None + } + + private static String buildNewTravisFile(BuildSystem buildSystem) { + return """ +language: java + +jdk: + - openjdk8 + +script: + - ${buildScripts[buildSystem.name()]} +""" + } + + private static void stageAndPushChanges(Project project, String branchName) { + Path projectPath = Paths.get(project.getPath()) + + // Stage changes + Utils.runGitCommand(projectPath, 'add', '.') + + // Commit changes + Utils.runGitCommand(projectPath, 'commit', '-m', 'S3M Handlers Analysis new branch') + + // Push changes + Utils.runGitCommand(projectPath, 'push', '--set-upstream', 'origin', branchName, '--force-with-lease') + } + + +} diff --git a/src/main/services/util/MergeCommitSummary.groovy b/src/main/services/util/MergeCommitSummary.groovy new file mode 100644 index 000000000..b7c6ed7f3 --- /dev/null +++ b/src/main/services/util/MergeCommitSummary.groovy @@ -0,0 +1,66 @@ +package services.util + +class MergeCommitSummary { + + int numberOfModifiedFiles + List numberOfConflicts + boolean handlersHaveSameOutputs + boolean handlersHaveSameConflicts + Map checkingBuilds + + List mergeScenarioSummaries + + MergeCommitSummary() { + this.numberOfModifiedFiles = 0 + this.numberOfConflicts = [0, 0, 0, 0, 0] + this.handlersHaveSameOutputs = true + this.handlersHaveSameConflicts = true + this.mergeScenarioSummaries = [] + this.checkingBuilds = [:] + } + + /** + * Add the merge scenario summary's information to this summary. + * @param scenarioSummary + */ + void addMergeSummary(MergeScenarioSummary scenarioSummary) { + numberOfModifiedFiles++ + addConflicts(scenarioSummary.numberOfConflicts) + + handlersHaveSameOutputs &= !scenarioSummary.getDifferenceBetweenMergeResults().contains(false) + handlersHaveSameConflicts &= !scenarioSummary.getDifferenceBetweenConflictSets().contains(false) + + mergeScenarioSummaries.add(scenarioSummary) + } + + /** + * Add a link for a Travis build. + * @param link + * @param mergeAlgorithm + */ + void markAsChecking(String link, String mergeAlgorithm) { + checkingBuilds[mergeAlgorithm] = link + } + + private void addConflicts(List localNumberOfConflicts) { + for (int i = 0; i < numberOfConflicts.size(); i++) { + numberOfConflicts[i] += localNumberOfConflicts[i] + } + } + + @Override + String toString() { + return "${numberOfModifiedFiles},${Utils.toStringList(numberOfConflicts, ',')},${handlersHaveSameOutputs},${handlersHaveSameConflicts},${toStringMap(checkingBuilds, ',')},,," + } + + private static String toStringMap(Map map, String separator) { + StringBuilder string = new StringBuilder() + + map.each { key, value -> + string.append(Utils.getHyperLink(value, key)).append(separator) + } + + return string.toString() + } + +} diff --git a/src/main/services/util/MergeConflict.groovy b/src/main/services/util/MergeConflict.groovy new file mode 100644 index 000000000..d9ebe7aa1 --- /dev/null +++ b/src/main/services/util/MergeConflict.groovy @@ -0,0 +1,77 @@ +package services.util + +import org.apache.commons.io.FileUtils +@Grab(group = 'commons-io', module = 'commons-io', version = '2.6') +import org.apache.commons.lang3.StringUtils + +import java.nio.charset.Charset +import java.nio.file.Path + +class MergeConflict { + + enum ConflictArea { + None, + Left, + Right + } + + public static MINE_CONFLICT_MARKER = "<<<<<< extractMergeConflicts(Path file) { + Set mergeConflicts = new HashSet() + + StringBuilder leftConflictingContent = new StringBuilder() + StringBuilder rightConflictingContent = new StringBuilder() + + ConflictArea conflictArea + conflictArea = ConflictArea.None + + Iterator mergeCodeLines = FileUtils.readLines(file.toFile(), Charset.defaultCharset()).iterator() + while (mergeCodeLines.hasNext()) { + String line = mergeCodeLines.next() + + /* See the following conditionals as a state machine. */ + if (StringUtils.deleteWhitespace(line).contains(MINE_CONFLICT_MARKER) && conflictArea == ConflictArea.None) { + conflictArea = ConflictArea.Left + } else if (StringUtils.deleteWhitespace(line).contains(CHANGE_CONFLICT_MARKER) && conflictArea == ConflictArea.Left) { + conflictArea = ConflictArea.Right + } else if (StringUtils.deleteWhitespace(line).contains(YOURS_CONFLICT_MARKER) && conflictArea == ConflictArea.Right) { + mergeConflicts.add(new MergeConflict(leftConflictingContent.toString(), rightConflictingContent.toString())) + conflictArea = ConflictArea.None + } else { + switch (conflictArea) { + case ConflictArea.Left: + leftConflictingContent.append(line).append('\n') + break + case ConflictArea.Right: + rightConflictingContent.append(line).append('\n') + break + default: // not in conflict area + break + } + } + } + return mergeConflicts + } + +} diff --git a/src/main/services/util/MergeScenarioSummary.groovy b/src/main/services/util/MergeScenarioSummary.groovy new file mode 100644 index 000000000..02c819551 --- /dev/null +++ b/src/main/services/util/MergeScenarioSummary.groovy @@ -0,0 +1,81 @@ +package services.util + +import org.apache.commons.lang3.StringUtils + +import services.outputProcessors.S3MOutputProcessor +import util.Handlers + +import java.nio.file.Path + +class MergeScenarioSummary { + + Path mergeScenario + List numberOfConflicts + List differenceBetweenMergeResults + List differenceBetweenConflictSets + + MergeScenarioSummary(Path mergeScenario) { + Set TMConflicts = getMergeConflicts(mergeScenario, Handlers.mergeResultPaths[0]) + Set CTConflicts = getMergeConflicts(mergeScenario, Handlers.mergeResultPaths[1]) + Set SFConflicts = getMergeConflicts(mergeScenario, Handlers.mergeResultPaths[2]) + Set MMConflicts = getMergeConflicts(mergeScenario, Handlers.mergeResultPaths[3]) + Set KBConflicts = getMergeConflicts(mergeScenario, Handlers.mergeResultPaths[4]) + + this.mergeScenario = Utils.getOutputPath().relativize(mergeScenario) + + this.numberOfConflicts = [ + TMConflicts.size(), + CTConflicts.size(), + SFConflicts.size(), + MMConflicts.size(), + KBConflicts.size() + ] + + this.differenceBetweenConflictSets = [ + CTConflicts == SFConflicts, + CTConflicts == SFConflicts, + CTConflicts == SFConflicts, + SFConflicts == MMConflicts, + SFConflicts == KBConflicts, + MMConflicts == KBConflicts + ] + + String CTText = getFileText(mergeScenario, Handlers.mergeResultPaths[1]) + String SFText = getFileText(mergeScenario, Handlers.mergeResultPaths[2]) + String MMText = getFileText(mergeScenario, Handlers.mergeResultPaths[3]) + String KBText = getFileText(mergeScenario, Handlers.mergeResultPaths[4]) + + this.differenceBetweenMergeResults = [ + equalsModuloWhitespace(CTText, SFText), + equalsModuloWhitespace(CTText, MMText), + equalsModuloWhitespace(CTText, KBText), + equalsModuloWhitespace(SFText, MMText), + equalsModuloWhitespace(SFText, KBText), + equalsModuloWhitespace(MMText, KBText) + ] + + } + + private static boolean equalsModuloWhitespace(String s1, String s2) { + return StringUtils.deleteWhitespace(s1) == StringUtils.deleteWhitespace(s2) + } + + private static Set getMergeConflicts(Path mergeScenario, String toResolve) { + Path mergeFile = mergeScenario.resolve(toResolve) + return MergeConflict.extractMergeConflicts(mergeFile) + } + + private static String getFileText(Path mergeScenario, String toResolve) { + Path mergeFile = mergeScenario.resolve(toResolve) + return mergeFile.getText() + } + + @Override + String toString() { + String mergeScenarioLink = Utils.getHyperLink(S3MOutputProcessor.ANALYSIS_REMOTE_URL + '/' + mergeScenario.toString(), mergeScenario.getFileName().toString()) + + return "${mergeScenarioLink},${Utils.toStringList(numberOfConflicts, ',')},${Utils.toStringList(differenceBetweenMergeResults, ',')},${Utils.toStringList(differenceBetweenConflictSets, ',')}" + } + + +} diff --git a/src/main/services/util/Utils.groovy b/src/main/services/util/Utils.groovy new file mode 100644 index 000000000..68c04c0b5 --- /dev/null +++ b/src/main/services/util/Utils.groovy @@ -0,0 +1,76 @@ +package services.util + +import app.MiningFramework +import project.MergeCommit +import project.Project +import util.ProcessRunner + +import java.nio.file.Path +import java.nio.file.Paths + +final class Utils { + + /** + * Runs a git command, waiting for it to finish. + * @param repositoryPath + * @param arguments + */ + static void runGitCommand(Path repositoryPath, String... arguments) { + Process gitCommand = ProcessRunner.startProcess(buildGitCommand(repositoryPath, arguments)) + gitCommand.getInputStream().eachLine { + } + gitCommand.waitFor() + } + + private static ProcessBuilder buildGitCommand(Path repositoryPath, String... arguments) { + ProcessBuilder gitCommand = ProcessRunner.buildProcess(repositoryPath.toString(), 'git') + gitCommand.command().addAll(arguments.toList()) + return gitCommand + } + + /** + * Equivalent to Paths.get(MiningFramework.arguments.getOutputPath()) + * @return a path to the output path given as argument + */ + static Path getOutputPath() { + return Paths.get(MiningFramework.arguments.getOutputPath()) + } + + /** + * @param project + * @param mergeCommit + * @return the output path resolved in the project/merge commit directory + */ + static Path commitFilesPath(Project project, MergeCommit mergeCommit) { + return getOutputPath().resolve(project.getName()).resolve(mergeCommit.getSHA()) + } + + /** + * @param list + * @param separator + * @return a concatenation of all the string representation of the elements of the list, separated by the separator + */ + static String toStringList(List list, String separator) { + if (list.isEmpty()) + return '' + + StringBuilder string = new StringBuilder() + + for (int i = 0; i < list.size() - 1; i++) { + string.append(list.get(i).toString()).append(separator) + } + string.append(list.last().toString()) + + return string.toString() + } + + /** + * @param link + * @param name + * @return a link in the format required by Google Sheets for hyperlinks, using {@code link} + * as link and {@code name} as its name in the cell + */ + static String getHyperLink(String link, String name) { + return "=HYPERLINK(\"${link}\";\"${name}\")" + } +} diff --git a/src/main/util/Handlers.groovy b/src/main/util/Handlers.groovy new file mode 100644 index 000000000..d32f17cd9 --- /dev/null +++ b/src/main/util/Handlers.groovy @@ -0,0 +1,9 @@ +package util + +enum Handlers { + Renaming + + static final Map mergeResultPaths = [0: 'textual.java', 1: 'Renaming/CT.java', 2: 'Renaming/SF.java', 3: 'Renaming/MM.java', 4: 'Renaming/KB.java'] + static final Map mergeAlgorithms = [0: 'TM', 1: 'CT', 2: 'SF', 3: 'MM', 4: 'KB'] + +} \ No newline at end of file