Merge remote-tracking branch 'origin/develop' into feature/internal-w…

…eb-viewer

# Conflicts:
#	cli/src/main/java/de/jplag/cli/CLI.java

.github/workflows/publish.yml

@@ -13,7 +13,7 @@ jobs:
           java-version: '21'
           distribution: 'temurin'
       - name: Set maven settings.xml
-        uses: whelk-io/maven-settings-xml-action@v21
+        uses: whelk-io/maven-settings-xml-action@v22
         with:
           servers: '[{ "id": "ossrh", "username": "jplag", "password": "${{ secrets.OSSRH_TOKEN }}" }]'
       - name: Import GPG key

README.md

@@ -22,8 +22,8 @@ In the following, a list of all supported languages with their supported languag
 | Language                                               |                                                                                Version | CLI Argument Name | [state](https://github.com/jplag/JPlag/wiki/2.-Supported-Languages) |  parser   |
 |--------------------------------------------------------|---------------------------------------------------------------------------------------:|-------------------|:-------------------------------------------------------------------:|:---------:|
 | [Java](https://www.java.com)                           |                                                                                     21 | java              |                               mature                                |   JavaC   |
-| [C/C++](https://isocpp.org)                            |                                                                                     11 | cpp               |                               legacy                                |  JavaCC   |
-| [C/C++](https://isocpp.org)                            |                                                                                     14 | cpp2              |                                beta                                 |  ANTLR 4  |
+| [C](https://isocpp.org)                                |                                                                                     11 | c                 |                               legacy                                |  JavaCC   |
+| [C++](https://isocpp.org)                              |                                                                                     14 | cpp               |                                beta                                 |  ANTLR 4  |
 | [C#](https://docs.microsoft.com/en-us/dotnet/csharp/)  |                                                                                      6 | csharp            |                                beta                                 |  ANTLR 4  |
 | [Go](https://go.dev)                                   |                                                                                   1.17 | golang            |                                beta                                 |  ANTLR 4  |
 | [Kotlin](https://kotlinlang.org)                       |                                                                                    1.3 | kotlin            |                                beta                                 |  ANTLR 4  |
@@ -147,8 +147,8 @@ Clustering
 
       --cluster-skip   Skips the clustering (default: false)
 Commands:
+  c
   cpp
-  cpp2
   csharp
   emf
   emf-model

cli/pom.xml

@@ -45,12 +45,12 @@
         </dependency>
         <dependency>
             <groupId>de.jplag</groupId>
-            <artifactId>cpp</artifactId>
+            <artifactId>c</artifactId>
             <version>${revision}</version>
         </dependency>
         <dependency>
             <groupId>de.jplag</groupId>
-            <artifactId>cpp2</artifactId>
+            <artifactId>cpp</artifactId>
             <version>${revision}</version>
         </dependency>
         <dependency>
@@ -129,6 +129,12 @@
             <artifactId>picocli</artifactId>
             <version>4.7.5</version>
         </dependency>
+
+        <dependency>
+            <groupId>me.tongfei</groupId>
+            <artifactId>progressbar</artifactId>
+            <version>0.10.0</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

cli/src/main/java/de/jplag/cli/CLI.java

@@ -26,9 +26,11 @@
 import de.jplag.Language;
 import de.jplag.cli.logger.CollectedLoggerFactory;
 import de.jplag.cli.server.ReportViewer;
+import de.jplag.cli.logger.TongfeiProgressBarProvider;
 import de.jplag.clustering.ClusteringOptions;
 import de.jplag.clustering.Preprocessing;
 import de.jplag.exceptions.ExitException;
+import de.jplag.logging.ProgressBarLogger;
 import de.jplag.merging.MergingOptions;
 import de.jplag.options.JPlagOptions;
 import de.jplag.options.LanguageOption;
@@ -68,6 +70,8 @@ public final class CLI {
 
     private static final String DESCRIPTION_PATTERN = "%nJPlag - %s%n%s%n%n";
 
+    private static final String DEFAULT_FILE_ENDING = ".zip";
+
     /**
      * Main class for using JPlag via the CLI.
      * @param args are the CLI arguments that will be passed to JPlag.
@@ -81,14 +85,20 @@ public static void main(String[] args) {
             ParseResult parseResult = cli.parseOptions(args);
 
             if (!parseResult.isUsageHelpRequested() && !(parseResult.subcommand() != null && parseResult.subcommand().isUsageHelpRequested())) {
+                ProgressBarLogger.setProgressBarProvider(new TongfeiProgressBarProvider());
                 switch (cli.options.mode) {
                     case RUN -> cli.runJPlag(parseResult);
                     case VIEW -> cli.runViewer(null);
                     case RUN_AND_VIEW -> cli.runViewer(cli.runJPlag(parseResult));
                 }
             }
-        } catch (ExitException | IOException exception) {
-            logger.error(exception.getMessage()); // do not pass exception here to keep log clean
+        } catch (ExitException | IOException exception) { // do not pass exceptions here to keep log clean
+            if (exception.getCause() != null) {
+                logger.error("{} - {}", exception.getMessage(), exception.getCause().getMessage());
+            } else {
+                logger.error(exception.getMessage());
+            }
+
             finalizeLogger();
             System.exit(1);
         }
@@ -106,9 +116,8 @@ public CLI() {
         this.commandLine.getHelpSectionMap().put(SECTION_KEY_OPTION_LIST, help -> help.optionList().lines().map(it -> {
             if (it.startsWith("  -")) {
                 return "    " + it;
-            } else {
-                return it;
             }
+            return it;
         }).collect(Collectors.joining(System.lineSeparator())));
 
         buildSubcommands().forEach(commandLine::addSubcommand);
@@ -121,10 +130,10 @@ public CLI() {
     public File runJPlag(ParseResult parseResult) throws ExitException, FileNotFoundException {
         JPlagOptions jplagOptions = buildOptionsFromArguments(parseResult);
         JPlagResult result = JPlag.run(jplagOptions);
-        File target = new File(getResultFolder() + ".zip");
+        File target = new File(getResultFilePath());
         ReportObjectFactory reportObjectFactory = new ReportObjectFactory(target);
         reportObjectFactory.createAndSaveReport(result);
-        OutputFileGenerator.generateCsvOutput(result, new File(getResultFolder()), this.options);
+        OutputFileGenerator.generateCsvOutput(result, new File(getResultFileBaseName()), this.options);
         return target;
     }
 
@@ -168,7 +177,7 @@ public ParseResult parseOptions(String... args) throws CliException {
             }
             return result;
         } catch (CommandLine.ParameterException e) {
-            if (e.getArgSpec().isOption() && Arrays.asList(((OptionSpec) e.getArgSpec()).names()).contains("-l")) {
+            if (e.getArgSpec() != null && e.getArgSpec().isOption() && Arrays.asList(((OptionSpec) e.getArgSpec()).names()).contains("-l")) {
                 throw new CliException(String.format(UNKOWN_LANGAUGE_EXCEPTION, e.getValue(),
                         String.join(", ", LanguageLoader.getAllAvailableLanguageIdentifiers())));
             }
@@ -206,33 +215,31 @@ public JPlagOptions buildOptionsFromArguments(ParseResult parseResult) throws Cl
         JPlagOptions jPlagOptions = new JPlagOptions(loadLanguage(parseResult), this.options.minTokenMatch, submissionDirectories,
                 oldSubmissionDirectories, null, this.options.advanced.subdirectory, suffixes, this.options.advanced.exclusionFileName,
                 JPlagOptions.DEFAULT_SIMILARITY_METRIC, this.options.advanced.similarityThreshold, this.options.shownComparisons, clusteringOptions,
-                this.options.advanced.debug, mergingOptions);
+                this.options.advanced.debug, mergingOptions, this.options.normalize);
 
         String baseCodePath = this.options.baseCode;
         File baseCodeDirectory = baseCodePath == null ? null : new File(baseCodePath);
         if (baseCodeDirectory == null || baseCodeDirectory.exists()) {
             return jPlagOptions.withBaseCodeSubmissionDirectory(baseCodeDirectory);
-        } else {
-            logger.warn("Using legacy partial base code API. Please migrate to new full path base code API.");
-            return jPlagOptions.withBaseCodeSubmissionName(baseCodePath);
         }
+        logger.warn("Using legacy partial base code API. Please migrate to new full path base code API.");
+        return jPlagOptions.withBaseCodeSubmissionName(baseCodePath);
     }
 
     private Language loadLanguage(ParseResult result) throws CliException {
-        if (result.subcommand() != null) {
-            ParseResult subcommandResult = result.subcommand();
-            Language language = LanguageLoader.getLanguage(subcommandResult.commandSpec().name())
-                    .orElseThrow(() -> new CliException(IMPOSSIBLE_EXCEPTION));
-            LanguageOptions languageOptions = language.getOptions();
-            languageOptions.getOptionsAsList().forEach(option -> {
-                if (subcommandResult.hasMatchedOption(option.getNameAsUnixParameter())) {
-                    option.setValue(subcommandResult.matchedOptionValue(option.getNameAsUnixParameter(), null));
-                }
-            });
-            return language;
-        } else {
+        if (result.subcommand() == null) {
             return this.options.language;
         }
+        ParseResult subcommandResult = result.subcommand();
+        Language language = LanguageLoader.getLanguage(subcommandResult.commandSpec().name())
+                .orElseThrow(() -> new CliException(IMPOSSIBLE_EXCEPTION));
+        LanguageOptions languageOptions = language.getOptions();
+        languageOptions.getOptionsAsList().forEach(option -> {
+            if (subcommandResult.hasMatchedOption(option.getNameAsUnixParameter())) {
+                option.setValue(subcommandResult.matchedOptionValue(option.getNameAsUnixParameter(), null));
+            }
+        });
+        return language;
     }
 
     private static ClusteringOptions getClusteringOptions(CliOptions options) {
@@ -274,7 +281,17 @@ private String generateDescription() {
         return String.format(DESCRIPTION_PATTERN, randomDescription, CREDITS);
     }
 
-    public String getResultFolder() {
-        return this.options.resultFolder;
+    private String getResultFilePath() {
+        String optionValue = this.options.resultFile;
+        if (optionValue.endsWith(DEFAULT_FILE_ENDING)) {
+            return optionValue;
+        } else {
+            return optionValue + DEFAULT_FILE_ENDING;
+        }
+    }
+
+    private String getResultFileBaseName() {
+        String defaultOutputFile = getResultFilePath();
+        return defaultOutputFile.substring(0, defaultOutputFile.length() - DEFAULT_FILE_ENDING.length());
     }
 }

cli/src/main/java/de/jplag/cli/CliOptions.java

@@ -7,6 +7,7 @@
 import de.jplag.clustering.ClusteringOptions;
 import de.jplag.clustering.algorithm.InterClusterSimilarity;
 import de.jplag.java.JavaLanguage;
+import de.jplag.merging.MergingOptions;
 import de.jplag.options.JPlagOptions;
 import de.jplag.options.SimilarityMetric;
 
@@ -50,8 +51,8 @@ public class CliOptions implements Runnable {
     public int shownComparisons = JPlagOptions.DEFAULT_SHOWN_COMPARISONS;
 
     @Option(names = {"-r",
-            "--result-directory"}, description = "Name of the directory in which the comparison results will be stored (default: ${DEFAULT-VALUE})%n")
-    public String resultFolder = "results";
+            "--result-file"}, description = "Name of the file in which the comparison results will be stored (default: ${DEFAULT-VALUE}). Missing .zip endings will be automatically added.%n")
+    public String resultFile = "results";
 
     @Option(names = {"--mode"}, description = "The mode to run jplag in")
     public JPlagMode mode = JPlagMode.RUN;
@@ -65,6 +66,9 @@ public class CliOptions implements Runnable {
     @ArgGroup(validate = false, heading = "Merging of neighboring matches to increase the similarity of concealed plagiarism:%n")
     public Merging merging = new Merging();
 
+    @Option(names = {"--normalize"}, description = "Activate the normalization of tokens. Supported for languages: Java, C++.")
+    public boolean normalize = false;
+
     /**
      * Empty run method, so picocli prints help automatically
      */
@@ -122,14 +126,15 @@ public static class ClusteringEnabled {
     }
 
     public static class Merging {
-        @Option(names = {"--match-merging"}, description = "Enables match merging (default: false)%n")
-        public boolean enabled;
+        @Option(names = {"--match-merging"}, description = "Enables match merging (default: ${DEFAULT-VALUE})%n")
+        public boolean enabled = MergingOptions.DEFAULT_ENABLED;
 
-        @Option(names = {"--neighbor-length"}, description = "Defines how short a match can be, to be considered (default: 2)%n")
-        public int minimumNeighborLength;
+        @Option(names = {"--neighbor-length"}, description = "Defines how short a match can be, to be considered (default: ${DEFAULT-VALUE})%n")
+        public int minimumNeighborLength = MergingOptions.DEFAULT_NEIGHBOR_LENGTH;
 
-        @Option(names = {"--gap-size"}, description = "Defines how many token there can be between two neighboring matches (default: 6)%n")
-        public int maximumGapSize;
+        @Option(names = {
+                "--gap-size"}, description = "Defines how many token there can be between two neighboring matches (default: ${DEFAULT-VALUE})%n")
+        public int maximumGapSize = MergingOptions.DEFAULT_GAP_SIZE;
 
     }
 

cli/src/main/java/de/jplag/cli/LanguageLoader.java

@@ -32,8 +32,9 @@ private LanguageLoader() {
      * @return the languages as unmodifiable map from identifier to language instance.
      */
     public static synchronized Map<String, Language> getAllAvailableLanguages() {
-        if (cachedLanguageInstances != null)
+        if (cachedLanguageInstances != null) {
             return cachedLanguageInstances;
+        }
 
         Map<String, Language> languages = new TreeMap<>();
 
@@ -61,8 +62,9 @@ public static synchronized Map<String, Language> getAllAvailableLanguages() {
      */
     public static Optional<Language> getLanguage(String identifier) {
         var language = getAllAvailableLanguages().get(identifier);
-        if (language == null)
+        if (language == null) {
             logger.warn("Attempt to load Language {} was not successful", identifier);
+        }
         return Optional.ofNullable(language);
     }
 

cli/src/main/java/de/jplag/cli/logger/TongfeiProgressBar.java

@@ -0,0 +1,24 @@
+package de.jplag.cli.logger;
+
+import de.jplag.logging.ProgressBar;
+
+/**
+ * A ProgressBar, that used the tongfei progress bar library underneath, to show progress bars on the cli.
+ */
+public class TongfeiProgressBar implements ProgressBar {
+    private final me.tongfei.progressbar.ProgressBar progressBar;
+
+    public TongfeiProgressBar(me.tongfei.progressbar.ProgressBar progressBar) {
+        this.progressBar = progressBar;
+    }
+
+    @Override
+    public void step(int number) {
+        this.progressBar.stepBy(number);
+    }
+
+    @Override
+    public void dispose() {
+        this.progressBar.close();
+    }
+}

cli/src/main/java/de/jplag/cli/logger/TongfeiProgressBarProvider.java

@@ -0,0 +1,20 @@
+package de.jplag.cli.logger;
+
+import de.jplag.logging.ProgressBar;
+import de.jplag.logging.ProgressBarProvider;
+import de.jplag.logging.ProgressBarType;
+
+import me.tongfei.progressbar.ProgressBarBuilder;
+import me.tongfei.progressbar.ProgressBarStyle;
+
+/**
+ * A ProgressBar provider, that used the tongfei progress bar library underneath, to show progress bars on the cli.
+ */
+public class TongfeiProgressBarProvider implements ProgressBarProvider {
+    @Override
+    public ProgressBar initProgressBar(ProgressBarType type, int totalSteps) {
+        me.tongfei.progressbar.ProgressBar progressBar = new ProgressBarBuilder().setTaskName(type.getDefaultText()).setInitialMax(totalSteps)
+                .setStyle(ProgressBarStyle.UNICODE_BLOCK).build();
+        return new TongfeiProgressBar(progressBar);
+    }
+}

cli/src/test/java/de/jplag/cli/MergingOptionsTest.java

@@ -0,0 +1,25 @@
+package de.jplag.cli;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import de.jplag.merging.MergingOptions;
+
+/**
+ * Test cases for the options of the match merging mechanism.
+ */
+class MergingOptionsTest extends CommandLineInterfaceTest {
+
+    @Test
+    @DisplayName("Test if default values are used when creating merging options from CLI")
+    void testMergingDefault() throws CliException {
+        buildOptionsFromCLI(defaultArguments());
+        assertNotNull(options.mergingOptions());
+        assertEquals(MergingOptions.DEFAULT_ENABLED, options.mergingOptions().enabled());
+        assertEquals(MergingOptions.DEFAULT_NEIGHBOR_LENGTH, options.mergingOptions().minimumNeighborLength());
+        assertEquals(MergingOptions.DEFAULT_GAP_SIZE, options.mergingOptions().maximumGapSize());
+    }
+}

core/src/main/java/de/jplag/JPlag.java

@@ -71,6 +71,9 @@ public static JPlagResult run(JPlagOptions options) throws ExitException {
         // Parse and validate submissions.
         SubmissionSetBuilder builder = new SubmissionSetBuilder(options);
         SubmissionSet submissionSet = builder.buildSubmissionSet();
+        if (options.normalize() && options.language().supportsNormalization() && options.language().requiresCoreNormalization()) {
+            submissionSet.normalizeSubmissions();
+        }
         int submissionCount = submissionSet.numberOfSubmissions();
         if (submissionCount < 2)
             throw new SubmissionException("Not enough valid submissions! (found " + submissionCount + " valid submissions)");
@@ -103,6 +106,10 @@ private static void logSkippedSubmissions(SubmissionSet submissionSet, JPlagOpti
     }
 
     private static void checkForConfigurationConsistency(JPlagOptions options) throws RootDirectoryException {
+        if (options.normalize() && !options.language().supportsNormalization()) {
+            logger.error(String.format("The language %s cannot be used with normalization.", options.language().getName()));
+        }
+
         List<String> duplicateNames = getDuplicateSubmissionFolderNames(options);
         if (duplicateNames.size() > 0) {
             throw new RootDirectoryException(String.format("Duplicate root directory names found: %s", String.join(", ", duplicateNames)));