Skip to content

Commit

Permalink
Merge pull request #2087 from TwoOfTwelve/feature/bachelorAlex-multiL…
Browse files Browse the repository at this point in the history
…anguage

Multi Language Program Support
  • Loading branch information
robinmaisch authored Dec 4, 2024
2 parents 918d9cf + f660c08 commit 0774cc7
Show file tree
Hide file tree
Showing 16 changed files with 263 additions and 9 deletions.
5 changes: 5 additions & 0 deletions cli/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,11 @@
<artifactId>llvmir</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>de.jplag</groupId>
<artifactId>multi-language</artifactId>
<version>${revision}</version>
</dependency>
<!-- CLI -->
<dependency>
<groupId>org.kohsuke.metainf-services</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import java.util.ArrayList;

import de.jplag.LanguageLoader;

/**
* Helper class for picocli to find all available languages.
*/
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package de.jplag.cli.options;

import de.jplag.Language;
import de.jplag.LanguageLoader;

import picocli.CommandLine;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
import java.util.stream.Collectors;

import de.jplag.Language;
import de.jplag.LanguageLoader;
import de.jplag.cli.CliException;
import de.jplag.cli.options.CliOptions;
import de.jplag.cli.options.LanguageLoader;
import de.jplag.options.LanguageOption;
import de.jplag.options.LanguageOptions;

Expand Down
9 changes: 6 additions & 3 deletions cli/src/test/java/de/jplag/cli/LanguageTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,16 @@
import org.junit.jupiter.params.provider.MethodSource;

import de.jplag.Language;
import de.jplag.LanguageLoader;
import de.jplag.cli.options.CliOptions;
import de.jplag.cli.options.LanguageLoader;
import de.jplag.cli.test.CliArgument;
import de.jplag.cli.test.CliTest;
import de.jplag.exceptions.ExitException;
import de.jplag.multilang.MultiLanguage;
import de.jplag.options.JPlagOptions;

class LanguageTest extends CliTest {
private static final List<Class<? extends Language>> ignoredLanguages = List.of(MultiLanguage.class);

@Test
void testDefaultLanguage() throws ExitException, IOException {
Expand All @@ -38,7 +40,7 @@ void testInvalidLanguage() {
@Test
void testLoading() {
var languages = LanguageLoader.getAllAvailableLanguages();
assertEquals(19, languages.size(), "Loaded Languages: " + languages.keySet());
assertEquals(20, languages.size(), "Loaded Languages: " + languages.keySet());
}

@ParameterizedTest
Expand All @@ -58,6 +60,7 @@ void testCustomSuffixes() throws ExitException, IOException {
}

public static Collection<Language> getAllLanguages() {
return LanguageLoader.getAllAvailableLanguages().values();
return LanguageLoader.getAllAvailableLanguages().values().stream().filter(language -> !ignoredLanguages.contains(language.getClass()))
.toList();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import java.io.IOException;

import de.jplag.Language;
import de.jplag.cli.options.LanguageLoader;
import de.jplag.LanguageLoader;

import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package de.jplag.cli.options;
package de.jplag;

import java.util.Collections;
import java.util.Map;
Expand All @@ -11,8 +11,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import de.jplag.Language;

/**
* This class contains methods to load {@link Language Languages}.
* @author Dominik Fuchss
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public class DefaultLanguageOption<T> implements LanguageOption<T> {
this.hasValue = true;
}

DefaultLanguageOption(OptionType<T> type, String description, String name) {
DefaultLanguageOption(OptionType<T> type, String name, String description) {
this(type, name, description, null);
this.hasValue = false;
}
Expand Down
26 changes: 26 additions & 0 deletions languages/multi-language/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>de.jplag</groupId>
<artifactId>languages</artifactId>
<version>${revision}</version>
</parent>
<artifactId>multi-language</artifactId>

<dependencies>
<dependency>
<groupId>de.jplag</groupId>
<artifactId>java</artifactId>
<version>${revision}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>de.jplag</groupId>
<artifactId>cpp</artifactId>
<version>${revision}</version>
<scope>test</scope>
</dependency>
</dependencies>

</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package de.jplag.multilang;

import java.io.File;
import java.util.Arrays;
import java.util.List;
import java.util.Set;

import org.kohsuke.MetaInfServices;

import de.jplag.Language;
import de.jplag.LanguageLoader;
import de.jplag.ParsingException;
import de.jplag.Token;
import de.jplag.options.LanguageOptions;

@MetaInfServices(Language.class)
public class MultiLanguage implements Language {
private final MultiLanguageOptions options;

public MultiLanguage() {
this.options = new MultiLanguageOptions();
}

@Override
public String[] suffixes() {
return LanguageLoader.getAllAvailableLanguages().values().stream().filter(it -> it != this).flatMap(it -> Arrays.stream(it.suffixes()))
.toArray(String[]::new);
}

@Override
public String getName() {
return "multi-language";
}

@Override
public String getIdentifier() {
return "multi";
}

@Override
public int minimumTokenMatch() {
return this.options.getLanguages().stream().mapToInt(Language::minimumTokenMatch).min().orElse(9);
}

@Override
public List<Token> parse(Set<File> files, boolean normalize) throws ParsingException {
MultiLanguageParser parser = new MultiLanguageParser(this.options);
return parser.parseFiles(files, normalize);
}

@Override
public LanguageOptions getOptions() {
return this.options;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package de.jplag.multilang;

import java.util.Arrays;
import java.util.List;

import de.jplag.Language;
import de.jplag.LanguageLoader;
import de.jplag.options.LanguageOption;
import de.jplag.options.LanguageOptions;
import de.jplag.options.OptionType;

public class MultiLanguageOptions extends LanguageOptions {
private static final String ERROR_LANGUAGE_NOT_FOUND = "The selected language %s could not be found";
private static final String ERROR_NOT_ENOUGH_LANGUAGES = "To use multi language specify at least 1 language";
private static final String OPTION_DESCRIPTION_LANGUAGES = "The languages that should be used. This is a ',' separated list";

private final LanguageOption<String> languageNames = createOption(OptionType.string(), "languages", OPTION_DESCRIPTION_LANGUAGES);
private List<Language> languages = null;

public List<Language> getLanguages() {
if (this.languages == null) {
if (languageNames.getValue() == null) {
throw new IllegalArgumentException(ERROR_NOT_ENOUGH_LANGUAGES);
}

this.languages = Arrays.stream(languageNames.getValue().split(","))
.map(name -> LanguageLoader.getLanguage(name)
.orElseThrow(() -> new IllegalArgumentException(String.format(ERROR_LANGUAGE_NOT_FOUND, name))))
.filter(language -> !language.getClass().equals(MultiLanguage.class)).toList();

if (this.languages.isEmpty()) {
throw new IllegalArgumentException(ERROR_NOT_ENOUGH_LANGUAGES);
}
}

return this.languages;
}

public LanguageOption<String> getLanguageNames() {
return this.languageNames;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package de.jplag.multilang;

import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.Set;

import de.jplag.Language;
import de.jplag.ParsingException;
import de.jplag.Token;

public class MultiLanguageParser {
private final List<Language> languages;

public MultiLanguageParser(MultiLanguageOptions options) {
this.languages = options.getLanguages();
}

public List<Token> parseFiles(Set<File> files, boolean normalize) throws ParsingException {
List<Token> results = new ArrayList<>();
for (File file : files) {
Optional<Language> language = findLanguageForFile(file);
if (language.isPresent()) {
results.addAll(language.get().parse(Set.of(file), normalize));
}
}
return results;
}

private Optional<Language> findLanguageForFile(File file) {
return this.languages.stream().filter(language -> Arrays.stream(language.suffixes()).anyMatch(suffix -> file.getName().endsWith(suffix)))
.findFirst();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package de.java.multilang;

import static de.jplag.SharedTokenType.FILE_END;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;

import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;

import de.jplag.ParsingException;
import de.jplag.Token;
import de.jplag.TokenType;
import de.jplag.cpp.CPPTokenType;
import de.jplag.java.JavaTokenType;
import de.jplag.multilang.MultiLanguage;
import de.jplag.multilang.MultiLanguageOptions;

class MultilangTest {
private static File testDataDirectory;
private static File javaCode;
private static File cppCode;

private static List<TokenType> expectedTokens = List.of(CPPTokenType.FUNCTION_BEGIN, CPPTokenType.RETURN, CPPTokenType.FUNCTION_END, FILE_END,
JavaTokenType.J_CLASS_BEGIN, JavaTokenType.J_CLASS_END, FILE_END);

@BeforeAll
static void setUp() throws IOException {
testDataDirectory = Files.createTempDirectory("multiLanguageTestData").toFile();
cppCode = new File(testDataDirectory, "CppCode.cpp");
javaCode = new File(testDataDirectory, "JavaCode.java");

MultilangTest.class.getResourceAsStream("/de/jplag/multilang/testDataSet/CppCode.cpp").transferTo(new FileOutputStream(cppCode));
MultilangTest.class.getResourceAsStream("/de/jplag/multilang/testDataSet/JavaCode.java").transferTo(new FileOutputStream(javaCode));
}

@Test
void testMultiLanguageParsing() throws ParsingException {
MultiLanguage languageModule = new MultiLanguage();
((MultiLanguageOptions) languageModule.getOptions()).getLanguageNames().setValue("java,cpp");

Set<File> sources = new TreeSet<>(List.of(javaCode, cppCode)); // Using TreeSet to ensure order of entries
List<Token> tokens = languageModule.parse(sources, false);

Assertions.assertEquals(expectedTokens, tokens.stream().map(Token::getType).toList());
}

@Test
void testNoLanguagesConfigured() {
MultiLanguage languageModule = new MultiLanguage();
Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
languageModule.parse(Set.of(javaCode, cppCode), false);
});
}

@Test
void testInvalidLanguage() {
MultiLanguage languageModule = new MultiLanguage();
((MultiLanguageOptions) languageModule.getOptions()).getLanguageNames().setValue("thisIsNotALanguage");

Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
languageModule.parse(Set.of(javaCode, cppCode), false);
});
}

@AfterAll
static void cleanUp() {
javaCode.delete();
cppCode.delete();
testDataDirectory.delete();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
int main() {
return 0;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
public class JavaCode {

}
1 change: 1 addition & 0 deletions languages/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
<module>typescript</module>
<module>javascript</module>
<module>llvmir</module>
<module>multi-language</module>
</modules>
<dependencies>
<dependency>
Expand Down

0 comments on commit 0774cc7

Please sign in to comment.