From dc43deda3bf517b1f5be1f5bbedde1d39fa5bde7 Mon Sep 17 00:00:00 2001 From: CodingPF Date: Sat, 14 Oct 2023 00:54:09 +0200 Subject: [PATCH] init importfilmlists --- MServer-Config.yaml | 19 ++- .../config/ImportFilmlistConfiguration.java | 67 +++++++++ .../mserver/base/config/MServerConfigDTO.java | 39 ++---- .../mserver/crawler/CrawlerManager.java | 116 ++++++++++++++-- .../mserver/ui/config/MServerConfigUI.java | 4 +- .../CrawlerManagerImportFilmlistsTest.java | 129 ++++++++++++++++++ .../crawler/CrawlerManagerLivestreamTest.java | 3 +- .../mserver/crawler/CrawlerManagerTest.java | 3 +- .../importFilmlist/FilmlistImportTest1.json | 7 + .../importFilmlist/FilmlistImportTest2.json | 6 + .../importFilmlist/FilmlistImportTest3.json | 7 + 11 files changed, 351 insertions(+), 49 deletions(-) create mode 100644 src/main/java/de/mediathekview/mserver/base/config/ImportFilmlistConfiguration.java create mode 100644 src/test/java/de/mediathekview/mserver/crawler/CrawlerManagerImportFilmlistsTest.java create mode 100644 src/test/resources/filmlists/importFilmlist/FilmlistImportTest1.json create mode 100644 src/test/resources/filmlists/importFilmlist/FilmlistImportTest2.json create mode 100644 src/test/resources/filmlists/importFilmlist/FilmlistImportTest3.json diff --git a/MServer-Config.yaml b/MServer-Config.yaml index f76c79ef4..bcc248ce9 100644 --- a/MServer-Config.yaml +++ b/MServer-Config.yaml @@ -83,8 +83,25 @@ writeFilmlistIdFileEnabled: true #The fimlist id file path filmlistIdFilePath: target/filmlists/filmlist.id + +# import additional filmlist sources +importFilmlistConfigurations : + - active: false + path: "someCrawlerlist.json" + format: OLD_JSON + createDiff: false + - active: false + path: "filme-org.xz" + format: OLD_JSON_COMPRESSED_XZ + createDiff: true + +# filmlistImporEnabled = true; +# filmlistImportFormat = FilmlistFormats.OLD_JSON_COMPRESSED_XZ; +# filmlistImportLocation = "https://verteiler1.mediathekview.de/Filmliste-akt.xz"; + + # Sets if a filmlist should be imported -filmlistImporEnabled: false +#filmlistImporEnabled: false # The format of the film list to import. # Possible are: JSON, OLD_JSON, JSON_COMPRESSED_XZ, OLD_JSON_COMPRESSED_XZ, JSON_COMPRESSED_GZIP, OLD_JSON_COMPRESSED_BZIP, JSON_COMPRESSED_GZIP, OLD_JSON_COMPRESSED_BZIP diff --git a/src/main/java/de/mediathekview/mserver/base/config/ImportFilmlistConfiguration.java b/src/main/java/de/mediathekview/mserver/base/config/ImportFilmlistConfiguration.java new file mode 100644 index 000000000..7fa8a1258 --- /dev/null +++ b/src/main/java/de/mediathekview/mserver/base/config/ImportFilmlistConfiguration.java @@ -0,0 +1,67 @@ +package de.mediathekview.mserver.base.config; + +import java.util.Objects; + +import de.mediathekview.mlib.filmlisten.FilmlistFormats; + +public class ImportFilmlistConfiguration { + private final Boolean active; + private final String path; + private final FilmlistFormats format; + private final Boolean createDiff; + + + public ImportFilmlistConfiguration(Boolean active, String path, FilmlistFormats format, Boolean createDiff) { + this.active = active; + this.path = path; + this.format = format; + this.createDiff = createDiff; + } + + public ImportFilmlistConfiguration() { + this.active = null; + this.path = null; + this.format = null; + this.createDiff = null; + } + + public Boolean isActive() { + return active; + } + public String getPath() { + return path; + } + public FilmlistFormats getFormat() { + return format; + } + public Boolean isCreateDiff() { + return createDiff; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof final ImportFilmlistConfiguration that)) { + return false; + } + if (!super.equals(obj)) { + return false; + } + return Objects.equals(isActive(), that.isActive()) + && Objects.equals(getPath(), that.getPath()) + && Objects.equals(getFormat(), that.getFormat()) + && Objects.equals(isCreateDiff(), that.isCreateDiff()); + } + + @Override + public int hashCode() { + return Objects.hash( + super.hashCode(), + isActive(), + getPath(), + getFormat(), + isCreateDiff()); + } +} diff --git a/src/main/java/de/mediathekview/mserver/base/config/MServerConfigDTO.java b/src/main/java/de/mediathekview/mserver/base/config/MServerConfigDTO.java index c23f4b762..342921b1f 100644 --- a/src/main/java/de/mediathekview/mserver/base/config/MServerConfigDTO.java +++ b/src/main/java/de/mediathekview/mserver/base/config/MServerConfigDTO.java @@ -18,6 +18,8 @@ public class MServerConfigDTO extends MServerBasicConfigDTO implements ConfigDTO private final String ignoreFilmlistPath; /** add livestreams from external list **/ private final ImportLivestreamConfiguration importLivestreamConfiguration; + /** add additional filmlist from external **/ + private final List ImportFilmlistConfigurations; /** The maximum amount of cpu threads to be used. */ private Integer maximumCpuThreads; /** @@ -32,12 +34,10 @@ public class MServerConfigDTO extends MServerBasicConfigDTO implements ConfigDTO private Set filmlistSaveFormats; private Map filmlistSavePaths; private Map filmlistDiffSavePaths; - private FilmlistFormats filmlistImportFormat; private String filmlistImportLocation; private MServerLogSettingsDTO logSettings; private Map crawlerURLs; private Map crawlerApiParams; - private Boolean filmlistImporEnabled; public MServerConfigDTO() { super(); @@ -79,16 +79,14 @@ public MServerConfigDTO() { filmlistSavePaths.put(FilmlistFormats.JSON_COMPRESSED_BZIP, "filmliste.json.bz"); filmlistSavePaths.put(FilmlistFormats.OLD_JSON_COMPRESSED_BZIP, "filmliste_old.json.bz"); - filmlistImporEnabled = true; - filmlistImportFormat = FilmlistFormats.OLD_JSON_COMPRESSED_XZ; - filmlistImportLocation = "https://verteiler1.mediathekview.de/Filmliste-akt.xz"; - writeFilmlistHashFileEnabled = false; filmlistHashFilePath = "filmlist.hash"; writeFilmlistIdFileEnabled = true; filmlistIdFilePath = "filmlist.id"; ignoreFilmlistPath = "ignoreFilmlist.txt"; importLivestreamConfiguration = new ImportLivestreamConfiguration(false, "live-streams.json", FilmlistFormats.OLD_JSON); + ImportFilmlistConfigurations = new ArrayList(); + //ImportFilmlistConfigurations.add(new ImportFilmlistConfiguration(false, "https://verteiler1.mediathekview.de/Filmliste-akt.xz", FilmlistFormats.OLD_JSON_COMPRESSED_XZ, true)); Arrays.stream(Sender.values()) .forEach(sender -> senderConfigurations.put(sender, new MServerBasicConfigDTO(this))); @@ -122,14 +120,6 @@ public void setFilmlistDiffSavePaths(final Map filmlist this.filmlistDiffSavePaths = filmlistDiffSavePaths; } - public FilmlistFormats getFilmlistImportFormat() { - return filmlistImportFormat; - } - - public void setFilmlistImportFormat(final FilmlistFormats filmlistImportFormat) { - this.filmlistImportFormat = filmlistImportFormat; - } - public String getFilmlistImportLocation() { return filmlistImportLocation; } @@ -242,6 +232,9 @@ public ImportLivestreamConfiguration getImportLivestreamConfiguration() { return importLivestreamConfiguration; } + public List getImportFilmlistConfigurations() { + return ImportFilmlistConfigurations; + } /** * Loads the {@link Sender} specific configuration and if it not exist creates one. @@ -255,14 +248,6 @@ public MServerBasicConfigDTO getSenderConfig(final Sender aSender) { return senderConfigurations.get(aSender); } - public Boolean getFilmlistImporEnabled() { - return filmlistImporEnabled; - } - - public void setFilmlistImporEnabled(final Boolean filmlistImporEnabled) { - this.filmlistImporEnabled = filmlistImporEnabled; - } - @Override public boolean equals(final Object o) { if (this == o) { @@ -284,17 +269,16 @@ public boolean equals(final Object o) { && Objects.equals(getFilmlistSaveFormats(), that.getFilmlistSaveFormats()) && Objects.equals(getFilmlistSavePaths(), that.getFilmlistSavePaths()) && Objects.equals(getFilmlistDiffSavePaths(), that.getFilmlistDiffSavePaths()) - && getFilmlistImportFormat() == that.getFilmlistImportFormat() && Objects.equals(getFilmlistImportLocation(), that.getFilmlistImportLocation()) && Objects.equals(getLogSettings(), that.getLogSettings()) && Objects.equals(getCrawlerURLs(), that.getCrawlerURLs()) - && Objects.equals(getFilmlistImporEnabled(), that.getFilmlistImporEnabled()) && Objects.equals(getWriteFilmlistHashFileEnabled(), that.getWriteFilmlistHashFileEnabled()) && Objects.equals(getFilmlistHashFilePath(), that.getFilmlistHashFilePath()) && Objects.equals(getWriteFilmlistIdFileEnabled(), that.getWriteFilmlistIdFileEnabled()) && Objects.equals(getFilmlistIdFilePath(), that.getFilmlistIdFilePath()) && Objects.equals(getIgnoreFilmslistPath(), that.getIgnoreFilmslistPath()) - && Objects.equals(getImportLivestreamConfiguration(), that.getImportLivestreamConfiguration()); + && Objects.equals(getImportLivestreamConfiguration(), that.getImportLivestreamConfiguration()) + && Objects.equals(getImportFilmlistConfigurations(), that.getImportFilmlistConfigurations()); } @Override @@ -310,17 +294,16 @@ public int hashCode() { getFilmlistSaveFormats(), getFilmlistSavePaths(), getFilmlistDiffSavePaths(), - getFilmlistImportFormat(), getFilmlistImportLocation(), getLogSettings(), getCrawlerURLs(), - getFilmlistImporEnabled(), getWriteFilmlistHashFileEnabled(), getFilmlistHashFilePath(), getWriteFilmlistIdFileEnabled(), getFilmlistIdFilePath(), getIgnoreFilmslistPath(), - getImportLivestreamConfiguration()); + getImportLivestreamConfiguration(), + getImportFilmlistConfigurations()); } public void initializeSenderConfigurations() { diff --git a/src/main/java/de/mediathekview/mserver/crawler/CrawlerManager.java b/src/main/java/de/mediathekview/mserver/crawler/CrawlerManager.java index 9ea05f9f3..696fd576a 100644 --- a/src/main/java/de/mediathekview/mserver/crawler/CrawlerManager.java +++ b/src/main/java/de/mediathekview/mserver/crawler/CrawlerManager.java @@ -2,10 +2,12 @@ import de.mediathekview.mlib.daten.Film; import de.mediathekview.mlib.daten.Filmlist; +import de.mediathekview.mlib.daten.Podcast; import de.mediathekview.mlib.daten.Sender; import de.mediathekview.mlib.filmlisten.FilmlistFormats; import de.mediathekview.mlib.filmlisten.FilmlistManager; import de.mediathekview.mlib.progress.ProgressListener; +import de.mediathekview.mserver.base.config.ImportFilmlistConfiguration; import de.mediathekview.mserver.base.config.MServerConfigDTO; import de.mediathekview.mserver.base.config.MServerConfigManager; import de.mediathekview.mserver.base.config.MServerCopySettings; @@ -37,6 +39,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.LocalDateTime; import java.util.*; import java.util.Map.Entry; import java.util.concurrent.*; @@ -65,7 +68,7 @@ public class CrawlerManager extends AbstractManager { private final FilmlistManager filmlistManager; private final Collection copyProgressListeners; - private Filmlist differenceList; + private final Filmlist differenceList; private CrawlerManager() { super(); @@ -77,6 +80,7 @@ private CrawlerManager() { crawlerMap = new EnumMap<>(Sender.class); filmlist = new Filmlist(); + differenceList = new Filmlist(); filmlistManager = FilmlistManager.getInstance(); copyProgressListeners = new ArrayList<>(); initializeCrawler(rootConfig); @@ -151,9 +155,12 @@ public void filterFilmlist() { * {@link MServerConfigDTO#getFilmlistImportLocation()}. */ public void importFilmlist() { - if (checkConfigForFilmlistImport()) { - importFilmlist(config.getFilmlistImportFormat(), config.getFilmlistImportLocation()); + for (ImportFilmlistConfiguration importFilmlistConfiguration : config.getImportFilmlistConfigurations()) { + if (checkImportFilmlistConfig(importFilmlistConfiguration) && importFilmlistConfiguration.isActive()) { + importFilmlist(importFilmlistConfiguration); + } } + } public void importLivestreamFilmlist() { @@ -188,18 +195,22 @@ public void importLivestreamFilmlist(final FilmlistFormats aFormat, final String * or https it tries to import from URL. Otherwise it tries to import * from the given Location as a file path. */ - public void importFilmlist(final FilmlistFormats aFormat, final String aFilmlistLocation) { + public void importFilmlist(final ImportFilmlistConfiguration importFilmlistConfiguration) { try { final Optional importedFilmlist; - if (aFilmlistLocation.startsWith(HTTP)) { - importedFilmlist = importFilmListFromURl(aFormat, aFilmlistLocation); + if (importFilmlistConfiguration.getPath().startsWith(HTTP)) { + importedFilmlist = importFilmListFromURl(importFilmlistConfiguration.getFormat(), importFilmlistConfiguration.getPath()); } else { - importedFilmlist = importFilmlistFromFile(aFormat, aFilmlistLocation); + importedFilmlist = importFilmlistFromFile(importFilmlistConfiguration.getFormat(), importFilmlistConfiguration.getPath()); + } + // + final Filmlist difflist = new Filmlist(UUID.randomUUID(), LocalDateTime.now()); + importedFilmlist.ifPresent(value -> addAllToFilmlist(mergeTwoFilmlists(filmlist,value),difflist)); + if (importFilmlistConfiguration.isCreateDiff()) { + addAllToFilmlist(difflist, differenceList); } - - importedFilmlist.ifPresent(value -> differenceList = filmlist.merge(value)); } catch (final IOException ioException) { - LOG.fatal(String.format(FILMLIST_IMPORT_ERROR_TEMPLATE, aFilmlistLocation), ioException); + LOG.fatal(String.format(FILMLIST_IMPORT_ERROR_TEMPLATE, importFilmlistConfiguration.getPath()), ioException); } } @@ -348,14 +359,23 @@ private boolean checkAllUsedFormatsHaveSavePaths() { return missingSavePathFormats.isEmpty(); } - private boolean checkConfigForFilmlistImport() { - if (config.getFilmlistImportFormat() == null) { - printMessage(ServerMessages.NO_FILMLIST_IMPORT_FORMAT_IN_CONFIG); + private boolean checkImportFilmlistConfig(ImportFilmlistConfiguration config) { + if (config.getPath() == null || config.getPath().isBlank() ) { + printMessage(ServerMessages.NO_FILMLIST_IMPORT_LOCATION_IN_CONFIG); return false; } - if (config.getFilmlistImportLocation() == null) { - printMessage(ServerMessages.NO_FILMLIST_IMPORT_LOCATION_IN_CONFIG); + if (config.getFormat() == null) { + printMessage(ServerMessages.NO_FILMLIST_IMPORT_FORMAT_IN_CONFIG); + return false; + } + // @TODO define message + if (config.isActive() == null) { + printMessage(ServerMessages.NO_FILMLIST_IMPORT_FORMAT_IN_CONFIG); + return false; + } + if (config.isCreateDiff() == null) { + printMessage(ServerMessages.NO_FILMLIST_IMPORT_FORMAT_IN_CONFIG); return false; } @@ -532,4 +552,70 @@ private void runCrawlers(final AbstractCrawler... aCrawlers) { public Filmlist getFilmlist() { return filmlist; } + // added to allow JUNIT tests + public Filmlist getDifferenceList() { + return differenceList; + } + + // move this to MLIB after PR merge + public static void addAllToFilmlist(final Filmlist source,final Filmlist target) { + target.addAllFilms(source.getFilms().values()); + target.addAllLivestreams(source.getLivestreams().values()); + target.addAllPodcasts(source.getPodcasts().values()); + } + + // move this to MLIB and replace merge function after PR merge + public static Filmlist mergeTwoFilmlists(final Filmlist aThis, final Filmlist aFilmlist) { + final Filmlist toBeAdded = new Filmlist(UUID.randomUUID(), LocalDateTime.now()); + final Filmlist diff = new Filmlist(UUID.randomUUID(), LocalDateTime.now()); + // add all from old list not in the new list + aFilmlist.getFilms().entrySet().stream() + .filter(e -> !containsFilm(aThis, e.getValue())) + .forEachOrdered(e -> toBeAdded.getFilms().put(e.getKey(), e.getValue())); + // the diff list contains all new entries (fresh list) which are not already in the old list + aThis.getFilms().entrySet().stream() + .filter(e -> !containsFilm(aFilmlist,e.getValue())) + .forEachOrdered(e -> diff.getFilms().put(e.getKey(), e.getValue())); + // add the history to the current list + aThis.getFilms().putAll(toBeAdded.getFilms()); + // + // the same for podcast + aFilmlist.getPodcasts().entrySet().stream() + .filter(e -> !containsPodcast(aThis,e.getValue())) + .forEachOrdered(e -> toBeAdded.getPodcasts().put(e.getKey(), e.getValue())); + aThis.getFilms().entrySet().stream() + .filter(e -> !containsPodcast(aFilmlist,e.getValue())) + .forEachOrdered(e -> diff.getPodcasts().put(e.getKey(), e.getValue())); + aThis.getPodcasts().putAll(toBeAdded.getPodcasts()); + // + return diff; + } + + public static boolean containsFilm(Filmlist athis, Film film) { + Optional check = athis.getFilms().entrySet().stream() + .filter(entry -> + film.getTitel().equalsIgnoreCase(entry.getValue().getTitel()) && + film.getThema().equalsIgnoreCase(entry.getValue().getThema()) && + film.getSender().equals(entry.getValue().getSender()) && + film.getDuration().equals(entry.getValue().getDuration())) + .map(Map.Entry::getValue) + .findFirst(); + + return check.isPresent(); + } + + public static boolean containsPodcast(Filmlist athis, Podcast prodcast) { + Optional check = athis.getPodcasts().entrySet().stream() + .filter(entry -> + prodcast.getTitel().equalsIgnoreCase(entry.getValue().getTitel()) && + prodcast.getThema().equalsIgnoreCase(entry.getValue().getThema()) && + prodcast.getSender().equals(entry.getValue().getSender()) && + prodcast.getDuration().equals(entry.getValue().getDuration())) + .map(Map.Entry::getValue) + .findFirst(); + + return check.isPresent(); + } + + } diff --git a/src/main/java/de/mediathekview/mserver/ui/config/MServerConfigUI.java b/src/main/java/de/mediathekview/mserver/ui/config/MServerConfigUI.java index 5a07fc7af..c57376524 100644 --- a/src/main/java/de/mediathekview/mserver/ui/config/MServerConfigUI.java +++ b/src/main/java/de/mediathekview/mserver/ui/config/MServerConfigUI.java @@ -98,9 +98,7 @@ private boolean logLevelInfoOrLower(final Level configLevel) { void start() { try { manager.start(); - if (config.getFilmlistImporEnabled() != null && config.getFilmlistImporEnabled()) { - manager.importFilmlist(); - } + manager.importFilmlist(); if (Boolean.TRUE.equals(config.getImportLivestreamConfiguration().isActive())) { manager.importLivestreamFilmlist(); } diff --git a/src/test/java/de/mediathekview/mserver/crawler/CrawlerManagerImportFilmlistsTest.java b/src/test/java/de/mediathekview/mserver/crawler/CrawlerManagerImportFilmlistsTest.java new file mode 100644 index 000000000..83588d493 --- /dev/null +++ b/src/test/java/de/mediathekview/mserver/crawler/CrawlerManagerImportFilmlistsTest.java @@ -0,0 +1,129 @@ +package de.mediathekview.mserver.crawler; + +import de.mediathekview.mlib.filmlisten.FilmlistFormats; +import de.mediathekview.mlib.messages.Message; +import de.mediathekview.mlib.messages.MessageTypes; +import de.mediathekview.mlib.messages.MessageUtil; +import de.mediathekview.mlib.messages.listener.MessageListener; +import de.mediathekview.mserver.base.config.ImportFilmlistConfiguration; +import de.mediathekview.mserver.testhelper.FileReader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.junit.jupiter.api.Assertions.fail; + + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.Date; +import java.util.stream.Stream; + + +public class CrawlerManagerImportFilmlistsTest implements MessageListener { + + private static final Logger LOG = LogManager.getLogger(CrawlerManagerImportFilmlistsTest.class); + private static final String TEMP_FOLDER_NAME_PATTERN = "MSERVER_TEST_%d"; + private static Path testFileFolderPath; + + static Stream getTestArgumentForFilmlistsInDifferentFormats() { + ImportFilmlistConfiguration import1 = new ImportFilmlistConfiguration( + true, + FileReader.getPath("./filmlists/importFilmlist/FilmlistImportTest1.json").toString(), + FilmlistFormats.OLD_JSON, + true); + ImportFilmlistConfiguration import2 = new ImportFilmlistConfiguration( + true, + FileReader.getPath("./filmlists/importFilmlist/FilmlistImportTest2.json").toString(), + FilmlistFormats.OLD_JSON, + true); + ImportFilmlistConfiguration import3 = new ImportFilmlistConfiguration( + true, + FileReader.getPath("./filmlists/importFilmlist/FilmlistImportTest3.json").toString(), + FilmlistFormats.OLD_JSON, + true); + ImportFilmlistConfiguration import4 = new ImportFilmlistConfiguration( + true, + FileReader.getPath("./filmlists/importFilmlist/FilmlistImportTest2.json").toString(), + FilmlistFormats.OLD_JSON, + false); + + return Stream.of( + arguments(import1, import1, 3, 0), // two times the same list should result in no additional films + arguments(import1, import2, 5, 3), // two different lists should result in the sum of both lists - the fist list is also diff since list two is "old" + arguments(import1, import3, 4, 1), // overlapping lists - one entry is new compared to the "old" list + arguments(import2, import3, 4, 1), // overlapping lists - one entry is new compared to the "old" list + arguments(import1, import4, 5, 0) // two different lists and not diff list active + + ); + } + + @ParameterizedTest + @Execution(ExecutionMode.SAME_THREAD) + @MethodSource("getTestArgumentForFilmlistsInDifferentFormats") + public void testSaveAndImport(final ImportFilmlistConfiguration initialList, final ImportFilmlistConfiguration additionalList, final int expectedSize, final int expectedDiffListSize) { + CrawlerManager crawlerManagerForEachRun = createEmptyCrawlerManager(); + crawlerManagerForEachRun.addMessageListener(this); + crawlerManagerForEachRun.importFilmlist(initialList); + crawlerManagerForEachRun.importFilmlist(additionalList); + assertThat(crawlerManagerForEachRun.getFilmlist().getFilms()).hasSize(expectedSize); + assertThat(crawlerManagerForEachRun.getDifferenceList().getFilms()).hasSize(expectedDiffListSize); + } + + public CrawlerManager createEmptyCrawlerManager() { + // reset singelton CrawlerManager to have an empty filmlist + Field instance; + try { + instance = CrawlerManager.class.getDeclaredField("instance"); + instance.setAccessible(true); + instance.set(null, null); + } catch (Exception e) { + fail("Exception mooking crawler manager: " + e.getMessage()); + } // + return CrawlerManager.getInstance(); + } + + @AfterAll + public static void deleteTempFiles() throws IOException { + Files.walk(testFileFolderPath) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + + @BeforeAll + public static void initTestData() throws Exception { + testFileFolderPath = Files.createTempDirectory(formatWithDate(TEMP_FOLDER_NAME_PATTERN)); + Files.createDirectory(testFileFolderPath.resolve("filmlists")); + } + + private static String formatWithDate(final String aPattern) { + return String.format(aPattern, new Date().getTime()); + } + + @Override + public void consumeMessage(final Message aMessage, final Object... aParameters) { + if (MessageTypes.FATAL_ERROR.equals(aMessage.getMessageType())) { + fail(String.format(MessageUtil.getInstance().loadMessageText(aMessage), aParameters)); + } else { + LOG.info( + String.format( + "%s: %s", + aMessage.getMessageType().name(), + String.format(MessageUtil.getInstance().loadMessageText(aMessage), aParameters))); + } + } + +} diff --git a/src/test/java/de/mediathekview/mserver/crawler/CrawlerManagerLivestreamTest.java b/src/test/java/de/mediathekview/mserver/crawler/CrawlerManagerLivestreamTest.java index 2c376b710..15ba06302 100644 --- a/src/test/java/de/mediathekview/mserver/crawler/CrawlerManagerLivestreamTest.java +++ b/src/test/java/de/mediathekview/mserver/crawler/CrawlerManagerLivestreamTest.java @@ -5,6 +5,7 @@ import de.mediathekview.mlib.messages.MessageTypes; import de.mediathekview.mlib.messages.MessageUtil; import de.mediathekview.mlib.messages.listener.MessageListener; +import de.mediathekview.mserver.base.config.ImportFilmlistConfiguration; import de.mediathekview.mserver.testhelper.FileReader; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -59,7 +60,7 @@ public void testSaveAndImport(final FilmlistFormats format, final String filmlis final Path filmListFilePath = FileReader.getPath(filmlistPath); final Path livestreamFilmListFilePath = FileReader.getPath(livestreamPath); crawlerManagerForEachRun.addMessageListener(this); - crawlerManagerForEachRun.importFilmlist(format, filmListFilePath.toAbsolutePath().toString()); + crawlerManagerForEachRun.importFilmlist(new ImportFilmlistConfiguration(true, filmListFilePath.toAbsolutePath().toString(), format, false)); // assertThat(crawlerManagerForEachRun.getFilmlist().getFilms()).hasSize(expectedInitialSize); // diff --git a/src/test/java/de/mediathekview/mserver/crawler/CrawlerManagerTest.java b/src/test/java/de/mediathekview/mserver/crawler/CrawlerManagerTest.java index 74c8ff5bb..3f278203e 100644 --- a/src/test/java/de/mediathekview/mserver/crawler/CrawlerManagerTest.java +++ b/src/test/java/de/mediathekview/mserver/crawler/CrawlerManagerTest.java @@ -5,6 +5,7 @@ import de.mediathekview.mlib.messages.MessageTypes; import de.mediathekview.mlib.messages.MessageUtil; import de.mediathekview.mlib.messages.listener.MessageListener; +import de.mediathekview.mserver.base.config.ImportFilmlistConfiguration; import de.mediathekview.mserver.testhelper.FileReader; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -107,7 +108,7 @@ public void testSaveAndImport() { final Path filmListFilePath = FileReader.getPath(filmlistPath); synchronized (CRAWLER_MANAGER) { CRAWLER_MANAGER.addMessageListener(this); - CRAWLER_MANAGER.importFilmlist(format, filmListFilePath.toAbsolutePath().toString()); + CRAWLER_MANAGER.importFilmlist(new ImportFilmlistConfiguration(true, filmListFilePath.toAbsolutePath().toString(), format, false)); assertThat(CRAWLER_MANAGER.getFilmlist().getFilms()).hasSize(expectedSize); CRAWLER_MANAGER.saveFilmlist(testFileFolderPath.resolve(filmlistPath), format); assertThat(testFileFolderPath.resolve(filmlistPath)).exists(); diff --git a/src/test/resources/filmlists/importFilmlist/FilmlistImportTest1.json b/src/test/resources/filmlists/importFilmlist/FilmlistImportTest1.json new file mode 100644 index 000000000..3bbf6487a --- /dev/null +++ b/src/test/resources/filmlists/importFilmlist/FilmlistImportTest1.json @@ -0,0 +1,7 @@ +{ +"Filmliste":["13.10.2023, 17:36","13.10.2023, 17:36","3.1.220"," [Vers.: 3.1.220 ]","78cd7afb-3e0e-4de9-a4ca-f858ec35bade"], +"Filmliste":["Sender","Thema","Titel","Datum","Zeit","Dauer","Größe [MB]","Beschreibung","Url","Website","Url Untertitel","Url RTMP","Url Klein","Url RTMP Klein","Url HD","Url RTMP HD","DatumL","Url History","Geo","neu"], +"X":["DW","Auf den Punkt","Hamas-Terror gegen Israel: Wie der Konflikt die Region weiter spaltet","12.10.2023","12:37:19","00:42:36","814","Nach den Terrorangriffen der Hamas bereitet Israel womöglich eine Bodenoffensive vor und fliegt Luftangriffe. Hat der Konflikt das Potenzial, die Region im ganzen Nahen Osten nachhaltig zu verändern? Unsere Gäste: Guido Steinberg (SWP), Daniel-Dylan Böhmer (Die Welt), Ofer Waldman (freier Journalist)","https://tvdownloaddw-a.akamaihd.net/dwtv_video/flv/qud/qud231012_gesamt_AVC_960x540.mp4","https://p.dw.com/p/4XQux","","","76|640x360.mp4","","76|1280x720.mp4","","1697107039","","","false"], +"X":["","DW Nachrichten","Terrorangriff der Hamas: Israelis bangen um Vermisste","11.10.2023","19:30:13","00:03:26","64","Bei ihrem Angriff auf Israel haben die militant-islamistischen Hamas-Terroristen Dutzende Geiseln entführt, viele davon auf einem Musikfestival in der Nähe des Gazastreifens. Noch immer werden zahlreiche Menschen vermisst. Ihre Familien und Freunde hoffen auf ein Lebenszeichen.","https://tvdownloaddw-a.akamaihd.net/dwtv_video/flv/jd/jd20231110_dhostages19c_AVC_960x540.mp4","https://p.dw.com/p/4XQRF","","","82|640x360.mp4","","82|1280x720.mp4","","1697045413","","","false"], +"X":["","","NATO-Ministertreffen zu Israel und Ukraine","11.10.2023","19:39:20","00:01:57","35","Könnte der aktuelle Brandherd im Nahen Osten vom Krieg in der Ukraine ablenken? Um das zu verhindern, reiste der ukrainische Präsident Wolodymyr Selenskyj persönlich nach Brüssel und warb um weitere militärische Hilfen für sein Land.","https://tvdownloaddw-a.akamaihd.net/dwtv_video/flv/jd/jd20231011_dminister19c_AVC_960x540.mp4","https://p.dw.com/p/4XQRj","","","82|640x360.mp4","","82|1280x720.mp4","","1697045960","","","false"] +} \ No newline at end of file diff --git a/src/test/resources/filmlists/importFilmlist/FilmlistImportTest2.json b/src/test/resources/filmlists/importFilmlist/FilmlistImportTest2.json new file mode 100644 index 000000000..6477f5050 --- /dev/null +++ b/src/test/resources/filmlists/importFilmlist/FilmlistImportTest2.json @@ -0,0 +1,6 @@ +{ + "Filmliste": ["30.05.2023, 19:21", "30.05.2023, 17:21", "3", "MSearch [Vers.: 3.1.212]", "6fe29b8c65d5c834e12164d667e7713e"], + "Filmliste": ["Sender", "Thema", "Titel", "Datum", "Zeit", "Dauer", "Größe [MB]", "Beschreibung", "Url", "Website", "Url Untertitel", "Url RTMP", "Url Klein", "Url RTMP Klein", "Url HD", "Url RTMP HD", "DatumL", "Url History", "Geo", "neu"], + "X": ["3Sat", "37 Grad", "37° Totgerast (Audiodeskription)", "30.05.2023", "22:15:00", "00:28:45", "405", "Wer plötzlich Angehörige bei einem Unfall verliert, wacht in einem anderen Leben auf. Antonia verlor ihre Eltern und ihre Schwester, Steffen seine Frau und seinen Sohn.", "https://rodlzdf-a.akamaihd.net/none/zdf/23/05/230530_2215_sendung_37g/2/230530_2215_sendung_37g_a3a4_2360k_p35v17.mp4", "https://www.3sat.de/gesellschaft/37-grad/37-totgerast-102.html", "", "", "101|808k_p11v17.mp4", "", "8|nrodlzdf-a.akamaihd.net/none/zdf/23/05/230530_2215_sendung_37g/2/230530_2215_sendung_37g_a3a4_6660k_p37v17.mp4", "", "1685477700", "", "", "false"], + "X": ["", "", "37°: Ein Gen verändert unser Leben (Audiodeskription)", "16.05.2023", "22:15:00", "00:28:41", "414", "Eine Diagnose, die alles verändert: Milo hat einen Gendefekt. Wie wird das Leben werden mit einem Kind, das extrem viel Betreuung braucht? \"37°\" begleitet die Familie.", "https://rodlzdf-a.akamaihd.net/none/zdf/23/05/230502_2215_sendung_37g/4/230502_2215_sendung_37g_a3a4_2360k_p35v17.mp4", "https://www.3sat.de/gesellschaft/37-grad/37-ein-gen-veraendert-unser-leben-100.html", "", "", "101|808k_p11v17.mp4", "", "8|nrodlzdf-a.akamaihd.net/none/zdf/23/05/230502_2215_sendung_37g/4/230502_2215_sendung_37g_a3a4_6660k_p37v17.mp4", "", "1684268100", "", "", "false"] +} diff --git a/src/test/resources/filmlists/importFilmlist/FilmlistImportTest3.json b/src/test/resources/filmlists/importFilmlist/FilmlistImportTest3.json new file mode 100644 index 000000000..919ab7c88 --- /dev/null +++ b/src/test/resources/filmlists/importFilmlist/FilmlistImportTest3.json @@ -0,0 +1,7 @@ +{ +"Filmliste": ["30.05.2023, 19:21", "30.05.2023, 17:21", "3", "MSearch [Vers.: 3.1.212]", "6fe29b8c65d5c834e12164d667e7713e"], +"Filmliste": ["Sender", "Thema", "Titel", "Datum", "Zeit", "Dauer", "Größe [MB]", "Beschreibung", "Url", "Website", "Url Untertitel", "Url RTMP", "Url Klein", "Url RTMP Klein", "Url HD", "Url RTMP HD", "DatumL", "Url History", "Geo", "neu"], +"X": ["3Sat", "37 Grad", "37° Totgerast (Audiodeskription)", "30.05.2023", "22:15:00", "00:28:45", "405", "Wer plötzlich Angehörige bei einem Unfall verliert, wacht in einem anderen Leben auf. Antonia verlor ihre Eltern und ihre Schwester, Steffen seine Frau und seinen Sohn.", "https://rodlzdf-a.akamaihd.net/none/zdf/23/05/230530_2215_sendung_37g/2/230530_2215_sendung_37g_a3a4_2360k_p35v17.mp4", "https://www.3sat.de/gesellschaft/37-grad/37-totgerast-102.html", "", "", "101|808k_p11v17.mp4", "", "8|nrodlzdf-a.akamaihd.net/none/zdf/23/05/230530_2215_sendung_37g/2/230530_2215_sendung_37g_a3a4_6660k_p37v17.mp4", "", "1685477700", "", "", "false"], +"X":["DW","Auf den Punkt","Hamas-Terror gegen Israel: Wie der Konflikt die Region weiter spaltet","12.10.2023","12:37:19","00:42:36","814","Nach den Terrorangriffen der Hamas bereitet Israel womöglich eine Bodenoffensive vor und fliegt Luftangriffe. Hat der Konflikt das Potenzial, die Region im ganzen Nahen Osten nachhaltig zu verändern? Unsere Gäste: Guido Steinberg (SWP), Daniel-Dylan Böhmer (Die Welt), Ofer Waldman (freier Journalist)","https://tvdownloaddw-a.akamaihd.net/dwtv_video/flv/qud/qud231012_gesamt_AVC_960x540.mp4","https://p.dw.com/p/4XQux","","","76|640x360.mp4","","76|1280x720.mp4","","1697107039","","","false"], +"X":["","DW Nachrichten","Terrorangriff der Hamas: Israelis bangen um Vermisste","11.10.2023","19:30:13","00:03:26","64","Bei ihrem Angriff auf Israel haben die militant-islamistischen Hamas-Terroristen Dutzende Geiseln entführt, viele davon auf einem Musikfestival in der Nähe des Gazastreifens. Noch immer werden zahlreiche Menschen vermisst. Ihre Familien und Freunde hoffen auf ein Lebenszeichen.","https://tvdownloaddw-a.akamaihd.net/dwtv_video/flv/jd/jd20231110_dhostages19c_AVC_960x540.mp4","https://p.dw.com/p/4XQRF","","","82|640x360.mp4","","82|1280x720.mp4","","1697045413","","","false"] +}