Skip to content

Commit

Permalink
update configurationmanager to support configuration file name
Browse files Browse the repository at this point in the history
codingPF committed Nov 5, 2023
1 parent cb44dfa commit 1460c7a
Showing 16 changed files with 101 additions and 92 deletions.
Original file line number Diff line number Diff line change
@@ -45,9 +45,19 @@ public class Log4JConfigurationFactory extends ConfigurationFactory {

private static MServerLogSettingsDTO logSettings;

public Log4JConfigurationFactory(MServerLogSettingsDTO logSettings) {
Log4JConfigurationFactory.logSettings = logSettings;
}


public Log4JConfigurationFactory() {
if (Log4JConfigurationFactory.logSettings == null) {
Log4JConfigurationFactory.logSettings = new MServerConfigManager(MServerConfigManager.DEFAULT_CONFIG_FILE).getConfig().getLogSettings();
}
}

static Configuration createConfiguration(
final String name, final ConfigurationBuilder<BuiltConfiguration> aBuilder) {
logSettings = new MServerConfigManager().getConfig().getLogSettings();

aBuilder.setConfigurationName(name);

Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@ public class MServerConfigDTO extends MServerBasicConfigDTO implements ConfigDTO
private final Boolean writeFilmlistIdFileEnabled;
private final String filmlistIdFilePath;
/** ignore certain film by title **/
private final String ignoreFilmlistPath;
private String ignoreFilmlistPath;
/** add livestreams from external list **/
private final ImportLivestreamConfiguration importLivestreamConfiguration;
/** add additional filmlist from external **/
@@ -147,6 +147,10 @@ public Map<FilmlistFormats, String> getFilmlistSavePaths() {
public void setFilmlistSavePaths(final Map<FilmlistFormats, String> filmlistSavePaths) {
this.filmlistSavePaths = filmlistSavePaths;
}

public void setIgnoreFilmlistPath(final String ignoreFilmlistPath) {
this.ignoreFilmlistPath = ignoreFilmlistPath;
}

public MServerLogSettingsDTO getLogSettings() {
return logSettings;
Original file line number Diff line number Diff line change
@@ -14,10 +14,6 @@ public MServerConfigManager(final String fileName) {
configFileName = fileName;
}

public MServerConfigManager() {
this(DEFAULT_CONFIG_FILE);
}

/**
* @param aSender The {@link Sender} for which the config will be loaded.
* @return The Sender specific config.
Original file line number Diff line number Diff line change
@@ -57,8 +57,8 @@ public class CrawlerManager extends AbstractManager {
private static final String FILMLIST_JSON_COMPRESSED_DEFAULT_NAME =
FILMLIST_JSON_DEFAULT_NAME + ".xz";
private static final Logger LOG = LogManager.getLogger(CrawlerManager.class);
private static CrawlerManager instance;
private final MServerConfigDTO config;
private final MServerConfigManager rootConfig;
private final ForkJoinPool forkJoinPool;
private final Filmlist filmlist;
private final IgnoreFilmFilter ingoreFilmFilter;
@@ -70,9 +70,9 @@ public class CrawlerManager extends AbstractManager {
private final Collection<ProgressListener> copyProgressListeners;
private final Filmlist differenceList;

private CrawlerManager() {
public CrawlerManager(MServerConfigManager aMServerConfigManager) {
super();
final MServerConfigManager rootConfig = new MServerConfigManager();
rootConfig = aMServerConfigManager;
config = rootConfig.getConfig();
ingoreFilmFilter = new IgnoreFilmFilter(config.getIgnoreFilmslistPath());
executorService = Executors.newFixedThreadPool(config.getMaximumCpuThreads());
@@ -86,11 +86,8 @@ private CrawlerManager() {
initializeCrawler(rootConfig);
}

public static CrawlerManager getInstance() {
if (instance == null) {
instance = new CrawlerManager();
}
return instance;
public MServerConfigManager getConfigManager() {
return rootConfig;
}

public void copyFilmlist() {
@@ -166,6 +163,7 @@ public void importLivestreamFilmlist() {
}

public void importLivestreamFilmlist(final FilmlistFormats aFormat, final String aFilmlistLocation) {
LOG.debug("importLivestreamFilmlist {}", aFilmlistLocation);
try {
final Optional<Filmlist> importedFilmlist;
if (aFilmlistLocation.startsWith(HTTP)) {
Original file line number Diff line number Diff line change
@@ -31,7 +31,7 @@ public IgnoreFilmFilter(String configFileNameAndPath) {
} else {
ignoreFilmTitles = read(configFileNameAndPath);
}
LOG.debug("ignoreFilmList setup with {} entries", size());
LOG.debug("ignoreFilmList {} setup with {} entries", configFileNameAndPath, size());
} catch (IOException e) {
LOG.error("Could not read ignorefilmlist from {} ",configFileNameAndPath, e);
}
Original file line number Diff line number Diff line change
@@ -5,7 +5,6 @@
import com.google.gson.GsonBuilder;

import de.mediathekview.mlib.daten.Sender;
import de.mediathekview.mserver.base.config.MServerConfigManager;
import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
import de.mediathekview.mserver.crawler.basic.AbstractRestTask;
import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;
@@ -25,9 +24,7 @@
public abstract class DWTaskBase<T, D extends CrawlerUrlDTO> extends AbstractRestTask<T, D> {
private static final Logger LOG = LogManager.getLogger(DWTaskBase.class);

private static final RateLimiter limiter =
RateLimiter.create(
new MServerConfigManager().getSenderConfig(Sender.DW).getMaximumRequestsPerSecond());
private static RateLimiter limiter = null;

private final transient GsonBuilder gsonBuilder;

@@ -81,6 +78,9 @@ private Response executeRequest(final WebTarget aTarget) {
request.header(
ZdfConstants.HEADER_AUTHENTIFICATION, AUTHORIZATION_BEARER + authKey.get());
}
if (limiter == null) {
limiter = RateLimiter.create(crawler.getRuntimeConfig().getSenderConfig(Sender.DW).getMaximumRequestsPerSecond());
}
limiter.acquire();
return request.header(HEADER_ACCEPT_ENCODING, ENCODING_GZIP).get();
}
Original file line number Diff line number Diff line change
@@ -73,9 +73,7 @@ protected FilmInfoDto mapToElement(final JsonObject jsonObject) {
}

private MServerConfigDTO getRuntimeConfig() {
return crawler
.map(AbstractCrawler::getRuntimeConfig)
.orElseGet(() -> new MServerConfigManager().getConfig());
return crawler.get().getRuntimeConfig();
}

private String createNexxCloudUrl(final String entityId) {
Original file line number Diff line number Diff line change
@@ -18,7 +18,6 @@
import org.jsoup.nodes.Node;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
Original file line number Diff line number Diff line change
@@ -14,9 +14,7 @@ public abstract class SrRateLimitedDocumentTask<T, D extends CrawlerUrlDTO>

private static final long serialVersionUID = -4077182368484515410L;

private static final RateLimiter LIMITER =
RateLimiter.create(
new MServerConfigManager().getSenderConfig(Sender.SR).getMaximumRequestsPerSecond());
private static RateLimiter LIMITER = null;

SrRateLimitedDocumentTask(
final AbstractCrawler crawler,
@@ -26,6 +24,9 @@ public abstract class SrRateLimitedDocumentTask<T, D extends CrawlerUrlDTO>

@Override
protected void processElement(final D urlDTO) {
if (LIMITER== null) {
LIMITER = RateLimiter.create(crawler.getRuntimeConfig().getSenderConfig(Sender.SR).getMaximumRequestsPerSecond());
}
LIMITER.acquire();
super.processElement(urlDTO);
}
Original file line number Diff line number Diff line change
@@ -4,7 +4,6 @@
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import de.mediathekview.mlib.daten.Sender;
import de.mediathekview.mserver.base.config.MServerConfigManager;
import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
import de.mediathekview.mserver.crawler.basic.AbstractRestTask;
import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;
@@ -22,9 +21,7 @@
public abstract class ZdfTaskBase<T, D extends CrawlerUrlDTO> extends AbstractRestTask<T, D> {
private static final Logger LOG = LogManager.getLogger(ZdfTaskBase.class);

private static final RateLimiter limiter =
RateLimiter.create(
new MServerConfigManager().getSenderConfig(Sender.ZDF).getMaximumRequestsPerSecond());
private static RateLimiter limiter = null;

private final GsonBuilder gsonBuilder;

@@ -76,6 +73,10 @@ private Response executeRequest(final WebTarget aTarget) {
request.header(
ZdfConstants.HEADER_AUTHENTIFICATION, AUTHORIZATION_BEARER + authKey.get());
}
if (limiter == null) {
limiter = RateLimiter.create(crawler.getRuntimeConfig().getSenderConfig(Sender.ZDF).getMaximumRequestsPerSecond());
}

limiter.acquire();
return request.header(HEADER_ACCEPT_ENCODING, ENCODING_GZIP).get();
}
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@

import de.mediathekview.mlib.messages.listener.LogMessageListener;
import de.mediathekview.mlib.messages.listener.MessageListener;
import de.mediathekview.mserver.base.config.MServerConfigDTO;
import de.mediathekview.mserver.base.config.Log4JConfigurationFactory;
import de.mediathekview.mserver.base.config.MServerConfigManager;
import de.mediathekview.mserver.base.config.MServerLogSettingsDTO;
import de.mediathekview.mserver.base.messages.ServerMessages;
@@ -13,34 +13,27 @@
import org.apache.logging.log4j.Logger;

import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;

public final class MServerConfigUI {

private static final Logger LOG = LogManager.getLogger(MServerConfigUI.class);
// logger setup in start
private Logger LOG = null;
private static final String CONFIG_FILE_NAME = "MServer-Config.yaml";
private static final String ARGUMENT_GCONF = "-gconf";
private final LogMessageListener logMessageListener;
private final MServerConfigDTO config;
private LogMessageListener logMessageListener;
private CrawlerManager manager;

public MServerConfigUI() {
super();
config = new MServerConfigManager().getConfig();
final MServerLogSettingsDTO logSettings = config.getLogSettings();
logSettings.setLogActivateConsole(true);

logMessageListener = new LogMessageListener();

final Level configLevel = logSettings.getLogLevelConsole();
if (configLevel == null || !logLevelInfoOrLower(configLevel)) {
logSettings.setLogLevelConsole(Level.INFO);
}
}

public static void main(final String[] args) {
@@ -79,10 +72,6 @@ private boolean interpretProgramArguments(final String[] aProgramAgruments) {

if (ARGUMENT_GCONF.equals(aProgramAgruments[0])) {
generateDefaultConfiguration();
} else {
logMessageListener.consumeMessage(
ServerMessages.UI_UNKNOWN_ARGUMENT, aProgramAgruments[0], ARGUMENT_GCONF);
return false;
}
}

@@ -99,9 +88,7 @@ void start() {
try {
manager.start();
manager.importFilmlist();
if (Boolean.TRUE.equals(config.getImportLivestreamConfiguration().isActive())) {
manager.importLivestreamFilmlist();
}
manager.importLivestreamFilmlist();
} finally {
manager.filterFilmlist();
manager.saveFilmlist();
@@ -114,10 +101,53 @@ void start() {
}

void start(final String[] aProgramAgruments) {
MServerConfigManager aMServerConfigManager = null;
if (aProgramAgruments.length > 0 && !ARGUMENT_GCONF.equals(aProgramAgruments[0])) {
String configFileName = aProgramAgruments[0];
if (configFileName.startsWith("http")) {
URL fileUrl;
try {
// get a copy of this file to use it as configuration file
fileUrl = new URL(configFileName);
String filename = Paths.get(fileUrl.getPath()).getFileName().toString();
MServerConfigUI.getRemoteFileToLocal(configFileName, filename);
configFileName = filename;
} catch (MalformedURLException e) {
e.printStackTrace();
}
}
aMServerConfigManager = new MServerConfigManager(configFileName);
} else {
aMServerConfigManager = new MServerConfigManager(MServerConfigManager.DEFAULT_CONFIG_FILE);
}
// here we set the correct configManager for all log4logger
// logsettings are stored static in our factory
new Log4JConfigurationFactory(aMServerConfigManager.getConfig().getLogSettings());
LOG = LogManager.getLogger(MServerConfigUI.class);
logMessageListener = new LogMessageListener();

if (interpretProgramArguments(aProgramAgruments)) {
manager = CrawlerManager.getInstance();
manager = new CrawlerManager(aMServerConfigManager);
final MServerLogSettingsDTO logSettings = aMServerConfigManager.getConfig().getLogSettings();
logSettings.setLogActivateConsole(true);
final Level configLevel = logSettings.getLogLevelConsole();
if (configLevel == null || !logLevelInfoOrLower(configLevel)) {
logSettings.setLogLevelConsole(Level.INFO);
}
addListeners();
start();
}
}

public static void getRemoteFileToLocal(String source, String target) {
try {
URL fileUrl = new URL(source);
try (InputStream in = fileUrl.openStream()) {
Path outputPath = Path.of(target);
Files.copy(in, outputPath, StandardCopyOption.REPLACE_EXISTING);
}
} catch (IOException e) {
e.printStackTrace(); // we do not have a logger yet
}
}
}
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@
import de.mediathekview.mlib.messages.MessageUtil;
import de.mediathekview.mlib.messages.listener.MessageListener;
import de.mediathekview.mserver.base.config.ImportFilmlistConfiguration;
import de.mediathekview.mserver.base.config.MServerConfigManager;
import de.mediathekview.mserver.testhelper.FileReader;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -24,7 +25,6 @@

import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Comparator;
@@ -34,7 +34,7 @@

public class CrawlerManagerImportFilmlistsTest implements MessageListener {

private static final Logger LOG = LogManager.getLogger(CrawlerManagerImportFilmlistsTest.class);
private Logger LOG;
private static final String TEMP_FOLDER_NAME_PATTERN = "MSERVER_TEST_%d";
private static Path testFileFolderPath;

@@ -87,16 +87,9 @@ void testSaveAndImport(final ImportFilmlistConfiguration initialList, final Impo
}

public CrawlerManager createEmptyCrawlerManager() {
// reset singelton CrawlerManager to have an empty filmlist
Field instance;
try {
instance = CrawlerManager.class.getDeclaredField("instance");
instance.setAccessible(true);
instance.set(null, null);
} catch (Exception e) {
fail("Exception mooking crawler manager: " + e.getMessage());
} //
return CrawlerManager.getInstance();
CrawlerManager cm = new CrawlerManager(new MServerConfigManager(MServerConfigManager.DEFAULT_CONFIG_FILE));
LOG = LogManager.getLogger(CrawlerManagerImportFilmlistsTest.class);
return cm;
}

@AfterAll
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@
import de.mediathekview.mlib.messages.MessageUtil;
import de.mediathekview.mlib.messages.listener.MessageListener;
import de.mediathekview.mserver.base.config.ImportFilmlistConfiguration;
import de.mediathekview.mserver.base.config.MServerConfigManager;
import de.mediathekview.mserver.testhelper.FileReader;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -24,7 +25,6 @@

import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Comparator;
@@ -74,16 +74,7 @@ public void testSaveAndImport(final FilmlistFormats format, final String filmlis
}

public CrawlerManager createEmptyCrawlerManager() {
// reset singelton CrawlerManager to have an empty filmlist
Field instance;
try {
instance = CrawlerManager.class.getDeclaredField("instance");
instance.setAccessible(true);
instance.set(null, null);
} catch (Exception e) {
fail("Exception mooking crawler manager: " + e.getMessage());
} //
return CrawlerManager.getInstance();
return new CrawlerManager(new MServerConfigManager(MServerConfigManager.DEFAULT_CONFIG_FILE));
}

@AfterAll
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@
import de.mediathekview.mlib.messages.MessageUtil;
import de.mediathekview.mlib.messages.listener.MessageListener;
import de.mediathekview.mserver.base.config.ImportFilmlistConfiguration;
import de.mediathekview.mserver.base.config.MServerConfigManager;
import de.mediathekview.mserver.testhelper.FileReader;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -18,7 +19,6 @@

import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
@@ -27,12 +27,11 @@
import java.util.Date;

import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;

@RunWith(Parameterized.class)
public class CrawlerManagerTest implements MessageListener {

private static final Logger LOG = LogManager.getLogger(CrawlerManagerTest.class);
private Logger LOG;
private static final String TEMP_FOLDER_NAME_PATTERN = "MSERVER_TEST_%d";
private static Path testFileFolderPath;

@@ -45,16 +44,8 @@ public CrawlerManagerTest(final String aFilmlistPath, final FilmlistFormats aFor
filmlistPath = aFilmlistPath;
expectedSize = aExpectedSize;
format = aFormat;
// reset singelton CrawlerManager
Field instance;
try {
instance = CrawlerManager.class.getDeclaredField("instance");
instance.setAccessible(true);
instance.set(null, null);
} catch (Exception e) {
fail("Exception mooking crawler manager: " + e.getMessage());
} //
CRAWLER_MANAGER = CrawlerManager.getInstance();
CRAWLER_MANAGER = new CrawlerManager(new MServerConfigManager(MServerConfigManager.DEFAULT_CONFIG_FILE));
LOG = LogManager.getLogger(CrawlerManagerTest.class);
}

@Parameterized.Parameters(name = "Test {index} Filmlist for {0} with {1}")
Original file line number Diff line number Diff line change
@@ -7,10 +7,8 @@
import org.hamcrest.Matchers;
import org.junit.Test;

import java.net.URLEncoder;
import java.util.Set;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.MatcherAssert.assertThat;

Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package de.mediathekview.mserver.crawler.zdf.parser;

import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;
import de.mediathekview.mserver.crawler.zdf.ZdfConstants;
import de.mediathekview.mserver.testhelper.FileReader;
import org.hamcrest.Matchers;
import org.jsoup.Jsoup;

0 comments on commit 1460c7a

Please sign in to comment.