From b964f8bccac7c40ffc08684bcaae61ffc6ff44db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Berenguel?= Date: Thu, 22 Aug 2024 17:12:26 -0300 Subject: [PATCH] #2304: adjust timeout for UFED chats --- iped-app/resources/config/conf/ParsingTaskConfig.txt | 8 +++++++- iped-app/resources/config/conf/metadataTypes.txt | 1 + .../java/iped/engine/config/ParsingTaskConfig.java | 10 ++++++++++ .../java/iped/engine/datasource/UfedXmlReader.java | 12 +++++++++++- .../main/java/iped/engine/io/FastPipedReader.java | 10 ++++------ .../src/main/java/iped/engine/io/ParsingReader.java | 9 ++++++++- 6 files changed, 41 insertions(+), 9 deletions(-) diff --git a/iped-app/resources/config/conf/ParsingTaskConfig.txt b/iped-app/resources/config/conf/ParsingTaskConfig.txt index b30a4953c5..fd24c591e9 100644 --- a/iped-app/resources/config/conf/ParsingTaskConfig.txt +++ b/iped-app/resources/config/conf/ParsingTaskConfig.txt @@ -17,7 +17,9 @@ parseUnknownFiles = true # For example, deleted and partially overwritten images could have searchable plain text into them. parseCorruptedFiles = true -# Minimum timeout (seconds) while parsing files. To this baseline will be added the result of timeOutPerMB * file size in MB (see next parameter). +# Minimum timeout (seconds) while parsing files. +# To this baseline will be added the result of timeOutPerMB * file size in MB +# or timeOutPerHundredMessages * messages_count / 100 (see next parameters). # After the timeout, the file's raw strings will be indexed. timeOut = 180 @@ -25,6 +27,10 @@ timeOut = 180 # Total_Timeout = timeOut + timeOutPerMB * timeOutPerMB = 2 +# Timeout (seconds) for each 100 messages in UFED Chat +# Total_Timeout = timeOut + timeOutPerHundredMessages * / 100 +timeOutPerHundredMessages = 2 + # Minimum size of raw strings extracted from unknown files for indexing. minRawStringSize = 4 diff --git a/iped-app/resources/config/conf/metadataTypes.txt b/iped-app/resources/config/conf/metadataTypes.txt index 3c1f666922..cdce71c501 100644 --- a/iped-app/resources/config/conf/metadataTypes.txt +++ b/iped-app/resources/config/conf/metadataTypes.txt @@ -4202,6 +4202,7 @@ ufed:GlobalNumberOfFiles = java.lang.Integer ufed:Header offset = java.lang.String ufed:Height = java.lang.Integer ufed:Identifier = java.lang.String +ufed:InstantMessagesCount = java.lang.Integer ufed:IsEmulatable = java.lang.String ufed:Label = java.lang.String ufed:LastAccessTime = java.util.Date diff --git a/iped-engine/src/main/java/iped/engine/config/ParsingTaskConfig.java b/iped-engine/src/main/java/iped/engine/config/ParsingTaskConfig.java index 545c88c434..b7475c5703 100644 --- a/iped-engine/src/main/java/iped/engine/config/ParsingTaskConfig.java +++ b/iped-engine/src/main/java/iped/engine/config/ParsingTaskConfig.java @@ -22,6 +22,7 @@ public class ParsingTaskConfig extends AbstractTaskPropertiesConfig { private boolean parseUnknownFiles = true; private int timeOut = 180; private int timeOutPerMB = 2; + private int timeOutPerHundredMessages = 2; private int minRawStringSize = 4; private boolean storeTextCacheOnDisk = true; private boolean sortPDFChars; @@ -85,6 +86,11 @@ public void processProperties(UTF8Properties properties) { timeOutPerMB = Integer.valueOf(value.trim()); } + value = properties.getProperty("timeOutPerHundredMessages"); //$NON-NLS-1$ + if (value != null) { + timeOutPerHundredMessages = Integer.valueOf(value.trim()); + } + value = properties.getProperty("minRawStringSize"); //$NON-NLS-1$ if (value != null) { minRawStringSize = Integer.valueOf(value.trim()); @@ -145,6 +151,10 @@ public int getTimeOutPerMB() { return timeOutPerMB; } + public int getTimeOutPerHundredMessages() { + return timeOutPerHundredMessages; + } + public boolean isStoreTextCacheOnDisk() { return storeTextCacheOnDisk; } diff --git a/iped-engine/src/main/java/iped/engine/datasource/UfedXmlReader.java b/iped-engine/src/main/java/iped/engine/datasource/UfedXmlReader.java index f740860616..ac030dbd02 100644 --- a/iped-engine/src/main/java/iped/engine/datasource/UfedXmlReader.java +++ b/iped-engine/src/main/java/iped/engine/datasource/UfedXmlReader.java @@ -123,6 +123,8 @@ public class UfedXmlReader extends DataSourceReader { private Set supportedApps = new HashSet(Arrays.asList(WhatsAppParser.WHATSAPP, TelegramParser.TELEGRAM, WhatsAppParser.WHATSAPP + " Business", WhatsAppParser.WHATSAPP + " (Dual App)")); + public static final Property UFED_IM_COUNT = Property.internalInteger(ExtraProperties.UFED_META_PREFIX + "InstantMessagesCount"); + private static Random random = new Random(); private static HashMap uisfMap = new HashMap<>(); @@ -392,6 +394,7 @@ private class XMLContentHandler implements ContentHandler { Object ignoreItemTree = null; boolean inChat = false; int numAttachments = 0; + int numInstantMessages = 0; String prevUfedId = null; private class XmlNode { @@ -580,8 +583,10 @@ public void startElement(String uri, String localName, String qName, Attributes Item item = new Item(); item.setExtraAttribute(ExtraProperties.DATASOURCE_READER, UfedXmlReader.class.getSimpleName()); String type = atts.getValue("type"); //$NON-NLS-1$ - if (type.equals("Chat")) + if (type.equals("Chat")) { inChat = true; + numInstantMessages = 0; + } String name = type + "_" + atts.getValue("id"); //$NON-NLS-1$ //$NON-NLS-2$ item.setName(name); String path = decodedFolder.getPath() + "/" + type + "/" + name; //$NON-NLS-1$ //$NON-NLS-2$ @@ -874,6 +879,7 @@ else if (parentItem.getMediaType().equals(MediaTypes.UFED_MESSAGE_MIME)) { if (TelegramParser.TELEGRAM.equalsIgnoreCase(source)) // $NON-NLS-1$ item.setMediaType(UFEDChatParser.UFED_CHAT_TELEGRAM); + item.getMetadata().set(UFED_IM_COUNT, numInstantMessages); item.setExtraAttribute(IndexItem.TREENODE, "true"); //$NON-NLS-1$ } if ("InstantMessage".equals(type) || "Email".equals(type) || "Call".equals(type) || "SMS".equals(type) //$NON-NLS-4$ @@ -911,6 +917,9 @@ else if (parentItem.getMediaType().equals(MediaTypes.UFED_MESSAGE_MIME)) { ignoreItemLocal = true; } } + if (!ignoreItemLocal) { + numInstantMessages++; + } } if (mergeInParentNode.contains(type) && itemSeq.size() > 0) { IItem parentItem = itemSeq.get(itemSeq.size() - 1); @@ -1125,6 +1134,7 @@ else if ("Bcc".equalsIgnoreCase(role)) //$NON-NLS-1$ if (MediaTypes.isInstanceOf(item.getMediaType(), UFEDChatParser.UFED_CHAT_MIME)) { inChat = false; ignoreItems = false; + numInstantMessages = 0; } } diff --git a/iped-engine/src/main/java/iped/engine/io/FastPipedReader.java b/iped-engine/src/main/java/iped/engine/io/FastPipedReader.java index 5fc3313685..3b0e432df4 100644 --- a/iped-engine/src/main/java/iped/engine/io/FastPipedReader.java +++ b/iped-engine/src/main/java/iped/engine/io/FastPipedReader.java @@ -25,8 +25,7 @@ public class FastPipedReader extends Reader { // utilizado para verificar se há comunicação entre reader e writer private int timer = 0; - private int timeOutBySize = 0; - private int minTimeout = 60; + private int totalTimeout = 60; private boolean timeoutPaused = false, timedOut = false; boolean closedByWriter = false; @@ -123,10 +122,9 @@ public FastPipedReader() { * if pipeSize less than 1. * @since 1.6 */ - public FastPipedReader(int pipeSize, int minTimeout, int timeOutBySize) { + public FastPipedReader(int pipeSize, int totalTimeout) { initPipe(pipeSize); - this.minTimeout = minTimeout; - this.timeOutBySize = timeOutBySize; + this.totalTimeout = totalTimeout; } public synchronized boolean setTimeoutPaused(boolean paused) { @@ -135,7 +133,7 @@ public synchronized boolean setTimeoutPaused(boolean paused) { } public int getTotalTimeout() { - return minTimeout + timeOutBySize; + return totalTimeout; } private void initPipe(int pipeSize) { diff --git a/iped-engine/src/main/java/iped/engine/io/ParsingReader.java b/iped-engine/src/main/java/iped/engine/io/ParsingReader.java index e67d72e63f..14b015bdd7 100644 --- a/iped-engine/src/main/java/iped/engine/io/ParsingReader.java +++ b/iped-engine/src/main/java/iped/engine/io/ParsingReader.java @@ -42,6 +42,7 @@ import iped.engine.config.ConfigurationManager; import iped.engine.config.ParsingTaskConfig; import iped.engine.core.QueuesProcessingOrder; +import iped.engine.datasource.UfedXmlReader; import iped.parsers.fork.ParsingTimeout; import iped.parsers.standard.StandardParser; import iped.parsers.util.CorruptedCarvedException; @@ -144,7 +145,13 @@ public ParsingReader(Parser parser, InputStream stream, Metadata metadata, Parse ParsingTaskConfig parsingConfig = ConfigurationManager.get().findObject(ParsingTaskConfig.class); timeOutBySize = (int) (length / 1000000) * parsingConfig.getTimeOutPerMB(); - pipedReader = new FastPipedReader(128 * 1024, parsingConfig.getTimeOut(), timeOutBySize); + int timeOutByMessages = 0; + Integer messagesCount = metadata.getInt(UfedXmlReader.UFED_IM_COUNT); + if (messagesCount != null) { + timeOutByMessages = messagesCount * parsingConfig.getTimeOutPerHundredMessages() / 100; + } + + pipedReader = new FastPipedReader(128 * 1024, parsingConfig.getTimeOut() + timeOutBySize + timeOutByMessages); this.reader = new BufferedReader(pipedReader); this.writer = new FastPipedWriter(pipedReader);