Merge remote-tracking branch 'origin/master' into ALeappTask

.gitignore

@@ -4,3 +4,4 @@ nbactions.xml
 /.classpath
 /.project
 .pydevproject
+/.idea/

ReleaseNotes.txt

@@ -1,3 +1,45 @@
+05/04/2024: IPED-4.1.6:
+Optimizations:
+#455:  Optimization of UFDR reader memory usage (@lfcnassif)
+#2084: Optimize expansion of certain RAR files (@lfcnassif)
+Fixes:
+#1932: Imagemagick portable missing vcomp140.dll dependency [4.1.5 regression] (@lfcnassif)
+#1940: Sometimes items are not included in their bookmarks, when generating a report from a multicase (@wladimirleite)
+#2037: Bookmarks may be lost in multicase reports (@wladimirleite)
+#1975: Processed files in mounted folders with incorrect extension being renamed when opened externally (@lfcnassif)
+#439:  Show warning in GUI for each evidence processed unsuccessfully (@lfcnassif)
+#2092: Incorrect directory tree when parsing partial RAR files (@lfcnassif)
+#1977: Telegram parser duplicating messages (@hauck-jvsh)
+#1921: WhatsApp recovered media messages may become duplicated (iOS only) (@wladimirleite)
+#2089: Failing to process Cellebrite XML Reports (@fmpfeifer, @lfcnassif)
+#2038: Aborting OutOfMemoryError caused by too many results from ItemSearcher called from UFEDChatParser (@lfcnassif, @wladimirleite)
+#2099: Non VMDK file being detected as VMDK causing processing to abort (@fsicoli)
+#2107: Aborting "IllegalArgumentException: DocValuesField "parentIds" is too large, must be <= 32766" caused by very long file name in a UFDR (@wladimirleite)
+#2110: Report generation aborting because of inconsistent data types while indexing a property (@wladimirleite, @lfcnassif)
+#2145: Rare NullPointerException from VideoThumbTask when extracting frames as subitems (@lfcnassif)
+#2141: lastId incorrectly computed after some evidence is removed from case (@lfcnassif)
+#1947: NullPointerException in LanguageDetectTask (@wladimirleite)
+#2077: DecoderException: Odd number of characters from ExportCSVTask if resuming processing (@lfcnassif)
+#1942: Audios not retried and skipped if specific errors happen in remote transcription service (@lfcnassif)
+#1945: Makes Wav2Vec2 transcription robust to new versions of python libraries (@lfcnassif)
+#2102: Weird line wraps when file names contain emojis (@wladimirleite)
+#1989: Parsing exception when searching for chat attachments (@lfcnassif)
+#2051: Stop condition for iped.parsers.whatsapp.Message.setThumbData() recursion (@aberenguel, @lfcnassif)
+#2024: Geopoints are being indexed wrongly inside opensearch (@hauck-jvsh)
+#1938: Fix PDF thumbnail when page is rotated (@wladimirleite)
+#1740: Clear selection not correctly updated on Map (@patrickdalla)
+#1929: Timeline panel needlessly call updateFileListing when clear filters button is pressed (@patrickdalla)
+#1685: Item from UFDR being extracted three times (@lfcnassif, @hauck-jvsh)
+#2041: Using an existing case output folder by accident makes the case unfinished (@wladimirleite)
+#1993: Pressing the "Pause" button during initialization phase cause an exception (@wladimirleite)
+#1988: Avoid searching for regexes in hash values and UUIDs (@wladimirleite)
+#1955: FileNotFoundException might be caused by race condition reading UFDR evidence (@lfcnassif)
+#2064: Negative estimated time to finish UFDR processing (@wladimirleite)
+#1950: Sometimes negative parse times are shown (@wladimirleite)
+#2120: Reduce the automatically set maximum for "-Xmx" from 32GB to 32500MB (@wladimirleite)
+#2008: Minor localization issues on UI (@lfcnassif)
+
+
 05/10/2023: IPED-4.1.5:
 Fixes:
 #1903: RCE vulnerability in libwebp dependency (@tc-wleite, @lfcnassif)

ThirdParty.txt

@@ -179,3 +179,7 @@ Zlib
 JUnRAR
 - Site: https://github.com/edmund-wagner/junrar
 - License: licenses/JUNRAR.txt
+
+Lottie-player
+- Site: https://github.com/LottieFiles/lottie-player/blob/master/LICENSE
+- License: licenses/LOTTIE-PLAYER.txt

iped-api/src/main/java/iped/data/IBookmarks.java

@@ -52,6 +52,7 @@ public interface IBookmarks extends Serializable {
 
     int getLastId();
 
+    @Deprecated
     int getTotalItens();
 
     int getTotalChecked();
@@ -102,6 +103,8 @@ public interface IBookmarks extends Serializable {
 
     KeyStroke getBookmarkKeyStroke(int bookmarkId);
 
+    void removeBookmarkKeyStroke(int bookmarkId);
+
     int getBookmarkCount(int bookmarkId);
 
     void setInReport(int bookmarkId, boolean inReport);

iped-api/src/main/java/iped/data/IItemReader.java

@@ -7,6 +7,8 @@
 import java.util.HashSet;
 import java.util.Map;
 
+import javax.imageio.stream.ImageInputStream;
+
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.mime.MediaType;
 
@@ -133,6 +135,8 @@ public interface IItemReader extends IStreamSource {
 
     public BufferedInputStream getBufferedInputStream() throws IOException;
 
+    public ImageInputStream getImageInputStream() throws IOException;
+
     /**
      * @return data da última modificação do arquivo
      */

iped-api/src/main/java/iped/data/IMultiBookmarks.java

@@ -25,8 +25,11 @@
  */
 public interface IMultiBookmarks extends Serializable {
 
+    @Deprecated
     void addBookmark(List<IItemId> ids, String bookmarkName);
 
+    void addBookmark(Set<IItemId> uniqueSelectedIds, String bookmarkName);
+
     void addToTypedWords(String texto);
 
     void renameBookmark(String oldBookmark, String newBookmark);
@@ -69,8 +72,11 @@ public interface IMultiBookmarks extends Serializable {
 
     void newBookmark(String bookmarkName);
 
+    @Deprecated
     void removeBookmark(List<IItemId> ids, String bookmarkName);
 
+    void removeBookmark(Set<IItemId> uniqueSelectedIds, String bookmarkName);
+
     void saveState();
 
     void saveState(boolean sync);
@@ -81,6 +87,8 @@ public interface IMultiBookmarks extends Serializable {
 
     KeyStroke getBookmarkKeyStroke(String bookmarkName);
 
+    void removeBookmarkKeyStroke(String bookmarkName);
+
     void checkAll();
 
     void setChecked(boolean value, IItemId item);

iped-api/src/main/java/iped/properties/ExtraProperties.java

@@ -49,7 +49,7 @@ public class ExtraProperties {
 
     public static final Property MESSAGE_DATE = COMMUNICATION_DATE;
 
-    public static final String PARTICIPANTS = "Participants";
+    public static final String PARTICIPANTS = COMMUNICATION_PREFIX + "Participants";
 
     public static final String GROUP_ID = "GroupID";
 
@@ -149,7 +149,7 @@ public class ExtraProperties {
 
     public static final String FACE_ENCODINGS = "face_encodings";
 
-    public static final List<String> COMMUNICATION_BASIC_PROPS = Arrays.asList(MESSAGE_SUBJECT, COMMUNICATION_DATE.getName(),
-            MESSAGE_BODY, COMMUNICATION_FROM, COMMUNICATION_TO, Message.MESSAGE_CC, Message.MESSAGE_BCC,
-            Message.MESSAGE_RECIPIENT_ADDRESS, MESSAGE_IS_ATTACHMENT, MESSAGE_ATTACHMENT_COUNT.getName());
+    public static final List<String> COMMUNICATION_BASIC_PROPS = Arrays.asList(MESSAGE_SUBJECT, MESSAGE_BODY,
+            Message.MESSAGE_CC, Message.MESSAGE_BCC, Message.MESSAGE_RECIPIENT_ADDRESS, MESSAGE_IS_ATTACHMENT,
+            MESSAGE_ATTACHMENT_COUNT.getName());
 }

iped-app/pom.xml

@@ -52,11 +52,6 @@
 	  		<artifactId>jfreechartextensions</artifactId>
 	  		<version>5d6b903557dac49a925a7a62d5c6592bd07482c5</version>
 	  	</dependency>
-	  	<dependency>
-            <groupId>org.roaringbitmap</groupId>
-            <artifactId>RoaringBitmap</artifactId>
-            <version>0.9.39</version>
-        </dependency>
     </dependencies>
 
     <build>
@@ -344,7 +339,7 @@
                                 <artifactItem>
                                     <groupId>tesseract</groupId>
                                     <artifactId>tesseract-zip</artifactId>
-                                    <version>5.3.2-24-g3922</version>
+                                    <version>5.3.2-24-g3922_1</version>
                                     <type>zip</type>
                                     <overWrite>false</overWrite>
                                     <outputDirectory>${tools.dir}</outputDirectory>

iped-app/resources/config/IPEDConfig.txt

@@ -1,4 +1,4 @@
-########################################################################
+########################################################################
 # Processing Settings
 ########################################################################
 
@@ -103,9 +103,9 @@ enableOCR = false
 # Configure it in conf/AudioTranscriptConfig.txt
 enableAudioTranscription = false
 
-# Enables carving. "addUnallocated" must be enabled to unallocated area be searched.
-# By default carving runs on almost every item in the case. 
-# File types to be searched and retrieved can be set in file "conf/CarvingConfig.txt"
+# Enables carving. "addUnallocated" must be enabled to scan unallocated space.
+# By default, our carving module scans many places in the evidence for deleted or embedded files. 
+# Places to scan and file types to recover can be configured in "conf/CarverConfig.xml"
 enableCarving = false
 
 # Enables carving that retrieves known files from the LED base, based on the beginning (64K) of the file.
@@ -129,6 +129,9 @@ enableImageSimilarity = false
 # If enabled, you can search for faces from the analysis interface, check the options menu.
 enableFaceRecognition = false
 
+# Enables searching for crypto hardware wallets.
+enableSearchHardwareWallets = true
+
 # Enables extraction of image frames from video files
 # Extraction settings can be modified in file "conf/VideoThumbsConfig.txt"
 enableVideoThumbs = true

iped-app/resources/config/LocalConfig.txt

@@ -2,7 +2,7 @@
 # Local environment configuration
 ########################################################################
 
-# Defines program localization/language. Currently there are localizations for 'en', 'pt-BR', 'it-IT', 'de-DE' & 'es-AR'
+# Defines program localization/language. Currently there are localizations for 'en', 'pt-BR', 'it-IT', 'de-DE', 'es-AR' and 'fr-FR'.
 locale = en
 
 # Temporary directory for processing: "default" uses the system temporary folder.

iped-app/resources/config/conf/CarverConfig.xml

@@ -188,13 +188,17 @@
         </carverType>
 
         <carverType>
-            <!-- May be improved: if a PDF is edited, changes are appended to the end of file and a new footer is added, so it should also accumulate footers for PDFs -->
             <name>PDF</name>
+            <carverClass>iped.carvers.custom.PDFCarver</carverClass>
             <signatures>
-                <headerSignature>%PDF-</headerSignature>
-                <footerSignature>%%EOF?</footerSignature>
+                <headerSignature>%PDF-1</headerSignature>
+                <headerSignature>%PDF-2</headerSignature>
+                <footerSignature>\0a%%EOF\0d\0a</footerSignature>
+                <footerSignature>\0d%%EOF\0d</footerSignature>
+                <footerSignature>\0a%%EOF\0a</footerSignature>
+                <footerSignature>\0a%%EOF</footerSignature>
             </signatures>
-            <minLength>1000</minLength>            
+            <minLength>1000</minLength>
             <maxLength>50000000</maxLength>
             <mediaType>application/pdf</mediaType>
         </carverType>

iped-app/resources/config/conf/CategoriesConfig.json

@@ -85,7 +85,7 @@
         {"name": "Jabber", "mimes":["application/x-jabber-chat"]},
         {"name": "Facebook", "mimes":["application/x-ufed-chat-preview-facebook"]},
         {"name": "Instagram", "mimes":["application/x-ufed-chat-preview-instagram"]},
-        {"name": "Discord", "mimes":["application/x-discord-index", "application/x-discord-chat"]},
+        {"name": "Discord", "mimes":["application/x-discord-chat"]},
         {"name": "Badoo", "mimes":["application/aleapp-badoochat"]},
         {"name": "Chat Activities", "mimes": ["application/x-ufed-chatactivity"]},
         {"name": "Others Chats", "mimes":["application/x-ufed-html-chats", "application/x-ufed-chats-txt", "application/x-ufed-chat", "application/x-ufed-chat-preview"]}
@@ -113,15 +113,16 @@
       {"name": "Ares Galaxy", "mimes": ["application/x-ares-galaxy","application/x-ares-galaxy-entry"]},
       {"name": "E-Mule", "mimes": ["application/x-emule", "application/x-emule-part-met", "application/x-emule-searches", "application/x-emule-preferences-ini", "application/x-emule-preferences-dat", "application/x-emule-known-met-entry", "application/x-emule-part-met-entry"]},
       {"name": "Shareaza", "mimes": ["application/x-shareaza-searches-dat", "application/x-shareaza-library-dat", "application/x-shareaza-library-dat-entry", "application/x-shareaza-download"]},
-      {"name": "Torrent", "mimes": ["application/x-bittorrent-resume-dat", "application/x-bittorrent"]},
+      {"name": "Torrent", "mimes": ["application/x-bittorrent-resume-dat","application/x-bittorrent-settings-dat", "application/x-bittorrent"]},
       {"name": "Other Peer-to-peer", "mimes": ["application/x-p2p"]}
     ]},
     {"name": "Browser Artifacts", "categories":[
       {"name": "Internet History", "mimes": ["application/x-msie-cache", "application/x-webcache", "application/x-webcache-table", "application/x-ufed-html-webhistory", "application/x-firefox-places", "application/x-firefox-history", "application/x-firefox-downloads", "application/x-chrome-sqlite", "application/x-chrome-history", "application/x-chrome-downloads", "application/x-chrome-searches", "application/x-edge-web-cache", "application/x-edge-history", "application/x-safari-sqlite", "application/x-safari-history", "application/x-safari-plist", "application/x-safari-downloads"]},
       {"name": "Internet History Entries", "mimes": ["application/x-ufed-visitedpage", "application/x-firefox-history-registry", "application/x-firefox-downloads-registry", "application/x-chrome-history-registry", "application/x-chrome-downloads-registry", "application/x-edge-history-registry", "application/x-safari-downloads-registry", "application/x-safari-history-registry", "application/aleapp-chrome-webhistory"]},
       {"name": "Web Bookmarks", "mimes": ["application/x-ufed-html-bookmarks", "application/x-ufed-webbookmark", "application/x-firefox-bookmarks", "application/x-firefox-bookmarks-registry"]},
       {"name": "Mozilla Firefox Saved Session", "mimes": ["application/x-firefox-savedsession"]},
-      {"name": "TorTCFragment", "mimes": ["application/x-tor-tc-fragment"]}
+      {"name": "TorTCFragment", "mimes": ["application/x-tor-tc-fragment"]},
+      {"name": "Chrome Cache", "mimes": ["application/x-chrome-cache-index", "application/x-discord-index", "application/x-discord-chat+json"]}
     ]}, 
 
     {"name": "Image Disks", "categories":[

iped-app/resources/config/conf/CategoriesToExpand.txt

@@ -18,6 +18,7 @@ GDrive Synced Files
 OLE files
 Georeferenced Files
 Peer-to-peer
+Chrome Cache
 #Event Files
 
 # Generates registry reports:

iped-app/resources/config/conf/CustomSignatures.xml

@@ -445,13 +445,25 @@
         <_comment>BitTorrent Client Resume.dat file</_comment>
         <sub-class-of type="application/x-p2p"/>
         <magic priority="50">
-    		<match value="d10:.fileguard" type="string" offset="0"/>
+    		<match value="d10:.fileguard" type="string" offset="0">
+	    		<match value="e4:info" type="string" offset="16:1024"/>
+    		</match>
 		</magic>
         <glob pattern="resume.dat" />
         <glob pattern="resume.dat.old" />
         <glob pattern="resume.dat.new" />
     </mime-type>
 
+    <mime-type type="application/x-bittorrent-settings-dat">
+        <_comment>BitTorrent Client Settings.dat file</_comment>
+        <sub-class-of type="application/x-p2p"/>
+        <magic priority="50">
+    		<match value="d10:.fileguard" type="string" offset="0">
+	    		<match value="e9:autostart" type="string" offset="16:1024"/>
+    		</match>
+		</magic>
+    </mime-type>
+
 	<mime-type type="application/x-bittorrent">
 		<sub-class-of type="application/x-p2p"/>
 		<magic priority="50">
@@ -1595,10 +1607,30 @@
         <glob pattern="*.lzfse"/>
     </mime-type>
 
+    <mime-type type="audio/mp4">
+        <alias type="audio/x-m4a"/>
+        <alias type="audio/x-mp4a"/>
+        <magic priority="60">
+            <match value="ftypM4A " type="string" offset="4"/>
+            <match value="ftypM4B " type="string" offset="4"/>
+            <match value="ftypF4A " type="string" offset="4"/>
+            <match value="ftypF4B " type="string" offset="4"/>
+            <match value="ftypmp42" type="string" offset="4">
+                <match value="M4A" type="string" offset="16"/>
+                <match value="m4a" type="string" offset="16"/>
+            </match>
+        </magic>
+        <glob pattern="*.mp4a"/>
+        <glob pattern="*.m4a"/>
+        <glob pattern="*.m4b"/>
+        <sub-class-of type="application/quicktime" />
+    </mime-type>
+
     <mime-type type="application/x-ofx-v1">
         <magic priority="50">
-            <match value="OFXHEADER:100" type="string" offset="0"/>
+            <match value="OFXHEADER:100" type="string" offset="0:1"/>
         </magic>
+        <sub-class-of type="text/plain"/>
         <glob pattern="*.ofx"/>
     </mime-type>
 
@@ -1615,5 +1647,10 @@
         </magic>
         <glob pattern="*.ofc"/>
     </mime-type>
+
+    <mime-type type="application/x-vlc-ini">
+        <sub-class-of type="text/x-ini"/>
+        <glob pattern="vlc-qt-interface.ini"/>
+    </mime-type>
 
 </mime-info>

iped-app/resources/config/conf/MakePreviewConfig.txt

@@ -7,7 +7,7 @@
 supportedMimes = application/x-msaccess; application/x-lnk; application/x-firefox-savedsession
 supportedMimes = application/x-sqlite3; application/sqlite-skype; application/x-win10-timeline; application/x-gdrive-cloud-graph; application/x-gdrive-snapshot
 supportedMimes = application/x-whatsapp-db; application/x-whatsapp-db-f; application/x-whatsapp-chatstorage; application/x-whatsapp-chatstorage-f; application/x-threema-chatstorage; application/x-shareaza-searches-dat; application/x-msie-cache
-supportedMimes = application/x-prefetch; text/x-vcard; application/x-bittorrent-resume-dat; application/x-bittorrent; application/x-emule-preferences-dat; application/vnd.android.package-archive
+supportedMimes = application/x-prefetch; text/x-vcard; application/x-emule-preferences-dat; application/vnd.android.package-archive; application/x-bittorrent-settings-dat
 
 # List of mimetypes which parsers insert links to other case items into preview
-supportedMimesWithLinks = application/x-emule; application/x-emule-part-met; application/x-ares-galaxy; application/x-shareaza-library-dat; application/x-shareaza-download
+supportedMimesWithLinks = application/x-emule; application/x-emule-part-met; application/x-ares-galaxy; application/x-shareaza-library-dat; application/x-shareaza-download; application/x-bittorrent-resume-dat; application/x-bittorrent

iped-app/resources/config/conf/OCRConfig.txt

@@ -5,7 +5,7 @@
 # You can use the parameter -ocr "bookmark_name" to restrict the OCR to a specific bookmark (just works when creating reports from cmd line).
 
 # Dictionary language to be used for OCR. You can specify multiple languages e.g. por+eng but you will pay an important processing cost.
-# Languages supported: eng, por, ita, deu, spa
+# Languages supported: eng, por, ita, deu, spa, fra
 OCRLanguage = por
 
 # Skip known files found in the hash lookup database.

iped-app/resources/config/conf/ParserConfig.xml

@@ -307,6 +307,8 @@
                 <param name="extractMessages" type="bool">true</param>
                 <param name="mergeBackups" type="bool">true</param>
                 <param name="linkMediasByNameAndApproxSizeFallback" type="bool">true</param>
+                <!--Set as 0 to disable long path matching fallback-->                
+                <param name="linkMediasByLongPathFallback" type="int">40</param>
                 <param name="downloadConnectionTimeout" type="int">500</param>
                 <param name="downloadReadTimeout" type="int">1000</param>
                 <param name="recoverDeletedRecords" type="bool">true</param>
@@ -350,6 +352,7 @@
             </params>
         </parser>
         <parser class="iped.parsers.bittorrent.BitTorrentResumeDatParser"></parser>
+        <parser class="iped.parsers.bittorrent.BitTorrentGenericDatParser"></parser>
         <parser class="iped.parsers.bittorrent.TorrentFileParser"></parser>
         <parser class="iped.parsers.lnk.LNKShortcutParser"></parser>
         <parser class="iped.parsers.misc.GenericOLEParser"></parser>
@@ -373,6 +376,7 @@
         <parser class="iped.parsers.mail.win10.Win10MailParser"></parser>
         <parser class="iped.parsers.discord.DiscordParser"></parser>
         <parser class="iped.parsers.apk.APKParser"></parser>
+        <parser class="iped.parsers.vlc.VLCIniParser"></parser>
 
     </parsers>