Reduced the memory foot print of the email parser

thunderbirdparser/ivy.xml

@@ -13,5 +13,6 @@
         <dependency conf="autopsy->default" org="org.apache.james" name="apache-mime4j-mbox-iterator" rev="0.8.0"/>
         <dependency conf="autopsy->default" org="com.googlecode.ez-vcard" name="ez-vcard" rev="0.10.5"/>
         <dependency conf="autopsy->default" org="com.github.mangstadt" name="vinnie" rev="2.0.2"/>
+        <dependency org="com.google.guava" name="guava" rev="19.0"/>
     </dependencies>
 </ivy-module>

thunderbirdparser/nbproject/project.properties

@@ -1,13 +1,19 @@
+file.reference.apache-mime4j-core-0.8.0-SNAPSHOT.jar=release/modules/ext/apache-mime4j-core-0.8.0-SNAPSHOT.jar
+file.reference.apache-mime4j-dom-0.8.0-SNAPSHOT.jar=release/modules/ext/apache-mime4j-dom-0.8.0-SNAPSHOT.jar
+file.reference.apache-mime4j-mbox-iterator-0.8.0-SNAPSHOT.jar=release/modules/ext/apache-mime4j-mbox-iterator-0.8.0-SNAPSHOT.jar
 file.reference.commons-lang3-3.8.1.jar=release/modules/ext/commons-lang3-3.8.1.jar
 file.reference.apache-mime4j-core-0.8.0.jar=release/modules/ext/apache-mime4j-core-0.8.0-SNAPSHOT.jar
 file.reference.apache-mime4j-dom-0.8.0.jar=release/modules/ext/apache-mime4j-dom-0.8.0-SNAPSHOT.jar
 file.reference.apache-mime4j-mbox-iterator-0.8.0.jar=release/modules/ext/apache-mime4j-mbox-iterator-0.8.0-SNAPSHOT.jar
+file.reference.guava-19.0.jar=release/modules/ext/guava-19.0.jar
 file.reference.java-libpst-1.0-SNAPSHOT.jar=release/modules/ext/java-libpst-1.0-SNAPSHOT.jar
 file.reference.ez-vcard-0.10.5.jar=release/modules/ext/ez-vcard-0.10.5.jar
 file.reference.vinnie-2.0.2.jar=release/modules/ext/vinnie-2.0.2.jar
 javac.source=1.8
 javac.compilerargs=-Xlint -Xlint:-serial
+javadoc.reference.guava-19.0.jar=release/modules/ext/guava-19.0-javadoc.jar
 license.file=../LICENSE-2.0.txt
 nbm.homepage=http://www.sleuthkit.org/autopsy/
 nbm.needs.restart=true
+source.reference.guava-19.0.jar=release/modules/ext/guava-19.0-sources.jar
 spec.version.base=4.0

thunderbirdparser/nbproject/project.xml

@@ -54,18 +54,6 @@
                 <runtime-relative-path>ext/commons-lang3-3.8.1.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/commons-lang3-3.8.1.jar</binary-origin>
             </class-path-extension>
-            <class-path-extension>
-                <runtime-relative-path>ext/apache-mime4j-core-0.8.0-SNAPSHOT.jar</runtime-relative-path>
-                <binary-origin>release/modules/ext/apache-mime4j-core-0.8.0-SNAPSHOT.jar</binary-origin>
-            </class-path-extension>
-            <class-path-extension>
-                <runtime-relative-path>ext/apache-mime4j-dom-0.8.0-SNAPSHOT.jar</runtime-relative-path>
-                <binary-origin>release/modules/ext/apache-mime4j-dom-0.8.0-SNAPSHOT.jar</binary-origin>
-            </class-path-extension>
-            <class-path-extension>
-                <runtime-relative-path>ext/java-libpst-1.0-SNAPSHOT.jar</runtime-relative-path>
-                <binary-origin>release/modules/ext/java-libpst-1.0-SNAPSHOT.jar</binary-origin>
-            </class-path-extension>
             <class-path-extension>
                 <runtime-relative-path>ext/apache-mime4j-mbox-iterator-0.8.0-SNAPSHOT.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/apache-mime4j-mbox-iterator-0.8.0-SNAPSHOT.jar</binary-origin>
@@ -74,6 +62,22 @@
                 <runtime-relative-path>ext/ez-vcard-0.10.5.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/ez-vcard-0.10.5.jar</binary-origin>
             </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/apache-mime4j-core-0.8.0-SNAPSHOT.jar</runtime-relative-path>
+                <binary-origin>release/modules/ext/apache-mime4j-core-0.8.0-SNAPSHOT.jar</binary-origin>
+            </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/java-libpst-1.0-SNAPSHOT.jar</runtime-relative-path>
+                <binary-origin>release/modules/ext/java-libpst-1.0-SNAPSHOT.jar</binary-origin>
+            </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/guava-19.0.jar</runtime-relative-path>
+                <binary-origin>release/modules/ext/guava-19.0.jar</binary-origin>
+            </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/apache-mime4j-dom-0.8.0-SNAPSHOT.jar</runtime-relative-path>
+                <binary-origin>release/modules/ext/apache-mime4j-dom-0.8.0-SNAPSHOT.jar</binary-origin>
+            </class-path-extension>
             <class-path-extension>
                 <runtime-relative-path>ext/vinnie-2.0.2.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/vinnie-2.0.2.jar</binary-origin>

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2011-2014 Basis Technology Corp.
+ * Copyright 2011-2019 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -33,6 +33,7 @@ import java.nio.charset.IllegalCharsetNameException;
 import java.nio.charset.StandardCharsets;
 import java.nio.charset.UnsupportedCharsetException;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 import java.util.UUID;
 import java.util.logging.Level;
@@ -57,22 +58,17 @@ import org.apache.tika.parser.txt.CharsetDetector;
 import org.apache.tika.parser.txt.CharsetMatch;
 import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
-import org.sleuthkit.autopsy.ingest.IngestServices;
 import org.sleuthkit.datamodel.TskData;
 import org.sleuthkit.datamodel.EncodedFileOutputStream;
 
 /**
- * A parser that extracts information about email messages and attachments from
+ * An Iterator for parsing mbox files.  Wraps an instance of MBoxEmailIterator.
- * a mbox file.
- *
- * @author jwallace
  */
-class MboxParser {
+class MboxParser implements Iterator<EmailMessage>{
 
     private static final Logger logger = Logger.getLogger(MboxParser.class.getName());
-    private DefaultMessageBuilder messageBuilder;
+    private final DefaultMessageBuilder messageBuilder;
-    private IngestServices services;
+    private final StringBuilder errors;
-    private StringBuilder errors;
 
     /**
      * The mime type string for html text.
@@ -83,9 +79,11 @@ class MboxParser {
      * The local path of the mbox file.
      */
     private String localPath;
+    
+    private Iterator<EmailMessage> emailIterator = null;
+    
 
-    MboxParser(IngestServices services, String localPath) {
+    private MboxParser( String localPath) {
-        this.services = services;
         this.localPath = localPath;
         messageBuilder = new DefaultMessageBuilder();
         MimeConfig config = MimeConfig.custom().setMaxLineLen(-1).build();
@@ -97,63 +95,76 @@ class MboxParser {
     static boolean isValidMimeTypeMbox(byte[] buffer) {
         return (new String(buffer)).startsWith("From "); //NON-NLS
     }
-
+    
     /**
-     * Parse the mbox file and get the email messages.
+     * Returns an instance of MBoxParser that will iterate and return EMailMessage
-     *
+     * objects with only the information needed for threading emails.
-     * @param mboxFile
+     * 
-     *
+     * @param localPath String path to the mboxFile
-     * @return a list of the email messages in the mbox file.
+     * @param mboxFile The mboxFile to parse
+     * @return Instance of MboxParser
      */
-    List<EmailMessage> parse(File mboxFile, long fileID) {
+    static MboxParser getThreadInfoIterator(String localPath, File mboxFile) {
+        MboxParser parser = new MboxParser(localPath);
+        parser.createIterator(mboxFile, 0, false);
+        return parser;
+    }
+    
+    /**
+     * Returns an instance of MBoxParser that will iterate "whole" EmailMessages.
+     * 
+     * @param localPath String path to the mboxFile
+     * @param mboxFile The mboxFile to parse
+     * @param fileID The fileID of the abstractFile that mboxFile was found
+     * @return Instance of MboxParser
+     */
+    static MboxParser getEmailIterator(String localPath, File mboxFile, long fileID) {
+        MboxParser parser = new MboxParser(localPath);
+        parser.createIterator(mboxFile, fileID, true);
+        
+        return parser;
+    }
+    
+    /**
+     * Creates the real Iterator object instance.
+     * 
+     * @param mboxFile  The mboxFile to parse
+     * @param fileID    The fileID of the abstractFile that mboxFile was found
+     * @param wholeMsg  True if EmailMessage should have the whole message, 
+     *                  not just the thread information.
+     */
+    private void createIterator(File mboxFile, long fileID, boolean wholeMsg) {
         // Detect possible charsets
         List<CharsetEncoder> encoders = getPossibleEncoders(mboxFile);
-
+        
-        CharsetEncoder theEncoder = null;
-        Iterable<CharBufferWrapper> mboxIterator = null;
         // Loop through the possible encoders and find the first one that works.
         // That will usually be one of the first ones.
         for (CharsetEncoder encoder : encoders) {
             try {
-                mboxIterator = MboxIterator.fromFile(mboxFile).charset(encoder.charset()).build();
+                Iterable<CharBufferWrapper> mboxIterable = MboxIterator.fromFile(mboxFile).charset(encoder.charset()).build();
-                theEncoder = encoder;
+                if(mboxIterable != null) {
+                    emailIterator = new MBoxEmailIterator(mboxIterable.iterator(), encoder, fileID, wholeMsg);
+                }
                 break;
             } catch (CharConversionException | UnsupportedCharsetException ex) {
                 // Not the right encoder
             } catch (IllegalArgumentException ex) {
                 // Not the right encoder
             } catch (IOException ex) {
-                logger.log(Level.WARNING, "couldn't find mbox file.", ex); //NON-NLS
+                logger.log(Level.WARNING, String.format("Failed to open mbox file: %s %d", mboxFile.getName(), fileID), ex); //NON-NLS
                 addErrorMessage(NbBundle.getMessage(this.getClass(), "MboxParser.parse.errMsg.failedToReadFile"));
-                return new ArrayList<>();
             }
         }
-
+    }
-        // If no encoders work, post an error message and return.
+    
-        if (mboxIterator == null || theEncoder == null) {
+    @Override
-            addErrorMessage(NbBundle.getMessage(this.getClass(), "MboxParser.parse.errMsg.couldntFindCharset"));
+    public boolean hasNext() {
-            return new ArrayList<>();
+        return emailIterator != null ? emailIterator.hasNext() : false;
-        }
+    }
-
+    
-        List<EmailMessage> emails = new ArrayList<>();
+    @Override
-        long failCount = 0;
+    public EmailMessage next() {
-
+        return emailIterator != null ? emailIterator.next() : null;
-        // Parse each message and extract an EmailMessage structure
-        for (CharBufferWrapper message : mboxIterator) {
-            try {
-                Message msg = messageBuilder.parseMessage(message.asInputStream(theEncoder.charset()));
-                emails.add(extractEmail(msg, fileID));
-            } catch (RuntimeException | IOException ex) {
-                logger.log(Level.WARNING, "Failed to get message from mbox: {0}", ex.getMessage()); //NON-NLS
-                failCount++;
-            }
-        }
-
-        if (failCount > 0) {
-            addErrorMessage(
-                    NbBundle.getMessage(this.getClass(), "MboxParser.parse.errMsg.failedToParseNMsgs", failCount));
-        }
-        return emails;
     }
 
     String getErrors() {
@@ -211,6 +222,44 @@ class MboxParser {
         
         return email;
     }
+    
+    /**
+     * Extract the subject, inReplyTo, message-ID and references from the Message
+     * object and returns them in a new EmailMessage object.
+     * 
+     * @param msg Message object
+     * 
+     * @return EmailMessage instance with only some of the message information 
+     */
+    private EmailMessage extractPartialEmail(Message msg) {
+        EmailMessage email = new EmailMessage();
+        email.setSubject(msg.getSubject());
+        email.setMessageID(msg.getMessageId());
+        
+        Field field = msg.getHeader().getField("in-reply-to"); //NON-NLS
+        String inReplyTo = null;
+
+        if (field != null) {
+            inReplyTo = field.getBody();
+            email.setInReplyToID(inReplyTo);
+        }
+
+        field = msg.getHeader().getField("references");
+        if (field != null) {
+            List<String> references = new ArrayList<>();
+            for (String id : field.getBody().split(">")) {
+                references.add(id.trim() + ">");
+            }
+
+            if (!references.contains(inReplyTo)) {
+                references.add(inReplyTo);
+            }
+
+            email.setReferences(references);
+        }
+        
+        return email;
+    }
 
     /**
      * Handle a multipart mime message. Recursively calls handleMultipart if one
@@ -333,7 +382,7 @@ class MboxParser {
             addErrorMessage(
                     NbBundle.getMessage(this.getClass(),
                             "MboxParser.handleAttch.errMsg.failedToCreateOnDisk", outPath));
-            logger.log(Level.INFO, "Failed to create file output stream for: " + outPath, ex); //NON-NLS
+            logger.log(Level.WARNING, "Failed to create file output stream for: " + outPath, ex); //NON-NLS
             return;
         }
 
@@ -346,14 +395,14 @@ class MboxParser {
                 // This could potentially be other types. Only seen this once.
             }
         } catch (IOException ex) {
-            logger.log(Level.INFO, "Failed to write mbox email attachment to disk.", ex); //NON-NLS
+            logger.log(Level.WARNING, "Failed to write mbox email attachment to disk.", ex); //NON-NLS
             addErrorMessage(NbBundle.getMessage(this.getClass(), "MboxParser.handleAttch.failedWriteToDisk", filename));
             return;
         } finally {
             try {
                 fos.close();
             } catch (IOException ex) {
-                logger.log(Level.INFO, "Failed to close file output stream", ex); //NON-NLS
+                logger.log(Level.WARNING, "Failed to close file output stream", ex); //NON-NLS
             }
         }
 
@@ -441,7 +490,7 @@ class MboxParser {
             try {
                 is.close();
             } catch (IOException ex) {
-                logger.log(Level.INFO, "Failed to close input stream"); //NON-NLS
+                logger.log(Level.WARNING, "Failed to close input stream"); //NON-NLS
             }
         }
     }
@@ -449,4 +498,45 @@ class MboxParser {
     private void addErrorMessage(String msg) {
         errors.append("<li>").append(msg).append("</li>"); //NON-NLS
     }
+    
+    /**
+     * An Interator for mbox email messages.
+     */
+    final class MBoxEmailIterator implements Iterator<EmailMessage> {
+        
+        private final Iterator<CharBufferWrapper> mboxIterator;
+        private final CharsetEncoder encoder;
+        private final long fileID;
+        private final boolean wholeMsg;
+        
+        MBoxEmailIterator(Iterator<CharBufferWrapper> mboxIter, CharsetEncoder encoder, long fileID, boolean wholeMsg) {
+            mboxIterator = mboxIter;
+            this.encoder = encoder;
+            this.fileID = fileID;
+            this.wholeMsg = wholeMsg;
+        }
+
+        @Override
+        public boolean hasNext() {
+            return (mboxIterator != null && encoder != null) ? mboxIterator.hasNext() : false;
+        }
+
+        @Override
+        public EmailMessage next() {
+            CharBufferWrapper messageBuffer = mboxIterator.next();
+            
+            try {
+                Message msg = messageBuilder.parseMessage(messageBuffer.asInputStream(encoder.charset()));
+                if(wholeMsg) {
+                    return extractEmail(msg, fileID);
+                } else {                
+                    return extractPartialEmail(msg);
+                }
+            } catch (RuntimeException | IOException ex) {
+                logger.log(Level.WARNING, "Failed to get message from mbox: {0}", ex.getMessage()); //NON-NLS
+            }
+            return null;
+        }
+        
+    }
 }

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2011-2014 Basis Technology Corp.
+ * Copyright 2011-2019 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,6 +18,7 @@
  */
 package org.sleuthkit.autopsy.thunderbirdparser;
 
+import com.google.common.collect.Iterables;
 import com.pff.PSTAttachment;
 import com.pff.PSTException;
 import com.pff.PSTFile;
@@ -29,6 +30,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Scanner;
 import java.util.logging.Level;
@@ -55,112 +57,193 @@ class PstParser {
      * First four bytes of a pst file.
      */
     private static int PST_HEADER = 0x2142444E;
-    private IngestServices services;
     /**
      * A map of PSTMessages to their Local path within the file's internal
      * directory structure.
      */
-    private List<EmailMessage> results;
+    private final StringBuilder errors;
-    private StringBuilder errors;
+    
-
+    private final IngestServices services;
+    
+    private PSTFile pstFile;
+    private long fileID;
+    
+    private int failureCount = 0;
+      
     PstParser(IngestServices services) {
-        results = new ArrayList<>();
-        this.services = services;
         errors = new StringBuilder();
+        this.services = services;
     }
 
     enum ParseResult {
 
         OK, ERROR, ENCRYPT;
     }
-
     /**
-     * Parse and extract email messages from the pst/ost file.
+     * Create an instance of PSTFile for the given File object. 
-     *
+     * 
-     * @param file A pst or ost file.
+     * The constructor for PSTFile object will throw a generic PSTException if the
-     *
+     * file is encrypted. 
-     * @return ParseResult: OK on success, ERROR on an error, ENCRYPT if failed
+     * <a href=https://github.com/rjohnsondev/java-libpst/blob/5436a7abc8ac8c1622bf5dba0f4f9428fdbcd634/src/main/java/com/pff/PSTFile.java> PSTFile.java</a>
-     *         because the file is encrypted.
+     * 
+     * @param file File to open
+     * @param fileID File id for use when creating the EmailMessage objects
+     * @return  ParserResult value OK if the PSTFile was successfully created, 
+     *          ENCRYPT will be returned for PSTExceptions that matches at specific
+     *          message or IllegalArgumentExceptions
      */
-    ParseResult parse(File file, long fileID) {
+    ParseResult open(File file, long fileID) {
-        PSTFile pstFile;
+        if(file == null) {
-        long failures;
+            return ParseResult.ERROR;
+        }
+        
         try {
             pstFile = new PSTFile(file);
-            failures = processFolder(pstFile.getRootFolder(), "\\", true, fileID);
+        } catch(PSTException ex) {
-            if (failures > 0) {
+            // This is the message thrown from the PSTFile constructor if it
-                addErrorMessage(
+            // detects that the file is encrypted. 
-                        NbBundle.getMessage(this.getClass(), "PstParser.parse.errMsg.failedToParseNMsgs", failures));
+            if(ex.getMessage().equals("Only unencrypted and compressable PST files are supported at this time")) { //NON-NLS
+                logger.log(Level.INFO, "Found encrypted PST file."); //NON-NLS
+                return ParseResult.ENCRYPT;
             }
-            return ParseResult.OK;
-        } catch (PSTException | IOException ex) {
             String msg = file.getName() + ": Failed to create internal java-libpst PST file to parse:\n" + ex.getMessage(); //NON-NLS
-            logger.log(Level.WARNING, msg);
+            logger.log(Level.WARNING, msg, ex);
             return ParseResult.ERROR;
-        } catch (IllegalArgumentException ex) {
+        } catch (IOException ex) {
+            String msg = file.getName() + ": Failed to create internal java-libpst PST file to parse:\n" + ex.getMessage(); //NON-NLS
+            logger.log(Level.WARNING, msg, ex);
+            return ParseResult.ERROR;
+        } catch (IllegalArgumentException ex) { // Not sure if this is true, was in previous version of code.
             logger.log(Level.INFO, "Found encrypted PST file."); //NON-NLS
             return ParseResult.ENCRYPT;
         }
+        
+        return ParseResult.OK;
     }
-
+    
     /**
-     * Get the results of the parsing.
+     * Creates an EmailMessage iterator for pstFile.  These Email objects will be
-     *
+     * complete and with all available information.
-     * @return
+     * 
+     * @return A instance of an EmailMessage Iterator
      */
-    List<EmailMessage> getResults() {
+    Iterator<EmailMessage> getEmailMessageIterator() {
-        return results;
+        if(pstFile == null) {
+            return null;
+        }
+        
+        Iterable<EmailMessage> iterable = null;
+        
+        try {
+            iterable = getEmaiMessageIterator(pstFile.getRootFolder(), "\\", fileID, true);
+        } catch (PSTException | IOException ex) {
+            logger.log(Level.WARNING, String.format("Exception thrown while parsing fileID: %d", fileID), ex);
+        }
+        
+        if(iterable == null) {
+            return null;
+        }
+        
+        return iterable.iterator();
     }
-
+    
+    /**
+     * Get a List of EmailMessages which contain only the information needed for
+     * threading the emails.
+     * 
+     * @return A list of EmailMessage or an empty list if non were found.
+     */
+    List<EmailMessage> getPartialEmailMessages() {
+        List<EmailMessage> messages = new ArrayList<>();
+        Iterator<EmailMessage> iterator = getPartialEmailMessageIterator();
+        if(iterator != null) {
+            while(iterator.hasNext()) {
+                messages.add(iterator.next());
+            }
+        }
+        
+        return messages;
+    }
+    
+    /**
+     * 
+     * @return 
+     */
     String getErrors() {
         return errors.toString();
     }
-
+    
-    /**
+    int getFailureCount() {
-     * Process this folder and all subfolders, adding every email found to
+        return failureCount;
-     * results. Accumulates the folder hierarchy path as it navigates the folder
+    }
-     * structure.
+     
-     *
+     /**
-     * @param folder The folder to navigate and process
+     * Get an Iterator to which will iterate over the PSTFile, but return EmailMessages
-     * @param path   The path to the folder within the pst/ost file's directory
+     * with only the information needed for putting the emails into threads.
-     *               structure
+     * 
-     *
+     * @return A EmailMessage iterator or null if no messages where found
-     * @throws PSTException
-     * @throws IOException
      */
-    private long processFolder(PSTFolder folder, String path, boolean root, long fileID) {
+    private Iterator<EmailMessage> getPartialEmailMessageIterator() {
-        String newPath = (root ? path : path + "\\" + folder.getDisplayName());
+        if(pstFile == null) {
-        long failCount = 0L; // Number of emails that failed
+            return null;
-        if (folder.hasSubfolders()) {
-            List<PSTFolder> subFolders;
-            try {
-                subFolders = folder.getSubFolders();
-            } catch (PSTException | IOException ex) {
-                subFolders = new ArrayList<>();
-                logger.log(Level.INFO, "java-libpst exception while getting subfolders: {0}", ex.getMessage()); //NON-NLS
-            }
-
-            for (PSTFolder f : subFolders) {
-                failCount += processFolder(f, newPath, false, fileID);
-            }
         }
-
+        
-        if (folder.getContentCount() != 0) {
+        Iterable<EmailMessage> iterable = null;
-            PSTMessage email;
+        
-            // A folder's children are always emails, never other folders.
+        try {
-            try {
+            iterable = getEmaiMessageIterator(pstFile.getRootFolder(), "\\", fileID, false);
-                while ((email = (PSTMessage) folder.getNextChild()) != null) {
+        } catch (PSTException | IOException ex) {
-                    results.add(extractEmailMessage(email, newPath, fileID));
+            logger.log(Level.WARNING, String.format("Exception thrown while parsing fileID: %d", fileID), ex);
-                }
-            } catch (PSTException | IOException ex) {
-                failCount++;
-                logger.log(Level.INFO, "java-libpst exception while getting emails from a folder: {0}", ex.getMessage()); //NON-NLS
-            }
         }
-
+        
-        return failCount;
+        if(iterable == null) {
+            return null;
+        }
+        
+        return iterable.iterator();
     }
 
+    /**
+     * Creates an Iterable object of Email messages for the given folder.
+     * 
+     * @param folder PSTFolder to process
+     * @param path String path to folder
+     * @param fileID FileID of the AbstractFile folder was found in
+     * @param partialEmail Whether or not fill the EMailMessage with all data
+     * 
+     * @return  An Iterable for iterating email message, or null if there were no 
+     *          messages or children in folder.
+     * 
+     * @throws PSTException
+     * @throws IOException 
+     */
+    private Iterable<EmailMessage> getEmaiMessageIterator(PSTFolder folder, String path, long fileID, boolean wholeMsg) throws PSTException, IOException {
+        Iterable<EmailMessage> iterable = null;
+        
+        if(folder.getContentCount() > 0) {
+            iterable = new PstEmailIterator(folder, path, fileID, wholeMsg).getIterable();
+        }
+        
+        if(folder.hasSubfolders()) {
+            List<PSTFolder> subFolders = folder.getSubFolders();
+            for(PSTFolder subFolder: subFolders) {
+                String newpath = path + "\\" + subFolder.getDisplayName();
+                Iterable<EmailMessage> subIterable = getEmaiMessageIterator(subFolder, newpath, fileID, wholeMsg);
+                if(subIterable == null) {
+                    continue;
+                }
+                
+                if(iterable != null) {   
+                    iterable = Iterables.concat(iterable,  subIterable);
+                } else {
+                    iterable = subIterable;
+                }
+                
+            }
+        }
+        
+        return iterable;
+    }
     /**
      * Create an EmailMessage from a PSTMessage.
      *
@@ -214,6 +297,33 @@ class PstParser {
 
         return email;
     }
+    
+    /**
+     * Create an EmailMessage from a PSTMessage with only the information
+     * needed for threading emails.
+     *
+     * @return EmailMessage object with only some information, not all of the msg.
+     */
+    private EmailMessage extractPartialEmailMessage(PSTMessage msg) {
+        EmailMessage email = new EmailMessage();
+        email.setSubject(msg.getSubject());
+        email.setId(msg.getDescriptorNodeId());
+        email.setMessageID(msg.getInternetMessageId());
+        String inReplyToID = msg.getInReplyToId();
+        email.setInReplyToID(inReplyToID);
+        List<String> references = extractReferences(msg.getTransportMessageHeaders());
+        if (inReplyToID != null && !inReplyToID.isEmpty()) {
+            if (references == null) {
+                references = new ArrayList<>();
+                references.add(inReplyToID);
+            } else if (!references.contains(inReplyToID)) {
+                references.add(inReplyToID);
+            }
+        }
+        email.setReferences(references);
+
+        return email;
+    }
 
     /**
      * Add the attachments within the PSTMessage to the EmailMessage.
@@ -388,5 +498,95 @@ class PstParser {
         }
 
         return null;
+    }
+
+    /**
+     * A iterator for processing the PST email folder structure and returning 
+     * instances of the EmailMessage object.
+     */
+    private final class PstEmailIterator implements Iterator<EmailMessage> {
+
+        private final PSTFolder folder;
+        private EmailMessage current;
+        private EmailMessage next;
+        
+        private final String currentPath;
+        private final long fileID;
+        private final boolean wholeMsg;
+        
+        /**
+         * Class constructor, initializes the "next" message;
+         * 
+         * @param folder PSTFolder object to iterate across
+         * @param path String path value to the location of folder
+         * @param fileID Long fileID of the abstract file this PSTFolder was found
+         */
+        PstEmailIterator(PSTFolder folder, String path, long fileID, boolean wholeMsg) {
+            this.folder = folder;
+            this.fileID = fileID;
+            this.currentPath = path;
+            this.wholeMsg = wholeMsg;
+            
+            if(folder.getContentCount() > 0) {
+                try {
+                    PSTMessage message = (PSTMessage)folder.getNextChild();
+                    if(message != null) {
+                        if(wholeMsg) {
+                            next = extractEmailMessage(message, currentPath, fileID);
+                        } else {
+                            next = extractPartialEmailMessage(message);
+                        }
+                    }
+                } catch (PSTException |  IOException ex) {
+                    failureCount++;
+                    logger.log(Level.WARNING, String.format("Unable to extract emails for path: %s file ID: %d ", path, fileID), ex);
+                }
+            } 
+        }
+        
+        @Override
+        public boolean hasNext() {
+            return next != null;
+        }
+
+        @Override
+        public EmailMessage next() {
+            
+            current = next;
+            
+            try {
+                PSTMessage message = (PSTMessage)folder.getNextChild();
+                if(message != null) {
+                    if(wholeMsg) {
+                        next = extractEmailMessage(message, currentPath, fileID);
+                    } else {
+                        next = extractPartialEmailMessage(message);
+                    }
+                } else {
+                    next = null;
+                }
+            } catch (PSTException |  IOException ex) {
+                logger.log(Level.WARNING, String.format("Unable to extract emails for path: %s file ID: %d ", currentPath, fileID), ex);
+                failureCount++;
+                next = null;
+            } 
+            
+            return current;
+        }
+        
+        /**
+         * Get a wrapped Iterable version of PstEmailIterator
+         * 
+         * @return Iterable wrapping this class
+         */
+        Iterable<EmailMessage> getIterable(){
+            return new Iterable<EmailMessage>(){
+                @Override
+                public Iterator<EmailMessage> iterator() {
+                    return PstEmailIterator.this;
+                }
+            };
+        }
+        
     }    
 }

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java

@@ -23,6 +23,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import java.util.logging.Level;
@@ -183,51 +184,53 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
         }
 
         PstParser parser = new PstParser(services);
-        PstParser.ParseResult result = parser.parse(file, abstractFile.getId());
+        PstParser.ParseResult result = parser.open(file, abstractFile.getId());
 
-        if (result == PstParser.ParseResult.OK) {
+        switch( result) {
-            // parse success: Process email and add artifacts
+            case OK:
-            processEmails(parser.getResults(), abstractFile);
+                processEmails(parser.getPartialEmailMessages(), parser.getEmailMessageIterator(), abstractFile);
-
+                break;
-        } else if (result == PstParser.ParseResult.ENCRYPT) {
-            // encrypted pst: Add encrypted file artifact
-            try {
-                BlackboardArtifact artifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
-                artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, EmailParserModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.encryptionFileLevel")));
 
+            case ENCRYPT:
+                // encrypted pst: Add encrypted file artifact
                 try {
-                    // index the artifact for keyword search
+                    BlackboardArtifact artifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
-                    blackboard.indexArtifact(artifact);
+                    artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, EmailParserModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.encryptionFileLevel")));
-                } catch (Blackboard.BlackboardException ex) {
-                    MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_processPst_indexError_message(), artifact.getDisplayName());
-                    logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS
-                }
 
-                services.fireModuleDataEvent(new ModuleDataEvent(EmailParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED));
+                    try {
-            } catch (TskCoreException ex) {
+                        // index the artifact for keyword search
-                logger.log(Level.INFO, "Failed to add encryption attribute to file: {0}", abstractFile.getName()); //NON-NLS
+                        blackboard.indexArtifact(artifact);
-            }
+                    } catch (Blackboard.BlackboardException ex) {
-        } else {
+                        MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_processPst_indexError_message(), artifact.getDisplayName());
-            // parsing error: log message
+                        logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS
-            postErrorMessage(
+                    }
-                    NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processPst.errProcFile.msg",
+
-                            abstractFile.getName()),
+                    services.fireModuleDataEvent(new ModuleDataEvent(EmailParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED));
-                    NbBundle.getMessage(this.getClass(),
+                } catch (TskCoreException ex) {
-                            "ThunderbirdMboxFileIngestModule.processPst.errProcFile.details"));
+                    logger.log(Level.INFO, "Failed to add encryption attribute to file: {0}", abstractFile.getName()); //NON-NLS
-            logger.log(Level.INFO, "PSTParser failed to parse {0}", abstractFile.getName()); //NON-NLS
+                }
-            return ProcessResult.ERROR;
+                break;
+            default:
+                // parsing error: log message
+                postErrorMessage(
+                        NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processPst.errProcFile.msg",
+                                abstractFile.getName()),
+                        NbBundle.getMessage(this.getClass(),
+                                "ThunderbirdMboxFileIngestModule.processPst.errProcFile.details"));
+                logger.log(Level.INFO, "PSTParser failed to parse {0}", abstractFile.getName()); //NON-NLS
+                return ProcessResult.ERROR;
         }
 
         if (file.delete() == false) {
             logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS
         }
 
-        String errors = parser.getErrors();
+//        String errors = parser.getErrors();
-        if (errors.isEmpty() == false) {
+//        if (errors.isEmpty() == false) {
-            postErrorMessage(
+//            postErrorMessage(
-                    NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processPst.errProcFile.msg2",
+//                    NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processPst.errProcFile.msg2",
-                            abstractFile.getName()), errors);
+//                            abstractFile.getName()), errors);
-        }
+//        }
 
         return ProcessResult.OK;
     }
@@ -281,21 +284,29 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
             return ProcessResult.OK;
         }
 
-        MboxParser parser = new MboxParser(services, emailFolder);
+        MboxParser emailIterator = MboxParser.getEmailIterator( emailFolder, file, abstractFile.getId());
-        List<EmailMessage> emails = parser.parse(file, abstractFile.getId());
+        List<EmailMessage> emails = new ArrayList<>();
-        processEmails(emails, abstractFile);
+        if(emailIterator != null) {
+            while(emailIterator.hasNext()) {
+                EmailMessage emailMessage = emailIterator.next();
+                if(emailMessage != null) {
+                    emails.add(emailMessage);
+                }
+            }
+            
+            String errors = emailIterator.getErrors();
+            if (!errors.isEmpty()) {
+                postErrorMessage(
+                        NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processMBox.errProcFile.msg2",
+                                abstractFile.getName()), errors);
+            }
+        }
+        processEmails(emails, MboxParser.getEmailIterator( emailFolder, file, abstractFile.getId()), abstractFile);
 
         if (file.delete() == false) {
             logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS
         }
 
-        String errors = parser.getErrors();
-        if (errors.isEmpty() == false) {
-            postErrorMessage(
-                    NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processMBox.errProcFile.msg2",
-                            abstractFile.getName()), errors);
-        }
-
         return ProcessResult.OK;
     }
     
@@ -434,7 +445,50 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
         context.addFilesToJob(derivedFiles);
         services.fireModuleDataEvent(new ModuleDataEvent(EmailParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG));
     }
+    
+    private void processEmails(List<EmailMessage> partialEmailsForThreading, Iterator<EmailMessage> fullMessageIterator, AbstractFile abstractFile) {
+        // Putting try/catch around this to catch any exception and still allow
+        // the creation of the artifacts to continue.
+        try{
+            EmailMessageThreader.threadMessages(partialEmailsForThreading, String.format("%d", abstractFile.getId()));
+        } catch(Exception ex) {
+            logger.log(Level.WARNING, String.format("Exception thrown parsing emails from %s", abstractFile.getName()), ex);
+        }
+        
+        List<AbstractFile> derivedFiles = new ArrayList<>();
 
+        int msgCnt = 0;
+        while(fullMessageIterator.hasNext()) {
+            EmailMessage current = fullMessageIterator.next();
+            
+            if(current == null) {
+                continue;
+            }
+
+            if(partialEmailsForThreading.size() > msgCnt) {
+                EmailMessage threaded = partialEmailsForThreading.get(msgCnt++);
+                
+                if(threaded.getMessageID().equals(current.getMessageID()) &&
+                        threaded.getSubject().equals(current.getSubject())) {
+                    current.setMessageThreadID(threaded.getMessageThreadID());
+                }
+            }
+            
+            BlackboardArtifact msgArtifact = addEmailArtifact(current, abstractFile);
+            
+            if ((msgArtifact != null) && (current.hasAttachment()))  {
+                derivedFiles.addAll(handleAttachments(current.getAttachments(), abstractFile, msgArtifact ));
+            }
+        }
+
+        if (derivedFiles.isEmpty() == false) {
+            for (AbstractFile derived : derivedFiles) {
+                services.fireModuleContentEvent(new ModuleContentEvent(derived));
+            }
+        }
+        context.addFilesToJob(derivedFiles);
+        services.fireModuleDataEvent(new ModuleDataEvent(EmailParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG));
+    }
     /**
      * Add the given attachments as derived files and reschedule them for
      * ingest.