5709: Email attachments

- Use CommunicationsArtifactsHelper to add the attachments to email messages.
 - Updated the DirectoryTree & DataResultsTable to not show message artifacts for 'new' cases, but continue to show them for older cases.

Core/nbproject/project.xml

@@ -345,6 +345,7 @@
                 <package>org.sleuthkit.autopsy.textextractors.configs</package>
                 <package>org.sleuthkit.autopsy.texttranslation</package>
                 <package>org.sleuthkit.datamodel</package>
+                <package>org.sleuthkit.datamodel.blackboardutils</package>
             </public-packages>
             <class-path-extension>
                 <runtime-relative-path>ext/commons-lang3-3.8.1.jar</runtime-relative-path>

Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java

@@ -41,7 +41,6 @@ import org.sleuthkit.autopsy.actions.DeleteFileBlackboardArtifactTagAction;
 import org.sleuthkit.autopsy.actions.DeleteFileContentTagAction;
 import org.sleuthkit.autopsy.coreutils.ContextMenuExtensionPoint;
 import org.sleuthkit.autopsy.coreutils.Logger;
-import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode.AbstractFilePropertyType;
 import org.sleuthkit.autopsy.datamodel.AbstractFsContentNode;
 import org.sleuthkit.autopsy.datamodel.BlackboardArtifactNode;
 import org.sleuthkit.autopsy.datamodel.DataModelActionsFactory;
@@ -258,16 +257,20 @@ public class DataResultFilterNode extends FilterNode {
 
         @Override
         protected Node[] createNodes(Node key) {
-            // filter out all non-message artifacts, if displaying the results from the Data Source tree
+            // if displaying the results from the Data Source tree
+            // filter out artifacts
+            // unless there are message artifacts with attachments as children
             BlackboardArtifact art = key.getLookup().lookup(BlackboardArtifact.class);
-            if (art != null
+            if (art != null && filterArtifacts) {
-                    && filterArtifacts
+                
-                    && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID()
+                if ((DirectoryTreeUtils.showMessagesInDirTree() == false)  || 
-                    && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE.getTypeID()) {
+                    (DirectoryTreeUtils.showMessagesInDirTree() &&
-                return new Node[]{};
+                        art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID() &&
-            }
+                        art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE.getTypeID()) ) {
-
+                             return new Node[]{};
-            return new Node[]{new DataResultFilterNode(key, sourceEm)};
+                        }
+                }
+             return new Node[]{new DataResultFilterNode(key, sourceEm)};
         }
     }
 

Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterNode.java

@@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.directorytree;
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.logging.Level;
 import javax.swing.Action;
@@ -33,17 +32,12 @@ import org.sleuthkit.autopsy.core.UserPreferences;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.datamodel.AbstractContentNode;
 import org.sleuthkit.autopsy.datamodel.BlackboardArtifactNode;
-import org.sleuthkit.autopsy.ingest.runIngestModuleWizard.RunIngestModulesAction;
 import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.BlackboardArtifact;
 import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
 import org.sleuthkit.datamodel.Content;
-import org.sleuthkit.datamodel.Directory;
-import org.sleuthkit.datamodel.Image;
 import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.datamodel.TskData;
-import org.sleuthkit.datamodel.VirtualDirectory;
-import org.sleuthkit.datamodel.Volume;
 
 /**
  * A node filter (decorator) that sets the actions for a node in the tree view
@@ -137,11 +131,18 @@ class DirectoryTreeFilterNode extends FilterNode {
                         numVisibleChildren--;
                     }
                 } else if (child instanceof BlackboardArtifact) {
-                    BlackboardArtifact bba = (BlackboardArtifact) child;
+                    
-
+                    if (DirectoryTreeUtils.showMessagesInDirTree()) {
-                    // Only message type artifacts are displayed in the tree
+                        // In older versions of Autopsy,  attachments were children of email/message artifacts
-                    if ((bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID())
+                        // and hence email/messages with attachments are shown in the directory tree.
-                            && (bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_MESSAGE.getTypeID())) {
+                        BlackboardArtifact bba = (BlackboardArtifact) child;
+                        // Only message type artifacts are displayed in the tree
+                        if ((bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID())
+                                && (bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_MESSAGE.getTypeID())) {
+                            numVisibleChildren--;
+                        }
+                    }
+                    else {
                         numVisibleChildren--;
                     }
                 }

Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeUtils.java

@@ -0,0 +1,60 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2019 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.sleuthkit.autopsy.directorytree;
+
+import org.sleuthkit.autopsy.casemodule.Case;
+import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber;
+
+/**
+ * Utility class for Directory tree.
+ * 
+ */
+final class DirectoryTreeUtils {
+    
+    private static final int ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER = 8;
+    private static final int ATTACHMENT_CHILDOF_MSG_MAX_DB_MINOR_VER = 4;
+     
+    
+     /**
+     * Prior to schema version 8.4, attachments were children of messages and
+     * hence messages with any attachment children are shown in the directory
+     * tree.
+     *
+     * At 8.4, attachments are tracked as an attribute, and the message artifact
+     * don't need to be shown in the directory tree.
+     *
+     * This method may be used to check the schema version and behave
+     * accordingly, in order to maintain backward compatibility.
+     *
+     * @return True if messages with attachment children should be shown in
+     * directory tree.
+     */
+    static boolean showMessagesInDirTree() {
+        boolean showMessagesInDirTree = true;
+        if (Case.isCaseOpen()) {
+            CaseDbSchemaVersionNumber version = Case.getCurrentCase().getSleuthkitCase().getDBSchemaCreationVersion();
+            showMessagesInDirTree
+                    = ((version.getMajor() < ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER)
+                    || (version.getMajor() == ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER && version.getMinor() < ATTACHMENT_CHILDOF_MSG_MAX_DB_MINOR_VER));
+        }
+        return showMessagesInDirTree;
+    }
+    
+}

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED

@@ -15,6 +15,7 @@ ThunderbirdMboxFileIngestModule.addArtifact.indexError.message=Failed to index e
 # {0} - file name
 # {1} - file ID
 ThunderbirdMboxFileIngestModule.errorMessage.outOfDiskSpace=Out of disk space. Cannot copy '{0}' (id={1}) to parse.
+ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message.
 ThunderbirdMboxFileIngestModule.moduleName=Email Parser
 ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case.
 ThunderbirdMboxFileIngestModule.processPst.errMsg.outOfDiskSpace=Out of disk space. Cannot copy {0} to parse.

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java

@@ -22,6 +22,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -58,6 +59,9 @@ import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.datamodel.TskData;
 import org.sleuthkit.datamodel.TskDataException;
 import org.sleuthkit.datamodel.TskException;
+import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper;
+import org.sleuthkit.datamodel.blackboardutils.FileAttachment;
+import org.sleuthkit.datamodel.blackboardutils.MessageAttachments;
 
 /**
  * File-level ingest module that detects MBOX, PST, and vCard files based on
@@ -70,6 +74,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
     private FileManager fileManager;
     private IngestJobContext context;
     private Blackboard blackboard;
+    private CommunicationArtifactsHelper communicationArtifactsHelper;
     
     private Case currentCase;
 
@@ -129,6 +134,15 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
             logger.log(Level.WARNING, null, ex);
         }
 
+        try {
+            communicationArtifactsHelper = new CommunicationArtifactsHelper(currentCase.getSleuthkitCase(),
+                    EmailParserModuleFactory.getModuleName(), abstractFile, Account.Type.EMAIL);
+        } catch (TskCoreException ex) {
+            logger.log(Level.SEVERE, String.format("Failed to create CommunicationArtifactsHelper for file %s", abstractFile.getName()), ex);
+            return ProcessResult.ERROR;
+        }
+
+       
         if (isMbox) {
             return processMBox(abstractFile);
         }
@@ -267,7 +281,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
         } else if (mboxParentDir.contains("/ImapMail/")) { //NON-NLS
             emailFolder = mboxParentDir.substring(mboxParentDir.indexOf("/ImapMail/") + 9); //NON-NLS
         }
-        emailFolder = emailFolder + mboxFileName;
+        emailFolder += mboxFileName;
         emailFolder = emailFolder.replaceAll(".sbd", ""); //NON-NLS
 
         String fileName;
@@ -487,8 +501,12 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
      *
      * @return List of attachments
      */
+    @NbBundle.Messages({
+    "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message."
+})
     private List<AbstractFile> handleAttachments(List<EmailMessage.Attachment> attachments, AbstractFile abstractFile, BlackboardArtifact messageArtifact) {
         List<AbstractFile> files = new ArrayList<>();
+        List<FileAttachment> fileAttachments = new ArrayList<>();
         for (EmailMessage.Attachment attach : attachments) {
             String filename = attach.getName();
             long crTime = attach.getCrTime();
@@ -501,12 +519,14 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
 
             try {
                 DerivedFile df = fileManager.addDerivedFile(filename, relPath,
-                        size, cTime, crTime, aTime, mTime, true, messageArtifact, "",
+                        size, cTime, crTime, aTime, mTime, true, abstractFile, "",
                         EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleVersion(), "", encodingType);
                 
                 associateAttachmentWithMesssge(messageArtifact, df);
                 
                 files.add(df);
+                
+                fileAttachments.add(new FileAttachment(df));
             } catch (TskCoreException ex) {
                 postErrorMessage(
                         NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.errMsg",
@@ -516,6 +536,17 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
                 logger.log(Level.INFO, "", ex);
             }
         }
+        
+       
+        try {
+            communicationArtifactsHelper.addAttachments(messageArtifact, new MessageAttachments(fileAttachments, Collections.emptyList()));
+        } catch (TskCoreException ex) {
+             postErrorMessage(
+                        NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg"),
+                        "");
+                logger.log(Level.INFO, "Failed to add attachments to email message.", ex);
+        }
+       
         return files;
     }