mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cleaned up EmailParser code

Browse Source
This commit is contained in:
Kelly Kelly 2021-01-27 17:13:28 -05:00
parent 4974955ffa
commit 2103ce95f3
2 changed files with 162 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java
View File

@ -1,7 +1,7 @@
/* /*
* Autopsy Forensic Browser * Autopsy Forensic Browser
* *
* Copyright 2019 Basis Technology Corp. * Copyright 2019-2020 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org * Contact: carrier <at> sleuthkit <dot> org
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
@ -31,7 +31,6 @@ import org.apache.james.mime4j.dom.Entity;
import org.apache.james.mime4j.dom.Message; import org.apache.james.mime4j.dom.Message;
import org.apache.james.mime4j.dom.MessageWriter; import org.apache.james.mime4j.dom.MessageWriter;
import org.apache.james.mime4j.dom.Multipart; import org.apache.james.mime4j.dom.Multipart;
import org.apache.james.mime4j.dom.SingleBody;
import org.apache.james.mime4j.dom.TextBody; import org.apache.james.mime4j.dom.TextBody;
import org.apache.james.mime4j.dom.address.AddressList; import org.apache.james.mime4j.dom.address.AddressList;
import org.apache.james.mime4j.dom.address.Mailbox; import org.apache.james.mime4j.dom.address.Mailbox;
@ -348,8 +347,6 @@ class MimeJ4MessageParser implements AutoCloseable{
logger.log(Level.WARNING, "Failed to create file output stream for: " + outPath, ex); //NON-NLS logger.log(Level.WARNING, "Failed to create file output stream for: " + outPath, ex); //NON-NLS
} }
} }
} }
/** /**

338
thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java
View File

@ -73,13 +73,14 @@ import org.sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments.Fil
* structure and metadata. * structure and metadata.
*/ */
public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()); private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName());
private final IngestServices services = IngestServices.getInstance(); private final IngestServices services = IngestServices.getInstance();
private FileManager fileManager; private FileManager fileManager;
private IngestJobContext context; private IngestJobContext context;
private Blackboard blackboard; private Blackboard blackboard;
private CommunicationArtifactsHelper communicationArtifactsHelper; private CommunicationArtifactsHelper communicationArtifactsHelper;
private static final int MBOX_SIZE_TO_SPLIT = 1048576000; private static final int MBOX_SIZE_TO_SPLIT = 1048576000;
private Case currentCase; private Case currentCase;
@ -90,7 +91,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
} }
@Override @Override
@Messages ({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."}) @Messages({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."})
public void startUp(IngestJobContext context) throws IngestModuleException { public void startUp(IngestJobContext context) throws IngestModuleException {
this.context = context; this.context = context;
try { try {
@ -113,8 +114,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
} }
//skip unalloc //skip unalloc
if ((abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) || if ((abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS))
(abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) { || (abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) {
return ProcessResult.OK; return ProcessResult.OK;
} }
@ -125,7 +126,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
// check its signature // check its signature
boolean isMbox = false; boolean isMbox = false;
boolean isEMLFile = false; boolean isEMLFile = false;
try { try {
byte[] t = new byte[64]; byte[] t = new byte[64];
if (abstractFile.getSize() > 64) { if (abstractFile.getSize() > 64) {
@ -138,15 +139,15 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
} catch (TskException ex) { } catch (TskException ex) {
logger.log(Level.WARNING, null, ex); logger.log(Level.WARNING, null, ex);
} }
boolean isPstFile = PstParser.isPstFile(abstractFile); boolean isPstFile = PstParser.isPstFile(abstractFile);
boolean isVcardFile = VcardParser.isVcardFile(abstractFile); boolean isVcardFile = VcardParser.isVcardFile(abstractFile);
if (context.fileIngestIsCancelled()) { if (context.fileIngestIsCancelled()) {
return ProcessResult.OK; return ProcessResult.OK;
} }
if (isMbox || isEMLFile || isPstFile || isVcardFile ) { if (isMbox || isEMLFile || isPstFile || isVcardFile) {
try { try {
communicationArtifactsHelper = new CommunicationArtifactsHelper(currentCase.getSleuthkitCase(), communicationArtifactsHelper = new CommunicationArtifactsHelper(currentCase.getSleuthkitCase(),
EmailParserModuleFactory.getModuleName(), abstractFile, Account.Type.EMAIL); EmailParserModuleFactory.getModuleName(), abstractFile, Account.Type.EMAIL);
@ -159,7 +160,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
if (isMbox) { if (isMbox) {
return processMBox(abstractFile); return processMBox(abstractFile);
} }
if (isEMLFile) { if (isEMLFile) {
return processEMLFile(abstractFile); return processEMLFile(abstractFile);
} }
@ -167,11 +168,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
if (isPstFile) { if (isPstFile) {
return processPst(abstractFile); return processPst(abstractFile);
} }
if (isVcardFile) { if (isVcardFile) {
return processVcard(abstractFile); return processVcard(abstractFile);
} }
return ProcessResult.OK; return ProcessResult.OK;
} }
@ -187,7 +188,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
String fileName; String fileName;
try { try {
fileName = getTempPath() + File.separator + abstractFile.getName() fileName = getTempPath() + File.separator + abstractFile.getName()
+ "-" + String.valueOf(abstractFile.getId()); + "-" + String.valueOf(abstractFile.getId());
} catch (NoCurrentCaseException ex) { } catch (NoCurrentCaseException ex) {
logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
return ProcessResult.ERROR; return ProcessResult.ERROR;
@ -204,8 +205,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
services.postMessage(msg); services.postMessage(msg);
return ProcessResult.OK; return ProcessResult.OK;
} }
try (PstParser parser = new PstParser(services)){ try (PstParser parser = new PstParser(services)) {
try { try {
ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled);
} catch (IOException ex) { } catch (IOException ex) {
@ -215,7 +216,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
PstParser.ParseResult result = parser.open(file, abstractFile.getId()); PstParser.ParseResult result = parser.open(file, abstractFile.getId());
switch( result) { switch (result) {
case OK: case OK:
Iterator<EmailMessage> pstMsgIterator = parser.getEmailMessageIterator(); Iterator<EmailMessage> pstMsgIterator = parser.getEmailMessageIterator();
if (pstMsgIterator != null) { if (pstMsgIterator != null) {
@ -239,20 +240,20 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
// encrypted pst: Add encrypted file artifact // encrypted pst: Add encrypted file artifact
try { try {
BlackboardArtifact artifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED); BlackboardArtifact artifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, EmailParserModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.encryptionFileLevel"))); artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, EmailParserModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.encryptionFileLevel")));
try { try {
// index the artifact for keyword search // index the artifact for keyword search
blackboard.postArtifact(artifact, EmailParserModuleFactory.getModuleName()); blackboard.postArtifact(artifact, EmailParserModuleFactory.getModuleName());
} catch (Blackboard.BlackboardException ex) { } catch (Blackboard.BlackboardException ex) {
MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_processPst_indexError_message(), artifact.getDisplayName()); MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_processPst_indexError_message(), artifact.getDisplayName());
logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS
}
} catch (TskCoreException ex) {
logger.log(Level.INFO, "Failed to add encryption attribute to file: {0}", abstractFile.getName()); //NON-NLS
} }
break; } catch (TskCoreException ex) {
logger.log(Level.INFO, "Failed to add encryption attribute to file: {0}", abstractFile.getName()); //NON-NLS
}
break;
default: default:
// parsing error: log message // parsing error: log message
postErrorMessage( postErrorMessage(
@ -263,8 +264,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
logger.log(Level.INFO, "PSTParser failed to parse {0}", abstractFile.getName()); //NON-NLS logger.log(Level.INFO, "PSTParser failed to parse {0}", abstractFile.getName()); //NON-NLS
return ProcessResult.ERROR; return ProcessResult.ERROR;
} }
} catch(Exception ex) { } catch (Exception ex) {
logger.log(Level.WARNING, String.format("Failed to close temp pst file %s", file.getAbsolutePath())); logger.log(Level.WARNING, String.format("Failed to close temp pst file %s", file.getAbsolutePath()));
} finally { } finally {
file.delete(); file.delete();
} }
@ -295,7 +296,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
String fileName; String fileName;
try { try {
fileName = getTempPath() + File.separator + abstractFile.getName() fileName = getTempPath() + File.separator + abstractFile.getName()
+ "-" + String.valueOf(abstractFile.getId()); + "-" + String.valueOf(abstractFile.getId());
} catch (NoCurrentCaseException ex) { } catch (NoCurrentCaseException ex) {
logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
return ProcessResult.ERROR; return ProcessResult.ERROR;
@ -314,7 +315,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
} }
if (abstractFile.getSize() < MBOX_SIZE_TO_SPLIT) { if (abstractFile.getSize() < MBOX_SIZE_TO_SPLIT) {
try { try {
ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled);
} catch (IOException ex) { } catch (IOException ex) {
@ -322,25 +323,25 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
return ProcessResult.OK; return ProcessResult.OK;
} }
try{ try {
processMboxFile(file, abstractFile, emailFolder); processMboxFile(file, abstractFile, emailFolder);
if (context.fileIngestIsCancelled()) { if (context.fileIngestIsCancelled()) {
return ProcessResult.OK; return ProcessResult.OK;
} }
}finally { } finally {
file.delete(); file.delete();
} }
} else { } else {
List<Long> mboxSplitOffsets = new ArrayList<>(); List<Long> mboxSplitOffsets = new ArrayList<>();
try{ try {
mboxSplitOffsets = findMboxSplitOffset(abstractFile, file); mboxSplitOffsets = findMboxSplitOffset(abstractFile, file);
} catch (IOException ex) { } catch (IOException ex) {
logger.log(Level.WARNING, String.format("Failed finding split offsets for mbox file {0}.", fileName), ex); //NON-NLS logger.log(Level.WARNING, String.format("Failed finding split offsets for mbox file {0}.", fileName), ex); //NON-NLS
return ProcessResult.OK; return ProcessResult.OK;
} }
long startingOffset = 0; long startingOffset = 0;
for (Long mboxSplitOffset : mboxSplitOffsets) { for (Long mboxSplitOffset : mboxSplitOffsets) {
File splitFile = new File(fileName + "-" + mboxSplitOffset); File splitFile = new File(fileName + "-" + mboxSplitOffset);
try { try {
@ -349,55 +350,54 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
logger.log(Level.WARNING, "Failed writing split mbox file to disk.", ex); //NON-NLS logger.log(Level.WARNING, "Failed writing split mbox file to disk.", ex); //NON-NLS
return ProcessResult.OK; return ProcessResult.OK;
} }
try{ try {
processMboxFile(splitFile, abstractFile, emailFolder); processMboxFile(splitFile, abstractFile, emailFolder);
startingOffset = mboxSplitOffset; startingOffset = mboxSplitOffset;
} finally { } finally {
splitFile.delete(); splitFile.delete();
} }
if (context.fileIngestIsCancelled()) { if (context.fileIngestIsCancelled()) {
return ProcessResult.OK; return ProcessResult.OK;
} }
} }
} }
return ProcessResult.OK; return ProcessResult.OK;
} }
private List<Long> findMboxSplitOffset(AbstractFile abstractFile, File file) throws IOException { private List<Long> findMboxSplitOffset(AbstractFile abstractFile, File file) throws IOException {
List<Long> mboxSplitOffset = new ArrayList<>(); List<Long> mboxSplitOffset = new ArrayList<>();
byte[] buffer = new byte[7]; byte[] buffer = new byte[7];
ReadContentInputStream in = new ReadContentInputStream(abstractFile); ReadContentInputStream in = new ReadContentInputStream(abstractFile);
in.skip(MBOX_SIZE_TO_SPLIT); in.skip(MBOX_SIZE_TO_SPLIT);
int len = in.read(buffer); int len = in.read(buffer);
while (len != -1) { while (len != -1) {
len = in.read(buffer); len = in.read(buffer);
if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114 && if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114
buffer[4] == 111 && buffer[5] == 109 && buffer[6] == 32) { && buffer[4] == 111 && buffer[5] == 109 && buffer[6] == 32) {
mboxSplitOffset.add(in.getCurPosition() - 5 ); mboxSplitOffset.add(in.getCurPosition() - 5);
in.skip(MBOX_SIZE_TO_SPLIT); in.skip(MBOX_SIZE_TO_SPLIT);
} }
} }
return mboxSplitOffset; return mboxSplitOffset;
} }
private void processMboxFile(File file, AbstractFile abstractFile, String emailFolder) { private void processMboxFile(File file, AbstractFile abstractFile, String emailFolder) {
try(MboxParser emailIterator = MboxParser.getEmailIterator( emailFolder, file, abstractFile.getId())) { try (MboxParser emailIterator = MboxParser.getEmailIterator(emailFolder, file, abstractFile.getId())) {
List<EmailMessage> emails = new ArrayList<>(); List<EmailMessage> emails = new ArrayList<>();
if(emailIterator != null) { if (emailIterator != null) {
while(emailIterator.hasNext()) { while (emailIterator.hasNext()) {
if (context.fileIngestIsCancelled()) { if (context.fileIngestIsCancelled()) {
return; return;
} }
EmailMessage emailMessage = emailIterator.next(); EmailMessage emailMessage = emailIterator.next();
if(emailMessage != null) { if (emailMessage != null) {
emails.add(emailMessage); emails.add(emailMessage);
} }
} }
@ -409,13 +409,13 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
abstractFile.getName()), errors); abstractFile.getName()), errors);
} }
} }
processEmails(emails, MboxParser.getEmailIterator( emailFolder, file, abstractFile.getId()), abstractFile); processEmails(emails, MboxParser.getEmailIterator(emailFolder, file, abstractFile.getId()), abstractFile);
} catch(Exception ex) { } catch (Exception ex) {
logger.log(Level.WARNING, String.format("Failed to close mbox temp file %s", file.getAbsolutePath())); logger.log(Level.WARNING, String.format("Failed to close mbox temp file %s", file.getAbsolutePath()));
} }
} }
/** /**
* Parse and extract data from a vCard file. * Parse and extract data from a vCard file.
* *
@ -439,8 +439,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
} }
return ProcessResult.OK; return ProcessResult.OK;
} }
private ProcessResult processEMLFile(AbstractFile abstractFile) { private ProcessResult processEMLFile(AbstractFile abstractFile) {
try { try {
EmailMessage message = EMLParser.parse(abstractFile); EmailMessage message = EMLParser.parse(abstractFile);
@ -451,16 +451,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
List<AbstractFile> derivedFiles = new ArrayList<>(); List<AbstractFile> derivedFiles = new ArrayList<>();
AccountFileInstanceCache accountFileInstanceCache = new AccountFileInstanceCache(abstractFile, currentCase); AccountFileInstanceCache accountFileInstanceCache = new AccountFileInstanceCache(abstractFile, currentCase);
// BlackboardArtifact msgArtifact = addEmailArtifact(message, abstractFile, accountFileInstanceCache);
createEmailArtifact(message, abstractFile, accountFileInstanceCache, derivedFiles); createEmailArtifact(message, abstractFile, accountFileInstanceCache, derivedFiles);
accountFileInstanceCache.clear(); accountFileInstanceCache.clear();
// if ((msgArtifact != null) && (message.hasAttachment())) {
// derivedFiles.addAll(handleAttachments(message.getAttachments(), abstractFile, msgArtifact));
// }
if (derivedFiles.isEmpty() == false) { if (derivedFiles.isEmpty() == false) {
for (AbstractFile derived : derivedFiles) { for (AbstractFile derived : derivedFiles) {
services.fireModuleContentEvent(new ModuleContentEvent(derived)); services.fireModuleContentEvent(new ModuleContentEvent(derived));
@ -497,7 +490,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
/** /**
* Get a module output folder. * Get a module output folder.
* *
* @throws NoCurrentCaseException if there is no open case. * @throws NoCurrentCaseException if there is no open case.
* *
* @return the module output folder * @return the module output folder
@ -531,49 +524,42 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
* @param fullMessageIterator * @param fullMessageIterator
* @param abstractFile * @param abstractFile
*/ */
private void processEmails(List<EmailMessage> partialEmailsForThreading, Iterator<EmailMessage> fullMessageIterator, private void processEmails(List<EmailMessage> partialEmailsForThreading, Iterator<EmailMessage> fullMessageIterator,
AbstractFile abstractFile) { AbstractFile abstractFile) {
// Create cache for accounts // Create cache for accounts
AccountFileInstanceCache accountFileInstanceCache = new AccountFileInstanceCache(abstractFile, currentCase); AccountFileInstanceCache accountFileInstanceCache = new AccountFileInstanceCache(abstractFile, currentCase);
// Putting try/catch around this to catch any exception and still allow // Putting try/catch around this to catch any exception and still allow
// the creation of the artifacts to continue. // the creation of the artifacts to continue.
try{ try {
EmailMessageThreader.threadMessages(partialEmailsForThreading); EmailMessageThreader.threadMessages(partialEmailsForThreading);
} catch(Exception ex) { } catch (Exception ex) {
logger.log(Level.WARNING, String.format("Exception thrown parsing emails from %s", abstractFile.getName()), ex); logger.log(Level.WARNING, String.format("Exception thrown parsing emails from %s", abstractFile.getName()), ex);
} }
List<AbstractFile> derivedFiles = new ArrayList<>(); List<AbstractFile> derivedFiles = new ArrayList<>();
int msgCnt = 0; int msgCnt = 0;
while(fullMessageIterator.hasNext()) { while (fullMessageIterator.hasNext()) {
if (context.fileIngestIsCancelled()) { if (context.fileIngestIsCancelled()) {
return; return;
} }
EmailMessage current = fullMessageIterator.next(); EmailMessage current = fullMessageIterator.next();
if(current == null) { if (current == null) {
continue; continue;
} }
if(partialEmailsForThreading.size() > msgCnt) { if (partialEmailsForThreading.size() > msgCnt) {
EmailMessage threaded = partialEmailsForThreading.get(msgCnt++); EmailMessage threaded = partialEmailsForThreading.get(msgCnt++);
if(threaded.getMessageID().equals(current.getMessageID()) && if (threaded.getMessageID().equals(current.getMessageID())
threaded.getSubject().equals(current.getSubject())) { && threaded.getSubject().equals(current.getSubject())) {
current.setMessageThreadID(threaded.getMessageThreadID()); current.setMessageThreadID(threaded.getMessageThreadID());
} }
} }
// BlackboardArtifact msgArtifact = addEmailArtifact(current, abstractFile, accountFileInstanceCache);
//
// if ((msgArtifact != null) && (current.hasAttachment())) {
// derivedFiles.addAll(handleAttachments(current.getAttachments(), abstractFile, msgArtifact ));
// }
createEmailArtifact(current, abstractFile, accountFileInstanceCache, derivedFiles); createEmailArtifact(current, abstractFile, accountFileInstanceCache, derivedFiles);
} }
@ -587,21 +573,21 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
} }
context.addFilesToJob(derivedFiles); context.addFilesToJob(derivedFiles);
} }
void createEmailArtifact(EmailMessage email, AbstractFile abstractFile, AccountFileInstanceCache accountFileInstanceCache, List<AbstractFile> derivedFiles) { void createEmailArtifact(EmailMessage email, AbstractFile abstractFile, AccountFileInstanceCache accountFileInstanceCache, List<AbstractFile> derivedFiles) {
BlackboardArtifact msgArtifact = addEmailArtifact(email, abstractFile, accountFileInstanceCache); BlackboardArtifact msgArtifact = addEmailArtifact(email, abstractFile, accountFileInstanceCache);
if ((msgArtifact != null) && (email.hasAttachment())) { if ((msgArtifact != null) && (email.hasAttachment())) {
derivedFiles.addAll(handleAttachments(email.getAttachments(), abstractFile, msgArtifact )); derivedFiles.addAll(handleAttachments(email.getAttachments(), abstractFile, msgArtifact));
for (EmailMessage.Attachment attach : email.getAttachments()) { for (EmailMessage.Attachment attach : email.getAttachments()) {
if(attach instanceof AttachedEmailMessage) { if (attach instanceof AttachedEmailMessage) {
createEmailArtifact(((AttachedEmailMessage) attach).getEmailMessage(), abstractFile, accountFileInstanceCache, derivedFiles); createEmailArtifact(((AttachedEmailMessage) attach).getEmailMessage(), abstractFile, accountFileInstanceCache, derivedFiles);
} }
} }
} }
} }
/** /**
* Add the given attachments as derived files and reschedule them for * Add the given attachments as derived files and reschedule them for
* ingest. * ingest.
@ -613,8 +599,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
* @return List of attachments * @return List of attachments
*/ */
@NbBundle.Messages({ @NbBundle.Messages({
"ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message." "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message."
}) })
private List<AbstractFile> handleAttachments(List<EmailMessage.Attachment> attachments, AbstractFile abstractFile, BlackboardArtifact messageArtifact) { private List<AbstractFile> handleAttachments(List<EmailMessage.Attachment> attachments, AbstractFile abstractFile, BlackboardArtifact messageArtifact) {
List<AbstractFile> files = new ArrayList<>(); List<AbstractFile> files = new ArrayList<>();
List<FileAttachment> fileAttachments = new ArrayList<>(); List<FileAttachment> fileAttachments = new ArrayList<>();
@ -632,11 +618,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
DerivedFile df = fileManager.addDerivedFile(filename, relPath, DerivedFile df = fileManager.addDerivedFile(filename, relPath,
size, cTime, crTime, aTime, mTime, true, abstractFile, "", size, cTime, crTime, aTime, mTime, true, abstractFile, "",
EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleVersion(), "", encodingType); EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleVersion(), "", encodingType);
associateAttachmentWithMesssge(messageArtifact, df); associateAttachmentWithMesssge(messageArtifact, df);
files.add(df); files.add(df);
fileAttachments.add(new FileAttachment(df)); fileAttachments.add(new FileAttachment(df));
} catch (TskCoreException ex) { } catch (TskCoreException ex) {
postErrorMessage( postErrorMessage(
@ -647,17 +633,16 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
logger.log(Level.INFO, "", ex); logger.log(Level.INFO, "", ex);
} }
} }
try { try {
communicationArtifactsHelper.addAttachments(messageArtifact, new MessageAttachments(fileAttachments, Collections.emptyList())); communicationArtifactsHelper.addAttachments(messageArtifact, new MessageAttachments(fileAttachments, Collections.emptyList()));
} catch (TskCoreException ex) { } catch (TskCoreException ex) {
postErrorMessage( postErrorMessage(
NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg"), NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg"),
""); "");
logger.log(Level.INFO, "Failed to add attachments to email message.", ex); logger.log(Level.INFO, "Failed to add attachments to email message.", ex);
} }
return files; return files;
} }
@ -673,32 +658,33 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
bba.addAttributes(attributes); //write out to bb bba.addAttributes(attributes); //write out to bb
return bba; return bba;
} }
/** /**
* Finds and returns a set of unique email addresses found in the input string * Finds and returns a set of unique email addresses found in the input
* string
* *
* @param input - input string, like the To/CC line from an email header * @param input - input string, like the To/CC line from an email header
* *
* @return Set<String>: set of email addresses found in the input string * @return Set<String>: set of email addresses found in the input string
*/ */
private Set<String> findEmailAddresess(String input) { private Set<String> findEmailAddresess(String input) {
Pattern p = Pattern.compile("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", Pattern p = Pattern.compile("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b",
Pattern.CASE_INSENSITIVE); Pattern.CASE_INSENSITIVE);
Matcher m = p.matcher(input); Matcher m = p.matcher(input);
Set<String> emailAddresses = new HashSet<>(); Set<String> emailAddresses = new HashSet<>();
while (m.find()) { while (m.find()) {
emailAddresses.add( m.group()); emailAddresses.add(m.group());
} }
return emailAddresses; return emailAddresses;
} }
/** /**
* Add a blackboard artifact for the given e-mail message. * Add a blackboard artifact for the given e-mail message.
* *
* @param email The e-mail message. * @param email The e-mail message.
* @param abstractFile The associated file. * @param abstractFile The associated file.
* @param accountFileInstanceCache The current cache of account instances. * @param accountFileInstanceCache The current cache of account instances.
* *
* @return The generated e-mail message artifact. * @return The generated e-mail message artifact.
*/ */
@Messages({"ThunderbirdMboxFileIngestModule.addArtifact.indexError.message=Failed to index email message detected artifact for keyword search."}) @Messages({"ThunderbirdMboxFileIngestModule.addArtifact.indexError.message=Failed to index email message detected artifact for keyword search."})
@ -722,35 +708,33 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
List<String> senderAddressList = new ArrayList<>(); List<String> senderAddressList = new ArrayList<>();
String senderAddress; String senderAddress;
senderAddressList.addAll(findEmailAddresess(from)); senderAddressList.addAll(findEmailAddresess(from));
if (context.fileIngestIsCancelled()) { if (context.fileIngestIsCancelled()) {
return null; return null;
} }
AccountFileInstance senderAccountInstance = null; AccountFileInstance senderAccountInstance = null;
if (senderAddressList.size() == 1) { if (senderAddressList.size() == 1) {
senderAddress = senderAddressList.get(0); senderAddress = senderAddressList.get(0);
try { try {
senderAccountInstance = accountFileInstanceCache.getAccountInstance(senderAddress); senderAccountInstance = accountFileInstanceCache.getAccountInstance(senderAddress);
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Failed to create account for email address " + senderAddress, ex); //NON-NLS
} }
catch(TskCoreException ex) { } else {
logger.log(Level.WARNING, "Failed to create account for email address " + senderAddress, ex); //NON-NLS logger.log(Level.WARNING, "Failed to find sender address, from = {0}", from); //NON-NLS
}
} }
else {
logger.log(Level.WARNING, "Failed to find sender address, from = {0}", from); //NON-NLS
}
if (context.fileIngestIsCancelled()) { if (context.fileIngestIsCancelled()) {
return null; return null;
} }
List<String> recipientAddresses = new ArrayList<>(); List<String> recipientAddresses = new ArrayList<>();
recipientAddresses.addAll(findEmailAddresess(to)); recipientAddresses.addAll(findEmailAddresess(to));
recipientAddresses.addAll(findEmailAddresess(cc)); recipientAddresses.addAll(findEmailAddresess(cc));
recipientAddresses.addAll(findEmailAddresess(bcc)); recipientAddresses.addAll(findEmailAddresess(bcc));
List<AccountFileInstance> recipientAccountInstances = new ArrayList<>(); List<AccountFileInstance> recipientAccountInstances = new ArrayList<>();
for (String addr : recipientAddresses) { for (String addr : recipientAddresses) {
if (context.fileIngestIsCancelled()) { if (context.fileIngestIsCancelled()) {
@ -759,56 +743,54 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
try { try {
AccountFileInstance recipientAccountInstance = accountFileInstanceCache.getAccountInstance(addr); AccountFileInstance recipientAccountInstance = accountFileInstanceCache.getAccountInstance(addr);
recipientAccountInstances.add(recipientAccountInstance); recipientAccountInstances.add(recipientAccountInstance);
} } catch (TskCoreException ex) {
catch(TskCoreException ex) {
logger.log(Level.WARNING, "Failed to create account for email address " + addr, ex); //NON-NLS logger.log(Level.WARNING, "Failed to create account for email address " + addr, ex); //NON-NLS
} }
} }
addArtifactAttribute(headers, ATTRIBUTE_TYPE.TSK_HEADERS, bbattributes); addArtifactAttribute(headers, ATTRIBUTE_TYPE.TSK_HEADERS, bbattributes);
addArtifactAttribute(from, ATTRIBUTE_TYPE.TSK_EMAIL_FROM, bbattributes); addArtifactAttribute(from, ATTRIBUTE_TYPE.TSK_EMAIL_FROM, bbattributes);
addArtifactAttribute(to, ATTRIBUTE_TYPE.TSK_EMAIL_TO, bbattributes); addArtifactAttribute(to, ATTRIBUTE_TYPE.TSK_EMAIL_TO, bbattributes);
addArtifactAttribute(subject, ATTRIBUTE_TYPE.TSK_SUBJECT, bbattributes); addArtifactAttribute(subject, ATTRIBUTE_TYPE.TSK_SUBJECT, bbattributes);
addArtifactAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_RCVD, bbattributes); addArtifactAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_RCVD, bbattributes);
addArtifactAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_SENT, bbattributes); addArtifactAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_SENT, bbattributes);
addArtifactAttribute(body, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_PLAIN, bbattributes); addArtifactAttribute(body, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_PLAIN, bbattributes);
addArtifactAttribute(((id < 0L) ? NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.notAvail") : String.valueOf(id)), addArtifactAttribute(((id < 0L) ? NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.notAvail") : String.valueOf(id)),
ATTRIBUTE_TYPE.TSK_MSG_ID, bbattributes); ATTRIBUTE_TYPE.TSK_MSG_ID, bbattributes);
addArtifactAttribute(((localPath.isEmpty() == false) ? localPath : ""), addArtifactAttribute(((localPath.isEmpty() == false) ? localPath : ""),
ATTRIBUTE_TYPE.TSK_PATH, bbattributes); ATTRIBUTE_TYPE.TSK_PATH, bbattributes);
addArtifactAttribute(cc, ATTRIBUTE_TYPE.TSK_EMAIL_CC, bbattributes); addArtifactAttribute(cc, ATTRIBUTE_TYPE.TSK_EMAIL_CC, bbattributes);
addArtifactAttribute(bodyHTML, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_HTML, bbattributes); addArtifactAttribute(bodyHTML, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_HTML, bbattributes);
addArtifactAttribute(rtf, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_RTF, bbattributes); addArtifactAttribute(rtf, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_RTF, bbattributes);
addArtifactAttribute(threadID, ATTRIBUTE_TYPE.TSK_THREAD_ID, bbattributes); addArtifactAttribute(threadID, ATTRIBUTE_TYPE.TSK_THREAD_ID, bbattributes);
try { try {
if (context.fileIngestIsCancelled()) { if (context.fileIngestIsCancelled()) {
return null; return null;
} }
bbart = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG); bbart = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG);
bbart.addAttributes(bbattributes); bbart.addAttributes(bbattributes);
if (context.fileIngestIsCancelled()) { if (context.fileIngestIsCancelled()) {
return null; return null;
} }
// Add account relationships // Add account relationships
currentCase.getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart,Relationship.Type.MESSAGE, dateL); currentCase.getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart, Relationship.Type.MESSAGE, dateL);
if (context.fileIngestIsCancelled()) { if (context.fileIngestIsCancelled()) {
return null; return null;
} }
try { try {
// index the artifact for keyword search // index the artifact for keyword search
blackboard.postArtifact(bbart, EmailParserModuleFactory.getModuleName()); blackboard.postArtifact(bbart, EmailParserModuleFactory.getModuleName());
} catch (Blackboard.BlackboardException ex) { } catch (Blackboard.BlackboardException ex) {
logger.log(Level.SEVERE, "Unable to index blackboard artifact " + bbart.getArtifactID(), ex); //NON-NLS logger.log(Level.SEVERE, "Unable to index blackboard artifact " + bbart.getArtifactID(), ex); //NON-NLS
MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_addArtifact_indexError_message(), bbart.getDisplayName()); MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_addArtifact_indexError_message(), bbart.getDisplayName());
@ -819,11 +801,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
return bbart; return bbart;
} }
/** /**
* Add an attribute of a specified type to a supplied Collection. * Add an attribute of a specified type to a supplied Collection.
* *
* @param stringVal The attribute value. * @param stringVal The attribute value.
* @param attrType The type of attribute to be added. * @param attrType The type of attribute to be added.
* @param bbattributes The Collection to which the attribute will be added. * @param bbattributes The Collection to which the attribute will be added.
*/ */
@ -835,7 +817,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
/** /**
* Add an attribute of a specified type to a supplied Collection. * Add an attribute of a specified type to a supplied Collection.
* *
* @param stringVal The attribute value. * @param stringVal The attribute value.
* @param attrType The type of attribute to be added. * @param attrType The type of attribute to be added.
* @param bbattributes The Collection to which the attribute will be added. * @param bbattributes The Collection to which the attribute will be added.
@ -845,10 +827,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), stringVal)); bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), stringVal));
} }
} }
/** /**
* Add an attribute of a specified type to a supplied Collection. * Add an attribute of a specified type to a supplied Collection.
* *
* @param longVal The attribute value. * @param longVal The attribute value.
* @param attrType The type of attribute to be added. * @param attrType The type of attribute to be added.
* @param bbattributes The Collection to which the attribute will be added. * @param bbattributes The Collection to which the attribute will be added.
@ -858,49 +840,51 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), longVal)); bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), longVal));
} }
} }
/** /**
* Cache for storing AccountFileInstance. * Cache for storing AccountFileInstance. The idea is that emails will be
* The idea is that emails will be used multiple times in a file and * used multiple times in a file and we shouldn't do a database lookup each
* we shouldn't do a database lookup each time. * time.
*/ */
static private class AccountFileInstanceCache { static private class AccountFileInstanceCache {
private final Map<String, AccountFileInstance> cacheMap; private final Map<String, AccountFileInstance> cacheMap;
private final AbstractFile file; private final AbstractFile file;
private final Case currentCase; private final Case currentCase;
/** /**
* Create a new cache. Caches are linked to a specific file. * Create a new cache. Caches are linked to a specific file.
*
* @param file * @param file
* @param currentCase * @param currentCase
*/ */
AccountFileInstanceCache(AbstractFile file, Case currentCase) { AccountFileInstanceCache(AbstractFile file, Case currentCase) {
cacheMap= new HashMap<>(); cacheMap = new HashMap<>();
this.file = file; this.file = file;
this.currentCase = currentCase; this.currentCase = currentCase;
} }
/** /**
* Get the account file instance from the cache or the database. * Get the account file instance from the cache or the database.
* *
* @param email The email for this account. * @param email The email for this account.
* *
* @return The corresponding AccountFileInstance * @return The corresponding AccountFileInstance
* *
* @throws TskCoreException * @throws TskCoreException
*/ */
AccountFileInstance getAccountInstance(String email) throws TskCoreException { AccountFileInstance getAccountInstance(String email) throws TskCoreException {
if (cacheMap.containsKey(email)) { if (cacheMap.containsKey(email)) {
return cacheMap.get(email); return cacheMap.get(email);
} }
AccountFileInstance accountInstance = AccountFileInstance accountInstance
currentCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, email, = currentCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, email,
EmailParserModuleFactory.getModuleName(), file); EmailParserModuleFactory.getModuleName(), file);
cacheMap.put(email, accountInstance); cacheMap.put(email, accountInstance);
return accountInstance; return accountInstance;
} }
/** /**
* Clears the cache. * Clears the cache.
*/ */
@ -908,10 +892,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
cacheMap.clear(); cacheMap.clear();
} }
} }
/** /**
* Post an error message for the user. * Post an error message for the user.
* *
* @param subj The error subject. * @param subj The error subject.
* @param details The error details. * @param details The error details.
*/ */
@ -922,7 +906,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
/** /**
* Get the IngestServices object. * Get the IngestServices object.
* *
* @return The IngestServices object. * @return The IngestServices object.
*/ */
IngestServices getServices() { IngestServices getServices() {
@ -933,5 +917,5 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
public void shutDown() { public void shutDown() {
// nothing to shut down // nothing to shut down
} }
} }