mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

update ThunderbirdMboxFileIngestModule for postArtifacts and cleanup

Browse Source
This commit is contained in:
millmanorama 2018-08-16 11:56:58 +02:00
parent 740e1b0e9a
commit 430fc68cea
2 changed files with 90 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
thunderbirdparser/nbproject/project.xml
View File

@ -39,6 +39,15 @@
<specification-version>10.12</specification-version> <specification-version>10.12</specification-version>
</run-dependency> </run-dependency>
</dependency> </dependency>
<dependency>
<code-name-base>org.sleuthkit.autopsy.corelibs</code-name-base>
<build-prerequisite/>
<compile-dependency/>
<run-dependency>
<release-version>3</release-version>
<specification-version>1.2</specification-version>
</run-dependency>
</dependency>
<dependency> <dependency>
<code-name-base>org.sleuthkit.autopsy.keywordsearch</code-name-base> <code-name-base>org.sleuthkit.autopsy.keywordsearch</code-name-base>
<build-prerequisite/> <build-prerequisite/>

166
thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java
View File

@ -28,6 +28,8 @@ import java.util.Set;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import static org.apache.commons.collections4.CollectionUtils.isNotEmpty;
import org.apache.commons.lang.StringUtils;
import org.openide.util.NbBundle; import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages; import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case;
@ -43,16 +45,19 @@ import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult;
import org.sleuthkit.autopsy.ingest.IngestMonitor; import org.sleuthkit.autopsy.ingest.IngestMonitor;
import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.ModuleContentEvent; import org.sleuthkit.autopsy.ingest.ModuleContentEvent;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.Account;
import org.sleuthkit.datamodel.AccountFileInstance; import org.sleuthkit.datamodel.AccountFileInstance;
import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.Blackboard;
import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact;
import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG;
import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED;
import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
import org.sleuthkit.datamodel.CommunicationsManager;
import org.sleuthkit.datamodel.DerivedFile; import org.sleuthkit.datamodel.DerivedFile;
import org.sleuthkit.datamodel.Relationship; import org.sleuthkit.datamodel.Relationship;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskDataException; import org.sleuthkit.datamodel.TskDataException;
@ -66,36 +71,39 @@ import org.sleuthkit.datamodel.TskException;
public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()); private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName());
private IngestServices services = IngestServices.getInstance(); private final IngestServices services = IngestServices.getInstance();
private FileManager fileManager; private FileManager fileManager;
private IngestJobContext context; private IngestJobContext context;
private Blackboard blackboard; private Blackboard blackboard;
private SleuthkitCase sleuthkitCase;
private CommunicationsManager communicationsManager;
ThunderbirdMboxFileIngestModule() { ThunderbirdMboxFileIngestModule() {
} }
@Override @Override
@Messages({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."}) @Messages({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case.",
"ThunderbirdMboxFileIngestModule.noCommsManager.errMsg=Exception while getting communications manager."})
public void startUp(IngestJobContext context) throws IngestModuleException { public void startUp(IngestJobContext context) throws IngestModuleException {
this.context = context; this.context = context;
try { try {
fileManager = Case.getCurrentCaseThrows().getServices().getFileManager(); Case currentCaseThrows = Case.getCurrentCaseThrows();
fileManager = currentCaseThrows.getServices().getFileManager();
sleuthkitCase = currentCaseThrows.getSleuthkitCase();
blackboard = sleuthkitCase.getBlackboard();
communicationsManager = sleuthkitCase.getCommunicationsManager();
} catch (NoCurrentCaseException ex) { } catch (NoCurrentCaseException ex) {
logger.log(Level.SEVERE, "Exception while getting open case.", ex); logger.log(Level.SEVERE, "Exception while getting open case.", ex);
throw new IngestModuleException(Bundle.ThunderbirdMboxFileIngestModule_noOpenCase_errMsg(), ex); throw new IngestModuleException(Bundle.ThunderbirdMboxFileIngestModule_noOpenCase_errMsg(), ex);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Exception while getting communications manager.", ex);
throw new IngestModuleException(Bundle.ThunderbirdMboxFileIngestModule_noCommsManager_errMsg(), ex);
} }
} }
@Override @Override
public ProcessResult process(AbstractFile abstractFile) { public ProcessResult process(AbstractFile abstractFile) {
try {
blackboard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard();
} catch (NoCurrentCaseException ex) {
logger.log(Level.SEVERE, "Exception while getting open case.", ex);
return ProcessResult.ERROR;
}
// skip known // skip known
if (abstractFile.getKnown().equals(TskData.FileKnown.KNOWN)) { if (abstractFile.getKnown().equals(TskData.FileKnown.KNOWN)) {
return ProcessResult.OK; return ProcessResult.OK;
@ -176,42 +184,44 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
PstParser parser = new PstParser(services); PstParser parser = new PstParser(services);
PstParser.ParseResult result = parser.parse(file, abstractFile.getId()); PstParser.ParseResult result = parser.parse(file, abstractFile.getId());
if (result == PstParser.ParseResult.OK) { switch (result) {
try { case OK:
// parse success: Process email and add artifacts
processEmails(parser.getResults(), abstractFile);
} catch (NoCurrentCaseException ex) {
logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
return ProcessResult.ERROR;
}
} else if (result == PstParser.ParseResult.ENCRYPT) {
// encrypted pst: Add encrypted file artifact
try {
BlackboardArtifact artifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, EmailParserModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.encryptionFileLevel")));
try { try {
// index the artifact for keyword search // parse success: Process email and add artifacts
blackboard.postArtifact(artifact, EmailParserModuleFactory.getModuleName()); processEmails(parser.getResults(), abstractFile);
} catch (Blackboard.BlackboardException ex) { } catch (NoCurrentCaseException ex) {
MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_processPst_indexError_message(), artifact.getDisplayName()); logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS return ProcessResult.ERROR;
} }
break;
case ENCRYPT:
// encrypted pst: Add encrypted file artifact
try {
BlackboardArtifact artifact = abstractFile.newArtifact(TSK_ENCRYPTION_DETECTED);
artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, EmailParserModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.encryptionFileLevel")));
services.fireModuleDataEvent(new ModuleDataEvent(EmailParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED)); try {
} catch (TskCoreException ex) { /* Post the encryption artifact, which will index it and
logger.log(Level.INFO, "Failed to add encryption attribute to file: {0}", abstractFile.getName()); //NON-NLS * send out a notification event to the
} */
} else { blackboard.postArtifact(artifact, EmailParserModuleFactory.getModuleName());
// parsing error: log message } catch (Blackboard.BlackboardException ex) {
postErrorMessage( MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_processPst_indexError_message(), artifact.getDisplayName());
NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processPst.errProcFile.msg", logger.log(Level.SEVERE, "Unable to post blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS
abstractFile.getName()), }
NbBundle.getMessage(this.getClass(), } catch (TskCoreException ex) {
"ThunderbirdMboxFileIngestModule.processPst.errProcFile.details")); logger.log(Level.INFO, "Failed to add encryption attribute to file: {0}", abstractFile.getName()); //NON-NLS
logger.log(Level.INFO, "PSTParser failed to parse {0}", abstractFile.getName()); //NON-NLS }
return ProcessResult.ERROR; break;
default:
// parsing error: log message
postErrorMessage(
NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processPst.errProcFile.msg",
abstractFile.getName()),
NbBundle.getMessage(this.getClass(),
"ThunderbirdMboxFileIngestModule.processPst.errProcFile.details"));
logger.log(Level.INFO, "PSTParser failed to parse {0}", abstractFile.getName()); //NON-NLS
return ProcessResult.ERROR;
} }
if (file.delete() == false) { if (file.delete() == false) {
@ -246,7 +256,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
} else if (mboxParentDir.contains("/ImapMail/")) { //NON-NLS } else if (mboxParentDir.contains("/ImapMail/")) { //NON-NLS
emailFolder = mboxParentDir.substring(mboxParentDir.indexOf("/ImapMail/") + 9); //NON-NLS emailFolder = mboxParentDir.substring(mboxParentDir.indexOf("/ImapMail/") + 9); //NON-NLS
} }
emailFolder = emailFolder + mboxFileName; emailFolder += mboxFileName;
emailFolder = emailFolder.replaceAll(".sbd", ""); //NON-NLS emailFolder = emailFolder.replaceAll(".sbd", ""); //NON-NLS
String fileName; String fileName;
@ -363,13 +373,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
} }
} }
if (derivedFiles.isEmpty() == false) { if (isNotEmpty(derivedFiles)) {
for (AbstractFile derived : derivedFiles) { derivedFiles.forEach(derived -> services.fireModuleContentEvent(new ModuleContentEvent(derived)));
services.fireModuleContentEvent(new ModuleContentEvent(derived));
}
} }
context.addFilesToJob(derivedFiles); context.addFilesToJob(derivedFiles);
services.fireModuleDataEvent(new ModuleDataEvent(EmailParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG));
} }
/** /**
@ -423,7 +430,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
Pattern p = Pattern.compile("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", Pattern p = Pattern.compile("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b",
Pattern.CASE_INSENSITIVE); Pattern.CASE_INSENSITIVE);
Matcher m = p.matcher(input); Matcher m = p.matcher(input);
Set<String> emailAddresses = new HashSet<String>(); Set<String> emailAddresses = new HashSet<>();
while (m.find()) { while (m.find()) {
emailAddresses.add(m.group()); emailAddresses.add(m.group());
} }
@ -447,43 +454,37 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
String bcc = email.getBcc(); String bcc = email.getBcc();
String from = email.getSender(); String from = email.getSender();
long dateL = email.getSentDate(); long dateL = email.getSentDate();
String headers = email.getHeaders();
String body = email.getTextBody();
String bodyHTML = email.getHtmlBody();
String rtf = email.getRtfBody();
String subject = email.getSubject();
long id = email.getId(); long id = email.getId();
String localPath = email.getLocalPath();
List<String> senderAddressList = new ArrayList<>(); List<String> senderAddressList = new ArrayList<>();
String senderAddress;
senderAddressList.addAll(findEmailAddresess(from)); senderAddressList.addAll(findEmailAddresess(from));
AccountFileInstance senderAccountInstance = null; AccountFileInstance senderAccountInstance = null;
Case openCase = Case.getCurrentCaseThrows();
if (senderAddressList.size() == 1) { if (senderAddressList.size() == 1) {
senderAddress = senderAddressList.get(0); String senderAddress = senderAddressList.get(0);
try { try {
senderAccountInstance = openCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, senderAddress, EmailParserModuleFactory.getModuleName(), abstractFile); senderAccountInstance = communicationsManager.
createAccountFileInstance(Account.Type.EMAIL, senderAddress, EmailParserModuleFactory.getModuleName(), abstractFile);
} catch (TskCoreException ex) { } catch (TskCoreException ex) {
logger.log(Level.WARNING, "Failed to create account for email address " + senderAddress, ex); //NON-NLS logger.log(Level.WARNING, "Failed to create account for email address " + senderAddress, ex); //NON-NLS
} }
} else { } else {
logger.log(Level.WARNING, "Failed to find sender address, from = " + from); //NON-NLS logger.log(Level.WARNING, "Failed to find sender address, from = {0}", from); //NON-NLS
} }
//TODO: should this be a set?
List<String> recipientAddresses = new ArrayList<>(); List<String> recipientAddresses = new ArrayList<>();
recipientAddresses.addAll(findEmailAddresess(to)); recipientAddresses.addAll(findEmailAddresess(to));
recipientAddresses.addAll(findEmailAddresess(cc)); recipientAddresses.addAll(findEmailAddresess(cc));
recipientAddresses.addAll(findEmailAddresess(bcc)); recipientAddresses.addAll(findEmailAddresess(bcc));
List<AccountFileInstance> recipientAccountInstances = new ArrayList<>(); List<AccountFileInstance> recipientAccountInstances = new ArrayList<>();
recipientAddresses.forEach((addr) -> { recipientAddresses.forEach(addr -> {
try { try {
AccountFileInstance recipientAccountInstance AccountFileInstance recipientAccountInstance = communicationsManager
= openCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, addr, .createAccountFileInstance(Account.Type.EMAIL, addr,
EmailParserModuleFactory.getModuleName(), abstractFile); EmailParserModuleFactory.getModuleName(), abstractFile);
recipientAccountInstances.add(recipientAccountInstance); recipientAccountInstances.add(recipientAccountInstance);
} catch (TskCoreException ex) { } catch (TskCoreException ex) {
@ -491,36 +492,35 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
} }
}); });
addEmailAttribute(headers, ATTRIBUTE_TYPE.TSK_HEADERS, bbattributes); addEmailAttribute(email.getHeaders(), ATTRIBUTE_TYPE.TSK_HEADERS, bbattributes);
addEmailAttribute(from, ATTRIBUTE_TYPE.TSK_EMAIL_FROM, bbattributes); addEmailAttribute(from, ATTRIBUTE_TYPE.TSK_EMAIL_FROM, bbattributes);
addEmailAttribute(to, ATTRIBUTE_TYPE.TSK_EMAIL_TO, bbattributes); addEmailAttribute(to, ATTRIBUTE_TYPE.TSK_EMAIL_TO, bbattributes);
addEmailAttribute(subject, ATTRIBUTE_TYPE.TSK_SUBJECT, bbattributes); addEmailAttribute(cc, ATTRIBUTE_TYPE.TSK_EMAIL_CC, bbattributes);
addEmailAttribute(email.getSubject(), ATTRIBUTE_TYPE.TSK_SUBJECT, bbattributes);
addEmailAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_RCVD, bbattributes); addEmailAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_RCVD, bbattributes);
addEmailAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_SENT, bbattributes); addEmailAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_SENT, bbattributes);
addEmailAttribute(email.getTextBody(), ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_PLAIN, bbattributes);
addEmailAttribute(body, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_PLAIN, bbattributes); addEmailAttribute(email.getHtmlBody(), ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_HTML, bbattributes);
addEmailAttribute(email.getRtfBody(), ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_RTF, bbattributes);
addEmailAttribute(((id < 0L) ? NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.notAvail") : String.valueOf(id)), addEmailAttribute(((id < 0L) ? NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.notAvail") : String.valueOf(id)),
ATTRIBUTE_TYPE.TSK_MSG_ID, bbattributes); ATTRIBUTE_TYPE.TSK_MSG_ID, bbattributes);
addEmailAttribute(((localPath.isEmpty() == false) ? localPath : "/foo/bar"), addEmailAttribute(StringUtils.defaultIfEmpty(email.getLocalPath(), "/foo/bar"), //TODO: really /foo/bar?
ATTRIBUTE_TYPE.TSK_PATH, bbattributes); ATTRIBUTE_TYPE.TSK_PATH, bbattributes);
addEmailAttribute(cc, ATTRIBUTE_TYPE.TSK_EMAIL_CC, bbattributes);
addEmailAttribute(bodyHTML, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_HTML, bbattributes);
addEmailAttribute(rtf, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_RTF, bbattributes);
try { try {
bbart = abstractFile.newArtifact(TSK_EMAIL_MSG);
bbart = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG);
bbart.addAttributes(bbattributes); bbart.addAttributes(bbattributes);
// Add account relationships // Add account relationships
openCase.getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart, Relationship.Type.MESSAGE, dateL); communicationsManager.addRelationships(senderAccountInstance, recipientAccountInstances, bbart, Relationship.Type.MESSAGE, dateL);
try { try {
// index the artifact for keyword search /*
* Post the artifact to the blackboard which will index it for
* keyword search and send a notification for the UI.
*/
blackboard.postArtifact(bbart, EmailParserModuleFactory.getModuleName()); blackboard.postArtifact(bbart, EmailParserModuleFactory.getModuleName());
} catch (Blackboard.BlackboardException ex) { } catch (Blackboard.BlackboardException ex) {
logger.log(Level.SEVERE, "Unable to index blackboard artifact " + bbart.getArtifactID(), ex); //NON-NLS logger.log(Level.SEVERE, "Unable to index blackboard artifact " + bbart.getArtifactID(), ex); //NON-NLS
@ -534,7 +534,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
} }
private void addEmailAttribute(String stringVal, ATTRIBUTE_TYPE attrType, Collection<BlackboardAttribute> bbattributes) { private void addEmailAttribute(String stringVal, ATTRIBUTE_TYPE attrType, Collection<BlackboardAttribute> bbattributes) {
if (stringVal.isEmpty() == false) { if (StringUtils.isNotBlank(stringVal)) {
bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), stringVal)); bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), stringVal));
} }
} }
@ -550,10 +550,6 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
services.postMessage(ingestMessage); services.postMessage(ingestMessage);
} }
IngestServices getServices() {
return services;
}
@Override @Override
public void shutDown() { public void shutDown() {
// nothing to shut down // nothing to shut down