mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-11 23:46:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

Browse Source
This commit is contained in:
Richard Cordovano 2013-07-23 11:14:10 -04:00
commit fc81760167
4 changed files with 139 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java
View File

@ -902,7 +902,7 @@ public class IngestManager {
abstractFileModulesRetValues.clear(); abstractFileModulesRetValues.clear();
} }
logger.log(Level.INFO, "IngestManager: Processing: {0}", fileToProcess.getName()); //logger.log(Level.INFO, "IngestManager: Processing: {0}", fileToProcess.getName());
progress.progress(fileToProcess.getName(), processedFiles); progress.progress(fileToProcess.getName(), processedFiles);
for (IngestModuleAbstractFile module : fileIngestTask.getModules()) { for (IngestModuleAbstractFile module : fileIngestTask.getModules()) {
//process the file with every file module //process the file with every file module

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
View File

@ -781,7 +781,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile {
} }
} }
} }
logger.log(Level.INFO, "Detected format: " + aFile.getName() + " " + detectedFormat); //logger.log(Level.INFO, "Detected format: " + aFile.getName() + " " + detectedFormat);
// we skip archive formats that are opened by the archive module. // we skip archive formats that are opened by the archive module.
// @@@ We could have a check here to see if the archive module was enabled though... // @@@ We could have a check here to see if the archive module was enabled though...

2
thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdEmailParser.java
View File

@ -94,7 +94,7 @@ public class ThunderbirdEmailParser {
return this.tika.detect(firstFewBytes, inDocName); return this.tika.detect(firstFewBytes, inDocName);
} }
public boolean isValidMimeTypeMbox(byte[] buffer) { static public boolean isValidMimeTypeMbox(byte[] buffer) {
return (new String(buffer)).startsWith("From "); return (new String(buffer)).startsWith("From ");
} }

269
thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java
View File

@ -1,7 +1,7 @@
/* /*
* Autopsy Forensic Browser * Autopsy Forensic Browser
* *
* Copyright 2011 Basis Technology Corp. * Copyright 2011-2013 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org * Contact: carrier <at> sleuthkit <dot> org
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
@ -29,30 +29,33 @@ import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.logging.Level; import java.util.logging.Level;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.tika.exception.TikaException; import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.Metadata;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.IngestModuleAbstract.*;
import org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
import org.sleuthkit.datamodel.ReadContentInputStream;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskException;
import org.xml.sax.SAXException;
import org.apache.commons.lang.StringEscapeUtils;
import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.autopsy.ingest.PipelineContext; import org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile;
import org.sleuthkit.autopsy.ingest.IngestModuleInit; import org.sleuthkit.autopsy.ingest.IngestModuleInit;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
import org.sleuthkit.autopsy.ingest.PipelineContext;
import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.ReadContentInputStream;
import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskException;
import org.xml.sax.SAXException;
/**
* File-level ingest module that detects MBOX files based on signature.
* Understands Thunderbird folder layout to provide additional structure and metadata.
*/
public class ThunderbirdMboxFileIngestModule extends IngestModuleAbstractFile { public class ThunderbirdMboxFileIngestModule extends IngestModuleAbstractFile {
private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()); private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName());
@ -73,91 +76,93 @@ public class ThunderbirdMboxFileIngestModule extends IngestModuleAbstractFile {
@Override @Override
public ProcessResult process(PipelineContext<IngestModuleAbstractFile>ingestContext, AbstractFile abstractFile) { public ProcessResult process(PipelineContext<IngestModuleAbstractFile>ingestContext, AbstractFile abstractFile) {
ThunderbirdEmailParser mbox = new ThunderbirdEmailParser();
boolean isMbox = false;
IngestModuleAbstractFile.ProcessResult hashDBResult =
services.getAbstractFileModuleResult(hashDBModuleName);
if (abstractFile.getKnown().equals( if (abstractFile.getKnown().equals(
TskData.FileKnown.KNOWN)) { TskData.FileKnown.KNOWN)) {
return ProcessResult.OK; //file is known, stop processing it return ProcessResult.OK; //file is known, stop processing it
} else if (hashDBResult == IngestModuleAbstractFile.ProcessResult.ERROR) { }
IngestModuleAbstractFile.ProcessResult hashDBResult =
services.getAbstractFileModuleResult(hashDBModuleName);
if (hashDBResult == IngestModuleAbstractFile.ProcessResult.ERROR) {
return ProcessResult.ERROR; //file has read error, stop processing it return ProcessResult.ERROR; //file has read error, stop processing it
} }
if (abstractFile.isVirtual()) { if (abstractFile.isVirtual()) {
return ProcessResult.OK; return ProcessResult.OK;
} }
boolean isMbox = false;
try { try {
byte[] t = new byte[64]; byte[] t = new byte[64];
if (abstractFile.getSize() > 64) { if (abstractFile.getSize() > 64) {
int byteRead = abstractFile.read(t, 0, 64); int byteRead = abstractFile.read(t, 0, 64);
if (byteRead > 0) { if (byteRead > 0) {
isMbox = mbox.isValidMimeTypeMbox(t); isMbox = ThunderbirdEmailParser.isValidMimeTypeMbox(t);
} }
} }
} catch (TskException ex) { } catch (TskException ex) {
logger.log(Level.WARNING, null, ex); logger.log(Level.WARNING, null, ex);
} }
if (isMbox == false) {
return ProcessResult.OK;
}
logger.log(Level.INFO, "ThunderbirdMboxFileIngestModule: Parsing {0}", abstractFile.getName());
String mboxName = abstractFile.getName();
String msfName = mboxName + ".msf";
//Long mboxId = fsContent.getId();
String mboxPath = abstractFile.getParentPath();
Long msfId = 0L;
currentCase = Case.getCurrentCase(); // get the most updated case
SleuthkitCase tskCase = currentCase.getSleuthkitCase();
if (isMbox) { try {
logger.log(Level.INFO, "ThunderbirdMboxFileIngestModule: Parsing {0}", abstractFile.getName()); ResultSet resultset = tskCase.runQuery("SELECT obj_id FROM tsk_files WHERE parent_path = '" + mboxPath + "' and name = '" + msfName + "'");
if (!resultset.next()) {
String mboxName = abstractFile.getName();
String msfName = mboxName + ".msf";
//Long mboxId = fsContent.getId();
String mboxPath = abstractFile.getParentPath();
Long msfId = 0L;
currentCase = Case.getCurrentCase(); // get the most updated case
SleuthkitCase tskCase = currentCase.getSleuthkitCase();
try {
ResultSet resultset = tskCase.runQuery("SELECT obj_id FROM tsk_files WHERE parent_path = '" + mboxPath + "' and name = '" + msfName + "'");
if (!resultset.next()) {
logger.log(Level.WARNING, "Could not find msf file in mbox dir: " + mboxPath + " file: " + msfName);
tskCase.closeRunQuery(resultset);
return ProcessResult.OK;
} else {
msfId = resultset.getLong(1);
tskCase.closeRunQuery(resultset);
}
} catch (SQLException ex) {
logger.log(Level.WARNING, "Could not find msf file in mbox dir: " + mboxPath + " file: " + msfName); logger.log(Level.WARNING, "Could not find msf file in mbox dir: " + mboxPath + " file: " + msfName);
} tskCase.closeRunQuery(resultset);
return ProcessResult.OK;
try {
Content msfContent = tskCase.getContentById(msfId);
if (msfContent != null) {
ContentUtils.writeToFile(msfContent, new File(currentCase.getTempDirectory() + File.separator + msfName));
}
} catch (IOException ex) {
logger.log(Level.WARNING, "Unable to obtain msf file for mbox parsing:" + msfName, ex);
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Unable to obtain msf file for mbox parsing:" + msfName, ex);
}
int index = 0;
String replace = "";
boolean a = mboxPath.indexOf("/ImapMail/") > 0;
boolean b = mboxPath.indexOf("/Mail/") > 0;
if (b == true) {
index = mboxPath.indexOf("/Mail/");
replace = "/Mail";
} else if (a == true) {
index = mboxPath.indexOf("/ImapMail/");
replace = "/ImapMail";
} else { } else {
replace = ""; msfId = resultset.getLong(1);
tskCase.closeRunQuery(resultset);
} }
String folderPath = mboxPath.substring(index);
folderPath = folderPath.replaceAll(replace, ""); } catch (SQLException ex) {
folderPath = folderPath + mboxName; logger.log(Level.WARNING, "Could not find msf file in mbox dir: " + mboxPath + " file: " + msfName);
folderPath = folderPath.replaceAll(".sbd", ""); }
try {
Content msfContent = tskCase.getContentById(msfId);
if (msfContent != null) {
ContentUtils.writeToFile(msfContent, new File(currentCase.getTempDirectory() + File.separator + msfName));
}
} catch (IOException ex) {
logger.log(Level.WARNING, "Unable to obtain msf file for mbox parsing:" + msfName, ex);
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Unable to obtain msf file for mbox parsing:" + msfName, ex);
}
int index = 0;
String replace = "";
boolean a = mboxPath.indexOf("/ImapMail/") > 0;
boolean b = mboxPath.indexOf("/Mail/") > 0;
if (b == true) {
index = mboxPath.indexOf("/Mail/");
replace = "/Mail";
} else if (a == true) {
index = mboxPath.indexOf("/ImapMail/");
replace = "/ImapMail";
} else {
replace = "";
}
String folderPath = mboxPath.substring(index);
folderPath = folderPath.replaceAll(replace, "");
folderPath = folderPath + mboxName;
folderPath = folderPath.replaceAll(".sbd", "");
// Reader reader = null; // Reader reader = null;
// try { // try {
// reader = new FileReader(currentCase.getTempDirectory() + File.separator + msfName); // reader = new FileReader(currentCase.getTempDirectory() + File.separator + msfName);
@ -170,66 +175,64 @@ public class ThunderbirdMboxFileIngestModule extends IngestModuleAbstractFile {
// String path = dict.getValue("81").toString(); // String path = dict.getValue("81").toString();
// String account = dict.getValue("8D").toString(); // String account = dict.getValue("8D").toString();
// } // }
String emailId = ""; String emailId = "";
String content = ""; String content = "";
String from = ""; String from = "";
String to = ""; String to = "";
String stringDate = ""; String stringDate = "";
Long date = 0L; Long date = 0L;
String subject = ""; String subject = "";
String cc = ""; String cc = "";
String bcc = ""; String bcc = "";
try { ThunderbirdEmailParser mbox = new ThunderbirdEmailParser();
ReadContentInputStream contentStream = new ReadContentInputStream(abstractFile); try {
mbox.parse(contentStream); ReadContentInputStream contentStream = new ReadContentInputStream(abstractFile);
HashMap<String, Map<String, String>> emailMap = new HashMap<String, Map<String, String>>(); mbox.parse(contentStream);
emailMap = mbox.getAllEmails(); HashMap<String, Map<String, String>> emailMap = new HashMap<String, Map<String, String>>();
for (Entry<String, Map<String, String>> entry : emailMap.entrySet()) { emailMap = mbox.getAllEmails();
Map<String, String> propertyMap = new HashMap<String, String>(); for (Entry<String, Map<String, String>> entry : emailMap.entrySet()) {
emailId = ((entry.getKey() != null) ? entry.getKey() : "Not Available"); Map<String, String> propertyMap = new HashMap<String, String>();
propertyMap = entry.getValue(); emailId = ((entry.getKey() != null) ? entry.getKey() : "Not Available");
content = ((propertyMap.get("content") != null) ? propertyMap.get("content") : ""); propertyMap = entry.getValue();
from = ((propertyMap.get(Metadata.AUTHOR) != null) ? propertyMap.get(Metadata.AUTHOR) : ""); content = ((propertyMap.get("content") != null) ? propertyMap.get("content") : "");
to = ((propertyMap.get(Metadata.MESSAGE_TO) != null) ? propertyMap.get(Metadata.MESSAGE_TO) : ""); from = ((propertyMap.get(Metadata.AUTHOR) != null) ? propertyMap.get(Metadata.AUTHOR) : "");
stringDate = ((propertyMap.get("date") != null) ? propertyMap.get("date") : ""); to = ((propertyMap.get(Metadata.MESSAGE_TO) != null) ? propertyMap.get(Metadata.MESSAGE_TO) : "");
if (!"".equals(stringDate)) { stringDate = ((propertyMap.get("date") != null) ? propertyMap.get("date") : "");
date = mbox.getDateCreated(stringDate); if (!"".equals(stringDate)) {
} date = mbox.getDateCreated(stringDate);
subject = ((propertyMap.get(Metadata.SUBJECT) != null) ? propertyMap.get(Metadata.SUBJECT) : "");
cc = ((propertyMap.get(Metadata.MESSAGE_CC) != null) ? propertyMap.get(Metadata.MESSAGE_CC) : "");
bcc = ((propertyMap.get(Metadata.MESSAGE_BCC) != null) ? propertyMap.get(Metadata.MESSAGE_BCC) : "");
Collection<BlackboardAttribute> bbattributes = new ArrayList<BlackboardAttribute>();
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_TO.getTypeID(), MODULE_NAME, to));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_CC.getTypeID(), MODULE_NAME, cc));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_BCC.getTypeID(), MODULE_NAME, bcc));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_FROM.getTypeID(), MODULE_NAME, from));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_PLAIN.getTypeID(), MODULE_NAME, content.replaceAll("\\<[^>]*>", "")));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_HTML.getTypeID(), MODULE_NAME, content));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_MSG_ID.getTypeID(), MODULE_NAME, StringEscapeUtils.escapeHtml(emailId)));
//bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_MSG_REPLY_ID.getTypeID(), MODULE_NAME, "",));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_RCVD.getTypeID(), MODULE_NAME, date));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_SENT.getTypeID(), MODULE_NAME, date));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SUBJECT.getTypeID(), MODULE_NAME, subject));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH.getTypeID(), MODULE_NAME, folderPath));
BlackboardArtifact bbart;
try {
bbart = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG);
bbart.addAttributes(bbattributes);
} catch (TskCoreException ex) {
Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()).log(Level.WARNING, null, ex);
}
services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG));
} }
} catch (FileNotFoundException ex) { subject = ((propertyMap.get(Metadata.SUBJECT) != null) ? propertyMap.get(Metadata.SUBJECT) : "");
Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()).log(Level.WARNING, null, ex); cc = ((propertyMap.get(Metadata.MESSAGE_CC) != null) ? propertyMap.get(Metadata.MESSAGE_CC) : "");
} catch (IOException ex) { bcc = ((propertyMap.get(Metadata.MESSAGE_BCC) != null) ? propertyMap.get(Metadata.MESSAGE_BCC) : "");
Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()).log(Level.WARNING, null, ex);
} catch (SAXException ex) { Collection<BlackboardAttribute> bbattributes = new ArrayList<BlackboardAttribute>();
Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()).log(Level.WARNING, null, ex); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_TO.getTypeID(), MODULE_NAME, to));
} catch (TikaException ex) { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_CC.getTypeID(), MODULE_NAME, cc));
Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()).log(Level.WARNING, null, ex); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_BCC.getTypeID(), MODULE_NAME, bcc));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_FROM.getTypeID(), MODULE_NAME, from));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_PLAIN.getTypeID(), MODULE_NAME, content.replaceAll("\\<[^>]*>", "")));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_HTML.getTypeID(), MODULE_NAME, content));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_MSG_ID.getTypeID(), MODULE_NAME, StringEscapeUtils.escapeHtml(emailId)));
//bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_MSG_REPLY_ID.getTypeID(), MODULE_NAME, "",));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_RCVD.getTypeID(), MODULE_NAME, date));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_SENT.getTypeID(), MODULE_NAME, date));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SUBJECT.getTypeID(), MODULE_NAME, subject));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH.getTypeID(), MODULE_NAME, folderPath));
BlackboardArtifact bbart;
try {
bbart = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG);
bbart.addAttributes(bbattributes);
} catch (TskCoreException ex) {
Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()).log(Level.WARNING, null, ex);
}
services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG));
} }
} catch (FileNotFoundException ex) {
Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()).log(Level.WARNING, null, ex);
} catch (IOException ex) {
Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()).log(Level.WARNING, null, ex);
} catch (SAXException | TikaException ex) {
Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()).log(Level.WARNING, null, ex);
} }
return ProcessResult.OK; return ProcessResult.OK;