diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdEmailParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdEmailParser.java index 11d2ca91b7..e130a5ba20 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdEmailParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdEmailParser.java @@ -1,3 +1,21 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2013 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.sleuthkit.autopsy.thunderbirdparser; import java.io.*; @@ -18,6 +36,10 @@ import org.apache.tika.sax.BodyContentHandler; import org.xml.sax.ContentHandler; import org.xml.sax.SAXException; +/** + * Parses an MBOX file. + * + */ public class ThunderbirdEmailParser { private InputStream stream; @@ -41,6 +63,10 @@ public class ThunderbirdEmailParser { this.tika = new Tika(); } + /** + * + * @param inStream String to MBX file + */ public ThunderbirdEmailParser(InputStream inStream) { this.tika = new Tika(); this.stream = inStream; @@ -61,11 +87,26 @@ public class ThunderbirdEmailParser { this.contentHandler = new BodyContentHandler(10*1024*1024); } + /** + * Parse data passed in via constructor + * @throws FileNotFoundException + * @throws IOException + * @throws SAXException + * @throws TikaException + */ public void parse() throws FileNotFoundException, IOException, SAXException, TikaException { init(); parser.parse(this.stream, this.contentHandler, this.metadata, context); } + /** + * Parse given MBX stream + * @param inStream stream of MBX file + * @throws FileNotFoundException + * @throws IOException + * @throws SAXException + * @throws TikaException + */ public void parse(InputStream inStream) throws FileNotFoundException, IOException, SAXException, TikaException { init(); parser.parseMbox(inStream, this.contentHandler, this.metadata, context); diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 926ea686a0..e8dcc2690e 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -18,11 +18,8 @@ */ package org.sleuthkit.autopsy.thunderbirdparser; -import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -34,7 +31,6 @@ import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Metadata; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile; import org.sleuthkit.autopsy.ingest.IngestModuleInit; import org.sleuthkit.autopsy.ingest.IngestServices; @@ -44,9 +40,7 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; -import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ReadContentInputStream; -import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskException; @@ -61,8 +55,6 @@ public class ThunderbirdMboxFileIngestModule extends IngestModuleAbstractFile { private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()); private static ThunderbirdMboxFileIngestModule instance = null; private IngestServices services; - private static int messageId = 0; - private Case currentCase; private static final String MODULE_NAME = "Thunderbird Parser"; private final String hashDBModuleName = "Hash Lookup"; final public static String MODULE_VERSION = "1.0"; @@ -76,21 +68,31 @@ public class ThunderbirdMboxFileIngestModule extends IngestModuleAbstractFile { @Override public ProcessResult process(PipelineContextingestContext, AbstractFile abstractFile) { - if (abstractFile.getKnown().equals( - TskData.FileKnown.KNOWN)) { - return ProcessResult.OK; //file is known, stop processing it - } + + // skip known + if (abstractFile.getKnown().equals(TskData.FileKnown.KNOWN)) { + return ProcessResult.OK; + } + + //skip unalloc + if(abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) { + return ProcessResult.OK; + } + //file has read error, stop processing it + // @@@ I don't really like this + // we don't know if Hash was run or if it had lookup errors IngestModuleAbstractFile.ProcessResult hashDBResult = services.getAbstractFileModuleResult(hashDBModuleName); if (hashDBResult == IngestModuleAbstractFile.ProcessResult.ERROR) { - return ProcessResult.ERROR; //file has read error, stop processing it + return ProcessResult.ERROR; } if (abstractFile.isVirtual()) { return ProcessResult.OK; } + // check its signature boolean isMbox = false; try { byte[] t = new byte[64]; @@ -110,102 +112,82 @@ public class ThunderbirdMboxFileIngestModule extends IngestModuleAbstractFile { logger.log(Level.INFO, "ThunderbirdMboxFileIngestModule: Parsing {0}", abstractFile.getName()); - String mboxName = abstractFile.getName(); - String msfName = mboxName + ".msf"; - //Long mboxId = fsContent.getId(); - String mboxPath = abstractFile.getParentPath(); - Long msfId = 0L; - currentCase = Case.getCurrentCase(); // get the most updated case - SleuthkitCase tskCase = currentCase.getSleuthkitCase(); + String mboxFileName = abstractFile.getName(); + String mboxParentDir = abstractFile.getParentPath(); - try { - ResultSet resultset = tskCase.runQuery("SELECT obj_id FROM tsk_files WHERE parent_path = '" + mboxPath + "' and name = '" + msfName + "'"); - if (!resultset.next()) { - logger.log(Level.WARNING, "Could not find msf file in mbox dir: " + mboxPath + " file: " + msfName); - tskCase.closeRunQuery(resultset); - return ProcessResult.OK; - } else { - msfId = resultset.getLong(1); - tskCase.closeRunQuery(resultset); - } - - } catch (SQLException ex) { - logger.log(Level.WARNING, "Could not find msf file in mbox dir: " + mboxPath + " file: " + msfName); - } - - try { - Content msfContent = tskCase.getContentById(msfId); - if (msfContent != null) { - ContentUtils.writeToFile(msfContent, new File(currentCase.getTempDirectory() + File.separator + msfName)); - } - } catch (IOException ex) { - logger.log(Level.WARNING, "Unable to obtain msf file for mbox parsing:" + msfName, ex); - } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Unable to obtain msf file for mbox parsing:" + msfName, ex); - } - int index = 0; - String replace = ""; - boolean a = mboxPath.indexOf("/ImapMail/") > 0; - boolean b = mboxPath.indexOf("/Mail/") > 0; - if (b == true) { - index = mboxPath.indexOf("/Mail/"); - replace = "/Mail"; - } else if (a == true) { - index = mboxPath.indexOf("/ImapMail/"); - replace = "/ImapMail"; - } else { - replace = ""; - - } - - String folderPath = mboxPath.substring(index); - folderPath = folderPath.replaceAll(replace, ""); - folderPath = folderPath + mboxName; - folderPath = folderPath.replaceAll(".sbd", ""); -// Reader reader = null; -// try { -// reader = new FileReader(currentCase.getTempDirectory() + File.separator + msfName); -// } catch (FileNotFoundException ex) { -// Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()).log(Level.WARNING, null, ex); + // Find the .msf file in the same folder + // BC: Commented out because results are not being used Oct '13 + //Long msfId = 0L; + //String msfName = mboxFileName + ".msf"; + //SleuthkitCase tskCase = currentCase.getSleuthkitCase(); + // @@@ We shouldn't bail out here if we dont' find it... +// try { +// // @@@ Replace this with a call to FileManager.findFiles() +// ResultSet resultset = tskCase.runQuery("SELECT obj_id FROM tsk_files WHERE parent_path = '" + mboxParentDir + "' and name = '" + msfName + "'"); +// if (!resultset.next()) { +// logger.log(Level.WARNING, "Could not find msf file in mbox dir: " + mboxParentDir + " file: " + msfName); +// tskCase.closeRunQuery(resultset); +// return ProcessResult.OK; +// } else { +// msfId = resultset.getLong(1); +// tskCase.closeRunQuery(resultset); // } -// MorkDocument morkDocument = new MorkDocument(reader); -// List dicts = morkDocument.getDicts(); -// for(Dict dict : dicts){ -// String path = dict.getValue("81").toString(); -// String account = dict.getValue("8D").toString(); -// } - String emailId = ""; - String content = ""; - String from = ""; - String to = ""; - String stringDate = ""; - Long date = 0L; - String subject = ""; - String cc = ""; - String bcc = ""; - ThunderbirdEmailParser mbox = new ThunderbirdEmailParser(); +// +// } catch (SQLException ex) { +// logger.log(Level.WARNING, "Could not find msf file in mbox dir: " + mboxParentDir + " file: " + msfName); +// } +// +// try { +// Content msfContent = tskCase.getContentById(msfId); +// if (msfContent != null) { +// ContentUtils.writeToFile(msfContent, new File(currentCase.getTempDirectory() + File.separator + msfName)); +// } +// } catch (IOException ex) { +// logger.log(Level.WARNING, "Unable to obtain msf file for mbox parsing:" + msfName, ex); +// } catch (TskCoreException ex) { +// logger.log(Level.WARNING, "Unable to obtain msf file for mbox parsing:" + msfName, ex); +// } + + + // use the local path to determine the e-mail folder structure + String emailFolder = ""; + // email folder is everything after "Mail" or ImapMail + if (mboxParentDir.contains("/Mail/")) { + emailFolder = mboxParentDir.substring(mboxParentDir.indexOf("/Mail/") + 5); + } + else if (mboxParentDir.contains("/ImapMail/")) { + emailFolder = mboxParentDir.substring(mboxParentDir.indexOf("/ImapMail/") + 9); + } + emailFolder = emailFolder + mboxFileName; + emailFolder = emailFolder.replaceAll(".sbd", ""); + try { ReadContentInputStream contentStream = new ReadContentInputStream(abstractFile); + ThunderbirdEmailParser mbox = new ThunderbirdEmailParser(); mbox.parse(contentStream); - HashMap> emailMap = new HashMap>(); - emailMap = mbox.getAllEmails(); + + HashMap>emailMap = mbox.getAllEmails(); for (Entry> entry : emailMap.entrySet()) { - Map propertyMap = new HashMap(); - emailId = ((entry.getKey() != null) ? entry.getKey() : "Not Available"); - propertyMap = entry.getValue(); - content = ((propertyMap.get("content") != null) ? propertyMap.get("content") : ""); - from = ((propertyMap.get(Metadata.AUTHOR) != null) ? propertyMap.get(Metadata.AUTHOR) : ""); - to = ((propertyMap.get(Metadata.MESSAGE_TO) != null) ? propertyMap.get(Metadata.MESSAGE_TO) : ""); - stringDate = ((propertyMap.get("date") != null) ? propertyMap.get("date") : ""); - if (!"".equals(stringDate)) { + /* @@@ I'd rather this code be cleaned up a bit so that we check if the value is + * set and then directly add it to the attribute. otherwise, we end up with a bunch + * of "" attribute values. + */ + Collection bbattributes = new ArrayList<>(); + String emailId = ((entry.getKey() != null) ? entry.getKey() : "Not Available"); + MappropertyMap = entry.getValue(); + String content = ((propertyMap.get("content") != null) ? propertyMap.get("content") : ""); + String from = ((propertyMap.get(Metadata.AUTHOR) != null) ? propertyMap.get(Metadata.AUTHOR) : ""); + String to = ((propertyMap.get(Metadata.MESSAGE_TO) != null) ? propertyMap.get(Metadata.MESSAGE_TO) : ""); + String stringDate = ((propertyMap.get("date") != null) ? propertyMap.get("date") : ""); + Long date = 0L; + if (stringDate.equals("") == false) { date = mbox.getDateCreated(stringDate); } - subject = ((propertyMap.get(Metadata.SUBJECT) != null) ? propertyMap.get(Metadata.SUBJECT) : ""); - cc = ((propertyMap.get(Metadata.MESSAGE_CC) != null) ? propertyMap.get(Metadata.MESSAGE_CC) : ""); - bcc = ((propertyMap.get(Metadata.MESSAGE_BCC) != null) ? propertyMap.get(Metadata.MESSAGE_BCC) : ""); - - Collection bbattributes = new ArrayList(); + String subject = ((propertyMap.get(Metadata.SUBJECT) != null) ? propertyMap.get(Metadata.SUBJECT) : ""); + String cc = ((propertyMap.get(Metadata.MESSAGE_CC) != null) ? propertyMap.get(Metadata.MESSAGE_CC) : ""); + String bcc = ((propertyMap.get(Metadata.MESSAGE_BCC) != null) ? propertyMap.get(Metadata.MESSAGE_BCC) : ""); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_TO.getTypeID(), MODULE_NAME, to)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_CC.getTypeID(), MODULE_NAME, cc)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_BCC.getTypeID(), MODULE_NAME, bcc)); @@ -217,7 +199,7 @@ public class ThunderbirdMboxFileIngestModule extends IngestModuleAbstractFile { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_RCVD.getTypeID(), MODULE_NAME, date)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_SENT.getTypeID(), MODULE_NAME, date)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SUBJECT.getTypeID(), MODULE_NAME, subject)); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH.getTypeID(), MODULE_NAME, folderPath)); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH.getTypeID(), MODULE_NAME, emailFolder)); BlackboardArtifact bbart; try { bbart = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG); @@ -240,9 +222,6 @@ public class ThunderbirdMboxFileIngestModule extends IngestModuleAbstractFile { @Override public void complete() { - logger.log(Level.INFO, "complete()"); - - //module specific cleanup due completion here } @Override @@ -263,18 +242,11 @@ public class ThunderbirdMboxFileIngestModule extends IngestModuleAbstractFile { @Override public void init(IngestModuleInit initContext) { - logger.log(Level.INFO, "init()"); services = IngestServices.getDefault(); - - currentCase = Case.getCurrentCase(); - //module specific initialization here } @Override public void stop() { - logger.log(Level.INFO, "stop()"); - - //module specific cleanup due interruption here } @Override diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxParser.java index 60b9f75ca6..ee61a6952d 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxParser.java @@ -37,6 +37,9 @@ import org.apache.tika.sax.BodyContentHandler; import org.xml.sax.ContentHandler; import org.xml.sax.SAXException; +/** + * Contains the logic to parse an MBOX file. + */ public class ThunderbirdMboxParser { /** Serial version UID */