diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED index b39a22c484..cdfd241886 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED @@ -1,4 +1,4 @@ -MboxParser.handleAttch.noOpenCase.errMsg=Exception while getting open case. +MimeJ4MessageParser.handleAttch.noOpenCase.errMsg=Exception while getting open case. OpenIDE-Module-Display-Category=Ingest Module OpenIDE-Module-Long-Description=Email Parser ingest module.\n\nThe module extracts MBOX and PST e-mail files and posts the results to the blackboard.\nIt knows about the Thunderbird folder structure for MBOX files. OpenIDE-Module-Name=Email Parser diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java new file mode 100755 index 0000000000..e0b50ffa8c --- /dev/null +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java @@ -0,0 +1,72 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.thunderbirdparser; + +import java.io.FileNotFoundException; +import java.io.IOException; +import org.apache.james.mime4j.MimeException; +import org.apache.james.mime4j.dom.Message; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.ReadContentInputStream; + +/** + * EML file parser. An .eml file contains a single email message. + * + */ +class EMLParser extends MimeJ4MessageParser { + + /** + * If the extention of the AbstractFile is eml and 'To:' is found close to + * the beginning of the file, then its probably an eml file. + * + * @param abFile AbstractFile to test + * @param buffer A byte buffer of the beginning of the file. + * + * @return True, if we think this is an eml file, false otherwise. + */ + static boolean isEMLFile(AbstractFile abFile, byte[] buffer) { + String ext = abFile.getNameExtension(); + boolean isEMLFile = ext != null && ext.equals("eml"); + if (isEMLFile) { + isEMLFile = (new String(buffer)).contains("To:"); //NON-NLS + } + + return isEMLFile; + } + + /** + * + * @param sourceFile AbstractFile source file for eml message + * @param localPath The local path to the eml file + * + * @return EmailMessage object for message in eml file + * + * @throws FileNotFoundException + * @throws IOException + * @throws MimeException + */ + static EmailMessage parse(AbstractFile sourceFile) throws FileNotFoundException, IOException, MimeException { + try (ReadContentInputStream fis = new ReadContentInputStream(sourceFile)) { + EMLParser parser = new EMLParser(); + parser.setLocalPath(sourceFile.getParentPath()); + Message mimeMsg = parser.getMessageBuilder().parseMessage(fis); + return parser.extractEmail(mimeMsg, "", sourceFile.getId()); + } + } +} diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java index 270e0ecdf8..a17294b15b 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java @@ -19,12 +19,10 @@ package org.sleuthkit.autopsy.thunderbirdparser; import java.io.BufferedInputStream; -import java.io.BufferedReader; import java.io.CharConversionException; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; @@ -35,60 +33,26 @@ import java.nio.charset.UnsupportedCharsetException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.UUID; import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; -import org.apache.james.mime4j.dom.BinaryBody; -import org.apache.james.mime4j.dom.Body; -import org.apache.james.mime4j.dom.Entity; import org.apache.james.mime4j.dom.Message; -import org.apache.james.mime4j.dom.Multipart; -import org.apache.james.mime4j.dom.TextBody; -import org.apache.james.mime4j.dom.address.AddressList; -import org.apache.james.mime4j.dom.address.Mailbox; -import org.apache.james.mime4j.dom.address.MailboxList; -import org.apache.james.mime4j.dom.field.ContentDispositionField; -import org.apache.james.mime4j.dom.field.ContentTypeField; import org.apache.james.mime4j.mboxiterator.CharBufferWrapper; import org.apache.james.mime4j.mboxiterator.MboxIterator; -import org.apache.james.mime4j.message.DefaultMessageBuilder; -import org.apache.james.mime4j.stream.Field; -import org.apache.james.mime4j.stream.MimeConfig; import org.apache.tika.parser.txt.CharsetDetector; import org.apache.tika.parser.txt.CharsetMatch; import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.coreutils.FileUtil; -import org.sleuthkit.datamodel.TskData; -import org.sleuthkit.datamodel.EncodedFileOutputStream; /** * An Iterator for parsing mbox files. Wraps an instance of MBoxEmailIterator. */ -class MboxParser implements Iterator { +class MboxParser extends MimeJ4MessageParser implements Iterator { private static final Logger logger = Logger.getLogger(MboxParser.class.getName()); - private final DefaultMessageBuilder messageBuilder; - private final List errorList = new ArrayList<>(); - - /** - * The mime type string for html text. - */ - private static final String HTML_TYPE = "text/html"; //NON-NLS - - /** - * The local path of the mbox file. - */ - private String localPath; private Iterator emailIterator = null; private MboxParser(String localPath) { - this.localPath = localPath; - messageBuilder = new DefaultMessageBuilder(); - MimeConfig config = MimeConfig.custom().setMaxLineLen(-1).build(); - // disable line length checks. - messageBuilder.setMimeEntityConfig(config); + setLocalPath(localPath); } static boolean isValidMimeTypeMbox(byte[] buffer) { @@ -170,276 +134,6 @@ class MboxParser implements Iterator { return emailIterator != null ? emailIterator.next() : null; } - String getErrors() { - String result = ""; - for (String msg: errorList) { - result += "
  • " + msg + "
    • "; - } - return result; - } - - /** - * Use the information stored in the given mime4j message to populate an - * EmailMessage. - * - * @param msg - * - * @return - */ - private EmailMessage extractEmail(Message msg, long fileID) { - EmailMessage email = new EmailMessage(); - // Basic Info - email.setSender(getAddresses(msg.getFrom())); - email.setRecipients(getAddresses(msg.getTo())); - email.setBcc(getAddresses(msg.getBcc())); - email.setCc(getAddresses(msg.getCc())); - email.setSubject(msg.getSubject()); - email.setSentDate(msg.getDate()); - email.setLocalPath(localPath); - email.setMessageID(msg.getMessageId()); - - Field field = msg.getHeader().getField("in-reply-to"); //NON-NLS - String inReplyTo = null; - - if (field != null) { - inReplyTo = field.getBody(); - email.setInReplyToID(inReplyTo); - } - - field = msg.getHeader().getField("references"); - if (field != null) { - List references = new ArrayList<>(); - for (String id : field.getBody().split(">")) { - references.add(id.trim() + ">"); - } - - if (!references.contains(inReplyTo)) { - references.add(inReplyTo); - } - - email.setReferences(references); - } - - // Body - if (msg.isMultipart()) { - handleMultipart(email, (Multipart) msg.getBody(), fileID); - } else { - handleTextBody(email, (TextBody) msg.getBody(), msg.getMimeType(), msg.getHeader().getFields()); - } - - return email; - } - - /** - * Extract the subject, inReplyTo, message-ID and references from the - * Message object and returns them in a new EmailMessage object. - * - * @param msg Message object - * - * @return EmailMessage instance with only some of the message information - */ - private EmailMessage extractPartialEmail(Message msg) { - EmailMessage email = new EmailMessage(); - email.setSubject(msg.getSubject()); - email.setMessageID(msg.getMessageId()); - - Field field = msg.getHeader().getField("in-reply-to"); //NON-NLS - String inReplyTo = null; - - if (field != null) { - inReplyTo = field.getBody(); - email.setInReplyToID(inReplyTo); - } - - field = msg.getHeader().getField("references"); - if (field != null) { - List references = new ArrayList<>(); - for (String id : field.getBody().split(">")) { - references.add(id.trim() + ">"); - } - - if (!references.contains(inReplyTo)) { - references.add(inReplyTo); - } - - email.setReferences(references); - } - - return email; - } - - /** - * Handle a multipart mime message. Recursively calls handleMultipart if one - * of the body parts is another multipart. Otherwise, calls the correct - * method to extract information out of each part of the body. - * - * @param email - * @param multi - */ - private void handleMultipart(EmailMessage email, Multipart multi, long fileID) { - List entities = multi.getBodyParts(); - for (int index = 0; index < entities.size(); index++) { - Entity e = entities.get(index); - if (e.isMultipart()) { - handleMultipart(email, (Multipart) e.getBody(), fileID); - } else if (e.getDispositionType() != null - && e.getDispositionType().equals(ContentDispositionField.DISPOSITION_TYPE_ATTACHMENT)) { - handleAttachment(email, e, fileID, index); - } else if (e.getMimeType().equals(HTML_TYPE) - || e.getMimeType().equals(ContentTypeField.TYPE_TEXT_PLAIN)) { - handleTextBody(email, (TextBody) e.getBody(), e.getMimeType(), e.getHeader().getFields()); - } else { - // Ignore other types. - } - } - } - - /** - * Extract text out of a body part of the message. - * - * Handles text and html mime types. Throws away all other types. (only - * other example I've seen is text/calendar) - * - * @param email - * @param tb - * @param type The Mime type of the body. - */ - private void handleTextBody(EmailMessage email, TextBody tb, String type, List fields) { - BufferedReader r; - try { - r = new BufferedReader(tb.getReader()); - StringBuilder bodyString = new StringBuilder(); - StringBuilder headersString = new StringBuilder(); - String line; - while ((line = r.readLine()) != null) { - bodyString.append(line).append("\n"); - } - - headersString.append("\n-----HEADERS-----\n"); - for (Field field : fields) { - String nextLine = field.getName() + ": " + field.getBody(); - headersString.append("\n").append(nextLine); - } - headersString.append("\n\n---END HEADERS--\n\n"); - - email.setHeaders(headersString.toString()); - - switch (type) { - case ContentTypeField.TYPE_TEXT_PLAIN: - email.setTextBody(bodyString.toString()); - break; - case HTML_TYPE: - email.setHtmlBody(bodyString.toString()); - break; - default: - // Not interested in other text types. - break; - } - } catch (IOException ex) { - logger.log(Level.WARNING, "Error getting text body of mbox message", ex); //NON-NLS - } - } - - /** - * Extract the attachment out of the given entity. Should only be called if - * e.getDispositionType() == "attachment" - * - * @param email - * @param e - */ - @NbBundle.Messages({"MboxParser.handleAttch.noOpenCase.errMsg=Exception while getting open case."}) - private void handleAttachment(EmailMessage email, Entity e, long fileID, int index) { - String outputDirPath; - String relModuleOutputPath; - try { - outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator; - relModuleOutputPath = ThunderbirdMboxFileIngestModule.getRelModuleOutputPath() + File.separator; - } catch (NoCurrentCaseException ex) { - addErrorMessage(Bundle.MboxParser_handleAttch_noOpenCase_errMsg()); - logger.log(Level.SEVERE, Bundle.MboxParser_handleAttch_noOpenCase_errMsg(), ex); //NON-NLS - return; - } - String filename = FileUtil.escapeFileName(e.getFilename()); - - // also had some crazy long names, so make random one if we get those. - // also from Japanese image that had encoded name - if (filename.length() > 64) { - filename = UUID.randomUUID().toString(); - } - - String uniqueFilename = fileID + "-" + index + "-" + email.getSentDate() + "-" + filename; - String outPath = outputDirPath + uniqueFilename; - EncodedFileOutputStream fos; - BinaryBody bb; - try { - fos = new EncodedFileOutputStream(new FileOutputStream(outPath), TskData.EncodingType.XOR1); - } catch (IOException ex) { - addErrorMessage( - NbBundle.getMessage(this.getClass(), - "MboxParser.handleAttch.errMsg.failedToCreateOnDisk", outPath)); - logger.log(Level.WARNING, "Failed to create file output stream for: " + outPath, ex); //NON-NLS - return; - } - - try { - Body b = e.getBody(); - if (b instanceof BinaryBody) { - bb = (BinaryBody) b; - bb.writeTo(fos); - } else { - // This could potentially be other types. Only seen this once. - } - } catch (IOException ex) { - logger.log(Level.WARNING, "Failed to write mbox email attachment to disk.", ex); //NON-NLS - addErrorMessage(NbBundle.getMessage(this.getClass(), "MboxParser.handleAttch.failedWriteToDisk", filename)); - return; - } finally { - try { - fos.close(); - } catch (IOException ex) { - logger.log(Level.WARNING, "Failed to close file output stream", ex); //NON-NLS - } - } - - EmailMessage.Attachment attach = new EmailMessage.Attachment(); - attach.setName(filename); - attach.setLocalPath(relModuleOutputPath + uniqueFilename); - attach.setSize(new File(outPath).length()); - attach.setEncodingType(TskData.EncodingType.XOR1); - email.addAttachment(attach); - } - - /** - * Get a String representation of the MailboxList (which is a list of email - * addresses). - * - * @param mailboxList - * - * @return - */ - private String getAddresses(MailboxList mailboxList) { - if (mailboxList == null) { - return ""; - } - StringBuilder addresses = new StringBuilder(); - for (Mailbox m : mailboxList) { - addresses.append(m.toString()).append("; "); - } - return addresses.toString(); - } - - /** - * Get a String representation of the AddressList (which is a list of email - * addresses). - * - * @param addressList - * - * @return - */ - private String getAddresses(AddressList addressList) { - return (addressList == null) ? "" : getAddresses(addressList.flatten()); - } - /** * Get a list of the possible encoders for the given mboxFile using Tika's * CharsetDetector. At a minimum, returns the standard built in charsets. @@ -489,11 +183,7 @@ class MboxParser implements Iterator { } } } - - private void addErrorMessage(String msg) { - errorList.add(msg); - } - + /** * An Interator for mbox email messages. */ @@ -521,9 +211,9 @@ class MboxParser implements Iterator { CharBufferWrapper messageBuffer = mboxIterator.next(); try { - Message msg = messageBuilder.parseMessage(messageBuffer.asInputStream(encoder.charset())); + Message msg = getMessageBuilder().parseMessage(messageBuffer.asInputStream(encoder.charset())); if (wholeMsg) { - return extractEmail(msg, fileID); + return extractEmail(msg, getLocalPath(), fileID); } else { return extractPartialEmail(msg); } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java new file mode 100755 index 0000000000..1114d71eba --- /dev/null +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java @@ -0,0 +1,379 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.thunderbirdparser; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.logging.Level; +import org.apache.james.mime4j.dom.BinaryBody; +import org.apache.james.mime4j.dom.Body; +import org.apache.james.mime4j.dom.Entity; +import org.apache.james.mime4j.dom.Message; +import org.apache.james.mime4j.dom.Multipart; +import org.apache.james.mime4j.dom.TextBody; +import org.apache.james.mime4j.dom.address.AddressList; +import org.apache.james.mime4j.dom.address.Mailbox; +import org.apache.james.mime4j.dom.address.MailboxList; +import org.apache.james.mime4j.dom.field.ContentDispositionField; +import org.apache.james.mime4j.dom.field.ContentTypeField; +import org.apache.james.mime4j.message.DefaultMessageBuilder; +import org.apache.james.mime4j.stream.Field; +import org.apache.james.mime4j.stream.MimeConfig; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.coreutils.FileUtil; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.EncodedFileOutputStream; +import org.sleuthkit.datamodel.TskData; + +/** + * Super class for email parsers that can use the james.mime4J.Message objects. + */ +class MimeJ4MessageParser { + + private static final Logger logger = Logger.getLogger(MimeJ4MessageParser.class.getName()); + + /** + * The mime type string for html text. + */ + private static final String HTML_TYPE = "text/html"; //NON-NLS + private DefaultMessageBuilder messageBuilder = null; + private final List errorList = new ArrayList<>(); + + /** + * The local path of the email message(s) file. + */ + private String localPath; + + DefaultMessageBuilder getMessageBuilder() { + if (messageBuilder == null) { + messageBuilder = new DefaultMessageBuilder(); + MimeConfig config = MimeConfig.custom().setMaxLineLen(-1).setMaxHeaderLen(-1).setMaxHeaderCount(-1).build(); + // disable line length checks. + messageBuilder.setMimeEntityConfig(config); + } + + return messageBuilder; + } + + /** + * Sets the local path of the email messages file. + * + * @param localPath Local path of the file the email messages + */ + final void setLocalPath(String localPath) { + this.localPath = localPath; + } + + /** + * Gets the local path. + * + * @return + */ + String getLocalPath() { + return localPath; + } + + /** + * Get a list of the parsing error message. + * + * @return String containing all of the parse error message. Empty string is + * returned if there are no error messages. + */ + String getErrors() { + String result = ""; + for (String msg : errorList) { + result += "
  • " + msg + "
    • "; + } + return result; + } + + /** + * Adds a message to the error Message list. + * + * @param msg Message to add to the list. + */ + void addErrorMessage(String msg) { + errorList.add(msg); + } + + /** + * Use the information stored in the given mime4j message to populate an + * EmailMessage. + * + * @param msg The Message to extract data from. + * + * @return EmailMessage for the Message. + */ + EmailMessage extractEmail(Message msg, String localPath, long sourceFileID) { + EmailMessage email = new EmailMessage(); + // Basic Info + email.setSender(getAddresses(msg.getFrom())); + email.setRecipients(getAddresses(msg.getTo())); + email.setBcc(getAddresses(msg.getBcc())); + email.setCc(getAddresses(msg.getCc())); + email.setSubject(msg.getSubject()); + email.setSentDate(msg.getDate()); + email.setLocalPath(localPath); + email.setMessageID(msg.getMessageId()); + + Field field = msg.getHeader().getField("in-reply-to"); //NON-NLS + String inReplyTo = null; + + if (field != null) { + inReplyTo = field.getBody(); + email.setInReplyToID(inReplyTo); + } + + field = msg.getHeader().getField("references"); + if (field != null) { + List references = new ArrayList<>(); + for (String id : field.getBody().split(">")) { + references.add(id.trim() + ">"); + } + + if (!references.contains(inReplyTo)) { + references.add(inReplyTo); + } + + email.setReferences(references); + } + + // Body + if (msg.isMultipart()) { + handleMultipart(email, (Multipart) msg.getBody(), sourceFileID); + } else { + handleTextBody(email, (TextBody) msg.getBody(), msg.getMimeType(), msg.getHeader().getFields()); + } + + return email; + } + + /** + * Extract the subject, inReplyTo, message-ID and references from the + * Message object and returns them in a new EmailMessage object. + * + * @param msg Message object + * + * @return EmailMessage instance with only some of the message information + */ + EmailMessage extractPartialEmail(Message msg) { + EmailMessage email = new EmailMessage(); + email.setSubject(msg.getSubject()); + email.setMessageID(msg.getMessageId()); + + Field field = msg.getHeader().getField("in-reply-to"); //NON-NLS + String inReplyTo = null; + + if (field != null) { + inReplyTo = field.getBody(); + email.setInReplyToID(inReplyTo); + } + + field = msg.getHeader().getField("references"); + if (field != null) { + List references = new ArrayList<>(); + for (String id : field.getBody().split(">")) { + references.add(id.trim() + ">"); + } + + if (!references.contains(inReplyTo)) { + references.add(inReplyTo); + } + + email.setReferences(references); + } + + return email; + } + + /** + * Handle a multipart mime message. Recursively calls handleMultipart if one + * of the body parts is another multipart. Otherwise, calls the correct + * method to extract information out of each part of the body. + * + * @param email + * @param multi + */ + private void handleMultipart(EmailMessage email, Multipart multi, long fileID) { + List entities = multi.getBodyParts(); + for (int index = 0; index < entities.size(); index++) { + Entity e = entities.get(index); + if (e.isMultipart()) { + handleMultipart(email, (Multipart) e.getBody(), fileID); + } else if (e.getDispositionType() != null + && e.getDispositionType().equals(ContentDispositionField.DISPOSITION_TYPE_ATTACHMENT)) { + handleAttachment(email, e, fileID, index); + } else if (e.getMimeType().equals(HTML_TYPE) + || e.getMimeType().equals(ContentTypeField.TYPE_TEXT_PLAIN)) { + handleTextBody(email, (TextBody) e.getBody(), e.getMimeType(), e.getHeader().getFields()); + } else { + // Ignore other types. + } + } + } + + /** + * Extract text out of a body part of the message. + * + * Handles text and html mime types. Throws away all other types. (only + * other example I've seen is text/calendar) + * + * @param email + * @param tb + * @param type The Mime type of the body. + */ + private void handleTextBody(EmailMessage email, TextBody tb, String type, List fields) { + BufferedReader r; + try { + r = new BufferedReader(tb.getReader()); + StringBuilder bodyString = new StringBuilder(); + StringBuilder headersString = new StringBuilder(); + String line; + while ((line = r.readLine()) != null) { + bodyString.append(line).append("\n"); + } + + headersString.append("\n-----HEADERS-----\n"); + for (Field field : fields) { + String nextLine = field.getName() + ": " + field.getBody(); + headersString.append("\n").append(nextLine); + } + headersString.append("\n\n---END HEADERS--\n\n"); + + email.setHeaders(headersString.toString()); + + switch (type) { + case ContentTypeField.TYPE_TEXT_PLAIN: + email.setTextBody(bodyString.toString()); + break; + case HTML_TYPE: + email.setHtmlBody(bodyString.toString()); + break; + default: + // Not interested in other text types. + break; + } + } catch (IOException ex) { + logger.log(Level.WARNING, "Error getting text body of mbox message", ex); //NON-NLS + } + } + + /** + * Extract the attachment out of the given entity. Should only be called if + * e.getDispositionType() == "attachment" + * + * @param email + * @param e + */ + @NbBundle.Messages({"MimeJ4MessageParser.handleAttch.noOpenCase.errMsg=Exception while getting open case."}) + private static void handleAttachment(EmailMessage email, Entity e, long fileID, int index) { + String outputDirPath; + String relModuleOutputPath; + try { + outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator; + relModuleOutputPath = ThunderbirdMboxFileIngestModule.getRelModuleOutputPath() + File.separator; + } catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, Bundle.MimeJ4MessageParser_handleAttch_noOpenCase_errMsg(), ex); //NON-NLS + return; + } + String filename = FileUtil.escapeFileName(e.getFilename()); + + // also had some crazy long names, so make random one if we get those. + // also from Japanese image that had encoded name + if (filename.length() > 64) { + filename = UUID.randomUUID().toString(); + } + + String uniqueFilename = fileID + "-" + index + "-" + email.getSentDate() + "-" + filename; + String outPath = outputDirPath + uniqueFilename; + EncodedFileOutputStream fos; + BinaryBody bb; + try { + fos = new EncodedFileOutputStream(new FileOutputStream(outPath), TskData.EncodingType.XOR1); + } catch (IOException ex) { + logger.log(Level.WARNING, "Failed to create file output stream for: " + outPath, ex); //NON-NLS + return; + } + + try { + Body b = e.getBody(); + if (b instanceof BinaryBody) { + bb = (BinaryBody) b; + bb.writeTo(fos); + } else { + // This could potentially be other types. Only seen this once. + } + } catch (IOException ex) { + logger.log(Level.WARNING, "Failed to write mbox email attachment to disk.", ex); //NON-NLS + return; + } finally { + try { + fos.close(); + } catch (IOException ex) { + logger.log(Level.WARNING, "Failed to close file output stream", ex); //NON-NLS + } + } + + EmailMessage.Attachment attach = new EmailMessage.Attachment(); + attach.setName(filename); + attach.setLocalPath(relModuleOutputPath + uniqueFilename); + attach.setSize(new File(outPath).length()); + attach.setEncodingType(TskData.EncodingType.XOR1); + email.addAttachment(attach); + } + + /** + * Get a String representation of the MailboxList (which is a list of email + * addresses). + * + * @param mailboxList + * + * @return String list of email addresses separated by a ; or empty string + * if no addresses were found. + */ + private static String getAddresses(MailboxList mailboxList) { + if (mailboxList == null) { + return ""; + } + StringBuilder addresses = new StringBuilder(); + for (Mailbox m : mailboxList) { + addresses.append(m.toString()).append("; "); + } + return addresses.toString(); + } + + /** + * Get a String representation of the AddressList (which is a list of email + * addresses). + * + * @param addressList + * + * @return String list of email addresses separated by a ; or empty string + * if no addresses were found. + */ + private static String getAddresses(AddressList addressList) { + return (addressList == null) ? "" : getAddresses(addressList.flatten()); + } +} diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 9687df82d1..f428b455cf 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -29,6 +29,7 @@ import java.util.Set; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.james.mime4j.MimeException; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; @@ -113,12 +114,15 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { // check its signature boolean isMbox = false; + boolean isEMLFile = false; + try { byte[] t = new byte[64]; if (abstractFile.getSize() > 64) { int byteRead = abstractFile.read(t, 0, 64); if (byteRead > 0) { isMbox = MboxParser.isValidMimeTypeMbox(t); + isEMLFile = EMLParser.isEMLFile(abstractFile, t); } } } catch (TskException ex) { @@ -128,6 +132,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { if (isMbox) { return processMBox(abstractFile); } + + if (isEMLFile) { + return processEMLFile(abstractFile); + } if (PstParser.isPstFile(abstractFile)) { return processPst(abstractFile); @@ -310,46 +318,47 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { "ThunderbirdMboxFileIngestModule.errorMessage.outOfDiskSpace=Out of disk space. Cannot copy '{0}' (id={1}) to parse." }) private ProcessResult processVcard(AbstractFile abstractFile) { - String fileName; - try { - fileName = getTempPath() + File.separator + abstractFile.getName() - + "-" + String.valueOf(abstractFile.getId()); - } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS - return ProcessResult.ERROR; - } - File file = new File(fileName); - - long freeSpace = services.getFreeDiskSpace(); - if ((freeSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN) && (abstractFile.getSize() >= freeSpace)) { - logger.log(Level.WARNING, String.format("Not enough disk space to write file '%s' (id=%d) to disk.", - abstractFile.getName(), abstractFile.getId())); //NON-NLS - IngestMessage msg = IngestMessage.createErrorMessage(EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleName(), - Bundle.ThunderbirdMboxFileIngestModule_errorMessage_outOfDiskSpace(abstractFile.getName(), abstractFile.getId())); - services.postMessage(msg); - return ProcessResult.OK; - } - - try { - ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); - } catch (IOException ex) { - logger.log(Level.WARNING, String.format("Failed writing the vCard file '%s' (id=%d) to disk.", - abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS - return ProcessResult.OK; - } - try { VcardParser parser = new VcardParser(currentCase, context); - parser.parse(file, abstractFile); + parser.parse(abstractFile); } catch (IOException | NoCurrentCaseException ex) { - logger.log(Level.WARNING, String.format("Exception while parsing the file '%s' (id=%d).", file.getName(), abstractFile.getId()), ex); //NON-NLS + logger.log(Level.WARNING, String.format("Exception while parsing the file '%s' (id=%d).", abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS return ProcessResult.OK; } - - if (file.delete() == false) { - logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS + return ProcessResult.OK; + } + + private ProcessResult processEMLFile(AbstractFile abstractFile) { + try { + EmailMessage message = EMLParser.parse(abstractFile); + + if (message == null) { + return ProcessResult.OK; + } + + List derivedFiles = new ArrayList<>(); + + BlackboardArtifact msgArtifact = addEmailArtifact(message, abstractFile); + + if ((msgArtifact != null) && (message.hasAttachment())) { + derivedFiles.addAll(handleAttachments(message.getAttachments(), abstractFile, msgArtifact)); + } + + if (derivedFiles.isEmpty() == false) { + for (AbstractFile derived : derivedFiles) { + services.fireModuleContentEvent(new ModuleContentEvent(derived)); + } + } + context.addFilesToJob(derivedFiles); + + } catch (IOException ex) { + logger.log(Level.WARNING, String.format("Error reading eml file %s", abstractFile.getName()), ex); + return ProcessResult.ERROR; + } catch (MimeException ex) { + logger.log(Level.WARNING, String.format("Error reading eml file %s", abstractFile.getName()), ex); + return ProcessResult.ERROR; } - + return ProcessResult.OK; } @@ -580,7 +589,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { addArtifactAttribute(((id < 0L) ? NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.notAvail") : String.valueOf(id)), ATTRIBUTE_TYPE.TSK_MSG_ID, bbattributes); - addArtifactAttribute(((localPath.isEmpty() == false) ? localPath : "/foo/bar"), + addArtifactAttribute(((localPath.isEmpty() == false) ? localPath : ""), ATTRIBUTE_TYPE.TSK_PATH, bbattributes); addArtifactAttribute(cc, ATTRIBUTE_TYPE.TSK_EMAIL_CC, bbattributes); diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java index facbce4794..84f4cd92c8 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java @@ -57,6 +57,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataSource; +import org.sleuthkit.datamodel.ReadContentInputStream; import org.sleuthkit.datamodel.Relationship; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -141,8 +142,8 @@ final class VcardParser { * file. * @throws NoCurrentCaseException If there is no open case. */ - void parse(File vcardFile, AbstractFile abstractFile) throws IOException, NoCurrentCaseException { - for (VCard vcard: Ezvcard.parse(vcardFile).all()) { + void parse(AbstractFile abstractFile) throws IOException, NoCurrentCaseException { + for (VCard vcard: Ezvcard.parse(new ReadContentInputStream(abstractFile)).all()) { addContactArtifact(vcard, abstractFile); } }