diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java index c44289491e..34ebfac8a6 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java @@ -290,6 +290,62 @@ public final class ContentUtils { return totalRead; } + /** + * Reads all the data from any content object and writes (extracts) it to a + * file, using a cancellation check instead of a Future object method. + * + * @param content Any content object. + * @param outputFile Will be created if it doesn't exist, and overwritten + * if it does + * @param cancelCheck A function used to check if the file write process + * should be terminated. + * @param startingOffset the starting offset to start reading the file + * @param endingOffset the ending offset to read of the file to write + * + * @return number of bytes extracted + * + * @throws IOException if file could not be written + */ + public static long writeToFile(Content content, java.io.File outputFile, + Supplier cancelCheck, long startingOffset, long endingOffset) throws IOException { + + InputStream in = new ReadContentInputStream(content); + long totalRead = 0; + try (FileOutputStream out = new FileOutputStream(outputFile, false)) { + long offsetSkipped = in.skip(startingOffset); + if (offsetSkipped != startingOffset) { + in.close(); + throw new IOException(String.format("Skipping file to starting offset {0} was not successful only skipped to offset {1}.", startingOffset, offsetSkipped)); + } + byte[] buffer = new byte[TO_FILE_BUFFER_SIZE]; + int len = in.read(buffer); + long writeFileLength = endingOffset - startingOffset; + writeFileLength = writeFileLength - TO_FILE_BUFFER_SIZE; + while (len != -1 && writeFileLength != 0) { + out.write(buffer, 0, len); + totalRead += len; + if (cancelCheck.get()) { + break; + } + if (writeFileLength > TO_FILE_BUFFER_SIZE) { + len = in.read(buffer); + writeFileLength = writeFileLength - TO_FILE_BUFFER_SIZE; + } else { + int writeLength = (int)writeFileLength; + byte[] lastBuffer = new byte[writeLength]; + len = in.read(lastBuffer); + out.write(lastBuffer, 0, len); + totalRead += len; + writeFileLength = 0; + } + } + + } finally { + in.close(); + } + return totalRead; + } + /** * Helper to ignore the '.' and '..' directories * diff --git a/thunderbirdparser/nbproject/project.xml b/thunderbirdparser/nbproject/project.xml index 29782066ac..5f285136a0 100644 --- a/thunderbirdparser/nbproject/project.xml +++ b/thunderbirdparser/nbproject/project.xml @@ -6,6 +6,15 @@ org.sleuthkit.autopsy.thunderbirdparser + + org.netbeans.api.progress + + + + 1 + 1.47.1 + + org.openide.util diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index e9db7287e8..0f9759819f 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -54,6 +54,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.DerivedFile; +import org.sleuthkit.datamodel.ReadContentInputStream; import org.sleuthkit.datamodel.Relationship; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -76,6 +77,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { private Blackboard blackboard; private CommunicationArtifactsHelper communicationArtifactsHelper; + private static final int MBOX_SIZE_TO_SPLIT = 1048576000; private Case currentCase; /** @@ -309,12 +311,75 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return ProcessResult.OK; } - try { - ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); - } catch (IOException ex) { - logger.log(Level.WARNING, "Failed writing mbox file to disk.", ex); //NON-NLS - return ProcessResult.OK; + if (abstractFile.getSize() < MBOX_SIZE_TO_SPLIT) { + + try { + ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); + } catch (IOException ex) { + logger.log(Level.WARNING, "Failed writing mbox file to disk.", ex); //NON-NLS + return ProcessResult.OK; + } + + processMboxFile(file, abstractFile, emailFolder); + + if (file.delete() == false) { + logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS + } + } else { + + List mboxSplitOffsets = new ArrayList<>(); + try{ + mboxSplitOffsets = findMboxSplitOffset(abstractFile, file); + } catch (IOException ex) { + logger.log(Level.WARNING, String.format("Failed finding split offsets for mbox file {0}.", fileName), ex); //NON-NLS + return ProcessResult.OK; + } + + long startingOffset = 0; + for (Long mboxSplitOffset : mboxSplitOffsets) { + File splitFile = new File(fileName + "-" + mboxSplitOffset); + try { + ContentUtils.writeToFile(abstractFile, splitFile, context::fileIngestIsCancelled, startingOffset, mboxSplitOffset); + } catch (IOException ex) { + logger.log(Level.WARNING, "Failed writing split mbox file to disk.", ex); //NON-NLS + return ProcessResult.OK; + } + processMboxFile(splitFile, abstractFile, emailFolder); + startingOffset = mboxSplitOffset; + if (splitFile.delete() == false) { + logger.log(Level.INFO, "Failed to delete temp file: {0}", splitFile); //NON-NLS + } + + } + } + + return ProcessResult.OK; + } + + private List findMboxSplitOffset(AbstractFile abstractFile, File file) throws IOException { + + List mboxSplitOffset = new ArrayList<>(); + + byte[] buffer = new byte[7]; + ReadContentInputStream in = new ReadContentInputStream(abstractFile); + in.skip(MBOX_SIZE_TO_SPLIT); + int len = in.read(buffer); + while (len != -1) { + len = in.read(buffer); + if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114 && + buffer[4] == 111 && buffer[5] == 109 && buffer[6] == 32) { + mboxSplitOffset.add(in.getCurPosition() - 5 ); + in.skip(MBOX_SIZE_TO_SPLIT); + } } + + return mboxSplitOffset; + + } + + + private void processMboxFile(File file, AbstractFile abstractFile, String emailFolder) { + MboxParser emailIterator = MboxParser.getEmailIterator( emailFolder, file, abstractFile.getId()); List emails = new ArrayList<>(); @@ -325,7 +390,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { emails.add(emailMessage); } } - + String errors = emailIterator.getErrors(); if (!errors.isEmpty()) { postErrorMessage( @@ -335,11 +400,6 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } processEmails(emails, MboxParser.getEmailIterator( emailFolder, file, abstractFile.getId()), abstractFile); - if (file.delete() == false) { - logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS - } - - return ProcessResult.OK; } /** @@ -755,4 +815,5 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { public void shutDown() { // nothing to shut down } + }