From aee5f25ccb7a8e925ffe26c11710357b7ac1bb6e Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Thu, 11 Jun 2020 21:35:41 -0400 Subject: [PATCH 01/13] Update ThunderbirdMboxFileIngestModule.java FInd offsets to break up mbox file. --- .../ThunderbirdMboxFileIngestModule.java | 131 ++++++++++++++++-- 1 file changed, 122 insertions(+), 9 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index e9db7287e8..0c63dbdc61 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -19,7 +19,9 @@ package org.sleuthkit.autopsy.thunderbirdparser; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -53,7 +55,9 @@ import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; +import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DerivedFile; +import org.sleuthkit.datamodel.ReadContentInputStream; import org.sleuthkit.datamodel.Relationship; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -76,6 +80,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { private Blackboard blackboard; private CommunicationArtifactsHelper communicationArtifactsHelper; + private static final int MBOX_SIZE_TO_SPLIT = 104857600; + private static final int TO_FILE_BUFFER_SIZE = 8192; +// private static final int MBOX_SIZE_TO_SPLIT = 2099751000; private Case currentCase; /** @@ -309,12 +316,72 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return ProcessResult.OK; } + if (abstractFile.getSize() < MBOX_SIZE_TO_SPLIT) { + + try { + ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); + } catch (IOException ex) { + logger.log(Level.WARNING, "Failed writing mbox file to disk.", ex); //NON-NLS + return ProcessResult.OK; + } + + processMboxFile(file, abstractFile, emailFolder); + + if (file.delete() == false) { + logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS + } + } else { + long startingOffset = 0; + List mboxSplitOffsets = findMboxSplitOffset(abstractFile, file); + for (Long mboxSplitOffset : mboxSplitOffsets) { + try { + writeToFile(abstractFile, file, context::fileIngestIsCancelled, startingOffset, mboxSplitOffset); + startingOffset = mboxSplitOffset; + } catch (IOException ex) { + logger.log(Level.WARNING, "Failed writing mbox file to disk.", ex); //NON-NLS + return ProcessResult.OK; + } + } + } + + return ProcessResult.OK; + } + + private List findMboxSplitOffset(AbstractFile abstractFile, File file) { + + List mboxSplitOffset = new ArrayList<>(); + long currentPos = 0; + +// try { +// ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); +// } catch (IOException ex) { +// logger.log(Level.WARNING, "Failed writing mbox file to disk.", ex); //NON-NLS +// return ProcessResult.OK; +// } try { - ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); + byte[] buffer = new byte[4]; + ReadContentInputStream in = new ReadContentInputStream(abstractFile); + long newPosition = in.skip(MBOX_SIZE_TO_SPLIT); + int len = in.read(buffer); + while (len != -1) { + len = in.read(buffer); + if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114) { + currentPos = in.getCurPosition() - 2; + mboxSplitOffset.add(currentPos); + newPosition = in.skip(MBOX_SIZE_TO_SPLIT + currentPos); + } + } } catch (IOException ex) { - logger.log(Level.WARNING, "Failed writing mbox file to disk.", ex); //NON-NLS - return ProcessResult.OK; + logger.log(Level.WARNING, "Failed writing mbox file to disk.", ex); //NON-NLS } + return mboxSplitOffset; + + + } + + + private void processMboxFile(File file, AbstractFile abstractFile, String emailFolder) { + MboxParser emailIterator = MboxParser.getEmailIterator( emailFolder, file, abstractFile.getId()); List emails = new ArrayList<>(); @@ -325,7 +392,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { emails.add(emailMessage); } } - + String errors = emailIterator.getErrors(); if (!errors.isEmpty()) { postErrorMessage( @@ -335,11 +402,6 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } processEmails(emails, MboxParser.getEmailIterator( emailFolder, file, abstractFile.getId()), abstractFile); - if (file.delete() == false) { - logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS - } - - return ProcessResult.OK; } /** @@ -755,4 +817,55 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { public void shutDown() { // nothing to shut down } + + /** + * Reads all the data from any content object and writes (extracts) it to a + * file, using a cancellation check instead of a Future object method. + * + * @param content Any content object. + * @param outputFile Will be created if it doesn't exist, and overwritten + * if it does + * @param cancelCheck A function used to check if the file write process + * should be terminated. + * @param startingOffset the starting offset to start reading the file + * @param endingOffset the ending offset to read of the file to write + * + * @return number of bytes extracted + * + * @throws IOException if file could not be written + */ + public static long writeToFile(Content content, java.io.File outputFile, + Supplier cancelCheck, long startingOffset, long endingOffset) throws IOException { + + long writeFileLength = endingOffset - startingOffset; + InputStream in = new ReadContentInputStream(content); + long totalRead = 0; + long newPosition = in.skip(startingOffset); + try (FileOutputStream out = new FileOutputStream(outputFile, false)) { + byte[] buffer = new byte[TO_FILE_BUFFER_SIZE]; + int len = in.read(buffer); + writeFileLength = writeFileLength - TO_FILE_BUFFER_SIZE; + while (len != -1 && writeFileLength == 0) { + out.write(buffer, 0, len); + totalRead += len; + if (cancelCheck.get()) { + break; + } + if (writeFileLength > TO_FILE_BUFFER_SIZE) { + len = in.read(buffer); + writeFileLength = writeFileLength - TO_FILE_BUFFER_SIZE; + } else { + int fileOffset = (int)(endingOffset - writeFileLength); + int writeLength = (int)writeFileLength; + len = in.read(buffer, fileOffset, writeLength); + writeFileLength = 0; + } + } + + } finally { + in.close(); + } + return totalRead; + } + } From d5d5825909429ad51d74d7ff432d353a036d3bf5 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Mon, 15 Jun 2020 16:44:19 -0400 Subject: [PATCH 02/13] Update ThunderbirdMboxFileIngestModule.java Initial commit of code for spliting large mbox files. --- .../ThunderbirdMboxFileIngestModule.java | 66 ++++++++++--------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 0c63dbdc61..555163e318 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -29,6 +29,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; +import java.util.function.Supplier; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -331,49 +332,52 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS } } else { - long startingOffset = 0; - List mboxSplitOffsets = findMboxSplitOffset(abstractFile, file); + + List mboxSplitOffsets = new ArrayList<>(); + try{ + mboxSplitOffsets = findMboxSplitOffset(abstractFile, file); + } catch (IOException ex) { + logger.log(Level.WARNING, String.format("Failed finding split offsets for mbox file {0}.", fileName), ex); //NON-NLS + return ProcessResult.OK; + } + + long startingOffset = 0; for (Long mboxSplitOffset : mboxSplitOffsets) { + File splitFile = new File(fileName + "-" + String.valueOf(mboxSplitOffset)); try { - writeToFile(abstractFile, file, context::fileIngestIsCancelled, startingOffset, mboxSplitOffset); - startingOffset = mboxSplitOffset; + writeToFile(abstractFile, splitFile, context::fileIngestIsCancelled, startingOffset, mboxSplitOffset); } catch (IOException ex) { - logger.log(Level.WARNING, "Failed writing mbox file to disk.", ex); //NON-NLS + logger.log(Level.WARNING, "Failed writing split mbox file to disk.", ex); //NON-NLS return ProcessResult.OK; } + processMboxFile(splitFile, abstractFile, emailFolder); + startingOffset = mboxSplitOffset; + if (file.delete() == false) { + logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS + } + } } return ProcessResult.OK; } - private List findMboxSplitOffset(AbstractFile abstractFile, File file) { + private List findMboxSplitOffset(AbstractFile abstractFile, File file) throws IOException { List mboxSplitOffset = new ArrayList<>(); - long currentPos = 0; -// try { -// ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); -// } catch (IOException ex) { -// logger.log(Level.WARNING, "Failed writing mbox file to disk.", ex); //NON-NLS -// return ProcessResult.OK; -// } - try { - byte[] buffer = new byte[4]; - ReadContentInputStream in = new ReadContentInputStream(abstractFile); - long newPosition = in.skip(MBOX_SIZE_TO_SPLIT); - int len = in.read(buffer); - while (len != -1) { - len = in.read(buffer); - if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114) { - currentPos = in.getCurPosition() - 2; - mboxSplitOffset.add(currentPos); - newPosition = in.skip(MBOX_SIZE_TO_SPLIT + currentPos); - } + byte[] buffer = new byte[4]; + ReadContentInputStream in = new ReadContentInputStream(abstractFile); + long newPosition = in.skip(MBOX_SIZE_TO_SPLIT); + int len = in.read(buffer); + while (len != -1) { + len = in.read(buffer); + if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114) { + mboxSplitOffset.add(in.getCurPosition() - 2); + newPosition = in.skip(MBOX_SIZE_TO_SPLIT); } - } catch (IOException ex) { - logger.log(Level.WARNING, "Failed writing mbox file to disk.", ex); //NON-NLS } + return mboxSplitOffset; @@ -845,7 +849,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { byte[] buffer = new byte[TO_FILE_BUFFER_SIZE]; int len = in.read(buffer); writeFileLength = writeFileLength - TO_FILE_BUFFER_SIZE; - while (len != -1 && writeFileLength == 0) { + while (len != -1 && writeFileLength != 0) { out.write(buffer, 0, len); totalRead += len; if (cancelCheck.get()) { @@ -855,9 +859,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { len = in.read(buffer); writeFileLength = writeFileLength - TO_FILE_BUFFER_SIZE; } else { - int fileOffset = (int)(endingOffset - writeFileLength); int writeLength = (int)writeFileLength; - len = in.read(buffer, fileOffset, writeLength); + byte[] lastBuffer = new byte[writeLength]; + len = in.read(lastBuffer); + out.write(lastBuffer, 0, len); + totalRead += len; writeFileLength = 0; } } From 45ac02a781647da2b2b541eaee294da77f23ca98 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Mon, 15 Jun 2020 16:46:31 -0400 Subject: [PATCH 03/13] Update ThunderbirdMboxFileIngestModule.java Update split size for testing --- .../thunderbirdparser/ThunderbirdMboxFileIngestModule.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 555163e318..619a0a3f2c 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -81,7 +81,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { private Blackboard blackboard; private CommunicationArtifactsHelper communicationArtifactsHelper; - private static final int MBOX_SIZE_TO_SPLIT = 104857600; + private static final int MBOX_SIZE_TO_SPLIT = 1048576000; private static final int TO_FILE_BUFFER_SIZE = 8192; // private static final int MBOX_SIZE_TO_SPLIT = 2099751000; private Case currentCase; @@ -379,8 +379,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } return mboxSplitOffset; - - + } From f41f5026f31ee4e61c8446e21b0c0934ae1d7615 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Tue, 16 Jun 2020 10:49:16 -0400 Subject: [PATCH 04/13] Update ThunderbirdMboxFileIngestModule.java Move writeToFile to ContentUtils and add buffer check to find end of email's --- .../ThunderbirdMboxFileIngestModule.java | 66 +++---------------- 1 file changed, 9 insertions(+), 57 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 619a0a3f2c..a5ab375638 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -29,7 +29,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; -import java.util.function.Supplier; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -345,7 +344,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { for (Long mboxSplitOffset : mboxSplitOffsets) { File splitFile = new File(fileName + "-" + String.valueOf(mboxSplitOffset)); try { - writeToFile(abstractFile, splitFile, context::fileIngestIsCancelled, startingOffset, mboxSplitOffset); + ContentUtils.writeToFile(abstractFile, splitFile, context::fileIngestIsCancelled, startingOffset, mboxSplitOffset); } catch (IOException ex) { logger.log(Level.WARNING, "Failed writing split mbox file to disk.", ex); //NON-NLS return ProcessResult.OK; @@ -365,17 +364,22 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { private List findMboxSplitOffset(AbstractFile abstractFile, File file) throws IOException { List mboxSplitOffset = new ArrayList<>(); + logger.log(Level.WARNING, String.format("starting find split")); //NON-NLS - byte[] buffer = new byte[4]; + byte[] buffer = new byte[7]; ReadContentInputStream in = new ReadContentInputStream(abstractFile); long newPosition = in.skip(MBOX_SIZE_TO_SPLIT); int len = in.read(buffer); while (len != -1) { len = in.read(buffer); - if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114) { - mboxSplitOffset.add(in.getCurPosition() - 2); + if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114 && + buffer[4] == 111 && buffer[5] == 109 && buffer[6] == 32) { + logger.log(Level.WARNING, String.format("found Offset")); //NON-NLS + + mboxSplitOffset.add(in.getCurPosition() - 5 ); newPosition = in.skip(MBOX_SIZE_TO_SPLIT); } +// in.skip(-5); } return mboxSplitOffset; @@ -821,56 +825,4 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { // nothing to shut down } - /** - * Reads all the data from any content object and writes (extracts) it to a - * file, using a cancellation check instead of a Future object method. - * - * @param content Any content object. - * @param outputFile Will be created if it doesn't exist, and overwritten - * if it does - * @param cancelCheck A function used to check if the file write process - * should be terminated. - * @param startingOffset the starting offset to start reading the file - * @param endingOffset the ending offset to read of the file to write - * - * @return number of bytes extracted - * - * @throws IOException if file could not be written - */ - public static long writeToFile(Content content, java.io.File outputFile, - Supplier cancelCheck, long startingOffset, long endingOffset) throws IOException { - - long writeFileLength = endingOffset - startingOffset; - InputStream in = new ReadContentInputStream(content); - long totalRead = 0; - long newPosition = in.skip(startingOffset); - try (FileOutputStream out = new FileOutputStream(outputFile, false)) { - byte[] buffer = new byte[TO_FILE_BUFFER_SIZE]; - int len = in.read(buffer); - writeFileLength = writeFileLength - TO_FILE_BUFFER_SIZE; - while (len != -1 && writeFileLength != 0) { - out.write(buffer, 0, len); - totalRead += len; - if (cancelCheck.get()) { - break; - } - if (writeFileLength > TO_FILE_BUFFER_SIZE) { - len = in.read(buffer); - writeFileLength = writeFileLength - TO_FILE_BUFFER_SIZE; - } else { - int writeLength = (int)writeFileLength; - byte[] lastBuffer = new byte[writeLength]; - len = in.read(lastBuffer); - out.write(lastBuffer, 0, len); - totalRead += len; - writeFileLength = 0; - } - } - - } finally { - in.close(); - } - return totalRead; - } - } From b55305b7e45c6b3466e0e7ffe2eb4a77fcf9380f Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Tue, 16 Jun 2020 10:50:00 -0400 Subject: [PATCH 05/13] Update ContentUtils.java Add new writeToFile to process a file with start and stop offsets --- .../autopsy/datamodel/ContentUtils.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java index c44289491e..1164ead6f8 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java @@ -290,6 +290,58 @@ public final class ContentUtils { return totalRead; } + /** + * Reads all the data from any content object and writes (extracts) it to a + * file, using a cancellation check instead of a Future object method. + * + * @param content Any content object. + * @param outputFile Will be created if it doesn't exist, and overwritten + * if it does + * @param cancelCheck A function used to check if the file write process + * should be terminated. + * @param startingOffset the starting offset to start reading the file + * @param endingOffset the ending offset to read of the file to write + * + * @return number of bytes extracted + * + * @throws IOException if file could not be written + */ + public static long writeToFile(Content content, java.io.File outputFile, + Supplier cancelCheck, long startingOffset, long endingOffset) throws IOException { + + long writeFileLength = endingOffset - startingOffset; + InputStream in = new ReadContentInputStream(content); + long totalRead = 0; + long newPosition = in.skip(startingOffset); + try (FileOutputStream out = new FileOutputStream(outputFile, false)) { + byte[] buffer = new byte[TO_FILE_BUFFER_SIZE]; + int len = in.read(buffer); + writeFileLength = writeFileLength - TO_FILE_BUFFER_SIZE; + while (len != -1 && writeFileLength != 0) { + out.write(buffer, 0, len); + totalRead += len; + if (cancelCheck.get()) { + break; + } + if (writeFileLength > TO_FILE_BUFFER_SIZE) { + len = in.read(buffer); + writeFileLength = writeFileLength - TO_FILE_BUFFER_SIZE; + } else { + int writeLength = (int)writeFileLength; + byte[] lastBuffer = new byte[writeLength]; + len = in.read(lastBuffer); + out.write(lastBuffer, 0, len); + totalRead += len; + writeFileLength = 0; + } + } + + } finally { + in.close(); + } + return totalRead; + } + /** * Helper to ignore the '.' and '..' directories * From 4eb03ccc9b390c5b13b64a74065d795fbfc9d52a Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Tue, 16 Jun 2020 11:19:08 -0400 Subject: [PATCH 06/13] Update ThunderbirdMboxFileIngestModule.java Add message to print offsets for later debugging --- .../thunderbirdparser/ThunderbirdMboxFileIngestModule.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index a5ab375638..506bfd4ded 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -351,7 +351,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } processMboxFile(splitFile, abstractFile, emailFolder); startingOffset = mboxSplitOffset; - if (file.delete() == false) { + if (splitFile.delete() == false) { logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS } @@ -374,12 +374,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { len = in.read(buffer); if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114 && buffer[4] == 111 && buffer[5] == 109 && buffer[6] == 32) { - logger.log(Level.WARNING, String.format("found Offset")); //NON-NLS - + logger.log(Level.WARNING, String.format("found Offset %d", in.getCurPosition())); //NON-NLS mboxSplitOffset.add(in.getCurPosition() - 5 ); newPosition = in.skip(MBOX_SIZE_TO_SPLIT); } -// in.skip(-5); } return mboxSplitOffset; From 0b48cee0260291890b2964964d7f21ec7efdddaf Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Wed, 17 Jun 2020 10:00:09 -0400 Subject: [PATCH 07/13] Update project.xml Added dependency to be able to compile. --- thunderbirdparser/nbproject/project.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/thunderbirdparser/nbproject/project.xml b/thunderbirdparser/nbproject/project.xml index 29782066ac..5f285136a0 100644 --- a/thunderbirdparser/nbproject/project.xml +++ b/thunderbirdparser/nbproject/project.xml @@ -6,6 +6,15 @@ org.sleuthkit.autopsy.thunderbirdparser + + org.netbeans.api.progress + + + + 1 + 1.47.1 + + org.openide.util From b4ebba4cda9b29029ba853babe8e7b0efdf564f1 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Wed, 17 Jun 2020 10:03:58 -0400 Subject: [PATCH 08/13] Update ThunderbirdMboxFileIngestModule.java Update log message --- .../thunderbirdparser/ThunderbirdMboxFileIngestModule.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 506bfd4ded..0342c07a7d 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -352,7 +352,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { processMboxFile(splitFile, abstractFile, emailFolder); startingOffset = mboxSplitOffset; if (splitFile.delete() == false) { - logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS + logger.log(Level.INFO, "Failed to delete temp file: {0}", splitFile); //NON-NLS } } @@ -374,7 +374,6 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { len = in.read(buffer); if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114 && buffer[4] == 111 && buffer[5] == 109 && buffer[6] == 32) { - logger.log(Level.WARNING, String.format("found Offset %d", in.getCurPosition())); //NON-NLS mboxSplitOffset.add(in.getCurPosition() - 5 ); newPosition = in.skip(MBOX_SIZE_TO_SPLIT); } From 5754b1d6772328c28d7879c83201d5451bf4260d Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Wed, 17 Jun 2020 10:21:06 -0400 Subject: [PATCH 09/13] Fix Codacy Fix Codacy issues --- .../org/sleuthkit/autopsy/datamodel/ContentUtils.java | 2 +- .../ThunderbirdMboxFileIngestModule.java | 11 +++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java index 1164ead6f8..cfe89fde92 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java @@ -312,7 +312,7 @@ public final class ContentUtils { long writeFileLength = endingOffset - startingOffset; InputStream in = new ReadContentInputStream(content); long totalRead = 0; - long newPosition = in.skip(startingOffset); + in.skip(startingOffset); try (FileOutputStream out = new FileOutputStream(outputFile, false)) { byte[] buffer = new byte[TO_FILE_BUFFER_SIZE]; int len = in.read(buffer); diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 0342c07a7d..934279443b 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -19,9 +19,7 @@ package org.sleuthkit.autopsy.thunderbirdparser; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -55,7 +53,6 @@ import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; -import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DerivedFile; import org.sleuthkit.datamodel.ReadContentInputStream; import org.sleuthkit.datamodel.Relationship; @@ -81,8 +78,6 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { private CommunicationArtifactsHelper communicationArtifactsHelper; private static final int MBOX_SIZE_TO_SPLIT = 1048576000; - private static final int TO_FILE_BUFFER_SIZE = 8192; -// private static final int MBOX_SIZE_TO_SPLIT = 2099751000; private Case currentCase; /** @@ -342,7 +337,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { long startingOffset = 0; for (Long mboxSplitOffset : mboxSplitOffsets) { - File splitFile = new File(fileName + "-" + String.valueOf(mboxSplitOffset)); + File splitFile = new File(fileName + "-" + mboxSplitOffset); try { ContentUtils.writeToFile(abstractFile, splitFile, context::fileIngestIsCancelled, startingOffset, mboxSplitOffset); } catch (IOException ex) { @@ -368,14 +363,14 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { byte[] buffer = new byte[7]; ReadContentInputStream in = new ReadContentInputStream(abstractFile); - long newPosition = in.skip(MBOX_SIZE_TO_SPLIT); + in.skip(MBOX_SIZE_TO_SPLIT); int len = in.read(buffer); while (len != -1) { len = in.read(buffer); if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114 && buffer[4] == 111 && buffer[5] == 109 && buffer[6] == 32) { mboxSplitOffset.add(in.getCurPosition() - 5 ); - newPosition = in.skip(MBOX_SIZE_TO_SPLIT); + in.skip(MBOX_SIZE_TO_SPLIT); } } From 91b3a9460bebbfa2e1f289a300b0808e7430e64e Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Wed, 17 Jun 2020 11:55:16 -0400 Subject: [PATCH 10/13] Codacy and Remove Log message Codacy Fixes and remove unneeded log message. --- Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java | 5 ++++- .../thunderbirdparser/ThunderbirdMboxFileIngestModule.java | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java index cfe89fde92..c341579db2 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java @@ -312,7 +312,10 @@ public final class ContentUtils { long writeFileLength = endingOffset - startingOffset; InputStream in = new ReadContentInputStream(content); long totalRead = 0; - in.skip(startingOffset); + long offsetSkipped = in.skip(startingOffset); + if (offsetSkipped != startingOffset) { + throw new IOException(String.format("Skipping file to starting offset {0} was not successful only skipped to offset {1}.", startingOffset, offsetSkipped)); + } try (FileOutputStream out = new FileOutputStream(outputFile, false)) { byte[] buffer = new byte[TO_FILE_BUFFER_SIZE]; int len = in.read(buffer); diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 934279443b..0f9759819f 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -359,7 +359,6 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { private List findMboxSplitOffset(AbstractFile abstractFile, File file) throws IOException { List mboxSplitOffset = new ArrayList<>(); - logger.log(Level.WARNING, String.format("starting find split")); //NON-NLS byte[] buffer = new byte[7]; ReadContentInputStream in = new ReadContentInputStream(abstractFile); From 547a7afe3816960d92011ba84b068206f7e7cd68 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Wed, 17 Jun 2020 13:46:55 -0400 Subject: [PATCH 11/13] Update ContentUtils.java More codacy happiness. --- Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java index c341579db2..b6bece787f 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java @@ -309,16 +309,16 @@ public final class ContentUtils { public static long writeToFile(Content content, java.io.File outputFile, Supplier cancelCheck, long startingOffset, long endingOffset) throws IOException { - long writeFileLength = endingOffset - startingOffset; InputStream in = new ReadContentInputStream(content); - long totalRead = 0; long offsetSkipped = in.skip(startingOffset); if (offsetSkipped != startingOffset) { throw new IOException(String.format("Skipping file to starting offset {0} was not successful only skipped to offset {1}.", startingOffset, offsetSkipped)); } + long totalRead = 0; try (FileOutputStream out = new FileOutputStream(outputFile, false)) { byte[] buffer = new byte[TO_FILE_BUFFER_SIZE]; int len = in.read(buffer); + long writeFileLength = endingOffset - startingOffset; writeFileLength = writeFileLength - TO_FILE_BUFFER_SIZE; while (len != -1 && writeFileLength != 0) { out.write(buffer, 0, len); From 243134296ef9c43713feee55b5509c8bead3ddc4 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Thu, 18 Jun 2020 10:50:59 -0400 Subject: [PATCH 12/13] Update ContentUtils.java Codacy fix --- Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java index b6bece787f..791064c20c 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java @@ -312,6 +312,7 @@ public final class ContentUtils { InputStream in = new ReadContentInputStream(content); long offsetSkipped = in.skip(startingOffset); if (offsetSkipped != startingOffset) { + in.close(); throw new IOException(String.format("Skipping file to starting offset {0} was not successful only skipped to offset {1}.", startingOffset, offsetSkipped)); } long totalRead = 0; From 03efd9d27c1d1261a5f6bd63968955de3a0acaac Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Thu, 18 Jun 2020 14:40:56 -0400 Subject: [PATCH 13/13] Update ContentUtils.java One more time for Codacy fix --- .../org/sleuthkit/autopsy/datamodel/ContentUtils.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java index 791064c20c..34ebfac8a6 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java @@ -310,13 +310,13 @@ public final class ContentUtils { Supplier cancelCheck, long startingOffset, long endingOffset) throws IOException { InputStream in = new ReadContentInputStream(content); - long offsetSkipped = in.skip(startingOffset); - if (offsetSkipped != startingOffset) { - in.close(); - throw new IOException(String.format("Skipping file to starting offset {0} was not successful only skipped to offset {1}.", startingOffset, offsetSkipped)); - } long totalRead = 0; try (FileOutputStream out = new FileOutputStream(outputFile, false)) { + long offsetSkipped = in.skip(startingOffset); + if (offsetSkipped != startingOffset) { + in.close(); + throw new IOException(String.format("Skipping file to starting offset {0} was not successful only skipped to offset {1}.", startingOffset, offsetSkipped)); + } byte[] buffer = new byte[TO_FILE_BUFFER_SIZE]; int len = in.read(buffer); long writeFileLength = endingOffset - startingOffset;