From 30a84fdceb14f8440aec15a3a37cd166d47ce8a3 Mon Sep 17 00:00:00 2001 From: sidheshenator Date: Tue, 19 May 2015 13:15:07 -0400 Subject: [PATCH 1/7] getFileType() added and used --- .../exif/ExifParserFileIngestModule.java | 10 +++- .../modules/filetypeid/FileTypeDetector.java | 46 +++++++++++++++++++ .../sevenzip/SevenZipIngestModule.java | 31 ++++--------- .../KeywordSearchIngestModule.java | 34 ++++---------- 4 files changed, 74 insertions(+), 47 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java index a045f48a5e..7cc563a7a2 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java @@ -34,6 +34,7 @@ import java.util.Collection; import java.util.Date; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; +import org.openide.util.Exceptions; import org.sleuthkit.autopsy.coreutils.ImageUtils; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.FileIngestModule; @@ -41,6 +42,7 @@ import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; +import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -63,6 +65,7 @@ public final class ExifParserFileIngestModule implements FileIngestModule { private volatile boolean filesToFire = false; private long jobId; private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); + private static FileTypeDetector fileTypeDetector; ExifParserFileIngestModule() { } @@ -71,6 +74,11 @@ public final class ExifParserFileIngestModule implements FileIngestModule { public void startUp(IngestJobContext context) throws IngestModuleException { jobId = context.getJobId(); refCounter.incrementAndGet(jobId); + try { + fileTypeDetector = new FileTypeDetector(); + } catch (FileTypeDetector.FileTypeDetectorInitException ex) { + logger.log(Level.WARNING, "Error initializing FileTypeDetector", ex); // NON-NLS + } } @@ -197,7 +205,7 @@ public final class ExifParserFileIngestModule implements FileIngestModule { * @return true if to be processed */ private boolean parsableFormat(AbstractFile f) { - return ImageUtils.isJpegFileHeader(f); + return fileTypeDetector.getFileType(f).equals("image/jpeg"); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java index bd7418d2a2..08631210ec 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java @@ -18,11 +18,14 @@ */ package org.sleuthkit.autopsy.modules.filetypeid; +import java.util.ArrayList; import java.util.Map; import java.util.SortedSet; +import java.util.logging.Level; import org.apache.tika.Tika; import org.apache.tika.mime.MediaType; import org.apache.tika.mime.MimeTypes; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -37,6 +40,7 @@ public class FileTypeDetector { private static final int BUFFER_SIZE = 64 * 1024; private final byte buffer[] = new byte[BUFFER_SIZE]; private final Map userDefinedFileTypes; + private static final Logger logger = Logger.getLogger(FileTypeDetector.class.getName()); /** * Constructs an object that detects the type of a file by an inspection of @@ -93,6 +97,48 @@ public class FileTypeDetector { return false; } + /** + * This method returns a string representing the mimetype of the provided + * abstractFile. Blackboard-lookup is performed to check if the mimetype has + * been already detected. If not, mimetype is determined using Apache Tika. + * + * @param abstractFile the file whose mimetype is to be determined. + * @return mimetype of the abstractFile is returned. Null value returned in + * case of error. + */ + public synchronized String getFileType(AbstractFile abstractFile) { + String identifiedFileType = null; + + // check BB + ArrayList attributes = null; + try { + attributes = abstractFile.getGenInfoAttributes(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FILE_TYPE_SIG); + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Error performing mimetype blackboard-lookup for " + abstractFile.getName(), ex); + } + for (BlackboardAttribute attribute : attributes) { + identifiedFileType = attribute.getValueString(); + break; + } + + if (identifiedFileType != null) { + return identifiedFileType; + } + + try { + // check UDF and TDF + identifiedFileType = detectAndPostToBlackboard(abstractFile); + if (identifiedFileType != null) { + return identifiedFileType; + } + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Error determining the mimetype for " + abstractFile.getName(), ex); + return null; + } + + return null; + } + /** * Detect the MIME type of a file, posting it to the blackboard if detection * succeeds. diff --git a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java index 9ccb53ae17..70413e532a 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java @@ -62,6 +62,7 @@ import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import net.sf.sevenzipjbinding.ArchiveFormat; import static net.sf.sevenzipjbinding.ArchiveFormat.RAR; +import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; /** * 7Zip ingest module extracts supported archives, adds extracted DerivedFiles, @@ -87,13 +88,10 @@ public final class SevenZipIngestModule implements FileIngestModule { private static final long MIN_FREE_DISK_SPACE = 1 * 1000 * 1000000L; //1GB //counts archive depth private ArchiveDepthCountTree archiveDepthCountTree; - //buffer for checking file headers and signatures - private static final int readHeaderSize = 4; - private final byte[] fileHeaderBuffer = new byte[readHeaderSize]; - private static final int ZIP_SIGNATURE_BE = 0x504B0304; private IngestJobContext context; private long jobId; private final static IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); + private static FileTypeDetector fileTypeDetector; SevenZipIngestModule() { } @@ -103,6 +101,12 @@ public final class SevenZipIngestModule implements FileIngestModule { this.context = context; jobId = context.getJobId(); + try { + fileTypeDetector = new FileTypeDetector(); + } catch (FileTypeDetector.FileTypeDetectorInitException ex) { + logger.log(Level.WARNING, "Error initializing FileTypeDetector", ex); // NON-NLS + } + final Case currentCase = Case.getCurrentCase(); moduleDirRelative = Case.getModulesOutputDirRelPath() + File.separator + ArchiveFileExtractorModuleFactory.getModuleName(); @@ -657,24 +661,7 @@ public final class SevenZipIngestModule implements FileIngestModule { * @return true if zip file, false otherwise */ private boolean isZipFileHeader(AbstractFile file) { - if (file.getSize() < readHeaderSize) { - return false; - } - - try { - int bytesRead = file.read(fileHeaderBuffer, 0, readHeaderSize); - if (bytesRead != readHeaderSize) { - return false; - } - } catch (TskCoreException ex) { - //ignore if can't read the first few bytes, not a ZIP - return false; - } - - ByteBuffer bytes = ByteBuffer.wrap(fileHeaderBuffer); - int signature = bytes.getInt(); - - return signature == ZIP_SIGNATURE_BE; + return fileTypeDetector.getFileType(file).equals("application/zip"); //NON-NLS } /** diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 80462dd1af..a891f2721b 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -74,7 +74,8 @@ public final class KeywordSearchIngestModule implements FileIngestModule { private final IngestServices services = IngestServices.getInstance(); private Ingester ingester = null; private Indexer indexer; - //only search images from current ingest, not images previously ingested/indexed + private static FileTypeDetector fileTypeDetector; +//only search images from current ingest, not images previously ingested/indexed //accessed read-only by searcher thread private boolean startedSearching = false; @@ -130,6 +131,11 @@ public final class KeywordSearchIngestModule implements FileIngestModule { jobId = context.getJobId(); dataSourceId = context.getDataSource().getId(); + try { + fileTypeDetector = new FileTypeDetector(); + } catch (FileTypeDetector.FileTypeDetectorInitException ex) { + logger.log(Level.WARNING, "Error initializing FileTypeDetector", ex); // NON-NLS + } ingester = Server.getIngester(); this.context = context; @@ -469,30 +475,10 @@ public final class KeywordSearchIngestModule implements FileIngestModule { return; } - - - // try to get the file type from the BB - String detectedFormat = null; - try { - ArrayList attributes = aFile.getGenInfoAttributes(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FILE_TYPE_SIG); - for (BlackboardAttribute attribute : attributes) { - detectedFormat = attribute.getValueString(); - break; - } - } catch (TskCoreException ex) { - } - // else, use FileType module to detect the format + String detectedFormat = fileTypeDetector.getFileType(aFile); if (detectedFormat == null) { - try { - detectedFormat = new FileTypeDetector().detectAndPostToBlackboard(aFile); - } catch (FileTypeDetector.FileTypeDetectorInitException | TskCoreException ex) { - logger.log(Level.WARNING, "Could not detect format using file type detector for file: {0}", aFile); //NON-NLS - return; - } - if (detectedFormat == null) { - logger.log(Level.WARNING, "Could not detect format using file type detector for file: {0}", aFile); //NON-NLS - return; - } + logger.log(Level.WARNING, "Could not detect format using fileTypeDetector for file: {0}", aFile); //NON-NLS + return; } // we skip archive formats that are opened by the archive module. From 715bc3c2cd66a3ccaf4f2e14aaf151c465a74bf6 Mon Sep 17 00:00:00 2001 From: sidheshenator Date: Tue, 19 May 2015 15:56:35 -0400 Subject: [PATCH 2/7] empty string returned in case of error --- .../modules/filetypeid/FileTypeDetector.java | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java index 08631210ec..d09c6a9ea0 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java @@ -103,40 +103,39 @@ public class FileTypeDetector { * been already detected. If not, mimetype is determined using Apache Tika. * * @param abstractFile the file whose mimetype is to be determined. - * @return mimetype of the abstractFile is returned. Null value returned in - * case of error. + * @return mimetype of the abstractFile is returned. Empty String returned + * in case of error. */ public synchronized String getFileType(AbstractFile abstractFile) { - String identifiedFileType = null; + String identifiedFileType = ""; // check BB - ArrayList attributes = null; try { - attributes = abstractFile.getGenInfoAttributes(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FILE_TYPE_SIG); + ArrayList attributes = abstractFile.getGenInfoAttributes(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FILE_TYPE_SIG); + for (BlackboardAttribute attribute : attributes) { + identifiedFileType = attribute.getValueString(); + break; + } + if (identifiedFileType != null && !identifiedFileType.isEmpty()) { + return identifiedFileType; + } } catch (TskCoreException ex) { logger.log(Level.WARNING, "Error performing mimetype blackboard-lookup for " + abstractFile.getName(), ex); } - for (BlackboardAttribute attribute : attributes) { - identifiedFileType = attribute.getValueString(); - break; - } - - if (identifiedFileType != null) { - return identifiedFileType; - } try { // check UDF and TDF identifiedFileType = detectAndPostToBlackboard(abstractFile); - if (identifiedFileType != null) { + if (identifiedFileType != null && !identifiedFileType.isEmpty()) { return identifiedFileType; } } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Error determining the mimetype for " + abstractFile.getName(), ex); - return null; + logger.log(Level.WARNING, "Error determining the mimetype for " + abstractFile.getName(), ex); // NON-NLS + return ""; // NON-NLS } - return null; + logger.log(Level.WARNING, "Unable to determine the mimetype for {0}", abstractFile.getName()); // NON-NLS + return ""; // NON-NLS } /** From d0739092727eb463cde9fa362dbd6a09d8dc85e6 Mon Sep 17 00:00:00 2001 From: sidheshenator Date: Tue, 19 May 2015 16:12:40 -0400 Subject: [PATCH 3/7] IngestModuleException thrown when FileTypeDetector not instantiated --- .../autopsy/modules/exif/ExifParserFileIngestModule.java | 1 + .../sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java | 1 + .../autopsy/keywordsearch/KeywordSearchIngestModule.java | 1 + 3 files changed, 3 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java index 7cc563a7a2..3462e3a146 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java @@ -78,6 +78,7 @@ public final class ExifParserFileIngestModule implements FileIngestModule { fileTypeDetector = new FileTypeDetector(); } catch (FileTypeDetector.FileTypeDetectorInitException ex) { logger.log(Level.WARNING, "Error initializing FileTypeDetector", ex); // NON-NLS + throw new IngestModuleException("Error initializing FileTypeDetector"); // NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java index 70413e532a..6c93b8ebed 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java @@ -105,6 +105,7 @@ public final class SevenZipIngestModule implements FileIngestModule { fileTypeDetector = new FileTypeDetector(); } catch (FileTypeDetector.FileTypeDetectorInitException ex) { logger.log(Level.WARNING, "Error initializing FileTypeDetector", ex); // NON-NLS + throw new IngestModuleException("Error initializing FileTypeDetector"); // NON-NLS } final Case currentCase = Case.getCurrentCase(); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index a891f2721b..56e39927e4 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -135,6 +135,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { fileTypeDetector = new FileTypeDetector(); } catch (FileTypeDetector.FileTypeDetectorInitException ex) { logger.log(Level.WARNING, "Error initializing FileTypeDetector", ex); // NON-NLS + throw new IngestModuleException("Error initializing FileTypeDetector"); // NON-NLS } ingester = Server.getIngester(); this.context = context; From 0046f727328923a2424230557aa87f0d7f083aef Mon Sep 17 00:00:00 2001 From: sidheshenator Date: Tue, 19 May 2015 16:39:42 -0400 Subject: [PATCH 4/7] Appropriate Levels logged. Unused imports removed --- .../autopsy/modules/exif/ExifParserFileIngestModule.java | 4 +--- .../autopsy/modules/sevenzip/SevenZipIngestModule.java | 3 +-- .../autopsy/keywordsearch/KeywordSearchIngestModule.java | 4 +--- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java index 3462e3a146..85d6bf49c2 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java @@ -34,8 +34,6 @@ import java.util.Collection; import java.util.Date; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; -import org.openide.util.Exceptions; -import org.sleuthkit.autopsy.coreutils.ImageUtils; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestJobContext; @@ -77,7 +75,7 @@ public final class ExifParserFileIngestModule implements FileIngestModule { try { fileTypeDetector = new FileTypeDetector(); } catch (FileTypeDetector.FileTypeDetectorInitException ex) { - logger.log(Level.WARNING, "Error initializing FileTypeDetector", ex); // NON-NLS + logger.log(Level.SEVERE, "Error initializing FileTypeDetector", ex); // NON-NLS throw new IngestModuleException("Error initializing FileTypeDetector"); // NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java index 6c93b8ebed..b4277230be 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java @@ -24,7 +24,6 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -104,7 +103,7 @@ public final class SevenZipIngestModule implements FileIngestModule { try { fileTypeDetector = new FileTypeDetector(); } catch (FileTypeDetector.FileTypeDetectorInitException ex) { - logger.log(Level.WARNING, "Error initializing FileTypeDetector", ex); // NON-NLS + logger.log(Level.SEVERE, "Error initializing FileTypeDetector", ex); // NON-NLS throw new IngestModuleException("Error initializing FileTypeDetector"); // NON-NLS } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 40fae2a471..bc87c20564 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -37,8 +37,6 @@ import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.BlackboardAttribute; -import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskData.FileKnown; @@ -134,7 +132,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { try { fileTypeDetector = new FileTypeDetector(); } catch (FileTypeDetector.FileTypeDetectorInitException ex) { - logger.log(Level.WARNING, "Error initializing FileTypeDetector", ex); // NON-NLS + logger.log(Level.SEVERE, "Error initializing FileTypeDetector", ex); // NON-NLS throw new IngestModuleException("Error initializing FileTypeDetector"); // NON-NLS } ingester = Server.getIngester(); From 22b50d55705532d1fd816649773baf0dee6b15fd Mon Sep 17 00:00:00 2001 From: sidheshenator Date: Wed, 20 May 2015 11:54:47 -0400 Subject: [PATCH 5/7] FileTypeDetector made non-static field in ingest modules --- .../autopsy/modules/exif/ExifParserFileIngestModule.java | 2 +- .../autopsy/modules/filetypeid/FileTypeDetector.java | 4 ++-- .../autopsy/modules/sevenzip/SevenZipIngestModule.java | 2 +- .../autopsy/keywordsearch/KeywordSearchIngestModule.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java index 85d6bf49c2..1f7ae3ed9a 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java @@ -63,7 +63,7 @@ public final class ExifParserFileIngestModule implements FileIngestModule { private volatile boolean filesToFire = false; private long jobId; private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); - private static FileTypeDetector fileTypeDetector; + private FileTypeDetector fileTypeDetector; ExifParserFileIngestModule() { } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java index d09c6a9ea0..6e775b93e8 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java @@ -106,7 +106,7 @@ public class FileTypeDetector { * @return mimetype of the abstractFile is returned. Empty String returned * in case of error. */ - public synchronized String getFileType(AbstractFile abstractFile) { + public String getFileType(AbstractFile abstractFile) { String identifiedFileType = ""; // check BB @@ -147,7 +147,7 @@ public class FileTypeDetector { * @return The MIME type name id detection was successful, null otherwise. * @throws TskCoreException if there is an error posting to the blackboard. */ - public synchronized String detectAndPostToBlackboard(AbstractFile file) throws TskCoreException { + public String detectAndPostToBlackboard(AbstractFile file) throws TskCoreException { String mimeType = detect(file); if (null != mimeType) { /** diff --git a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java index b4277230be..c72b5ec4f3 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java @@ -90,7 +90,7 @@ public final class SevenZipIngestModule implements FileIngestModule { private IngestJobContext context; private long jobId; private final static IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); - private static FileTypeDetector fileTypeDetector; + private FileTypeDetector fileTypeDetector; SevenZipIngestModule() { } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index bc87c20564..9bc0a6442b 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -72,7 +72,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { private final IngestServices services = IngestServices.getInstance(); private Ingester ingester = null; private Indexer indexer; - private static FileTypeDetector fileTypeDetector; + private FileTypeDetector fileTypeDetector; //only search images from current ingest, not images previously ingested/indexed //accessed read-only by searcher thread From 34c5a1d6c905c2377b12e713d43735567be6c2c4 Mon Sep 17 00:00:00 2001 From: sidheshenator Date: Fri, 22 May 2015 11:22:49 -0400 Subject: [PATCH 6/7] Resource bundle updated. Unalloc Unused blocks marked as octet-stream --- .../autopsy/modules/exif/Bundle.properties | 1 + .../modules/exif/ExifParserFileIngestModule.java | 5 +++-- .../modules/filetypeid/FileTypeDetector.java | 14 ++++++++++++-- .../modules/filetypeid/FileTypeIdIngestModule.java | 9 --------- .../autopsy/modules/sevenzip/Bundle.properties | 1 + .../modules/sevenzip/SevenZipIngestModule.java | 4 ++-- .../autopsy/keywordsearch/Bundle.properties | 1 + .../keywordsearch/KeywordSearchIngestModule.java | 4 ++-- 8 files changed, 22 insertions(+), 17 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/exif/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/exif/Bundle.properties index 2987fc2ae8..391cfdac7f 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/exif/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/exif/Bundle.properties @@ -6,3 +6,4 @@ OpenIDE-Module-Name=ExifParser OpenIDE-Module-Short-Description=Exif metadata ingest module ExifParserFileIngestModule.moduleName.text=Exif Parser ExifParserFileIngestModule.getDesc.text=Ingests JPEG files and retrieves their EXIF metadata. +ExifParserFileIngestModule.startUp.fileTypeDetectorInitializationException.msg=Error initializing the File Type Detector. \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java index 1f7ae3ed9a..f3f7ea3313 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java @@ -34,6 +34,7 @@ import java.util.Collection; import java.util.Date; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; +import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestJobContext; @@ -75,8 +76,8 @@ public final class ExifParserFileIngestModule implements FileIngestModule { try { fileTypeDetector = new FileTypeDetector(); } catch (FileTypeDetector.FileTypeDetectorInitException ex) { - logger.log(Level.SEVERE, "Error initializing FileTypeDetector", ex); // NON-NLS - throw new IngestModuleException("Error initializing FileTypeDetector"); // NON-NLS + logger.log(Level.SEVERE, NbBundle.getMessage(this.getClass(), "ExifParserFileIngestModule.startUp.fileTypeDetectorInitializationException.msg"), ex); + throw new IngestModuleException(NbBundle.getMessage(this.getClass(), "ExifParserFileIngestModule.startUp.fileTypeDetectorInitializationException.msg")); } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java index 6e775b93e8..4b8b9266e8 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java @@ -30,6 +30,7 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskData; /** * Detects the type of a file by an inspection of its contents. @@ -143,12 +144,21 @@ public class FileTypeDetector { * succeeds. * * @param file The file to test. - * @param moduleName The name of the module posting to the blackboard. * @return The MIME type name id detection was successful, null otherwise. * @throws TskCoreException if there is an error posting to the blackboard. */ public String detectAndPostToBlackboard(AbstractFile file) throws TskCoreException { - String mimeType = detect(file); + + String mimeType; + // Consistently mark unallocated and unused space as file type application/octet-stream + if ((file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) + || (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS) + || (file.isFile() == false)) { + mimeType = MimeTypes.OCTET_STREAM; + } else { + mimeType = detect(file); + } + if (null != mimeType) { /** * Add the file type attribute to the general info artifact. Note diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index f1fe3e26bd..142081a005 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -95,15 +95,6 @@ public class FileTypeIdIngestModule implements FileIngestModule { @Override public ProcessResult process(AbstractFile file) { - /** - * Skip unallocated space and unused blocks files. - */ - if ((file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) - || (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS) - || (file.isFile() == false)) { - return ProcessResult.OK; - } - /** * Skip known files if configured to do so. */ diff --git a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/Bundle.properties index f0540e3482..badd35146a 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/Bundle.properties @@ -29,3 +29,4 @@ SevenZipIngestModule.unpack.encrFileDetected.msg=Encrypted files in archive dete SevenZipIngestModule.unpack.encrFileDetected.details=Some files in archive\: {0} are encrypted. {1} extractor was unable to extract all files from this archive. SevenZipIngestModule.UnpackStream.write.exception.msg=Error writing unpacked file to\: {0} SevenZipIngestModule.UnpackedTree.exception.msg=Error adding a derived file to db\:{0} +SevenZipIngestModule.startUp.fileTypeDetectorInitializationException.msg=Error initializing the File Type Detector. diff --git a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java index c72b5ec4f3..59553ae89f 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java @@ -103,8 +103,8 @@ public final class SevenZipIngestModule implements FileIngestModule { try { fileTypeDetector = new FileTypeDetector(); } catch (FileTypeDetector.FileTypeDetectorInitException ex) { - logger.log(Level.SEVERE, "Error initializing FileTypeDetector", ex); // NON-NLS - throw new IngestModuleException("Error initializing FileTypeDetector"); // NON-NLS + logger.log(Level.SEVERE, NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.startUp.fileTypeDetectorInitializationException.msg"), ex); + throw new IngestModuleException(NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.startUp.fileTypeDetectorInitializationException.msg")); } final Case currentCase = Case.getCurrentCase(); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index 03c68141f3..7e784739a7 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -283,3 +283,4 @@ KeywordSearchModuleFactory.createFileIngestModule.exception.msg=Expected setting SearchRunner.Searcher.done.err.msg=Error performing keyword search KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.toolTipText=Fastest overall, but no results until the end KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.text=No periodic searches +KeywordSearchIngestModule.startUp.fileTypeDetectorInitializationException.msg=Error initializing the File Type Detector. diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 9bc0a6442b..5a4931574c 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -132,8 +132,8 @@ public final class KeywordSearchIngestModule implements FileIngestModule { try { fileTypeDetector = new FileTypeDetector(); } catch (FileTypeDetector.FileTypeDetectorInitException ex) { - logger.log(Level.SEVERE, "Error initializing FileTypeDetector", ex); // NON-NLS - throw new IngestModuleException("Error initializing FileTypeDetector"); // NON-NLS + logger.log(Level.SEVERE, NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.startUp.fileTypeDetectorInitializationException.msg"), ex); + throw new IngestModuleException(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.startUp.fileTypeDetectorInitializationException.msg")); } ingester = Server.getIngester(); this.context = context; From 4dd9db30845a0c1d5b7ed0450fa80f9973a07453 Mon Sep 17 00:00:00 2001 From: sidheshenator Date: Tue, 26 May 2015 09:43:45 -0400 Subject: [PATCH 7/7] Unalloc space consistently marked as octet/stream --- .../modules/filetypeid/FileTypeDetector.java | 17 ++++++++--------- .../filetypeid/FileTypeIdIngestModule.java | 1 - 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java index 4b8b9266e8..3f0b47d7f4 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java @@ -150,15 +150,7 @@ public class FileTypeDetector { public String detectAndPostToBlackboard(AbstractFile file) throws TskCoreException { String mimeType; - // Consistently mark unallocated and unused space as file type application/octet-stream - if ((file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) - || (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS) - || (file.isFile() == false)) { - mimeType = MimeTypes.OCTET_STREAM; - } else { - mimeType = detect(file); - } - + mimeType = detect(file); if (null != mimeType) { /** * Add the file type attribute to the general info artifact. Note @@ -180,6 +172,13 @@ public class FileTypeDetector { * @return The MIME type name id detection was successful, null otherwise. */ public String detect(AbstractFile file) throws TskCoreException { + // Consistently mark unallocated and unused space as file type application/octet-stream + if ((file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) + || (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS) + || (file.isFile() == false)) { + return MimeTypes.OCTET_STREAM; + } + String fileType = detectUserDefinedType(file); if (null == fileType) { try { diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index 142081a005..6625c7c616 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -27,7 +27,6 @@ import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskData.FileKnown; import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult; import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter;