From c4339f6e2ad0f6b86b10a18dc3565bedfde1baa3 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Wed, 15 Nov 2017 14:07:29 -0500 Subject: [PATCH 1/7] Hold off on writing md5, known status, and MIME type until the end of ingest --- .../autopsy/ingest/FileIngestPipeline.java | 11 +++++ .../modules/filetypeid/FileTypeDetector.java | 7 ++-- .../filetypeid/FileTypeIdIngestModule.java | 3 +- .../hashdatabase/HashDbIngestModule.java | 42 ++++++++++--------- 4 files changed, 39 insertions(+), 24 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 436418712a..2b5271fc9a 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -21,10 +21,14 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.logging.Level; import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.TskCoreException; /** * This class manages a sequence of file level ingest modules for a data source @@ -136,6 +140,13 @@ final class FileIngestPipeline { break; } } + + try{ + Case.getCurrentCase().getSleuthkitCase().setKnownAndFileTypeAndMD5(file); + } catch (TskCoreException ex){ + Logger.getLogger(FileIngestPipeline.class.getName()).log(Level.SEVERE, "Failed to save data", ex); //NON-NLS + } + file.close(); if (!this.job.isCancelled()) { IngestManager.getInstance().fireFileIngestDone(file); diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java index eb1ab33591..c771426109 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java @@ -187,7 +187,8 @@ public class FileTypeDetector { * writing the result to the case database. */ public String getFileType(AbstractFile file) throws TskCoreException { - return detect(file, true); + return file.getMIMEType(); + //return detect(file, true); } /** @@ -222,7 +223,7 @@ public class FileTypeDetector { * @throws TskCoreException If there is a problem writing the result to the * case database. */ - private String detect(AbstractFile file, boolean addToCaseDb) throws TskCoreException { + public String detect(AbstractFile file, boolean addToCaseDb) throws TskCoreException { /* * Check to see if the file has already been typed. This is the "check" * part of a check-then-act race condition (see note below). @@ -322,7 +323,7 @@ public class FileTypeDetector { /* * Add the MIME type to the files table in the case database. */ - Case.getCurrentCase().getSleuthkitCase().setFileMIMEType(file, mimeType); + //Case.getCurrentCase().getSleuthkitCase().setFileMIMEType(file, mimeType); } return mimeType; diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index ef4e0add71..4499139d1a 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -91,7 +91,8 @@ public class FileTypeIdIngestModule implements FileIngestModule { */ try { long startTime = System.currentTimeMillis(); - fileTypeDetector.getFileType(file); + String type = fileTypeDetector.detect(file, false); + file.setMIMEType(type); addToTotals(jobId, (System.currentTimeMillis() - startTime)); return ProcessResult.OK; } catch (Exception e) { diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java index 3dd7416872..b83c91ace5 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java @@ -204,20 +204,21 @@ public class HashDbIngestModule implements FileIngestModule { foundBad = true; totals.totalKnownBadCount.incrementAndGet(); - try { - skCase.setKnown(file, TskData.FileKnown.BAD); - } catch (TskException ex) { - logger.log(Level.WARNING, "Couldn't set notable state for file " + name + " - see sleuthkit log for details", ex); //NON-NLS - services.postMessage(IngestMessage.createErrorMessage( - HashLookupModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), - "HashDbIngestModule.hashLookupErrorMsg", - name), - NbBundle.getMessage(this.getClass(), - "HashDbIngestModule.settingKnownBadStateErr", - name))); - ret = ProcessResult.ERROR; - } + //try { + file.setKnown(TskData.FileKnown.BAD); + // skCase.setKnown(file, TskData.FileKnown.BAD); + //} catch (TskException ex) { + // logger.log(Level.WARNING, "Couldn't set notable state for file " + name + " - see sleuthkit log for details", ex); //NON-NLS + // services.postMessage(IngestMessage.createErrorMessage( + // HashLookupModuleFactory.getModuleName(), + // NbBundle.getMessage(this.getClass(), + // "HashDbIngestModule.hashLookupErrorMsg", + // name), + // NbBundle.getMessage(this.getClass(), + // "HashDbIngestModule.settingKnownBadStateErr", + // name))); + // ret = ProcessResult.ERROR; + //} String hashSetName = db.getHashSetName(); String comment = ""; @@ -261,13 +262,14 @@ public class HashDbIngestModule implements FileIngestModule { try { long lookupstart = System.currentTimeMillis(); if (db.lookupMD5Quick(file)) { - try { - skCase.setKnown(file, TskData.FileKnown.KNOWN); + //try { + file.setKnown(TskData.FileKnown.KNOWN); + //skCase.setKnown(file, TskData.FileKnown.KNOWN); break; - } catch (TskException ex) { - logger.log(Level.WARNING, "Couldn't set known state for file " + name + " - see sleuthkit log for details", ex); //NON-NLS - ret = ProcessResult.ERROR; - } + //} catch (TskException ex) { + // logger.log(Level.WARNING, "Couldn't set known state for file " + name + " - see sleuthkit log for details", ex); //NON-NLS + // ret = ProcessResult.ERROR; + //} } long delta = (System.currentTimeMillis() - lookupstart); totals.totalLookuptime.addAndGet(delta); From 013eb34f2411cc66681283f03c83f67607f119d1 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 28 Dec 2017 14:58:43 -0500 Subject: [PATCH 2/7] Adding save to database parameter --- .../corecomponents/MediaViewVideoPanel.java | 2 +- .../autopsy/coreutils/ImageUtils.java | 2 +- .../autopsy/ingest/FileIngestPipeline.java | 3 +- .../MSOfficeEmbeddedContentExtractor.java | 2 +- .../SevenZipExtractor.java | 2 +- .../EncryptionDetectionFileIngestModule.java | 2 +- .../exif/ExifParserFileIngestModule.java | 2 +- .../FileExtMismatchIngestModule.java | 2 +- .../modules/filetypeid/FileTypeDetector.java | 47 +++++++++++++++---- .../filetypeid/FileTypeIdIngestModule.java | 3 +- .../hashdatabase/HashDbIngestModule.java | 2 +- .../autopsy/imagegallery/FileTypeUtils.java | 4 +- .../KeywordSearchIngestModule.java | 2 +- 13 files changed, 53 insertions(+), 22 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/MediaViewVideoPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/MediaViewVideoPanel.java index b9ad634ff5..d6354543d3 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/MediaViewVideoPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/MediaViewVideoPanel.java @@ -153,7 +153,7 @@ public abstract class MediaViewVideoPanel extends JPanel implements FrameCapture if (AUDIO_EXTENSIONS.contains("." + extension) || getExtensionsList().contains("." + extension)) { SortedSet mimeTypes = new TreeSet<>(getMimeTypes()); try { - String mimeType = new FileTypeDetector().detect(file); + String mimeType = new FileTypeDetector().detect(file, false); if (nonNull(mimeType)) { return mimeTypes.contains(mimeType); } diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/ImageUtils.java b/Core/src/org/sleuthkit/autopsy/coreutils/ImageUtils.java index eabc56e3bd..e355267df4 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/ImageUtils.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/ImageUtils.java @@ -263,7 +263,7 @@ public class ImageUtils { return true; } else { try { - String mimeType = getFileTypeDetector().detect(file); + String mimeType = getFileTypeDetector().detect(file, false); if (StringUtils.isNotBlank(mimeTypePrefix) && mimeType.startsWith(mimeTypePrefix)) { return true; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 2b5271fc9a..ad6d7a9ba3 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -141,8 +141,9 @@ final class FileIngestPipeline { } } + // Save any properties that have not already been saved to the database try{ - Case.getCurrentCase().getSleuthkitCase().setKnownAndFileTypeAndMD5(file); + file.save(Case.getCurrentCase().getSleuthkitCase()); } catch (TskCoreException ex){ Logger.getLogger(FileIngestPipeline.class.getName()).log(Level.SEVERE, "Failed to save data", ex); //NON-NLS } diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/MSOfficeEmbeddedContentExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/MSOfficeEmbeddedContentExtractor.java index 61376d31f1..d40ff4701e 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/MSOfficeEmbeddedContentExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/MSOfficeEmbeddedContentExtractor.java @@ -136,7 +136,7 @@ class MSOfficeEmbeddedContentExtractor { */ boolean isContentExtractionSupported(AbstractFile abstractFile) { try { - String abstractFileMimeType = fileTypeDetector.getFileType(abstractFile); + String abstractFileMimeType = fileTypeDetector.getFileType(abstractFile, false); for (SupportedExtractionFormats s : SupportedExtractionFormats.values()) { if (s.toString().equals(abstractFileMimeType)) { abstractFileExtractionFormat = s; diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 900329d3cb..cdbcaa61b8 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -142,7 +142,7 @@ class SevenZipExtractor { */ boolean isSevenZipExtractionSupported(AbstractFile abstractFile) { try { - String abstractFileMimeType = fileTypeDetector.getFileType(abstractFile); + String abstractFileMimeType = fileTypeDetector.getFileType(abstractFile, false); for (SupportedArchiveExtractionFormats s : SupportedArchiveExtractionFormats.values()) { if (s.toString().equals(abstractFileMimeType)) { return true; diff --git a/Core/src/org/sleuthkit/autopsy/modules/encryptiondetection/EncryptionDetectionFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/encryptiondetection/EncryptionDetectionFileIngestModule.java index da4e91882a..3a2792a0c2 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/encryptiondetection/EncryptionDetectionFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/encryptiondetection/EncryptionDetectionFileIngestModule.java @@ -189,7 +189,7 @@ final class EncryptionDetectionFileIngestModule extends FileIngestModuleAdapter * Qualify the MIME type. */ try { - String mimeType = fileTypeDetector.getFileType(file); + String mimeType = fileTypeDetector.getFileType(file, false); if (mimeType != null && mimeType.equals("application/octet-stream")) { possiblyEncrypted = true; } diff --git a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java index a1d0809ae0..7540abfbd4 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java @@ -251,7 +251,7 @@ public final class ExifParserFileIngestModule implements FileIngestModule { */ private boolean parsableFormat(AbstractFile f) { try { - String mimeType = fileTypeDetector.getFileType(f); + String mimeType = fileTypeDetector.getFileType(f, false); if (mimeType != null) { return supportedMimeTypes.contains(mimeType); } else { diff --git a/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java index 5d557de8bf..89cc7a8617 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java @@ -170,7 +170,7 @@ public class FileExtMismatchIngestModule implements FileIngestModule { if (settings.skipFilesWithNoExtension() && currActualExt.isEmpty()) { return false; } - String currActualSigType = detector.getFileType(abstractFile); + String currActualSigType = detector.getFileType(abstractFile, false); if (currActualSigType == null) { return false; } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java index afaffd3370..80ed7bbee4 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java @@ -173,16 +173,21 @@ public class FileTypeDetector { private boolean isDetectableByTika(String mimeType) { return FileTypeDetector.getTikaDetectedTypes().contains(removeOptionalParameter(mimeType)); } - + /** * Gets the MIME type of a file, detecting it if it is not already known. If - * detection is necessary, the result is added to the case database. + * detection is necessary, the result is saved to the AbstractFile object + * and optionally added to the case database. * + * In general, saveToDatabase should be set to false - the ingest process + * handles the database save. + * * IMPORTANT: This method should only be called by ingest modules. All other * clients should call AbstractFile.getMIMEType, and may call * FileTypeDetector.detect, if AbstractFile.getMIMEType returns null. * * @param file The file. + * @param saveToDatabase True if the result should be saved to the database * * @return A MIME type name. If file type could not be detected or results * were uncertain, octet-stream is returned. @@ -190,9 +195,8 @@ public class FileTypeDetector { * @throws TskCoreException if detection is required and there is a problem * writing the result to the case database. */ - public String getFileType(AbstractFile file) throws TskCoreException { - return file.getMIMEType(); - //return detect(file, true); + public String getFileType(AbstractFile file, boolean saveToDatabase) throws TskCoreException { + return detect(file, saveToDatabase); } /** @@ -207,13 +211,16 @@ public class FileTypeDetector { * @throws TskCoreException If there is a problem writing the result to the * case database. */ - public String detect(AbstractFile file) throws TskCoreException { + private String detect(AbstractFile file) throws TskCoreException { return detect(file, false); } /** * Detects the MIME type of a file. The result is saved to the case database * only if the add to case database flag is set. + * + * Ingest modules should not set addToCaseDb to true - the ingest process + * handles the database save. * * @param file The file to test. * @param addToCaseDb Whether the MIME type should be added to the case @@ -321,9 +328,10 @@ public class FileTypeDetector { /* * Add the MIME type to the files table in the case database. */ - //Case.getCurrentCase().getSleuthkitCase().setFileMIMEType(file, mimeType); + Case.getCurrentCase().getSleuthkitCase().setFileMIMEType(file, mimeType); } + file.setMIMEType(mimeType); return mimeType; } @@ -475,7 +483,30 @@ public class FileTypeDetector { */ @Deprecated public String detectAndPostToBlackboard(AbstractFile file) throws TskCoreException { - return getFileType(file); + return getFileType(file, true); + } + + /** + * Gets the MIME type of a file, detecting it if it is not already known. If + * detection is necessary, the result is added to the case database. + * + * IMPORTANT: This method should only be called by ingest modules. All other + * clients should call AbstractFile.getMIMEType, and may call + * FileTypeDetector.detect, if AbstractFile.getMIMEType returns null. + * + * @param file The file. + * + * @return A MIME type name. If file type could not be detected or results + * were uncertain, octet-stream is returned. + * + * @throws TskCoreException if detection is required and there is a problem + * writing the result to the case database. + * + * @deprecated + */ + @Deprecated + public String getFileType(AbstractFile file) throws TskCoreException { + return detect(file, true); } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index 4499139d1a..69572e755c 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -91,8 +91,7 @@ public class FileTypeIdIngestModule implements FileIngestModule { */ try { long startTime = System.currentTimeMillis(); - String type = fileTypeDetector.detect(file, false); - file.setMIMEType(type); + fileTypeDetector.detect(file, false); addToTotals(jobId, (System.currentTimeMillis() - startTime)); return ProcessResult.OK; } catch (Exception e) { diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java index c2895dec86..0d24ce0e81 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java @@ -176,7 +176,7 @@ public class HashDbIngestModule implements FileIngestModule { if (md5Hash == null || md5Hash.isEmpty()) { try { long calcstart = System.currentTimeMillis(); - md5Hash = HashUtility.calculateMd5(file); + md5Hash = HashUtility.calculateMd5(file, false); long delta = (System.currentTimeMillis() - calcstart); totals.totalCalctime.addAndGet(delta); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/FileTypeUtils.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/FileTypeUtils.java index 83966ecfe4..aedb06e2f0 100755 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/FileTypeUtils.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/FileTypeUtils.java @@ -220,7 +220,7 @@ public enum FileTypeUtils { * mimetype could not be detected. */ static boolean hasDrawableMIMEType(AbstractFile file) throws TskCoreException, FileTypeDetector.FileTypeDetectorInitException { - String mimeType = getFileTypeDetector().detect(file).toLowerCase(); + String mimeType = getFileTypeDetector().detect(file, true).toLowerCase(); return isDrawableMimeType(mimeType) || (mimeType.equals("audio/x-aiff") && "tiff".equalsIgnoreCase(file.getNameExtension())); } @@ -235,7 +235,7 @@ public enum FileTypeUtils { */ public static boolean hasVideoMIMEType(AbstractFile file) { try { - String mimeType = getFileTypeDetector().detect(file).toLowerCase(); + String mimeType = getFileTypeDetector().detect(file, true).toLowerCase(); return mimeType.startsWith("video/") || videoMimeTypes.contains(mimeType); } catch (FileTypeDetector.FileTypeDetectorInitException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error determining MIME type of " + getContentPathSafe(file), ex); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index dbbca6394b..71e5967521 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -514,7 +514,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { if (context.fileIngestIsCancelled()) { return; } - fileType = fileTypeDetector.getFileType(aFile); + fileType = fileTypeDetector.getFileType(aFile, false); } catch (TskCoreException ex) { logger.log(Level.SEVERE, String.format("Could not detect format using fileTypeDetector for file: %s", aFile), ex); //NON-NLS return; From 083e51ca29444fcf1e5ef06a81df88a3cca9dabf Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 28 Dec 2017 15:23:21 -0500 Subject: [PATCH 3/7] Changing detect back to private --- .../sleuthkit/autopsy/corecomponents/MediaViewVideoPanel.java | 2 +- Core/src/org/sleuthkit/autopsy/coreutils/ImageUtils.java | 2 +- .../autopsy/modules/filetypeid/FileTypeDetector.java | 4 ++-- .../autopsy/modules/filetypeid/FileTypeIdIngestModule.java | 2 +- .../src/org/sleuthkit/autopsy/imagegallery/FileTypeUtils.java | 4 ++-- .../core/core.jar/org/netbeans/core/startup/Bundle.properties | 2 +- .../org/netbeans/core/windows/view/ui/Bundle.properties | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/MediaViewVideoPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/MediaViewVideoPanel.java index d6354543d3..b9ad634ff5 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/MediaViewVideoPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/MediaViewVideoPanel.java @@ -153,7 +153,7 @@ public abstract class MediaViewVideoPanel extends JPanel implements FrameCapture if (AUDIO_EXTENSIONS.contains("." + extension) || getExtensionsList().contains("." + extension)) { SortedSet mimeTypes = new TreeSet<>(getMimeTypes()); try { - String mimeType = new FileTypeDetector().detect(file, false); + String mimeType = new FileTypeDetector().detect(file); if (nonNull(mimeType)) { return mimeTypes.contains(mimeType); } diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/ImageUtils.java b/Core/src/org/sleuthkit/autopsy/coreutils/ImageUtils.java index e355267df4..eabc56e3bd 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/ImageUtils.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/ImageUtils.java @@ -263,7 +263,7 @@ public class ImageUtils { return true; } else { try { - String mimeType = getFileTypeDetector().detect(file, false); + String mimeType = getFileTypeDetector().detect(file); if (StringUtils.isNotBlank(mimeTypePrefix) && mimeType.startsWith(mimeTypePrefix)) { return true; } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java index 80ed7bbee4..a40d45b3e3 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java @@ -211,7 +211,7 @@ public class FileTypeDetector { * @throws TskCoreException If there is a problem writing the result to the * case database. */ - private String detect(AbstractFile file) throws TskCoreException { + public String detect(AbstractFile file) throws TskCoreException { return detect(file, false); } @@ -234,7 +234,7 @@ public class FileTypeDetector { * @throws TskCoreException If there is a problem writing the result to the * case database. */ - public String detect(AbstractFile file, boolean addToCaseDb) throws TskCoreException { + private String detect(AbstractFile file, boolean addToCaseDb) throws TskCoreException { /* * Check to see if the file has already been typed. This is the "check" * part of a check-then-act race condition (see note below). diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index 69572e755c..ed6698bd32 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -91,7 +91,7 @@ public class FileTypeIdIngestModule implements FileIngestModule { */ try { long startTime = System.currentTimeMillis(); - fileTypeDetector.detect(file, false); + fileTypeDetector.detect(file); addToTotals(jobId, (System.currentTimeMillis() - startTime)); return ProcessResult.OK; } catch (Exception e) { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/FileTypeUtils.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/FileTypeUtils.java index aedb06e2f0..83966ecfe4 100755 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/FileTypeUtils.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/FileTypeUtils.java @@ -220,7 +220,7 @@ public enum FileTypeUtils { * mimetype could not be detected. */ static boolean hasDrawableMIMEType(AbstractFile file) throws TskCoreException, FileTypeDetector.FileTypeDetectorInitException { - String mimeType = getFileTypeDetector().detect(file, true).toLowerCase(); + String mimeType = getFileTypeDetector().detect(file).toLowerCase(); return isDrawableMimeType(mimeType) || (mimeType.equals("audio/x-aiff") && "tiff".equalsIgnoreCase(file.getNameExtension())); } @@ -235,7 +235,7 @@ public enum FileTypeUtils { */ public static boolean hasVideoMIMEType(AbstractFile file) { try { - String mimeType = getFileTypeDetector().detect(file, true).toLowerCase(); + String mimeType = getFileTypeDetector().detect(file).toLowerCase(); return mimeType.startsWith("video/") || videoMimeTypes.contains(mimeType); } catch (FileTypeDetector.FileTypeDetectorInitException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error determining MIME type of " + getContentPathSafe(file), ex); diff --git a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties index 0de39782ca..7d140146fa 100644 --- a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties +++ b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Wed, 08 Nov 2017 17:45:11 -0500 +#Mon, 18 Dec 2017 14:43:20 -0500 LBL_splash_window_title=Starting Autopsy SPLASH_HEIGHT=314 SPLASH_WIDTH=538 diff --git a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties index fa55dddb62..2196ae7af5 100644 --- a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties +++ b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties @@ -1,4 +1,4 @@ #Updated by build script -#Wed, 08 Nov 2017 17:45:11 -0500 +#Mon, 18 Dec 2017 14:43:20 -0500 CTL_MainWindow_Title=Autopsy 4.5.0 CTL_MainWindow_Title_No_Project=Autopsy 4.5.0 From 1722d20426db7333c4cc485282390e39d165869b Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 28 Dec 2017 15:27:10 -0500 Subject: [PATCH 4/7] Cleanup --- .../hashdatabase/HashDbIngestModule.java | 27 +++---------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java index 0d24ce0e81..eda1830c68 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java @@ -205,21 +205,8 @@ public class HashDbIngestModule implements FileIngestModule { foundBad = true; totals.totalKnownBadCount.incrementAndGet(); - //try { - file.setKnown(TskData.FileKnown.BAD); - // skCase.setKnown(file, TskData.FileKnown.BAD); - //} catch (TskException ex) { - // logger.log(Level.WARNING, "Couldn't set notable state for file " + name + " - see sleuthkit log for details", ex); //NON-NLS - // services.postMessage(IngestMessage.createErrorMessage( - // HashLookupModuleFactory.getModuleName(), - // NbBundle.getMessage(this.getClass(), - // "HashDbIngestModule.hashLookupErrorMsg", - // name), - // NbBundle.getMessage(this.getClass(), - // "HashDbIngestModule.settingKnownBadStateErr", - // name))); - // ret = ProcessResult.ERROR; - //} + file.setKnown(TskData.FileKnown.BAD); + String hashSetName = db.getDisplayName(); String comment = ""; @@ -263,14 +250,8 @@ public class HashDbIngestModule implements FileIngestModule { try { long lookupstart = System.currentTimeMillis(); if (db.lookupMD5Quick(file)) { - //try { - file.setKnown(TskData.FileKnown.KNOWN); - //skCase.setKnown(file, TskData.FileKnown.KNOWN); - break; - //} catch (TskException ex) { - // logger.log(Level.WARNING, "Couldn't set known state for file " + name + " - see sleuthkit log for details", ex); //NON-NLS - // ret = ProcessResult.ERROR; - //} + file.setKnown(TskData.FileKnown.KNOWN); + break; } long delta = (System.currentTimeMillis() - lookupstart); totals.totalLookuptime.addAndGet(delta); From bfffa539f7224e4873a3bd4988aacf1ddf5c44f6 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 29 Dec 2017 11:55:29 -0500 Subject: [PATCH 5/7] Addressing review comments --- .../autopsy/ingest/FileIngestPipeline.java | 15 +++++++-------- .../MSOfficeEmbeddedContentExtractor.java | 2 +- .../embeddedfileextractor/SevenZipExtractor.java | 2 +- .../EncryptionDetectionFileIngestModule.java | 2 +- .../modules/exif/ExifParserFileIngestModule.java | 2 +- .../FileExtMismatchIngestModule.java | 2 +- .../modules/filetypeid/FileTypeDetector.java | 10 +++------- .../modules/hashdatabase/HashDbIngestModule.java | 1 + .../keywordsearch/KeywordSearchIngestModule.java | 2 +- 9 files changed, 17 insertions(+), 21 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index ad6d7a9ba3..7c0b1e58ed 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -141,17 +141,16 @@ final class FileIngestPipeline { } } - // Save any properties that have not already been saved to the database - try{ - file.save(Case.getCurrentCase().getSleuthkitCase()); - } catch (TskCoreException ex){ - Logger.getLogger(FileIngestPipeline.class.getName()).log(Level.SEVERE, "Failed to save data", ex); //NON-NLS - } - - file.close(); if (!this.job.isCancelled()) { + // Save any properties that have not already been saved to the database + try{ + file.save(Case.getCurrentCase().getSleuthkitCase()); + } catch (TskCoreException ex){ + Logger.getLogger(FileIngestPipeline.class.getName()).log(Level.SEVERE, "Failed to save data for file " + file.getId(), ex); //NON-NLS + } IngestManager.getInstance().fireFileIngestDone(file); } + file.close(); } FileIngestPipeline.ingestManager.setIngestTaskProgressCompleted(task); return errors; diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/MSOfficeEmbeddedContentExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/MSOfficeEmbeddedContentExtractor.java index d40ff4701e..1844173ca0 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/MSOfficeEmbeddedContentExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/MSOfficeEmbeddedContentExtractor.java @@ -136,7 +136,7 @@ class MSOfficeEmbeddedContentExtractor { */ boolean isContentExtractionSupported(AbstractFile abstractFile) { try { - String abstractFileMimeType = fileTypeDetector.getFileType(abstractFile, false); + String abstractFileMimeType = fileTypeDetector.detectFileType(abstractFile); for (SupportedExtractionFormats s : SupportedExtractionFormats.values()) { if (s.toString().equals(abstractFileMimeType)) { abstractFileExtractionFormat = s; diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index cdbcaa61b8..829b56312b 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -142,7 +142,7 @@ class SevenZipExtractor { */ boolean isSevenZipExtractionSupported(AbstractFile abstractFile) { try { - String abstractFileMimeType = fileTypeDetector.getFileType(abstractFile, false); + String abstractFileMimeType = fileTypeDetector.detectFileType(abstractFile); for (SupportedArchiveExtractionFormats s : SupportedArchiveExtractionFormats.values()) { if (s.toString().equals(abstractFileMimeType)) { return true; diff --git a/Core/src/org/sleuthkit/autopsy/modules/encryptiondetection/EncryptionDetectionFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/encryptiondetection/EncryptionDetectionFileIngestModule.java index 3a2792a0c2..928e7ccb36 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/encryptiondetection/EncryptionDetectionFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/encryptiondetection/EncryptionDetectionFileIngestModule.java @@ -189,7 +189,7 @@ final class EncryptionDetectionFileIngestModule extends FileIngestModuleAdapter * Qualify the MIME type. */ try { - String mimeType = fileTypeDetector.getFileType(file, false); + String mimeType = fileTypeDetector.detectFileType(file); if (mimeType != null && mimeType.equals("application/octet-stream")) { possiblyEncrypted = true; } diff --git a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java index 7540abfbd4..86deac1903 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java @@ -251,7 +251,7 @@ public final class ExifParserFileIngestModule implements FileIngestModule { */ private boolean parsableFormat(AbstractFile f) { try { - String mimeType = fileTypeDetector.getFileType(f, false); + String mimeType = fileTypeDetector.detectFileType(f); if (mimeType != null) { return supportedMimeTypes.contains(mimeType); } else { diff --git a/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java index 89cc7a8617..db4224a0a8 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java @@ -170,7 +170,7 @@ public class FileExtMismatchIngestModule implements FileIngestModule { if (settings.skipFilesWithNoExtension() && currActualExt.isEmpty()) { return false; } - String currActualSigType = detector.getFileType(abstractFile, false); + String currActualSigType = detector.detectFileType(abstractFile); if (currActualSigType == null) { return false; } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java index a40d45b3e3..b3d2b38759 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java @@ -178,16 +178,12 @@ public class FileTypeDetector { * Gets the MIME type of a file, detecting it if it is not already known. If * detection is necessary, the result is saved to the AbstractFile object * and optionally added to the case database. - * - * In general, saveToDatabase should be set to false - the ingest process - * handles the database save. * * IMPORTANT: This method should only be called by ingest modules. All other * clients should call AbstractFile.getMIMEType, and may call * FileTypeDetector.detect, if AbstractFile.getMIMEType returns null. * * @param file The file. - * @param saveToDatabase True if the result should be saved to the database * * @return A MIME type name. If file type could not be detected or results * were uncertain, octet-stream is returned. @@ -195,8 +191,8 @@ public class FileTypeDetector { * @throws TskCoreException if detection is required and there is a problem * writing the result to the case database. */ - public String getFileType(AbstractFile file, boolean saveToDatabase) throws TskCoreException { - return detect(file, saveToDatabase); + public String detectFileType(AbstractFile file) throws TskCoreException { + return detect(file, false); } /** @@ -483,7 +479,7 @@ public class FileTypeDetector { */ @Deprecated public String detectAndPostToBlackboard(AbstractFile file) throws TskCoreException { - return getFileType(file, true); + return detect(file, true); } /** diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java index eda1830c68..4cd87c3648 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java @@ -177,6 +177,7 @@ public class HashDbIngestModule implements FileIngestModule { try { long calcstart = System.currentTimeMillis(); md5Hash = HashUtility.calculateMd5(file, false); + file.setMd5Hash(md5Hash); long delta = (System.currentTimeMillis() - calcstart); totals.totalCalctime.addAndGet(delta); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 71e5967521..02968ddd56 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -514,7 +514,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { if (context.fileIngestIsCancelled()) { return; } - fileType = fileTypeDetector.getFileType(aFile, false); + fileType = fileTypeDetector.detectFileType(aFile); } catch (TskCoreException ex) { logger.log(Level.SEVERE, String.format("Could not detect format using fileTypeDetector for file: %s", aFile), ex); //NON-NLS return; From dce17a87be4dff1ce4c5bbe17fdd1f8b41b27d74 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 29 Dec 2017 12:43:09 -0500 Subject: [PATCH 6/7] Changing calculateMd5Hash --- .../autopsy/modules/hashdatabase/HashDbIngestModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java index 4cd87c3648..696a105416 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java @@ -176,7 +176,7 @@ public class HashDbIngestModule implements FileIngestModule { if (md5Hash == null || md5Hash.isEmpty()) { try { long calcstart = System.currentTimeMillis(); - md5Hash = HashUtility.calculateMd5(file, false); + md5Hash = HashUtility.calculateMd5Hash(file); file.setMd5Hash(md5Hash); long delta = (System.currentTimeMillis() - calcstart); totals.totalCalctime.addAndGet(delta); From a8b0c4f1bc520d38aa6108be192d8375324b73c7 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 29 Dec 2017 12:45:31 -0500 Subject: [PATCH 7/7] Cleanup --- .../sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java index b3d2b38759..dcfc75a2b3 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java @@ -177,7 +177,6 @@ public class FileTypeDetector { /** * Gets the MIME type of a file, detecting it if it is not already known. If * detection is necessary, the result is saved to the AbstractFile object - * and optionally added to the case database. * * IMPORTANT: This method should only be called by ingest modules. All other * clients should call AbstractFile.getMIMEType, and may call