Merge pull request #3347 from APriestman/3208_delayDbWrites

3208 delay writing to the database until the end of the ingest pipeline

Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java

@@ -21,10 +21,14 @@ package org.sleuthkit.autopsy.ingest;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
+import java.util.logging.Level;
 
 import org.openide.util.NbBundle;
+import org.sleuthkit.autopsy.casemodule.Case;
+import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
 import org.sleuthkit.datamodel.AbstractFile;
+import org.sleuthkit.datamodel.TskCoreException;
 
 /**
  * This class manages a sequence of file level ingest modules for a data source
@@ -136,10 +140,17 @@ final class FileIngestPipeline {
                     break;
                 }
             }
-            file.close();
+            
             if (!this.job.isCancelled()) {
+                // Save any properties that have not already been saved to the database
+                try{
+                    file.save(Case.getCurrentCase().getSleuthkitCase());
+                } catch (TskCoreException ex){
+                    Logger.getLogger(FileIngestPipeline.class.getName()).log(Level.SEVERE, "Failed to save data for file " + file.getId(), ex); //NON-NLS
+                }
                 IngestManager.getInstance().fireFileIngestDone(file);
             }
+            file.close();
         }
         FileIngestPipeline.ingestManager.setIngestTaskProgressCompleted(task);
         return errors;

Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/MSOfficeEmbeddedContentExtractor.java

@@ -136,7 +136,7 @@ class MSOfficeEmbeddedContentExtractor {
      */
     boolean isContentExtractionSupported(AbstractFile abstractFile) {
         try {
-            String abstractFileMimeType = fileTypeDetector.getFileType(abstractFile);
+            String abstractFileMimeType = fileTypeDetector.detectFileType(abstractFile);
             for (SupportedExtractionFormats s : SupportedExtractionFormats.values()) {
                 if (s.toString().equals(abstractFileMimeType)) {
                     abstractFileExtractionFormat = s;

Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java

@@ -142,7 +142,7 @@ class SevenZipExtractor {
      */
     boolean isSevenZipExtractionSupported(AbstractFile abstractFile) {
         try {
-            String abstractFileMimeType = fileTypeDetector.getFileType(abstractFile);
+            String abstractFileMimeType = fileTypeDetector.detectFileType(abstractFile);
             for (SupportedArchiveExtractionFormats s : SupportedArchiveExtractionFormats.values()) {
                 if (s.toString().equals(abstractFileMimeType)) {
                     return true;

Core/src/org/sleuthkit/autopsy/modules/encryptiondetection/EncryptionDetectionFileIngestModule.java

@@ -189,7 +189,7 @@ final class EncryptionDetectionFileIngestModule extends FileIngestModuleAdapter
                          * Qualify the MIME type.
                          */
                         try {
-                            String mimeType = fileTypeDetector.getFileType(file);
+                            String mimeType = fileTypeDetector.detectFileType(file);
                             if (mimeType != null && mimeType.equals("application/octet-stream")) {
                                 possiblyEncrypted = true;
                             }

Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java

@@ -251,7 +251,7 @@ public final class ExifParserFileIngestModule implements FileIngestModule {
      */
     private boolean parsableFormat(AbstractFile f) {
         try {
-            String mimeType = fileTypeDetector.getFileType(f);
+            String mimeType = fileTypeDetector.detectFileType(f);
             if (mimeType != null) {
                 return supportedMimeTypes.contains(mimeType);
             } else {

Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java

@@ -170,7 +170,7 @@ public class FileExtMismatchIngestModule implements FileIngestModule {
         if (settings.skipFilesWithNoExtension() && currActualExt.isEmpty()) {
             return false;
         }
-        String currActualSigType = detector.getFileType(abstractFile);
+        String currActualSigType = detector.detectFileType(abstractFile);
         if (currActualSigType == null) {
             return false;
         }

Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java

@@ -173,11 +173,11 @@ public class FileTypeDetector {
     private boolean isDetectableByTika(String mimeType) {
         return FileTypeDetector.getTikaDetectedTypes().contains(removeOptionalParameter(mimeType));
     }
-
+    
     /**
      * Gets the MIME type of a file, detecting it if it is not already known. If
-     * detection is necessary, the result is added to the case database.
+     * detection is necessary, the result is saved to the AbstractFile object
-     *
+     * 
      * IMPORTANT: This method should only be called by ingest modules. All other
      * clients should call AbstractFile.getMIMEType, and may call
      * FileTypeDetector.detect, if AbstractFile.getMIMEType returns null.
@@ -190,8 +190,8 @@ public class FileTypeDetector {
      * @throws TskCoreException if detection is required and there is a problem
      *                          writing the result to the case database.
      */
-    public String getFileType(AbstractFile file) throws TskCoreException {
+    public String detectFileType(AbstractFile file) throws TskCoreException {
-        return detect(file, true);
+        return detect(file, false);
     }
 
     /**
@@ -213,6 +213,9 @@ public class FileTypeDetector {
     /**
      * Detects the MIME type of a file. The result is saved to the case database
      * only if the add to case database flag is set.
+     * 
+     * Ingest modules should not set addToCaseDb to true - the ingest process
+     * handles the database save.
      *
      * @param file        The file to test.
      * @param addToCaseDb Whether the MIME type should be added to the case
@@ -323,6 +326,7 @@ public class FileTypeDetector {
             Case.getCurrentCase().getSleuthkitCase().setFileMIMEType(file, mimeType);
         }
 
+        file.setMIMEType(mimeType);
         return mimeType;
     }
 
@@ -474,7 +478,30 @@ public class FileTypeDetector {
      */
     @Deprecated
     public String detectAndPostToBlackboard(AbstractFile file) throws TskCoreException {
-        return getFileType(file);
+        return detect(file, true);
+    }
+    
+    /**
+     * Gets the MIME type of a file, detecting it if it is not already known. If
+     * detection is necessary, the result is added to the case database.
+     *
+     * IMPORTANT: This method should only be called by ingest modules. All other
+     * clients should call AbstractFile.getMIMEType, and may call
+     * FileTypeDetector.detect, if AbstractFile.getMIMEType returns null.
+     *
+     * @param file The file.
+     *
+     * @return A MIME type name. If file type could not be detected or results
+     *         were uncertain, octet-stream is returned.
+     *
+     * @throws TskCoreException if detection is required and there is a problem
+     *                          writing the result to the case database.
+     * 
+     * @deprecated
+     */
+    @Deprecated
+    public String getFileType(AbstractFile file) throws TskCoreException {
+        return detect(file, true);
     }
 
 }

Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java

@@ -91,7 +91,7 @@ public class FileTypeIdIngestModule implements FileIngestModule {
          */
         try {
             long startTime = System.currentTimeMillis();
-            fileTypeDetector.getFileType(file);
+            fileTypeDetector.detect(file);
             addToTotals(jobId, (System.currentTimeMillis() - startTime));
             return ProcessResult.OK;
         } catch (Exception e) {

Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java

@@ -176,7 +176,8 @@ public class HashDbIngestModule implements FileIngestModule {
         if (md5Hash == null || md5Hash.isEmpty()) {
             try {
                 long calcstart = System.currentTimeMillis();
-                md5Hash = HashUtility.calculateMd5(file);
+                md5Hash = HashUtility.calculateMd5Hash(file);
+                file.setMd5Hash(md5Hash);
                 long delta = (System.currentTimeMillis() - calcstart);
                 totals.totalCalctime.addAndGet(delta);
 
@@ -205,20 +206,8 @@ public class HashDbIngestModule implements FileIngestModule {
                     foundBad = true;
                     totals.totalKnownBadCount.incrementAndGet();
 
-                    try {
+                    file.setKnown(TskData.FileKnown.BAD);
-                        skCase.setKnown(file, TskData.FileKnown.BAD);
+
-                    } catch (TskException ex) {
-                        logger.log(Level.WARNING, "Couldn't set notable state for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
-                        services.postMessage(IngestMessage.createErrorMessage(
-                                HashLookupModuleFactory.getModuleName(),
-                                NbBundle.getMessage(this.getClass(),
-                                        "HashDbIngestModule.hashLookupErrorMsg",
-                                        name),
-                                NbBundle.getMessage(this.getClass(),
-                                        "HashDbIngestModule.settingKnownBadStateErr",
-                                        name)));
-                        ret = ProcessResult.ERROR;
-                    }
                     String hashSetName = db.getDisplayName();
 
                     String comment = "";
@@ -262,13 +251,8 @@ public class HashDbIngestModule implements FileIngestModule {
                 try {
                     long lookupstart = System.currentTimeMillis();
                     if (db.lookupMD5Quick(file)) {
-                        try {
+                        file.setKnown(TskData.FileKnown.KNOWN);
-                            skCase.setKnown(file, TskData.FileKnown.KNOWN);
+                        break;
-                            break;
-                        } catch (TskException ex) {
-                            logger.log(Level.WARNING, "Couldn't set known state for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
-                            ret = ProcessResult.ERROR;
-                        }
                     }
                     long delta = (System.currentTimeMillis() - lookupstart);
                     totals.totalLookuptime.addAndGet(delta);

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java

@@ -514,7 +514,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
                 if (context.fileIngestIsCancelled()) {
                     return;
                 }
-                fileType = fileTypeDetector.getFileType(aFile);
+                fileType = fileTypeDetector.detectFileType(aFile);
             } catch (TskCoreException ex) {
                 logger.log(Level.SEVERE, String.format("Could not detect format using fileTypeDetector for file: %s", aFile), ex); //NON-NLS
                 return;