mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3347 from APriestman/3208_delayDbWrites

Browse Source 
3208 delay writing to the database until the end of the ingest pipeline
This commit is contained in:
Richard Cordovano 2017-12-29 13:00:05 -05:00 committed by GitHub
commit 68a2948a39
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 58 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java
View File

@ -21,10 +21,14 @@ package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.logging.Level;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.TskCoreException;
/**
* This class manages a sequence of file level ingest modules for a data source
@ -136,10 +140,17 @@ final class FileIngestPipeline {
break;
}
}
file.close();
if (!this.job.isCancelled()) {
// Save any properties that have not already been saved to the database
try{
file.save(Case.getCurrentCase().getSleuthkitCase());
} catch (TskCoreException ex){
Logger.getLogger(FileIngestPipeline.class.getName()).log(Level.SEVERE, "Failed to save data for file " + file.getId(), ex); //NON-NLS
}
IngestManager.getInstance().fireFileIngestDone(file);
}
file.close();
}
FileIngestPipeline.ingestManager.setIngestTaskProgressCompleted(task);
return errors;

2
Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/MSOfficeEmbeddedContentExtractor.java
View File

@ -136,7 +136,7 @@ class MSOfficeEmbeddedContentExtractor {
*/
boolean isContentExtractionSupported(AbstractFile abstractFile) {
try {
String abstractFileMimeType = fileTypeDetector.getFileType(abstractFile);
String abstractFileMimeType = fileTypeDetector.detectFileType(abstractFile);
for (SupportedExtractionFormats s : SupportedExtractionFormats.values()) {
if (s.toString().equals(abstractFileMimeType)) {
abstractFileExtractionFormat = s;

2
Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java
View File

@ -142,7 +142,7 @@ class SevenZipExtractor {
*/
boolean isSevenZipExtractionSupported(AbstractFile abstractFile) {
try {
String abstractFileMimeType = fileTypeDetector.getFileType(abstractFile);
String abstractFileMimeType = fileTypeDetector.detectFileType(abstractFile);
for (SupportedArchiveExtractionFormats s : SupportedArchiveExtractionFormats.values()) {
if (s.toString().equals(abstractFileMimeType)) {
return true;

2
Core/src/org/sleuthkit/autopsy/modules/encryptiondetection/EncryptionDetectionFileIngestModule.java
View File

@ -189,7 +189,7 @@ final class EncryptionDetectionFileIngestModule extends FileIngestModuleAdapter
* Qualify the MIME type.
*/
try {
String mimeType = fileTypeDetector.getFileType(file);
String mimeType = fileTypeDetector.detectFileType(file);
if (mimeType != null && mimeType.equals("application/octet-stream")) {
possiblyEncrypted = true;
}

2
Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java
View File

@ -251,7 +251,7 @@ public final class ExifParserFileIngestModule implements FileIngestModule {
*/
private boolean parsableFormat(AbstractFile f) {
try {
String mimeType = fileTypeDetector.getFileType(f);
String mimeType = fileTypeDetector.detectFileType(f);
if (mimeType != null) {
return supportedMimeTypes.contains(mimeType);
} else {

2
Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java
View File

@ -170,7 +170,7 @@ public class FileExtMismatchIngestModule implements FileIngestModule {
if (settings.skipFilesWithNoExtension() && currActualExt.isEmpty()) {
return false;
}
String currActualSigType = detector.getFileType(abstractFile);
String currActualSigType = detector.detectFileType(abstractFile);
if (currActualSigType == null) {
return false;
}

35
Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java
View File

@ -176,7 +176,7 @@ public class FileTypeDetector {
/**
* Gets the MIME type of a file, detecting it if it is not already known. If
* detection is necessary, the result is added to the case database.
* detection is necessary, the result is saved to the AbstractFile object
*
* IMPORTANT: This method should only be called by ingest modules. All other
* clients should call AbstractFile.getMIMEType, and may call
@ -190,8 +190,8 @@ public class FileTypeDetector {
* @throws TskCoreException if detection is required and there is a problem
* writing the result to the case database.
*/
public String getFileType(AbstractFile file) throws TskCoreException {
return detect(file, true);
public String detectFileType(AbstractFile file) throws TskCoreException {
return detect(file, false);
}
/**
@ -214,6 +214,9 @@ public class FileTypeDetector {
* Detects the MIME type of a file. The result is saved to the case database
* only if the add to case database flag is set.
*
* Ingest modules should not set addToCaseDb to true - the ingest process
* handles the database save.
*
* @param file The file to test.
* @param addToCaseDb Whether the MIME type should be added to the case
* database. This flag is part of a partial workaround
@ -323,6 +326,7 @@ public class FileTypeDetector {
Case.getCurrentCase().getSleuthkitCase().setFileMIMEType(file, mimeType);
}
file.setMIMEType(mimeType);
return mimeType;
}
@ -474,7 +478,30 @@ public class FileTypeDetector {
*/
@Deprecated
public String detectAndPostToBlackboard(AbstractFile file) throws TskCoreException {
return getFileType(file);
return detect(file, true);
}
/**
* Gets the MIME type of a file, detecting it if it is not already known. If
* detection is necessary, the result is added to the case database.
*
* IMPORTANT: This method should only be called by ingest modules. All other
* clients should call AbstractFile.getMIMEType, and may call
* FileTypeDetector.detect, if AbstractFile.getMIMEType returns null.
*
* @param file The file.
*
* @return A MIME type name. If file type could not be detected or results
* were uncertain, octet-stream is returned.
*
* @throws TskCoreException if detection is required and there is a problem
* writing the result to the case database.
*
* @deprecated
*/
@Deprecated
public String getFileType(AbstractFile file) throws TskCoreException {
return detect(file, true);
}
}

2
Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java
View File

@ -91,7 +91,7 @@ public class FileTypeIdIngestModule implements FileIngestModule {
*/
try {
long startTime = System.currentTimeMillis();
fileTypeDetector.getFileType(file);
fileTypeDetector.detect(file);
addToTotals(jobId, (System.currentTimeMillis() - startTime));
return ProcessResult.OK;
} catch (Exception e) {

26
Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java
View File

@ -176,7 +176,8 @@ public class HashDbIngestModule implements FileIngestModule {
if (md5Hash == null || md5Hash.isEmpty()) {
try {
long calcstart = System.currentTimeMillis();
md5Hash = HashUtility.calculateMd5(file);
md5Hash = HashUtility.calculateMd5Hash(file);
file.setMd5Hash(md5Hash);
long delta = (System.currentTimeMillis() - calcstart);
totals.totalCalctime.addAndGet(delta);
@ -205,20 +206,8 @@ public class HashDbIngestModule implements FileIngestModule {
foundBad = true;
totals.totalKnownBadCount.incrementAndGet();
try {
skCase.setKnown(file, TskData.FileKnown.BAD);
} catch (TskException ex) {
logger.log(Level.WARNING, "Couldn't set notable state for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
services.postMessage(IngestMessage.createErrorMessage(
HashLookupModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.hashLookupErrorMsg",
name),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.settingKnownBadStateErr",
name)));
ret = ProcessResult.ERROR;
}
file.setKnown(TskData.FileKnown.BAD);
String hashSetName = db.getDisplayName();
String comment = "";
@ -262,13 +251,8 @@ public class HashDbIngestModule implements FileIngestModule {
try {
long lookupstart = System.currentTimeMillis();
if (db.lookupMD5Quick(file)) {
try {
skCase.setKnown(file, TskData.FileKnown.KNOWN);
file.setKnown(TskData.FileKnown.KNOWN);
break;
} catch (TskException ex) {
logger.log(Level.WARNING, "Couldn't set known state for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
ret = ProcessResult.ERROR;
}
}
long delta = (System.currentTimeMillis() - lookupstart);
totals.totalLookuptime.addAndGet(delta);

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
View File

@ -514,7 +514,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
if (context.fileIngestIsCancelled()) {
return;
}
fileType = fileTypeDetector.getFileType(aFile);
fileType = fileTypeDetector.detectFileType(aFile);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, String.format("Could not detect format using fileTypeDetector for file: %s", aFile), ex); //NON-NLS
return;