From d21bbcfaea180b00c061238c7f9f4016ddcf7f19 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 10 Nov 2016 11:17:44 -0500 Subject: [PATCH] Cycle through each DSP with non-zero confidence in decreasing order of confidence. --- .../autoingest/AutoIngestJobLogger.java | 26 +++++++++ .../autoingest/AutoIngestManager.java | 55 ++++++++++++------- 2 files changed, 61 insertions(+), 20 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java index 34e5b8fc9a..1ded9aff0a 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java @@ -191,6 +191,32 @@ final class AutoIngestJobLogger { void logDataSourceProcessorCancelled() throws AutoIngestJobLoggerException, InterruptedException { // RJCTODO: Is this used now? log(MessageCategory.WARNING, "Cancelled adding data source to case"); } + + /** + * Logs selection of a data source processor + * @param dsp Name of the data source processor + * @throws AutoIngestJobLoggerException if there is an error writing the log + * message. + * @throws InterruptedException if interrupted while blocked waiting + * to acquire an exclusive lock on the + * log file. + */ + void logDataSourceProcessorSelected(String dsp) throws AutoIngestJobLoggerException, InterruptedException{ + log(MessageCategory.INFO, "Using data source processor: " + dsp); + } + + /** + * Logs the failure of the selected data source processor. + * @param dsp Name of the data source processor + * @throws AutoIngestJobLoggerException if there is an error writing the log + * message. + * @throws InterruptedException if interrupted while blocked waiting + * to acquire an exclusive lock on the + * log file. + */ + void logDataSourceProcessorError(String dsp) throws AutoIngestJobLoggerException, InterruptedException{ + log(MessageCategory.ERROR, "Error processing with data source processor: " + dsp); + } /** * Logs the addition of a data source to the case database. diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index cba60d48c9..aa760e4e18 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -63,6 +63,7 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.logging.Level; +import java.util.stream.Collectors; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.ThreadSafe; @@ -2144,31 +2145,28 @@ public final class AutoIngestManager extends Observable implements PropertyChang DataSourceProcessorProgressMonitor progressMonitor = new DoNothingDSPProgressMonitor(); Path caseDirectoryPath = currentJob.getCaseDirectoryPath(); AutoIngestJobLogger jobLogger = new AutoIngestJobLogger(manifestPath, manifest.getDataSourceFileName(), caseDirectoryPath); - AutomatedIngestDataSourceProcessor selectedProcessor = null; try { caseForJob.notifyAddingDataSource(taskId); // lookup all AutomatedIngestDataSourceProcessors Collection processorCandidates = Lookup.getDefault().lookupAll(AutomatedIngestDataSourceProcessor.class); - int selectedProcessorConfidence = 0; + Map validDataSourceProcessorsMap = new HashMap<>(); for (AutomatedIngestDataSourceProcessor processor : processorCandidates) { - int confidence = 0; try { - confidence = processor.canProcess(dataSource.getPath()); + int confidence = processor.canProcess(dataSource.getPath()); + if(confidence > 0){ + validDataSourceProcessorsMap.put(processor, confidence); + } } catch (AutomatedIngestDataSourceProcessor.AutomatedIngestDataSourceProcessorException ex) { SYS_LOGGER.log(Level.SEVERE, "Exception while determining whether data source processor {0} can process {1}", new Object[]{processor.getDataSourceType(), dataSource.getPath()}); // rethrow the exception. It will get caught & handled upstream and will result in AIM auto-pause. throw ex; } - if (confidence > selectedProcessorConfidence) { - selectedProcessor = processor; - selectedProcessorConfidence = confidence; - } } // did we find a data source processor that can process the data source - if (selectedProcessor == null) { + if (validDataSourceProcessorsMap.isEmpty()) { // This should never happen. We should add all unsupported data sources as logical files. AutoIngestAlertFile.create(caseDirectoryPath); currentJob.setErrorsOccurred(true); @@ -2176,20 +2174,37 @@ public final class AutoIngestManager extends Observable implements PropertyChang SYS_LOGGER.log(Level.WARNING, "Unsupported data source {0} for {1}", new Object[]{dataSource.getPath(), manifestPath}); // NON-NLS return; } + + // Get an ordered list of data source processors to try + List validDataSourceProcessors = validDataSourceProcessorsMap.entrySet().stream() + .sorted(Map.Entry.comparingByValue().reversed()) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); synchronized (ingestLock) { - SYS_LOGGER.log(Level.INFO, "Identified data source type for {0} as {1}", new Object[]{manifestPath, selectedProcessor.getDataSourceType()}); - try { - selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), progressMonitor, callBack); - } catch (AutomatedIngestDataSourceProcessor.AutomatedIngestDataSourceProcessorException ex) { - AutoIngestAlertFile.create(caseDirectoryPath); - currentJob.setErrorsOccurred(true); - jobLogger.logFailedToAddDataSource(); - SYS_LOGGER.log(Level.SEVERE, "Exception while processing {0} with data source processor {1}", new Object[]{dataSource.getPath(), selectedProcessor.getDataSourceType()}); - // rethrow the exception. It will get caught & handled upstream and will result in AIM auto-pause. - throw ex; + // Try each DSP in decreasing order of confidence + for(AutomatedIngestDataSourceProcessor selectedProcessor:validDataSourceProcessors){ + jobLogger.logDataSourceProcessorSelected(selectedProcessor.getDataSourceType()); + SYS_LOGGER.log(Level.INFO, "Identified data source type for {0} as {1}", new Object[]{manifestPath, selectedProcessor.getDataSourceType()}); + try { + selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), progressMonitor, callBack); + ingestLock.wait(); + return; + } catch (AutomatedIngestDataSourceProcessor.AutomatedIngestDataSourceProcessorException ex) { + // Log that the current DSP failed and set the error flag. We consider it an error + // if a DSP fails even if a later one succeeds since we expected to be able to process + // the data source which each DSP on the list. + AutoIngestAlertFile.create(caseDirectoryPath); + currentJob.setErrorsOccurred(true); + jobLogger.logDataSourceProcessorError(selectedProcessor.getDataSourceType()); + SYS_LOGGER.log(Level.SEVERE, "Exception while processing {0} with data source processor {1}", new Object[]{dataSource.getPath(), selectedProcessor.getDataSourceType()}); + } } - ingestLock.wait(); + // If we get to this point, none of the processors were successful + SYS_LOGGER.log(Level.SEVERE, "All data source processors failed to process {0}", dataSource.getPath()); + jobLogger.logFailedToAddDataSource(); + // Throw an exception. It will get caught & handled upstream and will result in AIM auto-pause. + throw new AutomatedIngestDataSourceProcessor.AutomatedIngestDataSourceProcessorException("Failed to process " + dataSource.getPath() + " with all data source processors"); } } finally { currentJob.setDataSourceProcessor(null);