diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 9f43879b55..a779085813 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -37,7 +37,6 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.EnumSet; @@ -59,13 +58,11 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.stream.Collectors; import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.Immutable; -import javax.annotation.concurrent.ThreadSafe; -import org.openide.util.Exceptions; import org.openide.util.Lookup; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case.CaseType; import org.sleuthkit.autopsy.casemodule.CaseActionException; +import org.sleuthkit.autopsy.casemodule.CaseDetails; import org.sleuthkit.autopsy.casemodule.CaseMetadata; import org.sleuthkit.autopsy.coordinationservice.CoordinationService; import org.sleuthkit.autopsy.coordinationservice.CoordinationService.CoordinationServiceException; @@ -101,7 +98,6 @@ import org.sleuthkit.autopsy.ingest.IngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestJobStartResult; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestModuleError; -import org.sleuthkit.datamodel.Content; /** * An auto ingest manager is responsible for processing auto ingest jobs defined @@ -503,7 +499,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang } SYS_LOGGER.log(Level.INFO, "Starting input scan of {0}", rootInputDirectory); InputDirScanner scanner = new InputDirScanner(); - scanner.scan(); SYS_LOGGER.log(Level.INFO, "Completed input scan of {0}", rootInputDirectory); } @@ -559,12 +554,10 @@ public final class AutoIngestManager extends Observable implements PropertyChang if (!prioritizedJobs.isEmpty()) { ++maxPriority; for (AutoIngestJob job : prioritizedJobs) { - int oldPriority = job.getPriority(); - job.setPriority(maxPriority); try { this.updateCoordinationServiceNode(job); + job.setPriority(maxPriority); } catch (CoordinationServiceException | InterruptedException ex) { - job.setPriority(oldPriority); throw new AutoIngestManagerException("Error updating case priority", ex); } } @@ -615,14 +608,12 @@ public final class AutoIngestManager extends Observable implements PropertyChang */ if (null != prioritizedJob) { ++maxPriority; - int oldPriority = prioritizedJob.getPriority(); - prioritizedJob.setPriority(maxPriority); try { this.updateCoordinationServiceNode(prioritizedJob); } catch (CoordinationServiceException | InterruptedException ex) { - prioritizedJob.setPriority(oldPriority); throw new AutoIngestManagerException("Error updating job priority", ex); } + prioritizedJob.setPriority(maxPriority); } Collections.sort(pendingJobs, new AutoIngestJob.PriorityComparator()); @@ -1051,8 +1042,8 @@ public final class AutoIngestManager extends Observable implements PropertyChang if (null != manifest) { /* - * Update the mapping of case names to manifest paths that - * is used for case deletion. + * Update the mapping of case names to manifest paths that is + * used for case deletion. */ String caseName = manifest.getCaseName(); Path manifestPath = manifest.getFilePath(); @@ -1066,8 +1057,8 @@ public final class AutoIngestManager extends Observable implements PropertyChang } /* - * Add a job to the pending jobs queue, the completed jobs - * list, or do crashed job recovery, as required. + * Add a job to the pending jobs queue, the completed jobs list, + * or do crashed job recovery, as required. */ try { byte[] rawData = coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString()); @@ -1087,7 +1078,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang break; case DELETED: /* - * Ignore jobs marked as "deleted." + * Ignore jobs marked as "deleted." */ break; default: @@ -1247,48 +1238,38 @@ public final class AutoIngestManager extends Observable implements PropertyChang if (null != manifestLock) { SYS_LOGGER.log(Level.SEVERE, "Attempting crash recovery for {0}", manifestPath); try { + Path caseDirectoryPath = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName()); + /* * Create the recovery job. */ AutoIngestJob job = new AutoIngestJob(nodeData); int numberOfCrashes = job.getNumberOfCrashes(); - ++numberOfCrashes; - job.setNumberOfCrashes(numberOfCrashes); - job.setCompletedDate(new Date(0)); - Path caseDirectoryPath = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName()); + if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) { + ++numberOfCrashes; + job.setNumberOfCrashes(numberOfCrashes); + if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) { + job.setCompletedDate(new Date(0)); + } else { + job.setCompletedDate(Date.from(Instant.now())); + } + } + if (null != caseDirectoryPath) { job.setCaseDirectoryPath(caseDirectoryPath); job.setErrorsOccurred(true); - } else { - job.setErrorsOccurred(false); - } - - /* - * Update the coordination service node for the job. If - * this fails, leave the recovery to another host. - */ - try { - updateCoordinationServiceNode(job); - if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) { - newPendingJobsList.add(job); - } else { - newCompletedJobsList.add(new AutoIngestJob(nodeData)); - } - } catch (CoordinationServiceException ex) { - SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifestPath), ex); - return; - } - - /* - * Write the alert file and do the logging. - */ - if (null != caseDirectoryPath) { try { - AutoIngestAlertFile.create(nodeData.getCaseDirectoryPath()); + /* + * Write the alert file and do the logging. + */ + AutoIngestAlertFile.create(caseDirectoryPath); } catch (AutoIngestAlertFileException ex) { SYS_LOGGER.log(Level.SEVERE, String.format("Error creating alert file for crashed job for %s", manifestPath), ex); } + } else { + job.setErrorsOccurred(false); } + if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) { job.setProcessingStatus(AutoIngestJob.ProcessingStatus.PENDING); if (null != caseDirectoryPath) { @@ -1302,13 +1283,32 @@ public final class AutoIngestManager extends Observable implements PropertyChang job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED); if (null != caseDirectoryPath) { try { - new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), nodeData.getCaseDirectoryPath()).logCrashRecoveryNoRetry(); + new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), caseDirectoryPath).logCrashRecoveryNoRetry(); } catch (AutoIngestJobLoggerException ex) { SYS_LOGGER.log(Level.SEVERE, String.format("Error creating case auto ingest log entry for crashed job for %s", manifestPath), ex); } } } + /* + * Update the coordination service node for the job. If + * this fails, leave the recovery to another host. + */ + try { + updateCoordinationServiceNode(job); + } catch (CoordinationServiceException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifestPath), ex); + return; + } + + nodeData = new AutoIngestJobNodeData(job); + + if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) { + newPendingJobsList.add(job); + } else { + newCompletedJobsList.add(new AutoIngestJob(nodeData)); + } + } finally { try { manifestLock.release(); @@ -1459,7 +1459,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang */ private final class JobProcessingTask implements Runnable { - private static final String AUTO_INGEST_MODULE_OUTPUT_DIR = "AutoIngest"; private final Object ingestLock; private final Object pauseLock; @GuardedBy("pauseLock") @@ -2126,7 +2125,8 @@ public final class AutoIngestManager extends Observable implements PropertyChang Case.openAsCurrentCase(metadataFilePath.toString()); } else { caseDirectoryPath = PathUtils.createCaseFolderPath(rootOutputDirectory, caseName); - Case.createAsCurrentCase(caseDirectoryPath.toString(), caseName, "", "", CaseType.MULTI_USER_CASE); + CaseDetails caseDetails = new CaseDetails(caseName, "", "", "", "", ""); + Case.createAsCurrentCase(CaseType.MULTI_USER_CASE, caseDirectoryPath.toString(), caseDetails); /* * Sleep a bit before releasing the lock to ensure * that the new case folder is visible on the @@ -2222,7 +2222,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang return; } - DataSource dataSource = identifyDataSource(caseForJob); + DataSource dataSource = identifyDataSource(); if (null == dataSource) { currentJob.setProcessingStage(AutoIngestJob.Stage.COMPLETED, Date.from(Instant.now())); return; @@ -2275,7 +2275,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * interrupted while blocked, i.e., * if auto ingest is shutting down. */ - private DataSource identifyDataSource(Case caseForJob) throws AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { + private DataSource identifyDataSource() throws AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { Manifest manifest = currentJob.getManifest(); Path manifestPath = manifest.getFilePath(); SYS_LOGGER.log(Level.INFO, "Identifying data source for {0} ", manifestPath); @@ -2294,7 +2294,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang String deviceId = manifest.getDeviceId(); return new DataSource(deviceId, dataSourcePath); } - + /** * Passes the data source for the current job through a data source * processor that adds it to the case database. @@ -2317,28 +2317,21 @@ public final class AutoIngestManager extends Observable implements PropertyChang SYS_LOGGER.log(Level.INFO, "Adding data source for {0} ", manifestPath); currentJob.setProcessingStage(AutoIngestJob.Stage.ADDING_DATA_SOURCE, Date.from(Instant.now())); UUID taskId = UUID.randomUUID(); - DataSourceProcessorCallback callBack = new AddDataSourceCallback(caseForJob, dataSource, taskId); + DataSourceProcessorCallback callBack = new AddDataSourceCallback(caseForJob, dataSource, taskId, ingestLock); DataSourceProcessorProgressMonitor progressMonitor = new DoNothingDSPProgressMonitor(); Path caseDirectoryPath = currentJob.getCaseDirectoryPath(); AutoIngestJobLogger jobLogger = new AutoIngestJobLogger(manifestPath, manifest.getDataSourceFileName(), caseDirectoryPath); try { caseForJob.notifyAddingDataSource(taskId); - // lookup all AutomatedIngestDataSourceProcessors - Collection processorCandidates = Lookup.getDefault().lookupAll(AutoIngestDataSourceProcessor.class); - - Map validDataSourceProcessorsMap = new HashMap<>(); - for (AutoIngestDataSourceProcessor processor : processorCandidates) { - try { - int confidence = processor.canProcess(dataSource.getPath()); - if (confidence > 0) { - validDataSourceProcessorsMap.put(processor, confidence); - } - } catch (AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException ex) { - SYS_LOGGER.log(Level.SEVERE, "Exception while determining whether data source processor {0} can process {1}", new Object[]{processor.getDataSourceType(), dataSource.getPath()}); - // rethrow the exception. It will get caught & handled upstream and will result in AIM auto-pause. - throw ex; - } + Map validDataSourceProcessorsMap; + try { + // lookup all AutomatedIngestDataSourceProcessors and poll which ones are able to process the current data source + validDataSourceProcessorsMap = DataSourceProcessorUtility.getDataSourceProcessor(dataSource.getPath()); + } catch (AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException ex) { + SYS_LOGGER.log(Level.SEVERE, "Exception while determining best data source processor for {0}", dataSource.getPath()); + // rethrow the exception. It will get caught & handled upstream and will result in AIM auto-pause. + throw ex; } // did we find a data source processor that can process the data source @@ -2595,80 +2588,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang jobLogger.logFileExportError(); } } - - /** - * A "callback" that collects the results of running a data source - * processor on a data source and unblocks the job processing thread - * when the data source processor finishes running in its own thread. - */ - @Immutable - class AddDataSourceCallback extends DataSourceProcessorCallback { - - private final Case caseForJob; - private final DataSource dataSourceInfo; - private final UUID taskId; - - /** - * Constructs a "callback" that collects the results of running a - * data source processor on a data source and unblocks the job - * processing thread when the data source processor finishes running - * in its own thread. - * - * @param caseForJob The case for the current job. - * @param dataSourceInfo The data source - * @param taskId The task id to associate with ingest job - * events. - */ - AddDataSourceCallback(Case caseForJob, DataSource dataSourceInfo, UUID taskId) { - this.caseForJob = caseForJob; - this.dataSourceInfo = dataSourceInfo; - this.taskId = taskId; - } - - /** - * Called by the data source processor when it finishes running in - * its own thread. - * - * @param result The result code for the processing of - * the data source. - * @param errorMessages Any error messages generated during the - * processing of the data source. - * @param dataSourceContent The content produced by processing the - * data source. - */ - @Override - public void done(DataSourceProcessorCallback.DataSourceProcessorResult result, List errorMessages, List dataSourceContent) { - if (!dataSourceContent.isEmpty()) { - caseForJob.notifyDataSourceAdded(dataSourceContent.get(0), taskId); - } else { - caseForJob.notifyFailedAddingDataSource(taskId); - } - dataSourceInfo.setDataSourceProcessorOutput(result, errorMessages, dataSourceContent); - dataSourceContent.addAll(dataSourceContent); - synchronized (ingestLock) { - ingestLock.notify(); - } - } - - /** - * Called by the data source processor when it finishes running in - * its own thread, if that thread is the AWT (Abstract Window - * Toolkit) event dispatch thread (EDT). - * - * @param result The result code for the processing of - * the data source. - * @param errorMessages Any error messages generated during the - * processing of the data source. - * @param dataSourceContent The content produced by processing the - * data source. - */ - @Override - public void doneEDT(DataSourceProcessorCallback.DataSourceProcessorResult result, List errorMessages, List dataSources) { - done(result, errorMessages, dataSources); - } - - } - + /** * A data source processor progress monitor does nothing. There is * currently no mechanism for showing or recording data source processor @@ -3008,49 +2928,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang PARTIALLY_DELETED, FULLY_DELETED } - - @ThreadSafe - private static final class DataSource { - - private final String deviceId; - private final Path path; - private DataSourceProcessorResult resultCode; - private List errorMessages; - private List content; - - DataSource(String deviceId, Path path) { - this.deviceId = deviceId; - this.path = path; - } - - String getDeviceId() { - return deviceId; - } - - Path getPath() { - return this.path; - } - - synchronized void setDataSourceProcessorOutput(DataSourceProcessorResult result, List errorMessages, List content) { - this.resultCode = result; - this.errorMessages = new ArrayList<>(errorMessages); - this.content = new ArrayList<>(content); - } - - synchronized DataSourceProcessorResult getResultDataSourceProcessorResultCode() { - return resultCode; - } - - synchronized List getDataSourceProcessorErrorMessages() { - return new ArrayList<>(errorMessages); - } - - synchronized List getContent() { - return new ArrayList<>(content); - } - - } - + static final class AutoIngestManagerException extends Exception { private static final long serialVersionUID = 1L;