diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java index 268404faa0..96208f007f 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java @@ -211,10 +211,7 @@ class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel dataSources) { - IngestManager ingestManager = IngestManager.getInstance(); - for (Content dataSource : dataSources) { - ingestManager.startIngestJob(dataSource, this.settings, true); - } + IngestManager.getInstance().startIngestJobs(dataSources, this.settings, true); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 3a9a58a959..a336dd9bd9 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -22,6 +22,7 @@ import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -46,8 +47,8 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; /** - * Manages the creation and execution of ingest jobs, i.e., processing of data - * sources by ingest modules. + * Manages the creation and execution of ingest jobs, i.e., the processing of + * data sources by ingest modules. */ public class IngestManager { @@ -65,7 +66,7 @@ public class IngestManager { private final ExecutorService fileIngestThreadPool; private final ExecutorService fireIngestEventsThreadPool = Executors.newSingleThreadExecutor(); private final AtomicLong nextThreadId = new AtomicLong(0L); - private final ConcurrentHashMap> startIngestJobsCallables = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. + private final ConcurrentHashMap> ingestJobStarters = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final AtomicLong ingestErrorMessagePosts = new AtomicLong(0L); private final ConcurrentHashMap ingestThreadActivitySnapshots = new ConcurrentHashMap<>(); // Maps ingest thread ids to progress ingestThreadActivitySnapshots. private final ConcurrentHashMap ingestModuleRunTimes = new ConcurrentHashMap<>(); @@ -129,9 +130,10 @@ public class IngestManager { }; /** - * Gets the ingest manager. + * Gets the manager of the creation and execution of ingest jobs, i.e., the + * processing of data sources by ingest modules. * - * @return A singleton IngestManager object. + * @return A singleton ingest manager object. */ public synchronized static IngestManager getInstance() { if (instance == null) { @@ -156,15 +158,14 @@ public class IngestManager { } /** - * Starts an ingest job, i.e., processing by ingest modules, for a data - * source. + * Starts an ingest job, i.e., processing by ingest modules, for each data + * source in a collection of data sources. * - * @param dataSource The data source to be processed. + * @param dataSources The data sources to be processed. * @param settings The ingest job settings. - * @param doStartupErrorsMsgBox Whether or not to display ingest module - * startup errors in a message box. + * @param doMessageBoxes Whether or not to display message boxes for errors. */ - public synchronized void startIngestJob(Content dataSource, IngestJobSettings settings, boolean doStartupErrorsMsgBox) { + public synchronized void startIngestJobs(Collection dataSources, IngestJobSettings settings, boolean doMessageBoxes) { if (!isIngestRunning()) { clearIngestMessageBox(); } @@ -174,12 +175,12 @@ public class IngestManager { } long taskId = nextThreadId.incrementAndGet(); - Future task = startIngestJobsThreadPool.submit(new StartIngestJobsCallable(taskId, dataSource, settings, doStartupErrorsMsgBox)); - startIngestJobsCallables.put(taskId, task); + Future task = startIngestJobsThreadPool.submit(new IngestJobsStarter(taskId, dataSources, settings, doMessageBoxes)); + ingestJobStarters.put(taskId, task); } /** - * Queries whether any ingest jobs are in progress. + * Queries whether or not any ingest jobs are in progress. * * @return True or false. */ @@ -192,7 +193,7 @@ public class IngestManager { */ public void cancelAllIngestJobs() { // Stop creating new ingest jobs. - for (Future handle : startIngestJobsCallables.values()) { + for (Future handle : ingestJobStarters.values()) { handle.cancel(true); } @@ -250,7 +251,7 @@ public class IngestManager { } /** - * Remove an ingest job and ingest module event property change listener. + * Removes an ingest job and ingest module event property change listener. * * @param listener The PropertyChangeListener to unregister. * @deprecated Use removeIngestJobEventListener() and/or @@ -263,10 +264,11 @@ public class IngestManager { } /** - * Starts the ingest monitor and submits task execution tasks (Callable - * objects) to the data source ingest and file ingest thread pools. The task - * execution tasks are simple consumers that will normally run as long as - * the application runs + * Constructs a manager of the creation and execution of ingest jobs, i.e., + * the processing of data sources by ingest modules. The manager immediately + * submits ingest task executers (Callable objects) to the data source level + * ingest and file level ingest thread pools. The ingest task executers are + * simple consumers that will normally run as long as the application runs. */ private IngestManager() { startDataSourceIngestThread(); @@ -292,22 +294,22 @@ public class IngestManager { } /** - * Submits a ExecuteIngestTasksTask Callable to the data source ingest task + * Submits an ingest task executer Callable to the data source level ingest * thread pool. */ private void startDataSourceIngestThread() { long threadId = nextThreadId.incrementAndGet(); - dataSourceIngestThreadPool.submit(new ExecuteIngestTasksRunnable(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue())); + dataSourceIngestThreadPool.submit(new IngestTaskExecuter(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue())); ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); } /** - * Submits a ExecuteIngestTasksTask Callable to the data source ingest - * thread pool. + * Submits a ingest task executer Callable to the file level ingest thread + * pool. */ private void startFileIngestThread() { long threadId = nextThreadId.incrementAndGet(); - fileIngestThreadPool.submit(new ExecuteIngestTasksRunnable(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue())); + fileIngestThreadPool.submit(new IngestTaskExecuter(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue())); ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); } @@ -442,7 +444,7 @@ public class IngestManager { * @param ingestJobId The ingest job id. */ void fireIngestJobStarted(long ingestJobId) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestJobEventPublisher, IngestJobEvent.STARTED, ingestJobId, null)); + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestJobEventPublisher, IngestJobEvent.STARTED, ingestJobId, null)); } /** @@ -451,7 +453,7 @@ public class IngestManager { * @param ingestJobId The ingest job id. */ void fireIngestJobCompleted(long ingestJobId) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestJobEventPublisher, IngestJobEvent.COMPLETED, ingestJobId, null)); + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestJobEventPublisher, IngestJobEvent.COMPLETED, ingestJobId, null)); } /** @@ -460,7 +462,7 @@ public class IngestManager { * @param ingestJobId The ingest job id. */ void fireIngestJobCancelled(long ingestJobId) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestJobEventPublisher, IngestJobEvent.CANCELLED, ingestJobId, null)); + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestJobEventPublisher, IngestJobEvent.CANCELLED, ingestJobId, null)); } /** @@ -469,7 +471,7 @@ public class IngestManager { * @param file The file that is completed. */ void fireFileIngestDone(AbstractFile file) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestModuleEventPublisher, IngestModuleEvent.FILE_DONE, file.getId(), file)); + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestModuleEventPublisher, IngestModuleEvent.FILE_DONE, file.getId(), file)); } /** @@ -478,7 +480,7 @@ public class IngestManager { * @param moduleDataEvent A ModuleDataEvent with the details of the posting. */ void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestModuleEventPublisher, IngestModuleEvent.DATA_ADDED, moduleDataEvent, null)); + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestModuleEventPublisher, IngestModuleEvent.DATA_ADDED, moduleDataEvent, null)); } /** @@ -489,7 +491,7 @@ public class IngestManager { * content. */ void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) { - fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestModuleEventPublisher, IngestModuleEvent.CONTENT_CHANGED, moduleContentEvent, null)); + fireIngestEventsThreadPool.submit(new IngestEventPublisher(ingestModuleEventPublisher, IngestModuleEvent.CONTENT_CHANGED, moduleContentEvent, null)); } /** @@ -532,27 +534,31 @@ public class IngestManager { /** * Creates and starts an ingest job, i.e., processing by ingest modules, for - * a data source. + * each data source in a collection of data sources. */ - private final class StartIngestJobsCallable implements Callable { + private final class IngestJobsStarter implements Callable { private final long threadId; - private final Content dataSource; + private final Collection dataSources; private final IngestJobSettings settings; private final boolean doStartupErrorsMsgBox; private ProgressHandle progress; - StartIngestJobsCallable(long threadId, Content dataSource, IngestJobSettings settings, boolean doStartupErrorsMsgBox) { + IngestJobsStarter(long threadId, Collection dataSources, IngestJobSettings settings, boolean doMessageDialogs) { this.threadId = threadId; - this.dataSource = dataSource; + this.dataSources = dataSources; this.settings = settings; - this.doStartupErrorsMsgBox = doStartupErrorsMsgBox; + this.doStartupErrorsMsgBox = doMessageDialogs; } @Override public Void call() { try { - if (Thread.currentThread().isInterrupted()) { + /** + * Bail out if there is nothing to do or cancellation has been + * requested. + */ + if (this.dataSources.isEmpty() || Thread.currentThread().isInterrupted()) { return null; } @@ -569,63 +575,89 @@ public class IngestManager { "IngestManager.StartIngestJobsTask.run.cancelling", displayName)); } - Future handle = startIngestJobsCallables.remove(threadId); + Future handle = ingestJobStarters.remove(threadId); handle.cancel(true); return true; } }); - progress.switchToIndeterminate(); - progress.start(); + progress.start(dataSources.size()); /** - * Create and start an ingest job for the data source. + * Try to start the ingest jobs. */ - List errors = IngestJob.startJob(dataSource, this.settings); - if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { - // Report the errors to the user. They have already been logged. - StringBuilder moduleStartUpErrors = new StringBuilder(); - for (IngestModuleError error : errors) { - String moduleName = error.getModuleDisplayName(); - moduleStartUpErrors.append(moduleName); - moduleStartUpErrors.append(": "); - moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); - moduleStartUpErrors.append("\n"); - } - StringBuilder notifyMessage = new StringBuilder(); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); - notifyMessage.append("\n"); - notifyMessage.append(NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", - moduleStartUpErrors.toString())); - notifyMessage.append("\n\n"); - JOptionPane.showMessageDialog(null, notifyMessage.toString(), - NbBundle.getMessage(this.getClass(), - "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); - } - } catch (Exception ex) { - logger.log(Level.SEVERE, "Failed to create ingest job", ex); //NON-NLS - } finally { - progress.finish(); - startIngestJobsCallables.remove(threadId); - } + int workUnitsCompleted = 0; + for (Content dataSource : this.dataSources) { - return null; + /** + * Cancellation check. + */ + if (Thread.currentThread().isInterrupted()) { + return null; + } + + /** + * Add a "subtitle" to the display name of the progress bar + * to indicate an ingest job is being started for this data + * source. + */ + String progressMessage = NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.progressingDisplayName", + dataSource.getName()); + progress.progress(progressMessage); + + /** + * Start an ingest job for this data source. + */ + List errors = IngestJob.startJob(dataSource, this.settings); + if (!errors.isEmpty() && this.doStartupErrorsMsgBox) { + StringBuilder moduleStartUpErrors = new StringBuilder(); + for (IngestModuleError error : errors) { + String moduleName = error.getModuleDisplayName(); + moduleStartUpErrors.append(moduleName); + moduleStartUpErrors.append(": "); + moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); + moduleStartUpErrors.append("\n"); + } + StringBuilder notifyMessage = new StringBuilder(); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution")); + notifyMessage.append("\n"); + notifyMessage.append(NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList", + moduleStartUpErrors.toString())); + notifyMessage.append("\n\n"); + JOptionPane.showMessageDialog(null, notifyMessage.toString(), + NbBundle.getMessage(this.getClass(), + "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); + } + + progress.progress(progressMessage, ++workUnitsCompleted); + } + + return null; + + } finally { + if (null != progress) { + progress.finish(); + } + ingestJobStarters.remove(threadId); + } } + } /** * A consumer for an ingest task queue. */ - private final class ExecuteIngestTasksRunnable implements Runnable { + private final class IngestTaskExecuter implements Runnable { private final long threadId; private final IngestTaskQueue tasks; - ExecuteIngestTasksRunnable(long threadId, IngestTaskQueue tasks) { + IngestTaskExecuter(long threadId, IngestTaskQueue tasks) { this.threadId = threadId; this.tasks = tasks; } @@ -649,7 +681,7 @@ public class IngestManager { /** * Fires ingest events to ingest manager property change listeners. */ - private static final class FireIngestEventRunnable implements Runnable { + private static final class IngestEventPublisher implements Runnable { private final PropertyChangeSupport publisher; private final IngestJobEvent jobEvent; @@ -657,7 +689,7 @@ public class IngestManager { private final Object oldValue; private final Object newValue; - FireIngestEventRunnable(PropertyChangeSupport publisher, IngestJobEvent event, Object oldValue, Object newValue) { + IngestEventPublisher(PropertyChangeSupport publisher, IngestJobEvent event, Object oldValue, Object newValue) { this.publisher = publisher; this.jobEvent = event; this.moduleEvent = null; @@ -665,7 +697,7 @@ public class IngestManager { this.newValue = newValue; } - FireIngestEventRunnable(PropertyChangeSupport publisher, IngestModuleEvent event, Object oldValue, Object newValue) { + IngestEventPublisher(PropertyChangeSupport publisher, IngestModuleEvent event, Object oldValue, Object newValue) { this.publisher = publisher; this.jobEvent = null; this.moduleEvent = event; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java index f726a1d525..d8a7536ac1 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java @@ -198,14 +198,9 @@ public final class RunIngestModulesDialog extends JDialog { IngestJobSettings ingestJobSettings = this.ingestJobSettingsPanel.getSettings(); ingestJobSettings.save(); showWarnings(ingestJobSettings); - if (startIngestJob) { - IngestManager ingestManager = IngestManager.getInstance(); - for (Content dataSource : RunIngestModulesDialog.this.dataSources) { - ingestManager.startIngestJob(dataSource, ingestJobSettings, true); - } + IngestManager.getInstance().startIngestJobs(RunIngestModulesDialog.this.dataSources, ingestJobSettings, true); } - setVisible(false); dispose(); }