diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 6e1b921757..c244a16f7c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -920,7 +920,7 @@ final class DataSourceIngestJob { * * @return The currently running module, may be null. */ - DataSourceIngestPipeline.DataSourceIngestModuleDecorator getCurrentDataSourceIngestModule() { + DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() { if (null != this.currentDataSourceIngestPipeline) { return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule(); } else { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 734effc789..d4dd78ca34 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -28,13 +28,15 @@ import org.sleuthkit.datamodel.Content; * This class manages a sequence of data source level ingest modules. It starts * the modules, runs data sources through them, and shuts them down when data * source level ingest is complete. + *

+ * This class is not thread-safe. */ final class DataSourceIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); private final DataSourceIngestJob job; - private final List modules = new ArrayList<>(); - private volatile DataSourceIngestModuleDecorator currentModule; + private final List modules = new ArrayList<>(); + private volatile PipelineModule currentModule; /** * Constructs an object that manages a sequence of data source level ingest @@ -54,7 +56,7 @@ final class DataSourceIngestPipeline { */ for (IngestModuleTemplate template : moduleTemplates) { if (template.isDataSourceIngestModuleTemplate()) { - DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName()); + PipelineModule module = new PipelineModule(template.createDataSourceIngestModule(), template.getModuleName()); modules.add(module); } } @@ -76,7 +78,7 @@ final class DataSourceIngestPipeline { */ List startUp() { List errors = new ArrayList<>(); - for (DataSourceIngestModuleDecorator module : modules) { + for (PipelineModule module : modules) { try { module.startUp(new IngestJobContext(this.job)); } catch (Throwable ex) { // Catch-all exception firewall @@ -96,7 +98,7 @@ final class DataSourceIngestPipeline { List process(DataSourceIngestTask task) { List errors = new ArrayList<>(); Content dataSource = task.getDataSource(); - for (DataSourceIngestModuleDecorator module : modules) { + for (PipelineModule module : modules) { try { module.setStartTime(); this.currentModule = module; @@ -124,7 +126,7 @@ final class DataSourceIngestPipeline { /** * Gets the currently running module. */ - DataSourceIngestModuleDecorator getCurrentlyRunningModule() { + PipelineModule getCurrentlyRunningModule() { return this.currentModule; } @@ -132,7 +134,7 @@ final class DataSourceIngestPipeline { * This class decorates a data source level ingest module with a display * name and a start time. */ - static class DataSourceIngestModuleDecorator implements DataSourceIngestModule { + static class PipelineModule implements DataSourceIngestModule { private final DataSourceIngestModule module; private final String displayName; @@ -145,7 +147,7 @@ final class DataSourceIngestPipeline { * @param module The data source level ingest module to be decorated. * @param displayName The display name. */ - DataSourceIngestModuleDecorator(DataSourceIngestModule module, String displayName) { + PipelineModule(DataSourceIngestModule module, String displayName) { this.module = module; this.displayName = displayName; this.startTime = new Date(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 3bbdd7fa12..3e5b09f3ba 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -26,14 +26,17 @@ import org.sleuthkit.datamodel.AbstractFile; /** * This class manages a sequence of file level ingest modules. It starts the * modules, runs files through them, and shuts them down when file level ingest - * is complete. + * is complete. + *

+ * This class is not thread-safe. */ final class FileIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); private final DataSourceIngestJob job; - private final List modules = new ArrayList<>(); + private final List modules = new ArrayList<>(); private Date startTime; + private boolean running; /** * Constructs an object that manages a sequence of file level ingest @@ -53,24 +56,15 @@ final class FileIngestPipeline { */ for (IngestModuleTemplate template : moduleTemplates) { if (template.isFileIngestModuleTemplate()) { - FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); + PipelineModule module = new PipelineModule(template.createFileIngestModule(), template.getModuleName()); modules.add(module); } } } /** - * Returns the time when the pipeline began processing files. - * - * @return The file processing start time, may be null. - */ - Date getProcessingStartTime() { - return this.startTime; - } - - /** - * Queries whether or not the pipeline has been configured with at least one - * file level ingest module. + * Queries whether or not this pipeline has been configured with at least + * one file level ingest module. * * @return True or false. */ @@ -79,35 +73,54 @@ final class FileIngestPipeline { } /** - * Start up all of the modules in the pipeline. + * Starts up all of the modules in the pipeline. * - * @return List of errors or empty list if no errors + * @return List of start up errors, possibly empty. */ List startUp() { List errors = new ArrayList<>(); - for (FileIngestModuleDecorator module : this.modules) { + if (this.running) { + throw new IllegalStateException("Attempt to start up a pipeline that is already running"); //NON-NLS + } + + for (PipelineModule module : this.modules) { try { module.startUp(new IngestJobContext(this.job)); } catch (Throwable ex) { // Catch-all exception firewall errors.add(new IngestModuleError(module.getDisplayName(), ex)); } } + this.running = true; return errors; } + /** + * Returns the start up time of the pipeline. + * + * @return The file processing start time, may be null. + */ + Date getStartTime() { + return this.startTime; + } + /** * Runs a file through the ingest modules in sequential order. * * @param task A file level ingest task containing a file to be processed. - * @return A list of ingest module errors, possible empty. + * @return A list of processing errors, possible empty. */ List process(FileIngestTask task) { + if (!this.running) { + throw new IllegalStateException("Attempt to process a file with pipeline that is not running"); //NON-NLS + } + if (null == this.startTime) { this.startTime = new Date(); } + List errors = new ArrayList<>(); AbstractFile file = task.getFile(); - for (FileIngestModuleDecorator module : this.modules) { + for (PipelineModule module : this.modules) { try { FileIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName()); module.process(file); @@ -126,9 +139,18 @@ final class FileIngestPipeline { return errors; } + /** + * Shuts down all of the modules in the pipeline. + * + * @return A list of shut down errors, possibly empty. + */ List shutDown() { + if (!this.running) { + throw new IllegalStateException("Attempt to shut down a pipeline that is not running"); //NON-NLS + } + List errors = new ArrayList<>(); - for (FileIngestModuleDecorator module : this.modules) { + for (PipelineModule module : this.modules) { try { module.shutDown(); } catch (Throwable ex) { // Catch-all exception firewall @@ -138,10 +160,19 @@ final class FileIngestPipeline { return errors; } + /** + * Queries whether or not this file ingest level pipeline is running. + * + * @return True or false. + */ + boolean isRunning() { + return this.running; + } + /** * This class decorates a file level ingest module with a display name. */ - private static final class FileIngestModuleDecorator implements FileIngestModule { + private static final class PipelineModule implements FileIngestModule { private final FileIngestModule module; private final String displayName; @@ -153,7 +184,7 @@ final class FileIngestPipeline { * @param module The file level ingest module to be decorated. * @param displayName The display name. */ - FileIngestModuleDecorator(FileIngestModule module, String displayName) { + PipelineModule(FileIngestModule module, String displayName) { this.module = module; this.displayName = displayName; } @@ -168,7 +199,7 @@ final class FileIngestPipeline { } /** - * Gets a module name suitable for display in a UI. + * Gets display name of the decorated ingest module. * * @return The display name. */ diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index c7731aecb5..9c51e744d9 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -68,32 +68,34 @@ public final class IngestJob { * * @return The snapshot. */ - synchronized public Snapshot getSnapshot() { - // TODO: There are race conditions here that can be easily fixed by - // eliminating the child jobs per plan. + synchronized public ProgressSnapshot getSnapshot() { + /** + * There are race conditions in the code that follows, but they are not + * important because this is just a coarse-grained status report. If + * stale data is returned in any single snapshot, it will be corrected + * in subsequent snapshots. + */ DataSourceIngestModuleHandle moduleHandle = null; - Date fileAnalysisStartTime = null; + boolean fileIngestRunning = false; + Date fileIngestStartTime = null; for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { - /** - * Only one data source ingest module should be running at a time, - * so grab the first one. There is a race condition here. - */ - DataSourceIngestPipeline.DataSourceIngestModuleDecorator module = dataSourceJob.getCurrentDataSourceIngestModule(); + DataSourceIngestPipeline.PipelineModule module = dataSourceJob.getCurrentDataSourceIngestModule(); if (null != module) { moduleHandle = new DataSourceIngestModuleHandle(dataSourceJob.getId(), module); } - - + + // RJCTODO: For each data source job, check for a running flag and + // get the oldest start data for the start dates, if any. } - return new Snapshot(moduleHandle, fileAnalysisStartTime, this.cancelled); + return new ProgressSnapshot(moduleHandle, fileIngestRunning, fileIngestStartTime, this.cancelled); } /** * Gets snapshots of the state and performance of this ingest job's child * data source ingest jobs. * - * @return A list of data source ingest job snapshots. + * @return A list of data source ingest job progress snapshots. */ synchronized List getDetailedSnapshot() { List snapshots = new ArrayList<>(); @@ -104,29 +106,34 @@ public final class IngestJob { } /** + * Requests cancellation of a specific data source level ingest module. + * Returns immediately, but there may be a delay before the ingest module + * responds by stopping processing, if it is still running when the request + * is made. * - * @param module + * @param module The handle of the data source ingest module to be canceled, + * which can obtained from a progress snapshot. */ synchronized public void cancelIngestModule(DataSourceIngestModuleHandle module) { DataSourceIngestJob dataSourceJob = this.dataSourceJobs.get(module.dataSourceIngestJobId); - // RJCTODO: check equality, etc. + // RJCTODO: Pass through the cancellation request. } /** - * Requests cancellation of this ingest job, i.e., a shutdown of its data - * source level and file level ingest pipelines. + * Requests cancellation of the data source level and file level ingest + * modules of this ingest job. Returns immediately, but there may be a delay + * before all of the ingest modules respond by stopping processing. */ synchronized public void cancel() { for (DataSourceIngestJob job : this.dataSourceJobs.values()) { job.cancel(); } - this.cancelled = true; // RJCTODO: make children push cancellation up + this.cancelled = true; } /** - * Queries whether or not cancellation of this ingest job, i.e., a shutdown - * of its data source level and file level ingest pipelines, has been - * requested. + * Queries whether or not cancellation of the data source level and file + * level ingest modules of this ingest job has been requested. * * @return True or false. */ @@ -135,38 +142,72 @@ public final class IngestJob { } /** - * A thread-safe snapshot of ingest job progress. + * A snapshot of ingest job progress. */ - public static final class Snapshot { + public static final class ProgressSnapshot { private final DataSourceIngestModuleHandle dataSourceModule; - private final Date fileAnalysisStartTime; + private final boolean fileIngestRunning; + private final Date fileIngestStartTime; private final boolean cancelled; - private Snapshot(DataSourceIngestModuleHandle dataSourceModule, Date fileAnalysisStartTime, boolean cancelled) { + /** + * Constructs a snapshot of ingest job progress. + * + * @param dataSourceModule The currently running data source level + * ingest module, may be null + * @param fileIngestRunning Whether or not file ingest is currently + * running. + * @param fileIngestStartTime The start time of file level ingest, may + * be null + * @param cancelled Whether or not a cancellation request has been + * issued. + */ + private ProgressSnapshot(DataSourceIngestModuleHandle dataSourceModule, boolean fileIngestRunning, Date fileIngestStartTime, boolean cancelled) { this.dataSourceModule = dataSourceModule; - this.fileAnalysisStartTime = fileAnalysisStartTime; + this.fileIngestRunning = fileIngestRunning; + this.fileIngestStartTime = fileIngestStartTime; this.cancelled = cancelled; } + /** + * Gets a handle to the currently running data source level ingest + * module at the time the snapshot is taken. + * + * @return The handle, may be null. + */ public DataSourceIngestModuleHandle runningDataSourceIngestModule() { /** * It is safe to hand out this reference because the object is - * immutable and the mutable - * DataSourceIngestPipeline.DataSourceIngestModuleDecorator is - * hidden from public access. + * immutable. */ return this.dataSourceModule; } - public boolean fileAnalysisIsRunning() { - return (null != this.fileAnalysisStartTime); + /** + * Queries whether or not file level ingest is running at the time the + * snapshot is taken. + * + * @return True or false. + */ + public boolean fileIngestIsRunning() { + return this.fileIngestRunning; } - public Date fileAnalysisStartTime() { - return new Date(this.fileAnalysisStartTime.getTime()); + /** + * Gets the time that file level ingest started. + * + * @return The start time, may be null. + */ + public Date fileIngestStartTime() { + return new Date(this.fileIngestStartTime.getTime()); } + /** + * Queries whether or not a cancellation request has been issued. + * + * @return True or false. + */ public boolean isCancelled() { return this.cancelled; } @@ -181,20 +222,36 @@ public final class IngestJob { public static class DataSourceIngestModuleHandle { private final long dataSourceIngestJobId; - private final DataSourceIngestPipeline.DataSourceIngestModuleDecorator module; + private final DataSourceIngestPipeline.PipelineModule module; - private DataSourceIngestModuleHandle(long dataSourceIngestJobId, DataSourceIngestPipeline.DataSourceIngestModuleDecorator module) { + /** + * Constructs a handle to a data source level ingest module that can be + * used to get basic information about the module and to request + * cancellation of the module. + */ + private DataSourceIngestModuleHandle(long dataSourceIngestJobId, DataSourceIngestPipeline.PipelineModule module) { this.dataSourceIngestJobId = dataSourceIngestJobId; this.module = module; } + /** + * Gets the display name of the data source level ingest module + * associated with this handle. + * + * @return The display name. + */ public String displayName() { return this.module.getDisplayName(); } + /** + * Returns the time the data source level ingest module associated with + * this handle began processing. + * + * @return The module start time. + */ public Date startTime() { - // RJCTODO - return new Date(); + return this.module.getStartTime(); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 0602579188..13d4f4211e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -213,19 +213,15 @@ public class IngestManager { } /** - * Starts an ingest job, i.e., processing by ingest modules, for each data - * source in a collection of data sources. Note that if the provide UI - * argument is set to true, it is assumed this method is being called on the - * EDT and a worker thread will be dispatched to start the job. + * Starts an ingest job for a collection of data sources. * * @param dataSources The data sources to be processed. * @param settings The ingest job settings. - * @param provideUI Whether or not to support user interaction, e.g., - * showing message boxes and reporting progress through the NetBeans - * Progress API. + * @param doUI Whether or not to support user interaction, e.g., showing + * message boxes and reporting progress through the NetBeans Progress API. * @return The ingest job that was started */ - public synchronized void startIngestJobs(Collection dataSources, IngestJobSettings settings, boolean provideUI) { + public synchronized void startIngestJobs(Collection dataSources, IngestJobSettings settings, boolean doUI) { if (!isIngestRunning()) { clearIngestMessageBox(); } @@ -234,9 +230,17 @@ public class IngestManager { ingestMonitor.start(); } - long taskId = nextThreadId.incrementAndGet(); - Future task = startIngestJobsThreadPool.submit(new IngestJobsStarter(taskId, dataSources, settings, provideUI)); - ingestJobStarters.put(taskId, task); + if (doUI) { + /** + * Assume request is from code running on the EDT and dispatch to a + * worker thread. + */ + long taskId = nextThreadId.incrementAndGet(); + Future task = startIngestJobsThreadPool.submit(new IngestJobStarter(taskId, dataSources, settings, doUI)); + ingestJobStarters.put(taskId, task); + } else { + this.startJob(dataSources, settings); + } } /** @@ -245,6 +249,8 @@ public class IngestManager { * @return True or false. */ public boolean isIngestRunning() { + // RJCTODO: This may return the wrong answer if an IngestJobStarter has + // been dispatched to the startIngestJobsThreadPool. return !this.jobsById.isEmpty(); } @@ -640,10 +646,9 @@ public class IngestManager { } /** - * Creates and starts an ingest job, i.e., processing by ingest modules, for - * each data source in a collection of data sources. + * Creates and starts an ingest job for a collection of data sources. */ - private final class IngestJobsStarter implements Callable { + private final class IngestJobStarter implements Callable { private final long threadId; private final Collection dataSources; @@ -651,7 +656,7 @@ public class IngestManager { private final boolean doStartupErrorsMsgBox; private ProgressHandle progress; - IngestJobsStarter(long threadId, Collection dataSources, IngestJobSettings settings, boolean doMessageDialogs) { + IngestJobStarter(long threadId, Collection dataSources, IngestJobSettings settings, boolean doMessageDialogs) { this.threadId = threadId; this.dataSources = dataSources; this.settings = settings;