diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 97f082cce3..63152f4199 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -719,7 +719,7 @@ final class DataSourceIngestJob { * @param files A list of the files to add. */ void addFiles(List files) { - if (DataSourceIngestJob.Stages.FIRST == this.stage) { // RJCTODO: Is this synchronized correctly + if (DataSourceIngestJob.Stages.FIRST == this.stage) { for (AbstractFile file : files) { DataSourceIngestJob.taskScheduler.scheduleFileIngestTask(this, file); } @@ -970,7 +970,7 @@ final class DataSourceIngestJob { /** * Stores basic diagnostic statistics for a data source ingest job. */ - class Snapshot { + final class Snapshot { private final String dataSource; private final long jobId; @@ -1129,7 +1129,7 @@ final class DataSourceIngestJob { return this.tasksSnapshot.getRunningListSize(); } - boolean getCancelled() { + boolean isCancelled() { return this.jobCancelled; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 8218186b4c..961428148e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -20,10 +20,12 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.sleuthkit.datamodel.Content; @@ -38,7 +40,8 @@ public final class IngestJob { private static final AtomicLong nextId = new AtomicLong(0L); private final long id; private final Map dataSourceJobs; - private boolean cancelled; + private final AtomicInteger incompleteJobsCount; + private volatile boolean cancelled; /** * Constructs an ingest job that runs a collection of data sources through a @@ -51,11 +54,12 @@ public final class IngestJob { */ IngestJob(Collection dataSources, IngestJobSettings settings, boolean runInteractively) { this.id = IngestJob.nextId.getAndIncrement(); - this.dataSourceJobs = new HashMap<>(); + this.dataSourceJobs = new ConcurrentHashMap<>(); for (Content dataSource : dataSources) { DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, settings, runInteractively); this.dataSourceJobs.put(dataSourceIngestJob.getId(), dataSourceIngestJob); } + incompleteJobsCount = new AtomicInteger(dataSourceJobs.size()); } /** @@ -73,7 +77,7 @@ public final class IngestJob { * * @return True or false. */ - synchronized boolean hasIngestPipeline() { + boolean hasIngestPipeline() { for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { if (dataSourceJob.hasIngestPipeline()) { return true; @@ -88,16 +92,29 @@ public final class IngestJob { * * @return A collection of ingest module start up errors, empty on success. */ - synchronized List start() { + List start() { List errors = new ArrayList<>(); for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { errors.addAll(dataSourceJob.start()); if (!errors.isEmpty()) { - // RJCTODO: Need to let sucessfully started data source ingest - // jobs know they should shut down. break; } } + + /** + * TODO: Need to let successfully started data source ingest jobs know + * they should shut down. This means that the start up of the ingest + * module pipelines and the submission of ingest tasks should be + * separated. This cancellation is just a stop gap; fortunately, if + * startup is going to fail, it will likely fail for the first child + * data source ingest job. + */ + if (!errors.isEmpty()) { + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + dataSourceJob.cancel(); + } + } + return errors; } @@ -106,26 +123,8 @@ public final class IngestJob { * * @return The snapshot. */ - synchronized public ProgressSnapshot getSnapshot() { - DataSourceIngestModuleHandle moduleHandle = null; - boolean fileIngestIsRunning = false; - Date fileIngestStartTime = null; - for (DataSourceIngestJob.Snapshot snapshot : this.getDataSourceIngestJobSnapshots()) { - if (null == moduleHandle) { - DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule(); - if (null != module) { - moduleHandle = new DataSourceIngestModuleHandle(this.dataSourceJobs.get(snapshot.getJobId()), module); - } - } - if (snapshot.fileIngestIsRunning()) { - fileIngestIsRunning = true; - } - Date childFileIngestStartTime = snapshot.fileIngestStartTime(); - if (null != childFileIngestStartTime && (null == fileIngestStartTime || childFileIngestStartTime.before(fileIngestStartTime))) { - fileIngestStartTime = childFileIngestStartTime; - } - } - return new ProgressSnapshot(moduleHandle, fileIngestIsRunning, fileIngestStartTime, this.cancelled); + public ProgressSnapshot getSnapshot() { + return new ProgressSnapshot(); } /** @@ -134,7 +133,7 @@ public final class IngestJob { * * @return A list of data source ingest job progress snapshots. */ - synchronized List getDataSourceIngestJobSnapshots() { + List getDataSourceIngestJobSnapshots() { List snapshots = new ArrayList<>(); for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { snapshots.add(dataSourceJob.getSnapshot()); @@ -148,7 +147,7 @@ public final class IngestJob { * but there may be a delay before all of the ingest modules in the * pipelines respond by stopping processing. */ - synchronized public void cancel() { + public void cancel() { for (DataSourceIngestJob job : this.dataSourceJobs.values()) { job.cancel(); } @@ -161,7 +160,7 @@ public final class IngestJob { * * @return True or false. */ - synchronized public boolean isCancelled() { + public boolean isCancelled() { return this.cancelled; } @@ -171,9 +170,8 @@ public final class IngestJob { * * @param dataSourceIngestJob A completed data source ingest job. */ - synchronized void dataSourceJobFinished(DataSourceIngestJob dataSourceIngestJob) { - this.dataSourceJobs.remove(dataSourceIngestJob.getId()); - if (this.dataSourceJobs.isEmpty()) { + void dataSourceJobFinished(DataSourceIngestJob dataSourceIngestJob) { + if (incompleteJobsCount.decrementAndGet() == 0) { IngestManager.getInstance().finishIngestJob(this); } } @@ -181,30 +179,85 @@ public final class IngestJob { /** * A snapshot of the progress of an ingest job. */ - public static final class ProgressSnapshot { + public final class ProgressSnapshot { - private final DataSourceIngestModuleHandle dataSourceModule; - private final boolean fileIngestRunning; - private final Date fileIngestStartTime; - private final boolean cancelled; + private final List dataSourceProcessingSnapshots; + private DataSourceIngestModuleHandle dataSourceModule; + private boolean fileIngestRunning; + private Date fileIngestStartTime; + private final boolean jobCancelled; + + /** + * A snapshot of the progress of an ingest job on the processing of a + * data source. + */ + public final class DataSourceProcessingSnapshot { + + private final DataSourceIngestJob.Snapshot snapshot; + + private DataSourceProcessingSnapshot(DataSourceIngestJob.Snapshot snapshot) { + this.snapshot = snapshot; + } + + /** + * Gets the name of the data source that is the subject of this + * snapshot. + * + * @return A data source name string. + */ + public String getDataSource() { + return snapshot.getDataSource(); + } + + /** + * Indicates whether or not the processing of the data source that + * is the subject of this snapshot was canceled. + * + * @return True or false. + */ + public boolean isCancelled() { + return snapshot.isCancelled(); + } + + /** + * Gets a list of the display names of any canceled data source + * level ingest modules. + * + * @return A list of canceled data source level ingest module + * display names, possibly empty. + */ + public List getCancelledDataSourceIngestModules() { + return snapshot.getCancelledDataSourceIngestModules(); + } + + } /** * Constructs a snapshot of ingest job progress. - * - * @param dataSourceModule The currently running data source level - * ingest module, may be null. - * @param fileIngestRunning Whether or not file ingest is currently - * running. - * @param fileIngestStartTime The start time of file level ingest, may - * be null. - * @param cancelled Whether or not a cancellation request has been - * issued. */ - private ProgressSnapshot(DataSourceIngestModuleHandle dataSourceModule, boolean fileIngestRunning, Date fileIngestStartTime, boolean cancelled) { - this.dataSourceModule = dataSourceModule; - this.fileIngestRunning = fileIngestRunning; - this.fileIngestStartTime = fileIngestStartTime; - this.cancelled = cancelled; + private ProgressSnapshot() { + dataSourceModule = null; + fileIngestRunning = false; + fileIngestStartTime = null; + dataSourceProcessingSnapshots = new ArrayList<>(); + for (DataSourceIngestJob dataSourceJob : dataSourceJobs.values()) { + DataSourceIngestJob.Snapshot snapshot = dataSourceJob.getSnapshot(); + dataSourceProcessingSnapshots.add(new DataSourceProcessingSnapshot(snapshot)); + if (null == dataSourceModule) { + DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule(); + if (null != module) { + dataSourceModule = new DataSourceIngestModuleHandle(dataSourceJobs.get(snapshot.getJobId()), module); + } + } + if (snapshot.fileIngestIsRunning()) { + fileIngestRunning = true; + } + Date childFileIngestStartTime = snapshot.fileIngestStartTime(); + if (null != childFileIngestStartTime && (null == fileIngestStartTime || childFileIngestStartTime.before(fileIngestStartTime))) { + fileIngestStartTime = childFileIngestStartTime; + } + } + this.jobCancelled = cancelled; } /** @@ -243,7 +296,16 @@ public final class IngestJob { * @return True or false. */ public boolean isCancelled() { - return this.cancelled; + return this.jobCancelled; + } + + /** + * Gets snapshots of the progress processing individual data sources. + * + * @return The list of snapshots. + */ + public List getDataSourceSnapshots() { + return Collections.unmodifiableList(this.dataSourceProcessingSnapshots); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index b0cb987710..a1cb07858e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -404,7 +404,7 @@ public class IngestManager { boolean success = false; if (this.jobCreationIsEnabled) { - if (runInteractively && jobsById.isEmpty()) { // RJCTODO: This is sort of broken + if (runInteractively && jobsById.isEmpty()) { clearIngestMessageBox(); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index 55e19676aa..96784a86c7 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -149,7 +149,7 @@ class QueryResults { if (writeResult != null) { newArtifacts.add(writeResult.getArtifact()); if (notifyInbox) { - writeSingleFileInboxMessage(writeResult, hit.getContent()); // RJCTODO: Consider rewriting this message post code + writeSingleFileInboxMessage(writeResult, hit.getContent()); } } else { logger.log(Level.WARNING, "BB artifact for keyword hit not written, file: {0}, hit: {1}", new Object[]{hit.getContent(), keyword.toString()}); //NON-NLS