diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED index 9ceb1bae41..9de7eedde0 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED @@ -1,7 +1,7 @@ CTL_RunIngestAction=Run Ingest FileIngestPipeline_SaveResults_Activity=Saving Results # {0} - data source name -IngestJob.progress.analysisResultIngest.displayName=Analyzing analysis results from {0} +IngestJob_progress_analysisResultIngest_displayName=Analyzing analysis results from {0} IngestJobSettingsPanel.IngestModulesTableRenderer.info.message=A previous version of this ingest module has been run before on this data source. IngestJobSettingsPanel.IngestModulesTableRenderer.warning.message=This ingest module has been run before on this data source. IngestJobSettingsPanel.noPerRunSettings=The selected module has no per-run settings. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index c991e8f1f2..e16a889086 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -181,10 +181,10 @@ public final class IngestJob { * Starts data source level analysis for this job if it is running in * streaming ingest mode. */ - void processStreamingIngestDataSource() { + void addStreamedDataSource() { if (ingestMode == Mode.STREAMING) { if (ingestModuleExecutor != null) { - ingestModuleExecutor.startStreamingModeDataSourceAnalysis(); + ingestModuleExecutor.addStreamedDataSource(); } else { logger.log(Level.SEVERE, "Attempted to start data source analaysis with no ingest pipeline"); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index b3ceae48e7..758d15be02 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -33,6 +33,7 @@ import java.util.regex.Pattern; import java.util.stream.Stream; import javax.annotation.concurrent.GuardedBy; import javax.swing.JOptionPane; +import javax.swing.SwingUtilities; import org.netbeans.api.progress.ProgressHandle; import org.openide.util.Cancellable; import org.openide.util.NbBundle; @@ -41,7 +42,7 @@ import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.NetworkUtils; -import org.sleuthkit.autopsy.ingest.IngestTasksScheduler.IngestTasksSnapshot; +import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.IngestJobInfo; @@ -57,8 +58,9 @@ import org.sleuthkit.datamodel.DataArtifact; import org.sleuthkit.datamodel.DataSource; /** - * Executes an ingest job the orchestrating the construction, start up, running, - * and shut down of the ingest module pipelines for an ingest job. + * Executes an ingest job by orchestrating the construction, start up, running, + * and shut down of the ingest module pipelines that perform the ingest tasks + * for the job. */ final class IngestJobExecutor { @@ -73,10 +75,10 @@ final class IngestJobExecutor { private static final Pattern JYTHON_MODULE_REGEX = Pattern.compile("org\\.python\\.proxies\\.(.+?)\\$(.+?)(\\$[0-9]*)?$"); /* - * These fields are the identity of this object: the parent ingest job, the - * ingest job settings, and the data source to be analyzed by the ingest - * module pipelines. Optionally, there is a set of files to be analyzed, - * instead of analyzing all of the files in the data source. + * These fields are the identity of this object: the ingest job to be + * executed, the ingest job settings, and the data source to be analyzed by + * the ingest module pipelines. Optionally, there is a set of files to be + * analyzed instead of analyzing all of the files in the data source. */ private final IngestJob ingestJob; private final IngestJobSettings settings; @@ -98,12 +100,12 @@ final class IngestJobExecutor { * the number of file ingest threads in the ingest manager. References to * the file ingest pipelines are put into two collections, each with its own * purpose. A blocking queue allows file ingest threads to take and return - * file ingest pipelines, as they work through the file ingest tasks for one - * or more ingest jobs. Having the same number of pipelines as threads - * ensures that a file ingest thread will never be idle, as long as there - * are file ingest tasks still to do, regardless of the number of ingest - * jobs in progress. Additionally, a fixed list is used to cycle through the - * file ingest module pipelines when making ingest progress snapshots. + * file ingest pipeline copies, as they work through the file ingest tasks + * for the job. Having the same number of pipelines as file ingest threads + * ensures that a thread will never be idle, as long as there are file + * ingest tasks still to do, regardless of the number of ingest jobs in + * progress. Additionally, a fixed list is used to cycle through the file + * ingest module pipelines when making ingest progress snapshots. */ private final LinkedBlockingQueue fileIngestPipelinesQueue = new LinkedBlockingQueue<>(); private final List fileIngestPipelines = new ArrayList<>(); @@ -124,9 +126,9 @@ final class IngestJobExecutor { */ private static enum IngestJobStage { /* - * In this stage, the ingest module pipelines are constructed per the - * user's ingest job settings. This stage ends when all of the ingest - * module pipelines for the ingest job are ready to run. + * In this stage, the ingest module pipelines for the ingest job are + * constructed per the ingest job settings. This stage ends when all of + * the ingest module pipelines for the ingest job are ready to run. */ PIPELINES_START_UP, /* @@ -145,7 +147,8 @@ final class IngestJobExecutor { * ingest job is configured to have a data artifact and/or analysis * result ingest pipeline, those pipelines are also analyzing any data * artifacts and/or analysis results generated by the file and/or data - * source level ingest modules. + * source level ingest modules. This stage ends when all of the ingest + * tasks for the stage are completed. */ FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS, /** @@ -154,7 +157,8 @@ final class IngestJobExecutor { * is configured to have a data artifact and/or analysis result ingest * pipeline, those pipelines are also analyzing any data artifacts * and/or analysis results generated by the data source level ingest - * modules. + * modules. This stage ends when all of the ingest tasks for the stage + * are completed. */ LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS, /** @@ -163,22 +167,19 @@ final class IngestJobExecutor { */ PIPELINES_SHUT_DOWN }; + private volatile IngestJobStage stage = IngestJobExecutor.IngestJobStage.PIPELINES_START_UP; /* - * The stage field is volatile to allow it to be read by multiple threads. - * So the stage transition lock is used not to guard the stage field, but to - * coordinate stage transitions. + * The stage transition lock is used to coordinate stage transitions, not to + * guard the volatile stage field. */ - private volatile IngestJobStage stage = IngestJobExecutor.IngestJobStage.PIPELINES_START_UP; private final Object stageTransitionLock = new Object(); /* - * During each stage of an ingest job, the ingest job executor interacts - * with the ingest task scheduler to create ingest tasks for analyzing the - * data source, files, data artifacts, and analysis results that are the - * subject of the ingest job. The scheduler queues the ingest tasks for the - * ingest manager's ingest threads. The ingest tasks are the units of work - * for the ingest module pipelines. + * During each stage of the ingest job, the ingest job executor interacts + * with the ingest task scheduler to create ingest tasks for job. The + * scheduler queues the ingest tasks for the ingest manager's ingest + * threads. */ private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); @@ -193,6 +194,14 @@ final class IngestJobExecutor { * cancellation means that there can be a variable length delay between a * cancellation request and its fulfillment. Analysis already completed at * the time that cancellation occurs is NOT discarded. + * + * Note that the DataSourceIngestModule interface does not currently have a + * cancel() API. As a consequence, cancelling an individual data source + * ingest module requires setting and then unsetting a cancellation flag. + * Because of this, there is no ironclad guarantee that the correct module + * will be cancelled. We are relying on the module being long-running to + * avoid a race condition between module cancellation and the transition of + * the execution of a data source level ingest task to another module. */ private volatile boolean currentDataSourceIngestModuleCancelled; private final List cancelledDataSourceIngestModules = new CopyOnWriteArrayList<>(); @@ -201,30 +210,28 @@ final class IngestJobExecutor { /* * If running in the NetBeans thick client application version of Autopsy, - * NetBeans progress bars are used to display ingest job progress in the - * lower right hand corner of the main application window. A layer of - * abstraction to allow alternate representations of progress could be used - * here, as it is in other places in the application, to better decouple - * this object from the application's presentation layer. + * NetBeans progress handles (i.e., progress bars) are used to display + * ingest job progress in the lower right hand corner of the main + * application window. + * + * A layer of abstraction to allow alternate representations of progress + * could be used here, as it is in other places in the application (see + * implementations and usage of + * org.sleuthkit.autopsy.progress.ProgressIndicator interface). This would + * better decouple this object from the application's presentation layer. */ private final boolean usingNetBeansGUI; - private final Object dataSourceIngestProgressLock = new Object(); - @GuardedBy("dataSourceIngestProgressLock") + @ThreadConfined(type = ThreadConfined.ThreadType.AWT) private ProgressHandle dataSourceIngestProgressBar; - private final Object fileIngestProgressLock = new Object(); - @GuardedBy("fileIngestProgressLock") + @ThreadConfined(type = ThreadConfined.ThreadType.AWT) private final List filesInProgress = new ArrayList<>(); - @GuardedBy("fileIngestProgressLock") - private long estimatedFilesToProcess; - @GuardedBy("fileIngestProgressLock") - private long processedFiles; - @GuardedBy("fileIngestProgressLock") + private volatile long estimatedFilesToProcess; + private volatile long processedFiles; + @ThreadConfined(type = ThreadConfined.ThreadType.AWT) private ProgressHandle fileIngestProgressBar; - private final Object artifactIngestProgressLock = new Object(); - @GuardedBy("artifactIngestProgressLock") + @ThreadConfined(type = ThreadConfined.ThreadType.AWT) private ProgressHandle artifactIngestProgressBar; - private final Object resultIngestProgressLock = new Object(); - @GuardedBy("resultIngestProgressLock") + @ThreadConfined(type = ThreadConfined.ThreadType.AWT) private ProgressHandle resultIngestProgressBar; /* @@ -246,20 +253,21 @@ final class IngestJobExecutor { /** * Constructs an object that executes an ingest job by orchestrating the - * construction, start up, ingest task execution, and shut down of the - * ingest module pipelines for an ingest job. - * + * construction, start up, running, and shut down of the ingest module + * pipelines that perform the ingest tasks for the job. * * @param ingestJob The ingest job. - * @param dataSource The data source. + * @param dataSource The data source that is the subject of the ingest job. * @param files A subset of the files from the data source. If the list * is empty, ALL of the files in the data source are an * analyzed. * @param settings The ingest job settings. * - * @throws InterruptedException Exception thrown if the thread in which the - * pipeline is being created is interrupted. + * @throws InterruptedException The exception is thrown if the thread in + * which the pipeline is being created is + * interrupted. */ + // RJCTODO: Refactor so that only the job is passed in and the other params are obtained from the job. IngestJobExecutor(IngestJob ingestJob, Content dataSource, List files, IngestJobSettings settings) throws InterruptedException { if (!(dataSource instanceof DataSource)) { throw new IllegalArgumentException("Passed dataSource that does not implement the DataSource interface"); //NON-NLS @@ -577,8 +585,8 @@ final class IngestJobExecutor { } /** - * Determnines which inges job stage to start in and starts up the ingest - * module pipelines. + * Determnines which ingest job stage to start in and starts up the ingest + * module pipelines for all of the stages. * * @return A collection of ingest module startup errors, empty on success. */ @@ -600,13 +608,12 @@ final class IngestJobExecutor { } /** - * Starts up the ingest module pipelines in this ingest. Note that all of - * the child pipelines are started so that any and all start up errors can - * be returned to the caller. It is important to capture all of the errors, - * because the ingest job will be automatically cancelled and the errors - * will be reported to the user so either the issues can be addressed or the - * modules that can't start up can be disabled before the ingest job is - * attempted again. + * Starts up the ingest module pipelines. Note that ALL of the pipelines are + * started, so that any and all start up errors can be returned to the + * caller. It is important to capture all of the errors, because the ingest + * job will be automatically cancelled, and the errors will be reported to + * the user. This allows the user to either address the issues, or to + * disable the modules that can't start up, and attempt the job again. * * @return A list of ingest module startup errors, empty on success. */ @@ -702,50 +709,59 @@ final class IngestJobExecutor { /** * Starts analysis for a batch mode ingest job. For a batch mode job, all of * the files in the data source (excepting carved and derived files) have - * already been added to the case database by the data source processor and - * analysis starts in the file and high priority data source level analysis - * stage. + * already been added to the case database by the data source processor + * (DSP) and analysis starts in the file and high priority data source level + * analysis stage. */ private void startBatchModeAnalysis() { synchronized (stageTransitionLock) { - logInfoMessage(String.format("Starting analysis in batch mode for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), ingestJob.getId())); //NON-NLS + logInfoMessage("Starting ingest job in batch mode"); //NON-NLS stage = IngestJobStage.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS; if (hasFileIngestModules()) { /* - * Do a count of the files the data source processor has added - * to the case database. This number will be used to estimate - * how many files remain to be analyzed as each file ingest task - * is completed. + * Do an estimate of the total number of files to be analyzed. + * This will be used to estimate of how many files remain to be + * analyzed as each file ingest task is completed. The numbers + * are estimates because analysis can add carved files and/or + * derived files to the job. */ - long filesToProcess; if (files.isEmpty()) { - filesToProcess = dataSource.accept(new GetFilesCountVisitor()); + /* + * Do a count of the files the data source processor (DSP) + * has added to the case database. + */ + estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor()); } else { - filesToProcess = files.size(); - } - synchronized (fileIngestProgressLock) { - estimatedFilesToProcess = filesToProcess; + /* + * Otherwise, this job is analyzing a user-specified subset + * of the files in the data source. + */ + estimatedFilesToProcess = files.size(); } + startFileIngestProgressBar(); } - if (usingNetBeansGUI) { + if (hasHighPriorityDataSourceIngestModules()) { + startDataSourceIngestProgressBar(); + } + + if (hasDataArtifactIngestModules()) { /* - * Start ingest progress bars in the lower right hand corner of - * the main application window. + * Note that even if there are no other ingest module pipelines, + * analysis of any data artifacts already in the case database + * will be performed. */ - if (hasFileIngestModules()) { - startFileIngestProgressBar(); - } - if (hasHighPriorityDataSourceIngestModules()) { - startDataSourceIngestProgressBar(); - } - if (hasDataArtifactIngestModules()) { - startDataArtifactIngestProgressBar(); - } - if (hasAnalysisResultIngestModules()) { - startAnalysisResultIngestProgressBar(); - } + startDataArtifactIngestProgressBar(); + } + + if (hasAnalysisResultIngestModules()) { + /* + * Note that even if there are no other ingest module pipelines, + * analysis of any analysis results already in the case database + * will be performed. + */ + startAnalysisResultIngestProgressBar(); } /* @@ -755,65 +771,86 @@ final class IngestJobExecutor { currentDataSourceIngestPipeline = highPriorityDataSourceIngestPipeline; /* - * Schedule ingest tasks and then immediately check for stage - * completion. This is necessary because it is possible that zero - * tasks will actually make it to task execution due to the file - * filter or other ingest job settings. In that case, there will - * never be a stage completion check in an ingest thread executing - * an ingest task, so such a job would run forever without a check - * here. + * Schedule ingest tasks. */ if (!files.isEmpty() && hasFileIngestModules()) { + /* + * If only analyzing a subset of the files in the data source, + * only schedule file ingest tasks for those files. Data + * artifact ingest tasks will be scheduled as data artifacts + * produced by the file analysis are posted to the blackboard. + */ taskScheduler.scheduleFileIngestTasks(this, files); } else if (hasHighPriorityDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { + /* + * Otherwise, schedule tasks based on the ingest job settings, + * i.e., the enabled ingest modules. + */ taskScheduler.scheduleIngestTasks(this); } + + /* + * Check for stage completion. This is necessary because it is + * possible that none of the tasks that were just scheduled will + * actually make it to task execution, due to the file filter or + * other ingest job settings. If that happens, there will never be + * another stage completion check for this job in an ingest thread + * executing an ingest task, so such a job would run forever without + * a check here. + */ checkForStageCompleted(); } } /** - * Starts analysis for a streaming mode ingest job. For a streaming mode - * job, the data source processor streams files in as it adds them to the - * case database and file analysis can begin before data source level - * analysis. + * Starts analysis for a streaming mode ingest job. Streaming mode is + * typically used to allow a data source processor (DSP) to streams file to + * this ingest job executor as it adds the files to the case database. This + * alternative to waiting until the DSP completes its processing allows file + * level analysis to begin before data source level analysis. */ private void startStreamingModeAnalysis() { synchronized (stageTransitionLock) { - logInfoMessage("Starting data source level analysis in streaming mode"); //NON-NLS + logInfoMessage("Starting ingest job in streaming mode"); //NON-NLS stage = IngestJobStage.STREAMED_FILE_ANALYSIS_ONLY; - if (usingNetBeansGUI) { + if (hasFileIngestModules()) { /* - * Start ingest progress bars in the lower right hand corner of - * the main application window. + * Start the file ingest progress bar, but do not schedule any + * file or data source ingest tasks. File ingest tasks will + * instead be scheduled as files are streamed in via + * addStreamedFiles(), and a data source ingest task will be + * scheduled later, via addStreamedDataSource(). + * + * Note that because estimated files remaining to process still + * has its initial value of zero, the fle ingest progress bar + * will start in the "indeterminate" state. A rough estimate of + * the files to be processed will be computed later, when all of + * the files have been added to the case database, as signalled + * by a call to the addStreamedDataSource(). */ - if (hasFileIngestModules()) { - /* - * Note that because estimated files remaining to process - * still has its initial value of zero, the progress bar - * will start in the "indeterminate" state. An estimate of - * the files to process can be computed later, when all of - * the files have been added ot the case database. - */ - startFileIngestProgressBar(); - } - if (hasDataArtifactIngestModules()) { - startDataArtifactIngestProgressBar(); - } - if (hasAnalysisResultIngestModules()) { - startAnalysisResultIngestProgressBar(); - } + estimatedFilesToProcess = 0; + startFileIngestProgressBar(); } - if (hasDataArtifactIngestModules()) { // RJCTODO - /* - * Schedule artifact ingest tasks for any artifacts currently in - * the case database. This needs to be done before any files or - * the data source are streamed in to avoid analyzing the data - * artifacts added to the case database by those tasks twice. - */ + /* + * Start the data artifact and analysis result progress bars and + * schedule ingest tasks for any data artifacts and analysis results + * currently in the case database. This needs to be done BEFORE any + * files or the data source are streamed in. This is necessary to + * avoid analyzing any artifacts or results added to the case + * database by the file and data source ingest tasks twice. To + * ensure this happens, a streaming mode ingest job is created and + * started in IngestManager.openIngestStream(), before the ingest + * manager returns the IngestStream object that the client uses to + * call addStreamedFiles() and addStreamedDataSource(). + */ + if (hasDataArtifactIngestModules()) { + startDataArtifactIngestProgressBar(); taskScheduler.scheduleDataArtifactIngestTasks(this); + } + if (hasAnalysisResultIngestModules()) { + startAnalysisResultIngestProgressBar(); taskScheduler.scheduleAnalysisResultIngestTasks(this); } } @@ -821,10 +858,10 @@ final class IngestJobExecutor { /** * Signals in streaming mode that all of the files have been added to the - * case database and streamed in, and the data source is now ready for - * analysis. + * case database and streamed in to this ingest job executor, and the data + * source is now ready for analysis. */ - void startStreamingModeDataSourceAnalysis() { + void addStreamedDataSource() { synchronized (stageTransitionLock) { logInfoMessage("Starting full first stage analysis in streaming mode"); //NON-NLS stage = IngestJobExecutor.IngestJobStage.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS; @@ -832,43 +869,36 @@ final class IngestJobExecutor { if (hasFileIngestModules()) { /* - * Do a count of the files the data source processor has added - * to the case database. This number will be used to estimate - * how many files remain to be analyzed as each file ingest task - * is completed. + * For ingest job progress reporting purposes, do a count of the + * files the data source processor has added to the case + * database. */ - long filesToProcess = dataSource.accept(new GetFilesCountVisitor()); - synchronized (fileIngestProgressLock) { - estimatedFilesToProcess = filesToProcess; - if (usingNetBeansGUI && fileIngestProgressBar != null) { - fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); - } - } - } - - if (usingNetBeansGUI) { - /* - * Start a data source level ingest progress bar in the lower - * right hand corner of the main application window. The file - * and data artifact ingest progress bars were already started - * in startStreamingModeAnalysis(). - */ - if (hasHighPriorityDataSourceIngestModules()) { - startDataSourceIngestProgressBar(); - } + estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor()); + switchFileIngestProgressBarToDeterminate(); } currentDataSourceIngestPipeline = highPriorityDataSourceIngestPipeline; if (hasHighPriorityDataSourceIngestModules()) { + /* + * Start a data source level ingest progress bar in the lower + * right hand corner of the main application window. The file, + * data artifact, and analysis result ingest progress bars were + * already started in startStreamingModeAnalysis(). + */ + startDataSourceIngestProgressBar(); + + /* + * Schedule a task for the data source. + */ IngestJobExecutor.taskScheduler.scheduleDataSourceIngestTask(this); } else { /* - * If no data source level ingest task is scheduled at this time - * and all of the file level and artifact ingest tasks scheduled - * during the initial file streaming stage have already - * executed, there will never be a stage completion check in an - * ingest thread executing an ingest task, so such a job would - * run forever without a check here. + * If no data source level ingest task is scheduled at this + * time, and all of the file level and artifact ingest tasks + * scheduled during the initial file streaming stage have + * already been executed, there will never be a stage completion + * check in an ingest thread executing an ingest task for this + * job, so such a job would run forever without a check here. */ checkForStageCompleted(); } @@ -881,13 +911,9 @@ final class IngestJobExecutor { private void startLowPriorityDataSourceAnalysis() { synchronized (stageTransitionLock) { if (hasLowPriorityDataSourceIngestModules()) { - logInfoMessage(String.format("Starting low priority data source analysis for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), ingestJob.getId())); //NON-NLS + logInfoMessage("Starting low priority data source analysis"); //NON-NLS stage = IngestJobExecutor.IngestJobStage.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS; - - if (usingNetBeansGUI) { - startDataSourceIngestProgressBar(); - } - + startDataSourceIngestProgressBar(); currentDataSourceIngestPipeline = lowPriorityDataSourceIngestPipeline; taskScheduler.scheduleDataSourceIngestTask(this); } @@ -895,77 +921,71 @@ final class IngestJobExecutor { } /** - * Starts a data artifacts analysis NetBeans progress bar in the lower right - * hand corner of the main application window. The progress bar provides the - * user with a task cancellation button. Pressing it cancels the ingest job. - * Analysis already completed at the time that cancellation occurs is NOT - * discarded. + * Starts a NetBeans progress bar for data artifacts analysis in the lower + * right hand corner of the main application window. The progress bar + * provides the user with a task cancellation button. Pressing it cancels + * the entire ingest job. Analysis already completed at the time that + * cancellation occurs is NOT discarded. */ private void startDataArtifactIngestProgressBar() { if (usingNetBeansGUI) { - artifactIngestProgressBar = startArtifactIngestProgressBar(artifactIngestProgressLock, NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", dataSource.getName())); - } - } - - /** - * Starts a analysis results analysis NetBeans progress bar in the lower - * right hand corner of the main application window. The progress bar - * provides the user with a task cancellation button. Pressing it cancels - * the ingest job. Analysis already completed at the time that cancellation - * occurs is NOT discarded. - */ - @NbBundle.Messages({ - "# {0} - data source name", "IngestJob.progress.analysisResultIngest.displayName=Analyzing analysis results from {0}" - }) - private void startAnalysisResultIngestProgressBar() { - if (usingNetBeansGUI) { - resultIngestProgressBar = startArtifactIngestProgressBar(resultIngestProgressLock, Bundle.IngestJob_progress_analysisResultIngest_displayName(dataSource.getName())); - } - } - - /** - * Starts a data artifacts or analysis results analysis NetBeans progress - * bar in the lower right hand corner of the main application window. The - * progress bar provides the user with a task cancellation button. Pressing - * it cancels the ingest job. Analysis already completed at the time that - * cancellation occurs is NOT discarded. - * - * @param progressBarLock The lock for the progress bar. - * @param displayName The display name for the progress bar. - * - * @return The progress bar. - */ - private ProgressHandle startArtifactIngestProgressBar(Object progressBarLock, String displayName) { - ProgressHandle progressBar = null; - if (usingNetBeansGUI) { - synchronized (progressBarLock) { - progressBar = ProgressHandle.createHandle(displayName, new Cancellable() { + SwingUtilities.invokeLater(() -> { + artifactIngestProgressBar = ProgressHandle.createHandle(NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", this.dataSource.getName()), new Cancellable() { @Override public boolean cancel() { - IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + new Thread(() -> { + IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + }).start(); return true; } }); - progressBar.start(); - progressBar.switchToIndeterminate(); - } + artifactIngestProgressBar.start(); + artifactIngestProgressBar.switchToIndeterminate(); + }); } - return progressBar; } /** - * Starts a data source level analysis NetBeans progress bar in the lower + * Starts a NetBeans progress bar for analysis results analysis in the lower * right hand corner of the main application window. The progress bar * provides the user with a task cancellation button. Pressing it cancels - * either the currently running data source level ingest module or the + * the entire ingest job. Analysis already completed at the time that + * cancellation occurs is NOT discarded. + */ + @NbBundle.Messages({ + "# {0} - data source name", + "IngestJob_progress_analysisResultIngest_displayName=Analyzing analysis results from {0}" + }) + private void startAnalysisResultIngestProgressBar() { + if (usingNetBeansGUI) { + SwingUtilities.invokeLater(() -> { + resultIngestProgressBar = ProgressHandle.createHandle(Bundle.IngestJob_progress_analysisResultIngest_displayName(dataSource.getName()), new Cancellable() { + @Override + public boolean cancel() { + new Thread(() -> { + IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + }).start(); + return true; + } + }); + resultIngestProgressBar.start(); + resultIngestProgressBar.switchToIndeterminate(); + }); + } + } + + /** + * Starts a NetBeans progress bar for data source level analysis in the + * lower right hand corner of the main application window. The progress bar + * provides the user with a task cancellation button. Pressing it cancels + * either the currently running data source level ingest module, or the * entire ingest job. Analysis already completed at the time that * cancellation occurs is NOT discarded. */ private void startDataSourceIngestProgressBar() { if (usingNetBeansGUI) { - synchronized (dataSourceIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", dataSource.getName()); - dataSourceIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { + SwingUtilities.invokeLater(() -> { + dataSourceIngestProgressBar = ProgressHandle.createHandle(NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", dataSource.getName()), new Cancellable() { @Override public boolean cancel() { /* @@ -980,40 +1000,88 @@ final class IngestJobExecutor { String dialogTitle = NbBundle.getMessage(IngestJobExecutor.this.getClass(), "IngestJob.cancellationDialog.title"); JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); if (panel.cancelAllDataSourceIngestModules()) { - IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + new Thread(() -> { + IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + }).start(); } else { - IngestJobExecutor.this.cancelCurrentDataSourceIngestModule(); + new Thread(() -> { + IngestJobExecutor.this.cancelCurrentDataSourceIngestModule(); + }).start(); } return true; } }); dataSourceIngestProgressBar.start(); dataSourceIngestProgressBar.switchToIndeterminate(); - } + }); } } /** - * Starts a file analysis NetBeans progress bar in the lower right hand + * Starts a NetBeans progress bar for file analysis in the lower right hand * corner of the main application window. The progress bar provides the user - * with a task cancellation button. Pressing it cancels the ingest job. - * Analysis already completed at the time that cancellation occurs is NOT - * discarded. + * with a task cancellation button. Pressing it cancels the entire ingest + * job. Analysis already completed at the time that cancellation occurs is + * NOT discarded. */ private void startFileIngestProgressBar() { if (usingNetBeansGUI) { - synchronized (fileIngestProgressLock) { - String displayName = NbBundle.getMessage(getClass(), "IngestJob.progress.fileIngest.displayName", dataSource.getName()); - fileIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { + SwingUtilities.invokeLater(() -> { + fileIngestProgressBar = ProgressHandle.createHandle(NbBundle.getMessage(getClass(), "IngestJob.progress.fileIngest.displayName", dataSource.getName()), new Cancellable() { @Override public boolean cancel() { - IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + new Thread(() -> { + IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + }).start(); return true; } }); fileIngestProgressBar.start(); - fileIngestProgressBar.switchToDeterminate((int) this.estimatedFilesToProcess); - } + fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); + }); + } + } + + /** + * Finishes the first stage ingest progress bars. + */ + private void finishFirstStageProgressBars() { + if (usingNetBeansGUI) { + SwingUtilities.invokeLater(() -> { + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; + } + + if (fileIngestProgressBar != null) { + fileIngestProgressBar.finish(); + fileIngestProgressBar = null; + } + }); + } + } + + /** + * Finishes all of the ingest progress bars. + */ + private void finishAllProgressBars() { + if (usingNetBeansGUI) { + SwingUtilities.invokeLater(() -> { + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; + } + + if (fileIngestProgressBar != null) { + fileIngestProgressBar.finish(); + fileIngestProgressBar = null; + } + + if (artifactIngestProgressBar != null) { + artifactIngestProgressBar.finish(); + artifactIngestProgressBar = null; + } + }); } } @@ -1054,21 +1122,7 @@ final class IngestJobExecutor { shutDownIngestModulePipeline(pipeline); } - if (usingNetBeansGUI) { - synchronized (dataSourceIngestProgressLock) { - if (dataSourceIngestProgressBar != null) { - dataSourceIngestProgressBar.finish(); - dataSourceIngestProgressBar = null; - } - } - - synchronized (fileIngestProgressLock) { - if (fileIngestProgressBar != null) { - fileIngestProgressBar.finish(); - fileIngestProgressBar = null; - } - } - } + finishFirstStageProgressBars(); if (!jobCancelled && hasLowPriorityDataSourceIngestModules()) { startLowPriorityDataSourceAnalysis(); @@ -1079,46 +1133,16 @@ final class IngestJobExecutor { } /** - * Shuts down the ingest module pipelines and progress bars. + * Shuts down the ingest module pipelines and ingest job progress bars. */ private void shutDown() { synchronized (stageTransitionLock) { - logInfoMessage("Finished all tasks"); //NON-NLS + logInfoMessage("Finished all ingest tasks"); //NON-NLS stage = IngestJobExecutor.IngestJobStage.PIPELINES_SHUT_DOWN; - shutDownIngestModulePipeline(currentDataSourceIngestPipeline); shutDownIngestModulePipeline(artifactIngestPipeline); shutDownIngestModulePipeline(resultIngestPipeline); - - if (usingNetBeansGUI) { - synchronized (dataSourceIngestProgressLock) { - if (dataSourceIngestProgressBar != null) { - dataSourceIngestProgressBar.finish(); - dataSourceIngestProgressBar = null; - } - } - - synchronized (fileIngestProgressLock) { - if (fileIngestProgressBar != null) { - fileIngestProgressBar.finish(); - fileIngestProgressBar = null; - } - } - - synchronized (artifactIngestProgressLock) { - if (artifactIngestProgressBar != null) { - artifactIngestProgressBar.finish(); - artifactIngestProgressBar = null; - } - } - - synchronized (resultIngestProgressLock) { - if (resultIngestProgressBar != null) { - resultIngestProgressBar.finish(); - resultIngestProgressBar = null; - } - } - } + finishAllProgressBars(); if (ingestJobInfo != null) { if (jobCancelled) { @@ -1165,7 +1189,7 @@ final class IngestJobExecutor { * data source level ingest module pipeline (high-priority or low-priority). * * @param task A data source ingest task encapsulating the data source and - * the data source ingest pipeline to use to execute the task. + * the data source ingest pipeline. */ void execute(DataSourceIngestTask task) { try { @@ -1187,7 +1211,7 @@ final class IngestJobExecutor { * ingest module pipeline. * * @param task A file ingest task encapsulating the file and the file ingest - * pipeline to use to execute the task. + * pipeline. */ void execute(FileIngestTask task) { try { @@ -1195,8 +1219,8 @@ final class IngestJobExecutor { FileIngestPipeline pipeline = fileIngestPipelinesQueue.take(); if (!pipeline.isEmpty()) { /* - * Get the file from the task. If the file was "streamed," - * the task may only have the file object ID and a trip to + * Get the file from the task. If the file was streamed in, + * the task may only have the file object ID, and a trip to * the case database will be required. */ AbstractFile file; @@ -1210,46 +1234,24 @@ final class IngestJobExecutor { return; } - synchronized (fileIngestProgressLock) { - ++processedFiles; - if (usingNetBeansGUI) { - if (processedFiles <= estimatedFilesToProcess) { - fileIngestProgressBar.progress(file.getName(), (int) processedFiles); - } else { - fileIngestProgressBar.progress(file.getName(), (int) estimatedFilesToProcess); - } - filesInProgress.add(file.getName()); - } - } - /** - * Run the file through the modules in the pipeline. + * Run the file through the modules in the file ingest + * pipeline. */ + final String fileName = file.getName(); + processedFiles++; + updateFileIngestProgressForFileTaskStarted(fileName); List errors = new ArrayList<>(); errors.addAll(pipeline.performTask(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors, file); } - - if (usingNetBeansGUI && !jobCancelled) { - synchronized (fileIngestProgressLock) { - /** - * Update the file ingest progress bar again, in - * case the file was being displayed. - */ - filesInProgress.remove(file.getName()); - if (filesInProgress.size() > 0) { - fileIngestProgressBar.progress(filesInProgress.get(0)); - } else { - fileIngestProgressBar.progress(""); - } - } - } + updateFileProgressBarForFileTaskCompleted(fileName); } fileIngestPipelinesQueue.put(pipeline); } } catch (InterruptedException ex) { - logger.log(Level.SEVERE, String.format("Unexpected interrupt of file ingest thread during execution of file ingest job (file obj ID = %d)", task.getFileId()), ex); + logger.log(Level.SEVERE, String.format("File ingest thread interrupted during execution of file ingest job (file object ID = %d, thread ID = %d)", task.getFileId(), task.getThreadId()), ex); Thread.currentThread().interrupt(); } finally { taskScheduler.notifyTaskCompleted(task); @@ -1262,8 +1264,7 @@ final class IngestJobExecutor { * the data artifact ingest module pipeline. * * @param task A data artifact ingest task encapsulating the data artifact - * and the data artifact ingest pipeline to use to execute the - * task. + * and the data artifact ingest pipeline. */ void execute(DataArtifactIngestTask task) { try { @@ -1285,8 +1286,7 @@ final class IngestJobExecutor { * through the analysis result ingest module pipeline. * * @param task An analysis result ingest task encapsulating the analysis - * result and the analysis result ingest pipeline to use to - * execute the task. + * result and the analysis result ingest pipeline. */ void execute(AnalysisResultIngestTask task) { try { @@ -1304,8 +1304,7 @@ final class IngestJobExecutor { } /** - * Adds some streamed files for analysis as part of a streaming mode ingest - * job. + * Streams in files for analysis as part of a streaming mode ingest job. * * @param fileObjIds The object IDs of the files. */ @@ -1320,7 +1319,8 @@ final class IngestJobExecutor { } /** - * Adds additional files (e.g., extracted or carved files) for analysis. + * Adds additional files produced by ingest modules (e.g., extracted or + * carved files) for analysis. * * @param files A list of the files to add. */ @@ -1336,7 +1336,7 @@ final class IngestJobExecutor { * The intended clients of this method are ingest modules running code * in an ingest thread that is holding a reference to a "primary" ingest * task that was the source of the files, in which case a completion - * check would not be necessary, so this is a bit of defensive + * check would not be necessary, so this check is a bit of defensive * programming. */ checkForStageCompleted(); @@ -1392,100 +1392,206 @@ final class IngestJobExecutor { /** * Updates the display name shown on the current data source level ingest - * progress bar for this job. + * progress bar for this job, if the job has not been cancelled. This is + * intended to be called by data source level ingest modules and the display + * name should reference the ingest module name. * * @param displayName The new display name. */ void updateDataSourceIngestProgressBarDisplayName(String displayName) { if (usingNetBeansGUI && !jobCancelled) { - synchronized (dataSourceIngestProgressLock) { + SwingUtilities.invokeLater(() -> { if (dataSourceIngestProgressBar != null) { dataSourceIngestProgressBar.setDisplayName(displayName); } - } + }); } } /** * Switches the current data source level ingest progress bar to determinate - * mode. This should be called if the total work units to process the data - * source is known. + * mode, if the job has not been cancelled. This should be called if the + * total work units to process the data source is known. This is intended to + * be called by data source level ingest modules in conjunction with + * updateDataSourceIngestProgressBarDisplayName(). * * @param workUnits Total number of work units for the processing of the * data source. */ void switchDataSourceIngestProgressBarToDeterminate(int workUnits) { if (usingNetBeansGUI && !jobCancelled) { - synchronized (dataSourceIngestProgressLock) { + SwingUtilities.invokeLater(() -> { if (dataSourceIngestProgressBar != null) { dataSourceIngestProgressBar.switchToDeterminate(workUnits); } - } + }); } } /** * Switches the current data source level ingest progress bar to - * indeterminate mode. This should be called if the total work units to - * process the data source is unknown. + * indeterminate mode, if the job has not been cancelled. This should be + * called if the total work units to process the data source is unknown. + * This is intended to be called by data source level ingest modules in + * conjunction with updateDataSourceIngestProgressBarDisplayName(). */ void switchDataSourceIngestProgressBarToIndeterminate() { if (usingNetBeansGUI && !jobCancelled) { - synchronized (dataSourceIngestProgressLock) { + SwingUtilities.invokeLater(() -> { if (dataSourceIngestProgressBar != null) { dataSourceIngestProgressBar.switchToIndeterminate(); } - } + }); } } /** * Updates the current data source level ingest progress bar with the number - * of work units performed, if in the determinate mode. + * of work units performed, if in the determinate mode, and the job has not + * been cancelled. This is intended to be called by data source level ingest + * modules that have called + * switchDataSourceIngestProgressBarToDeterminate(). * * @param workUnits Number of work units performed. */ void advanceDataSourceIngestProgressBar(int workUnits) { if (usingNetBeansGUI && !jobCancelled) { - synchronized (dataSourceIngestProgressLock) { + SwingUtilities.invokeLater(() -> { if (dataSourceIngestProgressBar != null) { dataSourceIngestProgressBar.progress("", workUnits); } - } + }); } } /** * Updates the current data source level ingest progress bar with a new task - * name, where the task name is the "subtitle" under the display name. + * name, where the task name is the "subtitle" under the display name, if + * the job has not been cancelled. * * @param currentTask The task name. */ void advanceDataSourceIngestProgressBar(String currentTask) { if (usingNetBeansGUI && !jobCancelled) { - synchronized (dataSourceIngestProgressLock) { + SwingUtilities.invokeLater(() -> { if (dataSourceIngestProgressBar != null) { dataSourceIngestProgressBar.progress(currentTask); } - } + }); } } /** * Updates the current data source level ingest progress bar with a new task - * name and the number of work units performed, if in the determinate mode. - * The task name is the "subtitle" under the display name. + * name and the number of work units performed, if in the determinate mode, + * and the job has not been cancelled. The task name is the "subtitle" under + * the display name. * * @param currentTask The task name. * @param workUnits Number of work units performed. */ void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { if (usingNetBeansGUI && !jobCancelled) { - synchronized (dataSourceIngestProgressLock) { + SwingUtilities.invokeLater(() -> { if (dataSourceIngestProgressBar != null) { dataSourceIngestProgressBar.progress(currentTask, workUnits); } - } + }); + } + } + + /** + * Switches the file ingest progress bar to determinate mode, using the + * estimated number of files to process as the number of work units. + */ + private void switchFileIngestProgressBarToDeterminate() { + if (usingNetBeansGUI) { + SwingUtilities.invokeLater(() -> { + if (fileIngestProgressBar != null) { + fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); + } + }); + } + } + + /** + * Updates the current file ingest progress bar upon start of analysis of a + * file, if the job has not been cancelled. + * + * @param fileName The name of the file. + */ + private void updateFileIngestProgressForFileTaskStarted(String fileName) { + if (usingNetBeansGUI && !jobCancelled) { + SwingUtilities.invokeLater(() -> { + /* + * If processedFiles exceeds estimatedFilesToProcess, i.e., the + * max work units set for the progress bar, the progress bar + * will go into an infinite loop throwing + * IllegalArgumentExceptions in the EDT (NetBeans bug). Also, a + * check-then-act race condition needs to be avoided here. This + * can be done without guarding processedFiles and + * estimatedFilesToProcess with the same lock because + * estimatedFilesToProcess does not change after it is used to + * switch the progress bar to determinate mode. + */ + long processedFilesCapture = processedFiles; + if (processedFilesCapture <= estimatedFilesToProcess) { + fileIngestProgressBar.progress(fileName, (int) processedFilesCapture); + } else { + fileIngestProgressBar.progress(fileName, (int) estimatedFilesToProcess); + } + filesInProgress.add(fileName); + }); + } + } + + /** + * Updates the current file ingest progress bar upon completion of analysis + * of a file, if the job has not been cancelled. + * + * @param fileName The name of the file. + */ + private void updateFileProgressBarForFileTaskCompleted(String fileName) { + if (usingNetBeansGUI && !jobCancelled) { + SwingUtilities.invokeLater(() -> { + filesInProgress.remove(fileName); + /* + * Display the name of another file in progress, or the empty + * string if there are none. + */ + if (filesInProgress.size() > 0) { + fileIngestProgressBar.progress(filesInProgress.get(0)); + } else { + fileIngestProgressBar.progress(""); // NON-NLS + } + }); + } + } + + /** + * Displays a "cancelling" message on all of the current ingest message + * progress bars. + */ + private void displayCancellingProgressMessages() { + if (usingNetBeansGUI) { + SwingUtilities.invokeLater(() -> { + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", dataSource.getName())); + dataSourceIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); + } + if (fileIngestProgressBar != null) { + fileIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.fileIngest.displayName", dataSource.getName())); + fileIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); + } + if (artifactIngestProgressBar != null) { + artifactIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.dataArtifactIngest.displayName", dataSource.getName())); + artifactIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); + } + if (resultIngestProgressBar != null) { + resultIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.resultArtifactIngest.displayName", dataSource.getName())); + resultIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); + } + }); } } @@ -1494,6 +1600,15 @@ final class IngestJobExecutor { * ingest in order to stop the currently executing data source level ingest * module is in effect for this job. * + * Note that the DataSourceIngestModule interface does not currently have a + * cancel() API. As a consequence, cancelling an individual data source + * ingest module requires setting and then unsetting the + * currentDataSourceIngestModuleCancelled flag. Because of this, there is no + * ironclad guarantee that the correct module will be cancelled. We are + * relying on the module being long-running to avoid a race condition + * between module cancellation and the transition of the execution of a data + * source level ingest task to another module. + * * @return True or false. */ boolean currentDataSourceIngestModuleIsCancelled() { @@ -1502,27 +1617,37 @@ final class IngestJobExecutor { /** * Rescinds a temporary cancellation of data source level ingest that was - * used to stop a single data source level ingest module for this job. + * used to stop a single data source level ingest module for this job. The + * data source ingest progress bar is reset, if the job has not been + * cancelled. + * + * Note that the DataSourceIngestModule interface does not currently have a + * cancel() API. As a consequence, cancelling an individual data source + * ingest module requires setting and then unsetting the + * currentDataSourceIngestModuleCancelled flag. Because of this, there is no + * ironclad guarantee that the correct module will be cancelled. We are + * relying on the module being long-running to avoid a race condition + * between module cancellation and the transition of the execution of a data + * source level ingest task to another module. * * @param moduleDisplayName The display name of the module that was stopped. */ void currentDataSourceIngestModuleCancellationCompleted(String moduleDisplayName) { currentDataSourceIngestModuleCancelled = false; cancelledDataSourceIngestModules.add(moduleDisplayName); - - if (usingNetBeansGUI) { - /** - * A new progress bar must be created because the cancel button of - * the previously constructed component is disabled by NetBeans when - * the user selects the "OK" button of the cancellation confirmation - * dialog popped up by NetBeans when the progress bar cancel button - * is pressed. - */ - synchronized (dataSourceIngestProgressLock) { + if (usingNetBeansGUI && !jobCancelled) { + SwingUtilities.invokeLater(() -> { + /** + * A new progress bar must be created because the cancel button + * of the previously constructed component is disabled by + * NetBeans when the user selects the "OK" button of the + * cancellation confirmation dialog popped up by NetBeans when + * the progress bar cancel button is pressed. + */ dataSourceIngestProgressBar.finish(); dataSourceIngestProgressBar = null; startDataSourceIngestProgressBar(); - } + }); } } @@ -1542,68 +1667,47 @@ final class IngestJobExecutor { /** * Requests a temporary cancellation of data source level ingest for this * job in order to stop the currently executing data source ingest module. + * + * Note that the DataSourceIngestModule interface does not currently have a + * cancel() API. As a consequence, cancelling an individual data source + * ingest module requires setting and then unsetting the + * currentDataSourceIngestModuleCancelled flag. Because of this, there is no + * ironclad guarantee that the correct module will be cancelled. We are + * relying on the module being long-running to avoid a race condition + * between module cancellation and the transition of the execution of a data + * source level ingest task to another module. */ void cancelCurrentDataSourceIngestModule() { currentDataSourceIngestModuleCancelled = true; } /** - * Requests cancellation of ingest, i.e., a shutdown of the data source - * level and file level ingest pipelines. + * Requests cancellation of the ingest job. All pending ingest tasks for the + * job will be cancelled, but any tasks already in progress in ingest + * threads will run to completion. This could take a while if the ingest + * modules executing the tasks are not checking the ingest job cancellation + * flag via the ingest joib context. * * @param reason The cancellation reason. */ void cancel(IngestJob.CancellationReason reason) { jobCancelled = true; cancellationReason = reason; + displayCancellingProgressMessages(); IngestJobExecutor.taskScheduler.cancelPendingFileTasksForIngestJob(getIngestJobId()); - - if (usingNetBeansGUI) { - synchronized (dataSourceIngestProgressLock) { - if (dataSourceIngestProgressBar != null) { - dataSourceIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", dataSource.getName())); - dataSourceIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); - } - } - - synchronized (fileIngestProgressLock) { - if (null != fileIngestProgressBar) { - fileIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.fileIngest.displayName", dataSource.getName())); - fileIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); - } - } - - synchronized (artifactIngestProgressLock) { - if (null != artifactIngestProgressBar) { - artifactIngestProgressBar.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", dataSource.getName())); - artifactIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); - } - } - - synchronized (resultIngestProgressLock) { - if (null != resultIngestProgressBar) { - resultIngestProgressBar.setDisplayName(Bundle.IngestJob_progress_analysisResultIngest_displayName(dataSource.getName())); - resultIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); - } - } - } - synchronized (threadRegistrationLock) { for (Thread thread : pausedIngestThreads) { thread.interrupt(); } pausedIngestThreads.clear(); } - - /* - * If a data source had no tasks in progress it may now be complete. - */ checkForStageCompleted(); } /** - * Queries whether or not cancellation, i.e., a shut down of the data source - * level and file level ingest pipelines for this job, has been requested. + * Queries whether or not cancellation of the ingest job has been requested. + * Ingest modules executing ingest tasks for this job should check this flag + * frequently via the ingest job context. * * @return True or false. */ @@ -1612,9 +1716,9 @@ final class IngestJobExecutor { } /** - * Gets the reason this job was cancelled. + * If the ingest job was cancelled, gets the reason this job was cancelled. * - * @return The cancellation reason, may be not cancelled. + * @return The cancellation reason, may be "not cancelled." */ IngestJob.CancellationReason getCancellationReason() { return cancellationReason; @@ -1627,7 +1731,7 @@ final class IngestJobExecutor { * @param message The message. */ private void logInfoMessage(String message) { - logger.log(Level.INFO, String.format("%s (data source = %s, object Id = %d, job id = %d)", message, dataSource.getName(), dataSource.getId(), getIngestJobId())); //NON-NLS + logger.log(Level.INFO, String.format("%s (data source = %s, data source object Id = %d, job id = %d)", message, dataSource.getName(), dataSource.getId(), getIngestJobId())); //NON-NLS } /** @@ -1639,7 +1743,7 @@ final class IngestJobExecutor { * @param throwable The throwable associated with the error. */ private void logErrorMessage(Level level, String message, Throwable throwable) { - logger.log(level, String.format("%s (data source = %s, object Id = %d, ingest job id = %d)", message, dataSource.getName(), dataSource.getId(), getIngestJobId()), throwable); //NON-NLS + logger.log(level, String.format("%s (data source = %s, data source object Id = %d, ingest job id = %d)", message, dataSource.getName(), dataSource.getId(), getIngestJobId()), throwable); //NON-NLS } /** @@ -1650,7 +1754,7 @@ final class IngestJobExecutor { * @param message The message. */ private void logErrorMessage(Level level, String message) { - logger.log(level, String.format("%s (data source = %s, object Id = %d, ingest job id %d)", message, dataSource.getName(), dataSource.getId(), getIngestJobId())); //NON-NLS + logger.log(level, String.format("%s (data source = %s, data source object Id = %d, ingest job id %d)", message, dataSource.getName(), dataSource.getId(), getIngestJobId())); //NON-NLS } /** @@ -1672,7 +1776,7 @@ final class IngestJobExecutor { */ private void logIngestModuleErrors(List errors, AbstractFile file) { for (IngestModuleError error : errors) { - logErrorMessage(Level.SEVERE, String.format("%s experienced an error during analysis while processing file %s, object ID %d", error.getModuleDisplayName(), file.getName(), file.getId()), error.getThrowable()); //NON-NLS + logErrorMessage(Level.SEVERE, String.format("%s experienced an error during analysis while processing file %s (object ID = %d)", error.getModuleDisplayName(), file.getName(), file.getId()), error.getThrowable()); //NON-NLS } } @@ -1705,16 +1809,13 @@ final class IngestJobExecutor { long processedFilesCount = 0; long estimatedFilesToProcessCount = 0; long snapShotTime = new Date().getTime(); - IngestTasksSnapshot tasksSnapshot = null; + IngestTasksScheduler.IngestTasksSnapshot tasksSnapshot = null; if (includeIngestTasksSnapshot) { - synchronized (fileIngestProgressLock) { - processedFilesCount = processedFiles; - estimatedFilesToProcessCount = estimatedFilesToProcess; - snapShotTime = new Date().getTime(); - } + processedFilesCount = processedFiles; + estimatedFilesToProcessCount = estimatedFilesToProcess; + snapShotTime = new Date().getTime(); tasksSnapshot = taskScheduler.getTasksSnapshotForJob(getIngestJobId()); } - return new Snapshot( dataSource.getName(), getIngestJobId(), diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java index 2d00727858..fe43bb12b3 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java @@ -67,7 +67,7 @@ class IngestJobInputStream implements IngestStream { @Override public synchronized void close() { closed = true; - ingestJob.processStreamingIngestDataSource(); + ingestJob.addStreamedDataSource(); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 32e306e6f3..ea978112e0 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -1021,7 +1021,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { } /** - * Creates and starts an ingest job for a collection of data sources. + * Creates and starts an ingest job. */ private final class StartIngestJobTask implements Callable { diff --git a/InternalPythonModules/GPX_Module/GPX_Parser_Module.py b/InternalPythonModules/GPX_Module/GPX_Parser_Module.py index 375652b6c4..a1deaa447e 100644 --- a/InternalPythonModules/GPX_Module/GPX_Parser_Module.py +++ b/InternalPythonModules/GPX_Module/GPX_Parser_Module.py @@ -134,7 +134,7 @@ class GPXParserFileIngestModule(FileIngestModule): # Create a GeoArtifactsHelper for this file. geoArtifactHelper = GeoArtifactsHelper( - self.skCase, self.moduleName, None, file, context.getJobId()) + self.skCase, self.moduleName, None, file, self.context.getJobId()) if self.writeDebugMsgs: self.log(Level.INFO, "Processing " + file.getUniquePath() + @@ -213,7 +213,7 @@ class GPXParserFileIngestModule(FileIngestModule): art = file.newDataArtifact(BlackboardArtifact.Type(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_BOOKMARK), attributes) - self.blackboard.postArtifact(art, self.moduleName, context.getJobId()) + self.blackboard.postArtifact(art, self.moduleName, self.context.getJobId()) except Blackboard.BlackboardException as e: self.log(Level.SEVERE, "Error posting GPS bookmark artifact for " + diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java index 4dfc014598..c01a81ff66 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java @@ -31,15 +31,18 @@ import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.swing.SwingUtilities; import javax.swing.SwingWorker; import org.netbeans.api.progress.ProgressHandle; import org.openide.nodes.ChildFactory; import org.openide.nodes.Children; import org.openide.nodes.Node; +import org.openide.util.Cancellable; import org.openide.util.NbBundle; import org.openide.util.lookup.Lookups; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode; @@ -89,7 +92,8 @@ class AdHocSearchChildFactory extends ChildFactory { * Constructor * * @param queryRequests Query results - * @param saveResults Flag whether to save search results as KWS artifacts. + * @param saveResults Flag whether to save search results as KWS + * artifacts. */ AdHocSearchChildFactory(Collection queryRequests, boolean saveResults) { this.queryRequests = queryRequests; @@ -129,7 +133,7 @@ class AdHocSearchChildFactory extends ChildFactory { createFlatKeys(queryRequest.getQuery(), toPopulate); } - + // If there were no hits, make a single Node that will display that // no results were found. if (toPopulate.isEmpty()) { @@ -176,7 +180,7 @@ class AdHocSearchChildFactory extends ChildFactory { * Get file properties. */ Map properties = new LinkedHashMap<>(); - + /** * Add a snippet property, if available. */ @@ -204,7 +208,6 @@ class AdHocSearchChildFactory extends ChildFactory { properties.put(LOCATION.toString(), contentName); } - String hitName; BlackboardArtifact artifact = null; if (hit.isArtifactHit()) { @@ -414,21 +417,35 @@ class AdHocSearchChildFactory extends ChildFactory { this.saveResults = saveResults; } - protected void finalizeWorker() { - deregisterWriter(this); - EventQueue.invokeLater(progress::finish); - } - @Override protected Void doInBackground() throws Exception { - registerWriter(this); //register (synchronized on class) outside of writerLock to prevent deadlock - final String queryStr = query.getQueryString(); - final String queryDisp = queryStr.length() > QUERY_DISPLAY_LEN ? queryStr.substring(0, QUERY_DISPLAY_LEN - 1) + " ..." : queryStr; try { - progress = ProgressHandle.createHandle(NbBundle.getMessage(this.getClass(), "KeywordSearchResultFactory.progress.saving", queryDisp), () -> BlackboardResultWriter.this.cancel(true)); - hits.process(progress, null, this, false, saveResults, null); + if (RuntimeProperties.runningWithGUI()) { + final String queryStr = query.getQueryString(); + final String queryDisp = queryStr.length() > QUERY_DISPLAY_LEN ? queryStr.substring(0, QUERY_DISPLAY_LEN - 1) + " ..." : queryStr; + SwingUtilities.invokeLater(() -> { + progress = ProgressHandle.createHandle( + NbBundle.getMessage(this.getClass(), "KeywordSearchResultFactory.progress.saving", queryDisp), + new Cancellable() { + @Override + public boolean cancel() { + //progress.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg")); + logger.log(Level.INFO, "Ad hoc search cancelled by user"); //NON-NLS + new Thread(() -> { + BlackboardResultWriter.this.cancel(true); + }).start(); + return true; + } + }); + }); + } + registerWriter(this); //register (synchronized on class) outside of writerLock to prevent deadlock + hits.process(this, false, saveResults, null); } finally { - finalizeWorker(); + deregisterWriter(this); + if (RuntimeProperties.runningWithGUI() && progress != null) { + EventQueue.invokeLater(progress::finish); + } } return null; } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java old mode 100644 new mode 100755 index 9d6c413199..bb2fbe189a --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java @@ -38,15 +38,15 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; -import org.netbeans.api.progress.aggregate.AggregateProgressFactory; -import org.netbeans.api.progress.aggregate.AggregateProgressHandle; -import org.netbeans.api.progress.aggregate.ProgressContributor; +import org.netbeans.api.progress.ProgressHandle; import org.openide.util.Cancellable; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; +import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.StopWatch; +import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestServices; @@ -447,13 +447,14 @@ final class IngestSearchRunner { /** * Searcher has private copies/snapshots of the lists and keywords */ - private SearchJobInfo job; - private List keywords; //keywords to search - private List keywordListNames; // lists currently being searched - private List keywordLists; - private Map keywordToList; //keyword to list name mapping - private AggregateProgressHandle progressGroup; - private final Logger logger = Logger.getLogger(IngestSearchRunner.Searcher.class.getName()); + private final SearchJobInfo job; + private final List keywords; //keywords to search + private final List keywordListNames; // lists currently being searched + private final List keywordLists; + private final Map keywordToList; //keyword to list name mapping + private final boolean usingNetBeansGUI; + @ThreadConfined(type = ThreadConfined.ThreadType.AWT) + private ProgressHandle progressIndicator; private boolean finalRun = false; Searcher(SearchJobInfo job) { @@ -463,6 +464,7 @@ final class IngestSearchRunner { keywordToList = new HashMap<>(); keywordLists = new ArrayList<>(); //keywords are populated as searcher runs + usingNetBeansGUI = RuntimeProperties.runningWithGUI(); } Searcher(SearchJobInfo job, boolean finalRun) { @@ -473,77 +475,86 @@ final class IngestSearchRunner { @Override @Messages("SearchRunner.query.exception.msg=Error performing query:") protected Object doInBackground() throws Exception { - final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") - + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); - final String pgDisplayName = displayName + (" (" + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") + ")"); - progressGroup = AggregateProgressFactory.createSystemHandle(pgDisplayName, null, new Cancellable() { - @Override - public boolean cancel() { - logger.log(Level.INFO, "Cancelling the searcher by user."); //NON-NLS - if (progressGroup != null) { - progressGroup.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg")); - } - progressGroup.finish(); - return IngestSearchRunner.Searcher.this.cancel(true); - } - }, null); - - updateKeywords(); - - ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()]; - int i = 0; - for (Keyword keywordQuery : keywords) { - subProgresses[i] = AggregateProgressFactory.createProgressContributor(keywordQuery.getSearchTerm()); - progressGroup.addContributor(subProgresses[i]); - i++; - } - - progressGroup.start(); - final StopWatch stopWatch = new StopWatch(); stopWatch.start(); try { - progressGroup.setDisplayName(displayName); - - int keywordsSearched = 0; + if (usingNetBeansGUI) { + /* + * If running in the NetBeans thick client application + * version of Autopsy, NetBeans progress handles (i.e., + * progress bars) are used to display search progress in the + * lower right hand corner of the main application window. + * + * A layer of abstraction to allow alternate representations + * of progress could be used here, as it is in other places + * in the application (see implementations and usage of + * org.sleuthkit.autopsy.progress.ProgressIndicator + * interface), to better decouple keyword search from the + * application's presentation layer. + */ + SwingUtilities.invokeAndWait(() -> { + final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") + + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); + progressIndicator = ProgressHandle.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + progressIndicator.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg")); + logger.log(Level.INFO, "Search cancelled by user"); //NON-NLS + new Thread(() -> { + IngestSearchRunner.Searcher.this.cancel(true); + }).start(); + return true; + } + }); + progressIndicator.start(); + progressIndicator.switchToIndeterminate(); + }); + } + updateKeywords(); for (Keyword keyword : keywords) { - if (this.isCancelled() || this.job.getJobContext().fileIngestIsCancelled()) { - logger.log(Level.INFO, "Cancel detected, bailing before new keyword processed: {0}", keyword.getSearchTerm()); //NON-NLS + if (isCancelled() || job.getJobContext().fileIngestIsCancelled()) { + logger.log(Level.INFO, "Cancellation requested, exiting before new keyword processed: {0}", keyword.getSearchTerm()); //NON-NLS return null; } - logger.log(Level.INFO, String.format("Performing keyword query for search job %d: %s", job.getJobId(), keyword.getSearchTerm())); //NON-NLS - final KeywordList keywordList = keywordToList.get(keyword); - - //new subProgress will be active after the initial query - //when we know number of hits to start() with - if (keywordsSearched > 0) { - subProgresses[keywordsSearched - 1].finish(); + KeywordList keywordList = keywordToList.get(keyword); + if (usingNetBeansGUI) { + String searchTermStr = keyword.getSearchTerm(); + if (searchTermStr.length() > 50) { + searchTermStr = searchTermStr.substring(0, 49) + "..."; + } + final String progressMessage = keywordList.getName() + ": " + searchTermStr; + SwingUtilities.invokeLater(() -> { + progressIndicator.progress(progressMessage); + }); } - KeywordSearchQuery keywordSearchQuery = KeywordSearchUtil.getQueryForKeyword(keyword, keywordList); - // Filtering //limit search to currently ingested data sources //set up a filter with 1 or more image ids OR'ed - final KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, job.getDataSourceId()); + KeywordSearchQuery keywordSearchQuery = KeywordSearchUtil.getQueryForKeyword(keyword, keywordList); + KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, job.getDataSourceId()); keywordSearchQuery.addFilter(dataSourceFilter); - QueryResults queryResults; - // Do the actual search + QueryResults queryResults; try { queryResults = keywordSearchQuery.performQuery(); } catch (KeywordSearchModuleException | NoOpenCoreException ex) { logger.log(Level.SEVERE, "Error performing query: " + keyword.getSearchTerm(), ex); //NON-NLS - MessageNotifyUtil.Notify.error(Bundle.SearchRunner_query_exception_msg() + keyword.getSearchTerm(), ex.getCause().getMessage()); + if (usingNetBeansGUI) { + final String userMessage = Bundle.SearchRunner_query_exception_msg() + keyword.getSearchTerm(); + SwingUtilities.invokeLater(() -> { + MessageNotifyUtil.Notify.error(userMessage, ex.getCause().getMessage()); + }); + } //no reason to continue with next query if recovery failed //or wait for recovery to kick in and run again later //likely case has closed and threads are being interrupted return null; } catch (CancellationException e) { - logger.log(Level.INFO, "Cancel detected, bailing during keyword query: {0}", keyword.getSearchTerm()); //NON-NLS + logger.log(Level.INFO, "Cancellation requested, exiting during keyword query: {0}", keyword.getSearchTerm()); //NON-NLS return null; } @@ -552,42 +563,26 @@ final class IngestSearchRunner { QueryResults newResults = filterResults(queryResults); if (!newResults.getKeywords().isEmpty()) { - - // Write results to BB - //scale progress bar more more granular, per result sub-progress, within per keyword - int totalUnits = newResults.getKeywords().size(); - subProgresses[keywordsSearched].start(totalUnits); - int unitProgress = 0; - String queryDisplayStr = keyword.getSearchTerm(); - if (queryDisplayStr.length() > 50) { - queryDisplayStr = queryDisplayStr.substring(0, 49) + "..."; - } - subProgresses[keywordsSearched].progress(keywordList.getName() + ": " + queryDisplayStr, unitProgress); - // Create blackboard artifacts - newResults.process(null, subProgresses[keywordsSearched], this, keywordList.getIngestMessages(), true, job.getJobId()); - - } //if has results - - //reset the status text before it goes away - subProgresses[keywordsSearched].progress(""); - - ++keywordsSearched; - - } //for each keyword - - } //end try block - catch (Exception ex) { - logger.log(Level.WARNING, "searcher exception occurred", ex); //NON-NLS - } finally { - try { - finalizeSearcher(); - stopWatch.stop(); - logger.log(Level.INFO, "Searcher took {0} secs to run (final = {1})", new Object[]{stopWatch.getElapsedTimeSecs(), this.finalRun}); //NON-NLS - } finally { - // In case a thread is waiting on this worker to be done - job.searchNotify(); + newResults.process(this, keywordList.getIngestMessages(), true, job.getJobId()); + } } + } catch (Exception ex) { + logger.log(Level.WARNING, "Error occurred during keyword search", ex); //NON-NLS + } finally { + if (progressIndicator != null) { + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + progressIndicator.finish(); + progressIndicator = null; + } + }); + } + stopWatch.stop(); + logger.log(Level.INFO, "Searcher took {0} secs to run (final = {1})", new Object[]{stopWatch.getElapsedTimeSecs(), this.finalRun}); //NON-NLS + // In case a thread is waiting on this worker to be done + job.searchNotify(); } return null; @@ -613,20 +608,6 @@ final class IngestSearchRunner { } } - /** - * Performs the cleanup that needs to be done right AFTER - * doInBackground() returns without relying on done() method that is not - * guaranteed to run. - */ - private void finalizeSearcher() { - SwingUtilities.invokeLater(new Runnable() { - @Override - public void run() { - progressGroup.finish(); - } - }); - } - /** * This method filters out all of the hits found in earlier periodic * searches and returns only the results found by the most recent diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index 40ed7db43d..313bad3e60 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.logging.Level; +import javax.swing.SwingUtilities; import javax.swing.SwingWorker; import org.apache.commons.lang.StringUtils; import org.netbeans.api.progress.ProgressHandle; @@ -32,6 +33,7 @@ import org.netbeans.api.progress.aggregate.ProgressContributor; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.EscapeUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestMessage; @@ -51,6 +53,8 @@ import org.sleuthkit.datamodel.TskCoreException; * about the search hits to the ingest inbox, and publishing an event to notify * subscribers of the blackboard posts. */ + + class QueryResults { private static final Logger logger = Logger.getLogger(QueryResults.class.getName()); @@ -131,10 +135,6 @@ class QueryResults { * All calls to the addResult method MUST be completed before calling this * method. * - * @param progress A progress indicator that reports the number of - * keywords processed. Can be null. - * @param subProgress A progress contributor that reports the keyword - * currently being processed. Can be null. * @param worker The SwingWorker that is being used to do the * processing, will be checked for task cancellation * before processing each keyword. @@ -145,19 +145,7 @@ class QueryResults { * @param ingestJobId The numeric identifier of the ingest job within which * the artifacts are being created, may be null. */ - void process(ProgressHandle progress, ProgressContributor subProgress, SwingWorker worker, boolean notifyInbox, boolean saveResults, Long ingestJobId) { - /* - * Initialize the progress indicator to the number of keywords that will - * be processed. - */ - if (null != progress) { - progress.start(getKeywords().size()); - } - - /* - * Process the keyword hits for each keyword. - */ - int keywordsProcessed = 0; + void process(SwingWorker worker, boolean notifyInbox, boolean saveResults, Long ingestJobId) { final Collection hitArtifacts = new ArrayList<>(); for (final Keyword keyword : getKeywords()) { /* @@ -165,22 +153,7 @@ class QueryResults { */ if (worker.isCancelled()) { logger.log(Level.INFO, "Processing cancelled, exiting before processing search term {0}", keyword.getSearchTerm()); //NON-NLS - break; - } - - /* - * Update the progress indicator and the show the current keyword - * via the progress contributor. - */ - if (progress != null) { - progress.progress(keyword.toString(), keywordsProcessed); - } - if (subProgress != null) { - String hitDisplayStr = keyword.getSearchTerm(); - if (hitDisplayStr.length() > 50) { - hitDisplayStr = hitDisplayStr.substring(0, 49) + "..."; - } - subProgress.progress(query.getKeywordList().getName() + ": " + hitDisplayStr, keywordsProcessed); + return; } /* @@ -202,7 +175,7 @@ class QueryResults { snippet = LuceneQuery.querySnippet(snippetQuery, hit.getSolrObjectId(), hit.getChunkId(), !query.isLiteral(), true); } catch (NoOpenCoreException e) { logger.log(Level.SEVERE, "Solr core closed while executing snippet query " + snippetQuery, e); //NON-NLS - break; // Stop processing. + return; // Stop processing. } catch (Exception e) { logger.log(Level.SEVERE, "Error executing snippet query " + snippetQuery, e); //NON-NLS continue; // Try processing the next hit. @@ -242,8 +215,6 @@ class QueryResults { } } } - - ++keywordsProcessed; } /* @@ -298,69 +269,74 @@ class QueryResults { * @throws TskCoreException If there is a problem generating or send the * inbox message. */ - private void writeSingleFileInboxMessage(BlackboardArtifact artifact, Content hitContent) throws TskCoreException { - StringBuilder subjectSb = new StringBuilder(1024); - if (!query.isLiteral()) { - subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExpHitLbl")); - } else { - subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLbl")); - } + private void writeSingleFileInboxMessage(final BlackboardArtifact artifact, final Content hitContent) throws TskCoreException { + if (artifact != null && hitContent != null && RuntimeProperties.runningWithGUI()) { + final StringBuilder subjectSb = new StringBuilder(1024); + if (!query.isLiteral()) { + subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExpHitLbl")); + } else { + subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLbl")); + } - StringBuilder detailsSb = new StringBuilder(1024); - String uniqueKey = null; - BlackboardAttribute attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD)); - if (attr != null) { - final String keyword = attr.getValueString(); - subjectSb.append(keyword); - uniqueKey = keyword.toLowerCase(); - detailsSb.append(""); //NON-NLS - detailsSb.append(""); //NON-NLS - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitThLbl")); - detailsSb.append(""); //NON-NLS - detailsSb.append(""); //NON-NLS - } + final StringBuilder detailsSb = new StringBuilder(1024); + String uniqueKey = null; + BlackboardAttribute attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD)); + if (attr != null) { + final String keyword = attr.getValueString(); + subjectSb.append(keyword); + uniqueKey = keyword.toLowerCase(); + detailsSb.append("
").append(EscapeUtil.escapeHtml(keyword)).append("
"); //NON-NLS + detailsSb.append(""); //NON-NLS + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitThLbl")); + detailsSb.append(""); //NON-NLS + detailsSb.append(""); //NON-NLS + } - //preview - attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW)); - if (attr != null) { - detailsSb.append(""); //NON-NLS - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.previewThLbl")); - detailsSb.append(""); //NON-NLS - detailsSb.append(""); //NON-NLS - } - - //file - detailsSb.append(""); //NON-NLS - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.fileThLbl")); - if (hitContent instanceof AbstractFile) { - AbstractFile hitFile = (AbstractFile) hitContent; - detailsSb.append(""); //NON-NLS - } else { - detailsSb.append(""); //NON-NLS - } - detailsSb.append(""); //NON-NLS - - //list - attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME)); - if (attr != null) { - detailsSb.append(""); //NON-NLS - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.listThLbl")); - detailsSb.append(""); //NON-NLS - detailsSb.append(""); //NON-NLS - } - - //regex - if (!query.isLiteral()) { - attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP)); + //preview + attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW)); if (attr != null) { detailsSb.append(""); //NON-NLS - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExThLbl")); + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.previewThLbl")); + detailsSb.append(""); //NON-NLS + detailsSb.append(""); //NON-NLS + } + + //file + detailsSb.append(""); //NON-NLS + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.fileThLbl")); + if (hitContent instanceof AbstractFile) { + AbstractFile hitFile = (AbstractFile) hitContent; + detailsSb.append(""); //NON-NLS + } else { + detailsSb.append(""); //NON-NLS + } + detailsSb.append(""); //NON-NLS + + //list + attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME)); + if (attr != null) { + detailsSb.append(""); //NON-NLS + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.listThLbl")); detailsSb.append(""); //NON-NLS detailsSb.append(""); //NON-NLS } - } - detailsSb.append("
").append(EscapeUtil.escapeHtml(keyword)).append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(hitFile.getParentPath()).append(hitFile.getName()).append("").append(hitContent.getName()).append("
").append(attr.getValueString()).append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(hitFile.getParentPath()).append(hitFile.getName()).append("").append(hitContent.getName()).append("
").append(attr.getValueString()).append("
"); //NON-NLS - IngestServices.getInstance().postMessage(IngestMessage.createDataMessage(MODULE_NAME, subjectSb.toString(), detailsSb.toString(), uniqueKey, artifact)); + //regex + if (!query.isLiteral()) { + attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP)); + if (attr != null) { + detailsSb.append(""); //NON-NLS + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExThLbl")); + detailsSb.append("").append(attr.getValueString()).append(""); //NON-NLS + detailsSb.append(""); //NON-NLS + } + } + detailsSb.append(""); //NON-NLS + + final String key = uniqueKey; // Might be null, but that's supported. + SwingUtilities.invokeLater(() -> { + IngestServices.getInstance().postMessage(IngestMessage.createDataMessage(MODULE_NAME, subjectSb.toString(), detailsSb.toString(), key, artifact)); + }); + } } } diff --git a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties index b144f07890..87be1c8010 100644 --- a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties +++ b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Fri, 19 Nov 2021 17:01:30 -0500 +#Wed, 01 Dec 2021 12:53:03 -0500 LBL_splash_window_title=Starting Autopsy SPLASH_HEIGHT=314 SPLASH_WIDTH=538 diff --git a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties index d3404a4493..bfb787467d 100644 --- a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties +++ b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties @@ -1,4 +1,4 @@ #Updated by build script -#Fri, 19 Nov 2021 17:01:30 -0500 +#Wed, 01 Dec 2021 12:53:03 -0500 CTL_MainWindow_Title=Autopsy 4.19.2 CTL_MainWindow_Title_No_Project=Autopsy 4.19.2