From 70ffd4ae11b7c573b27ed7d996200ef2fc3d8da7 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 24 Nov 2021 11:35:17 -0500 Subject: [PATCH 01/23] 8168 capture IngestJobContext reference in GPX module --- InternalPythonModules/GPX_Module/GPX_Parser_Module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/InternalPythonModules/GPX_Module/GPX_Parser_Module.py b/InternalPythonModules/GPX_Module/GPX_Parser_Module.py index 375652b6c4..a1deaa447e 100644 --- a/InternalPythonModules/GPX_Module/GPX_Parser_Module.py +++ b/InternalPythonModules/GPX_Module/GPX_Parser_Module.py @@ -134,7 +134,7 @@ class GPXParserFileIngestModule(FileIngestModule): # Create a GeoArtifactsHelper for this file. geoArtifactHelper = GeoArtifactsHelper( - self.skCase, self.moduleName, None, file, context.getJobId()) + self.skCase, self.moduleName, None, file, self.context.getJobId()) if self.writeDebugMsgs: self.log(Level.INFO, "Processing " + file.getUniquePath() + @@ -213,7 +213,7 @@ class GPXParserFileIngestModule(FileIngestModule): art = file.newDataArtifact(BlackboardArtifact.Type(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_BOOKMARK), attributes) - self.blackboard.postArtifact(art, self.moduleName, context.getJobId()) + self.blackboard.postArtifact(art, self.moduleName, self.context.getJobId()) except Blackboard.BlackboardException as e: self.log(Level.SEVERE, "Error posting GPS bookmark artifact for " + From 817bbdd093434a642f02b86be69c7a4409f4051d Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 29 Nov 2021 12:56:07 -0500 Subject: [PATCH 02/23] 8202 ingest prog bars on EDT; job cancel off EDT --- .../autopsy/ingest/IngestJobExecutor.java | 322 +++++++++--------- .../keywordsearch/IngestSearchRunner.java | 49 +-- 2 files changed, 197 insertions(+), 174 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 7a93b15b22..b3691903c1 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -33,6 +33,7 @@ import java.util.regex.Pattern; import java.util.stream.Stream; import javax.annotation.concurrent.GuardedBy; import javax.swing.JOptionPane; +import javax.swing.SwingUtilities; import org.netbeans.api.progress.ProgressHandle; import org.openide.util.Cancellable; import org.openide.util.NbBundle; @@ -41,6 +42,7 @@ import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.NetworkUtils; +import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.ingest.IngestTasksScheduler.IngestJobTasksSnapshot; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; @@ -190,21 +192,26 @@ final class IngestJobExecutor { /* * If running in the NetBeans thick client application version of Autopsy, - * NetBeans progress bars are used to display ingest job progress in the - * lower right hand corner of the main application window. A layer of - * abstraction to allow alternate representations of progress could be used - * here, as it is in other places in the application, to better decouple - * this object from the application's presentation layer. + * NetBeans progress handles (i.e., progress bars) are used to display + * ingest job progress in the lower right hand corner of the main + * application window. + * + * A layer of abstraction to allow alternate representations of progress + * could be used here, as it is in other places in the application (see + * implementations and usage of + * org.sleuthkit.autopsy.progress.ProgressIndicator interface), to better + * decouple this object from the application's presentation layer. */ + private volatile long estimatedFilesToProcess; + private volatile long processedFiles; private final boolean usingNetBeansGUI; - private final Object dataSourceIngestProgressLock = new Object(); - private ProgressHandle dataSourceIngestProgressBar; - private final Object fileIngestProgressLock = new Object(); + @ThreadConfined(type = ThreadConfined.ThreadType.AWT) private final List filesInProgress = new ArrayList<>(); - private long estimatedFilesToProcess; - private long processedFiles; + @ThreadConfined(type = ThreadConfined.ThreadType.AWT) + private ProgressHandle dataSourceIngestProgressBar; + @ThreadConfined(type = ThreadConfined.ThreadType.AWT) private ProgressHandle fileIngestProgressBar; - private final Object artifactIngestProgressLock = new Object(); + @ThreadConfined(type = ThreadConfined.ThreadType.AWT) private ProgressHandle artifactIngestProgressBar; /* @@ -674,14 +681,10 @@ final class IngestJobExecutor { * how many files remain to be analyzed as each file ingest task * is completed. */ - long filesToProcess; if (files.isEmpty()) { - filesToProcess = dataSource.accept(new GetFilesCountVisitor()); + estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor()); } else { - filesToProcess = files.size(); - } - synchronized (fileIngestProgressLock) { - estimatedFilesToProcess = filesToProcess; + estimatedFilesToProcess = files.size(); } } @@ -781,17 +784,21 @@ final class IngestJobExecutor { if (hasFileIngestModules()) { /* - * Do a count of the files the data source processor has added - * to the case database. This number will be used to estimate - * how many files remain to be analyzed as each file ingest task - * is completed. + * For ingest job progress reporting purposes, do a count of the + * files the data source processor has added to the case + * database. This number will be used to estimate how many files + * remain to be analyzed as each file ingest task is completed. + * The estimate will likely be an over-estimate, since some of + * the files will have already been "streamed" to this job and + * processed. */ - long filesToProcess = dataSource.accept(new GetFilesCountVisitor()); - synchronized (fileIngestProgressLock) { - estimatedFilesToProcess = filesToProcess; - if (usingNetBeansGUI && fileIngestProgressBar != null) { - fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); - } + estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor()); + if (usingNetBeansGUI) { + SwingUtilities.invokeLater(() -> { + if (fileIngestProgressBar != null) { + fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); + } + }); } } @@ -852,18 +859,20 @@ final class IngestJobExecutor { */ private void startArtifactIngestProgressBar() { if (usingNetBeansGUI) { - synchronized (artifactIngestProgressLock) { + SwingUtilities.invokeLater(() -> { String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", this.dataSource.getName()); artifactIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { - IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + new Thread(() -> { + IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + }).start(); return true; } }); artifactIngestProgressBar.start(); artifactIngestProgressBar.switchToIndeterminate(); - } + }); } } @@ -876,34 +885,57 @@ final class IngestJobExecutor { * cancellation occurs is NOT discarded. */ private void startDataSourceIngestProgressBar() { - if (usingNetBeansGUI) { - synchronized (dataSourceIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", dataSource.getName()); - dataSourceIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - /* - * The user has already pressed the cancel button on - * this progress bar, and the OK button of a cancelation - * confirmation dialog supplied by NetBeans. Find out - * whether the user wants to cancel only the currently - * executing data source ingest module or the entire - * ingest job. - */ - DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); - String dialogTitle = NbBundle.getMessage(IngestJobExecutor.this.getClass(), "IngestJob.cancellationDialog.title"); - JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); - if (panel.cancelAllDataSourceIngestModules()) { + SwingUtilities.invokeLater(() -> { + String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", dataSource.getName()); + dataSourceIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + /* + * The user has already pressed the cancel button on this + * progress bar, and the OK button of a cancelation + * confirmation dialog supplied by NetBeans. Find out + * whether the user wants to cancel only the currently + * executing data source ingest module or the entire ingest + * job. + */ + DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); + String dialogTitle = NbBundle.getMessage(IngestJobExecutor.this.getClass(), "IngestJob.cancellationDialog.title"); + JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); + if (panel.cancelAllDataSourceIngestModules()) { + new Thread(() -> { IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); - } else { + }).start(); + } else { + new Thread(() -> { IngestJobExecutor.this.cancelCurrentDataSourceIngestModule(); - } - return true; + }).start(); } - }); - dataSourceIngestProgressBar.start(); - dataSourceIngestProgressBar.switchToIndeterminate(); - } + return true; + } + }); + dataSourceIngestProgressBar.start(); + dataSourceIngestProgressBar.switchToIndeterminate(); + }); + } + + private void finishProgressIndicators() { + if (usingNetBeansGUI) { + SwingUtilities.invokeLater(() -> { + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; + } + + if (fileIngestProgressBar != null) { + fileIngestProgressBar.finish(); + fileIngestProgressBar = null; + } + + if (artifactIngestProgressBar != null) { + artifactIngestProgressBar.finish(); + artifactIngestProgressBar = null; + } + }); } } @@ -915,20 +947,20 @@ final class IngestJobExecutor { * discarded. */ private void startFileIngestProgressBar() { - if (usingNetBeansGUI) { - synchronized (fileIngestProgressLock) { - String displayName = NbBundle.getMessage(getClass(), "IngestJob.progress.fileIngest.displayName", dataSource.getName()); - fileIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { + SwingUtilities.invokeLater(() -> { + String displayName = NbBundle.getMessage(getClass(), "IngestJob.progress.fileIngest.displayName", dataSource.getName()); + fileIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + new Thread(() -> { IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); - return true; - } - }); - fileIngestProgressBar.start(); - fileIngestProgressBar.switchToDeterminate((int) this.estimatedFilesToProcess); - } - } + }).start(); + return true; + } + }); + fileIngestProgressBar.start(); + fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); + }); } /** @@ -969,19 +1001,17 @@ final class IngestJobExecutor { } if (usingNetBeansGUI) { - synchronized (dataSourceIngestProgressLock) { + SwingUtilities.invokeLater(() -> { if (dataSourceIngestProgressBar != null) { dataSourceIngestProgressBar.finish(); dataSourceIngestProgressBar = null; } - } - synchronized (fileIngestProgressLock) { if (fileIngestProgressBar != null) { fileIngestProgressBar.finish(); fileIngestProgressBar = null; } - } + }); } if (!jobCancelled && hasLowPriorityDataSourceIngestModules()) { @@ -993,7 +1023,8 @@ final class IngestJobExecutor { } /** - * Shuts down the ingest module pipelines and progress bars. + * Shuts down the ingest module pipelines and ingest job progress + * indicators. */ private void shutDown() { synchronized (stageTransitionLock) { @@ -1002,29 +1033,7 @@ final class IngestJobExecutor { shutDownIngestModulePipeline(currentDataSourceIngestPipeline); shutDownIngestModulePipeline(artifactIngestPipeline); - - if (usingNetBeansGUI) { - synchronized (dataSourceIngestProgressLock) { - if (dataSourceIngestProgressBar != null) { - dataSourceIngestProgressBar.finish(); - dataSourceIngestProgressBar = null; - } - } - - synchronized (fileIngestProgressLock) { - if (fileIngestProgressBar != null) { - fileIngestProgressBar.finish(); - fileIngestProgressBar = null; - } - } - - synchronized (artifactIngestProgressLock) { - if (artifactIngestProgressBar != null) { - artifactIngestProgressBar.finish(); - artifactIngestProgressBar = null; - } - } - } + finishProgressIndicators(); if (ingestJobInfo != null) { if (jobCancelled) { @@ -1114,16 +1123,17 @@ final class IngestJobExecutor { return; } - synchronized (fileIngestProgressLock) { - ++processedFiles; - if (usingNetBeansGUI) { + final String fileName = file.getName(); + processedFiles++; + if (usingNetBeansGUI) { + SwingUtilities.invokeLater(() -> { if (processedFiles <= estimatedFilesToProcess) { - fileIngestProgressBar.progress(file.getName(), (int) processedFiles); + fileIngestProgressBar.progress(fileName, (int) processedFiles); } else { - fileIngestProgressBar.progress(file.getName(), (int) estimatedFilesToProcess); + fileIngestProgressBar.progress(fileName, (int) estimatedFilesToProcess); } - filesInProgress.add(file.getName()); - } + filesInProgress.add(fileName); + }); } /** @@ -1136,18 +1146,18 @@ final class IngestJobExecutor { } if (usingNetBeansGUI && !jobCancelled) { - synchronized (fileIngestProgressLock) { + SwingUtilities.invokeLater(() -> { /** * Update the file ingest progress bar again, in * case the file was being displayed. */ - filesInProgress.remove(file.getName()); + filesInProgress.remove(fileName); if (filesInProgress.size() > 0) { fileIngestProgressBar.progress(filesInProgress.get(0)); } else { fileIngestProgressBar.progress(""); } - } + }); } } fileIngestPipelinesQueue.put(pipeline); @@ -1254,11 +1264,11 @@ final class IngestJobExecutor { */ void updateDataSourceIngestProgressBarDisplayName(String displayName) { if (usingNetBeansGUI && !jobCancelled) { - synchronized (dataSourceIngestProgressLock) { + SwingUtilities.invokeLater(() -> { if (dataSourceIngestProgressBar != null) { dataSourceIngestProgressBar.setDisplayName(displayName); } - } + }); } } @@ -1272,11 +1282,11 @@ final class IngestJobExecutor { */ void switchDataSourceIngestProgressBarToDeterminate(int workUnits) { if (usingNetBeansGUI && !jobCancelled) { - synchronized (dataSourceIngestProgressLock) { + SwingUtilities.invokeLater(() -> { if (dataSourceIngestProgressBar != null) { dataSourceIngestProgressBar.switchToDeterminate(workUnits); } - } + }); } } @@ -1287,11 +1297,11 @@ final class IngestJobExecutor { */ void switchDataSourceIngestProgressBarToIndeterminate() { if (usingNetBeansGUI && !jobCancelled) { - synchronized (dataSourceIngestProgressLock) { + SwingUtilities.invokeLater(() -> { if (dataSourceIngestProgressBar != null) { dataSourceIngestProgressBar.switchToIndeterminate(); } - } + }); } } @@ -1303,11 +1313,11 @@ final class IngestJobExecutor { */ void advanceDataSourceIngestProgressBar(int workUnits) { if (usingNetBeansGUI && !jobCancelled) { - synchronized (dataSourceIngestProgressLock) { + SwingUtilities.invokeLater(() -> { if (dataSourceIngestProgressBar != null) { dataSourceIngestProgressBar.progress("", workUnits); } - } + }); } } @@ -1319,11 +1329,11 @@ final class IngestJobExecutor { */ void advanceDataSourceIngestProgressBar(String currentTask) { if (usingNetBeansGUI && !jobCancelled) { - synchronized (dataSourceIngestProgressLock) { + SwingUtilities.invokeLater(() -> { if (dataSourceIngestProgressBar != null) { dataSourceIngestProgressBar.progress(currentTask); } - } + }); } } @@ -1337,11 +1347,11 @@ final class IngestJobExecutor { */ void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { if (usingNetBeansGUI && !jobCancelled) { - synchronized (dataSourceIngestProgressLock) { + SwingUtilities.invokeLater(() -> { if (dataSourceIngestProgressBar != null) { dataSourceIngestProgressBar.progress(currentTask, workUnits); } - } + }); } } @@ -1367,18 +1377,18 @@ final class IngestJobExecutor { cancelledDataSourceIngestModules.add(moduleDisplayName); if (usingNetBeansGUI) { - /** - * A new progress bar must be created because the cancel button of - * the previously constructed component is disabled by NetBeans when - * the user selects the "OK" button of the cancellation confirmation - * dialog popped up by NetBeans when the progress bar cancel button - * is pressed. - */ - synchronized (dataSourceIngestProgressLock) { + SwingUtilities.invokeLater(() -> { + /** + * A new progress bar must be created because the cancel button + * of the previously constructed component is disabled by + * NetBeans when the user selects the "OK" button of the + * cancellation confirmation dialog popped up by NetBeans when + * the progress bar cancel button is pressed. + */ dataSourceIngestProgressBar.finish(); dataSourceIngestProgressBar = null; startDataSourceIngestProgressBar(); - } + }); } } @@ -1404,8 +1414,11 @@ final class IngestJobExecutor { } /** - * Requests cancellation of ingest, i.e., a shutdown of the data source - * level and file level ingest pipelines. + * Requests cancellation of the ingest job. All pending ingest tasks for the + * job will be cancelled, but any tasks already in progress in ingest + * threads will run to completion. This could take a while if the ingest + * modules executing the tasks are not checking the ingest job cancellation + * flag via the ingest joib context. * * @param reason The cancellation reason. */ @@ -1415,19 +1428,17 @@ final class IngestJobExecutor { IngestJobExecutor.taskScheduler.cancelPendingFileTasksForIngestJob(this); if (usingNetBeansGUI) { - synchronized (dataSourceIngestProgressLock) { + SwingUtilities.invokeLater(() -> { if (dataSourceIngestProgressBar != null) { dataSourceIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", dataSource.getName())); dataSourceIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); } - } - synchronized (this.fileIngestProgressLock) { - if (null != this.fileIngestProgressBar) { - this.fileIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.fileIngest.displayName", dataSource.getName())); - this.fileIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); + if (fileIngestProgressBar != null) { + fileIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.fileIngest.displayName", dataSource.getName())); + fileIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); } - } + }); } synchronized (threadRegistrationLock) { @@ -1437,15 +1448,13 @@ final class IngestJobExecutor { pausedIngestThreads.clear(); } - /* - * If a data source had no tasks in progress it may now be complete. - */ checkForStageCompleted(); } /** - * Queries whether or not cancellation, i.e., a shut down of the data source - * level and file level ingest pipelines for this job, has been requested. + * Queries whether or not cancellation of the ingest job has been requested. + * Ingest modules executing ingest tasks for this job should check this flag + * frequently via the ingest job context. * * @return True or false. */ @@ -1454,9 +1463,9 @@ final class IngestJobExecutor { } /** - * Gets the reason this job was cancelled. + * If the ingest job was cancelled, gets the reason this job was cancelled. * - * @return The cancellation reason, may be not cancelled. + * @return The cancellation reason, may be "not cancelled." */ IngestJob.CancellationReason getCancellationReason() { return cancellationReason; @@ -1549,20 +1558,25 @@ final class IngestJobExecutor { long snapShotTime = new Date().getTime(); IngestJobTasksSnapshot tasksSnapshot = null; if (includeIngestTasksSnapshot) { - synchronized (fileIngestProgressLock) { - processedFilesCount = processedFiles; - estimatedFilesToProcessCount = estimatedFilesToProcess; - snapShotTime = new Date().getTime(); - } + processedFilesCount = processedFiles; + estimatedFilesToProcessCount = estimatedFilesToProcess; + snapShotTime = new Date().getTime(); tasksSnapshot = taskScheduler.getTasksSnapshotForJob(getIngestJobId()); } - - return new Snapshot(dataSource.getName(), - getIngestJobId(), createTime, + return new Snapshot( + dataSource.getName(), + getIngestJobId(), + createTime, getCurrentDataSourceIngestModule(), - fileIngestRunning, fileIngestStartTime, - jobCancelled, cancellationReason, cancelledDataSourceIngestModules, - processedFilesCount, estimatedFilesToProcessCount, snapShotTime, tasksSnapshot); + fileIngestRunning, + fileIngestStartTime, + jobCancelled, + cancellationReason, + cancelledDataSourceIngestModules, + processedFilesCount, + estimatedFilesToProcessCount, + snapShotTime, + tasksSnapshot); } /** diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java index 383abbd3af..b31ba6ca33 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java @@ -248,7 +248,8 @@ final class IngestSearchRunner { } /** - * Task to perform periodic searches for each job (does a single index commit first) + * Task to perform periodic searches for each job (does a single index + * commit first) */ private final class PeriodicSearchTask implements Runnable { @@ -296,24 +297,23 @@ final class IngestSearchRunner { NbBundle.getMessage(this.getClass(), "SearchRunner.Searcher.done.err.msg"), ex.getMessage())); }// catch and ignore if we were cancelled - catch (java.util.concurrent.CancellationException ex) { + catch (java.util.concurrent.CancellationException ex) { } } } stopWatch.stop(); logger.log(Level.INFO, "All periodic searches cumulatively took {0} secs", stopWatch.getElapsedTimeSecs()); //NON-NLS - + // calculate "hold off" time recalculateUpdateIntervalTime(stopWatch.getElapsedTimeSecs()); // ELDEBUG - + // schedule next PeriodicSearchTask jobProcessingTaskFuture = jobProcessingExecutor.schedule(new PeriodicSearchTask(), currentUpdateIntervalMs, MILLISECONDS); - + // exit this thread return; } - - + private void recalculateUpdateIntervalTime(long lastSerchTimeSec) { // If periodic search takes more than 1/4 of the current periodic search interval, then double the search interval if (lastSerchTimeSec * 1000 < currentUpdateIntervalMs / 4) { @@ -321,7 +321,7 @@ final class IngestSearchRunner { } // double the search interval currentUpdateIntervalMs = currentUpdateIntervalMs * 2; - logger.log(Level.WARNING, "Last periodic search took {0} sec. Increasing search interval to {1} sec", new Object[]{lastSerchTimeSec, currentUpdateIntervalMs/1000}); + logger.log(Level.WARNING, "Last periodic search took {0} sec. Increasing search interval to {1} sec", new Object[]{lastSerchTimeSec, currentUpdateIntervalMs / 1000}); return; } } @@ -484,26 +484,35 @@ final class IngestSearchRunner { progressGroup.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg")); } progressGroup.finish(); - return IngestSearchRunner.Searcher.this.cancel(true); + new Thread(() -> { + IngestSearchRunner.Searcher.this.cancel(true); + }).start(); + return true; } - }, null); + }, + null); updateKeywords(); - ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()]; - int i = 0; - for (Keyword keywordQuery : keywords) { - subProgresses[i] = AggregateProgressFactory.createProgressContributor(keywordQuery.getSearchTerm()); - progressGroup.addContributor(subProgresses[i]); - i++; - } - - progressGroup.start(); + SwingUtilities.invokeLater(() -> { + ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()]; + int i = 0; + for (Keyword keywordQuery : keywords) { + subProgresses[i] = AggregateProgressFactory.createProgressContributor(keywordQuery.getSearchTerm()); + progressGroup.addContributor(subProgresses[i]); + i++; + } + progressGroup.start(); + }); final StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + try { - progressGroup.setDisplayName(displayName); + SwingUtilities.invokeLater(() -> { + progressGroup.setDisplayName(displayName); + }); int keywordsSearched = 0; From cd3f1a58ec24bff4ba43af0ac49d41281ba225c5 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 29 Nov 2021 13:54:17 -0500 Subject: [PATCH 03/23] 8202 ingest prog bars on EDT; job cancel off EDT --- .../org/sleuthkit/autopsy/ingest/IngestJobExecutor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index b3691903c1..8e0f6c7493 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -202,14 +202,14 @@ final class IngestJobExecutor { * org.sleuthkit.autopsy.progress.ProgressIndicator interface), to better * decouple this object from the application's presentation layer. */ - private volatile long estimatedFilesToProcess; - private volatile long processedFiles; private final boolean usingNetBeansGUI; @ThreadConfined(type = ThreadConfined.ThreadType.AWT) - private final List filesInProgress = new ArrayList<>(); - @ThreadConfined(type = ThreadConfined.ThreadType.AWT) private ProgressHandle dataSourceIngestProgressBar; @ThreadConfined(type = ThreadConfined.ThreadType.AWT) + private final List filesInProgress = new ArrayList<>(); + private volatile long estimatedFilesToProcess; + private volatile long processedFiles; + @ThreadConfined(type = ThreadConfined.ThreadType.AWT) private ProgressHandle fileIngestProgressBar; @ThreadConfined(type = ThreadConfined.ThreadType.AWT) private ProgressHandle artifactIngestProgressBar; From 087c70d492d66eb016c6e5876f24fedcb23a12d6 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 29 Nov 2021 14:39:48 -0500 Subject: [PATCH 04/23] 8202 ingest prog bars on EDT; job cancel off EDT --- .../sleuthkit/autopsy/ingest/IngestJob.java | 2 +- .../autopsy/ingest/IngestJobExecutor.java | 214 +++++++++--------- .../autopsy/ingest/IngestManager.java | 2 +- 3 files changed, 113 insertions(+), 105 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 37e4b549ee..a508110b99 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -173,7 +173,7 @@ public final class IngestJob { void processStreamingIngestDataSource() { if (ingestMode == Mode.STREAMING) { if (ingestModuleExecutor != null) { - ingestModuleExecutor.startStreamingModeDataSourceAnalysis(); + ingestModuleExecutor.addStreamedDataSource(); } else { logger.log(Level.SEVERE, "Attempted to start data source analaysis with no ingest pipeline"); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 8e0f6c7493..160173c9c8 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -541,7 +541,7 @@ final class IngestJobExecutor { } /** - * Determnines which inges job stage to start in and starts up the ingest + * Determnines which ingets job stage to start in and starts up the ingest * module pipelines. * * @return A collection of ingest module startup errors, empty on success. @@ -671,7 +671,7 @@ final class IngestJobExecutor { */ private void startBatchModeAnalysis() { synchronized (stageTransitionLock) { - logInfoMessage(String.format("Starting analysis in batch mode for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), ingestJob.getId())); //NON-NLS + logInfoMessage("Starting ingest job in batch mode"); //NON-NLS stage = IngestJobStage.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS; if (hasFileIngestModules()) { @@ -686,22 +686,15 @@ final class IngestJobExecutor { } else { estimatedFilesToProcess = files.size(); } + startFileIngestProgressBar(); } - if (usingNetBeansGUI) { - /* - * Start ingest progress bars in the lower right hand corner of - * the main application window. - */ - if (hasFileIngestModules()) { - startFileIngestProgressBar(); - } - if (hasHighPriorityDataSourceIngestModules()) { - startDataSourceIngestProgressBar(); - } - if (hasDataArtifactIngestModules()) { - startArtifactIngestProgressBar(); - } + if (hasHighPriorityDataSourceIngestModules()) { + startDataSourceIngestProgressBar(); + } + + if (hasDataArtifactIngestModules()) { + startArtifactIngestProgressBar(); } /* @@ -711,60 +704,69 @@ final class IngestJobExecutor { currentDataSourceIngestPipeline = highPriorityDataSourceIngestPipeline; /* - * Schedule ingest tasks and then immediately check for stage - * completion. This is necessary because it is possible that zero - * tasks will actually make it to task execution due to the file - * filter or other ingest job settings. In that case, there will - * never be a stage completion check in an ingest thread executing - * an ingest task, so such a job would run forever without a check - * here. + * Schedule ingest tasks. */ if (!files.isEmpty() && hasFileIngestModules()) { taskScheduler.scheduleFileIngestTasks(this, files); } else if (hasHighPriorityDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { taskScheduler.scheduleIngestTasks(this); } + + /* + * Check for stage completion. This is necessary because it is + * possible that none of the tasks that were just scheduled will + * actually make it to task execution due to the file filter or + * other ingest job settings. In that case, there will never be a + * stage completion check in an ingest thread executing an ingest + * task, so such a job would run forever without a check here. + */ checkForStageCompleted(); } } /** * Starts analysis for a streaming mode ingest job. For a streaming mode - * job, the data source processor streams files in as it adds them to the - * case database and file analysis can begin before data source level - * analysis. + * job, a data source processor streams files to this ingest job executor as + * it adds the files to the case database, and file level analysis can begin + * before data source level analysis. */ private void startStreamingModeAnalysis() { synchronized (stageTransitionLock) { - logInfoMessage("Starting data source level analysis in streaming mode"); //NON-NLS + logInfoMessage("Starting ingest job in streaming mode"); //NON-NLS stage = IngestJobStage.STREAMED_FILE_ANALYSIS_ONLY; - if (usingNetBeansGUI) { + if (hasFileIngestModules()) { /* - * Start ingest progress bars in the lower right hand corner of - * the main application window. + * Start the file ingest progress bar, but do not schedule any + * file or data source ingest tasks. File ingest tasks will + * instead be scheduled as files are streamed in via + * addStreamedFiles(), and a data source ingest task will be + * scheduled later, via addStreamedDataSource(). + * + * Note that because estimated files remaining to process still + * has its initial value of zero, the fle ingest progress bar + * will start in the "indeterminate" state. A rough estimate of + * the files to processed will be computed later, when all of + * the files have been added to the case database, as signaled + * by a call to the addStreamedDataSource(). */ - if (hasFileIngestModules()) { - /* - * Note that because estimated files remaining to process - * still has its initial value of zero, the progress bar - * will start in the "indeterminate" state. An estimate of - * the files to process can be computed later, when all of - * the files have been added ot the case database. - */ - startFileIngestProgressBar(); - } - if (hasDataArtifactIngestModules()) { - startArtifactIngestProgressBar(); - } + estimatedFilesToProcess = 0; + startFileIngestProgressBar(); } if (hasDataArtifactIngestModules()) { + startArtifactIngestProgressBar(); + /* * Schedule artifact ingest tasks for any artifacts currently in * the case database. This needs to be done before any files or * the data source are streamed in to avoid analyzing the data * artifacts added to the case database by those tasks twice. + * This constraint is implemented by restricting construction of + * a streaming mode IngestJob to + * IngestManager.openIngestStream(), which constructs and starts + * the job before returning the IngestStream that is used to + * stream in the files and data source. */ taskScheduler.scheduleDataArtifactIngestTasks(this); } @@ -776,7 +778,7 @@ final class IngestJobExecutor { * case database and streamed in, and the data source is now ready for * analysis. */ - void startStreamingModeDataSourceAnalysis() { + void addStreamedDataSource() { synchronized (stageTransitionLock) { logInfoMessage("Starting full first stage analysis in streaming mode"); //NON-NLS stage = IngestJobExecutor.IngestJobStage.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS; @@ -859,20 +861,22 @@ final class IngestJobExecutor { */ private void startArtifactIngestProgressBar() { if (usingNetBeansGUI) { - SwingUtilities.invokeLater(() -> { - String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", this.dataSource.getName()); - artifactIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - new Thread(() -> { - IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); - }).start(); - return true; - } + if (usingNetBeansGUI) { + SwingUtilities.invokeLater(() -> { + String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", this.dataSource.getName()); + artifactIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + new Thread(() -> { + IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + }).start(); + return true; + } + }); + artifactIngestProgressBar.start(); + artifactIngestProgressBar.switchToIndeterminate(); }); - artifactIngestProgressBar.start(); - artifactIngestProgressBar.switchToIndeterminate(); - }); + } } } @@ -885,37 +889,39 @@ final class IngestJobExecutor { * cancellation occurs is NOT discarded. */ private void startDataSourceIngestProgressBar() { - SwingUtilities.invokeLater(() -> { - String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", dataSource.getName()); - dataSourceIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - /* - * The user has already pressed the cancel button on this - * progress bar, and the OK button of a cancelation - * confirmation dialog supplied by NetBeans. Find out - * whether the user wants to cancel only the currently - * executing data source ingest module or the entire ingest - * job. - */ - DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); - String dialogTitle = NbBundle.getMessage(IngestJobExecutor.this.getClass(), "IngestJob.cancellationDialog.title"); - JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); - if (panel.cancelAllDataSourceIngestModules()) { - new Thread(() -> { - IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); - }).start(); - } else { - new Thread(() -> { - IngestJobExecutor.this.cancelCurrentDataSourceIngestModule(); - }).start(); + if (usingNetBeansGUI) { + SwingUtilities.invokeLater(() -> { + String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", dataSource.getName()); + dataSourceIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + /* + * The user has already pressed the cancel button on + * this progress bar, and the OK button of a cancelation + * confirmation dialog supplied by NetBeans. Find out + * whether the user wants to cancel only the currently + * executing data source ingest module or the entire + * ingest job. + */ + DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); + String dialogTitle = NbBundle.getMessage(IngestJobExecutor.this.getClass(), "IngestJob.cancellationDialog.title"); + JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); + if (panel.cancelAllDataSourceIngestModules()) { + new Thread(() -> { + IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + }).start(); + } else { + new Thread(() -> { + IngestJobExecutor.this.cancelCurrentDataSourceIngestModule(); + }).start(); + } + return true; } - return true; - } + }); + dataSourceIngestProgressBar.start(); + dataSourceIngestProgressBar.switchToIndeterminate(); }); - dataSourceIngestProgressBar.start(); - dataSourceIngestProgressBar.switchToIndeterminate(); - }); + } } private void finishProgressIndicators() { @@ -947,20 +953,22 @@ final class IngestJobExecutor { * discarded. */ private void startFileIngestProgressBar() { - SwingUtilities.invokeLater(() -> { - String displayName = NbBundle.getMessage(getClass(), "IngestJob.progress.fileIngest.displayName", dataSource.getName()); - fileIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - new Thread(() -> { - IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); - }).start(); - return true; - } + if (usingNetBeansGUI) { + SwingUtilities.invokeLater(() -> { + String displayName = NbBundle.getMessage(getClass(), "IngestJob.progress.fileIngest.displayName", dataSource.getName()); + fileIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + new Thread(() -> { + IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + }).start(); + return true; + } + }); + fileIngestProgressBar.start(); + fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); }); - fileIngestProgressBar.start(); - fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); - }); + } } /** @@ -1478,7 +1486,7 @@ final class IngestJobExecutor { * @param message The message. */ private void logInfoMessage(String message) { - logger.log(Level.INFO, String.format("%s (data source = %s, object Id = %d, job id = %d)", message, dataSource.getName(), dataSource.getId(), getIngestJobId())); //NON-NLS + logger.log(Level.INFO, String.format("%s (data source = %s, data source object Id = %d, job id = %d)", message, dataSource.getName(), dataSource.getId(), getIngestJobId())); //NON-NLS } /** @@ -1490,7 +1498,7 @@ final class IngestJobExecutor { * @param throwable The throwable associated with the error. */ private void logErrorMessage(Level level, String message, Throwable throwable) { - logger.log(level, String.format("%s (data source = %s, object Id = %d, ingest job id = %d)", message, dataSource.getName(), dataSource.getId(), getIngestJobId()), throwable); //NON-NLS + logger.log(level, String.format("%s (data source = %s, data source object Id = %d, ingest job id = %d)", message, dataSource.getName(), dataSource.getId(), getIngestJobId()), throwable); //NON-NLS } /** @@ -1501,7 +1509,7 @@ final class IngestJobExecutor { * @param message The message. */ private void logErrorMessage(Level level, String message) { - logger.log(level, String.format("%s (data source = %s, object Id = %d, ingest job id %d)", message, dataSource.getName(), dataSource.getId(), getIngestJobId())); //NON-NLS + logger.log(level, String.format("%s (data source = %s, data source object Id = %d, ingest job id %d)", message, dataSource.getName(), dataSource.getId(), getIngestJobId())); //NON-NLS } /** @@ -1523,7 +1531,7 @@ final class IngestJobExecutor { */ private void logIngestModuleErrors(List errors, AbstractFile file) { for (IngestModuleError error : errors) { - logErrorMessage(Level.SEVERE, String.format("%s experienced an error during analysis while processing file %s, object ID %d", error.getModuleDisplayName(), file.getName(), file.getId()), error.getThrowable()); //NON-NLS + logErrorMessage(Level.SEVERE, String.format("%s experienced an error during analysis while processing file %s (object ID = %d)", error.getModuleDisplayName(), file.getName(), file.getId()), error.getThrowable()); //NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 2c87487232..0dc2597481 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -1050,7 +1050,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { } /** - * Creates and starts an ingest job for a collection of data sources. + * Creates and starts an ingest job. */ private final class StartIngestJobTask implements Callable { From 280580042ee7cf54c200d0b228540cb0a6828105 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 29 Nov 2021 15:13:14 -0500 Subject: [PATCH 05/23] 8202 ingest prog bars on EDT; job cancel off EDT --- .../autopsy/ingest/IngestJobExecutor.java | 190 ++++++++++-------- 1 file changed, 103 insertions(+), 87 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 160173c9c8..43fed4ce58 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -795,38 +795,31 @@ final class IngestJobExecutor { * processed. */ estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor()); - if (usingNetBeansGUI) { - SwingUtilities.invokeLater(() -> { - if (fileIngestProgressBar != null) { - fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); - } - }); - } + switchFileIngestProgressBarToDeterminate(); } - if (usingNetBeansGUI) { + currentDataSourceIngestPipeline = highPriorityDataSourceIngestPipeline; + if (hasHighPriorityDataSourceIngestModules()) { /* * Start a data source level ingest progress bar in the lower * right hand corner of the main application window. The file * and data artifact ingest progress bars were already started * in startStreamingModeAnalysis(). */ - if (hasHighPriorityDataSourceIngestModules()) { - startDataSourceIngestProgressBar(); - } - } + startDataSourceIngestProgressBar(); - currentDataSourceIngestPipeline = highPriorityDataSourceIngestPipeline; - if (hasHighPriorityDataSourceIngestModules()) { + /* + * Schedule a task for the data source. + */ IngestJobExecutor.taskScheduler.scheduleDataSourceIngestTask(this); } else { /* - * If no data source level ingest task is scheduled at this time - * and all of the file level and artifact ingest tasks scheduled - * during the initial file streaming stage have already - * executed, there will never be a stage completion check in an - * ingest thread executing an ingest task, so such a job would - * run forever without a check here. + * If no data source level ingest task is scheduled at this + * time, and all of the file level and artifact ingest tasks + * scheduled during the initial file streaming stage have + * already been executed, there will never be a stage completion + * check in an ingest thread executing an ingest task, so such a + * job would run forever without a check here. */ checkForStageCompleted(); } @@ -839,13 +832,9 @@ final class IngestJobExecutor { private void startLowPriorityDataSourceAnalysis() { synchronized (stageTransitionLock) { if (hasLowPriorityDataSourceIngestModules()) { - logInfoMessage(String.format("Starting low priority data source analysis for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), ingestJob.getId())); //NON-NLS + logInfoMessage("Starting low priority data source analysis"); //NON-NLS stage = IngestJobExecutor.IngestJobStage.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS; - - if (usingNetBeansGUI) { - startDataSourceIngestProgressBar(); - } - + startDataSourceIngestProgressBar(); currentDataSourceIngestPipeline = lowPriorityDataSourceIngestPipeline; taskScheduler.scheduleDataSourceIngestTask(this); } @@ -853,38 +842,36 @@ final class IngestJobExecutor { } /** - * Starts a data artifacts analysis NetBeans progress bar in the lower right - * hand corner of the main application window. The progress bar provides the - * user with a task cancellation button. Pressing it cancels the ingest job. - * Analysis already completed at the time that cancellation occurs is NOT - * discarded. + * Starts a NetBeans progress bar for data artifacts analysis in the lower + * right hand corner of the main application window. The progress bar + * provides the user with a task cancellation button. Pressing it cancels + * the ingest job. Analysis already completed at the time that cancellation + * occurs is NOT discarded. */ private void startArtifactIngestProgressBar() { if (usingNetBeansGUI) { - if (usingNetBeansGUI) { - SwingUtilities.invokeLater(() -> { - String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", this.dataSource.getName()); - artifactIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - new Thread(() -> { - IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); - }).start(); - return true; - } - }); - artifactIngestProgressBar.start(); - artifactIngestProgressBar.switchToIndeterminate(); + SwingUtilities.invokeLater(() -> { + String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", this.dataSource.getName()); + artifactIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + new Thread(() -> { + IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + }).start(); + return true; + } }); - } + artifactIngestProgressBar.start(); + artifactIngestProgressBar.switchToIndeterminate(); + }); } } /** - * Starts a data source level analysis NetBeans progress bar in the lower - * right hand corner of the main application window. The progress bar + * Starts a NetBeans progress bar for data source level analysis in the + * lower right hand corner of the main application window. The progress bar * provides the user with a task cancellation button. Pressing it cancels - * either the currently running data source level ingest module or the + * either the currently running data source level ingest module, or the * entire ingest job. Analysis already completed at the time that * cancellation occurs is NOT discarded. */ @@ -924,29 +911,8 @@ final class IngestJobExecutor { } } - private void finishProgressIndicators() { - if (usingNetBeansGUI) { - SwingUtilities.invokeLater(() -> { - if (dataSourceIngestProgressBar != null) { - dataSourceIngestProgressBar.finish(); - dataSourceIngestProgressBar = null; - } - - if (fileIngestProgressBar != null) { - fileIngestProgressBar.finish(); - fileIngestProgressBar = null; - } - - if (artifactIngestProgressBar != null) { - artifactIngestProgressBar.finish(); - artifactIngestProgressBar = null; - } - }); - } - } - /** - * Starts a file analysis NetBeans progress bar in the lower right hand + * Starts a NetBeans progress bar for file analysis in the lower right hand * corner of the main application window. The progress bar provides the user * with a task cancellation button. Pressing it cancels the ingest job. * Analysis already completed at the time that cancellation occurs is NOT @@ -971,6 +937,49 @@ final class IngestJobExecutor { } } + /** + * Finishes the first stage progress bars. + */ + private void finishFirstStageProgressBars() { + if (usingNetBeansGUI) { + SwingUtilities.invokeLater(() -> { + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; + } + + if (fileIngestProgressBar != null) { + fileIngestProgressBar.finish(); + fileIngestProgressBar = null; + } + }); + } + } + + /** + * Finishes all current progress bars. + */ + private void finishProgressBars() { + if (usingNetBeansGUI) { + SwingUtilities.invokeLater(() -> { + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; + } + + if (fileIngestProgressBar != null) { + fileIngestProgressBar.finish(); + fileIngestProgressBar = null; + } + + if (artifactIngestProgressBar != null) { + artifactIngestProgressBar.finish(); + artifactIngestProgressBar = null; + } + }); + } + } + /** * Checks to see if the ingest tasks for the current stage of this job are * completed and does a stage transition if they are. @@ -1008,19 +1017,7 @@ final class IngestJobExecutor { shutDownIngestModulePipeline(pipeline); } - if (usingNetBeansGUI) { - SwingUtilities.invokeLater(() -> { - if (dataSourceIngestProgressBar != null) { - dataSourceIngestProgressBar.finish(); - dataSourceIngestProgressBar = null; - } - - if (fileIngestProgressBar != null) { - fileIngestProgressBar.finish(); - fileIngestProgressBar = null; - } - }); - } + finishFirstStageProgressBars(); if (!jobCancelled && hasLowPriorityDataSourceIngestModules()) { startLowPriorityDataSourceAnalysis(); @@ -1041,7 +1038,7 @@ final class IngestJobExecutor { shutDownIngestModulePipeline(currentDataSourceIngestPipeline); shutDownIngestModulePipeline(artifactIngestPipeline); - finishProgressIndicators(); + finishProgressBars(); if (ingestJobInfo != null) { if (jobCancelled) { @@ -1363,6 +1360,20 @@ final class IngestJobExecutor { } } + /** + * Switches the file ingest progress bar to determinate mode, using the + * estimated number of files to process as the number of work units. + */ + private void switchFileIngestProgressBarToDeterminate() { + if (usingNetBeansGUI) { + SwingUtilities.invokeLater(() -> { + if (fileIngestProgressBar != null) { + fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); + } + }); + } + } + /** * Queries whether or not a temporary cancellation of data source level * ingest in order to stop the currently executing data source level ingest @@ -1383,7 +1394,6 @@ final class IngestJobExecutor { void currentDataSourceIngestModuleCancellationCompleted(String moduleDisplayName) { currentDataSourceIngestModuleCancelled = false; cancelledDataSourceIngestModules.add(moduleDisplayName); - if (usingNetBeansGUI) { SwingUtilities.invokeLater(() -> { /** @@ -1433,7 +1443,6 @@ final class IngestJobExecutor { void cancel(IngestJob.CancellationReason reason) { jobCancelled = true; cancellationReason = reason; - IngestJobExecutor.taskScheduler.cancelPendingFileTasksForIngestJob(this); if (usingNetBeansGUI) { SwingUtilities.invokeLater(() -> { @@ -1446,9 +1455,16 @@ final class IngestJobExecutor { fileIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.fileIngest.displayName", dataSource.getName())); fileIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); } + + if (artifactIngestProgressBar != null) { + artifactIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.dataArtifactIngest.displayName", dataSource.getName())); + artifactIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); + } }); } + IngestJobExecutor.taskScheduler.cancelPendingFileTasksForIngestJob(this); + synchronized (threadRegistrationLock) { for (Thread thread : pausedIngestThreads) { thread.interrupt(); From c4b703ae0c54d72b784bbe70e99461f02baa2963 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 29 Nov 2021 15:14:59 -0500 Subject: [PATCH 06/23] 8202 ingest prog bars on EDT; job cancel off EDT --- .../keywordsearch/IngestSearchRunner.java | 49 ++++++++----------- 1 file changed, 20 insertions(+), 29 deletions(-) mode change 100644 => 100755 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java old mode 100644 new mode 100755 index b31ba6ca33..383abbd3af --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java @@ -248,8 +248,7 @@ final class IngestSearchRunner { } /** - * Task to perform periodic searches for each job (does a single index - * commit first) + * Task to perform periodic searches for each job (does a single index commit first) */ private final class PeriodicSearchTask implements Runnable { @@ -297,23 +296,24 @@ final class IngestSearchRunner { NbBundle.getMessage(this.getClass(), "SearchRunner.Searcher.done.err.msg"), ex.getMessage())); }// catch and ignore if we were cancelled - catch (java.util.concurrent.CancellationException ex) { + catch (java.util.concurrent.CancellationException ex) { } } } stopWatch.stop(); logger.log(Level.INFO, "All periodic searches cumulatively took {0} secs", stopWatch.getElapsedTimeSecs()); //NON-NLS - + // calculate "hold off" time recalculateUpdateIntervalTime(stopWatch.getElapsedTimeSecs()); // ELDEBUG - + // schedule next PeriodicSearchTask jobProcessingTaskFuture = jobProcessingExecutor.schedule(new PeriodicSearchTask(), currentUpdateIntervalMs, MILLISECONDS); - + // exit this thread return; } - + + private void recalculateUpdateIntervalTime(long lastSerchTimeSec) { // If periodic search takes more than 1/4 of the current periodic search interval, then double the search interval if (lastSerchTimeSec * 1000 < currentUpdateIntervalMs / 4) { @@ -321,7 +321,7 @@ final class IngestSearchRunner { } // double the search interval currentUpdateIntervalMs = currentUpdateIntervalMs * 2; - logger.log(Level.WARNING, "Last periodic search took {0} sec. Increasing search interval to {1} sec", new Object[]{lastSerchTimeSec, currentUpdateIntervalMs / 1000}); + logger.log(Level.WARNING, "Last periodic search took {0} sec. Increasing search interval to {1} sec", new Object[]{lastSerchTimeSec, currentUpdateIntervalMs/1000}); return; } } @@ -484,35 +484,26 @@ final class IngestSearchRunner { progressGroup.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg")); } progressGroup.finish(); - new Thread(() -> { - IngestSearchRunner.Searcher.this.cancel(true); - }).start(); - return true; + return IngestSearchRunner.Searcher.this.cancel(true); } - }, - null); + }, null); updateKeywords(); - SwingUtilities.invokeLater(() -> { - ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()]; - int i = 0; - for (Keyword keywordQuery : keywords) { - subProgresses[i] = AggregateProgressFactory.createProgressContributor(keywordQuery.getSearchTerm()); - progressGroup.addContributor(subProgresses[i]); - i++; - } - progressGroup.start(); - }); + ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()]; + int i = 0; + for (Keyword keywordQuery : keywords) { + subProgresses[i] = AggregateProgressFactory.createProgressContributor(keywordQuery.getSearchTerm()); + progressGroup.addContributor(subProgresses[i]); + i++; + } + + progressGroup.start(); final StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - try { - SwingUtilities.invokeLater(() -> { - progressGroup.setDisplayName(displayName); - }); + progressGroup.setDisplayName(displayName); int keywordsSearched = 0; From ce634ce909c3ffee0eac4c592bd8f4bb090e1cb4 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 29 Nov 2021 16:50:36 -0500 Subject: [PATCH 07/23] 8202 ingest prog bars on EDT; job cancel off EDT --- .../autopsy/ingest/IngestJobExecutor.java | 159 +++++++++++------- 1 file changed, 95 insertions(+), 64 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 43fed4ce58..197b35ef2c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -959,7 +959,7 @@ final class IngestJobExecutor { /** * Finishes all current progress bars. */ - private void finishProgressBars() { + private void finishAllProgressBars() { if (usingNetBeansGUI) { SwingUtilities.invokeLater(() -> { if (dataSourceIngestProgressBar != null) { @@ -1038,7 +1038,7 @@ final class IngestJobExecutor { shutDownIngestModulePipeline(currentDataSourceIngestPipeline); shutDownIngestModulePipeline(artifactIngestPipeline); - finishProgressBars(); + finishAllProgressBars(); if (ingestJobInfo != null) { if (jobCancelled) { @@ -1114,7 +1114,7 @@ final class IngestJobExecutor { if (!pipeline.isEmpty()) { /* * Get the file from the task. If the file was "streamed," - * the task may only have the file object ID and a trip to + * the task may only have the file object ID, and a trip to * the case database will be required. */ AbstractFile file; @@ -1128,47 +1128,24 @@ final class IngestJobExecutor { return; } + /** + * Run the file through the modules in the file ingest + * pipeline. + */ final String fileName = file.getName(); processedFiles++; - if (usingNetBeansGUI) { - SwingUtilities.invokeLater(() -> { - if (processedFiles <= estimatedFilesToProcess) { - fileIngestProgressBar.progress(fileName, (int) processedFiles); - } else { - fileIngestProgressBar.progress(fileName, (int) estimatedFilesToProcess); - } - filesInProgress.add(fileName); - }); - } - - /** - * Run the file through the modules in the pipeline. - */ + updateFileIngestProgressForFileTaskStarted(fileName); List errors = new ArrayList<>(); errors.addAll(pipeline.performTask(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors, file); } - - if (usingNetBeansGUI && !jobCancelled) { - SwingUtilities.invokeLater(() -> { - /** - * Update the file ingest progress bar again, in - * case the file was being displayed. - */ - filesInProgress.remove(fileName); - if (filesInProgress.size() > 0) { - fileIngestProgressBar.progress(filesInProgress.get(0)); - } else { - fileIngestProgressBar.progress(""); - } - }); - } + updateFileProgressBarForFileTaskCompleted(fileName); } fileIngestPipelinesQueue.put(pipeline); } } catch (InterruptedException ex) { - logger.log(Level.SEVERE, String.format("Unexpected interrupt of file ingest thread during execution of file ingest job (file obj ID = %d)", task.getFileId()), ex); + logger.log(Level.SEVERE, String.format("Unexpected interrupt of file ingest thread during execution of file ingest job (file object ID = %d, thread ID = %d)", task.getFileId(), task.getThreadId()), ex); Thread.currentThread().interrupt(); } finally { taskScheduler.notifyTaskCompleted(task); @@ -1263,7 +1240,7 @@ final class IngestJobExecutor { /** * Updates the display name shown on the current data source level ingest - * progress bar for this job. + * progress bar for this job, if the job has not been cancelled. * * @param displayName The new display name. */ @@ -1279,8 +1256,8 @@ final class IngestJobExecutor { /** * Switches the current data source level ingest progress bar to determinate - * mode. This should be called if the total work units to process the data - * source is known. + * mode, if the job has not been cancelled. This should be called if the + * total work units to process the data source is known. * * @param workUnits Total number of work units for the processing of the * data source. @@ -1297,8 +1274,8 @@ final class IngestJobExecutor { /** * Switches the current data source level ingest progress bar to - * indeterminate mode. This should be called if the total work units to - * process the data source is unknown. + * indeterminate mode, if the job has not been cancelled. This should be + * called if the total work units to process the data source is unknown. */ void switchDataSourceIngestProgressBarToIndeterminate() { if (usingNetBeansGUI && !jobCancelled) { @@ -1312,7 +1289,8 @@ final class IngestJobExecutor { /** * Updates the current data source level ingest progress bar with the number - * of work units performed, if in the determinate mode. + * of work units performed, if in the determinate mode, and the job has not + * been cancelled. * * @param workUnits Number of work units performed. */ @@ -1328,7 +1306,8 @@ final class IngestJobExecutor { /** * Updates the current data source level ingest progress bar with a new task - * name, where the task name is the "subtitle" under the display name. + * name, where the task name is the "subtitle" under the display name, if + * the job has not been cancelled. * * @param currentTask The task name. */ @@ -1344,8 +1323,9 @@ final class IngestJobExecutor { /** * Updates the current data source level ingest progress bar with a new task - * name and the number of work units performed, if in the determinate mode. - * The task name is the "subtitle" under the display name. + * name and the number of work units performed, if in the determinate mode, + * and the job has not been cancelled. The task name is the "subtitle" under + * the display name. * * @param currentTask The task name. * @param workUnits Number of work units performed. @@ -1374,6 +1354,74 @@ final class IngestJobExecutor { } } + /** + * Updates the current file ingest progress bar upon start of analysis of a + * file, if the job has not been cancelled, if the job has not been + * cancelled. + * + * @param fileName The name of the file. + */ + private void updateFileIngestProgressForFileTaskStarted(String fileName) { + if (usingNetBeansGUI && !jobCancelled) { + SwingUtilities.invokeLater(() -> { + if (processedFiles <= estimatedFilesToProcess) { + fileIngestProgressBar.progress(fileName, (int) processedFiles); + } else { + fileIngestProgressBar.progress(fileName, (int) estimatedFilesToProcess); + } + filesInProgress.add(fileName); + }); + } + } + + /** + * Updates the current file ingest progress bar upon completion of analysis + * of a file, if the job has not been cancelled. + * + * @param fileName The name of the file. + */ + private void updateFileProgressBarForFileTaskCompleted(String fileName) { + if (usingNetBeansGUI && !jobCancelled) { + SwingUtilities.invokeLater(() -> { + filesInProgress.remove(fileName); + /* + * Display the name of another file in progress, or the empty + * string if there are none. + */ + if (filesInProgress.size() > 0) { + fileIngestProgressBar.progress(filesInProgress.get(0)); + } else { + fileIngestProgressBar.progress(""); // NON-NLS + } + }); + } + } + + /** + * Displays a "cancelling" message on all of the current ingest message + * progress bars. + */ + private void displayCancellingProgressMessage() { + if (usingNetBeansGUI) { + SwingUtilities.invokeLater(() -> { + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", dataSource.getName())); + dataSourceIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); + } + + if (fileIngestProgressBar != null) { + fileIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.fileIngest.displayName", dataSource.getName())); + fileIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); + } + + if (artifactIngestProgressBar != null) { + artifactIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.dataArtifactIngest.displayName", dataSource.getName())); + artifactIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); + } + }); + } + } + /** * Queries whether or not a temporary cancellation of data source level * ingest in order to stop the currently executing data source level ingest @@ -1387,14 +1435,16 @@ final class IngestJobExecutor { /** * Rescinds a temporary cancellation of data source level ingest that was - * used to stop a single data source level ingest module for this job. + * used to stop a single data source level ingest module for this job. The + * data source ingest progress bar is reset, if the job has not been + * cancelled. * * @param moduleDisplayName The display name of the module that was stopped. */ void currentDataSourceIngestModuleCancellationCompleted(String moduleDisplayName) { currentDataSourceIngestModuleCancelled = false; cancelledDataSourceIngestModules.add(moduleDisplayName); - if (usingNetBeansGUI) { + if (usingNetBeansGUI && !jobCancelled) { SwingUtilities.invokeLater(() -> { /** * A new progress bar must be created because the cancel button @@ -1443,26 +1493,7 @@ final class IngestJobExecutor { void cancel(IngestJob.CancellationReason reason) { jobCancelled = true; cancellationReason = reason; - - if (usingNetBeansGUI) { - SwingUtilities.invokeLater(() -> { - if (dataSourceIngestProgressBar != null) { - dataSourceIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", dataSource.getName())); - dataSourceIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); - } - - if (fileIngestProgressBar != null) { - fileIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.fileIngest.displayName", dataSource.getName())); - fileIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); - } - - if (artifactIngestProgressBar != null) { - artifactIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.dataArtifactIngest.displayName", dataSource.getName())); - artifactIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); - } - }); - } - + displayCancellingProgressMessage(); IngestJobExecutor.taskScheduler.cancelPendingFileTasksForIngestJob(this); synchronized (threadRegistrationLock) { From c80d81675f324c90f14dc52a45a33eb142df87d2 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 29 Nov 2021 16:58:46 -0500 Subject: [PATCH 08/23] 8202 ingest prog bars on EDT; job cancel off EDT --- Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java | 2 +- Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index a508110b99..350096d626 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -170,7 +170,7 @@ public final class IngestJob { * Starts data source level analysis for this job if it is running in * streaming ingest mode. */ - void processStreamingIngestDataSource() { + void addStreamedDataSource() { if (ingestMode == Mode.STREAMING) { if (ingestModuleExecutor != null) { ingestModuleExecutor.addStreamedDataSource(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java index 2d00727858..fe43bb12b3 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java @@ -67,7 +67,7 @@ class IngestJobInputStream implements IngestStream { @Override public synchronized void close() { closed = true; - ingestJob.processStreamingIngestDataSource(); + ingestJob.addStreamedDataSource(); } @Override From 256f97a6fcc52699a950247e8d30fd77032f9620 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 30 Nov 2021 11:01:13 -0500 Subject: [PATCH 09/23] 8202 ingest prog bars on EDT; job cancel off EDT --- .../src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 197b35ef2c..858d3b5bc5 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -704,7 +704,11 @@ final class IngestJobExecutor { currentDataSourceIngestPipeline = highPriorityDataSourceIngestPipeline; /* - * Schedule ingest tasks. + * Schedule ingest tasks. If only analyzing a subset of the files in + * the data source, the current assumption is that only file ingest + * task need to be scheduled. Data artifact ingest tasks will be + * scheduled as data artifacts produced by the file analysis are + * posted to the blackboard. */ if (!files.isEmpty() && hasFileIngestModules()) { taskScheduler.scheduleFileIngestTasks(this, files); From 097f7fdd4e58977de64426774957744b830b3719 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 30 Nov 2021 11:16:55 -0500 Subject: [PATCH 10/23] 8202 ingest prog bars on EDT; job cancel off EDT --- .../autopsy/ingest/IngestJobExecutor.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 858d3b5bc5..9eeb220cc2 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -706,9 +706,9 @@ final class IngestJobExecutor { /* * Schedule ingest tasks. If only analyzing a subset of the files in * the data source, the current assumption is that only file ingest - * task need to be scheduled. Data artifact ingest tasks will be - * scheduled as data artifacts produced by the file analysis are - * posted to the blackboard. + * tasks for those files need to be scheduled. Data artifact ingest + * tasks will be scheduled as data artifacts produced by the file + * analysis are posted to the blackboard. */ if (!files.isEmpty() && hasFileIngestModules()) { taskScheduler.scheduleFileIngestTasks(this, files); @@ -719,10 +719,11 @@ final class IngestJobExecutor { /* * Check for stage completion. This is necessary because it is * possible that none of the tasks that were just scheduled will - * actually make it to task execution due to the file filter or - * other ingest job settings. In that case, there will never be a - * stage completion check in an ingest thread executing an ingest - * task, so such a job would run forever without a check here. + * actually make it to task execution, due to the file filter or + * other ingest job settings. If that happens, there will never be + * another stage completion check for this job in an ingest thread + * executing an ingest task, so such a job would run forever without + * a check here. */ checkForStageCompleted(); } @@ -769,8 +770,9 @@ final class IngestJobExecutor { * This constraint is implemented by restricting construction of * a streaming mode IngestJob to * IngestManager.openIngestStream(), which constructs and starts - * the job before returning the IngestStream that is used to - * stream in the files and data source. + * the job before returning the IngestStream. This means that + * the code in this method will run before addStreamedFiles() or + * addStreamedDataSource() can be called via the IngestStream. */ taskScheduler.scheduleDataArtifactIngestTasks(this); } @@ -822,8 +824,8 @@ final class IngestJobExecutor { * time, and all of the file level and artifact ingest tasks * scheduled during the initial file streaming stage have * already been executed, there will never be a stage completion - * check in an ingest thread executing an ingest task, so such a - * job would run forever without a check here. + * check in an ingest thread executing an ingest task for this + * job, so such a job would run forever without a check here. */ checkForStageCompleted(); } From ea16779d82212bfb16c14fec993d46e973cc4688 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 30 Nov 2021 12:24:06 -0500 Subject: [PATCH 11/23] 8202 ingest prog bars on EDT; job cancel off EDT --- .../AdHocSearchChildFactory.java | 47 ++-- .../keywordsearch/IngestSearchRunner.java | 201 ++++++++---------- .../autopsy/keywordsearch/QueryResults.java | 160 ++++++-------- 3 files changed, 193 insertions(+), 215 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java index 4dfc014598..c01a81ff66 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java @@ -31,15 +31,18 @@ import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.swing.SwingUtilities; import javax.swing.SwingWorker; import org.netbeans.api.progress.ProgressHandle; import org.openide.nodes.ChildFactory; import org.openide.nodes.Children; import org.openide.nodes.Node; +import org.openide.util.Cancellable; import org.openide.util.NbBundle; import org.openide.util.lookup.Lookups; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode; @@ -89,7 +92,8 @@ class AdHocSearchChildFactory extends ChildFactory { * Constructor * * @param queryRequests Query results - * @param saveResults Flag whether to save search results as KWS artifacts. + * @param saveResults Flag whether to save search results as KWS + * artifacts. */ AdHocSearchChildFactory(Collection queryRequests, boolean saveResults) { this.queryRequests = queryRequests; @@ -129,7 +133,7 @@ class AdHocSearchChildFactory extends ChildFactory { createFlatKeys(queryRequest.getQuery(), toPopulate); } - + // If there were no hits, make a single Node that will display that // no results were found. if (toPopulate.isEmpty()) { @@ -176,7 +180,7 @@ class AdHocSearchChildFactory extends ChildFactory { * Get file properties. */ Map properties = new LinkedHashMap<>(); - + /** * Add a snippet property, if available. */ @@ -204,7 +208,6 @@ class AdHocSearchChildFactory extends ChildFactory { properties.put(LOCATION.toString(), contentName); } - String hitName; BlackboardArtifact artifact = null; if (hit.isArtifactHit()) { @@ -414,21 +417,35 @@ class AdHocSearchChildFactory extends ChildFactory { this.saveResults = saveResults; } - protected void finalizeWorker() { - deregisterWriter(this); - EventQueue.invokeLater(progress::finish); - } - @Override protected Void doInBackground() throws Exception { - registerWriter(this); //register (synchronized on class) outside of writerLock to prevent deadlock - final String queryStr = query.getQueryString(); - final String queryDisp = queryStr.length() > QUERY_DISPLAY_LEN ? queryStr.substring(0, QUERY_DISPLAY_LEN - 1) + " ..." : queryStr; try { - progress = ProgressHandle.createHandle(NbBundle.getMessage(this.getClass(), "KeywordSearchResultFactory.progress.saving", queryDisp), () -> BlackboardResultWriter.this.cancel(true)); - hits.process(progress, null, this, false, saveResults, null); + if (RuntimeProperties.runningWithGUI()) { + final String queryStr = query.getQueryString(); + final String queryDisp = queryStr.length() > QUERY_DISPLAY_LEN ? queryStr.substring(0, QUERY_DISPLAY_LEN - 1) + " ..." : queryStr; + SwingUtilities.invokeLater(() -> { + progress = ProgressHandle.createHandle( + NbBundle.getMessage(this.getClass(), "KeywordSearchResultFactory.progress.saving", queryDisp), + new Cancellable() { + @Override + public boolean cancel() { + //progress.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg")); + logger.log(Level.INFO, "Ad hoc search cancelled by user"); //NON-NLS + new Thread(() -> { + BlackboardResultWriter.this.cancel(true); + }).start(); + return true; + } + }); + }); + } + registerWriter(this); //register (synchronized on class) outside of writerLock to prevent deadlock + hits.process(this, false, saveResults, null); } finally { - finalizeWorker(); + deregisterWriter(this); + if (RuntimeProperties.runningWithGUI() && progress != null) { + EventQueue.invokeLater(progress::finish); + } } return null; } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java index 383abbd3af..8c7c0ffa0a 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java @@ -38,15 +38,16 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; -import org.netbeans.api.progress.aggregate.AggregateProgressFactory; +import org.netbeans.api.progress.ProgressHandle; import org.netbeans.api.progress.aggregate.AggregateProgressHandle; -import org.netbeans.api.progress.aggregate.ProgressContributor; import org.openide.util.Cancellable; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; +import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.StopWatch; +import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestServices; @@ -248,7 +249,8 @@ final class IngestSearchRunner { } /** - * Task to perform periodic searches for each job (does a single index commit first) + * Task to perform periodic searches for each job (does a single index + * commit first) */ private final class PeriodicSearchTask implements Runnable { @@ -296,24 +298,23 @@ final class IngestSearchRunner { NbBundle.getMessage(this.getClass(), "SearchRunner.Searcher.done.err.msg"), ex.getMessage())); }// catch and ignore if we were cancelled - catch (java.util.concurrent.CancellationException ex) { + catch (java.util.concurrent.CancellationException ex) { } } } stopWatch.stop(); logger.log(Level.INFO, "All periodic searches cumulatively took {0} secs", stopWatch.getElapsedTimeSecs()); //NON-NLS - + // calculate "hold off" time recalculateUpdateIntervalTime(stopWatch.getElapsedTimeSecs()); // ELDEBUG - + // schedule next PeriodicSearchTask jobProcessingTaskFuture = jobProcessingExecutor.schedule(new PeriodicSearchTask(), currentUpdateIntervalMs, MILLISECONDS); - + // exit this thread return; } - - + private void recalculateUpdateIntervalTime(long lastSerchTimeSec) { // If periodic search takes more than 1/4 of the current periodic search interval, then double the search interval if (lastSerchTimeSec * 1000 < currentUpdateIntervalMs / 4) { @@ -321,7 +322,7 @@ final class IngestSearchRunner { } // double the search interval currentUpdateIntervalMs = currentUpdateIntervalMs * 2; - logger.log(Level.WARNING, "Last periodic search took {0} sec. Increasing search interval to {1} sec", new Object[]{lastSerchTimeSec, currentUpdateIntervalMs/1000}); + logger.log(Level.WARNING, "Last periodic search took {0} sec. Increasing search interval to {1} sec", new Object[]{lastSerchTimeSec, currentUpdateIntervalMs / 1000}); return; } } @@ -447,13 +448,15 @@ final class IngestSearchRunner { /** * Searcher has private copies/snapshots of the lists and keywords */ - private SearchJobInfo job; - private List keywords; //keywords to search - private List keywordListNames; // lists currently being searched - private List keywordLists; - private Map keywordToList; //keyword to list name mapping + private final SearchJobInfo job; + private final List keywords; //keywords to search + private final List keywordListNames; // lists currently being searched + private final List keywordLists; + private final Map keywordToList; //keyword to list name mapping + private final boolean usingNetBeansGUI; + @ThreadConfined(type = ThreadConfined.ThreadType.AWT) + private ProgressHandle progressIndicator; private AggregateProgressHandle progressGroup; - private final Logger logger = Logger.getLogger(IngestSearchRunner.Searcher.class.getName()); private boolean finalRun = false; Searcher(SearchJobInfo job) { @@ -463,6 +466,7 @@ final class IngestSearchRunner { keywordToList = new HashMap<>(); keywordLists = new ArrayList<>(); //keywords are populated as searcher runs + usingNetBeansGUI = RuntimeProperties.runningWithGUI(); } Searcher(SearchJobInfo job, boolean finalRun) { @@ -473,76 +477,88 @@ final class IngestSearchRunner { @Override @Messages("SearchRunner.query.exception.msg=Error performing query:") protected Object doInBackground() throws Exception { - final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") - + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); - final String pgDisplayName = displayName + (" (" + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") + ")"); - progressGroup = AggregateProgressFactory.createSystemHandle(pgDisplayName, null, new Cancellable() { - @Override - public boolean cancel() { - logger.log(Level.INFO, "Cancelling the searcher by user."); //NON-NLS - if (progressGroup != null) { - progressGroup.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg")); - } - progressGroup.finish(); - return IngestSearchRunner.Searcher.this.cancel(true); - } - }, null); - - updateKeywords(); - - ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()]; - int i = 0; - for (Keyword keywordQuery : keywords) { - subProgresses[i] = AggregateProgressFactory.createProgressContributor(keywordQuery.getSearchTerm()); - progressGroup.addContributor(subProgresses[i]); - i++; + if (usingNetBeansGUI) { + /* + * If running in the NetBeans thick client application version + * of Autopsy, NetBeans progress handles (i.e., progress bars) + * are used to display search progress in the lower right hand + * corner of the main application window. + * + * A layer of abstraction to allow alternate representations of + * progress could be used here, as it is in other places in the + * application (see implementations and usage of + * org.sleuthkit.autopsy.progress.ProgressIndicator interface), + * to better decouple keyword search from the application's + * presentation layer. + */ + final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") + + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); + SwingUtilities.invokeLater(() -> { + progressIndicator = ProgressHandle.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + progressIndicator.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg")); + logger.log(Level.INFO, "Search cancelled by user"); //NON-NLS + new Thread(() -> { + IngestSearchRunner.Searcher.this.cancel(true); + }).start(); + return true; + } + }); + progressIndicator.start(); + progressIndicator.switchToIndeterminate(); + }); } - progressGroup.start(); - final StopWatch stopWatch = new StopWatch(); stopWatch.start(); try { - progressGroup.setDisplayName(displayName); - - int keywordsSearched = 0; - + updateKeywords(); for (Keyword keyword : keywords) { - if (this.isCancelled() || this.job.getJobContext().fileIngestIsCancelled()) { - logger.log(Level.INFO, "Cancel detected, bailing before new keyword processed: {0}", keyword.getSearchTerm()); //NON-NLS + if (isCancelled() || job.getJobContext().fileIngestIsCancelled()) { + logger.log(Level.INFO, "Cancellation requested, exiting before new keyword processed: {0}", keyword.getSearchTerm()); //NON-NLS return null; } - final KeywordList keywordList = keywordToList.get(keyword); - - //new subProgress will be active after the initial query - //when we know number of hits to start() with - if (keywordsSearched > 0) { - subProgresses[keywordsSearched - 1].finish(); + KeywordList keywordList = keywordToList.get(keyword); + if (usingNetBeansGUI) { + String searchTermStr = keyword.getSearchTerm(); + if (searchTermStr.length() > 50) { + searchTermStr = searchTermStr.substring(0, 49) + "..."; + } else { + searchTermStr = searchTermStr; + } + final String progressMessage = keywordList.getName() + ": " + searchTermStr; + SwingUtilities.invokeLater(() -> { + progressIndicator.progress(progressMessage); + }); } - KeywordSearchQuery keywordSearchQuery = KeywordSearchUtil.getQueryForKeyword(keyword, keywordList); - // Filtering //limit search to currently ingested data sources //set up a filter with 1 or more image ids OR'ed - final KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, job.getDataSourceId()); + KeywordSearchQuery keywordSearchQuery = KeywordSearchUtil.getQueryForKeyword(keyword, keywordList); + KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, job.getDataSourceId()); keywordSearchQuery.addFilter(dataSourceFilter); - QueryResults queryResults; - // Do the actual search + QueryResults queryResults; try { queryResults = keywordSearchQuery.performQuery(); } catch (KeywordSearchModuleException | NoOpenCoreException ex) { logger.log(Level.SEVERE, "Error performing query: " + keyword.getSearchTerm(), ex); //NON-NLS - MessageNotifyUtil.Notify.error(Bundle.SearchRunner_query_exception_msg() + keyword.getSearchTerm(), ex.getCause().getMessage()); + if (usingNetBeansGUI) { + final String userMessage = Bundle.SearchRunner_query_exception_msg() + keyword.getSearchTerm(); + SwingUtilities.invokeLater(() -> { + MessageNotifyUtil.Notify.error(userMessage, ex.getCause().getMessage()); + }); + } //no reason to continue with next query if recovery failed //or wait for recovery to kick in and run again later //likely case has closed and threads are being interrupted return null; } catch (CancellationException e) { - logger.log(Level.INFO, "Cancel detected, bailing during keyword query: {0}", keyword.getSearchTerm()); //NON-NLS + logger.log(Level.INFO, "Cancellation requested, exiting during keyword query: {0}", keyword.getSearchTerm()); //NON-NLS return null; } @@ -551,42 +567,25 @@ final class IngestSearchRunner { QueryResults newResults = filterResults(queryResults); if (!newResults.getKeywords().isEmpty()) { - - // Write results to BB - //scale progress bar more more granular, per result sub-progress, within per keyword - int totalUnits = newResults.getKeywords().size(); - subProgresses[keywordsSearched].start(totalUnits); - int unitProgress = 0; - String queryDisplayStr = keyword.getSearchTerm(); - if (queryDisplayStr.length() > 50) { - queryDisplayStr = queryDisplayStr.substring(0, 49) + "..."; - } - subProgresses[keywordsSearched].progress(keywordList.getName() + ": " + queryDisplayStr, unitProgress); - // Create blackboard artifacts - newResults.process(null, subProgresses[keywordsSearched], this, keywordList.getIngestMessages(), true, job.getJobId()); - - } //if has results - - //reset the status text before it goes away - subProgresses[keywordsSearched].progress(""); - - ++keywordsSearched; - - } //for each keyword - - } //end try block - catch (Exception ex) { - logger.log(Level.WARNING, "searcher exception occurred", ex); //NON-NLS - } finally { - try { - finalizeSearcher(); - stopWatch.stop(); - logger.log(Level.INFO, "Searcher took {0} secs to run (final = {1})", new Object[]{stopWatch.getElapsedTimeSecs(), this.finalRun}); //NON-NLS - } finally { - // In case a thread is waiting on this worker to be done - job.searchNotify(); + newResults.process(this, keywordList.getIngestMessages(), true, job.getJobId()); + } } + } catch (Exception ex) { + logger.log(Level.WARNING, "Error occurred during keyword search", ex); //NON-NLS + } finally { + if (progressGroup != null) { + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + progressGroup.finish(); + } + }); + } + stopWatch.stop(); + logger.log(Level.INFO, "Searcher took {0} secs to run (final = {1})", new Object[]{stopWatch.getElapsedTimeSecs(), this.finalRun}); //NON-NLS + // In case a thread is waiting on this worker to be done + job.searchNotify(); } return null; @@ -612,20 +611,6 @@ final class IngestSearchRunner { } } - /** - * Performs the cleanup that needs to be done right AFTER - * doInBackground() returns without relying on done() method that is not - * guaranteed to run. - */ - private void finalizeSearcher() { - SwingUtilities.invokeLater(new Runnable() { - @Override - public void run() { - progressGroup.finish(); - } - }); - } - /** * This method filters out all of the hits found in earlier periodic * searches and returns only the results found by the most recent diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index 40ed7db43d..d58f9c1a5c 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.logging.Level; +import javax.swing.SwingUtilities; import javax.swing.SwingWorker; import org.apache.commons.lang.StringUtils; import org.netbeans.api.progress.ProgressHandle; @@ -32,6 +33,7 @@ import org.netbeans.api.progress.aggregate.ProgressContributor; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.EscapeUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestMessage; @@ -51,6 +53,8 @@ import org.sleuthkit.datamodel.TskCoreException; * about the search hits to the ingest inbox, and publishing an event to notify * subscribers of the blackboard posts. */ + + class QueryResults { private static final Logger logger = Logger.getLogger(QueryResults.class.getName()); @@ -131,10 +135,6 @@ class QueryResults { * All calls to the addResult method MUST be completed before calling this * method. * - * @param progress A progress indicator that reports the number of - * keywords processed. Can be null. - * @param subProgress A progress contributor that reports the keyword - * currently being processed. Can be null. * @param worker The SwingWorker that is being used to do the * processing, will be checked for task cancellation * before processing each keyword. @@ -145,19 +145,7 @@ class QueryResults { * @param ingestJobId The numeric identifier of the ingest job within which * the artifacts are being created, may be null. */ - void process(ProgressHandle progress, ProgressContributor subProgress, SwingWorker worker, boolean notifyInbox, boolean saveResults, Long ingestJobId) { - /* - * Initialize the progress indicator to the number of keywords that will - * be processed. - */ - if (null != progress) { - progress.start(getKeywords().size()); - } - - /* - * Process the keyword hits for each keyword. - */ - int keywordsProcessed = 0; + void process(SwingWorker worker, boolean notifyInbox, boolean saveResults, Long ingestJobId) { final Collection hitArtifacts = new ArrayList<>(); for (final Keyword keyword : getKeywords()) { /* @@ -165,22 +153,7 @@ class QueryResults { */ if (worker.isCancelled()) { logger.log(Level.INFO, "Processing cancelled, exiting before processing search term {0}", keyword.getSearchTerm()); //NON-NLS - break; - } - - /* - * Update the progress indicator and the show the current keyword - * via the progress contributor. - */ - if (progress != null) { - progress.progress(keyword.toString(), keywordsProcessed); - } - if (subProgress != null) { - String hitDisplayStr = keyword.getSearchTerm(); - if (hitDisplayStr.length() > 50) { - hitDisplayStr = hitDisplayStr.substring(0, 49) + "..."; - } - subProgress.progress(query.getKeywordList().getName() + ": " + hitDisplayStr, keywordsProcessed); + return; } /* @@ -202,7 +175,7 @@ class QueryResults { snippet = LuceneQuery.querySnippet(snippetQuery, hit.getSolrObjectId(), hit.getChunkId(), !query.isLiteral(), true); } catch (NoOpenCoreException e) { logger.log(Level.SEVERE, "Solr core closed while executing snippet query " + snippetQuery, e); //NON-NLS - break; // Stop processing. + return; // Stop processing. } catch (Exception e) { logger.log(Level.SEVERE, "Error executing snippet query " + snippetQuery, e); //NON-NLS continue; // Try processing the next hit. @@ -242,8 +215,6 @@ class QueryResults { } } } - - ++keywordsProcessed; } /* @@ -298,69 +269,74 @@ class QueryResults { * @throws TskCoreException If there is a problem generating or send the * inbox message. */ - private void writeSingleFileInboxMessage(BlackboardArtifact artifact, Content hitContent) throws TskCoreException { - StringBuilder subjectSb = new StringBuilder(1024); - if (!query.isLiteral()) { - subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExpHitLbl")); - } else { - subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLbl")); - } + private void writeSingleFileInboxMessage(final BlackboardArtifact artifact, final Content hitContent) throws TskCoreException { + if (artifact != null && hitContent != null && RuntimeProperties.runningWithGUI()) { + final StringBuilder subjectSb = new StringBuilder(1024); + if (!query.isLiteral()) { + subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExpHitLbl")); + } else { + subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLbl")); + } - StringBuilder detailsSb = new StringBuilder(1024); - String uniqueKey = null; - BlackboardAttribute attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD)); - if (attr != null) { - final String keyword = attr.getValueString(); - subjectSb.append(keyword); - uniqueKey = keyword.toLowerCase(); - detailsSb.append(""); //NON-NLS - detailsSb.append(""); //NON-NLS - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitThLbl")); - detailsSb.append(""); //NON-NLS - detailsSb.append(""); //NON-NLS - } + final StringBuilder detailsSb = new StringBuilder(1024); + String uniqueKey = null; + BlackboardAttribute attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD)); + if (attr != null) { + final String keyword = attr.getValueString(); + subjectSb.append(keyword); + uniqueKey = keyword.toLowerCase(); + detailsSb.append("
").append(EscapeUtil.escapeHtml(keyword)).append("
"); //NON-NLS + detailsSb.append(""); //NON-NLS + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitThLbl")); + detailsSb.append(""); //NON-NLS + detailsSb.append(""); //NON-NLS + } - //preview - attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW)); - if (attr != null) { - detailsSb.append(""); //NON-NLS - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.previewThLbl")); - detailsSb.append(""); //NON-NLS - detailsSb.append(""); //NON-NLS - } - - //file - detailsSb.append(""); //NON-NLS - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.fileThLbl")); - if (hitContent instanceof AbstractFile) { - AbstractFile hitFile = (AbstractFile) hitContent; - detailsSb.append(""); //NON-NLS - } else { - detailsSb.append(""); //NON-NLS - } - detailsSb.append(""); //NON-NLS - - //list - attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME)); - if (attr != null) { - detailsSb.append(""); //NON-NLS - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.listThLbl")); - detailsSb.append(""); //NON-NLS - detailsSb.append(""); //NON-NLS - } - - //regex - if (!query.isLiteral()) { - attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP)); + //preview + attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW)); if (attr != null) { detailsSb.append(""); //NON-NLS - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExThLbl")); + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.previewThLbl")); + detailsSb.append(""); //NON-NLS + detailsSb.append(""); //NON-NLS + } + + //file + detailsSb.append(""); //NON-NLS + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.fileThLbl")); + if (hitContent instanceof AbstractFile) { + AbstractFile hitFile = (AbstractFile) hitContent; + detailsSb.append(""); //NON-NLS + } else { + detailsSb.append(""); //NON-NLS + } + detailsSb.append(""); //NON-NLS + + //list + attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME)); + if (attr != null) { + detailsSb.append(""); //NON-NLS + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.listThLbl")); detailsSb.append(""); //NON-NLS detailsSb.append(""); //NON-NLS } - } - detailsSb.append("
").append(EscapeUtil.escapeHtml(keyword)).append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(hitFile.getParentPath()).append(hitFile.getName()).append("").append(hitContent.getName()).append("
").append(attr.getValueString()).append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(hitFile.getParentPath()).append(hitFile.getName()).append("").append(hitContent.getName()).append("
").append(attr.getValueString()).append("
"); //NON-NLS - IngestServices.getInstance().postMessage(IngestMessage.createDataMessage(MODULE_NAME, subjectSb.toString(), detailsSb.toString(), uniqueKey, artifact)); + //regex + if (!query.isLiteral()) { + attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP)); + if (attr != null) { + detailsSb.append(""); //NON-NLS + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExThLbl")); + detailsSb.append("").append(attr.getValueString()).append(""); //NON-NLS + detailsSb.append(""); //NON-NLS + } + } + detailsSb.append(""); //NON-NLS + + String key = uniqueKey; // RC: Might be null, does this work? + SwingUtilities.invokeLater(() -> { + IngestServices.getInstance().postMessage(IngestMessage.createDataMessage(MODULE_NAME, subjectSb.toString(), detailsSb.toString(), key, artifact)); + }); + } } } From 65392209d3b3a988751a0bba415c6487feaf55ee Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 30 Nov 2021 13:21:47 -0500 Subject: [PATCH 12/23] 8202 KWS cancellation out of EDT, progress ops in EDT --- .../keywordsearch/IngestSearchRunner.java | 75 +++++++++---------- 1 file changed, 36 insertions(+), 39 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java index 8c7c0ffa0a..1174e6fc05 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java @@ -39,7 +39,6 @@ import java.util.logging.Level; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; import org.netbeans.api.progress.ProgressHandle; -import org.netbeans.api.progress.aggregate.AggregateProgressHandle; import org.openide.util.Cancellable; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; @@ -456,7 +455,6 @@ final class IngestSearchRunner { private final boolean usingNetBeansGUI; @ThreadConfined(type = ThreadConfined.ThreadType.AWT) private ProgressHandle progressIndicator; - private AggregateProgressHandle progressGroup; private boolean finalRun = false; Searcher(SearchJobInfo job) { @@ -477,42 +475,42 @@ final class IngestSearchRunner { @Override @Messages("SearchRunner.query.exception.msg=Error performing query:") protected Object doInBackground() throws Exception { - if (usingNetBeansGUI) { - /* - * If running in the NetBeans thick client application version - * of Autopsy, NetBeans progress handles (i.e., progress bars) - * are used to display search progress in the lower right hand - * corner of the main application window. - * - * A layer of abstraction to allow alternate representations of - * progress could be used here, as it is in other places in the - * application (see implementations and usage of - * org.sleuthkit.autopsy.progress.ProgressIndicator interface), - * to better decouple keyword search from the application's - * presentation layer. - */ - final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") - + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); - SwingUtilities.invokeLater(() -> { - progressIndicator = ProgressHandle.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - progressIndicator.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg")); - logger.log(Level.INFO, "Search cancelled by user"); //NON-NLS - new Thread(() -> { - IngestSearchRunner.Searcher.this.cancel(true); - }).start(); - return true; - } - }); - progressIndicator.start(); - progressIndicator.switchToIndeterminate(); - }); - } - final StopWatch stopWatch = new StopWatch(); stopWatch.start(); try { + if (usingNetBeansGUI) { + /* + * If running in the NetBeans thick client application + * version of Autopsy, NetBeans progress handles (i.e., + * progress bars) are used to display search progress in the + * lower right hand corner of the main application window. + * + * A layer of abstraction to allow alternate representations + * of progress could be used here, as it is in other places + * in the application (see implementations and usage of + * org.sleuthkit.autopsy.progress.ProgressIndicator + * interface), to better decouple keyword search from the + * application's presentation layer. + */ + final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") + + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); + SwingUtilities.invokeAndWait(() -> { + progressIndicator = ProgressHandle.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + progressIndicator.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg")); + logger.log(Level.INFO, "Search cancelled by user"); //NON-NLS + new Thread(() -> { + IngestSearchRunner.Searcher.this.cancel(true); + }).start(); + return true; + } + }); + progressIndicator.start(); + progressIndicator.switchToIndeterminate(); + }); + } + updateKeywords(); for (Keyword keyword : keywords) { if (isCancelled() || job.getJobContext().fileIngestIsCancelled()) { @@ -525,8 +523,6 @@ final class IngestSearchRunner { String searchTermStr = keyword.getSearchTerm(); if (searchTermStr.length() > 50) { searchTermStr = searchTermStr.substring(0, 49) + "..."; - } else { - searchTermStr = searchTermStr; } final String progressMessage = keywordList.getName() + ": " + searchTermStr; SwingUtilities.invokeLater(() -> { @@ -574,11 +570,12 @@ final class IngestSearchRunner { } catch (Exception ex) { logger.log(Level.WARNING, "Error occurred during keyword search", ex); //NON-NLS } finally { - if (progressGroup != null) { + if (progressIndicator != null) { SwingUtilities.invokeLater(new Runnable() { @Override public void run() { - progressGroup.finish(); + progressIndicator.finish(); + progressIndicator = null; } }); } From 4d964e627a8551dcb96a76aa794d90adf4712516 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 30 Nov 2021 16:12:16 -0500 Subject: [PATCH 13/23] 8202 address review comments --- .../autopsy/ingest/IngestJobExecutor.java | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 9eeb220cc2..78f7238df4 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -541,7 +541,7 @@ final class IngestJobExecutor { } /** - * Determnines which ingets job stage to start in and starts up the ingest + * Determnines which ingest job stage to start in and starts up the ingest * module pipelines. * * @return A collection of ingest module startup errors, empty on success. @@ -676,14 +676,23 @@ final class IngestJobExecutor { if (hasFileIngestModules()) { /* - * Do a count of the files the data source processor has added - * to the case database. This number will be used to estimate - * how many files remain to be analyzed as each file ingest task - * is completed. + * Do an estimate of the total number of files to be analyzed. + * This number will be used to estimate of how many files remain + * to be analyzed as each file ingest task is completed. The + * numbers are estimates because file analysis can add carved + * files and/or derived files. */ if (files.isEmpty()) { + /* + * Do a count of the files the data source processor has + * added to the case database. + */ estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor()); } else { + /* + * Use the number of files in the specified subset of all of + * the files for the data source. + */ estimatedFilesToProcess = files.size(); } startFileIngestProgressBar(); @@ -794,11 +803,7 @@ final class IngestJobExecutor { /* * For ingest job progress reporting purposes, do a count of the * files the data source processor has added to the case - * database. This number will be used to estimate how many files - * remain to be analyzed as each file ingest task is completed. - * The estimate will likely be an over-estimate, since some of - * the files will have already been "streamed" to this job and - * processed. + * database. */ estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor()); switchFileIngestProgressBarToDeterminate(); @@ -1204,6 +1209,7 @@ final class IngestJobExecutor { void addFiles(List files) { if (stage.equals(IngestJobStage.STREAMED_FILE_ANALYSIS_ONLY) || stage.equals(IngestJobStage.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { + estimatedFilesToProcess += files.size(); taskScheduler.fastTrackFileIngestTasks(this, files); } else { logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); @@ -1362,19 +1368,17 @@ final class IngestJobExecutor { /** * Updates the current file ingest progress bar upon start of analysis of a - * file, if the job has not been cancelled, if the job has not been - * cancelled. + * file, if the job has not been cancelled. * * @param fileName The name of the file. */ private void updateFileIngestProgressForFileTaskStarted(String fileName) { if (usingNetBeansGUI && !jobCancelled) { SwingUtilities.invokeLater(() -> { - if (processedFiles <= estimatedFilesToProcess) { - fileIngestProgressBar.progress(fileName, (int) processedFiles); - } else { - fileIngestProgressBar.progress(fileName, (int) estimatedFilesToProcess); - } + /* + * Note that if processedFiles exceeds estimatedFilesToProcess + */ + fileIngestProgressBar.progress(fileName, (int) processedFiles); filesInProgress.add(fileName); }); } From 1aa1c80b53e4096938efd354b4ef50ab97e563d3 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 30 Nov 2021 16:20:58 -0500 Subject: [PATCH 14/23] 8202 KWS cancellation out of EDT, progress ops in EDT --- .../sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java index 1174e6fc05..bb2fbe189a 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java @@ -492,9 +492,9 @@ final class IngestSearchRunner { * interface), to better decouple keyword search from the * application's presentation layer. */ - final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") - + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); SwingUtilities.invokeAndWait(() -> { + final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") + + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); progressIndicator = ProgressHandle.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { From e7c3eb14bb2a17ac123866fefb4d9ef26835499d Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 30 Nov 2021 16:30:55 -0500 Subject: [PATCH 15/23] 8202 KWS cancellation out of EDT, progress ops in EDT --- .../src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index d58f9c1a5c..313bad3e60 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -333,7 +333,7 @@ class QueryResults { } detailsSb.append(""); //NON-NLS - String key = uniqueKey; // RC: Might be null, does this work? + final String key = uniqueKey; // Might be null, but that's supported. SwingUtilities.invokeLater(() -> { IngestServices.getInstance().postMessage(IngestMessage.createDataMessage(MODULE_NAME, subjectSb.toString(), detailsSb.toString(), key, artifact)); }); From f0eed25f2178836231a0730af8c9ef816c5b5768 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 30 Nov 2021 17:06:19 -0500 Subject: [PATCH 16/23] 8202 KWS cancellation out of EDT, progress ops in EDT --- Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 78f7238df4..fd331dfab7 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -1209,7 +1209,6 @@ final class IngestJobExecutor { void addFiles(List files) { if (stage.equals(IngestJobStage.STREAMED_FILE_ANALYSIS_ONLY) || stage.equals(IngestJobStage.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { - estimatedFilesToProcess += files.size(); taskScheduler.fastTrackFileIngestTasks(this, files); } else { logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); From 1804f4cb1c9f4c1085d5c39ea4c8ad0cf7515b2b Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 30 Nov 2021 17:54:37 -0500 Subject: [PATCH 17/23] 8202 fix progress bar IllegalArgumentException --- .../sleuthkit/autopsy/ingest/IngestJobExecutor.java | 10 +++++++++- .../org/netbeans/core/startup/Bundle.properties | 2 +- .../netbeans/core/windows/view/ui/Bundle.properties | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index fd331dfab7..31df9b39f6 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -1375,8 +1375,16 @@ final class IngestJobExecutor { if (usingNetBeansGUI && !jobCancelled) { SwingUtilities.invokeLater(() -> { /* - * Note that if processedFiles exceeds estimatedFilesToProcess + * Note that if processedFiles exceeds estimatedFilesToProcess, + * the progress bar will go into an infinte loop throwing + * IllegalArgumentExceptions in the EDT. */ + long processedFilesCapture = processedFiles; + if (processedFiles <= estimatedFilesToProcess) { + fileIngestProgressBar.progress(fileName, (int) processedFilesCapture); + } else { + fileIngestProgressBar.progress(fileName, (int) estimatedFilesToProcess); + } fileIngestProgressBar.progress(fileName, (int) processedFiles); filesInProgress.add(fileName); }); diff --git a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties index cd253dc3cb..e9620c3d7a 100644 --- a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties +++ b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Thu, 30 Sep 2021 19:36:31 -0400 +#Tue, 30 Nov 2021 17:19:50 -0500 LBL_splash_window_title=Starting Autopsy SPLASH_HEIGHT=314 SPLASH_WIDTH=538 diff --git a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties index 2d02262803..f591caf623 100644 --- a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties +++ b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties @@ -1,4 +1,4 @@ #Updated by build script -#Thu, 30 Sep 2021 19:36:31 -0400 +#Tue, 30 Nov 2021 17:19:50 -0500 CTL_MainWindow_Title=Autopsy 4.19.2 CTL_MainWindow_Title_No_Project=Autopsy 4.19.2 From 73e4f647b1f5a2dda8fe2a99daa1d5cd86d694f8 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 30 Nov 2021 17:57:02 -0500 Subject: [PATCH 18/23] 8202 fix progress bar IllegalArgumentException --- Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 31df9b39f6..83c705acb1 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -1376,7 +1376,8 @@ final class IngestJobExecutor { SwingUtilities.invokeLater(() -> { /* * Note that if processedFiles exceeds estimatedFilesToProcess, - * the progress bar will go into an infinte loop throwing + * i.e., max work units fopr the progress bar, the progress bar + * will go into an infinte loop throwing * IllegalArgumentExceptions in the EDT. */ long processedFilesCapture = processedFiles; From f7e1886a71ca6ec313aa7dea846208b0867d1cb0 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 30 Nov 2021 17:57:34 -0500 Subject: [PATCH 19/23] 8202 fix progress bar IllegalArgumentException --- Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 83c705acb1..f397dd55d9 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -1376,7 +1376,7 @@ final class IngestJobExecutor { SwingUtilities.invokeLater(() -> { /* * Note that if processedFiles exceeds estimatedFilesToProcess, - * i.e., max work units fopr the progress bar, the progress bar + * i.e., max work units for the progress bar, the progress bar * will go into an infinte loop throwing * IllegalArgumentExceptions in the EDT. */ From b28b777ff49a6eec9cb45928a967f2ac1046a4ab Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 30 Nov 2021 17:58:48 -0500 Subject: [PATCH 20/23] 8202 fix progress bar IllegalArgumentException --- Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index f397dd55d9..f49d6ecb97 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -1376,8 +1376,8 @@ final class IngestJobExecutor { SwingUtilities.invokeLater(() -> { /* * Note that if processedFiles exceeds estimatedFilesToProcess, - * i.e., max work units for the progress bar, the progress bar - * will go into an infinte loop throwing + * i.e., the max work units set for the progress bar, the + * progress bar will go into an infinite loop throwing * IllegalArgumentExceptions in the EDT. */ long processedFilesCapture = processedFiles; From 2b9b608bcf19893eb9e5899e688ce28637b47cbc Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 30 Nov 2021 17:59:40 -0500 Subject: [PATCH 21/23] 8202 fix progress bar IllegalArgumentException --- Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index f49d6ecb97..660df7326b 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -1381,7 +1381,7 @@ final class IngestJobExecutor { * IllegalArgumentExceptions in the EDT. */ long processedFilesCapture = processedFiles; - if (processedFiles <= estimatedFilesToProcess) { + if (processedFilesCapture <= estimatedFilesToProcess) { fileIngestProgressBar.progress(fileName, (int) processedFilesCapture); } else { fileIngestProgressBar.progress(fileName, (int) estimatedFilesToProcess); From 220a0a7cbb7cc464f30bcdb44d4a59e880ad9ee3 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 30 Nov 2021 18:03:48 -0500 Subject: [PATCH 22/23] 8202 fix progress bar IllegalArgumentException --- .../sleuthkit/autopsy/ingest/IngestJobExecutor.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 660df7326b..1bfb817cde 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -1375,10 +1375,15 @@ final class IngestJobExecutor { if (usingNetBeansGUI && !jobCancelled) { SwingUtilities.invokeLater(() -> { /* - * Note that if processedFiles exceeds estimatedFilesToProcess, - * i.e., the max work units set for the progress bar, the - * progress bar will go into an infinite loop throwing - * IllegalArgumentExceptions in the EDT. + * If processedFiles exceeds estimatedFilesToProcess, i.e., the + * max work units set for the progress bar, the progress bar + * will go into an infinite loop throwing + * IllegalArgumentExceptions in the EDT (NetBeans bug). Also, a + * check-then-act race condition needs to be avoided here. This + * can be done without guarding processedFiles and + * estimatedFilesToProcess with the same lock because + * estimatedFilesToProcess does not change after it is used to + * switch the progress bar to determinate mode. */ long processedFilesCapture = processedFiles; if (processedFilesCapture <= estimatedFilesToProcess) { From b591883ff9a0d4725d5f5b12fac1e393e88f98a2 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 30 Nov 2021 18:53:28 -0500 Subject: [PATCH 23/23] 8202 fix progress bar IllegalArgumentException --- Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 1bfb817cde..47b4d9b601 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -1391,7 +1391,6 @@ final class IngestJobExecutor { } else { fileIngestProgressBar.progress(fileName, (int) estimatedFilesToProcess); } - fileIngestProgressBar.progress(fileName, (int) processedFiles); filesInProgress.add(fileName); }); }