From ed42d5e9650778dec8235e26ae36a7cae92c1ace Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 15 Nov 2021 15:40:36 -0500 Subject: [PATCH 01/19] 7529 KWS artifact ingest module --- .../KeywordSearchArtifactIngestModule.java | 63 +++++++++++++++++++ .../KeywordSearchModuleFactory.java | 14 ++++- .../keywordsearch/SolrSearchService.java | 22 ------- 3 files changed, 76 insertions(+), 23 deletions(-) create mode 100755 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchArtifactIngestModule.java diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchArtifactIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchArtifactIngestModule.java new file mode 100755 index 0000000000..f01417f980 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchArtifactIngestModule.java @@ -0,0 +1,63 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021-2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import java.util.logging.Level; +import org.openide.util.Lookup; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.ingest.DataArtifactIngestModule; +import org.sleuthkit.autopsy.ingest.IngestJobContext; +import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.DataArtifact; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * A data artifact ingest module that indexes text for keyword search. All + * keyword searching of indexed text, from both data artifacts and files, + * including the final keyword search of an ingest job, is done in the last + * instance of the companion keyword search file ingest module. + */ +public class KeywordSearchArtifactIngestModule implements DataArtifactIngestModule { + + private static final Logger LOGGER = Logger.getLogger(KeywordSearchIngestModule.class.getName()); + private static final int TSK_ASSOCIATED_OBJECT_TYPE_ID = BlackboardArtifact.Type.TSK_ASSOCIATED_OBJECT.getTypeID(); + private IngestJobContext context; + private KeywordSearchService searchService; + + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + this.context = context; + searchService = Lookup.getDefault().lookup(KeywordSearchService.class); + } + + @Override + public ProcessResult process(DataArtifact artifact) { + try { + if (artifact.getType().getTypeID() != TSK_ASSOCIATED_OBJECT_TYPE_ID) { + searchService.index(artifact); + } + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, String.format("Error indexing data artifact '%s' (job ID=%d)", artifact, context.getJobId()), ex); //NON-NLS + return ProcessResult.ERROR; + } + return ProcessResult.OK; + } + +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchModuleFactory.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchModuleFactory.java index 17c760323a..8f63977945 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchModuleFactory.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchModuleFactory.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,6 +25,7 @@ import java.util.List; import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; import org.sleuthkit.autopsy.coreutils.Version; +import org.sleuthkit.autopsy.ingest.DataArtifactIngestModule; import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestModuleFactory; @@ -121,4 +122,15 @@ public class KeywordSearchModuleFactory extends IngestModuleFactoryAdapter { } return new KeywordSearchIngestModule((KeywordSearchJobSettings) settings); } + + @Override + public boolean isDataArtifactIngestModuleFactory() { + return true; + } + + @Override + public DataArtifactIngestModule createDataArtifactIngestModule(IngestModuleIngestJobSettings settings) { + return new KeywordSearchArtifactIngestModule(); + } + } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index e5c3f8ae03..02a71d6aa1 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -411,28 +411,6 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { } } - /** - * Event handler for ArtifactsPostedEvents from SleuthkitCase. - * - * @param event The ArtifactsPostedEvent to handle. - */ - @NbBundle.Messages("SolrSearchService.indexingError=Unable to index blackboard artifact.") - @Subscribe - void handleNewArtifacts(Blackboard.ArtifactsPostedEvent event) { - for (BlackboardArtifact artifact : event.getArtifacts()) { - if ((artifact.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID()) && // don't index KWH bc it's based on existing indexed text - (artifact.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_ASSOCIATED_OBJECT.getTypeID())){ //don't index AO bc it has only an artifact ID - no useful text - try { - index(artifact); - } catch (TskCoreException ex) { - //TODO: is this the right error handling? - logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS - MessageNotifyUtil.Notify.error(Bundle.SolrSearchService_indexingError(), artifact.getDisplayName()); - } - } - } - } - /** * Adds an artifact to the keyword search text index as a concantenation of * all of its attributes. From 8691b9ce310b2e1d82dbe1e47a434246b4a92694 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 16 Nov 2021 13:21:48 -0500 Subject: [PATCH 02/19] 7529 KWS artifact ingest module --- .../keywordsearch/SolrSearchService.java | 33 ++++--------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 02a71d6aa1..5cb0d3ae20 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2015-2020 Basis Technology Corp. + * Copyright 2015-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +18,6 @@ */ package org.sleuthkit.autopsy.keywordsearch; -import com.google.common.eventbus.Subscribe; import java.io.File; import java.io.IOException; import java.io.Reader; @@ -36,14 +35,12 @@ import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.CaseMetadata; import org.sleuthkit.autopsy.coreutils.FileUtil; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException; import org.sleuthkit.autopsy.progress.ProgressIndicator; import org.sleuthkit.autopsy.textextractors.TextExtractor; import org.sleuthkit.autopsy.textextractors.TextExtractorFactory; -import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.TskCoreException; @@ -66,13 +63,10 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { /** * Indexes the given content for keyword search. * - * IMPORTANT: Currently, there are two correct uses for this code: - * - * 1) Indexing an artifact created during while either the file level ingest - * module pipeline or the first stage data source level ingest module - * pipeline of an ingest job is running. - * - * 2) Indexing a report. + * IMPORTANT: This indexes the given content, but does not execute a keyword + * search. For the text of the content to be searched, the indexing has to + * occur either in the context of an ingest job configured for keyword + * search, or in the context of an ad hoc keyword search. * * @param content The content to index. * @@ -80,19 +74,6 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { */ @Override public void index(Content content) throws TskCoreException { - /* - * TODO (JIRA-1099): The following code has some issues that need to be - * resolved. For artifacts, it is assumed that the posting of artifacts - * is only occuring during an ingest job with an enabled keyword search - * ingest module handling index commits; it also assumes that the - * artifacts are only posted by modules in the either the file level - * ingest pipeline or the first stage data source level ingest pipeline, - * so that the artifacts will be searched during a periodic or final - * keyword search. It also assumes that the only other type of Content - * for which this API will be called are Reports generated at a time - * when doing a commit is required and desirable, i.e., in a context - * other than an ingest job. - */ if (content == null) { return; } @@ -152,7 +133,7 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { if (host == null || host.isEmpty()) { throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.MissingHostname")); //NON-NLS } - try { + try { KeywordSearch.getServer().connectToSolrServer(host, Integer.toString(port)); } catch (SolrServerException ex) { logger.log(Level.SEVERE, "Unable to connect to Solr server. Host: " + host + ", port: " + port, ex); @@ -232,7 +213,7 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { logger.log(Level.WARNING, NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.exceptionMessage.noCurrentSolrCore")); throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, - "SolrSearchService.exceptionMessage.noCurrentSolrCore")); + "SolrSearchService.exceptionMessage.noCurrentSolrCore")); } // delete index(es) for this case From ff24a5587d97ccb66fb361481204b5d96930956d Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 17 Nov 2021 08:43:20 -0500 Subject: [PATCH 03/19] 7529 KWS artifact ingest module --- .../ingest/AnalysisResultIngestModule.java | 46 +++++++++++++++++++ .../ingest/DataArtifactIngestModule.java | 2 +- .../autopsy/ingest/IngestModuleFactory.java | 39 ++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestModule.java diff --git a/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestModule.java new file mode 100755 index 0000000000..6bb475b0ad --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestModule.java @@ -0,0 +1,46 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021-2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import org.sleuthkit.datamodel.AnalysisResult; + +/** + * Interface that must be implemented by all ingest modules that process + * analysis results. + */ +public interface AnalysisResultIngestModule extends IngestModule { + + /** + * Processes an analysis result. + * + * IMPORTANT: In addition to returning ProcessResult.OK or + * ProcessResult.ERROR, modules should log all errors using methods provided + * by the org.sleuthkit.autopsy.coreutils.Logger class. Log messages should + * include the name and object ID of the data being processed. If an + * exception has been caught by the module, the exception should be sent to + * the Logger along with the log message so that a stack trace will appear + * in the application log. + * + * @param result The analysis result to process. + * + * @return A result code indicating success or failure of the processing. + */ + ProcessResult process(AnalysisResult result); + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java index a37816d6be..f38a4a3ebf 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2021 Basis Technology Corp. + * Copyright 2021-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java index e473086d18..361f0a9626 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java @@ -271,4 +271,43 @@ public interface IngestModuleFactory { throw new UnsupportedOperationException(); } + /** + * Queries the factory to determine if it is capable of creating analysis + * result ingest modules. + * + * @return True or false. + */ + default boolean isAnalysisResultIngestModuleFactory() { + return false; + } + + /** + * Creates an analysis result ingest module instance. + *

+ * Autopsy will generally use the factory to several instances of each type + * of module for each ingest job it performs. Completing an ingest job + * entails processing a single data source (e.g., a disk image) and all of + * the files from the data source, including files extracted from archives + * and any unallocated space (made to look like a series of files). The data + * source is passed through one or more pipelines of data source ingest + * modules. The files are passed through one or more pipelines of file + * ingest modules. + *

+ * The ingest framework may use multiple threads to complete an ingest job, + * but it is guaranteed that there will be no more than one module instance + * per thread. However, if the module instances must share resources, the + * modules are responsible for synchronizing access to the shared resources + * and doing reference counting as required to release those resources + * correctly. Also, more than one ingest job may be in progress at any given + * time. This must also be taken into consideration when sharing resources + * between module instances. modules. + * + * @param settings The settings for the ingest job. + * + * @return A file ingest module instance. + */ + default AnalysisResultIngestModule createAnalysisResultIngestModule(IngestModuleIngestJobSettings settings) { + throw new UnsupportedOperationException(); + } + } From a0b91a914da49673c4e431a82ead6229e0d459e2 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 18 Nov 2021 11:58:15 -0500 Subject: [PATCH 04/19] 7529 KWS artifact ingest module --- .../ingest/DataArtifactIngestPipeline.java | 2 +- .../ingest/DataArtifactIngestTask.java | 2 +- .../autopsy/ingest/IngestJobExecutor.java | 53 +++++++++++++++++-- .../autopsy/ingest/IngestModuleTemplate.java | 8 +++ .../autopsy/ingest/IngestTasksScheduler.java | 14 ++++- .../KeywordSearchModuleFactory.java | 2 +- ....java => KwsDataArtifactIngestModule.java} | 8 +-- 7 files changed, 78 insertions(+), 11 deletions(-) rename KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/{KeywordSearchArtifactIngestModule.java => KwsDataArtifactIngestModule.java} (87%) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java index 865b29cb59..0b2cd293a7 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2021 Basis Technology Corp. + * Copyright 2021-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java index 2a43146ca5..dbee5ded46 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2021 Basis Technology Corp. + * Copyright 2021-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 7a93b15b22..76588e7c5c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -112,6 +112,11 @@ final class IngestJobExecutor { */ private DataArtifactIngestPipeline artifactIngestPipeline; + /* + * There is at most one analysis result ingest module pipeline. + */ + private AnalysisResultIngestPipeline resultIngestPipeline; + /* * The construction, start up, execution, and shut down of the ingest module * pipelines for an ingest job is done in stages. @@ -364,6 +369,7 @@ final class IngestJobExecutor { List secondStageDataSourcePipelineTemplate = createIngestPipelineTemplate(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageTwoDataSourceIngestPipelineConfig()); List filePipelineTemplate = createIngestPipelineTemplate(javaFileModuleTemplates, jythonFileModuleTemplates, pipelineConfig.getFileIngestPipelineConfig()); List artifactPipelineTemplate = new ArrayList<>(); + List resultsPipelineTemplate = new ArrayList<>(); /** * Add any ingest module templates remaining in the buckets to the @@ -376,6 +382,7 @@ final class IngestJobExecutor { addToIngestPipelineTemplate(firstStageDataSourcePipelineTemplate, javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates); addToIngestPipelineTemplate(filePipelineTemplate, javaFileModuleTemplates, jythonFileModuleTemplates); addToIngestPipelineTemplate(artifactPipelineTemplate, javaArtifactModuleTemplates, jythonArtifactModuleTemplates); + addToIngestPipelineTemplate(resultsPipelineTemplate, javaArtifactModuleTemplates, jythonArtifactModuleTemplates); /** * Construct the ingest module pipelines from the ingest module pipeline @@ -390,6 +397,7 @@ final class IngestJobExecutor { fileIngestPipelines.add(pipeline); } artifactIngestPipeline = new DataArtifactIngestPipeline(this, artifactPipelineTemplate); + resultIngestPipeline = new AnalysisResultIngestPipeline(this, resultsPipelineTemplate); } /** @@ -533,6 +541,16 @@ final class IngestJobExecutor { return (artifactIngestPipeline.isEmpty() == false); } + /** + * Checks to see if there is at least one analysis result ingest module to + * run. + * + * @return True or false. + */ + boolean hasAnalysisResultIngestModules() { + return (resultIngestPipeline.isEmpty() == false); + } + /** * Determnines which inges job stage to start in and starts up the ingest * module pipelines. @@ -583,6 +601,7 @@ final class IngestJobExecutor { } } errors.addAll(startUpIngestModulePipeline(artifactIngestPipeline)); + errors.addAll(startUpIngestModulePipeline(resultIngestPipeline)); return errors; } @@ -1002,6 +1021,7 @@ final class IngestJobExecutor { shutDownIngestModulePipeline(currentDataSourceIngestPipeline); shutDownIngestModulePipeline(artifactIngestPipeline); + shutDownIngestModulePipeline(resultIngestPipeline); if (usingNetBeansGUI) { synchronized (dataSourceIngestProgressLock) { @@ -1070,7 +1090,8 @@ final class IngestJobExecutor { * Passes the data source for the ingest job through the currently active * data source level ingest module pipeline (high-priority or low-priority). * - * @param task A data source ingest task wrapping the data source. + * @param task A data source ingest task encapsulating the data source and + * the data source ingest pipeline to use to execute the task. */ void execute(DataSourceIngestTask task) { try { @@ -1091,7 +1112,8 @@ final class IngestJobExecutor { * Passes a file from the data source for the ingest job through a file * ingest module pipeline. * - * @param task A file ingest task wrapping the file. + * @param task A file ingest task encapsulating the file and the file ingest + * pipeline to use to execute the task. */ void execute(FileIngestTask task) { try { @@ -1165,7 +1187,9 @@ final class IngestJobExecutor { * Passes a data artifact from the data source for the ingest job through * the data artifact ingest module pipeline. * - * @param task A data artifact ingest task wrapping the data artifact. + * @param task A data artifact ingest task encapsulating the data artifact + * and the data artifact ingest pipeline to use to execute the + * task. */ void execute(DataArtifactIngestTask task) { try { @@ -1182,6 +1206,29 @@ final class IngestJobExecutor { } } + /** + * Passes an analyisis result from the data source for the ingest job + * through the analysis result ingest module pipeline. + * + * @param task An analysis result ingest task encapsulating the analysis + * result and the analysis result ingest pipeline to use to + * execute the task. + */ + void execute(AnalysisResultIngestTask task) { + try { + if (!isCancelled() && !resultIngestPipeline.isEmpty()) { + List errors = new ArrayList<>(); + errors.addAll(resultIngestPipeline.performTask(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + } + } finally { + taskScheduler.notifyTaskCompleted(task); + checkForStageCompleted(); + } + } + /** * Adds some streamed files for analysis as part of a streaming mode ingest * job. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java index 26285f6439..4aadb62929 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java @@ -93,6 +93,14 @@ public final class IngestModuleTemplate { return moduleFactory.createDataArtifactIngestModule(settings); } + public boolean isAnalysisResultIngestModuleTemplate() { + return moduleFactory.isAnalysisResultIngestModuleFactory(); + } + + public AnalysisResultIngestModule createAnalysisResultIngestModule() { + return moduleFactory.createAnalysisResultIngestModule(settings); + } + public void setEnabled(boolean enabled) { this.enabled = enabled; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 63d45ae3e0..c2ff2c9ce3 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -67,6 +67,7 @@ final class IngestTasksScheduler { private final Queue streamedFileIngestTasksQueue; private final IngestTaskTrackingQueue fileIngestTasksQueue; private final IngestTaskTrackingQueue artifactIngestTasksQueue; + private final IngestTaskTrackingQueue resultIngestTasksQueue; /** * Gets the ingest tasks scheduler singleton that creates ingest tasks for @@ -92,6 +93,7 @@ final class IngestTasksScheduler { fileIngestTasksQueue = new IngestTaskTrackingQueue(); streamedFileIngestTasksQueue = new LinkedList<>(); artifactIngestTasksQueue = new IngestTaskTrackingQueue(); + resultIngestTasksQueue = new IngestTaskTrackingQueue(); } /** @@ -365,6 +367,16 @@ final class IngestTasksScheduler { artifactIngestTasksQueue.taskCompleted(task); } + /** + * Allows an ingest thread to notify this ingest task scheduler that a data + * artifact ingest task has been completed. + * + * @param task The completed task. + */ + synchronized void notifyTaskCompleted(AnalysisResultIngestTask task) { + resultIngestTasksQueue.taskCompleted(task); + } + /** * Queries the task scheduler to determine whether or not all of the ingest * tasks for an ingest job have been completed. @@ -1137,7 +1149,7 @@ final class IngestTasksScheduler { long getRunningListSize() { return runningListSize; } - + long getArtifactsQueueSize() { return artifactsQueueSize; } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchModuleFactory.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchModuleFactory.java index 8f63977945..db45655f31 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchModuleFactory.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchModuleFactory.java @@ -130,7 +130,7 @@ public class KeywordSearchModuleFactory extends IngestModuleFactoryAdapter { @Override public DataArtifactIngestModule createDataArtifactIngestModule(IngestModuleIngestJobSettings settings) { - return new KeywordSearchArtifactIngestModule(); + return new KwsDataArtifactIngestModule(); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchArtifactIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KwsDataArtifactIngestModule.java similarity index 87% rename from KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchArtifactIngestModule.java rename to KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KwsDataArtifactIngestModule.java index f01417f980..fe4cac8b4f 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchArtifactIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KwsDataArtifactIngestModule.java @@ -30,11 +30,11 @@ import org.sleuthkit.datamodel.TskCoreException; /** * A data artifact ingest module that indexes text for keyword search. All - * keyword searching of indexed text, from both data artifacts and files, - * including the final keyword search of an ingest job, is done in the last - * instance of the companion keyword search file ingest module. + * keyword searching of indexed text, whether from files, data artifacts, or + * analysis results, including the final keyword search of an ingest job, is + * done in the last instance of the companion keyword search file ingest module. */ -public class KeywordSearchArtifactIngestModule implements DataArtifactIngestModule { +public class KwsDataArtifactIngestModule implements DataArtifactIngestModule { private static final Logger LOGGER = Logger.getLogger(KeywordSearchIngestModule.class.getName()); private static final int TSK_ASSOCIATED_OBJECT_TYPE_ID = BlackboardArtifact.Type.TSK_ASSOCIATED_OBJECT.getTypeID(); From 08e5ca61ac59dbf23c367ccb55fc4803a3ffa3e2 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 19 Nov 2021 13:42:56 -0500 Subject: [PATCH 05/19] 7529 KWS artifact ingest module --- .../ingest/AnalysisResultIngestPipeline.java | 90 ++++ .../ingest/AnalysisResultIngestTask.java | 59 +++ .../sleuthkit/autopsy/ingest/IngestJob.java | 11 + .../autopsy/ingest/IngestJobExecutor.java | 111 +++- .../autopsy/ingest/IngestManager.java | 15 +- .../autopsy/ingest/IngestTasksScheduler.java | 482 +++++++++++------- .../sleuthkit/autopsy/ingest/Snapshot.java | 12 +- .../KwsAnalysisResultIngestModule.java | 64 +++ 8 files changed, 637 insertions(+), 207 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestPipeline.java create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestTask.java create mode 100755 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KwsAnalysisResultIngestModule.java diff --git a/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestPipeline.java new file mode 100755 index 0000000000..f5d1519cdc --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestPipeline.java @@ -0,0 +1,90 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021-2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.List; +import java.util.Optional; +import org.sleuthkit.datamodel.AnalysisResult; + +/** + * A pipeline of analysis result ingest modules used to perform analysis result + * ingest tasks for an ingest job. + */ +public class AnalysisResultIngestPipeline extends IngestPipeline { + + /** + * Constructs a pipeline of analysis result ingest modules used to perform + * analysis result ingest tasks for an ingest job. + * + * @param ingestJobExecutor The ingest job executor for this pipeline. + * @param moduleTemplates The ingest module templates to be used to + * construct the ingest modules for this pipeline. + * May be an empty list if this type of pipeline is + * not needed for the ingest job. + */ + AnalysisResultIngestPipeline(IngestJobExecutor ingestJobExecutor, List moduleTemplates) { + super(ingestJobExecutor, moduleTemplates); + } + + @Override + Optional> acceptModuleTemplate(IngestModuleTemplate template) { + Optional> module = Optional.empty(); + if (template.isAnalysisResultIngestModuleTemplate()) { + AnalysisResultIngestModule ingestModule = template.createAnalysisResultIngestModule(); + module = Optional.of(new AnalysisResultIngestPipelineModule(ingestModule, template.getModuleName())); + } + return module; + } + + @Override + void prepareForTask(AnalysisResultIngestTask task) throws IngestPipelineException { + } + + @Override + void cleanUpAfterTask(AnalysisResultIngestTask task) throws IngestPipelineException { + } + + /** + * A decorator that adds ingest infrastructure operations to a data artifact + * ingest module. + */ + static final class AnalysisResultIngestPipelineModule extends IngestPipeline.PipelineModule { + + private final AnalysisResultIngestModule module; + + /** + * Constructs a decorator that adds ingest infrastructure operations to + * a data artifact ingest module. + * + * @param module The module. + * @param displayName The display name of the module. + */ + AnalysisResultIngestPipelineModule(AnalysisResultIngestModule module, String displayName) { + super(module, displayName); + this.module = module; + } + + @Override + void process(IngestJobExecutor ingestJobExecutor, AnalysisResultIngestTask task) throws IngestModule.IngestModuleException { + AnalysisResult result = task.getAnalysisResult(); + module.process(result); + } + + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestTask.java new file mode 100755 index 0000000000..2b1c750b75 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestTask.java @@ -0,0 +1,59 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021-2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import org.sleuthkit.datamodel.AnalysisResult; + +/** + * An analysis result ingest task that will be executed by an ingest thread + * using a given ingest job executor. + */ +final class AnalysisResultIngestTask extends IngestTask { + + private final AnalysisResult analysisResult; + + /** + * Constructs analysis result ingest task that will be executed by an ingest + * thread using a given ingest job executor. + * + * @param ingestJobExecutor The ingest job executor to use to execute the + * task. + * @param analysisResult The analysis result to be processed. + */ + AnalysisResultIngestTask(IngestJobExecutor ingestJobExecutor, AnalysisResult analysisResult) { + super(ingestJobExecutor); + this.analysisResult = analysisResult; + } + + /** + * Gets the analysis result for this task. + * + * @return The analysis result. + */ + AnalysisResult getAnalysisResult() { + return analysisResult; + } + + @Override + void execute(long threadId) { + super.setThreadId(threadId); + getIngestJobExecutor().execute(this); + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 37e4b549ee..c991e8f1f2 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -27,6 +27,7 @@ import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.AnalysisResult; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataArtifact; @@ -166,6 +167,16 @@ public final class IngestJob { ingestModuleExecutor.addDataArtifacts(dataArtifacts); } + /** + * Adds one or more analysis results to this ingest job for processing by + * its analysis result ingest modules. + * + * @param results The analysis results. + */ + void addAnalysisResults(List results) { + ingestModuleExecutor.addAnalysisResults(results); + } + /** * Starts data source level analysis for this job if it is running in * streaming ingest mode. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 76588e7c5c..2e91e66977 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -41,7 +41,7 @@ import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.NetworkUtils; -import org.sleuthkit.autopsy.ingest.IngestTasksScheduler.IngestJobTasksSnapshot; +import org.sleuthkit.autopsy.ingest.IngestTasksScheduler.IngestTasksSnapshot; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.IngestJobInfo; @@ -52,12 +52,14 @@ import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.modules.interestingitems.FilesSet; import org.sleuthkit.autopsy.python.FactoryClassNameNormalizer; +import org.sleuthkit.datamodel.AnalysisResult; import org.sleuthkit.datamodel.DataArtifact; import org.sleuthkit.datamodel.DataSource; /** - * Manages the construction, start up, execution, and shut down of the ingest - * module pipelines for an ingest job. + * Executes an ingest job by orchestrating the construction, start up, ingest + * task execution, and shut down of the ingest module pipelines for an ingest + * job. */ final class IngestJobExecutor { @@ -211,6 +213,8 @@ final class IngestJobExecutor { private ProgressHandle fileIngestProgressBar; private final Object artifactIngestProgressLock = new Object(); private ProgressHandle artifactIngestProgressBar; + private final Object resultIngestProgressLock = new Object(); + private ProgressHandle resultIngestProgressBar; /* * The ingest job details that are stored to the case database are tracked @@ -230,8 +234,10 @@ final class IngestJobExecutor { private final Set pausedIngestThreads = new HashSet<>(); /** - * Constructs an object that manages the construction, start up, execution, - * and shut down of the ingest module pipelines for an ingest job. + * Constructs an object that executes an ingest job by orchestrating the + * construction, start up, ingest task execution, and shut down of the + * ingest module pipelines for an ingest job. + * * * @param ingestJob The ingest job. * @param dataSource The data source. @@ -718,6 +724,9 @@ final class IngestJobExecutor { if (hasDataArtifactIngestModules()) { startArtifactIngestProgressBar(); } + if (hasAnalysisResultIngestModules()) { + startResultIngestProgressBar(); + } } /* @@ -773,6 +782,9 @@ final class IngestJobExecutor { if (hasDataArtifactIngestModules()) { startArtifactIngestProgressBar(); } + if (hasAnalysisResultIngestModules()) { + startResultIngestProgressBar(); + } } if (hasDataArtifactIngestModules()) { @@ -783,6 +795,7 @@ final class IngestJobExecutor { * artifacts added to the case database by those tasks twice. */ taskScheduler.scheduleDataArtifactIngestTasks(this); + taskScheduler.scheduleAnalysisResultIngestTasks(this); } } } @@ -871,17 +884,51 @@ final class IngestJobExecutor { */ private void startArtifactIngestProgressBar() { if (usingNetBeansGUI) { - synchronized (artifactIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", this.dataSource.getName()); - artifactIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { + String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", dataSource.getName()); + startArtifactIngestProgressBar(artifactIngestProgressLock, artifactIngestProgressBar, displayName); + } + } + + /** + * Starts a data artifacts analysis NetBeans progress bar in the lower right + * hand corner of the main application window. The progress bar provides the + * user with a task cancellation button. Pressing it cancels the ingest job. + * Analysis already completed at the time that cancellation occurs is NOT + * discarded. + */ + @NbBundle.Messages({ + "# {0} - data source name", "IngestJob.progress.analysisResultIngest.displayName=Analyzing data artifacts from {0}" + }) + private void startResultIngestProgressBar() { + if (usingNetBeansGUI) { + String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", dataSource.getName()); + startArtifactIngestProgressBar(resultIngestProgressLock, resultIngestProgressBar, displayName); + } + } + + /** + * Starts a data artifacts or analysis results analysis NetBeans progress + * bar in the lower right hand corner of the main application window. The + * progress bar provides the user with a task cancellation button. Pressing + * it cancels the ingest job. Analysis already completed at the time that + * cancellation occurs is NOT discarded. + * + * @param progressBarLock The lock for the progress bar. + * @param progressBar The progress bar. + * @param displayName The display name for the progress bar. + */ + private void startArtifactIngestProgressBar(Object progressBarLock, ProgressHandle progressBar, String displayName) { + if (usingNetBeansGUI) { + synchronized (progressBarLock) { + progressBar = ProgressHandle.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); return true; } }); - artifactIngestProgressBar.start(); - artifactIngestProgressBar.switchToIndeterminate(); + progressBar.start(); + progressBar.switchToIndeterminate(); } } } @@ -1044,6 +1091,13 @@ final class IngestJobExecutor { artifactIngestProgressBar = null; } } + + synchronized (resultIngestProgressLock) { + if (resultIngestProgressBar != null) { + resultIngestProgressBar.finish(); + resultIngestProgressBar = null; + } + } } if (ingestJobInfo != null) { @@ -1280,15 +1334,40 @@ final class IngestJobExecutor { || stage.equals(IngestJobStage.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { taskScheduler.scheduleDataArtifactIngestTasks(this, artifactsToAnalyze); } else { - logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); + logErrorMessage(Level.SEVERE, "Adding data artifacts to job during stage " + stage.toString() + " not supported"); } /** * The intended clients of this method are ingest modules running code * in an ingest thread that is holding a reference to a "primary" ingest - * task that was the source of the files, in which case a completion - * check would not be necessary, so this is a bit of defensive - * programming. + * task that was the source of the data artifacts, in which case a + * completion check would not be necessary, so this is a bit of + * defensive programming. + */ + checkForStageCompleted(); + } + + /** + * Adds analysis results for analysis. + * + * @param results The analysis results. + */ + void addAnalysisResults(List results) { + List resultsToAnalyze = new ArrayList<>(results); + if (stage.equals(IngestJobStage.STREAMED_FILE_ANALYSIS_ONLY) + || stage.equals(IngestJobStage.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS) + || stage.equals(IngestJobStage.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { + taskScheduler.scheduleAnalysisResultIngestTasks(this, resultsToAnalyze); + } else { + logErrorMessage(Level.SEVERE, "Adding analysis results to job during stage " + stage.toString() + " not supported"); + } + + /** + * The intended clients of this method are ingest modules running code + * in an ingest thread that is holding a reference to a "primary" ingest + * task that was the source of the analysis results, in which case a + * completion check would not be necessary, so this is a bit of + * defensive programming. */ checkForStageCompleted(); } @@ -1459,7 +1538,7 @@ final class IngestJobExecutor { void cancel(IngestJob.CancellationReason reason) { jobCancelled = true; cancellationReason = reason; - IngestJobExecutor.taskScheduler.cancelPendingFileTasksForIngestJob(this); + IngestJobExecutor.taskScheduler.cancelPendingFileTasksForIngestJob(getIngestJobId()); if (usingNetBeansGUI) { synchronized (dataSourceIngestProgressLock) { @@ -1594,7 +1673,7 @@ final class IngestJobExecutor { long processedFilesCount = 0; long estimatedFilesToProcessCount = 0; long snapShotTime = new Date().getTime(); - IngestJobTasksSnapshot tasksSnapshot = null; + IngestTasksSnapshot tasksSnapshot = null; if (includeIngestTasksSnapshot) { synchronized (fileIngestProgressLock) { processedFilesCount = processedFiles; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 2c87487232..6267946a7c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -70,6 +70,7 @@ import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisCompletedEvent; import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisStartedEvent; import org.sleuthkit.autopsy.ingest.events.FileAnalyzedEvent; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.AnalysisResult; import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.Content; @@ -194,7 +195,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { resultIngestTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-results-ingest-%d").build()); //NON-NLS; threadId = nextIngestManagerTaskId.incrementAndGet(); - resultIngestTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getResultIngestTaskQueue())); + resultIngestTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getDataArtifactIngestTaskQueue())); // RJCTODO // ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); // RJCTODO: Where is the shut down code? @@ -301,13 +302,16 @@ public class IngestManager implements IngestProgressSnapshotProvider { * job for possible analysis. */ List newDataArtifacts = new ArrayList<>(); + List newAnalysisResults = new ArrayList<>(); Collection newArtifacts = tskEvent.getArtifacts(); for (BlackboardArtifact artifact : newArtifacts) { if (artifact instanceof DataArtifact) { newDataArtifacts.add((DataArtifact) artifact); + } else { + newAnalysisResults.add((AnalysisResult) artifact); } } - if (!newDataArtifacts.isEmpty()) { + if (!newDataArtifacts.isEmpty() || !newAnalysisResults.isEmpty()) { IngestJob ingestJob = null; Optional ingestJobId = tskEvent.getIngestJobId(); if (ingestJobId.isPresent()) { @@ -379,7 +383,12 @@ public class IngestManager implements IngestProgressSnapshotProvider { } } if (ingestJob != null) { - ingestJob.addDataArtifacts(newDataArtifacts); + if (!newDataArtifacts.isEmpty()) { + ingestJob.addDataArtifacts(newDataArtifacts); + } + if (!newAnalysisResults.isEmpty()) { + ingestJob.addAnalysisResults(newAnalysisResults); + } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index c2ff2c9ce3..29816d224e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -39,6 +39,7 @@ import javax.annotation.concurrent.ThreadSafe; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.AnalysisResult; import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataArtifact; @@ -122,56 +123,71 @@ final class IngestTasksScheduler { * * @return The queue. */ - BlockingIngestTaskQueue getResultIngestTaskQueue() { + BlockingIngestTaskQueue getDataArtifactIngestTaskQueue() { return artifactIngestTasksQueue; } /** - * Schedules ingest tasks based on the types of ingest modules that the - * ingest pipeline that will exedute tasks has. Scheduling these tasks - * atomically means that it is valid to call currentTasksAreCompleted() - * immediately after calling this method. Note that the may cause some or - * even all of any file tasks to be discarded. + * Gets the analysis result ingest tasks queue. This queue is a blocking + * queue consumed by the ingest manager's analysis result ingest thread. * - * @param ingestPipeline The ingest pipeline that will execute the scheduled - * tasks. A reference to the pipeline is added to each - * task so that when the task is dequeued by an ingest - * thread the task can pass the target Content of the - * task to the pipeline for processing by the - * pipeline's ingest modules. + * @return The queue. */ - synchronized void scheduleIngestTasks(IngestJobExecutor ingestPipeline) { - if (!ingestPipeline.isCancelled()) { - if (ingestPipeline.hasDataSourceIngestModules()) { - scheduleDataSourceIngestTask(ingestPipeline); + BlockingIngestTaskQueue getAnalysisResultIngestTaskQueue() { + return resultIngestTasksQueue; + } + + /** + * Schedules ingest tasks for an ingest job based on the types of ingest + * modules that will be used by the ingest job executor. + * + * Scheduling these tasks atomically means that it is valid to call + * currentTasksAreCompleted() immediately after calling this method. An + * example of where this is relevant would be an ingest job that only + * requires the processing of files, and that has an ingest filter in its + * settings that causes the ingest task scheduler to discard all of the file + * tasks. + * + * @param executor The ingest job executor that will execute the scheduled + * tasks. A reference to the executor is added to each task + * so that when the task is dequeued by an ingest thread, + * the task can pass its target item to the executor for + * processing by the executor's ingest module pipelines. + */ + synchronized void scheduleIngestTasks(IngestJobExecutor executor) { + if (!executor.isCancelled()) { + if (executor.hasDataSourceIngestModules()) { + scheduleDataSourceIngestTask(executor); } - if (ingestPipeline.hasFileIngestModules()) { - scheduleFileIngestTasks(ingestPipeline, Collections.emptyList()); + if (executor.hasFileIngestModules()) { + scheduleFileIngestTasks(executor, Collections.emptyList()); } - if (ingestPipeline.hasDataArtifactIngestModules()) { - scheduleDataArtifactIngestTasks(ingestPipeline); + if (executor.hasDataArtifactIngestModules()) { + scheduleDataArtifactIngestTasks(executor); + } + if (executor.hasAnalysisResultIngestModules()) { + scheduleAnalysisResultIngestTasks(executor); } } } /** * Schedules a data source level ingest task for an ingest job. The data - * source is obtained from the ingest pipeline passed in. + * source is obtained from the ingest ingest job executor passed in. * - * @param ingestPipeline The ingest pipeline that will execute the scheduled - * task. A reference to the pipeline is added to the - * task so that when the task is dequeued by an ingest - * thread the task can pass the target Content of the - * task to the pipeline for processing by the - * pipeline's ingest modules. + * @param executor The ingest job executor that will execute the scheduled + * tasks. A reference to the executor is added to each task + * so that when the task is dequeued by an ingest thread, + * the task can pass its target item to the executor for + * processing by the executor's ingest module pipelines. */ - synchronized void scheduleDataSourceIngestTask(IngestJobExecutor ingestPipeline) { - if (!ingestPipeline.isCancelled()) { - DataSourceIngestTask task = new DataSourceIngestTask(ingestPipeline); + synchronized void scheduleDataSourceIngestTask(IngestJobExecutor executor) { + if (!executor.isCancelled()) { + DataSourceIngestTask task = new DataSourceIngestTask(executor); try { dataSourceIngestTasksQueue.putLast(task); } catch (InterruptedException ex) { - IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while blocked adding a task to the data source level ingest task queue (pipelineId={%d)", ingestPipeline.getIngestJobId()), ex); + IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while blocked adding a task to the data source level ingest task queue (ingest job ID={%d)", executor.getIngestJobId()), ex); Thread.currentThread().interrupt(); } } @@ -180,28 +196,27 @@ final class IngestTasksScheduler { /** * Schedules file tasks for either all the files, or a given subset of the * files, for a data source. The data source is obtained from the ingest - * pipeline passed in. + * ingest job executor passed in. * - * @param ingestPipeline The ingest pipeline that will execute the scheduled - * tasks. A reference to the pipeline is added to each - * task so that when the task is dequeued by an ingest - * thread the task can pass the target Content of the - * task to the pipeline for processing by the - * pipeline's ingest modules. - * @param files A subset of the files from the data source; if - * empty, then all if the files from the data source - * are candidates for scheduling. + * @param executor The ingest job executor that will execute the scheduled + * tasks. A reference to the executor is added to each task + * so that when the task is dequeued by an ingest thread, + * the task can pass its target item to the executor for + * processing by the executor's ingest module pipelines. + * @param files A subset of the files from the data source; if empty, + * then all if the files from the data source are candidates + * for scheduling. */ - synchronized void scheduleFileIngestTasks(IngestJobExecutor ingestPipeline, Collection files) { - if (!ingestPipeline.isCancelled()) { + synchronized void scheduleFileIngestTasks(IngestJobExecutor executor, Collection files) { + if (!executor.isCancelled()) { Collection candidateFiles; if (files.isEmpty()) { - candidateFiles = getTopLevelFiles(ingestPipeline.getDataSource()); + candidateFiles = getTopLevelFiles(executor.getDataSource()); } else { candidateFiles = files; } for (AbstractFile file : candidateFiles) { - FileIngestTask task = new FileIngestTask(ingestPipeline, file); + FileIngestTask task = new FileIngestTask(executor, file); if (IngestTasksScheduler.shouldEnqueueFileTask(task)) { topLevelFileIngestTasksQueue.add(task); } @@ -214,16 +229,15 @@ final class IngestTasksScheduler { * Schedules file tasks for a collection of "streamed" files for a streaming * ingest job. * - * @param ingestPipeline The ingest pipeline for the job. A reference to the - * pipeline is added to each task so that when the - * task is dequeued by an ingest thread and the task's - * execute() method is called, execute() can pass the - * target Content of the task to the pipeline for - * processing by the pipeline's ingest modules. - * @param files A list of file object IDs for the streamed files. + * @param executor The ingest job executor that will execute the scheduled + * tasks. A reference to the executor is added to each task + * so that when the task is dequeued by an ingest thread, + * the task can pass its target item to the executor for + * processing by the executor's ingest module pipelines. + * @param files A list of file object IDs for the streamed files. */ - synchronized void scheduleStreamedFileIngestTasks(IngestJobExecutor ingestPipeline, List fileIds) { - if (!ingestPipeline.isCancelled()) { + synchronized void scheduleStreamedFileIngestTasks(IngestJobExecutor executor, List fileIds) { + if (!executor.isCancelled()) { for (long id : fileIds) { /* * Create the file ingest task. Note that we do not do the @@ -232,7 +246,7 @@ final class IngestTasksScheduler { * file filter will be applied before the file task makes it to * the task queue consumed by the file ingest threads. */ - FileIngestTask task = new FileIngestTask(ingestPipeline, id); + FileIngestTask task = new FileIngestTask(executor, id); streamedFileIngestTasksQueue.add(task); } refillFileIngestTasksQueue(); @@ -246,16 +260,15 @@ final class IngestTasksScheduler { * be used to schedule files that are products of ingest module processing, * e.g., extracted files and carved files. * - * @param ingestPipeline The ingest pipeline for the job. A reference to the - * pipeline is added to each task so that when the - * task is dequeued by an ingest thread and the task's - * execute() method is called, execute() can pass the - * target Content of the task to the pipeline for - * processing by the pipeline's ingest modules. - * @param files The files. + * @param executor The ingest job executor that will execute the scheduled + * tasks. A reference to the executor is added to each task + * so that when the task is dequeued by an ingest thread, + * the task can pass its target item to the executor for + * processing by the executor's ingest module pipelines. + * @param files The files. */ - synchronized void fastTrackFileIngestTasks(IngestJobExecutor ingestPipeline, Collection files) { - if (!ingestPipeline.isCancelled()) { + synchronized void fastTrackFileIngestTasks(IngestJobExecutor executor, Collection files) { + if (!executor.isCancelled()) { /* * Put the files directly into the queue for the file ingest * threads, if they pass the file filter for the job. The files are @@ -265,12 +278,12 @@ final class IngestTasksScheduler { * in progress. */ for (AbstractFile file : files) { - FileIngestTask fileTask = new FileIngestTask(ingestPipeline, file); + FileIngestTask fileTask = new FileIngestTask(executor, file); if (shouldEnqueueFileTask(fileTask)) { try { fileIngestTasksQueue.putFirst(fileTask); } catch (InterruptedException ex) { - DataSource dataSource = ingestPipeline.getDataSource(); + DataSource dataSource = executor.getDataSource(); logger.log(Level.WARNING, String.format("Interrupted while enqueuing file tasks for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS Thread.currentThread().interrupt(); return; @@ -283,51 +296,73 @@ final class IngestTasksScheduler { /** * Schedules data artifact ingest tasks for any data artifacts that have * already been added to the case database for a data source. The data - * source is obtained from the ingest pipeline passed in. + * source is obtained from the ingest job executor passed in. * - * @param ingestPipeline The ingest pipeline for the job. A reference to the - * pipeline is added to each task so that when the - * task is dequeued by an ingest thread and the task's - * execute() method is called, execute() can pass the - * target Content of the task to the pipeline for - * processing by the pipeline's ingest modules. + * @param executor The ingest job executor that will execute the scheduled + * tasks. A reference to the executor is added to each task + * so that when the task is dequeued by an ingest thread, + * the task can pass its target item to the executor for + * processing by the executor's ingest module pipelines. */ - synchronized void scheduleDataArtifactIngestTasks(IngestJobExecutor ingestPipeline) { - if (!ingestPipeline.isCancelled()) { + synchronized void scheduleDataArtifactIngestTasks(IngestJobExecutor executor) { + if (!executor.isCancelled()) { Blackboard blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard(); try { - List artifacts = blackboard.getDataArtifacts(ingestPipeline.getDataSource().getId(), null); - scheduleDataArtifactIngestTasks(ingestPipeline, artifacts); + List artifacts = blackboard.getDataArtifacts(executor.getDataSource().getId(), null); + scheduleDataArtifactIngestTasks(executor, artifacts); } catch (TskCoreException ex) { - DataSource dataSource = ingestPipeline.getDataSource(); + DataSource dataSource = executor.getDataSource(); logger.log(Level.SEVERE, String.format("Failed to retrieve data artifacts for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS } } } + /** + * Schedules analysis result ingest tasks for any analysis result that have + * already been added to the case database for a data source. The data + * source is obtained from the ingest job executor passed in. + * + * @param executor The ingest job executor that will execute the scheduled + * tasks. A reference to the executor is added to each task + * so that when the task is dequeued by an ingest thread, + * the task can pass its target item to the executor for + * processing by the executor's ingest module pipelines. + */ + synchronized void scheduleAnalysisResultIngestTasks(IngestJobExecutor executor) { + if (!executor.isCancelled()) { + Blackboard blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard(); + try { + List results = blackboard.getAnalysisResults(executor.getDataSource().getId(), null); + scheduleAnalysisResultIngestTasks(executor, results); + } catch (TskCoreException ex) { + DataSource dataSource = executor.getDataSource(); + logger.log(Level.SEVERE, String.format("Failed to retrieve analysis results for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS + } + } + } + /** * Schedules data artifact ingest tasks for an ingest job. This method is * intended to be used to schedule artifacts that are products of ingest * module processing. * - * @param ingestPipeline The ingest pipeline for the job. A reference to the - * pipeline is added to each task so that when the - * task is dequeued by an ingest thread and the task's - * execute() method is called, execute() can pass the - * target Content of the task to the pipeline for - * processing by the pipeline's ingest modules. - * @param artifacts A subset of the data artifacts from the data - * source; if empty, then all of the data artifacts - * from the data source will be scheduled. + * @param executor The ingest job executor that will execute the scheduled + * tasks. A reference to the executor is added to each task + * so that when the task is dequeued by an ingest thread, + * the task can pass its target item to the executor for + * processing by the executor's ingest module pipelines. + * @param artifacts A subset of the data artifacts from the data source; if + * empty, then all of the data artifacts from the data + * source will be scheduled. */ - synchronized void scheduleDataArtifactIngestTasks(IngestJobExecutor ingestPipeline, List artifacts) { - if (!ingestPipeline.isCancelled()) { + synchronized void scheduleDataArtifactIngestTasks(IngestJobExecutor executor, List artifacts) { + if (!executor.isCancelled()) { for (DataArtifact artifact : artifacts) { - DataArtifactIngestTask task = new DataArtifactIngestTask(ingestPipeline, artifact); + DataArtifactIngestTask task = new DataArtifactIngestTask(executor, artifact); try { - this.artifactIngestTasksQueue.putLast(task); + artifactIngestTasksQueue.putLast(task); } catch (InterruptedException ex) { - DataSource dataSource = ingestPipeline.getDataSource(); + DataSource dataSource = executor.getDataSource(); logger.log(Level.WARNING, String.format("Interrupted while enqueuing data artifact tasks for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS Thread.currentThread().interrupt(); break; @@ -336,6 +371,36 @@ final class IngestTasksScheduler { } } + /** + * Schedules data artifact ingest tasks for an ingest job. This method is + * intended to be used to schedule artifacts that are products of ingest + * module processing. + * + * @param executor The ingest job executor that will execute the scheduled + * tasks. A reference to the executor is added to each task + * so that when the task is dequeued by an ingest thread, + * the task can pass its target item to the executor for + * processing by the executor's ingest module pipelines. + * @param results A subset of the data artifacts from the data source; if + * empty, then all of the data artifacts from the data + * source will be scheduled. + */ + synchronized void scheduleAnalysisResultIngestTasks(IngestJobExecutor executor, List results) { + if (!executor.isCancelled()) { + for (AnalysisResult result : results) { + AnalysisResultIngestTask task = new AnalysisResultIngestTask(executor, result); + try { + resultIngestTasksQueue.putLast(task); + } catch (InterruptedException ex) { + DataSource dataSource = executor.getDataSource(); + logger.log(Level.WARNING, String.format("Interrupted while enqueuing analysis results tasks for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS + Thread.currentThread().interrupt(); + break; + } + } + } + } + /** * Allows an ingest thread to notify this ingest task scheduler that a data * source level task has been completed. @@ -381,18 +446,19 @@ final class IngestTasksScheduler { * Queries the task scheduler to determine whether or not all of the ingest * tasks for an ingest job have been completed. * - * @param ingestPipeline The ingest pipeline for the job. + * @param executor The ingest job executor. * * @return True or false. */ - synchronized boolean currentTasksAreCompleted(IngestJobExecutor ingestPipeline) { - long pipelineId = ingestPipeline.getIngestJobId(); - return !(dataSourceIngestTasksQueue.hasTasksForJob(pipelineId) - || hasTasksForJob(topLevelFileIngestTasksQueue, pipelineId) - || hasTasksForJob(batchedFileIngestTasksQueue, pipelineId) - || hasTasksForJob(streamedFileIngestTasksQueue, pipelineId) - || fileIngestTasksQueue.hasTasksForJob(pipelineId) - || artifactIngestTasksQueue.hasTasksForJob(pipelineId)); + synchronized boolean currentTasksAreCompleted(IngestJobExecutor executor) { + long ingestJobId = executor.getIngestJobId(); + return !(dataSourceIngestTasksQueue.hasTasksForJob(ingestJobId) + || hasTasksForJob(topLevelFileIngestTasksQueue, ingestJobId) + || hasTasksForJob(batchedFileIngestTasksQueue, ingestJobId) + || hasTasksForJob(streamedFileIngestTasksQueue, ingestJobId) + || fileIngestTasksQueue.hasTasksForJob(ingestJobId) + || artifactIngestTasksQueue.hasTasksForJob(ingestJobId) + || resultIngestTasksQueue.hasTasksForJob(ingestJobId)); } /** @@ -412,13 +478,12 @@ final class IngestTasksScheduler { * in the batch root file tasks queue and any directories in the batch root * children file tasks queue. * - * @param ingestJobPipeline The ingest pipeline for the job. + * @param ingestJobId The ingest job ID. */ - synchronized void cancelPendingFileTasksForIngestJob(IngestJobExecutor ingestJobPipeline) { - long jobId = ingestJobPipeline.getIngestJobId(); - removeTasksForJob(topLevelFileIngestTasksQueue, jobId); - removeTasksForJob(batchedFileIngestTasksQueue, jobId); - removeTasksForJob(streamedFileIngestTasksQueue, jobId); + synchronized void cancelPendingFileTasksForIngestJob(long ingestJobId) { + removeTasksForJob(topLevelFileIngestTasksQueue, ingestJobId); + removeTasksForJob(batchedFileIngestTasksQueue, ingestJobId); + removeTasksForJob(streamedFileIngestTasksQueue, ingestJobId); } /** @@ -707,14 +772,14 @@ final class IngestTasksScheduler { * Checks whether or not a collection of ingest tasks includes a task for a * given ingest job. * - * @param tasks The tasks. - * @param pipelineId The ID of the ingest pipeline for the job. + * @param tasks The tasks. + * @param ingestJobId The ingest job ID. * * @return True if there are no tasks for the job, false otherwise. */ - synchronized private static boolean hasTasksForJob(Collection tasks, long pipelineId) { + synchronized private static boolean hasTasksForJob(Collection tasks, long ingestJobId) { for (IngestTask task : tasks) { - if (task.getIngestJobExecutor().getIngestJobId() == pipelineId) { + if (task.getIngestJobExecutor().getIngestJobId() == ingestJobId) { return true; } } @@ -725,14 +790,14 @@ final class IngestTasksScheduler { * Removes all of the ingest tasks associated with an ingest job from a * collection of tasks. * - * @param tasks The tasks. - * @param pipelineId The ID of the ingest pipeline for the job. + * @param tasks The tasks. + * @param ingestJobId The ingest job ID. */ - private static void removeTasksForJob(Collection tasks, long pipelineId) { + private static void removeTasksForJob(Collection tasks, long ingestJobId) { Iterator iterator = tasks.iterator(); while (iterator.hasNext()) { IngestTask task = iterator.next(); - if (task.getIngestJobExecutor().getIngestJobId() == pipelineId) { + if (task.getIngestJobExecutor().getIngestJobId() == ingestJobId) { iterator.remove(); } } @@ -742,15 +807,15 @@ final class IngestTasksScheduler { * Counts the number of ingest tasks in a collection of tasks for a given * ingest job. * - * @param tasks The tasks. - * @param pipelineId The ID of the ingest pipeline for the job. + * @param tasks The tasks. + * @param ingestJobId The ingest job ID. * * @return The count. */ - private static int countTasksForJob(Collection tasks, long pipelineId) { + private static int countTasksForJob(Collection tasks, long ingestJobId) { int count = 0; for (IngestTask task : tasks) { - if (task.getIngestJobExecutor().getIngestJobId() == pipelineId) { + if (task.getIngestJobExecutor().getIngestJobId() == ingestJobId) { count++; } } @@ -761,18 +826,22 @@ final class IngestTasksScheduler { * Returns a snapshot of the states of the tasks in progress for an ingest * job. * - * @param jobId The identifier assigned to the job. + * @param ingestJobId The ingest job ID. * - * @return + * @return The snaphot. */ - synchronized IngestJobTasksSnapshot getTasksSnapshotForJob(long jobId) { - return new IngestJobTasksSnapshot(jobId, dataSourceIngestTasksQueue.countQueuedTasksForJob(jobId), - countTasksForJob(topLevelFileIngestTasksQueue, jobId), - countTasksForJob(batchedFileIngestTasksQueue, jobId), - fileIngestTasksQueue.countQueuedTasksForJob(jobId), - dataSourceIngestTasksQueue.countRunningTasksForJob(jobId) + fileIngestTasksQueue.countRunningTasksForJob(jobId) + artifactIngestTasksQueue.countRunningTasksForJob(jobId), - countTasksForJob(streamedFileIngestTasksQueue, jobId), - artifactIngestTasksQueue.countQueuedTasksForJob(jobId)); + synchronized IngestTasksSnapshot getTasksSnapshotForJob(long ingestJobId) { + return new IngestTasksSnapshot( + ingestJobId, + dataSourceIngestTasksQueue.countQueuedTasksForJob(ingestJobId), + countTasksForJob(topLevelFileIngestTasksQueue, ingestJobId), + countTasksForJob(batchedFileIngestTasksQueue, ingestJobId), + fileIngestTasksQueue.countQueuedTasksForJob(ingestJobId), + countTasksForJob(streamedFileIngestTasksQueue, ingestJobId), + artifactIngestTasksQueue.countQueuedTasksForJob(ingestJobId), + artifactIngestTasksQueue.countQueuedTasksForJob(ingestJobId), + dataSourceIngestTasksQueue.countRunningTasksForJob(ingestJobId) + fileIngestTasksQueue.countRunningTasksForJob(ingestJobId) + artifactIngestTasksQueue.countRunningTasksForJob(ingestJobId) + resultIngestTasksQueue.countRunningTasksForJob(ingestJobId) + ); } /** @@ -1025,98 +1094,107 @@ final class IngestTasksScheduler { * Checks whether there are any ingest tasks are queued and/or running * for a given ingest job. * - * @param pipelineId The ID of the ingest pipeline for the job. + * @param ingestJobId The ingest job ID. * - * @return + * @return True or false. */ - boolean hasTasksForJob(long pipelineId) { + boolean hasTasksForJob(long ingestJobId) { synchronized (this) { - return IngestTasksScheduler.hasTasksForJob(queuedTasks, pipelineId) || IngestTasksScheduler.hasTasksForJob(tasksInProgress, pipelineId); + return IngestTasksScheduler.hasTasksForJob(queuedTasks, ingestJobId) || IngestTasksScheduler.hasTasksForJob(tasksInProgress, ingestJobId); } } /** * Gets a count of the queued ingest tasks for a given ingest job. * - * @param pipelineId The ID of the ingest pipeline for the job. + * @param ingestJobId The ingest job ID. * - * @return + * @return The count. */ - int countQueuedTasksForJob(long pipelineId) { + int countQueuedTasksForJob(long ingestJobId) { synchronized (this) { - return IngestTasksScheduler.countTasksForJob(queuedTasks, pipelineId); + return IngestTasksScheduler.countTasksForJob(queuedTasks, ingestJobId); } } /** * Gets a count of the running ingest tasks for a given ingest job. * - * @param pipelineId The ID of the ingest pipeline for the job. + * @param ingestJobId The ingest job ID. * - * @return + * @return The count. */ - int countRunningTasksForJob(long pipelineId) { + int countRunningTasksForJob(long ingestJobId) { synchronized (this) { - return IngestTasksScheduler.countTasksForJob(tasksInProgress, pipelineId); + return IngestTasksScheduler.countTasksForJob(tasksInProgress, ingestJobId); } } } /** - * A snapshot of ingest tasks data for an ingest job. + * A snapshot of the sizes of the ingest task lists and queues for a given + * ingest job. */ - static final class IngestJobTasksSnapshot implements Serializable { + static final class IngestTasksSnapshot implements Serializable { private static final long serialVersionUID = 1L; - private final long jobId; - private final long dsQueueSize; + private final long ingestJobId; + private final long dataSourceQueueSize; private final long rootQueueSize; private final long dirQueueSize; private final long fileQueueSize; - private final long runningListSize; - private final long streamingQueueSize; + private final long inProgressListSize; + private final long streamedFileQueueSize; private final long artifactsQueueSize; + private final long resultsQueueSize; /** - * RJCTODO + * Constructs a snapshot of the sizes of the ingest task lists and + * queues for a given ingest job. * - * Constructs a snapshot of ingest tasks data for an ingest job. - * - * @param jobId The identifier associated with the job. - * @param dsQueueSize - * @param rootQueueSize - * @param dirQueueSize - * @param fileQueueSize - * @param runningListSize - * @param streamingQueueSize - * @param artifactsQueueSize + * @param ingestJobId The ingest job ID. + * @param dataSourceQueueSize The number of queued ingest tasks for + * data sources. + * @param rootQueueSize The number of queued ingest tasks for + * "root" file system objects. + * @param dirQueueSize The number of queued ingest tasks for + * directories. + * @param fileQueueSize The number of queued ingest tasks for + * files. + * @param inProgressListSize The number of ingest tasks in progress. + * @param streamedFileQueueSize The number of queued ingest tasks for + * streamed files. + * @param artifactsQueueSize The number of queued ingest tasks for + * data artifacts. + * @param resultsQueueSize The number of queued ingest tasks for + * analysis results. */ - IngestJobTasksSnapshot(long jobId, long dsQueueSize, long rootQueueSize, long dirQueueSize, long fileQueueSize, - long runningListSize, long streamingQueueSize, long artifactsQueueSize) { - this.jobId = jobId; - this.dsQueueSize = dsQueueSize; + IngestTasksSnapshot(long ingestJobId, long dataSourceQueueSize, long rootQueueSize, long dirQueueSize, long fileQueueSize, long inProgressListSize, long streamedFileQueueSize, long artifactsQueueSize, long resultsQueueSize) { + this.ingestJobId = ingestJobId; + this.dataSourceQueueSize = dataSourceQueueSize; this.rootQueueSize = rootQueueSize; this.dirQueueSize = dirQueueSize; this.fileQueueSize = fileQueueSize; - this.runningListSize = runningListSize; - this.streamingQueueSize = streamingQueueSize; + this.inProgressListSize = inProgressListSize; + this.streamedFileQueueSize = streamedFileQueueSize; this.artifactsQueueSize = artifactsQueueSize; + this.resultsQueueSize = resultsQueueSize; } /** - * Gets the identifier associated with the ingest job for which this - * snapshot was created. + * Gets the ingest job ID of the ingest job for which this snapshot was + * created. * - * @return The ingest job identifier. + * @return The ingest job ID. */ - long getJobId() { - return jobId; + long getIngestJobId() { + return ingestJobId; } /** - * Gets the number of file ingest tasks associated with the job that are - * in the root directories queue. + * Gets the number of file ingest tasks for this job that are in the + * file system root objects queue. * * @return The tasks count. */ @@ -1125,35 +1203,75 @@ final class IngestTasksScheduler { } /** - * Gets the number of file ingest tasks associated with the job that are - * in the root directories queue. + * Gets the number of file ingest tasks for this job that are in the + * ditrectories queue. * * @return The tasks count. */ - long getDirectoryTasksQueueSize() { + long getDirQueueSize() { return dirQueueSize; } + /** + * Gets the number of file ingest tasks for this job that are in the + * files queue. + * + * @return The tasks count. + */ long getFileQueueSize() { return fileQueueSize; } - long getStreamingQueueSize() { - return streamingQueueSize; + /** + * Gets the number of file ingest tasks for this job that are in the + * streamed files queue. + * + * @return The tasks count. + */ + long getStreamedFilesQueueSize() { + return streamedFileQueueSize; } - long getDsQueueSize() { - return dsQueueSize; - } - - long getRunningListSize() { - return runningListSize; + /** + * Gets the number of data source ingest tasks for this job that are in + * the data sources queue. + * + * @return The tasks count. + */ + long getDataSourceQueueSize() { + return dataSourceQueueSize; } + /** + * Gets the number of data artifact ingest tasks for this job that are + * in the data artifacts queue. + * + * @return The tasks count. + */ long getArtifactsQueueSize() { return artifactsQueueSize; } + /** + * Gets the number of analysis result ingest tasks for this job that are + * in the analysis results queue. + * + * @return The tasks count. + */ + long getResultsQueueSize() { + return resultsQueueSize; + } + + /** + * Gets the number of ingest tasks for this job that are in the tasks in + * progress list. + * + * @return The tasks count. + */ + long getProgressListSize() { + return inProgressListSize; + } + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java b/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java index d67e3f2441..9c2e6debcc 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java @@ -39,7 +39,7 @@ public final class Snapshot implements Serializable { private final Date fileIngestStartTime; private final long processedFiles; private final long estimatedFilesToProcess; - private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot; + private final IngestTasksScheduler.IngestTasksSnapshot tasksSnapshot; transient private final boolean jobCancelled; transient private final IngestJob.CancellationReason jobCancellationReason; transient private final List cancelledDataSourceModules; @@ -52,7 +52,7 @@ public final class Snapshot implements Serializable { boolean fileIngestRunning, Date fileIngestStartTime, boolean jobCancelled, IngestJob.CancellationReason cancellationReason, List cancelledModules, long processedFiles, long estimatedFilesToProcess, - long snapshotTime, IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot) { + long snapshotTime, IngestTasksScheduler.IngestTasksSnapshot tasksSnapshot) { this.dataSource = dataSourceName; this.jobId = jobId; this.jobStartTime = jobStartTime; @@ -162,7 +162,7 @@ public final class Snapshot implements Serializable { if (null == this.tasksSnapshot) { return 0; } - return this.tasksSnapshot.getDirectoryTasksQueueSize(); + return this.tasksSnapshot.getDirQueueSize(); } long getFileQueueSize() { @@ -176,21 +176,21 @@ public final class Snapshot implements Serializable { if (null == this.tasksSnapshot) { return 0; } - return this.tasksSnapshot.getDsQueueSize(); + return this.tasksSnapshot.getDataSourceQueueSize(); } long getStreamingQueueSize() { if (null == this.tasksSnapshot) { return 0; } - return this.tasksSnapshot.getStreamingQueueSize(); + return this.tasksSnapshot.getStreamedFilesQueueSize(); } long getRunningListSize() { if (null == this.tasksSnapshot) { return 0; } - return this.tasksSnapshot.getRunningListSize(); + return this.tasksSnapshot.getProgressListSize(); } long getArtifactTasksQueueSize() { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KwsAnalysisResultIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KwsAnalysisResultIngestModule.java new file mode 100755 index 0000000000..b82c4d91fe --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KwsAnalysisResultIngestModule.java @@ -0,0 +1,64 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021-2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import java.util.logging.Level; +import org.openide.util.Lookup; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.ingest.AnalysisResultIngestModule; +import org.sleuthkit.autopsy.ingest.IngestJobContext; +import org.sleuthkit.autopsy.ingest.IngestModule; +import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService; +import org.sleuthkit.datamodel.AnalysisResult; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * An analysis result ingest module that indexes text for keyword search. All + * keyword searching of indexed text, whether from files, data artifacts, or + * analysis results, including the final keyword search of an ingest job, is + * done in the last instance of the companion keyword search file ingest module. + */ +public class KwsAnalysisResultIngestModule implements AnalysisResultIngestModule { + + private static final Logger LOGGER = Logger.getLogger(KeywordSearchIngestModule.class.getName()); + private static final int TSK_KEYWORD_HIT_TYPE_ID = BlackboardArtifact.Type.TSK_KEYWORD_HIT.getTypeID(); + private IngestJobContext context; + private KeywordSearchService searchService; + + @Override + public void startUp(IngestJobContext context) throws IngestModule.IngestModuleException { + this.context = context; + searchService = Lookup.getDefault().lookup(KeywordSearchService.class); + } + + @Override + public IngestModule.ProcessResult process(AnalysisResult result) { + try { + if (result.getType().getTypeID() != TSK_KEYWORD_HIT_TYPE_ID) { + searchService.index(result); + } + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, String.format("Error indexing analysis result '%s' (job ID=%d)", result, context.getJobId()), ex); //NON-NLS + return IngestModule.ProcessResult.ERROR; + } + return IngestModule.ProcessResult.OK; + } + +} From 75463dc9f05c0928ca9661cb75b9770cde61e3d3 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Sat, 20 Nov 2021 06:23:48 -0500 Subject: [PATCH 06/19] 7529 KWS artifact ingest module --- .../autopsy/ingest/Bundle.properties | 1 + .../autopsy/ingest/Bundle.properties-MERGED | 3 + .../autopsy/ingest/IngestJobExecutor.java | 22 +++-- .../ingest/IngestProgressSnapshotPanel.java | 6 +- .../autopsy/ingest/IngestTasksScheduler.java | 97 ++++++++++++------- .../sleuthkit/autopsy/ingest/Snapshot.java | 7 ++ .../configuration/Bundle.properties-MERGED | 4 + .../netbeans/core/startup/Bundle.properties | 2 +- .../core/windows/view/ui/Bundle.properties | 2 +- 9 files changed, 98 insertions(+), 46 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties index cda1ec1503..4069a2c7f2 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties @@ -93,6 +93,7 @@ IngestJobTableModel.colName.rootQueued=Roots Queued IngestJobTableModel.colName.streamingQueued=Streamed Files Queued IngestJobTableModel.colName.dsQueued=DS Queued IngestJobTableModel.colName.artifactsQueued=Artifacts Queued +IngestJobTableModel.colName.resultsQueued=Results Queued ModuleTableModel.colName.module=Module ModuleTableModel.colName.duration=Duration IngestJobSettingsPanel.jButtonSelectAll.text=Select All diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED index 11fbd0a9d8..d208a9c363 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED @@ -1,5 +1,7 @@ CTL_RunIngestAction=Run Ingest FileIngestPipeline_SaveResults_Activity=Saving Results +# {0} - data source name +IngestJob.progress.analysisResultIngest.displayName=Analyzing data artifacts from {0} IngestJobSettingsPanel.IngestModulesTableRenderer.info.message=A previous version of this ingest module has been run before on this data source. IngestJobSettingsPanel.IngestModulesTableRenderer.warning.message=This ingest module has been run before on this data source. IngestJobSettingsPanel.noPerRunSettings=The selected module has no per-run settings. @@ -109,6 +111,7 @@ IngestJobTableModel.colName.rootQueued=Roots Queued IngestJobTableModel.colName.streamingQueued=Streamed Files Queued IngestJobTableModel.colName.dsQueued=DS Queued IngestJobTableModel.colName.artifactsQueued=Artifacts Queued +IngestJobTableModel.colName.resultsQueued=Results Queued ModuleTableModel.colName.module=Module ModuleTableModel.colName.duration=Duration IngestJobSettingsPanel.jButtonSelectAll.text=Select All diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 2e91e66977..9ed0aff23d 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -1006,7 +1006,7 @@ final class IngestJobExecutor { if (stage == IngestJobStage.STREAMED_FILE_ANALYSIS_ONLY) { return; } - if (taskScheduler.currentTasksAreCompleted(this)) { + if (taskScheduler.currentTasksAreCompleted(getIngestJobId())) { switch (stage) { case FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS: finishFileAndHighPriorityDataSrcAnalysis(); @@ -1307,7 +1307,7 @@ final class IngestJobExecutor { void addFiles(List files) { if (stage.equals(IngestJobStage.STREAMED_FILE_ANALYSIS_ONLY) || stage.equals(IngestJobStage.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { - taskScheduler.fastTrackFileIngestTasks(this, files); + taskScheduler.scheduleHighPriorityFileIngestTasks(this, files); } else { logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); } @@ -1683,12 +1683,20 @@ final class IngestJobExecutor { tasksSnapshot = taskScheduler.getTasksSnapshotForJob(getIngestJobId()); } - return new Snapshot(dataSource.getName(), - getIngestJobId(), createTime, + return new Snapshot( + dataSource.getName(), + getIngestJobId(), + createTime, getCurrentDataSourceIngestModule(), - fileIngestRunning, fileIngestStartTime, - jobCancelled, cancellationReason, cancelledDataSourceIngestModules, - processedFilesCount, estimatedFilesToProcessCount, snapShotTime, tasksSnapshot); + fileIngestRunning, + fileIngestStartTime, + jobCancelled, + cancellationReason, + cancelledDataSourceIngestModules, + processedFilesCount, + estimatedFilesToProcessCount, + snapShotTime, + tasksSnapshot); } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index b15f5723b5..317d90ff97 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -179,7 +179,8 @@ class IngestProgressSnapshotPanel extends javax.swing.JPanel { NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.rootQueued"), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.streamingQueued"), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.dsQueued"), - NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.artifactsQueued")}; + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.artifactsQueued"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.resultsQueued")}; private List jobSnapshots; @@ -249,6 +250,9 @@ class IngestProgressSnapshotPanel extends javax.swing.JPanel { case 11: cellValue = snapShot.getArtifactTasksQueueSize(); break; + case 12: + cellValue = snapShot.getResultTasksQueueSize(); + break; default: cellValue = null; break; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 29816d224e..8c1f9f3307 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -148,6 +148,9 @@ final class IngestTasksScheduler { * settings that causes the ingest task scheduler to discard all of the file * tasks. * + * RJCTODO: Return a count of scheduled tasks or even just a boolean; let + * the caller know if file filters, etc., caused no tasks to be scheduled. + * * @param executor The ingest job executor that will execute the scheduled * tasks. A reference to the executor is added to each task * so that when the task is dequeued by an ingest thread, @@ -164,7 +167,7 @@ final class IngestTasksScheduler { } if (executor.hasDataArtifactIngestModules()) { scheduleDataArtifactIngestTasks(executor); - } + } if (executor.hasAnalysisResultIngestModules()) { scheduleAnalysisResultIngestTasks(executor); } @@ -194,8 +197,8 @@ final class IngestTasksScheduler { } /** - * Schedules file tasks for either all the files, or a given subset of the - * files, for a data source. The data source is obtained from the ingest + * Schedules file tasks for either all of the files, or a given subset of + * the files, for a data source. The data source is obtained from the ingest * ingest job executor passed in. * * @param executor The ingest job executor that will execute the scheduled @@ -204,7 +207,7 @@ final class IngestTasksScheduler { * the task can pass its target item to the executor for * processing by the executor's ingest module pipelines. * @param files A subset of the files from the data source; if empty, - * then all if the files from the data source are candidates + * then all of the files from the data source are candidates * for scheduling. */ synchronized void scheduleFileIngestTasks(IngestJobExecutor executor, Collection files) { @@ -267,7 +270,7 @@ final class IngestTasksScheduler { * processing by the executor's ingest module pipelines. * @param files The files. */ - synchronized void fastTrackFileIngestTasks(IngestJobExecutor executor, Collection files) { + synchronized void scheduleHighPriorityFileIngestTasks(IngestJobExecutor executor, Collection files) { if (!executor.isCancelled()) { /* * Put the files directly into the queue for the file ingest @@ -450,8 +453,7 @@ final class IngestTasksScheduler { * * @return True or false. */ - synchronized boolean currentTasksAreCompleted(IngestJobExecutor executor) { - long ingestJobId = executor.getIngestJobId(); + synchronized boolean currentTasksAreCompleted(Long ingestJobId) { return !(dataSourceIngestTasksQueue.hasTasksForJob(ingestJobId) || hasTasksForJob(topLevelFileIngestTasksQueue, ingestJobId) || hasTasksForJob(batchedFileIngestTasksQueue, ingestJobId) @@ -498,7 +500,9 @@ final class IngestTasksScheduler { List topLevelFiles = new ArrayList<>(); Collection rootObjects = dataSource.accept(new GetRootDirectoryVisitor()); if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { - // The data source is itself a file to be processed. + /* + * The data source is itself a file to be processed. + */ topLevelFiles.add((AbstractFile) dataSource); } else { for (AbstractFile root : rootObjects) { @@ -506,12 +510,17 @@ final class IngestTasksScheduler { try { children = root.getChildren(); if (children.isEmpty()) { - // Add the root object itself, it could be an unallocated - // space file, or a child of a volume or an image. + /* + * Add the root object itself, it could be an + * unallocated space file, or a child of a volume or an + * image. + */ topLevelFiles.add(root); } else { - // The root object is a file system root directory, get - // the files within it. + /* + * The root object is a file system root directory, get + * the files within it. + */ for (Content child : children) { if (child instanceof AbstractFile) { topLevelFiles.add((AbstractFile) child); @@ -623,7 +632,8 @@ final class IngestTasksScheduler { AbstractFile file = null; try { file = nextTask.getFile(); - for (Content child : file.getChildren()) { + List children = file.getChildren(); + for (Content child : children) { if (child instanceof AbstractFile) { AbstractFile childFile = (AbstractFile) child; FileIngestTask childTask = new FileIngestTask(nextTask.getIngestJobExecutor(), childFile); @@ -663,8 +673,10 @@ final class IngestTasksScheduler { return false; } - // Skip the task if the file is actually the pseudo-file for the parent - // or current directory. + /* + * Skip the task if the file is actually the pseudo-file for the parent + * or current directory. + */ String fileName = file.getName(); if (fileName.equals(".") || fileName.equals("..")) { @@ -687,12 +699,16 @@ final class IngestTasksScheduler { return false; } - // Skip the task if the file is one of a select group of special, large - // NTFS or FAT file system files. + /* + * Skip the task if the file is one of a select group of special, large + * NTFS or FAT file system files. + */ if (file instanceof org.sleuthkit.datamodel.File) { final org.sleuthkit.datamodel.File f = (org.sleuthkit.datamodel.File) file; - // Get the type of the file system, if any, that owns the file. + /* + * Get the type of the file system, if any, that owns the file. + */ TskData.TSK_FS_TYPE_ENUM fsType = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_UNSUPP; try { FileSystem fs = f.getFileSystem(); @@ -703,12 +719,16 @@ final class IngestTasksScheduler { logger.log(Level.SEVERE, "Error querying file system for " + f, ex); //NON-NLS } - // If the file system is not NTFS or FAT, don't skip the file. + /* + * If the file system is not NTFS or FAT, don't skip the file. + */ if ((fsType.getValue() & FAT_NTFS_FLAGS) == 0) { return true; } - // Find out whether the file is in a root directory. + /* + * Find out whether the file is in a root directory. + */ boolean isInRootDir = false; try { AbstractFile parent = f.getParentDirectory(); @@ -721,9 +741,11 @@ final class IngestTasksScheduler { logger.log(Level.WARNING, "Error querying parent directory for" + f.getName(), ex); //NON-NLS } - // If the file is in the root directory of an NTFS or FAT file - // system, check its meta-address and check its name for the '$' - // character and a ':' character (not a default attribute). + /* + * If the file is in the root directory of an NTFS or FAT file + * system, check its meta-address and check its name for the '$' + * character and a ':' character (not a default attribute). + */ if (isInRootDir && f.getMetaAddr() < 32) { String name = f.getName(); if (name.length() > 0 && name.charAt(0) == '$' && name.contains(":")) { @@ -839,7 +861,7 @@ final class IngestTasksScheduler { fileIngestTasksQueue.countQueuedTasksForJob(ingestJobId), countTasksForJob(streamedFileIngestTasksQueue, ingestJobId), artifactIngestTasksQueue.countQueuedTasksForJob(ingestJobId), - artifactIngestTasksQueue.countQueuedTasksForJob(ingestJobId), + resultIngestTasksQueue.countQueuedTasksForJob(ingestJobId), dataSourceIngestTasksQueue.countRunningTasksForJob(ingestJobId) + fileIngestTasksQueue.countRunningTasksForJob(ingestJobId) + artifactIngestTasksQueue.countRunningTasksForJob(ingestJobId) + resultIngestTasksQueue.countRunningTasksForJob(ingestJobId) ); } @@ -852,20 +874,27 @@ final class IngestTasksScheduler { @Override public int compare(FileIngestTask q1, FileIngestTask q2) { - // In practice the case where one or both calls to getFile() fails - // should never occur since such tasks would not be added to the queue. + /* + * In practice the case where one or both calls to getFile() fails + * should never occur since such tasks would not be added to the + * queue. + */ AbstractFile file1 = null; AbstractFile file2 = null; try { file1 = q1.getFile(); } catch (TskCoreException ex) { - // Do nothing - the exception has been logged elsewhere + /* + * Do nothing - the exception has been logged elsewhere + */ } try { file2 = q2.getFile(); } catch (TskCoreException ex) { - // Do nothing - the exception has been logged elsewhere + /* + * Do nothing - the exception has been logged elsewhere + */ } if (file1 == null) { @@ -910,15 +939,11 @@ final class IngestTasksScheduler { static final List HIGH_PRI_PATHS = new ArrayList<>(); /* - * prioritize root directory folders based on the assumption that we + * Prioritize root directory folders based on the assumption that we * are looking for user content. Other types of investigations may * want different priorities. */ - static /* - * prioritize root directory folders based on the assumption that we - * are looking for user content. Other types of investigations may - * want different priorities. - */ { + static { // these files have no structure, so they go last //unalloc files are handled as virtual files in getPriority() //LAST_PRI_PATHS.schedule(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE)); @@ -1170,16 +1195,16 @@ final class IngestTasksScheduler { * @param resultsQueueSize The number of queued ingest tasks for * analysis results. */ - IngestTasksSnapshot(long ingestJobId, long dataSourceQueueSize, long rootQueueSize, long dirQueueSize, long fileQueueSize, long inProgressListSize, long streamedFileQueueSize, long artifactsQueueSize, long resultsQueueSize) { + IngestTasksSnapshot(long ingestJobId, long dataSourceQueueSize, long rootQueueSize, long dirQueueSize, long fileQueueSize, long streamedFileQueueSize, long artifactsQueueSize, long resultsQueueSize, long inProgressListSize) { this.ingestJobId = ingestJobId; this.dataSourceQueueSize = dataSourceQueueSize; this.rootQueueSize = rootQueueSize; this.dirQueueSize = dirQueueSize; this.fileQueueSize = fileQueueSize; - this.inProgressListSize = inProgressListSize; this.streamedFileQueueSize = streamedFileQueueSize; this.artifactsQueueSize = artifactsQueueSize; this.resultsQueueSize = resultsQueueSize; + this.inProgressListSize = inProgressListSize; } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java b/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java index 9c2e6debcc..5ab6fda8bf 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java @@ -200,6 +200,13 @@ public final class Snapshot implements Serializable { return tasksSnapshot.getArtifactsQueueSize(); } + long getResultTasksQueueSize() { + if (tasksSnapshot == null) { + return 0; + } + return tasksSnapshot.getResultsQueueSize(); + } + boolean isCancelled() { return this.jobCancelled; } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/Bundle.properties-MERGED b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/Bundle.properties-MERGED index 86fd175181..854c57bed1 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/Bundle.properties-MERGED +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/Bundle.properties-MERGED @@ -65,15 +65,19 @@ DayOfTheWeekRenderer_Tuesday_Label=Tuesday DayOfTheWeekRenderer_Wednesday_Label=Wednesday GeneralOptionsPanelController.moduleErr.msg=A module caused an error listening to GeneralOptionsPanelController updates. See log to determine which module. Some data could be incomplete. GeneralOptionsPanelController.moduleErr=Module Error +# {0} - errorMessage MultiUserTestTool.criticalError=Critical error running data source processor on test data source: {0} MultiUserTestTool.errorStartingIngestJob=Ingest manager error while starting ingest job +# {0} - cancellationReason MultiUserTestTool.ingestCancelled=Ingest cancelled due to {0} MultiUserTestTool.ingestSettingsError=Failed to analyze data source due to ingest settings errors MultiUserTestTool.noContent=Test data source failed to produce content +# {0} - serviceName MultiUserTestTool.serviceDown=Multi User service is down: {0} MultiUserTestTool.startupError=Failed to analyze data source due to ingest job startup error MultiUserTestTool.unableAddFileAsDataSource=Unable to add test file as data source to case MultiUserTestTool.unableCreatFile=Unable to create a file in case output directory +# {0} - serviceName MultiUserTestTool.unableToCheckService=Unable to check Multi User service state: {0} MultiUserTestTool.unableToCreateCase=Unable to create case MultiUserTestTool.unableToInitializeDatabase=Case database was not successfully initialized diff --git a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties index cd253dc3cb..b144f07890 100644 --- a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties +++ b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Thu, 30 Sep 2021 19:36:31 -0400 +#Fri, 19 Nov 2021 17:01:30 -0500 LBL_splash_window_title=Starting Autopsy SPLASH_HEIGHT=314 SPLASH_WIDTH=538 diff --git a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties index 2d02262803..d3404a4493 100644 --- a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties +++ b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties @@ -1,4 +1,4 @@ #Updated by build script -#Thu, 30 Sep 2021 19:36:31 -0400 +#Fri, 19 Nov 2021 17:01:30 -0500 CTL_MainWindow_Title=Autopsy 4.19.2 CTL_MainWindow_Title_No_Project=Autopsy 4.19.2 From 82fe22856dd7f3f8e960c196706599bcbc7cb11e Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 22 Nov 2021 15:46:30 -0500 Subject: [PATCH 07/19] 7529 KWS artifact ingest module --- .../autopsy/ingest/Bundle.properties-MERGED | 2 +- .../autopsy/ingest/IngestJobExecutor.java | 127 +++++++++++------- .../autopsy/ingest/IngestManager.java | 23 ++-- .../KeywordSearchModuleFactory.java | 14 +- 4 files changed, 100 insertions(+), 66 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED index d208a9c363..9ceb1bae41 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED @@ -1,7 +1,7 @@ CTL_RunIngestAction=Run Ingest FileIngestPipeline_SaveResults_Activity=Saving Results # {0} - data source name -IngestJob.progress.analysisResultIngest.displayName=Analyzing data artifacts from {0} +IngestJob.progress.analysisResultIngest.displayName=Analyzing analysis results from {0} IngestJobSettingsPanel.IngestModulesTableRenderer.info.message=A previous version of this ingest module has been run before on this data source. IngestJobSettingsPanel.IngestModulesTableRenderer.warning.message=This ingest module has been run before on this data source. IngestJobSettingsPanel.noPerRunSettings=The selected module has no per-run settings. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 9ed0aff23d..2cfcf2572f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -57,9 +57,8 @@ import org.sleuthkit.datamodel.DataArtifact; import org.sleuthkit.datamodel.DataSource; /** - * Executes an ingest job by orchestrating the construction, start up, ingest - * task execution, and shut down of the ingest module pipelines for an ingest - * job. + * Executes an ingest job by orchestrating the construction, start up, running, + * and shut down of the ingest module pipelines for an ingest job. */ final class IngestJobExecutor { @@ -75,9 +74,9 @@ final class IngestJobExecutor { /* * These fields are the identity of this object: the parent ingest job, the - * user's ingest job settings, and the data source to be analyzed by the - * ingest module pipelines. Optionally, there is a set of files to be - * analyzed instead of analyzing all of the files in the data source. + * ingest job settings, and the data source to be analyzed by the ingest + * module pipelines. Optionally, there is a set of files to be analyzed, + * instead of analyzing all of the files in the data source. */ private final IngestJob ingestJob; private final IngestJobSettings settings; @@ -99,12 +98,12 @@ final class IngestJobExecutor { * the number of file ingest threads in the ingest manager. References to * the file ingest pipelines are put into two collections, each with its own * purpose. A blocking queue allows file ingest threads to take and return - * file ingest pipelines as they work through the file ingest tasks for one + * file ingest pipelines, as they work through the file ingest tasks for one * or more ingest jobs. Having the same number of pipelines as threads - * ensures that a file ingest thread will never be idle as long as there are - * file ingest tasks still to do, regardless of the number of ingest jobs in - * progress. Additionally, a fixed list is used to cycle through the file - * ingest module pipelines to make ingest progress snapshots. + * ensures that a file ingest thread will never be idle, as long as there + * are file ingest tasks still to do, regardless of the number of ingest + * jobs in progress. Additionally, a fixed list is used to cycle through the + * file ingest module pipelines when making ingest progress snapshots. */ private final LinkedBlockingQueue fileIngestPipelinesQueue = new LinkedBlockingQueue<>(); private final List fileIngestPipelines = new ArrayList<>(); @@ -120,7 +119,7 @@ final class IngestJobExecutor { private AnalysisResultIngestPipeline resultIngestPipeline; /* - * The construction, start up, execution, and shut down of the ingest module + * The construction, start up, running, and shut down of the ingest module * pipelines for an ingest job is done in stages. */ private static enum IngestJobStage { @@ -134,29 +133,33 @@ final class IngestJobExecutor { * This stage is unique to a streaming mode ingest job. In this stage, * file ingest module pipelines are analyzing files streamed to them via * addStreamedFiles(). If the ingest job is configured to have a data - * artifact ingest pipeline, that pipeline is also analyzing any data - * artifacts generated by the file ingest modules. This stage ends when + * artifact and/or analysis result ingest pipeline, those pipelines are + * also analyzing any data artifacts and/or analysis results generated + * by the file ingest modules. This stage ends when * addStreamedDataSource() is called. */ STREAMED_FILE_ANALYSIS_ONLY, /* * In this stage, file ingest module pipelines and/or a pipeline of * higher-priority data source level ingest modules are running. If the - * ingest job is configured to have a data artifact ingest pipeline, - * that pipeline is also analyzing any data artifacts generated by the - * file and/or data source level ingest modules. + * ingest job is configured to have a data artifact and/or analysis + * result ingest pipeline, those pipelines are also analyzing any data + * artifacts and/or analysis results generated by the file and/or data + * source level ingest modules. */ FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS, /** * In this stage, a pipeline of lower-priority, usually long-running * data source level ingest ingest modules is running. If the ingest job - * is configured to have a data artifact ingest pipeline, that pipeline - * is also analyzing any data artifacts generated by the data source - * level ingest modules. + * is configured to have a data artifact and/or analysis result ingest + * pipeline, those pipelines are also analyzing any data artifacts + * and/or analysis results generated by the data source level ingest + * modules. */ LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS, /** - * In this stage, The pipeline is shutting down its ingest modules. + * In this stage, the ingest job executor is shutting down its ingest + * modules. */ PIPELINES_SHUT_DOWN }; @@ -170,11 +173,12 @@ final class IngestJobExecutor { private final Object stageTransitionLock = new Object(); /* - * During each stage of the ingest job, this object interacts with the - * ingest task scheduler to create ingest tasks for analyzing the data - * source, files and data artifacts that are the subject of the ingest job. - * The scheduler queues the tasks for the ingest manager's ingest threads. - * The ingest tasks are the units of work for the ingest module pipelines. + * During each stage of an ingest job, the ingest job executor interacts + * with the ingest task scheduler to create ingest tasks for analyzing the + * data source, files, data artifacts, and analysis results that are the + * subject of the ingest job. The scheduler queues the ingest tasks for the + * ingest manager's ingest threads. The ingest tasks are the units of work + * for the ingest module pipelines. */ private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); @@ -205,15 +209,22 @@ final class IngestJobExecutor { */ private final boolean usingNetBeansGUI; private final Object dataSourceIngestProgressLock = new Object(); + @GuardedBy("dataSourceIngestProgressLock") private ProgressHandle dataSourceIngestProgressBar; private final Object fileIngestProgressLock = new Object(); + @GuardedBy("fileIngestProgressLock") private final List filesInProgress = new ArrayList<>(); + @GuardedBy("fileIngestProgressLock") private long estimatedFilesToProcess; + @GuardedBy("fileIngestProgressLock") private long processedFiles; + @GuardedBy("fileIngestProgressLock") private ProgressHandle fileIngestProgressBar; private final Object artifactIngestProgressLock = new Object(); + @GuardedBy("artifactIngestProgressLock") private ProgressHandle artifactIngestProgressBar; private final Object resultIngestProgressLock = new Object(); + @GuardedBy("resultIngestProgressLock") private ProgressHandle resultIngestProgressBar; /* @@ -363,6 +374,9 @@ final class IngestJobExecutor { if (template.isDataArtifactIngestModuleTemplate()) { addModuleTemplateToSortingMap(javaArtifactModuleTemplates, jythonArtifactModuleTemplates, template); } + if (template.isAnalysisResultIngestModuleTemplate()) { + addModuleTemplateToSortingMap(javaArtifactModuleTemplates, jythonArtifactModuleTemplates, template); + } } /** @@ -784,7 +798,7 @@ final class IngestJobExecutor { } if (hasAnalysisResultIngestModules()) { startResultIngestProgressBar(); - } + } } if (hasDataArtifactIngestModules()) { @@ -884,8 +898,7 @@ final class IngestJobExecutor { */ private void startArtifactIngestProgressBar() { if (usingNetBeansGUI) { - String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", dataSource.getName()); - startArtifactIngestProgressBar(artifactIngestProgressLock, artifactIngestProgressBar, displayName); + artifactIngestProgressBar = startArtifactIngestProgressBar(artifactIngestProgressLock, NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", dataSource.getName())); } } @@ -897,12 +910,11 @@ final class IngestJobExecutor { * discarded. */ @NbBundle.Messages({ - "# {0} - data source name", "IngestJob.progress.analysisResultIngest.displayName=Analyzing data artifacts from {0}" + "# {0} - data source name", "IngestJob.progress.analysisResultIngest.displayName=Analyzing analysis results from {0}" }) private void startResultIngestProgressBar() { if (usingNetBeansGUI) { - String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", dataSource.getName()); - startArtifactIngestProgressBar(resultIngestProgressLock, resultIngestProgressBar, displayName); + resultIngestProgressBar = startArtifactIngestProgressBar(resultIngestProgressLock, Bundle.IngestJob_progress_analysisResultIngest_displayName(dataSource.getName())); } } @@ -914,10 +926,12 @@ final class IngestJobExecutor { * cancellation occurs is NOT discarded. * * @param progressBarLock The lock for the progress bar. - * @param progressBar The progress bar. * @param displayName The display name for the progress bar. + * + * @return The progress bar. */ - private void startArtifactIngestProgressBar(Object progressBarLock, ProgressHandle progressBar, String displayName) { + private ProgressHandle startArtifactIngestProgressBar(Object progressBarLock, String displayName) { + ProgressHandle progressBar = null; if (usingNetBeansGUI) { synchronized (progressBarLock) { progressBar = ProgressHandle.createHandle(displayName, new Cancellable() { @@ -931,6 +945,7 @@ final class IngestJobExecutor { progressBar.switchToIndeterminate(); } } + return progressBar; } /** @@ -1325,14 +1340,13 @@ final class IngestJobExecutor { /** * Adds data artifacts for analysis. * - * @param artifacts + * @param artifacts The data artifacts. */ void addDataArtifacts(List artifacts) { - List artifactsToAnalyze = new ArrayList<>(artifacts); if (stage.equals(IngestJobStage.STREAMED_FILE_ANALYSIS_ONLY) || stage.equals(IngestJobStage.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS) || stage.equals(IngestJobStage.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { - taskScheduler.scheduleDataArtifactIngestTasks(this, artifactsToAnalyze); + taskScheduler.scheduleDataArtifactIngestTasks(this, artifacts); } else { logErrorMessage(Level.SEVERE, "Adding data artifacts to job during stage " + stage.toString() + " not supported"); } @@ -1353,11 +1367,10 @@ final class IngestJobExecutor { * @param results The analysis results. */ void addAnalysisResults(List results) { - List resultsToAnalyze = new ArrayList<>(results); if (stage.equals(IngestJobStage.STREAMED_FILE_ANALYSIS_ONLY) || stage.equals(IngestJobStage.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS) || stage.equals(IngestJobStage.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { - taskScheduler.scheduleAnalysisResultIngestTasks(this, resultsToAnalyze); + taskScheduler.scheduleAnalysisResultIngestTasks(this, results); } else { logErrorMessage(Level.SEVERE, "Adding analysis results to job during stage " + stage.toString() + " not supported"); } @@ -1548,10 +1561,24 @@ final class IngestJobExecutor { } } - synchronized (this.fileIngestProgressLock) { - if (null != this.fileIngestProgressBar) { - this.fileIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.fileIngest.displayName", dataSource.getName())); - this.fileIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); + synchronized (fileIngestProgressLock) { + if (null != fileIngestProgressBar) { + fileIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.fileIngest.displayName", dataSource.getName())); + fileIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); + } + } + + synchronized (artifactIngestProgressLock) { + if (null != artifactIngestProgressBar) { + artifactIngestProgressBar.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", dataSource.getName())); + artifactIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); + } + } + + synchronized (resultIngestProgressLock) { + if (null != resultIngestProgressBar) { + resultIngestProgressBar.setDisplayName(Bundle.IngestJob_progress_analysisResultIngest_displayName(dataSource.getName())); + resultIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); } } } @@ -1685,17 +1712,17 @@ final class IngestJobExecutor { return new Snapshot( dataSource.getName(), - getIngestJobId(), + getIngestJobId(), createTime, getCurrentDataSourceIngestModule(), - fileIngestRunning, + fileIngestRunning, fileIngestStartTime, - jobCancelled, - cancellationReason, + jobCancelled, + cancellationReason, cancelledDataSourceIngestModules, - processedFilesCount, - estimatedFilesToProcessCount, - snapShotTime, + processedFilesCount, + estimatedFilesToProcessCount, + snapShotTime, tasksSnapshot); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 6267946a7c..993f10ea0b 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -137,7 +137,8 @@ public class IngestManager implements IngestProgressSnapshotProvider { private final Map ingestJobsById = new HashMap<>(); private final ExecutorService dataSourceLevelIngestJobTasksExecutor; private final ExecutorService fileLevelIngestJobTasksExecutor; - private final ExecutorService resultIngestTasksExecutor; + private final ExecutorService dataArtifactIngestTasksExecutor; + private final ExecutorService analysisResultIngestTasksExecutor; private final ExecutorService eventPublishingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-ingest-events-%d").build()); //NON-NLS; private final IngestMonitor ingestMonitor = new IngestMonitor(); private final ServicesMonitor servicesMonitor = ServicesMonitor.getInstance(); @@ -170,21 +171,11 @@ public class IngestManager implements IngestProgressSnapshotProvider { * the processing of data sources by ingest modules. */ private IngestManager() { - /* - * Submit a single Runnable ingest manager task for processing data - * source level ingest job tasks to the data source level ingest job - * tasks executor. - */ dataSourceLevelIngestJobTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-data-source-ingest-%d").build()); //NON-NLS; long threadId = nextIngestManagerTaskId.incrementAndGet(); dataSourceLevelIngestJobTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue())); ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); - /* - * Submit a configurable number of Runnable ingest manager tasks for - * processing file level ingest job tasks to the file level ingest job - * tasks executor. - */ numberOfFileIngestThreads = UserPreferences.numberOfFileIngestThreads(); fileLevelIngestJobTasksExecutor = Executors.newFixedThreadPool(numberOfFileIngestThreads, new ThreadFactoryBuilder().setNameFormat("IM-file-ingest-%d").build()); //NON-NLS for (int i = 0; i < numberOfFileIngestThreads; ++i) { @@ -193,9 +184,15 @@ public class IngestManager implements IngestProgressSnapshotProvider { ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); } - resultIngestTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-results-ingest-%d").build()); //NON-NLS; + dataArtifactIngestTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-data-artifact-ingest-%d").build()); //NON-NLS; threadId = nextIngestManagerTaskId.incrementAndGet(); - resultIngestTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getDataArtifactIngestTaskQueue())); + dataArtifactIngestTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getDataArtifactIngestTaskQueue())); + + analysisResultIngestTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-analysis-result-ingest-%d").build()); //NON-NLS; + threadId = nextIngestManagerTaskId.incrementAndGet(); + analysisResultIngestTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getAnalysisResultIngestTaskQueue())); + + // RJCTODO // ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); // RJCTODO: Where is the shut down code? diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchModuleFactory.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchModuleFactory.java index db45655f31..c6b2c39f88 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchModuleFactory.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchModuleFactory.java @@ -25,8 +25,8 @@ import java.util.List; import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; import org.sleuthkit.autopsy.coreutils.Version; +import org.sleuthkit.autopsy.ingest.AnalysisResultIngestModule; import org.sleuthkit.autopsy.ingest.DataArtifactIngestModule; -import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestModuleFactory; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; @@ -38,7 +38,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; * searching. */ @ServiceProvider(service = IngestModuleFactory.class) -public class KeywordSearchModuleFactory extends IngestModuleFactoryAdapter { +public class KeywordSearchModuleFactory implements IngestModuleFactory { private static final HashSet defaultDisabledKeywordListNames = new HashSet<>(Arrays.asList("Phone Numbers", "IP Addresses", "URLs", "Credit Card Numbers")); //NON-NLS private KeywordSearchJobSettingsPanel jobSettingsPanel = null; @@ -133,4 +133,14 @@ public class KeywordSearchModuleFactory extends IngestModuleFactoryAdapter { return new KwsDataArtifactIngestModule(); } + @Override + public boolean isAnalysisResultIngestModuleFactory() { + return true; + } + + @Override + public AnalysisResultIngestModule createAnalysisResultIngestModule(IngestModuleIngestJobSettings settings) { + return new KwsAnalysisResultIngestModule(); + } + } From b033df0b15315ba5abc7abcf26b132cf24fef025 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 22 Nov 2021 15:48:10 -0500 Subject: [PATCH 08/19] 7529 KWS artifact ingest module --- Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 993f10ea0b..642327278e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -191,7 +191,6 @@ public class IngestManager implements IngestProgressSnapshotProvider { analysisResultIngestTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-analysis-result-ingest-%d").build()); //NON-NLS; threadId = nextIngestManagerTaskId.incrementAndGet(); analysisResultIngestTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getAnalysisResultIngestTaskQueue())); - // RJCTODO // ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); From 60d3ffb7342573750d20afeba3e44dd5a114600b Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 23 Nov 2021 14:05:45 -0500 Subject: [PATCH 09/19] 7529 KWS artifact ingest module --- .../ingest/AnalysisResultIngestPipeline.java | 2 + .../ingest/AnalysisResultIngestTask.java | 6 +- .../ingest/DataArtifactIngestPipeline.java | 2 + .../ingest/DataArtifactIngestTask.java | 4 +- .../autopsy/ingest/DataSourceIngestTask.java | 4 +- .../autopsy/ingest/FileIngestTask.java | 9 +- .../autopsy/ingest/IngestJobExecutor.java | 24 +-- .../autopsy/ingest/IngestManager.java | 149 +++++++----------- .../ingest/IngestProgressSnapshotPanel.java | 2 +- .../sleuthkit/autopsy/ingest/IngestTask.java | 33 +++- 10 files changed, 120 insertions(+), 115 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestPipeline.java index f5d1519cdc..14b7a76cae 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestPipeline.java @@ -58,6 +58,7 @@ public class AnalysisResultIngestPipeline extends IngestPipeline Date: Tue, 23 Nov 2021 15:23:59 -0500 Subject: [PATCH 10/19] 7529 KWS artifact ingest module --- .../ingest/AnalysisResultIngestPipeline.java | 7 ++++--- .../autopsy/ingest/IngestJobExecutor.java | 19 ++++++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestPipeline.java index 14b7a76cae..92497922ba 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestPipeline.java @@ -62,8 +62,8 @@ public class AnalysisResultIngestPipeline extends IngestPipeline { @@ -71,7 +71,7 @@ public class AnalysisResultIngestPipeline extends IngestPipeline jythonFileModuleTemplates = new LinkedHashMap<>(); Map javaArtifactModuleTemplates = new LinkedHashMap<>(); Map jythonArtifactModuleTemplates = new LinkedHashMap<>(); + Map javaResultModuleTemplates = new LinkedHashMap<>(); + Map jythonResultModuleTemplates = new LinkedHashMap<>(); for (IngestModuleTemplate template : enabledTemplates) { if (template.isDataSourceIngestModuleTemplate()) { addModuleTemplateToSortingMap(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, template); @@ -375,14 +377,16 @@ final class IngestJobExecutor { addModuleTemplateToSortingMap(javaArtifactModuleTemplates, jythonArtifactModuleTemplates, template); } if (template.isAnalysisResultIngestModuleTemplate()) { - addModuleTemplateToSortingMap(javaArtifactModuleTemplates, jythonArtifactModuleTemplates, template); + addModuleTemplateToSortingMap(javaResultModuleTemplates, jythonResultModuleTemplates, template); } } /** * Take the module templates that have pipeline configuration entries * out of the buckets and add them to ingest module pipeline templates - * in the order prescribed by the pipeline configuration. + * in the order prescribed by the pipeline configuration. There is + * currently no pipeline configuration file support for data artifact or + * analysis result ingest module pipelines. */ IngestPipelinesConfiguration pipelineConfig = IngestPipelinesConfiguration.getInstance(); List firstStageDataSourcePipelineTemplate = createIngestPipelineTemplate(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageOneDataSourceIngestPipelineConfig()); @@ -396,13 +400,13 @@ final class IngestJobExecutor { * appropriate ingest module pipeline templates. Data source level * ingest modules templates that were not listed in the pipeline * configuration are added to the first stage data source pipeline - * template, Java modules are added before Jython modules and Core + * template, Java modules are added before Jython modules, and Core * Autopsy modules are added before third party modules. */ addToIngestPipelineTemplate(firstStageDataSourcePipelineTemplate, javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates); addToIngestPipelineTemplate(filePipelineTemplate, javaFileModuleTemplates, jythonFileModuleTemplates); addToIngestPipelineTemplate(artifactPipelineTemplate, javaArtifactModuleTemplates, jythonArtifactModuleTemplates); - addToIngestPipelineTemplate(resultsPipelineTemplate, javaArtifactModuleTemplates, jythonArtifactModuleTemplates); + addToIngestPipelineTemplate(resultsPipelineTemplate, javaResultModuleTemplates, jythonResultModuleTemplates); /** * Construct the ingest module pipelines from the ingest module pipeline @@ -502,7 +506,8 @@ final class IngestJobExecutor { return hasFileIngestModules() || hasHighPriorityDataSourceIngestModules() || hasLowPriorityDataSourceIngestModules() - || hasDataArtifactIngestModules(); + || hasDataArtifactIngestModules() + || hasAnalysisResultIngestModules(); } /** @@ -581,7 +586,7 @@ final class IngestJobExecutor { List errors = startUpIngestModulePipelines(); if (errors.isEmpty()) { recordIngestJobStartUpInfo(); - if (hasHighPriorityDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { + if (hasHighPriorityDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules() || hasAnalysisResultIngestModules()) { if (ingestJob.getIngestMode() == IngestJob.Mode.STREAMING) { startStreamingModeAnalysis(); } else { @@ -801,7 +806,7 @@ final class IngestJobExecutor { } } - if (hasDataArtifactIngestModules()) { + if (hasDataArtifactIngestModules()) { // RJCTODO /* * Schedule artifact ingest tasks for any artifacts currently in * the case database. This needs to be done before any files or From 8b483cf1b1763380175cd6c48f84e0f92a936ca7 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 23 Nov 2021 15:27:51 -0500 Subject: [PATCH 11/19] 7529 KWS artifact ingest module --- .../autopsy/ingest/AnalysisResultIngestTask.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestTask.java index 0cbba2a0b7..99f3412c24 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/AnalysisResultIngestTask.java @@ -29,8 +29,8 @@ final class AnalysisResultIngestTask extends IngestTask { private final AnalysisResult analysisResult; /** - * Constructs analysis result ingest task that will be executed by an ingest - * thread using a given ingest job executor. + * Constructs an analysis result ingest task that will be executed by an + * ingest thread using a given ingest job executor. * * @param ingestJobExecutor The ingest job executor to use to execute the * task. @@ -39,8 +39,8 @@ final class AnalysisResultIngestTask extends IngestTask { AnalysisResultIngestTask(IngestJobExecutor ingestJobExecutor, AnalysisResult analysisResult) { super(analysisResult.getName(), ingestJobExecutor); this.analysisResult = analysisResult; - } - + } + /** * Gets the analysis result for this task. * From e9fd2b4e61b92fb0d424a7889fa5648ca3c91bb4 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 24 Nov 2021 13:07:33 -0500 Subject: [PATCH 12/19] 7529 KWS artifact ingest module --- .../keywordsearch/IngestSearchRunner.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java index 383abbd3af..9d6c413199 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java @@ -248,7 +248,8 @@ final class IngestSearchRunner { } /** - * Task to perform periodic searches for each job (does a single index commit first) + * Task to perform periodic searches for each job (does a single index + * commit first) */ private final class PeriodicSearchTask implements Runnable { @@ -296,24 +297,23 @@ final class IngestSearchRunner { NbBundle.getMessage(this.getClass(), "SearchRunner.Searcher.done.err.msg"), ex.getMessage())); }// catch and ignore if we were cancelled - catch (java.util.concurrent.CancellationException ex) { + catch (java.util.concurrent.CancellationException ex) { } } } stopWatch.stop(); logger.log(Level.INFO, "All periodic searches cumulatively took {0} secs", stopWatch.getElapsedTimeSecs()); //NON-NLS - + // calculate "hold off" time recalculateUpdateIntervalTime(stopWatch.getElapsedTimeSecs()); // ELDEBUG - + // schedule next PeriodicSearchTask jobProcessingTaskFuture = jobProcessingExecutor.schedule(new PeriodicSearchTask(), currentUpdateIntervalMs, MILLISECONDS); - + // exit this thread return; } - - + private void recalculateUpdateIntervalTime(long lastSerchTimeSec) { // If periodic search takes more than 1/4 of the current periodic search interval, then double the search interval if (lastSerchTimeSec * 1000 < currentUpdateIntervalMs / 4) { @@ -321,7 +321,7 @@ final class IngestSearchRunner { } // double the search interval currentUpdateIntervalMs = currentUpdateIntervalMs * 2; - logger.log(Level.WARNING, "Last periodic search took {0} sec. Increasing search interval to {1} sec", new Object[]{lastSerchTimeSec, currentUpdateIntervalMs/1000}); + logger.log(Level.WARNING, "Last periodic search took {0} sec. Increasing search interval to {1} sec", new Object[]{lastSerchTimeSec, currentUpdateIntervalMs / 1000}); return; } } @@ -513,6 +513,7 @@ final class IngestSearchRunner { return null; } + logger.log(Level.INFO, String.format("Performing keyword query for search job %d: %s", job.getJobId(), keyword.getSearchTerm())); //NON-NLS final KeywordList keywordList = keywordToList.get(keyword); //new subProgress will be active after the initial query From ea7c961d1a0df228804861d28751f899f4529c14 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 2 Dec 2021 12:10:41 -0500 Subject: [PATCH 13/19] 7529 KWS artifact ingest module --- .../autopsy/ingest/IngestJobExecutor.java | 57 ++++++++----------- .../autopsy/keywordsearch/QueryResults.java | 4 -- 2 files changed, 25 insertions(+), 36 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 758d15be02..3a6c8aed07 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -1080,6 +1080,11 @@ final class IngestJobExecutor { if (artifactIngestProgressBar != null) { artifactIngestProgressBar.finish(); artifactIngestProgressBar = null; + } + + if (resultIngestProgressBar != null) { + resultIngestProgressBar.finish(); + resultIngestProgressBar = null; } }); } @@ -1320,7 +1325,12 @@ final class IngestJobExecutor { /** * Adds additional files produced by ingest modules (e.g., extracted or - * carved files) for analysis. + * carved files) for analysis. The intended clients of this method are + * ingest modules running code in an ingest thread that has not yet notified + * the ingest task scheduler that the the primary ingest task that is the + * source of the files is completed. This means that the new tasks will be + * scheduled BEFORE the primary task has been removed from the scheduler's + * running tasks list. * * @param files A list of the files to add. */ @@ -1331,19 +1341,15 @@ final class IngestJobExecutor { } else { logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); } - - /** - * The intended clients of this method are ingest modules running code - * in an ingest thread that is holding a reference to a "primary" ingest - * task that was the source of the files, in which case a completion - * check would not be necessary, so this check is a bit of defensive - * programming. - */ - checkForStageCompleted(); } /** - * Adds data artifacts for analysis. + * Adds data artifacts for analysis. The intended clients of this method are + * ingest modules running code in an ingest thread that has not yet notified + * the ingest task scheduler that the the primary ingest task that is the + * source of the data artifacts is completed. This means that the new tasks + * will be scheduled BEFORE the primary task has been removed from the + * scheduler's running tasks list. * * @param artifacts The data artifacts. */ @@ -1353,21 +1359,17 @@ final class IngestJobExecutor { || stage.equals(IngestJobStage.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { taskScheduler.scheduleDataArtifactIngestTasks(this, artifacts); } else { - logErrorMessage(Level.SEVERE, "Adding data artifacts to job during stage " + stage.toString() + " not supported"); + logErrorMessage(Level.SEVERE, "Attempt to add data artifacts to job during stage " + stage.toString() + " not supported"); } - - /** - * The intended clients of this method are ingest modules running code - * in an ingest thread that is holding a reference to a "primary" ingest - * task that was the source of the data artifacts, in which case a - * completion check would not be necessary, so this is a bit of - * defensive programming. - */ - checkForStageCompleted(); } /** - * Adds analysis results for analysis. + * Adds analysis results for analysis. The intended clients of this method + * are ingest modules running code in an ingest thread that has not yet + * notified the ingest task scheduler that the the primary ingest task that + * is the source of the analysis results is completed. This means that the + * new tasks will be scheduled BEFORE the primary task has been removed from + * the scheduler's running tasks list. * * @param results The analysis results. */ @@ -1377,17 +1379,8 @@ final class IngestJobExecutor { || stage.equals(IngestJobStage.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { taskScheduler.scheduleAnalysisResultIngestTasks(this, results); } else { - logErrorMessage(Level.SEVERE, "Adding analysis results to job during stage " + stage.toString() + " not supported"); + logErrorMessage(Level.SEVERE, "Attempt to add analysis results to job during stage " + stage.toString() + " not supported"); } - - /** - * The intended clients of this method are ingest modules running code - * in an ingest thread that is holding a reference to a "primary" ingest - * task that was the source of the analysis results, in which case a - * completion check would not be necessary, so this is a bit of - * defensive programming. - */ - checkForStageCompleted(); } /** diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index 313bad3e60..d79b76bcca 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -28,8 +28,6 @@ import java.util.logging.Level; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; import org.apache.commons.lang.StringUtils; -import org.netbeans.api.progress.ProgressHandle; -import org.netbeans.api.progress.aggregate.ProgressContributor; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -53,8 +51,6 @@ import org.sleuthkit.datamodel.TskCoreException; * about the search hits to the ingest inbox, and publishing an event to notify * subscribers of the blackboard posts. */ - - class QueryResults { private static final Logger logger = Logger.getLogger(QueryResults.class.getName()); From e79cea58ac15fd5f81ed942116419ffae7e0f002 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 7 Dec 2021 14:44:57 -0500 Subject: [PATCH 14/19] 7529 KWS artifact ingest module --- .../autopsy/ingest/IngestJobExecutor.java | 389 ++++++++--------- .../autopsy/ingest/IngestTasksScheduler.java | 38 -- .../AdHocSearchChildFactory.java | 2 +- .../keywordsearch/IngestSearchRunner.java | 393 +++++++++--------- .../autopsy/keywordsearch/QueryResults.java | 8 +- 5 files changed, 410 insertions(+), 420 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 3a6c8aed07..55a71da48e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.HashSet; import java.util.LinkedHashMap; @@ -27,6 +28,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -90,12 +92,7 @@ final class IngestJobExecutor { * There are separate pipelines for high-priority and low priority data * source level ingest modules. These pipelines are run sequentially, not * simultaneously. - */ - private DataSourceIngestPipeline highPriorityDataSourceIngestPipeline; - private DataSourceIngestPipeline lowPriorityDataSourceIngestPipeline; - private volatile DataSourceIngestPipeline currentDataSourceIngestPipeline; - - /* + * * There are one or more identical file ingest module pipelines, based on * the number of file ingest threads in the ingest manager. References to * the file ingest pipelines are put into two collections, each with its own @@ -106,80 +103,74 @@ final class IngestJobExecutor { * ingest tasks still to do, regardless of the number of ingest jobs in * progress. Additionally, a fixed list is used to cycle through the file * ingest module pipelines when making ingest progress snapshots. - */ - private final LinkedBlockingQueue fileIngestPipelinesQueue = new LinkedBlockingQueue<>(); - private final List fileIngestPipelines = new ArrayList<>(); - - /* + * * There is at most one data artifact ingest module pipeline. - */ - private DataArtifactIngestPipeline artifactIngestPipeline; - - /* + * * There is at most one analysis result ingest module pipeline. */ - private AnalysisResultIngestPipeline resultIngestPipeline; + private DataSourceIngestPipeline highPriorityDataSourceIngestPipeline; + private DataSourceIngestPipeline lowPriorityDataSourceIngestPipeline; + private volatile DataSourceIngestPipeline currentDataSourceIngestPipeline; + private final LinkedBlockingQueue fileIngestPipelinesQueue = new LinkedBlockingQueue<>(); + private final List fileIngestPipelines = new ArrayList<>(); + private DataArtifactIngestPipeline dataArtifactIngestPipeline; + private AnalysisResultIngestPipeline analysisResultIngestPipeline; /* - * The construction, start up, running, and shut down of the ingest module - * pipelines for an ingest job is done in stages. + * An ingest job transistion through several states during its execution. */ - private static enum IngestJobStage { + private static enum IngestJobState { /* - * In this stage, the ingest module pipelines for the ingest job are - * constructed per the ingest job settings. This stage ends when all of - * the ingest module pipelines for the ingest job are ready to run. + * In this once-per-job state, the ingest module pipelines for the + * ingest job are constructed per the ingest job settings. This state + * ends when all of the ingest module pipelines for the ingest job are + * ready to run. */ - PIPELINES_START_UP, + PIPELINES_STARTING_UP, /* - * This stage is unique to a streaming mode ingest job. In this stage, + * This state is unique to a streaming mode ingest job. In this state, * file ingest module pipelines are analyzing files streamed to them via * addStreamedFiles(). If the ingest job is configured to have a data * artifact and/or analysis result ingest pipeline, those pipelines are * also analyzing any data artifacts and/or analysis results generated - * by the file ingest modules. This stage ends when - * addStreamedDataSource() is called. + * by the file ingest modules. The transition out of this state occurs + * when addStreamedDataSource() is called. */ STREAMED_FILE_ANALYSIS_ONLY, /* - * In this stage, file ingest module pipelines and/or a pipeline of + * In this state, file ingest module pipelines and/or a pipeline of * higher-priority data source level ingest modules are running. If the * ingest job is configured to have a data artifact and/or analysis * result ingest pipeline, those pipelines are also analyzing any data * artifacts and/or analysis results generated by the file and/or data - * source level ingest modules. This stage ends when all of the ingest - * tasks for the stage are completed. + * source level ingest modules. The transition out of this state occurs + * when all of the currently scheduled ingest tasks are completed. */ FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS, /** - * In this stage, a pipeline of lower-priority, usually long-running + * In this state, a pipeline of lower-priority, usually long-running * data source level ingest ingest modules is running. If the ingest job * is configured to have a data artifact and/or analysis result ingest * pipeline, those pipelines are also analyzing any data artifacts * and/or analysis results generated by the data source level ingest - * modules. This stage ends when all of the ingest tasks for the stage - * are completed. + * modules. he transition out of this state occurs when all of the + * currently scheduled ingest tasks are completed. */ LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS, /** - * In this stage, the ingest job executor is shutting down its ingest - * modules. + * In this state, the ingest job executor is shutting down ingest + * modules pipelines, either during transitions between states in which + * analysis is performed, or at the end of the ingest job. */ - PIPELINES_SHUT_DOWN + PIPELINES_SHUTTING_DOWN }; - private volatile IngestJobStage stage = IngestJobExecutor.IngestJobStage.PIPELINES_START_UP; + private final ReentrantReadWriteLock jobStateLock = new ReentrantReadWriteLock(); + private volatile IngestJobState jobState = IngestJobExecutor.IngestJobState.PIPELINES_STARTING_UP; /* - * The stage transition lock is used to coordinate stage transitions, not to - * guard the volatile stage field. - */ - private final Object stageTransitionLock = new Object(); - - /* - * During each stage of the ingest job, the ingest job executor interacts - * with the ingest task scheduler to create ingest tasks for job. The - * scheduler queues the ingest tasks for the ingest manager's ingest - * threads. + * The ingest job executor interacts with the ingest task scheduler to + * create ingest tasks for job. The scheduler queues the ingest tasks for + * the ingest manager's ingest threads. */ private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); @@ -267,11 +258,11 @@ final class IngestJobExecutor { * which the pipeline is being created is * interrupted. */ - // RJCTODO: Refactor so that only the job is passed in and the other params are obtained from the job. IngestJobExecutor(IngestJob ingestJob, Content dataSource, List files, IngestJobSettings settings) throws InterruptedException { if (!(dataSource instanceof DataSource)) { throw new IllegalArgumentException("Passed dataSource that does not implement the DataSource interface"); //NON-NLS } + // RJCTODO: Refactor so that only the job is passed in and the other params are obtained from the job. this.ingestJob = ingestJob; this.dataSource = (DataSource) dataSource; this.files = new ArrayList<>(); @@ -279,8 +270,13 @@ final class IngestJobExecutor { this.settings = settings; usingNetBeansGUI = RuntimeProperties.runningWithGUI(); createTime = new Date().getTime(); - stage = IngestJobStage.PIPELINES_START_UP; - createIngestModulePipelines(); + jobStateLock.writeLock().lock(); + try { + jobState = IngestJobState.PIPELINES_STARTING_UP; + createIngestModulePipelines(); + } finally { + jobStateLock.writeLock().unlock(); + } } /** @@ -428,8 +424,8 @@ final class IngestJobExecutor { fileIngestPipelinesQueue.put(pipeline); fileIngestPipelines.add(pipeline); } - artifactIngestPipeline = new DataArtifactIngestPipeline(this, artifactPipelineTemplate); - resultIngestPipeline = new AnalysisResultIngestPipeline(this, resultsPipelineTemplate); + dataArtifactIngestPipeline = new DataArtifactIngestPipeline(this, artifactPipelineTemplate); + analysisResultIngestPipeline = new AnalysisResultIngestPipeline(this, resultsPipelineTemplate); } /** @@ -505,33 +501,6 @@ final class IngestJobExecutor { return settings.getFileFilter(); } - /** - * Checks to see if there is at least one ingest module to run. - * - * @return True or false. - */ - boolean hasIngestModules() { - return hasFileIngestModules() - || hasHighPriorityDataSourceIngestModules() - || hasLowPriorityDataSourceIngestModules() - || hasDataArtifactIngestModules() - || hasAnalysisResultIngestModules(); - } - - /** - * Checks to see if there is at least one data source level ingest module to - * run. - * - * @return True or false. - */ - boolean hasDataSourceIngestModules() { - if (stage == IngestJobStage.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS) { - return hasLowPriorityDataSourceIngestModules(); - } else { - return hasHighPriorityDataSourceIngestModules(); - } - } - /** * Checks to see if there is at least one high priority data source level * ingest module to run. @@ -539,7 +508,12 @@ final class IngestJobExecutor { * @return True or false. */ private boolean hasHighPriorityDataSourceIngestModules() { - return (highPriorityDataSourceIngestPipeline.isEmpty() == false); + jobStateLock.readLock().lock(); + try { + return (highPriorityDataSourceIngestPipeline.isEmpty() == false); + } finally { + jobStateLock.readLock().unlock(); + } } /** @@ -549,7 +523,12 @@ final class IngestJobExecutor { * @return True or false. */ private boolean hasLowPriorityDataSourceIngestModules() { - return (lowPriorityDataSourceIngestPipeline.isEmpty() == false); + jobStateLock.readLock().lock(); + try { + return (lowPriorityDataSourceIngestPipeline.isEmpty() == false); + } finally { + jobStateLock.readLock().unlock(); + } } /** @@ -557,11 +536,16 @@ final class IngestJobExecutor { * * @return True or false. */ - boolean hasFileIngestModules() { - if (!fileIngestPipelines.isEmpty()) { - return !fileIngestPipelines.get(0).isEmpty(); + private boolean hasFileIngestModules() { + jobStateLock.readLock().lock(); + try { + if (!fileIngestPipelines.isEmpty()) { + return !fileIngestPipelines.get(0).isEmpty(); + } + return false; + } finally { + jobStateLock.readLock().unlock(); } - return false; } /** @@ -570,8 +554,13 @@ final class IngestJobExecutor { * * @return True or false. */ - boolean hasDataArtifactIngestModules() { - return (artifactIngestPipeline.isEmpty() == false); + private boolean hasDataArtifactIngestModules() { + jobStateLock.readLock().lock(); + try { + return (dataArtifactIngestPipeline.isEmpty() == false); + } finally { + jobStateLock.readLock().unlock(); + } } /** @@ -580,8 +569,13 @@ final class IngestJobExecutor { * * @return True or false. */ - boolean hasAnalysisResultIngestModules() { - return (resultIngestPipeline.isEmpty() == false); + private boolean hasAnalysisResultIngestModules() { + jobStateLock.readLock().lock(); + try { + return (analysisResultIngestPipeline.isEmpty() == false); + } finally { + jobStateLock.readLock().unlock(); + } } /** @@ -632,8 +626,8 @@ final class IngestJobExecutor { break; } } - errors.addAll(startUpIngestModulePipeline(artifactIngestPipeline)); - errors.addAll(startUpIngestModulePipeline(resultIngestPipeline)); + errors.addAll(startUpIngestModulePipeline(dataArtifactIngestPipeline)); + errors.addAll(startUpIngestModulePipeline(analysisResultIngestPipeline)); return errors; } @@ -714,9 +708,11 @@ final class IngestJobExecutor { * analysis stage. */ private void startBatchModeAnalysis() { - synchronized (stageTransitionLock) { + jobStateLock.writeLock().lock(); + try { logInfoMessage("Starting ingest job in batch mode"); //NON-NLS - stage = IngestJobStage.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS; + jobState = IngestJobState.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS; + currentDataSourceIngestPipeline = highPriorityDataSourceIngestPipeline; if (hasFileIngestModules()) { /* @@ -732,17 +728,20 @@ final class IngestJobExecutor { * has added to the case database. */ estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor()); + taskScheduler.scheduleFileIngestTasks(this, files); } else { /* * Otherwise, this job is analyzing a user-specified subset * of the files in the data source. */ estimatedFilesToProcess = files.size(); + taskScheduler.scheduleFileIngestTasks(this, Collections.emptyList()); } startFileIngestProgressBar(); } if (hasHighPriorityDataSourceIngestModules()) { + taskScheduler.scheduleDataSourceIngestTask(this); startDataSourceIngestProgressBar(); } @@ -752,6 +751,7 @@ final class IngestJobExecutor { * analysis of any data artifacts already in the case database * will be performed. */ + taskScheduler.scheduleDataArtifactIngestTasks(this); startDataArtifactIngestProgressBar(); } @@ -761,34 +761,10 @@ final class IngestJobExecutor { * analysis of any analysis results already in the case database * will be performed. */ + taskScheduler.scheduleAnalysisResultIngestTasks(this); startAnalysisResultIngestProgressBar(); } - /* - * Make the high priority data source level ingest module pipeline - * the current data source level ingest module pipeline. - */ - currentDataSourceIngestPipeline = highPriorityDataSourceIngestPipeline; - - /* - * Schedule ingest tasks. - */ - if (!files.isEmpty() && hasFileIngestModules()) { - /* - * If only analyzing a subset of the files in the data source, - * only schedule file ingest tasks for those files. Data - * artifact ingest tasks will be scheduled as data artifacts - * produced by the file analysis are posted to the blackboard. - */ - taskScheduler.scheduleFileIngestTasks(this, files); - } else if (hasHighPriorityDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { - /* - * Otherwise, schedule tasks based on the ingest job settings, - * i.e., the enabled ingest modules. - */ - taskScheduler.scheduleIngestTasks(this); - } - /* * Check for stage completion. This is necessary because it is * possible that none of the tasks that were just scheduled will @@ -799,6 +775,9 @@ final class IngestJobExecutor { * a check here. */ checkForStageCompleted(); + + } finally { + jobStateLock.writeLock().unlock(); } } @@ -810,9 +789,10 @@ final class IngestJobExecutor { * level analysis to begin before data source level analysis. */ private void startStreamingModeAnalysis() { - synchronized (stageTransitionLock) { + jobStateLock.writeLock().lock(); + try { logInfoMessage("Starting ingest job in streaming mode"); //NON-NLS - stage = IngestJobStage.STREAMED_FILE_ANALYSIS_ONLY; + jobState = IngestJobState.STREAMED_FILE_ANALYSIS_ONLY; if (hasFileIngestModules()) { /* @@ -823,7 +803,7 @@ final class IngestJobExecutor { * scheduled later, via addStreamedDataSource(). * * Note that because estimated files remaining to process still - * has its initial value of zero, the fle ingest progress bar + * has its initial value of zero, the file ingest progress bar * will start in the "indeterminate" state. A rough estimate of * the files to be processed will be computed later, when all of * the files have been added to the case database, as signalled @@ -833,26 +813,38 @@ final class IngestJobExecutor { startFileIngestProgressBar(); } - /* - * Start the data artifact and analysis result progress bars and - * schedule ingest tasks for any data artifacts and analysis results - * currently in the case database. This needs to be done BEFORE any - * files or the data source are streamed in. This is necessary to - * avoid analyzing any artifacts or results added to the case - * database by the file and data source ingest tasks twice. To - * ensure this happens, a streaming mode ingest job is created and - * started in IngestManager.openIngestStream(), before the ingest - * manager returns the IngestStream object that the client uses to - * call addStreamedFiles() and addStreamedDataSource(). - */ if (hasDataArtifactIngestModules()) { + /* + * Start the data artifact progress bar and schedule ingest + * tasks for any data artifacts currently in the case database. + * This needs to be done BEFORE any files or the data source are + * streamed in to ensure that any data artifacts added to the + * case database by the file and data source ingest tasks are + * not analyzed twice. This works here because the ingest + * manager has not yet returned the ingest stream object that is + * used to call addStreamedFiles() and addStreamedDataSource(). + */ startDataArtifactIngestProgressBar(); taskScheduler.scheduleDataArtifactIngestTasks(this); } + if (hasAnalysisResultIngestModules()) { + /* + * Start the analysis result progress bar and schedule ingest + * tasks for any analysis results currently in the case + * database. This needs to be done BEFORE any files or the data + * source are streamed in to ensure that any analysis results + * added to the case database by the file and data source ingest + * tasks are not analyzed twice. This works here because the + * ingest manager has not yet returned the ingest stream object + * that is used to call addStreamedFiles() and + * addStreamedDataSource(). + */ startAnalysisResultIngestProgressBar(); taskScheduler.scheduleAnalysisResultIngestTasks(this); } + } finally { + jobStateLock.writeLock().unlock(); } } @@ -862,10 +854,10 @@ final class IngestJobExecutor { * source is now ready for analysis. */ void addStreamedDataSource() { - synchronized (stageTransitionLock) { + jobStateLock.writeLock().lock(); + try { logInfoMessage("Starting full first stage analysis in streaming mode"); //NON-NLS - stage = IngestJobExecutor.IngestJobStage.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS; - currentDataSourceIngestPipeline = highPriorityDataSourceIngestPipeline; + jobState = IngestJobExecutor.IngestJobState.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS; if (hasFileIngestModules()) { /* @@ -902,6 +894,8 @@ final class IngestJobExecutor { */ checkForStageCompleted(); } + } finally { + jobStateLock.writeLock().unlock(); } } @@ -909,14 +903,17 @@ final class IngestJobExecutor { * Starts low priority data source analysis. */ private void startLowPriorityDataSourceAnalysis() { - synchronized (stageTransitionLock) { + jobStateLock.writeLock().lock(); + try { if (hasLowPriorityDataSourceIngestModules()) { logInfoMessage("Starting low priority data source analysis"); //NON-NLS - stage = IngestJobExecutor.IngestJobStage.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS; - startDataSourceIngestProgressBar(); + jobState = IngestJobExecutor.IngestJobState.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS; currentDataSourceIngestPipeline = lowPriorityDataSourceIngestPipeline; + startDataSourceIngestProgressBar(); taskScheduler.scheduleDataSourceIngestTask(this); } + } finally { + jobStateLock.writeLock().unlock(); } } @@ -1080,8 +1077,8 @@ final class IngestJobExecutor { if (artifactIngestProgressBar != null) { artifactIngestProgressBar.finish(); artifactIngestProgressBar = null; - } - + } + if (resultIngestProgressBar != null) { resultIngestProgressBar.finish(); resultIngestProgressBar = null; @@ -1095,12 +1092,13 @@ final class IngestJobExecutor { * completed and does a stage transition if they are. */ private void checkForStageCompleted() { - synchronized (stageTransitionLock) { - if (stage == IngestJobStage.STREAMED_FILE_ANALYSIS_ONLY) { + jobStateLock.writeLock().lock(); + try { + if (jobState == IngestJobState.STREAMED_FILE_ANALYSIS_ONLY) { return; } if (taskScheduler.currentTasksAreCompleted(getIngestJobId())) { - switch (stage) { + switch (jobState) { case FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS: finishFileAndHighPriorityDataSrcAnalysis(); break; @@ -1109,6 +1107,8 @@ final class IngestJobExecutor { break; } } + } finally { + jobStateLock.writeLock().unlock(); } } @@ -1118,22 +1118,24 @@ final class IngestJobExecutor { * level analysis stage, if appropriate. */ private void finishFileAndHighPriorityDataSrcAnalysis() { - synchronized (stageTransitionLock) { - logInfoMessage("Finished file and high-priority data source analysis"); //NON-NLS - + jobStateLock.writeLock().lock(); + try { + jobState = IngestJobState.PIPELINES_SHUTTING_DOWN; shutDownIngestModulePipeline(currentDataSourceIngestPipeline); while (!fileIngestPipelinesQueue.isEmpty()) { FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll(); shutDownIngestModulePipeline(pipeline); } - finishFirstStageProgressBars(); + logInfoMessage("Finished file and high-priority data source analysis"); //NON-NLS if (!jobCancelled && hasLowPriorityDataSourceIngestModules()) { startLowPriorityDataSourceAnalysis(); } else { shutDown(); } + } finally { + jobStateLock.writeLock().unlock(); } } @@ -1141,37 +1143,33 @@ final class IngestJobExecutor { * Shuts down the ingest module pipelines and ingest job progress bars. */ private void shutDown() { - synchronized (stageTransitionLock) { + jobStateLock.writeLock().lock(); + try { logInfoMessage("Finished all ingest tasks"); //NON-NLS - stage = IngestJobExecutor.IngestJobStage.PIPELINES_SHUT_DOWN; + jobState = IngestJobExecutor.IngestJobState.PIPELINES_SHUTTING_DOWN; shutDownIngestModulePipeline(currentDataSourceIngestPipeline); - shutDownIngestModulePipeline(artifactIngestPipeline); - shutDownIngestModulePipeline(resultIngestPipeline); + shutDownIngestModulePipeline(dataArtifactIngestPipeline); + shutDownIngestModulePipeline(analysisResultIngestPipeline); + finishAllProgressBars(); - if (ingestJobInfo != null) { - if (jobCancelled) { - try { + try { + if (ingestJobInfo != null) { + if (jobCancelled) { ingestJobInfo.setIngestJobStatus(IngestJobStatusType.CANCELLED); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); - } - } else { - try { + } else { ingestJobInfo.setIngestJobStatus(IngestJobStatusType.COMPLETED); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); } - } - try { ingestJobInfo.setEndDateTime(new Date()); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); } + } catch (TskCoreException ex) { + logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); } - } - ingestJob.notifyIngestPipelinesShutDown(); + ingestJob.notifyIngestPipelinesShutDown(); + } finally { + jobStateLock.writeLock().unlock(); + } } /** @@ -1197,6 +1195,7 @@ final class IngestJobExecutor { * the data source ingest pipeline. */ void execute(DataSourceIngestTask task) { + jobStateLock.readLock().lock(); try { if (!isCancelled()) { List errors = new ArrayList<>(); @@ -1207,6 +1206,7 @@ final class IngestJobExecutor { } } finally { taskScheduler.notifyTaskCompleted(task); + jobStateLock.readLock().unlock(); checkForStageCompleted(); } } @@ -1219,6 +1219,7 @@ final class IngestJobExecutor { * pipeline. */ void execute(FileIngestTask task) { + jobStateLock.readLock().lock(); try { if (!isCancelled()) { FileIngestPipeline pipeline = fileIngestPipelinesQueue.take(); @@ -1260,6 +1261,7 @@ final class IngestJobExecutor { Thread.currentThread().interrupt(); } finally { taskScheduler.notifyTaskCompleted(task); + jobStateLock.readLock().unlock(); checkForStageCompleted(); } } @@ -1272,16 +1274,18 @@ final class IngestJobExecutor { * and the data artifact ingest pipeline. */ void execute(DataArtifactIngestTask task) { + jobStateLock.readLock().lock(); try { - if (!isCancelled() && !artifactIngestPipeline.isEmpty()) { + if (!isCancelled() && !dataArtifactIngestPipeline.isEmpty()) { List errors = new ArrayList<>(); - errors.addAll(artifactIngestPipeline.performTask(task)); + errors.addAll(dataArtifactIngestPipeline.performTask(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors); } } } finally { taskScheduler.notifyTaskCompleted(task); + jobStateLock.readLock().unlock(); checkForStageCompleted(); } } @@ -1294,16 +1298,18 @@ final class IngestJobExecutor { * result and the analysis result ingest pipeline. */ void execute(AnalysisResultIngestTask task) { + jobStateLock.readLock().lock(); try { - if (!isCancelled() && !resultIngestPipeline.isEmpty()) { + if (!isCancelled() && !analysisResultIngestPipeline.isEmpty()) { List errors = new ArrayList<>(); - errors.addAll(resultIngestPipeline.performTask(task)); + errors.addAll(analysisResultIngestPipeline.performTask(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors); } } } finally { taskScheduler.notifyTaskCompleted(task); + jobStateLock.readLock().unlock(); checkForStageCompleted(); } } @@ -1314,11 +1320,11 @@ final class IngestJobExecutor { * @param fileObjIds The object IDs of the files. */ void addStreamedFiles(List fileObjIds) { - if (hasFileIngestModules()) { - if (stage.equals(IngestJobStage.STREAMED_FILE_ANALYSIS_ONLY)) { + if (!isCancelled() && hasFileIngestModules()) { + if (jobState.equals(IngestJobState.STREAMED_FILE_ANALYSIS_ONLY)) { IngestJobExecutor.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); } else { - logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); + logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + jobState.toString() + " not supported"); } } } @@ -1335,11 +1341,12 @@ final class IngestJobExecutor { * @param files A list of the files to add. */ void addFiles(List files) { - if (stage.equals(IngestJobStage.STREAMED_FILE_ANALYSIS_ONLY) - || stage.equals(IngestJobStage.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { - taskScheduler.scheduleHighPriorityFileIngestTasks(this, files); - } else { - logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); + if (!isCancelled() && hasFileIngestModules()) { + if (jobState.equals(IngestJobState.STREAMED_FILE_ANALYSIS_ONLY) || jobState.equals(IngestJobState.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { + taskScheduler.scheduleHighPriorityFileIngestTasks(this, files); + } else { + logErrorMessage(Level.SEVERE, "Adding files to job during stage " + jobState.toString() + " not supported"); + } } } @@ -1354,12 +1361,12 @@ final class IngestJobExecutor { * @param artifacts The data artifacts. */ void addDataArtifacts(List artifacts) { - if (stage.equals(IngestJobStage.STREAMED_FILE_ANALYSIS_ONLY) - || stage.equals(IngestJobStage.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS) - || stage.equals(IngestJobStage.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { - taskScheduler.scheduleDataArtifactIngestTasks(this, artifacts); - } else { - logErrorMessage(Level.SEVERE, "Attempt to add data artifacts to job during stage " + stage.toString() + " not supported"); + if (!isCancelled() && hasDataArtifactIngestModules()) { + if (jobState.equals(IngestJobState.STREAMED_FILE_ANALYSIS_ONLY) || jobState.equals(IngestJobState.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS) || jobState.equals(IngestJobState.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { + taskScheduler.scheduleDataArtifactIngestTasks(this, artifacts); + } else { + logErrorMessage(Level.SEVERE, "Attempt to add data artifacts to job during stage " + jobState.toString() + " not supported"); + } } } @@ -1374,12 +1381,12 @@ final class IngestJobExecutor { * @param results The analysis results. */ void addAnalysisResults(List results) { - if (stage.equals(IngestJobStage.STREAMED_FILE_ANALYSIS_ONLY) - || stage.equals(IngestJobStage.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS) - || stage.equals(IngestJobStage.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { - taskScheduler.scheduleAnalysisResultIngestTasks(this, results); - } else { - logErrorMessage(Level.SEVERE, "Attempt to add analysis results to job during stage " + stage.toString() + " not supported"); + if (!isCancelled() && hasAnalysisResultIngestModules()) { + if (jobState.equals(IngestJobState.STREAMED_FILE_ANALYSIS_ONLY) || jobState.equals(IngestJobState.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS) || jobState.equals(IngestJobState.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { + taskScheduler.scheduleAnalysisResultIngestTasks(this, results); + } else { + logErrorMessage(Level.SEVERE, "Attempt to add analysis results to job during stage " + jobState.toString() + " not supported"); + } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 8c1f9f3307..d7609e63c7 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.ingest; import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.Deque; import java.util.Iterator; @@ -137,43 +136,6 @@ final class IngestTasksScheduler { return resultIngestTasksQueue; } - /** - * Schedules ingest tasks for an ingest job based on the types of ingest - * modules that will be used by the ingest job executor. - * - * Scheduling these tasks atomically means that it is valid to call - * currentTasksAreCompleted() immediately after calling this method. An - * example of where this is relevant would be an ingest job that only - * requires the processing of files, and that has an ingest filter in its - * settings that causes the ingest task scheduler to discard all of the file - * tasks. - * - * RJCTODO: Return a count of scheduled tasks or even just a boolean; let - * the caller know if file filters, etc., caused no tasks to be scheduled. - * - * @param executor The ingest job executor that will execute the scheduled - * tasks. A reference to the executor is added to each task - * so that when the task is dequeued by an ingest thread, - * the task can pass its target item to the executor for - * processing by the executor's ingest module pipelines. - */ - synchronized void scheduleIngestTasks(IngestJobExecutor executor) { - if (!executor.isCancelled()) { - if (executor.hasDataSourceIngestModules()) { - scheduleDataSourceIngestTask(executor); - } - if (executor.hasFileIngestModules()) { - scheduleFileIngestTasks(executor, Collections.emptyList()); - } - if (executor.hasDataArtifactIngestModules()) { - scheduleDataArtifactIngestTasks(executor); - } - if (executor.hasAnalysisResultIngestModules()) { - scheduleAnalysisResultIngestTasks(executor); - } - } - } - /** * Schedules a data source level ingest task for an ingest job. The data * source is obtained from the ingest ingest job executor passed in. diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java index c01a81ff66..dc633e8795 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java @@ -440,7 +440,7 @@ class AdHocSearchChildFactory extends ChildFactory { }); } registerWriter(this); //register (synchronized on class) outside of writerLock to prevent deadlock - hits.process(this, false, saveResults, null); + hits.process(this, false, saveResults, saveResults, null); } finally { deregisterWriter(this); if (RuntimeProperties.runningWithGUI() && progress != null) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java index bb2fbe189a..8a68bae0c7 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java @@ -36,6 +36,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import static java.util.concurrent.TimeUnit.MILLISECONDS; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; +import javax.annotation.concurrent.GuardedBy; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; import org.netbeans.api.progress.ProgressHandle; @@ -52,34 +53,44 @@ import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestServices; /** - * Singleton keyword search manager: Launches search threads for each job and - * performs commits, both on timed intervals. + * Performs periodic and final keyword searches for ingest jobs. Periodic + * searches are done in background tasks. This represents a careful working + * around of the contract for IngestModule.process(). Final searches are done + * synchronously in the calling thread, as required by the contract for + * IngestModule.shutDown(). */ final class IngestSearchRunner { private static final Logger logger = Logger.getLogger(IngestSearchRunner.class.getName()); private static IngestSearchRunner instance = null; - private IngestServices services = IngestServices.getInstance(); + private final IngestServices services = IngestServices.getInstance(); private Ingester ingester = null; private long currentUpdateIntervalMs; - private volatile boolean periodicSearchTaskRunning = false; - private Future jobProcessingTaskFuture; - private final ScheduledThreadPoolExecutor jobProcessingExecutor; + private volatile boolean periodicSearchTaskRunning; + private volatile Future periodicSearchTaskHandle; + private final ScheduledThreadPoolExecutor periodicSearchTaskExecutor; private static final int NUM_SEARCH_SCHEDULING_THREADS = 1; - private static final String SEARCH_SCHEDULER_THREAD_NAME = "periodic-search-scheduler-%d"; + private static final String SEARCH_SCHEDULER_THREAD_NAME = "periodic-search-scheduling-%d"; + private final Map jobs = new ConcurrentHashMap<>(); // Ingest job ID to search job info + private final boolean usingNetBeansGUI = RuntimeProperties.runningWithGUI(); - // maps a jobID to the search - private Map jobs = new ConcurrentHashMap<>(); - - IngestSearchRunner() { + /* + * Constructs a singleton object that performs periodic and final keyword + * searches for ingest jobs. Periodic searches are done in background tasks. + * This represents a careful working around of the contract for + * IngestModule.process(). Final searches are done synchronously in the + * calling thread, as required by the contract for IngestModule.shutDown(). + */ + private IngestSearchRunner() { currentUpdateIntervalMs = ((long) KeywordSearchSettings.getUpdateFrequency().getTime()) * 60 * 1000; ingester = Ingester.getDefault(); - jobProcessingExecutor = new ScheduledThreadPoolExecutor(NUM_SEARCH_SCHEDULING_THREADS, new ThreadFactoryBuilder().setNameFormat(SEARCH_SCHEDULER_THREAD_NAME).build()); + periodicSearchTaskExecutor = new ScheduledThreadPoolExecutor(NUM_SEARCH_SCHEDULING_THREADS, new ThreadFactoryBuilder().setNameFormat(SEARCH_SCHEDULER_THREAD_NAME).build()); } /** + * Gets the ingest search runner singleton. * - * @return the singleton object + * @return The ingest search runner. */ public static synchronized IngestSearchRunner getInstance() { if (instance == null) { @@ -89,72 +100,75 @@ final class IngestSearchRunner { } /** + * Starts the search job for an ingest job. * - * @param jobContext - * @param keywordListNames + * @param jobContext The ingest job context. + * @param keywordListNames The names of the keyword search lists for the + * ingest job. */ public synchronized void startJob(IngestJobContext jobContext, List keywordListNames) { long jobId = jobContext.getJobId(); if (jobs.containsKey(jobId) == false) { - logger.log(Level.INFO, "Adding job {0}", jobId); //NON-NLS SearchJobInfo jobData = new SearchJobInfo(jobContext, keywordListNames); jobs.put(jobId, jobData); } - // keep track of how many threads / module instances from this job have asked for this + /* + * Keep track of the number of keyword search file ingest modules that + * are doing analysis for the ingest job, i.e., that have called this + * method. This is needed by endJob(). + */ jobs.get(jobId).incrementModuleReferenceCount(); - // start the timer, if needed + /* + * Start a periodic search task in the + */ if ((jobs.size() > 0) && (periodicSearchTaskRunning == false)) { - // reset the default periodic search frequency to the user setting - logger.log(Level.INFO, "Resetting periodic search time out to default value"); //NON-NLS currentUpdateIntervalMs = ((long) KeywordSearchSettings.getUpdateFrequency().getTime()) * 60 * 1000; - jobProcessingTaskFuture = jobProcessingExecutor.schedule(new PeriodicSearchTask(), currentUpdateIntervalMs, MILLISECONDS); + periodicSearchTaskHandle = periodicSearchTaskExecutor.schedule(new PeriodicSearchTask(), currentUpdateIntervalMs, MILLISECONDS); periodicSearchTaskRunning = true; } } /** - * Perform normal finishing of searching for this job, including one last - * commit and search. Blocks until the final search is complete. + * Finishes a search job for an ingest job. * - * @param jobId + * @param jobId The ingest job ID. */ public synchronized void endJob(long jobId) { + /* + * Only complete the job if this is the last keyword search file ingest + * module doing annalysis for this job. + */ SearchJobInfo job; - boolean readyForFinalSearch = false; job = jobs.get(jobId); if (job == null) { - return; + return; // RJCTODO: SEVERE } - - // Only do final search if this is the last module/thread in this job to call endJob() - if (job.decrementModuleReferenceCount() == 0) { + if (job.decrementModuleReferenceCount() != 0) { jobs.remove(jobId); - readyForFinalSearch = true; } - if (readyForFinalSearch) { - logger.log(Level.INFO, "Commiting search index before final search for search job {0}", job.getJobId()); //NON-NLS - commit(); - doFinalSearch(job); //this will block until it's done + /* + * Commit the index and do the final search. The final search is done in + * the ingest thread that shutDown() on the keyword search file ingest + * module, per the contract of IngestModule.shutDwon(). + */ + logger.log(Level.INFO, "Commiting search index before final search for search job {0}", job.getJobId()); //NON-NLS + commit(); + logger.log(Level.INFO, "Starting final search for search job {0}", job.getJobId()); //NON-NLS + doFinalSearch(job); + logger.log(Level.INFO, "Final search for search job {0} completed", job.getJobId()); //NON-NLS - // new jobs could have been added while we were doing final search - if (jobs.isEmpty()) { - // no more jobs left. stop the PeriodicSearchTask. - // A new one will be created for future jobs. - logger.log(Level.INFO, "No more search jobs. Stopping periodic search task"); //NON-NLS - periodicSearchTaskRunning = false; - jobProcessingTaskFuture.cancel(true); - } + if (jobs.isEmpty()) { + cancelPeriodicSearchSchedulingTask(); } } /** - * Immediate stop and removal of job from SearchRunner. Cancels the - * associated search worker if it's still running. + * Stops the search job for an ingest job. * - * @param jobId + * @param jobId The ingest job ID. */ public synchronized void stopJob(long jobId) { logger.log(Level.INFO, "Stopping search job {0}", jobId); //NON-NLS @@ -166,7 +180,10 @@ final class IngestSearchRunner { return; } - //stop currentSearcher + /* + * Request cancellation of the current keyword search, whether it is a + * preiodic search or a final search. + */ IngestSearchRunner.Searcher currentSearcher = job.getCurrentSearcher(); if ((currentSearcher != null) && (!currentSearcher.isDone())) { logger.log(Level.INFO, "Cancelling search job {0}", jobId); //NON-NLS @@ -176,19 +193,16 @@ final class IngestSearchRunner { jobs.remove(jobId); if (jobs.isEmpty()) { - // no more jobs left. stop the PeriodicSearchTask. - // A new one will be created for future jobs. - logger.log(Level.INFO, "No more search jobs. Stopping periodic search task"); //NON-NLS - periodicSearchTaskRunning = false; - jobProcessingTaskFuture.cancel(true); + cancelPeriodicSearchSchedulingTask(); } } /** - * Add these lists to all of the jobs. Used when user wants to search for a - * list while ingest has already started. + * Adds the given keyword list names to the set of keyword lists to be + * searched by ALL keyword search jobs. This supports adding one or more + * keyword search lists to ingest jobs already in progress. * - * @param keywordListNames + * @param keywordListNames The n ames of the additional keyword lists. */ public synchronized void addKeywordListsToAllJobs(List keywordListNames) { for (String listName : keywordListNames) { @@ -200,155 +214,171 @@ final class IngestSearchRunner { } /** - * Commits index and notifies listeners of index update + * Commits the Solr index for the current case and publishes an event + * indicating the current number of indexed items (this is no longer just + * files). */ private void commit() { ingester.commit(); - // Signal a potential change in number of text_ingested files + /* + * Publish an event advertising the number of indexed items. Note that + * this is no longer the number of indexed files, since the text of many + * items in addition to files is indexed. + */ try { final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles(); KeywordSearch.fireNumIndexedFilesChange(null, numIndexedFiles); } catch (NoOpenCoreException | KeywordSearchModuleException ex) { - logger.log(Level.SEVERE, "Error executing Solr query to check number of indexed files", ex); //NON-NLS + logger.log(Level.SEVERE, "Error executing Solr query for number of indexed files", ex); //NON-NLS } } /** - * A final search waits for any still-running workers, and then executes a - * new one and waits until that is done. + * Performs the final keyword search for an ingest job. The search is done + * synchronously, as required by the contract for IngestModule.shutDown(). * - * @param job + * @param job The keyword search job info. */ private void doFinalSearch(SearchJobInfo job) { - // Run one last search as there are probably some new files committed - logger.log(Level.INFO, "Starting final search for search job {0}", job.getJobId()); //NON-NLS if (!job.getKeywordListNames().isEmpty()) { try { - // In case this job still has a worker running, wait for it to finish - logger.log(Level.INFO, "Checking for previous search for search job {0} before executing final search", job.getJobId()); //NON-NLS + /* + * Wait for any periodic searches being done in a SwingWorker + * pool thread to finish. + */ job.waitForCurrentWorker(); - IngestSearchRunner.Searcher finalSearcher = new IngestSearchRunner.Searcher(job, true); - job.setCurrentSearcher(finalSearcher); //save the ref - logger.log(Level.INFO, "Kicking off final search for search job {0}", job.getJobId()); //NON-NLS - finalSearcher.execute(); //start thread - - // block until the search is complete - logger.log(Level.INFO, "Waiting for final search for search job {0}", job.getJobId()); //NON-NLS - finalSearcher.get(); - logger.log(Level.INFO, "Final search for search job {0} completed", job.getJobId()); //NON-NLS - + job.setCurrentSearcher(finalSearcher); + /* + * Do the final search synchronously on the current ingest + * thread, per the contract specified + */ + finalSearcher.doInBackground(); } catch (InterruptedException | CancellationException ex) { logger.log(Level.INFO, "Final search for search job {0} interrupted or cancelled", job.getJobId()); //NON-NLS - } catch (ExecutionException ex) { + } catch (Exception ex) { logger.log(Level.SEVERE, String.format("Final search for search job %d failed", job.getJobId()), ex); //NON-NLS } } } /** - * Task to perform periodic searches for each job (does a single index - * commit first) + * Cancels the current periodic search scheduling task. */ - private final class PeriodicSearchTask implements Runnable { - - private final Logger logger = Logger.getLogger(IngestSearchRunner.PeriodicSearchTask.class.getName()); - - @Override - public void run() { - // If no jobs then cancel the task. If more job(s) come along, a new task will start up. - if (jobs.isEmpty() || jobProcessingTaskFuture.isCancelled()) { - logger.log(Level.INFO, "Exiting periodic search task"); //NON-NLS - periodicSearchTaskRunning = false; - return; - } - - commit(); - - logger.log(Level.INFO, "Starting periodic searches"); - final StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - // NOTE: contents of "jobs" ConcurrentHashMap can be modified in stopJob() and endJob() while we are inside this loop - for (Iterator> iterator = jobs.entrySet().iterator(); iterator.hasNext();) { - SearchJobInfo job = iterator.next().getValue(); - - if (jobProcessingTaskFuture.isCancelled()) { - logger.log(Level.INFO, "Search has been cancelled. Exiting periodic search task."); //NON-NLS - periodicSearchTaskRunning = false; - return; - } - - // If no lists or the worker is already running then skip it - if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) { - // Spawn a search thread for each job - logger.log(Level.INFO, "Executing periodic search for search job {0}", job.getJobId()); - Searcher searcher = new Searcher(job); // SwingWorker - job.setCurrentSearcher(searcher); //save the ref - searcher.execute(); //start thread - job.setWorkerRunning(true); - - try { - // wait for the searcher to finish - searcher.get(); - } catch (InterruptedException | ExecutionException ex) { - logger.log(Level.SEVERE, "Error performing keyword search: {0}", ex.getMessage()); //NON-NLS - services.postMessage(IngestMessage.createErrorMessage(KeywordSearchModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), - "SearchRunner.Searcher.done.err.msg"), ex.getMessage())); - }// catch and ignore if we were cancelled - catch (java.util.concurrent.CancellationException ex) { - } - } - } - stopWatch.stop(); - logger.log(Level.INFO, "All periodic searches cumulatively took {0} secs", stopWatch.getElapsedTimeSecs()); //NON-NLS - - // calculate "hold off" time - recalculateUpdateIntervalTime(stopWatch.getElapsedTimeSecs()); // ELDEBUG - - // schedule next PeriodicSearchTask - jobProcessingTaskFuture = jobProcessingExecutor.schedule(new PeriodicSearchTask(), currentUpdateIntervalMs, MILLISECONDS); - - // exit this thread - return; - } - - private void recalculateUpdateIntervalTime(long lastSerchTimeSec) { - // If periodic search takes more than 1/4 of the current periodic search interval, then double the search interval - if (lastSerchTimeSec * 1000 < currentUpdateIntervalMs / 4) { - return; - } - // double the search interval - currentUpdateIntervalMs = currentUpdateIntervalMs * 2; - logger.log(Level.WARNING, "Last periodic search took {0} sec. Increasing search interval to {1} sec", new Object[]{lastSerchTimeSec, currentUpdateIntervalMs / 1000}); - return; + private synchronized void cancelPeriodicSearchSchedulingTask() { + if (periodicSearchTaskHandle != null) { + logger.log(Level.INFO, "No more search jobs, stopping periodic search scheduling"); //NON-NLS + periodicSearchTaskHandle.cancel(true); + periodicSearchTaskRunning = false; } } /** - * Data structure to keep track of keyword lists, current results, and - * search running status for each jobid + * Task that runs in ScheduledThreadPoolExecutor to periodically start and + * wait for keyword search tasks for each keyword search job in progress. + * The keyword search tasks for individual ingest jobs are implemented as + * SwingWorkers to support legacy APIs. + */ + private final class PeriodicSearchTask implements Runnable { + + @Override + public void run() { + /* + * If there are no more jobs or this task has been cancelled, exit. + */ + if (jobs.isEmpty() || periodicSearchTaskHandle.isCancelled()) { + logger.log(Level.INFO, "Periodic search scheduling task has been cancelled, exiting"); //NON-NLS + periodicSearchTaskRunning = false; + return; + } + + /* + * Commit the Solr index for the current case before doing the + * searches. + */ + commit(); + + /* + * Do a keyword search for each ingest job in progress. When the + * searches are done, recalculate the "hold off" time between + * searches to prevent back-to-back periodic searches and schedule + * the nect periodic search task. + */ + final StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + for (Iterator> iterator = jobs.entrySet().iterator(); iterator.hasNext();) { + SearchJobInfo job = iterator.next().getValue(); + + if (periodicSearchTaskHandle.isCancelled()) { + logger.log(Level.INFO, "Periodic search scheduling task has been cancelled, exiting"); //NON-NLS + periodicSearchTaskRunning = false; + return; + } + + if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) { + logger.log(Level.INFO, "Starting periodic search for search job {0}", job.getJobId()); + Searcher searcher = new Searcher(job, false); + job.setCurrentSearcher(searcher); + searcher.execute(); + job.setWorkerRunning(true); + try { + searcher.get(); + } catch (InterruptedException | ExecutionException ex) { + logger.log(Level.SEVERE, String.format("Error performing keyword search for ingest job %d", job.getJobId()), ex); //NON-NLS + services.postMessage(IngestMessage.createErrorMessage( + KeywordSearchModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), "SearchRunner.Searcher.done.err.msg"), ex.getMessage())); + } catch (java.util.concurrent.CancellationException ex) { + logger.log(Level.SEVERE, String.format("Keyword search for ingest job %d cancelled", job.getJobId()), ex); //NON-NLS + } + } + } + stopWatch.stop(); + logger.log(Level.INFO, "Periodic searches for all ingest jobs cumulatively took {0} secs", stopWatch.getElapsedTimeSecs()); //NON-NLS + recalculateUpdateIntervalTime(stopWatch.getElapsedTimeSecs()); // ELDEBUG + periodicSearchTaskHandle = periodicSearchTaskExecutor.schedule(new PeriodicSearchTask(), currentUpdateIntervalMs, MILLISECONDS); + } + + /** + * Sets the time interval between periodic keyword searches to avoid + * running back-to-back searches. If the most recent round of searches + * took longer that 1/4 of the current interval, doubles the interval. + * + * @param lastSerchTimeSec The time in seconds used to execute the most + * recent round of keword searches. + */ + private void recalculateUpdateIntervalTime(long lastSerchTimeSec) { + if (lastSerchTimeSec * 1000 < currentUpdateIntervalMs / 4) { + return; + } + currentUpdateIntervalMs *= 2; + logger.log(Level.WARNING, "Last periodic search took {0} sec. Increasing search interval to {1} sec", new Object[]{lastSerchTimeSec, currentUpdateIntervalMs / 1000}); + } + } + + /** + * A data structure to keep track of the keyword lists, current results, and + * search running status for an ingest job. */ private class SearchJobInfo { private final IngestJobContext jobContext; private final long jobId; private final long dataSourceId; - // mutable state: private volatile boolean workerRunning; - private List keywordListNames; //guarded by SearchJobInfo.this - - // Map of keyword to the object ids that contain a hit - private Map> currentResults; //guarded by SearchJobInfo.this + @GuardedBy("this") + private final List keywordListNames; + @GuardedBy("this") + private final Map> currentResults; // Keyword to object IDs of items with hits private IngestSearchRunner.Searcher currentSearcher; - private AtomicLong moduleReferenceCount = new AtomicLong(0); - private final Object finalSearchLock = new Object(); //used for a condition wait + private final AtomicLong moduleReferenceCount = new AtomicLong(0); + private final Object finalSearchLock = new Object(); private SearchJobInfo(IngestJobContext jobContext, List keywordListNames) { this.jobContext = jobContext; - this.jobId = jobContext.getJobId(); - this.dataSourceId = jobContext.getDataSource().getId(); + jobId = jobContext.getJobId(); + dataSourceId = jobContext.getDataSource().getId(); this.keywordListNames = new ArrayList<>(keywordListNames); currentResults = new HashMap<>(); workerRunning = false; @@ -410,41 +440,40 @@ final class IngestSearchRunner { } /** - * In case this job still has a worker running, wait for it to finish + * Waits for the current search task to complete. * * @throws InterruptedException */ private void waitForCurrentWorker() throws InterruptedException { synchronized (finalSearchLock) { while (workerRunning) { - logger.log(Level.INFO, "Waiting for previous worker to finish"); //NON-NLS - finalSearchLock.wait(); //wait() releases the lock - logger.log(Level.INFO, "Notified previous worker finished"); //NON-NLS + logger.log(Level.INFO, String.format("Waiting for previous search task for job %d to finish", jobId)); //NON-NLS + finalSearchLock.wait(); + logger.log(Level.INFO, String.format("Notified previous search task for job %d to finish", jobId)); //NON-NLS } } } /** - * Unset workerRunning and wake up thread(s) waiting on finalSearchLock + * Signals any threads waiting on the current search task to complete. */ private void searchNotify() { synchronized (finalSearchLock) { - logger.log(Level.INFO, "Notifying after finishing search"); //NON-NLS workerRunning = false; finalSearchLock.notify(); } } } - /** - * Searcher responsible for searching the current index and writing results - * to blackboard and the inbox. Also, posts results to listeners as Ingest - * data events. Searches entire index, and keeps track of only new results - * to report and save. Runs as a background thread. + /* + * A SwingWorker responsible for searching the Solr index of the current + * case for the keywords for an ingest job. Keyword hit analysis results are + * created and posted to the blackboard and notifications are sent to the + * ingest inbox. */ private final class Searcher extends SwingWorker { - /** + /* * Searcher has private copies/snapshots of the lists and keywords */ private final SearchJobInfo job; @@ -452,31 +481,22 @@ final class IngestSearchRunner { private final List keywordListNames; // lists currently being searched private final List keywordLists; private final Map keywordToList; //keyword to list name mapping - private final boolean usingNetBeansGUI; @ThreadConfined(type = ThreadConfined.ThreadType.AWT) private ProgressHandle progressIndicator; private boolean finalRun = false; - Searcher(SearchJobInfo job) { + Searcher(SearchJobInfo job, boolean finalRun) { this.job = job; + this.finalRun = finalRun; keywordListNames = job.getKeywordListNames(); keywords = new ArrayList<>(); keywordToList = new HashMap<>(); keywordLists = new ArrayList<>(); - //keywords are populated as searcher runs - usingNetBeansGUI = RuntimeProperties.runningWithGUI(); - } - - Searcher(SearchJobInfo job, boolean finalRun) { - this(job); - this.finalRun = finalRun; } @Override @Messages("SearchRunner.query.exception.msg=Error performing query:") protected Object doInBackground() throws Exception { - final StopWatch stopWatch = new StopWatch(); - stopWatch.start(); try { if (usingNetBeansGUI) { /* @@ -564,11 +584,11 @@ final class IngestSearchRunner { if (!newResults.getKeywords().isEmpty()) { // Create blackboard artifacts - newResults.process(this, keywordList.getIngestMessages(), true, job.getJobId()); + newResults.process(this, keywordList.getIngestMessages(), true, finalRun, job.getJobId()); } } } catch (Exception ex) { - logger.log(Level.WARNING, "Error occurred during keyword search", ex); //NON-NLS + logger.log(Level.SEVERE, String.format("Error performing keyword search for ingest job %d", job.getJobId()), ex); //NON-NLS } finally { if (progressIndicator != null) { SwingUtilities.invokeLater(new Runnable() { @@ -579,8 +599,6 @@ final class IngestSearchRunner { } }); } - stopWatch.stop(); - logger.log(Level.INFO, "Searcher took {0} secs to run (final = {1})", new Object[]{stopWatch.getElapsedTimeSecs(), this.finalRun}); //NON-NLS // In case a thread is waiting on this worker to be done job.searchNotify(); } @@ -681,4 +699,5 @@ final class IngestSearchRunner { return newResults; } } + } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index d79b76bcca..955e8ba278 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -137,11 +137,13 @@ class QueryResults { * @param notifyInbox Whether or not to write a message to the ingest * messages inbox if there is a keyword hit in the text * exrtacted from the text source object. - * @param saveResults Flag whether to save search results as KWS artifacts. + * @param saveResults Whether or not to create keyword hit analysis results. + * @param postResults Whether or not to post any keyword hit analysis + * results created. * @param ingestJobId The numeric identifier of the ingest job within which * the artifacts are being created, may be null. */ - void process(SwingWorker worker, boolean notifyInbox, boolean saveResults, Long ingestJobId) { + void process(SwingWorker worker, boolean notifyInbox, boolean saveResults, boolean postResults, Long ingestJobId) { final Collection hitArtifacts = new ArrayList<>(); for (final Keyword keyword : getKeywords()) { /* @@ -217,7 +219,7 @@ class QueryResults { * Post the artifacts to the blackboard which will publish an event to * notify subscribers of the new artifacts. */ - if (!hitArtifacts.isEmpty()) { + if (!hitArtifacts.isEmpty() && postResults) { try { SleuthkitCase tskCase = Case.getCurrentCaseThrows().getSleuthkitCase(); Blackboard blackboard = tskCase.getBlackboard(); From 3ad0a994bcb48d96e2717e4732214b2fcd1c82ca Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 7 Dec 2021 16:20:13 -0500 Subject: [PATCH 15/19] 7529 KWS artifact ingest module --- .../src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java | 6 +++--- .../autopsy/keywordsearch/Bundle.properties-MERGED | 1 - .../sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java | 4 +++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 55a71da48e..b0f8b8c512 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -1300,12 +1300,12 @@ final class IngestJobExecutor { void execute(AnalysisResultIngestTask task) { jobStateLock.readLock().lock(); try { - if (!isCancelled() && !analysisResultIngestPipeline.isEmpty()) { + if (!isCancelled() && !analysisResultIngestPipeline.isEmpty()) { List errors = new ArrayList<>(); errors.addAll(analysisResultIngestPipeline.performTask(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors); - } + } } } finally { taskScheduler.notifyTaskCompleted(task); @@ -1588,7 +1588,7 @@ final class IngestJobExecutor { artifactIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); } if (resultIngestProgressBar != null) { - resultIngestProgressBar.setDisplayName(NbBundle.getMessage(getClass(), "IngestJob.progress.resultArtifactIngest.displayName", dataSource.getName())); + resultIngestProgressBar.setDisplayName(Bundle.IngestJob_progress_analysisResultIngest_displayName(dataSource.getName())); resultIngestProgressBar.progress(NbBundle.getMessage(getClass(), "IngestJob.progress.cancelling")); } }); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index 388f951276..1509902680 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -371,7 +371,6 @@ SolrSearchService.exceptionMessage.noCurrentSolrCore=IndexMetadata did not conta SolrSearchService.exceptionMessage.noIndexMetadata=Unable to create IndexMetaData from case directory: {0} # {0} - collection name SolrSearchService.exceptionMessage.unableToDeleteCollection=Unable to delete collection {0} -SolrSearchService.indexingError=Unable to index blackboard artifact. SolrSearchService.ServiceName=Solr Keyword Search Service SolrSearchService.DeleteDataSource.msg=Error Deleting Solr data for data source id {0} DropdownSingleTermSearchPanel.dataSourceCheckBox.text=Restrict search to the selected data sources: diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java index 8a68bae0c7..71e4c6d561 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java @@ -518,7 +518,9 @@ final class IngestSearchRunner { progressIndicator = ProgressHandle.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { - progressIndicator.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg")); + if (progressIndicator != null) { + progressIndicator.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg")); + } logger.log(Level.INFO, "Search cancelled by user"); //NON-NLS new Thread(() -> { IngestSearchRunner.Searcher.this.cancel(true); From 682a6d3aeda4b607f0e29d7d9961cf7572733e62 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 7 Dec 2021 23:04:27 -0500 Subject: [PATCH 16/19] 7529 KWS artifact ingest module --- .../autopsy/ingest/IngestJobExecutor.java | 43 ++++--------------- .../AdHocSearchChildFactory.java | 2 +- .../keywordsearch/IngestSearchRunner.java | 2 +- .../autopsy/keywordsearch/QueryResults.java | 6 +-- 4 files changed, 13 insertions(+), 40 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index b0f8b8c512..4a15bf2e4f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -508,12 +508,7 @@ final class IngestJobExecutor { * @return True or false. */ private boolean hasHighPriorityDataSourceIngestModules() { - jobStateLock.readLock().lock(); - try { - return (highPriorityDataSourceIngestPipeline.isEmpty() == false); - } finally { - jobStateLock.readLock().unlock(); - } + return (highPriorityDataSourceIngestPipeline.isEmpty() == false); } /** @@ -523,12 +518,7 @@ final class IngestJobExecutor { * @return True or false. */ private boolean hasLowPriorityDataSourceIngestModules() { - jobStateLock.readLock().lock(); - try { - return (lowPriorityDataSourceIngestPipeline.isEmpty() == false); - } finally { - jobStateLock.readLock().unlock(); - } + return (lowPriorityDataSourceIngestPipeline.isEmpty() == false); } /** @@ -537,15 +527,10 @@ final class IngestJobExecutor { * @return True or false. */ private boolean hasFileIngestModules() { - jobStateLock.readLock().lock(); - try { - if (!fileIngestPipelines.isEmpty()) { - return !fileIngestPipelines.get(0).isEmpty(); - } - return false; - } finally { - jobStateLock.readLock().unlock(); + if (!fileIngestPipelines.isEmpty()) { + return !fileIngestPipelines.get(0).isEmpty(); } + return false; } /** @@ -555,12 +540,7 @@ final class IngestJobExecutor { * @return True or false. */ private boolean hasDataArtifactIngestModules() { - jobStateLock.readLock().lock(); - try { - return (dataArtifactIngestPipeline.isEmpty() == false); - } finally { - jobStateLock.readLock().unlock(); - } + return (dataArtifactIngestPipeline.isEmpty() == false); } /** @@ -570,12 +550,7 @@ final class IngestJobExecutor { * @return True or false. */ private boolean hasAnalysisResultIngestModules() { - jobStateLock.readLock().lock(); - try { - return (analysisResultIngestPipeline.isEmpty() == false); - } finally { - jobStateLock.readLock().unlock(); - } + return (analysisResultIngestPipeline.isEmpty() == false); } /** @@ -1300,12 +1275,12 @@ final class IngestJobExecutor { void execute(AnalysisResultIngestTask task) { jobStateLock.readLock().lock(); try { - if (!isCancelled() && !analysisResultIngestPipeline.isEmpty()) { + if (!isCancelled() && !analysisResultIngestPipeline.isEmpty()) { List errors = new ArrayList<>(); errors.addAll(analysisResultIngestPipeline.performTask(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors); - } + } } } finally { taskScheduler.notifyTaskCompleted(task); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java index dc633e8795..c01a81ff66 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchChildFactory.java @@ -440,7 +440,7 @@ class AdHocSearchChildFactory extends ChildFactory { }); } registerWriter(this); //register (synchronized on class) outside of writerLock to prevent deadlock - hits.process(this, false, saveResults, saveResults, null); + hits.process(this, false, saveResults, null); } finally { deregisterWriter(this); if (RuntimeProperties.runningWithGUI() && progress != null) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java index 71e4c6d561..9cd33a8167 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java @@ -586,7 +586,7 @@ final class IngestSearchRunner { if (!newResults.getKeywords().isEmpty()) { // Create blackboard artifacts - newResults.process(this, keywordList.getIngestMessages(), true, finalRun, job.getJobId()); + newResults.process(this, keywordList.getIngestMessages(), true, job.getJobId()); } } } catch (Exception ex) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index 955e8ba278..5492f768d8 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -138,12 +138,10 @@ class QueryResults { * messages inbox if there is a keyword hit in the text * exrtacted from the text source object. * @param saveResults Whether or not to create keyword hit analysis results. - * @param postResults Whether or not to post any keyword hit analysis - * results created. * @param ingestJobId The numeric identifier of the ingest job within which * the artifacts are being created, may be null. */ - void process(SwingWorker worker, boolean notifyInbox, boolean saveResults, boolean postResults, Long ingestJobId) { + void process(SwingWorker worker, boolean notifyInbox, boolean saveResults, Long ingestJobId) { final Collection hitArtifacts = new ArrayList<>(); for (final Keyword keyword : getKeywords()) { /* @@ -219,7 +217,7 @@ class QueryResults { * Post the artifacts to the blackboard which will publish an event to * notify subscribers of the new artifacts. */ - if (!hitArtifacts.isEmpty() && postResults) { + if (!hitArtifacts.isEmpty()) { try { SleuthkitCase tskCase = Case.getCurrentCaseThrows().getSleuthkitCase(); Blackboard blackboard = tskCase.getBlackboard(); From dd39d31dd47e2b0c4aa0c8f616c4f9d6c1a69621 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 7 Dec 2021 23:22:22 -0500 Subject: [PATCH 17/19] 8213 Python modules context vars --- InternalPythonModules/android/browserlocation.py | 4 ++-- InternalPythonModules/android/cachelocation.py | 4 ++-- InternalPythonModules/android/viber.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/InternalPythonModules/android/browserlocation.py b/InternalPythonModules/android/browserlocation.py index ab0d52f709..c8c36ecff3 100644 --- a/InternalPythonModules/android/browserlocation.py +++ b/InternalPythonModules/android/browserlocation.py @@ -62,7 +62,7 @@ class BrowserLocationAnalyzer(general.AndroidComponentAnalyzer): try: jFile = File(Case.getCurrentCase().getTempDirectory(), str(abstractFile.getId()) + abstractFile.getName()) ContentUtils.writeToFile(abstractFile, jFile, context.dataSourceIngestIsCancelled) - self.__findGeoLocationsInDB(jFile.toString(), abstractFile) + self.__findGeoLocationsInDB(jFile.toString(), abstractFile, context) except Exception as ex: self._logger.log(Level.SEVERE, "Error parsing browser location files", ex) self._logger.log(Level.SEVERE, traceback.format_exc()) @@ -70,7 +70,7 @@ class BrowserLocationAnalyzer(general.AndroidComponentAnalyzer): # Error finding browser location files. pass - def __findGeoLocationsInDB(self, databasePath, abstractFile): + def __findGeoLocationsInDB(self, databasePath, abstractFile, context): if not databasePath: return diff --git a/InternalPythonModules/android/cachelocation.py b/InternalPythonModules/android/cachelocation.py index 15879e99b8..4938f930ce 100644 --- a/InternalPythonModules/android/cachelocation.py +++ b/InternalPythonModules/android/cachelocation.py @@ -66,7 +66,7 @@ class CacheLocationAnalyzer(general.AndroidComponentAnalyzer): try: jFile = File(Case.getCurrentCase().getTempDirectory(), str(abstractFile.getId()) + abstractFile.getName()) ContentUtils.writeToFile(abstractFile, jFile, context.dataSourceIngestIsCancelled) - self.__findGeoLocationsInFile(jFile, abstractFile) + self.__findGeoLocationsInFile(jFile, abstractFile, context) except Exception as ex: self._logger.log(Level.SEVERE, "Error parsing cached location files", ex) self._logger.log(Level.SEVERE, traceback.format_exc()) @@ -74,7 +74,7 @@ class CacheLocationAnalyzer(general.AndroidComponentAnalyzer): # Error finding cached location files. pass - def __findGeoLocationsInFile(self, file, abstractFile): + def __findGeoLocationsInFile(self, file, abstractFile, context): try: # code to parse the cache.wifi and cache.cell taken from https://forensics.spreitzenbarth.de/2011/10/28/decoding-cache-cell-and-cache-wifi-files/ diff --git a/InternalPythonModules/android/viber.py b/InternalPythonModules/android/viber.py index 9626f5d285..2aa18cdec1 100644 --- a/InternalPythonModules/android/viber.py +++ b/InternalPythonModules/android/viber.py @@ -92,7 +92,7 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): helper = CommunicationArtifactsHelper( current_case.getSleuthkitCase(), self._PARSER_NAME, contact_and_calllog_db.getDBFile(), Account.Type.VIBER, context.getJobId()) - self.parse_contacts(contact_and_calllog_db, helper) + self.parse_contacts(contact_and_calllog_db, helper, context) self.parse_calllogs(contact_and_calllog_db, helper) #Extract TSK_MESSAGE information @@ -113,7 +113,7 @@ class ViberAnalyzer(general.AndroidComponentAnalyzer): for contact_and_calllog_db in contact_and_calllog_dbs: contact_and_calllog_db.close() - def parse_contacts(self, contacts_db, helper): + def parse_contacts(self, contacts_db, helper, context): try: contacts_parser = ViberContactsParser(contacts_db) while contacts_parser.next(): From 3edb33b8f3142576b7147e20cfcf4f62a0c8045e Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 8 Dec 2021 10:07:15 -0500 Subject: [PATCH 18/19] 7529 no error msg for ignored tasks --- .../autopsy/ingest/IngestJobExecutor.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 4a15bf2e4f..01a38d7440 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -117,7 +117,7 @@ final class IngestJobExecutor { private AnalysisResultIngestPipeline analysisResultIngestPipeline; /* - * An ingest job transistion through several states during its execution. + * An ingest job transistions through several states during its execution. */ private static enum IngestJobState { /* @@ -1357,10 +1357,29 @@ final class IngestJobExecutor { */ void addAnalysisResults(List results) { if (!isCancelled() && hasAnalysisResultIngestModules()) { - if (jobState.equals(IngestJobState.STREAMED_FILE_ANALYSIS_ONLY) || jobState.equals(IngestJobState.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS) || jobState.equals(IngestJobState.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS)) { - taskScheduler.scheduleAnalysisResultIngestTasks(this, results); - } else { - logErrorMessage(Level.SEVERE, "Attempt to add analysis results to job during stage " + jobState.toString() + " not supported"); + switch (jobState) { + case STREAMED_FILE_ANALYSIS_ONLY: + case FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS: + case LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS: + taskScheduler.scheduleAnalysisResultIngestTasks(this, results); + break; + case PIPELINES_SHUTTING_DOWN: + /* + * Don't log an error if there is an attempt to add an + * analysis result ingest task in a pipeline shut down + * state. This is a work around for dealing with analysis + * results generated by a final keyword search carried out + * during ingest module shut down by simply ignoring them. + * Other ideas were to add a startShutDown() phase to the + * ingest module life cycle (complicated), or to add a flag + * to keyword hit processing to suppress posting the keyword + * hit analysis results to the blackboard during a final + * search (API changes required to allow firing of the event + * to make any GUI refresh). + */ + break; + default: + logErrorMessage(Level.SEVERE, "Attempt to add analysis results to job during stage " + jobState.toString() + " not supported"); } } } From 5a397163391532d3d67a613686696e0aa4920008 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 10 Dec 2021 09:45:49 -0500 Subject: [PATCH 19/19] logic fix --- .../org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java index 8804560afa..f4e856b82e 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/ViewsDAO.java @@ -828,17 +828,17 @@ public class ViewsDAO extends AbstractDAO { dsId = af.getDataSourceObjectId(); // create an extension mapping if extension present - if (StringUtils.isBlank(af.getNameExtension()) || !TSK_FS_NAME_TYPE_ENUM.REG.equals(af.getDirType())) { + if (!StringUtils.isBlank(af.getNameExtension()) && TSK_FS_NAME_TYPE_ENUM.REG.equals(af.getDirType())) { evtExtFilters = EXTENSION_FILTER_MAP.getOrDefault("." + af.getNameExtension(), Collections.emptySet()); } // create a mime type mapping if mime type present - if (StringUtils.isBlank(af.getMIMEType()) || !TSK_FS_NAME_TYPE_ENUM.REG.equals(af.getDirType()) || !getMimeDbFilesTypes().contains(af.getType())) { + if (!StringUtils.isBlank(af.getMIMEType()) && TSK_FS_NAME_TYPE_ENUM.REG.equals(af.getDirType()) && getMimeDbFilesTypes().contains(af.getType())) { evtMimeType = af.getMIMEType(); } // create a size mapping if size present in filters - if (TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS.equals(af.getType())) { + if (!TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS.equals(af.getType())) { evtFileSize = Stream.of(FileSizeFilter.values()) .filter(filter -> af.getSize() >= filter.getMinBound() && (filter.getMaxBound() == null || af.getSize() < filter.getMaxBound())) .findFirst()