7529 KWS artifact ingest module

This commit is contained in:
Richard Cordovano 2021-12-01 13:09:29 -05:00
commit 1071446a87
11 changed files with 700 additions and 625 deletions

View File

@ -1,7 +1,7 @@
CTL_RunIngestAction=Run Ingest
FileIngestPipeline_SaveResults_Activity=Saving Results
# {0} - data source name
IngestJob.progress.analysisResultIngest.displayName=Analyzing analysis results from {0}
IngestJob_progress_analysisResultIngest_displayName=Analyzing analysis results from {0}
IngestJobSettingsPanel.IngestModulesTableRenderer.info.message=A previous version of this ingest module has been run before on this data source.
IngestJobSettingsPanel.IngestModulesTableRenderer.warning.message=This ingest module has been run before on this data source.
IngestJobSettingsPanel.noPerRunSettings=The selected module has no per-run settings.

View File

@ -181,10 +181,10 @@ public final class IngestJob {
* Starts data source level analysis for this job if it is running in
* streaming ingest mode.
*/
void processStreamingIngestDataSource() {
void addStreamedDataSource() {
if (ingestMode == Mode.STREAMING) {
if (ingestModuleExecutor != null) {
ingestModuleExecutor.startStreamingModeDataSourceAnalysis();
ingestModuleExecutor.addStreamedDataSource();
} else {
logger.log(Level.SEVERE, "Attempted to start data source analaysis with no ingest pipeline");
}

File diff suppressed because it is too large Load Diff

View File

@ -67,7 +67,7 @@ class IngestJobInputStream implements IngestStream {
@Override
public synchronized void close() {
closed = true;
ingestJob.processStreamingIngestDataSource();
ingestJob.addStreamedDataSource();
}
@Override

View File

@ -1021,7 +1021,7 @@ public class IngestManager implements IngestProgressSnapshotProvider {
}
/**
* Creates and starts an ingest job for a collection of data sources.
* Creates and starts an ingest job.
*/
private final class StartIngestJobTask implements Callable<Void> {

View File

@ -134,7 +134,7 @@ class GPXParserFileIngestModule(FileIngestModule):
# Create a GeoArtifactsHelper for this file.
geoArtifactHelper = GeoArtifactsHelper(
self.skCase, self.moduleName, None, file, context.getJobId())
self.skCase, self.moduleName, None, file, self.context.getJobId())
if self.writeDebugMsgs:
self.log(Level.INFO, "Processing " + file.getUniquePath() +
@ -213,7 +213,7 @@ class GPXParserFileIngestModule(FileIngestModule):
art = file.newDataArtifact(BlackboardArtifact.Type(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_BOOKMARK), attributes)
self.blackboard.postArtifact(art, self.moduleName, context.getJobId())
self.blackboard.postArtifact(art, self.moduleName, self.context.getJobId())
except Blackboard.BlackboardException as e:
self.log(Level.SEVERE, "Error posting GPS bookmark artifact for " +

View File

@ -31,15 +31,18 @@ import java.util.concurrent.ExecutionException;
import java.util.logging.Level;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.swing.SwingUtilities;
import javax.swing.SwingWorker;
import org.netbeans.api.progress.ProgressHandle;
import org.openide.nodes.ChildFactory;
import org.openide.nodes.Children;
import org.openide.nodes.Node;
import org.openide.util.Cancellable;
import org.openide.util.NbBundle;
import org.openide.util.lookup.Lookups;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.core.RuntimeProperties;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode;
@ -89,7 +92,8 @@ class AdHocSearchChildFactory extends ChildFactory<KeyValue> {
* Constructor
*
* @param queryRequests Query results
* @param saveResults Flag whether to save search results as KWS artifacts.
* @param saveResults Flag whether to save search results as KWS
* artifacts.
*/
AdHocSearchChildFactory(Collection<AdHocQueryRequest> queryRequests, boolean saveResults) {
this.queryRequests = queryRequests;
@ -204,7 +208,6 @@ class AdHocSearchChildFactory extends ChildFactory<KeyValue> {
properties.put(LOCATION.toString(), contentName);
}
String hitName;
BlackboardArtifact artifact = null;
if (hit.isArtifactHit()) {
@ -414,21 +417,35 @@ class AdHocSearchChildFactory extends ChildFactory<KeyValue> {
this.saveResults = saveResults;
}
protected void finalizeWorker() {
deregisterWriter(this);
EventQueue.invokeLater(progress::finish);
}
@Override
protected Void doInBackground() throws Exception {
registerWriter(this); //register (synchronized on class) outside of writerLock to prevent deadlock
try {
if (RuntimeProperties.runningWithGUI()) {
final String queryStr = query.getQueryString();
final String queryDisp = queryStr.length() > QUERY_DISPLAY_LEN ? queryStr.substring(0, QUERY_DISPLAY_LEN - 1) + " ..." : queryStr;
try {
progress = ProgressHandle.createHandle(NbBundle.getMessage(this.getClass(), "KeywordSearchResultFactory.progress.saving", queryDisp), () -> BlackboardResultWriter.this.cancel(true));
hits.process(progress, null, this, false, saveResults, null);
SwingUtilities.invokeLater(() -> {
progress = ProgressHandle.createHandle(
NbBundle.getMessage(this.getClass(), "KeywordSearchResultFactory.progress.saving", queryDisp),
new Cancellable() {
@Override
public boolean cancel() {
//progress.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg"));
logger.log(Level.INFO, "Ad hoc search cancelled by user"); //NON-NLS
new Thread(() -> {
BlackboardResultWriter.this.cancel(true);
}).start();
return true;
}
});
});
}
registerWriter(this); //register (synchronized on class) outside of writerLock to prevent deadlock
hits.process(this, false, saveResults, null);
} finally {
finalizeWorker();
deregisterWriter(this);
if (RuntimeProperties.runningWithGUI() && progress != null) {
EventQueue.invokeLater(progress::finish);
}
}
return null;
}

View File

@ -38,15 +38,15 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import javax.swing.SwingUtilities;
import javax.swing.SwingWorker;
import org.netbeans.api.progress.aggregate.AggregateProgressFactory;
import org.netbeans.api.progress.aggregate.AggregateProgressHandle;
import org.netbeans.api.progress.aggregate.ProgressContributor;
import org.netbeans.api.progress.ProgressHandle;
import org.openide.util.Cancellable;
import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.core.RuntimeProperties;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.coreutils.StopWatch;
import org.sleuthkit.autopsy.coreutils.ThreadConfined;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestServices;
@ -447,13 +447,14 @@ final class IngestSearchRunner {
/**
* Searcher has private copies/snapshots of the lists and keywords
*/
private SearchJobInfo job;
private List<Keyword> keywords; //keywords to search
private List<String> keywordListNames; // lists currently being searched
private List<KeywordList> keywordLists;
private Map<Keyword, KeywordList> keywordToList; //keyword to list name mapping
private AggregateProgressHandle progressGroup;
private final Logger logger = Logger.getLogger(IngestSearchRunner.Searcher.class.getName());
private final SearchJobInfo job;
private final List<Keyword> keywords; //keywords to search
private final List<String> keywordListNames; // lists currently being searched
private final List<KeywordList> keywordLists;
private final Map<Keyword, KeywordList> keywordToList; //keyword to list name mapping
private final boolean usingNetBeansGUI;
@ThreadConfined(type = ThreadConfined.ThreadType.AWT)
private ProgressHandle progressIndicator;
private boolean finalRun = false;
Searcher(SearchJobInfo job) {
@ -463,6 +464,7 @@ final class IngestSearchRunner {
keywordToList = new HashMap<>();
keywordLists = new ArrayList<>();
//keywords are populated as searcher runs
usingNetBeansGUI = RuntimeProperties.runningWithGUI();
}
Searcher(SearchJobInfo job, boolean finalRun) {
@ -473,77 +475,86 @@ final class IngestSearchRunner {
@Override
@Messages("SearchRunner.query.exception.msg=Error performing query:")
protected Object doInBackground() throws Exception {
final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName")
+ (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : "");
final String pgDisplayName = displayName + (" (" + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") + ")");
progressGroup = AggregateProgressFactory.createSystemHandle(pgDisplayName, null, new Cancellable() {
@Override
public boolean cancel() {
logger.log(Level.INFO, "Cancelling the searcher by user."); //NON-NLS
if (progressGroup != null) {
progressGroup.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg"));
}
progressGroup.finish();
return IngestSearchRunner.Searcher.this.cancel(true);
}
}, null);
updateKeywords();
ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()];
int i = 0;
for (Keyword keywordQuery : keywords) {
subProgresses[i] = AggregateProgressFactory.createProgressContributor(keywordQuery.getSearchTerm());
progressGroup.addContributor(subProgresses[i]);
i++;
}
progressGroup.start();
final StopWatch stopWatch = new StopWatch();
stopWatch.start();
try {
progressGroup.setDisplayName(displayName);
int keywordsSearched = 0;
if (usingNetBeansGUI) {
/*
* If running in the NetBeans thick client application
* version of Autopsy, NetBeans progress handles (i.e.,
* progress bars) are used to display search progress in the
* lower right hand corner of the main application window.
*
* A layer of abstraction to allow alternate representations
* of progress could be used here, as it is in other places
* in the application (see implementations and usage of
* org.sleuthkit.autopsy.progress.ProgressIndicator
* interface), to better decouple keyword search from the
* application's presentation layer.
*/
SwingUtilities.invokeAndWait(() -> {
final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName")
+ (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : "");
progressIndicator = ProgressHandle.createHandle(displayName, new Cancellable() {
@Override
public boolean cancel() {
progressIndicator.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg"));
logger.log(Level.INFO, "Search cancelled by user"); //NON-NLS
new Thread(() -> {
IngestSearchRunner.Searcher.this.cancel(true);
}).start();
return true;
}
});
progressIndicator.start();
progressIndicator.switchToIndeterminate();
});
}
updateKeywords();
for (Keyword keyword : keywords) {
if (this.isCancelled() || this.job.getJobContext().fileIngestIsCancelled()) {
logger.log(Level.INFO, "Cancel detected, bailing before new keyword processed: {0}", keyword.getSearchTerm()); //NON-NLS
if (isCancelled() || job.getJobContext().fileIngestIsCancelled()) {
logger.log(Level.INFO, "Cancellation requested, exiting before new keyword processed: {0}", keyword.getSearchTerm()); //NON-NLS
return null;
}
logger.log(Level.INFO, String.format("Performing keyword query for search job %d: %s", job.getJobId(), keyword.getSearchTerm())); //NON-NLS
final KeywordList keywordList = keywordToList.get(keyword);
//new subProgress will be active after the initial query
//when we know number of hits to start() with
if (keywordsSearched > 0) {
subProgresses[keywordsSearched - 1].finish();
KeywordList keywordList = keywordToList.get(keyword);
if (usingNetBeansGUI) {
String searchTermStr = keyword.getSearchTerm();
if (searchTermStr.length() > 50) {
searchTermStr = searchTermStr.substring(0, 49) + "...";
}
final String progressMessage = keywordList.getName() + ": " + searchTermStr;
SwingUtilities.invokeLater(() -> {
progressIndicator.progress(progressMessage);
});
}
KeywordSearchQuery keywordSearchQuery = KeywordSearchUtil.getQueryForKeyword(keyword, keywordList);
// Filtering
//limit search to currently ingested data sources
//set up a filter with 1 or more image ids OR'ed
final KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, job.getDataSourceId());
KeywordSearchQuery keywordSearchQuery = KeywordSearchUtil.getQueryForKeyword(keyword, keywordList);
KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, job.getDataSourceId());
keywordSearchQuery.addFilter(dataSourceFilter);
QueryResults queryResults;
// Do the actual search
QueryResults queryResults;
try {
queryResults = keywordSearchQuery.performQuery();
} catch (KeywordSearchModuleException | NoOpenCoreException ex) {
logger.log(Level.SEVERE, "Error performing query: " + keyword.getSearchTerm(), ex); //NON-NLS
MessageNotifyUtil.Notify.error(Bundle.SearchRunner_query_exception_msg() + keyword.getSearchTerm(), ex.getCause().getMessage());
if (usingNetBeansGUI) {
final String userMessage = Bundle.SearchRunner_query_exception_msg() + keyword.getSearchTerm();
SwingUtilities.invokeLater(() -> {
MessageNotifyUtil.Notify.error(userMessage, ex.getCause().getMessage());
});
}
//no reason to continue with next query if recovery failed
//or wait for recovery to kick in and run again later
//likely case has closed and threads are being interrupted
return null;
} catch (CancellationException e) {
logger.log(Level.INFO, "Cancel detected, bailing during keyword query: {0}", keyword.getSearchTerm()); //NON-NLS
logger.log(Level.INFO, "Cancellation requested, exiting during keyword query: {0}", keyword.getSearchTerm()); //NON-NLS
return null;
}
@ -552,43 +563,27 @@ final class IngestSearchRunner {
QueryResults newResults = filterResults(queryResults);
if (!newResults.getKeywords().isEmpty()) {
// Write results to BB
//scale progress bar more more granular, per result sub-progress, within per keyword
int totalUnits = newResults.getKeywords().size();
subProgresses[keywordsSearched].start(totalUnits);
int unitProgress = 0;
String queryDisplayStr = keyword.getSearchTerm();
if (queryDisplayStr.length() > 50) {
queryDisplayStr = queryDisplayStr.substring(0, 49) + "...";
}
subProgresses[keywordsSearched].progress(keywordList.getName() + ": " + queryDisplayStr, unitProgress);
// Create blackboard artifacts
newResults.process(null, subProgresses[keywordsSearched], this, keywordList.getIngestMessages(), true, job.getJobId());
} //if has results
//reset the status text before it goes away
subProgresses[keywordsSearched].progress("");
++keywordsSearched;
} //for each keyword
} //end try block
catch (Exception ex) {
logger.log(Level.WARNING, "searcher exception occurred", ex); //NON-NLS
newResults.process(this, keywordList.getIngestMessages(), true, job.getJobId());
}
}
} catch (Exception ex) {
logger.log(Level.WARNING, "Error occurred during keyword search", ex); //NON-NLS
} finally {
try {
finalizeSearcher();
if (progressIndicator != null) {
SwingUtilities.invokeLater(new Runnable() {
@Override
public void run() {
progressIndicator.finish();
progressIndicator = null;
}
});
}
stopWatch.stop();
logger.log(Level.INFO, "Searcher took {0} secs to run (final = {1})", new Object[]{stopWatch.getElapsedTimeSecs(), this.finalRun}); //NON-NLS
} finally {
// In case a thread is waiting on this worker to be done
job.searchNotify();
}
}
return null;
}
@ -613,20 +608,6 @@ final class IngestSearchRunner {
}
}
/**
* Performs the cleanup that needs to be done right AFTER
* doInBackground() returns without relying on done() method that is not
* guaranteed to run.
*/
private void finalizeSearcher() {
SwingUtilities.invokeLater(new Runnable() {
@Override
public void run() {
progressGroup.finish();
}
});
}
/**
* This method filters out all of the hits found in earlier periodic
* searches and returns only the results found by the most recent

View File

@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import javax.swing.SwingUtilities;
import javax.swing.SwingWorker;
import org.apache.commons.lang.StringUtils;
import org.netbeans.api.progress.ProgressHandle;
@ -32,6 +33,7 @@ import org.netbeans.api.progress.aggregate.ProgressContributor;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.core.RuntimeProperties;
import org.sleuthkit.autopsy.coreutils.EscapeUtil;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.ingest.IngestMessage;
@ -51,6 +53,8 @@ import org.sleuthkit.datamodel.TskCoreException;
* about the search hits to the ingest inbox, and publishing an event to notify
* subscribers of the blackboard posts.
*/
class QueryResults {
private static final Logger logger = Logger.getLogger(QueryResults.class.getName());
@ -131,10 +135,6 @@ class QueryResults {
* All calls to the addResult method MUST be completed before calling this
* method.
*
* @param progress A progress indicator that reports the number of
* keywords processed. Can be null.
* @param subProgress A progress contributor that reports the keyword
* currently being processed. Can be null.
* @param worker The SwingWorker that is being used to do the
* processing, will be checked for task cancellation
* before processing each keyword.
@ -145,19 +145,7 @@ class QueryResults {
* @param ingestJobId The numeric identifier of the ingest job within which
* the artifacts are being created, may be null.
*/
void process(ProgressHandle progress, ProgressContributor subProgress, SwingWorker<?, ?> worker, boolean notifyInbox, boolean saveResults, Long ingestJobId) {
/*
* Initialize the progress indicator to the number of keywords that will
* be processed.
*/
if (null != progress) {
progress.start(getKeywords().size());
}
/*
* Process the keyword hits for each keyword.
*/
int keywordsProcessed = 0;
void process(SwingWorker<?, ?> worker, boolean notifyInbox, boolean saveResults, Long ingestJobId) {
final Collection<BlackboardArtifact> hitArtifacts = new ArrayList<>();
for (final Keyword keyword : getKeywords()) {
/*
@ -165,22 +153,7 @@ class QueryResults {
*/
if (worker.isCancelled()) {
logger.log(Level.INFO, "Processing cancelled, exiting before processing search term {0}", keyword.getSearchTerm()); //NON-NLS
break;
}
/*
* Update the progress indicator and the show the current keyword
* via the progress contributor.
*/
if (progress != null) {
progress.progress(keyword.toString(), keywordsProcessed);
}
if (subProgress != null) {
String hitDisplayStr = keyword.getSearchTerm();
if (hitDisplayStr.length() > 50) {
hitDisplayStr = hitDisplayStr.substring(0, 49) + "...";
}
subProgress.progress(query.getKeywordList().getName() + ": " + hitDisplayStr, keywordsProcessed);
return;
}
/*
@ -202,7 +175,7 @@ class QueryResults {
snippet = LuceneQuery.querySnippet(snippetQuery, hit.getSolrObjectId(), hit.getChunkId(), !query.isLiteral(), true);
} catch (NoOpenCoreException e) {
logger.log(Level.SEVERE, "Solr core closed while executing snippet query " + snippetQuery, e); //NON-NLS
break; // Stop processing.
return; // Stop processing.
} catch (Exception e) {
logger.log(Level.SEVERE, "Error executing snippet query " + snippetQuery, e); //NON-NLS
continue; // Try processing the next hit.
@ -242,8 +215,6 @@ class QueryResults {
}
}
}
++keywordsProcessed;
}
/*
@ -298,15 +269,16 @@ class QueryResults {
* @throws TskCoreException If there is a problem generating or send the
* inbox message.
*/
private void writeSingleFileInboxMessage(BlackboardArtifact artifact, Content hitContent) throws TskCoreException {
StringBuilder subjectSb = new StringBuilder(1024);
private void writeSingleFileInboxMessage(final BlackboardArtifact artifact, final Content hitContent) throws TskCoreException {
if (artifact != null && hitContent != null && RuntimeProperties.runningWithGUI()) {
final StringBuilder subjectSb = new StringBuilder(1024);
if (!query.isLiteral()) {
subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExpHitLbl"));
} else {
subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLbl"));
}
StringBuilder detailsSb = new StringBuilder(1024);
final StringBuilder detailsSb = new StringBuilder(1024);
String uniqueKey = null;
BlackboardAttribute attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD));
if (attr != null) {
@ -361,6 +333,10 @@ class QueryResults {
}
detailsSb.append("</table>"); //NON-NLS
IngestServices.getInstance().postMessage(IngestMessage.createDataMessage(MODULE_NAME, subjectSb.toString(), detailsSb.toString(), uniqueKey, artifact));
final String key = uniqueKey; // Might be null, but that's supported.
SwingUtilities.invokeLater(() -> {
IngestServices.getInstance().postMessage(IngestMessage.createDataMessage(MODULE_NAME, subjectSb.toString(), detailsSb.toString(), key, artifact));
});
}
}
}

View File

@ -1,5 +1,5 @@
#Updated by build script
#Fri, 19 Nov 2021 17:01:30 -0500
#Wed, 01 Dec 2021 12:53:03 -0500
LBL_splash_window_title=Starting Autopsy
SPLASH_HEIGHT=314
SPLASH_WIDTH=538

View File

@ -1,4 +1,4 @@
#Updated by build script
#Fri, 19 Nov 2021 17:01:30 -0500
#Wed, 01 Dec 2021 12:53:03 -0500
CTL_MainWindow_Title=Autopsy 4.19.2
CTL_MainWindow_Title_No_Project=Autopsy 4.19.2