mirror of
https://github.com/overcuriousity/autopsy-flatpak.git
synced 2025-07-15 09:17:42 +00:00
Remove stuff from KeywordSearchIngestModule that is now in SearchRunner, and add calls to SearchRunner.
This commit is contained in:
Samuel H. Kenyon
parent
3ebe81c6fa
commit
bafbd19ade
@ -98,20 +98,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
|
|||||||
private static final Logger logger = Logger.getLogger(KeywordSearchIngestModule.class.getName());
|
private static final Logger logger = Logger.getLogger(KeywordSearchIngestModule.class.getName());
|
||||||
private IngestServices services = IngestServices.getInstance();
|
private IngestServices services = IngestServices.getInstance();
|
||||||
private Ingester ingester = null;
|
private Ingester ingester = null;
|
||||||
private volatile boolean commitIndex = false; //whether to commit index next time
|
|
||||||
private volatile boolean runSearcher = false; //whether to run searcher next time
|
|
||||||
private Timer commitTimer;
|
|
||||||
private Timer searchTimer;
|
|
||||||
private Indexer indexer;
|
private Indexer indexer;
|
||||||
private Searcher currentSearcher;
|
|
||||||
private Searcher finalSearcher;
|
|
||||||
private volatile boolean searcherDone = true; //mark as done, until it's inited
|
|
||||||
private Map<Keyword, List<Long>> currentResults;
|
|
||||||
//only search images from current ingest, not images previously ingested/indexed
|
//only search images from current ingest, not images previously ingested/indexed
|
||||||
//accessed read-only by searcher thread
|
//accessed read-only by searcher thread
|
||||||
private Set<Long> curDataSourceIds;
|
|
||||||
private static final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); //use fairness policy
|
|
||||||
private static final Lock searcherLock = rwLock.writeLock();
|
|
||||||
private AtomicInteger messageID = new AtomicInteger(0);
|
private AtomicInteger messageID = new AtomicInteger(0);
|
||||||
private boolean processedFiles;
|
private boolean processedFiles;
|
||||||
private SleuthkitCase caseHandle = null;
|
private SleuthkitCase caseHandle = null;
|
||||||
@ -120,7 +109,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
|
|||||||
private final KeywordSearchJobSettings settings;
|
private final KeywordSearchJobSettings settings;
|
||||||
private boolean initialized = false;
|
private boolean initialized = false;
|
||||||
private Tika tikaFormatDetector;
|
private Tika tikaFormatDetector;
|
||||||
private long jobId = 0; ///@todo where does jobID come from?
|
private long jobId = 0; ///@todo get from IngestJobContext
|
||||||
|
private long dataSourceId;
|
||||||
|
|
||||||
private enum IngestStatus {
|
private enum IngestStatus {
|
||||||
|
|
||||||
@ -213,11 +203,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
|
|||||||
}
|
}
|
||||||
|
|
||||||
processedFiles = false;
|
processedFiles = false;
|
||||||
searcherDone = true; //make sure to start the initial currentSearcher
|
|
||||||
//keeps track of all results per run not to repeat reporting the same hits
|
|
||||||
currentResults = new HashMap<>();
|
|
||||||
|
|
||||||
curDataSourceIds = new HashSet<>();
|
|
||||||
|
|
||||||
indexer = new Indexer();
|
indexer = new Indexer();
|
||||||
|
|
||||||
@ -225,13 +210,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
|
|||||||
logger.log(Level.INFO, "Using commit interval (ms): {0}", updateIntervalMs);
|
logger.log(Level.INFO, "Using commit interval (ms): {0}", updateIntervalMs);
|
||||||
logger.log(Level.INFO, "Using searcher interval (ms): {0}", updateIntervalMs);
|
logger.log(Level.INFO, "Using searcher interval (ms): {0}", updateIntervalMs);
|
||||||
|
|
||||||
commitTimer = new Timer(updateIntervalMs, new CommitTimerAction());
|
|
||||||
searchTimer = new Timer(updateIntervalMs, new SearchTimerAction());
|
|
||||||
|
|
||||||
initialized = true;
|
initialized = true;
|
||||||
|
|
||||||
commitTimer.start();
|
|
||||||
searchTimer.start();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -245,8 +224,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
|
|||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
//add data source id of the file to the set, keeping track of images being ingested
|
//add data source id of the file to the set, keeping track of images being ingested
|
||||||
final long fileSourceId = caseHandle.getFileDataSource(abstractFile);
|
///@todo this should come from IngestJobContext
|
||||||
curDataSourceIds.add(fileSourceId);
|
dataSourceId = caseHandle.getFileDataSource(abstractFile);
|
||||||
|
|
||||||
} catch (TskCoreException ex) {
|
} catch (TskCoreException ex) {
|
||||||
logger.log(Level.SEVERE, "Error getting image id of file processed by keyword search: " + abstractFile.getName(), ex);
|
logger.log(Level.SEVERE, "Error getting image id of file processed by keyword search: " + abstractFile.getName(), ex);
|
||||||
@ -265,12 +244,13 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
|
|||||||
|
|
||||||
processedFiles = true;
|
processedFiles = true;
|
||||||
|
|
||||||
//check if it's time to commit after previous processing
|
|
||||||
checkRunCommitSearch();
|
|
||||||
|
|
||||||
//index the file and content (if the content is supported)
|
//index the file and content (if the content is supported)
|
||||||
indexer.indexFile(abstractFile, true);
|
indexer.indexFile(abstractFile, true);
|
||||||
|
|
||||||
|
// Start searching if it hasn't started already
|
||||||
|
List<String> keywordListNames = settings.getNamesOfEnabledKeyWordLists();
|
||||||
|
SearchRunner.getInstance().startJob(jobId, dataSourceId, keywordListNames);
|
||||||
|
|
||||||
return ProcessResult.OK;
|
return ProcessResult.OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -288,34 +268,12 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
|
|||||||
stop();
|
stop();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
//
|
|
||||||
// commitTimer.stop();
|
|
||||||
//
|
|
||||||
// //NOTE, we let the 1 before last searcher complete fully, and enqueue the last one
|
|
||||||
//
|
|
||||||
// //cancel searcher timer, ensure unwanted searcher does not start
|
|
||||||
// //before we start the final one
|
|
||||||
// if (searchTimer.isRunning()) {
|
|
||||||
// searchTimer.stop();
|
|
||||||
// }
|
|
||||||
// runSearcher = false;
|
|
||||||
//
|
|
||||||
// logger.log(Level.INFO, "Running final index commit and search");
|
|
||||||
// //final commit
|
|
||||||
// commit();
|
|
||||||
|
|
||||||
// Remove from the search list and trigger final commit and final search
|
// Remove from the search list and trigger final commit and final search
|
||||||
SearchRunner.getInstance().endJob(jobId);
|
SearchRunner.getInstance().endJob(jobId);
|
||||||
|
|
||||||
postIndexSummary();
|
postIndexSummary();
|
||||||
|
|
||||||
//run one last search as there are probably some new files committed
|
|
||||||
// List<String> keywordLists = settings.getNamesOfEnabledKeyWordLists();
|
|
||||||
// if (!keywordLists.isEmpty() && processedFiles == true) {
|
|
||||||
// finalSearcher = new Searcher(keywordLists, true); //final searcher run
|
|
||||||
// finalSearcher.execute();
|
|
||||||
// }
|
|
||||||
|
|
||||||
//log number of files / chunks in index
|
//log number of files / chunks in index
|
||||||
//signal a potential change in number of text_ingested files
|
//signal a potential change in number of text_ingested files
|
||||||
try {
|
try {
|
||||||
@ -326,8 +284,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
|
|||||||
} catch (NoOpenCoreException | KeywordSearchModuleException ex) {
|
} catch (NoOpenCoreException | KeywordSearchModuleException ex) {
|
||||||
logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files/chunks: ", ex);
|
logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files/chunks: ", ex);
|
||||||
}
|
}
|
||||||
|
|
||||||
//cleanup done in final searcher
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -336,21 +292,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
|
|||||||
private void stop() {
|
private void stop() {
|
||||||
logger.log(Level.INFO, "stop()");
|
logger.log(Level.INFO, "stop()");
|
||||||
|
|
||||||
//stop timer
|
SearchRunner.getInstance().stopJob(jobId);
|
||||||
commitTimer.stop();
|
|
||||||
//stop currentSearcher
|
|
||||||
if (currentSearcher != null) {
|
|
||||||
currentSearcher.cancel(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
//cancel searcher timer, ensure unwanted searcher does not start
|
|
||||||
if (searchTimer.isRunning()) {
|
|
||||||
searchTimer.stop();
|
|
||||||
}
|
|
||||||
runSearcher = false;
|
|
||||||
|
|
||||||
//commit uncommited files, don't search again
|
|
||||||
commit();
|
|
||||||
|
|
||||||
cleanup();
|
cleanup();
|
||||||
}
|
}
|
||||||
@ -360,15 +302,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
|
|||||||
*/
|
*/
|
||||||
private void cleanup() {
|
private void cleanup() {
|
||||||
ingestStatus.clear();
|
ingestStatus.clear();
|
||||||
currentResults.clear();
|
|
||||||
curDataSourceIds.clear();
|
|
||||||
currentSearcher = null;
|
|
||||||
//finalSearcher = null; //do not collect, might be finalizing
|
|
||||||
|
|
||||||
commitTimer.stop();
|
|
||||||
searchTimer.stop();
|
|
||||||
commitTimer = null;
|
|
||||||
//searchTimer = null; // do not collect, final searcher might still be running, in which case it throws an exception
|
|
||||||
|
|
||||||
textExtractors.clear();
|
textExtractors.clear();
|
||||||
textExtractors = null;
|
textExtractors = null;
|
||||||
@ -379,19 +312,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
|
|||||||
initialized = false;
|
initialized = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Commits index and notifies listeners of index update
|
|
||||||
*/
|
|
||||||
private void commit() {
|
|
||||||
if (initialized) {
|
|
||||||
logger.log(Level.INFO, "Commiting index");
|
|
||||||
ingester.commit();
|
|
||||||
logger.log(Level.INFO, "Index comitted");
|
|
||||||
//signal a potential change in number of text_ingested files
|
|
||||||
indexChangeNotify();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Posts inbox message with summary of text_ingested files
|
* Posts inbox message with summary of text_ingested files
|
||||||
*/
|
*/
|
||||||
@ -423,7 +343,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
|
|||||||
error_io++;
|
error_io++;
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
;
|
;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -447,73 +367,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper method to notify listeners on index update
|
|
||||||
*/
|
|
||||||
private void indexChangeNotify() {
|
|
||||||
//signal a potential change in number of text_ingested files
|
|
||||||
try {
|
|
||||||
final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles();
|
|
||||||
KeywordSearch.fireNumIndexedFilesChange(null, new Integer(numIndexedFiles));
|
|
||||||
} catch (NoOpenCoreException | KeywordSearchModuleException ex) {
|
|
||||||
logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files: ", ex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if time to commit, if so, run commit. Then run search if search
|
|
||||||
* timer is also set.
|
|
||||||
*/
|
|
||||||
void checkRunCommitSearch() {
|
|
||||||
if (commitIndex) {
|
|
||||||
logger.log(Level.INFO, "Commiting index");
|
|
||||||
commit();
|
|
||||||
commitIndex = false;
|
|
||||||
|
|
||||||
//after commit, check if time to run searcher
|
|
||||||
//NOTE commit/searcher timings don't need to align
|
|
||||||
//in worst case, we will run search next time after commit timer goes off, or at the end of ingest
|
|
||||||
if (searcherDone && runSearcher) {
|
|
||||||
//start search if previous not running
|
|
||||||
List<String> keywordLists = settings.getNamesOfEnabledKeyWordLists();
|
|
||||||
if (!keywordLists.isEmpty()) {
|
|
||||||
currentSearcher = new Searcher(keywordLists);
|
|
||||||
currentSearcher.execute();//searcher will stop timer and restart timer when done
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* CommitTimerAction to run by commitTimer Sets a flag to indicate we are
|
|
||||||
* ready for commit
|
|
||||||
*/
|
|
||||||
private class CommitTimerAction implements ActionListener {
|
|
||||||
|
|
||||||
private final Logger logger = Logger.getLogger(CommitTimerAction.class.getName());
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void actionPerformed(ActionEvent e) {
|
|
||||||
commitIndex = true;
|
|
||||||
logger.log(Level.INFO, "CommitTimer awake");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* SearchTimerAction to run by searchTimer Sets a flag to indicate we are
|
|
||||||
* ready to search
|
|
||||||
*/
|
|
||||||
private class SearchTimerAction implements ActionListener {
|
|
||||||
|
|
||||||
private final Logger logger = Logger.getLogger(SearchTimerAction.class.getName());
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void actionPerformed(ActionEvent e) {
|
|
||||||
runSearcher = true;
|
|
||||||
logger.log(Level.INFO, "SearchTimer awake");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* File indexer, processes and indexes known/allocated files,
|
* File indexer, processes and indexes known/allocated files,
|
||||||
* unknown/unallocated files and directories accordingly
|
* unknown/unallocated files and directories accordingly
|
||||||
@ -695,420 +548,4 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Searcher responsible for searching the current index and writing results
|
|
||||||
* to blackboard and the inbox. Also, posts results to listeners as Ingest
|
|
||||||
* data events. Searches entire index, and keeps track of only new results
|
|
||||||
* to report and save. Runs as a background thread.
|
|
||||||
*/
|
|
||||||
private final class Searcher extends SwingWorker<Object, Void> {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Searcher has private copies/snapshots of the lists and keywords
|
|
||||||
*/
|
|
||||||
private List<Keyword> keywords; //keywords to search
|
|
||||||
private List<String> keywordLists; // lists currently being searched
|
|
||||||
private Map<String, KeywordList> keywordToList; //keyword to list name mapping
|
|
||||||
private AggregateProgressHandle progressGroup;
|
|
||||||
private final Logger logger = Logger.getLogger(Searcher.class.getName());
|
|
||||||
private boolean finalRun = false;
|
|
||||||
|
|
||||||
Searcher(List<String> keywordLists) {
|
|
||||||
this.keywordLists = new ArrayList<>(keywordLists);
|
|
||||||
this.keywords = new ArrayList<>();
|
|
||||||
this.keywordToList = new HashMap<>();
|
|
||||||
//keywords are populated as searcher runs
|
|
||||||
}
|
|
||||||
|
|
||||||
Searcher(List<String> keywordLists, boolean finalRun) {
|
|
||||||
this(keywordLists);
|
|
||||||
this.finalRun = finalRun;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected Object doInBackground() throws Exception {
|
|
||||||
if (finalRun) {
|
|
||||||
logger.log(Level.INFO, "Pending start of new (final) searcher");
|
|
||||||
} else {
|
|
||||||
logger.log(Level.INFO, "Pending start of new searcher");
|
|
||||||
}
|
|
||||||
|
|
||||||
final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName")
|
|
||||||
+ (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : "");
|
|
||||||
progressGroup = AggregateProgressFactory.createSystemHandle(displayName + (" ("
|
|
||||||
+ NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") + ")"), null, new Cancellable() {
|
|
||||||
@Override
|
|
||||||
public boolean cancel() {
|
|
||||||
logger.log(Level.INFO, "Cancelling the searcher by user.");
|
|
||||||
if (progressGroup != null) {
|
|
||||||
progressGroup.setDisplayName(displayName + " (" + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.cancelMsg") + "...)");
|
|
||||||
}
|
|
||||||
return Searcher.this.cancel(true);
|
|
||||||
}
|
|
||||||
}, null);
|
|
||||||
|
|
||||||
updateKeywords();
|
|
||||||
|
|
||||||
ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()];
|
|
||||||
int i = 0;
|
|
||||||
for (Keyword keywordQuery : keywords) {
|
|
||||||
subProgresses[i] =
|
|
||||||
AggregateProgressFactory.createProgressContributor(keywordQuery.getQuery());
|
|
||||||
progressGroup.addContributor(subProgresses[i]);
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
|
|
||||||
progressGroup.start();
|
|
||||||
|
|
||||||
//block to ensure previous searcher is completely done with doInBackground()
|
|
||||||
//even after previous searcher cancellation, we need to check this
|
|
||||||
searcherLock.lock();
|
|
||||||
final StopWatch stopWatch = new StopWatch();
|
|
||||||
stopWatch.start();
|
|
||||||
try {
|
|
||||||
logger.log(Level.INFO, "Started a new searcher");
|
|
||||||
progressGroup.setDisplayName(displayName);
|
|
||||||
//make sure other searchers are not spawned
|
|
||||||
searcherDone = false;
|
|
||||||
runSearcher = false;
|
|
||||||
if (searchTimer.isRunning()) {
|
|
||||||
searchTimer.stop();
|
|
||||||
}
|
|
||||||
|
|
||||||
int keywordsSearched = 0;
|
|
||||||
|
|
||||||
//updateKeywords();
|
|
||||||
|
|
||||||
for (Keyword keywordQuery : keywords) {
|
|
||||||
if (this.isCancelled()) {
|
|
||||||
logger.log(Level.INFO, "Cancel detected, bailing before new keyword processed: {0}", keywordQuery.getQuery());
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
final String queryStr = keywordQuery.getQuery();
|
|
||||||
final KeywordList list = keywordToList.get(queryStr);
|
|
||||||
final String listName = list.getName();
|
|
||||||
|
|
||||||
//new subProgress will be active after the initial query
|
|
||||||
//when we know number of hits to start() with
|
|
||||||
if (keywordsSearched > 0) {
|
|
||||||
subProgresses[keywordsSearched - 1].finish();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
KeywordSearchQuery del = null;
|
|
||||||
|
|
||||||
boolean isRegex = !keywordQuery.isLiteral();
|
|
||||||
if (isRegex) {
|
|
||||||
del = new TermComponentQuery(keywordQuery);
|
|
||||||
} else {
|
|
||||||
del = new LuceneQuery(keywordQuery);
|
|
||||||
del.escape();
|
|
||||||
}
|
|
||||||
|
|
||||||
//limit search to currently ingested data sources
|
|
||||||
//set up a filter with 1 or more image ids OR'ed
|
|
||||||
final KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, curDataSourceIds);
|
|
||||||
del.addFilter(dataSourceFilter);
|
|
||||||
|
|
||||||
Map<String, List<ContentHit>> queryResult;
|
|
||||||
|
|
||||||
try {
|
|
||||||
queryResult = del.performQuery();
|
|
||||||
} catch (NoOpenCoreException ex) {
|
|
||||||
logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), ex);
|
|
||||||
//no reason to continue with next query if recovery failed
|
|
||||||
//or wait for recovery to kick in and run again later
|
|
||||||
//likely case has closed and threads are being interrupted
|
|
||||||
return null;
|
|
||||||
} catch (CancellationException e) {
|
|
||||||
logger.log(Level.INFO, "Cancel detected, bailing during keyword query: {0}", keywordQuery.getQuery());
|
|
||||||
return null;
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), e);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// calculate new results but substracting results already obtained in this ingest
|
|
||||||
// this creates a map of each keyword to the list of unique files that have that hit.
|
|
||||||
Map<Keyword, List<ContentHit>> newResults = filterResults(queryResult, isRegex);
|
|
||||||
|
|
||||||
if (!newResults.isEmpty()) {
|
|
||||||
|
|
||||||
//write results to BB
|
|
||||||
|
|
||||||
//new artifacts created, to report to listeners
|
|
||||||
Collection<BlackboardArtifact> newArtifacts = new ArrayList<>();
|
|
||||||
|
|
||||||
//scale progress bar more more granular, per result sub-progress, within per keyword
|
|
||||||
int totalUnits = newResults.size();
|
|
||||||
subProgresses[keywordsSearched].start(totalUnits);
|
|
||||||
int unitProgress = 0;
|
|
||||||
String queryDisplayStr = keywordQuery.getQuery();
|
|
||||||
if (queryDisplayStr.length() > 50) {
|
|
||||||
queryDisplayStr = queryDisplayStr.substring(0, 49) + "...";
|
|
||||||
}
|
|
||||||
subProgresses[keywordsSearched].progress(listName + ": " + queryDisplayStr, unitProgress);
|
|
||||||
|
|
||||||
|
|
||||||
/* cycle through the keywords returned -- only one unless it was a regexp */
|
|
||||||
for (final Keyword hitTerm : newResults.keySet()) {
|
|
||||||
//checking for cancellation between results
|
|
||||||
if (this.isCancelled()) {
|
|
||||||
logger.log(Level.INFO, "Cancel detected, bailing before new hit processed for query: {0}", keywordQuery.getQuery());
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// update progress display
|
|
||||||
String hitDisplayStr = hitTerm.getQuery();
|
|
||||||
if (hitDisplayStr.length() > 50) {
|
|
||||||
hitDisplayStr = hitDisplayStr.substring(0, 49) + "...";
|
|
||||||
}
|
|
||||||
subProgresses[keywordsSearched].progress(listName + ": " + hitDisplayStr, unitProgress);
|
|
||||||
//subProgresses[keywordsSearched].progress(unitProgress);
|
|
||||||
|
|
||||||
// this returns the unique files in the set with the first chunk that has a hit
|
|
||||||
Map<AbstractFile, Integer> contentHitsFlattened = ContentHit.flattenResults(newResults.get(hitTerm));
|
|
||||||
for (final AbstractFile hitFile : contentHitsFlattened.keySet()) {
|
|
||||||
|
|
||||||
// get the snippet for the first hit in the file
|
|
||||||
String snippet;
|
|
||||||
final String snippetQuery = KeywordSearchUtil.escapeLuceneQuery(hitTerm.getQuery());
|
|
||||||
int chunkId = contentHitsFlattened.get(hitFile);
|
|
||||||
try {
|
|
||||||
snippet = LuceneQuery.querySnippet(snippetQuery, hitFile.getId(), chunkId, isRegex, true);
|
|
||||||
} catch (NoOpenCoreException e) {
|
|
||||||
logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e);
|
|
||||||
//no reason to continue
|
|
||||||
return null;
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// write the blackboard artifact for this keyword in this file
|
|
||||||
KeywordWriteResult written = del.writeToBlackBoard(hitTerm.getQuery(), hitFile, snippet, listName);
|
|
||||||
if (written == null) {
|
|
||||||
logger.log(Level.WARNING, "BB artifact for keyword hit not written, file: {0}, hit: {1}", new Object[]{hitFile, hitTerm.toString()});
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
newArtifacts.add(written.getArtifact());
|
|
||||||
|
|
||||||
//generate an ingest inbox message for this keyword in this file
|
|
||||||
if (list.getIngestMessages()) {
|
|
||||||
StringBuilder subjectSb = new StringBuilder();
|
|
||||||
StringBuilder detailsSb = new StringBuilder();
|
|
||||||
//final int hitFiles = newResults.size();
|
|
||||||
|
|
||||||
if (!keywordQuery.isLiteral()) {
|
|
||||||
subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExpHitLbl"));
|
|
||||||
} else {
|
|
||||||
subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLbl"));
|
|
||||||
}
|
|
||||||
//subjectSb.append("<");
|
|
||||||
String uniqueKey = null;
|
|
||||||
BlackboardAttribute attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID());
|
|
||||||
if (attr != null) {
|
|
||||||
final String keyword = attr.getValueString();
|
|
||||||
subjectSb.append(keyword);
|
|
||||||
uniqueKey = keyword.toLowerCase();
|
|
||||||
}
|
|
||||||
|
|
||||||
//subjectSb.append(">");
|
|
||||||
//String uniqueKey = queryStr;
|
|
||||||
|
|
||||||
//details
|
|
||||||
detailsSb.append("<table border='0' cellpadding='4' width='280'>");
|
|
||||||
//hit
|
|
||||||
detailsSb.append("<tr>");
|
|
||||||
detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLThLbl"));
|
|
||||||
detailsSb.append("<td>").append(EscapeUtil.escapeHtml(attr.getValueString())).append("</td>");
|
|
||||||
detailsSb.append("</tr>");
|
|
||||||
|
|
||||||
//preview
|
|
||||||
attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID());
|
|
||||||
if (attr != null) {
|
|
||||||
detailsSb.append("<tr>");
|
|
||||||
detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.previewThLbl"));
|
|
||||||
detailsSb.append("<td>").append(EscapeUtil.escapeHtml(attr.getValueString())).append("</td>");
|
|
||||||
detailsSb.append("</tr>");
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
//file
|
|
||||||
detailsSb.append("<tr>");
|
|
||||||
detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.fileThLbl"));
|
|
||||||
detailsSb.append("<td>").append(hitFile.getParentPath()).append(hitFile.getName()).append("</td>");
|
|
||||||
|
|
||||||
detailsSb.append("</tr>");
|
|
||||||
|
|
||||||
|
|
||||||
//list
|
|
||||||
attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID());
|
|
||||||
detailsSb.append("<tr>");
|
|
||||||
detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.listThLbl"));
|
|
||||||
detailsSb.append("<td>").append(attr.getValueString()).append("</td>");
|
|
||||||
detailsSb.append("</tr>");
|
|
||||||
|
|
||||||
//regex
|
|
||||||
if (!keywordQuery.isLiteral()) {
|
|
||||||
attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID());
|
|
||||||
if (attr != null) {
|
|
||||||
detailsSb.append("<tr>");
|
|
||||||
detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExThLbl"));
|
|
||||||
detailsSb.append("<td>").append(attr.getValueString()).append("</td>");
|
|
||||||
detailsSb.append("</tr>");
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
detailsSb.append("</table>");
|
|
||||||
|
|
||||||
services.postMessage(IngestMessage.createDataMessage(messageID.getAndIncrement(), KeywordSearchModuleFactory.getModuleName(), subjectSb.toString(), detailsSb.toString(), uniqueKey, written.getArtifact()));
|
|
||||||
}
|
|
||||||
} //for each file hit
|
|
||||||
|
|
||||||
++unitProgress;
|
|
||||||
|
|
||||||
}//for each hit term
|
|
||||||
|
|
||||||
//update artifact browser
|
|
||||||
if (!newArtifacts.isEmpty()) {
|
|
||||||
services.fireModuleDataEvent(new ModuleDataEvent(KeywordSearchModuleFactory.getModuleName(), ARTIFACT_TYPE.TSK_KEYWORD_HIT, newArtifacts));
|
|
||||||
}
|
|
||||||
} //if has results
|
|
||||||
|
|
||||||
//reset the status text before it goes away
|
|
||||||
subProgresses[keywordsSearched].progress("");
|
|
||||||
|
|
||||||
++keywordsSearched;
|
|
||||||
|
|
||||||
} //for each keyword
|
|
||||||
|
|
||||||
} //end try block
|
|
||||||
catch (Exception ex) {
|
|
||||||
logger.log(Level.WARNING, "searcher exception occurred", ex);
|
|
||||||
} finally {
|
|
||||||
try {
|
|
||||||
finalizeSearcher();
|
|
||||||
stopWatch.stop();
|
|
||||||
logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs());
|
|
||||||
} finally {
|
|
||||||
searcherLock.unlock();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void done() {
|
|
||||||
// call get to see if there were any errors
|
|
||||||
try {
|
|
||||||
get();
|
|
||||||
} catch (InterruptedException | ExecutionException e) {
|
|
||||||
logger.log(Level.SEVERE, "Error performing keyword search: " + e.getMessage());
|
|
||||||
services.postMessage(IngestMessage.createErrorMessage(messageID.getAndIncrement(), KeywordSearchModuleFactory.getModuleName(), "Error performing keyword search", e.getMessage()));
|
|
||||||
} // catch and ignore if we were cancelled
|
|
||||||
catch (java.util.concurrent.CancellationException ex) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sync-up the updated keywords from the currently used lists in the XML
|
|
||||||
*/
|
|
||||||
private void updateKeywords() {
|
|
||||||
KeywordSearchListsXML loader = KeywordSearchListsXML.getCurrent();
|
|
||||||
|
|
||||||
this.keywords.clear();
|
|
||||||
this.keywordToList.clear();
|
|
||||||
|
|
||||||
for (String name : this.keywordLists) {
|
|
||||||
KeywordList list = loader.getList(name);
|
|
||||||
for (Keyword k : list.getKeywords()) {
|
|
||||||
this.keywords.add(k);
|
|
||||||
this.keywordToList.put(k.getQuery(), list);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
//perform all essential cleanup that needs to be done right AFTER doInBackground() returns
|
|
||||||
//without relying on done() method that is not guaranteed to run after background thread completes
|
|
||||||
//NEED to call this method always right before doInBackground() returns
|
|
||||||
/**
|
|
||||||
* Performs the cleanup that needs to be done right AFTER
|
|
||||||
* doInBackground() returns without relying on done() method that is not
|
|
||||||
* guaranteed to run after background thread completes REQUIRED to call
|
|
||||||
* this method always right before doInBackground() returns
|
|
||||||
*/
|
|
||||||
private void finalizeSearcher() {
|
|
||||||
logger.log(Level.INFO, "Searcher finalizing");
|
|
||||||
SwingUtilities.invokeLater(new Runnable() {
|
|
||||||
@Override
|
|
||||||
public void run() {
|
|
||||||
progressGroup.finish();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
searcherDone = true; //next currentSearcher can start
|
|
||||||
|
|
||||||
if (finalRun) {
|
|
||||||
//this is the final searcher
|
|
||||||
logger.log(Level.INFO, "The final searcher in this ingest done.");
|
|
||||||
|
|
||||||
//run module cleanup
|
|
||||||
cleanup();
|
|
||||||
} else {
|
|
||||||
//start counting time for a new searcher to start
|
|
||||||
//unless final searcher is pending
|
|
||||||
if (finalSearcher == null) {
|
|
||||||
//we need a new Timer object, because restarting previus will not cause firing of the action
|
|
||||||
final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000;
|
|
||||||
searchTimer = new Timer(updateIntervalMs, new SearchTimerAction());
|
|
||||||
searchTimer.start();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//calculate new results but substracting results already obtained in this ingest
|
|
||||||
//update currentResults map with the new results
|
|
||||||
private Map<Keyword, List<ContentHit>> filterResults(Map<String, List<ContentHit>> queryResult, boolean isRegex) {
|
|
||||||
Map<Keyword, List<ContentHit>> newResults = new HashMap<>();
|
|
||||||
|
|
||||||
for (String termResult : queryResult.keySet()) {
|
|
||||||
List<ContentHit> queryTermResults = queryResult.get(termResult);
|
|
||||||
|
|
||||||
//translate to list of IDs that we keep track of
|
|
||||||
List<Long> queryTermResultsIDs = new ArrayList<>();
|
|
||||||
for (ContentHit ch : queryTermResults) {
|
|
||||||
queryTermResultsIDs.add(ch.getId());
|
|
||||||
}
|
|
||||||
|
|
||||||
Keyword termResultK = new Keyword(termResult, !isRegex);
|
|
||||||
List<Long> curTermResults = currentResults.get(termResultK);
|
|
||||||
if (curTermResults == null) {
|
|
||||||
currentResults.put(termResultK, queryTermResultsIDs);
|
|
||||||
newResults.put(termResultK, queryTermResults);
|
|
||||||
} else {
|
|
||||||
//some AbstractFile hits already exist for this keyword
|
|
||||||
for (ContentHit res : queryTermResults) {
|
|
||||||
if (!curTermResults.contains(res.getId())) {
|
|
||||||
//add to new results
|
|
||||||
List<ContentHit> newResultsFs = newResults.get(termResultK);
|
|
||||||
if (newResultsFs == null) {
|
|
||||||
newResultsFs = new ArrayList<>();
|
|
||||||
newResults.put(termResultK, newResultsFs);
|
|
||||||
}
|
|
||||||
newResultsFs.add(res);
|
|
||||||
curTermResults.add(res.getId());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return newResults;
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -121,6 +121,26 @@ public final class SearchRunner {
|
|||||||
doFinalSearch(job);
|
doFinalSearch(job);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void stopJob(long jobId) {
|
||||||
|
SearchJobInfo job;
|
||||||
|
synchronized(this) {
|
||||||
|
job = jobs.get(jobId);
|
||||||
|
if (job == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
///@todo
|
||||||
|
//stop currentSearcher
|
||||||
|
// if (currentSearcher != null) {
|
||||||
|
// currentSearcher.cancel(true);
|
||||||
|
// }
|
||||||
|
|
||||||
|
jobs.remove(jobId);
|
||||||
|
}
|
||||||
|
|
||||||
|
commit();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Add this list to all of the jobs
|
* Add this list to all of the jobs
|
||||||
* @param keywordListName
|
* @param keywordListName
|
||||||
@ -201,6 +221,7 @@ public final class SearchRunner {
|
|||||||
// Spawn a search thread for each job
|
// Spawn a search thread for each job
|
||||||
for(Entry<Long, SearchJobInfo> j : jobs.entrySet()) {
|
for(Entry<Long, SearchJobInfo> j : jobs.entrySet()) {
|
||||||
SearchJobInfo job = j.getValue();
|
SearchJobInfo job = j.getValue();
|
||||||
|
// If no lists or the worker is already running then skip it
|
||||||
if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) {
|
if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) {
|
||||||
Searcher s = new Searcher(job);
|
Searcher s = new Searcher(job);
|
||||||
s.execute();
|
s.execute();
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user