keyword search ingest: more granular cancellation

This commit is contained in:
adam-m 2012-02-24 19:12:35 -05:00
parent 7a581bf487
commit 3cc94caeca

View File

@ -70,8 +70,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
private volatile int messageID = 0; private volatile int messageID = 0;
private volatile boolean finalRun = false; private volatile boolean finalRun = false;
private SleuthkitCase caseHandle = null; private SleuthkitCase caseHandle = null;
// TODO: use a more robust method than checking file extension to determine
// TODO: use a more robust method than checking file extension to determine
// whether to try a file // whether to try a file
// supported extensions list from http://www.lucidimagination.com/devzone/technical-articles/content-extraction-tika // supported extensions list from http://www.lucidimagination.com/devzone/technical-articles/content-extraction-tika
static final String[] ingestibleExtensions = {"tar", "jar", "zip", "bzip2", static final String[] ingestibleExtensions = {"tar", "jar", "zip", "bzip2",
@ -81,8 +80,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
public enum IngestStatus { public enum IngestStatus {
INGESTED, EXTRACTED_INGESTED, SKIPPED, INGESTED, EXTRACTED_INGESTED, SKIPPED,};
};
private Map<Long, IngestStatus> ingestStatus; private Map<Long, IngestStatus> ingestStatus;
private Map<String, List<FsContent>> reportedHits; //already reported hits private Map<String, List<FsContent>> reportedHits; //already reported hits
@ -102,7 +100,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
commit(); commit();
commitIndex = false; commitIndex = false;
indexChangeNotify(); indexChangeNotify();
updateKeywords(); updateKeywords();
//start search if previous not running //start search if previous not running
if (keywords != null && !keywords.isEmpty() && searcherDone) { if (keywords != null && !keywords.isEmpty() && searcherDone) {
@ -183,7 +181,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
ingestStatus = new HashMap<Long, IngestStatus>(); ingestStatus = new HashMap<Long, IngestStatus>();
reportedHits = new HashMap<String, List<FsContent>>(); reportedHits = new HashMap<String, List<FsContent>>();
keywords = new ArrayList<Keyword>(); keywords = new ArrayList<Keyword>();
keywordLists = new ArrayList<String>(); keywordLists = new ArrayList<String>();
@ -224,16 +222,17 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
public boolean isConfigurable() { public boolean isConfigurable() {
return true; return true;
} }
@Override @Override
public boolean hasBackgroundJobsRunning() { public boolean hasBackgroundJobsRunning() {
if (searcher != null && searcherDone == false) { if (searcher != null && searcherDone == false) {
return true; return true;
} else {
return false;
} }
else return false;
//no need to check timer thread //no need to check timer thread
} }
private void commit() { private void commit() {
@ -290,37 +289,39 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
*/ */
private void initKeywords() { private void initKeywords() {
KeywordSearchListsXML loader = KeywordSearchListsXML.getCurrent(); KeywordSearchListsXML loader = KeywordSearchListsXML.getCurrent();
keywords.clear(); keywords.clear();
keywordLists.clear(); keywordLists.clear();
for(KeywordSearchList list : loader.getListsL()){ for (KeywordSearchList list : loader.getListsL()) {
if(list.getUseForIngest()) if (list.getUseForIngest()) {
keywordLists.add(list.getName()); keywordLists.add(list.getName());
keywords.addAll(list.getKeywords()); }
keywords.addAll(list.getKeywords());
} }
} }
/** /**
* Retrieve the updated keyword search lists from the XML loader * Retrieve the updated keyword search lists from the XML loader
*/ */
private void updateKeywords() { private void updateKeywords() {
KeywordSearchListsXML loader = KeywordSearchListsXML.getCurrent(); KeywordSearchListsXML loader = KeywordSearchListsXML.getCurrent();
keywords.clear(); keywords.clear();
for(String name : keywordLists) { for (String name : keywordLists) {
keywords.addAll(loader.getList(name).getKeywords()); keywords.addAll(loader.getList(name).getKeywords());
} }
} }
List<String> getKeywordLists() { List<String> getKeywordLists() {
return keywordLists == null ? new ArrayList<String>() : keywordLists; return keywordLists == null ? new ArrayList<String>() : keywordLists;
} }
void addToKeywordLists(String name) { void addToKeywordLists(String name) {
if(!keywordLists.contains(name)) if (!keywordLists.contains(name)) {
keywordLists.add(name); keywordLists.add(name);
}
} }
//CommitTimer wakes up every interval ms //CommitTimer wakes up every interval ms
@ -510,6 +511,9 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
//write results to BB //write results to BB
Collection<BlackboardArtifact> newArtifacts = new ArrayList<BlackboardArtifact>(); //new artifacts to report Collection<BlackboardArtifact> newArtifacts = new ArrayList<BlackboardArtifact>(); //new artifacts to report
for (FsContent hitFile : newResults) { for (FsContent hitFile : newResults) {
if (this.isCancelled()) {
return null;
}
Collection<KeywordWriteResult> written = del.writeToBlackBoard(hitFile); Collection<KeywordWriteResult> written = del.writeToBlackBoard(hitFile);
for (KeywordWriteResult res : written) { for (KeywordWriteResult res : written) {
newArtifacts.add(res.getArtifact()); newArtifacts.add(res.getArtifact());
@ -527,7 +531,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
subjectSb.append(keyword); subjectSb.append(keyword);
uniqueKey = keyword; uniqueKey = keyword;
} }
subjectSb.append(">"); subjectSb.append(">");
//String uniqueKey = queryStr; //String uniqueKey = queryStr;
@ -545,7 +549,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
detailsSb.append("<br />"); detailsSb.append("<br />");
} }
} }
//file //file
detailsSb.append("File: "); detailsSb.append("File: ");
detailsSb.append(hitFile.getParentPath()).append(hitFile.getName()); detailsSb.append(hitFile.getParentPath()).append(hitFile.getName());