From 9d22f5c518063108972f062d5e174f2961fef78e Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 26 Mar 2014 19:05:20 -0400 Subject: [PATCH 01/38] Added unfinished skeleton of new SearchRunner. --- .../autopsy/keywordsearch/SearchRunner.java | 543 ++++++++++++++++++ 1 file changed, 543 insertions(+) create mode 100644 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java new file mode 100644 index 0000000000..f190891cd1 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -0,0 +1,543 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011 - 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.sleuthkit.autopsy.keywordsearch; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.logging.Level; +import javax.swing.SwingUtilities; +import javax.swing.SwingWorker; +import javax.swing.Timer; +import org.netbeans.api.progress.aggregate.AggregateProgressFactory; +import org.netbeans.api.progress.aggregate.AggregateProgressHandle; +import org.netbeans.api.progress.aggregate.ProgressContributor; +import org.openide.util.Cancellable; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.EscapeUtil; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.StopWatch; +import org.sleuthkit.autopsy.ingest.IngestMessage; +import org.sleuthkit.autopsy.ingest.IngestServices; +import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardAttribute; + +/** + * + */ +public final class SearchRunner { + private static final Logger logger = Logger.getLogger(SearchRunner.class.getName()); + private static SearchRunner instance = null; + private Map jobs = new HashMap<>(); + + public static synchronized SearchRunner getInstance() { + if (instance == null) { + instance = new SearchRunner(); + } + return instance; + } + + ///@todo Can we get the keyword lists here instead of passing them in? + public void startJob(long jobId, long dataSourceId, List keywordLists) { + if (!jobs.containsKey(jobId)) { + SearchRunnerJob runnerJob = new SearchRunnerJob(jobId, dataSourceId); + runnerJob.setKeywordLists(keywordLists); + jobs.put(jobId, runnerJob); + + ///@todo start it up + } + } + + public void endJob(long jobId) { + ///@todo remove this comment if this is ok. otherwise we might need to do module counting + + ///@todo stuff + SearchRunnerJob runnerJob = jobs.get(jobId); + if (runnerJob != null) { + runnerJob.doFinalSearch(); + } + } + + private void doFinalSearch() { + + } + + ///@todo currentResults? + private class SearchRunnerJob { + private long jobId; + private long dataSourceId; + private List keywordLists = new ArrayList<>(); // lists currently being searched + private List keywords; //keywords to search + private Map> currentResults; + private SearchRunner.Searcher currentSearcher; + private SearchRunner.Searcher finalSearcher; + private List keywordListNames = null; + + public SearchRunnerJob(long jobId, long dataSourceId) { + this.jobId = jobId; + this.dataSourceId = dataSourceId; + } + + public void setKeywordLists(List keywordLists) { + this.keywordLists = keywordLists; + //List keywordListNames = settings.getNamesOfEnabledKeyWordLists(); + } + + public List getKeywordLists() { + return keywordLists; + } + + public void doFinalSearch() { + finalSearcher = new SearchRunner.Searcher(keywordListNames, true); + finalSearcher.execute(); + } + } + + /** + * Searcher responsible for searching the current index and writing results + * to blackboard and the inbox. Also, posts results to listeners as Ingest + * data events. Searches entire index, and keeps track of only new results + * to report and save. Runs as a background thread. + */ + private final class Searcher extends SwingWorker { + + /** + * Searcher has private copies/snapshots of the lists and keywords + */ + private List keywords; //keywords to search + private List keywordLists; // lists currently being searched + private Map keywordToList; //keyword to list name mapping + private AggregateProgressHandle progressGroup; + private final Logger logger = Logger.getLogger(SearchRunner.Searcher.class.getName()); + private boolean finalRun = false; + private Timer searchTimer; + private IngestServices services = IngestServices.getDefault(); + //private static final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); //use fairness policy + //private static final Lock searcherLock = rwLock.writeLock(); + + Searcher(List keywordLists) { + this.keywordLists = new ArrayList<>(keywordLists); + this.keywords = new ArrayList<>(); + this.keywordToList = new HashMap<>(); + //keywords are populated as searcher runs + } + + Searcher(List keywordLists, boolean finalRun) { + this(keywordLists); + this.finalRun = finalRun; + } + + @Override + protected Object doInBackground() throws Exception { + if (finalRun) { + logger.log(Level.INFO, "Pending start of new (final) searcher"); + } else { + logger.log(Level.INFO, "Pending start of new searcher"); + } + + final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") + + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); + progressGroup = AggregateProgressFactory.createSystemHandle(displayName + (" (" + + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") + ")"), null, new Cancellable() { + @Override + public boolean cancel() { + logger.log(Level.INFO, "Cancelling the searcher by user."); + if (progressGroup != null) { + progressGroup.setDisplayName(displayName + " (" + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.cancelMsg") + "...)"); + } + return SearchRunner.Searcher.this.cancel(true); + } + }, null); + + updateKeywords(); + + ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()]; + int i = 0; + for (Keyword keywordQuery : keywords) { + subProgresses[i] = + AggregateProgressFactory.createProgressContributor(keywordQuery.getQuery()); + progressGroup.addContributor(subProgresses[i]); + i++; + } + + progressGroup.start(); + + //block to ensure previous searcher is completely done with doInBackground() + //even after previous searcher cancellation, we need to check this + searcherLock.lock(); + final StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + try { + logger.log(Level.INFO, "Started a new searcher"); + progressGroup.setDisplayName(displayName); + //make sure other searchers are not spawned + searcherDone = false; + runSearcher = false; + if (searchTimer.isRunning()) { + searchTimer.stop(); + } + + int keywordsSearched = 0; + + //updateKeywords(); + + for (Keyword keywordQuery : keywords) { + if (this.isCancelled()) { + logger.log(Level.INFO, "Cancel detected, bailing before new keyword processed: {0}", keywordQuery.getQuery()); + return null; + } + + final String queryStr = keywordQuery.getQuery(); + final KeywordList list = keywordToList.get(queryStr); + final String listName = list.getName(); + + //new subProgress will be active after the initial query + //when we know number of hits to start() with + if (keywordsSearched > 0) { + subProgresses[keywordsSearched - 1].finish(); + } + + + KeywordSearchQuery del = null; + + boolean isRegex = !keywordQuery.isLiteral(); + if (isRegex) { + del = new TermComponentQuery(keywordQuery); + } else { + del = new LuceneQuery(keywordQuery); + del.escape(); + } + + //limit search to currently ingested data sources + //set up a filter with 1 or more image ids OR'ed + final KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, curDataSourceIds); + del.addFilter(dataSourceFilter); + + Map> queryResult; + + try { + queryResult = del.performQuery(); + } catch (NoOpenCoreException ex) { + logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), ex); + //no reason to continue with next query if recovery failed + //or wait for recovery to kick in and run again later + //likely case has closed and threads are being interrupted + return null; + } catch (CancellationException e) { + logger.log(Level.INFO, "Cancel detected, bailing during keyword query: {0}", keywordQuery.getQuery()); + return null; + } catch (Exception e) { + logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), e); + continue; + } + + // calculate new results but substracting results already obtained in this ingest + // this creates a map of each keyword to the list of unique files that have that hit. + Map> newResults = filterResults(queryResult, isRegex); + + if (!newResults.isEmpty()) { + + //write results to BB + + //new artifacts created, to report to listeners + Collection newArtifacts = new ArrayList<>(); + + //scale progress bar more more granular, per result sub-progress, within per keyword + int totalUnits = newResults.size(); + subProgresses[keywordsSearched].start(totalUnits); + int unitProgress = 0; + String queryDisplayStr = keywordQuery.getQuery(); + if (queryDisplayStr.length() > 50) { + queryDisplayStr = queryDisplayStr.substring(0, 49) + "..."; + } + subProgresses[keywordsSearched].progress(listName + ": " + queryDisplayStr, unitProgress); + + + /* cycle through the keywords returned -- only one unless it was a regexp */ + for (final Keyword hitTerm : newResults.keySet()) { + //checking for cancellation between results + if (this.isCancelled()) { + logger.log(Level.INFO, "Cancel detected, bailing before new hit processed for query: {0}", keywordQuery.getQuery()); + return null; + } + + // update progress display + String hitDisplayStr = hitTerm.getQuery(); + if (hitDisplayStr.length() > 50) { + hitDisplayStr = hitDisplayStr.substring(0, 49) + "..."; + } + subProgresses[keywordsSearched].progress(listName + ": " + hitDisplayStr, unitProgress); + //subProgresses[keywordsSearched].progress(unitProgress); + + // this returns the unique files in the set with the first chunk that has a hit + Map contentHitsFlattened = ContentHit.flattenResults(newResults.get(hitTerm)); + for (final AbstractFile hitFile : contentHitsFlattened.keySet()) { + + // get the snippet for the first hit in the file + String snippet; + final String snippetQuery = KeywordSearchUtil.escapeLuceneQuery(hitTerm.getQuery()); + int chunkId = contentHitsFlattened.get(hitFile); + try { + snippet = LuceneQuery.querySnippet(snippetQuery, hitFile.getId(), chunkId, isRegex, true); + } catch (NoOpenCoreException e) { + logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e); + //no reason to continue + return null; + } catch (Exception e) { + logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e); + continue; + } + + // write the blackboard artifact for this keyword in this file + KeywordWriteResult written = del.writeToBlackBoard(hitTerm.getQuery(), hitFile, snippet, listName); + if (written == null) { + logger.log(Level.WARNING, "BB artifact for keyword hit not written, file: {0}, hit: {1}", new Object[]{hitFile, hitTerm.toString()}); + continue; + } + + newArtifacts.add(written.getArtifact()); + + //generate an ingest inbox message for this keyword in this file + if (list.getIngestMessages()) { + StringBuilder subjectSb = new StringBuilder(); + StringBuilder detailsSb = new StringBuilder(); + //final int hitFiles = newResults.size(); + + if (!keywordQuery.isLiteral()) { + subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExpHitLbl")); + } else { + subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLbl")); + } + //subjectSb.append("<"); + String uniqueKey = null; + BlackboardAttribute attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID()); + if (attr != null) { + final String keyword = attr.getValueString(); + subjectSb.append(keyword); + uniqueKey = keyword.toLowerCase(); + } + + //subjectSb.append(">"); + //String uniqueKey = queryStr; + + //details + detailsSb.append(""); + //hit + detailsSb.append(""); + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLThLbl")); + detailsSb.append(""); + detailsSb.append(""); + + //preview + attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID()); + if (attr != null) { + detailsSb.append(""); + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.previewThLbl")); + detailsSb.append(""); + detailsSb.append(""); + + } + + //file + detailsSb.append(""); + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.fileThLbl")); + detailsSb.append(""); + + detailsSb.append(""); + + + //list + attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID()); + detailsSb.append(""); + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.listThLbl")); + detailsSb.append(""); + detailsSb.append(""); + + //regex + if (!keywordQuery.isLiteral()) { + attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID()); + if (attr != null) { + detailsSb.append(""); + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExThLbl")); + detailsSb.append(""); + detailsSb.append(""); + + } + } + detailsSb.append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(hitFile.getParentPath()).append(hitFile.getName()).append("
").append(attr.getValueString()).append("
").append(attr.getValueString()).append("
"); + + services.postMessage(IngestMessage.createDataMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), subjectSb.toString(), detailsSb.toString(), uniqueKey, written.getArtifact())); + } + } //for each file hit + + ++unitProgress; + + }//for each hit term + + //update artifact browser + if (!newArtifacts.isEmpty()) { + services.fireModuleDataEvent(new ModuleDataEvent(KeywordSearchModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT, newArtifacts)); + } + } //if has results + + //reset the status text before it goes away + subProgresses[keywordsSearched].progress(""); + + ++keywordsSearched; + + } //for each keyword + + } //end try block + catch (Exception ex) { + logger.log(Level.WARNING, "searcher exception occurred", ex); + } finally { + try { + finalizeSearcher(); + stopWatch.stop(); + logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs()); + } finally { + searcherLock.unlock(); + } + } + + return null; + } + + @Override + protected void done() { + // call get to see if there were any errors + try { + get(); + } catch (InterruptedException | ExecutionException e) { + logger.log(Level.SEVERE, "Error performing keyword search: " + e.getMessage()); + services.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), "Error performing keyword search", e.getMessage())); + } // catch and ignore if we were cancelled + catch (java.util.concurrent.CancellationException ex) { + } + } + + /** + * Sync-up the updated keywords from the currently used lists in the XML + */ + private void updateKeywords() { + KeywordSearchListsXML loader = KeywordSearchListsXML.getCurrent(); + + this.keywords.clear(); + this.keywordToList.clear(); + + for (String name : this.keywordLists) { + KeywordList list = loader.getList(name); + for (Keyword k : list.getKeywords()) { + this.keywords.add(k); + this.keywordToList.put(k.getQuery(), list); + } + } + + + } + + //perform all essential cleanup that needs to be done right AFTER doInBackground() returns + //without relying on done() method that is not guaranteed to run after background thread completes + //NEED to call this method always right before doInBackground() returns + /** + * Performs the cleanup that needs to be done right AFTER + * doInBackground() returns without relying on done() method that is not + * guaranteed to run after background thread completes REQUIRED to call + * this method always right before doInBackground() returns + */ + private void finalizeSearcher() { + logger.log(Level.INFO, "Searcher finalizing"); + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + progressGroup.finish(); + } + }); + searcherDone = true; //next currentSearcher can start + + if (finalRun) { + //this is the final searcher + logger.log(Level.INFO, "The final searcher in this ingest done."); + + //run module cleanup + cleanup(); + } else { + //start counting time for a new searcher to start + //unless final searcher is pending + if (finalSearcher == null) { + //we need a new Timer object, because restarting previus will not cause firing of the action + final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; + searchTimer = new Timer(updateIntervalMs, new KeywordSearchIngestModule.SearchTimerAction()); + searchTimer.start(); + } + } + } + + //calculate new results but substracting results already obtained in this ingest + //update currentResults map with the new results + private Map> filterResults(Map> queryResult, boolean isRegex) { + Map> newResults = new HashMap<>(); + + for (String termResult : queryResult.keySet()) { + List queryTermResults = queryResult.get(termResult); + + //translate to list of IDs that we keep track of + List queryTermResultsIDs = new ArrayList<>(); + for (ContentHit ch : queryTermResults) { + queryTermResultsIDs.add(ch.getId()); + } + + Keyword termResultK = new Keyword(termResult, !isRegex); + List curTermResults = currentResults.get(termResultK); + if (curTermResults == null) { + currentResults.put(termResultK, queryTermResultsIDs); + newResults.put(termResultK, queryTermResults); + } else { + //some AbstractFile hits already exist for this keyword + for (ContentHit res : queryTermResults) { + if (!curTermResults.contains(res.getId())) { + //add to new results + List newResultsFs = newResults.get(termResultK); + if (newResultsFs == null) { + newResultsFs = new ArrayList<>(); + newResults.put(termResultK, newResultsFs); + } + newResultsFs.add(res); + curTermResults.add(res.getId()); + } + } + } + } + + return newResults; + + } + } + +} From a0372e9854b49d7a9eac6b62fc0d4c6cc9fb7112 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Fri, 28 Mar 2014 18:08:41 -0400 Subject: [PATCH 02/38] SearchRunner refactoring. --- .../KeywordSearchIngestModule.java | 71 ++--- .../autopsy/keywordsearch/SearchRunner.java | 251 ++++++++---------- 2 files changed, 150 insertions(+), 172 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 867e59eb16..7e9da0f9ba 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -286,30 +286,33 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme stop(); return; } +// +// commitTimer.stop(); +// +// //NOTE, we let the 1 before last searcher complete fully, and enqueue the last one +// +// //cancel searcher timer, ensure unwanted searcher does not start +// //before we start the final one +// if (searchTimer.isRunning()) { +// searchTimer.stop(); +// } +// runSearcher = false; +// +// logger.log(Level.INFO, "Running final index commit and search"); +// //final commit +// commit(); - commitTimer.stop(); - - //NOTE, we let the 1 before last searcher complete fully, and enqueue the last one - - //cancel searcher timer, ensure unwanted searcher does not start - //before we start the final one - if (searchTimer.isRunning()) { - searchTimer.stop(); - } - runSearcher = false; - - logger.log(Level.INFO, "Running final index commit and search"); - //final commit - commit(); - - postIndexSummary(); - + // Remove from the search list and trigger final commit and final search + SearchRunner.getInstance().endJob(jobId); + + postIndexSummary(); + //run one last search as there are probably some new files committed - List keywordLists = settings.getNamesOfEnabledKeyWordLists(); - if (!keywordLists.isEmpty() && processedFiles == true) { - finalSearcher = new Searcher(keywordLists, true); //final searcher run - finalSearcher.execute(); - } +// List keywordLists = settings.getNamesOfEnabledKeyWordLists(); +// if (!keywordLists.isEmpty() && processedFiles == true) { +// finalSearcher = new Searcher(keywordLists, true); //final searcher run +// finalSearcher.execute(); +// } //log number of files / chunks in index //signal a potential change in number of text_ingested files @@ -374,18 +377,18 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme initialized = false; } - /** - * Commits index and notifies listeners of index update - */ - private void commit() { - if (initialized) { - logger.log(Level.INFO, "Commiting index"); - ingester.commit(); - logger.log(Level.INFO, "Index comitted"); - //signal a potential change in number of text_ingested files - indexChangeNotify(); - } - } +// /** +// * Commits index and notifies listeners of index update +// */ +// private void commit() { +// if (initialized) { +// logger.log(Level.INFO, "Commiting index"); +// ingester.commit(); +// logger.log(Level.INFO, "Index comitted"); +// //signal a potential change in number of text_ingested files +// indexChangeNotify(); +// } +// } /** * Posts inbox message with summary of text_ingested files diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index f190891cd1..941f82333e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -49,13 +49,24 @@ import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; /** - * + * Singleton keyword search manager + * Launches search threads for each job and performs commits, + * both on timed intervals. */ public final class SearchRunner { private static final Logger logger = Logger.getLogger(SearchRunner.class.getName()); private static SearchRunner instance = null; - private Map jobs = new HashMap<>(); + private IngestServices services = IngestServices.getInstance(); + private Ingester ingester = null; + private boolean initialized = false; + private Map jobs = new HashMap<>(); + private Timer searchTimer; + SearchRunner() { + ingester = Server.getIngester(); + initialized = true; + } + public static synchronized SearchRunner getInstance() { if (instance == null) { instance = new SearchRunner(); @@ -63,62 +74,85 @@ public final class SearchRunner { return instance; } - ///@todo Can we get the keyword lists here instead of passing them in? - public void startJob(long jobId, long dataSourceId, List keywordLists) { + public synchronized void startJob(long jobId, long dataSourceId, List keywordListNames) { if (!jobs.containsKey(jobId)) { - SearchRunnerJob runnerJob = new SearchRunnerJob(jobId, dataSourceId); - runnerJob.setKeywordLists(keywordLists); - jobs.put(jobId, runnerJob); - - ///@todo start it up + SearchJobInfo jobData = new SearchJobInfo(jobId, dataSourceId, keywordListNames); + jobs.put(jobId, jobData); } } - public void endJob(long jobId) { - ///@todo remove this comment if this is ok. otherwise we might need to do module counting + public void endJob(long jobId) { + SearchJobInfo job; + synchronized(this) { + job = jobs.get(jobId); + if (job == null) { + return; + } + jobs.remove(jobId); + } + doFinalSearch(job); + } + + /** + * Commits index and notifies listeners of index update + */ + private void commit() { + if (initialized) { + logger.log(Level.INFO, "Commiting index"); + ingester.commit(); + logger.log(Level.INFO, "Index comitted"); + + //signal a potential change in number of text_ingested files + try { + final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles(); + KeywordSearch.fireNumIndexedFilesChange(null, new Integer(numIndexedFiles)); + } catch (NoOpenCoreException | KeywordSearchModuleException ex) { + logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files: ", ex); + } + } + } + + private void doFinalSearch(SearchJobInfo job) { + //cancel searcher timer, ensure unwanted searcher does not start + //before we start the final one + if (searchTimer.isRunning()) { + searchTimer.stop(); + } + + logger.log(Level.INFO, "Running final index commit and search for jobid {0}", jobId); + commit(); + + //run one last search as there are probably some new files committed + if (!job.getKeywordListNames().isEmpty()) { + SearchRunner.Searcher finalSearcher = new SearchRunner.Searcher(job.getKeywordListNames(), true); + finalSearcher.execute(); + } + } + + private class SearchJobInfo { + private long jobId; + private long dataSourceId; + private List keywordListNames = new ArrayList<>(); + + public SearchJobInfo(long jobId, long dataSourceId, List keywordListNames) { + this.jobId = jobId; + this.dataSourceId = dataSourceId; + this.keywordListNames = keywordListNames; + } - ///@todo stuff - SearchRunnerJob runnerJob = jobs.get(jobId); - if (runnerJob != null) { - runnerJob.doFinalSearch(); + long getDataSourceId() { + return dataSourceId; + } + + List getKeywordListNames() { + return keywordListNames; } } - private void doFinalSearch() { + private synchronized void setJobSearcherDone(long jobId, boolean flag) { } - ///@todo currentResults? - private class SearchRunnerJob { - private long jobId; - private long dataSourceId; - private List keywordLists = new ArrayList<>(); // lists currently being searched - private List keywords; //keywords to search - private Map> currentResults; - private SearchRunner.Searcher currentSearcher; - private SearchRunner.Searcher finalSearcher; - private List keywordListNames = null; - - public SearchRunnerJob(long jobId, long dataSourceId) { - this.jobId = jobId; - this.dataSourceId = dataSourceId; - } - - public void setKeywordLists(List keywordLists) { - this.keywordLists = keywordLists; - //List keywordListNames = settings.getNamesOfEnabledKeyWordLists(); - } - - public List getKeywordLists() { - return keywordLists; - } - - public void doFinalSearch() { - finalSearcher = new SearchRunner.Searcher(keywordListNames, true); - finalSearcher.execute(); - } - } - /** * Searcher responsible for searching the current index and writing results * to blackboard and the inbox. Also, posts results to listeners as Ingest @@ -136,20 +170,18 @@ public final class SearchRunner { private AggregateProgressHandle progressGroup; private final Logger logger = Logger.getLogger(SearchRunner.Searcher.class.getName()); private boolean finalRun = false; - private Timer searchTimer; - private IngestServices services = IngestServices.getDefault(); - //private static final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); //use fairness policy - //private static final Lock searcherLock = rwLock.writeLock(); - - Searcher(List keywordLists) { + private long jobId; + + Searcher(List keywordLists, long jobId) { this.keywordLists = new ArrayList<>(keywordLists); - this.keywords = new ArrayList<>(); - this.keywordToList = new HashMap<>(); + this.jobId = jobId; + keywords = new ArrayList<>(); + keywordToList = new HashMap<>(); //keywords are populated as searcher runs } - Searcher(List keywordLists, boolean finalRun) { - this(keywordLists); + Searcher(List keywordLists, long jobId, boolean finalRun) { + this(keywordLists, jobId); this.finalRun = finalRun; } @@ -161,15 +193,15 @@ public final class SearchRunner { logger.log(Level.INFO, "Pending start of new searcher"); } - final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") - + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); + final String displayName = NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.displayName") + + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.finalizeMsg")) : ""); progressGroup = AggregateProgressFactory.createSystemHandle(displayName + (" (" - + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") + ")"), null, new Cancellable() { + + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.pendingMsg") + ")"), null, new Cancellable() { @Override public boolean cancel() { logger.log(Level.INFO, "Cancelling the searcher by user."); if (progressGroup != null) { - progressGroup.setDisplayName(displayName + " (" + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.cancelMsg") + "...)"); + progressGroup.setDisplayName(displayName + " (" + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg") + "...)"); } return SearchRunner.Searcher.this.cancel(true); } @@ -190,15 +222,16 @@ public final class SearchRunner { //block to ensure previous searcher is completely done with doInBackground() //even after previous searcher cancellation, we need to check this - searcherLock.lock(); + //searcherLock.lock(); final StopWatch stopWatch = new StopWatch(); stopWatch.start(); try { logger.log(Level.INFO, "Started a new searcher"); progressGroup.setDisplayName(displayName); //make sure other searchers are not spawned - searcherDone = false; - runSearcher = false; + setJobSearcherDone(jobId, false); + //searcherDone = false; + //runSearcher = false; if (searchTimer.isRunning()) { searchTimer.stop(); } @@ -330,9 +363,9 @@ public final class SearchRunner { //final int hitFiles = newResults.size(); if (!keywordQuery.isLiteral()) { - subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExpHitLbl")); + subjectSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.regExpHitLbl")); } else { - subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLbl")); + subjectSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.kwHitLbl")); } //subjectSb.append("<"); String uniqueKey = null; @@ -350,7 +383,7 @@ public final class SearchRunner { detailsSb.append(""); //hit detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLThLbl")); + detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.kwHitLThLbl")); detailsSb.append(""); detailsSb.append(""); @@ -358,7 +391,7 @@ public final class SearchRunner { attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID()); if (attr != null) { detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.previewThLbl")); + detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.previewThLbl")); detailsSb.append(""); detailsSb.append(""); @@ -366,7 +399,7 @@ public final class SearchRunner { //file detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.fileThLbl")); + detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.fileThLbl")); detailsSb.append(""); detailsSb.append(""); @@ -375,7 +408,7 @@ public final class SearchRunner { //list attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID()); detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.listThLbl")); + detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.listThLbl")); detailsSb.append(""); detailsSb.append(""); @@ -384,7 +417,7 @@ public final class SearchRunner { attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID()); if (attr != null) { detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExThLbl")); + detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.regExThLbl")); detailsSb.append(""); detailsSb.append(""); @@ -422,7 +455,7 @@ public final class SearchRunner { stopWatch.stop(); logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs()); } finally { - searcherLock.unlock(); + //searcherLock.unlock(); } } @@ -462,9 +495,6 @@ public final class SearchRunner { } - //perform all essential cleanup that needs to be done right AFTER doInBackground() returns - //without relying on done() method that is not guaranteed to run after background thread completes - //NEED to call this method always right before doInBackground() returns /** * Performs the cleanup that needs to be done right AFTER * doInBackground() returns without relying on done() method that is not @@ -473,71 +503,16 @@ public final class SearchRunner { */ private void finalizeSearcher() { logger.log(Level.INFO, "Searcher finalizing"); - SwingUtilities.invokeLater(new Runnable() { - @Override - public void run() { - progressGroup.finish(); - } - }); - searcherDone = true; //next currentSearcher can start - - if (finalRun) { - //this is the final searcher - logger.log(Level.INFO, "The final searcher in this ingest done."); - - //run module cleanup - cleanup(); - } else { - //start counting time for a new searcher to start - //unless final searcher is pending - if (finalSearcher == null) { - //we need a new Timer object, because restarting previus will not cause firing of the action - final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; - searchTimer = new Timer(updateIntervalMs, new KeywordSearchIngestModule.SearchTimerAction()); - searchTimer.start(); - } - } +// SwingUtilities.invokeLater(new Runnable() { +// @Override +// public void run() { +// progressGroup.finish(); +// } +// }); + + setJobSearcherDone(jobId, true); } - //calculate new results but substracting results already obtained in this ingest - //update currentResults map with the new results - private Map> filterResults(Map> queryResult, boolean isRegex) { - Map> newResults = new HashMap<>(); - - for (String termResult : queryResult.keySet()) { - List queryTermResults = queryResult.get(termResult); - - //translate to list of IDs that we keep track of - List queryTermResultsIDs = new ArrayList<>(); - for (ContentHit ch : queryTermResults) { - queryTermResultsIDs.add(ch.getId()); - } - - Keyword termResultK = new Keyword(termResult, !isRegex); - List curTermResults = currentResults.get(termResultK); - if (curTermResults == null) { - currentResults.put(termResultK, queryTermResultsIDs); - newResults.put(termResultK, queryTermResults); - } else { - //some AbstractFile hits already exist for this keyword - for (ContentHit res : queryTermResults) { - if (!curTermResults.contains(res.getId())) { - //add to new results - List newResultsFs = newResults.get(termResultK); - if (newResultsFs == null) { - newResultsFs = new ArrayList<>(); - newResults.put(termResultK, newResultsFs); - } - newResultsFs.add(res); - curTermResults.add(res.getId()); - } - } - } - } - - return newResults; - - } } } From 6a78944e6149af3ee92a55906425628b6b32bcc4 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Mon, 31 Mar 2014 16:24:56 -0400 Subject: [PATCH 03/38] Latest compilable experimental SearchRunner. --- .../KeywordSearchIngestModule.java | 27 +-- .../autopsy/keywordsearch/SearchRunner.java | 162 +++++++++++++----- 2 files changed, 132 insertions(+), 57 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 7e9da0f9ba..9a63cd2d55 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -119,7 +119,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private final KeywordSearchJobSettings settings; private boolean initialized = false; private Tika tikaFormatDetector; - + private long jobId = 0; ///@todo where does jobID come from? + private enum IngestStatus { TEXT_INGESTED, /// Text was extracted by knowing file type and text_ingested @@ -377,18 +378,18 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme initialized = false; } -// /** -// * Commits index and notifies listeners of index update -// */ -// private void commit() { -// if (initialized) { -// logger.log(Level.INFO, "Commiting index"); -// ingester.commit(); -// logger.log(Level.INFO, "Index comitted"); -// //signal a potential change in number of text_ingested files -// indexChangeNotify(); -// } -// } + /** + * Commits index and notifies listeners of index update + */ + private void commit() { + if (initialized) { + logger.log(Level.INFO, "Commiting index"); + ingester.commit(); + logger.log(Level.INFO, "Index comitted"); + //signal a potential change in number of text_ingested files + indexChangeNotify(); + } + } /** * Posts inbox message with summary of text_ingested files diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 941f82333e..1f8271dfea 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -20,17 +20,18 @@ package org.sleuthkit.autopsy.keywordsearch; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; -import javax.swing.SwingUtilities; import javax.swing.SwingWorker; import javax.swing.Timer; import org.netbeans.api.progress.aggregate.AggregateProgressFactory; @@ -50,20 +51,26 @@ import org.sleuthkit.datamodel.BlackboardAttribute; /** * Singleton keyword search manager - * Launches search threads for each job and performs commits, - * both on timed intervals. + * Launches search threads for each job and performs commits, both on timed + * intervals. */ public final class SearchRunner { private static final Logger logger = Logger.getLogger(SearchRunner.class.getName()); + private AtomicInteger messageID = new AtomicInteger(0); private static SearchRunner instance = null; private IngestServices services = IngestServices.getInstance(); private Ingester ingester = null; private boolean initialized = false; private Map jobs = new HashMap<>(); - private Timer searchTimer; - + private Timer updateTimer; + private Map> currentResults; + SearchRunner() { - ingester = Server.getIngester(); + ingester = Server.getIngester(); + + final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; + updateTimer = new Timer(updateIntervalMs, new SearchRunner.UpdateTimerAction()); + initialized = true; } @@ -77,22 +84,29 @@ public final class SearchRunner { public synchronized void startJob(long jobId, long dataSourceId, List keywordListNames) { if (!jobs.containsKey(jobId)) { SearchJobInfo jobData = new SearchJobInfo(jobId, dataSourceId, keywordListNames); - jobs.put(jobId, jobData); + jobs.put(jobId, jobData); + } + + if (jobs.size() > 0) { + if (!updateTimer.isRunning()) { + updateTimer.start(); + } } } public void endJob(long jobId) { - SearchJobInfo job; + SearchJobInfo copy; synchronized(this) { - job = jobs.get(jobId); + SearchJobInfo job = jobs.get(jobId); if (job == null) { return; } + copy = new SearchJobInfo(job); jobs.remove(jobId); } - doFinalSearch(job); + doFinalSearch(copy); } - + /** * Commits index and notifies listeners of index update */ @@ -115,29 +129,64 @@ public final class SearchRunner { private void doFinalSearch(SearchJobInfo job) { //cancel searcher timer, ensure unwanted searcher does not start //before we start the final one - if (searchTimer.isRunning()) { - searchTimer.stop(); + if (updateTimer.isRunning()) { + updateTimer.stop(); } - logger.log(Level.INFO, "Running final index commit and search for jobid {0}", jobId); + logger.log(Level.INFO, "Running final index commit and search for jobid {0}", job.getJobId()); commit(); //run one last search as there are probably some new files committed if (!job.getKeywordListNames().isEmpty()) { - SearchRunner.Searcher finalSearcher = new SearchRunner.Searcher(job.getKeywordListNames(), true); + SearchRunner.Searcher finalSearcher = new SearchRunner.Searcher(job, true); finalSearcher.execute(); } } + + /** + * Timer triggered re-search for each job + * (does a single index commit first) + */ + private class UpdateTimerAction implements ActionListener { + private final Logger logger = Logger.getLogger(SearchRunner.UpdateTimerAction.class.getName()); + + @Override + public void actionPerformed(ActionEvent e) { + logger.log(Level.INFO, "Commiting index"); + commit(); + + // Spawn a search thread for each job + ///@todo Don't spawn a new thread if a job still has the previous one running + logger.log(Level.INFO, "Launching searchers"); + for(Entry j : jobs.entrySet()) { + SearchJobInfo copy = new SearchJobInfo(j.getValue()); + Searcher s = new Searcher(copy, true); + } + } + } + + /** + * Simple data structure so we can keep track of keyword lists and data sources for each jobid + * Provides a copy constructor for defensive copying + */ private class SearchJobInfo { private long jobId; private long dataSourceId; - private List keywordListNames = new ArrayList<>(); + private List keywordListNames; public SearchJobInfo(long jobId, long dataSourceId, List keywordListNames) { this.jobId = jobId; this.dataSourceId = dataSourceId; - this.keywordListNames = keywordListNames; + this.keywordListNames = new ArrayList<>(keywordListNames); + } + + public SearchJobInfo(SearchJobInfo src) { + this(src.jobId, src.dataSourceId, src.keywordListNames); + } + + long getJobId() { + return jobId; } long getDataSourceId() { @@ -148,10 +197,6 @@ public final class SearchRunner { return keywordListNames; } } - - private synchronized void setJobSearcherDone(long jobId, boolean flag) { - - } /** * Searcher responsible for searching the current index and writing results @@ -164,24 +209,24 @@ public final class SearchRunner { /** * Searcher has private copies/snapshots of the lists and keywords */ + private SearchJobInfo job; private List keywords; //keywords to search private List keywordLists; // lists currently being searched private Map keywordToList; //keyword to list name mapping private AggregateProgressHandle progressGroup; private final Logger logger = Logger.getLogger(SearchRunner.Searcher.class.getName()); private boolean finalRun = false; - private long jobId; - Searcher(List keywordLists, long jobId) { - this.keywordLists = new ArrayList<>(keywordLists); - this.jobId = jobId; + Searcher(SearchJobInfo job) { + this.job = job; + this.keywordLists = job.getKeywordListNames(); keywords = new ArrayList<>(); keywordToList = new HashMap<>(); //keywords are populated as searcher runs } - Searcher(List keywordLists, long jobId, boolean finalRun) { - this(keywordLists, jobId); + Searcher(SearchJobInfo job, boolean finalRun) { + this(job); this.finalRun = finalRun; } @@ -228,18 +273,9 @@ public final class SearchRunner { try { logger.log(Level.INFO, "Started a new searcher"); progressGroup.setDisplayName(displayName); - //make sure other searchers are not spawned - setJobSearcherDone(jobId, false); - //searcherDone = false; - //runSearcher = false; - if (searchTimer.isRunning()) { - searchTimer.stop(); - } int keywordsSearched = 0; - //updateKeywords(); - for (Keyword keywordQuery : keywords) { if (this.isCancelled()) { logger.log(Level.INFO, "Cancel detected, bailing before new keyword processed: {0}", keywordQuery.getQuery()); @@ -269,7 +305,7 @@ public final class SearchRunner { //limit search to currently ingested data sources //set up a filter with 1 or more image ids OR'ed - final KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, curDataSourceIds); + final KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, job.getDataSourceId()); del.addFilter(dataSourceFilter); Map> queryResult; @@ -290,7 +326,7 @@ public final class SearchRunner { continue; } - // calculate new results but substracting results already obtained in this ingest + // calculate new results by substracting results already obtained in this ingest // this creates a map of each keyword to the list of unique files that have that hit. Map> newResults = filterResults(queryResult, isRegex); @@ -425,7 +461,7 @@ public final class SearchRunner { } detailsSb.append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(hitFile.getParentPath()).append(hitFile.getName()).append("
").append(attr.getValueString()).append("
").append(attr.getValueString()).append("
"); - services.postMessage(IngestMessage.createDataMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), subjectSb.toString(), detailsSb.toString(), uniqueKey, written.getArtifact())); + services.postMessage(IngestMessage.createDataMessage(messageID.incrementAndGet(), KeywordSearchModuleFactory.getModuleName(), subjectSb.toString(), detailsSb.toString(), uniqueKey, written.getArtifact())); } } //for each file hit @@ -469,7 +505,7 @@ public final class SearchRunner { get(); } catch (InterruptedException | ExecutionException e) { logger.log(Level.SEVERE, "Error performing keyword search: " + e.getMessage()); - services.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), "Error performing keyword search", e.getMessage())); + services.postMessage(IngestMessage.createErrorMessage(messageID.incrementAndGet(), KeywordSearchModuleFactory.getModuleName(), "Error performing keyword search", e.getMessage())); } // catch and ignore if we were cancelled catch (java.util.concurrent.CancellationException ex) { } @@ -509,10 +545,48 @@ public final class SearchRunner { // progressGroup.finish(); // } // }); - - setJobSearcherDone(jobId, true); } + //calculate new results but substracting results already obtained in this ingest + //update currentResults map with the new results + private Map> filterResults(Map> queryResult, boolean isRegex) { + Map> newResults = new HashMap<>(); + + for (String termResult : queryResult.keySet()) { + List queryTermResults = queryResult.get(termResult); + + //translate to list of IDs that we keep track of + List queryTermResultsIDs = new ArrayList<>(); + for (ContentHit ch : queryTermResults) { + queryTermResultsIDs.add(ch.getId()); + } + + Keyword termResultK = new Keyword(termResult, !isRegex); + List curTermResults = currentResults.get(termResultK); + if (curTermResults == null) { + currentResults.put(termResultK, queryTermResultsIDs); + newResults.put(termResultK, queryTermResults); + } else { + //some AbstractFile hits already exist for this keyword + for (ContentHit res : queryTermResults) { + if (!curTermResults.contains(res.getId())) { + //add to new results + List newResultsFs = newResults.get(termResultK); + if (newResultsFs == null) { + newResultsFs = new ArrayList<>(); + newResults.put(termResultK, newResultsFs); + } + newResultsFs.add(res); + curTermResults.add(res.getId()); + } + } + } + } + + return newResults; + + } + } } From 96404a16f488453ddc708348a8553a3a4e5a367d Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Mon, 31 Mar 2014 16:56:37 -0400 Subject: [PATCH 04/38] Turn messageID into an AtomicInteger obj --- .../keywordsearch/KeywordSearchIngestModule.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 9a63cd2d55..00985671bd 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.logging.Level; @@ -111,7 +112,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private Set curDataSourceIds; private static final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); //use fairness policy private static final Lock searcherLock = rwLock.writeLock(); - private volatile int messageID = 0; // RJCTODO: Despite volatile, this is not thread safe, uses increment (not atomic) + private AtomicInteger messageID = new AtomicInteger(0); private boolean processedFiles; private SleuthkitCase caseHandle = null; private static List textExtractors; @@ -158,7 +159,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg"); logger.log(Level.SEVERE, msg); String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg); - services.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), msg, details)); + services.postMessage(IngestMessage.createErrorMessage(messageID.getAndIncrement(), KeywordSearchModuleFactory.getModuleName(), msg, details)); throw new IngestModuleException(msg); } } catch (KeywordSearchModuleException ex) { @@ -166,7 +167,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme //this means Solr is not properly initialized String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg"); String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg); - services.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), msg, details)); + services.postMessage(IngestMessage.createErrorMessage(messageID.getAndIncrement(), KeywordSearchModuleFactory.getModuleName(), msg, details)); throw new IngestModuleException(msg); } try { @@ -207,7 +208,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme } } if (!hasKeywordsForSearch) { - services.postMessage(IngestMessage.createWarningMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.noKwInLstMsg"), + services.postMessage(IngestMessage.createWarningMessage(messageID.getAndIncrement(), KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.noKwInLstMsg"), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.onlyIdxKwSkipMsg"))); } @@ -436,7 +437,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme msg.append(""); String indexStats = msg.toString(); logger.log(Level.INFO, "Keyword Indexing Completed: {0}", indexStats); - services.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxResultsLbl"), indexStats)); + services.postMessage(IngestMessage.createMessage(messageID.getAndIncrement(), MessageType.INFO, KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxResultsLbl"), indexStats)); if (error_index > 0) { MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxErrsTitle"), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxErrMsgFiles", error_index)); @@ -963,7 +964,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme } detailsSb.append(""); - services.postMessage(IngestMessage.createDataMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), subjectSb.toString(), detailsSb.toString(), uniqueKey, written.getArtifact())); + services.postMessage(IngestMessage.createDataMessage(messageID.getAndIncrement(), KeywordSearchModuleFactory.getModuleName(), subjectSb.toString(), detailsSb.toString(), uniqueKey, written.getArtifact())); } } //for each file hit @@ -1007,7 +1008,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme get(); } catch (InterruptedException | ExecutionException e) { logger.log(Level.SEVERE, "Error performing keyword search: " + e.getMessage()); - services.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), "Error performing keyword search", e.getMessage())); + services.postMessage(IngestMessage.createErrorMessage(messageID.getAndIncrement(), KeywordSearchModuleFactory.getModuleName(), "Error performing keyword search", e.getMessage())); } // catch and ignore if we were cancelled catch (java.util.concurrent.CancellationException ex) { } From c4cf08fb42c485c72d2af3107e12a1a2c46b64e6 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Mon, 31 Mar 2014 17:52:19 -0400 Subject: [PATCH 05/38] improve locking --- .../autopsy/keywordsearch/SearchRunner.java | 77 ++++++++++++------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 1f8271dfea..c10b474621 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -32,6 +32,7 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; +import javax.swing.SwingUtilities; import javax.swing.SwingWorker; import javax.swing.Timer; import org.netbeans.api.progress.aggregate.AggregateProgressFactory; @@ -50,7 +51,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; /** - * Singleton keyword search manager + * Singleton keyword search manager: * Launches search threads for each job and performs commits, both on timed * intervals. */ @@ -59,11 +60,12 @@ public final class SearchRunner { private AtomicInteger messageID = new AtomicInteger(0); private static SearchRunner instance = null; private IngestServices services = IngestServices.getInstance(); - private Ingester ingester = null; + private Ingester ingester = null; //guarded by "this" private boolean initialized = false; - private Map jobs = new HashMap<>(); private Timer updateTimer; - private Map> currentResults; + private Map> currentResults; + private Map jobs = new HashMap<>(); //guarded by "this" + SearchRunner() { ingester = Server.getIngester(); @@ -74,6 +76,10 @@ public final class SearchRunner { initialized = true; } + /** + * + * @return the singleton object + */ public static synchronized SearchRunner getInstance() { if (instance == null) { instance = new SearchRunner(); @@ -81,6 +87,12 @@ public final class SearchRunner { return instance; } + /** + * + * @param jobId + * @param dataSourceId + * @param keywordListNames + */ public synchronized void startJob(long jobId, long dataSourceId, List keywordListNames) { if (!jobs.containsKey(jobId)) { SearchJobInfo jobData = new SearchJobInfo(jobId, dataSourceId, keywordListNames); @@ -94,6 +106,10 @@ public final class SearchRunner { } } + /** + * + * @param jobId + */ public void endJob(long jobId) { SearchJobInfo copy; synchronized(this) { @@ -104,6 +120,8 @@ public final class SearchRunner { copy = new SearchJobInfo(job); jobs.remove(jobId); } + + commit(); doFinalSearch(copy); } @@ -113,10 +131,12 @@ public final class SearchRunner { private void commit() { if (initialized) { logger.log(Level.INFO, "Commiting index"); - ingester.commit(); + synchronized(this) { + ingester.commit(); + } logger.log(Level.INFO, "Index comitted"); - //signal a potential change in number of text_ingested files + // Signal a potential change in number of text_ingested files try { final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles(); KeywordSearch.fireNumIndexedFilesChange(null, new Integer(numIndexedFiles)); @@ -126,17 +146,19 @@ public final class SearchRunner { } } + /** + * Passes arg directly into a Searcher, so providing a copy is a good idea. + * @param job + */ private void doFinalSearch(SearchJobInfo job) { - //cancel searcher timer, ensure unwanted searcher does not start - //before we start the final one + // Cancel timer to ensure unwanted searchers do not start before we + // start the final one if (updateTimer.isRunning()) { updateTimer.stop(); } - logger.log(Level.INFO, "Running final index commit and search for jobid {0}", job.getJobId()); - commit(); - - //run one last search as there are probably some new files committed + // Run one last search as there are probably some new files committed + logger.log(Level.INFO, "Running final search for jobid {0}", job.getJobId()); if (!job.getKeywordListNames().isEmpty()) { SearchRunner.Searcher finalSearcher = new SearchRunner.Searcher(job, true); finalSearcher.execute(); @@ -145,23 +167,24 @@ public final class SearchRunner { /** - * Timer triggered re-search for each job - * (does a single index commit first) + * Timer triggered re-search for each job (does a single index commit first) */ private class UpdateTimerAction implements ActionListener { private final Logger logger = Logger.getLogger(SearchRunner.UpdateTimerAction.class.getName()); @Override public void actionPerformed(ActionEvent e) { - logger.log(Level.INFO, "Commiting index"); commit(); - // Spawn a search thread for each job - ///@todo Don't spawn a new thread if a job still has the previous one running - logger.log(Level.INFO, "Launching searchers"); - for(Entry j : jobs.entrySet()) { - SearchJobInfo copy = new SearchJobInfo(j.getValue()); - Searcher s = new Searcher(copy, true); + synchronized(SearchRunner.this) { + // Spawn a search thread for each job + ///@todo Don't spawn a new thread if a job still has the previous one running + logger.log(Level.INFO, "Launching searchers"); + for(Entry j : jobs.entrySet()) { + SearchJobInfo copy = new SearchJobInfo(j.getValue()); + Searcher s = new Searcher(copy, true); + s.execute(); + } } } } @@ -539,12 +562,12 @@ public final class SearchRunner { */ private void finalizeSearcher() { logger.log(Level.INFO, "Searcher finalizing"); -// SwingUtilities.invokeLater(new Runnable() { -// @Override -// public void run() { -// progressGroup.finish(); -// } -// }); + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + progressGroup.finish(); + } + }); } //calculate new results but substracting results already obtained in this ingest From ed9c83acf59f2c350fce133098f419b43fefd76a Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Mon, 31 Mar 2014 18:06:48 -0400 Subject: [PATCH 06/38] Added addKeywordListToAllJobs method --- .../autopsy/keywordsearch/SearchRunner.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index c10b474621..0361195fae 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -125,6 +125,19 @@ public final class SearchRunner { doFinalSearch(copy); } + /** + * Add this list to all of the jobs + * @param keywordListName + */ + public void addKeywordListToAllJobs(String keywordListName) { + logger.log(Level.INFO, "Adding keyword list {0} to all jobs", keywordListName); + synchronized(this) { + for(Entry j : jobs.entrySet()) { + j.getValue().getKeywordListNames().add(keywordListName); + } + } + } + /** * Commits index and notifies listeners of index update */ @@ -175,11 +188,11 @@ public final class SearchRunner { @Override public void actionPerformed(ActionEvent e) { commit(); - + + logger.log(Level.INFO, "Launching searchers"); synchronized(SearchRunner.this) { // Spawn a search thread for each job ///@todo Don't spawn a new thread if a job still has the previous one running - logger.log(Level.INFO, "Launching searchers"); for(Entry j : jobs.entrySet()) { SearchJobInfo copy = new SearchJobInfo(j.getValue()); Searcher s = new Searcher(copy, true); From ce2d42542b1004cf6429303fa2fe9e5efdad5b3d Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 1 Apr 2014 11:54:45 -0400 Subject: [PATCH 07/38] Final search now blocks until that thread is done. --- .../sleuthkit/autopsy/keywordsearch/SearchRunner.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 0361195fae..4f72cec793 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -39,6 +39,7 @@ import org.netbeans.api.progress.aggregate.AggregateProgressFactory; import org.netbeans.api.progress.aggregate.AggregateProgressHandle; import org.netbeans.api.progress.aggregate.ProgressContributor; import org.openide.util.Cancellable; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.EscapeUtil; import org.sleuthkit.autopsy.coreutils.Logger; @@ -175,7 +176,13 @@ public final class SearchRunner { if (!job.getKeywordListNames().isEmpty()) { SearchRunner.Searcher finalSearcher = new SearchRunner.Searcher(job, true); finalSearcher.execute(); - } + try { + // block until the search is complete + finalSearcher.get(); + } catch (InterruptedException | ExecutionException ex) { + logger.log(Level.WARNING, "Job {1} final search thread failed: {2}", new Object[]{job.getJobId(), ex}); + } + } } From a5d5691b16b037aa55e6fe5d47c174ad900da475 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 1 Apr 2014 14:57:27 -0400 Subject: [PATCH 08/38] Don't start a searcher thread unless that job has keyword lists. --- .../autopsy/keywordsearch/SearchRunner.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 4f72cec793..683d0ea337 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -39,7 +39,6 @@ import org.netbeans.api.progress.aggregate.AggregateProgressFactory; import org.netbeans.api.progress.aggregate.AggregateProgressHandle; import org.netbeans.api.progress.aggregate.ProgressContributor; import org.openide.util.Cancellable; -import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.EscapeUtil; import org.sleuthkit.autopsy.coreutils.Logger; @@ -201,17 +200,19 @@ public final class SearchRunner { // Spawn a search thread for each job ///@todo Don't spawn a new thread if a job still has the previous one running for(Entry j : jobs.entrySet()) { - SearchJobInfo copy = new SearchJobInfo(j.getValue()); - Searcher s = new Searcher(copy, true); - s.execute(); + if (!j.getValue().getKeywordListNames().isEmpty()) { + SearchJobInfo copy = new SearchJobInfo(j.getValue()); + Searcher s = new Searcher(copy, true); + s.execute(); + } } } } } /** - * Simple data structure so we can keep track of keyword lists and data sources for each jobid - * Provides a copy constructor for defensive copying + * Simple data structure so we can keep track of keyword lists and data + * sources for each jobid */ private class SearchJobInfo { private long jobId; From d7f255044c5aa26ec827a7f692eb1854922d7444 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 1 Apr 2014 15:41:56 -0400 Subject: [PATCH 09/38] Concurrency refactoring; SearchJobInfo now controls currentResults. --- .../autopsy/keywordsearch/SearchRunner.java | 69 +++++++++++-------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 683d0ea337..8a57e9b461 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -63,9 +63,7 @@ public final class SearchRunner { private Ingester ingester = null; //guarded by "this" private boolean initialized = false; private Timer updateTimer; - private Map> currentResults; - private Map jobs = new HashMap<>(); //guarded by "this" - + private Map jobs = new HashMap<>(); //guarded by "this" SearchRunner() { ingester = Server.getIngester(); @@ -111,18 +109,17 @@ public final class SearchRunner { * @param jobId */ public void endJob(long jobId) { - SearchJobInfo copy; + SearchJobInfo job; synchronized(this) { - SearchJobInfo job = jobs.get(jobId); + job = jobs.get(jobId); if (job == null) { return; } - copy = new SearchJobInfo(job); jobs.remove(jobId); } commit(); - doFinalSearch(copy); + doFinalSearch(job); } /** @@ -133,7 +130,7 @@ public final class SearchRunner { logger.log(Level.INFO, "Adding keyword list {0} to all jobs", keywordListName); synchronized(this) { for(Entry j : jobs.entrySet()) { - j.getValue().getKeywordListNames().add(keywordListName); + j.getValue().addKeywordListName(keywordListName); } } } @@ -160,7 +157,7 @@ public final class SearchRunner { } /** - * Passes arg directly into a Searcher, so providing a copy is a good idea. + * * @param job */ private void doFinalSearch(SearchJobInfo job) { @@ -201,8 +198,7 @@ public final class SearchRunner { ///@todo Don't spawn a new thread if a job still has the previous one running for(Entry j : jobs.entrySet()) { if (!j.getValue().getKeywordListNames().isEmpty()) { - SearchJobInfo copy = new SearchJobInfo(j.getValue()); - Searcher s = new Searcher(copy, true); + Searcher s = new Searcher(j.getValue(), true); s.execute(); } } @@ -211,34 +207,51 @@ public final class SearchRunner { } /** - * Simple data structure so we can keep track of keyword lists and data - * sources for each jobid + * Data structure to keep track of keyword lists, current results, and search + * running status for each jobid */ private class SearchJobInfo { - private long jobId; - private long dataSourceId; - private List keywordListNames; - + private final long jobId; + private final long dataSourceId; + // mutable state: + private volatile boolean workerRunning; + private List keywordListNames; //guarded by SearchJobInfo.this + private Map> currentResults; //guarded by SearchJobInfo.this + public SearchJobInfo(long jobId, long dataSourceId, List keywordListNames) { this.jobId = jobId; this.dataSourceId = dataSourceId; this.keywordListNames = new ArrayList<>(keywordListNames); + currentResults = new HashMap<>(); + workerRunning = false; } - - public SearchJobInfo(SearchJobInfo src) { - this(src.jobId, src.dataSourceId, src.keywordListNames); - } - - long getJobId() { + + public long getJobId() { return jobId; } - long getDataSourceId() { + public long getDataSourceId() { return dataSourceId; } - List getKeywordListNames() { - return keywordListNames; + public synchronized List getKeywordListNames() { + return new ArrayList<>(keywordListNames); + } + + public synchronized void addKeywordListName(String keywordListName) { + keywordListNames.add(keywordListName); + } + + public synchronized List currentKeywordResults(Keyword k) { + return currentResults.get(k); + } + + public synchronized void addKeywordResults(Keyword k, List resultsIDs) { + currentResults.put(k, resultsIDs); + } + + boolean isWorkerRunning() { + return workerRunning; } } @@ -606,9 +619,9 @@ public final class SearchRunner { } Keyword termResultK = new Keyword(termResult, !isRegex); - List curTermResults = currentResults.get(termResultK); + List curTermResults = job.currentKeywordResults(termResultK); if (curTermResults == null) { - currentResults.put(termResultK, queryTermResultsIDs); + job.addKeywordResults(termResultK, queryTermResultsIDs); newResults.put(termResultK, queryTermResults); } else { //some AbstractFile hits already exist for this keyword From 2c006bec7d4b0297c0d3f20d14db5c3d15d346fe Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 1 Apr 2014 16:15:43 -0400 Subject: [PATCH 10/38] Don't start a new worker if the workerRunning flag is set --- .../autopsy/keywordsearch/SearchRunner.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 8a57e9b461..2b77ceef6a 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -169,7 +169,7 @@ public final class SearchRunner { // Run one last search as there are probably some new files committed logger.log(Level.INFO, "Running final search for jobid {0}", job.getJobId()); - if (!job.getKeywordListNames().isEmpty()) { + if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) { SearchRunner.Searcher finalSearcher = new SearchRunner.Searcher(job, true); finalSearcher.execute(); try { @@ -195,11 +195,12 @@ public final class SearchRunner { logger.log(Level.INFO, "Launching searchers"); synchronized(SearchRunner.this) { // Spawn a search thread for each job - ///@todo Don't spawn a new thread if a job still has the previous one running for(Entry j : jobs.entrySet()) { - if (!j.getValue().getKeywordListNames().isEmpty()) { - Searcher s = new Searcher(j.getValue(), true); + SearchJobInfo job = j.getValue(); + if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) { + Searcher s = new Searcher(job, true); s.execute(); + job.setWorkerRunning(true); } } } @@ -250,9 +251,13 @@ public final class SearchRunner { currentResults.put(k, resultsIDs); } - boolean isWorkerRunning() { + public boolean isWorkerRunning() { return workerRunning; } + + public void setWorkerRunning(boolean flag) { + workerRunning = flag; + } } /** @@ -322,9 +327,6 @@ public final class SearchRunner { progressGroup.start(); - //block to ensure previous searcher is completely done with doInBackground() - //even after previous searcher cancellation, we need to check this - //searcherLock.lock(); final StopWatch stopWatch = new StopWatch(); stopWatch.start(); try { @@ -546,6 +548,7 @@ public final class SearchRunner { try { finalizeSearcher(); stopWatch.stop(); + job.setWorkerRunning(false); logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs()); } finally { //searcherLock.unlock(); @@ -584,15 +587,12 @@ public final class SearchRunner { this.keywordToList.put(k.getQuery(), list); } } - - } /** * Performs the cleanup that needs to be done right AFTER * doInBackground() returns without relying on done() method that is not - * guaranteed to run after background thread completes REQUIRED to call - * this method always right before doInBackground() returns + * guaranteed to run. */ private void finalizeSearcher() { logger.log(Level.INFO, "Searcher finalizing"); From 93e466a0b6417769796a2ffe0fa7fd1e76e61941 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 1 Apr 2014 16:40:16 -0400 Subject: [PATCH 11/38] doFinalSearch() should now wait if the given job still has a search worker running before launching the final search worker. --- .../autopsy/keywordsearch/SearchRunner.java | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 2b77ceef6a..3de92d4ea1 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -169,10 +169,17 @@ public final class SearchRunner { // Run one last search as there are probably some new files committed logger.log(Level.INFO, "Running final search for jobid {0}", job.getJobId()); - if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) { - SearchRunner.Searcher finalSearcher = new SearchRunner.Searcher(job, true); - finalSearcher.execute(); - try { + if (!job.getKeywordListNames().isEmpty()) { + try { + // In case this job still has a worker running, wait for it to finish + synchronized(this) { + while(job.isWorkerRunning()) { + this.wait(); + } + } + + SearchRunner.Searcher finalSearcher = new SearchRunner.Searcher(job, true); + finalSearcher.execute(); // block until the search is complete finalSearcher.get(); } catch (InterruptedException | ExecutionException ex) { @@ -198,7 +205,7 @@ public final class SearchRunner { for(Entry j : jobs.entrySet()) { SearchJobInfo job = j.getValue(); if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) { - Searcher s = new Searcher(job, true); + Searcher s = new Searcher(job); s.execute(); job.setWorkerRunning(true); } @@ -256,7 +263,7 @@ public final class SearchRunner { } public void setWorkerRunning(boolean flag) { - workerRunning = flag; + workerRunning = flag; } } @@ -549,6 +556,12 @@ public final class SearchRunner { finalizeSearcher(); stopWatch.stop(); job.setWorkerRunning(false); + + // In case the enclosing class instance is waiting on this worker to be done + synchronized(SearchRunner.this) { + SearchRunner.this.notify(); + } + logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs()); } finally { //searcherLock.unlock(); From 3ebe81c6fac54724b38d993edd18d293b7c5252b Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 1 Apr 2014 17:27:23 -0400 Subject: [PATCH 12/38] A bit of cleanup --- .../autopsy/keywordsearch/SearchRunner.java | 32 +++++-------------- 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 3de92d4ea1..9c864d558e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -17,7 +17,6 @@ * limitations under the License. */ - package org.sleuthkit.autopsy.keywordsearch; import java.awt.event.ActionEvent; @@ -126,13 +125,11 @@ public final class SearchRunner { * Add this list to all of the jobs * @param keywordListName */ - public void addKeywordListToAllJobs(String keywordListName) { + public synchronized void addKeywordListToAllJobs(String keywordListName) { logger.log(Level.INFO, "Adding keyword list {0} to all jobs", keywordListName); - synchronized(this) { - for(Entry j : jobs.entrySet()) { - j.getValue().addKeywordListName(keywordListName); - } - } + for(Entry j : jobs.entrySet()) { + j.getValue().addKeywordListName(keywordListName); + } } /** @@ -309,8 +306,8 @@ public final class SearchRunner { final String displayName = NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.displayName") + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.finalizeMsg")) : ""); - progressGroup = AggregateProgressFactory.createSystemHandle(displayName + (" (" - + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.pendingMsg") + ")"), null, new Cancellable() { + final String pgDisplayName = displayName + (" (" + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.pendingMsg") + ")"); + progressGroup = AggregateProgressFactory.createSystemHandle(pgDisplayName, null, new Cancellable() { @Override public boolean cancel() { logger.log(Level.INFO, "Cancelling the searcher by user."); @@ -326,8 +323,7 @@ public final class SearchRunner { ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()]; int i = 0; for (Keyword keywordQuery : keywords) { - subProgresses[i] = - AggregateProgressFactory.createProgressContributor(keywordQuery.getQuery()); + subProgresses[i] = AggregateProgressFactory.createProgressContributor(keywordQuery.getQuery()); progressGroup.addContributor(subProgresses[i]); i++; } @@ -358,7 +354,6 @@ public final class SearchRunner { subProgresses[keywordsSearched - 1].finish(); } - KeywordSearchQuery del = null; boolean isRegex = !keywordQuery.isLiteral(); @@ -413,8 +408,7 @@ public final class SearchRunner { } subProgresses[keywordsSearched].progress(listName + ": " + queryDisplayStr, unitProgress); - - /* cycle through the keywords returned -- only one unless it was a regexp */ + // cycle through the keywords returned -- only one unless it was a regexp for (final Keyword hitTerm : newResults.keySet()) { //checking for cancellation between results if (this.isCancelled()) { @@ -428,7 +422,6 @@ public final class SearchRunner { hitDisplayStr = hitDisplayStr.substring(0, 49) + "..."; } subProgresses[keywordsSearched].progress(listName + ": " + hitDisplayStr, unitProgress); - //subProgresses[keywordsSearched].progress(unitProgress); // this returns the unique files in the set with the first chunk that has a hit Map contentHitsFlattened = ContentHit.flattenResults(newResults.get(hitTerm)); @@ -462,14 +455,12 @@ public final class SearchRunner { if (list.getIngestMessages()) { StringBuilder subjectSb = new StringBuilder(); StringBuilder detailsSb = new StringBuilder(); - //final int hitFiles = newResults.size(); if (!keywordQuery.isLiteral()) { subjectSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.regExpHitLbl")); } else { subjectSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.kwHitLbl")); } - //subjectSb.append("<"); String uniqueKey = null; BlackboardAttribute attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID()); if (attr != null) { @@ -478,9 +469,6 @@ public final class SearchRunner { uniqueKey = keyword.toLowerCase(); } - //subjectSb.append(">"); - //String uniqueKey = queryStr; - //details detailsSb.append(""); //hit @@ -496,17 +484,14 @@ public final class SearchRunner { detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.previewThLbl")); detailsSb.append(""); detailsSb.append(""); - } //file detailsSb.append(""); detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.fileThLbl")); detailsSb.append(""); - detailsSb.append(""); - //list attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID()); detailsSb.append(""); @@ -522,7 +507,6 @@ public final class SearchRunner { detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.regExThLbl")); detailsSb.append(""); detailsSb.append(""); - } } detailsSb.append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(hitFile.getParentPath()).append(hitFile.getName()).append("
").append(attr.getValueString()).append("
"); From bafbd19ade193b54b8111cf47a2b034610f1bf7b Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 1 Apr 2014 18:21:00 -0400 Subject: [PATCH 13/38] Remove stuff from KeywordSearchIngestModule that is now in SearchRunner, and add calls to SearchRunner. --- .../KeywordSearchIngestModule.java | 589 +----------------- .../autopsy/keywordsearch/SearchRunner.java | 21 + 2 files changed, 34 insertions(+), 576 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 00985671bd..0a7238099b 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -98,20 +98,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private static final Logger logger = Logger.getLogger(KeywordSearchIngestModule.class.getName()); private IngestServices services = IngestServices.getInstance(); private Ingester ingester = null; - private volatile boolean commitIndex = false; //whether to commit index next time - private volatile boolean runSearcher = false; //whether to run searcher next time - private Timer commitTimer; - private Timer searchTimer; private Indexer indexer; - private Searcher currentSearcher; - private Searcher finalSearcher; - private volatile boolean searcherDone = true; //mark as done, until it's inited - private Map> currentResults; //only search images from current ingest, not images previously ingested/indexed //accessed read-only by searcher thread - private Set curDataSourceIds; - private static final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); //use fairness policy - private static final Lock searcherLock = rwLock.writeLock(); private AtomicInteger messageID = new AtomicInteger(0); private boolean processedFiles; private SleuthkitCase caseHandle = null; @@ -120,8 +109,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private final KeywordSearchJobSettings settings; private boolean initialized = false; private Tika tikaFormatDetector; - private long jobId = 0; ///@todo where does jobID come from? - + private long jobId = 0; ///@todo get from IngestJobContext + private long dataSourceId; + private enum IngestStatus { TEXT_INGESTED, /// Text was extracted by knowing file type and text_ingested @@ -213,25 +203,14 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme } processedFiles = false; - searcherDone = true; //make sure to start the initial currentSearcher - //keeps track of all results per run not to repeat reporting the same hits - currentResults = new HashMap<>(); - - curDataSourceIds = new HashSet<>(); indexer = new Indexer(); final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; logger.log(Level.INFO, "Using commit interval (ms): {0}", updateIntervalMs); logger.log(Level.INFO, "Using searcher interval (ms): {0}", updateIntervalMs); - - commitTimer = new Timer(updateIntervalMs, new CommitTimerAction()); - searchTimer = new Timer(updateIntervalMs, new SearchTimerAction()); - + initialized = true; - - commitTimer.start(); - searchTimer.start(); } @Override @@ -245,8 +224,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme } try { //add data source id of the file to the set, keeping track of images being ingested - final long fileSourceId = caseHandle.getFileDataSource(abstractFile); - curDataSourceIds.add(fileSourceId); + ///@todo this should come from IngestJobContext + dataSourceId = caseHandle.getFileDataSource(abstractFile); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error getting image id of file processed by keyword search: " + abstractFile.getName(), ex); @@ -265,12 +244,13 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme processedFiles = true; - //check if it's time to commit after previous processing - checkRunCommitSearch(); - //index the file and content (if the content is supported) indexer.indexFile(abstractFile, true); + // Start searching if it hasn't started already + List keywordListNames = settings.getNamesOfEnabledKeyWordLists(); + SearchRunner.getInstance().startJob(jobId, dataSourceId, keywordListNames); + return ProcessResult.OK; } @@ -288,34 +268,12 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme stop(); return; } -// -// commitTimer.stop(); -// -// //NOTE, we let the 1 before last searcher complete fully, and enqueue the last one -// -// //cancel searcher timer, ensure unwanted searcher does not start -// //before we start the final one -// if (searchTimer.isRunning()) { -// searchTimer.stop(); -// } -// runSearcher = false; -// -// logger.log(Level.INFO, "Running final index commit and search"); -// //final commit -// commit(); // Remove from the search list and trigger final commit and final search SearchRunner.getInstance().endJob(jobId); postIndexSummary(); - //run one last search as there are probably some new files committed -// List keywordLists = settings.getNamesOfEnabledKeyWordLists(); -// if (!keywordLists.isEmpty() && processedFiles == true) { -// finalSearcher = new Searcher(keywordLists, true); //final searcher run -// finalSearcher.execute(); -// } - //log number of files / chunks in index //signal a potential change in number of text_ingested files try { @@ -326,8 +284,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme } catch (NoOpenCoreException | KeywordSearchModuleException ex) { logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files/chunks: ", ex); } - - //cleanup done in final searcher } /** @@ -336,22 +292,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private void stop() { logger.log(Level.INFO, "stop()"); - //stop timer - commitTimer.stop(); - //stop currentSearcher - if (currentSearcher != null) { - currentSearcher.cancel(true); - } - - //cancel searcher timer, ensure unwanted searcher does not start - if (searchTimer.isRunning()) { - searchTimer.stop(); - } - runSearcher = false; - - //commit uncommited files, don't search again - commit(); - + SearchRunner.getInstance().stopJob(jobId); + cleanup(); } @@ -360,15 +302,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme */ private void cleanup() { ingestStatus.clear(); - currentResults.clear(); - curDataSourceIds.clear(); - currentSearcher = null; - //finalSearcher = null; //do not collect, might be finalizing - - commitTimer.stop(); - searchTimer.stop(); - commitTimer = null; - //searchTimer = null; // do not collect, final searcher might still be running, in which case it throws an exception textExtractors.clear(); textExtractors = null; @@ -379,19 +312,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme initialized = false; } - /** - * Commits index and notifies listeners of index update - */ - private void commit() { - if (initialized) { - logger.log(Level.INFO, "Commiting index"); - ingester.commit(); - logger.log(Level.INFO, "Index comitted"); - //signal a potential change in number of text_ingested files - indexChangeNotify(); - } - } - /** * Posts inbox message with summary of text_ingested files */ @@ -423,7 +343,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme error_io++; break; default: - ; + ; } } @@ -447,73 +367,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme } } - /** - * Helper method to notify listeners on index update - */ - private void indexChangeNotify() { - //signal a potential change in number of text_ingested files - try { - final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles(); - KeywordSearch.fireNumIndexedFilesChange(null, new Integer(numIndexedFiles)); - } catch (NoOpenCoreException | KeywordSearchModuleException ex) { - logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files: ", ex); - } - } - - /** - * Check if time to commit, if so, run commit. Then run search if search - * timer is also set. - */ - void checkRunCommitSearch() { - if (commitIndex) { - logger.log(Level.INFO, "Commiting index"); - commit(); - commitIndex = false; - - //after commit, check if time to run searcher - //NOTE commit/searcher timings don't need to align - //in worst case, we will run search next time after commit timer goes off, or at the end of ingest - if (searcherDone && runSearcher) { - //start search if previous not running - List keywordLists = settings.getNamesOfEnabledKeyWordLists(); - if (!keywordLists.isEmpty()) { - currentSearcher = new Searcher(keywordLists); - currentSearcher.execute();//searcher will stop timer and restart timer when done - } - } - } - } - - /** - * CommitTimerAction to run by commitTimer Sets a flag to indicate we are - * ready for commit - */ - private class CommitTimerAction implements ActionListener { - - private final Logger logger = Logger.getLogger(CommitTimerAction.class.getName()); - - @Override - public void actionPerformed(ActionEvent e) { - commitIndex = true; - logger.log(Level.INFO, "CommitTimer awake"); - } - } - - /** - * SearchTimerAction to run by searchTimer Sets a flag to indicate we are - * ready to search - */ - private class SearchTimerAction implements ActionListener { - - private final Logger logger = Logger.getLogger(SearchTimerAction.class.getName()); - - @Override - public void actionPerformed(ActionEvent e) { - runSearcher = true; - logger.log(Level.INFO, "SearchTimer awake"); - } - } - /** * File indexer, processes and indexes known/allocated files, * unknown/unallocated files and directories accordingly @@ -695,420 +548,4 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme } } - /** - * Searcher responsible for searching the current index and writing results - * to blackboard and the inbox. Also, posts results to listeners as Ingest - * data events. Searches entire index, and keeps track of only new results - * to report and save. Runs as a background thread. - */ - private final class Searcher extends SwingWorker { - - /** - * Searcher has private copies/snapshots of the lists and keywords - */ - private List keywords; //keywords to search - private List keywordLists; // lists currently being searched - private Map keywordToList; //keyword to list name mapping - private AggregateProgressHandle progressGroup; - private final Logger logger = Logger.getLogger(Searcher.class.getName()); - private boolean finalRun = false; - - Searcher(List keywordLists) { - this.keywordLists = new ArrayList<>(keywordLists); - this.keywords = new ArrayList<>(); - this.keywordToList = new HashMap<>(); - //keywords are populated as searcher runs - } - - Searcher(List keywordLists, boolean finalRun) { - this(keywordLists); - this.finalRun = finalRun; - } - - @Override - protected Object doInBackground() throws Exception { - if (finalRun) { - logger.log(Level.INFO, "Pending start of new (final) searcher"); - } else { - logger.log(Level.INFO, "Pending start of new searcher"); - } - - final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") - + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); - progressGroup = AggregateProgressFactory.createSystemHandle(displayName + (" (" - + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") + ")"), null, new Cancellable() { - @Override - public boolean cancel() { - logger.log(Level.INFO, "Cancelling the searcher by user."); - if (progressGroup != null) { - progressGroup.setDisplayName(displayName + " (" + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.cancelMsg") + "...)"); - } - return Searcher.this.cancel(true); - } - }, null); - - updateKeywords(); - - ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()]; - int i = 0; - for (Keyword keywordQuery : keywords) { - subProgresses[i] = - AggregateProgressFactory.createProgressContributor(keywordQuery.getQuery()); - progressGroup.addContributor(subProgresses[i]); - i++; - } - - progressGroup.start(); - - //block to ensure previous searcher is completely done with doInBackground() - //even after previous searcher cancellation, we need to check this - searcherLock.lock(); - final StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - try { - logger.log(Level.INFO, "Started a new searcher"); - progressGroup.setDisplayName(displayName); - //make sure other searchers are not spawned - searcherDone = false; - runSearcher = false; - if (searchTimer.isRunning()) { - searchTimer.stop(); - } - - int keywordsSearched = 0; - - //updateKeywords(); - - for (Keyword keywordQuery : keywords) { - if (this.isCancelled()) { - logger.log(Level.INFO, "Cancel detected, bailing before new keyword processed: {0}", keywordQuery.getQuery()); - return null; - } - - final String queryStr = keywordQuery.getQuery(); - final KeywordList list = keywordToList.get(queryStr); - final String listName = list.getName(); - - //new subProgress will be active after the initial query - //when we know number of hits to start() with - if (keywordsSearched > 0) { - subProgresses[keywordsSearched - 1].finish(); - } - - - KeywordSearchQuery del = null; - - boolean isRegex = !keywordQuery.isLiteral(); - if (isRegex) { - del = new TermComponentQuery(keywordQuery); - } else { - del = new LuceneQuery(keywordQuery); - del.escape(); - } - - //limit search to currently ingested data sources - //set up a filter with 1 or more image ids OR'ed - final KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, curDataSourceIds); - del.addFilter(dataSourceFilter); - - Map> queryResult; - - try { - queryResult = del.performQuery(); - } catch (NoOpenCoreException ex) { - logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), ex); - //no reason to continue with next query if recovery failed - //or wait for recovery to kick in and run again later - //likely case has closed and threads are being interrupted - return null; - } catch (CancellationException e) { - logger.log(Level.INFO, "Cancel detected, bailing during keyword query: {0}", keywordQuery.getQuery()); - return null; - } catch (Exception e) { - logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), e); - continue; - } - - // calculate new results but substracting results already obtained in this ingest - // this creates a map of each keyword to the list of unique files that have that hit. - Map> newResults = filterResults(queryResult, isRegex); - - if (!newResults.isEmpty()) { - - //write results to BB - - //new artifacts created, to report to listeners - Collection newArtifacts = new ArrayList<>(); - - //scale progress bar more more granular, per result sub-progress, within per keyword - int totalUnits = newResults.size(); - subProgresses[keywordsSearched].start(totalUnits); - int unitProgress = 0; - String queryDisplayStr = keywordQuery.getQuery(); - if (queryDisplayStr.length() > 50) { - queryDisplayStr = queryDisplayStr.substring(0, 49) + "..."; - } - subProgresses[keywordsSearched].progress(listName + ": " + queryDisplayStr, unitProgress); - - - /* cycle through the keywords returned -- only one unless it was a regexp */ - for (final Keyword hitTerm : newResults.keySet()) { - //checking for cancellation between results - if (this.isCancelled()) { - logger.log(Level.INFO, "Cancel detected, bailing before new hit processed for query: {0}", keywordQuery.getQuery()); - return null; - } - - // update progress display - String hitDisplayStr = hitTerm.getQuery(); - if (hitDisplayStr.length() > 50) { - hitDisplayStr = hitDisplayStr.substring(0, 49) + "..."; - } - subProgresses[keywordsSearched].progress(listName + ": " + hitDisplayStr, unitProgress); - //subProgresses[keywordsSearched].progress(unitProgress); - - // this returns the unique files in the set with the first chunk that has a hit - Map contentHitsFlattened = ContentHit.flattenResults(newResults.get(hitTerm)); - for (final AbstractFile hitFile : contentHitsFlattened.keySet()) { - - // get the snippet for the first hit in the file - String snippet; - final String snippetQuery = KeywordSearchUtil.escapeLuceneQuery(hitTerm.getQuery()); - int chunkId = contentHitsFlattened.get(hitFile); - try { - snippet = LuceneQuery.querySnippet(snippetQuery, hitFile.getId(), chunkId, isRegex, true); - } catch (NoOpenCoreException e) { - logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e); - //no reason to continue - return null; - } catch (Exception e) { - logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e); - continue; - } - - // write the blackboard artifact for this keyword in this file - KeywordWriteResult written = del.writeToBlackBoard(hitTerm.getQuery(), hitFile, snippet, listName); - if (written == null) { - logger.log(Level.WARNING, "BB artifact for keyword hit not written, file: {0}, hit: {1}", new Object[]{hitFile, hitTerm.toString()}); - continue; - } - - newArtifacts.add(written.getArtifact()); - - //generate an ingest inbox message for this keyword in this file - if (list.getIngestMessages()) { - StringBuilder subjectSb = new StringBuilder(); - StringBuilder detailsSb = new StringBuilder(); - //final int hitFiles = newResults.size(); - - if (!keywordQuery.isLiteral()) { - subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExpHitLbl")); - } else { - subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLbl")); - } - //subjectSb.append("<"); - String uniqueKey = null; - BlackboardAttribute attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID()); - if (attr != null) { - final String keyword = attr.getValueString(); - subjectSb.append(keyword); - uniqueKey = keyword.toLowerCase(); - } - - //subjectSb.append(">"); - //String uniqueKey = queryStr; - - //details - detailsSb.append(""); - //hit - detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLThLbl")); - detailsSb.append(""); - detailsSb.append(""); - - //preview - attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID()); - if (attr != null) { - detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.previewThLbl")); - detailsSb.append(""); - detailsSb.append(""); - - } - - //file - detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.fileThLbl")); - detailsSb.append(""); - - detailsSb.append(""); - - - //list - attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID()); - detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.listThLbl")); - detailsSb.append(""); - detailsSb.append(""); - - //regex - if (!keywordQuery.isLiteral()) { - attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID()); - if (attr != null) { - detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExThLbl")); - detailsSb.append(""); - detailsSb.append(""); - - } - } - detailsSb.append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(hitFile.getParentPath()).append(hitFile.getName()).append("
").append(attr.getValueString()).append("
").append(attr.getValueString()).append("
"); - - services.postMessage(IngestMessage.createDataMessage(messageID.getAndIncrement(), KeywordSearchModuleFactory.getModuleName(), subjectSb.toString(), detailsSb.toString(), uniqueKey, written.getArtifact())); - } - } //for each file hit - - ++unitProgress; - - }//for each hit term - - //update artifact browser - if (!newArtifacts.isEmpty()) { - services.fireModuleDataEvent(new ModuleDataEvent(KeywordSearchModuleFactory.getModuleName(), ARTIFACT_TYPE.TSK_KEYWORD_HIT, newArtifacts)); - } - } //if has results - - //reset the status text before it goes away - subProgresses[keywordsSearched].progress(""); - - ++keywordsSearched; - - } //for each keyword - - } //end try block - catch (Exception ex) { - logger.log(Level.WARNING, "searcher exception occurred", ex); - } finally { - try { - finalizeSearcher(); - stopWatch.stop(); - logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs()); - } finally { - searcherLock.unlock(); - } - } - - return null; - } - - @Override - protected void done() { - // call get to see if there were any errors - try { - get(); - } catch (InterruptedException | ExecutionException e) { - logger.log(Level.SEVERE, "Error performing keyword search: " + e.getMessage()); - services.postMessage(IngestMessage.createErrorMessage(messageID.getAndIncrement(), KeywordSearchModuleFactory.getModuleName(), "Error performing keyword search", e.getMessage())); - } // catch and ignore if we were cancelled - catch (java.util.concurrent.CancellationException ex) { - } - } - - /** - * Sync-up the updated keywords from the currently used lists in the XML - */ - private void updateKeywords() { - KeywordSearchListsXML loader = KeywordSearchListsXML.getCurrent(); - - this.keywords.clear(); - this.keywordToList.clear(); - - for (String name : this.keywordLists) { - KeywordList list = loader.getList(name); - for (Keyword k : list.getKeywords()) { - this.keywords.add(k); - this.keywordToList.put(k.getQuery(), list); - } - } - - - } - - //perform all essential cleanup that needs to be done right AFTER doInBackground() returns - //without relying on done() method that is not guaranteed to run after background thread completes - //NEED to call this method always right before doInBackground() returns - /** - * Performs the cleanup that needs to be done right AFTER - * doInBackground() returns without relying on done() method that is not - * guaranteed to run after background thread completes REQUIRED to call - * this method always right before doInBackground() returns - */ - private void finalizeSearcher() { - logger.log(Level.INFO, "Searcher finalizing"); - SwingUtilities.invokeLater(new Runnable() { - @Override - public void run() { - progressGroup.finish(); - } - }); - searcherDone = true; //next currentSearcher can start - - if (finalRun) { - //this is the final searcher - logger.log(Level.INFO, "The final searcher in this ingest done."); - - //run module cleanup - cleanup(); - } else { - //start counting time for a new searcher to start - //unless final searcher is pending - if (finalSearcher == null) { - //we need a new Timer object, because restarting previus will not cause firing of the action - final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; - searchTimer = new Timer(updateIntervalMs, new SearchTimerAction()); - searchTimer.start(); - } - } - } - - //calculate new results but substracting results already obtained in this ingest - //update currentResults map with the new results - private Map> filterResults(Map> queryResult, boolean isRegex) { - Map> newResults = new HashMap<>(); - - for (String termResult : queryResult.keySet()) { - List queryTermResults = queryResult.get(termResult); - - //translate to list of IDs that we keep track of - List queryTermResultsIDs = new ArrayList<>(); - for (ContentHit ch : queryTermResults) { - queryTermResultsIDs.add(ch.getId()); - } - - Keyword termResultK = new Keyword(termResult, !isRegex); - List curTermResults = currentResults.get(termResultK); - if (curTermResults == null) { - currentResults.put(termResultK, queryTermResultsIDs); - newResults.put(termResultK, queryTermResults); - } else { - //some AbstractFile hits already exist for this keyword - for (ContentHit res : queryTermResults) { - if (!curTermResults.contains(res.getId())) { - //add to new results - List newResultsFs = newResults.get(termResultK); - if (newResultsFs == null) { - newResultsFs = new ArrayList<>(); - newResults.put(termResultK, newResultsFs); - } - newResultsFs.add(res); - curTermResults.add(res.getId()); - } - } - } - } - - return newResults; - - } - } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 9c864d558e..3002051c0b 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -121,6 +121,26 @@ public final class SearchRunner { doFinalSearch(job); } + public void stopJob(long jobId) { + SearchJobInfo job; + synchronized(this) { + job = jobs.get(jobId); + if (job == null) { + return; + } + + ///@todo + //stop currentSearcher +// if (currentSearcher != null) { +// currentSearcher.cancel(true); +// } + + jobs.remove(jobId); + } + + commit(); + } + /** * Add this list to all of the jobs * @param keywordListName @@ -201,6 +221,7 @@ public final class SearchRunner { // Spawn a search thread for each job for(Entry j : jobs.entrySet()) { SearchJobInfo job = j.getValue(); + // If no lists or the worker is already running then skip it if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) { Searcher s = new Searcher(job); s.execute(); From 977d47fdf973433daedd8bdc075bc463d461c1c5 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 2 Apr 2014 17:26:44 -0400 Subject: [PATCH 14/38] Keep track of current searcher refs and cancel them in stop(). --- .../autopsy/keywordsearch/SearchRunner.java | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 3002051c0b..990fb87397 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -104,7 +104,8 @@ public final class SearchRunner { } /** - * + * Perform normal finishing of searching for this job, including one last + * commit and search. Blocks until the final search is complete. * @param jobId */ public void endJob(long jobId) { @@ -118,9 +119,15 @@ public final class SearchRunner { } commit(); - doFinalSearch(job); + doFinalSearch(job); //this will block until it's done } + + /** + * Immediate stop and removal of job from SearchRunner. Cancels the + * associated search worker if it's still running. + * @param jobId + */ public void stopJob(long jobId) { SearchJobInfo job; synchronized(this) { @@ -129,11 +136,11 @@ public final class SearchRunner { return; } - ///@todo //stop currentSearcher -// if (currentSearcher != null) { -// currentSearcher.cancel(true); -// } + SearchRunner.Searcher currentSearcher = job.getCurrentSearcher(); + if ((currentSearcher != null) && (!currentSearcher.isDone())) { + currentSearcher.cancel(true); + } jobs.remove(jobId); } @@ -174,7 +181,8 @@ public final class SearchRunner { } /** - * + * A final search waits for any still-running workers, and then executes a + * new one and waits until that is done. * @param job */ private void doFinalSearch(SearchJobInfo job) { @@ -196,9 +204,12 @@ public final class SearchRunner { } SearchRunner.Searcher finalSearcher = new SearchRunner.Searcher(job, true); - finalSearcher.execute(); + job.setCurrentSearcher(finalSearcher); //save the ref + finalSearcher.execute(); //start thread + // block until the search is complete - finalSearcher.get(); + finalSearcher.get(); + } catch (InterruptedException | ExecutionException ex) { logger.log(Level.WARNING, "Job {1} final search thread failed: {2}", new Object[]{job.getJobId(), ex}); } @@ -223,8 +234,9 @@ public final class SearchRunner { SearchJobInfo job = j.getValue(); // If no lists or the worker is already running then skip it if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) { - Searcher s = new Searcher(job); - s.execute(); + Searcher searcher = new Searcher(job); + job.setCurrentSearcher(searcher); //save the ref + searcher.execute(); //start thread job.setWorkerRunning(true); } } @@ -243,6 +255,7 @@ public final class SearchRunner { private volatile boolean workerRunning; private List keywordListNames; //guarded by SearchJobInfo.this private Map> currentResults; //guarded by SearchJobInfo.this + private SearchRunner.Searcher currentSearcher; public SearchJobInfo(long jobId, long dataSourceId, List keywordListNames) { this.jobId = jobId; @@ -250,6 +263,7 @@ public final class SearchRunner { this.keywordListNames = new ArrayList<>(keywordListNames); currentResults = new HashMap<>(); workerRunning = false; + currentSearcher = null; } public long getJobId() { @@ -283,6 +297,14 @@ public final class SearchRunner { public void setWorkerRunning(boolean flag) { workerRunning = flag; } + + public synchronized SearchRunner.Searcher getCurrentSearcher() { + return currentSearcher; + } + + public synchronized void setCurrentSearcher(SearchRunner.Searcher searchRunner) { + currentSearcher = searchRunner; + } } /** @@ -306,7 +328,7 @@ public final class SearchRunner { Searcher(SearchJobInfo job) { this.job = job; - this.keywordLists = job.getKeywordListNames(); + this.keywordLists = job.getKeywordListNames(); keywords = new ArrayList<>(); keywordToList = new HashMap<>(); //keywords are populated as searcher runs From 004b22ffc24913595fa3754ce36d8560ca5fd6fc Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 2 Apr 2014 17:31:06 -0400 Subject: [PATCH 15/38] Remove unused imports --- .../KeywordSearchIngestModule.java | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 0a7238099b..a8e510abdb 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -18,37 +18,19 @@ */ package org.sleuthkit.autopsy.keywordsearch; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.concurrent.CancellationException; -import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; -import javax.swing.SwingUtilities; -import javax.swing.SwingWorker; -import javax.swing.Timer; import org.apache.tika.Tika; -import org.netbeans.api.progress.aggregate.AggregateProgressFactory; -import org.netbeans.api.progress.aggregate.AggregateProgressHandle; -import org.netbeans.api.progress.aggregate.ProgressContributor; -import org.openide.util.Cancellable; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.coreutils.EscapeUtil; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; -import org.sleuthkit.autopsy.coreutils.StopWatch; import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT; import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestServices; @@ -56,11 +38,7 @@ import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; import org.sleuthkit.autopsy.ingest.IngestJobContext; -import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException; -import org.sleuthkit.datamodel.BlackboardArtifact; -import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; -import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.ReadContentInputStream; import org.sleuthkit.datamodel.SleuthkitCase; From 44837e905fc1c7d78a88796d4446f5d275695126 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 2 Apr 2014 18:53:59 -0400 Subject: [PATCH 16/38] Remove calls to module.checkRunCommitSearch(). --- .../autopsy/keywordsearch/AbstractFileHtmlExtract.java | 4 ---- .../autopsy/keywordsearch/AbstractFileStringExtract.java | 4 ---- .../autopsy/keywordsearch/AbstractFileTikaTextExtract.java | 4 ---- 3 files changed, 12 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java index e013ec2fac..7af85f7e3f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java @@ -177,10 +177,6 @@ import org.sleuthkit.datamodel.ReadContentInputStream; + sourceFile.getName() + "' (id: " + sourceFile.getId() + ").", ingEx); throw ingEx; //need to rethrow/return to signal error and move on } - - //check if need invoke commit/search between chunks - //not to delay commit if timer has gone off - module.checkRunCommitSearch(); } } catch (IOException ex) { logger.log(Level.WARNING, "Unable to read content stream from " + sourceFile.getId() + ": " + sourceFile.getName(), ex); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java index 4d82827c6a..6f96ff2495 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java @@ -147,10 +147,6 @@ class AbstractFileStringExtract implements AbstractFileExtract { throw ingEx; //need to rethrow/return to signal error and move on } - //check if need invoke commit/search between chunks - //not to delay commit if timer has gone off - module.checkRunCommitSearch(); - //debug.close(); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java index 4f52805753..e19be18a5c 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java @@ -223,10 +223,6 @@ class AbstractFileTikaTextExtract implements AbstractFileExtract { + sourceFile.getName() + "' (id: " + sourceFile.getId() + ").", ingEx); throw ingEx; //need to rethrow/return to signal error and move on } - - //check if need invoke commit/search between chunks - //not to delay commit if timer has gone off - module.checkRunCommitSearch(); } } catch (IOException ex) { final String msg = "Exception: Unable to read Tika content stream from " + sourceFile.getId() + ": " + sourceFile.getName(); From 6ae82a69bc3d6bbb4bebcf863751304fa7515e81 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 2 Apr 2014 19:19:13 -0400 Subject: [PATCH 17/38] Fix bundle names. --- .../org/sleuthkit/autopsy/keywordsearch/SearchRunner.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 990fb87397..4be60c08b3 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -347,9 +347,9 @@ public final class SearchRunner { logger.log(Level.INFO, "Pending start of new searcher"); } - final String displayName = NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.displayName") - + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.finalizeMsg")) : ""); - final String pgDisplayName = displayName + (" (" + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.pendingMsg") + ")"); + final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") + + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); + final String pgDisplayName = displayName + (" (" + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") + ")"); progressGroup = AggregateProgressFactory.createSystemHandle(pgDisplayName, null, new Cancellable() { @Override public boolean cancel() { From ad6613601133e99a8356aa09a5ef37430f392b2d Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 2 Apr 2014 20:19:30 -0400 Subject: [PATCH 18/38] Placeholders for missing bundle properties. --- .../autopsy/keywordsearch/SearchRunner.java | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 4be60c08b3..cc3cb7a807 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -500,9 +500,11 @@ public final class SearchRunner { StringBuilder detailsSb = new StringBuilder(); if (!keywordQuery.isLiteral()) { - subjectSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.regExpHitLbl")); + //subjectSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.regExpHitLbl")); + subjectSb.append("RegEx"); ///@todo make a bundle prop for this } else { - subjectSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.kwHitLbl")); + //subjectSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.kwHitLbl")); + subjectSb.append("Keyword"); ///@todo make a bundle prop for this } String uniqueKey = null; BlackboardAttribute attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID()); @@ -516,7 +518,8 @@ public final class SearchRunner { detailsSb.append(""); //hit detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.kwHitLThLbl")); + //detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.kwHitLThLbl")); + detailsSb.append("Keyword"); ///@todo make a bundle prop for this detailsSb.append(""); detailsSb.append(""); @@ -524,21 +527,25 @@ public final class SearchRunner { attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID()); if (attr != null) { detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.previewThLbl")); + //detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.previewThLbl")); + detailsSb.append("Preview"); ///@todo make a bundle prop for this detailsSb.append(""); detailsSb.append(""); } //file detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.fileThLbl")); + //detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.fileThLbl")); + detailsSb.append("File"); ///@todo make a bundle prop for this detailsSb.append(""); detailsSb.append(""); //list attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID()); detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.listThLbl")); + //detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.listThLbl")); + detailsSb.append("List"); ///@todo make a bundle prop for this + detailsSb.append(""); detailsSb.append(""); @@ -547,7 +554,9 @@ public final class SearchRunner { attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID()); if (attr != null) { detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.regExThLbl")); + //detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.regExThLbl")); + detailsSb.append("RegEx"); ///@todo make a bundle prop for this + detailsSb.append(""); detailsSb.append(""); } From 5b7c84d3661b8b5c1298fe4723261dc19f2f55db Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Thu, 3 Apr 2014 13:38:53 -0400 Subject: [PATCH 19/38] Get jobID from IngestJobContext arg --- .../autopsy/keywordsearch/KeywordSearchIngestModule.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index a8e510abdb..e748f7befd 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -87,7 +87,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private final KeywordSearchJobSettings settings; private boolean initialized = false; private Tika tikaFormatDetector; - private long jobId = 0; ///@todo get from IngestJobContext + private long jobId; private long dataSourceId; private enum IngestStatus { @@ -114,11 +114,10 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme public void startUp(IngestJobContext context) throws IngestModuleException { logger.log(Level.INFO, "init()"); initialized = false; - + + jobId = context.getJobId(); caseHandle = Case.getCurrentCase().getSleuthkitCase(); - tikaFormatDetector = new Tika(); - ingester = Server.getIngester(); final Server server = KeywordSearch.getServer(); From 799208c2f754f854d31a658e68e64560967e3abd Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Thu, 3 Apr 2014 13:59:51 -0400 Subject: [PATCH 20/38] Missing strings added to Bundle.properties --- .../autopsy/keywordsearch/Bundle.properties | 7 ++++++ .../autopsy/keywordsearch/SearchRunner.java | 23 ++++++------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index 040e9e0c03..e5f6e5e984 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -263,3 +263,10 @@ KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.text_1=1 minute (faster KeywordSearchGlobalSearchSettingsPanel.chunksLabel.text=Chunks in keyword index: KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.toolTipText=5 minutes (overall ingest time will be longer) KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.text=5 minutes (default) +KeywordSearchIngestModule.regExpHitLbl=Reg Ex +KeywordSearchIngestModule.kwHitLbl=Keyword +KeywordSearchIngestModule.kwHitThLbl=Keyword +KeywordSearchIngestModule.previewThLbl=Preview +KeywordSearchIngestModule.fileThLbl=File +KeywordSearchIngestModule.listThLbl=List +KeywordSearchIngestModule.regExThLbl=Reg Ex diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index cc3cb7a807..b71d69dab1 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -500,11 +500,9 @@ public final class SearchRunner { StringBuilder detailsSb = new StringBuilder(); if (!keywordQuery.isLiteral()) { - //subjectSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.regExpHitLbl")); - subjectSb.append("RegEx"); ///@todo make a bundle prop for this + subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExpHitLbl")); } else { - //subjectSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.kwHitLbl")); - subjectSb.append("Keyword"); ///@todo make a bundle prop for this + subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLbl")); } String uniqueKey = null; BlackboardAttribute attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID()); @@ -518,8 +516,7 @@ public final class SearchRunner { detailsSb.append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(hitFile.getParentPath()).append(hitFile.getName()).append("
").append(attr.getValueString()).append("
").append(attr.getValueString()).append("
"); //hit detailsSb.append(""); - //detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.kwHitLThLbl")); - detailsSb.append("Keyword"); ///@todo make a bundle prop for this + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitThLbl")); detailsSb.append(""); detailsSb.append(""); @@ -527,25 +524,21 @@ public final class SearchRunner { attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID()); if (attr != null) { detailsSb.append(""); - //detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.previewThLbl")); - detailsSb.append("Preview"); ///@todo make a bundle prop for this + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.previewThLbl")); detailsSb.append(""); detailsSb.append(""); } //file detailsSb.append(""); - //detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.fileThLbl")); - detailsSb.append("File"); ///@todo make a bundle prop for this + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.fileThLbl")); detailsSb.append(""); detailsSb.append(""); //list attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID()); detailsSb.append(""); - //detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.listThLbl")); - detailsSb.append("List"); ///@todo make a bundle prop for this - + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.listThLbl")); detailsSb.append(""); detailsSb.append(""); @@ -554,9 +547,7 @@ public final class SearchRunner { attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID()); if (attr != null) { detailsSb.append(""); - //detailsSb.append(NbBundle.getMessage(this.getClass(), "SearchRunner.regExThLbl")); - detailsSb.append("RegEx"); ///@todo make a bundle prop for this - + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExThLbl")); detailsSb.append(""); detailsSb.append(""); } From b16dfba6b95c7b73e1444ed9422cf812553b0bfe Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Thu, 3 Apr 2014 15:55:03 -0400 Subject: [PATCH 21/38] Add colon and trailing space to some of the bundle props --- .../src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index e5f6e5e984..a608ce590f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -263,8 +263,8 @@ KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.text_1=1 minute (faster KeywordSearchGlobalSearchSettingsPanel.chunksLabel.text=Chunks in keyword index: KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.toolTipText=5 minutes (overall ingest time will be longer) KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.text=5 minutes (default) -KeywordSearchIngestModule.regExpHitLbl=Reg Ex -KeywordSearchIngestModule.kwHitLbl=Keyword +KeywordSearchIngestModule.regExpHitLbl=Reg Ex hit: +KeywordSearchIngestModule.kwHitLbl=Keyword hit: KeywordSearchIngestModule.kwHitThLbl=Keyword KeywordSearchIngestModule.previewThLbl=Preview KeywordSearchIngestModule.fileThLbl=File From a5c4b9d518c980b8d1e5ec46e221ad1152773735 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Thu, 3 Apr 2014 15:56:42 -0400 Subject: [PATCH 22/38] Adjust final search condition/wait to use its own lock object. --- .../autopsy/keywordsearch/SearchRunner.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index b71d69dab1..c7c65df96d 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -63,6 +63,7 @@ public final class SearchRunner { private boolean initialized = false; private Timer updateTimer; private Map jobs = new HashMap<>(); //guarded by "this" + private static final Object finalSearchLock = new Object(); //used for a condition wait SearchRunner() { ingester = Server.getIngester(); @@ -93,12 +94,12 @@ public final class SearchRunner { public synchronized void startJob(long jobId, long dataSourceId, List keywordListNames) { if (!jobs.containsKey(jobId)) { SearchJobInfo jobData = new SearchJobInfo(jobId, dataSourceId, keywordListNames); - jobs.put(jobId, jobData); + jobs.put(jobId, jobData); } if (jobs.size() > 0) { if (!updateTimer.isRunning()) { - updateTimer.start(); + updateTimer.start(); } } } @@ -195,11 +196,11 @@ public final class SearchRunner { // Run one last search as there are probably some new files committed logger.log(Level.INFO, "Running final search for jobid {0}", job.getJobId()); if (!job.getKeywordListNames().isEmpty()) { - try { + try { // In case this job still has a worker running, wait for it to finish - synchronized(this) { + synchronized(finalSearchLock) { while(job.isWorkerRunning()) { - this.wait(); + finalSearchLock.wait(); } } @@ -582,11 +583,11 @@ public final class SearchRunner { try { finalizeSearcher(); stopWatch.stop(); - job.setWorkerRunning(false); // In case the enclosing class instance is waiting on this worker to be done - synchronized(SearchRunner.this) { - SearchRunner.this.notify(); + synchronized(SearchRunner.finalSearchLock) { + job.setWorkerRunning(false); + SearchRunner.finalSearchLock.notify(); } logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs()); From 5a77a3102eeb8816ec1ae5b2b6c4be419c740981 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Thu, 3 Apr 2014 16:16:45 -0400 Subject: [PATCH 23/38] Move notify into the final finally(). --- .../autopsy/keywordsearch/SearchRunner.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index c7c65df96d..cdd1f4a4b0 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -56,13 +56,13 @@ import org.sleuthkit.datamodel.BlackboardAttribute; */ public final class SearchRunner { private static final Logger logger = Logger.getLogger(SearchRunner.class.getName()); - private AtomicInteger messageID = new AtomicInteger(0); + private AtomicInteger messageID = new AtomicInteger(0); private static SearchRunner instance = null; private IngestServices services = IngestServices.getInstance(); private Ingester ingester = null; //guarded by "this" private boolean initialized = false; private Timer updateTimer; - private Map jobs = new HashMap<>(); //guarded by "this" + private Map jobs = new HashMap<>(); //guarded by "this" private static final Object finalSearchLock = new Object(); //used for a condition wait SearchRunner() { @@ -236,7 +236,7 @@ public final class SearchRunner { // If no lists or the worker is already running then skip it if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) { Searcher searcher = new Searcher(job); - job.setCurrentSearcher(searcher); //save the ref + job.setCurrentSearcher(searcher); //save the ref searcher.execute(); //start thread job.setWorkerRunning(true); } @@ -582,17 +582,15 @@ public final class SearchRunner { } finally { try { finalizeSearcher(); - stopWatch.stop(); + stopWatch.stop(); + logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs()); + } finally { // In case the enclosing class instance is waiting on this worker to be done synchronized(SearchRunner.finalSearchLock) { job.setWorkerRunning(false); SearchRunner.finalSearchLock.notify(); } - - logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs()); - } finally { - //searcherLock.unlock(); } } From 7913ecf25d3eed4df3f8c40de181cd26f49a8029 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Thu, 3 Apr 2014 17:25:23 -0400 Subject: [PATCH 24/38] Only do final search if this is the last module in this job to call endJob() --- .../KeywordSearchIngestModule.java | 15 ++++----- .../autopsy/keywordsearch/SearchRunner.java | 32 +++++++++++++++---- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index e748f7befd..e13e0fdd5b 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -80,7 +80,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme //only search images from current ingest, not images previously ingested/indexed //accessed read-only by searcher thread private AtomicInteger messageID = new AtomicInteger(0); - private boolean processedFiles; + private boolean startedSearching = false; private SleuthkitCase caseHandle = null; private static List textExtractors; private static AbstractFileStringExtract stringExtractor; @@ -88,7 +88,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private boolean initialized = false; private Tika tikaFormatDetector; private long jobId; - private long dataSourceId; + private long dataSourceId; private enum IngestStatus { @@ -179,8 +179,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.onlyIdxKwSkipMsg"))); } - processedFiles = false; - indexer = new Indexer(); final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; @@ -219,14 +217,15 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme return ProcessResult.OK; } - processedFiles = true; - //index the file and content (if the content is supported) indexer.indexFile(abstractFile, true); // Start searching if it hasn't started already - List keywordListNames = settings.getNamesOfEnabledKeyWordLists(); - SearchRunner.getInstance().startJob(jobId, dataSourceId, keywordListNames); + if (!startedSearching) { + List keywordListNames = settings.getNamesOfEnabledKeyWordLists(); + SearchRunner.getInstance().startJob(jobId, dataSourceId, keywordListNames); + startedSearching = true; + } return ProcessResult.OK; } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index cdd1f4a4b0..8c6e8b3ce4 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -30,6 +30,7 @@ import java.util.Map.Entry; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; @@ -97,6 +98,8 @@ public final class SearchRunner { jobs.put(jobId, jobData); } + jobs.get(jobId).incrementModuleReferenceCount(); + if (jobs.size() > 0) { if (!updateTimer.isRunning()) { updateTimer.start(); @@ -111,16 +114,24 @@ public final class SearchRunner { */ public void endJob(long jobId) { SearchJobInfo job; + boolean readyForFinalSearch = false; synchronized(this) { job = jobs.get(jobId); if (job == null) { return; - } - jobs.remove(jobId); + } + + // Only do final search if this is the last module in this job to call endJob() + if(job.decrementModuleReferenceCount() == 0) { + jobs.remove(jobId); + readyForFinalSearch = true; + } + } + + if (readyForFinalSearch) { + commit(); + doFinalSearch(job); //this will block until it's done } - - commit(); - doFinalSearch(job); //this will block until it's done } @@ -257,7 +268,8 @@ public final class SearchRunner { private List keywordListNames; //guarded by SearchJobInfo.this private Map> currentResults; //guarded by SearchJobInfo.this private SearchRunner.Searcher currentSearcher; - + private AtomicLong moduleReferenceCount = new AtomicLong(0); + public SearchJobInfo(long jobId, long dataSourceId, List keywordListNames) { this.jobId = jobId; this.dataSourceId = dataSourceId; @@ -306,6 +318,14 @@ public final class SearchRunner { public synchronized void setCurrentSearcher(SearchRunner.Searcher searchRunner) { currentSearcher = searchRunner; } + + public void incrementModuleReferenceCount() { + moduleReferenceCount.incrementAndGet(); + } + + public long decrementModuleReferenceCount() { + return moduleReferenceCount.decrementAndGet(); + } } /** From 66a53a3f0654451286b54f32724ca92eadc32155 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Thu, 3 Apr 2014 19:15:17 -0400 Subject: [PATCH 25/38] Eyeball widget now sends new lists to ingest jobs via the SearchRunner singleton. --- .../KeywordSearchListsViewerPanel.java | 3 ++- .../autopsy/keywordsearch/SearchRunner.java | 16 ++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java index de6d93a918..d0f564fcf5 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java @@ -141,7 +141,8 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer { searchAddListener = new ActionListener() { @Override public void actionPerformed(ActionEvent e) { - if (ingestRunning) { + if (ingestRunning) { + SearchRunner.getInstance().addKeywordListsToAllJobs(listsTableModel.getSelectedLists()); logger.log(Level.INFO, "Submitted enqueued lists to ingest"); } else { searchAction(e); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 8c6e8b3ce4..d53b925d29 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -161,13 +161,15 @@ public final class SearchRunner { } /** - * Add this list to all of the jobs + * Add these lists to all of the jobs * @param keywordListName */ - public synchronized void addKeywordListToAllJobs(String keywordListName) { - logger.log(Level.INFO, "Adding keyword list {0} to all jobs", keywordListName); - for(Entry j : jobs.entrySet()) { - j.getValue().addKeywordListName(keywordListName); + public synchronized void addKeywordListsToAllJobs(List keywordListNames) { + for(String listName : keywordListNames) { + logger.log(Level.INFO, "Adding keyword list {0} to all jobs", listName); + for(Entry j : jobs.entrySet()) { + j.getValue().addKeywordListName(listName); + } } } @@ -292,7 +294,9 @@ public final class SearchRunner { } public synchronized void addKeywordListName(String keywordListName) { - keywordListNames.add(keywordListName); + if (!keywordListNames.contains(keywordListName)) { + keywordListNames.add(keywordListName); + } } public synchronized List currentKeywordResults(Keyword k) { From 341fbfc94eea431900f64d611cb81c2eefed95f0 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Fri, 4 Apr 2014 14:59:05 -0400 Subject: [PATCH 26/38] Use a util timer set as a daemon, not a swing timer. --- .../autopsy/keywordsearch/SearchRunner.java | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 270200ae91..c13c6280f1 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -19,8 +19,6 @@ package org.sleuthkit.autopsy.keywordsearch; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -29,12 +27,12 @@ import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; -import javax.swing.Timer; +import java.util.Timer; +import java.util.TimerTask; import org.netbeans.api.progress.aggregate.AggregateProgressFactory; import org.netbeans.api.progress.aggregate.AggregateProgressHandle; import org.netbeans.api.progress.aggregate.ProgressContributor; @@ -57,11 +55,11 @@ import org.sleuthkit.datamodel.BlackboardAttribute; */ public final class SearchRunner { private static final Logger logger = Logger.getLogger(SearchRunner.class.getName()); - private AtomicInteger messageID = new AtomicInteger(0); private static SearchRunner instance = null; private IngestServices services = IngestServices.getInstance(); private Ingester ingester = null; //guarded by "this" private boolean initialized = false; + private volatile boolean updateTimerRunning = false; private Timer updateTimer; private Map jobs = new HashMap<>(); //guarded by "this" private static final Object finalSearchLock = new Object(); //used for a condition wait @@ -69,8 +67,7 @@ public final class SearchRunner { SearchRunner() { ingester = Server.getIngester(); - final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; - updateTimer = new Timer(updateIntervalMs, new SearchRunner.UpdateTimerAction()); + updateTimer = new Timer(true); // run as a daemon initialized = true; } @@ -100,9 +97,12 @@ public final class SearchRunner { jobs.get(jobId).incrementModuleReferenceCount(); - if (jobs.size() > 0) { - if (!updateTimer.isRunning()) { - updateTimer.start(); + if (jobs.size() > 0) { + final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; + if (!updateTimerRunning) { + logger.log(Level.INFO, "Scheduling update daemon"); + updateTimer.scheduleAtFixedRate(new UpdateTimerTask(), updateIntervalMs, updateIntervalMs); + updateTimerRunning = true; } } } @@ -126,9 +126,9 @@ public final class SearchRunner { jobs.remove(jobId); readyForFinalSearch = true; } - } - - if (readyForFinalSearch) { + } + + if (readyForFinalSearch) { commit(); doFinalSearch(job); //this will block until it's done } @@ -146,8 +146,8 @@ public final class SearchRunner { job = jobs.get(jobId); if (job == null) { return; - } - + } + //stop currentSearcher SearchRunner.Searcher currentSearcher = job.getCurrentSearcher(); if ((currentSearcher != null) && (!currentSearcher.isDone())) { @@ -178,7 +178,7 @@ public final class SearchRunner { */ private void commit() { if (initialized) { - logger.log(Level.INFO, "Commiting index"); + logger.log(Level.INFO, "Committing index"); synchronized(this) { ingester.commit(); } @@ -200,12 +200,6 @@ public final class SearchRunner { * @param job */ private void doFinalSearch(SearchJobInfo job) { - // Cancel timer to ensure unwanted searchers do not start before we - // start the final one - if (updateTimer.isRunning()) { - updateTimer.stop(); - } - // Run one last search as there are probably some new files committed logger.log(Level.INFO, "Running final search for jobid {0}", job.getJobId()); if (!job.getKeywordListNames().isEmpty()) { @@ -234,11 +228,17 @@ public final class SearchRunner { /** * Timer triggered re-search for each job (does a single index commit first) */ - private class UpdateTimerAction implements ActionListener { - private final Logger logger = Logger.getLogger(SearchRunner.UpdateTimerAction.class.getName()); + private class UpdateTimerTask extends TimerTask { + private final Logger logger = Logger.getLogger(SearchRunner.UpdateTimerTask.class.getName()); @Override - public void actionPerformed(ActionEvent e) { + public void run() { + if (jobs.isEmpty()) { + this.cancel(); //terminate this timer task + updateTimerRunning = false; + return; + } + commit(); logger.log(Level.INFO, "Launching searchers"); From ea19685e3ed3f9670d0f46d2f3030df58e7649cc Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Fri, 4 Apr 2014 14:59:35 -0400 Subject: [PATCH 27/38] extractors changed to non-static --- .../autopsy/keywordsearch/KeywordSearchIngestModule.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 27a46ff067..e61e7aedba 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -83,8 +83,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private AtomicInteger messageID = new AtomicInteger(0); private boolean startedSearching = false; private SleuthkitCase caseHandle = null; - private static List textExtractors; - private static AbstractFileStringExtract stringExtractor; + private List textExtractors; + private AbstractFileStringExtract stringExtractor; private final KeywordSearchJobSettings settings; private boolean initialized = false; private Tika tikaFormatDetector; @@ -181,10 +181,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme } indexer = new Indexer(); - - final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; - logger.log(Level.INFO, "Using commit interval (ms): {0}", updateIntervalMs); - logger.log(Level.INFO, "Using searcher interval (ms): {0}", updateIntervalMs); initialized = true; } From 44d11e1862be606205eed83d264da9cc558b5923 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Fri, 4 Apr 2014 17:31:28 -0400 Subject: [PATCH 28/38] Comments, logging, and changed ingester lock. --- .../sleuthkit/autopsy/keywordsearch/SearchRunner.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index c13c6280f1..261e31baad 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -57,7 +57,7 @@ public final class SearchRunner { private static final Logger logger = Logger.getLogger(SearchRunner.class.getName()); private static SearchRunner instance = null; private IngestServices services = IngestServices.getInstance(); - private Ingester ingester = null; //guarded by "this" + private Ingester ingester = null; //guarded by "ingester" private boolean initialized = false; private volatile boolean updateTimerRunning = false; private Timer updateTimer; @@ -91,6 +91,7 @@ public final class SearchRunner { */ public synchronized void startJob(long jobId, long dataSourceId, List keywordListNames) { if (!jobs.containsKey(jobId)) { + logger.log(Level.INFO, "Adding job {0}", jobId); SearchJobInfo jobData = new SearchJobInfo(jobId, dataSourceId, keywordListNames); jobs.put(jobId, jobData); } @@ -113,6 +114,7 @@ public final class SearchRunner { * @param jobId */ public void endJob(long jobId) { + logger.log(Level.INFO, "Ending job {0}", jobId); SearchJobInfo job; boolean readyForFinalSearch = false; synchronized(this) { @@ -141,6 +143,8 @@ public final class SearchRunner { * @param jobId */ public void stopJob(long jobId) { + logger.log(Level.INFO, "Stopping job"); + SearchJobInfo job; synchronized(this) { job = jobs.get(jobId); @@ -179,7 +183,7 @@ public final class SearchRunner { private void commit() { if (initialized) { logger.log(Level.INFO, "Committing index"); - synchronized(this) { + synchronized(ingester) { ingester.commit(); } logger.log(Level.INFO, "Index comitted"); @@ -207,7 +211,7 @@ public final class SearchRunner { // In case this job still has a worker running, wait for it to finish synchronized(finalSearchLock) { while(job.isWorkerRunning()) { - finalSearchLock.wait(); + finalSearchLock.wait(); //wait() releases the lock } } From 8248bf66e7fc0f4ec5dbeca26bb284d8c7467216 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Fri, 4 Apr 2014 17:58:52 -0400 Subject: [PATCH 29/38] Give each job a finalSearchLock so that one job doesn't interfere with another. --- .../autopsy/keywordsearch/SearchRunner.java | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 261e31baad..3400b720d9 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -62,7 +62,6 @@ public final class SearchRunner { private volatile boolean updateTimerRunning = false; private Timer updateTimer; private Map jobs = new HashMap<>(); //guarded by "this" - private static final Object finalSearchLock = new Object(); //used for a condition wait SearchRunner() { ingester = Server.getIngester(); @@ -209,11 +208,7 @@ public final class SearchRunner { if (!job.getKeywordListNames().isEmpty()) { try { // In case this job still has a worker running, wait for it to finish - synchronized(finalSearchLock) { - while(job.isWorkerRunning()) { - finalSearchLock.wait(); //wait() releases the lock - } - } + job.waitForCurrentWorker(); SearchRunner.Searcher finalSearcher = new SearchRunner.Searcher(job, true); job.setCurrentSearcher(finalSearcher); //save the ref @@ -237,6 +232,7 @@ public final class SearchRunner { @Override public void run() { + // If no jobs then cancel the task. If more job(s) come along, a new task will start up. if (jobs.isEmpty()) { this.cancel(); //terminate this timer task updateTimerRunning = false; @@ -275,6 +271,7 @@ public final class SearchRunner { private Map> currentResults; //guarded by SearchJobInfo.this private SearchRunner.Searcher currentSearcher; private AtomicLong moduleReferenceCount = new AtomicLong(0); + private final Object finalSearchLock = new Object(); //used for a condition wait public SearchJobInfo(long jobId, long dataSourceId, List keywordListNames) { this.jobId = jobId; @@ -333,7 +330,29 @@ public final class SearchRunner { public long decrementModuleReferenceCount() { return moduleReferenceCount.decrementAndGet(); - } + } + + /** In case this job still has a worker running, wait for it to finish + * + * @throws InterruptedException + */ + public void waitForCurrentWorker() throws InterruptedException { + synchronized(finalSearchLock) { + while(workerRunning) { + finalSearchLock.wait(); //wait() releases the lock + } + } + } + + /** + * Unset workerRunning and wake up thread(s) waiting on finalSearchLock + */ + public void searchNotify() { + synchronized(finalSearchLock) { + workerRunning = false; + finalSearchLock.notify(); + } + } } /** @@ -614,11 +633,8 @@ public final class SearchRunner { logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs()); } finally { - // In case the enclosing class instance is waiting on this worker to be done - synchronized(SearchRunner.finalSearchLock) { - job.setWorkerRunning(false); - SearchRunner.finalSearchLock.notify(); - } + // In case a thread is waiting on this worker to be done + job.searchNotify(); } } From 3a28422f5fe67917b9d810ec98f06f90170baff8 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Fri, 4 Apr 2014 17:59:48 -0400 Subject: [PATCH 30/38] Remove initialized variable --- .../autopsy/keywordsearch/SearchRunner.java | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 3400b720d9..f2c31e6652 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -58,17 +58,13 @@ public final class SearchRunner { private static SearchRunner instance = null; private IngestServices services = IngestServices.getInstance(); private Ingester ingester = null; //guarded by "ingester" - private boolean initialized = false; private volatile boolean updateTimerRunning = false; private Timer updateTimer; private Map jobs = new HashMap<>(); //guarded by "this" SearchRunner() { ingester = Server.getIngester(); - updateTimer = new Timer(true); // run as a daemon - - initialized = true; } /** @@ -180,20 +176,18 @@ public final class SearchRunner { * Commits index and notifies listeners of index update */ private void commit() { - if (initialized) { - logger.log(Level.INFO, "Committing index"); - synchronized(ingester) { - ingester.commit(); - } - logger.log(Level.INFO, "Index comitted"); + logger.log(Level.INFO, "Committing index"); + synchronized(ingester) { + ingester.commit(); + } + logger.log(Level.INFO, "Index comitted"); - // Signal a potential change in number of text_ingested files - try { - final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles(); - KeywordSearch.fireNumIndexedFilesChange(null, new Integer(numIndexedFiles)); - } catch (NoOpenCoreException | KeywordSearchModuleException ex) { - logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files: ", ex); - } + // Signal a potential change in number of text_ingested files + try { + final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles(); + KeywordSearch.fireNumIndexedFilesChange(null, new Integer(numIndexedFiles)); + } catch (NoOpenCoreException | KeywordSearchModuleException ex) { + logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files: ", ex); } } From f6f5ad700283212f69bdf920350a2330a6a2b55b Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Fri, 4 Apr 2014 18:37:18 -0400 Subject: [PATCH 31/38] Some cleanup and an extra logging statement. --- .../autopsy/keywordsearch/KeywordSearchIngestModule.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index e61e7aedba..1a2afb329f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -24,7 +24,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; @@ -80,7 +79,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme //only search images from current ingest, not images previously ingested/indexed //accessed read-only by searcher thread - private AtomicInteger messageID = new AtomicInteger(0); private boolean startedSearching = false; private SleuthkitCase caseHandle = null; private List textExtractors; @@ -196,7 +194,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme } try { //add data source id of the file to the set, keeping track of images being ingested - ///@todo this should come from IngestJobContext dataSourceId = caseHandle.getFileDataSource(abstractFile); } catch (TskCoreException ex) { @@ -238,6 +235,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme } if (ingestJobCancelled) { + logger.log(Level.INFO, "Ingest job cancelled"); stop(); return; } From 98593fe7b1efb3c82fd1205f3e58c9cbaeb2365c Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Fri, 4 Apr 2014 18:37:42 -0400 Subject: [PATCH 32/38] More thread-safe --- .../src/org/sleuthkit/autopsy/keywordsearch/Ingester.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java index 12d9999b01..bdb304dabf 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java @@ -53,7 +53,6 @@ import org.sleuthkit.datamodel.ContentVisitor; import org.sleuthkit.datamodel.DerivedFile; import org.sleuthkit.datamodel.Directory; import org.sleuthkit.datamodel.File; -import org.sleuthkit.datamodel.FsContent; import org.sleuthkit.datamodel.LayoutFile; import org.sleuthkit.datamodel.LocalFile; import org.sleuthkit.datamodel.ReadContentInputStream; @@ -66,7 +65,7 @@ import org.sleuthkit.datamodel.TskCoreException; class Ingester { private static final Logger logger = Logger.getLogger(Ingester.class.getName()); - private boolean uncommitedIngests = false; + private volatile boolean uncommitedIngests = false; private final ExecutorService upRequestExecutor = Executors.newSingleThreadExecutor(); private final Server solrServer = KeywordSearch.getServer(); private final GetContentFieldsV getContentFieldsV = new GetContentFieldsV(); @@ -74,8 +73,7 @@ class Ingester { //for ingesting chunk as SolrInputDocument (non-content-streaming, by-pass tika) //TODO use a streaming way to add content to /update handler - private final static int MAX_DOC_CHUNK_SIZE = 1024*1024; - private final byte[] docChunkContentBuf = new byte[MAX_DOC_CHUNK_SIZE]; + private static final int MAX_DOC_CHUNK_SIZE = 1024*1024; private static final String docContentEncoding = "UTF-8"; @@ -286,6 +284,7 @@ class Ingester { throw new IngesterException(msg); } + final byte[] docChunkContentBuf = new byte[MAX_DOC_CHUNK_SIZE]; SolrInputDocument updateDoc = new SolrInputDocument(); for (String key : fields.keySet()) { From 652ff6d3063e75a486aebbac0effd9f1c4c6ad93 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Mon, 7 Apr 2014 13:32:44 -0400 Subject: [PATCH 33/38] Give the timer daemon a name (useful for debugging). --- .../src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index f2c31e6652..6e2c52dbd3 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -64,7 +64,7 @@ public final class SearchRunner { SearchRunner() { ingester = Server.getIngester(); - updateTimer = new Timer(true); // run as a daemon + updateTimer = new Timer("SearchRunner update timer", true); // run as a daemon } /** From 94e8bb1c9518d6d5f2122855aa0926cd42599edb Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 8 Apr 2014 15:42:59 -0400 Subject: [PATCH 34/38] Some naming cleanup. --- .../sleuthkit/autopsy/keywordsearch/SearchRunner.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 6e2c52dbd3..aa08b8b60e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -362,7 +362,7 @@ public final class SearchRunner { */ private SearchJobInfo job; private List keywords; //keywords to search - private List keywordLists; // lists currently being searched + private List keywordListNames; // lists currently being searched private Map keywordToList; //keyword to list name mapping private AggregateProgressHandle progressGroup; private final Logger logger = Logger.getLogger(SearchRunner.Searcher.class.getName()); @@ -370,7 +370,7 @@ public final class SearchRunner { Searcher(SearchJobInfo job) { this.job = job; - this.keywordLists = job.getKeywordListNames(); + keywordListNames = job.getKeywordListNames(); keywords = new ArrayList<>(); keywordToList = new HashMap<>(); //keywords are populated as searcher runs @@ -654,10 +654,11 @@ public final class SearchRunner { private void updateKeywords() { KeywordSearchListsXML loader = KeywordSearchListsXML.getCurrent(); - this.keywords.clear(); - this.keywordToList.clear(); + keywords.clear(); + keywordToList.clear(); + - for (String name : this.keywordLists) { + for (String name : keywordListNames) { KeywordList list = loader.getList(name); for (Keyword k : list.getKeywords()) { this.keywords.add(k); From 97b724350fc7d38aa0ae50f6066fe4bb7475b0a7 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 8 Apr 2014 16:05:13 -0400 Subject: [PATCH 35/38] Extra log info --- .../keywordsearch/KeywordSearchIngestModule.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 1a2afb329f..205fd36035 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; @@ -88,6 +89,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private Tika tikaFormatDetector; private long jobId; private long dataSourceId; + private static AtomicInteger instanceCount = new AtomicInteger(0); //just used for logging + private int instanceNum = 0; private enum IngestStatus { @@ -102,6 +105,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme KeywordSearchIngestModule(KeywordSearchJobSettings settings) { this.settings = settings; + instanceNum = instanceCount.getAndIncrement(); } /** @@ -111,7 +115,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme */ @Override public void startUp(IngestJobContext context) throws IngestModuleException { - logger.log(Level.INFO, "init()"); + logger.log(Level.INFO, "Initializing instance {0}", instanceNum); initialized = false; jobId = context.getJobId(); @@ -179,13 +183,11 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme } indexer = new Indexer(); - initialized = true; } @Override public ProcessResult process(AbstractFile abstractFile) { - if (initialized == false) //error initializing indexing/Solr { logger.log(Level.WARNING, "Skipping processing, module not initialized, file: {0}", abstractFile.getName()); @@ -230,6 +232,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme */ @Override public void shutDown(boolean ingestJobCancelled) { + logger.log(Level.INFO, "Instance {0}", instanceNum); + if (initialized == false) { return; } From bb29624635856251376d61f07bd77f46c82257f4 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 8 Apr 2014 16:05:27 -0400 Subject: [PATCH 36/38] Make the ctor synchronized --- .../sleuthkit/autopsy/keywordsearch/KeywordSearchListsXML.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsXML.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsXML.java index 6a6bd9fc87..929ec49ac3 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsXML.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsXML.java @@ -66,7 +66,7 @@ final class KeywordSearchListsXML extends KeywordSearchListsAbstract { * RJCTODO: Move this one to the manager * @return */ - static KeywordSearchListsXML getCurrent() { + static synchronized KeywordSearchListsXML getCurrent() { if (currentInstance == null) { currentInstance = new KeywordSearchListsXML(CUR_LISTS_FILE); currentInstance.reload(); From 0437977226598122438f0823e431eadca875981e Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 9 Apr 2014 15:30:45 -0400 Subject: [PATCH 37/38] moved commit() up in stopJob() and removed some INFO logging --- .../autopsy/keywordsearch/SearchRunner.java | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 6e2c52dbd3..3faaede87d 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -96,7 +96,6 @@ public final class SearchRunner { if (jobs.size() > 0) { final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; if (!updateTimerRunning) { - logger.log(Level.INFO, "Scheduling update daemon"); updateTimer.scheduleAtFixedRate(new UpdateTimerTask(), updateIntervalMs, updateIntervalMs); updateTimerRunning = true; } @@ -109,7 +108,6 @@ public final class SearchRunner { * @param jobId */ public void endJob(long jobId) { - logger.log(Level.INFO, "Ending job {0}", jobId); SearchJobInfo job; boolean readyForFinalSearch = false; synchronized(this) { @@ -138,7 +136,8 @@ public final class SearchRunner { * @param jobId */ public void stopJob(long jobId) { - logger.log(Level.INFO, "Stopping job"); + logger.log(Level.INFO, "Stopping job {0}", jobId); + commit(); SearchJobInfo job; synchronized(this) { @@ -154,9 +153,7 @@ public final class SearchRunner { } jobs.remove(jobId); - } - - commit(); + } } /** @@ -176,11 +173,9 @@ public final class SearchRunner { * Commits index and notifies listeners of index update */ private void commit() { - logger.log(Level.INFO, "Committing index"); synchronized(ingester) { ingester.commit(); } - logger.log(Level.INFO, "Index comitted"); // Signal a potential change in number of text_ingested files try { @@ -235,7 +230,6 @@ public final class SearchRunner { commit(); - logger.log(Level.INFO, "Launching searchers"); synchronized(SearchRunner.this) { // Spawn a search thread for each job for(Entry j : jobs.entrySet()) { @@ -383,12 +377,6 @@ public final class SearchRunner { @Override protected Object doInBackground() throws Exception { - if (finalRun) { - logger.log(Level.INFO, "Pending start of new (final) searcher"); - } else { - logger.log(Level.INFO, "Pending start of new searcher"); - } - final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); final String pgDisplayName = displayName + (" (" + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") + ")"); @@ -418,7 +406,6 @@ public final class SearchRunner { final StopWatch stopWatch = new StopWatch(); stopWatch.start(); try { - logger.log(Level.INFO, "Started a new searcher"); progressGroup.setDisplayName(displayName); int keywordsSearched = 0; @@ -672,7 +659,6 @@ public final class SearchRunner { * guaranteed to run. */ private void finalizeSearcher() { - logger.log(Level.INFO, "Searcher finalizing"); SwingUtilities.invokeLater(new Runnable() { @Override public void run() { From b5d06e67ce3a1633dea80d1c7607974dd791193f Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 9 Apr 2014 15:36:11 -0400 Subject: [PATCH 38/38] simplify the loop in addKeywordListsToAllJobs() --- .../src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 3faaede87d..aa0e6bcd46 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -163,8 +163,8 @@ public final class SearchRunner { public synchronized void addKeywordListsToAllJobs(List keywordListNames) { for(String listName : keywordListNames) { logger.log(Level.INFO, "Adding keyword list {0} to all jobs", listName); - for(Entry j : jobs.entrySet()) { - j.getValue().addKeywordListName(listName); + for(SearchJobInfo j : jobs.values()) { + j.addKeywordListName(listName); } } }
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(hitFile.getParentPath()).append(hitFile.getName()).append("
").append(attr.getValueString()).append("
").append(attr.getValueString()).append("