mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-08 06:09:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

- Fix to show stats and file ingest dialog after ingest complete and also all background threads (keyword search) completed.

Browse Source 
- fix to not run complete() portion of keyword search service in some cases, when service was not selected to run
This commit is contained in:
adam-m 2012-03-19 18:54:12 -04:00
parent 15409ce3a9
commit c928788bb6
8 changed files with 130 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestService.java
View File

@ -18,6 +18,7 @@
*/
package org.sleuthkit.autopsy.hashdatabase;
import java.beans.PropertyChangeListener;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Collections;
@ -207,6 +208,11 @@ public class HashDbIngestService implements IngestServiceFsContent {
return false;
}
@Override
public boolean backgroundJobsCompleteListener(PropertyChangeListener l) {
return false;
}
@Override
public boolean hasSimpleConfiguration() {
return false;

68
Ingest/src/org/sleuthkit/autopsy/ingest/IngestManager.java
View File

@ -18,6 +18,7 @@
*/
package org.sleuthkit.autopsy.ingest;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeSupport;
import java.text.DateFormat;
@ -139,7 +140,7 @@ public class IngestManager {
* @param images images to execute services on
*/
void execute(final List<IngestServiceAbstract> services, final List<Image> images) {
logger.log(Level.INFO, "Will enqueue number of images: " + images.size());
logger.log(Level.INFO, "Will enqueue number of images: " + images.size() + " to " + services.size() + " services.");
if (!isIngestRunning()) {
ui.clearMessages();
@ -443,8 +444,9 @@ public class IngestManager {
*/
public static List<IngestServiceImage> enumerateImageServices() {
List<IngestServiceImage> ret = new ArrayList<IngestServiceImage>();
for (IngestServiceImage list : Lookup.getDefault().lookupAll(IngestServiceImage.class))
for (IngestServiceImage list : Lookup.getDefault().lookupAll(IngestServiceImage.class)) {
ret.add(list);
}
return ret;
}
@ -453,8 +455,9 @@ public class IngestManager {
*/
public static List<IngestServiceFsContent> enumerateFsContentServices() {
List<IngestServiceFsContent> ret = new ArrayList<IngestServiceFsContent>();
for (IngestServiceFsContent list : Lookup.getDefault().lookupAll(IngestServiceFsContent.class))
for (IngestServiceFsContent list : Lookup.getDefault().lookupAll(IngestServiceFsContent.class)) {
ret.add(list);
}
return ret;
}
@ -750,7 +753,7 @@ public class IngestManager {
public synchronized String toString() {
return "FsContentQueue, size: " + Integer.toString(fsContentUnits.size());
}
public String printQueue() {
StringBuilder sb = new StringBuilder();
for (QueueUnit<FsContent, IngestServiceFsContent> u : fsContentUnits) {
@ -903,7 +906,7 @@ public class IngestManager {
hash = 37 * hash + (this.services != null ? this.services.hashCode() : 0);
return hash;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@ -1134,18 +1137,63 @@ public class IngestManager {
handleInterruption();
logger.log(Level.SEVERE, "Fatal error during ingest.", ex);
} finally {
stats.end();
//stats.end();
progress.finish();
if (!this.isCancelled()) {
logger.log(Level.INFO, "Summary Report: " + stats.toString());
ui.displayReport(stats.toHtmlString());
//logger.log(Level.INFO, "Summary Report: " + stats.toString());
//ui.displayReport(stats.toHtmlString());
new FsServicesComplete(stats);
}
initMainProgress(0);
}
}
/**
* Ensures that all background threads are done
* then finalize the stats and show dialog
*/
private class FsServicesComplete {
private IngestManagerStats stats; //ongoing stats
private List<IngestServiceAbstract> running = new ArrayList<IngestServiceAbstract>();
FsServicesComplete(IngestManagerStats stats) {
this.stats = stats;
for (IngestServiceAbstract s : fsContentServices) {
if (s.backgroundJobsCompleteListener(new PropertyChangeListener() {
@Override
public void propertyChange(PropertyChangeEvent evt) {
if (evt.getPropertyName().equals(IngestServiceAbstract.BCKGRND_JOBS_COMPLETED_EVT)) {
IngestServiceAbstract service = (IngestServiceAbstract) evt.getNewValue();
running.remove(service);
if (running.isEmpty()) {
showStats();
}
}
}
})) {
running.add(s);
}
}
//no listeners registered since no services running any longer
if (running.isEmpty()) {
showStats();
}
}
void showStats() {
stats.end();
logger.log(Level.INFO, "Summary Report: " + stats.toString());
ui.displayReport(stats.toHtmlString());
}
}
private void handleInterruption() {
for (IngestServiceFsContent s : fsContentServices) {
s.stop();
@ -1254,9 +1302,9 @@ public class IngestManager {
progress.progress(serviceName + " " + imageName, ++processed);
}
}
//logger.log(Level.INFO, fsContentQueue.printQueue());
progress.progress("Sorting files", processed);
sortFsContents();
}

13
Ingest/src/org/sleuthkit/autopsy/ingest/IngestServiceAbstract.java
View File

@ -19,12 +19,15 @@
package org.sleuthkit.autopsy.ingest;
import java.beans.PropertyChangeListener;
/**
* Base interface for ingest services
*/
public interface IngestServiceAbstract {
public enum ServiceType {Image, FsContent};
public static final String BCKGRND_JOBS_COMPLETED_EVT = "BCKGRND_JOBS_COMPLETED_EVT";
/**
* notification from manager that brand new processing should be initiated.
@ -66,6 +69,16 @@ public interface IngestServiceAbstract {
*/
public boolean hasBackgroundJobsRunning();
/**
* Register listener to notify when all background jobs have completed and the service
* has truly finished. The service should first check if it has threads running, and then register the listener, all in atomic operation.
* The event fired off should be BCKGRND_JOBS_COMPLETED_EVT, with the instance of IngestServiceAbstract in the newValue parameter.
*
* @param l listener
* @return true if listener registered, false otherwise (i.e. no background jobs were running)
*/
public boolean backgroundJobsCompleteListener(PropertyChangeListener l);
/**
* @return does this service have a simple configuration?

7
Ingest/src/org/sleuthkit/autopsy/ingest/example/ExampleFsContentIngestService.java
View File

@ -18,9 +18,9 @@
*/
package org.sleuthkit.autopsy.ingest.example;
import java.beans.PropertyChangeListener;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.swing.JPanel;
import org.sleuthkit.autopsy.ingest.IngestManagerProxy;
import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType;
@ -117,6 +117,11 @@ public class ExampleFsContentIngestService implements IngestServiceFsContent {
return false;
}
@Override
public boolean backgroundJobsCompleteListener(PropertyChangeListener l) {
return false;
}
@Override
public void saveAdvancedConfiguration() {
}

6
Ingest/src/org/sleuthkit/autopsy/ingest/example/ExampleImageIngestService.java
View File

@ -18,6 +18,7 @@
*/
package org.sleuthkit.autopsy.ingest.example;
import java.beans.PropertyChangeListener;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.sleuthkit.autopsy.ingest.IngestImageWorkerController;
@ -147,6 +148,11 @@ public final class ExampleImageIngestService implements IngestServiceImage {
return false;
}
@Override
public boolean backgroundJobsCompleteListener(PropertyChangeListener l) {
return false;
}
@Override
public void saveAdvancedConfiguration() {
}

1
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java
View File

@ -48,6 +48,7 @@ class Ingester {
}
@Override
@SuppressWarnings("FinalizeDeclaration")
protected void finalize() throws Throwable {
super.finalize();

36
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java
View File

@ -18,6 +18,8 @@
*/
package org.sleuthkit.autopsy.keywordsearch;
import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeSupport;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@ -25,7 +27,6 @@ import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.swing.JPanel;
import javax.swing.SwingUtilities;
import javax.swing.SwingWorker;
import org.apache.commons.lang.StringEscapeUtils;
@ -38,6 +39,7 @@ import org.sleuthkit.autopsy.ingest.IngestManager;
import org.sleuthkit.autopsy.ingest.IngestManagerProxy;
import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType;
import org.sleuthkit.autopsy.ingest.IngestServiceAbstract;
import org.sleuthkit.autopsy.ingest.IngestServiceFsContent;
import org.sleuthkit.autopsy.ingest.ServiceDataEvent;
import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException;
@ -68,11 +70,15 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
private Indexer indexer;
private SwingWorker searcher;
private volatile boolean searcherDone = true;
private static PropertyChangeSupport pcs = null;
private Map<Keyword, List<FsContent>> currentResults;
private volatile int messageID = 0;
private boolean processedFiles;
private volatile boolean finalRun = false;
private volatile boolean finalRunComplete = false;
private final String hashDBServiceName = "Hash Lookup";
private SleuthkitCase caseHandle = null;
// TODO: use a more robust method than checking file extension to determine
// whether to try a file
// supported extensions list from http://www.lucidimagination.com/devzone/technical-articles/content-extraction-tika
@ -108,6 +114,9 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
//notify depending service that keyword search (would) encountered error for this file
return ProcessResult.ERROR;
}
if (processedFiles == false)
processedFiles = true;
//check if time to commit and previous search is not running
//commiting while searching causes performance issues
@ -151,11 +160,12 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
updateKeywords();
//run one last search as there are probably some new files committed
if (keywords != null && !keywords.isEmpty()) {
if (keywords != null && !keywords.isEmpty() && processedFiles == true) {
finalRun = true;
searcher = new Searcher(keywords);
searcher.execute();
} else {
finalRunComplete = true;
managerProxy.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, this, "Completed"));
}
//postSummary();
@ -191,6 +201,9 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
caseHandle = Case.getCurrentCase().getSleuthkitCase();
this.managerProxy = managerProxy;
//this deregisters previously registered listeners at every init()
pcs = new PropertyChangeSupport(KeywordSearchIngestService.class);
final Server.Core solrCore = KeywordSearch.getServer().getCore();
ingester = solrCore.getIngester();
@ -209,7 +222,9 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
managerProxy.postMessage(IngestMessage.createWarningMessage(++messageID, instance, "No keywords in keyword list.", "Only indexing will be done and and keyword search will be skipped (it can be executed later again as ingest or using toolbar search feature)."));
}
processedFiles = false;
finalRun = false;
finalRunComplete = false;
searcherDone = true; //make sure to start the initial searcher
//keeps track of all results per run not to repeat reporting the same hits
currentResults = new HashMap<Keyword, List<FsContent>>();
@ -271,6 +286,17 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
//no need to check timer thread
}
@Override
public synchronized boolean backgroundJobsCompleteListener(PropertyChangeListener l) {
if (finalRunComplete == true)
return false;
else {
pcs.addPropertyChangeListener(l);
return true;
}
}
private void commit() {
ingester.commit();
@ -468,11 +494,12 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
if (fsContent.getSize() < MAX_STRING_EXTRACT_SIZE) {
if (!extractAndIngest(fsContent)) {
logger.log(Level.INFO, "Failed to extract strings and ingest, file '" + fsContent.getName() + "' (id: " + fsContent.getId() + ").");
ingestStatus.put(fsContent.getId(), IngestStatus.SKIPPED);
} else {
ingestStatus.put(fsContent.getId(), IngestStatus.EXTRACTED_INGESTED);
}
} else {
ingestStatus.put(fsContent.getId(), IngestStatus.SKIPPED);
//ingestStatus.put(fsContent.getId(), IngestStatus.SKIPPED);
}
}
}
@ -499,6 +526,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
@Override
public boolean cancel() {
finalRunComplete = true;
return Searcher.this.cancel(true);
}
});
@ -655,9 +683,11 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
//logger.log(Level.INFO, "Finished search");
if (finalRun) {
finalRunComplete = true;
keywords.clear();
keywordLists.clear();
managerProxy.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, KeywordSearchIngestService.instance, "Completed"));
pcs.firePropertyChange(IngestServiceAbstract.BCKGRND_JOBS_COMPLETED_EVT, null, KeywordSearchIngestService.this);
}
}
}

8
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestService.java
View File

@ -18,12 +18,12 @@
*/
package org.sleuthkit.autopsy.recentactivity;
import java.beans.PropertyChangeListener;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.swing.JPanel;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.ingest.IngestImageWorkerController;
import org.sleuthkit.autopsy.ingest.IngestManager;
@ -162,4 +162,10 @@ public final class RAImageIngestService implements IngestServiceImage {
public boolean hasBackgroundJobsRunning() {
return false;
}
@Override
public boolean backgroundJobsCompleteListener(PropertyChangeListener l) {
return false;
}
}